Lithium update (#5896)

* Lithium update

* Lithium update

* Lithium update

* Lithium: small optimizations.

* Lithium: add pgo.

* Lithium: fix llvm-profdata path.

* Lithium: fix clang install.

* Lithium: Fix dockerfiles.

* Lithium: Fix test.

* Lithium: try to fix profile on travis.

* Lithium: factor compilation in a sh script.

* Lithium: profile on  8081 port.

* Lithium: LTO.

frameworks/C++/lithium/compile_and_start.sh

@@ -0,0 +1,22 @@
+#! /bin/sh
+
+DB_FLAG=$1
+COMMIT=ba3849b86e004798faf91d64a1a9550e379ac53a
+
+if [ $DB_FLAG = "TFB_MYSQL" ]; then
+  CXX_FLAGS="-I /usr/include/mariadb  -lmariadbclient "
+  wget https://raw.githubusercontent.com/matt-42/lithium/$COMMIT/single_headers/lithium_mysql.hh
+elif [ $DB_FLAG = "TFB_PGSQL" ]; then
+  CXX_FLAGS="-I/usr/include/postgresql -I /usr/include/postgresql/12/server -lpthread -lpq"
+  wget https://raw.githubusercontent.com/matt-42/lithium/$COMMIT/single_headers/lithium_pgsql.hh
+fi
+
+
+wget https://raw.githubusercontent.com/matt-42/lithium/$COMMIT/single_headers/lithium_http_backend.hh
+
+clang++ -fprofile-instr-generate=./profile.prof -flto -DPROFILE_MODE -DNDEBUG -D$DB_FLAG -O3 -march=native -std=c++17 ./lithium.cc $CXX_FLAGS -lpthread -lboost_context -lssl -lcrypto -o /lithium_tbf
+/lithium_tbf tfb-database 8081
+llvm-profdata-10 merge -output=./profile.pgo  ./profile.prof
+clang++ -fprofile-instr-use=./profile.pgo -flto -DNDEBUG -D$DB_FLAG -O3 -march=native -std=c++17 ./lithium.cc $CXX_FLAGS -lpthread -lboost_context -lssl -lcrypto -o /lithium_tbf
+
+/lithium_tbf tfb-database 8080

frameworks/C++/lithium/lithium-postgres.dockerfile

@@ -1,15 +1,8 @@
 FROM buildpack-deps:focal
 
 RUN apt-get update -yqq
-RUN apt-get install -yqq g++-9 libboost-dev postgresql-server-dev-all libpq-dev wget libboost-context-dev
+RUN apt-get install -yqq clang libboost-dev postgresql-server-dev-all libpq-dev wget libboost-context-dev
 
 COPY ./ ./
 
-ENV COMMIT=876f869c8a1cba74e906296da83a10957afb941e
-
-RUN wget https://raw.githubusercontent.com/matt-42/lithium/$COMMIT/single_headers/lithium_pgsql.hh
-RUN wget https://raw.githubusercontent.com/matt-42/lithium/$COMMIT/single_headers/lithium_http_backend.hh
-
-RUN g++ -DTFB_PGSQL -O3 -DNDEBUG -march=native -std=c++17 ./lithium.cc -I/usr/include/postgresql -I /usr/include/postgresql/12/server -lpthread -lpq -lboost_context -lssl -lcrypto -o /lithium_tbf
-
-CMD /lithium_tbf tfb-database 8080
+CMD ./compile_and_start.sh TFB_PGSQL

frameworks/C++/lithium/lithium.cc

@@ -24,35 +24,18 @@ void escape_html_entities(B& buffer, const std::string& data)
     }
 }
 
-void tune_n_sql_connections(int& nc_to_tune, std::string http_req, int port, int min, int max) {
-
-  std::cout << std::endl << "Benchmark " << http_req << std::endl;
-
+#ifdef PROFILE_MODE
+void siege(int port) {
   auto sockets = http_benchmark_connect(512, port);
-
-  float max_req_per_s = 0;
-  int best_nconn = 2;
-  for (int i = 0; i <= 7; i++)
-  {
-    int nc = min + (max - min) * i / 7;
-    nc_to_tune = nc;
-
-    // Warmup.
-    http_benchmark(sockets, 4, 200, http_req);
-    float req_per_s = http_benchmark(sockets, 4, 1000, http_req);
-    std::cout << nc << " -> " << req_per_s << " req/s." << std::endl;
-    if (req_per_s > max_req_per_s)
-    {
-      max_req_per_s = req_per_s;
-      best_nconn = nc;
-    }
-  }
-
+  http_benchmark(sockets, 2, 1000, "GET /json HTTP/1.1\r\n\r\n");
+  http_benchmark(sockets, 2, 1000, "GET /plaintext HTTP/1.1\r\n\r\n");
+  http_benchmark(sockets, 2, 1000, "GET /db HTTP/1.1\r\n\r\n");
+  http_benchmark(sockets, 2, 1000, "GET /queries?N=20 HTTP/1.1\r\n\r\n");
+  http_benchmark(sockets, 2, 1000, "GET /fortunes HTTP/1.1\r\n\r\n");
+  http_benchmark(sockets, 2, 1000, "GET /updates?N=20 HTTP/1.1\r\n\r\n");
   http_benchmark_close(sockets);
-
-  std::cout << "best: " << best_nconn << " (" << max_req_per_s << " req/s)."<< std::endl;
-  nc_to_tune = best_nconn;
 }
+#endif
 
 int main(int argc, char* argv[]) {
 
@@ -112,7 +95,7 @@ int main(int argc, char* argv[]) {
   };
   my_api.get("/db") = [&](http_request& request, http_response& response) {
     sql_db.max_async_connections_per_thread_ = db_nconn;
-    response.write_json(random_numbers.connect(request.fiber).find_one(s::id = 1 + rand() % 10000).value());
+    response.write_json(*random_numbers.connect(request.fiber).find_one(s::id = 1 + rand() % 10000));
   };
 
   my_api.get("/queries") = [&](http_request& request, http_response& response) {
@@ -125,7 +108,7 @@ int main(int argc, char* argv[]) {
     auto c = random_numbers.connect(request.fiber);
     std::vector<decltype(random_numbers.all_fields())> numbers(N);
     for (int i = 0; i < N; i++)
-      numbers[i] = c.find_one(s::id = 1 + rand() % 10000).value();
+      numbers[i] = *c.find_one(s::id = 1 + rand() % 10000);
 
     response.write_json(numbers);
   };
@@ -147,7 +130,7 @@ int main(int argc, char* argv[]) {
 #endif
       for (int i = 0; i < N; i++)
       {
-        numbers[i] = c.find_one(s::id = 1 + rand() % 10000).value();
+        numbers[i] = *c.find_one(s::id = 1 + rand() % 10000);
         numbers[i].randomNumber = 1 + rand() % 10000;
       }
 
@@ -192,8 +175,18 @@ int main(int argc, char* argv[]) {
     response.write(ss.to_string_view());
   };
 
+#ifndef PROFILE_MODE
   // Start the server for the Techempower benchmark.
   http_serve(my_api, port, s::nthreads = nthreads);
+#else
+  std::thread server_thread([&] {
+    http_serve(my_api, port, s::nthreads = nprocs);
+  });
+  usleep(2e6);
+  siege(port);
+  li::quit_signal_catched = true;
+  server_thread.join();
 
+#endif
   return 0;
 }

frameworks/C++/lithium/lithium.dockerfile

@@ -1,15 +1,8 @@
 FROM buildpack-deps:focal
 
 RUN apt-get update -yqq
-RUN apt-get install -yqq libboost-dev libmariadb-dev wget  libboost-context-dev g++-9
+RUN apt-get install -yqq libboost-dev libmariadb-dev wget  libboost-context-dev clang
 
 COPY ./ ./
 
-ENV COMMIT=876f869c8a1cba74e906296da83a10957afb941e
-
-RUN wget https://raw.githubusercontent.com/matt-42/lithium/$COMMIT/single_headers/lithium_mysql.hh
-RUN wget https://raw.githubusercontent.com/matt-42/lithium/$COMMIT/single_headers/lithium_http_backend.hh
-
-RUN g++ -DNDEBUG -DTFB_MYSQL -O3 -march=native -std=c++17 ./lithium.cc -I /usr/include/mariadb -lpthread -lmariadbclient -lboost_context -lssl -lcrypto -o /lithium_tbf
-
-CMD /lithium_tbf tfb-database 8080
+CMD ./compile_and_start.sh TFB_MYSQL