H2O: Yet another batch of changes (#3821)

* Optimize the cached queries test - the cache implementation
provided by libh2o does not support batch operations, so when
the cache is multithreaded, the locking cost is paid for every
entry. The solution is to use a non-concurrent cache, and to
wrap all interfaces with custom locking. The trade-off is that
critical sections become longer.
* Adjust the number of database connections in the cloud
environment.
* Expand the documentation.
* Fix a file descriptor leak.

frameworks/C/h2o/README.md

@@ -1,20 +1,28 @@
 # h2o
 
-This is a framework implementation using the [H2O](https://h2o.examp1e.net/) HTTP server.
+This is a framework implementation using the [H2O](https://h2o.examp1e.net) HTTP server. It builds directly on top of `libh2o` instead of running the standalone server.
 
 ## Requirements
 
-CMake, H2O, libpq, mustache-c, OpenSSL, YAJL
+[CMake](https://cmake.org), [H2O](https://h2o.examp1e.net), [libpq](https://www.postgresql.org), [mustache-c](https://github.com/x86-64/mustache-c), [OpenSSL](https://www.openssl.org), [YAJL](https://lloyd.github.io/yajl)
 
 ## Performance issues
 
+### Database tests
+
+`libpq` does not support command pipelining, and implementing anything equivalent on top of it conflicts with the requirements.
+
+### Database updates
+
+In the Citrine environment the database connection settings that improve the performance on the updates test make the other database results worse, and vice versa.
+
 ### Plaintext
 
-H2O performs at least one system call per pipelined response.
+`libh2o` performs at least one system call per pipelined response.
 
 ### Cached queries
 
-The in-memory caching mechanism provided by libh2o uses mutexes even for read-only access, so when the application is running multithreaded, cache usage is serialized.
+Most of the operations that the in-memory caching mechanism provided by `libh2o` supports modify the cache (in particular, `h2o_cache_fetch()` updates a list of least recently used entries, and may remove expired ones), so when the application is running multithreaded, cache access must be serialized.
 
 ## Contact
 

frameworks/C/h2o/src/main.c

@@ -80,6 +80,7 @@ static void free_global_data(global_data_t *global_data)
 	if (global_data->world_cache)
 		h2o_cache_destroy(global_data->world_cache);
 
+	CHECK_ERROR(pthread_mutex_destroy, &global_data->world_cache_lock);
 	h2o_config_dispose(&global_data->h2o_config);
 
 	if (global_data->ssl_ctx)
@@ -132,7 +133,8 @@ static int initialize_global_data(const config_t *config, global_data_t *global_
 	CHECK_ERRNO_RETURN(global_data->signal_fd, signalfd, -1, &signals, SFD_NONBLOCK | SFD_CLOEXEC);
 	global_data->fortunes_template = get_fortunes_template(config->template_path);
 	h2o_config_init(&global_data->h2o_config);
-	global_data->world_cache = h2o_cache_create(H2O_CACHE_FLAG_MULTITHREADED,
+	CHECK_ERROR(pthread_mutex_init, &global_data->world_cache_lock, NULL);
+	global_data->world_cache = h2o_cache_create(0,
 	                                            config->world_cache_capacity,
 	                                            config->world_cache_duration,
 	                                            free_world_cache_entry);
@@ -177,6 +179,7 @@ static int initialize_global_data(const config_t *config, global_data_t *global_
 	}
 
 error:
+	close(global_data->signal_fd);
 	free_global_data(global_data);
 	return EXIT_FAILURE;
 }

frameworks/C/h2o/src/utility.h

@@ -22,6 +22,7 @@
 #define UTILITY_H_
 
 #include <h2o.h>
+#include <pthread.h>
 #include <stdint.h>
 #include <h2o/cache.h>
 #include <openssl/ssl.h>
@@ -73,6 +74,7 @@ typedef struct {
 	int signal_fd;
 	bool shutdown;
 	h2o_globalconf_t h2o_config;
+	pthread_mutex_t world_cache_lock;
 } global_data_t;
 
 typedef struct {

frameworks/C/h2o/src/world.c

@@ -20,6 +20,7 @@
 #include <assert.h>
 #include <ctype.h>
 #include <h2o.h>
+#include <pthread.h>
 #include <stdbool.h>
 #include <stddef.h>
 #include <stdint.h>
@@ -89,6 +90,7 @@ static int do_multiple_queries(bool do_update, bool use_cache, h2o_req_t *req);
 static void do_updates(multiple_query_ctx_t *query_ctx);
 static void fetch_from_cache(uint64_t now,
                              h2o_cache_t *world_cache,
+                             pthread_mutex_t *world_cache_lock,
                              multiple_query_ctx_t *query_ctx);
 static size_t get_query_number(h2o_req_t *req);
 static void initialize_ids(size_t num_query, query_result_t *res, unsigned int *seed);
@@ -197,6 +199,7 @@ static int do_multiple_queries(bool do_update, bool use_cache, h2o_req_t *req)
 		if (use_cache) {
 			fetch_from_cache(h2o_now(ctx->event_loop.h2o_ctx.loop),
 			                 ctx->global_data->world_cache,
+			                 &ctx->global_data->world_cache_lock,
 			                 query_ctx);
 
 			if (query_ctx->num_result == query_ctx->num_query) {
@@ -309,10 +312,13 @@ error:
 
 static void fetch_from_cache(uint64_t now,
                              h2o_cache_t *world_cache,
+                             pthread_mutex_t *world_cache_lock,
                              multiple_query_ctx_t *query_ctx)
 {
 	h2o_iovec_t key = {.len = sizeof(query_ctx->res->id)};
 
+	CHECK_ERROR(pthread_mutex_lock, world_cache_lock);
+
 	for (size_t i = 0; i < query_ctx->num_query; i++) {
 		key.base = (char *) &query_ctx->res[i].id;
 
@@ -326,6 +332,8 @@ static void fetch_from_cache(uint64_t now,
 			h2o_cache_release(world_cache, r);
 		}
 	}
+
+	CHECK_ERROR(pthread_mutex_unlock, world_cache_lock);
 }
 
 static size_t get_query_number(h2o_req_t *req)
@@ -402,11 +410,13 @@ static result_return_t on_multiple_query_result(db_query_param_t *param, PGresul
 				const h2o_iovec_t value = {.base = (char *) r, .len = sizeof(*r)};
 
 				*r = query_ctx->res[query_ctx->num_result];
+				CHECK_ERROR(pthread_mutex_lock, &ctx->global_data->world_cache_lock);
 				h2o_cache_set(ctx->global_data->world_cache,
 				              h2o_now(ctx->event_loop.h2o_ctx.loop),
 				              key,
 				              0,
 				              value);
+				CHECK_ERROR(pthread_mutex_unlock, &ctx->global_data->world_cache_lock);
 			}
 		}
 

frameworks/C/h2o/start-servers.sh

@@ -18,7 +18,7 @@ if [[ "$CPU_COUNT" -gt 16 ]]; then
 else
 	echo "Running h2o_app in the cloud environment."
 	USE_PROCESSES=false
-	DB_CONN=4
+	DB_CONN=8
 fi
 
 build_h2o_app()