H2O: Reuse JSON generators across HTTP requests (#2713)

Previously, whenever a memory allocation for the sake of a JSON
generator was performed, the application used the memory pool
associated with the HTTP request, so the generator was freed
after the response was sent (since the memory pool was
subsequently destroyed), and a new one was created for the next
request. In this way the memory management overhead for the JSON
generator was reduced.

However, it has been discovered that some of the memory
allocation sizes associated with the JSON generator are above
the limit imposed by libh2o, so the latter resorts to calling
malloc() directly, which defeats the purpose of using a memory
pool in the first place.

The solution is to separate the memory management for the JSON
generators from the memory pools managed by libh2o, and to reuse
generators across HTTP requests. After a response is sent, the
state of each generator is fully reset (as if the generator has
just been constructed), so that when it is reused, the new JSON
response is created from scratch. Reinitializing the generator
is much cheaper than freeing it and allocating a new one.

Since this approach increases the memory usage of the
application, the maximum number of JSON generators that can be
reused is configurable. Any generator above that limit will be
deallocated immediately as soon as it is no longer used.

frameworks/C/h2o/setup.sh

@@ -71,7 +71,7 @@ rm -rf "$BUILD_DIR"
 echo "Maximum database connections per thread: $DB_CONN"
 
 if "$CLOUD_ENVIRONMENT"; then
-	run_h2o_app "0-$((NUM_PROC - 1))" "${H2O_APP_HOME}/bin" "${H2O_APP_HOME}/share/h2o_app"
+	run_h2o_app 0 "${H2O_APP_HOME}/bin" "${H2O_APP_HOME}/share/h2o_app"
 else
 	for ((i = 0; i < PHYSICAL_ENVIRONMENT_THREADS; i++)); do
 		run_h2o_app "$i" "${H2O_APP_HOME}/bin" "${H2O_APP_HOME}/share/h2o_app" -t1

frameworks/C/h2o/src/main.c

@@ -43,7 +43,7 @@
 #define USAGE_MESSAGE \
 	"Usage:\n%s [-a <max connections accepted simultaneously>] [-b <bind address>] " \
 	"[-c <certificate file>] [-d <database connection string>] [-f fortunes template file path] " \
-	"[-k <private key file>] [-l <log path>] " \
+	"[-j <max reused JSON generators>] [-k <private key file>] [-l <log path>] " \
 	"[-m <max database connections per thread>] [-p <port>] " \
 	"[-q <max enqueued database queries per thread>] [-r <root directory>] " \
 	"[-s <HTTPS port>] [-t <thread number>]\n"
@@ -165,10 +165,12 @@ error:
 static int parse_options(int argc, char *argv[], config_t *config)
 {
 	memset(config, 0, sizeof(*config));
+	// Need to set the default value here because 0 is a valid input value.
+	config->max_json_generator = 32;
 	opterr = 0;
 
 	while (1) {
-		const int opt = getopt(argc, argv, "?a:b:c:d:f:k:l:m:p:q:r:s:t:");
+		const int opt = getopt(argc, argv, "?a:b:c:d:f:j:k:l:m:p:q:r:s:t:");
 
 		if (opt == -1)
 			break;
@@ -204,6 +206,9 @@ static int parse_options(int argc, char *argv[], config_t *config)
 			case 'f':
 				config->template_path = optarg;
 				break;
+			case 'j':
+				PARSE_NUMBER(config->max_json_generator);
+				break;
 			case 'k':
 				config->key = optarg;
 				break;

frameworks/C/h2o/src/request_handler.c

@@ -28,6 +28,7 @@
 #include "error.h"
 #include "fortune.h"
 #include "request_handler.h"
+#include "thread.h"
 #include "utility.h"
 #include "world.h"
 
@@ -41,13 +42,17 @@ static int json_serializer(struct st_h2o_handler_t *self, h2o_req_t *req)
 {
 	IGNORE_FUNCTION_PARAMETER(self);
 
-	const yajl_gen gen = get_json_generator(&req->pool);
+	thread_context_t * const ctx = H2O_STRUCT_FROM_MEMBER(thread_context_t,
+	                                                      event_loop.h2o_ctx,
+	                                                      req->conn->ctx);
+	json_generator_t * const gen = get_json_generator(&ctx->json_generator,
+	                                                  &ctx->json_generator_num);
 
 	if (gen) {
-		CHECK_YAJL_STATUS(yajl_gen_map_open, gen);
-		CHECK_YAJL_STATUS(yajl_gen_string, gen, YAJL_STRLIT("message"));
-		CHECK_YAJL_STATUS(yajl_gen_string, gen, YAJL_STRLIT(HELLO_RESPONSE));
-		CHECK_YAJL_STATUS(yajl_gen_map_close, gen);
+		CHECK_YAJL_STATUS(yajl_gen_map_open, gen->gen);
+		CHECK_YAJL_STATUS(yajl_gen_string, gen->gen, YAJL_STRLIT("message"));
+		CHECK_YAJL_STATUS(yajl_gen_string, gen->gen, YAJL_STRLIT(HELLO_RESPONSE));
+		CHECK_YAJL_STATUS(yajl_gen_map_close, gen->gen);
 
 		// The response is small enough, so that it is simpler to copy it
 		// instead of doing a delayed deallocation of the JSON generator.
@@ -55,7 +60,8 @@ static int json_serializer(struct st_h2o_handler_t *self, h2o_req_t *req)
 			return 0;
 
 error_yajl:
-		yajl_gen_free(gen);
+		// If there is a problem with the generator, don't reuse it.
+		free_json_generator(gen, NULL, NULL, 0);
 	}
 
 	send_error(INTERNAL_SERVER_ERROR, REQ_ERROR, req);
@@ -178,20 +184,27 @@ void send_error(http_status_code_t status_code, const char *body, h2o_req_t *req
 	h2o_send_error_generic(req, status_code, status_code_to_string(status_code), body, 0);
 }
 
-int send_json_response(yajl_gen gen, bool free_gen, h2o_req_t *req)
+int send_json_response(json_generator_t *gen, bool free_gen, h2o_req_t *req)
 {
 	const unsigned char *buf;
 	size_t len;
 	int ret = EXIT_FAILURE;
 
-	if (yajl_gen_get_buf(gen, &buf, &len) == yajl_gen_status_ok) {
+	if (yajl_gen_get_buf(gen->gen, &buf, &len) == yajl_gen_status_ok) {
 		set_default_response_param(JSON, len, req);
 		ret = EXIT_SUCCESS;
 
 		if (free_gen) {
+			thread_context_t * const ctx = H2O_STRUCT_FROM_MEMBER(thread_context_t,
+			                                                      event_loop.h2o_ctx,
+			                                                      req->conn->ctx);
+
 			// h2o_send_inline() makes a copy of its input.
 			h2o_send_inline(req, (char *) buf, len);
-			yajl_gen_free(gen);
+			free_json_generator(gen,
+			                    &ctx->json_generator,
+			                    &ctx->json_generator_num,
+			                    ctx->config->max_json_generator);
 		}
 		else {
 			h2o_generator_t generator;

frameworks/C/h2o/src/request_handler.h

@@ -25,6 +25,8 @@
 #include <stdbool.h>
 #include <yajl/yajl_gen.h>
 
+#include "utility.h"
+
 #define REQ_ERROR "request error\n"
 
 typedef enum {
@@ -47,7 +49,7 @@ const char *get_query_param(const char *query,
                             size_t param_len);
 void register_request_handlers(h2o_hostconf_t *hostconf, h2o_access_log_filehandle_t *log_handle);
 void send_error(http_status_code_t status_code, const char *body, h2o_req_t *req);
-int send_json_response(yajl_gen gen, bool free_gen, h2o_req_t *req);
+int send_json_response(json_generator_t *gen, bool free_gen, h2o_req_t *req);
 void send_service_unavailable_error(const char *body, h2o_req_t *req);
 void set_default_response_param(content_type_t content_type,
                                 size_t content_length,

frameworks/C/h2o/src/thread.c

@@ -32,6 +32,7 @@
 #include "error.h"
 #include "event_loop.h"
 #include "thread.h"
+#include "utility.h"
 
 static void *run_thread(void *arg);
 
@@ -50,6 +51,16 @@ void free_thread_context(thread_context_t *ctx)
 {
 	free_database_state(ctx->event_loop.h2o_ctx.loop, &ctx->db_state);
 	free_event_loop(&ctx->event_loop, &ctx->global_thread_data->h2o_receiver);
+
+	if (ctx->json_generator)
+		do {
+			json_generator_t * const gen = H2O_STRUCT_FROM_MEMBER(json_generator_t,
+			                                                      l,
+			                                                      ctx->json_generator);
+
+			ctx->json_generator = gen->l.next;
+			free_json_generator(gen, NULL, NULL, 0);
+		} while (ctx->json_generator);
 }
 
 global_thread_data_t *initialize_global_thread_data(const config_t *config,

frameworks/C/h2o/src/thread.h

@@ -47,6 +47,8 @@ struct thread_context_t {
 	// global_thread_data contains config and global_data as well,
 	// but keep copies here to avoid some pointer chasing.
 	global_thread_data_t *global_thread_data;
+	list_t *json_generator;
+	size_t json_generator_num;
 	unsigned random_seed;
 	pid_t tid;
 	db_state_t db_state;

frameworks/C/h2o/src/utility.c

@@ -18,78 +18,58 @@
 */
 
 #include <assert.h>
-#include <h2o.h>
-#include <limits.h>
-#include <stdalign.h>
 #include <stdint.h>
 #include <stdlib.h>
-#include <string.h>
 #include <yajl/yajl_gen.h>
 
+#include "list.h"
 #include "utility.h"
 
-#define HEADER_SIZE (MAX(sizeof(intmax_t), sizeof(void(*)(void))))
-
-static void mem_pool_free(void *ctx, void *ptr);
-static void *mem_pool_malloc(void *ctx, size_t sz);
-static void *mem_pool_realloc(void *ctx, void *ptr, size_t sz);
-
-static void mem_pool_free(void *ctx, void *ptr)
-{
-	// The memory pool will free all allocations in one go.
-	IGNORE_FUNCTION_PARAMETER(ctx);
-	IGNORE_FUNCTION_PARAMETER(ptr);
-}
-
-static void *mem_pool_malloc(void *ctx, size_t sz)
+void free_json_generator(json_generator_t *gen, list_t **pool, size_t *gen_num, size_t max_gen)
 {
-	void *ret = NULL;
-
-	if (SIZE_MAX - sz >= HEADER_SIZE) {
-		size_t * const p = h2o_mem_alloc_pool(ctx, sz + HEADER_SIZE);
-
-		*p = sz;
-		ret = (char *) p + HEADER_SIZE;
-		// check alignment
-		assert(!(((uintptr_t) ret) & (alignof(intmax_t) - 1)));
-		assert(!(((uintptr_t) ret) & (alignof(void(*)(void)) - 1)));
+	if (gen) {
+		assert(!pool || gen_num);
+
+		if (pool && *gen_num < max_gen) {
+			yajl_gen_reset(gen->gen, NULL);
+			yajl_gen_clear(gen->gen);
+			gen->l.next = *pool;
+			*pool = &gen->l;
+			(*gen_num)++;
+		}
+		else {
+			yajl_gen_free(gen->gen);
+			free(gen);
+		}
 	}
-
-	return ret;
 }
 
-static void *mem_pool_realloc(void *ctx, void *ptr, size_t sz)
+json_generator_t *get_json_generator(list_t **pool, size_t *gen_num)
 {
-	void *ret;
+	json_generator_t *ret;
 
-	if (ptr) {
-		const size_t old_sz = *(const size_t *)((const char *) ptr - HEADER_SIZE);
+	if (pool && *pool) {
+		assert(gen_num && *gen_num);
+		ret = H2O_STRUCT_FROM_MEMBER(json_generator_t, l, *pool);
+		*pool = ret->l.next;
+		(*gen_num)--;
+	}
+	else {
+		ret = malloc(sizeof(*ret));
 
-		if (sz > old_sz) {
-			ret = mem_pool_malloc(ctx, sz);
+		if (ret) {
+			ret->gen = yajl_gen_alloc(NULL);
 
-			if (ret)
-				memcpy(ret, ptr, old_sz);
+			if (!ret->gen) {
+				free(ret);
+				ret = NULL;
+			}
 		}
-		else
-			ret = ptr;
 	}
-	else
-		ret = mem_pool_malloc(ctx, sz);
 
 	return ret;
 }
 
-yajl_gen get_json_generator(h2o_mem_pool_t *pool)
-{
-	const yajl_alloc_funcs mem_pool_alloc_funcs = {mem_pool_malloc,
-	                                               mem_pool_realloc,
-	                                               mem_pool_free,
-	                                               pool};
-
-	return yajl_gen_alloc(&mem_pool_alloc_funcs);
-}
-
 uint32_t get_random_number(uint32_t max_rand, unsigned int *seed)
 {
 	assert(max_rand <= (uint32_t) RAND_MAX);

frameworks/C/h2o/src/utility.h

@@ -28,6 +28,8 @@
 #include <yajl/yajl_gen.h>
 #include <mustache.h>
 
+#include "list.h"
+
 #define ARRAY_SIZE(a) (sizeof(a) / sizeof(*(a)))
 // mainly used to silence compiler warnings about unused function parameters
 #define IGNORE_FUNCTION_PARAMETER(p) ((void) (p))
@@ -50,6 +52,7 @@ typedef struct {
 	const char *template_path;
 	size_t max_accept;
 	size_t max_db_conn_num;
+	size_t max_json_generator;
 	size_t max_query_num;
 	size_t thread_num;
 	uint16_t https_port;
@@ -68,11 +71,13 @@ typedef struct {
 	h2o_globalconf_t h2o_config;
 } global_data_t;
 
-// Call yajl_gen_free() on the result, even though the JSON generator
-// uses a memory pool; in this way the code remains correct if the
-// underlying memory allocator is changed (e.g. for debugging purposes).
-yajl_gen get_json_generator(h2o_mem_pool_t *pool);
+typedef struct {
+	list_t l;
+	yajl_gen gen;
+} json_generator_t;
 
+void free_json_generator(json_generator_t *gen, list_t **pool, size_t *gen_num, size_t max_gen);
+json_generator_t *get_json_generator(list_t **pool, size_t *gen_num);
 uint32_t get_random_number(uint32_t max_rand, unsigned int *seed);
 
 #endif // UTILITY_H_

frameworks/C/h2o/src/world.c

@@ -68,7 +68,7 @@ typedef struct {
 } single_query_ctx_t;
 
 struct multiple_query_ctx_t {
-	yajl_gen gen; // also an error flag
+	json_generator_t *gen; // also an error flag
 	h2o_req_t *req;
 	query_param_t *query_param;
 	size_t num_query;
@@ -95,15 +95,23 @@ static void process_result(PGresult *result, query_result_t *out);
 static int serialize_item(uint32_t id, uint32_t random_number, yajl_gen gen);
 static void serialize_items(const query_result_t *res,
                             size_t num_result,
-                            yajl_gen gen,
+                            json_generator_t **gen,
                             h2o_req_t *req);
 
 static void cleanup_multiple_query(void *data)
 {
 	const multiple_query_ctx_t * const query_ctx = data;
 
-	if (query_ctx->gen)
-		yajl_gen_free(query_ctx->gen);
+	if (query_ctx->gen) {
+		thread_context_t * const ctx = H2O_STRUCT_FROM_MEMBER(thread_context_t,
+		                                                      event_loop.h2o_ctx,
+		                                                      query_ctx->req->conn->ctx);
+
+		free_json_generator(query_ctx->gen,
+		                    &ctx->json_generator,
+		                    &ctx->json_generator_num,
+		                    ctx->config->max_json_generator);
+	}
 }
 
 static int compare_items(const void *x, const void *y)
@@ -184,7 +192,7 @@ static int do_multiple_queries(bool do_update, h2o_req_t *req)
 		}
 
 		// Create a JSON generator while the queries are processed.
-		query_ctx->gen = get_json_generator(&req->pool);
+		query_ctx->gen = get_json_generator(&ctx->json_generator, &ctx->json_generator_num);
 
 		if (!query_ctx->gen)
 			send_error(INTERNAL_SERVER_ERROR, REQ_ERROR, req);
@@ -305,7 +313,14 @@ static void on_multiple_query_error(db_query_param_t *param, const char *error_s
 	multiple_query_ctx_t * const query_ctx = query_param->ctx;
 
 	if (query_ctx->gen) {
-		yajl_gen_free(query_ctx->gen);
+		thread_context_t * const ctx = H2O_STRUCT_FROM_MEMBER(thread_context_t,
+		                                                      event_loop.h2o_ctx,
+		                                                      query_ctx->req->conn->ctx);
+
+		free_json_generator(query_ctx->gen,
+		                    &ctx->json_generator,
+		                    &ctx->json_generator_num,
+		                    ctx->config->max_json_generator);
 		query_ctx->gen = NULL;
 		send_error(BAD_GATEWAY, error_string, query_ctx->req);
 	}
@@ -341,7 +356,10 @@ static result_return_t on_multiple_query_result(db_query_param_t *param, PGresul
 				return DONE;
 			}
 
-			yajl_gen_free(query_ctx->gen);
+			free_json_generator(query_ctx->gen,
+			                    &ctx->json_generator,
+			                    &ctx->json_generator_num,
+			                    ctx->config->max_json_generator);
 			query_ctx->gen = NULL;
 			send_service_unavailable_error(DB_REQ_ERROR, query_ctx->req);
 		}
@@ -349,7 +367,10 @@ static result_return_t on_multiple_query_result(db_query_param_t *param, PGresul
 			if (query_ctx->do_update)
 				do_updates(query_ctx);
 			else
-				serialize_items(query_ctx->res, query_ctx->num_result, query_ctx->gen, query_ctx->req);
+				serialize_items(query_ctx->res,
+				                query_ctx->num_result,
+				                &query_ctx->gen,
+				                query_ctx->req);
 		}
 
 		h2o_mem_release_shared(query_ctx);
@@ -371,7 +392,14 @@ static void on_multiple_query_timeout(db_query_param_t *param)
 	multiple_query_ctx_t * const query_ctx = query_param->ctx;
 
 	if (query_ctx->gen) {
-		yajl_gen_free(query_ctx->gen);
+		thread_context_t * const ctx = H2O_STRUCT_FROM_MEMBER(thread_context_t,
+		                                                      event_loop.h2o_ctx,
+		                                                      query_ctx->req->conn->ctx);
+
+		free_json_generator(query_ctx->gen,
+		                    &ctx->json_generator,
+		                    &ctx->json_generator_num,
+		                    ctx->config->max_json_generator);
 		query_ctx->gen = NULL;
 		send_error(GATEWAY_TIMEOUT, DB_TIMEOUT_ERROR, query_ctx->req);
 	}
@@ -403,16 +431,21 @@ static result_return_t on_single_query_result(db_query_param_t *param, PGresult
 		random_number = ntohl(random_number);
 		PQclear(result);
 
-		const yajl_gen gen = get_json_generator(&query_ctx->req->pool);
+		thread_context_t * const ctx = H2O_STRUCT_FROM_MEMBER(thread_context_t,
+		                                                      event_loop.h2o_ctx,
+		                                                      query_ctx->req->conn->ctx);
+		json_generator_t * const gen = get_json_generator(&ctx->json_generator,
+		                                                  &ctx->json_generator_num);
 
 		if (gen) {
 			// The response is small enough, so that it is simpler to copy it
 			// instead of doing a delayed deallocation of the JSON generator.
-			if (!serialize_item(ntohl(query_ctx->id), random_number, gen) &&
+			if (!serialize_item(ntohl(query_ctx->id), random_number, gen->gen) &&
 			    !send_json_response(gen, true, query_ctx->req))
 				return DONE;
 
-			yajl_gen_free(gen);
+			// If there is a problem with the generator, don't reuse it.
+			free_json_generator(gen, NULL, NULL, 0);
 		}
 
 		send_error(INTERNAL_SERVER_ERROR, REQ_ERROR, query_ctx->req);
@@ -440,7 +473,7 @@ static result_return_t on_update_result(db_query_param_t *param, PGresult *resul
 
 	if (PQresultStatus(result) == PGRES_COMMAND_OK) {
 		query_ctx->num_query_in_progress--;
-		serialize_items(query_ctx->res, query_ctx->num_result, query_ctx->gen, query_ctx->req);
+		serialize_items(query_ctx->res, query_ctx->num_result, &query_ctx->gen, query_ctx->req);
 		h2o_mem_release_shared(query_ctx);
 	}
 	else {
@@ -495,21 +528,24 @@ error_yajl:
 
 static void serialize_items(const query_result_t *res,
                             size_t num_result,
-                            yajl_gen gen,
+                            json_generator_t **gen,
                             h2o_req_t *req)
 {
-	CHECK_YAJL_STATUS(yajl_gen_array_open, gen);
+	CHECK_YAJL_STATUS(yajl_gen_array_open, (*gen)->gen);
 
 	for (size_t i = 0; i < num_result; i++)
-		if (serialize_item(res[i].id, res[i].random_number, gen))
+		if (serialize_item(res[i].id, res[i].random_number, (*gen)->gen))
 			goto error_yajl;
 
-	CHECK_YAJL_STATUS(yajl_gen_array_close, gen);
+	CHECK_YAJL_STATUS(yajl_gen_array_close, (*gen)->gen);
 
-	if (!send_json_response(gen, false, req))
+	if (!send_json_response(*gen, false, req))
 		return;
 
 error_yajl:
+	// If there is a problem with the generator, don't reuse it.
+	free_json_generator(*gen, NULL, NULL, 0);
+	*gen = NULL;
 	send_error(INTERNAL_SERVER_ERROR, REQ_ERROR, req);
 }