Merge pull request #38944 from Wavesonics/http-gzip

HttpRequest now handles gzipping response bodies

core/io/compression.cpp

@@ -180,8 +180,95 @@ int Compression::decompress(uint8_t *p_dst, int p_dst_max_size, const uint8_t *p
 	ERR_FAIL_V(-1);
 }
 
+/**
+	This will handle both Gzip and Deflat streams. It will automatically allocate the output buffer into the provided p_dst_vect Vector.
+	This is required for compressed data who's final uncompressed size is unknown, as is the case for HTTP response bodies.
+	This is much slower however than using Compression::decompress because it may result in multiple full copies of the output buffer.
+*/
+int Compression::decompress_dynamic(Vector<uint8_t> *p_dst_vect, int p_max_dst_size, const uint8_t *p_src, int p_src_size, Mode p_mode) {
+	int ret;
+	uint8_t *dst = nullptr;
+	int out_mark = 0;
+	z_stream strm;
+
+	ERR_FAIL_COND_V(p_src_size <= 0, Z_DATA_ERROR);
+
+	// This function only supports GZip and Deflate
+	int window_bits = p_mode == MODE_DEFLATE ? 15 : 15 + 16;
+	ERR_FAIL_COND_V(p_mode != MODE_DEFLATE && p_mode != MODE_GZIP, Z_ERRNO);
+
+	// Initialize the stream
+	strm.zalloc = Z_NULL;
+	strm.zfree = Z_NULL;
+	strm.opaque = Z_NULL;
+	strm.avail_in = 0;
+	strm.next_in = Z_NULL;
+
+	int err = inflateInit2(&strm, window_bits);
+	ERR_FAIL_COND_V(err != Z_OK, -1);
+
+	// Setup the stream inputs
+	strm.next_in = (Bytef *)p_src;
+	strm.avail_in = p_src_size;
+
+	// Ensure the destination buffer is empty
+	p_dst_vect->resize(0);
+
+	// decompress until deflate stream ends or end of file
+	do {
+		// Add another chunk size to the output buffer
+		// This forces a copy of the whole buffer
+		p_dst_vect->resize(p_dst_vect->size() + gzip_chunk);
+		// Get pointer to the actual output buffer
+		dst = p_dst_vect->ptrw();
+
+		// Set the stream to the new output stream
+		// Since it was copied, we need to reset the stream to the new buffer
+		strm.next_out = &(dst[out_mark]);
+		strm.avail_out = gzip_chunk;
+
+		// run inflate() on input until output buffer is full and needs to be resized
+		// or input runs out
+		do {
+			ret = inflate(&strm, Z_SYNC_FLUSH);
+
+			switch (ret) {
+				case Z_NEED_DICT:
+					ret = Z_DATA_ERROR;
+					[[fallthrough]];
+				case Z_DATA_ERROR:
+				case Z_MEM_ERROR:
+				case Z_STREAM_ERROR:
+					WARN_PRINT(strm.msg);
+					(void)inflateEnd(&strm);
+					p_dst_vect->resize(0);
+					return ret;
+			}
+		} while (strm.avail_out > 0 && strm.avail_in > 0);
+
+		out_mark += gzip_chunk;
+
+		// Encorce max output size
+		if (p_max_dst_size > -1 && strm.total_out > (uint64_t)p_max_dst_size) {
+			(void)inflateEnd(&strm);
+			p_dst_vect->resize(0);
+			return Z_BUF_ERROR;
+		}
+	} while (ret != Z_STREAM_END);
+
+	// If all done successfully, resize the output if it's larger than the actual output
+	if (ret == Z_STREAM_END && (unsigned long)p_dst_vect->size() > strm.total_out) {
+		p_dst_vect->resize(strm.total_out);
+	}
+
+	// clean up and return
+	(void)inflateEnd(&strm);
+	return ret == Z_STREAM_END ? Z_OK : Z_DATA_ERROR;
+}
+
 int Compression::zlib_level = Z_DEFAULT_COMPRESSION;
 int Compression::gzip_level = Z_DEFAULT_COMPRESSION;
 int Compression::zstd_level = 3;
 bool Compression::zstd_long_distance_matching = false;
 int Compression::zstd_window_log_size = 27; // ZSTD_WINDOWLOG_LIMIT_DEFAULT
+int Compression::gzip_chunk = 16384;

core/io/compression.h

@@ -32,6 +32,7 @@
 #define COMPRESSION_H
 
 #include "core/typedefs.h"
+#include "core/vector.h"
 
 class Compression {
 public:
@@ -40,6 +41,7 @@ public:
 	static int zstd_level;
 	static bool zstd_long_distance_matching;
 	static int zstd_window_log_size;
+	static int gzip_chunk;
 
 	enum Mode {
 		MODE_FASTLZ,
@@ -51,6 +53,7 @@ public:
 	static int compress(uint8_t *p_dst, const uint8_t *p_src, int p_src_size, Mode p_mode = MODE_ZSTD);
 	static int get_max_compressed_buffer_size(int p_src_size, Mode p_mode = MODE_ZSTD);
 	static int decompress(uint8_t *p_dst, int p_dst_max_size, const uint8_t *p_src, int p_src_size, Mode p_mode = MODE_ZSTD);
+	static int decompress_dynamic(Vector<uint8_t> *p_dst_vect, int p_max_dst_size, const uint8_t *p_src, int p_src_size, Mode p_mode);
 
 	Compression() {}
 };

core/variant_call.cpp

@@ -712,6 +712,23 @@ struct _VariantCall {
 		r_ret = decompressed;
 	}
 
+	static void _call_PackedByteArray_decompress_dynamic(Variant &r_ret, Variant &p_self, const Variant **p_args) {
+		Variant::PackedArrayRef<uint8_t> *ba = reinterpret_cast<Variant::PackedArrayRef<uint8_t> *>(p_self._data.packed_array);
+		PackedByteArray decompressed;
+		int max_output_size = (int)(*p_args[0]);
+		Compression::Mode mode = (Compression::Mode)(int)(*p_args[1]);
+
+		int result = Compression::decompress_dynamic(&decompressed, max_output_size, ba->array.ptr(), ba->array.size(), mode);
+
+		if (result == OK) {
+			r_ret = decompressed;
+		} else {
+			decompressed.clear();
+			r_ret = decompressed;
+			ERR_FAIL_MSG("Decompression failed.");
+		}
+	}
+
 	static void _call_PackedByteArray_hex_encode(Variant &r_ret, Variant &p_self, const Variant **p_args) {
 		Variant::PackedArrayRef<uint8_t> *ba = reinterpret_cast<Variant::PackedArrayRef<uint8_t> *>(p_self._data.packed_array);
 		if (ba->array.size() == 0) {
@@ -2177,6 +2194,7 @@ void register_variant_methods() {
 	ADDFUNC0R(PACKED_BYTE_ARRAY, STRING, PackedByteArray, hex_encode, varray());
 	ADDFUNC1R(PACKED_BYTE_ARRAY, PACKED_BYTE_ARRAY, PackedByteArray, compress, INT, "compression_mode", varray(0));
 	ADDFUNC2R(PACKED_BYTE_ARRAY, PACKED_BYTE_ARRAY, PackedByteArray, decompress, INT, "buffer_size", INT, "compression_mode", varray(0));
+	ADDFUNC2R(PACKED_BYTE_ARRAY, PACKED_BYTE_ARRAY, PackedByteArray, decompress_dynamic, INT, "max_output_size", INT, "compression_mode", varray(0));
 
 	ADDFUNC0R(PACKED_INT32_ARRAY, INT, PackedInt32Array, size, varray());
 	ADDFUNC0R(PACKED_INT32_ARRAY, BOOL, PackedInt32Array, empty, varray());

doc/classes/HTTPRequest.xml

@@ -64,6 +64,11 @@
 		    add_child(texture_rect)
 		    texture_rect.texture = texture
 		[/codeblock]
+
+		[b]Gzipped response bodies[/b]
+		HttpRequest will automatically handle decompression of response bodies.
+		A "Accept-Encoding" header will be automatically added to each of your requests, unless one is already specified.
+		Any response with a "Content-Encoding: gzip" header will automatically be decompressed and delivered to you as a uncompressed bytes.
 		[b]Note:[/b] When performing HTTP requests from a project exported to HTML5, keep in mind the remote server may not allow requests from foreign origins due to [url=https://developer.mozilla.org/en-US/docs/Web/HTTP/CORS]CORS[/url]. If you host the server in question, you should modify its backend to allow requests from foreign origins by adding the [code]Access-Control-Allow-Origin: *[/code] HTTP header.
 	</description>
 	<tutorials>
@@ -119,10 +124,28 @@
 				[b]Note:[/b] The [code]request_data[/code] parameter is ignored if [code]method[/code] is [constant HTTPClient.METHOD_GET]. This is because GET methods can't contain request data. As a workaround, you can pass request data as a query string in the URL. See [method String.http_escape] for an example.
 			</description>
 		</method>
+		<method name="request_raw">
+			<return type="int" enum="Error">
+			</return>
+			<argument index="0" name="url" type="String">
+			</argument>
+			<argument index="1" name="custom_headers" type="PackedStringArray" default="PackedStringArray(  )">
+			</argument>
+			<argument index="2" name="ssl_validate_domain" type="bool" default="true">
+			</argument>
+			<argument index="3" name="method" type="int" enum="HTTPClient.Method" default="0">
+			</argument>
+			<argument index="4" name="request_data_raw" type="PackedByteArray" default="PackedByteArray()">
+			</argument>
+			<description>
+				Creates request on the underlying [HTTPClient] using a raw array of bytes for the request body. If there is no configuration errors, it tries to connect using [method HTTPClient.connect_to_host] and passes parameters onto [method HTTPClient.request].
+				Returns [constant OK] if request is successfully created. (Does not imply that the server has responded), [constant ERR_UNCONFIGURED] if not in the tree, [constant ERR_BUSY] if still processing previous request, [constant ERR_INVALID_PARAMETER] if given string is not a valid URL format, or [constant ERR_CANT_CONNECT] if not using thread and the [HTTPClient] cannot connect to host.
+			</description>
+		</method>
 	</methods>
 	<members>
 		<member name="body_size_limit" type="int" setter="set_body_size_limit" getter="get_body_size_limit" default="-1">
-			Maximum allowed size for response bodies.
+			Maximum allowed size for response bodies. If the response body is compressed, this will be used as the maximum allowed size for the decompressed body.
 		</member>
 		<member name="download_chunk_size" type="int" setter="set_download_chunk_size" getter="get_download_chunk_size" default="4096">
 			The size of the buffer used and maximum bytes to read per iteration. See [member HTTPClient.read_chunk_size].
@@ -139,6 +162,12 @@
 		<member name="use_threads" type="bool" setter="set_use_threads" getter="is_using_threads" default="false">
 			If [code]true[/code], multithreading is used to improve performance.
 		</member>
+		<member name="accept_gzip" type="bool" setter="set_accept_gzip" getter="is_accepting_gzip" default="true">
+			If [code]true[/code], this header will be added to each request: [code]Accept-Encoding: gzip, deflate[/code] telling servers that it's okay to compress response bodies.
+			Any Reponse body declaring a [code]Content-Encoding[/code] of either [code]gzip[/code] or [code]deflate[/code] will then be automatically decompressed, and the uncompressed bytes will be delivered via [code]request_completed[/code].
+			If the user has specified their own [code]Accept-Encoding[/code] header, then no header will be added regaurdless of [code]accept_gzip[/code].
+			If [code]false[/code] no header will be added, and no decompression will be performed on response bodies. The raw bytes of the response body will be returned via [code]request_completed[/code].
+		</member>
 	</members>
 	<signals>
 		<signal name="request_completed">

doc/classes/PackedByteArray.xml

@@ -57,6 +57,20 @@
 				Returns a new [PackedByteArray] with the data decompressed. Set [code]buffer_size[/code] to the size of the uncompressed data. Set the compression mode using one of [enum File.CompressionMode]'s constants.
 			</description>
 		</method>
+		<method name="decompress_dynamic">
+			<return type="PackedByteArray">
+			</return>
+			<argument index="0" name="max_output_size" type="int" default="0">
+			</argument>
+			<argument index="1" name="compression_mode" type="int" default="0">
+			</argument>
+			<description>
+				Returns a new [PackedByteArray] with the data decompressed. Set the compression mode using one of [enum File.CompressionMode]'s constants. [b]This method only accepts gzip and deflate compression modes.[/b]
+				This method is potentially slower than [code]decompress[/code], as it may have to re-allocate it's output buffer multiple times while decompressing, where as [code]decompress[/code] knows it's output buffer size from the begining.
+
+				GZIP has a maximal compression ratio of 1032:1, meaning it's very possible for a small compressed payload to decompress to a potentially very large output. To guard against this, you may provide a maximum size this function is allowed to allocate in bytes via [code]max_output_size[/code]. Passing -1 will allow for unbounded output. If any positive value is passed, and the decompression exceeds that ammount in bytes, then an error will be returned.
+			</description>
+		</method>
 		<method name="empty">
 			<return type="bool">
 			</return>

scene/main/http_request.cpp

@@ -29,6 +29,8 @@
 /*************************************************************************/
 
 #include "http_request.h"
+#include "core/io/compression.h"
+#include "core/ustring.h"
 
 void HTTPRequest::_redirect_request(const String &p_new_url) {
 }
@@ -82,7 +84,51 @@ Error HTTPRequest::_parse_url(const String &p_url) {
 	return OK;
 }
 
+bool HTTPRequest::has_header(const PackedStringArray &p_headers, const String &p_header_name) {
+	bool exists = false;
+
+	String lower_case_header_name = p_header_name.to_lower();
+	for (int i = 0; i < p_headers.size() && !exists; i++) {
+		String sanitized = p_headers[i].strip_edges().to_lower();
+		if (sanitized.begins_with(lower_case_header_name)) {
+			exists = true;
+		}
+	}
+
+	return exists;
+}
+
+String HTTPRequest::get_header_value(const PackedStringArray &p_headers, const String &p_header_name) {
+	String value = "";
+
+	String lowwer_case_header_name = p_header_name.to_lower();
+	for (int i = 0; i < p_headers.size(); i++) {
+		if (p_headers[i].find(":", 0) >= 0) {
+			Vector<String> parts = p_headers[i].split(":", false, 1);
+			if (parts[0].strip_edges().to_lower() == lowwer_case_header_name) {
+				value = parts[1].strip_edges();
+				break;
+			}
+		}
+	}
+
+	return value;
+}
+
 Error HTTPRequest::request(const String &p_url, const Vector<String> &p_custom_headers, bool p_ssl_validate_domain, HTTPClient::Method p_method, const String &p_request_data) {
+	// Copy the string into a raw buffer
+	Vector<uint8_t> raw_data;
+
+	CharString charstr = p_request_data.utf8();
+	size_t len = charstr.length();
+	raw_data.resize(len);
+	uint8_t *w = raw_data.ptrw();
+	copymem(w, charstr.ptr(), len);
+
+	return request_raw(p_url, p_custom_headers, p_ssl_validate_domain, p_method, raw_data);
+}
+
+Error HTTPRequest::request_raw(const String &p_url, const Vector<String> &p_custom_headers, bool p_ssl_validate_domain, HTTPClient::Method p_method, const Vector<uint8_t> &p_request_data_raw) {
 	ERR_FAIL_COND_V(!is_inside_tree(), ERR_UNCONFIGURED);
 	ERR_FAIL_COND_V_MSG(requesting, ERR_BUSY, "HTTPRequest is processing a request. Wait for completion or cancel it before attempting a new one.");
 
@@ -102,7 +148,14 @@ Error HTTPRequest::request(const String &p_url, const Vector<String> &p_custom_h
 
 	headers = p_custom_headers;
 
-	request_data = p_request_data;
+	if (accept_gzip) {
+		// If the user has specified a different Accept-Encoding, don't overwrite it
+		if (!has_header(headers, "Accept-Encoding")) {
+			headers.push_back("Accept-Encoding: gzip, deflate");
+		}
+	}
+
+	request_data = p_request_data_raw;
 
 	requesting = true;
 
@@ -288,7 +341,7 @@ bool HTTPRequest::_update_connection() {
 			} else {
 				// Did not request yet, do request
 
-				Error err = client->request(method, request_string, headers, request_data);
+				Error err = client->request_raw(method, request_string, headers, request_data);
 				if (err != OK) {
 					call_deferred("_request_done", RESULT_CONNECTION_ERROR, 0, PackedStringArray(), PackedByteArray());
 					return true;
@@ -382,9 +435,47 @@ bool HTTPRequest::_update_connection() {
 	ERR_FAIL_V(false);
 }
 
-void HTTPRequest::_request_done(int p_status, int p_code, const PackedStringArray &headers, const PackedByteArray &p_data) {
+void HTTPRequest::_request_done(int p_status, int p_code, const PackedStringArray &p_headers, const PackedByteArray &p_data) {
 	cancel_request();
-	emit_signal("request_completed", p_status, p_code, headers, p_data);
+
+	// Determine if the request body is compressed
+	bool is_compressed;
+	String content_encoding = get_header_value(p_headers, "Content-Encoding").to_lower();
+	Compression::Mode mode;
+	if (content_encoding == "gzip") {
+		mode = Compression::Mode::MODE_GZIP;
+		is_compressed = true;
+	} else if (content_encoding == "deflate") {
+		mode = Compression::Mode::MODE_DEFLATE;
+		is_compressed = true;
+	} else {
+		is_compressed = false;
+	}
+
+	const PackedByteArray *data = NULL;
+
+	if (accept_gzip && is_compressed && p_data.size() > 0) {
+		// Decompress request body
+		PackedByteArray *decompressed = memnew(PackedByteArray);
+		int result = Compression::decompress_dynamic(decompressed, body_size_limit, p_data.ptr(), p_data.size(), mode);
+		if (result == OK) {
+			data = decompressed;
+		} else if (result == -5) {
+			WARN_PRINT("Decompressed size of HTTP response body exceeded body_size_limit");
+			p_status = RESULT_BODY_SIZE_LIMIT_EXCEEDED;
+			// Just return the raw data if we failed to decompress it
+			data = &p_data;
+		} else {
+			WARN_PRINT("Failed to decompress HTTP response body");
+			p_status = RESULT_BODY_DECOMPRESS_FAILED;
+			// Just return the raw data if we failed to decompress it
+			data = &p_data;
+		}
+	} else {
+		data = &p_data;
+	}
+
+	emit_signal("request_completed", p_status, p_code, p_headers, *data);
 }
 
 void HTTPRequest::_notification(int p_what) {
@@ -415,6 +506,14 @@ bool HTTPRequest::is_using_threads() const {
 	return use_threads;
 }
 
+void HTTPRequest::set_accept_gzip(bool p_gzip) {
+	accept_gzip = p_gzip;
+}
+
+bool HTTPRequest::is_accepting_gzip() const {
+	return accept_gzip;
+}
+
 void HTTPRequest::set_body_size_limit(int p_bytes) {
 	ERR_FAIL_COND(get_http_client_status() != HTTPClient::STATUS_DISCONNECTED);
 
@@ -481,6 +580,7 @@ void HTTPRequest::_timeout() {
 
 void HTTPRequest::_bind_methods() {
 	ClassDB::bind_method(D_METHOD("request", "url", "custom_headers", "ssl_validate_domain", "method", "request_data"), &HTTPRequest::request, DEFVAL(PackedStringArray()), DEFVAL(true), DEFVAL(HTTPClient::METHOD_GET), DEFVAL(String()));
+	ClassDB::bind_method(D_METHOD("request_raw", "url", "custom_headers", "ssl_validate_domain", "method", "request_data_raw"), &HTTPRequest::request_raw, DEFVAL(PackedStringArray()), DEFVAL(true), DEFVAL(HTTPClient::METHOD_GET), DEFVAL(PackedByteArray()));
 	ClassDB::bind_method(D_METHOD("cancel_request"), &HTTPRequest::cancel_request);
 
 	ClassDB::bind_method(D_METHOD("get_http_client_status"), &HTTPRequest::get_http_client_status);
@@ -488,6 +588,9 @@ void HTTPRequest::_bind_methods() {
 	ClassDB::bind_method(D_METHOD("set_use_threads", "enable"), &HTTPRequest::set_use_threads);
 	ClassDB::bind_method(D_METHOD("is_using_threads"), &HTTPRequest::is_using_threads);
 
+	ClassDB::bind_method(D_METHOD("set_accept_gzip", "enable"), &HTTPRequest::set_accept_gzip);
+	ClassDB::bind_method(D_METHOD("is_accepting_gzip"), &HTTPRequest::is_accepting_gzip);
+
 	ClassDB::bind_method(D_METHOD("set_body_size_limit", "bytes"), &HTTPRequest::set_body_size_limit);
 	ClassDB::bind_method(D_METHOD("get_body_size_limit"), &HTTPRequest::get_body_size_limit);
 
@@ -512,6 +615,7 @@ void HTTPRequest::_bind_methods() {
 	ADD_PROPERTY(PropertyInfo(Variant::STRING, "download_file", PROPERTY_HINT_FILE), "set_download_file", "get_download_file");
 	ADD_PROPERTY(PropertyInfo(Variant::INT, "download_chunk_size", PROPERTY_HINT_RANGE, "256,16777216"), "set_download_chunk_size", "get_download_chunk_size");
 	ADD_PROPERTY(PropertyInfo(Variant::BOOL, "use_threads"), "set_use_threads", "is_using_threads");
+	ADD_PROPERTY(PropertyInfo(Variant::BOOL, "accept_gzip"), "set_accept_gzip", "is_accepting_gzip");
 	ADD_PROPERTY(PropertyInfo(Variant::INT, "body_size_limit", PROPERTY_HINT_RANGE, "-1,2000000000"), "set_body_size_limit", "get_body_size_limit");
 	ADD_PROPERTY(PropertyInfo(Variant::INT, "max_redirects", PROPERTY_HINT_RANGE, "-1,64"), "set_max_redirects", "get_max_redirects");
 	ADD_PROPERTY(PropertyInfo(Variant::INT, "timeout", PROPERTY_HINT_RANGE, "0,86400"), "set_timeout", "get_timeout");
@@ -544,6 +648,7 @@ HTTPRequest::HTTPRequest() {
 	got_response = false;
 	validate_ssl = false;
 	use_ssl = false;
+	accept_gzip = true;
 	response_code = 0;
 	request_sent = false;
 	requesting = false;

scene/main/http_request.h

@@ -50,6 +50,7 @@ public:
 		RESULT_SSL_HANDSHAKE_ERROR,
 		RESULT_NO_RESPONSE,
 		RESULT_BODY_SIZE_LIMIT_EXCEEDED,
+		RESULT_BODY_DECOMPRESS_FAILED,
 		RESULT_REQUEST_FAILED,
 		RESULT_DOWNLOAD_FILE_CANT_OPEN,
 		RESULT_DOWNLOAD_FILE_WRITE_ERROR,
@@ -68,12 +69,13 @@ private:
 	bool validate_ssl;
 	bool use_ssl;
 	HTTPClient::Method method;
-	String request_data;
+	Vector<uint8_t> request_data;
 
 	bool request_sent;
 	Ref<HTTPClient> client;
 	PackedByteArray body;
 	volatile bool use_threads;
+	bool accept_gzip;
 
 	bool got_response;
 	int response_code;
@@ -102,12 +104,15 @@ private:
 	Error _parse_url(const String &p_url);
 	Error _request();
 
+	bool has_header(const PackedStringArray &p_headers, const String &p_header_name);
+	String get_header_value(const PackedStringArray &p_headers, const String &header_name);
+
 	volatile bool thread_done;
 	volatile bool thread_request_quit;
 
 	Thread *thread;
 
-	void _request_done(int p_status, int p_code, const PackedStringArray &headers, const PackedByteArray &p_data);
+	void _request_done(int p_status, int p_code, const PackedStringArray &p_headers, const PackedByteArray &p_data);
 	static void _thread_func(void *p_userdata);
 
 protected:
@@ -116,12 +121,16 @@ protected:
 
 public:
 	Error request(const String &p_url, const Vector<String> &p_custom_headers = Vector<String>(), bool p_ssl_validate_domain = true, HTTPClient::Method p_method = HTTPClient::METHOD_GET, const String &p_request_data = ""); //connects to a full url and perform request
+	Error request_raw(const String &p_url, const Vector<String> &p_custom_headers = Vector<String>(), bool p_ssl_validate_domain = true, HTTPClient::Method p_method = HTTPClient::METHOD_GET, const Vector<uint8_t> &p_request_data_raw = Vector<uint8_t>()); //connects to a full url and perform request
 	void cancel_request();
 	HTTPClient::Status get_http_client_status() const;
 
 	void set_use_threads(bool p_use);
 	bool is_using_threads() const;
 
+	void set_accept_gzip(bool p_gzip);
+	bool is_accepting_gzip() const;
+
 	void set_download_file(const String &p_file);
 	String get_download_file() const;