Merge pull request #63740 from Faless/net/4.x_http_request_decompress

[HTTP] Implement streaming decompression.

core/io/stream_peer_gzip.cpp

@@ -0,0 +1,209 @@
+/*************************************************************************/
+/*  stream_peer_gzip.cpp                                                 */
+/*************************************************************************/
+/*                       This file is part of:                           */
+/*                           GODOT ENGINE                                */
+/*                      https://godotengine.org                          */
+/*************************************************************************/
+/* Copyright (c) 2007-2022 Juan Linietsky, Ariel Manzur.                 */
+/* Copyright (c) 2014-2022 Godot Engine contributors (cf. AUTHORS.md).   */
+/*                                                                       */
+/* Permission is hereby granted, free of charge, to any person obtaining */
+/* a copy of this software and associated documentation files (the       */
+/* "Software"), to deal in the Software without restriction, including   */
+/* without limitation the rights to use, copy, modify, merge, publish,   */
+/* distribute, sublicense, and/or sell copies of the Software, and to    */
+/* permit persons to whom the Software is furnished to do so, subject to */
+/* the following conditions:                                             */
+/*                                                                       */
+/* The above copyright notice and this permission notice shall be        */
+/* included in all copies or substantial portions of the Software.       */
+/*                                                                       */
+/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,       */
+/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF    */
+/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*/
+/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY  */
+/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,  */
+/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE     */
+/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.                */
+/*************************************************************************/
+
+#include "core/io/stream_peer_gzip.h"
+
+#include "core/io/zip_io.h"
+#include <zlib.h>
+
+void StreamPeerGZIP::_bind_methods() {
+	ClassDB::bind_method(D_METHOD("start_compression", "use_deflate", "buffer_size"), &StreamPeerGZIP::start_compression, DEFVAL(false), DEFVAL(65535));
+	ClassDB::bind_method(D_METHOD("start_decompression", "use_deflate", "buffer_size"), &StreamPeerGZIP::start_decompression, DEFVAL(false), DEFVAL(65535));
+	ClassDB::bind_method(D_METHOD("finish"), &StreamPeerGZIP::finish);
+	ClassDB::bind_method(D_METHOD("clear"), &StreamPeerGZIP::clear);
+}
+
+StreamPeerGZIP::StreamPeerGZIP() {
+}
+
+StreamPeerGZIP::~StreamPeerGZIP() {
+	_close();
+}
+
+void StreamPeerGZIP::_close() {
+	if (ctx) {
+		z_stream *strm = (z_stream *)ctx;
+		if (compressing) {
+			deflateEnd(strm);
+		} else {
+			inflateEnd(strm);
+		}
+		memfree(strm);
+		ctx = nullptr;
+	}
+}
+
+void StreamPeerGZIP::clear() {
+	_close();
+	rb.clear();
+	buffer.clear();
+}
+
+Error StreamPeerGZIP::start_compression(bool p_is_deflate, int buffer_size) {
+	return _start(true, p_is_deflate, buffer_size);
+}
+
+Error StreamPeerGZIP::start_decompression(bool p_is_deflate, int buffer_size) {
+	return _start(false, p_is_deflate, buffer_size);
+}
+
+Error StreamPeerGZIP::_start(bool p_compress, bool p_is_deflate, int buffer_size) {
+	ERR_FAIL_COND_V(ctx != nullptr, ERR_ALREADY_IN_USE);
+	clear();
+	compressing = p_compress;
+	rb.resize(nearest_shift(buffer_size - 1));
+	buffer.resize(1024);
+
+	// Create ctx.
+	ctx = memalloc(sizeof(z_stream));
+	z_stream &strm = *(z_stream *)ctx;
+	strm.next_in = Z_NULL;
+	strm.avail_in = 0;
+	strm.zalloc = zipio_alloc;
+	strm.zfree = zipio_free;
+	strm.opaque = Z_NULL;
+	int window_bits = p_is_deflate ? 15 : (15 + 16);
+	int err = Z_OK;
+	int level = Z_DEFAULT_COMPRESSION;
+	if (compressing) {
+		err = deflateInit2(&strm, level, Z_DEFLATED, window_bits, 8, Z_DEFAULT_STRATEGY);
+	} else {
+		err = inflateInit2(&strm, window_bits);
+	}
+	ERR_FAIL_COND_V(err != Z_OK, FAILED);
+	return OK;
+}
+
+Error StreamPeerGZIP::_process(uint8_t *p_dst, int p_dst_size, const uint8_t *p_src, int p_src_size, int &r_consumed, int &r_out, bool p_close) {
+	ERR_FAIL_COND_V(!ctx, ERR_UNCONFIGURED);
+	z_stream &strm = *(z_stream *)ctx;
+	strm.avail_in = p_src_size;
+	strm.avail_out = p_dst_size;
+	strm.next_in = (Bytef *)p_src;
+	strm.next_out = (Bytef *)p_dst;
+	int flush = p_close ? Z_FINISH : Z_NO_FLUSH;
+	if (compressing) {
+		int err = deflate(&strm, flush);
+		ERR_FAIL_COND_V(err != (p_close ? Z_STREAM_END : Z_OK), FAILED);
+	} else {
+		int err = inflate(&strm, flush);
+		ERR_FAIL_COND_V(err != Z_OK && err != Z_STREAM_END, FAILED);
+	}
+	r_out = p_dst_size - strm.avail_out;
+	r_consumed = p_src_size - strm.avail_in;
+	return OK;
+}
+
+Error StreamPeerGZIP::put_data(const uint8_t *p_data, int p_bytes) {
+	int wrote = 0;
+	Error err = put_partial_data(p_data, p_bytes, wrote);
+	if (err != OK) {
+		return err;
+	}
+	ERR_FAIL_COND_V(p_bytes != wrote, ERR_OUT_OF_MEMORY);
+	return OK;
+}
+
+Error StreamPeerGZIP::put_partial_data(const uint8_t *p_data, int p_bytes, int &r_sent) {
+	ERR_FAIL_COND_V(!ctx, ERR_UNCONFIGURED);
+	ERR_FAIL_COND_V(p_bytes < 0, ERR_INVALID_PARAMETER);
+
+	// Ensure we have enough space in temporary buffer.
+	if (buffer.size() < p_bytes) {
+		buffer.resize(p_bytes);
+	}
+
+	r_sent = 0;
+	while (r_sent < p_bytes && rb.space_left() > 1024) { // Keep the ring buffer size meaningful.
+		int sent = 0;
+		int to_write = 0;
+		// Compress or decompress
+		Error err = _process(buffer.ptrw(), MIN(buffer.size(), rb.space_left()), p_data + r_sent, p_bytes - r_sent, sent, to_write);
+		if (err != OK) {
+			return err;
+		}
+		// When decompressing, we might need to do another round.
+		r_sent += sent;
+
+		// We can't write more than this buffer is full.
+		if (sent == 0 && to_write == 0) {
+			return OK;
+		}
+		if (to_write) {
+			// Copy to ring buffer.
+			int wrote = rb.write(buffer.ptr(), to_write);
+			ERR_FAIL_COND_V(wrote != to_write, ERR_BUG);
+		}
+	}
+	return OK;
+}
+
+Error StreamPeerGZIP::get_data(uint8_t *p_buffer, int p_bytes) {
+	int received = 0;
+	Error err = get_partial_data(p_buffer, p_bytes, received);
+	if (err != OK) {
+		return err;
+	}
+	ERR_FAIL_COND_V(p_bytes != received, ERR_UNAVAILABLE);
+	return OK;
+}
+
+Error StreamPeerGZIP::get_partial_data(uint8_t *p_buffer, int p_bytes, int &r_received) {
+	ERR_FAIL_COND_V(p_bytes < 0, ERR_INVALID_PARAMETER);
+
+	r_received = MIN(p_bytes, rb.data_left());
+	if (r_received == 0) {
+		return OK;
+	}
+	int received = rb.read(p_buffer, r_received);
+	ERR_FAIL_COND_V(received != r_received, ERR_BUG);
+	return OK;
+}
+
+int StreamPeerGZIP::get_available_bytes() const {
+	return rb.data_left();
+}
+
+Error StreamPeerGZIP::finish() {
+	ERR_FAIL_COND_V(!ctx || !compressing, ERR_UNAVAILABLE);
+	// Ensure we have enough space in temporary buffer.
+	if (buffer.size() < 1024) {
+		buffer.resize(1024); // 1024 should be more than enough.
+	}
+	int consumed = 0;
+	int to_write = 0;
+	Error err = _process(buffer.ptrw(), 1024, nullptr, 0, consumed, to_write, true); // compress
+	if (err != OK) {
+		return err;
+	}
+	int wrote = rb.write(buffer.ptr(), to_write);
+	ERR_FAIL_COND_V(wrote != to_write, ERR_OUT_OF_MEMORY);
+	return OK;
+}

core/io/stream_peer_gzip.h

@@ -0,0 +1,76 @@
+/*************************************************************************/
+/*  stream_peer_gzip.h                                                   */
+/*************************************************************************/
+/*                       This file is part of:                           */
+/*                           GODOT ENGINE                                */
+/*                      https://godotengine.org                          */
+/*************************************************************************/
+/* Copyright (c) 2007-2022 Juan Linietsky, Ariel Manzur.                 */
+/* Copyright (c) 2014-2022 Godot Engine contributors (cf. AUTHORS.md).   */
+/*                                                                       */
+/* Permission is hereby granted, free of charge, to any person obtaining */
+/* a copy of this software and associated documentation files (the       */
+/* "Software"), to deal in the Software without restriction, including   */
+/* without limitation the rights to use, copy, modify, merge, publish,   */
+/* distribute, sublicense, and/or sell copies of the Software, and to    */
+/* permit persons to whom the Software is furnished to do so, subject to */
+/* the following conditions:                                             */
+/*                                                                       */
+/* The above copyright notice and this permission notice shall be        */
+/* included in all copies or substantial portions of the Software.       */
+/*                                                                       */
+/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,       */
+/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF    */
+/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*/
+/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY  */
+/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,  */
+/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE     */
+/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.                */
+/*************************************************************************/
+
+#ifndef STREAM_PEER_GZIP_H
+#define STREAM_PEER_GZIP_H
+
+#include "core/io/stream_peer.h"
+
+#include "core/core_bind.h"
+#include "core/io/compression.h"
+#include "core/templates/ring_buffer.h"
+
+class StreamPeerGZIP : public StreamPeer {
+	GDCLASS(StreamPeerGZIP, StreamPeer);
+
+private:
+	void *ctx = nullptr; // Will hold our z_stream instance.
+	bool compressing = true;
+
+	RingBuffer<uint8_t> rb;
+	Vector<uint8_t> buffer;
+
+	Error _process(uint8_t *p_dst, int p_dst_size, const uint8_t *p_src, int p_src_size, int &r_consumed, int &r_out, bool p_close = false);
+	void _close();
+	Error _start(bool p_compress, bool p_is_deflate, int buffer_size = 65535);
+
+protected:
+	static void _bind_methods();
+
+public:
+	Error start_compression(bool p_is_deflate, int buffer_size = 65535);
+	Error start_decompression(bool p_is_deflate, int buffer_size = 65535);
+
+	Error finish();
+	void clear();
+
+	virtual Error put_data(const uint8_t *p_data, int p_bytes) override;
+	virtual Error put_partial_data(const uint8_t *p_data, int p_bytes, int &r_sent) override;
+
+	virtual Error get_data(uint8_t *p_buffer, int p_bytes) override;
+	virtual Error get_partial_data(uint8_t *p_buffer, int p_bytes, int &r_received) override;
+
+	virtual int get_available_bytes() const override;
+
+	StreamPeerGZIP();
+	~StreamPeerGZIP();
+};
+
+#endif // STREAM_PEER_GZIP_H

core/register_core_types.cpp

@@ -58,6 +58,7 @@
 #include "core/io/resource_format_binary.h"
 #include "core/io/resource_importer.h"
 #include "core/io/resource_uid.h"
+#include "core/io/stream_peer_gzip.h"
 #include "core/io/stream_peer_tls.h"
 #include "core/io/tcp_server.h"
 #include "core/io/translation_loader_po.h"
@@ -184,6 +185,7 @@ void register_core_types() {
 	GDREGISTER_ABSTRACT_CLASS(StreamPeer);
 	GDREGISTER_CLASS(StreamPeerExtension);
 	GDREGISTER_CLASS(StreamPeerBuffer);
+	GDREGISTER_CLASS(StreamPeerGZIP);
 	GDREGISTER_CLASS(StreamPeerTCP);
 	GDREGISTER_CLASS(TCPServer);
 

doc/classes/StreamPeerGZIP.xml

@@ -0,0 +1,42 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<class name="StreamPeerGZIP" inherits="StreamPeer" is_experimental="true" version="4.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="../class.xsd">
+	<brief_description>
+		Stream peer handling GZIP and deflate compression/decompresison.
+	</brief_description>
+	<description>
+		This class allows to compress or decompress data using GZIP/deflate in a streaming fashion. This is particularly useful when compressing or decompressing files that has to be sent through the network without having to allocate them all in memory.
+		After starting the stream via [method start_compression] (or [method start_decompression]), calling [method StreamPeer.put_partial_data] on this stream will compress (or decompress) the data, writing it to the internal buffer. Calling [method StreamPeer.get_available_bytes] will return the pending bytes in the internal buffer, and [method StreamPeer.get_partial_data] will retrieve the compressed (or decompressed) bytes from it. When the stream is over, you must call [method finish] to ensure the internal buffer is properly flushed (make sure to call [method StreamPeer.get_available_bytes] on last time to check if more data needs to be read after that).
+	</description>
+	<tutorials>
+	</tutorials>
+	<methods>
+		<method name="clear">
+			<return type="void" />
+			<description>
+				Clears this stream, resetting the internal state.
+			</description>
+		</method>
+		<method name="finish">
+			<return type="int" enum="Error" />
+			<description>
+				Finalizes the stream, compressing or decompressing any buffered chunk left.
+			</description>
+		</method>
+		<method name="start_compression">
+			<return type="int" enum="Error" />
+			<param index="0" name="use_deflate" type="bool" default="false" />
+			<param index="1" name="buffer_size" type="int" default="65535" />
+			<description>
+				Start the stream in compression mode with the given [param buffer_size], if [param use_deflate] is [code]true[/code] uses deflate instead of GZIP.
+			</description>
+		</method>
+		<method name="start_decompression">
+			<return type="int" enum="Error" />
+			<param index="0" name="use_deflate" type="bool" default="false" />
+			<param index="1" name="buffer_size" type="int" default="65535" />
+			<description>
+				Start the stream in decompression mode with the given [param buffer_size], if [param use_deflate] is [code]true[/code] uses deflate instead of GZIP.
+			</description>
+		</method>
+	</methods>
+</class>

scene/main/http_request.cpp

@@ -32,9 +32,6 @@
 #include "core/io/compression.h"
 #include "scene/main/timer.h"
 
-void HTTPRequest::_redirect_request(const String &p_new_url) {
-}
-
 Error HTTPRequest::_request() {
 	return client->connect_to_host(url, port, use_tls, validate_tls);
 }
@@ -48,6 +45,7 @@ Error HTTPRequest::_parse_url(const String &p_url) {
 	body_len = -1;
 	body.clear();
 	downloaded.set(0);
+	final_body_size.set(0);
 	redirections = 0;
 
 	String scheme;
@@ -153,7 +151,7 @@ Error HTTPRequest::request_raw(const String &p_url, const Vector<String> &p_cust
 		client->set_blocking_mode(false);
 		err = _request();
 		if (err != OK) {
-			call_deferred(SNAME("_request_done"), RESULT_CANT_CONNECT, 0, PackedStringArray(), PackedByteArray());
+			_defer_done(RESULT_CANT_CONNECT, 0, PackedStringArray(), PackedByteArray());
 			return ERR_CANT_CONNECT;
 		}
 
@@ -169,7 +167,7 @@ void HTTPRequest::_thread_func(void *p_userdata) {
 	Error err = hr->_request();
 
 	if (err != OK) {
-		hr->call_deferred(SNAME("_request_done"), RESULT_CANT_CONNECT, 0, PackedStringArray(), PackedByteArray());
+		hr->_defer_done(RESULT_CANT_CONNECT, 0, PackedStringArray(), PackedByteArray());
 	} else {
 		while (!hr->thread_request_quit.is_set()) {
 			bool exit = hr->_update_connection();
@@ -198,6 +196,7 @@ void HTTPRequest::cancel_request() {
 	}
 
 	file.unref();
+	decompressor.unref();
 	client->close();
 	body.clear();
 	got_response = false;
@@ -208,7 +207,7 @@ void HTTPRequest::cancel_request() {
 
 bool HTTPRequest::_handle_response(bool *ret_value) {
 	if (!client->has_response()) {
-		call_deferred(SNAME("_request_done"), RESULT_NO_RESPONSE, 0, PackedStringArray(), PackedByteArray());
+		_defer_done(RESULT_NO_RESPONSE, 0, PackedStringArray(), PackedByteArray());
 		*ret_value = true;
 		return true;
 	}
@@ -219,6 +218,9 @@ bool HTTPRequest::_handle_response(bool *ret_value) {
 	client->get_response_headers(&rheaders);
 	response_headers.clear();
 	downloaded.set(0);
+	final_body_size.set(0);
+	decompressor.unref();
+
 	for (const String &E : rheaders) {
 		response_headers.push_back(E);
 	}
@@ -227,7 +229,7 @@ bool HTTPRequest::_handle_response(bool *ret_value) {
 		// Handle redirect.
 
 		if (max_redirects >= 0 && redirections >= max_redirects) {
-			call_deferred(SNAME("_request_done"), RESULT_REDIRECT_LIMIT_REACHED, response_code, response_headers, PackedByteArray());
+			_defer_done(RESULT_REDIRECT_LIMIT_REACHED, response_code, response_headers, PackedByteArray());
 			*ret_value = true;
 			return true;
 		}
@@ -259,6 +261,7 @@ bool HTTPRequest::_handle_response(bool *ret_value) {
 				body_len = -1;
 				body.clear();
 				downloaded.set(0);
+				final_body_size.set(0);
 				redirections = new_redirs;
 				*ret_value = false;
 				return true;
@@ -266,13 +269,26 @@ bool HTTPRequest::_handle_response(bool *ret_value) {
 		}
 	}
 
+	// Check if we need to start streaming decompression.
+	String content_encoding;
+	if (accept_gzip) {
+		content_encoding = get_header_value(response_headers, "Content-Encoding").to_lower();
+	}
+	if (content_encoding == "gzip") {
+		decompressor.instantiate();
+		decompressor->start_decompression(false, get_download_chunk_size() * 2);
+	} else if (content_encoding == "deflate") {
+		decompressor.instantiate();
+		decompressor->start_decompression(true, get_download_chunk_size() * 2);
+	}
+
 	return false;
 }
 
 bool HTTPRequest::_update_connection() {
 	switch (client->get_status()) {
 		case HTTPClient::STATUS_DISCONNECTED: {
-			call_deferred(SNAME("_request_done"), RESULT_CANT_CONNECT, 0, PackedStringArray(), PackedByteArray());
+			_defer_done(RESULT_CANT_CONNECT, 0, PackedStringArray(), PackedByteArray());
 			return true; // End it, since it's disconnected.
 		} break;
 		case HTTPClient::STATUS_RESOLVING: {
@@ -281,7 +297,7 @@ bool HTTPRequest::_update_connection() {
 			return false;
 		} break;
 		case HTTPClient::STATUS_CANT_RESOLVE: {
-			call_deferred(SNAME("_request_done"), RESULT_CANT_RESOLVE, 0, PackedStringArray(), PackedByteArray());
+			_defer_done(RESULT_CANT_RESOLVE, 0, PackedStringArray(), PackedByteArray());
 			return true;
 
 		} break;
@@ -291,7 +307,7 @@ bool HTTPRequest::_update_connection() {
 			return false;
 		} break; // Connecting to IP.
 		case HTTPClient::STATUS_CANT_CONNECT: {
-			call_deferred(SNAME("_request_done"), RESULT_CANT_CONNECT, 0, PackedStringArray(), PackedByteArray());
+			_defer_done(RESULT_CANT_CONNECT, 0, PackedStringArray(), PackedByteArray());
 			return true;
 
 		} break;
@@ -306,16 +322,16 @@ bool HTTPRequest::_update_connection() {
 						return ret_value;
 					}
 
-					call_deferred(SNAME("_request_done"), RESULT_SUCCESS, response_code, response_headers, PackedByteArray());
+					_defer_done(RESULT_SUCCESS, response_code, response_headers, PackedByteArray());
 					return true;
 				}
 				if (body_len < 0) {
 					// Chunked transfer is done.
-					call_deferred(SNAME("_request_done"), RESULT_SUCCESS, response_code, response_headers, body);
+					_defer_done(RESULT_SUCCESS, response_code, response_headers, body);
 					return true;
 				}
 
-				call_deferred(SNAME("_request_done"), RESULT_CHUNKED_BODY_SIZE_MISMATCH, response_code, response_headers, PackedByteArray());
+				_defer_done(RESULT_CHUNKED_BODY_SIZE_MISMATCH, response_code, response_headers, PackedByteArray());
 				return true;
 				// Request might have been done.
 			} else {
@@ -324,7 +340,7 @@ bool HTTPRequest::_update_connection() {
 				int size = request_data.size();
 				Error err = client->request(method, request_string, headers, size > 0 ? request_data.ptr() : nullptr, size);
 				if (err != OK) {
-					call_deferred(SNAME("_request_done"), RESULT_CONNECTION_ERROR, 0, PackedStringArray(), PackedByteArray());
+					_defer_done(RESULT_CONNECTION_ERROR, 0, PackedStringArray(), PackedByteArray());
 					return true;
 				}
 
@@ -347,7 +363,7 @@ bool HTTPRequest::_update_connection() {
 				}
 
 				if (!client->is_response_chunked() && client->get_response_body_length() == 0) {
-					call_deferred(SNAME("_request_done"), RESULT_SUCCESS, response_code, response_headers, PackedByteArray());
+					_defer_done(RESULT_SUCCESS, response_code, response_headers, PackedByteArray());
 					return true;
 				}
 
@@ -356,14 +372,14 @@ bool HTTPRequest::_update_connection() {
 				body_len = client->get_response_body_length();
 
 				if (body_size_limit >= 0 && body_len > body_size_limit) {
-					call_deferred(SNAME("_request_done"), RESULT_BODY_SIZE_LIMIT_EXCEEDED, response_code, response_headers, PackedByteArray());
+					_defer_done(RESULT_BODY_SIZE_LIMIT_EXCEEDED, response_code, response_headers, PackedByteArray());
 					return true;
 				}
 
 				if (!download_to_file.is_empty()) {
 					file = FileAccess::open(download_to_file, FileAccess::WRITE);
 					if (file.is_null()) {
-						call_deferred(SNAME("_request_done"), RESULT_DOWNLOAD_FILE_CANT_OPEN, response_code, response_headers, PackedByteArray());
+						_defer_done(RESULT_DOWNLOAD_FILE_CANT_OPEN, response_code, response_headers, PackedByteArray());
 						return true;
 					}
 				}
@@ -375,14 +391,33 @@ bool HTTPRequest::_update_connection() {
 			}
 
 			PackedByteArray chunk = client->read_response_body_chunk();
+			downloaded.add(chunk.size());
+
+			// Decompress chunk if needed.
+			if (decompressor.is_valid()) {
+				Error err = decompressor->put_data(chunk.ptr(), chunk.size());
+				if (err == OK) {
+					chunk.resize(decompressor->get_available_bytes());
+					err = decompressor->get_data(chunk.ptrw(), chunk.size());
+				}
+				if (err != OK) {
+					_defer_done(RESULT_BODY_DECOMPRESS_FAILED, response_code, response_headers, PackedByteArray());
+					return true;
+				}
+			}
+			final_body_size.add(chunk.size());
+
+			if (body_size_limit >= 0 && final_body_size.get() > body_size_limit) {
+				_defer_done(RESULT_BODY_SIZE_LIMIT_EXCEEDED, response_code, response_headers, PackedByteArray());
+				return true;
+			}
 
 			if (chunk.size()) {
-				downloaded.add(chunk.size());
 				if (file.is_valid()) {
 					const uint8_t *r = chunk.ptr();
 					file->store_buffer(r, chunk.size());
 					if (file->get_error() != OK) {
-						call_deferred(SNAME("_request_done"), RESULT_DOWNLOAD_FILE_WRITE_ERROR, response_code, response_headers, PackedByteArray());
+						_defer_done(RESULT_DOWNLOAD_FILE_WRITE_ERROR, response_code, response_headers, PackedByteArray());
 						return true;
 					}
 				} else {
@@ -390,19 +425,14 @@ bool HTTPRequest::_update_connection() {
 				}
 			}
 
-			if (body_size_limit >= 0 && downloaded.get() > body_size_limit) {
-				call_deferred(SNAME("_request_done"), RESULT_BODY_SIZE_LIMIT_EXCEEDED, response_code, response_headers, PackedByteArray());
-				return true;
-			}
-
 			if (body_len >= 0) {
 				if (downloaded.get() == body_len) {
-					call_deferred(SNAME("_request_done"), RESULT_SUCCESS, response_code, response_headers, body);
+					_defer_done(RESULT_SUCCESS, response_code, response_headers, body);
 					return true;
 				}
 			} else if (client->get_status() == HTTPClient::STATUS_DISCONNECTED) {
 				// We read till EOF, with no errors. Request is done.
-				call_deferred(SNAME("_request_done"), RESULT_SUCCESS, response_code, response_headers, body);
+				_defer_done(RESULT_SUCCESS, response_code, response_headers, body);
 				return true;
 			}
 
@@ -410,11 +440,11 @@ bool HTTPRequest::_update_connection() {
 
 		} break; // Request resulted in body: break which must be read.
 		case HTTPClient::STATUS_CONNECTION_ERROR: {
-			call_deferred(SNAME("_request_done"), RESULT_CONNECTION_ERROR, 0, PackedStringArray(), PackedByteArray());
+			_defer_done(RESULT_CONNECTION_ERROR, 0, PackedStringArray(), PackedByteArray());
 			return true;
 		} break;
 		case HTTPClient::STATUS_TLS_HANDSHAKE_ERROR: {
-			call_deferred(SNAME("_request_done"), RESULT_TLS_HANDSHAKE_ERROR, 0, PackedStringArray(), PackedByteArray());
+			_defer_done(RESULT_TLS_HANDSHAKE_ERROR, 0, PackedStringArray(), PackedByteArray());
 			return true;
 		} break;
 	}
@@ -422,41 +452,13 @@ bool HTTPRequest::_update_connection() {
 	ERR_FAIL_V(false);
 }
 
+void HTTPRequest::_defer_done(int p_status, int p_code, const PackedStringArray &p_headers, const PackedByteArray &p_data) {
+	call_deferred(SNAME("_request_done"), p_status, p_code, p_headers, p_data);
+}
+
 void HTTPRequest::_request_done(int p_status, int p_code, const PackedStringArray &p_headers, const PackedByteArray &p_data) {
 	cancel_request();
 
-	// Determine if the request body is compressed.
-	bool is_compressed;
-	String content_encoding = get_header_value(p_headers, "Content-Encoding").to_lower();
-	Compression::Mode mode;
-	if (content_encoding == "gzip") {
-		mode = Compression::Mode::MODE_GZIP;
-		is_compressed = true;
-	} else if (content_encoding == "deflate") {
-		mode = Compression::Mode::MODE_DEFLATE;
-		is_compressed = true;
-	} else {
-		is_compressed = false;
-	}
-
-	if (accept_gzip && is_compressed && p_data.size() > 0) {
-		// Decompress request body
-		PackedByteArray decompressed;
-		int result = Compression::decompress_dynamic(&decompressed, body_size_limit, p_data.ptr(), p_data.size(), mode);
-		if (result == OK) {
-			emit_signal(SNAME("request_completed"), p_status, p_code, p_headers, decompressed);
-			return;
-		} else if (result == -5) {
-			WARN_PRINT("Decompressed size of HTTP response body exceeded body_size_limit");
-			p_status = RESULT_BODY_SIZE_LIMIT_EXCEEDED;
-			// Just return the raw data if we failed to decompress it.
-		} else {
-			WARN_PRINT("Failed to decompress HTTP response body");
-			p_status = RESULT_BODY_DECOMPRESS_FAILED;
-			// Just return the raw data if we failed to decompress it.
-		}
-	}
-
 	emit_signal(SNAME("request_completed"), p_status, p_code, p_headers, p_data);
 }
 
@@ -566,7 +568,7 @@ double HTTPRequest::get_timeout() {
 
 void HTTPRequest::_timeout() {
 	cancel_request();
-	call_deferred(SNAME("_request_done"), RESULT_TIMEOUT, 0, PackedStringArray(), PackedByteArray());
+	_defer_done(RESULT_TIMEOUT, 0, PackedStringArray(), PackedByteArray());
 }
 
 void HTTPRequest::_bind_methods() {
@@ -594,7 +596,6 @@ void HTTPRequest::_bind_methods() {
 	ClassDB::bind_method(D_METHOD("get_downloaded_bytes"), &HTTPRequest::get_downloaded_bytes);
 	ClassDB::bind_method(D_METHOD("get_body_size"), &HTTPRequest::get_body_size);
 
-	ClassDB::bind_method(D_METHOD("_redirect_request"), &HTTPRequest::_redirect_request);
 	ClassDB::bind_method(D_METHOD("_request_done"), &HTTPRequest::_request_done);
 
 	ClassDB::bind_method(D_METHOD("set_timeout", "timeout"), &HTTPRequest::set_timeout);

scene/main/http_request.h

@@ -32,6 +32,7 @@
 #define HTTP_REQUEST_H
 
 #include "core/io/http_client.h"
+#include "core/io/stream_peer_gzip.h"
 #include "core/os/thread.h"
 #include "core/templates/safe_refcount.h"
 #include "scene/main/node.h"
@@ -84,10 +85,12 @@ private:
 
 	String download_to_file;
 
+	Ref<StreamPeerGZIP> decompressor;
 	Ref<FileAccess> file;
 
 	int body_len = -1;
 	SafeNumeric<int> downloaded;
+	SafeNumeric<int> final_body_size;
 	int body_size_limit = -1;
 
 	int redirections = 0;
@@ -113,6 +116,7 @@ private:
 
 	Thread thread;
 
+	void _defer_done(int p_status, int p_code, const PackedStringArray &p_headers, const PackedByteArray &p_data);
 	void _request_done(int p_status, int p_code, const PackedStringArray &p_headers, const PackedByteArray &p_data);
 	static void _thread_func(void *p_userdata);