Merge pull request #47308 from Faless/net/4.x_http_client_request_and_custom

[Net] Unify HTTPClient `requests` make it a custom class.

core/io/http_client.cpp

@@ -30,9 +30,6 @@
 
 #include "http_client.h"
 
-#include "core/io/stream_peer_ssl.h"
-#include "core/version.h"
-
 const char *HTTPClient::_methods[METHOD_MAX] = {
 	"GET",
 	"HEAD",
@@ -45,698 +42,23 @@ const char *HTTPClient::_methods[METHOD_MAX] = {
 	"PATCH"
 };
 
-#ifndef JAVASCRIPT_ENABLED
-Error HTTPClient::connect_to_host(const String &p_host, int p_port, bool p_ssl, bool p_verify_host) {
-	close();
-
-	conn_port = p_port;
-	conn_host = p_host;
-
-	ssl = p_ssl;
-	ssl_verify_host = p_verify_host;
-
-	String host_lower = conn_host.to_lower();
-	if (host_lower.begins_with("http://")) {
-		conn_host = conn_host.substr(7, conn_host.length() - 7);
-	} else if (host_lower.begins_with("https://")) {
-		ssl = true;
-		conn_host = conn_host.substr(8, conn_host.length() - 8);
-	}
-
-	ERR_FAIL_COND_V(conn_host.length() < HOST_MIN_LEN, ERR_INVALID_PARAMETER);
-
-	if (conn_port < 0) {
-		if (ssl) {
-			conn_port = PORT_HTTPS;
-		} else {
-			conn_port = PORT_HTTP;
-		}
-	}
-
-	connection = tcp_connection;
-
-	if (conn_host.is_valid_ip_address()) {
-		// Host contains valid IP
-		Error err = tcp_connection->connect_to_host(IPAddress(conn_host), p_port);
-		if (err) {
-			status = STATUS_CANT_CONNECT;
-			return err;
-		}
-
-		status = STATUS_CONNECTING;
-	} else {
-		// Host contains hostname and needs to be resolved to IP
-		resolving = IP::get_singleton()->resolve_hostname_queue_item(conn_host);
-		status = STATUS_RESOLVING;
+HTTPClient *HTTPClient::create() {
+	if (_create) {
+		return _create();
 	}
-
-	return OK;
+	return nullptr;
 }
 
-void HTTPClient::set_connection(const Ref<StreamPeer> &p_connection) {
-	ERR_FAIL_COND_MSG(p_connection.is_null(), "Connection is not a reference to a valid StreamPeer object.");
-
-	if (ssl) {
-		ERR_FAIL_NULL_MSG(Object::cast_to<StreamPeerSSL>(p_connection.ptr()),
-				"Connection is not a reference to a valid StreamPeerSSL object.");
-	}
-
-	if (connection == p_connection) {
-		return;
-	}
-
-	close();
-	connection = p_connection;
-	status = STATUS_CONNECTED;
+Error HTTPClient::_request_raw(Method p_method, const String &p_url, const Vector<String> &p_headers, const Vector<uint8_t> &p_body) {
+	int size = p_body.size();
+	return request(p_method, p_url, p_headers, size > 0 ? p_body.ptr() : nullptr, size);
 }
 
-Ref<StreamPeer> HTTPClient::get_connection() const {
-	return connection;
+Error HTTPClient::_request(Method p_method, const String &p_url, const Vector<String> &p_headers, const String &p_body) {
+	int size = p_body.length();
+	return request(p_method, p_url, p_headers, size > 0 ? (const uint8_t *)p_body.utf8().get_data() : nullptr, size);
 }
 
-static bool _check_request_url(HTTPClient::Method p_method, const String &p_url) {
-	switch (p_method) {
-		case HTTPClient::METHOD_CONNECT: {
-			// Authority in host:port format, as in RFC7231
-			int pos = p_url.find_char(':');
-			return 0 < pos && pos < p_url.length() - 1;
-		}
-		case HTTPClient::METHOD_OPTIONS: {
-			if (p_url == "*") {
-				return true;
-			}
-			[[fallthrough]];
-		}
-		default:
-			// Absolute path or absolute URL
-			return p_url.begins_with("/") || p_url.begins_with("http://") || p_url.begins_with("https://");
-	}
-}
-
-Error HTTPClient::request_raw(Method p_method, const String &p_url, const Vector<String> &p_headers, const Vector<uint8_t> &p_body) {
-	ERR_FAIL_INDEX_V(p_method, METHOD_MAX, ERR_INVALID_PARAMETER);
-	ERR_FAIL_COND_V(!_check_request_url(p_method, p_url), ERR_INVALID_PARAMETER);
-	ERR_FAIL_COND_V(status != STATUS_CONNECTED, ERR_INVALID_PARAMETER);
-	ERR_FAIL_COND_V(connection.is_null(), ERR_INVALID_DATA);
-
-	String request = String(_methods[p_method]) + " " + p_url + " HTTP/1.1\r\n";
-	bool add_host = true;
-	bool add_clen = p_body.size() > 0;
-	bool add_uagent = true;
-	bool add_accept = true;
-	for (int i = 0; i < p_headers.size(); i++) {
-		request += p_headers[i] + "\r\n";
-		if (add_host && p_headers[i].findn("Host:") == 0) {
-			add_host = false;
-		}
-		if (add_clen && p_headers[i].findn("Content-Length:") == 0) {
-			add_clen = false;
-		}
-		if (add_uagent && p_headers[i].findn("User-Agent:") == 0) {
-			add_uagent = false;
-		}
-		if (add_accept && p_headers[i].findn("Accept:") == 0) {
-			add_accept = false;
-		}
-	}
-	if (add_host) {
-		if ((ssl && conn_port == PORT_HTTPS) || (!ssl && conn_port == PORT_HTTP)) {
-			// Don't append the standard ports
-			request += "Host: " + conn_host + "\r\n";
-		} else {
-			request += "Host: " + conn_host + ":" + itos(conn_port) + "\r\n";
-		}
-	}
-	if (add_clen) {
-		request += "Content-Length: " + itos(p_body.size()) + "\r\n";
-		// Should it add utf8 encoding?
-	}
-	if (add_uagent) {
-		request += "User-Agent: GodotEngine/" + String(VERSION_FULL_BUILD) + " (" + OS::get_singleton()->get_name() + ")\r\n";
-	}
-	if (add_accept) {
-		request += "Accept: */*\r\n";
-	}
-	request += "\r\n";
-	CharString cs = request.utf8();
-
-	Vector<uint8_t> data;
-	data.resize(cs.length());
-	{
-		uint8_t *data_write = data.ptrw();
-		for (int i = 0; i < cs.length(); i++) {
-			data_write[i] = cs[i];
-		}
-	}
-
-	data.append_array(p_body);
-
-	const uint8_t *r = data.ptr();
-	Error err = connection->put_data(&r[0], data.size());
-
-	if (err) {
-		close();
-		status = STATUS_CONNECTION_ERROR;
-		return err;
-	}
-
-	status = STATUS_REQUESTING;
-	head_request = p_method == METHOD_HEAD;
-
-	return OK;
-}
-
-Error HTTPClient::request(Method p_method, const String &p_url, const Vector<String> &p_headers, const String &p_body) {
-	ERR_FAIL_INDEX_V(p_method, METHOD_MAX, ERR_INVALID_PARAMETER);
-	ERR_FAIL_COND_V(!_check_request_url(p_method, p_url), ERR_INVALID_PARAMETER);
-	ERR_FAIL_COND_V(status != STATUS_CONNECTED, ERR_INVALID_PARAMETER);
-	ERR_FAIL_COND_V(connection.is_null(), ERR_INVALID_DATA);
-
-	String request = String(_methods[p_method]) + " " + p_url + " HTTP/1.1\r\n";
-	bool add_host = true;
-	bool add_uagent = true;
-	bool add_accept = true;
-	bool add_clen = p_body.length() > 0;
-	for (int i = 0; i < p_headers.size(); i++) {
-		request += p_headers[i] + "\r\n";
-		if (add_host && p_headers[i].findn("Host:") == 0) {
-			add_host = false;
-		}
-		if (add_clen && p_headers[i].findn("Content-Length:") == 0) {
-			add_clen = false;
-		}
-		if (add_uagent && p_headers[i].findn("User-Agent:") == 0) {
-			add_uagent = false;
-		}
-		if (add_accept && p_headers[i].findn("Accept:") == 0) {
-			add_accept = false;
-		}
-	}
-	if (add_host) {
-		if ((ssl && conn_port == PORT_HTTPS) || (!ssl && conn_port == PORT_HTTP)) {
-			// Don't append the standard ports
-			request += "Host: " + conn_host + "\r\n";
-		} else {
-			request += "Host: " + conn_host + ":" + itos(conn_port) + "\r\n";
-		}
-	}
-	if (add_clen) {
-		request += "Content-Length: " + itos(p_body.utf8().length()) + "\r\n";
-		// Should it add utf8 encoding?
-	}
-	if (add_uagent) {
-		request += "User-Agent: GodotEngine/" + String(VERSION_FULL_BUILD) + " (" + OS::get_singleton()->get_name() + ")\r\n";
-	}
-	if (add_accept) {
-		request += "Accept: */*\r\n";
-	}
-	request += "\r\n";
-	request += p_body;
-
-	CharString cs = request.utf8();
-	Error err = connection->put_data((const uint8_t *)cs.ptr(), cs.length());
-	if (err) {
-		close();
-		status = STATUS_CONNECTION_ERROR;
-		return err;
-	}
-
-	status = STATUS_REQUESTING;
-	head_request = p_method == METHOD_HEAD;
-
-	return OK;
-}
-
-bool HTTPClient::has_response() const {
-	return response_headers.size() != 0;
-}
-
-bool HTTPClient::is_response_chunked() const {
-	return chunked;
-}
-
-int HTTPClient::get_response_code() const {
-	return response_num;
-}
-
-Error HTTPClient::get_response_headers(List<String> *r_response) {
-	if (!response_headers.size()) {
-		return ERR_INVALID_PARAMETER;
-	}
-
-	for (int i = 0; i < response_headers.size(); i++) {
-		r_response->push_back(response_headers[i]);
-	}
-
-	response_headers.clear();
-
-	return OK;
-}
-
-void HTTPClient::close() {
-	if (tcp_connection->get_status() != StreamPeerTCP::STATUS_NONE) {
-		tcp_connection->disconnect_from_host();
-	}
-
-	connection.unref();
-	status = STATUS_DISCONNECTED;
-	head_request = false;
-	if (resolving != IP::RESOLVER_INVALID_ID) {
-		IP::get_singleton()->erase_resolve_item(resolving);
-		resolving = IP::RESOLVER_INVALID_ID;
-	}
-
-	response_headers.clear();
-	response_str.clear();
-	body_size = -1;
-	body_left = 0;
-	chunk_left = 0;
-	chunk_trailer_part = false;
-	read_until_eof = false;
-	response_num = 0;
-	handshaking = false;
-}
-
-Error HTTPClient::poll() {
-	switch (status) {
-		case STATUS_RESOLVING: {
-			ERR_FAIL_COND_V(resolving == IP::RESOLVER_INVALID_ID, ERR_BUG);
-
-			IP::ResolverStatus rstatus = IP::get_singleton()->get_resolve_item_status(resolving);
-			switch (rstatus) {
-				case IP::RESOLVER_STATUS_WAITING:
-					return OK; // Still resolving
-
-				case IP::RESOLVER_STATUS_DONE: {
-					IPAddress host = IP::get_singleton()->get_resolve_item_address(resolving);
-					Error err = tcp_connection->connect_to_host(host, conn_port);
-					IP::get_singleton()->erase_resolve_item(resolving);
-					resolving = IP::RESOLVER_INVALID_ID;
-					if (err) {
-						status = STATUS_CANT_CONNECT;
-						return err;
-					}
-
-					status = STATUS_CONNECTING;
-				} break;
-				case IP::RESOLVER_STATUS_NONE:
-				case IP::RESOLVER_STATUS_ERROR: {
-					IP::get_singleton()->erase_resolve_item(resolving);
-					resolving = IP::RESOLVER_INVALID_ID;
-					close();
-					status = STATUS_CANT_RESOLVE;
-					return ERR_CANT_RESOLVE;
-				} break;
-			}
-		} break;
-		case STATUS_CONNECTING: {
-			StreamPeerTCP::Status s = tcp_connection->get_status();
-			switch (s) {
-				case StreamPeerTCP::STATUS_CONNECTING: {
-					return OK;
-				} break;
-				case StreamPeerTCP::STATUS_CONNECTED: {
-					if (ssl) {
-						Ref<StreamPeerSSL> ssl;
-						if (!handshaking) {
-							// Connect the StreamPeerSSL and start handshaking
-							ssl = Ref<StreamPeerSSL>(StreamPeerSSL::create());
-							ssl->set_blocking_handshake_enabled(false);
-							Error err = ssl->connect_to_stream(tcp_connection, ssl_verify_host, conn_host);
-							if (err != OK) {
-								close();
-								status = STATUS_SSL_HANDSHAKE_ERROR;
-								return ERR_CANT_CONNECT;
-							}
-							connection = ssl;
-							handshaking = true;
-						} else {
-							// We are already handshaking, which means we can use your already active SSL connection
-							ssl = static_cast<Ref<StreamPeerSSL>>(connection);
-							if (ssl.is_null()) {
-								close();
-								status = STATUS_SSL_HANDSHAKE_ERROR;
-								return ERR_CANT_CONNECT;
-							}
-
-							ssl->poll(); // Try to finish the handshake
-						}
-
-						if (ssl->get_status() == StreamPeerSSL::STATUS_CONNECTED) {
-							// Handshake has been successful
-							handshaking = false;
-							status = STATUS_CONNECTED;
-							return OK;
-						} else if (ssl->get_status() != StreamPeerSSL::STATUS_HANDSHAKING) {
-							// Handshake has failed
-							close();
-							status = STATUS_SSL_HANDSHAKE_ERROR;
-							return ERR_CANT_CONNECT;
-						}
-						// ... we will need to poll more for handshake to finish
-					} else {
-						status = STATUS_CONNECTED;
-					}
-					return OK;
-				} break;
-				case StreamPeerTCP::STATUS_ERROR:
-				case StreamPeerTCP::STATUS_NONE: {
-					close();
-					status = STATUS_CANT_CONNECT;
-					return ERR_CANT_CONNECT;
-				} break;
-			}
-		} break;
-		case STATUS_BODY:
-		case STATUS_CONNECTED: {
-			// Check if we are still connected
-			if (ssl) {
-				Ref<StreamPeerSSL> tmp = connection;
-				tmp->poll();
-				if (tmp->get_status() != StreamPeerSSL::STATUS_CONNECTED) {
-					status = STATUS_CONNECTION_ERROR;
-					return ERR_CONNECTION_ERROR;
-				}
-			} else if (tcp_connection->get_status() != StreamPeerTCP::STATUS_CONNECTED) {
-				status = STATUS_CONNECTION_ERROR;
-				return ERR_CONNECTION_ERROR;
-			}
-			// Connection established, requests can now be made
-			return OK;
-		} break;
-		case STATUS_REQUESTING: {
-			while (true) {
-				uint8_t byte;
-				int rec = 0;
-				Error err = _get_http_data(&byte, 1, rec);
-				if (err != OK) {
-					close();
-					status = STATUS_CONNECTION_ERROR;
-					return ERR_CONNECTION_ERROR;
-				}
-
-				if (rec == 0) {
-					return OK; // Still requesting, keep trying!
-				}
-
-				response_str.push_back(byte);
-				int rs = response_str.size();
-				if (
-						(rs >= 2 && response_str[rs - 2] == '\n' && response_str[rs - 1] == '\n') ||
-						(rs >= 4 && response_str[rs - 4] == '\r' && response_str[rs - 3] == '\n' && response_str[rs - 2] == '\r' && response_str[rs - 1] == '\n')) {
-					// End of response, parse.
-					response_str.push_back(0);
-					String response;
-					response.parse_utf8((const char *)response_str.ptr());
-					Vector<String> responses = response.split("\n");
-					body_size = -1;
-					chunked = false;
-					body_left = 0;
-					chunk_left = 0;
-					chunk_trailer_part = false;
-					read_until_eof = false;
-					response_str.clear();
-					response_headers.clear();
-					response_num = RESPONSE_OK;
-
-					// Per the HTTP 1.1 spec, keep-alive is the default.
-					// Not following that specification breaks standard implementations.
-					// Broken web servers should be fixed.
-					bool keep_alive = true;
-
-					for (int i = 0; i < responses.size(); i++) {
-						String header = responses[i].strip_edges();
-						String s = header.to_lower();
-						if (s.length() == 0) {
-							continue;
-						}
-						if (s.begins_with("content-length:")) {
-							body_size = s.substr(s.find(":") + 1, s.length()).strip_edges().to_int();
-							body_left = body_size;
-
-						} else if (s.begins_with("transfer-encoding:")) {
-							String encoding = header.substr(header.find(":") + 1, header.length()).strip_edges();
-							if (encoding == "chunked") {
-								chunked = true;
-							}
-						} else if (s.begins_with("connection: close")) {
-							keep_alive = false;
-						}
-
-						if (i == 0 && responses[i].begins_with("HTTP")) {
-							String num = responses[i].get_slicec(' ', 1);
-							response_num = num.to_int();
-						} else {
-							response_headers.push_back(header);
-						}
-					}
-
-					// This is a HEAD request, we won't receive anything.
-					if (head_request) {
-						body_size = 0;
-						body_left = 0;
-					}
-
-					if (body_size != -1 || chunked) {
-						status = STATUS_BODY;
-					} else if (!keep_alive) {
-						read_until_eof = true;
-						status = STATUS_BODY;
-					} else {
-						status = STATUS_CONNECTED;
-					}
-					return OK;
-				}
-			}
-		} break;
-		case STATUS_DISCONNECTED: {
-			return ERR_UNCONFIGURED;
-		} break;
-		case STATUS_CONNECTION_ERROR:
-		case STATUS_SSL_HANDSHAKE_ERROR: {
-			return ERR_CONNECTION_ERROR;
-		} break;
-		case STATUS_CANT_CONNECT: {
-			return ERR_CANT_CONNECT;
-		} break;
-		case STATUS_CANT_RESOLVE: {
-			return ERR_CANT_RESOLVE;
-		} break;
-	}
-
-	return OK;
-}
-
-int HTTPClient::get_response_body_length() const {
-	return body_size;
-}
-
-PackedByteArray HTTPClient::read_response_body_chunk() {
-	ERR_FAIL_COND_V(status != STATUS_BODY, PackedByteArray());
-
-	PackedByteArray ret;
-	Error err = OK;
-
-	if (chunked) {
-		while (true) {
-			if (chunk_trailer_part) {
-				// We need to consume the trailer part too or keep-alive will break
-				uint8_t b;
-				int rec = 0;
-				err = _get_http_data(&b, 1, rec);
-
-				if (rec == 0) {
-					break;
-				}
-
-				chunk.push_back(b);
-				int cs = chunk.size();
-				if ((cs >= 2 && chunk[cs - 2] == '\r' && chunk[cs - 1] == '\n')) {
-					if (cs == 2) {
-						// Finally over
-						chunk_trailer_part = false;
-						status = STATUS_CONNECTED;
-						chunk.clear();
-						break;
-					} else {
-						// We do not process nor return the trailer data
-						chunk.clear();
-					}
-				}
-			} else if (chunk_left == 0) {
-				// Reading length
-				uint8_t b;
-				int rec = 0;
-				err = _get_http_data(&b, 1, rec);
-
-				if (rec == 0) {
-					break;
-				}
-
-				chunk.push_back(b);
-
-				if (chunk.size() > 32) {
-					ERR_PRINT("HTTP Invalid chunk hex len");
-					status = STATUS_CONNECTION_ERROR;
-					break;
-				}
-
-				if (chunk.size() > 2 && chunk[chunk.size() - 2] == '\r' && chunk[chunk.size() - 1] == '\n') {
-					int len = 0;
-					for (int i = 0; i < chunk.size() - 2; i++) {
-						char c = chunk[i];
-						int v = 0;
-						if (c >= '0' && c <= '9') {
-							v = c - '0';
-						} else if (c >= 'a' && c <= 'f') {
-							v = c - 'a' + 10;
-						} else if (c >= 'A' && c <= 'F') {
-							v = c - 'A' + 10;
-						} else {
-							ERR_PRINT("HTTP Chunk len not in hex!!");
-							status = STATUS_CONNECTION_ERROR;
-							break;
-						}
-						len <<= 4;
-						len |= v;
-						if (len > (1 << 24)) {
-							ERR_PRINT("HTTP Chunk too big!! >16mb");
-							status = STATUS_CONNECTION_ERROR;
-							break;
-						}
-					}
-
-					if (len == 0) {
-						// End reached!
-						chunk_trailer_part = true;
-						chunk.clear();
-						break;
-					}
-
-					chunk_left = len + 2;
-					chunk.resize(chunk_left);
-				}
-			} else {
-				int rec = 0;
-				err = _get_http_data(&chunk.write[chunk.size() - chunk_left], chunk_left, rec);
-				if (rec == 0) {
-					break;
-				}
-				chunk_left -= rec;
-
-				if (chunk_left == 0) {
-					if (chunk[chunk.size() - 2] != '\r' || chunk[chunk.size() - 1] != '\n') {
-						ERR_PRINT("HTTP Invalid chunk terminator (not \\r\\n)");
-						status = STATUS_CONNECTION_ERROR;
-						break;
-					}
-
-					ret.resize(chunk.size() - 2);
-					uint8_t *w = ret.ptrw();
-					memcpy(w, chunk.ptr(), chunk.size() - 2);
-					chunk.clear();
-				}
-
-				break;
-			}
-		}
-
-	} else {
-		int to_read = !read_until_eof ? MIN(body_left, read_chunk_size) : read_chunk_size;
-		ret.resize(to_read);
-		int _offset = 0;
-		while (to_read > 0) {
-			int rec = 0;
-			{
-				uint8_t *w = ret.ptrw();
-				err = _get_http_data(w + _offset, to_read, rec);
-			}
-			if (rec <= 0) { // Ended up reading less
-				ret.resize(_offset);
-				break;
-			} else {
-				_offset += rec;
-				to_read -= rec;
-				if (!read_until_eof) {
-					body_left -= rec;
-				}
-			}
-			if (err != OK) {
-				break;
-			}
-		}
-	}
-
-	if (err != OK) {
-		close();
-
-		if (err == ERR_FILE_EOF) {
-			status = STATUS_DISCONNECTED; // Server disconnected
-		} else {
-			status = STATUS_CONNECTION_ERROR;
-		}
-	} else if (body_left == 0 && !chunked && !read_until_eof) {
-		status = STATUS_CONNECTED;
-	}
-
-	return ret;
-}
-
-HTTPClient::Status HTTPClient::get_status() const {
-	return status;
-}
-
-void HTTPClient::set_blocking_mode(bool p_enable) {
-	blocking = p_enable;
-}
-
-bool HTTPClient::is_blocking_mode_enabled() const {
-	return blocking;
-}
-
-Error HTTPClient::_get_http_data(uint8_t *p_buffer, int p_bytes, int &r_received) {
-	if (blocking) {
-		// We can't use StreamPeer.get_data, since when reaching EOF we will get an
-		// error without knowing how many bytes we received.
-		Error err = ERR_FILE_EOF;
-		int read = 0;
-		int left = p_bytes;
-		r_received = 0;
-		while (left > 0) {
-			err = connection->get_partial_data(p_buffer + r_received, left, read);
-			if (err == OK) {
-				r_received += read;
-			} else if (err == ERR_FILE_EOF) {
-				r_received += read;
-				return err;
-			} else {
-				return err;
-			}
-			left -= read;
-		}
-		return err;
-	} else {
-		return connection->get_partial_data(p_buffer, p_bytes, r_received);
-	}
-}
-
-void HTTPClient::set_read_chunk_size(int p_size) {
-	ERR_FAIL_COND(p_size < 256 || p_size > (1 << 24));
-	read_chunk_size = p_size;
-}
-
-int HTTPClient::get_read_chunk_size() const {
-	return read_chunk_size;
-}
-
-HTTPClient::HTTPClient() {
-	tcp_connection.instantiate();
-}
-
-HTTPClient::~HTTPClient() {}
-
-#endif // #ifndef JAVASCRIPT_ENABLED
-
 String HTTPClient::query_string_from_dict(const Dictionary &p_dict) {
 	String query = "";
 	Array keys = p_dict.keys();
@@ -802,8 +124,8 @@ void HTTPClient::_bind_methods() {
 	ClassDB::bind_method(D_METHOD("connect_to_host", "host", "port", "use_ssl", "verify_host"), &HTTPClient::connect_to_host, DEFVAL(-1), DEFVAL(false), DEFVAL(true));
 	ClassDB::bind_method(D_METHOD("set_connection", "connection"), &HTTPClient::set_connection);
 	ClassDB::bind_method(D_METHOD("get_connection"), &HTTPClient::get_connection);
-	ClassDB::bind_method(D_METHOD("request_raw", "method", "url", "headers", "body"), &HTTPClient::request_raw);
-	ClassDB::bind_method(D_METHOD("request", "method", "url", "headers", "body"), &HTTPClient::request, DEFVAL(String()));
+	ClassDB::bind_method(D_METHOD("request_raw", "method", "url", "headers", "body"), &HTTPClient::_request_raw);
+	ClassDB::bind_method(D_METHOD("request", "method", "url", "headers", "body"), &HTTPClient::_request, DEFVAL(String()));
 	ClassDB::bind_method(D_METHOD("close"), &HTTPClient::close);
 
 	ClassDB::bind_method(D_METHOD("has_response"), &HTTPClient::has_response);

core/io/http_client.h

@@ -142,7 +142,7 @@ public:
 
 	};
 
-private:
+protected:
 	static const char *_methods[METHOD_MAX];
 	static const int HOST_MIN_LEN = 4;
 
@@ -152,79 +152,48 @@ private:
 
 	};
 
-#ifndef JAVASCRIPT_ENABLED
-	Status status = STATUS_DISCONNECTED;
-	IP::ResolverID resolving = IP::RESOLVER_INVALID_ID;
-	int conn_port = -1;
-	String conn_host;
-	bool ssl = false;
-	bool ssl_verify_host = false;
-	bool blocking = false;
-	bool handshaking = false;
-	bool head_request = false;
-
-	Vector<uint8_t> response_str;
-
-	bool chunked = false;
-	Vector<uint8_t> chunk;
-	int chunk_left = 0;
-	bool chunk_trailer_part = false;
-	int body_size = -1;
-	int body_left = 0;
-	bool read_until_eof = false;
-
-	Ref<StreamPeerTCP> tcp_connection;
-	Ref<StreamPeer> connection;
-
-	int response_num = 0;
-	Vector<String> response_headers;
-	// 64 KiB by default (favors fast download speeds at the cost of memory usage).
-	int read_chunk_size = 65536;
-
-	Error _get_http_data(uint8_t *p_buffer, int p_bytes, int &r_received);
-
-#else
-#include "platform/javascript/http_client.h.inc"
-#endif
-
 	PackedStringArray _get_response_headers();
 	Dictionary _get_response_headers_as_dictionary();
+	Error _request_raw(Method p_method, const String &p_url, const Vector<String> &p_headers, const Vector<uint8_t> &p_body);
+	Error _request(Method p_method, const String &p_url, const Vector<String> &p_headers, const String &p_body = String());
+
+	static HTTPClient *(*_create)();
 
 	static void _bind_methods();
 
 public:
-	Error connect_to_host(const String &p_host, int p_port = -1, bool p_ssl = false, bool p_verify_host = true);
+	static HTTPClient *create();
 
-	void set_connection(const Ref<StreamPeer> &p_connection);
-	Ref<StreamPeer> get_connection() const;
+	String query_string_from_dict(const Dictionary &p_dict);
 
-	Error request_raw(Method p_method, const String &p_url, const Vector<String> &p_headers, const Vector<uint8_t> &p_body);
-	Error request(Method p_method, const String &p_url, const Vector<String> &p_headers, const String &p_body = String());
+	virtual Error request(Method p_method, const String &p_url, const Vector<String> &p_headers, const uint8_t *p_body, int p_body_size) = 0;
+	virtual Error connect_to_host(const String &p_host, int p_port = -1, bool p_ssl = false, bool p_verify_host = true) = 0;
 
-	void close();
+	virtual void set_connection(const Ref<StreamPeer> &p_connection) = 0;
+	virtual Ref<StreamPeer> get_connection() const = 0;
 
-	Status get_status() const;
+	virtual void close() = 0;
 
-	bool has_response() const;
-	bool is_response_chunked() const;
-	int get_response_code() const;
-	Error get_response_headers(List<String> *r_response);
-	int get_response_body_length() const;
+	virtual Status get_status() const = 0;
 
-	PackedByteArray read_response_body_chunk(); // Can't get body as partial text because of most encodings UTF8, gzip, etc.
+	virtual bool has_response() const = 0;
+	virtual bool is_response_chunked() const = 0;
+	virtual int get_response_code() const = 0;
+	virtual Error get_response_headers(List<String> *r_response) = 0;
+	virtual int get_response_body_length() const = 0;
 
-	void set_blocking_mode(bool p_enable); // Useful mostly if running in a thread
-	bool is_blocking_mode_enabled() const;
+	virtual PackedByteArray read_response_body_chunk() = 0; // Can't get body as partial text because of most encodings UTF8, gzip, etc.
 
-	void set_read_chunk_size(int p_size);
-	int get_read_chunk_size() const;
+	virtual void set_blocking_mode(bool p_enable) = 0; // Useful mostly if running in a thread
+	virtual bool is_blocking_mode_enabled() const = 0;
 
-	Error poll();
+	virtual void set_read_chunk_size(int p_size) = 0;
+	virtual int get_read_chunk_size() const = 0;
 
-	String query_string_from_dict(const Dictionary &p_dict);
+	virtual Error poll() = 0;
 
-	HTTPClient();
-	~HTTPClient();
+	HTTPClient() {}
+	virtual ~HTTPClient() {}
 };
 
 VARIANT_ENUM_CAST(HTTPClient::ResponseCode)

core/io/http_client_tcp.cpp

@@ -0,0 +1,666 @@
+/*************************************************************************/
+/*  http_client_tcp.cpp                                                  */
+/*************************************************************************/
+/*                       This file is part of:                           */
+/*                           GODOT ENGINE                                */
+/*                      https://godotengine.org                          */
+/*************************************************************************/
+/* Copyright (c) 2007-2021 Juan Linietsky, Ariel Manzur.                 */
+/* Copyright (c) 2014-2021 Godot Engine contributors (cf. AUTHORS.md).   */
+/*                                                                       */
+/* Permission is hereby granted, free of charge, to any person obtaining */
+/* a copy of this software and associated documentation files (the       */
+/* "Software"), to deal in the Software without restriction, including   */
+/* without limitation the rights to use, copy, modify, merge, publish,   */
+/* distribute, sublicense, and/or sell copies of the Software, and to    */
+/* permit persons to whom the Software is furnished to do so, subject to */
+/* the following conditions:                                             */
+/*                                                                       */
+/* The above copyright notice and this permission notice shall be        */
+/* included in all copies or substantial portions of the Software.       */
+/*                                                                       */
+/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,       */
+/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF    */
+/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*/
+/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY  */
+/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,  */
+/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE     */
+/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.                */
+/*************************************************************************/
+
+#ifndef JAVASCRIPT_ENABLED
+
+#include "http_client_tcp.h"
+
+#include "core/io/stream_peer_ssl.h"
+#include "core/version.h"
+
+HTTPClient *HTTPClientTCP::_create_func() {
+	return memnew(HTTPClientTCP);
+}
+
+Error HTTPClientTCP::connect_to_host(const String &p_host, int p_port, bool p_ssl, bool p_verify_host) {
+	close();
+
+	conn_port = p_port;
+	conn_host = p_host;
+
+	ssl = p_ssl;
+	ssl_verify_host = p_verify_host;
+
+	String host_lower = conn_host.to_lower();
+	if (host_lower.begins_with("http://")) {
+		conn_host = conn_host.substr(7, conn_host.length() - 7);
+	} else if (host_lower.begins_with("https://")) {
+		ssl = true;
+		conn_host = conn_host.substr(8, conn_host.length() - 8);
+	}
+
+	ERR_FAIL_COND_V(conn_host.length() < HOST_MIN_LEN, ERR_INVALID_PARAMETER);
+
+	if (conn_port < 0) {
+		if (ssl) {
+			conn_port = PORT_HTTPS;
+		} else {
+			conn_port = PORT_HTTP;
+		}
+	}
+
+	connection = tcp_connection;
+
+	if (conn_host.is_valid_ip_address()) {
+		// Host contains valid IP
+		Error err = tcp_connection->connect_to_host(IPAddress(conn_host), p_port);
+		if (err) {
+			status = STATUS_CANT_CONNECT;
+			return err;
+		}
+
+		status = STATUS_CONNECTING;
+	} else {
+		// Host contains hostname and needs to be resolved to IP
+		resolving = IP::get_singleton()->resolve_hostname_queue_item(conn_host);
+		status = STATUS_RESOLVING;
+	}
+
+	return OK;
+}
+
+void HTTPClientTCP::set_connection(const Ref<StreamPeer> &p_connection) {
+	ERR_FAIL_COND_MSG(p_connection.is_null(), "Connection is not a reference to a valid StreamPeer object.");
+
+	if (ssl) {
+		ERR_FAIL_NULL_MSG(Object::cast_to<StreamPeerSSL>(p_connection.ptr()),
+				"Connection is not a reference to a valid StreamPeerSSL object.");
+	}
+
+	if (connection == p_connection) {
+		return;
+	}
+
+	close();
+	connection = p_connection;
+	status = STATUS_CONNECTED;
+}
+
+Ref<StreamPeer> HTTPClientTCP::get_connection() const {
+	return connection;
+}
+
+static bool _check_request_url(HTTPClientTCP::Method p_method, const String &p_url) {
+	switch (p_method) {
+		case HTTPClientTCP::METHOD_CONNECT: {
+			// Authority in host:port format, as in RFC7231
+			int pos = p_url.find_char(':');
+			return 0 < pos && pos < p_url.length() - 1;
+		}
+		case HTTPClientTCP::METHOD_OPTIONS: {
+			if (p_url == "*") {
+				return true;
+			}
+			[[fallthrough]];
+		}
+		default:
+			// Absolute path or absolute URL
+			return p_url.begins_with("/") || p_url.begins_with("http://") || p_url.begins_with("https://");
+	}
+}
+
+Error HTTPClientTCP::request(Method p_method, const String &p_url, const Vector<String> &p_headers, const uint8_t *p_body, int p_body_size) {
+	ERR_FAIL_INDEX_V(p_method, METHOD_MAX, ERR_INVALID_PARAMETER);
+	ERR_FAIL_COND_V(!_check_request_url(p_method, p_url), ERR_INVALID_PARAMETER);
+	ERR_FAIL_COND_V(status != STATUS_CONNECTED, ERR_INVALID_PARAMETER);
+	ERR_FAIL_COND_V(connection.is_null(), ERR_INVALID_DATA);
+
+	String request = String(_methods[p_method]) + " " + p_url + " HTTP/1.1\r\n";
+	bool add_host = true;
+	bool add_clen = p_body_size > 0;
+	bool add_uagent = true;
+	bool add_accept = true;
+	for (int i = 0; i < p_headers.size(); i++) {
+		request += p_headers[i] + "\r\n";
+		if (add_host && p_headers[i].findn("Host:") == 0) {
+			add_host = false;
+		}
+		if (add_clen && p_headers[i].findn("Content-Length:") == 0) {
+			add_clen = false;
+		}
+		if (add_uagent && p_headers[i].findn("User-Agent:") == 0) {
+			add_uagent = false;
+		}
+		if (add_accept && p_headers[i].findn("Accept:") == 0) {
+			add_accept = false;
+		}
+	}
+	if (add_host) {
+		if ((ssl && conn_port == PORT_HTTPS) || (!ssl && conn_port == PORT_HTTP)) {
+			// Don't append the standard ports
+			request += "Host: " + conn_host + "\r\n";
+		} else {
+			request += "Host: " + conn_host + ":" + itos(conn_port) + "\r\n";
+		}
+	}
+	if (add_clen) {
+		request += "Content-Length: " + itos(p_body_size) + "\r\n";
+		// Should it add utf8 encoding?
+	}
+	if (add_uagent) {
+		request += "User-Agent: GodotEngine/" + String(VERSION_FULL_BUILD) + " (" + OS::get_singleton()->get_name() + ")\r\n";
+	}
+	if (add_accept) {
+		request += "Accept: */*\r\n";
+	}
+	request += "\r\n";
+	CharString cs = request.utf8();
+
+	Vector<uint8_t> data;
+	data.resize(cs.length() + p_body_size);
+	memcpy(data.ptrw(), cs.get_data(), cs.length());
+	if (p_body_size > 0) {
+		memcpy(data.ptrw() + cs.length(), p_body, p_body_size);
+	}
+
+	// TODO Implement non-blocking requests.
+	Error err = connection->put_data(data.ptr(), data.size());
+
+	if (err) {
+		close();
+		status = STATUS_CONNECTION_ERROR;
+		return err;
+	}
+
+	status = STATUS_REQUESTING;
+	head_request = p_method == METHOD_HEAD;
+
+	return OK;
+}
+
+bool HTTPClientTCP::has_response() const {
+	return response_headers.size() != 0;
+}
+
+bool HTTPClientTCP::is_response_chunked() const {
+	return chunked;
+}
+
+int HTTPClientTCP::get_response_code() const {
+	return response_num;
+}
+
+Error HTTPClientTCP::get_response_headers(List<String> *r_response) {
+	if (!response_headers.size()) {
+		return ERR_INVALID_PARAMETER;
+	}
+
+	for (int i = 0; i < response_headers.size(); i++) {
+		r_response->push_back(response_headers[i]);
+	}
+
+	response_headers.clear();
+
+	return OK;
+}
+
+void HTTPClientTCP::close() {
+	if (tcp_connection->get_status() != StreamPeerTCP::STATUS_NONE) {
+		tcp_connection->disconnect_from_host();
+	}
+
+	connection.unref();
+	status = STATUS_DISCONNECTED;
+	head_request = false;
+	if (resolving != IP::RESOLVER_INVALID_ID) {
+		IP::get_singleton()->erase_resolve_item(resolving);
+		resolving = IP::RESOLVER_INVALID_ID;
+	}
+
+	response_headers.clear();
+	response_str.clear();
+	body_size = -1;
+	body_left = 0;
+	chunk_left = 0;
+	chunk_trailer_part = false;
+	read_until_eof = false;
+	response_num = 0;
+	handshaking = false;
+}
+
+Error HTTPClientTCP::poll() {
+	switch (status) {
+		case STATUS_RESOLVING: {
+			ERR_FAIL_COND_V(resolving == IP::RESOLVER_INVALID_ID, ERR_BUG);
+
+			IP::ResolverStatus rstatus = IP::get_singleton()->get_resolve_item_status(resolving);
+			switch (rstatus) {
+				case IP::RESOLVER_STATUS_WAITING:
+					return OK; // Still resolving
+
+				case IP::RESOLVER_STATUS_DONE: {
+					IPAddress host = IP::get_singleton()->get_resolve_item_address(resolving);
+					Error err = tcp_connection->connect_to_host(host, conn_port);
+					IP::get_singleton()->erase_resolve_item(resolving);
+					resolving = IP::RESOLVER_INVALID_ID;
+					if (err) {
+						status = STATUS_CANT_CONNECT;
+						return err;
+					}
+
+					status = STATUS_CONNECTING;
+				} break;
+				case IP::RESOLVER_STATUS_NONE:
+				case IP::RESOLVER_STATUS_ERROR: {
+					IP::get_singleton()->erase_resolve_item(resolving);
+					resolving = IP::RESOLVER_INVALID_ID;
+					close();
+					status = STATUS_CANT_RESOLVE;
+					return ERR_CANT_RESOLVE;
+				} break;
+			}
+		} break;
+		case STATUS_CONNECTING: {
+			StreamPeerTCP::Status s = tcp_connection->get_status();
+			switch (s) {
+				case StreamPeerTCP::STATUS_CONNECTING: {
+					return OK;
+				} break;
+				case StreamPeerTCP::STATUS_CONNECTED: {
+					if (ssl) {
+						Ref<StreamPeerSSL> ssl;
+						if (!handshaking) {
+							// Connect the StreamPeerSSL and start handshaking
+							ssl = Ref<StreamPeerSSL>(StreamPeerSSL::create());
+							ssl->set_blocking_handshake_enabled(false);
+							Error err = ssl->connect_to_stream(tcp_connection, ssl_verify_host, conn_host);
+							if (err != OK) {
+								close();
+								status = STATUS_SSL_HANDSHAKE_ERROR;
+								return ERR_CANT_CONNECT;
+							}
+							connection = ssl;
+							handshaking = true;
+						} else {
+							// We are already handshaking, which means we can use your already active SSL connection
+							ssl = static_cast<Ref<StreamPeerSSL>>(connection);
+							if (ssl.is_null()) {
+								close();
+								status = STATUS_SSL_HANDSHAKE_ERROR;
+								return ERR_CANT_CONNECT;
+							}
+
+							ssl->poll(); // Try to finish the handshake
+						}
+
+						if (ssl->get_status() == StreamPeerSSL::STATUS_CONNECTED) {
+							// Handshake has been successful
+							handshaking = false;
+							status = STATUS_CONNECTED;
+							return OK;
+						} else if (ssl->get_status() != StreamPeerSSL::STATUS_HANDSHAKING) {
+							// Handshake has failed
+							close();
+							status = STATUS_SSL_HANDSHAKE_ERROR;
+							return ERR_CANT_CONNECT;
+						}
+						// ... we will need to poll more for handshake to finish
+					} else {
+						status = STATUS_CONNECTED;
+					}
+					return OK;
+				} break;
+				case StreamPeerTCP::STATUS_ERROR:
+				case StreamPeerTCP::STATUS_NONE: {
+					close();
+					status = STATUS_CANT_CONNECT;
+					return ERR_CANT_CONNECT;
+				} break;
+			}
+		} break;
+		case STATUS_BODY:
+		case STATUS_CONNECTED: {
+			// Check if we are still connected
+			if (ssl) {
+				Ref<StreamPeerSSL> tmp = connection;
+				tmp->poll();
+				if (tmp->get_status() != StreamPeerSSL::STATUS_CONNECTED) {
+					status = STATUS_CONNECTION_ERROR;
+					return ERR_CONNECTION_ERROR;
+				}
+			} else if (tcp_connection->get_status() != StreamPeerTCP::STATUS_CONNECTED) {
+				status = STATUS_CONNECTION_ERROR;
+				return ERR_CONNECTION_ERROR;
+			}
+			// Connection established, requests can now be made
+			return OK;
+		} break;
+		case STATUS_REQUESTING: {
+			while (true) {
+				uint8_t byte;
+				int rec = 0;
+				Error err = _get_http_data(&byte, 1, rec);
+				if (err != OK) {
+					close();
+					status = STATUS_CONNECTION_ERROR;
+					return ERR_CONNECTION_ERROR;
+				}
+
+				if (rec == 0) {
+					return OK; // Still requesting, keep trying!
+				}
+
+				response_str.push_back(byte);
+				int rs = response_str.size();
+				if (
+						(rs >= 2 && response_str[rs - 2] == '\n' && response_str[rs - 1] == '\n') ||
+						(rs >= 4 && response_str[rs - 4] == '\r' && response_str[rs - 3] == '\n' && response_str[rs - 2] == '\r' && response_str[rs - 1] == '\n')) {
+					// End of response, parse.
+					response_str.push_back(0);
+					String response;
+					response.parse_utf8((const char *)response_str.ptr());
+					Vector<String> responses = response.split("\n");
+					body_size = -1;
+					chunked = false;
+					body_left = 0;
+					chunk_left = 0;
+					chunk_trailer_part = false;
+					read_until_eof = false;
+					response_str.clear();
+					response_headers.clear();
+					response_num = RESPONSE_OK;
+
+					// Per the HTTP 1.1 spec, keep-alive is the default.
+					// Not following that specification breaks standard implementations.
+					// Broken web servers should be fixed.
+					bool keep_alive = true;
+
+					for (int i = 0; i < responses.size(); i++) {
+						String header = responses[i].strip_edges();
+						String s = header.to_lower();
+						if (s.length() == 0) {
+							continue;
+						}
+						if (s.begins_with("content-length:")) {
+							body_size = s.substr(s.find(":") + 1, s.length()).strip_edges().to_int();
+							body_left = body_size;
+
+						} else if (s.begins_with("transfer-encoding:")) {
+							String encoding = header.substr(header.find(":") + 1, header.length()).strip_edges();
+							if (encoding == "chunked") {
+								chunked = true;
+							}
+						} else if (s.begins_with("connection: close")) {
+							keep_alive = false;
+						}
+
+						if (i == 0 && responses[i].begins_with("HTTP")) {
+							String num = responses[i].get_slicec(' ', 1);
+							response_num = num.to_int();
+						} else {
+							response_headers.push_back(header);
+						}
+					}
+
+					// This is a HEAD request, we won't receive anything.
+					if (head_request) {
+						body_size = 0;
+						body_left = 0;
+					}
+
+					if (body_size != -1 || chunked) {
+						status = STATUS_BODY;
+					} else if (!keep_alive) {
+						read_until_eof = true;
+						status = STATUS_BODY;
+					} else {
+						status = STATUS_CONNECTED;
+					}
+					return OK;
+				}
+			}
+		} break;
+		case STATUS_DISCONNECTED: {
+			return ERR_UNCONFIGURED;
+		} break;
+		case STATUS_CONNECTION_ERROR:
+		case STATUS_SSL_HANDSHAKE_ERROR: {
+			return ERR_CONNECTION_ERROR;
+		} break;
+		case STATUS_CANT_CONNECT: {
+			return ERR_CANT_CONNECT;
+		} break;
+		case STATUS_CANT_RESOLVE: {
+			return ERR_CANT_RESOLVE;
+		} break;
+	}
+
+	return OK;
+}
+
+int HTTPClientTCP::get_response_body_length() const {
+	return body_size;
+}
+
+PackedByteArray HTTPClientTCP::read_response_body_chunk() {
+	ERR_FAIL_COND_V(status != STATUS_BODY, PackedByteArray());
+
+	PackedByteArray ret;
+	Error err = OK;
+
+	if (chunked) {
+		while (true) {
+			if (chunk_trailer_part) {
+				// We need to consume the trailer part too or keep-alive will break
+				uint8_t b;
+				int rec = 0;
+				err = _get_http_data(&b, 1, rec);
+
+				if (rec == 0) {
+					break;
+				}
+
+				chunk.push_back(b);
+				int cs = chunk.size();
+				if ((cs >= 2 && chunk[cs - 2] == '\r' && chunk[cs - 1] == '\n')) {
+					if (cs == 2) {
+						// Finally over
+						chunk_trailer_part = false;
+						status = STATUS_CONNECTED;
+						chunk.clear();
+						break;
+					} else {
+						// We do not process nor return the trailer data
+						chunk.clear();
+					}
+				}
+			} else if (chunk_left == 0) {
+				// Reading length
+				uint8_t b;
+				int rec = 0;
+				err = _get_http_data(&b, 1, rec);
+
+				if (rec == 0) {
+					break;
+				}
+
+				chunk.push_back(b);
+
+				if (chunk.size() > 32) {
+					ERR_PRINT("HTTP Invalid chunk hex len");
+					status = STATUS_CONNECTION_ERROR;
+					break;
+				}
+
+				if (chunk.size() > 2 && chunk[chunk.size() - 2] == '\r' && chunk[chunk.size() - 1] == '\n') {
+					int len = 0;
+					for (int i = 0; i < chunk.size() - 2; i++) {
+						char c = chunk[i];
+						int v = 0;
+						if (c >= '0' && c <= '9') {
+							v = c - '0';
+						} else if (c >= 'a' && c <= 'f') {
+							v = c - 'a' + 10;
+						} else if (c >= 'A' && c <= 'F') {
+							v = c - 'A' + 10;
+						} else {
+							ERR_PRINT("HTTP Chunk len not in hex!!");
+							status = STATUS_CONNECTION_ERROR;
+							break;
+						}
+						len <<= 4;
+						len |= v;
+						if (len > (1 << 24)) {
+							ERR_PRINT("HTTP Chunk too big!! >16mb");
+							status = STATUS_CONNECTION_ERROR;
+							break;
+						}
+					}
+
+					if (len == 0) {
+						// End reached!
+						chunk_trailer_part = true;
+						chunk.clear();
+						break;
+					}
+
+					chunk_left = len + 2;
+					chunk.resize(chunk_left);
+				}
+			} else {
+				int rec = 0;
+				err = _get_http_data(&chunk.write[chunk.size() - chunk_left], chunk_left, rec);
+				if (rec == 0) {
+					break;
+				}
+				chunk_left -= rec;
+
+				if (chunk_left == 0) {
+					if (chunk[chunk.size() - 2] != '\r' || chunk[chunk.size() - 1] != '\n') {
+						ERR_PRINT("HTTP Invalid chunk terminator (not \\r\\n)");
+						status = STATUS_CONNECTION_ERROR;
+						break;
+					}
+
+					ret.resize(chunk.size() - 2);
+					uint8_t *w = ret.ptrw();
+					memcpy(w, chunk.ptr(), chunk.size() - 2);
+					chunk.clear();
+				}
+
+				break;
+			}
+		}
+
+	} else {
+		int to_read = !read_until_eof ? MIN(body_left, read_chunk_size) : read_chunk_size;
+		ret.resize(to_read);
+		int _offset = 0;
+		while (to_read > 0) {
+			int rec = 0;
+			{
+				uint8_t *w = ret.ptrw();
+				err = _get_http_data(w + _offset, to_read, rec);
+			}
+			if (rec <= 0) { // Ended up reading less
+				ret.resize(_offset);
+				break;
+			} else {
+				_offset += rec;
+				to_read -= rec;
+				if (!read_until_eof) {
+					body_left -= rec;
+				}
+			}
+			if (err != OK) {
+				break;
+			}
+		}
+	}
+
+	if (err != OK) {
+		close();
+
+		if (err == ERR_FILE_EOF) {
+			status = STATUS_DISCONNECTED; // Server disconnected
+		} else {
+			status = STATUS_CONNECTION_ERROR;
+		}
+	} else if (body_left == 0 && !chunked && !read_until_eof) {
+		status = STATUS_CONNECTED;
+	}
+
+	return ret;
+}
+
+HTTPClientTCP::Status HTTPClientTCP::get_status() const {
+	return status;
+}
+
+void HTTPClientTCP::set_blocking_mode(bool p_enable) {
+	blocking = p_enable;
+}
+
+bool HTTPClientTCP::is_blocking_mode_enabled() const {
+	return blocking;
+}
+
+Error HTTPClientTCP::_get_http_data(uint8_t *p_buffer, int p_bytes, int &r_received) {
+	if (blocking) {
+		// We can't use StreamPeer.get_data, since when reaching EOF we will get an
+		// error without knowing how many bytes we received.
+		Error err = ERR_FILE_EOF;
+		int read = 0;
+		int left = p_bytes;
+		r_received = 0;
+		while (left > 0) {
+			err = connection->get_partial_data(p_buffer + r_received, left, read);
+			if (err == OK) {
+				r_received += read;
+			} else if (err == ERR_FILE_EOF) {
+				r_received += read;
+				return err;
+			} else {
+				return err;
+			}
+			left -= read;
+		}
+		return err;
+	} else {
+		return connection->get_partial_data(p_buffer, p_bytes, r_received);
+	}
+}
+
+void HTTPClientTCP::set_read_chunk_size(int p_size) {
+	ERR_FAIL_COND(p_size < 256 || p_size > (1 << 24));
+	read_chunk_size = p_size;
+}
+
+int HTTPClientTCP::get_read_chunk_size() const {
+	return read_chunk_size;
+}
+
+HTTPClientTCP::HTTPClientTCP() {
+	tcp_connection.instantiate();
+}
+
+HTTPClient *(*HTTPClient::_create)() = HTTPClientTCP::_create_func;
+
+#endif // #ifndef JAVASCRIPT_ENABLED

platform/javascript/http_client.h.inc → core/io/http_client_tcp.h

@@ -1,5 +1,5 @@
 /*************************************************************************/
-/*  http_client.h.inc                                                    */
+/*  http_client_tcp.h                                                    */
 /*************************************************************************/
 /*                       This file is part of:                           */
 /*                           GODOT ENGINE                                */
@@ -28,24 +28,65 @@
 /* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.                */
 /*************************************************************************/
 
-// HTTPClient's additional private members in the javascript platform
+#ifndef HTTP_CLIENT_TCP_H
+#define HTTP_CLIENT_TCP_H
 
-Error make_request(Method p_method, const String &p_url, const Vector<String> &p_headers, const uint8_t *p_body, int p_body_len);
-static void _parse_headers(int p_len, const char **p_headers, void *p_ref);
+#include "http_client.h"
 
-int js_id = 0;
-// 64 KiB by default (favors fast download speeds at the cost of memory usage).
-int read_limit = 65536;
-Status status = STATUS_DISCONNECTED;
+class HTTPClientTCP : public HTTPClient {
+private:
+	Status status = STATUS_DISCONNECTED;
+	IP::ResolverID resolving = IP::RESOLVER_INVALID_ID;
+	int conn_port = -1;
+	String conn_host;
+	bool ssl = false;
+	bool ssl_verify_host = false;
+	bool blocking = false;
+	bool handshaking = false;
+	bool head_request = false;
 
-String host;
-int port = -1;
-bool use_tls = false;
+	Vector<uint8_t> response_str;
 
-int polled_response_code = 0;
-Vector<String> response_headers;
-Vector<uint8_t> response_buffer;
+	bool chunked = false;
+	Vector<uint8_t> chunk;
+	int chunk_left = 0;
+	bool chunk_trailer_part = false;
+	int body_size = -1;
+	int body_left = 0;
+	bool read_until_eof = false;
 
-#ifdef DEBUG_ENABLED
-uint64_t last_polling_frame = 0;
-#endif
+	Ref<StreamPeerTCP> tcp_connection;
+	Ref<StreamPeer> connection;
+
+	int response_num = 0;
+	Vector<String> response_headers;
+	// 64 KiB by default (favors fast download speeds at the cost of memory usage).
+	int read_chunk_size = 65536;
+
+	Error _get_http_data(uint8_t *p_buffer, int p_bytes, int &r_received);
+
+public:
+	static HTTPClient *_create_func();
+
+	Error request(Method p_method, const String &p_url, const Vector<String> &p_headers, const uint8_t *p_body, int p_body_size) override;
+
+	Error connect_to_host(const String &p_host, int p_port = -1, bool p_ssl = false, bool p_verify_host = true) override;
+	void set_connection(const Ref<StreamPeer> &p_connection) override;
+	Ref<StreamPeer> get_connection() const override;
+	void close() override;
+	Status get_status() const override;
+	bool has_response() const override;
+	bool is_response_chunked() const override;
+	int get_response_code() const override;
+	Error get_response_headers(List<String> *r_response) override;
+	int get_response_body_length() const override;
+	PackedByteArray read_response_body_chunk() override;
+	void set_blocking_mode(bool p_enable) override;
+	bool is_blocking_mode_enabled() const override;
+	void set_read_chunk_size(int p_size) override;
+	int get_read_chunk_size() const override;
+	Error poll() override;
+	HTTPClientTCP();
+};
+
+#endif // HTTP_CLIENT_TCP_H

core/register_core_types.cpp

@@ -153,14 +153,20 @@ void register_core_types() {
 	ClassDB::register_class<InputEventPanGesture>();
 	ClassDB::register_class<InputEventMIDI>();
 
+	// Network
+	ClassDB::register_virtual_class<IP>();
+
 	ClassDB::register_virtual_class<StreamPeer>();
 	ClassDB::register_class<StreamPeerBuffer>();
 	ClassDB::register_class<StreamPeerTCP>();
 	ClassDB::register_class<TCPServer>();
+
+	ClassDB::register_virtual_class<PacketPeer>();
+	ClassDB::register_class<PacketPeerStream>();
 	ClassDB::register_class<PacketPeerUDP>();
 	ClassDB::register_class<UDPServer>();
-	ClassDB::register_custom_instance_class<PacketPeerDTLS>();
-	ClassDB::register_custom_instance_class<DTLSServer>();
+
+	ClassDB::register_custom_instance_class<HTTPClient>();
 
 	// Crypto
 	ClassDB::register_class<HashingContext>();
@@ -170,22 +176,20 @@ void register_core_types() {
 	ClassDB::register_custom_instance_class<HMACContext>();
 	ClassDB::register_custom_instance_class<Crypto>();
 	ClassDB::register_custom_instance_class<StreamPeerSSL>();
+	ClassDB::register_custom_instance_class<PacketPeerDTLS>();
+	ClassDB::register_custom_instance_class<DTLSServer>();
 
 	resource_format_saver_crypto.instantiate();
 	ResourceSaver::add_resource_format_saver(resource_format_saver_crypto);
 	resource_format_loader_crypto.instantiate();
 	ResourceLoader::add_resource_format_loader(resource_format_loader_crypto);
 
-	ClassDB::register_virtual_class<IP>();
-	ClassDB::register_virtual_class<PacketPeer>();
-	ClassDB::register_class<PacketPeerStream>();
 	ClassDB::register_virtual_class<NetworkedMultiplayerPeer>();
 	ClassDB::register_class<MultiplayerAPI>();
 	ClassDB::register_class<MainLoop>();
 	ClassDB::register_class<Translation>();
 	ClassDB::register_class<OptimizedTranslation>();
 	ClassDB::register_class<UndoRedo>();
-	ClassDB::register_class<HTTPClient>();
 	ClassDB::register_class<TriangleMesh>();
 
 	ClassDB::register_class<ResourceFormatLoader>();

platform/javascript/http_client_javascript.cpp

@@ -28,45 +28,19 @@
 /* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.                */
 /*************************************************************************/
 
-#include "core/io/http_client.h"
+#include "http_client_javascript.h"
 
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include "stddef.h"
-
-typedef enum {
-	GODOT_JS_FETCH_STATE_REQUESTING = 0,
-	GODOT_JS_FETCH_STATE_BODY = 1,
-	GODOT_JS_FETCH_STATE_DONE = 2,
-	GODOT_JS_FETCH_STATE_ERROR = -1,
-} godot_js_fetch_state_t;
-
-extern int godot_js_fetch_create(const char *p_method, const char *p_url, const char **p_headers, int p_headers_len, const uint8_t *p_body, int p_body_len);
-extern int godot_js_fetch_read_headers(int p_id, void (*parse_callback)(int p_size, const char **p_headers, void *p_ref), void *p_ref);
-extern int godot_js_fetch_read_chunk(int p_id, uint8_t *p_buf, int p_buf_size);
-extern void godot_js_fetch_free(int p_id);
-extern godot_js_fetch_state_t godot_js_fetch_state_get(int p_id);
-extern int godot_js_fetch_body_length_get(int p_id);
-extern int godot_js_fetch_http_status_get(int p_id);
-extern int godot_js_fetch_is_chunked(int p_id);
-
-#ifdef __cplusplus
-}
-#endif
-
-void HTTPClient::_parse_headers(int p_len, const char **p_headers, void *p_ref) {
-	HTTPClient *client = static_cast<HTTPClient *>(p_ref);
+void HTTPClientJavaScript::_parse_headers(int p_len, const char **p_headers, void *p_ref) {
+	HTTPClientJavaScript *client = static_cast<HTTPClientJavaScript *>(p_ref);
 	for (int i = 0; i < p_len; i++) {
 		client->response_headers.push_back(String::utf8(p_headers[i]));
 	}
 }
 
-Error HTTPClient::connect_to_host(const String &p_host, int p_port, bool p_ssl, bool p_verify_host) {
+Error HTTPClientJavaScript::connect_to_host(const String &p_host, int p_port, bool p_ssl, bool p_verify_host) {
 	close();
 	if (p_ssl && !p_verify_host) {
-		WARN_PRINT("Disabling HTTPClient's host verification is not supported for the HTML5 platform, host will be verified");
+		WARN_PRINT("Disabling HTTPClientJavaScript's host verification is not supported for the HTML5 platform, host will be verified");
 	}
 
 	port = p_port;
@@ -97,15 +71,15 @@ Error HTTPClient::connect_to_host(const String &p_host, int p_port, bool p_ssl,
 	return OK;
 }
 
-void HTTPClient::set_connection(const Ref<StreamPeer> &p_connection) {
-	ERR_FAIL_MSG("Accessing an HTTPClient's StreamPeer is not supported for the HTML5 platform.");
+void HTTPClientJavaScript::set_connection(const Ref<StreamPeer> &p_connection) {
+	ERR_FAIL_MSG("Accessing an HTTPClientJavaScript's StreamPeer is not supported for the HTML5 platform.");
 }
 
-Ref<StreamPeer> HTTPClient::get_connection() const {
-	ERR_FAIL_V_MSG(REF(), "Accessing an HTTPClient's StreamPeer is not supported for the HTML5 platform.");
+Ref<StreamPeer> HTTPClientJavaScript::get_connection() const {
+	ERR_FAIL_V_MSG(REF(), "Accessing an HTTPClientJavaScript's StreamPeer is not supported for the HTML5 platform.");
 }
 
-Error HTTPClient::make_request(Method p_method, const String &p_url, const Vector<String> &p_headers, const uint8_t *p_body, int p_body_len) {
+Error HTTPClientJavaScript::request(Method p_method, const String &p_url, const Vector<String> &p_headers, const uint8_t *p_body, int p_body_len) {
 	ERR_FAIL_INDEX_V(p_method, METHOD_MAX, ERR_INVALID_PARAMETER);
 	ERR_FAIL_COND_V_MSG(p_method == METHOD_TRACE || p_method == METHOD_CONNECT, ERR_UNAVAILABLE, "HTTP methods TRACE and CONNECT are not supported for the HTML5 platform.");
 	ERR_FAIL_COND_V(status != STATUS_CONNECTED, ERR_INVALID_PARAMETER);
@@ -128,22 +102,7 @@ Error HTTPClient::make_request(Method p_method, const String &p_url, const Vecto
 	return OK;
 }
 
-Error HTTPClient::request_raw(Method p_method, const String &p_url, const Vector<String> &p_headers, const Vector<uint8_t> &p_body) {
-	if (p_body.is_empty()) {
-		return make_request(p_method, p_url, p_headers, nullptr, 0);
-	}
-	return make_request(p_method, p_url, p_headers, p_body.ptr(), p_body.size());
-}
-
-Error HTTPClient::request(Method p_method, const String &p_url, const Vector<String> &p_headers, const String &p_body) {
-	if (p_body.is_empty()) {
-		return make_request(p_method, p_url, p_headers, nullptr, 0);
-	}
-	const CharString cs = p_body.utf8();
-	return make_request(p_method, p_url, p_headers, (const uint8_t *)cs.get_data(), cs.size() - 1);
-}
-
-void HTTPClient::close() {
+void HTTPClientJavaScript::close() {
 	host = "";
 	port = -1;
 	use_tls = false;
@@ -157,23 +116,23 @@ void HTTPClient::close() {
 	}
 }
 
-HTTPClient::Status HTTPClient::get_status() const {
+HTTPClientJavaScript::Status HTTPClientJavaScript::get_status() const {
 	return status;
 }
 
-bool HTTPClient::has_response() const {
+bool HTTPClientJavaScript::has_response() const {
 	return response_headers.size() > 0;
 }
 
-bool HTTPClient::is_response_chunked() const {
+bool HTTPClientJavaScript::is_response_chunked() const {
 	return godot_js_fetch_is_chunked(js_id);
 }
 
-int HTTPClient::get_response_code() const {
+int HTTPClientJavaScript::get_response_code() const {
 	return polled_response_code;
 }
 
-Error HTTPClient::get_response_headers(List<String> *r_response) {
+Error HTTPClientJavaScript::get_response_headers(List<String> *r_response) {
 	if (!response_headers.size()) {
 		return ERR_INVALID_PARAMETER;
 	}
@@ -184,11 +143,11 @@ Error HTTPClient::get_response_headers(List<String> *r_response) {
 	return OK;
 }
 
-int HTTPClient::get_response_body_length() const {
+int HTTPClientJavaScript::get_response_body_length() const {
 	return godot_js_fetch_body_length_get(js_id);
 }
 
-PackedByteArray HTTPClient::read_response_body_chunk() {
+PackedByteArray HTTPClientJavaScript::read_response_body_chunk() {
 	ERR_FAIL_COND_V(status != STATUS_BODY, PackedByteArray());
 
 	if (response_buffer.size() != read_limit) {
@@ -213,23 +172,23 @@ PackedByteArray HTTPClient::read_response_body_chunk() {
 	return chunk;
 }
 
-void HTTPClient::set_blocking_mode(bool p_enable) {
-	ERR_FAIL_COND_MSG(p_enable, "HTTPClient blocking mode is not supported for the HTML5 platform.");
+void HTTPClientJavaScript::set_blocking_mode(bool p_enable) {
+	ERR_FAIL_COND_MSG(p_enable, "HTTPClientJavaScript blocking mode is not supported for the HTML5 platform.");
 }
 
-bool HTTPClient::is_blocking_mode_enabled() const {
+bool HTTPClientJavaScript::is_blocking_mode_enabled() const {
 	return false;
 }
 
-void HTTPClient::set_read_chunk_size(int p_size) {
+void HTTPClientJavaScript::set_read_chunk_size(int p_size) {
 	read_limit = p_size;
 }
 
-int HTTPClient::get_read_chunk_size() const {
+int HTTPClientJavaScript::get_read_chunk_size() const {
 	return read_limit;
 }
 
-Error HTTPClient::poll() {
+Error HTTPClientJavaScript::poll() {
 	switch (status) {
 		case STATUS_DISCONNECTED:
 			return ERR_UNCONFIGURED;
@@ -263,7 +222,7 @@ Error HTTPClient::poll() {
 #ifdef DEBUG_ENABLED
 			// forcing synchronous requests is not possible on the web
 			if (last_polling_frame == Engine::get_singleton()->get_process_frames()) {
-				WARN_PRINT("HTTPClient polled multiple times in one frame, "
+				WARN_PRINT("HTTPClientJavaScript polled multiple times in one frame, "
 						   "but request cannot progress more than once per "
 						   "frame on the HTML5 platform.");
 			}
@@ -294,9 +253,15 @@ Error HTTPClient::poll() {
 	return OK;
 }
 
-HTTPClient::HTTPClient() {
+HTTPClient *HTTPClientJavaScript::_create_func() {
+	return memnew(HTTPClientJavaScript);
+}
+
+HTTPClient *(*HTTPClient::_create)() = HTTPClientJavaScript::_create_func;
+
+HTTPClientJavaScript::HTTPClientJavaScript() {
 }
 
-HTTPClient::~HTTPClient() {
+HTTPClientJavaScript::~HTTPClientJavaScript() {
 	close();
 }

platform/javascript/http_client_javascript.h

@@ -0,0 +1,108 @@
+/*************************************************************************/
+/*  http_client_javascript.h                                             */
+/*************************************************************************/
+/*                       This file is part of:                           */
+/*                           GODOT ENGINE                                */
+/*                      https://godotengine.org                          */
+/*************************************************************************/
+/* Copyright (c) 2007-2021 Juan Linietsky, Ariel Manzur.                 */
+/* Copyright (c) 2014-2021 Godot Engine contributors (cf. AUTHORS.md).   */
+/*                                                                       */
+/* Permission is hereby granted, free of charge, to any person obtaining */
+/* a copy of this software and associated documentation files (the       */
+/* "Software"), to deal in the Software without restriction, including   */
+/* without limitation the rights to use, copy, modify, merge, publish,   */
+/* distribute, sublicense, and/or sell copies of the Software, and to    */
+/* permit persons to whom the Software is furnished to do so, subject to */
+/* the following conditions:                                             */
+/*                                                                       */
+/* The above copyright notice and this permission notice shall be        */
+/* included in all copies or substantial portions of the Software.       */
+/*                                                                       */
+/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,       */
+/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF    */
+/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*/
+/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY  */
+/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,  */
+/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE     */
+/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.                */
+/*************************************************************************/
+
+#ifndef HTTP_CLIENT_JAVASCRIPT_H
+#define HTTP_CLIENT_JAVASCRIPT_H
+
+#include "core/io/http_client.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "stddef.h"
+
+typedef enum {
+	GODOT_JS_FETCH_STATE_REQUESTING = 0,
+	GODOT_JS_FETCH_STATE_BODY = 1,
+	GODOT_JS_FETCH_STATE_DONE = 2,
+	GODOT_JS_FETCH_STATE_ERROR = -1,
+} godot_js_fetch_state_t;
+
+extern int godot_js_fetch_create(const char *p_method, const char *p_url, const char **p_headers, int p_headers_len, const uint8_t *p_body, int p_body_len);
+extern int godot_js_fetch_read_headers(int p_id, void (*parse_callback)(int p_size, const char **p_headers, void *p_ref), void *p_ref);
+extern int godot_js_fetch_read_chunk(int p_id, uint8_t *p_buf, int p_buf_size);
+extern void godot_js_fetch_free(int p_id);
+extern godot_js_fetch_state_t godot_js_fetch_state_get(int p_id);
+extern int godot_js_fetch_body_length_get(int p_id);
+extern int godot_js_fetch_http_status_get(int p_id);
+extern int godot_js_fetch_is_chunked(int p_id);
+
+#ifdef __cplusplus
+}
+#endif
+
+class HTTPClientJavaScript : public HTTPClient {
+private:
+	int js_id = 0;
+	Status status = STATUS_DISCONNECTED;
+
+	// 64 KiB by default (favors fast download speeds at the cost of memory usage).
+	int read_limit = 65536;
+
+	String host;
+	int port = -1;
+	bool use_tls = false;
+
+	int polled_response_code = 0;
+	Vector<String> response_headers;
+	Vector<uint8_t> response_buffer;
+
+#ifdef DEBUG_ENABLED
+	uint64_t last_polling_frame = 0;
+#endif
+
+	static void _parse_headers(int p_len, const char **p_headers, void *p_ref);
+
+public:
+	static HTTPClient *_create_func();
+
+	Error request(Method p_method, const String &p_url, const Vector<String> &p_headers, const uint8_t *p_body, int p_body_size) override;
+
+	Error connect_to_host(const String &p_host, int p_port = -1, bool p_ssl = false, bool p_verify_host = true) override;
+	void set_connection(const Ref<StreamPeer> &p_connection) override;
+	Ref<StreamPeer> get_connection() const override;
+	void close() override;
+	Status get_status() const override;
+	bool has_response() const override;
+	bool is_response_chunked() const override;
+	int get_response_code() const override;
+	Error get_response_headers(List<String> *r_response) override;
+	int get_response_body_length() const override;
+	PackedByteArray read_response_body_chunk() override;
+	void set_blocking_mode(bool p_enable) override;
+	bool is_blocking_mode_enabled() const override;
+	void set_read_chunk_size(int p_size) override;
+	int get_read_chunk_size() const override;
+	Error poll() override;
+	HTTPClientJavaScript();
+	~HTTPClientJavaScript();
+};
+#endif // HTTP_CLIENT_JAVASCRIPT_H

scene/main/http_request.cpp

@@ -322,7 +322,8 @@ bool HTTPRequest::_update_connection() {
 			} else {
 				// Did not request yet, do request
 
-				Error err = client->request_raw(method, request_string, headers, request_data);
+				int size = request_data.size();
+				Error err = client->request(method, request_string, headers, size > 0 ? request_data.ptr() : nullptr, size);
 				if (err != OK) {
 					call_deferred("_request_done", RESULT_CONNECTION_ERROR, 0, PackedStringArray(), PackedByteArray());
 					return true;
@@ -627,7 +628,7 @@ void HTTPRequest::_bind_methods() {
 }
 
 HTTPRequest::HTTPRequest() {
-	client.instantiate();
+	client = Ref<HTTPClient>(HTTPClient::create());
 	timer = memnew(Timer);
 	timer->set_one_shot(true);
 	timer->connect("timeout", callable_mp(this, &HTTPRequest::_timeout));