diff options
-rw-r--r-- | core/io/compression.cpp | 87 | ||||
-rw-r--r-- | core/io/compression.h | 3 | ||||
-rw-r--r-- | core/variant_call.cpp | 18 | ||||
-rw-r--r-- | doc/classes/HTTPRequest.xml | 31 | ||||
-rw-r--r-- | doc/classes/PackedByteArray.xml | 14 | ||||
-rw-r--r-- | scene/main/http_request.cpp | 113 | ||||
-rw-r--r-- | scene/main/http_request.h | 13 |
7 files changed, 272 insertions, 7 deletions
diff --git a/core/io/compression.cpp b/core/io/compression.cpp index 99ca8107e4..7480262835 100644 --- a/core/io/compression.cpp +++ b/core/io/compression.cpp @@ -180,8 +180,95 @@ int Compression::decompress(uint8_t *p_dst, int p_dst_max_size, const uint8_t *p ERR_FAIL_V(-1); } +/** + This will handle both Gzip and Deflat streams. It will automatically allocate the output buffer into the provided p_dst_vect Vector. + This is required for compressed data who's final uncompressed size is unknown, as is the case for HTTP response bodies. + This is much slower however than using Compression::decompress because it may result in multiple full copies of the output buffer. +*/ +int Compression::decompress_dynamic(Vector<uint8_t> *p_dst_vect, int p_max_dst_size, const uint8_t *p_src, int p_src_size, Mode p_mode) { + int ret; + uint8_t *dst = nullptr; + int out_mark = 0; + z_stream strm; + + ERR_FAIL_COND_V(p_src_size <= 0, Z_DATA_ERROR); + + // This function only supports GZip and Deflate + int window_bits = p_mode == MODE_DEFLATE ? 15 : 15 + 16; + ERR_FAIL_COND_V(p_mode != MODE_DEFLATE && p_mode != MODE_GZIP, Z_ERRNO); + + // Initialize the stream + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + strm.opaque = Z_NULL; + strm.avail_in = 0; + strm.next_in = Z_NULL; + + int err = inflateInit2(&strm, window_bits); + ERR_FAIL_COND_V(err != Z_OK, -1); + + // Setup the stream inputs + strm.next_in = (Bytef *)p_src; + strm.avail_in = p_src_size; + + // Ensure the destination buffer is empty + p_dst_vect->resize(0); + + // decompress until deflate stream ends or end of file + do { + // Add another chunk size to the output buffer + // This forces a copy of the whole buffer + p_dst_vect->resize(p_dst_vect->size() + gzip_chunk); + // Get pointer to the actual output buffer + dst = p_dst_vect->ptrw(); + + // Set the stream to the new output stream + // Since it was copied, we need to reset the stream to the new buffer + strm.next_out = &(dst[out_mark]); + strm.avail_out = gzip_chunk; + + // run inflate() on input until output buffer is full and needs to be resized + // or input runs out + do { + ret = inflate(&strm, Z_SYNC_FLUSH); + + switch (ret) { + case Z_NEED_DICT: + ret = Z_DATA_ERROR; + [[fallthrough]]; + case Z_DATA_ERROR: + case Z_MEM_ERROR: + case Z_STREAM_ERROR: + WARN_PRINT(strm.msg); + (void)inflateEnd(&strm); + p_dst_vect->resize(0); + return ret; + } + } while (strm.avail_out > 0 && strm.avail_in > 0); + + out_mark += gzip_chunk; + + // Encorce max output size + if (p_max_dst_size > -1 && strm.total_out > (uint64_t)p_max_dst_size) { + (void)inflateEnd(&strm); + p_dst_vect->resize(0); + return Z_BUF_ERROR; + } + } while (ret != Z_STREAM_END); + + // If all done successfully, resize the output if it's larger than the actual output + if (ret == Z_STREAM_END && (unsigned long)p_dst_vect->size() > strm.total_out) { + p_dst_vect->resize(strm.total_out); + } + + // clean up and return + (void)inflateEnd(&strm); + return ret == Z_STREAM_END ? Z_OK : Z_DATA_ERROR; +} + int Compression::zlib_level = Z_DEFAULT_COMPRESSION; int Compression::gzip_level = Z_DEFAULT_COMPRESSION; int Compression::zstd_level = 3; bool Compression::zstd_long_distance_matching = false; int Compression::zstd_window_log_size = 27; // ZSTD_WINDOWLOG_LIMIT_DEFAULT +int Compression::gzip_chunk = 16384; diff --git a/core/io/compression.h b/core/io/compression.h index f195f96ba5..c103fa8eae 100644 --- a/core/io/compression.h +++ b/core/io/compression.h @@ -32,6 +32,7 @@ #define COMPRESSION_H #include "core/typedefs.h" +#include "core/vector.h" class Compression { public: @@ -40,6 +41,7 @@ public: static int zstd_level; static bool zstd_long_distance_matching; static int zstd_window_log_size; + static int gzip_chunk; enum Mode { MODE_FASTLZ, @@ -51,6 +53,7 @@ public: static int compress(uint8_t *p_dst, const uint8_t *p_src, int p_src_size, Mode p_mode = MODE_ZSTD); static int get_max_compressed_buffer_size(int p_src_size, Mode p_mode = MODE_ZSTD); static int decompress(uint8_t *p_dst, int p_dst_max_size, const uint8_t *p_src, int p_src_size, Mode p_mode = MODE_ZSTD); + static int decompress_dynamic(Vector<uint8_t> *p_dst_vect, int p_max_dst_size, const uint8_t *p_src, int p_src_size, Mode p_mode); Compression() {} }; diff --git a/core/variant_call.cpp b/core/variant_call.cpp index d150ae28d7..0ebb2f04a1 100644 --- a/core/variant_call.cpp +++ b/core/variant_call.cpp @@ -712,6 +712,23 @@ struct _VariantCall { r_ret = decompressed; } + static void _call_PackedByteArray_decompress_dynamic(Variant &r_ret, Variant &p_self, const Variant **p_args) { + Variant::PackedArrayRef<uint8_t> *ba = reinterpret_cast<Variant::PackedArrayRef<uint8_t> *>(p_self._data.packed_array); + PackedByteArray decompressed; + int max_output_size = (int)(*p_args[0]); + Compression::Mode mode = (Compression::Mode)(int)(*p_args[1]); + + int result = Compression::decompress_dynamic(&decompressed, max_output_size, ba->array.ptr(), ba->array.size(), mode); + + if (result == OK) { + r_ret = decompressed; + } else { + decompressed.clear(); + r_ret = decompressed; + ERR_FAIL_MSG("Decompression failed."); + } + } + static void _call_PackedByteArray_hex_encode(Variant &r_ret, Variant &p_self, const Variant **p_args) { Variant::PackedArrayRef<uint8_t> *ba = reinterpret_cast<Variant::PackedArrayRef<uint8_t> *>(p_self._data.packed_array); if (ba->array.size() == 0) { @@ -2177,6 +2194,7 @@ void register_variant_methods() { ADDFUNC0R(PACKED_BYTE_ARRAY, STRING, PackedByteArray, hex_encode, varray()); ADDFUNC1R(PACKED_BYTE_ARRAY, PACKED_BYTE_ARRAY, PackedByteArray, compress, INT, "compression_mode", varray(0)); ADDFUNC2R(PACKED_BYTE_ARRAY, PACKED_BYTE_ARRAY, PackedByteArray, decompress, INT, "buffer_size", INT, "compression_mode", varray(0)); + ADDFUNC2R(PACKED_BYTE_ARRAY, PACKED_BYTE_ARRAY, PackedByteArray, decompress_dynamic, INT, "max_output_size", INT, "compression_mode", varray(0)); ADDFUNC0R(PACKED_INT32_ARRAY, INT, PackedInt32Array, size, varray()); ADDFUNC0R(PACKED_INT32_ARRAY, BOOL, PackedInt32Array, empty, varray()); diff --git a/doc/classes/HTTPRequest.xml b/doc/classes/HTTPRequest.xml index 4801af07d3..2ba2249d21 100644 --- a/doc/classes/HTTPRequest.xml +++ b/doc/classes/HTTPRequest.xml @@ -64,6 +64,11 @@ add_child(texture_rect) texture_rect.texture = texture [/codeblock] + + [b]Gzipped response bodies[/b] + HttpRequest will automatically handle decompression of response bodies. + A "Accept-Encoding" header will be automatically added to each of your requests, unless one is already specified. + Any response with a "Content-Encoding: gzip" header will automatically be decompressed and delivered to you as a uncompressed bytes. [b]Note:[/b] When performing HTTP requests from a project exported to HTML5, keep in mind the remote server may not allow requests from foreign origins due to [url=https://developer.mozilla.org/en-US/docs/Web/HTTP/CORS]CORS[/url]. If you host the server in question, you should modify its backend to allow requests from foreign origins by adding the [code]Access-Control-Allow-Origin: *[/code] HTTP header. </description> <tutorials> @@ -119,10 +124,28 @@ [b]Note:[/b] The [code]request_data[/code] parameter is ignored if [code]method[/code] is [constant HTTPClient.METHOD_GET]. This is because GET methods can't contain request data. As a workaround, you can pass request data as a query string in the URL. See [method String.http_escape] for an example. </description> </method> + <method name="request_raw"> + <return type="int" enum="Error"> + </return> + <argument index="0" name="url" type="String"> + </argument> + <argument index="1" name="custom_headers" type="PackedStringArray" default="PackedStringArray( )"> + </argument> + <argument index="2" name="ssl_validate_domain" type="bool" default="true"> + </argument> + <argument index="3" name="method" type="int" enum="HTTPClient.Method" default="0"> + </argument> + <argument index="4" name="request_data_raw" type="PackedByteArray" default="PackedByteArray()"> + </argument> + <description> + Creates request on the underlying [HTTPClient] using a raw array of bytes for the request body. If there is no configuration errors, it tries to connect using [method HTTPClient.connect_to_host] and passes parameters onto [method HTTPClient.request]. + Returns [constant OK] if request is successfully created. (Does not imply that the server has responded), [constant ERR_UNCONFIGURED] if not in the tree, [constant ERR_BUSY] if still processing previous request, [constant ERR_INVALID_PARAMETER] if given string is not a valid URL format, or [constant ERR_CANT_CONNECT] if not using thread and the [HTTPClient] cannot connect to host. + </description> + </method> </methods> <members> <member name="body_size_limit" type="int" setter="set_body_size_limit" getter="get_body_size_limit" default="-1"> - Maximum allowed size for response bodies. + Maximum allowed size for response bodies. If the response body is compressed, this will be used as the maximum allowed size for the decompressed body. </member> <member name="download_chunk_size" type="int" setter="set_download_chunk_size" getter="get_download_chunk_size" default="4096"> The size of the buffer used and maximum bytes to read per iteration. See [member HTTPClient.read_chunk_size]. @@ -139,6 +162,12 @@ <member name="use_threads" type="bool" setter="set_use_threads" getter="is_using_threads" default="false"> If [code]true[/code], multithreading is used to improve performance. </member> + <member name="accept_gzip" type="bool" setter="set_accept_gzip" getter="is_accepting_gzip" default="true"> + If [code]true[/code], this header will be added to each request: [code]Accept-Encoding: gzip, deflate[/code] telling servers that it's okay to compress response bodies. + Any Reponse body declaring a [code]Content-Encoding[/code] of either [code]gzip[/code] or [code]deflate[/code] will then be automatically decompressed, and the uncompressed bytes will be delivered via [code]request_completed[/code]. + If the user has specified their own [code]Accept-Encoding[/code] header, then no header will be added regaurdless of [code]accept_gzip[/code]. + If [code]false[/code] no header will be added, and no decompression will be performed on response bodies. The raw bytes of the response body will be returned via [code]request_completed[/code]. + </member> </members> <signals> <signal name="request_completed"> diff --git a/doc/classes/PackedByteArray.xml b/doc/classes/PackedByteArray.xml index b1c4e54854..7bfc502fbb 100644 --- a/doc/classes/PackedByteArray.xml +++ b/doc/classes/PackedByteArray.xml @@ -57,6 +57,20 @@ Returns a new [PackedByteArray] with the data decompressed. Set [code]buffer_size[/code] to the size of the uncompressed data. Set the compression mode using one of [enum File.CompressionMode]'s constants. </description> </method> + <method name="decompress_dynamic"> + <return type="PackedByteArray"> + </return> + <argument index="0" name="max_output_size" type="int" default="0"> + </argument> + <argument index="1" name="compression_mode" type="int" default="0"> + </argument> + <description> + Returns a new [PackedByteArray] with the data decompressed. Set the compression mode using one of [enum File.CompressionMode]'s constants. [b]This method only accepts gzip and deflate compression modes.[/b] + This method is potentially slower than [code]decompress[/code], as it may have to re-allocate it's output buffer multiple times while decompressing, where as [code]decompress[/code] knows it's output buffer size from the begining. + + GZIP has a maximal compression ratio of 1032:1, meaning it's very possible for a small compressed payload to decompress to a potentially very large output. To guard against this, you may provide a maximum size this function is allowed to allocate in bytes via [code]max_output_size[/code]. Passing -1 will allow for unbounded output. If any positive value is passed, and the decompression exceeds that ammount in bytes, then an error will be returned. + </description> + </method> <method name="empty"> <return type="bool"> </return> diff --git a/scene/main/http_request.cpp b/scene/main/http_request.cpp index 82ee4dde50..2b506b686b 100644 --- a/scene/main/http_request.cpp +++ b/scene/main/http_request.cpp @@ -29,6 +29,8 @@ /*************************************************************************/ #include "http_request.h" +#include "core/io/compression.h" +#include "core/ustring.h" void HTTPRequest::_redirect_request(const String &p_new_url) { } @@ -82,7 +84,51 @@ Error HTTPRequest::_parse_url(const String &p_url) { return OK; } +bool HTTPRequest::has_header(const PackedStringArray &p_headers, const String &p_header_name) { + bool exists = false; + + String lower_case_header_name = p_header_name.to_lower(); + for (int i = 0; i < p_headers.size() && !exists; i++) { + String sanitized = p_headers[i].strip_edges().to_lower(); + if (sanitized.begins_with(lower_case_header_name)) { + exists = true; + } + } + + return exists; +} + +String HTTPRequest::get_header_value(const PackedStringArray &p_headers, const String &p_header_name) { + String value = ""; + + String lowwer_case_header_name = p_header_name.to_lower(); + for (int i = 0; i < p_headers.size(); i++) { + if (p_headers[i].find(":", 0) >= 0) { + Vector<String> parts = p_headers[i].split(":", false, 1); + if (parts[0].strip_edges().to_lower() == lowwer_case_header_name) { + value = parts[1].strip_edges(); + break; + } + } + } + + return value; +} + Error HTTPRequest::request(const String &p_url, const Vector<String> &p_custom_headers, bool p_ssl_validate_domain, HTTPClient::Method p_method, const String &p_request_data) { + // Copy the string into a raw buffer + Vector<uint8_t> raw_data; + + CharString charstr = p_request_data.utf8(); + size_t len = charstr.length(); + raw_data.resize(len); + uint8_t *w = raw_data.ptrw(); + copymem(w, charstr.ptr(), len); + + return request_raw(p_url, p_custom_headers, p_ssl_validate_domain, p_method, raw_data); +} + +Error HTTPRequest::request_raw(const String &p_url, const Vector<String> &p_custom_headers, bool p_ssl_validate_domain, HTTPClient::Method p_method, const Vector<uint8_t> &p_request_data_raw) { ERR_FAIL_COND_V(!is_inside_tree(), ERR_UNCONFIGURED); ERR_FAIL_COND_V_MSG(requesting, ERR_BUSY, "HTTPRequest is processing a request. Wait for completion or cancel it before attempting a new one."); @@ -102,7 +148,14 @@ Error HTTPRequest::request(const String &p_url, const Vector<String> &p_custom_h headers = p_custom_headers; - request_data = p_request_data; + if (accept_gzip) { + // If the user has specified a different Accept-Encoding, don't overwrite it + if (!has_header(headers, "Accept-Encoding")) { + headers.push_back("Accept-Encoding: gzip, deflate"); + } + } + + request_data = p_request_data_raw; requesting = true; @@ -288,7 +341,7 @@ bool HTTPRequest::_update_connection() { } else { // Did not request yet, do request - Error err = client->request(method, request_string, headers, request_data); + Error err = client->request_raw(method, request_string, headers, request_data); if (err != OK) { call_deferred("_request_done", RESULT_CONNECTION_ERROR, 0, PackedStringArray(), PackedByteArray()); return true; @@ -382,9 +435,47 @@ bool HTTPRequest::_update_connection() { ERR_FAIL_V(false); } -void HTTPRequest::_request_done(int p_status, int p_code, const PackedStringArray &headers, const PackedByteArray &p_data) { +void HTTPRequest::_request_done(int p_status, int p_code, const PackedStringArray &p_headers, const PackedByteArray &p_data) { cancel_request(); - emit_signal("request_completed", p_status, p_code, headers, p_data); + + // Determine if the request body is compressed + bool is_compressed; + String content_encoding = get_header_value(p_headers, "Content-Encoding").to_lower(); + Compression::Mode mode; + if (content_encoding == "gzip") { + mode = Compression::Mode::MODE_GZIP; + is_compressed = true; + } else if (content_encoding == "deflate") { + mode = Compression::Mode::MODE_DEFLATE; + is_compressed = true; + } else { + is_compressed = false; + } + + const PackedByteArray *data = NULL; + + if (accept_gzip && is_compressed && p_data.size() > 0) { + // Decompress request body + PackedByteArray *decompressed = memnew(PackedByteArray); + int result = Compression::decompress_dynamic(decompressed, body_size_limit, p_data.ptr(), p_data.size(), mode); + if (result == OK) { + data = decompressed; + } else if (result == -5) { + WARN_PRINT("Decompressed size of HTTP response body exceeded body_size_limit"); + p_status = RESULT_BODY_SIZE_LIMIT_EXCEEDED; + // Just return the raw data if we failed to decompress it + data = &p_data; + } else { + WARN_PRINT("Failed to decompress HTTP response body"); + p_status = RESULT_BODY_DECOMPRESS_FAILED; + // Just return the raw data if we failed to decompress it + data = &p_data; + } + } else { + data = &p_data; + } + + emit_signal("request_completed", p_status, p_code, p_headers, *data); } void HTTPRequest::_notification(int p_what) { @@ -415,6 +506,14 @@ bool HTTPRequest::is_using_threads() const { return use_threads; } +void HTTPRequest::set_accept_gzip(bool p_gzip) { + accept_gzip = p_gzip; +} + +bool HTTPRequest::is_accepting_gzip() const { + return accept_gzip; +} + void HTTPRequest::set_body_size_limit(int p_bytes) { ERR_FAIL_COND(get_http_client_status() != HTTPClient::STATUS_DISCONNECTED); @@ -481,6 +580,7 @@ void HTTPRequest::_timeout() { void HTTPRequest::_bind_methods() { ClassDB::bind_method(D_METHOD("request", "url", "custom_headers", "ssl_validate_domain", "method", "request_data"), &HTTPRequest::request, DEFVAL(PackedStringArray()), DEFVAL(true), DEFVAL(HTTPClient::METHOD_GET), DEFVAL(String())); + ClassDB::bind_method(D_METHOD("request_raw", "url", "custom_headers", "ssl_validate_domain", "method", "request_data_raw"), &HTTPRequest::request_raw, DEFVAL(PackedStringArray()), DEFVAL(true), DEFVAL(HTTPClient::METHOD_GET), DEFVAL(PackedByteArray())); ClassDB::bind_method(D_METHOD("cancel_request"), &HTTPRequest::cancel_request); ClassDB::bind_method(D_METHOD("get_http_client_status"), &HTTPRequest::get_http_client_status); @@ -488,6 +588,9 @@ void HTTPRequest::_bind_methods() { ClassDB::bind_method(D_METHOD("set_use_threads", "enable"), &HTTPRequest::set_use_threads); ClassDB::bind_method(D_METHOD("is_using_threads"), &HTTPRequest::is_using_threads); + ClassDB::bind_method(D_METHOD("set_accept_gzip", "enable"), &HTTPRequest::set_accept_gzip); + ClassDB::bind_method(D_METHOD("is_accepting_gzip"), &HTTPRequest::is_accepting_gzip); + ClassDB::bind_method(D_METHOD("set_body_size_limit", "bytes"), &HTTPRequest::set_body_size_limit); ClassDB::bind_method(D_METHOD("get_body_size_limit"), &HTTPRequest::get_body_size_limit); @@ -512,6 +615,7 @@ void HTTPRequest::_bind_methods() { ADD_PROPERTY(PropertyInfo(Variant::STRING, "download_file", PROPERTY_HINT_FILE), "set_download_file", "get_download_file"); ADD_PROPERTY(PropertyInfo(Variant::INT, "download_chunk_size", PROPERTY_HINT_RANGE, "256,16777216"), "set_download_chunk_size", "get_download_chunk_size"); ADD_PROPERTY(PropertyInfo(Variant::BOOL, "use_threads"), "set_use_threads", "is_using_threads"); + ADD_PROPERTY(PropertyInfo(Variant::BOOL, "accept_gzip"), "set_accept_gzip", "is_accepting_gzip"); ADD_PROPERTY(PropertyInfo(Variant::INT, "body_size_limit", PROPERTY_HINT_RANGE, "-1,2000000000"), "set_body_size_limit", "get_body_size_limit"); ADD_PROPERTY(PropertyInfo(Variant::INT, "max_redirects", PROPERTY_HINT_RANGE, "-1,64"), "set_max_redirects", "get_max_redirects"); ADD_PROPERTY(PropertyInfo(Variant::INT, "timeout", PROPERTY_HINT_RANGE, "0,86400"), "set_timeout", "get_timeout"); @@ -544,6 +648,7 @@ HTTPRequest::HTTPRequest() { got_response = false; validate_ssl = false; use_ssl = false; + accept_gzip = true; response_code = 0; request_sent = false; requesting = false; diff --git a/scene/main/http_request.h b/scene/main/http_request.h index 1409965d45..2e8931120b 100644 --- a/scene/main/http_request.h +++ b/scene/main/http_request.h @@ -50,6 +50,7 @@ public: RESULT_SSL_HANDSHAKE_ERROR, RESULT_NO_RESPONSE, RESULT_BODY_SIZE_LIMIT_EXCEEDED, + RESULT_BODY_DECOMPRESS_FAILED, RESULT_REQUEST_FAILED, RESULT_DOWNLOAD_FILE_CANT_OPEN, RESULT_DOWNLOAD_FILE_WRITE_ERROR, @@ -68,12 +69,13 @@ private: bool validate_ssl; bool use_ssl; HTTPClient::Method method; - String request_data; + Vector<uint8_t> request_data; bool request_sent; Ref<HTTPClient> client; PackedByteArray body; volatile bool use_threads; + bool accept_gzip; bool got_response; int response_code; @@ -102,12 +104,15 @@ private: Error _parse_url(const String &p_url); Error _request(); + bool has_header(const PackedStringArray &p_headers, const String &p_header_name); + String get_header_value(const PackedStringArray &p_headers, const String &header_name); + volatile bool thread_done; volatile bool thread_request_quit; Thread *thread; - void _request_done(int p_status, int p_code, const PackedStringArray &headers, const PackedByteArray &p_data); + void _request_done(int p_status, int p_code, const PackedStringArray &p_headers, const PackedByteArray &p_data); static void _thread_func(void *p_userdata); protected: @@ -116,12 +121,16 @@ protected: public: Error request(const String &p_url, const Vector<String> &p_custom_headers = Vector<String>(), bool p_ssl_validate_domain = true, HTTPClient::Method p_method = HTTPClient::METHOD_GET, const String &p_request_data = ""); //connects to a full url and perform request + Error request_raw(const String &p_url, const Vector<String> &p_custom_headers = Vector<String>(), bool p_ssl_validate_domain = true, HTTPClient::Method p_method = HTTPClient::METHOD_GET, const Vector<uint8_t> &p_request_data_raw = Vector<uint8_t>()); //connects to a full url and perform request void cancel_request(); HTTPClient::Status get_http_client_status() const; void set_use_threads(bool p_use); bool is_using_threads() const; + void set_accept_gzip(bool p_gzip); + bool is_accepting_gzip() const; + void set_download_file(const String &p_file); String get_download_file() const; |