HTTPRequest now accepts gzip

Added request_raw to HttpRequest
Added decompress_dynamic to Compression class
Added decompress_dynamic to BytePoolArray

Merge doc fix

revert

2 files changed, 90 insertions, 0 deletions

diff --git a/core/io/compression.cpp b/core/io/compression.cpp
index 99ca8107e4..7480262835 100644
--- a/core/io/compression.cpp
+++ b/core/io/compression.cpp
@@ -180,8 +180,95 @@ int Compression::decompress(uint8_t *p_dst, int p_dst_max_size, const uint8_t *p
 	ERR_FAIL_V(-1);
 }
 
+/**
+	This will handle both Gzip and Deflat streams. It will automatically allocate the output buffer into the provided p_dst_vect Vector.
+	This is required for compressed data who's final uncompressed size is unknown, as is the case for HTTP response bodies.
+	This is much slower however than using Compression::decompress because it may result in multiple full copies of the output buffer.
+*/
+int Compression::decompress_dynamic(Vector<uint8_t> *p_dst_vect, int p_max_dst_size, const uint8_t *p_src, int p_src_size, Mode p_mode) {
+	int ret;
+	uint8_t *dst = nullptr;
+	int out_mark = 0;
+	z_stream strm;
+
+	ERR_FAIL_COND_V(p_src_size <= 0, Z_DATA_ERROR);
+
+	// This function only supports GZip and Deflate
+	int window_bits = p_mode == MODE_DEFLATE ? 15 : 15 + 16;
+	ERR_FAIL_COND_V(p_mode != MODE_DEFLATE && p_mode != MODE_GZIP, Z_ERRNO);
+
+	// Initialize the stream
+	strm.zalloc = Z_NULL;
+	strm.zfree = Z_NULL;
+	strm.opaque = Z_NULL;
+	strm.avail_in = 0;
+	strm.next_in = Z_NULL;
+
+	int err = inflateInit2(&strm, window_bits);
+	ERR_FAIL_COND_V(err != Z_OK, -1);
+
+	// Setup the stream inputs
+	strm.next_in = (Bytef *)p_src;
+	strm.avail_in = p_src_size;
+
+	// Ensure the destination buffer is empty
+	p_dst_vect->resize(0);
+
+	// decompress until deflate stream ends or end of file
+	do {
+		// Add another chunk size to the output buffer
+		// This forces a copy of the whole buffer
+		p_dst_vect->resize(p_dst_vect->size() + gzip_chunk);
+		// Get pointer to the actual output buffer
+		dst = p_dst_vect->ptrw();
+
+		// Set the stream to the new output stream
+		// Since it was copied, we need to reset the stream to the new buffer
+		strm.next_out = &(dst[out_mark]);
+		strm.avail_out = gzip_chunk;
+
+		// run inflate() on input until output buffer is full and needs to be resized
+		// or input runs out
+		do {
+			ret = inflate(&strm, Z_SYNC_FLUSH);
+
+			switch (ret) {
+				case Z_NEED_DICT:
+					ret = Z_DATA_ERROR;
+					[[fallthrough]];
+				case Z_DATA_ERROR:
+				case Z_MEM_ERROR:
+				case Z_STREAM_ERROR:
+					WARN_PRINT(strm.msg);
+					(void)inflateEnd(&strm);
+					p_dst_vect->resize(0);
+					return ret;
+			}
+		} while (strm.avail_out > 0 && strm.avail_in > 0);
+
+		out_mark += gzip_chunk;
+
+		// Encorce max output size
+		if (p_max_dst_size > -1 && strm.total_out > (uint64_t)p_max_dst_size) {
+			(void)inflateEnd(&strm);
+			p_dst_vect->resize(0);
+			return Z_BUF_ERROR;
+		}
+	} while (ret != Z_STREAM_END);
+
+	// If all done successfully, resize the output if it's larger than the actual output
+	if (ret == Z_STREAM_END && (unsigned long)p_dst_vect->size() > strm.total_out) {
+		p_dst_vect->resize(strm.total_out);
+	}
+
+	// clean up and return
+	(void)inflateEnd(&strm);
+	return ret == Z_STREAM_END ? Z_OK : Z_DATA_ERROR;
+}
+
 int Compression::zlib_level = Z_DEFAULT_COMPRESSION;
 int Compression::gzip_level = Z_DEFAULT_COMPRESSION;
 int Compression::zstd_level = 3;
 bool Compression::zstd_long_distance_matching = false;
 int Compression::zstd_window_log_size = 27; // ZSTD_WINDOWLOG_LIMIT_DEFAULT
+int Compression::gzip_chunk = 16384;
diff --git a/core/io/compression.h b/core/io/compression.h
index f195f96ba5..c103fa8eae 100644
--- a/core/io/compression.h
+++ b/core/io/compression.h
@@ -32,6 +32,7 @@
 #define COMPRESSION_H
 
 #include "core/typedefs.h"
+#include "core/vector.h"
 
 class Compression {
 public:
@@ -40,6 +41,7 @@ public:
 	static int zstd_level;
 	static bool zstd_long_distance_matching;
 	static int zstd_window_log_size;
+	static int gzip_chunk;
 
 	enum Mode {
 		MODE_FASTLZ,
@@ -51,6 +53,7 @@ public:
 	static int compress(uint8_t *p_dst, const uint8_t *p_src, int p_src_size, Mode p_mode = MODE_ZSTD);
 	static int get_max_compressed_buffer_size(int p_src_size, Mode p_mode = MODE_ZSTD);
 	static int decompress(uint8_t *p_dst, int p_dst_max_size, const uint8_t *p_src, int p_src_size, Mode p_mode = MODE_ZSTD);
+	static int decompress_dynamic(Vector<uint8_t> *p_dst_vect, int p_max_dst_size, const uint8_t *p_src, int p_src_size, Mode p_mode);
 
 	Compression() {}
 };