summaryrefslogtreecommitdiff
path: root/core
diff options
context:
space:
mode:
Diffstat (limited to 'core')
-rw-r--r--core/io/compression.cpp87
-rw-r--r--core/io/compression.h3
-rw-r--r--core/variant_call.cpp62
3 files changed, 131 insertions, 21 deletions
diff --git a/core/io/compression.cpp b/core/io/compression.cpp
index 99ca8107e4..7480262835 100644
--- a/core/io/compression.cpp
+++ b/core/io/compression.cpp
@@ -180,8 +180,95 @@ int Compression::decompress(uint8_t *p_dst, int p_dst_max_size, const uint8_t *p
ERR_FAIL_V(-1);
}
+/**
+ This will handle both Gzip and Deflat streams. It will automatically allocate the output buffer into the provided p_dst_vect Vector.
+ This is required for compressed data who's final uncompressed size is unknown, as is the case for HTTP response bodies.
+ This is much slower however than using Compression::decompress because it may result in multiple full copies of the output buffer.
+*/
+int Compression::decompress_dynamic(Vector<uint8_t> *p_dst_vect, int p_max_dst_size, const uint8_t *p_src, int p_src_size, Mode p_mode) {
+ int ret;
+ uint8_t *dst = nullptr;
+ int out_mark = 0;
+ z_stream strm;
+
+ ERR_FAIL_COND_V(p_src_size <= 0, Z_DATA_ERROR);
+
+ // This function only supports GZip and Deflate
+ int window_bits = p_mode == MODE_DEFLATE ? 15 : 15 + 16;
+ ERR_FAIL_COND_V(p_mode != MODE_DEFLATE && p_mode != MODE_GZIP, Z_ERRNO);
+
+ // Initialize the stream
+ strm.zalloc = Z_NULL;
+ strm.zfree = Z_NULL;
+ strm.opaque = Z_NULL;
+ strm.avail_in = 0;
+ strm.next_in = Z_NULL;
+
+ int err = inflateInit2(&strm, window_bits);
+ ERR_FAIL_COND_V(err != Z_OK, -1);
+
+ // Setup the stream inputs
+ strm.next_in = (Bytef *)p_src;
+ strm.avail_in = p_src_size;
+
+ // Ensure the destination buffer is empty
+ p_dst_vect->resize(0);
+
+ // decompress until deflate stream ends or end of file
+ do {
+ // Add another chunk size to the output buffer
+ // This forces a copy of the whole buffer
+ p_dst_vect->resize(p_dst_vect->size() + gzip_chunk);
+ // Get pointer to the actual output buffer
+ dst = p_dst_vect->ptrw();
+
+ // Set the stream to the new output stream
+ // Since it was copied, we need to reset the stream to the new buffer
+ strm.next_out = &(dst[out_mark]);
+ strm.avail_out = gzip_chunk;
+
+ // run inflate() on input until output buffer is full and needs to be resized
+ // or input runs out
+ do {
+ ret = inflate(&strm, Z_SYNC_FLUSH);
+
+ switch (ret) {
+ case Z_NEED_DICT:
+ ret = Z_DATA_ERROR;
+ [[fallthrough]];
+ case Z_DATA_ERROR:
+ case Z_MEM_ERROR:
+ case Z_STREAM_ERROR:
+ WARN_PRINT(strm.msg);
+ (void)inflateEnd(&strm);
+ p_dst_vect->resize(0);
+ return ret;
+ }
+ } while (strm.avail_out > 0 && strm.avail_in > 0);
+
+ out_mark += gzip_chunk;
+
+ // Encorce max output size
+ if (p_max_dst_size > -1 && strm.total_out > (uint64_t)p_max_dst_size) {
+ (void)inflateEnd(&strm);
+ p_dst_vect->resize(0);
+ return Z_BUF_ERROR;
+ }
+ } while (ret != Z_STREAM_END);
+
+ // If all done successfully, resize the output if it's larger than the actual output
+ if (ret == Z_STREAM_END && (unsigned long)p_dst_vect->size() > strm.total_out) {
+ p_dst_vect->resize(strm.total_out);
+ }
+
+ // clean up and return
+ (void)inflateEnd(&strm);
+ return ret == Z_STREAM_END ? Z_OK : Z_DATA_ERROR;
+}
+
int Compression::zlib_level = Z_DEFAULT_COMPRESSION;
int Compression::gzip_level = Z_DEFAULT_COMPRESSION;
int Compression::zstd_level = 3;
bool Compression::zstd_long_distance_matching = false;
int Compression::zstd_window_log_size = 27; // ZSTD_WINDOWLOG_LIMIT_DEFAULT
+int Compression::gzip_chunk = 16384;
diff --git a/core/io/compression.h b/core/io/compression.h
index f195f96ba5..c103fa8eae 100644
--- a/core/io/compression.h
+++ b/core/io/compression.h
@@ -32,6 +32,7 @@
#define COMPRESSION_H
#include "core/typedefs.h"
+#include "core/vector.h"
class Compression {
public:
@@ -40,6 +41,7 @@ public:
static int zstd_level;
static bool zstd_long_distance_matching;
static int zstd_window_log_size;
+ static int gzip_chunk;
enum Mode {
MODE_FASTLZ,
@@ -51,6 +53,7 @@ public:
static int compress(uint8_t *p_dst, const uint8_t *p_src, int p_src_size, Mode p_mode = MODE_ZSTD);
static int get_max_compressed_buffer_size(int p_src_size, Mode p_mode = MODE_ZSTD);
static int decompress(uint8_t *p_dst, int p_dst_max_size, const uint8_t *p_src, int p_src_size, Mode p_mode = MODE_ZSTD);
+ static int decompress_dynamic(Vector<uint8_t> *p_dst_vect, int p_max_dst_size, const uint8_t *p_src, int p_src_size, Mode p_mode);
Compression() {}
};
diff --git a/core/variant_call.cpp b/core/variant_call.cpp
index 7a1fdbaafe..0ebb2f04a1 100644
--- a/core/variant_call.cpp
+++ b/core/variant_call.cpp
@@ -630,14 +630,14 @@ struct _VariantCall {
VCALL_LOCALMEM0R(Array, min);
static void _call_PackedByteArray_get_string_from_ascii(Variant &r_ret, Variant &p_self, const Variant **p_args) {
- PackedByteArray *ba = reinterpret_cast<PackedByteArray *>(p_self._data._mem);
+ Variant::PackedArrayRef<uint8_t> *ba = reinterpret_cast<Variant::PackedArrayRef<uint8_t> *>(p_self._data.packed_array);
String s;
- if (ba->size() > 0) {
- const uint8_t *r = ba->ptr();
+ if (ba->array.size() > 0) {
+ const uint8_t *r = ba->array.ptr();
CharString cs;
- cs.resize(ba->size() + 1);
- copymem(cs.ptrw(), r, ba->size());
- cs[ba->size()] = 0;
+ cs.resize(ba->array.size() + 1);
+ copymem(cs.ptrw(), r, ba->array.size());
+ cs[ba->array.size()] = 0;
s = cs.get_data();
}
@@ -645,11 +645,12 @@ struct _VariantCall {
}
static void _call_PackedByteArray_get_string_from_utf8(Variant &r_ret, Variant &p_self, const Variant **p_args) {
- PackedByteArray *ba = reinterpret_cast<PackedByteArray *>(p_self._data._mem);
+ Variant::PackedArrayRef<uint8_t> *ba = reinterpret_cast<Variant::PackedArrayRef<uint8_t> *>(p_self._data.packed_array);
+
String s;
- if (ba->size() > 0) {
- const uint8_t *r = ba->ptr();
- s.parse_utf8((const char *)r, ba->size());
+ if (ba->array.size() > 0) {
+ const uint8_t *r = ba->array.ptr();
+ s.parse_utf8((const char *)r, ba->array.size());
}
r_ret = s;
}
@@ -675,22 +676,23 @@ struct _VariantCall {
}
static void _call_PackedByteArray_compress(Variant &r_ret, Variant &p_self, const Variant **p_args) {
- PackedByteArray *ba = reinterpret_cast<PackedByteArray *>(p_self._data._mem);
+ Variant::PackedArrayRef<uint8_t> *ba = reinterpret_cast<Variant::PackedArrayRef<uint8_t> *>(p_self._data.packed_array);
PackedByteArray compressed;
- if (ba->size() > 0) {
- Compression::Mode mode = (Compression::Mode)(int)(*p_args[0]);
- compressed.resize(Compression::get_max_compressed_buffer_size(ba->size(), mode));
- int result = Compression::compress(compressed.ptrw(), ba->ptr(), ba->size(), mode);
+ if (ba->array.size() > 0) {
+ Compression::Mode mode = (Compression::Mode)(int)(*p_args[0]);
+ compressed.resize(Compression::get_max_compressed_buffer_size(ba->array.size(), mode));
+ int result = Compression::compress(compressed.ptrw(), ba->array.ptr(), ba->array.size(), mode);
result = result >= 0 ? result : 0;
compressed.resize(result);
}
+
r_ret = compressed;
}
static void _call_PackedByteArray_decompress(Variant &r_ret, Variant &p_self, const Variant **p_args) {
- PackedByteArray *ba = reinterpret_cast<PackedByteArray *>(p_self._data._mem);
+ Variant::PackedArrayRef<uint8_t> *ba = reinterpret_cast<Variant::PackedArrayRef<uint8_t> *>(p_self._data.packed_array);
PackedByteArray decompressed;
Compression::Mode mode = (Compression::Mode)(int)(*p_args[1]);
@@ -702,7 +704,7 @@ struct _VariantCall {
}
decompressed.resize(buffer_size);
- int result = Compression::decompress(decompressed.ptrw(), buffer_size, ba->ptr(), ba->size(), mode);
+ int result = Compression::decompress(decompressed.ptrw(), buffer_size, ba->array.ptr(), ba->array.size(), mode);
result = result >= 0 ? result : 0;
decompressed.resize(result);
@@ -710,14 +712,31 @@ struct _VariantCall {
r_ret = decompressed;
}
+ static void _call_PackedByteArray_decompress_dynamic(Variant &r_ret, Variant &p_self, const Variant **p_args) {
+ Variant::PackedArrayRef<uint8_t> *ba = reinterpret_cast<Variant::PackedArrayRef<uint8_t> *>(p_self._data.packed_array);
+ PackedByteArray decompressed;
+ int max_output_size = (int)(*p_args[0]);
+ Compression::Mode mode = (Compression::Mode)(int)(*p_args[1]);
+
+ int result = Compression::decompress_dynamic(&decompressed, max_output_size, ba->array.ptr(), ba->array.size(), mode);
+
+ if (result == OK) {
+ r_ret = decompressed;
+ } else {
+ decompressed.clear();
+ r_ret = decompressed;
+ ERR_FAIL_MSG("Decompression failed.");
+ }
+ }
+
static void _call_PackedByteArray_hex_encode(Variant &r_ret, Variant &p_self, const Variant **p_args) {
- PackedByteArray *ba = reinterpret_cast<PackedByteArray *>(p_self._data._mem);
- if (ba->size() == 0) {
+ Variant::PackedArrayRef<uint8_t> *ba = reinterpret_cast<Variant::PackedArrayRef<uint8_t> *>(p_self._data.packed_array);
+ if (ba->array.size() == 0) {
r_ret = String();
return;
}
- const uint8_t *r = ba->ptr();
- String s = String::hex_encode_buffer(&r[0], ba->size());
+ const uint8_t *r = ba->array.ptr();
+ String s = String::hex_encode_buffer(&r[0], ba->array.size());
r_ret = s;
}
@@ -2175,6 +2194,7 @@ void register_variant_methods() {
ADDFUNC0R(PACKED_BYTE_ARRAY, STRING, PackedByteArray, hex_encode, varray());
ADDFUNC1R(PACKED_BYTE_ARRAY, PACKED_BYTE_ARRAY, PackedByteArray, compress, INT, "compression_mode", varray(0));
ADDFUNC2R(PACKED_BYTE_ARRAY, PACKED_BYTE_ARRAY, PackedByteArray, decompress, INT, "buffer_size", INT, "compression_mode", varray(0));
+ ADDFUNC2R(PACKED_BYTE_ARRAY, PACKED_BYTE_ARRAY, PackedByteArray, decompress_dynamic, INT, "max_output_size", INT, "compression_mode", varray(0));
ADDFUNC0R(PACKED_INT32_ARRAY, INT, PackedInt32Array, size, varray());
ADDFUNC0R(PACKED_INT32_ARRAY, BOOL, PackedInt32Array, empty, varray());
generated by cgit v1.2.3 (git 2.25.1) at 2025-10-05 04:57:37 +0000