summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorFabio Alessandrelli <fabio.alessandrelli@gmail.com>2022-07-30 11:43:05 +0200
committerFabio Alessandrelli <fabio.alessandrelli@gmail.com>2022-09-20 18:39:10 +0200
commit273ba0794f022748ed5084ee74020849443af7d7 (patch)
treeb5364e954353d7b0a6ded0469b8d3ee6cb80cfcb
parent6f5704d86f95171ba8b6b2ac9f56e284c4d35d7a (diff)
[IO] Add StreamPeerGZIP for streaming compression/decompression.
Putting data results in writing the compressed/decompressed output into a RingBuffer. You can retrieve the content of the RingBuffer via get_data. Support both gzip and deflate.
-rw-r--r--core/io/stream_peer_gzip.cpp209
-rw-r--r--core/io/stream_peer_gzip.h76
-rw-r--r--core/register_core_types.cpp2
-rw-r--r--doc/classes/StreamPeerGZIP.xml42
4 files changed, 329 insertions, 0 deletions
diff --git a/core/io/stream_peer_gzip.cpp b/core/io/stream_peer_gzip.cpp
new file mode 100644
index 0000000000..ca8be2d62e
--- /dev/null
+++ b/core/io/stream_peer_gzip.cpp
@@ -0,0 +1,209 @@
+/*************************************************************************/
+/* stream_peer_gzip.cpp */
+/*************************************************************************/
+/* This file is part of: */
+/* GODOT ENGINE */
+/* https://godotengine.org */
+/*************************************************************************/
+/* Copyright (c) 2007-2022 Juan Linietsky, Ariel Manzur. */
+/* Copyright (c) 2014-2022 Godot Engine contributors (cf. AUTHORS.md). */
+/* */
+/* Permission is hereby granted, free of charge, to any person obtaining */
+/* a copy of this software and associated documentation files (the */
+/* "Software"), to deal in the Software without restriction, including */
+/* without limitation the rights to use, copy, modify, merge, publish, */
+/* distribute, sublicense, and/or sell copies of the Software, and to */
+/* permit persons to whom the Software is furnished to do so, subject to */
+/* the following conditions: */
+/* */
+/* The above copyright notice and this permission notice shall be */
+/* included in all copies or substantial portions of the Software. */
+/* */
+/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */
+/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */
+/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*/
+/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */
+/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */
+/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */
+/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */
+/*************************************************************************/
+
+#include "core/io/stream_peer_gzip.h"
+
+#include "core/io/zip_io.h"
+#include <zlib.h>
+
+void StreamPeerGZIP::_bind_methods() {
+ ClassDB::bind_method(D_METHOD("start_compression", "use_deflate", "buffer_size"), &StreamPeerGZIP::start_compression, DEFVAL(false), DEFVAL(65535));
+ ClassDB::bind_method(D_METHOD("start_decompression", "use_deflate", "buffer_size"), &StreamPeerGZIP::start_decompression, DEFVAL(false), DEFVAL(65535));
+ ClassDB::bind_method(D_METHOD("finish"), &StreamPeerGZIP::finish);
+ ClassDB::bind_method(D_METHOD("clear"), &StreamPeerGZIP::clear);
+}
+
+StreamPeerGZIP::StreamPeerGZIP() {
+}
+
+StreamPeerGZIP::~StreamPeerGZIP() {
+ _close();
+}
+
+void StreamPeerGZIP::_close() {
+ if (ctx) {
+ z_stream *strm = (z_stream *)ctx;
+ if (compressing) {
+ deflateEnd(strm);
+ } else {
+ inflateEnd(strm);
+ }
+ memfree(strm);
+ ctx = nullptr;
+ }
+}
+
+void StreamPeerGZIP::clear() {
+ _close();
+ rb.clear();
+ buffer.clear();
+}
+
+Error StreamPeerGZIP::start_compression(bool p_is_deflate, int buffer_size) {
+ return _start(true, p_is_deflate, buffer_size);
+}
+
+Error StreamPeerGZIP::start_decompression(bool p_is_deflate, int buffer_size) {
+ return _start(false, p_is_deflate, buffer_size);
+}
+
+Error StreamPeerGZIP::_start(bool p_compress, bool p_is_deflate, int buffer_size) {
+ ERR_FAIL_COND_V(ctx != nullptr, ERR_ALREADY_IN_USE);
+ clear();
+ compressing = p_compress;
+ rb.resize(nearest_shift(buffer_size - 1));
+ buffer.resize(1024);
+
+ // Create ctx.
+ ctx = memalloc(sizeof(z_stream));
+ z_stream &strm = *(z_stream *)ctx;
+ strm.next_in = Z_NULL;
+ strm.avail_in = 0;
+ strm.zalloc = zipio_alloc;
+ strm.zfree = zipio_free;
+ strm.opaque = Z_NULL;
+ int window_bits = p_is_deflate ? 15 : (15 + 16);
+ int err = Z_OK;
+ int level = Z_DEFAULT_COMPRESSION;
+ if (compressing) {
+ err = deflateInit2(&strm, level, Z_DEFLATED, window_bits, 8, Z_DEFAULT_STRATEGY);
+ } else {
+ err = inflateInit2(&strm, window_bits);
+ }
+ ERR_FAIL_COND_V(err != Z_OK, FAILED);
+ return OK;
+}
+
+Error StreamPeerGZIP::_process(uint8_t *p_dst, int p_dst_size, const uint8_t *p_src, int p_src_size, int &r_consumed, int &r_out, bool p_close) {
+ ERR_FAIL_COND_V(!ctx, ERR_UNCONFIGURED);
+ z_stream &strm = *(z_stream *)ctx;
+ strm.avail_in = p_src_size;
+ strm.avail_out = p_dst_size;
+ strm.next_in = (Bytef *)p_src;
+ strm.next_out = (Bytef *)p_dst;
+ int flush = p_close ? Z_FINISH : Z_NO_FLUSH;
+ if (compressing) {
+ int err = deflate(&strm, flush);
+ ERR_FAIL_COND_V(err != (p_close ? Z_STREAM_END : Z_OK), FAILED);
+ } else {
+ int err = inflate(&strm, flush);
+ ERR_FAIL_COND_V(err != Z_OK && err != Z_STREAM_END, FAILED);
+ }
+ r_out = p_dst_size - strm.avail_out;
+ r_consumed = p_src_size - strm.avail_in;
+ return OK;
+}
+
+Error StreamPeerGZIP::put_data(const uint8_t *p_data, int p_bytes) {
+ int wrote = 0;
+ Error err = put_partial_data(p_data, p_bytes, wrote);
+ if (err != OK) {
+ return err;
+ }
+ ERR_FAIL_COND_V(p_bytes != wrote, ERR_OUT_OF_MEMORY);
+ return OK;
+}
+
+Error StreamPeerGZIP::put_partial_data(const uint8_t *p_data, int p_bytes, int &r_sent) {
+ ERR_FAIL_COND_V(!ctx, ERR_UNCONFIGURED);
+ ERR_FAIL_COND_V(p_bytes < 0, ERR_INVALID_PARAMETER);
+
+ // Ensure we have enough space in temporary buffer.
+ if (buffer.size() < p_bytes) {
+ buffer.resize(p_bytes);
+ }
+
+ r_sent = 0;
+ while (r_sent < p_bytes && rb.space_left() > 1024) { // Keep the ring buffer size meaningful.
+ int sent = 0;
+ int to_write = 0;
+ // Compress or decompress
+ Error err = _process(buffer.ptrw(), MIN(buffer.size(), rb.space_left()), p_data + r_sent, p_bytes - r_sent, sent, to_write);
+ if (err != OK) {
+ return err;
+ }
+ // When decompressing, we might need to do another round.
+ r_sent += sent;
+
+ // We can't write more than this buffer is full.
+ if (sent == 0 && to_write == 0) {
+ return OK;
+ }
+ if (to_write) {
+ // Copy to ring buffer.
+ int wrote = rb.write(buffer.ptr(), to_write);
+ ERR_FAIL_COND_V(wrote != to_write, ERR_BUG);
+ }
+ }
+ return OK;
+}
+
+Error StreamPeerGZIP::get_data(uint8_t *p_buffer, int p_bytes) {
+ int received = 0;
+ Error err = get_partial_data(p_buffer, p_bytes, received);
+ if (err != OK) {
+ return err;
+ }
+ ERR_FAIL_COND_V(p_bytes != received, ERR_UNAVAILABLE);
+ return OK;
+}
+
+Error StreamPeerGZIP::get_partial_data(uint8_t *p_buffer, int p_bytes, int &r_received) {
+ ERR_FAIL_COND_V(p_bytes < 0, ERR_INVALID_PARAMETER);
+
+ r_received = MIN(p_bytes, rb.data_left());
+ if (r_received == 0) {
+ return OK;
+ }
+ int received = rb.read(p_buffer, r_received);
+ ERR_FAIL_COND_V(received != r_received, ERR_BUG);
+ return OK;
+}
+
+int StreamPeerGZIP::get_available_bytes() const {
+ return rb.data_left();
+}
+
+Error StreamPeerGZIP::finish() {
+ ERR_FAIL_COND_V(!ctx || !compressing, ERR_UNAVAILABLE);
+ // Ensure we have enough space in temporary buffer.
+ if (buffer.size() < 1024) {
+ buffer.resize(1024); // 1024 should be more than enough.
+ }
+ int consumed = 0;
+ int to_write = 0;
+ Error err = _process(buffer.ptrw(), 1024, nullptr, 0, consumed, to_write, true); // compress
+ if (err != OK) {
+ return err;
+ }
+ int wrote = rb.write(buffer.ptr(), to_write);
+ ERR_FAIL_COND_V(wrote != to_write, ERR_OUT_OF_MEMORY);
+ return OK;
+}
diff --git a/core/io/stream_peer_gzip.h b/core/io/stream_peer_gzip.h
new file mode 100644
index 0000000000..5bafdbca9b
--- /dev/null
+++ b/core/io/stream_peer_gzip.h
@@ -0,0 +1,76 @@
+/*************************************************************************/
+/* stream_peer_gzip.h */
+/*************************************************************************/
+/* This file is part of: */
+/* GODOT ENGINE */
+/* https://godotengine.org */
+/*************************************************************************/
+/* Copyright (c) 2007-2022 Juan Linietsky, Ariel Manzur. */
+/* Copyright (c) 2014-2022 Godot Engine contributors (cf. AUTHORS.md). */
+/* */
+/* Permission is hereby granted, free of charge, to any person obtaining */
+/* a copy of this software and associated documentation files (the */
+/* "Software"), to deal in the Software without restriction, including */
+/* without limitation the rights to use, copy, modify, merge, publish, */
+/* distribute, sublicense, and/or sell copies of the Software, and to */
+/* permit persons to whom the Software is furnished to do so, subject to */
+/* the following conditions: */
+/* */
+/* The above copyright notice and this permission notice shall be */
+/* included in all copies or substantial portions of the Software. */
+/* */
+/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */
+/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */
+/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*/
+/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */
+/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */
+/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */
+/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */
+/*************************************************************************/
+
+#ifndef STREAM_PEER_GZIP_H
+#define STREAM_PEER_GZIP_H
+
+#include "core/io/stream_peer.h"
+
+#include "core/core_bind.h"
+#include "core/io/compression.h"
+#include "core/templates/ring_buffer.h"
+
+class StreamPeerGZIP : public StreamPeer {
+ GDCLASS(StreamPeerGZIP, StreamPeer);
+
+private:
+ void *ctx = nullptr; // Will hold our z_stream instance.
+ bool compressing = true;
+
+ RingBuffer<uint8_t> rb;
+ Vector<uint8_t> buffer;
+
+ Error _process(uint8_t *p_dst, int p_dst_size, const uint8_t *p_src, int p_src_size, int &r_consumed, int &r_out, bool p_close = false);
+ void _close();
+ Error _start(bool p_compress, bool p_is_deflate, int buffer_size = 65535);
+
+protected:
+ static void _bind_methods();
+
+public:
+ Error start_compression(bool p_is_deflate, int buffer_size = 65535);
+ Error start_decompression(bool p_is_deflate, int buffer_size = 65535);
+
+ Error finish();
+ void clear();
+
+ virtual Error put_data(const uint8_t *p_data, int p_bytes) override;
+ virtual Error put_partial_data(const uint8_t *p_data, int p_bytes, int &r_sent) override;
+
+ virtual Error get_data(uint8_t *p_buffer, int p_bytes) override;
+ virtual Error get_partial_data(uint8_t *p_buffer, int p_bytes, int &r_received) override;
+
+ virtual int get_available_bytes() const override;
+
+ StreamPeerGZIP();
+ ~StreamPeerGZIP();
+};
+
+#endif // STREAM_PEER_GZIP_H
diff --git a/core/register_core_types.cpp b/core/register_core_types.cpp
index 0233afe199..64cc6f2aab 100644
--- a/core/register_core_types.cpp
+++ b/core/register_core_types.cpp
@@ -58,6 +58,7 @@
#include "core/io/resource_format_binary.h"
#include "core/io/resource_importer.h"
#include "core/io/resource_uid.h"
+#include "core/io/stream_peer_gzip.h"
#include "core/io/stream_peer_tls.h"
#include "core/io/tcp_server.h"
#include "core/io/translation_loader_po.h"
@@ -184,6 +185,7 @@ void register_core_types() {
GDREGISTER_ABSTRACT_CLASS(StreamPeer);
GDREGISTER_CLASS(StreamPeerExtension);
GDREGISTER_CLASS(StreamPeerBuffer);
+ GDREGISTER_CLASS(StreamPeerGZIP);
GDREGISTER_CLASS(StreamPeerTCP);
GDREGISTER_CLASS(TCPServer);
diff --git a/doc/classes/StreamPeerGZIP.xml b/doc/classes/StreamPeerGZIP.xml
new file mode 100644
index 0000000000..71dd36160d
--- /dev/null
+++ b/doc/classes/StreamPeerGZIP.xml
@@ -0,0 +1,42 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<class name="StreamPeerGZIP" inherits="StreamPeer" is_experimental="true" version="4.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="../class.xsd">
+ <brief_description>
+ Stream peer handling GZIP and deflate compression/decompresison.
+ </brief_description>
+ <description>
+ This class allows to compress or decompress data using GZIP/deflate in a streaming fashion. This is particularly useful when compressing or decompressing files that has to be sent through the network without having to allocate them all in memory.
+ After starting the stream via [method start_compression] (or [method start_decompression]), calling [method StreamPeer.put_partial_data] on this stream will compress (or decompress) the data, writing it to the internal buffer. Calling [method StreamPeer.get_available_bytes] will return the pending bytes in the internal buffer, and [method StreamPeer.get_partial_data] will retrieve the compressed (or decompressed) bytes from it. When the stream is over, you must call [method finish] to ensure the internal buffer is properly flushed (make sure to call [method StreamPeer.get_available_bytes] on last time to check if more data needs to be read after that).
+ </description>
+ <tutorials>
+ </tutorials>
+ <methods>
+ <method name="clear">
+ <return type="void" />
+ <description>
+ Clears this stream, resetting the internal state.
+ </description>
+ </method>
+ <method name="finish">
+ <return type="int" enum="Error" />
+ <description>
+ Finalizes the stream, compressing or decompressing any buffered chunk left.
+ </description>
+ </method>
+ <method name="start_compression">
+ <return type="int" enum="Error" />
+ <param index="0" name="use_deflate" type="bool" default="false" />
+ <param index="1" name="buffer_size" type="int" default="65535" />
+ <description>
+ Start the stream in compression mode with the given [param buffer_size], if [param use_deflate] is [code]true[/code] uses deflate instead of GZIP.
+ </description>
+ </method>
+ <method name="start_decompression">
+ <return type="int" enum="Error" />
+ <param index="0" name="use_deflate" type="bool" default="false" />
+ <param index="1" name="buffer_size" type="int" default="65535" />
+ <description>
+ Start the stream in decompression mode with the given [param buffer_size], if [param use_deflate] is [code]true[/code] uses deflate instead of GZIP.
+ </description>
+ </method>
+ </methods>
+</class>