diff options
257 files changed, 9218 insertions, 3913 deletions
diff --git a/.gitignore b/.gitignore index 6af581040c..8d1aad947d 100644 --- a/.gitignore +++ b/.gitignore @@ -383,3 +383,6 @@ ruby.png snow.png updown.png gcov.css + +# https://clangd.llvm.org/ cache folder +.clangd/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 1f8f8bd676..88c9e1a8a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -384,6 +384,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Ogg samples now have an icon in the editor, like WAV samples. - Camera2D drag margins are now disabled by default. - If porting a project from Godot 3.1 where drag margins were used, these must be enabled manually again. +- The Camera2D Offset property now ignores the Limit property. + - To get the old behavior back, move the camera itself instead of changing the offset. - `Camera.project_position()` now requires a second `depth` argument to determine the distance of the point from the camera. - To get the old behavior back, pass the Camera's `near` property value as the second argument. - `Skeleton.set_bone_global_pose()` was replaced by `Skeleton.set_bone_global_pose_override()`. diff --git a/COPYRIGHT.txt b/COPYRIGHT.txt index 8ccdecd307..6122452b88 100644 --- a/COPYRIGHT.txt +++ b/COPYRIGHT.txt @@ -338,7 +338,7 @@ License: Expat Files: ./thirdparty/tinyexr/ Comment: TinyEXR -Copyright: 2014-2019, Syoyo Fujita +Copyright: 2014-2020, Syoyo Fujita 2002, Industrial Light & Magic, a division of Lucas Digital Ltd. LLC License: BSD-3-clause @@ -380,7 +380,7 @@ License: Zlib Files: ./thirdparty/zstd/ Comment: Zstandard -Copyright: 2016-2019, Facebook, Inc. +Copyright: 2016-2020, Facebook, Inc. License: BSD-3-clause @@ -14,7 +14,6 @@ generous deed immortalized in the next stable release of Godot Engine. Gamblify <https://www.gamblify.com> Heroic Labs <https://heroiclabs.com> - Interblock <http://interblockgaming.com> ## Gold sponsors diff --git a/SConstruct b/SConstruct index 64a4c48eff..b7b0321039 100644 --- a/SConstruct +++ b/SConstruct @@ -13,8 +13,6 @@ from collections import OrderedDict # Local import methods import glsl_builders -import version -from platform_methods import run_in_subprocess # Scan possible build platforms diff --git a/core/SCsub b/core/SCsub index 40ee78d3ea..59fe63b4b3 100644 --- a/core/SCsub +++ b/core/SCsub @@ -4,7 +4,6 @@ Import("env") import core_builders import make_binders -from platform_methods import run_in_subprocess env.core_sources = [] @@ -122,6 +121,7 @@ if env["builtin_zstd"]: "compress/zstdmt_compress.c", "compress/zstd_compress_literals.c", "compress/zstd_compress_sequences.c", + "compress/zstd_compress_superblock.c", "decompress/huf_decompress.c", "decompress/zstd_ddict.c", "decompress/zstd_decompress_block.c", diff --git a/core/image.cpp b/core/image.cpp index e2f353698f..b8a443eed2 100644 --- a/core/image.cpp +++ b/core/image.cpp @@ -363,6 +363,82 @@ void Image::get_mipmap_offset_size_and_dimensions(int p_mipmap, int &r_ofs, int r_size = ofs2 - ofs; } +Image::Image3DValidateError Image::validate_3d_image(Image::Format p_format, int p_width, int p_height, int p_depth, bool p_mipmaps, const Vector<Ref<Image>> &p_images) { + int w = p_width; + int h = p_height; + int d = p_depth; + + int arr_ofs = 0; + + while (true) { + for (int i = 0; i < d; i++) { + int idx = i + arr_ofs; + if (idx >= p_images.size()) { + return VALIDATE_3D_ERR_MISSING_IMAGES; + } + if (p_images[idx].is_null() || p_images[idx]->empty()) { + return VALIDATE_3D_ERR_IMAGE_EMPTY; + } + if (p_images[idx]->get_format() != p_format) { + return VALIDATE_3D_ERR_IMAGE_FORMAT_MISMATCH; + } + if (p_images[idx]->get_width() != w || p_images[idx]->get_height() != h) { + return VALIDATE_3D_ERR_IMAGE_SIZE_MISMATCH; + } + if (p_images[idx]->has_mipmaps()) { + return VALIDATE_3D_ERR_IMAGE_HAS_MIPMAPS; + } + } + + arr_ofs += d; + + if (!p_mipmaps) { + break; + } + + if (w == 1 && h == 1 && d == 1) { + break; + } + + w = MAX(1, w >> 1); + h = MAX(1, h >> 1); + d = MAX(1, d >> 1); + } + + if (arr_ofs != p_images.size()) { + return VALIDATE_3D_ERR_EXTRA_IMAGES; + } + + return VALIDATE_3D_OK; +} + +String Image::get_3d_image_validation_error_text(Image3DValidateError p_error) { + switch (p_error) { + case VALIDATE_3D_OK: { + return TTR("Ok"); + } break; + case VALIDATE_3D_ERR_IMAGE_EMPTY: { + return TTR("Empty Image found"); + } break; + case VALIDATE_3D_ERR_MISSING_IMAGES: { + return TTR("Missing Images"); + } break; + case VALIDATE_3D_ERR_EXTRA_IMAGES: { + return TTR("Too many Images"); + } break; + case VALIDATE_3D_ERR_IMAGE_SIZE_MISMATCH: { + return TTR("Image size mismatch"); + } break; + case VALIDATE_3D_ERR_IMAGE_FORMAT_MISMATCH: { + return TTR("Image format mismatch"); + } break; + case VALIDATE_3D_ERR_IMAGE_HAS_MIPMAPS: { + return TTR("Image has included mipmaps"); + } break; + } + return String(); +} + int Image::get_width() const { return width; } diff --git a/core/image.h b/core/image.h index d2572b072e..06794c7fed 100644 --- a/core/image.h +++ b/core/image.h @@ -230,6 +230,19 @@ public: void get_mipmap_offset_and_size(int p_mipmap, int &r_ofs, int &r_size) const; //get where the mipmap begins in data void get_mipmap_offset_size_and_dimensions(int p_mipmap, int &r_ofs, int &r_size, int &w, int &h) const; //get where the mipmap begins in data + enum Image3DValidateError { + VALIDATE_3D_OK, + VALIDATE_3D_ERR_IMAGE_EMPTY, + VALIDATE_3D_ERR_MISSING_IMAGES, + VALIDATE_3D_ERR_EXTRA_IMAGES, + VALIDATE_3D_ERR_IMAGE_SIZE_MISMATCH, + VALIDATE_3D_ERR_IMAGE_FORMAT_MISMATCH, + VALIDATE_3D_ERR_IMAGE_HAS_MIPMAPS, + }; + + static Image3DValidateError validate_3d_image(Format p_format, int p_width, int p_height, int p_depth, bool p_mipmaps, const Vector<Ref<Image>> &p_images); + static String get_3d_image_validation_error_text(Image3DValidateError p_error); + /** * Resize the image, using the preferred interpolation method. */ diff --git a/core/input/SCsub b/core/input/SCsub index f40978911b..740398b266 100644 --- a/core/input/SCsub +++ b/core/input/SCsub @@ -2,7 +2,6 @@ Import("env") -from platform_methods import run_in_subprocess import input_builders diff --git a/core/list.h b/core/list.h index f850db5241..1cef3c484d 100644 --- a/core/list.h +++ b/core/list.h @@ -395,28 +395,38 @@ public: ERR_FAIL_COND(p_A->data != _data); ERR_FAIL_COND(p_B->data != _data); + if (p_A == p_B) { + return; + } Element *A_prev = p_A->prev_ptr; Element *A_next = p_A->next_ptr; + Element *B_prev = p_B->prev_ptr; + Element *B_next = p_B->next_ptr; - p_A->next_ptr = p_B->next_ptr; - p_A->prev_ptr = p_B->prev_ptr; - - p_B->next_ptr = A_next; - p_B->prev_ptr = A_prev; - - if (p_A->prev_ptr) { - p_A->prev_ptr->next_ptr = p_A; + if (A_prev) { + A_prev->next_ptr = p_B; + } else { + _data->first = p_B; } - if (p_A->next_ptr) { - p_A->next_ptr->prev_ptr = p_A; + if (B_prev) { + B_prev->next_ptr = p_A; + } else { + _data->first = p_A; } - - if (p_B->prev_ptr) { - p_B->prev_ptr->next_ptr = p_B; + if (A_next) { + A_next->prev_ptr = p_B; + } else { + _data->last = p_B; } - if (p_B->next_ptr) { - p_B->next_ptr->prev_ptr = p_B; + if (B_next) { + B_next->prev_ptr = p_A; + } else { + _data->last = p_A; } + p_A->prev_ptr = A_next == p_B ? p_B : B_prev; + p_A->next_ptr = B_next == p_A ? p_B : B_next; + p_B->prev_ptr = B_next == p_A ? p_A : A_prev; + p_B->next_ptr = A_next == p_B ? p_A : A_next; } /** * copy the list diff --git a/core/math/math_funcs.h b/core/math/math_funcs.h index 9f8d4da5b3..24b42790f0 100644 --- a/core/math/math_funcs.h +++ b/core/math/math_funcs.h @@ -46,7 +46,8 @@ class Math { public: Math() {} // useless to instance - static const uint64_t RANDOM_MAX = 0xFFFFFFFF; + // Not using 'RANDOM_MAX' to avoid conflict with system headers on some OSes (at least NetBSD). + static const uint64_t RANDOM_32BIT_MAX = 0xFFFFFFFF; static _ALWAYS_INLINE_ double sin(double p_x) { return ::sin(p_x); } static _ALWAYS_INLINE_ float sin(float p_x) { return ::sinf(p_x); } @@ -283,8 +284,8 @@ public: static void randomize(); static uint32_t rand_from_seed(uint64_t *seed); static uint32_t rand(); - static _ALWAYS_INLINE_ double randd() { return (double)rand() / (double)Math::RANDOM_MAX; } - static _ALWAYS_INLINE_ float randf() { return (float)rand() / (float)Math::RANDOM_MAX; } + static _ALWAYS_INLINE_ double randd() { return (double)rand() / (double)Math::RANDOM_32BIT_MAX; } + static _ALWAYS_INLINE_ float randf() { return (float)rand() / (float)Math::RANDOM_32BIT_MAX; } static double random(double from, double to); static float random(float from, float to); diff --git a/core/math/random_pcg.h b/core/math/random_pcg.h index 8fd5a056fa..09b13ab74d 100644 --- a/core/math/random_pcg.h +++ b/core/math/random_pcg.h @@ -31,12 +31,12 @@ #ifndef RANDOM_PCG_H #define RANDOM_PCG_H -#include <math.h> - #include "core/math/math_defs.h" #include "thirdparty/misc/pcg.h" +#include <math.h> + #if defined(__GNUC__) #define CLZ32(x) __builtin_clz(x) #elif defined(_MSC_VER) @@ -67,7 +67,6 @@ class RandomPCG { public: static const uint64_t DEFAULT_SEED = 12047754176567800795U; static const uint64_t DEFAULT_INC = PCG_DEFAULT_INC_64; - static const uint64_t RANDOM_MAX = 0xFFFFFFFF; RandomPCG(uint64_t p_seed = DEFAULT_SEED, uint64_t p_inc = DEFAULT_INC); diff --git a/core/object.h b/core/object.h index f9a12da8f6..12ef600dfc 100644 --- a/core/object.h +++ b/core/object.h @@ -658,7 +658,7 @@ public: Variant call(const StringName &p_name, VARIANT_ARG_LIST); // C++ helper void notification(int p_notification, bool p_reversed = false); - String to_string(); + virtual String to_string(); //used mainly by script, get and set all INCLUDING string virtual Variant getvar(const Variant &p_key, bool *r_valid = nullptr) const; diff --git a/core/register_core_types.cpp b/core/register_core_types.cpp index 4f094dd6c6..a2b3f75bea 100644 --- a/core/register_core_types.cpp +++ b/core/register_core_types.cpp @@ -232,7 +232,7 @@ void register_core_types() { } void register_core_settings() { - //since in register core types, globals may not e present + // Since in register core types, globals may not be present. GLOBAL_DEF("network/limits/tcp/connect_timeout_seconds", (30)); ProjectSettings::get_singleton()->set_custom_property_info("network/limits/tcp/connect_timeout_seconds", PropertyInfo(Variant::INT, "network/limits/tcp/connect_timeout_seconds", PROPERTY_HINT_RANGE, "1,1800,1")); GLOBAL_DEF_RST("network/limits/packet_peer_stream/max_buffer_po2", (16)); diff --git a/core/translation_po.h b/core/translation_po.h index 730635f63d..88830210ef 100644 --- a/core/translation_po.h +++ b/core/translation_po.h @@ -42,7 +42,7 @@ class TranslationPO : public Translation { // TLDR: Maps context to a list of source strings and translated strings. In PO terms, maps msgctxt to a list of msgid and msgstr. // The first key corresponds to context, and the second key (of the contained HashMap) corresponds to source string. // The value Vector<StringName> in the second map stores the translated strings. Index 0, 1, 2 matches msgstr[0], msgstr[1], msgstr[2]... in the case of plurals. - // Otherwise index 0 mathes to msgstr in a singular translation. + // Otherwise index 0 matches to msgstr in a singular translation. // Strings without context have "" as first key. HashMap<StringName, HashMap<StringName, Vector<StringName>>> translation_map; diff --git a/core/variant_op.cpp b/core/variant_op.cpp index 95b488230d..ec4eea05bf 100644 --- a/core/variant_op.cpp +++ b/core/variant_op.cpp @@ -245,22 +245,22 @@ bool Variant::booleanize() const { _RETURN(p_a._data.m_type); \ } -#define DEFAULT_OP_NUM_VEC(m_prefix, m_op_name, m_name, m_op, m_type) \ - CASE_TYPE(m_prefix, m_op_name, m_name) { \ - if (p_b.type == INT) \ - _RETURN(p_a._data.m_type m_op p_b._data._int); \ - if (p_b.type == FLOAT) \ - _RETURN(p_a._data.m_type m_op p_b._data._float); \ - if (p_b.type == VECTOR2) \ - _RETURN(p_a._data.m_type m_op *reinterpret_cast<const Vector2 *>(p_b._data._mem)); \ - if (p_b.type == VECTOR3) \ - _RETURN(p_a._data.m_type m_op *reinterpret_cast<const Vector3 *>(p_b._data._mem)); \ - if (p_b.type == VECTOR2I) \ - _RETURN(p_a._data.m_type m_op *reinterpret_cast<const Vector2 *>(p_b._data._mem)); \ - if (p_b.type == VECTOR3I) \ - _RETURN(p_a._data.m_type m_op *reinterpret_cast<const Vector3 *>(p_b._data._mem)); \ - \ - _RETURN_FAIL \ +#define DEFAULT_OP_NUM_VEC(m_prefix, m_op_name, m_name, m_op, m_type) \ + CASE_TYPE(m_prefix, m_op_name, m_name) { \ + if (p_b.type == INT) \ + _RETURN(p_a._data.m_type m_op p_b._data._int); \ + if (p_b.type == FLOAT) \ + _RETURN(p_a._data.m_type m_op p_b._data._float); \ + if (p_b.type == VECTOR2) \ + _RETURN(p_a._data.m_type m_op *reinterpret_cast<const Vector2 *>(p_b._data._mem)); \ + if (p_b.type == VECTOR3) \ + _RETURN(p_a._data.m_type m_op *reinterpret_cast<const Vector3 *>(p_b._data._mem)); \ + if (p_b.type == VECTOR2I) \ + _RETURN(p_a._data.m_type m_op *reinterpret_cast<const Vector2i *>(p_b._data._mem)); \ + if (p_b.type == VECTOR3I) \ + _RETURN(p_a._data.m_type m_op *reinterpret_cast<const Vector3i *>(p_b._data._mem)); \ + \ + _RETURN_FAIL \ } #define DEFAULT_OP_STR_REV(m_prefix, m_op_name, m_name, m_op, m_type) \ diff --git a/core/vector.h b/core/vector.h index 5fb630c21c..5a61f0eae3 100644 --- a/core/vector.h +++ b/core/vector.h @@ -82,7 +82,7 @@ public: _FORCE_INLINE_ bool empty() const { return _cowdata.empty(); } _FORCE_INLINE_ T get(int p_index) { return _cowdata.get(p_index); } - _FORCE_INLINE_ const T get(int p_index) const { return _cowdata.get(p_index); } + _FORCE_INLINE_ const T &get(int p_index) const { return _cowdata.get(p_index); } _FORCE_INLINE_ void set(int p_index, const T &p_elem) { _cowdata.set(p_index, p_elem); } _FORCE_INLINE_ int size() const { return _cowdata.size(); } Error resize(int p_size) { return _cowdata.resize(p_size); } diff --git a/doc/classes/AABB.xml b/doc/classes/AABB.xml index cbc04ca672..faa380ff33 100644 --- a/doc/classes/AABB.xml +++ b/doc/classes/AABB.xml @@ -4,7 +4,9 @@ Axis-Aligned Bounding Box. </brief_description> <description> - AABB consists of a position, a size, and several utility functions. It is typically used for fast overlap tests. + [AABB] consists of a position, a size, and several utility functions. It is typically used for fast overlap tests. + It uses floating-point coordinates. The 2D counterpart to [AABB] is [Rect2]. + [b]Note:[/b] Unlike [Rect2], [AABB] does not have a variant that uses integer coordinates. </description> <tutorials> <link title="Math tutorial index">https://docs.godotengine.org/en/latest/tutorials/math/index.html</link> diff --git a/doc/classes/Array.xml b/doc/classes/Array.xml index 20579e0159..8bbb0817ea 100644 --- a/doc/classes/Array.xml +++ b/doc/classes/Array.xml @@ -292,7 +292,7 @@ <return type="Variant"> </return> <description> - Removes and returns the first element of the array. Returns [code]null[/code] if the array is empty, wwithout printing an error message. + Removes and returns the first element of the array. Returns [code]null[/code] if the array is empty, without printing an error message. </description> </method> <method name="push_back"> diff --git a/doc/classes/AudioEffectDistortion.xml b/doc/classes/AudioEffectDistortion.xml index 3cfeaadb23..24a145b0f3 100644 --- a/doc/classes/AudioEffectDistortion.xml +++ b/doc/classes/AudioEffectDistortion.xml @@ -2,13 +2,14 @@ <class name="AudioEffectDistortion" inherits="AudioEffect" version="4.0"> <brief_description> Adds a distortion audio effect to an Audio bus. - Modify the sound to make it dirty. + Modify the sound to make it distorted. </brief_description> <description> - Modify the sound and make it dirty. Different types are available: clip, tan, lo-fi (bit crushing), overdrive, or waveshape. + Different types are available: clip, tan, lo-fi (bit crushing), overdrive, or waveshape. By distorting the waveform the frequency content change, which will often make the sound "crunchy" or "abrasive". For games, it can simulate sound coming from some saturated device or speaker very efficiently. </description> <tutorials> + <link title="Audio buses">https://docs.godotengine.org/en/latest/tutorials/audio/audio_buses.html</link> </tutorials> <methods> </methods> diff --git a/doc/classes/AudioEffectFilter.xml b/doc/classes/AudioEffectFilter.xml index f548fb49cc..293848d204 100644 --- a/doc/classes/AudioEffectFilter.xml +++ b/doc/classes/AudioEffectFilter.xml @@ -7,6 +7,7 @@ Allows frequencies other than the [member cutoff_hz] to pass. </description> <tutorials> + <link title="Audio buses">https://docs.godotengine.org/en/latest/tutorials/audio/audio_buses.html</link> </tutorials> <methods> </methods> @@ -20,7 +21,7 @@ Gain amount of the frequencies after the filter. </member> <member name="resonance" type="float" setter="set_resonance" getter="get_resonance" default="0.5"> - Amount of boost in the overtones near the cutoff frequency. + Amount of boost in the frequency range near the cutoff frequency. </member> </members> <constants> diff --git a/doc/classes/BaseButton.xml b/doc/classes/BaseButton.xml index 3812b45b13..bf4d9383ac 100644 --- a/doc/classes/BaseButton.xml +++ b/doc/classes/BaseButton.xml @@ -65,8 +65,8 @@ <member name="pressed" type="bool" setter="set_pressed" getter="is_pressed" default="false"> If [code]true[/code], the button's state is pressed. Means the button is pressed down or toggled (if [member toggle_mode] is active). </member> - <member name="shortcut" type="ShortCut" setter="set_shortcut" getter="get_shortcut"> - [ShortCut] associated to the button. + <member name="shortcut" type="Shortcut" setter="set_shortcut" getter="get_shortcut"> + [Shortcut] associated to the button. </member> <member name="shortcut_in_tooltip" type="bool" setter="set_shortcut_in_tooltip" getter="is_shortcut_in_tooltip_enabled" default="true"> If [code]true[/code], the button will add information about its shortcut in the tooltip. diff --git a/doc/classes/CodeEdit.xml b/doc/classes/CodeEdit.xml new file mode 100644 index 0000000000..f6bc9e2cca --- /dev/null +++ b/doc/classes/CodeEdit.xml @@ -0,0 +1,203 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<class name="CodeEdit" inherits="TextEdit" version="4.0"> + <brief_description> + </brief_description> + <description> + </description> + <tutorials> + </tutorials> + <methods> + <method name="clear_bookmarked_lines"> + <return type="void"> + </return> + <description> + </description> + </method> + <method name="clear_breakpointed_lines"> + <return type="void"> + </return> + <description> + </description> + </method> + <method name="clear_executing_lines"> + <return type="void"> + </return> + <description> + </description> + </method> + <method name="get_bookmarked_lines" qualifiers="const"> + <return type="Array"> + </return> + <description> + </description> + </method> + <method name="get_breakpointed_lines" qualifiers="const"> + <return type="Array"> + </return> + <description> + </description> + </method> + <method name="get_executing_lines" qualifiers="const"> + <return type="Array"> + </return> + <description> + </description> + </method> + <method name="is_line_bookmarked" qualifiers="const"> + <return type="bool"> + </return> + <argument index="0" name="line" type="int"> + </argument> + <description> + </description> + </method> + <method name="is_line_breakpointed" qualifiers="const"> + <return type="bool"> + </return> + <argument index="0" name="line" type="int"> + </argument> + <description> + </description> + </method> + <method name="is_line_executing" qualifiers="const"> + <return type="bool"> + </return> + <argument index="0" name="line" type="int"> + </argument> + <description> + </description> + </method> + <method name="set_line_as_bookmarked"> + <return type="void"> + </return> + <argument index="0" name="line" type="int"> + </argument> + <argument index="1" name="bookmarked" type="bool"> + </argument> + <description> + </description> + </method> + <method name="set_line_as_breakpoint"> + <return type="void"> + </return> + <argument index="0" name="line" type="int"> + </argument> + <argument index="1" name="breakpointed" type="bool"> + </argument> + <description> + </description> + </method> + <method name="set_line_as_executing"> + <return type="void"> + </return> + <argument index="0" name="line" type="int"> + </argument> + <argument index="1" name="executing" type="bool"> + </argument> + <description> + </description> + </method> + </methods> + <members> + <member name="draw_bookmarks" type="bool" setter="set_draw_bookmarks_gutter" getter="is_drawing_bookmarks_gutter" default="false"> + </member> + <member name="draw_breakpoints_gutter" type="bool" setter="set_draw_breakpoints_gutter" getter="is_drawing_breakpoints_gutter" default="false"> + </member> + <member name="draw_executing_lines" type="bool" setter="set_draw_executing_lines_gutter" getter="is_drawing_executing_lines_gutter" default="false"> + </member> + <member name="draw_fold_gutter" type="bool" setter="set_draw_fold_gutter" getter="is_drawing_fold_gutter" default="false"> + </member> + <member name="draw_line_numbers" type="bool" setter="set_draw_line_numbers" getter="is_draw_line_numbers_enabled" default="false"> + </member> + <member name="zero_pad_line_numbers" type="bool" setter="set_line_numbers_zero_padded" getter="is_line_numbers_zero_padded" default="false"> + </member> + </members> + <signals> + <signal name="breakpoint_toggled"> + <argument index="0" name="line" type="int"> + </argument> + <description> + </description> + </signal> + </signals> + <constants> + </constants> + <theme_items> + <theme_item name="background_color" type="Color" default="Color( 0, 0, 0, 0 )"> + </theme_item> + <theme_item name="bookmark" type="Texture2D"> + </theme_item> + <theme_item name="bookmark_color" type="Color" default="Color( 0.5, 0.64, 1, 0.8 )"> + </theme_item> + <theme_item name="brace_mismatch_color" type="Color" default="Color( 1, 0.2, 0.2, 1 )"> + </theme_item> + <theme_item name="breakpoint" type="Texture2D"> + </theme_item> + <theme_item name="breakpoint_color" type="Color" default="Color( 0.9, 0.29, 0.3, 1 )"> + </theme_item> + <theme_item name="can_fold" type="Texture2D"> + </theme_item> + <theme_item name="caret_background_color" type="Color" default="Color( 0, 0, 0, 1 )"> + </theme_item> + <theme_item name="caret_color" type="Color" default="Color( 0.88, 0.88, 0.88, 1 )"> + </theme_item> + <theme_item name="code_folding_color" type="Color" default="Color( 0.8, 0.8, 0.8, 0.8 )"> + </theme_item> + <theme_item name="completion" type="StyleBox"> + </theme_item> + <theme_item name="completion_background_color" type="Color" default="Color( 0.17, 0.16, 0.2, 1 )"> + </theme_item> + <theme_item name="completion_existing_color" type="Color" default="Color( 0.87, 0.87, 0.87, 0.13 )"> + </theme_item> + <theme_item name="completion_font_color" type="Color" default="Color( 0.67, 0.67, 0.67, 1 )"> + </theme_item> + <theme_item name="completion_lines" type="int" default="7"> + </theme_item> + <theme_item name="completion_max_width" type="int" default="50"> + </theme_item> + <theme_item name="completion_scroll_color" type="Color" default="Color( 1, 1, 1, 1 )"> + </theme_item> + <theme_item name="completion_scroll_width" type="int" default="3"> + </theme_item> + <theme_item name="completion_selected_color" type="Color" default="Color( 0.26, 0.26, 0.27, 1 )"> + </theme_item> + <theme_item name="current_line_color" type="Color" default="Color( 0.25, 0.25, 0.26, 0.8 )"> + </theme_item> + <theme_item name="executing_line" type="Texture2D"> + </theme_item> + <theme_item name="executing_line_color" type="Color" default="Color( 0.98, 0.89, 0.27, 1 )"> + </theme_item> + <theme_item name="focus" type="StyleBox"> + </theme_item> + <theme_item name="folded" type="Texture2D"> + </theme_item> + <theme_item name="font" type="Font"> + </theme_item> + <theme_item name="font_color" type="Color" default="Color( 0.88, 0.88, 0.88, 1 )"> + </theme_item> + <theme_item name="font_color_readonly" type="Color" default="Color( 0.88, 0.88, 0.88, 0.5 )"> + </theme_item> + <theme_item name="font_color_selected" type="Color" default="Color( 0, 0, 0, 1 )"> + </theme_item> + <theme_item name="line_number_color" type="Color" default="Color( 0.67, 0.67, 0.67, 0.4 )"> + </theme_item> + <theme_item name="line_spacing" type="int" default="4"> + </theme_item> + <theme_item name="mark_color" type="Color" default="Color( 1, 0.4, 0.4, 0.4 )"> + </theme_item> + <theme_item name="normal" type="StyleBox"> + </theme_item> + <theme_item name="read_only" type="StyleBox"> + </theme_item> + <theme_item name="safe_line_number_color" type="Color" default="Color( 0.67, 0.78, 0.67, 0.6 )"> + </theme_item> + <theme_item name="selection_color" type="Color" default="Color( 0.49, 0.49, 0.49, 1 )"> + </theme_item> + <theme_item name="space" type="Texture2D"> + </theme_item> + <theme_item name="tab" type="Texture2D"> + </theme_item> + <theme_item name="word_highlighted_color" type="Color" default="Color( 0.8, 0.9, 0.9, 0.15 )"> + </theme_item> + </theme_items> +</class> diff --git a/doc/classes/DisplayServer.xml b/doc/classes/DisplayServer.xml index 814c232668..28194587ab 100644 --- a/doc/classes/DisplayServer.xml +++ b/doc/classes/DisplayServer.xml @@ -901,6 +901,30 @@ <description> </description> </method> + <method name="window_set_mouse_passthrough"> + <return type="void"> + </return> + <argument index="0" name="region" type="PackedVector2Array"> + </argument> + <argument index="1" name="window_id" type="int" default="0"> + </argument> + <description> + Sets a polygonal region of the window which accepts mouse events. Mouse events outside the region will be passed through. + Passing an empty array will disable passthrough support (all mouse events will be intercepted by the window, which is the default behavior). + [codeblock] + # Set region, using Path2D node. + DisplayServer.window_set_mouse_passthrough($Path2D.curve.get_baked_points()) + + # Set region, using Polygon2D node. + DisplayServer.window_set_mouse_passthrough($Polygon2D.polygon) + + # Reset region to default. + DisplayServer.window_set_mouse_passthrough([]) + [/codeblock] + [b]Note:[/b] On Windows, the portion of a window that lies outside the region is not drawn, while on Linux and macOS it is. + [b]Note:[/b] This method is implemented on Linux, macOS and Windows. + </description> + </method> <method name="window_set_position"> <return type="void"> </return> diff --git a/doc/classes/GPUParticles3D.xml b/doc/classes/GPUParticles3D.xml index 3fc9e73ccf..8444610f49 100644 --- a/doc/classes/GPUParticles3D.xml +++ b/doc/classes/GPUParticles3D.xml @@ -18,6 +18,22 @@ Returns the axis-aligned bounding box that contains all the particles that are active in the current frame. </description> </method> + <method name="emit_particle"> + <return type="void"> + </return> + <argument index="0" name="xform" type="Transform"> + </argument> + <argument index="1" name="velocity" type="Vector3"> + </argument> + <argument index="2" name="color" type="Color"> + </argument> + <argument index="3" name="custom" type="Color"> + </argument> + <argument index="4" name="flags" type="int"> + </argument> + <description> + </description> + </method> <method name="get_draw_pass_mesh" qualifiers="const"> <return type="Mesh"> </return> @@ -101,6 +117,8 @@ <member name="speed_scale" type="float" setter="set_speed_scale" getter="get_speed_scale" default="1.0"> Speed scaling ratio. A value of [code]0[/code] can be used to pause the particles. </member> + <member name="sub_emitter" type="NodePath" setter="set_sub_emitter" getter="get_sub_emitter" default="NodePath("")"> + </member> <member name="visibility_aabb" type="AABB" setter="set_visibility_aabb" getter="get_visibility_aabb" default="AABB( -4, -4, -4, 8, 8, 8 )"> The [AABB] that determines the area of the world part of which needs to be visible on screen for the particle system to be active. </member> @@ -115,6 +133,16 @@ <constant name="DRAW_ORDER_VIEW_DEPTH" value="2" enum="DrawOrder"> Particles are drawn in order of depth. </constant> + <constant name="EMIT_FLAG_POSITION" value="1" enum="EmitFlags"> + </constant> + <constant name="EMIT_FLAG_ROTATION_SCALE" value="2" enum="EmitFlags"> + </constant> + <constant name="EMIT_FLAG_VELOCITY" value="4" enum="EmitFlags"> + </constant> + <constant name="EMIT_FLAG_COLOR" value="8" enum="EmitFlags"> + </constant> + <constant name="EMIT_FLAG_CUSTOM" value="16" enum="EmitFlags"> + </constant> <constant name="MAX_DRAW_PASSES" value="4"> Maximum number of draw passes supported. </constant> diff --git a/doc/classes/HTTPRequest.xml b/doc/classes/HTTPRequest.xml index 2ba2249d21..8cc7ecfbe3 100644 --- a/doc/classes/HTTPRequest.xml +++ b/doc/classes/HTTPRequest.xml @@ -135,7 +135,7 @@ </argument> <argument index="3" name="method" type="int" enum="HTTPClient.Method" default="0"> </argument> - <argument index="4" name="request_data_raw" type="PackedByteArray" default="PackedByteArray()"> + <argument index="4" name="request_data_raw" type="PackedByteArray" default="PackedByteArray( )"> </argument> <description> Creates request on the underlying [HTTPClient] using a raw array of bytes for the request body. If there is no configuration errors, it tries to connect using [method HTTPClient.connect_to_host] and passes parameters onto [method HTTPClient.request]. @@ -144,6 +144,12 @@ </method> </methods> <members> + <member name="accept_gzip" type="bool" setter="set_accept_gzip" getter="is_accepting_gzip" default="true"> + If [code]true[/code], this header will be added to each request: [code]Accept-Encoding: gzip, deflate[/code] telling servers that it's okay to compress response bodies. + Any Response body declaring a [code]Content-Encoding[/code] of either [code]gzip[/code] or [code]deflate[/code] will then be automatically decompressed, and the uncompressed bytes will be delivered via [code]request_completed[/code]. + If the user has specified their own [code]Accept-Encoding[/code] header, then no header will be added regaurdless of [code]accept_gzip[/code]. + If [code]false[/code] no header will be added, and no decompression will be performed on response bodies. The raw bytes of the response body will be returned via [code]request_completed[/code]. + </member> <member name="body_size_limit" type="int" setter="set_body_size_limit" getter="get_body_size_limit" default="-1"> Maximum allowed size for response bodies. If the response body is compressed, this will be used as the maximum allowed size for the decompressed body. </member> @@ -162,12 +168,6 @@ <member name="use_threads" type="bool" setter="set_use_threads" getter="is_using_threads" default="false"> If [code]true[/code], multithreading is used to improve performance. </member> - <member name="accept_gzip" type="bool" setter="set_accept_gzip" getter="is_accepting_gzip" default="true"> - If [code]true[/code], this header will be added to each request: [code]Accept-Encoding: gzip, deflate[/code] telling servers that it's okay to compress response bodies. - Any Reponse body declaring a [code]Content-Encoding[/code] of either [code]gzip[/code] or [code]deflate[/code] will then be automatically decompressed, and the uncompressed bytes will be delivered via [code]request_completed[/code]. - If the user has specified their own [code]Accept-Encoding[/code] header, then no header will be added regaurdless of [code]accept_gzip[/code]. - If [code]false[/code] no header will be added, and no decompression will be performed on response bodies. The raw bytes of the response body will be returned via [code]request_completed[/code]. - </member> </members> <signals> <signal name="request_completed"> @@ -208,19 +208,21 @@ <constant name="RESULT_BODY_SIZE_LIMIT_EXCEEDED" value="7" enum="Result"> Request exceeded its maximum size limit, see [member body_size_limit]. </constant> - <constant name="RESULT_REQUEST_FAILED" value="8" enum="Result"> + <constant name="RESULT_BODY_DECOMPRESS_FAILED" value="8" enum="Result"> + </constant> + <constant name="RESULT_REQUEST_FAILED" value="9" enum="Result"> Request failed (currently unused). </constant> - <constant name="RESULT_DOWNLOAD_FILE_CANT_OPEN" value="9" enum="Result"> + <constant name="RESULT_DOWNLOAD_FILE_CANT_OPEN" value="10" enum="Result"> HTTPRequest couldn't open the download file. </constant> - <constant name="RESULT_DOWNLOAD_FILE_WRITE_ERROR" value="10" enum="Result"> + <constant name="RESULT_DOWNLOAD_FILE_WRITE_ERROR" value="11" enum="Result"> HTTPRequest couldn't write to the download file. </constant> - <constant name="RESULT_REDIRECT_LIMIT_REACHED" value="11" enum="Result"> + <constant name="RESULT_REDIRECT_LIMIT_REACHED" value="12" enum="Result"> Request reached its maximum redirect limit, see [member max_redirects]. </constant> - <constant name="RESULT_TIMEOUT" value="12" enum="Result"> + <constant name="RESULT_TIMEOUT" value="13" enum="Result"> </constant> </constants> </class> diff --git a/doc/classes/Image.xml b/doc/classes/Image.xml index 5aa5de1dae..20be20db34 100644 --- a/doc/classes/Image.xml +++ b/doc/classes/Image.xml @@ -233,7 +233,7 @@ <return type="PackedByteArray"> </return> <description> - Returns the image's raw data. + Returns a copy of the image's raw data. </description> </method> <method name="get_format" qualifiers="const"> diff --git a/doc/classes/ImageTexture3D.xml b/doc/classes/ImageTexture3D.xml new file mode 100644 index 0000000000..d05082487d --- /dev/null +++ b/doc/classes/ImageTexture3D.xml @@ -0,0 +1,39 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<class name="ImageTexture3D" inherits="Texture3D" version="4.0"> + <brief_description> + </brief_description> + <description> + </description> + <tutorials> + </tutorials> + <methods> + <method name="create"> + <return type="int" enum="Error"> + </return> + <argument index="0" name="format" type="int" enum="Image.Format"> + </argument> + <argument index="1" name="width" type="int"> + </argument> + <argument index="2" name="height" type="int"> + </argument> + <argument index="3" name="depth" type="int"> + </argument> + <argument index="4" name="use_mipmaps" type="bool"> + </argument> + <argument index="5" name="data" type="Image[]"> + </argument> + <description> + </description> + </method> + <method name="update"> + <return type="void"> + </return> + <argument index="0" name="data" type="Image[]"> + </argument> + <description> + </description> + </method> + </methods> + <constants> + </constants> +</class> diff --git a/doc/classes/JavaScript.xml b/doc/classes/JavaScript.xml index d2cb558385..c707a72ee8 100644 --- a/doc/classes/JavaScript.xml +++ b/doc/classes/JavaScript.xml @@ -5,6 +5,7 @@ </brief_description> <description> The JavaScript singleton is implemented only in the HTML5 export. It's used to access the browser's JavaScript context. This allows interaction with embedding pages or calling third-party JavaScript APIs. + [b]Note:[/b] This singleton can be disabled at build-time to improve security. By default, the JavaScript singleton is enabled. Official export templates also have the JavaScript singleton enabled. See [url=https://docs.godotengine.org/en/latest/development/compiling/compiling_for_web.html]Compiling for the Web[/url] in the documentation for more information. </description> <tutorials> <link title="Exporting for the Web: Calling JavaScript from script">https://docs.godotengine.org/en/latest/getting_started/workflow/export/exporting_for_web.html#calling-javascript-from-script</link> diff --git a/doc/classes/Mesh.xml b/doc/classes/Mesh.xml index 367099e455..1efa72a2b2 100644 --- a/doc/classes/Mesh.xml +++ b/doc/classes/Mesh.xml @@ -44,7 +44,7 @@ <return type="AABB"> </return> <description> - Returns the smallest [AABB] enclosing this mesh. Not affected by [code]custom_aabb[/code]. + Returns the smallest [AABB] enclosing this mesh in local space. Not affected by [code]custom_aabb[/code]. See also [method VisualInstance3D.get_transformed_aabb]. [b]Note:[/b] This is only implemented for [ArrayMesh] and [PrimitiveMesh]. </description> </method> diff --git a/doc/classes/MultiMesh.xml b/doc/classes/MultiMesh.xml index 8a6c560cdd..e1b3ffa2e4 100644 --- a/doc/classes/MultiMesh.xml +++ b/doc/classes/MultiMesh.xml @@ -18,7 +18,7 @@ <return type="AABB"> </return> <description> - Returns the visibility axis-aligned bounding box. + Returns the visibility axis-aligned bounding box in local space. See also [method VisualInstance3D.get_transformed_aabb]. </description> </method> <method name="get_instance_color" qualifiers="const"> diff --git a/doc/classes/Node.xml b/doc/classes/Node.xml index b342fc0813..1548800901 100644 --- a/doc/classes/Node.xml +++ b/doc/classes/Node.xml @@ -525,7 +525,7 @@ ┠╴Menu ┃ ┠╴Label ┃ ┖╴Camera2D - ┖-SplashScreen + ┖╴SplashScreen ┖╴Camera2D [/codeblock] </description> diff --git a/doc/classes/Object.xml b/doc/classes/Object.xml index 2395ccd211..50d91c7943 100644 --- a/doc/classes/Object.xml +++ b/doc/classes/Object.xml @@ -395,7 +395,7 @@ <argument index="0" name="name" type="String"> </argument> <description> - Removes a given entry from the object's metadata. + Removes a given entry from the object's metadata. See also [method set_meta]. </description> </method> <method name="set"> @@ -464,7 +464,8 @@ <argument index="1" name="value" type="Variant"> </argument> <description> - Adds or changes a given entry in the object's metadata. Metadata are serialized, and can take any [Variant] value. + Adds, changes or removes a given entry in the object's metadata. Metadata are serialized and can take any [Variant] value. + To remove a given entry from the object's metadata, use [method remove_meta]. Metadata is also removed if its value is set to [code]null[/code]. This means you can also use [code]set_meta("name", null)[/code] to remove metadata for [code]"name"[/code]. </description> </method> <method name="set_script"> diff --git a/doc/classes/PackedByteArray.xml b/doc/classes/PackedByteArray.xml index 7bfc502fbb..0b43522bce 100644 --- a/doc/classes/PackedByteArray.xml +++ b/doc/classes/PackedByteArray.xml @@ -60,15 +60,14 @@ <method name="decompress_dynamic"> <return type="PackedByteArray"> </return> - <argument index="0" name="max_output_size" type="int" default="0"> + <argument index="0" name="max_output_size" type="int"> </argument> <argument index="1" name="compression_mode" type="int" default="0"> </argument> <description> Returns a new [PackedByteArray] with the data decompressed. Set the compression mode using one of [enum File.CompressionMode]'s constants. [b]This method only accepts gzip and deflate compression modes.[/b] - This method is potentially slower than [code]decompress[/code], as it may have to re-allocate it's output buffer multiple times while decompressing, where as [code]decompress[/code] knows it's output buffer size from the begining. - - GZIP has a maximal compression ratio of 1032:1, meaning it's very possible for a small compressed payload to decompress to a potentially very large output. To guard against this, you may provide a maximum size this function is allowed to allocate in bytes via [code]max_output_size[/code]. Passing -1 will allow for unbounded output. If any positive value is passed, and the decompression exceeds that ammount in bytes, then an error will be returned. + This method is potentially slower than [code]decompress[/code], as it may have to re-allocate it's output buffer multiple times while decompressing, where as [code]decompress[/code] knows it's output buffer size from the beginning. + GZIP has a maximal compression ratio of 1032:1, meaning it's very possible for a small compressed payload to decompress to a potentially very large output. To guard against this, you may provide a maximum size this function is allowed to allocate in bytes via [code]max_output_size[/code]. Passing -1 will allow for unbounded output. If any positive value is passed, and the decompression exceeds that amount in bytes, then an error will be returned. </description> </method> <method name="empty"> @@ -89,21 +88,21 @@ <return type="String"> </return> <description> - Converts UTF-16 encoded array to [String]. If the BOM is missing, system endianness is assumed. Returns empty string if source array is not vaild UTF-16 string. + Converts UTF-16 encoded array to [String]. If the BOM is missing, system endianness is assumed. Returns empty string if source array is not valid UTF-16 string. </description> </method> <method name="get_string_from_utf32"> <return type="String"> </return> <description> - Converts UTF-32 encoded array to [String]. System endianness is assumed. Returns empty string if source array is not vaild UTF-32 string. + Converts UTF-32 encoded array to [String]. System endianness is assumed. Returns empty string if source array is not valid UTF-32 string. </description> </method> <method name="get_string_from_utf8"> <return type="String"> </return> <description> - Converts UTF-8 encoded array to [String]. Slower than [method get_string_from_ascii] but supports UTF-8 encoded data. Use this function if you are unsure about the source of the data. For user input this function should always be preferred. Returns empty string if source array is not vaild UTF-8 string. + Converts UTF-8 encoded array to [String]. Slower than [method get_string_from_ascii] but supports UTF-8 encoded data. Use this function if you are unsure about the source of the data. For user input this function should always be preferred. Returns empty string if source array is not valid UTF-8 string. </description> </method> <method name="has"> diff --git a/doc/classes/ParticlesMaterial.xml b/doc/classes/ParticlesMaterial.xml index d04ac5bdce..f6fa3cf38b 100644 --- a/doc/classes/ParticlesMaterial.xml +++ b/doc/classes/ParticlesMaterial.xml @@ -243,6 +243,14 @@ <member name="spread" type="float" setter="set_spread" getter="get_spread" default="45.0"> Each particle's initial direction range from [code]+spread[/code] to [code]-spread[/code] degrees. Applied to X/Z plane and Y/Z planes. </member> + <member name="sub_emitter_amount_at_end" type="int" setter="set_sub_emitter_amount_at_end" getter="get_sub_emitter_amount_at_end"> + </member> + <member name="sub_emitter_frequency" type="float" setter="set_sub_emitter_frequency" getter="get_sub_emitter_frequency"> + </member> + <member name="sub_emitter_keep_velocity" type="bool" setter="set_sub_emitter_keep_velocity" getter="get_sub_emitter_keep_velocity" default="false"> + </member> + <member name="sub_emitter_mode" type="int" setter="set_sub_emitter_mode" getter="get_sub_emitter_mode" enum="ParticlesMaterial.SubEmitterMode" default="0"> + </member> <member name="tangential_accel" type="float" setter="set_param" getter="get_param" default="0.0"> Tangential acceleration applied to each particle. Tangential acceleration is perpendicular to the particle's velocity giving the particles a swirling motion. </member> @@ -252,15 +260,6 @@ <member name="tangential_accel_random" type="float" setter="set_param_randomness" getter="get_param_randomness" default="0.0"> Tangential acceleration randomness ratio. </member> - <member name="trail_color_modifier" type="GradientTexture" setter="set_trail_color_modifier" getter="get_trail_color_modifier"> - Trail particles' color will vary along this [GradientTexture]. - </member> - <member name="trail_divisor" type="int" setter="set_trail_divisor" getter="get_trail_divisor" default="1"> - Emitter will emit [code]amount[/code] divided by [code]trail_divisor[/code] particles. The remaining particles will be used as trail(s). - </member> - <member name="trail_size_modifier" type="CurveTexture" setter="set_trail_size_modifier" getter="get_trail_size_modifier"> - Trail particles' size will vary along this [CurveTexture]. - </member> </members> <constants> <constant name="PARAM_INITIAL_LINEAR_VELOCITY" value="0" enum="Parameter"> @@ -332,5 +331,15 @@ <constant name="EMISSION_SHAPE_MAX" value="5" enum="EmissionShape"> Represents the size of the [enum EmissionShape] enum. </constant> + <constant name="SUB_EMITTER_DISABLED" value="0" enum="SubEmitterMode"> + </constant> + <constant name="SUB_EMITTER_CONSTANT" value="1" enum="SubEmitterMode"> + </constant> + <constant name="SUB_EMITTER_AT_END" value="2" enum="SubEmitterMode"> + </constant> + <constant name="SUB_EMITTER_AT_COLLISION" value="3" enum="SubEmitterMode"> + </constant> + <constant name="SUB_EMITTER_MAX" value="4" enum="SubEmitterMode"> + </constant> </constants> </class> diff --git a/doc/classes/PopupMenu.xml b/doc/classes/PopupMenu.xml index 2af0f500a0..b1ec9a222a 100644 --- a/doc/classes/PopupMenu.xml +++ b/doc/classes/PopupMenu.xml @@ -29,14 +29,14 @@ <method name="add_check_shortcut"> <return type="void"> </return> - <argument index="0" name="shortcut" type="ShortCut"> + <argument index="0" name="shortcut" type="Shortcut"> </argument> <argument index="1" name="id" type="int" default="-1"> </argument> <argument index="2" name="global" type="bool" default="false"> </argument> <description> - Adds a new checkable item and assigns the specified [ShortCut] to it. Sets the label of the checkbox to the [ShortCut]'s name. + Adds a new checkable item and assigns the specified [Shortcut] to it. Sets the label of the checkbox to the [Shortcut]'s name. An [code]id[/code] can optionally be provided. If no [code]id[/code] is provided, one will be created from the index. [b]Note:[/b] Checkable items just display a checkmark, but don't have any built-in checking behavior and must be checked/unchecked manually. See [method set_item_checked] for more info on how to control it. </description> @@ -63,14 +63,14 @@ </return> <argument index="0" name="texture" type="Texture2D"> </argument> - <argument index="1" name="shortcut" type="ShortCut"> + <argument index="1" name="shortcut" type="Shortcut"> </argument> <argument index="2" name="id" type="int" default="-1"> </argument> <argument index="3" name="global" type="bool" default="false"> </argument> <description> - Adds a new checkable item and assigns the specified [ShortCut] and icon [code]texture[/code] to it. Sets the label of the checkbox to the [ShortCut]'s name. + Adds a new checkable item and assigns the specified [Shortcut] and icon [code]texture[/code] to it. Sets the label of the checkbox to the [Shortcut]'s name. An [code]id[/code] can optionally be provided. If no [code]id[/code] is provided, one will be created from the index. [b]Note:[/b] Checkable items just display a checkmark, but don't have any built-in checking behavior and must be checked/unchecked manually. See [method set_item_checked] for more info on how to control it. </description> @@ -111,7 +111,7 @@ </return> <argument index="0" name="texture" type="Texture2D"> </argument> - <argument index="1" name="shortcut" type="ShortCut"> + <argument index="1" name="shortcut" type="Shortcut"> </argument> <argument index="2" name="id" type="int" default="-1"> </argument> @@ -126,14 +126,14 @@ </return> <argument index="0" name="texture" type="Texture2D"> </argument> - <argument index="1" name="shortcut" type="ShortCut"> + <argument index="1" name="shortcut" type="Shortcut"> </argument> <argument index="2" name="id" type="int" default="-1"> </argument> <argument index="3" name="global" type="bool" default="false"> </argument> <description> - Adds a new item and assigns the specified [ShortCut] and icon [code]texture[/code] to it. Sets the label of the checkbox to the [ShortCut]'s name. + Adds a new item and assigns the specified [Shortcut] and icon [code]texture[/code] to it. Sets the label of the checkbox to the [Shortcut]'s name. An [code]id[/code] can optionally be provided. If no [code]id[/code] is provided, one will be created from the index. </description> </method> @@ -188,14 +188,14 @@ <method name="add_radio_check_shortcut"> <return type="void"> </return> - <argument index="0" name="shortcut" type="ShortCut"> + <argument index="0" name="shortcut" type="Shortcut"> </argument> <argument index="1" name="id" type="int" default="-1"> </argument> <argument index="2" name="global" type="bool" default="false"> </argument> <description> - Adds a new radio check button and assigns a [ShortCut] to it. Sets the label of the checkbox to the [ShortCut]'s name. + Adds a new radio check button and assigns a [Shortcut] to it. Sets the label of the checkbox to the [Shortcut]'s name. An [code]id[/code] can optionally be provided. If no [code]id[/code] is provided, one will be created from the index. [b]Note:[/b] Checkable items just display a checkmark, but don't have any built-in checking behavior and must be checked/unchecked manually. See [method set_item_checked] for more info on how to control it. </description> @@ -212,14 +212,14 @@ <method name="add_shortcut"> <return type="void"> </return> - <argument index="0" name="shortcut" type="ShortCut"> + <argument index="0" name="shortcut" type="Shortcut"> </argument> <argument index="1" name="id" type="int" default="-1"> </argument> <argument index="2" name="global" type="bool" default="false"> </argument> <description> - Adds a [ShortCut]. + Adds a [Shortcut]. An [code]id[/code] can optionally be provided. If no [code]id[/code] is provided, one will be created from the index. </description> </method> @@ -303,12 +303,12 @@ </description> </method> <method name="get_item_shortcut" qualifiers="const"> - <return type="ShortCut"> + <return type="Shortcut"> </return> <argument index="0" name="idx" type="int"> </argument> <description> - Returns the [ShortCut] associated with the specified [code]idx[/code] item. + Returns the [Shortcut] associated with the specified [code]idx[/code] item. </description> </method> <method name="get_item_submenu" qualifiers="const"> @@ -521,12 +521,12 @@ </return> <argument index="0" name="idx" type="int"> </argument> - <argument index="1" name="shortcut" type="ShortCut"> + <argument index="1" name="shortcut" type="Shortcut"> </argument> <argument index="2" name="global" type="bool" default="false"> </argument> <description> - Sets a [ShortCut] for the specified item [code]idx[/code]. + Sets a [Shortcut] for the specified item [code]idx[/code]. </description> </method> <method name="set_item_shortcut_disabled"> @@ -537,7 +537,7 @@ <argument index="1" name="disabled" type="bool"> </argument> <description> - Disables the [ShortCut] of the specified index [code]idx[/code]. + Disables the [Shortcut] of the specified index [code]idx[/code]. </description> </method> <method name="set_item_submenu"> diff --git a/doc/classes/ProjectSettings.xml b/doc/classes/ProjectSettings.xml index 38d65f6338..ed8eddda07 100644 --- a/doc/classes/ProjectSettings.xml +++ b/doc/classes/ProjectSettings.xml @@ -873,7 +873,7 @@ Size of the hash table used for the broad-phase 2D hash grid algorithm. </member> <member name="physics/2d/cell_size" type="int" setter="" getter="" default="128"> - Cell size used for the broad-phase 2D hash grid algorithm. + Cell size used for the broad-phase 2D hash grid algorithm (in pixels). </member> <member name="physics/2d/default_angular_damp" type="float" setter="" getter="" default="1.0"> The default angular damp in 2D. @@ -1199,7 +1199,7 @@ <member name="rendering/vulkan/staging_buffer/texture_upload_region_size_px" type="int" setter="" getter="" default="64"> </member> <member name="world/2d/cell_size" type="int" setter="" getter="" default="100"> - Cell size used for the 2D hash grid that [VisibilityNotifier2D] uses. + Cell size used for the 2D hash grid that [VisibilityNotifier2D] uses (in pixels). </member> </members> <constants> diff --git a/doc/classes/RayCast2D.xml b/doc/classes/RayCast2D.xml index db4b9a0383..e30d7df63f 100644 --- a/doc/classes/RayCast2D.xml +++ b/doc/classes/RayCast2D.xml @@ -4,8 +4,8 @@ Query the closest object intersecting a ray. </brief_description> <description> - A RayCast represents a line from its origin to its destination position, [code]cast_to[/code]. It is used to query the 2D space in order to find the closest object along the path of the ray. - RayCast2D can ignore some objects by adding them to the exception list via [code]add_exception[/code], by setting proper filtering with collision layers, or by filtering object types with type masks. + A RayCast represents a line from its origin to its destination position, [member target_position]. It is used to query the 2D space in order to find the closest object along the path of the ray. + RayCast2D can ignore some objects by adding them to the exception list via [method add_exception], by setting proper filtering with collision layers, or by filtering object types with type masks. RayCast2D can be configured to report collisions with [Area2D]s ([member collide_with_areas]) and/or [PhysicsBody2D]s ([member collide_with_bodies]). Only enabled raycasts will be able to query the space and report collisions. RayCast2D calculates intersection every physics frame (see [Node]), and the result is cached so it can be used later until the next frame. If multiple queries are required between physics frames (or during the same frame) use [method force_raycast_update] after adjusting the raycast. @@ -123,9 +123,6 @@ </method> </methods> <members> - <member name="cast_to" type="Vector2" setter="set_cast_to" getter="get_cast_to" default="Vector2( 0, 50 )"> - The ray's destination point, relative to the RayCast's [code]position[/code]. - </member> <member name="collide_with_areas" type="bool" setter="set_collide_with_areas" getter="is_collide_with_areas_enabled" default="false"> If [code]true[/code], collision with [Area2D]s will be reported. </member> @@ -141,6 +138,9 @@ <member name="exclude_parent" type="bool" setter="set_exclude_parent_body" getter="get_exclude_parent_body" default="true"> If [code]true[/code], the parent node will be excluded from collision detection. </member> + <member name="target_position" type="Vector2" setter="set_target_position" getter="get_target_position" default="Vector2( 0, 50 )"> + The ray's destination point, relative to the RayCast's [code]position[/code]. + </member> </members> <constants> </constants> diff --git a/doc/classes/RayCast3D.xml b/doc/classes/RayCast3D.xml index 1e61664a7d..1d8edf0adb 100644 --- a/doc/classes/RayCast3D.xml +++ b/doc/classes/RayCast3D.xml @@ -4,8 +4,8 @@ Query the closest object intersecting a ray. </brief_description> <description> - A RayCast represents a line from its origin to its destination position, [code]cast_to[/code]. It is used to query the 3D space in order to find the closest object along the path of the ray. - RayCast3D can ignore some objects by adding them to the exception list via [code]add_exception[/code] or by setting proper filtering with collision layers and masks. + A RayCast represents a line from its origin to its destination position, [member target_position]. It is used to query the 3D space in order to find the closest object along the path of the ray. + RayCast3D can ignore some objects by adding them to the exception list via [method add_exception] or by setting proper filtering with collision layers and masks. RayCast3D can be configured to report collisions with [Area3D]s ([member collide_with_areas]) and/or [PhysicsBody3D]s ([member collide_with_bodies]). Only enabled raycasts will be able to query the space and report collisions. RayCast3D calculates intersection every physics frame (see [Node]), and the result is cached so it can be used later until the next frame. If multiple queries are required between physics frames (or during the same frame), use [method force_raycast_update] after adjusting the raycast. @@ -126,9 +126,6 @@ </method> </methods> <members> - <member name="cast_to" type="Vector3" setter="set_cast_to" getter="get_cast_to" default="Vector3( 0, -1, 0 )"> - The ray's destination point, relative to the RayCast's [code]position[/code]. - </member> <member name="collide_with_areas" type="bool" setter="set_collide_with_areas" getter="is_collide_with_areas_enabled" default="false"> If [code]true[/code], collision with [Area3D]s will be reported. </member> @@ -144,6 +141,9 @@ <member name="exclude_parent" type="bool" setter="set_exclude_parent_body" getter="get_exclude_parent_body" default="true"> If [code]true[/code], collisions will be ignored for this RayCast3D's immediate parent. </member> + <member name="target_position" type="Vector3" setter="set_target_position" getter="get_target_position" default="Vector3( 0, -1, 0 )"> + The ray's destination point, relative to the RayCast's [code]position[/code]. + </member> </members> <constants> </constants> diff --git a/doc/classes/Rect2.xml b/doc/classes/Rect2.xml index fe05f14fa1..2d1685871d 100644 --- a/doc/classes/Rect2.xml +++ b/doc/classes/Rect2.xml @@ -5,7 +5,8 @@ </brief_description> <description> [Rect2] consists of a position, a size, and several utility functions. It is typically used for fast overlap tests. - It uses floating point coordinates. + It uses floating-point coordinates. If you need integer coordinates, use [Rect2i] instead. + The 3D counterpart to [Rect2] is [AABB]. </description> <tutorials> <link title="Math tutorial index">https://docs.godotengine.org/en/latest/tutorials/math/index.html</link> diff --git a/doc/classes/Rect2i.xml b/doc/classes/Rect2i.xml index 2fdfe7c24b..809f385f6e 100644 --- a/doc/classes/Rect2i.xml +++ b/doc/classes/Rect2i.xml @@ -5,7 +5,7 @@ </brief_description> <description> [Rect2i] consists of a position, a size, and several utility functions. It is typically used for fast overlap tests. - It uses integer coordinates. + It uses integer coordinates. If you need floating-point coordinates, use [Rect2] instead. </description> <tutorials> <link title="Math tutorial index">https://docs.godotengine.org/en/latest/tutorials/math/index.html</link> diff --git a/doc/classes/ShortCut.xml b/doc/classes/Shortcut.xml index 9a2a761969..55bbb083c9 100644 --- a/doc/classes/ShortCut.xml +++ b/doc/classes/Shortcut.xml @@ -1,5 +1,5 @@ <?xml version="1.0" encoding="UTF-8" ?> -<class name="ShortCut" inherits="Resource" version="4.0"> +<class name="Shortcut" inherits="Resource" version="4.0"> <brief_description> A shortcut for binding input. </brief_description> diff --git a/doc/classes/SpriteFrames.xml b/doc/classes/SpriteFrames.xml index 516ae25e92..1c7d84c5a2 100644 --- a/doc/classes/SpriteFrames.xml +++ b/doc/classes/SpriteFrames.xml @@ -54,7 +54,7 @@ <argument index="0" name="anim" type="StringName"> </argument> <description> - If [code]true[/code], the given animation will loop. + Returns [code]true[/code] if the given animation is configured to loop when it finishes playing. Otherwise, returns [code]false[/code]. </description> </method> <method name="get_animation_names" qualifiers="const"> diff --git a/doc/classes/StreamTexture3D.xml b/doc/classes/StreamTexture3D.xml new file mode 100644 index 0000000000..7054a4ee99 --- /dev/null +++ b/doc/classes/StreamTexture3D.xml @@ -0,0 +1,25 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<class name="StreamTexture3D" inherits="Texture3D" version="4.0"> + <brief_description> + </brief_description> + <description> + </description> + <tutorials> + </tutorials> + <methods> + <method name="load"> + <return type="int" enum="Error"> + </return> + <argument index="0" name="path" type="String"> + </argument> + <description> + </description> + </method> + </methods> + <members> + <member name="load_path" type="String" setter="load" getter="get_load_path" default=""""> + </member> + </members> + <constants> + </constants> +</class> diff --git a/doc/classes/Tabs.xml b/doc/classes/Tabs.xml index 3fc1db9dc6..ef1f370185 100644 --- a/doc/classes/Tabs.xml +++ b/doc/classes/Tabs.xml @@ -36,6 +36,13 @@ Returns [code]true[/code] if the offset buttons (the ones that appear when there's not enough space for all tabs) are visible. </description> </method> + <method name="get_previous_tab" qualifiers="const"> + <return type="int"> + </return> + <description> + Returns the previously active tab index. + </description> + </method> <method name="get_select_with_rmb" qualifiers="const"> <return type="bool"> </return> diff --git a/doc/classes/TextEdit.xml b/doc/classes/TextEdit.xml index ccc99dc8e3..a23a4936f8 100644 --- a/doc/classes/TextEdit.xml +++ b/doc/classes/TextEdit.xml @@ -9,6 +9,14 @@ <tutorials> </tutorials> <methods> + <method name="add_gutter"> + <return type="void"> + </return> + <argument index="0" name="at" type="int" default="-1"> + </argument> + <description> + </description> + </method> <method name="can_fold" qualifiers="const"> <return type="bool"> </return> @@ -112,11 +120,34 @@ Folds the given line, if possible (see [method can_fold]). </description> </method> - <method name="get_breakpoints" qualifiers="const"> - <return type="Array"> + <method name="get_gutter_count" qualifiers="const"> + <return type="int"> + </return> + <description> + </description> + </method> + <method name="get_gutter_name" qualifiers="const"> + <return type="String"> + </return> + <argument index="0" name="gutter" type="int"> + </argument> + <description> + </description> + </method> + <method name="get_gutter_type" qualifiers="const"> + <return type="int" enum="TextEdit.GutterType"> </return> + <argument index="0" name="gutter" type="int"> + </argument> + <description> + </description> + </method> + <method name="get_gutter_width" qualifiers="const"> + <return type="int"> + </return> + <argument index="0" name="gutter" type="int"> + </argument> <description> - Returns an array containing the line number of each breakpoint. </description> </method> <method name="get_line" qualifiers="const"> @@ -135,6 +166,46 @@ Returns the amount of total lines in the text. </description> </method> + <method name="get_line_gutter_icon" qualifiers="const"> + <return type="Texture2D"> + </return> + <argument index="0" name="line" type="int"> + </argument> + <argument index="1" name="gutter" type="int"> + </argument> + <description> + </description> + </method> + <method name="get_line_gutter_item_color"> + <return type="Color"> + </return> + <argument index="0" name="line" type="int"> + </argument> + <argument index="1" name="gutter" type="int"> + </argument> + <description> + </description> + </method> + <method name="get_line_gutter_metadata" qualifiers="const"> + <return type="Variant"> + </return> + <argument index="0" name="line" type="int"> + </argument> + <argument index="1" name="gutter" type="int"> + </argument> + <description> + </description> + </method> + <method name="get_line_gutter_text" qualifiers="const"> + <return type="String"> + </return> + <argument index="0" name="line" type="int"> + </argument> + <argument index="1" name="gutter" type="int"> + </argument> + <description> + </description> + </method> <method name="get_menu" qualifiers="const"> <return type="PopupMenu"> </return> @@ -202,6 +273,40 @@ Returns whether the line at the specified index is folded or not. </description> </method> + <method name="is_gutter_clickable" qualifiers="const"> + <return type="bool"> + </return> + <argument index="0" name="gutter" type="int"> + </argument> + <description> + </description> + </method> + <method name="is_gutter_drawn" qualifiers="const"> + <return type="bool"> + </return> + <argument index="0" name="gutter" type="int"> + </argument> + <description> + </description> + </method> + <method name="is_gutter_overwritable" qualifiers="const"> + <return type="bool"> + </return> + <argument index="0" name="gutter" type="int"> + </argument> + <description> + </description> + </method> + <method name="is_line_gutter_clickable" qualifiers="const"> + <return type="bool"> + </return> + <argument index="0" name="line" type="int"> + </argument> + <argument index="1" name="gutter" type="int"> + </argument> + <description> + </description> + </method> <method name="is_line_hidden" qualifiers="const"> <return type="bool"> </return> @@ -241,11 +346,12 @@ Perform redo operation. </description> </method> - <method name="remove_breakpoints"> + <method name="remove_gutter"> <return type="void"> </return> + <argument index="0" name="gutter" type="int"> + </argument> <description> - Removes all the breakpoints. This will not fire the [signal breakpoint_toggled] signal. </description> </method> <method name="search" qualifiers="const"> @@ -295,6 +401,78 @@ If [member selecting_enabled] is [code]false[/code], no selection will occur. </description> </method> + <method name="set_gutter_clickable"> + <return type="void"> + </return> + <argument index="0" name="gutter" type="int"> + </argument> + <argument index="1" name="clickable" type="bool"> + </argument> + <description> + </description> + </method> + <method name="set_gutter_custom_draw"> + <return type="void"> + </return> + <argument index="0" name="column" type="int"> + </argument> + <argument index="1" name="object" type="Object"> + </argument> + <argument index="2" name="callback" type="StringName"> + </argument> + <description> + </description> + </method> + <method name="set_gutter_draw"> + <return type="void"> + </return> + <argument index="0" name="gutter" type="int"> + </argument> + <argument index="1" name="draw" type="bool"> + </argument> + <description> + </description> + </method> + <method name="set_gutter_name"> + <return type="void"> + </return> + <argument index="0" name="gutter" type="int"> + </argument> + <argument index="1" name="name" type="String"> + </argument> + <description> + </description> + </method> + <method name="set_gutter_overwritable"> + <return type="void"> + </return> + <argument index="0" name="gutter" type="int"> + </argument> + <argument index="1" name="overwritable" type="bool"> + </argument> + <description> + </description> + </method> + <method name="set_gutter_type"> + <return type="void"> + </return> + <argument index="0" name="gutter" type="int"> + </argument> + <argument index="1" name="type" type="int" enum="TextEdit.GutterType"> + </argument> + <description> + </description> + </method> + <method name="set_gutter_width"> + <return type="void"> + </return> + <argument index="0" name="gutter" type="int"> + </argument> + <argument index="1" name="width" type="int"> + </argument> + <description> + </description> + </method> <method name="set_line"> <return type="void"> </return> @@ -317,6 +495,66 @@ If [code]true[/code], hides the line of the specified index. </description> </method> + <method name="set_line_gutter_clickable"> + <return type="void"> + </return> + <argument index="0" name="line" type="int"> + </argument> + <argument index="1" name="gutter" type="int"> + </argument> + <argument index="2" name="clickable" type="bool"> + </argument> + <description> + </description> + </method> + <method name="set_line_gutter_icon"> + <return type="void"> + </return> + <argument index="0" name="line" type="int"> + </argument> + <argument index="1" name="gutter" type="int"> + </argument> + <argument index="2" name="icon" type="Texture2D"> + </argument> + <description> + </description> + </method> + <method name="set_line_gutter_item_color"> + <return type="void"> + </return> + <argument index="0" name="line" type="int"> + </argument> + <argument index="1" name="gutter" type="int"> + </argument> + <argument index="2" name="color" type="Color"> + </argument> + <description> + </description> + </method> + <method name="set_line_gutter_metadata"> + <return type="void"> + </return> + <argument index="0" name="line" type="int"> + </argument> + <argument index="1" name="gutter" type="int"> + </argument> + <argument index="2" name="metadata" type="Variant"> + </argument> + <description> + </description> + </method> + <method name="set_line_gutter_text"> + <return type="void"> + </return> + <argument index="0" name="line" type="int"> + </argument> + <argument index="1" name="gutter" type="int"> + </argument> + <argument index="2" name="text" type="String"> + </argument> + <description> + </description> + </method> <method name="toggle_fold_line"> <return type="void"> </return> @@ -351,9 +589,6 @@ </method> </methods> <members> - <member name="breakpoint_gutter" type="bool" setter="set_breakpoint_gutter_enabled" getter="is_breakpoint_gutter_enabled" default="false"> - If [code]true[/code], the breakpoint gutter is visible. - </member> <member name="caret_blink" type="bool" setter="cursor_set_blink_enabled" getter="cursor_get_blink_enabled" default="false"> If [code]true[/code], the caret (visual cursor) blinks. </member> @@ -378,9 +613,6 @@ If [code]true[/code], the "tab" character will have a visible representation. </member> <member name="focus_mode" type="int" setter="set_focus_mode" getter="get_focus_mode" override="true" enum="Control.FocusMode" default="2" /> - <member name="fold_gutter" type="bool" setter="set_draw_fold_gutter" getter="is_drawing_fold_gutter" default="false"> - If [code]true[/code], the fold gutter is visible. This enables folding groups of indented lines. - </member> <member name="hiding_enabled" type="bool" setter="set_hiding_enabled" getter="is_hiding_enabled" default="false"> If [code]true[/code], all lines that have been set to hidden by [method set_line_as_hidden], will not be visible. </member> @@ -416,9 +648,6 @@ <member name="shortcut_keys_enabled" type="bool" setter="set_shortcut_keys_enabled" getter="is_shortcut_keys_enabled" default="true"> If [code]true[/code], shortcut keys for context menu items are enabled, even if the context menu is disabled. </member> - <member name="show_line_numbers" type="bool" setter="set_show_line_numbers" getter="is_show_line_numbers_enabled" default="false"> - If [code]true[/code], line numbers are displayed to the left of the text. - </member> <member name="smooth_scrolling" type="bool" setter="set_smooth_scroll_enable" getter="is_smooth_scroll_enabled" default="false"> If [code]true[/code], sets the [code]step[/code] of the scrollbars to [code]0.25[/code] which results in smoother scrolling. </member> @@ -438,29 +667,31 @@ </member> </members> <signals> - <signal name="breakpoint_toggled"> - <argument index="0" name="row" type="int"> - </argument> + <signal name="cursor_changed"> <description> - Emitted when a breakpoint is placed via the breakpoint gutter. + Emitted when the cursor changes. </description> </signal> - <signal name="cursor_changed"> + <signal name="gutter_added"> <description> - Emitted when the cursor changes. </description> </signal> - <signal name="info_clicked"> - <argument index="0" name="row" type="int"> + <signal name="gutter_clicked"> + <argument index="0" name="line" type="int"> </argument> - <argument index="1" name="info" type="String"> + <argument index="1" name="gutter" type="int"> </argument> <description> - Emitted when the info icon is clicked. </description> </signal> - <signal name="line_edited_from"> - <argument index="0" name="line" type="int"> + <signal name="gutter_removed"> + <description> + </description> + </signal> + <signal name="lines_edited_from"> + <argument index="0" name="from_line" type="int"> + </argument> + <argument index="1" name="to_line" type="int"> </argument> <description> </description> @@ -501,6 +732,12 @@ <constant name="SEARCH_BACKWARDS" value="4" enum="SearchFlags"> Search from end to beginning. </constant> + <constant name="GUTTER_TYPE_STRING" value="0" enum="GutterType"> + </constant> + <constant name="GUTTER_TPYE_ICON" value="1" enum="GutterType"> + </constant> + <constant name="GUTTER_TPYE_CUSTOM" value="2" enum="GutterType"> + </constant> <constant name="MENU_CUT" value="0" enum="MenuItems"> Cuts (copies and clears) the selected text. </constant> @@ -530,14 +767,8 @@ <theme_item name="background_color" type="Color" default="Color( 0, 0, 0, 0 )"> Sets the background [Color] of this [TextEdit]. [member syntax_highlighting] has to be enabled. </theme_item> - <theme_item name="bookmark_color" type="Color" default="Color( 0.08, 0.49, 0.98, 1 )"> - Sets the [Color] of the bookmark marker. [member syntax_highlighting] has to be enabled. - </theme_item> <theme_item name="brace_mismatch_color" type="Color" default="Color( 1, 0.2, 0.2, 1 )"> </theme_item> - <theme_item name="breakpoint_color" type="Color" default="Color( 0.8, 0.8, 0.4, 0.2 )"> - Sets the [Color] of the breakpoints. [member breakpoint_gutter] has to be enabled. - </theme_item> <theme_item name="caret_background_color" type="Color" default="Color( 0, 0, 0, 1 )"> </theme_item> <theme_item name="caret_color" type="Color" default="Color( 0.88, 0.88, 0.88, 1 )"> @@ -565,14 +796,8 @@ <theme_item name="current_line_color" type="Color" default="Color( 0.25, 0.25, 0.26, 0.8 )"> Sets the [Color] of the breakpoints. [member breakpoint_gutter] has to be enabled. </theme_item> - <theme_item name="executing_line_color" type="Color" default="Color( 0.2, 0.8, 0.2, 0.4 )"> - </theme_item> <theme_item name="focus" type="StyleBox"> </theme_item> - <theme_item name="fold" type="Texture2D"> - </theme_item> - <theme_item name="folded" type="Texture2D"> - </theme_item> <theme_item name="font" type="Font"> Sets the default [Font]. </theme_item> @@ -584,9 +809,6 @@ <theme_item name="font_color_selected" type="Color" default="Color( 0, 0, 0, 1 )"> Sets the [Color] of the selected text. [member override_selected_font_color] has to be enabled. </theme_item> - <theme_item name="line_number_color" type="Color" default="Color( 0.67, 0.67, 0.67, 0.4 )"> - Sets the [Color] of the line numbers. [member show_line_numbers] has to be enabled. - </theme_item> <theme_item name="line_spacing" type="int" default="4"> Sets the spacing between the lines. </theme_item> @@ -599,8 +821,6 @@ <theme_item name="read_only" type="StyleBox"> Sets the [StyleBox] of this [TextEdit] when [member readonly] is enabled. </theme_item> - <theme_item name="safe_line_number_color" type="Color" default="Color( 0.67, 0.78, 0.67, 0.6 )"> - </theme_item> <theme_item name="selection_color" type="Color" default="Color( 0.49, 0.49, 0.49, 1 )"> Sets the highlight [Color] of text selections. </theme_item> diff --git a/doc/classes/Texture3D.xml b/doc/classes/Texture3D.xml new file mode 100644 index 0000000000..85e940716d --- /dev/null +++ b/doc/classes/Texture3D.xml @@ -0,0 +1,49 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<class name="Texture3D" inherits="Texture" version="4.0"> + <brief_description> + </brief_description> + <description> + </description> + <tutorials> + </tutorials> + <methods> + <method name="get_data" qualifiers="const"> + <return type="Image[]"> + </return> + <description> + </description> + </method> + <method name="get_depth" qualifiers="const"> + <return type="int"> + </return> + <description> + </description> + </method> + <method name="get_format" qualifiers="const"> + <return type="int" enum="Image.Format"> + </return> + <description> + </description> + </method> + <method name="get_height" qualifiers="const"> + <return type="int"> + </return> + <description> + </description> + </method> + <method name="get_width" qualifiers="const"> + <return type="int"> + </return> + <description> + </description> + </method> + <method name="has_mipmaps" qualifiers="const"> + <return type="bool"> + </return> + <description> + </description> + </method> + </methods> + <constants> + </constants> +</class> diff --git a/doc/classes/VisualInstance3D.xml b/doc/classes/VisualInstance3D.xml index 6451b3f330..01d569d9c8 100644 --- a/doc/classes/VisualInstance3D.xml +++ b/doc/classes/VisualInstance3D.xml @@ -13,7 +13,7 @@ <return type="AABB"> </return> <description> - Returns the [AABB] (also known as the bounding box) for this [VisualInstance3D]. + Returns the [AABB] (also known as the bounding box) for this [VisualInstance3D]. See also [method get_transformed_aabb]. </description> </method> <method name="get_base" qualifiers="const"> @@ -44,7 +44,7 @@ </return> <description> Returns the transformed [AABB] (also known as the bounding box) for this [VisualInstance3D]. - Transformed in this case means the [AABB] plus the position, rotation, and scale of the [Node3D]'s [Transform]. + Transformed in this case means the [AABB] plus the position, rotation, and scale of the [Node3D]'s [Transform]. See also [method get_aabb]. </description> </method> <method name="set_base"> diff --git a/doc/classes/VisualShader.xml b/doc/classes/VisualShader.xml index 12954beb43..f03550bd5e 100644 --- a/doc/classes/VisualShader.xml +++ b/doc/classes/VisualShader.xml @@ -209,10 +209,13 @@ <constant name="TYPE_LIGHT" value="2" enum="Type"> A shader for light calculations. </constant> - <constant name="TYPE_COMPUTE" value="3" enum="Type"> - A compute shader, to use the GPU for abstract computation. + <constant name="TYPE_EMIT" value="3" enum="Type"> </constant> - <constant name="TYPE_MAX" value="4" enum="Type"> + <constant name="TYPE_PROCESS" value="4" enum="Type"> + </constant> + <constant name="TYPE_END" value="5" enum="Type"> + </constant> + <constant name="TYPE_MAX" value="6" enum="Type"> Represents the size of the [enum Type] enum. </constant> <constant name="NODE_ID_INVALID" value="-1"> diff --git a/doc/classes/VisualShaderNode.xml b/doc/classes/VisualShaderNode.xml index 6327ab534f..3bb527f3d4 100644 --- a/doc/classes/VisualShaderNode.xml +++ b/doc/classes/VisualShaderNode.xml @@ -57,6 +57,12 @@ Emitted when the node requests an editor refresh. Currently called only in setter of [member VisualShaderNodeTexture.source], [VisualShaderNodeTexture], and [VisualShaderNodeCubemap] (and their derivatives). </description> </signal> + <signal name="show_port_preview"> + <argument index="0" name="port_id" type="int"> + </argument> + <description> + </description> + </signal> </signals> <constants> <constant name="PORT_TYPE_SCALAR" value="0" enum="PortType"> diff --git a/doc/classes/VisualShaderNodeTexture.xml b/doc/classes/VisualShaderNodeTexture.xml index 8e389e0b40..0c83ffffe4 100644 --- a/doc/classes/VisualShaderNodeTexture.xml +++ b/doc/classes/VisualShaderNodeTexture.xml @@ -1,7 +1,7 @@ <?xml version="1.0" encoding="UTF-8" ?> <class name="VisualShaderNodeTexture" inherits="VisualShaderNode" version="4.0"> <brief_description> - Performs a texture lookup within the visual shader graph. + Performs a 2D texture lookup within the visual shader graph. </brief_description> <description> Performs a lookup operation on the provided texture, with support for multiple texture sources to choose from. diff --git a/doc/classes/VisualShaderNodeTexture3D.xml b/doc/classes/VisualShaderNodeTexture3D.xml new file mode 100644 index 0000000000..17929e823e --- /dev/null +++ b/doc/classes/VisualShaderNodeTexture3D.xml @@ -0,0 +1,20 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<class name="VisualShaderNodeTexture3D" inherits="VisualShaderNodeSample3D" version="4.0"> + <brief_description> + Performs a 3D texture lookup within the visual shader graph. + </brief_description> + <description> + Performs a lookup operation on the provided texture, with support for multiple texture sources to choose from. + </description> + <tutorials> + </tutorials> + <methods> + </methods> + <members> + <member name="texture" type="Texture3D" setter="set_texture" getter="get_texture"> + A source texture. Used if [member VisualShaderNodeSample3D.source] is set to [constant VisualShaderNodeSample3D.SOURCE_TEXTURE]. + </member> + </members> + <constants> + </constants> +</class> diff --git a/doc/classes/VisualShaderNodeTexture3DUniform.xml b/doc/classes/VisualShaderNodeTexture3DUniform.xml new file mode 100644 index 0000000000..d9e9acf117 --- /dev/null +++ b/doc/classes/VisualShaderNodeTexture3DUniform.xml @@ -0,0 +1,15 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<class name="VisualShaderNodeTexture3DUniform" inherits="VisualShaderNodeTextureUniform" version="4.0"> + <brief_description> + Provides a 3D texture uniform within the visual shader graph. + </brief_description> + <description> + Translated to [code]uniform sampler3D[/code] in the shader language. + </description> + <tutorials> + </tutorials> + <methods> + </methods> + <constants> + </constants> +</class> diff --git a/doc/tools/makerst.py b/doc/tools/makerst.py index b594f652b3..ed147f31cd 100755 --- a/doc/tools/makerst.py +++ b/doc/tools/makerst.py @@ -1041,7 +1041,7 @@ def make_footer(): # type: () -> str return ( ".. |virtual| replace:: :abbr:`virtual (This method should typically be overridden by the user to have any effect.)`\n" ".. |const| replace:: :abbr:`const (This method has no side effects. It doesn't modify any of the instance's member variables.)`\n" - ".. |vararg| replace:: :abbr:`vararg (This method accepts any number of arguments after the ones described here.)`" + ".. |vararg| replace:: :abbr:`vararg (This method accepts any number of arguments after the ones described here.)`\n" ) # fmt: on diff --git a/doc/translations/classes.pot b/doc/translations/classes.pot index 0e89e02762..7f352bce28 100644 --- a/doc/translations/classes.pot +++ b/doc/translations/classes.pot @@ -41208,7 +41208,7 @@ msgid "" "result, it should not be depended upon for reproducible random streams " "across Godot versions.\n" "To generate a random float number (within a given range) based on a time-" -"dependant seed:\n" +"dependent seed:\n" "[codeblock]\n" "var rng = RandomNumberGenerator.new()\n" "func _ready():\n" diff --git a/drivers/unix/os_unix.cpp b/drivers/unix/os_unix.cpp index 6b98a344dc..96c338f86b 100644 --- a/drivers/unix/os_unix.cpp +++ b/drivers/unix/os_unix.cpp @@ -48,7 +48,7 @@ #include <mach/mach_time.h> #endif -#if defined(__FreeBSD__) || defined(__OpenBSD__) +#if defined(__FreeBSD__) || defined(__OpenBSD__) || defined(__NetBSD__) #include <sys/param.h> #include <sys/sysctl.h> #endif @@ -477,7 +477,7 @@ String OS_Unix::get_executable_path() const { return OS::get_executable_path(); } return b; -#elif defined(__OpenBSD__) +#elif defined(__OpenBSD__) || defined(__NetBSD__) char resolved_path[MAXPATHLEN]; realpath(OS::get_executable_path().utf8().get_data(), resolved_path); diff --git a/drivers/unix/thread_posix.cpp b/drivers/unix/thread_posix.cpp index aa1b5019ca..285088342b 100644 --- a/drivers/unix/thread_posix.cpp +++ b/drivers/unix/thread_posix.cpp @@ -29,17 +29,17 @@ /*************************************************************************/ #include "thread_posix.h" -#include "core/script_language.h" #if (defined(UNIX_ENABLED) || defined(PTHREAD_ENABLED)) && !defined(NO_THREADS) +#include "core/os/memory.h" +#include "core/safe_refcount.h" +#include "core/script_language.h" + #ifdef PTHREAD_BSD_SET_NAME #include <pthread_np.h> #endif -#include "core/os/memory.h" -#include "core/safe_refcount.h" - static void _thread_id_key_destr_callback(void *p_value) { memdelete(static_cast<Thread::ID *>(p_value)); } @@ -126,6 +126,8 @@ Error ThreadPosix::set_name_func_posix(const String &p_name) { #ifdef PTHREAD_BSD_SET_NAME pthread_set_name_np(running_thread, p_name.utf8().get_data()); int err = 0; // Open/FreeBSD ignore errors in this function +#elif defined(PTHREAD_NETBSD_SET_NAME) + int err = pthread_setname_np(running_thread, "%s", const_cast<char *>(p_name.utf8().get_data())); #else int err = pthread_setname_np(running_thread, p_name.utf8().get_data()); #endif // PTHREAD_BSD_SET_NAME diff --git a/editor/code_editor.cpp b/editor/code_editor.cpp index 37db3ba780..ede6dde239 100644 --- a/editor/code_editor.cpp +++ b/editor/code_editor.cpp @@ -40,7 +40,7 @@ #include "scene/gui/separator.h" #include "scene/resources/dynamic_font.h" -void GotoLineDialog::popup_find_line(TextEdit *p_edit) { +void GotoLineDialog::popup_find_line(CodeEdit *p_edit) { text_editor = p_edit; line->set_text(itos(text_editor->cursor_get_line())); @@ -113,7 +113,7 @@ void FindReplaceBar::_unhandled_input(const Ref<InputEvent> &p_event) { } Control *focus_owner = get_focus_owner(); - if (text_edit->has_focus() || (focus_owner && vbc_lineedit->is_a_parent_of(focus_owner))) { + if (text_editor->has_focus() || (focus_owner && vbc_lineedit->is_a_parent_of(focus_owner))) { bool accepted = true; switch (k->get_keycode()) { @@ -135,20 +135,20 @@ bool FindReplaceBar::_search(uint32_t p_flags, int p_from_line, int p_from_col) int line, col; String text = get_search_text(); - bool found = text_edit->search(text, p_flags, p_from_line, p_from_col, line, col); + bool found = text_editor->search(text, p_flags, p_from_line, p_from_col, line, col); if (found) { if (!preserve_cursor) { - text_edit->unfold_line(line); - text_edit->cursor_set_line(line, false); - text_edit->cursor_set_column(col + text.length(), false); - text_edit->center_viewport_to_cursor(); - text_edit->select(line, col, line, col + text.length()); + text_editor->unfold_line(line); + text_editor->cursor_set_line(line, false); + text_editor->cursor_set_column(col + text.length(), false); + text_editor->center_viewport_to_cursor(); + text_editor->select(line, col, line, col + text.length()); } - text_edit->set_search_text(text); - text_edit->set_search_flags(p_flags); - text_edit->set_current_search_result(line, col); + text_editor->set_search_text(text); + text_editor->set_search_flags(p_flags); + text_editor->set_current_search_result(line, col); result_line = line; result_col = col; @@ -158,9 +158,9 @@ bool FindReplaceBar::_search(uint32_t p_flags, int p_from_line, int p_from_col) results_count = 0; result_line = -1; result_col = -1; - text_edit->set_search_text(""); - text_edit->set_search_flags(p_flags); - text_edit->set_current_search_result(line, col); + text_editor->set_search_text(""); + text_editor->set_search_flags(p_flags); + text_editor->set_current_search_result(line, col); } _update_matches_label(); @@ -169,67 +169,67 @@ bool FindReplaceBar::_search(uint32_t p_flags, int p_from_line, int p_from_col) } void FindReplaceBar::_replace() { - bool selection_enabled = text_edit->is_selection_active(); + bool selection_enabled = text_editor->is_selection_active(); Point2i selection_begin, selection_end; if (selection_enabled) { - selection_begin = Point2i(text_edit->get_selection_from_line(), text_edit->get_selection_from_column()); - selection_end = Point2i(text_edit->get_selection_to_line(), text_edit->get_selection_to_column()); + selection_begin = Point2i(text_editor->get_selection_from_line(), text_editor->get_selection_from_column()); + selection_end = Point2i(text_editor->get_selection_to_line(), text_editor->get_selection_to_column()); } String replace_text = get_replace_text(); int search_text_len = get_search_text().length(); - text_edit->begin_complex_operation(); + text_editor->begin_complex_operation(); if (selection_enabled && is_selection_only()) { // To restrict search_current() to selected region - text_edit->cursor_set_line(selection_begin.width); - text_edit->cursor_set_column(selection_begin.height); + text_editor->cursor_set_line(selection_begin.width); + text_editor->cursor_set_column(selection_begin.height); } if (search_current()) { - text_edit->unfold_line(result_line); - text_edit->select(result_line, result_col, result_line, result_col + search_text_len); + text_editor->unfold_line(result_line); + text_editor->select(result_line, result_col, result_line, result_col + search_text_len); if (selection_enabled && is_selection_only()) { Point2i match_from(result_line, result_col); Point2i match_to(result_line, result_col + search_text_len); if (!(match_from < selection_begin || match_to > selection_end)) { - text_edit->insert_text_at_cursor(replace_text); + text_editor->insert_text_at_cursor(replace_text); if (match_to.x == selection_end.x) { // Adjust selection bounds if necessary selection_end.y += replace_text.length() - search_text_len; } } } else { - text_edit->insert_text_at_cursor(replace_text); + text_editor->insert_text_at_cursor(replace_text); } } - text_edit->end_complex_operation(); + text_editor->end_complex_operation(); results_count = -1; if (selection_enabled && is_selection_only()) { // Reselect in order to keep 'Replace' restricted to selection - text_edit->select(selection_begin.x, selection_begin.y, selection_end.x, selection_end.y); + text_editor->select(selection_begin.x, selection_begin.y, selection_end.x, selection_end.y); } else { - text_edit->deselect(); + text_editor->deselect(); } } void FindReplaceBar::_replace_all() { - text_edit->disconnect("text_changed", callable_mp(this, &FindReplaceBar::_editor_text_changed)); + text_editor->disconnect("text_changed", callable_mp(this, &FindReplaceBar::_editor_text_changed)); // Line as x so it gets priority in comparison, column as y. - Point2i orig_cursor(text_edit->cursor_get_line(), text_edit->cursor_get_column()); + Point2i orig_cursor(text_editor->cursor_get_line(), text_editor->cursor_get_column()); Point2i prev_match = Point2(-1, -1); - bool selection_enabled = text_edit->is_selection_active(); + bool selection_enabled = text_editor->is_selection_active(); Point2i selection_begin, selection_end; if (selection_enabled) { - selection_begin = Point2i(text_edit->get_selection_from_line(), text_edit->get_selection_from_column()); - selection_end = Point2i(text_edit->get_selection_to_line(), text_edit->get_selection_to_column()); + selection_begin = Point2i(text_editor->get_selection_from_line(), text_editor->get_selection_from_column()); + selection_end = Point2i(text_editor->get_selection_to_line(), text_editor->get_selection_to_column()); } - int vsval = text_edit->get_v_scroll(); + int vsval = text_editor->get_v_scroll(); - text_edit->cursor_set_line(0); - text_edit->cursor_set_column(0); + text_editor->cursor_set_line(0); + text_editor->cursor_set_column(0); String replace_text = get_replace_text(); int search_text_len = get_search_text().length(); @@ -238,11 +238,11 @@ void FindReplaceBar::_replace_all() { replace_all_mode = true; - text_edit->begin_complex_operation(); + text_editor->begin_complex_operation(); if (selection_enabled && is_selection_only()) { - text_edit->cursor_set_line(selection_begin.width); - text_edit->cursor_set_column(selection_begin.height); + text_editor->cursor_set_line(selection_begin.width); + text_editor->cursor_set_column(selection_begin.height); } if (search_current()) { do { @@ -256,8 +256,8 @@ void FindReplaceBar::_replace_all() { prev_match = Point2i(result_line, result_col + replace_text.length()); - text_edit->unfold_line(result_line); - text_edit->select(result_line, result_col, result_line, match_to.y); + text_editor->unfold_line(result_line); + text_editor->select(result_line, result_col, result_line, match_to.y); if (selection_enabled && is_selection_only()) { if (match_from < selection_begin || match_to > selection_end) { @@ -265,48 +265,48 @@ void FindReplaceBar::_replace_all() { } // Replace but adjust selection bounds. - text_edit->insert_text_at_cursor(replace_text); + text_editor->insert_text_at_cursor(replace_text); if (match_to.x == selection_end.x) { selection_end.y += replace_text.length() - search_text_len; } } else { // Just replace. - text_edit->insert_text_at_cursor(replace_text); + text_editor->insert_text_at_cursor(replace_text); } rc++; } while (search_next()); } - text_edit->end_complex_operation(); + text_editor->end_complex_operation(); replace_all_mode = false; // Restore editor state (selection, cursor, scroll). - text_edit->cursor_set_line(orig_cursor.x); - text_edit->cursor_set_column(orig_cursor.y); + text_editor->cursor_set_line(orig_cursor.x); + text_editor->cursor_set_column(orig_cursor.y); if (selection_enabled && is_selection_only()) { // Reselect. - text_edit->select(selection_begin.x, selection_begin.y, selection_end.x, selection_end.y); + text_editor->select(selection_begin.x, selection_begin.y, selection_end.x, selection_end.y); } else { - text_edit->deselect(); + text_editor->deselect(); } - text_edit->set_v_scroll(vsval); + text_editor->set_v_scroll(vsval); matches_label->add_theme_color_override("font_color", rc > 0 ? get_theme_color("font_color", "Label") : get_theme_color("error_color", "Editor")); matches_label->set_text(vformat(TTR("%d replaced."), rc)); - text_edit->call_deferred("connect", "text_changed", Callable(this, "_editor_text_changed")); + text_editor->call_deferred("connect", "text_changed", Callable(this, "_editor_text_changed")); results_count = -1; } void FindReplaceBar::_get_search_from(int &r_line, int &r_col) { - r_line = text_edit->cursor_get_line(); - r_col = text_edit->cursor_get_column(); + r_line = text_editor->cursor_get_line(); + r_col = text_editor->cursor_get_column(); - if (text_edit->is_selection_active() && is_selection_only()) { + if (text_editor->is_selection_active() && is_selection_only()) { return; } @@ -327,7 +327,7 @@ void FindReplaceBar::_update_results_count() { return; } - String full_text = text_edit->get_text(); + String full_text = text_editor->get_text(); int from_pos = 0; @@ -399,7 +399,7 @@ bool FindReplaceBar::search_prev() { int line, col; _get_search_from(line, col); - if (text_edit->is_selection_active()) { + if (text_editor->is_selection_active()) { col--; // Skip currently selected word. } @@ -407,9 +407,9 @@ bool FindReplaceBar::search_prev() { if (col < 0) { line -= 1; if (line < 0) { - line = text_edit->get_line_count() - 1; + line = text_editor->get_line_count() - 1; } - col = text_edit->get_line(line).length(); + col = text_editor->get_line(line).length(); } return _search(flags, line, col); @@ -440,9 +440,9 @@ bool FindReplaceBar::search_next() { if (line == result_line && col == result_col) { col += text.length(); - if (col > text_edit->get_line(line).length()) { + if (col > text_editor->get_line(line).length()) { line += 1; - if (line >= text_edit->get_line_count()) { + if (line >= text_editor->get_line_count()) { line = 0; } col = 0; @@ -454,10 +454,10 @@ bool FindReplaceBar::search_next() { void FindReplaceBar::_hide_bar() { if (replace_text->has_focus() || search_text->has_focus()) { - text_edit->grab_focus(); + text_editor->grab_focus(); } - text_edit->set_search_text(""); + text_editor->set_search_text(""); result_line = -1; result_col = -1; hide(); @@ -477,8 +477,8 @@ void FindReplaceBar::_show_search(bool p_focus_replace, bool p_show_only) { search_text->call_deferred("grab_focus"); } - if (text_edit->is_selection_active() && !selection_only->is_pressed()) { - search_text->set_text(text_edit->get_selection_text()); + if (text_editor->is_selection_active() && !selection_only->is_pressed()) { + search_text->set_text(text_editor->get_selection_text()); } if (!get_search_text().empty()) { @@ -511,9 +511,9 @@ void FindReplaceBar::popup_replace() { hbc_option_replace->show(); } - selection_only->set_pressed((text_edit->is_selection_active() && text_edit->get_selection_from_line() < text_edit->get_selection_to_line())); + selection_only->set_pressed((text_editor->is_selection_active() && text_editor->get_selection_from_line() < text_editor->get_selection_to_line())); - _show_search(is_visible() || text_edit->is_selection_active()); + _show_search(is_visible() || text_editor->is_selection_active()); } void FindReplaceBar::_search_options_changed(bool p_pressed) { @@ -544,7 +544,7 @@ void FindReplaceBar::_search_text_entered(const String &p_text) { } void FindReplaceBar::_replace_text_entered(const String &p_text) { - if (selection_only->is_pressed() && text_edit->is_selection_active()) { + if (selection_only->is_pressed() && text_editor->is_selection_active()) { _replace_all(); _hide_bar(); } else if (Input::get_singleton()->is_key_pressed(KEY_SHIFT)) { @@ -579,10 +579,10 @@ void FindReplaceBar::set_error(const String &p_label) { emit_signal("error", p_label); } -void FindReplaceBar::set_text_edit(TextEdit *p_text_edit) { +void FindReplaceBar::set_text_edit(CodeEdit *p_text_edit) { results_count = -1; - text_edit = p_text_edit; - text_edit->connect("text_changed", callable_mp(this, &FindReplaceBar::_editor_text_changed)); + text_editor = p_text_edit; + text_editor->connect("text_changed", callable_mp(this, &FindReplaceBar::_editor_text_changed)); } void FindReplaceBar::_bind_methods() { @@ -932,11 +932,9 @@ void CodeTextEditor::update_editor_settings() { text_editor->set_v_scroll_speed(EditorSettings::get_singleton()->get("text_editor/navigation/v_scroll_speed")); text_editor->set_draw_minimap(EditorSettings::get_singleton()->get("text_editor/navigation/show_minimap")); text_editor->set_minimap_width((int)EditorSettings::get_singleton()->get("text_editor/navigation/minimap_width") * EDSCALE); - text_editor->set_show_line_numbers(EditorSettings::get_singleton()->get("text_editor/appearance/show_line_numbers")); + text_editor->set_draw_line_numbers(EditorSettings::get_singleton()->get("text_editor/appearance/show_line_numbers")); text_editor->set_line_numbers_zero_padded(EditorSettings::get_singleton()->get("text_editor/appearance/line_numbers_zero_padded")); - text_editor->set_bookmark_gutter_enabled(EditorSettings::get_singleton()->get("text_editor/appearance/show_bookmark_gutter")); - text_editor->set_breakpoint_gutter_enabled(EditorSettings::get_singleton()->get("text_editor/appearance/show_breakpoint_gutter")); - text_editor->set_draw_info_gutter(EditorSettings::get_singleton()->get("text_editor/appearance/show_info_gutter")); + text_editor->set_draw_bookmarks_gutter(EditorSettings::get_singleton()->get("text_editor/appearance/show_bookmark_gutter")); text_editor->set_hiding_enabled(EditorSettings::get_singleton()->get("text_editor/appearance/code_folding")); text_editor->set_draw_fold_gutter(EditorSettings::get_singleton()->get("text_editor/appearance/code_folding")); text_editor->set_wrap_enabled(EditorSettings::get_singleton()->get("text_editor/appearance/word_wrap")); @@ -1390,11 +1388,11 @@ void CodeTextEditor::goto_line_centered(int p_line) { } void CodeTextEditor::set_executing_line(int p_line) { - text_editor->set_executing_line(p_line); + text_editor->set_line_as_executing(p_line, true); } void CodeTextEditor::clear_executing_line() { - text_editor->clear_executing_line(); + text_editor->clear_executing_lines(); } Variant CodeTextEditor::get_edit_state() { @@ -1405,8 +1403,8 @@ Variant CodeTextEditor::get_edit_state() { state["column"] = text_editor->cursor_get_column(); state["row"] = text_editor->cursor_get_line(); - state["selection"] = get_text_edit()->is_selection_active(); - if (get_text_edit()->is_selection_active()) { + state["selection"] = get_text_editor()->is_selection_active(); + if (get_text_editor()->is_selection_active()) { state["selection_from_line"] = text_editor->get_selection_from_line(); state["selection_from_column"] = text_editor->get_selection_from_column(); state["selection_to_line"] = text_editor->get_selection_to_line(); @@ -1414,8 +1412,8 @@ Variant CodeTextEditor::get_edit_state() { } state["folded_lines"] = text_editor->get_folded_lines(); - state["breakpoints"] = text_editor->get_breakpoints_array(); - state["bookmarks"] = text_editor->get_bookmarks_array(); + state["breakpoints"] = text_editor->get_breakpointed_lines(); + state["bookmarks"] = text_editor->get_bookmarked_lines(); Ref<EditorSyntaxHighlighter> syntax_highlighter = text_editor->get_syntax_highlighter(); state["syntax_highlighter"] = syntax_highlighter->_get_name(); @@ -1453,7 +1451,7 @@ void CodeTextEditor::set_edit_state(const Variant &p_state) { if (state.has("bookmarks")) { Array bookmarks = state["bookmarks"]; for (int i = 0; i < bookmarks.size(); i++) { - text_editor->set_line_as_bookmark(bookmarks[i], true); + text_editor->set_line_as_bookmarked(bookmarks[i], true); } } } @@ -1591,27 +1589,26 @@ void CodeTextEditor::set_warning_nb(int p_warning_nb) { void CodeTextEditor::toggle_bookmark() { int line = text_editor->cursor_get_line(); - text_editor->set_line_as_bookmark(line, !text_editor->is_line_set_as_bookmark(line)); + text_editor->set_line_as_bookmarked(line, !text_editor->is_line_bookmarked(line)); } void CodeTextEditor::goto_next_bookmark() { - List<int> bmarks; - text_editor->get_bookmarks(&bmarks); + Array bmarks = text_editor->get_bookmarked_lines(); if (bmarks.size() <= 0) { return; } int line = text_editor->cursor_get_line(); - if (line >= bmarks[bmarks.size() - 1]) { + if (line >= (int)bmarks[bmarks.size() - 1]) { text_editor->unfold_line(bmarks[0]); text_editor->cursor_set_line(bmarks[0]); text_editor->center_viewport_to_cursor(); } else { - for (List<int>::Element *E = bmarks.front(); E; E = E->next()) { - int bline = E->get(); - if (bline > line) { - text_editor->unfold_line(bline); - text_editor->cursor_set_line(bline); + for (int i = 0; i < bmarks.size(); i++) { + int bmark_line = bmarks[i]; + if (bmark_line > line) { + text_editor->unfold_line(bmark_line); + text_editor->cursor_set_line(bmark_line); text_editor->center_viewport_to_cursor(); return; } @@ -1620,23 +1617,22 @@ void CodeTextEditor::goto_next_bookmark() { } void CodeTextEditor::goto_prev_bookmark() { - List<int> bmarks; - text_editor->get_bookmarks(&bmarks); + Array bmarks = text_editor->get_bookmarked_lines(); if (bmarks.size() <= 0) { return; } int line = text_editor->cursor_get_line(); - if (line <= bmarks[0]) { + if (line <= (int)bmarks[0]) { text_editor->unfold_line(bmarks[bmarks.size() - 1]); text_editor->cursor_set_line(bmarks[bmarks.size() - 1]); text_editor->center_viewport_to_cursor(); } else { - for (List<int>::Element *E = bmarks.back(); E; E = E->prev()) { - int bline = E->get(); - if (bline < line) { - text_editor->unfold_line(bline); - text_editor->cursor_set_line(bline); + for (int i = bmarks.size(); i >= 0; i--) { + int bmark_line = bmarks[i]; + if (bmark_line < line) { + text_editor->unfold_line(bmark_line); + text_editor->cursor_set_line(bmark_line); text_editor->center_viewport_to_cursor(); return; } @@ -1645,12 +1641,7 @@ void CodeTextEditor::goto_prev_bookmark() { } void CodeTextEditor::remove_all_bookmarks() { - List<int> bmarks; - text_editor->get_bookmarks(&bmarks); - - for (List<int>::Element *E = bmarks.front(); E; E = E->next()) { - text_editor->set_line_as_bookmark(E->get(), false); - } + text_editor->clear_bookmarked_lines(); } void CodeTextEditor::_bind_methods() { @@ -1681,7 +1672,7 @@ CodeTextEditor::CodeTextEditor() { ED_SHORTCUT("script_editor/zoom_out", TTR("Zoom Out"), KEY_MASK_CMD | KEY_MINUS); ED_SHORTCUT("script_editor/reset_zoom", TTR("Reset Zoom"), KEY_MASK_CMD | KEY_0); - text_editor = memnew(TextEdit); + text_editor = memnew(CodeEdit); add_child(text_editor); text_editor->set_v_size_flags(SIZE_EXPAND_FILL); @@ -1693,7 +1684,7 @@ CodeTextEditor::CodeTextEditor() { find_replace_bar->set_text_edit(text_editor); - text_editor->set_show_line_numbers(true); + text_editor->set_draw_line_numbers(true); text_editor->set_brace_matching(true); text_editor->set_auto_indent(true); diff --git a/editor/code_editor.h b/editor/code_editor.h index 450c85c64b..b38170cbf5 100644 --- a/editor/code_editor.h +++ b/editor/code_editor.h @@ -34,9 +34,9 @@ #include "editor/editor_plugin.h" #include "scene/gui/check_box.h" #include "scene/gui/check_button.h" +#include "scene/gui/code_edit.h" #include "scene/gui/dialogs.h" #include "scene/gui/line_edit.h" -#include "scene/gui/text_edit.h" #include "scene/main/timer.h" class GotoLineDialog : public ConfirmationDialog { @@ -45,15 +45,15 @@ class GotoLineDialog : public ConfirmationDialog { Label *line_label; LineEdit *line; - TextEdit *text_editor; + CodeEdit *text_editor; virtual void ok_pressed() override; public: - void popup_find_line(TextEdit *p_edit); + void popup_find_line(CodeEdit *p_edit); int get_line() const; - void set_text_editor(TextEdit *p_text_editor); + void set_text_editor(CodeEdit *p_text_editor); GotoLineDialog(); }; @@ -77,7 +77,7 @@ class FindReplaceBar : public HBoxContainer { HBoxContainer *hbc_button_replace; HBoxContainer *hbc_option_replace; - TextEdit *text_edit; + CodeEdit *text_editor; int result_line; int result_col; @@ -120,7 +120,7 @@ public: bool is_selection_only() const; void set_error(const String &p_label); - void set_text_edit(TextEdit *p_text_edit); + void set_text_edit(CodeEdit *p_text_edit); void popup_search(bool p_show_only = false); void popup_replace(); @@ -137,7 +137,7 @@ typedef void (*CodeTextEditorCodeCompleteFunc)(void *p_ud, const String &p_code, class CodeTextEditor : public VBoxContainer { GDCLASS(CodeTextEditor, VBoxContainer); - TextEdit *text_editor; + CodeEdit *text_editor; FindReplaceBar *find_replace_bar; HBoxContainer *status_bar; @@ -240,7 +240,7 @@ public: void set_error(const String &p_error); void set_error_pos(int p_line, int p_column); void update_line_and_column() { _line_col_changed(); } - TextEdit *get_text_edit() { return text_editor; } + CodeEdit *get_text_editor() { return text_editor; } FindReplaceBar *get_find_replace_bar() { return find_replace_bar; } virtual void apply_code() {} void goto_error(); diff --git a/editor/connections_dialog.cpp b/editor/connections_dialog.cpp index 20d29d47f4..d1661fd7b3 100644 --- a/editor/connections_dialog.cpp +++ b/editor/connections_dialog.cpp @@ -754,8 +754,9 @@ void ConnectionsDock::_open_connection_dialog(ConnectDialog::ConnectionData cToE Node *dst = static_cast<Node *>(cToEdit.target); if (src && dst) { + const String &signalname = cToEdit.signal; connect_dialog->set_title(TTR("Edit Connection:") + cToEdit.signal); - connect_dialog->popup_centered(); + connect_dialog->popup_dialog(signalname); connect_dialog->init(cToEdit, true); } } diff --git a/editor/editor_builders.py b/editor/editor_builders.py index ea32e24f6e..86c5c87a68 100644 --- a/editor/editor_builders.py +++ b/editor/editor_builders.py @@ -54,7 +54,6 @@ def make_fonts_header(target, source, env): g.write("#define _EDITOR_FONTS_H\n") # saving uncompressed, since freetype will reference from memory pointer - xl_names = [] for i in range(len(source)): with open(source[i], "rb") as f: buf = f.read() diff --git a/editor/editor_node.cpp b/editor/editor_node.cpp index e90f30496c..3a52657862 100644 --- a/editor/editor_node.cpp +++ b/editor/editor_node.cpp @@ -157,6 +157,7 @@ #include "editor/plugins/sprite_frames_editor_plugin.h" #include "editor/plugins/style_box_editor_plugin.h" #include "editor/plugins/text_editor.h" +#include "editor/plugins/texture_3d_editor_plugin.h" #include "editor/plugins/texture_editor_plugin.h" #include "editor/plugins/texture_layered_editor_plugin.h" #include "editor/plugins/texture_region_editor_plugin.h" @@ -2818,9 +2819,9 @@ void EditorNode::_discard_changes(const String &p_str) { } void EditorNode::_update_file_menu_opened() { - Ref<ShortCut> close_scene_sc = ED_GET_SHORTCUT("editor/close_scene"); + Ref<Shortcut> close_scene_sc = ED_GET_SHORTCUT("editor/close_scene"); close_scene_sc->set_name(TTR("Close Scene")); - Ref<ShortCut> reopen_closed_scene_sc = ED_GET_SHORTCUT("editor/reopen_closed_scene"); + Ref<Shortcut> reopen_closed_scene_sc = ED_GET_SHORTCUT("editor/reopen_closed_scene"); reopen_closed_scene_sc->set_name(TTR("Reopen Closed Scene")); PopupMenu *pop = file_menu->get_popup(); pop->set_item_disabled(pop->get_item_index(FILE_OPEN_PREV), previous_scenes.empty()); @@ -4713,10 +4714,10 @@ void EditorNode::_scene_tab_input(const Ref<InputEvent> &p_input) { scene_tabs_context_menu->add_item(TTR("Play This Scene"), RUN_PLAY_SCENE); scene_tabs_context_menu->add_separator(); - Ref<ShortCut> close_tab_sc = ED_GET_SHORTCUT("editor/close_scene"); + Ref<Shortcut> close_tab_sc = ED_GET_SHORTCUT("editor/close_scene"); close_tab_sc->set_name(TTR("Close Tab")); scene_tabs_context_menu->add_shortcut(close_tab_sc, FILE_CLOSE); - Ref<ShortCut> undo_close_tab_sc = ED_GET_SHORTCUT("editor/reopen_closed_scene"); + Ref<Shortcut> undo_close_tab_sc = ED_GET_SHORTCUT("editor/reopen_closed_scene"); undo_close_tab_sc->set_name(TTR("Undo Close Tab")); scene_tabs_context_menu->add_shortcut(undo_close_tab_sc, FILE_OPEN_PREV); if (previous_scenes.empty()) { @@ -5620,10 +5621,10 @@ EditorNode::EditorNode() { import_cubemap_array->set_mode(ResourceImporterLayeredTexture::MODE_CUBEMAP_ARRAY); ResourceFormatImporter::get_singleton()->add_importer(import_cubemap_array); - /*Ref<ResourceImporterLayeredTexture> import_3d; + Ref<ResourceImporterLayeredTexture> import_3d; import_3d.instance(); import_3d->set_mode(ResourceImporterLayeredTexture::MODE_3D); - ResourceFormatImporter::get_singleton()->add_importer(import_3d);*/ + ResourceFormatImporter::get_singleton()->add_importer(import_3d); Ref<ResourceImporterImage> import_image; import_image.instance(); @@ -6197,7 +6198,9 @@ EditorNode::EditorNode() { #else p->add_shortcut(ED_SHORTCUT("editor/fullscreen_mode", TTR("Toggle Fullscreen"), KEY_MASK_SHIFT | KEY_F11), SETTINGS_TOGGLE_FULLSCREEN); #endif -#ifdef WINDOWS_ENABLED +#if defined(WINDOWS_ENABLED) && defined(WINDOWS_SUBSYSTEM_CONSOLE) + // The console can only be toggled if the application was built for the console subsystem, + // not the GUI subsystem. p->add_item(TTR("Toggle System Console"), SETTINGS_TOGGLE_CONSOLE); #endif p->add_separator(); @@ -6622,6 +6625,7 @@ EditorNode::EditorNode() { add_editor_plugin(memnew(CurveEditorPlugin(this))); add_editor_plugin(memnew(TextureEditorPlugin(this))); add_editor_plugin(memnew(TextureLayeredEditorPlugin(this))); + add_editor_plugin(memnew(Texture3DEditorPlugin(this))); add_editor_plugin(memnew(AudioStreamEditorPlugin(this))); add_editor_plugin(memnew(AudioBusesEditorPlugin(audio_bus_editor))); add_editor_plugin(memnew(Skeleton3DEditorPlugin(this))); diff --git a/editor/editor_settings.cpp b/editor/editor_settings.cpp index 0aefef7018..ac27c4a837 100644 --- a/editor/editor_settings.cpp +++ b/editor/editor_settings.cpp @@ -77,7 +77,7 @@ bool EditorSettings::_set_only(const StringName &p_name, const Variant &p_value) String name = arr[i]; Ref<InputEvent> shortcut = arr[i + 1]; - Ref<ShortCut> sc; + Ref<Shortcut> sc; sc.instance(); sc->set_shortcut(shortcut); add_shortcut(name, sc); @@ -120,8 +120,8 @@ bool EditorSettings::_get(const StringName &p_name, Variant &r_ret) const { if (p_name.operator String() == "shortcuts") { Array arr; - for (const Map<String, Ref<ShortCut>>::Element *E = shortcuts.front(); E; E = E->next()) { - Ref<ShortCut> sc = E->get(); + for (const Map<String, Ref<Shortcut>>::Element *E = shortcuts.front(); E; E = E->next()) { + Ref<Shortcut> sc = E->get(); if (optimize_save) { if (!sc->has_meta("original")) { @@ -445,7 +445,6 @@ void EditorSettings::_load_defaults(Ref<ConfigFile> p_extra_config) { _initial_set("text_editor/appearance/show_line_numbers", true); _initial_set("text_editor/appearance/line_numbers_zero_padded", false); _initial_set("text_editor/appearance/show_bookmark_gutter", true); - _initial_set("text_editor/appearance/show_breakpoint_gutter", true); _initial_set("text_editor/appearance/show_info_gutter", true); _initial_set("text_editor/appearance/code_folding", true); _initial_set("text_editor/appearance/word_wrap", false); @@ -707,8 +706,8 @@ void EditorSettings::_load_default_text_editor_theme() { _initial_set("text_editor/highlighting/member_variable_color", Color(0.9, 0.31, 0.35)); _initial_set("text_editor/highlighting/mark_color", Color(1.0, 0.4, 0.4, 0.4)); _initial_set("text_editor/highlighting/bookmark_color", Color(0.08, 0.49, 0.98)); - _initial_set("text_editor/highlighting/breakpoint_color", Color(0.8, 0.8, 0.4, 0.2)); - _initial_set("text_editor/highlighting/executing_line_color", Color(0.2, 0.8, 0.2, 0.4)); + _initial_set("text_editor/highlighting/breakpoint_color", Color(0.9, 0.29, 0.3)); + _initial_set("text_editor/highlighting/executing_line_color", Color(0.98, 0.89, 0.27)); _initial_set("text_editor/highlighting/code_folding_color", Color(0.8, 0.8, 0.8, 0.8)); _initial_set("text_editor/highlighting/search_result_color", Color(0.05, 0.25, 0.05, 1)); _initial_set("text_editor/highlighting/search_result_border_color", Color(0.41, 0.61, 0.91, 0.38)); @@ -1481,50 +1480,50 @@ String EditorSettings::get_editor_layouts_config() const { // Shortcuts -void EditorSettings::add_shortcut(const String &p_name, Ref<ShortCut> &p_shortcut) { +void EditorSettings::add_shortcut(const String &p_name, Ref<Shortcut> &p_shortcut) { shortcuts[p_name] = p_shortcut; } bool EditorSettings::is_shortcut(const String &p_name, const Ref<InputEvent> &p_event) const { - const Map<String, Ref<ShortCut>>::Element *E = shortcuts.find(p_name); + const Map<String, Ref<Shortcut>>::Element *E = shortcuts.find(p_name); ERR_FAIL_COND_V_MSG(!E, false, "Unknown Shortcut: " + p_name + "."); return E->get()->is_shortcut(p_event); } -Ref<ShortCut> EditorSettings::get_shortcut(const String &p_name) const { - const Map<String, Ref<ShortCut>>::Element *E = shortcuts.find(p_name); +Ref<Shortcut> EditorSettings::get_shortcut(const String &p_name) const { + const Map<String, Ref<Shortcut>>::Element *E = shortcuts.find(p_name); if (!E) { - return Ref<ShortCut>(); + return Ref<Shortcut>(); } return E->get(); } void EditorSettings::get_shortcut_list(List<String> *r_shortcuts) { - for (const Map<String, Ref<ShortCut>>::Element *E = shortcuts.front(); E; E = E->next()) { + for (const Map<String, Ref<Shortcut>>::Element *E = shortcuts.front(); E; E = E->next()) { r_shortcuts->push_back(E->key()); } } -Ref<ShortCut> ED_GET_SHORTCUT(const String &p_path) { +Ref<Shortcut> ED_GET_SHORTCUT(const String &p_path) { if (!EditorSettings::get_singleton()) { return nullptr; } - Ref<ShortCut> sc = EditorSettings::get_singleton()->get_shortcut(p_path); + Ref<Shortcut> sc = EditorSettings::get_singleton()->get_shortcut(p_path); ERR_FAIL_COND_V_MSG(!sc.is_valid(), sc, "Used ED_GET_SHORTCUT with invalid shortcut: " + p_path + "."); return sc; } -struct ShortCutMapping { +struct ShortcutMapping { const char *path; uint32_t keycode; }; -Ref<ShortCut> ED_SHORTCUT(const String &p_path, const String &p_name, uint32_t p_keycode) { +Ref<Shortcut> ED_SHORTCUT(const String &p_path, const String &p_name, uint32_t p_keycode) { #ifdef OSX_ENABLED // Use Cmd+Backspace as a general replacement for Delete shortcuts on macOS if (p_keycode == KEY_DELETE) { @@ -1545,7 +1544,7 @@ Ref<ShortCut> ED_SHORTCUT(const String &p_path, const String &p_name, uint32_t p } if (!EditorSettings::get_singleton()) { - Ref<ShortCut> sc; + Ref<Shortcut> sc; sc.instance(); sc->set_name(p_name); sc->set_shortcut(ie); @@ -1553,7 +1552,7 @@ Ref<ShortCut> ED_SHORTCUT(const String &p_path, const String &p_name, uint32_t p return sc; } - Ref<ShortCut> sc = EditorSettings::get_singleton()->get_shortcut(p_path); + Ref<Shortcut> sc = EditorSettings::get_singleton()->get_shortcut(p_path); if (sc.is_valid()) { sc->set_name(p_name); //keep name (the ones that come from disk have no name) sc->set_meta("original", ie); //to compare against changes diff --git a/editor/editor_settings.h b/editor/editor_settings.h index 13aebb7ea6..4896fb58db 100644 --- a/editor/editor_settings.h +++ b/editor/editor_settings.h @@ -85,7 +85,7 @@ private: int last_order; Ref<Resource> clipboard; - Map<String, Ref<ShortCut>> shortcuts; + Map<String, Ref<Shortcut>> shortcuts; String resource_path; String settings_dir; @@ -182,9 +182,9 @@ public: Vector<String> get_script_templates(const String &p_extension, const String &p_custom_path = String()); String get_editor_layouts_config() const; - void add_shortcut(const String &p_name, Ref<ShortCut> &p_shortcut); + void add_shortcut(const String &p_name, Ref<Shortcut> &p_shortcut); bool is_shortcut(const String &p_name, const Ref<InputEvent> &p_event) const; - Ref<ShortCut> get_shortcut(const String &p_name) const; + Ref<Shortcut> get_shortcut(const String &p_name) const; void get_shortcut_list(List<String> *r_shortcuts); void notify_changes(); @@ -203,7 +203,7 @@ Variant _EDITOR_DEF(const String &p_setting, const Variant &p_default, bool p_re Variant _EDITOR_GET(const String &p_setting); #define ED_IS_SHORTCUT(p_name, p_ev) (EditorSettings::get_singleton()->is_shortcut(p_name, p_ev)) -Ref<ShortCut> ED_SHORTCUT(const String &p_path, const String &p_name, uint32_t p_keycode = 0); -Ref<ShortCut> ED_GET_SHORTCUT(const String &p_path); +Ref<Shortcut> ED_SHORTCUT(const String &p_path, const String &p_name, uint32_t p_keycode = 0); +Ref<Shortcut> ED_GET_SHORTCUT(const String &p_path); #endif // EDITOR_SETTINGS_H diff --git a/editor/editor_themes.cpp b/editor/editor_themes.cpp index 8d54bc8021..79525ced51 100644 --- a/editor/editor_themes.cpp +++ b/editor/editor_themes.cpp @@ -218,8 +218,15 @@ void editor_register_and_generate_icons(Ref<Theme> p_theme, bool p_dark_theme = // Generate icons. if (!p_only_thumbs) { for (int i = 0; i < editor_icons_count; i++) { + float icon_scale = EDSCALE; + + // Always keep the DefaultProjectIcon at the default size + if (strcmp(editor_icons_names[i], "DefaultProjectIcon") == 0) { + icon_scale = 1.0f; + } + const int is_exception = exceptions.has(editor_icons_names[i]); - const Ref<ImageTexture> icon = editor_generate_icon(i, !is_exception); + const Ref<ImageTexture> icon = editor_generate_icon(i, !is_exception, icon_scale); p_theme->set_icon(editor_icons_names[i], "EditorIcons", icon); } @@ -868,12 +875,24 @@ Ref<Theme> create_editor_theme(const Ref<Theme> p_theme) { theme->set_constant("side_margin", "TabContainer", 0); theme->set_icon("tab", "TextEdit", theme->get_icon("GuiTab", "EditorIcons")); theme->set_icon("space", "TextEdit", theme->get_icon("GuiSpace", "EditorIcons")); - theme->set_icon("folded", "TextEdit", theme->get_icon("GuiTreeArrowRight", "EditorIcons")); - theme->set_icon("fold", "TextEdit", theme->get_icon("GuiTreeArrowDown", "EditorIcons")); theme->set_color("font_color", "TextEdit", font_color); theme->set_color("caret_color", "TextEdit", font_color); theme->set_color("selection_color", "TextEdit", font_color_selection); + // CodeEdit + theme->set_stylebox("normal", "CodeEdit", style_widget); + theme->set_stylebox("focus", "CodeEdit", style_widget_hover); + theme->set_stylebox("read_only", "CodeEdit", style_widget_disabled); + theme->set_constant("side_margin", "TabContainer", 0); + theme->set_icon("tab", "CodeEdit", theme->get_icon("GuiTab", "EditorIcons")); + theme->set_icon("space", "CodeEdit", theme->get_icon("GuiSpace", "EditorIcons")); + theme->set_icon("folded", "CodeEdit", theme->get_icon("GuiTreeArrowRight", "EditorIcons")); + theme->set_icon("can_fold", "CodeEdit", theme->get_icon("GuiTreeArrowDown", "EditorIcons")); + theme->set_icon("executing_line", "CodeEdit", theme->get_icon("MainPlay", "EditorIcons")); + theme->set_color("font_color", "CodeEdit", font_color); + theme->set_color("caret_color", "CodeEdit", font_color); + theme->set_color("selection_color", "CodeEdit", font_color_selection); + // H/VSplitContainer theme->set_stylebox("bg", "VSplitContainer", make_stylebox(theme->get_icon("GuiVsplitBg", "EditorIcons"), 1, 1, 1, 1)); theme->set_stylebox("bg", "HSplitContainer", make_stylebox(theme->get_icon("GuiHsplitBg", "EditorIcons"), 1, 1, 1, 1)); @@ -1179,7 +1198,7 @@ Ref<Theme> create_editor_theme(const Ref<Theme> p_theme) { const Color mark_color = Color(error_color.r, error_color.g, error_color.b, 0.3); const Color bookmark_color = Color(0.08, 0.49, 0.98); const Color breakpoint_color = error_color; - const Color executing_line_color = Color(0.2, 0.8, 0.2, 0.4); + const Color executing_line_color = Color(0.98, 0.89, 0.27); const Color code_folding_color = alpha3; const Color search_result_color = alpha1; const Color search_result_border_color = Color(0.41, 0.61, 0.91, 0.38); diff --git a/editor/icons/ShortCut.svg b/editor/icons/Shortcut.svg index 4ef16f0401..4ef16f0401 100644 --- a/editor/icons/ShortCut.svg +++ b/editor/icons/Shortcut.svg diff --git a/editor/import/resource_importer_layered_texture.cpp b/editor/import/resource_importer_layered_texture.cpp index f954931cee..bbf62596d0 100644 --- a/editor/import/resource_importer_layered_texture.cpp +++ b/editor/import/resource_importer_layered_texture.cpp @@ -70,7 +70,7 @@ String ResourceImporterLayeredTexture::get_visible_name() const { return "CubemapArray"; } break; case MODE_3D: { - return "3D"; + return "Texture3D"; } break; } @@ -156,15 +156,103 @@ void ResourceImporterLayeredTexture::get_import_options(List<ImportOption> *r_op } void ResourceImporterLayeredTexture::_save_tex(Vector<Ref<Image>> p_images, const String &p_to_path, int p_compress_mode, float p_lossy, Image::CompressMode p_vram_compression, Image::CompressSource p_csource, Image::UsedChannels used_channels, bool p_mipmaps, bool p_force_po2) { - for (int i = 0; i < p_images.size(); i++) { - if (p_force_po2) { - p_images.write[i]->resize_to_po2(); + Vector<Ref<Image>> mipmap_images; //for 3D + + if (mode == MODE_3D) { + //3D saves in its own way + + for (int i = 0; i < p_images.size(); i++) { + if (p_images.write[i]->has_mipmaps()) { + p_images.write[i]->clear_mipmaps(); + } + + if (p_force_po2) { + p_images.write[i]->resize_to_po2(); + } } if (p_mipmaps) { - p_images.write[i]->generate_mipmaps(); - } else { - p_images.write[i]->clear_mipmaps(); + Vector<Ref<Image>> parent_images = p_images; + //create 3D mipmaps, this is horrible, though not used very often + int w = p_images[0]->get_width(); + int h = p_images[0]->get_height(); + int d = p_images.size(); + + while (w > 1 || h > 1 || d > 1) { + Vector<Ref<Image>> mipmaps; + int mm_w = MAX(1, w >> 1); + int mm_h = MAX(1, h >> 1); + int mm_d = MAX(1, d >> 1); + + for (int i = 0; i < mm_d; i++) { + Ref<Image> mm; + mm.instance(); + mm->create(mm_w, mm_h, false, p_images[0]->get_format()); + Vector3 pos; + pos.z = float(i) * float(d) / float(mm_d) + 0.5; + for (int x = 0; x < mm_w; x++) { + for (int y = 0; y < mm_h; y++) { + pos.x = float(x) * float(w) / float(mm_w) + 0.5; + pos.y = float(y) * float(h) / float(mm_h) + 0.5; + + Vector3i posi = Vector3i(pos); + Vector3 fract = pos - Vector3(posi); + Vector3i posi_n = posi; + if (posi_n.x < w - 1) { + posi_n.x++; + } + if (posi_n.y < h - 1) { + posi_n.y++; + } + if (posi_n.z < d - 1) { + posi_n.z++; + } + + Color c000 = parent_images[posi.z]->get_pixel(posi.x, posi.y); + Color c100 = parent_images[posi.z]->get_pixel(posi_n.x, posi.y); + Color c010 = parent_images[posi.z]->get_pixel(posi.x, posi_n.y); + Color c110 = parent_images[posi.z]->get_pixel(posi_n.x, posi_n.y); + Color c001 = parent_images[posi_n.z]->get_pixel(posi.x, posi.y); + Color c101 = parent_images[posi_n.z]->get_pixel(posi_n.x, posi.y); + Color c011 = parent_images[posi_n.z]->get_pixel(posi.x, posi_n.y); + Color c111 = parent_images[posi_n.z]->get_pixel(posi_n.x, posi_n.y); + + Color cx00 = c000.lerp(c100, fract.x); + Color cx01 = c001.lerp(c101, fract.x); + Color cx10 = c010.lerp(c110, fract.x); + Color cx11 = c011.lerp(c111, fract.x); + + Color cy0 = cx00.lerp(cx10, fract.y); + Color cy1 = cx01.lerp(cx11, fract.y); + + Color cz = cy0.lerp(cy1, fract.z); + + mm->set_pixel(x, y, cz); + } + } + + mipmaps.push_back(mm); + } + + w = mm_w; + h = mm_h; + d = mm_d; + + mipmap_images.append_array(mipmaps); + parent_images = mipmaps; + } + } + } else { + for (int i = 0; i < p_images.size(); i++) { + if (p_force_po2) { + p_images.write[i]->resize_to_po2(); + } + + if (p_mipmaps) { + p_images.write[i]->generate_mipmaps(); + } else { + p_images.write[i]->clear_mipmaps(); + } } } @@ -175,13 +263,12 @@ void ResourceImporterLayeredTexture::_save_tex(Vector<Ref<Image>> p_images, cons f->store_8('L'); f->store_32(StreamTextureLayered::FORMAT_VERSION); - f->store_32(p_images.size()); + f->store_32(p_images.size()); //2d layers or 3d depth f->store_32(mode); - f->store_32(0); //dataformat - f->store_32(0); //mipmap limit + f->store_32(0); - //reserved f->store_32(0); + f->store_32(mipmap_images.size()); // amount of mipmaps f->store_32(0); f->store_32(0); @@ -189,6 +276,10 @@ void ResourceImporterLayeredTexture::_save_tex(Vector<Ref<Image>> p_images, cons ResourceImporterTexture::save_to_stex_format(f, p_images[i], ResourceImporterTexture::CompressMode(p_compress_mode), used_channels, p_vram_compression, p_lossy); } + for (int i = 0; i < mipmap_images.size(); i++) { + ResourceImporterTexture::save_to_stex_format(f, mipmap_images[i], ResourceImporterTexture::CompressMode(p_compress_mode), used_channels, p_vram_compression, p_lossy); + } + f->close(); } diff --git a/editor/import/resource_importer_layered_texture.h b/editor/import/resource_importer_layered_texture.h index 2d50889e9e..b54923be00 100644 --- a/editor/import/resource_importer_layered_texture.h +++ b/editor/import/resource_importer_layered_texture.h @@ -93,10 +93,6 @@ private: static const char *compression_formats[]; protected: - static void _texture_reimport_srgb(const Ref<StreamTexture2D> &p_tex); - static void _texture_reimport_3d(const Ref<StreamTexture2D> &p_tex); - static void _texture_reimport_normal(const Ref<StreamTexture2D> &p_tex); - static ResourceImporterLayeredTexture *singleton; public: diff --git a/editor/node_3d_editor_gizmos.cpp b/editor/node_3d_editor_gizmos.cpp index 2ddcf3d877..6e5fb6389d 100644 --- a/editor/node_3d_editor_gizmos.cpp +++ b/editor/node_3d_editor_gizmos.cpp @@ -1912,7 +1912,7 @@ void RayCast3DGizmoPlugin::redraw(EditorNode3DGizmo *p_gizmo) { Vector<Vector3> lines; lines.push_back(Vector3()); - lines.push_back(raycast->get_cast_to()); + lines.push_back(raycast->get_target_position()); const Ref<StandardMaterial3D> material = get_material(raycast->is_enabled() ? "shape_material" : "shape_material_disabled", p_gizmo); diff --git a/editor/plugins/canvas_item_editor_plugin.h b/editor/plugins/canvas_item_editor_plugin.h index ea58fb1e36..859e80befe 100644 --- a/editor/plugins/canvas_item_editor_plugin.h +++ b/editor/plugins/canvas_item_editor_plugin.h @@ -400,11 +400,11 @@ private: Ref<Texture2D> select_handle; Ref<Texture2D> anchor_handle; - Ref<ShortCut> drag_pivot_shortcut; - Ref<ShortCut> set_pivot_shortcut; - Ref<ShortCut> multiply_grid_step_shortcut; - Ref<ShortCut> divide_grid_step_shortcut; - Ref<ShortCut> pan_view_shortcut; + Ref<Shortcut> drag_pivot_shortcut; + Ref<Shortcut> set_pivot_shortcut; + Ref<Shortcut> multiply_grid_step_shortcut; + Ref<Shortcut> divide_grid_step_shortcut; + Ref<Shortcut> pan_view_shortcut; bool _is_node_locked(const Node *p_node); bool _is_node_movable(const Node *p_node, bool p_popup_warning = false); diff --git a/editor/plugins/mesh_instance_3d_editor_plugin.cpp b/editor/plugins/mesh_instance_3d_editor_plugin.cpp index 1b65987af0..5b241deab0 100644 --- a/editor/plugins/mesh_instance_3d_editor_plugin.cpp +++ b/editor/plugins/mesh_instance_3d_editor_plugin.cpp @@ -138,6 +138,7 @@ void MeshInstance3DEditor::_menu_option(int p_option) { CollisionShape3D *cshape = memnew(CollisionShape3D); cshape->set_shape(shape); + cshape->set_transform(node->get_transform()); Node *owner = node->get_owner(); diff --git a/editor/plugins/node_3d_editor_plugin.cpp b/editor/plugins/node_3d_editor_plugin.cpp index 952487c13c..d28bbadf39 100644 --- a/editor/plugins/node_3d_editor_plugin.cpp +++ b/editor/plugins/node_3d_editor_plugin.cpp @@ -2246,7 +2246,7 @@ Point2i Node3DEditorViewport::_get_warped_mouse_motion(const Ref<InputEventMouse } static bool is_shortcut_pressed(const String &p_path) { - Ref<ShortCut> shortcut = ED_GET_SHORTCUT(p_path); + Ref<Shortcut> shortcut = ED_GET_SHORTCUT(p_path); if (shortcut.is_null()) { return false; } diff --git a/editor/plugins/script_editor_plugin.cpp b/editor/plugins/script_editor_plugin.cpp index 20eef1cebd..be8ddf789b 100644 --- a/editor/plugins/script_editor_plugin.cpp +++ b/editor/plugins/script_editor_plugin.cpp @@ -1585,15 +1585,14 @@ void ScriptEditor::get_breakpoints(List<String> *p_breakpoints) { continue; } - List<int> bpoints; - se->get_breakpoints(&bpoints); String base = script->get_path(); if (base.begins_with("local://") || base == "") { continue; } - for (List<int>::Element *E = bpoints.front(); E; E = E->next()) { - p_breakpoints->push_back(base + ":" + itos(E->get() + 1)); + Array bpoints = se->get_breakpoints(); + for (int j = 0; j < bpoints.size(); j++) { + p_breakpoints->push_back(base + ":" + itos((int)bpoints[j] + 1)); } } } diff --git a/editor/plugins/script_editor_plugin.h b/editor/plugins/script_editor_plugin.h index 1234ebd267..c2b0b458eb 100644 --- a/editor/plugins/script_editor_plugin.h +++ b/editor/plugins/script_editor_plugin.h @@ -151,7 +151,7 @@ public: virtual void ensure_focus() = 0; virtual void tag_saved_version() = 0; virtual void reload(bool p_soft) {} - virtual void get_breakpoints(List<int> *p_breakpoints) = 0; + virtual Array get_breakpoints() = 0; virtual void add_callback(const String &p_function, PackedStringArray p_args) = 0; virtual void update_settings() = 0; virtual void set_debugger_active(bool p_active) = 0; diff --git a/editor/plugins/script_text_editor.cpp b/editor/plugins/script_text_editor.cpp index 4b89ca1216..7feb7cb3d3 100644 --- a/editor/plugins/script_text_editor.cpp +++ b/editor/plugins/script_text_editor.cpp @@ -111,7 +111,7 @@ ConnectionInfoDialog::ConnectionInfoDialog() { Vector<String> ScriptTextEditor::get_functions() { String errortxt; int line = -1, col; - TextEdit *te = code_editor->get_text_edit(); + CodeEdit *te = code_editor->get_text_editor(); String text = te->get_text(); List<String> fnc; @@ -130,9 +130,9 @@ void ScriptTextEditor::apply_code() { if (script.is_null()) { return; } - script->set_source_code(code_editor->get_text_edit()->get_text()); + script->set_source_code(code_editor->get_text_editor()->get_text()); script->update_exports(); - code_editor->get_text_edit()->get_syntax_highlighter()->update_cache(); + code_editor->get_text_editor()->get_syntax_highlighter()->update_cache(); } RES ScriptTextEditor::get_edited_resource() const { @@ -145,9 +145,9 @@ void ScriptTextEditor::set_edited_resource(const RES &p_res) { script = p_res; - code_editor->get_text_edit()->set_text(script->get_source_code()); - code_editor->get_text_edit()->clear_undo_history(); - code_editor->get_text_edit()->tag_saved_version(); + code_editor->get_text_editor()->set_text(script->get_source_code()); + code_editor->get_text_editor()->clear_undo_history(); + code_editor->get_text_editor()->tag_saved_version(); emit_signal("name_changed"); code_editor->update_line_and_column(); @@ -167,9 +167,19 @@ void ScriptTextEditor::enable_editor() { } void ScriptTextEditor::_load_theme_settings() { - TextEdit *text_edit = code_editor->get_text_edit(); + CodeEdit *text_edit = code_editor->get_text_editor(); text_edit->clear_keywords(); + Color updated_safe_line_number_color = EDITOR_GET("text_editor/highlighting/safe_line_number_color"); + if (updated_safe_line_number_color != safe_line_number_color) { + safe_line_number_color = updated_safe_line_number_color; + for (int i = 0; i < text_edit->get_line_count(); i++) { + if (text_edit->get_line_gutter_item_color(i, line_number_gutter) != default_line_number_color) { + text_edit->set_line_gutter_item_color(i, line_number_gutter, safe_line_number_color); + } + } + } + Color background_color = EDITOR_GET("text_editor/highlighting/background_color"); Color completion_background_color = EDITOR_GET("text_editor/highlighting/completion_background_color"); Color completion_selected_color = EDITOR_GET("text_editor/highlighting/completion_selected_color"); @@ -178,7 +188,6 @@ void ScriptTextEditor::_load_theme_settings() { Color completion_font_color = EDITOR_GET("text_editor/highlighting/completion_font_color"); Color text_color = EDITOR_GET("text_editor/highlighting/text_color"); Color line_number_color = EDITOR_GET("text_editor/highlighting/line_number_color"); - Color safe_line_number_color = EDITOR_GET("text_editor/highlighting/safe_line_number_color"); Color caret_color = EDITOR_GET("text_editor/highlighting/caret_color"); Color caret_background_color = EDITOR_GET("text_editor/highlighting/caret_background_color"); Color text_selected_color = EDITOR_GET("text_editor/highlighting/text_selected_color"); @@ -203,7 +212,6 @@ void ScriptTextEditor::_load_theme_settings() { text_edit->add_theme_color_override("completion_font_color", completion_font_color); text_edit->add_theme_color_override("font_color", text_color); text_edit->add_theme_color_override("line_number_color", line_number_color); - text_edit->add_theme_color_override("safe_line_number_color", safe_line_number_color); text_edit->add_theme_color_override("caret_color", caret_color); text_edit->add_theme_color_override("caret_background_color", caret_background_color); text_edit->add_theme_color_override("font_color_selected", text_selected_color); @@ -233,7 +241,7 @@ void ScriptTextEditor::_set_theme_for_script() { return; } - TextEdit *text_edit = code_editor->get_text_edit(); + CodeEdit *text_edit = code_editor->get_text_editor(); text_edit->get_syntax_highlighter()->update_cache(); /* add keywords for auto completion */ @@ -284,10 +292,10 @@ void ScriptTextEditor::_show_warnings_panel(bool p_show) { void ScriptTextEditor::_warning_clicked(Variant p_line) { if (p_line.get_type() == Variant::INT) { - code_editor->get_text_edit()->cursor_set_line(p_line.operator int64_t()); + code_editor->get_text_editor()->cursor_set_line(p_line.operator int64_t()); } else if (p_line.get_type() == Variant::DICTIONARY) { Dictionary meta = p_line.operator Dictionary(); - code_editor->get_text_edit()->insert_at("# warning-ignore:" + meta["code"].operator String(), meta["line"].operator int64_t() - 1); + code_editor->get_text_editor()->insert_at("# warning-ignore:" + meta["code"].operator String(), meta["line"].operator int64_t() - 1); _validate_script(); } } @@ -295,7 +303,7 @@ void ScriptTextEditor::_warning_clicked(Variant p_line) { void ScriptTextEditor::reload_text() { ERR_FAIL_COND(script.is_null()); - TextEdit *te = code_editor->get_text_edit(); + CodeEdit *te = code_editor->get_text_editor(); int column = te->cursor_get_column(); int row = te->cursor_get_line(); int h = te->get_h_scroll(); @@ -313,20 +321,20 @@ void ScriptTextEditor::reload_text() { } void ScriptTextEditor::add_callback(const String &p_function, PackedStringArray p_args) { - String code = code_editor->get_text_edit()->get_text(); + String code = code_editor->get_text_editor()->get_text(); int pos = script->get_language()->find_function(p_function, code); if (pos == -1) { //does not exist - code_editor->get_text_edit()->deselect(); - pos = code_editor->get_text_edit()->get_line_count() + 2; + code_editor->get_text_editor()->deselect(); + pos = code_editor->get_text_editor()->get_line_count() + 2; String func = script->get_language()->make_function("", p_function, p_args); //code=code+func; - code_editor->get_text_edit()->cursor_set_line(pos + 1); - code_editor->get_text_edit()->cursor_set_column(1000000); //none shall be that big - code_editor->get_text_edit()->insert_text_at_cursor("\n\n" + func); + code_editor->get_text_editor()->cursor_set_line(pos + 1); + code_editor->get_text_editor()->cursor_set_column(1000000); //none shall be that big + code_editor->get_text_editor()->insert_text_at_cursor("\n\n" + func); } - code_editor->get_text_edit()->cursor_set_line(pos); - code_editor->get_text_edit()->cursor_set_column(1); + code_editor->get_text_editor()->cursor_set_line(pos); + code_editor->get_text_editor()->cursor_set_column(1); } bool ScriptTextEditor::show_members_overview() { @@ -334,12 +342,13 @@ bool ScriptTextEditor::show_members_overview() { } void ScriptTextEditor::update_settings() { + code_editor->get_text_editor()->set_gutter_draw(connection_gutter, EditorSettings::get_singleton()->get("text_editor/appearance/show_info_gutter")); code_editor->update_editor_settings(); } bool ScriptTextEditor::is_unsaved() { const bool unsaved = - code_editor->get_text_edit()->get_version() != code_editor->get_text_edit()->get_saved_version() || + code_editor->get_text_editor()->get_version() != code_editor->get_text_editor()->get_saved_version() || script->get_path().empty(); // In memory. return unsaved; } @@ -385,7 +394,7 @@ void ScriptTextEditor::convert_indent_to_tabs() { } void ScriptTextEditor::tag_saved_version() { - code_editor->get_text_edit()->tag_saved_version(); + code_editor->get_text_editor()->tag_saved_version(); } void ScriptTextEditor::goto_line(int p_line, bool p_with_error) { @@ -409,7 +418,7 @@ void ScriptTextEditor::clear_executing_line() { } void ScriptTextEditor::ensure_focus() { - code_editor->get_text_edit()->grab_focus(); + code_editor->get_text_editor()->grab_focus(); } String ScriptTextEditor::get_name() { @@ -443,7 +452,7 @@ Ref<Texture2D> ScriptTextEditor::get_theme_icon() { void ScriptTextEditor::_validate_script() { String errortxt; int line = -1, col; - TextEdit *te = code_editor->get_text_edit(); + CodeEdit *te = code_editor->get_text_editor(); String text = te->get_text(); List<String> fnc; @@ -540,16 +549,16 @@ void ScriptTextEditor::_validate_script() { te->set_line_as_marked(i, line == i); if (highlight_safe) { if (safe_lines.has(i + 1)) { - te->set_line_as_safe(i, true); + te->set_line_gutter_item_color(i, line_number_gutter, safe_line_number_color); last_is_safe = true; } else if (last_is_safe && (te->is_line_comment(i) || te->get_line(i).strip_edges().empty())) { - te->set_line_as_safe(i, true); + te->set_line_gutter_item_color(i, line_number_gutter, safe_line_number_color); } else { - te->set_line_as_safe(i, false); + te->set_line_gutter_item_color(i, line_number_gutter, default_line_number_color); last_is_safe = false; } } else { - te->set_line_as_safe(i, false); + te->set_line_gutter_item_color(line, 1, default_line_number_color); } } @@ -566,7 +575,7 @@ void ScriptTextEditor::_update_bookmark_list() { bookmarks_menu->add_shortcut(ED_GET_SHORTCUT("script_text_editor/goto_next_bookmark"), BOOKMARK_GOTO_NEXT); bookmarks_menu->add_shortcut(ED_GET_SHORTCUT("script_text_editor/goto_previous_bookmark"), BOOKMARK_GOTO_PREV); - Array bookmark_list = code_editor->get_text_edit()->get_bookmarks_array(); + Array bookmark_list = code_editor->get_text_editor()->get_bookmarked_lines(); if (bookmark_list.size() == 0) { return; } @@ -576,7 +585,7 @@ void ScriptTextEditor::_update_bookmark_list() { for (int i = 0; i < bookmark_list.size(); i++) { // Strip edges to remove spaces or tabs. // Also replace any tabs by spaces, since we can't print tabs in the menu. - String line = code_editor->get_text_edit()->get_line(bookmark_list[i]).replace("\t", " ").strip_edges(); + String line = code_editor->get_text_editor()->get_line(bookmark_list[i]).replace("\t", " ").strip_edges(); // Limit the size of the line if too big. if (line.length() > 50) { @@ -593,7 +602,7 @@ void ScriptTextEditor::_bookmark_item_pressed(int p_idx) { _edit_option(bookmarks_menu->get_item_id(p_idx)); } else { code_editor->goto_line(bookmarks_menu->get_item_metadata(p_idx)); - code_editor->get_text_edit()->call_deferred("center_viewport_to_cursor"); //Need to be deferred, because goto uses call_deferred(). + code_editor->get_text_editor()->call_deferred("center_viewport_to_cursor"); //Need to be deferred, because goto uses call_deferred(). } } @@ -704,7 +713,7 @@ void ScriptTextEditor::_code_complete_script(const String &p_code, List<ScriptCo String hint; Error err = script->get_language()->complete_code(p_code, script->get_path(), base, r_options, r_force, hint); if (err == OK) { - code_editor->get_text_edit()->set_code_hint(hint); + code_editor->get_text_editor()->set_code_hint(hint); } } @@ -717,7 +726,7 @@ void ScriptTextEditor::_update_breakpoint_list() { breakpoints_menu->add_shortcut(ED_GET_SHORTCUT("script_text_editor/goto_next_breakpoint"), DEBUG_GOTO_NEXT_BREAKPOINT); breakpoints_menu->add_shortcut(ED_GET_SHORTCUT("script_text_editor/goto_previous_breakpoint"), DEBUG_GOTO_PREV_BREAKPOINT); - Array breakpoint_list = code_editor->get_text_edit()->get_breakpoints_array(); + Array breakpoint_list = code_editor->get_text_editor()->get_breakpointed_lines(); if (breakpoint_list.size() == 0) { return; } @@ -727,7 +736,7 @@ void ScriptTextEditor::_update_breakpoint_list() { for (int i = 0; i < breakpoint_list.size(); i++) { // Strip edges to remove spaces or tabs. // Also replace any tabs by spaces, since we can't print tabs in the menu. - String line = code_editor->get_text_edit()->get_line(breakpoint_list[i]).replace("\t", " ").strip_edges(); + String line = code_editor->get_text_editor()->get_line(breakpoint_list[i]).replace("\t", " ").strip_edges(); // Limit the size of the line if too big. if (line.length() > 50) { @@ -744,12 +753,12 @@ void ScriptTextEditor::_breakpoint_item_pressed(int p_idx) { _edit_option(breakpoints_menu->get_item_id(p_idx)); } else { code_editor->goto_line(breakpoints_menu->get_item_metadata(p_idx)); - code_editor->get_text_edit()->call_deferred("center_viewport_to_cursor"); //Need to be deferred, because goto uses call_deferred(). + code_editor->get_text_editor()->call_deferred("center_viewport_to_cursor"); //Need to be deferred, because goto uses call_deferred(). } } void ScriptTextEditor::_breakpoint_toggled(int p_row) { - EditorDebuggerNode::get_singleton()->set_breakpoint(script->get_path(), p_row + 1, code_editor->get_text_edit()->is_line_set_as_breakpoint(p_row)); + EditorDebuggerNode::get_singleton()->set_breakpoint(script->get_path(), p_row + 1, code_editor->get_text_editor()->is_line_breakpointed(p_row)); } void ScriptTextEditor::_lookup_symbol(const String &p_symbol, int p_row, int p_column) { @@ -771,7 +780,7 @@ void ScriptTextEditor::_lookup_symbol(const String &p_symbol, int p_row, int p_c EditorNode::get_singleton()->load_resource(p_symbol); } - } else if (script->get_language()->lookup_code(code_editor->get_text_edit()->get_text_for_lookup_completion(), p_symbol, script->get_path(), base, result) == OK) { + } else if (script->get_language()->lookup_code(code_editor->get_text_editor()->get_text_for_lookup_completion(), p_symbol, script->get_path(), base, result) == OK) { _goto_line(p_row); result.class_name = result.class_name.trim_prefix("_"); @@ -866,7 +875,7 @@ void ScriptTextEditor::_lookup_symbol(const String &p_symbol, int p_row, int p_c } void ScriptTextEditor::_validate_symbol(const String &p_symbol) { - TextEdit *text_edit = code_editor->get_text_edit(); + CodeEdit *text_edit = code_editor->get_text_editor(); Node *base = get_tree()->get_edited_scene_root(); if (base) { @@ -874,7 +883,7 @@ void ScriptTextEditor::_validate_symbol(const String &p_symbol) { } ScriptLanguage::LookupResult result; - if (ScriptServer::is_global_class(p_symbol) || p_symbol.is_resource_file() || script->get_language()->lookup_code(code_editor->get_text_edit()->get_text_for_lookup_completion(), p_symbol, script->get_path(), base, result) == OK || (ProjectSettings::get_singleton()->has_autoload(p_symbol) && ProjectSettings::get_singleton()->get_autoload(p_symbol).is_singleton)) { + if (ScriptServer::is_global_class(p_symbol) || p_symbol.is_resource_file() || script->get_language()->lookup_code(code_editor->get_text_editor()->get_text_for_lookup_completion(), p_symbol, script->get_path(), base, result) == OK || (ProjectSettings::get_singleton()->has_autoload(p_symbol) && ProjectSettings::get_singleton()->get_autoload(p_symbol).is_singleton)) { text_edit->set_highlighted_word(p_symbol); } else if (p_symbol.is_rel_path()) { String path = _get_absolute_path(p_symbol); @@ -902,8 +911,15 @@ void ScriptTextEditor::update_toggle_scripts_button() { } void ScriptTextEditor::_update_connected_methods() { - TextEdit *text_edit = code_editor->get_text_edit(); - text_edit->clear_info_icons(); + CodeEdit *text_edit = code_editor->get_text_editor(); + for (int i = 0; i < text_edit->get_line_count(); i++) { + if (text_edit->get_line_gutter_metadata(i, connection_gutter) == "") { + continue; + } + text_edit->set_line_gutter_metadata(i, connection_gutter, ""); + text_edit->set_line_gutter_icon(i, connection_gutter, nullptr); + text_edit->set_line_gutter_clickable(i, connection_gutter, false); + } missing_connections.clear(); if (!script_is_valid) { @@ -943,8 +959,10 @@ void ScriptTextEditor::_update_connected_methods() { for (int j = 0; j < functions.size(); j++) { String name = functions[j].get_slice(":", 0); if (name == connection.callable.get_method()) { - line = functions[j].get_slice(":", 1).to_int(); - text_edit->set_line_info_icon(line - 1, get_parent_control()->get_theme_icon("Slot", "EditorIcons"), connection.callable.get_method()); + line = functions[j].get_slice(":", 1).to_int() - 1; + text_edit->set_line_gutter_metadata(line, connection_gutter, connection.callable.get_method()); + text_edit->set_line_gutter_icon(line, connection_gutter, get_parent_control()->get_theme_icon("Slot", "EditorIcons")); + text_edit->set_line_gutter_clickable(line, connection_gutter, true); methods_found.insert(connection.callable.get_method()); break; } @@ -974,18 +992,41 @@ void ScriptTextEditor::_update_connected_methods() { } } -void ScriptTextEditor::_lookup_connections(int p_row, String p_method) { +void ScriptTextEditor::_update_gutter_indexes() { + for (int i = 0; i < code_editor->get_text_editor()->get_gutter_count(); i++) { + if (code_editor->get_text_editor()->get_gutter_name(i) == "connection_gutter") { + connection_gutter = i; + continue; + } + + if (code_editor->get_text_editor()->get_gutter_name(i) == "line_numbers") { + line_number_gutter = i; + continue; + } + } +} + +void ScriptTextEditor::_gutter_clicked(int p_line, int p_gutter) { + if (p_gutter != connection_gutter) { + return; + } + + String method = code_editor->get_text_editor()->get_line_gutter_metadata(p_line, p_gutter); + if (method == "") { + return; + } + Node *base = get_tree()->get_edited_scene_root(); if (!base) { return; } Vector<Node *> nodes = _find_all_node_for_script(base, base, script); - connection_info_dialog->popup_connections(p_method, nodes); + connection_info_dialog->popup_connections(method, nodes); } void ScriptTextEditor::_edit_option(int p_op) { - TextEdit *tx = code_editor->get_text_edit(); + CodeEdit *tx = code_editor->get_text_editor(); switch (p_op) { case EDIT_UNDO: { @@ -1109,7 +1150,7 @@ void ScriptTextEditor::_edit_option(int p_op) { } break; case EDIT_EVALUATE: { Expression expression; - Vector<String> lines = code_editor->get_text_edit()->get_selection_text().split("\n"); + Vector<String> lines = code_editor->get_text_editor()->get_selection_text().split("\n"); PackedStringArray results; for (int i = 0; i < lines.size(); i++) { @@ -1128,9 +1169,9 @@ void ScriptTextEditor::_edit_option(int p_op) { } } - code_editor->get_text_edit()->begin_complex_operation(); //prevents creating a two-step undo - code_editor->get_text_edit()->insert_text_at_cursor(String("\n").join(results)); - code_editor->get_text_edit()->end_complex_operation(); + code_editor->get_text_editor()->begin_complex_operation(); //prevents creating a two-step undo + code_editor->get_text_editor()->insert_text_at_cursor(String("\n").join(results)); + code_editor->get_text_editor()->end_complex_operation(); } break; case SEARCH_FIND: { code_editor->get_find_replace_bar()->popup_search(); @@ -1145,14 +1186,14 @@ void ScriptTextEditor::_edit_option(int p_op) { code_editor->get_find_replace_bar()->popup_replace(); } break; case SEARCH_IN_FILES: { - String selected_text = code_editor->get_text_edit()->get_selection_text(); + String selected_text = code_editor->get_text_editor()->get_selection_text(); // Yep, because it doesn't make sense to instance this dialog for every single script open... // So this will be delegated to the ScriptEditor. emit_signal("search_in_files_requested", selected_text); } break; case REPLACE_IN_FILES: { - String selected_text = code_editor->get_text_edit()->get_selection_text(); + String selected_text = code_editor->get_text_editor()->get_selection_text(); emit_signal("replace_in_files_requested", selected_text); } break; @@ -1177,24 +1218,22 @@ void ScriptTextEditor::_edit_option(int p_op) { } break; case DEBUG_TOGGLE_BREAKPOINT: { int line = tx->cursor_get_line(); - bool dobreak = !tx->is_line_set_as_breakpoint(line); + bool dobreak = !tx->is_line_breakpointed(line); tx->set_line_as_breakpoint(line, dobreak); EditorDebuggerNode::get_singleton()->set_breakpoint(script->get_path(), line + 1, dobreak); } break; case DEBUG_REMOVE_ALL_BREAKPOINTS: { - List<int> bpoints; - tx->get_breakpoints(&bpoints); + Array bpoints = tx->get_breakpointed_lines(); - for (List<int>::Element *E = bpoints.front(); E; E = E->next()) { - int line = E->get(); - bool dobreak = !tx->is_line_set_as_breakpoint(line); + for (int i = 0; i < bpoints.size(); i++) { + int line = bpoints[i]; + bool dobreak = !tx->is_line_breakpointed(line); tx->set_line_as_breakpoint(line, dobreak); EditorDebuggerNode::get_singleton()->set_breakpoint(script->get_path(), line + 1, dobreak); } } break; case DEBUG_GOTO_NEXT_BREAKPOINT: { - List<int> bpoints; - tx->get_breakpoints(&bpoints); + Array bpoints = tx->get_breakpointed_lines(); if (bpoints.size() <= 0) { return; } @@ -1202,13 +1241,13 @@ void ScriptTextEditor::_edit_option(int p_op) { int line = tx->cursor_get_line(); // wrap around - if (line >= bpoints[bpoints.size() - 1]) { + if (line >= (int)bpoints[bpoints.size() - 1]) { tx->unfold_line(bpoints[0]); tx->cursor_set_line(bpoints[0]); tx->center_viewport_to_cursor(); } else { - for (List<int>::Element *E = bpoints.front(); E; E = E->next()) { - int bline = E->get(); + for (int i = 0; i < bpoints.size(); i++) { + int bline = bpoints[i]; if (bline > line) { tx->unfold_line(bline); tx->cursor_set_line(bline); @@ -1220,21 +1259,20 @@ void ScriptTextEditor::_edit_option(int p_op) { } break; case DEBUG_GOTO_PREV_BREAKPOINT: { - List<int> bpoints; - tx->get_breakpoints(&bpoints); + Array bpoints = tx->get_breakpointed_lines(); if (bpoints.size() <= 0) { return; } int line = tx->cursor_get_line(); // wrap around - if (line <= bpoints[0]) { + if (line <= (int)bpoints[0]) { tx->unfold_line(bpoints[bpoints.size() - 1]); tx->cursor_set_line(bpoints[bpoints.size() - 1]); tx->center_viewport_to_cursor(); } else { - for (List<int>::Element *E = bpoints.back(); E; E = E->prev()) { - int bline = E->get(); + for (int i = bpoints.size(); i >= 0; i--) { + int bline = bpoints[i]; if (bline < line) { tx->unfold_line(bline); tx->cursor_set_line(bline); @@ -1303,7 +1341,7 @@ void ScriptTextEditor::set_syntax_highlighter(Ref<EditorSyntaxHighlighter> p_hig el = el->next(); } - TextEdit *te = code_editor->get_text_edit(); + CodeEdit *te = code_editor->get_text_editor(); p_highlighter->_set_edited_resource(script); te->set_syntax_highlighter(p_highlighter); } @@ -1312,6 +1350,16 @@ void ScriptTextEditor::_change_syntax_highlighter(int p_idx) { set_syntax_highlighter(highlighters[highlighter_menu->get_item_text(p_idx)]); } +void ScriptTextEditor::_notification(int p_what) { + switch (p_what) { + case NOTIFICATION_THEME_CHANGED: { + code_editor->get_text_editor()->set_gutter_width(connection_gutter, code_editor->get_text_editor()->get_row_height()); + } break; + default: + break; + } +} + void ScriptTextEditor::_bind_methods() { ClassDB::bind_method("_update_connected_methods", &ScriptTextEditor::_update_connected_methods); @@ -1331,7 +1379,7 @@ void ScriptTextEditor::clear_edit_menu() { } void ScriptTextEditor::reload(bool p_soft) { - TextEdit *te = code_editor->get_text_edit(); + CodeEdit *te = code_editor->get_text_editor(); Ref<Script> scr = script; if (scr.is_null()) { return; @@ -1342,12 +1390,12 @@ void ScriptTextEditor::reload(bool p_soft) { scr->get_language()->reload_tool_script(scr, soft); } -void ScriptTextEditor::get_breakpoints(List<int> *p_breakpoints) { - code_editor->get_text_edit()->get_breakpoints(p_breakpoints); +Array ScriptTextEditor::get_breakpoints() { + return code_editor->get_text_editor()->get_breakpointed_lines(); } void ScriptTextEditor::set_tooltip_request_func(String p_method, Object *p_obj) { - code_editor->get_text_edit()->set_tooltip_request_func(p_obj, p_method, this); + code_editor->get_text_editor()->set_tooltip_request_func(p_obj, p_method, this); } void ScriptTextEditor::set_debugger_active(bool p_active) { @@ -1393,7 +1441,7 @@ static Node *_find_script_node(Node *p_edited_scene, Node *p_current_node, const void ScriptTextEditor::drop_data_fw(const Point2 &p_point, const Variant &p_data, Control *p_from) { Dictionary d = p_data; - TextEdit *te = code_editor->get_text_edit(); + CodeEdit *te = code_editor->get_text_editor(); int row, col; te->_get_mouse_pos(p_point, row, col); @@ -1466,7 +1514,7 @@ void ScriptTextEditor::_text_edit_gui_input(const Ref<InputEvent> &ev) { Point2 local_pos; bool create_menu = false; - TextEdit *tx = code_editor->get_text_edit(); + CodeEdit *tx = code_editor->get_text_editor(); if (mb.is_valid() && mb->get_button_index() == BUTTON_RIGHT && mb->is_pressed()) { local_pos = mb->get_global_position() - tx->get_global_position(); create_menu = true; @@ -1519,7 +1567,7 @@ void ScriptTextEditor::_text_edit_gui_input(const Ref<InputEvent> &ev) { base = _find_node_for_script(base, base, script); } ScriptLanguage::LookupResult result; - if (script->get_language()->lookup_code(code_editor->get_text_edit()->get_text_for_lookup_completion(), word_at_pos, script->get_path(), base, result) == OK) { + if (script->get_language()->lookup_code(code_editor->get_text_editor()->get_text_for_lookup_completion(), word_at_pos, script->get_path(), base, result) == OK) { open_docs = true; } } @@ -1567,17 +1615,17 @@ void ScriptTextEditor::_color_changed(const Color &p_color) { new_args = String("(" + rtos(p_color.r) + ", " + rtos(p_color.g) + ", " + rtos(p_color.b) + ", " + rtos(p_color.a) + ")"); } - String line = code_editor->get_text_edit()->get_line(color_position.x); + String line = code_editor->get_text_editor()->get_line(color_position.x); int color_args_pos = line.find(color_args, color_position.y); String line_with_replaced_args = line; line_with_replaced_args.erase(color_args_pos, color_args.length()); line_with_replaced_args = line_with_replaced_args.insert(color_args_pos, new_args); color_args = new_args; - code_editor->get_text_edit()->begin_complex_operation(); - code_editor->get_text_edit()->set_line(color_position.x, line_with_replaced_args); - code_editor->get_text_edit()->end_complex_operation(); - code_editor->get_text_edit()->update(); + code_editor->get_text_editor()->begin_complex_operation(); + code_editor->get_text_editor()->set_line(color_position.x, line_with_replaced_args); + code_editor->get_text_editor()->end_complex_operation(); + code_editor->get_text_editor()->update(); } void ScriptTextEditor::_make_context_menu(bool p_selection, bool p_color, bool p_foldable, bool p_open_docs, bool p_goto_definition, Vector2 p_pos) { @@ -1636,12 +1684,15 @@ void ScriptTextEditor::_enable_code_editor() { code_editor->connect("show_warnings_panel", callable_mp(this, &ScriptTextEditor::_show_warnings_panel)); code_editor->connect("validate_script", callable_mp(this, &ScriptTextEditor::_validate_script)); code_editor->connect("load_theme_settings", callable_mp(this, &ScriptTextEditor::_load_theme_settings)); - code_editor->get_text_edit()->connect("breakpoint_toggled", callable_mp(this, &ScriptTextEditor::_breakpoint_toggled)); - code_editor->get_text_edit()->connect("symbol_lookup", callable_mp(this, &ScriptTextEditor::_lookup_symbol)); - code_editor->get_text_edit()->connect("symbol_validate", callable_mp(this, &ScriptTextEditor::_validate_symbol)); - code_editor->get_text_edit()->connect("info_clicked", callable_mp(this, &ScriptTextEditor::_lookup_connections)); - code_editor->get_text_edit()->connect("gui_input", callable_mp(this, &ScriptTextEditor::_text_edit_gui_input)); + code_editor->get_text_editor()->connect("breakpoint_toggled", callable_mp(this, &ScriptTextEditor::_breakpoint_toggled)); + code_editor->get_text_editor()->connect("symbol_lookup", callable_mp(this, &ScriptTextEditor::_lookup_symbol)); + code_editor->get_text_editor()->connect("symbol_validate", callable_mp(this, &ScriptTextEditor::_validate_symbol)); + code_editor->get_text_editor()->connect("gutter_added", callable_mp(this, &ScriptTextEditor::_update_gutter_indexes)); + code_editor->get_text_editor()->connect("gutter_removed", callable_mp(this, &ScriptTextEditor::_update_gutter_indexes)); + code_editor->get_text_editor()->connect("gutter_clicked", callable_mp(this, &ScriptTextEditor::_gutter_clicked)); + code_editor->get_text_editor()->connect("gui_input", callable_mp(this, &ScriptTextEditor::_text_edit_gui_input)); code_editor->show_toggle_scripts_button(); + _update_gutter_indexes(); editor_box->add_child(warnings_panel); warnings_panel->add_theme_font_override( @@ -1758,6 +1809,16 @@ ScriptTextEditor::ScriptTextEditor() { code_editor->set_code_complete_func(_code_complete_scripts, this); code_editor->set_v_size_flags(SIZE_EXPAND_FILL); + code_editor->get_text_editor()->set_draw_breakpoints_gutter(true); + code_editor->get_text_editor()->set_draw_executing_lines_gutter(true); + + connection_gutter = 1; + code_editor->get_text_editor()->add_gutter(connection_gutter); + code_editor->get_text_editor()->set_gutter_name(connection_gutter, "connection_gutter"); + code_editor->get_text_editor()->set_gutter_draw(connection_gutter, false); + code_editor->get_text_editor()->set_gutter_overwritable(connection_gutter, true); + code_editor->get_text_editor()->set_gutter_type(connection_gutter, TextEdit::GUTTER_TPYE_ICON); + warnings_panel = memnew(RichTextLabel); warnings_panel->set_custom_minimum_size(Size2(0, 100 * EDSCALE)); warnings_panel->set_h_size_flags(SIZE_EXPAND_FILL); @@ -1768,12 +1829,12 @@ ScriptTextEditor::ScriptTextEditor() { update_settings(); - code_editor->get_text_edit()->set_callhint_settings( + code_editor->get_text_editor()->set_callhint_settings( EditorSettings::get_singleton()->get("text_editor/completion/put_callhint_tooltip_below_current_line"), EditorSettings::get_singleton()->get("text_editor/completion/callhint_tooltip_offset")); - code_editor->get_text_edit()->set_select_identifiers_on_hover(true); - code_editor->get_text_edit()->set_context_menu_enabled(false); + code_editor->get_text_editor()->set_select_identifiers_on_hover(true); + code_editor->get_text_editor()->set_context_menu_enabled(false); context_menu = memnew(PopupMenu); @@ -1816,7 +1877,7 @@ ScriptTextEditor::ScriptTextEditor() { connection_info_dialog = memnew(ConnectionInfoDialog); - code_editor->get_text_edit()->set_drag_forwarding(this); + code_editor->get_text_editor()->set_drag_forwarding(this); } ScriptTextEditor::~ScriptTextEditor() { diff --git a/editor/plugins/script_text_editor.h b/editor/plugins/script_text_editor.h index e931c9fdc6..1e436fbe65 100644 --- a/editor/plugins/script_text_editor.h +++ b/editor/plugins/script_text_editor.h @@ -81,6 +81,14 @@ class ScriptTextEditor : public ScriptEditorBase { ScriptEditorQuickOpen *quick_open = nullptr; ConnectionInfoDialog *connection_info_dialog = nullptr; + int connection_gutter = -1; + void _gutter_clicked(int p_line, int p_gutter); + void _update_gutter_indexes(); + + int line_number_gutter = -1; + Color default_line_number_color = Color(1, 1, 1); + Color safe_line_number_color = Color(1, 1, 1); + PopupPanel *color_panel = nullptr; ColorPicker *color_picker = nullptr; Vector2 color_position; @@ -154,6 +162,7 @@ protected: void _show_warnings_panel(bool p_show); void _warning_clicked(Variant p_line); + void _notification(int p_what); static void _bind_methods(); Map<String, Ref<EditorSyntaxHighlighter>> highlighters; @@ -169,8 +178,6 @@ protected: void _lookup_symbol(const String &p_symbol, int p_row, int p_column); void _validate_symbol(const String &p_symbol); - void _lookup_connections(int p_row, String p_method); - void _convert_case(CodeTextEditor::CaseStyle p_case); Variant get_drag_data_fw(const Point2 &p_point, Control *p_from); @@ -211,7 +218,7 @@ public: virtual void clear_executing_line() override; virtual void reload(bool p_soft) override; - virtual void get_breakpoints(List<int> *p_breakpoints) override; + virtual Array get_breakpoints() override; virtual void add_callback(const String &p_function, PackedStringArray p_args) override; virtual void update_settings() override; diff --git a/editor/plugins/shader_editor_plugin.cpp b/editor/plugins/shader_editor_plugin.cpp index 2a7f3f0656..29db284b44 100644 --- a/editor/plugins/shader_editor_plugin.cpp +++ b/editor/plugins/shader_editor_plugin.cpp @@ -55,8 +55,8 @@ void ShaderTextEditor::set_edited_shader(const Ref<Shader> &p_shader) { _load_theme_settings(); - get_text_edit()->set_text(p_shader->get_code()); - get_text_edit()->clear_undo_history(); + get_text_editor()->set_text(p_shader->get_code()); + get_text_editor()->clear_undo_history(); _validate_script(); _line_col_changed(); @@ -65,7 +65,7 @@ void ShaderTextEditor::set_edited_shader(const Ref<Shader> &p_shader) { void ShaderTextEditor::reload_text() { ERR_FAIL_COND(shader.is_null()); - TextEdit *te = get_text_edit(); + CodeEdit *te = get_text_editor(); int column = te->cursor_get_column(); int row = te->cursor_get_line(); int h = te->get_h_scroll(); @@ -107,29 +107,29 @@ void ShaderTextEditor::_load_theme_settings() { Color search_result_color = EDITOR_GET("text_editor/highlighting/search_result_color"); Color search_result_border_color = EDITOR_GET("text_editor/highlighting/search_result_border_color"); - get_text_edit()->add_theme_color_override("background_color", background_color); - get_text_edit()->add_theme_color_override("completion_background_color", completion_background_color); - get_text_edit()->add_theme_color_override("completion_selected_color", completion_selected_color); - get_text_edit()->add_theme_color_override("completion_existing_color", completion_existing_color); - get_text_edit()->add_theme_color_override("completion_scroll_color", completion_scroll_color); - get_text_edit()->add_theme_color_override("completion_font_color", completion_font_color); - get_text_edit()->add_theme_color_override("font_color", text_color); - get_text_edit()->add_theme_color_override("line_number_color", line_number_color); - get_text_edit()->add_theme_color_override("caret_color", caret_color); - get_text_edit()->add_theme_color_override("caret_background_color", caret_background_color); - get_text_edit()->add_theme_color_override("font_color_selected", text_selected_color); - get_text_edit()->add_theme_color_override("selection_color", selection_color); - get_text_edit()->add_theme_color_override("brace_mismatch_color", brace_mismatch_color); - get_text_edit()->add_theme_color_override("current_line_color", current_line_color); - get_text_edit()->add_theme_color_override("line_length_guideline_color", line_length_guideline_color); - get_text_edit()->add_theme_color_override("word_highlighted_color", word_highlighted_color); - get_text_edit()->add_theme_color_override("mark_color", mark_color); - get_text_edit()->add_theme_color_override("bookmark_color", bookmark_color); - get_text_edit()->add_theme_color_override("breakpoint_color", breakpoint_color); - get_text_edit()->add_theme_color_override("executing_line_color", executing_line_color); - get_text_edit()->add_theme_color_override("code_folding_color", code_folding_color); - get_text_edit()->add_theme_color_override("search_result_color", search_result_color); - get_text_edit()->add_theme_color_override("search_result_border_color", search_result_border_color); + get_text_editor()->add_theme_color_override("background_color", background_color); + get_text_editor()->add_theme_color_override("completion_background_color", completion_background_color); + get_text_editor()->add_theme_color_override("completion_selected_color", completion_selected_color); + get_text_editor()->add_theme_color_override("completion_existing_color", completion_existing_color); + get_text_editor()->add_theme_color_override("completion_scroll_color", completion_scroll_color); + get_text_editor()->add_theme_color_override("completion_font_color", completion_font_color); + get_text_editor()->add_theme_color_override("font_color", text_color); + get_text_editor()->add_theme_color_override("line_number_color", line_number_color); + get_text_editor()->add_theme_color_override("caret_color", caret_color); + get_text_editor()->add_theme_color_override("caret_background_color", caret_background_color); + get_text_editor()->add_theme_color_override("font_color_selected", text_selected_color); + get_text_editor()->add_theme_color_override("selection_color", selection_color); + get_text_editor()->add_theme_color_override("brace_mismatch_color", brace_mismatch_color); + get_text_editor()->add_theme_color_override("current_line_color", current_line_color); + get_text_editor()->add_theme_color_override("line_length_guideline_color", line_length_guideline_color); + get_text_editor()->add_theme_color_override("word_highlighted_color", word_highlighted_color); + get_text_editor()->add_theme_color_override("mark_color", mark_color); + get_text_editor()->add_theme_color_override("bookmark_color", bookmark_color); + get_text_editor()->add_theme_color_override("breakpoint_color", breakpoint_color); + get_text_editor()->add_theme_color_override("executing_line_color", executing_line_color); + get_text_editor()->add_theme_color_override("code_folding_color", code_folding_color); + get_text_editor()->add_theme_color_override("search_result_color", search_result_color); + get_text_editor()->add_theme_color_override("search_result_border_color", search_result_border_color); syntax_highlighter->set_number_color(EDITOR_GET("text_editor/highlighting/number_color")); syntax_highlighter->set_symbol_color(EDITOR_GET("text_editor/highlighting/symbol_color")); @@ -176,7 +176,7 @@ void ShaderTextEditor::_load_theme_settings() { } void ShaderTextEditor::_check_shader_mode() { - String type = ShaderLanguage::get_shader_type(get_text_edit()->get_text()); + String type = ShaderLanguage::get_shader_type(get_text_editor()->get_text()); Shader::Mode mode; @@ -189,7 +189,7 @@ void ShaderTextEditor::_check_shader_mode() { } if (shader->get_mode() != mode) { - shader->set_code(get_text_edit()->get_text()); + shader->set_code(get_text_editor()->get_text()); _load_theme_settings(); } } @@ -207,13 +207,13 @@ void ShaderTextEditor::_code_complete_script(const String &p_code, List<ScriptCo sl.complete(p_code, ShaderTypes::get_singleton()->get_functions(RenderingServer::ShaderMode(shader->get_mode())), ShaderTypes::get_singleton()->get_modes(RenderingServer::ShaderMode(shader->get_mode())), ShaderTypes::get_singleton()->get_types(), _get_global_variable_type, r_options, calltip); - get_text_edit()->set_code_hint(calltip); + get_text_editor()->set_code_hint(calltip); } void ShaderTextEditor::_validate_script() { _check_shader_mode(); - String code = get_text_edit()->get_text(); + String code = get_text_editor()->get_text(); //List<StringName> params; //shader->get_param_list(¶ms); @@ -225,14 +225,14 @@ void ShaderTextEditor::_validate_script() { String error_text = "error(" + itos(sl.get_error_line()) + "): " + sl.get_error_text(); set_error(error_text); set_error_pos(sl.get_error_line() - 1, 0); - for (int i = 0; i < get_text_edit()->get_line_count(); i++) { - get_text_edit()->set_line_as_marked(i, false); + for (int i = 0; i < get_text_editor()->get_line_count(); i++) { + get_text_editor()->set_line_as_marked(i, false); } - get_text_edit()->set_line_as_marked(sl.get_error_line() - 1, true); + get_text_editor()->set_line_as_marked(sl.get_error_line() - 1, true); } else { - for (int i = 0; i < get_text_edit()->get_line_count(); i++) { - get_text_edit()->set_line_as_marked(i, false); + for (int i = 0; i < get_text_editor()->get_line_count(); i++) { + get_text_editor()->set_line_as_marked(i, false); } set_error(""); } @@ -245,7 +245,7 @@ void ShaderTextEditor::_bind_methods() { ShaderTextEditor::ShaderTextEditor() { syntax_highlighter.instance(); - get_text_edit()->set_syntax_highlighter(syntax_highlighter); + get_text_editor()->set_syntax_highlighter(syntax_highlighter); } /*** SCRIPT EDITOR ******/ @@ -253,22 +253,22 @@ ShaderTextEditor::ShaderTextEditor() { void ShaderEditor::_menu_option(int p_option) { switch (p_option) { case EDIT_UNDO: { - shader_editor->get_text_edit()->undo(); + shader_editor->get_text_editor()->undo(); } break; case EDIT_REDO: { - shader_editor->get_text_edit()->redo(); + shader_editor->get_text_editor()->redo(); } break; case EDIT_CUT: { - shader_editor->get_text_edit()->cut(); + shader_editor->get_text_editor()->cut(); } break; case EDIT_COPY: { - shader_editor->get_text_edit()->copy(); + shader_editor->get_text_editor()->copy(); } break; case EDIT_PASTE: { - shader_editor->get_text_edit()->paste(); + shader_editor->get_text_editor()->paste(); } break; case EDIT_SELECT_ALL: { - shader_editor->get_text_edit()->select_all(); + shader_editor->get_text_editor()->select_all(); } break; case EDIT_MOVE_LINE_UP: { shader_editor->move_lines_up(); @@ -281,7 +281,7 @@ void ShaderEditor::_menu_option(int p_option) { return; } - TextEdit *tx = shader_editor->get_text_edit(); + CodeEdit *tx = shader_editor->get_text_editor(); tx->indent_left(); } break; @@ -290,7 +290,7 @@ void ShaderEditor::_menu_option(int p_option) { return; } - TextEdit *tx = shader_editor->get_text_edit(); + CodeEdit *tx = shader_editor->get_text_editor(); tx->indent_right(); } break; @@ -309,7 +309,7 @@ void ShaderEditor::_menu_option(int p_option) { } break; case EDIT_COMPLETE: { - shader_editor->get_text_edit()->query_code_comple(); + shader_editor->get_text_editor()->query_code_comple(); } break; case SEARCH_FIND: { shader_editor->get_find_replace_bar()->popup_search(); @@ -324,7 +324,7 @@ void ShaderEditor::_menu_option(int p_option) { shader_editor->get_find_replace_bar()->popup_replace(); } break; case SEARCH_GOTO_LINE: { - goto_line_dialog->popup_find_line(shader_editor->get_text_edit()); + goto_line_dialog->popup_find_line(shader_editor->get_text_editor()); } break; case BOOKMARK_TOGGLE: { shader_editor->toggle_bookmark(); @@ -343,7 +343,7 @@ void ShaderEditor::_menu_option(int p_option) { } break; } if (p_option != SEARCH_FIND && p_option != SEARCH_REPLACE && p_option != SEARCH_GOTO_LINE) { - shader_editor->get_text_edit()->call_deferred("grab_focus"); + shader_editor->get_text_editor()->call_deferred("grab_focus"); } } @@ -358,28 +358,11 @@ void ShaderEditor::_params_changed() { } void ShaderEditor::_editor_settings_changed() { - shader_editor->get_text_edit()->set_auto_brace_completion(EditorSettings::get_singleton()->get("text_editor/completion/auto_brace_complete")); - shader_editor->get_text_edit()->set_scroll_pass_end_of_file(EditorSettings::get_singleton()->get("text_editor/cursor/scroll_past_end_of_file")); - shader_editor->get_text_edit()->set_indent_size(EditorSettings::get_singleton()->get("text_editor/indent/size")); - shader_editor->get_text_edit()->set_indent_using_spaces(EditorSettings::get_singleton()->get("text_editor/indent/type")); - shader_editor->get_text_edit()->set_auto_indent(EditorSettings::get_singleton()->get("text_editor/indent/auto_indent")); - shader_editor->get_text_edit()->set_draw_tabs(EditorSettings::get_singleton()->get("text_editor/indent/draw_tabs")); - shader_editor->get_text_edit()->set_draw_spaces(EditorSettings::get_singleton()->get("text_editor/indent/draw_spaces")); - shader_editor->get_text_edit()->set_show_line_numbers(EditorSettings::get_singleton()->get("text_editor/appearance/show_line_numbers")); - shader_editor->get_text_edit()->set_highlight_all_occurrences(EditorSettings::get_singleton()->get("text_editor/highlighting/highlight_all_occurrences")); - shader_editor->get_text_edit()->set_highlight_current_line(EditorSettings::get_singleton()->get("text_editor/highlighting/highlight_current_line")); - shader_editor->get_text_edit()->cursor_set_blink_enabled(EditorSettings::get_singleton()->get("text_editor/cursor/caret_blink")); - shader_editor->get_text_edit()->cursor_set_blink_speed(EditorSettings::get_singleton()->get("text_editor/cursor/caret_blink_speed")); - shader_editor->get_text_edit()->add_theme_constant_override("line_spacing", EditorSettings::get_singleton()->get("text_editor/theme/line_spacing")); - shader_editor->get_text_edit()->cursor_set_block_mode(EditorSettings::get_singleton()->get("text_editor/cursor/block_caret")); - shader_editor->get_text_edit()->set_smooth_scroll_enabled(EditorSettings::get_singleton()->get("text_editor/navigation/smooth_scrolling")); - shader_editor->get_text_edit()->set_v_scroll_speed(EditorSettings::get_singleton()->get("text_editor/navigation/v_scroll_speed")); - shader_editor->get_text_edit()->set_draw_minimap(EditorSettings::get_singleton()->get("text_editor/navigation/show_minimap")); - shader_editor->get_text_edit()->set_minimap_width((int)EditorSettings::get_singleton()->get("text_editor/navigation/minimap_width") * EDSCALE); - shader_editor->get_text_edit()->set_show_line_length_guidelines(EditorSettings::get_singleton()->get("text_editor/appearance/show_line_length_guidelines")); - shader_editor->get_text_edit()->set_line_length_guideline_soft_column(EditorSettings::get_singleton()->get("text_editor/appearance/line_length_guideline_soft_column")); - shader_editor->get_text_edit()->set_line_length_guideline_hard_column(EditorSettings::get_singleton()->get("text_editor/appearance/line_length_guideline_hard_column")); - shader_editor->get_text_edit()->set_breakpoint_gutter_enabled(false); + shader_editor->update_editor_settings(); + + shader_editor->get_text_editor()->add_theme_constant_override("line_spacing", EditorSettings::get_singleton()->get("text_editor/theme/line_spacing")); + shader_editor->get_text_editor()->set_draw_breakpoints_gutter(false); + shader_editor->get_text_editor()->set_draw_executing_lines_gutter(false); } void ShaderEditor::_bind_methods() { @@ -466,7 +449,7 @@ void ShaderEditor::save_external_data(const String &p_str) { void ShaderEditor::apply_shaders() { if (shader.is_valid()) { String shader_code = shader->get_code(); - String editor_code = shader_editor->get_text_edit()->get_text(); + String editor_code = shader_editor->get_text_editor()->get_text(); if (shader_code != editor_code) { shader->set_code(editor_code); shader->set_edited(true); @@ -480,7 +463,7 @@ void ShaderEditor::_text_edit_gui_input(const Ref<InputEvent> &ev) { if (mb.is_valid()) { if (mb->get_button_index() == BUTTON_RIGHT && mb->is_pressed()) { int col, row; - TextEdit *tx = shader_editor->get_text_edit(); + CodeEdit *tx = shader_editor->get_text_editor(); tx->_get_mouse_pos(mb->get_global_position() - tx->get_global_position(), row, col); tx->set_right_click_moves_caret(EditorSettings::get_singleton()->get("text_editor/cursor/right_click_moves_caret")); @@ -507,7 +490,7 @@ void ShaderEditor::_text_edit_gui_input(const Ref<InputEvent> &ev) { Ref<InputEventKey> k = ev; if (k.is_valid() && k->is_pressed() && k->get_keycode() == KEY_MENU) { - TextEdit *tx = shader_editor->get_text_edit(); + CodeEdit *tx = shader_editor->get_text_editor(); _make_context_menu(tx->is_selection_active(), (get_global_transform().inverse() * tx->get_global_transform()).xform(tx->_get_cursor_pixel_pos())); context_menu->grab_focus(); } @@ -521,7 +504,7 @@ void ShaderEditor::_update_bookmark_list() { bookmarks_menu->add_shortcut(ED_GET_SHORTCUT("script_text_editor/goto_next_bookmark"), BOOKMARK_GOTO_NEXT); bookmarks_menu->add_shortcut(ED_GET_SHORTCUT("script_text_editor/goto_previous_bookmark"), BOOKMARK_GOTO_PREV); - Array bookmark_list = shader_editor->get_text_edit()->get_bookmarks_array(); + Array bookmark_list = shader_editor->get_text_editor()->get_bookmarked_lines(); if (bookmark_list.size() == 0) { return; } @@ -529,7 +512,7 @@ void ShaderEditor::_update_bookmark_list() { bookmarks_menu->add_separator(); for (int i = 0; i < bookmark_list.size(); i++) { - String line = shader_editor->get_text_edit()->get_line(bookmark_list[i]).strip_edges(); + String line = shader_editor->get_text_editor()->get_line(bookmark_list[i]).strip_edges(); // Limit the size of the line if too big. if (line.length() > 50) { line = line.substr(0, 50); @@ -581,13 +564,13 @@ ShaderEditor::ShaderEditor(EditorNode *p_node) { shader_editor->connect("script_changed", callable_mp(this, &ShaderEditor::apply_shaders)); EditorSettings::get_singleton()->connect("settings_changed", callable_mp(this, &ShaderEditor::_editor_settings_changed)); - shader_editor->get_text_edit()->set_callhint_settings( + shader_editor->get_text_editor()->set_callhint_settings( EditorSettings::get_singleton()->get("text_editor/completion/put_callhint_tooltip_below_current_line"), EditorSettings::get_singleton()->get("text_editor/completion/callhint_tooltip_offset")); - shader_editor->get_text_edit()->set_select_identifiers_on_hover(true); - shader_editor->get_text_edit()->set_context_menu_enabled(false); - shader_editor->get_text_edit()->connect("gui_input", callable_mp(this, &ShaderEditor::_text_edit_gui_input)); + shader_editor->get_text_editor()->set_select_identifiers_on_hover(true); + shader_editor->get_text_editor()->set_context_menu_enabled(false); + shader_editor->get_text_editor()->connect("gui_input", callable_mp(this, &ShaderEditor::_text_edit_gui_input)); shader_editor->update_editor_settings(); diff --git a/editor/plugins/text_editor.cpp b/editor/plugins/text_editor.cpp index 82e231e396..8935b698b6 100644 --- a/editor/plugins/text_editor.cpp +++ b/editor/plugins/text_editor.cpp @@ -50,7 +50,7 @@ void TextEditor::set_syntax_highlighter(Ref<EditorSyntaxHighlighter> p_highlight el = el->next(); } - TextEdit *te = code_editor->get_text_edit(); + CodeEdit *te = code_editor->get_text_editor(); te->set_syntax_highlighter(p_highlighter); } @@ -59,7 +59,7 @@ void TextEditor::_change_syntax_highlighter(int p_idx) { } void TextEditor::_load_theme_settings() { - TextEdit *text_edit = code_editor->get_text_edit(); + CodeEdit *text_edit = code_editor->get_text_editor(); text_edit->get_syntax_highlighter()->update_cache(); Color background_color = EDITOR_GET("text_editor/highlighting/background_color"); @@ -147,9 +147,9 @@ void TextEditor::set_edited_resource(const RES &p_res) { text_file = p_res; - code_editor->get_text_edit()->set_text(text_file->get_text()); - code_editor->get_text_edit()->clear_undo_history(); - code_editor->get_text_edit()->tag_saved_version(); + code_editor->get_text_editor()->set_text(text_file->get_text()); + code_editor->get_text_editor()->clear_undo_history(); + code_editor->get_text_editor()->tag_saved_version(); emit_signal("name_changed"); code_editor->update_line_and_column(); @@ -171,13 +171,14 @@ void TextEditor::add_callback(const String &p_function, PackedStringArray p_args void TextEditor::set_debugger_active(bool p_active) { } -void TextEditor::get_breakpoints(List<int> *p_breakpoints) { +Array TextEditor::get_breakpoints() { + return Array(); } void TextEditor::reload_text() { ERR_FAIL_COND(text_file.is_null()); - TextEdit *te = code_editor->get_text_edit(); + CodeEdit *te = code_editor->get_text_editor(); int column = te->cursor_get_column(); int row = te->cursor_get_line(); int h = te->get_h_scroll(); @@ -207,7 +208,7 @@ void TextEditor::_update_bookmark_list() { bookmarks_menu->add_shortcut(ED_GET_SHORTCUT("script_text_editor/goto_next_bookmark"), BOOKMARK_GOTO_NEXT); bookmarks_menu->add_shortcut(ED_GET_SHORTCUT("script_text_editor/goto_previous_bookmark"), BOOKMARK_GOTO_PREV); - Array bookmark_list = code_editor->get_text_edit()->get_bookmarks_array(); + Array bookmark_list = code_editor->get_text_editor()->get_bookmarked_lines(); if (bookmark_list.size() == 0) { return; } @@ -215,7 +216,7 @@ void TextEditor::_update_bookmark_list() { bookmarks_menu->add_separator(); for (int i = 0; i < bookmark_list.size(); i++) { - String line = code_editor->get_text_edit()->get_line(bookmark_list[i]).strip_edges(); + String line = code_editor->get_text_editor()->get_line(bookmark_list[i]).strip_edges(); // Limit the size of the line if too big. if (line.length() > 50) { line = line.substr(0, 50); @@ -235,12 +236,12 @@ void TextEditor::_bookmark_item_pressed(int p_idx) { } void TextEditor::apply_code() { - text_file->set_text(code_editor->get_text_edit()->get_text()); + text_file->set_text(code_editor->get_text_editor()->get_text()); } bool TextEditor::is_unsaved() { const bool unsaved = - code_editor->get_text_edit()->get_version() != code_editor->get_text_edit()->get_saved_version() || + code_editor->get_text_editor()->get_version() != code_editor->get_text_editor()->get_saved_version() || text_file->get_path().empty(); // In memory. return unsaved; } @@ -280,7 +281,7 @@ void TextEditor::convert_indent_to_tabs() { } void TextEditor::tag_saved_version() { - code_editor->get_text_edit()->tag_saved_version(); + code_editor->get_text_editor()->tag_saved_version(); } void TextEditor::goto_line(int p_line, bool p_with_error) { @@ -300,7 +301,7 @@ void TextEditor::clear_executing_line() { } void TextEditor::ensure_focus() { - code_editor->get_text_edit()->grab_focus(); + code_editor->get_text_editor()->grab_focus(); } Vector<String> TextEditor::get_functions() { @@ -316,7 +317,7 @@ void TextEditor::update_settings() { } void TextEditor::set_tooltip_request_func(String p_method, Object *p_obj) { - code_editor->get_text_edit()->set_tooltip_request_func(p_obj, p_method, this); + code_editor->get_text_editor()->set_tooltip_request_func(p_obj, p_method, this); } Control *TextEditor::get_edit_menu() { @@ -328,7 +329,7 @@ void TextEditor::clear_edit_menu() { } void TextEditor::_edit_option(int p_op) { - TextEdit *tx = code_editor->get_text_edit(); + CodeEdit *tx = code_editor->get_text_editor(); switch (p_op) { case EDIT_UNDO: { @@ -416,14 +417,14 @@ void TextEditor::_edit_option(int p_op) { code_editor->get_find_replace_bar()->popup_replace(); } break; case SEARCH_IN_FILES: { - String selected_text = code_editor->get_text_edit()->get_selection_text(); + String selected_text = code_editor->get_text_editor()->get_selection_text(); // Yep, because it doesn't make sense to instance this dialog for every single script open... // So this will be delegated to the ScriptEditor. emit_signal("search_in_files_requested", selected_text); } break; case REPLACE_IN_FILES: { - String selected_text = code_editor->get_text_edit()->get_selection_text(); + String selected_text = code_editor->get_text_editor()->get_selection_text(); emit_signal("replace_in_files_requested", selected_text); } break; @@ -470,7 +471,7 @@ void TextEditor::_text_edit_gui_input(const Ref<InputEvent> &ev) { if (mb.is_valid()) { if (mb->get_button_index() == BUTTON_RIGHT) { int col, row; - TextEdit *tx = code_editor->get_text_edit(); + CodeEdit *tx = code_editor->get_text_editor(); tx->_get_mouse_pos(mb->get_global_position() - tx->get_global_position(), row, col); tx->set_right_click_moves_caret(EditorSettings::get_singleton()->get("text_editor/cursor/right_click_moves_caret")); @@ -503,7 +504,7 @@ void TextEditor::_text_edit_gui_input(const Ref<InputEvent> &ev) { Ref<InputEventKey> k = ev; if (k.is_valid() && k->is_pressed() && k->get_keycode() == KEY_MENU) { - TextEdit *tx = code_editor->get_text_edit(); + CodeEdit *tx = code_editor->get_text_editor(); int line = tx->cursor_get_line(); _make_context_menu(tx->is_selection_active(), tx->can_fold(line), tx->is_folded(line), (get_global_transform().inverse() * tx->get_global_transform()).xform(tx->_get_cursor_pixel_pos())); context_menu->grab_focus(); @@ -552,8 +553,8 @@ TextEditor::TextEditor() { update_settings(); - code_editor->get_text_edit()->set_context_menu_enabled(false); - code_editor->get_text_edit()->connect("gui_input", callable_mp(this, &TextEditor::_text_edit_gui_input)); + code_editor->get_text_editor()->set_context_menu_enabled(false); + code_editor->get_text_editor()->connect("gui_input", callable_mp(this, &TextEditor::_text_edit_gui_input)); context_menu = memnew(PopupMenu); add_child(context_menu); @@ -649,7 +650,7 @@ TextEditor::TextEditor() { goto_line_dialog = memnew(GotoLineDialog); add_child(goto_line_dialog); - code_editor->get_text_edit()->set_drag_forwarding(this); + code_editor->get_text_editor()->set_drag_forwarding(this); } TextEditor::~TextEditor() { diff --git a/editor/plugins/text_editor.h b/editor/plugins/text_editor.h index f3e9e599cf..ea425bd033 100644 --- a/editor/plugins/text_editor.h +++ b/editor/plugins/text_editor.h @@ -119,7 +119,7 @@ public: virtual Variant get_edit_state() override; virtual void set_edit_state(const Variant &p_state) override; virtual Vector<String> get_functions() override; - virtual void get_breakpoints(List<int> *p_breakpoints) override; + virtual Array get_breakpoints() override; virtual void goto_line(int p_line, bool p_with_error = false) override; void goto_line_selection(int p_line, int p_begin, int p_end); virtual void set_executing_line(int p_line) override; diff --git a/editor/plugins/texture_3d_editor_plugin.cpp b/editor/plugins/texture_3d_editor_plugin.cpp new file mode 100644 index 0000000000..ba2eef8484 --- /dev/null +++ b/editor/plugins/texture_3d_editor_plugin.cpp @@ -0,0 +1,213 @@ +/*************************************************************************/ +/* texture_3d_editor_plugin.cpp */ +/*************************************************************************/ +/* This file is part of: */ +/* GODOT ENGINE */ +/* https://godotengine.org */ +/*************************************************************************/ +/* Copyright (c) 2007-2020 Juan Linietsky, Ariel Manzur. */ +/* Copyright (c) 2014-2020 Godot Engine contributors (cf. AUTHORS.md). */ +/* */ +/* Permission is hereby granted, free of charge, to any person obtaining */ +/* a copy of this software and associated documentation files (the */ +/* "Software"), to deal in the Software without restriction, including */ +/* without limitation the rights to use, copy, modify, merge, publish, */ +/* distribute, sublicense, and/or sell copies of the Software, and to */ +/* permit persons to whom the Software is furnished to do so, subject to */ +/* the following conditions: */ +/* */ +/* The above copyright notice and this permission notice shall be */ +/* included in all copies or substantial portions of the Software. */ +/* */ +/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */ +/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */ +/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*/ +/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */ +/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */ +/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */ +/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +/*************************************************************************/ + +#include "texture_3d_editor_plugin.h" + +#include "core/io/resource_loader.h" +#include "core/project_settings.h" +#include "editor/editor_settings.h" + +void Texture3DEditor::_gui_input(Ref<InputEvent> p_event) { +} + +void Texture3DEditor::_texture_rect_draw() { + texture_rect->draw_rect(Rect2(Point2(), texture_rect->get_size()), Color(1, 1, 1, 1)); +} + +void Texture3DEditor::_notification(int p_what) { + if (p_what == NOTIFICATION_READY) { + //get_scene()->connect("node_removed",this,"_node_removed"); + } + if (p_what == NOTIFICATION_RESIZED) { + _texture_rect_update_area(); + } + + if (p_what == NOTIFICATION_DRAW) { + Ref<Texture2D> checkerboard = get_theme_icon("Checkerboard", "EditorIcons"); + Size2 size = get_size(); + + draw_texture_rect(checkerboard, Rect2(Point2(), size), true); + } +} + +void Texture3DEditor::_changed_callback(Object *p_changed, const char *p_prop) { + if (!is_visible()) { + return; + } + update(); +} + +void Texture3DEditor::_update_material() { + material->set_shader_param("layer", (layer->get_value() + 0.5) / texture->get_depth()); + material->set_shader_param("tex", texture->get_rid()); + + String format = Image::get_format_name(texture->get_format()); + + String text; + text = itos(texture->get_width()) + "x" + itos(texture->get_height()) + "x" + itos(texture->get_depth()) + " " + format; + + info->set_text(text); +} + +void Texture3DEditor::_make_shaders() { + String shader_3d = "" + "shader_type canvas_item;\n" + "uniform sampler3D tex;\n" + "uniform float layer;\n" + "void fragment() {\n" + " COLOR = textureLod(tex,vec3(UV,layer),0.0);\n" + "}"; + + shader.instance(); + shader->set_code(shader_3d); + material.instance(); + material->set_shader(shader); +} + +void Texture3DEditor::_texture_rect_update_area() { + Size2 size = get_size(); + int tex_width = texture->get_width() * size.height / texture->get_height(); + int tex_height = size.height; + + if (tex_width > size.width) { + tex_width = size.width; + tex_height = texture->get_height() * tex_width / texture->get_width(); + } + + // Prevent the texture from being unpreviewable after the rescale, so that we can still see something + if (tex_height <= 0) { + tex_height = 1; + } + if (tex_width <= 0) { + tex_width = 1; + } + + int ofs_x = (size.width - tex_width) / 2; + int ofs_y = (size.height - tex_height) / 2; + + texture_rect->set_position(Vector2(ofs_x, ofs_y)); + texture_rect->set_size(Vector2(tex_width, tex_height)); +} + +void Texture3DEditor::edit(Ref<Texture3D> p_texture) { + if (!texture.is_null()) { + texture->remove_change_receptor(this); + } + + texture = p_texture; + + if (!texture.is_null()) { + if (shader.is_null()) { + _make_shaders(); + } + + texture->add_change_receptor(this); + update(); + texture_rect->set_material(material); + setting = true; + layer->set_max(texture->get_depth() - 1); + layer->set_value(0); + layer->show(); + _update_material(); + setting = false; + _texture_rect_update_area(); + } else { + hide(); + } +} + +void Texture3DEditor::_bind_methods() { + ClassDB::bind_method(D_METHOD("_gui_input"), &Texture3DEditor::_gui_input); + ClassDB::bind_method(D_METHOD("_layer_changed"), &Texture3DEditor::_layer_changed); +} + +Texture3DEditor::Texture3DEditor() { + set_texture_repeat(TextureRepeat::TEXTURE_REPEAT_ENABLED); + set_custom_minimum_size(Size2(1, 150)); + texture_rect = memnew(Control); + texture_rect->connect("draw", callable_mp(this, &Texture3DEditor::_texture_rect_draw)); + texture_rect->set_mouse_filter(MOUSE_FILTER_IGNORE); + add_child(texture_rect); + + layer = memnew(SpinBox); + layer->set_step(1); + layer->set_max(100); + add_child(layer); + layer->set_anchor(MARGIN_RIGHT, 1); + layer->set_anchor(MARGIN_LEFT, 1); + layer->set_h_grow_direction(GROW_DIRECTION_BEGIN); + layer->set_modulate(Color(1, 1, 1, 0.8)); + info = memnew(Label); + add_child(info); + info->set_anchor(MARGIN_RIGHT, 1); + info->set_anchor(MARGIN_LEFT, 1); + info->set_anchor(MARGIN_BOTTOM, 1); + info->set_anchor(MARGIN_TOP, 1); + info->set_h_grow_direction(GROW_DIRECTION_BEGIN); + info->set_v_grow_direction(GROW_DIRECTION_BEGIN); + info->add_theme_color_override("font_color", Color(1, 1, 1, 1)); + info->add_theme_color_override("font_color_shadow", Color(0, 0, 0, 0.5)); + info->add_theme_color_override("font_color_shadow", Color(0, 0, 0, 0.5)); + info->add_theme_constant_override("shadow_as_outline", 1); + info->add_theme_constant_override("shadow_offset_x", 2); + info->add_theme_constant_override("shadow_offset_y", 2); + + setting = false; + layer->connect("value_changed", Callable(this, "_layer_changed")); +} + +Texture3DEditor::~Texture3DEditor() { + if (!texture.is_null()) { + texture->remove_change_receptor(this); + } +} + +// +bool EditorInspectorPlugin3DTexture::can_handle(Object *p_object) { + return Object::cast_to<Texture3D>(p_object) != nullptr; +} + +void EditorInspectorPlugin3DTexture::parse_begin(Object *p_object) { + Texture3D *texture = Object::cast_to<Texture3D>(p_object); + if (!texture) { + return; + } + Ref<Texture3D> m(texture); + + Texture3DEditor *editor = memnew(Texture3DEditor); + editor->edit(m); + add_custom_control(editor); +} + +Texture3DEditorPlugin::Texture3DEditorPlugin(EditorNode *p_node) { + Ref<EditorInspectorPlugin3DTexture> plugin; + plugin.instance(); + add_inspector_plugin(plugin); +} diff --git a/editor/plugins/texture_3d_editor_plugin.h b/editor/plugins/texture_3d_editor_plugin.h new file mode 100644 index 0000000000..4fbf47ecfe --- /dev/null +++ b/editor/plugins/texture_3d_editor_plugin.h @@ -0,0 +1,93 @@ +/*************************************************************************/ +/* texture_3d_editor_plugin.h */ +/*************************************************************************/ +/* This file is part of: */ +/* GODOT ENGINE */ +/* https://godotengine.org */ +/*************************************************************************/ +/* Copyright (c) 2007-2020 Juan Linietsky, Ariel Manzur. */ +/* Copyright (c) 2014-2020 Godot Engine contributors (cf. AUTHORS.md). */ +/* */ +/* Permission is hereby granted, free of charge, to any person obtaining */ +/* a copy of this software and associated documentation files (the */ +/* "Software"), to deal in the Software without restriction, including */ +/* without limitation the rights to use, copy, modify, merge, publish, */ +/* distribute, sublicense, and/or sell copies of the Software, and to */ +/* permit persons to whom the Software is furnished to do so, subject to */ +/* the following conditions: */ +/* */ +/* The above copyright notice and this permission notice shall be */ +/* included in all copies or substantial portions of the Software. */ +/* */ +/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */ +/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */ +/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*/ +/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */ +/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */ +/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */ +/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +/*************************************************************************/ + +#ifndef TEXTURE_3D_EDITOR_PLUGIN_H +#define TEXTURE_3D_EDITOR_PLUGIN_H + +#include "editor/editor_node.h" +#include "editor/editor_plugin.h" +#include "scene/resources/shader.h" +#include "scene/resources/texture.h" + +class Texture3DEditor : public Control { + GDCLASS(Texture3DEditor, Control); + + SpinBox *layer; + Label *info; + Ref<Texture3D> texture; + + Ref<Shader> shader; + Ref<ShaderMaterial> material; + + Control *texture_rect; + + void _make_shaders(); + + void _update_material(); + bool setting; + void _layer_changed(double) { + if (!setting) { + _update_material(); + } + } + + void _texture_rect_update_area(); + void _texture_rect_draw(); + +protected: + void _notification(int p_what); + void _gui_input(Ref<InputEvent> p_event); + void _changed_callback(Object *p_changed, const char *p_prop) override; + static void _bind_methods(); + +public: + void edit(Ref<Texture3D> p_texture); + Texture3DEditor(); + ~Texture3DEditor(); +}; + +class EditorInspectorPlugin3DTexture : public EditorInspectorPlugin { + GDCLASS(EditorInspectorPlugin3DTexture, EditorInspectorPlugin); + +public: + virtual bool can_handle(Object *p_object) override; + virtual void parse_begin(Object *p_object) override; +}; + +class Texture3DEditorPlugin : public EditorPlugin { + GDCLASS(Texture3DEditorPlugin, EditorPlugin); + +public: + virtual String get_name() const override { return "Texture3D"; } + + Texture3DEditorPlugin(EditorNode *p_node); +}; + +#endif // TEXTURE_EDITOR_PLUGIN_H diff --git a/editor/plugins/tile_map_editor_plugin.cpp b/editor/plugins/tile_map_editor_plugin.cpp index e71485e9fc..8cd8aaf277 100644 --- a/editor/plugins/tile_map_editor_plugin.cpp +++ b/editor/plugins/tile_map_editor_plugin.cpp @@ -802,7 +802,6 @@ void TileMapEditor::_draw_cell(Control *p_viewport, int p_cell, const Point2i &p r.size = node->get_tileset()->autotile_get_size(p_cell); r.position += (r.size + Vector2(spacing, spacing)) * offset; } - Size2 sc = p_xform.get_scale(); Size2 cell_size = node->get_cell_size(); bool centered_texture = node->is_centered_textures_enabled(); bool compatibility_mode_enabled = node->is_compatibility_mode_enabled(); @@ -838,12 +837,12 @@ void TileMapEditor::_draw_cell(Control *p_viewport, int p_cell, const Point2i &p } if (p_flip_h) { - sc.x *= -1.0; + rect.size.x *= -1.0; tile_ofs.x *= -1.0; } if (p_flip_v) { - sc.y *= -1.0; + rect.size.y *= -1.0; tile_ofs.y *= -1.0; } diff --git a/editor/plugins/visual_shader_editor_plugin.cpp b/editor/plugins/visual_shader_editor_plugin.cpp index 9810fbee2c..6a70f973c3 100644 --- a/editor/plugins/visual_shader_editor_plugin.cpp +++ b/editor/plugins/visual_shader_editor_plugin.cpp @@ -60,6 +60,572 @@ void VisualShaderNodePlugin::_bind_methods() { /////////////////// +static Ref<StyleBoxEmpty> make_empty_stylebox(float p_margin_left = -1, float p_margin_top = -1, float p_margin_right = -1, float p_margin_bottom = -1) { + Ref<StyleBoxEmpty> style(memnew(StyleBoxEmpty)); + style->set_default_margin(MARGIN_LEFT, p_margin_left * EDSCALE); + style->set_default_margin(MARGIN_RIGHT, p_margin_right * EDSCALE); + style->set_default_margin(MARGIN_BOTTOM, p_margin_bottom * EDSCALE); + style->set_default_margin(MARGIN_TOP, p_margin_top * EDSCALE); + return style; +} + +/////////////////// + +VisualShaderGraphPlugin::VisualShaderGraphPlugin() { +} + +void VisualShaderGraphPlugin::_bind_methods() { + ClassDB::bind_method("add_node", &VisualShaderGraphPlugin::add_node); + ClassDB::bind_method("remove_node", &VisualShaderGraphPlugin::remove_node); + ClassDB::bind_method("connect_nodes", &VisualShaderGraphPlugin::connect_nodes); + ClassDB::bind_method("disconnect_nodes", &VisualShaderGraphPlugin::disconnect_nodes); + ClassDB::bind_method("set_node_position", &VisualShaderGraphPlugin::set_node_position); + ClassDB::bind_method("set_node_size", &VisualShaderGraphPlugin::set_node_size); + ClassDB::bind_method("show_port_preview", &VisualShaderGraphPlugin::show_port_preview); + ClassDB::bind_method("update_node", &VisualShaderGraphPlugin::update_node); + ClassDB::bind_method("update_node_deferred", &VisualShaderGraphPlugin::update_node_deferred); + ClassDB::bind_method("set_input_port_default_value", &VisualShaderGraphPlugin::set_input_port_default_value); + ClassDB::bind_method("set_uniform_name", &VisualShaderGraphPlugin::set_uniform_name); +} + +void VisualShaderGraphPlugin::register_shader(VisualShader *p_shader) { + visual_shader = Ref<VisualShader>(p_shader); +} + +void VisualShaderGraphPlugin::set_connections(List<VisualShader::Connection> &p_connections) { + connections = p_connections; +} + +void VisualShaderGraphPlugin::show_port_preview(VisualShader::Type p_type, int p_node_id, int p_port_id) { + if (visual_shader->get_shader_type() == p_type && links.has(p_node_id)) { + for (Map<int, Port>::Element *E = links[p_node_id].output_ports.front(); E; E = E->next()) { + E->value().preview_button->set_pressed(false); + } + + if (links[p_node_id].preview_visible && !is_dirty() && links[p_node_id].preview_box != nullptr) { + links[p_node_id].graph_node->remove_child(links[p_node_id].preview_box); + memdelete(links[p_node_id].preview_box); + links[p_node_id].graph_node->set_size(Vector2(-1, -1)); + links[p_node_id].preview_visible = false; + } + + if (p_port_id != -1) { + if (is_dirty()) { + links[p_node_id].preview_pos = links[p_node_id].graph_node->get_child_count(); + } + + VBoxContainer *vbox = memnew(VBoxContainer); + links[p_node_id].graph_node->add_child(vbox); + if (links[p_node_id].preview_pos != -1) { + links[p_node_id].graph_node->move_child(vbox, links[p_node_id].preview_pos); + } + + Control *offset = memnew(Control); + offset->set_custom_minimum_size(Size2(0, 5 * EDSCALE)); + vbox->add_child(offset); + + VisualShaderNodePortPreview *port_preview = memnew(VisualShaderNodePortPreview); + port_preview->setup(visual_shader, visual_shader->get_shader_type(), p_node_id, p_port_id); + port_preview->set_h_size_flags(Control::SIZE_SHRINK_CENTER); + vbox->add_child(port_preview); + links[p_node_id].preview_visible = true; + links[p_node_id].preview_box = vbox; + links[p_node_id].output_ports[p_port_id].preview_button->set_pressed(true); + } + } +} + +void VisualShaderGraphPlugin::update_node_deferred(VisualShader::Type p_type, int p_node_id) { + call_deferred("update_node", p_type, p_node_id); +} + +void VisualShaderGraphPlugin::update_node(VisualShader::Type p_type, int p_node_id) { + if (p_type != visual_shader->get_shader_type() || !links.has(p_node_id)) { + return; + } + remove_node(p_type, p_node_id); + add_node(p_type, p_node_id); +} + +void VisualShaderGraphPlugin::set_input_port_default_value(VisualShader::Type p_type, int p_node_id, int p_port_id, Variant p_value) { + if (p_type != visual_shader->get_shader_type() || !links.has(p_node_id)) { + return; + } + + Button *button = links[p_node_id].input_ports[p_port_id].default_input_button; + + switch (p_value.get_type()) { + case Variant::COLOR: { + button->set_custom_minimum_size(Size2(30, 0) * EDSCALE); + if (!button->is_connected("draw", callable_mp(VisualShaderEditor::get_singleton(), &VisualShaderEditor::_draw_color_over_button))) { + button->connect("draw", callable_mp(VisualShaderEditor::get_singleton(), &VisualShaderEditor::_draw_color_over_button), varray(button, p_value)); + } + } break; + case Variant::BOOL: { + button->set_text(((bool)p_value) ? "true" : "false"); + } break; + case Variant::INT: + case Variant::FLOAT: { + button->set_text(String::num(p_value, 4)); + } break; + case Variant::VECTOR3: { + Vector3 v = p_value; + button->set_text(String::num(v.x, 3) + "," + String::num(v.y, 3) + "," + String::num(v.z, 3)); + } break; + default: { + } + } +} + +void VisualShaderGraphPlugin::set_uniform_name(VisualShader::Type p_type, int p_node_id, const String &p_name) { + if (visual_shader->get_shader_type() == p_type && links.has(p_node_id) && links[p_node_id].uniform_name != nullptr) { + links[p_node_id].uniform_name->set_text(p_name); + } +} + +void VisualShaderGraphPlugin::register_default_input_button(int p_node_id, int p_port_id, Button *p_button) { + links[p_node_id].input_ports.insert(p_port_id, { p_button }); +} + +void VisualShaderGraphPlugin::update_uniform_refs() { + for (Map<int, Link>::Element *E = links.front(); E; E = E->next()) { + VisualShaderNodeUniformRef *ref = Object::cast_to<VisualShaderNodeUniformRef>(E->get().visual_node); + if (ref) { + remove_node(E->get().type, E->key()); + add_node(E->get().type, E->key()); + } + } +} + +VisualShader::Type VisualShaderGraphPlugin::get_shader_type() const { + return visual_shader->get_shader_type(); +} + +void VisualShaderGraphPlugin::set_node_position(VisualShader::Type p_type, int p_id, const Vector2 &p_position) { + if (visual_shader->get_shader_type() == p_type && links.has(p_id)) { + links[p_id].graph_node->set_offset(p_position); + } +} + +void VisualShaderGraphPlugin::set_node_size(VisualShader::Type p_type, int p_id, const Vector2 &p_size) { + if (visual_shader->get_shader_type() == p_type && links.has(p_id)) { + links[p_id].graph_node->set_size(p_size); + } +} + +bool VisualShaderGraphPlugin::is_preview_visible(int p_id) const { + return links[p_id].preview_visible; +} + +void VisualShaderGraphPlugin::clear_links() { + links.clear(); +} + +bool VisualShaderGraphPlugin::is_dirty() const { + return dirty; +} + +void VisualShaderGraphPlugin::make_dirty(bool p_enabled) { + dirty = p_enabled; +} + +void VisualShaderGraphPlugin::register_link(VisualShader::Type p_type, int p_id, VisualShaderNode *p_visual_node, GraphNode *p_graph_node) { + links.insert(p_id, { p_type, p_visual_node, p_graph_node, p_visual_node->get_output_port_for_preview() != -1, -1, Map<int, InputPort>(), Map<int, Port>(), nullptr, nullptr }); +} + +void VisualShaderGraphPlugin::register_output_port(int p_node_id, int p_port, TextureButton *p_button) { + links[p_node_id].output_ports.insert(p_port, { p_button }); +} + +void VisualShaderGraphPlugin::register_uniform_name(int p_node_id, LineEdit *p_uniform_name) { + links[p_node_id].uniform_name = p_uniform_name; +} + +void VisualShaderGraphPlugin::add_node(VisualShader::Type p_type, int p_id) { + if (p_type != visual_shader->get_shader_type()) { + return; + } + + Control *offset; + + static Ref<StyleBoxEmpty> label_style = make_empty_stylebox(2, 1, 2, 1); + + static const Color type_color[6] = { + Color(0.38, 0.85, 0.96), // scalar (float) + Color(0.49, 0.78, 0.94), // scalar (int) + Color(0.84, 0.49, 0.93), // vector + Color(0.55, 0.65, 0.94), // boolean + Color(0.96, 0.66, 0.43), // transform + Color(1.0, 1.0, 0.0), // sampler + }; + + Ref<VisualShaderNode> vsnode = visual_shader->get_node(p_type, p_id); + + Ref<VisualShaderNodeGroupBase> group_node = Object::cast_to<VisualShaderNodeGroupBase>(vsnode.ptr()); + bool is_group = !group_node.is_null(); + Size2 size = Size2(0, 0); + + Ref<VisualShaderNodeExpression> expression_node = Object::cast_to<VisualShaderNodeExpression>(group_node.ptr()); + bool is_expression = !expression_node.is_null(); + String expression = ""; + + GraphNode *node = memnew(GraphNode); + register_link(p_type, p_id, vsnode.ptr(), node); + + if (is_group) { + size = group_node->get_size(); + + node->set_resizable(true); + node->connect("resize_request", callable_mp(VisualShaderEditor::get_singleton(), &VisualShaderEditor::_node_resized), varray((int)p_type, p_id)); + } + if (is_expression) { + expression = expression_node->get_expression(); + } + + node->set_offset(visual_shader->get_node_position(p_type, p_id)); + node->set_title(vsnode->get_caption()); + node->set_name(itos(p_id)); + + if (p_id >= 2) { + node->set_show_close_button(true); + node->connect("close_request", callable_mp(VisualShaderEditor::get_singleton(), &VisualShaderEditor::_delete_request), varray(p_id), CONNECT_DEFERRED); + } + + node->connect("dragged", callable_mp(VisualShaderEditor::get_singleton(), &VisualShaderEditor::_node_dragged), varray(p_id)); + + Control *custom_editor = nullptr; + int port_offset = 0; + + if (is_group) { + port_offset += 2; + } + + Ref<VisualShaderNodeUniform> uniform = vsnode; + if (uniform.is_valid()) { + VisualShaderEditor::get_singleton()->graph->add_child(node); + VisualShaderEditor::get_singleton()->_update_created_node(node); + + LineEdit *uniform_name = memnew(LineEdit); + register_uniform_name(p_id, uniform_name); + uniform_name->set_text(uniform->get_uniform_name()); + node->add_child(uniform_name); + uniform_name->connect("text_entered", callable_mp(VisualShaderEditor::get_singleton(), &VisualShaderEditor::_uniform_line_edit_changed), varray(p_id)); + uniform_name->connect("focus_exited", callable_mp(VisualShaderEditor::get_singleton(), &VisualShaderEditor::_uniform_line_edit_focus_out), varray(uniform_name, p_id)); + + if (vsnode->get_input_port_count() == 0 && vsnode->get_output_port_count() == 1 && vsnode->get_output_port_name(0) == "") { + //shortcut + VisualShaderNode::PortType port_right = vsnode->get_output_port_type(0); + node->set_slot(0, false, VisualShaderNode::PORT_TYPE_SCALAR, Color(), true, port_right, type_color[port_right]); + if (!vsnode->is_use_prop_slots()) { + return; + } + } + port_offset++; + } + + for (int i = 0; i < VisualShaderEditor::get_singleton()->plugins.size(); i++) { + vsnode->set_meta("id", p_id); + vsnode->set_meta("shader_type", (int)p_type); + custom_editor = VisualShaderEditor::get_singleton()->plugins.write[i]->create_editor(visual_shader, vsnode); + vsnode->remove_meta("id"); + vsnode->remove_meta("shader_type"); + if (custom_editor) { + if (vsnode->is_show_prop_names()) { + custom_editor->call_deferred("_show_prop_names", true); + } + break; + } + } + + if (custom_editor && !vsnode->is_use_prop_slots() && vsnode->get_output_port_count() > 0 && vsnode->get_output_port_name(0) == "" && (vsnode->get_input_port_count() == 0 || vsnode->get_input_port_name(0) == "")) { + //will be embedded in first port + } else if (custom_editor) { + port_offset++; + node->add_child(custom_editor); + if (vsnode->is_use_prop_slots()) { + return; + } + custom_editor = nullptr; + } + + if (is_group) { + offset = memnew(Control); + offset->set_custom_minimum_size(Size2(0, 6 * EDSCALE)); + node->add_child(offset); + + if (group_node->is_editable()) { + HBoxContainer *hb2 = memnew(HBoxContainer); + + Button *add_input_btn = memnew(Button); + add_input_btn->set_text(TTR("Add Input")); + add_input_btn->connect("pressed", callable_mp(VisualShaderEditor::get_singleton(), &VisualShaderEditor::_add_input_port), varray(p_id, group_node->get_free_input_port_id(), VisualShaderNode::PORT_TYPE_VECTOR, "input" + itos(group_node->get_free_input_port_id())), CONNECT_DEFERRED); + hb2->add_child(add_input_btn); + + hb2->add_spacer(); + + Button *add_output_btn = memnew(Button); + add_output_btn->set_text(TTR("Add Output")); + add_output_btn->connect("pressed", callable_mp(VisualShaderEditor::get_singleton(), &VisualShaderEditor::_add_output_port), varray(p_id, group_node->get_free_output_port_id(), VisualShaderNode::PORT_TYPE_VECTOR, "output" + itos(group_node->get_free_output_port_id())), CONNECT_DEFERRED); + hb2->add_child(add_output_btn); + + node->add_child(hb2); + } + } + + for (int i = 0; i < MAX(vsnode->get_input_port_count(), vsnode->get_output_port_count()); i++) { + if (vsnode->is_port_separator(i)) { + node->add_child(memnew(HSeparator)); + port_offset++; + } + + bool valid_left = i < vsnode->get_input_port_count(); + VisualShaderNode::PortType port_left = VisualShaderNode::PORT_TYPE_SCALAR; + bool port_left_used = false; + String name_left; + if (valid_left) { + name_left = vsnode->get_input_port_name(i); + port_left = vsnode->get_input_port_type(i); + for (List<VisualShader::Connection>::Element *E = connections.front(); E; E = E->next()) { + if (E->get().to_node == p_id && E->get().to_port == i) { + port_left_used = true; + } + } + } + + bool valid_right = i < vsnode->get_output_port_count(); + VisualShaderNode::PortType port_right = VisualShaderNode::PORT_TYPE_SCALAR; + String name_right; + if (valid_right) { + name_right = vsnode->get_output_port_name(i); + port_right = vsnode->get_output_port_type(i); + } + + HBoxContainer *hb = memnew(HBoxContainer); + hb->add_theme_constant_override("separation", 7 * EDSCALE); + + Variant default_value; + + if (valid_left && !port_left_used) { + default_value = vsnode->get_input_port_default_value(i); + } + + Button *button = memnew(Button); + hb->add_child(button); + register_default_input_button(p_id, i, button); + button->connect("pressed", callable_mp(VisualShaderEditor::get_singleton(), &VisualShaderEditor::_edit_port_default_input), varray(button, p_id, i)); + if (default_value.get_type() != Variant::NIL) { // only a label + set_input_port_default_value(p_type, p_id, i, default_value); + } else { + button->hide(); + } + + if (i == 0 && custom_editor) { + hb->add_child(custom_editor); + custom_editor->set_h_size_flags(Control::SIZE_EXPAND_FILL); + } else { + if (valid_left) { + if (is_group) { + OptionButton *type_box = memnew(OptionButton); + hb->add_child(type_box); + type_box->add_item(TTR("Float")); + type_box->add_item(TTR("Int")); + type_box->add_item(TTR("Vector")); + type_box->add_item(TTR("Boolean")); + type_box->add_item(TTR("Transform")); + type_box->add_item(TTR("Sampler")); + type_box->select(group_node->get_input_port_type(i)); + type_box->set_custom_minimum_size(Size2(100 * EDSCALE, 0)); + type_box->connect("item_selected", callable_mp(VisualShaderEditor::get_singleton(), &VisualShaderEditor::_change_input_port_type), varray(p_id, i), CONNECT_DEFERRED); + + LineEdit *name_box = memnew(LineEdit); + hb->add_child(name_box); + name_box->set_custom_minimum_size(Size2(65 * EDSCALE, 0)); + name_box->set_h_size_flags(Control::SIZE_EXPAND_FILL); + name_box->set_text(name_left); + name_box->connect("text_entered", callable_mp(VisualShaderEditor::get_singleton(), &VisualShaderEditor::_change_input_port_name), varray(name_box, p_id, i)); + name_box->connect("focus_exited", callable_mp(VisualShaderEditor::get_singleton(), &VisualShaderEditor::_port_name_focus_out), varray(name_box, p_id, i, false)); + + Button *remove_btn = memnew(Button); + remove_btn->set_icon(EditorNode::get_singleton()->get_gui_base()->get_theme_icon("Remove", "EditorIcons")); + remove_btn->set_tooltip(TTR("Remove") + " " + name_left); + remove_btn->connect("pressed", callable_mp(VisualShaderEditor::get_singleton(), &VisualShaderEditor::_remove_input_port), varray(p_id, i), CONNECT_DEFERRED); + hb->add_child(remove_btn); + } else { + Label *label = memnew(Label); + label->set_text(name_left); + label->add_theme_style_override("normal", label_style); //more compact + hb->add_child(label); + + if (vsnode->get_input_port_default_hint(i) != "" && !port_left_used) { + Label *hint_label = memnew(Label); + hint_label->set_text("[" + vsnode->get_input_port_default_hint(i) + "]"); + hint_label->add_theme_color_override("font_color", VisualShaderEditor::get_singleton()->get_theme_color("font_color_readonly", "TextEdit")); + hint_label->add_theme_style_override("normal", label_style); + hb->add_child(hint_label); + } + } + } + + if (!is_group) { + hb->add_spacer(); + } + + if (valid_right) { + if (is_group) { + Button *remove_btn = memnew(Button); + remove_btn->set_icon(EditorNode::get_singleton()->get_gui_base()->get_theme_icon("Remove", "EditorIcons")); + remove_btn->set_tooltip(TTR("Remove") + " " + name_left); + remove_btn->connect("pressed", callable_mp(VisualShaderEditor::get_singleton(), &VisualShaderEditor::_remove_output_port), varray(p_id, i), CONNECT_DEFERRED); + hb->add_child(remove_btn); + + LineEdit *name_box = memnew(LineEdit); + hb->add_child(name_box); + name_box->set_custom_minimum_size(Size2(65 * EDSCALE, 0)); + name_box->set_h_size_flags(Control::SIZE_EXPAND_FILL); + name_box->set_text(name_right); + name_box->connect("text_entered", callable_mp(VisualShaderEditor::get_singleton(), &VisualShaderEditor::_change_output_port_name), varray(name_box, p_id, i)); + name_box->connect("focus_exited", callable_mp(VisualShaderEditor::get_singleton(), &VisualShaderEditor::_port_name_focus_out), varray(name_box, p_id, i, true)); + + OptionButton *type_box = memnew(OptionButton); + hb->add_child(type_box); + type_box->add_item(TTR("Float")); + type_box->add_item(TTR("Int")); + type_box->add_item(TTR("Vector")); + type_box->add_item(TTR("Boolean")); + type_box->add_item(TTR("Transform")); + type_box->select(group_node->get_output_port_type(i)); + type_box->set_custom_minimum_size(Size2(100 * EDSCALE, 0)); + type_box->connect("item_selected", callable_mp(VisualShaderEditor::get_singleton(), &VisualShaderEditor::_change_output_port_type), varray(p_id, i), CONNECT_DEFERRED); + } else { + Label *label = memnew(Label); + label->set_text(name_right); + label->add_theme_style_override("normal", label_style); //more compact + hb->add_child(label); + } + } + } + + if (valid_right && visual_shader->get_shader_type() == VisualShader::TYPE_FRAGMENT && port_right != VisualShaderNode::PORT_TYPE_TRANSFORM && port_right != VisualShaderNode::PORT_TYPE_SAMPLER) { + TextureButton *preview = memnew(TextureButton); + preview->set_toggle_mode(true); + preview->set_normal_texture(VisualShaderEditor::get_singleton()->get_theme_icon("GuiVisibilityHidden", "EditorIcons")); + preview->set_pressed_texture(VisualShaderEditor::get_singleton()->get_theme_icon("GuiVisibilityVisible", "EditorIcons")); + preview->set_v_size_flags(Control::SIZE_SHRINK_CENTER); + + register_output_port(p_id, i, preview); + + preview->connect("pressed", callable_mp(VisualShaderEditor::get_singleton(), &VisualShaderEditor::_preview_select_port), varray(p_id, i), CONNECT_DEFERRED); + hb->add_child(preview); + } + + if (is_group) { + offset = memnew(Control); + offset->set_custom_minimum_size(Size2(0, 5 * EDSCALE)); + node->add_child(offset); + port_offset++; + } + + node->add_child(hb); + + node->set_slot(i + port_offset, valid_left, port_left, type_color[port_left], valid_right, port_right, type_color[port_right]); + } + + if (vsnode->get_output_port_for_preview() >= 0) { + show_port_preview(p_type, p_id, vsnode->get_output_port_for_preview()); + } + + offset = memnew(Control); + offset->set_custom_minimum_size(Size2(0, 4 * EDSCALE)); + node->add_child(offset); + + String error = vsnode->get_warning(visual_shader->get_mode(), p_type); + if (error != String()) { + Label *error_label = memnew(Label); + error_label->add_theme_color_override("font_color", VisualShaderEditor::get_singleton()->get_theme_color("error_color", "Editor")); + error_label->set_text(error); + node->add_child(error_label); + } + + if (is_expression) { + CodeEdit *expression_box = memnew(CodeEdit); + Ref<CodeHighlighter> expression_syntax_highlighter; + expression_syntax_highlighter.instance(); + expression_node->set_control(expression_box, 0); + node->add_child(expression_box); + + Color background_color = EDITOR_GET("text_editor/highlighting/background_color"); + Color text_color = EDITOR_GET("text_editor/highlighting/text_color"); + Color keyword_color = EDITOR_GET("text_editor/highlighting/keyword_color"); + Color comment_color = EDITOR_GET("text_editor/highlighting/comment_color"); + Color symbol_color = EDITOR_GET("text_editor/highlighting/symbol_color"); + Color function_color = EDITOR_GET("text_editor/highlighting/function_color"); + Color number_color = EDITOR_GET("text_editor/highlighting/number_color"); + Color members_color = EDITOR_GET("text_editor/highlighting/member_variable_color"); + + expression_box->set_syntax_highlighter(expression_syntax_highlighter); + expression_box->add_theme_color_override("background_color", background_color); + + for (List<String>::Element *E = VisualShaderEditor::get_singleton()->keyword_list.front(); E; E = E->next()) { + expression_syntax_highlighter->add_keyword_color(E->get(), keyword_color); + } + + expression_box->add_theme_font_override("font", VisualShaderEditor::get_singleton()->get_theme_font("expression", "EditorFonts")); + expression_box->add_theme_color_override("font_color", text_color); + expression_syntax_highlighter->set_number_color(number_color); + expression_syntax_highlighter->set_symbol_color(symbol_color); + expression_syntax_highlighter->set_function_color(function_color); + expression_syntax_highlighter->set_member_variable_color(members_color); + expression_syntax_highlighter->add_color_region("/*", "*/", comment_color, false); + expression_syntax_highlighter->add_color_region("//", "", comment_color, true); + + expression_box->set_text(expression); + expression_box->set_context_menu_enabled(false); + expression_box->set_draw_line_numbers(true); + + expression_box->connect("focus_exited", callable_mp(VisualShaderEditor::get_singleton(), &VisualShaderEditor::_expression_focus_out), varray(expression_box, p_id)); + } + + if (!uniform.is_valid()) { + VisualShaderEditor::get_singleton()->graph->add_child(node); + VisualShaderEditor::get_singleton()->_update_created_node(node); + if (is_group) { + call_deferred("_set_node_size", (int)p_type, p_id, size); + } + } +} + +void VisualShaderGraphPlugin::remove_node(VisualShader::Type p_type, int p_id) { + if (visual_shader->get_shader_type() == p_type && links.has(p_id)) { + links[p_id].graph_node->get_parent()->remove_child(links[p_id].graph_node); + memdelete(links[p_id].graph_node); + links.erase(p_id); + } +} + +void VisualShaderGraphPlugin::connect_nodes(VisualShader::Type p_type, int p_from_node, int p_from_port, int p_to_node, int p_to_port) { + if (visual_shader->get_shader_type() == p_type) { + VisualShaderEditor::get_singleton()->graph->connect_node(itos(p_from_node), p_from_port, itos(p_to_node), p_to_port); + if (links[p_to_node].input_ports.has(p_to_port) && links[p_to_node].input_ports[p_to_port].default_input_button != nullptr) { + links[p_to_node].input_ports[p_to_port].default_input_button->hide(); + } + } +} + +void VisualShaderGraphPlugin::disconnect_nodes(VisualShader::Type p_type, int p_from_node, int p_from_port, int p_to_node, int p_to_port) { + if (visual_shader->get_shader_type() == p_type) { + VisualShaderEditor::get_singleton()->graph->disconnect_node(itos(p_from_node), p_from_port, itos(p_to_node), p_to_port); + if (links[p_to_node].input_ports.has(p_to_port) && links[p_to_node].input_ports[p_to_port].default_input_button != nullptr && links[p_to_node].visual_node->get_input_port_default_value(p_to_port).get_type() != Variant::NIL) { + links[p_to_node].input_ports[p_to_port].default_input_button->show(); + set_input_port_default_value(p_type, p_to_node, p_to_port, links[p_to_node].visual_node->get_input_port_default_value(p_to_port)); + } + } +} + +VisualShaderGraphPlugin::~VisualShaderGraphPlugin() { +} + +///////////////// + void VisualShaderEditor::edit(VisualShader *p_visual_shader) { bool changed = false; if (p_visual_shader) { @@ -71,6 +637,7 @@ void VisualShaderEditor::edit(VisualShader *p_visual_shader) { } } visual_shader = Ref<VisualShader>(p_visual_shader); + graph_plugin->register_shader(visual_shader.ptr()); if (!visual_shader->is_connected("changed", callable_mp(this, &VisualShaderEditor::_update_preview))) { visual_shader->connect("changed", callable_mp(this, &VisualShaderEditor::_update_preview)); } @@ -81,19 +648,7 @@ void VisualShaderEditor::edit(VisualShader *p_visual_shader) { } #endif visual_shader->set_graph_offset(graph->get_scroll_ofs() / EDSCALE); - - if (visual_shader->get_mode() == VisualShader::MODE_PARTICLES) { - edit_type_standart->set_visible(false); - edit_type_particles->set_visible(true); - edit_type = edit_type_particles; - particles_mode = true; - } else { - edit_type_particles->set_visible(false); - edit_type_standart->set_visible(true); - edit_type = edit_type_standart; - particles_mode = false; - } - visual_shader->set_shader_type(get_current_shader_type()); + _set_mode(visual_shader->get_mode()); } else { if (visual_shader.is_valid()) { if (visual_shader->is_connected("changed", callable_mp(this, &VisualShaderEditor::_update_preview))) { @@ -110,8 +665,8 @@ void VisualShaderEditor::edit(VisualShader *p_visual_shader) { _clear_buffer(); _update_options_menu(); _update_preview(); + _update_graph(); } - _update_graph(); } } @@ -401,6 +956,21 @@ void VisualShaderEditor::_update_options_menu() { } } +void VisualShaderEditor::_set_mode(int p_which) { + if (p_which == VisualShader::MODE_PARTICLES) { + edit_type_standart->set_visible(false); + edit_type_particles->set_visible(true); + edit_type = edit_type_particles; + particles_mode = true; + } else { + edit_type_particles->set_visible(false); + edit_type_standart->set_visible(true); + edit_type = edit_type_standart; + particles_mode = false; + } + visual_shader->set_shader_type(get_current_shader_type()); +} + Size2 VisualShaderEditor::get_minimum_size() const { return Size2(10, 200); } @@ -415,15 +985,6 @@ void VisualShaderEditor::_draw_color_over_button(Object *obj, Color p_color) { button->draw_rect(Rect2(normal->get_offset(), button->get_size() - normal->get_minimum_size()), p_color); } -static Ref<StyleBoxEmpty> make_empty_stylebox(float p_margin_left = -1, float p_margin_top = -1, float p_margin_right = -1, float p_margin_bottom = -1) { - Ref<StyleBoxEmpty> style(memnew(StyleBoxEmpty)); - style->set_default_margin(MARGIN_LEFT, p_margin_left * EDSCALE); - style->set_default_margin(MARGIN_RIGHT, p_margin_right * EDSCALE); - style->set_default_margin(MARGIN_BOTTOM, p_margin_bottom * EDSCALE); - style->set_default_margin(MARGIN_TOP, p_margin_top * EDSCALE); - return style; -} - void VisualShaderEditor::_update_created_node(GraphNode *node) { if (EditorSettings::get_singleton()->get("interface/theme/use_graph_node_headers")) { Ref<StyleBoxFlat> sb = node->get_theme_stylebox("frame", "GraphNode"); @@ -447,50 +1008,9 @@ void VisualShaderEditor::_update_created_node(GraphNode *node) { } } -void VisualShaderEditor::_update_graph() { - if (updating) { - return; - } - - if (visual_shader.is_null()) { - return; - } - - graph->set_scroll_ofs(visual_shader->get_graph_offset() * EDSCALE); - - VisualShader::Type type = get_current_shader_type(); - - graph->clear_connections(); - //erase all nodes - for (int i = 0; i < graph->get_child_count(); i++) { - if (Object::cast_to<GraphNode>(graph->get_child(i))) { - Node *node = graph->get_child(i); - graph->remove_child(node); - memdelete(node); - i--; - } - } - - static const Color type_color[6] = { - Color(0.38, 0.85, 0.96), // scalar (float) - Color(0.49, 0.78, 0.94), // scalar (int) - Color(0.84, 0.49, 0.93), // vector - Color(0.55, 0.65, 0.94), // boolean - Color(0.96, 0.66, 0.43), // transform - Color(1.0, 1.0, 0.0), // sampler - }; - - List<VisualShader::Connection> connections; - visual_shader->get_node_connections(type, &connections); - - Ref<StyleBoxEmpty> label_style = make_empty_stylebox(2, 1, 2, 1); - - Vector<int> nodes = visual_shader->get_node_list(type); - +void VisualShaderEditor::_update_uniforms(bool p_update_refs) { VisualShaderNodeUniformRef::clear_uniforms(); - // scan for all uniforms - for (int t = 0; t < VisualShader::TYPE_MAX; t++) { Vector<int> tnodes = visual_shader->get_node_list((VisualShader::Type)t); for (int i = 0; i < tnodes.size(); i++) { @@ -525,389 +1045,73 @@ void VisualShaderEditor::_update_graph() { } } } + if (p_update_refs) { + graph_plugin->update_uniform_refs(); + } +} - Control *offset; - - for (int n_i = 0; n_i < nodes.size(); n_i++) { - Vector2 position = visual_shader->get_node_position(type, nodes[n_i]); - Ref<VisualShaderNode> vsnode = visual_shader->get_node(type, nodes[n_i]); - - Ref<VisualShaderNodeGroupBase> group_node = Object::cast_to<VisualShaderNodeGroupBase>(vsnode.ptr()); - bool is_group = !group_node.is_null(); - Size2 size = Size2(0, 0); - - Ref<VisualShaderNodeExpression> expression_node = Object::cast_to<VisualShaderNodeExpression>(group_node.ptr()); - bool is_expression = !expression_node.is_null(); - String expression = ""; - - GraphNode *node = memnew(GraphNode); - visual_shader->set_graph_node(type, nodes[n_i], node); - - if (is_group) { - size = group_node->get_size(); - - node->set_resizable(true); - node->connect("resize_request", callable_mp(this, &VisualShaderEditor::_node_resized), varray((int)type, nodes[n_i])); - } - if (is_expression) { - expression = expression_node->get_expression(); - } - - node->set_offset(position); - - node->set_title(vsnode->get_caption()); - node->set_name(itos(nodes[n_i])); - - if (nodes[n_i] >= 2) { - node->set_show_close_button(true); - node->connect("close_request", callable_mp(this, &VisualShaderEditor::_delete_request), varray(nodes[n_i]), CONNECT_DEFERRED); - } - - node->connect("dragged", callable_mp(this, &VisualShaderEditor::_node_dragged), varray(nodes[n_i])); - - Control *custom_editor = nullptr; - int port_offset = 0; - - if (is_group) { - port_offset += 2; - } - - Ref<VisualShaderNodeUniform> uniform = vsnode; - Ref<VisualShaderNodeFloatUniform> float_uniform = vsnode; - Ref<VisualShaderNodeIntUniform> int_uniform = vsnode; - Ref<VisualShaderNodeVec3Uniform> vec3_uniform = vsnode; - Ref<VisualShaderNodeColorUniform> color_uniform = vsnode; - Ref<VisualShaderNodeBooleanUniform> bool_uniform = vsnode; - Ref<VisualShaderNodeTransformUniform> transform_uniform = vsnode; - if (uniform.is_valid()) { - graph->add_child(node); - _update_created_node(node); - - LineEdit *uniform_name = memnew(LineEdit); - uniform_name->set_text(uniform->get_uniform_name()); - node->add_child(uniform_name); - uniform_name->connect("text_entered", callable_mp(this, &VisualShaderEditor::_line_edit_changed), varray(uniform_name, nodes[n_i])); - uniform_name->connect("focus_exited", callable_mp(this, &VisualShaderEditor::_line_edit_focus_out), varray(uniform_name, nodes[n_i])); - - String error = vsnode->get_warning(visual_shader->get_mode(), type); - if (error != String()) { - offset = memnew(Control); - offset->set_custom_minimum_size(Size2(0, 4 * EDSCALE)); - node->add_child(offset); - Label *error_label = memnew(Label); - error_label->add_theme_color_override("font_color", get_theme_color("error_color", "Editor")); - error_label->set_text(error); - node->add_child(error_label); - } - - if (vsnode->get_input_port_count() == 0 && vsnode->get_output_port_count() == 1 && vsnode->get_output_port_name(0) == "") { - //shortcut - VisualShaderNode::PortType port_right = vsnode->get_output_port_type(0); - node->set_slot(0, false, VisualShaderNode::PORT_TYPE_SCALAR, Color(), true, port_right, type_color[port_right]); - if (!float_uniform.is_valid() && !int_uniform.is_valid() && !vec3_uniform.is_valid() && !color_uniform.is_valid() && !bool_uniform.is_valid() && !transform_uniform.is_valid()) { - continue; - } - } - port_offset++; - } - - for (int i = 0; i < plugins.size(); i++) { - custom_editor = plugins.write[i]->create_editor(visual_shader, vsnode); - if (custom_editor) { - break; - } - } - - if (custom_editor && !float_uniform.is_valid() && !int_uniform.is_valid() && !vec3_uniform.is_valid() && !bool_uniform.is_valid() && !transform_uniform.is_valid() && vsnode->get_output_port_count() > 0 && vsnode->get_output_port_name(0) == "" && (vsnode->get_input_port_count() == 0 || vsnode->get_input_port_name(0) == "")) { - //will be embedded in first port - } else if (custom_editor) { - port_offset++; - node->add_child(custom_editor); - if (color_uniform.is_valid()) { - custom_editor->call_deferred("_show_prop_names", true); - } - if (float_uniform.is_valid() || int_uniform.is_valid() || vec3_uniform.is_valid() || bool_uniform.is_valid() || transform_uniform.is_valid()) { - custom_editor->call_deferred("_show_prop_names", true); - continue; - } - custom_editor = nullptr; - } - - if (is_group) { - offset = memnew(Control); - offset->set_custom_minimum_size(Size2(0, 6 * EDSCALE)); - node->add_child(offset); - - if (group_node->is_editable()) { - HBoxContainer *hb2 = memnew(HBoxContainer); - - Button *add_input_btn = memnew(Button); - add_input_btn->set_text(TTR("Add Input")); - add_input_btn->connect("pressed", callable_mp(this, &VisualShaderEditor::_add_input_port), varray(nodes[n_i], group_node->get_free_input_port_id(), VisualShaderNode::PORT_TYPE_VECTOR, "input" + itos(group_node->get_free_input_port_id())), CONNECT_DEFERRED); - hb2->add_child(add_input_btn); - - hb2->add_spacer(); - - Button *add_output_btn = memnew(Button); - add_output_btn->set_text(TTR("Add Output")); - add_output_btn->connect("pressed", callable_mp(this, &VisualShaderEditor::_add_output_port), varray(nodes[n_i], group_node->get_free_output_port_id(), VisualShaderNode::PORT_TYPE_VECTOR, "output" + itos(group_node->get_free_output_port_id())), CONNECT_DEFERRED); - hb2->add_child(add_output_btn); - - node->add_child(hb2); - } - } - - for (int i = 0; i < MAX(vsnode->get_input_port_count(), vsnode->get_output_port_count()); i++) { - if (vsnode->is_port_separator(i)) { - node->add_child(memnew(HSeparator)); - port_offset++; - } - - bool valid_left = i < vsnode->get_input_port_count(); - VisualShaderNode::PortType port_left = VisualShaderNode::PORT_TYPE_SCALAR; - bool port_left_used = false; - String name_left; - if (valid_left) { - name_left = vsnode->get_input_port_name(i); - port_left = vsnode->get_input_port_type(i); - for (List<VisualShader::Connection>::Element *E = connections.front(); E; E = E->next()) { - if (E->get().to_node == nodes[n_i] && E->get().to_port == i) { - port_left_used = true; - } - } - } - - bool valid_right = i < vsnode->get_output_port_count(); - VisualShaderNode::PortType port_right = VisualShaderNode::PORT_TYPE_SCALAR; - String name_right; - if (valid_right) { - name_right = vsnode->get_output_port_name(i); - port_right = vsnode->get_output_port_type(i); - } - - HBoxContainer *hb = memnew(HBoxContainer); - hb->add_theme_constant_override("separation", 7 * EDSCALE); - - Variant default_value; - - if (valid_left && !port_left_used) { - default_value = vsnode->get_input_port_default_value(i); - } - - if (default_value.get_type() != Variant::NIL) { // only a label - Button *button = memnew(Button); - hb->add_child(button); - button->connect("pressed", callable_mp(this, &VisualShaderEditor::_edit_port_default_input), varray(button, nodes[n_i], i)); - - switch (default_value.get_type()) { - case Variant::COLOR: { - button->set_custom_minimum_size(Size2(30, 0) * EDSCALE); - button->connect("draw", callable_mp(this, &VisualShaderEditor::_draw_color_over_button), varray(button, default_value)); - } break; - case Variant::BOOL: { - button->set_text(((bool)default_value) ? "true" : "false"); - } break; - case Variant::INT: - case Variant::FLOAT: { - button->set_text(String::num(default_value, 4)); - } break; - case Variant::VECTOR3: { - Vector3 v = default_value; - button->set_text(String::num(v.x, 3) + "," + String::num(v.y, 3) + "," + String::num(v.z, 3)); - } break; - default: { - } - } - } - - if (i == 0 && custom_editor) { - hb->add_child(custom_editor); - custom_editor->set_h_size_flags(SIZE_EXPAND_FILL); - } else { - if (valid_left) { - if (is_group) { - OptionButton *type_box = memnew(OptionButton); - hb->add_child(type_box); - type_box->add_item(TTR("Float")); - type_box->add_item(TTR("Int")); - type_box->add_item(TTR("Vector")); - type_box->add_item(TTR("Boolean")); - type_box->add_item(TTR("Transform")); - type_box->add_item(TTR("Sampler")); - type_box->select(group_node->get_input_port_type(i)); - type_box->set_custom_minimum_size(Size2(100 * EDSCALE, 0)); - type_box->connect("item_selected", callable_mp(this, &VisualShaderEditor::_change_input_port_type), varray(nodes[n_i], i), CONNECT_DEFERRED); - - LineEdit *name_box = memnew(LineEdit); - hb->add_child(name_box); - name_box->set_custom_minimum_size(Size2(65 * EDSCALE, 0)); - name_box->set_h_size_flags(SIZE_EXPAND_FILL); - name_box->set_text(name_left); - name_box->connect("text_entered", callable_mp(this, &VisualShaderEditor::_change_input_port_name), varray(name_box, nodes[n_i], i)); - name_box->connect("focus_exited", callable_mp(this, &VisualShaderEditor::_port_name_focus_out), varray(name_box, nodes[n_i], i, false)); - - Button *remove_btn = memnew(Button); - remove_btn->set_icon(EditorNode::get_singleton()->get_gui_base()->get_theme_icon("Remove", "EditorIcons")); - remove_btn->set_tooltip(TTR("Remove") + " " + name_left); - remove_btn->connect("pressed", callable_mp(this, &VisualShaderEditor::_remove_input_port), varray(nodes[n_i], i), CONNECT_DEFERRED); - hb->add_child(remove_btn); - } else { - Label *label = memnew(Label); - label->set_text(name_left); - label->add_theme_style_override("normal", label_style); //more compact - hb->add_child(label); - - if (vsnode->get_input_port_default_hint(i) != "" && !port_left_used) { - Label *hint_label = memnew(Label); - hint_label->set_text("[" + vsnode->get_input_port_default_hint(i) + "]"); - hint_label->add_theme_color_override("font_color", get_theme_color("font_color_readonly", "TextEdit")); - hint_label->add_theme_style_override("normal", label_style); - hb->add_child(hint_label); - } - } - } - - if (!is_group) { - hb->add_spacer(); - } +void VisualShaderEditor::_update_uniform_refs(Set<String> &p_deleted_names) { + for (int i = 0; i < VisualShader::TYPE_MAX; i++) { + VisualShader::Type type = VisualShader::Type(i); - if (valid_right) { - if (is_group) { - Button *remove_btn = memnew(Button); - remove_btn->set_icon(EditorNode::get_singleton()->get_gui_base()->get_theme_icon("Remove", "EditorIcons")); - remove_btn->set_tooltip(TTR("Remove") + " " + name_left); - remove_btn->connect("pressed", callable_mp(this, &VisualShaderEditor::_remove_output_port), varray(nodes[n_i], i), CONNECT_DEFERRED); - hb->add_child(remove_btn); - - LineEdit *name_box = memnew(LineEdit); - hb->add_child(name_box); - name_box->set_custom_minimum_size(Size2(65 * EDSCALE, 0)); - name_box->set_h_size_flags(SIZE_EXPAND_FILL); - name_box->set_text(name_right); - name_box->connect("text_entered", callable_mp(this, &VisualShaderEditor::_change_output_port_name), varray(name_box, nodes[n_i], i)); - name_box->connect("focus_exited", callable_mp(this, &VisualShaderEditor::_port_name_focus_out), varray(name_box, nodes[n_i], i, true)); - - OptionButton *type_box = memnew(OptionButton); - hb->add_child(type_box); - type_box->add_item(TTR("Float")); - type_box->add_item(TTR("Int")); - type_box->add_item(TTR("Vector")); - type_box->add_item(TTR("Boolean")); - type_box->add_item(TTR("Transform")); - type_box->select(group_node->get_output_port_type(i)); - type_box->set_custom_minimum_size(Size2(100 * EDSCALE, 0)); - type_box->connect("item_selected", callable_mp(this, &VisualShaderEditor::_change_output_port_type), varray(nodes[n_i], i), CONNECT_DEFERRED); - } else { - Label *label = memnew(Label); - label->set_text(name_right); - label->add_theme_style_override("normal", label_style); //more compact - hb->add_child(label); + Vector<int> nodes = visual_shader->get_node_list(type); + for (int j = 0; j < nodes.size(); j++) { + if (j > 0) { + Ref<VisualShaderNodeUniformRef> ref = visual_shader->get_node(type, nodes[j]); + if (ref.is_valid()) { + if (p_deleted_names.has(ref->get_uniform_name())) { + undo_redo->add_do_method(ref.ptr(), "set_uniform_name", "[None]"); + undo_redo->add_undo_method(ref.ptr(), "set_uniform_name", ref->get_uniform_name()); + undo_redo->add_do_method(graph_plugin.ptr(), "update_node", VisualShader::Type(i), nodes[j]); + undo_redo->add_undo_method(graph_plugin.ptr(), "update_node", VisualShader::Type(i), nodes[j]); } } } - - if (valid_right && edit_type->get_selected() == VisualShader::TYPE_FRAGMENT && port_right != VisualShaderNode::PORT_TYPE_TRANSFORM && port_right != VisualShaderNode::PORT_TYPE_SAMPLER) { - TextureButton *preview = memnew(TextureButton); - preview->set_toggle_mode(true); - preview->set_normal_texture(get_theme_icon("GuiVisibilityHidden", "EditorIcons")); - preview->set_pressed_texture(get_theme_icon("GuiVisibilityVisible", "EditorIcons")); - preview->set_v_size_flags(SIZE_SHRINK_CENTER); - - if (vsnode->get_output_port_for_preview() == i) { - preview->set_pressed(true); - } - - preview->connect("pressed", callable_mp(this, &VisualShaderEditor::_preview_select_port), varray(nodes[n_i], i), CONNECT_DEFERRED); - hb->add_child(preview); - } - - if (is_group) { - offset = memnew(Control); - offset->set_custom_minimum_size(Size2(0, 5 * EDSCALE)); - node->add_child(offset); - port_offset++; - } - - node->add_child(hb); - - node->set_slot(i + port_offset, valid_left, port_left, type_color[port_left], valid_right, port_right, type_color[port_right]); } + } +} - if (vsnode->get_output_port_for_preview() >= 0) { - int port_type = vsnode->get_output_port_type(vsnode->get_output_port_for_preview()); +void VisualShaderEditor::_update_graph() { + if (updating) { + return; + } - if (port_type != VisualShaderNode::PORT_TYPE_TRANSFORM && port_type != VisualShaderNode::PORT_TYPE_SAMPLER) { - offset = memnew(Control); - offset->set_custom_minimum_size(Size2(0, 5 * EDSCALE)); - node->add_child(offset); + if (visual_shader.is_null()) { + return; + } - VisualShaderNodePortPreview *port_preview = memnew(VisualShaderNodePortPreview); - port_preview->setup(visual_shader, type, nodes[n_i], vsnode->get_output_port_for_preview()); - port_preview->set_h_size_flags(SIZE_SHRINK_CENTER); - node->add_child(port_preview); - } - } + graph->set_scroll_ofs(visual_shader->get_graph_offset() * EDSCALE); - offset = memnew(Control); - offset->set_custom_minimum_size(Size2(0, 4 * EDSCALE)); - node->add_child(offset); + VisualShader::Type type = get_current_shader_type(); - String error = vsnode->get_warning(visual_shader->get_mode(), type); - if (error != String()) { - Label *error_label = memnew(Label); - error_label->add_theme_color_override("font_color", get_theme_color("error_color", "Editor")); - error_label->set_text(error); - node->add_child(error_label); + graph->clear_connections(); + //erase all nodes + for (int i = 0; i < graph->get_child_count(); i++) { + if (Object::cast_to<GraphNode>(graph->get_child(i))) { + Node *node = graph->get_child(i); + graph->remove_child(node); + memdelete(node); + i--; } + } - if (is_expression) { - TextEdit *expression_box = memnew(TextEdit); - Ref<CodeHighlighter> expression_syntax_highlighter; - expression_syntax_highlighter.instance(); - expression_node->set_control(expression_box, 0); - node->add_child(expression_box); - - Color background_color = EDITOR_GET("text_editor/highlighting/background_color"); - Color text_color = EDITOR_GET("text_editor/highlighting/text_color"); - Color keyword_color = EDITOR_GET("text_editor/highlighting/keyword_color"); - Color comment_color = EDITOR_GET("text_editor/highlighting/comment_color"); - Color symbol_color = EDITOR_GET("text_editor/highlighting/symbol_color"); - Color function_color = EDITOR_GET("text_editor/highlighting/function_color"); - Color number_color = EDITOR_GET("text_editor/highlighting/number_color"); - Color members_color = EDITOR_GET("text_editor/highlighting/member_variable_color"); - - expression_box->set_syntax_highlighter(expression_syntax_highlighter); - expression_box->add_theme_color_override("background_color", background_color); - - for (List<String>::Element *E = keyword_list.front(); E; E = E->next()) { - expression_syntax_highlighter->add_keyword_color(E->get(), keyword_color); - } + List<VisualShader::Connection> connections; + visual_shader->get_node_connections(type, &connections); + graph_plugin->set_connections(connections); - expression_box->add_theme_font_override("font", get_theme_font("expression", "EditorFonts")); - expression_box->add_theme_color_override("font_color", text_color); - expression_syntax_highlighter->set_number_color(number_color); - expression_syntax_highlighter->set_symbol_color(symbol_color); - expression_syntax_highlighter->set_function_color(function_color); - expression_syntax_highlighter->set_member_variable_color(members_color); - expression_syntax_highlighter->add_color_region("/*", "*/", comment_color, false); - expression_syntax_highlighter->add_color_region("//", "", comment_color, true); + Vector<int> nodes = visual_shader->get_node_list(type); - expression_box->set_text(expression); - expression_box->set_context_menu_enabled(false); - expression_box->set_show_line_numbers(true); + _update_uniforms(false); - expression_box->connect("focus_exited", callable_mp(this, &VisualShaderEditor::_expression_focus_out), varray(expression_box, nodes[n_i])); - } + graph_plugin->clear_links(); + graph_plugin->make_dirty(true); - if (!uniform.is_valid()) { - graph->add_child(node); - _update_created_node(node); - if (is_group) { - call_deferred("_set_node_size", (int)type, nodes[n_i], size); - } - } + for (int n_i = 0; n_i < nodes.size(); n_i++) { + graph_plugin->add_node(type, nodes[n_i]); } + graph_plugin->make_dirty(false); + for (List<VisualShader::Connection>::Element *E = connections.front(); E; E = E->next()) { int from = E->get().from_node; int from_idx = E->get().from_port; @@ -1110,14 +1314,14 @@ void VisualShaderEditor::_remove_output_port(int p_node, int p_port) { undo_redo->commit_action(); } -void VisualShaderEditor::_expression_focus_out(Object *text_edit, int p_node) { +void VisualShaderEditor::_expression_focus_out(Object *code_edit, int p_node) { VisualShader::Type type = get_current_shader_type(); Ref<VisualShaderNodeExpression> node = visual_shader->get_node(type, p_node); if (node.is_null()) { return; } - TextEdit *expression_box = Object::cast_to<TextEdit>(text_edit); + CodeEdit *expression_box = Object::cast_to<CodeEdit>(code_edit); if (node->get_expression() == expression_box->get_text()) { return; @@ -1204,19 +1408,19 @@ void VisualShaderEditor::_preview_select_port(int p_node, int p_port) { if (node.is_null()) { return; } - + int prev_port = node->get_output_port_for_preview(); if (node->get_output_port_for_preview() == p_port) { p_port = -1; //toggle it } - undo_redo->create_action(TTR("Set Uniform Name")); + undo_redo->create_action(p_port == -1 ? TTR("Hide Port Preview") : TTR("Show Port Preview")); undo_redo->add_do_method(node.ptr(), "set_output_port_for_preview", p_port); - undo_redo->add_undo_method(node.ptr(), "set_output_port_for_preview", node->get_output_port_for_preview()); - undo_redo->add_do_method(this, "_update_graph"); - undo_redo->add_undo_method(this, "_update_graph"); + undo_redo->add_undo_method(node.ptr(), "set_output_port_for_preview", prev_port); + undo_redo->add_do_method(graph_plugin.ptr(), "show_port_preview", (int)type, p_node, p_port); + undo_redo->add_undo_method(graph_plugin.ptr(), "show_port_preview", (int)type, p_node, prev_port); undo_redo->commit_action(); } -void VisualShaderEditor::_line_edit_changed(const String &p_text, Object *line_edit, int p_node_id) { +void VisualShaderEditor::_uniform_line_edit_changed(const String &p_text, int p_node_id) { VisualShader::Type type = get_current_shader_type(); Ref<VisualShaderNodeUniform> node = visual_shader->get_node(type, p_node_id); @@ -1224,21 +1428,28 @@ void VisualShaderEditor::_line_edit_changed(const String &p_text, Object *line_e String validated_name = visual_shader->validate_uniform_name(p_text, node); - updating = true; + if (validated_name == node->get_uniform_name()) { + return; + } + undo_redo->create_action(TTR("Set Uniform Name")); undo_redo->add_do_method(node.ptr(), "set_uniform_name", validated_name); undo_redo->add_undo_method(node.ptr(), "set_uniform_name", node->get_uniform_name()); - undo_redo->add_do_method(this, "_update_graph"); - undo_redo->add_undo_method(this, "_update_graph"); - undo_redo->commit_action(); - updating = false; + undo_redo->add_do_method(graph_plugin.ptr(), "set_uniform_name", type, p_node_id, validated_name); + undo_redo->add_undo_method(graph_plugin.ptr(), "set_uniform_name", type, p_node_id, node->get_uniform_name()); + + undo_redo->add_do_method(this, "_update_uniforms", true); + undo_redo->add_undo_method(this, "_update_uniforms", true); + + Set<String> changed_names; + changed_names.insert(node->get_uniform_name()); + _update_uniform_refs(changed_names); - Object::cast_to<LineEdit>(line_edit)->set_text(validated_name); + undo_redo->commit_action(); } -void VisualShaderEditor::_line_edit_focus_out(Object *line_edit, int p_node_id) { - String text = Object::cast_to<LineEdit>(line_edit)->get_text(); - _line_edit_changed(text, line_edit, p_node_id); +void VisualShaderEditor::_uniform_line_edit_focus_out(Object *line_edit, int p_node_id) { + _uniform_line_edit_changed(Object::cast_to<LineEdit>(line_edit)->get_text(), p_node_id); } void VisualShaderEditor::_port_name_focus_out(Object *line_edit, int p_node_id, int p_port_id, bool p_output) { @@ -1302,8 +1513,8 @@ void VisualShaderEditor::_port_edited() { undo_redo->create_action(TTR("Set Input Default Port")); undo_redo->add_do_method(vsn.ptr(), "set_input_port_default_value", editing_port, value); undo_redo->add_undo_method(vsn.ptr(), "set_input_port_default_value", editing_port, vsn->get_input_port_default_value(editing_port)); - undo_redo->add_do_method(this, "_update_graph"); - undo_redo->add_undo_method(this, "_update_graph"); + undo_redo->add_do_method(graph_plugin.ptr(), "set_input_port_default_value", type, editing_node, editing_port, value); + undo_redo->add_undo_method(graph_plugin.ptr(), "set_input_port_default_value", type, editing_node, editing_port, vsn->get_input_port_default_value(editing_port)); undo_redo->commit_action(); property_editor->hide(); @@ -1483,6 +1694,8 @@ VisualShaderNode *VisualShaderEditor::_add_node(int p_idx, int p_op_idx) { undo_redo->create_action(TTR("Add Node to Visual Shader")); undo_redo->add_do_method(visual_shader.ptr(), "add_node", type, vsnode, position, id_to_use); undo_redo->add_undo_method(visual_shader.ptr(), "remove_node", type, id_to_use); + undo_redo->add_do_method(graph_plugin.ptr(), "add_node", type, id_to_use); + undo_redo->add_undo_method(graph_plugin.ptr(), "remove_node", type, id_to_use); VisualShaderNodeExpression *expr = Object::cast_to<VisualShaderNodeExpression>(vsnode.ptr()); if (expr) { @@ -1497,6 +1710,8 @@ VisualShaderNode *VisualShaderEditor::_add_node(int p_idx, int p_op_idx) { if (visual_shader->is_port_types_compatible(vsnode->get_output_port_type(_from_slot), visual_shader->get_node(type, to_node)->get_input_port_type(to_slot))) { undo_redo->add_do_method(visual_shader.ptr(), "connect_nodes", type, _from_node, _from_slot, to_node, to_slot); undo_redo->add_undo_method(visual_shader.ptr(), "disconnect_nodes", type, _from_node, _from_slot, to_node, to_slot); + undo_redo->add_do_method(graph_plugin.ptr(), "connect_nodes", type, _from_node, _from_slot, to_node, to_slot); + undo_redo->add_undo_method(graph_plugin.ptr(), "disconnect_nodes", type, _from_node, _from_slot, to_node, to_slot); } } } else if (from_node != -1 && from_slot != -1) { @@ -1505,14 +1720,20 @@ VisualShaderNode *VisualShaderEditor::_add_node(int p_idx, int p_op_idx) { int _to_slot = 0; if (visual_shader->is_port_types_compatible(visual_shader->get_node(type, from_node)->get_output_port_type(from_slot), vsnode->get_input_port_type(_to_slot))) { - undo_redo->add_do_method(visual_shader.ptr(), "connect_nodes", type, from_node, from_slot, _to_node, _to_slot); undo_redo->add_undo_method(visual_shader.ptr(), "disconnect_nodes", type, from_node, from_slot, _to_node, _to_slot); + undo_redo->add_do_method(visual_shader.ptr(), "connect_nodes", type, from_node, from_slot, _to_node, _to_slot); + undo_redo->add_undo_method(graph_plugin.ptr(), "disconnect_nodes", type, from_node, from_slot, _to_node, _to_slot); + undo_redo->add_do_method(graph_plugin.ptr(), "connect_nodes", type, from_node, from_slot, _to_node, _to_slot); } } } - undo_redo->add_do_method(this, "_update_graph"); - undo_redo->add_undo_method(this, "_update_graph"); + VisualShaderNodeUniform *uniform = Object::cast_to<VisualShaderNodeUniform>(vsnode.ptr()); + if (uniform) { + undo_redo->add_do_method(this, "_update_uniforms", true); + undo_redo->add_undo_method(this, "_update_uniforms", true); + } + undo_redo->commit_action(); return vsnode.ptr(); } @@ -1523,6 +1744,8 @@ void VisualShaderEditor::_node_dragged(const Vector2 &p_from, const Vector2 &p_t undo_redo->create_action(TTR("Node Moved")); undo_redo->add_do_method(visual_shader.ptr(), "set_node_position", type, p_node, p_to); undo_redo->add_undo_method(visual_shader.ptr(), "set_node_position", type, p_node, p_from); + undo_redo->add_do_method(graph_plugin.ptr(), "set_node_position", type, p_node, p_to); + undo_redo->add_undo_method(graph_plugin.ptr(), "set_node_position", type, p_node, p_from); undo_redo->commit_action(); } @@ -1545,13 +1768,15 @@ void VisualShaderEditor::_connection_request(const String &p_from, int p_from_in if (E->get().to_node == to && E->get().to_port == p_to_index) { undo_redo->add_do_method(visual_shader.ptr(), "disconnect_nodes", type, E->get().from_node, E->get().from_port, E->get().to_node, E->get().to_port); undo_redo->add_undo_method(visual_shader.ptr(), "connect_nodes", type, E->get().from_node, E->get().from_port, E->get().to_node, E->get().to_port); + undo_redo->add_do_method(graph_plugin.ptr(), "disconnect_nodes", type, E->get().from_node, E->get().from_port, E->get().to_node, E->get().to_port); + undo_redo->add_undo_method(graph_plugin.ptr(), "connect_nodes", type, E->get().from_node, E->get().from_port, E->get().to_node, E->get().to_port); } } undo_redo->add_do_method(visual_shader.ptr(), "connect_nodes", type, from, p_from_index, to, p_to_index); undo_redo->add_undo_method(visual_shader.ptr(), "disconnect_nodes", type, from, p_from_index, to, p_to_index); - undo_redo->add_do_method(this, "_update_graph"); - undo_redo->add_undo_method(this, "_update_graph"); + undo_redo->add_do_method(graph_plugin.ptr(), "connect_nodes", type, from, p_from_index, to, p_to_index); + undo_redo->add_undo_method(graph_plugin.ptr(), "disconnect_nodes", type, from, p_from_index, to, p_to_index); undo_redo->commit_action(); } @@ -1567,8 +1792,8 @@ void VisualShaderEditor::_disconnection_request(const String &p_from, int p_from undo_redo->create_action(TTR("Nodes Disconnected")); undo_redo->add_do_method(visual_shader.ptr(), "disconnect_nodes", type, from, p_from_index, to, p_to_index); undo_redo->add_undo_method(visual_shader.ptr(), "connect_nodes", type, from, p_from_index, to, p_to_index); - undo_redo->add_do_method(this, "_update_graph"); - undo_redo->add_undo_method(this, "_update_graph"); + undo_redo->add_do_method(graph_plugin.ptr(), "disconnect_nodes", type, from, p_from_index, to, p_to_index); + undo_redo->add_undo_method(graph_plugin.ptr(), "connect_nodes", type, from, p_from_index, to, p_to_index); undo_redo->commit_action(); //updating = false; } @@ -1590,8 +1815,18 @@ void VisualShaderEditor::_delete_request(int which) { Ref<VisualShaderNode> node = Ref<VisualShaderNode>(visual_shader->get_node(type, which)); undo_redo->create_action(TTR("Delete Node")); + + List<VisualShader::Connection> conns; + visual_shader->get_node_connections(type, &conns); + for (List<VisualShader::Connection>::Element *E = conns.front(); E; E = E->next()) { + if (E->get().from_node == which || E->get().to_node == which) { + undo_redo->add_do_method(graph_plugin.ptr(), "disconnect_nodes", type, E->get().from_node, E->get().from_port, E->get().to_node, E->get().to_port); + } + } + undo_redo->add_do_method(visual_shader.ptr(), "remove_node", type, which); undo_redo->add_undo_method(visual_shader.ptr(), "add_node", type, node, visual_shader->get_node_position(type, which), which); + undo_redo->add_undo_method(graph_plugin.ptr(), "add_node", type, which); undo_redo->add_do_method(this, "_clear_buffer"); undo_redo->add_undo_method(this, "_clear_buffer"); @@ -1610,17 +1845,26 @@ void VisualShaderEditor::_delete_request(int which) { undo_redo->add_undo_method(expression, "set_expression", expression->get_expression()); } - List<VisualShader::Connection> conns; - visual_shader->get_node_connections(type, &conns); - for (List<VisualShader::Connection>::Element *E = conns.front(); E; E = E->next()) { if (E->get().from_node == which || E->get().to_node == which) { undo_redo->add_undo_method(visual_shader.ptr(), "connect_nodes", type, E->get().from_node, E->get().from_port, E->get().to_node, E->get().to_port); + undo_redo->add_undo_method(graph_plugin.ptr(), "connect_nodes", type, E->get().from_node, E->get().from_port, E->get().to_node, E->get().to_port); } } + // delete a node from the graph + undo_redo->add_do_method(graph_plugin.ptr(), "remove_node", type, which); + + VisualShaderNodeUniform *uniform = Object::cast_to<VisualShaderNodeUniform>(node.ptr()); + if (uniform) { + undo_redo->add_do_method(this, "_update_uniforms", true); + undo_redo->add_undo_method(this, "_update_uniforms", true); + + Set<String> uniform_names; + uniform_names.insert(uniform->get_uniform_name()); + + _update_uniform_refs(uniform_names); + } - undo_redo->add_do_method(this, "_update_graph"); - undo_redo->add_undo_method(this, "_update_graph"); undo_redo->commit_action(); } @@ -1879,12 +2123,13 @@ void VisualShaderEditor::_dup_paste_nodes(int p_type, int p_pasted_type, List<in Ref<VisualShaderNode> dupli = node->duplicate(); undo_redo->add_do_method(visual_shader.ptr(), "add_node", type, dupli, visual_shader->get_node_position(pasted_type, E->get()) + p_offset, id_from); - undo_redo->add_undo_method(visual_shader.ptr(), "remove_node", type, id_from); + undo_redo->add_do_method(graph_plugin.ptr(), "add_node", type, id_from); // duplicate size, inputs and outputs if node is group Ref<VisualShaderNodeGroupBase> group = Object::cast_to<VisualShaderNodeGroupBase>(node.ptr()); if (!group.is_null()) { undo_redo->add_do_method(dupli.ptr(), "set_size", group->get_size()); + undo_redo->add_do_method(graph_plugin.ptr(), "set_node_size", type, id_from, group->get_size()); undo_redo->add_do_method(dupli.ptr(), "set_inputs", group->get_inputs()); undo_redo->add_do_method(dupli.ptr(), "set_outputs", group->get_outputs()); } @@ -1905,12 +2150,19 @@ void VisualShaderEditor::_dup_paste_nodes(int p_type, int p_pasted_type, List<in continue; } if (connection_remap.has(E->get().from_node) && connection_remap.has(E->get().to_node)) { - undo_redo->add_do_method(visual_shader.ptr(), "connect_nodes_forced", type, connection_remap[E->get().from_node], E->get().from_port, connection_remap[E->get().to_node], E->get().to_port); + undo_redo->add_do_method(visual_shader.ptr(), "connect_nodes", type, connection_remap[E->get().from_node], E->get().from_port, connection_remap[E->get().to_node], E->get().to_port); + undo_redo->add_do_method(graph_plugin.ptr(), "connect_nodes", type, connection_remap[E->get().from_node], E->get().from_port, connection_remap[E->get().to_node], E->get().to_port); + undo_redo->add_undo_method(graph_plugin.ptr(), "disconnect_nodes", type, connection_remap[E->get().from_node], E->get().from_port, connection_remap[E->get().to_node], E->get().to_port); } } - undo_redo->add_do_method(this, "_update_graph"); - undo_redo->add_undo_method(this, "_update_graph"); + id_from = base_id; + for (List<int>::Element *E = r_nodes.front(); E; E = E->next()) { + undo_redo->add_undo_method(visual_shader.ptr(), "remove_node", type, id_from); + undo_redo->add_undo_method(graph_plugin.ptr(), "remove_node", type, id_from); + id_from++; + } + undo_redo->commit_action(); if (p_select) { @@ -2001,11 +2253,25 @@ void VisualShaderEditor::_delete_nodes() { undo_redo->create_action(TTR("Delete Nodes")); + List<VisualShader::Connection> conns; + visual_shader->get_node_connections(type, &conns); + + for (List<int>::Element *F = to_erase.front(); F; F = F->next()) { + for (List<VisualShader::Connection>::Element *E = conns.front(); E; E = E->next()) { + if (E->get().from_node == F->get() || E->get().to_node == F->get()) { + undo_redo->add_do_method(graph_plugin.ptr(), "disconnect_nodes", type, E->get().from_node, E->get().from_port, E->get().to_node, E->get().to_port); + } + } + } + + Set<String> uniform_names; + for (List<int>::Element *F = to_erase.front(); F; F = F->next()) { Ref<VisualShaderNode> node = visual_shader->get_node(type, F->get()); undo_redo->add_do_method(visual_shader.ptr(), "remove_node", type, F->get()); undo_redo->add_undo_method(visual_shader.ptr(), "add_node", type, node, visual_shader->get_node_position(type, F->get()), F->get()); + undo_redo->add_undo_method(graph_plugin.ptr(), "add_node", type, F->get()); undo_redo->add_do_method(this, "_clear_buffer"); undo_redo->add_undo_method(this, "_clear_buffer"); @@ -2023,10 +2289,12 @@ void VisualShaderEditor::_delete_nodes() { if (expression) { undo_redo->add_undo_method(expression, "set_expression", expression->get_expression()); } - } - List<VisualShader::Connection> conns; - visual_shader->get_node_connections(type, &conns); + VisualShaderNodeUniform *uniform = Object::cast_to<VisualShaderNodeUniform>(node.ptr()); + if (uniform) { + uniform_names.insert(uniform->get_uniform_name()); + } + } List<VisualShader::Connection> used_conns; for (List<int>::Element *F = to_erase.front(); F; F = F->next()) { @@ -2041,54 +2309,78 @@ void VisualShaderEditor::_delete_nodes() { } if (!cancel) { undo_redo->add_undo_method(visual_shader.ptr(), "connect_nodes", type, E->get().from_node, E->get().from_port, E->get().to_node, E->get().to_port); + undo_redo->add_undo_method(graph_plugin.ptr(), "connect_nodes", type, E->get().from_node, E->get().from_port, E->get().to_node, E->get().to_port); used_conns.push_back(E->get()); } } } } - undo_redo->add_do_method(this, "_update_graph"); - undo_redo->add_undo_method(this, "_update_graph"); + // delete nodes from the graph + for (List<int>::Element *F = to_erase.front(); F; F = F->next()) { + undo_redo->add_do_method(graph_plugin.ptr(), "remove_node", type, F->get()); + } + + // update uniform refs if any uniform has been deleted + if (uniform_names.size() > 0) { + undo_redo->add_do_method(this, "_update_uniforms", true); + undo_redo->add_undo_method(this, "_update_uniforms", true); + + _update_uniform_refs(uniform_names); + } + undo_redo->commit_action(); } void VisualShaderEditor::_mode_selected(int p_id) { - visual_shader->set_shader_type(VisualShader::Type(p_id)); + visual_shader->set_shader_type(particles_mode ? VisualShader::Type(p_id + 3) : VisualShader::Type(p_id)); _update_options_menu(); _update_graph(); } -void VisualShaderEditor::_input_select_item(Ref<VisualShaderNodeInput> input, String name) { - String prev_name = input->get_input_name(); +void VisualShaderEditor::_input_select_item(Ref<VisualShaderNodeInput> p_input, String p_name) { + String prev_name = p_input->get_input_name(); - if (name == prev_name) { + if (p_name == prev_name) { return; } - bool type_changed = input->get_input_type_by_name(name) != input->get_input_type_by_name(prev_name); + bool type_changed = p_input->get_input_type_by_name(p_name) != p_input->get_input_type_by_name(prev_name); UndoRedo *undo_redo = EditorNode::get_singleton()->get_undo_redo(); undo_redo->create_action(TTR("Visual Shader Input Type Changed")); - undo_redo->add_do_method(input.ptr(), "set_input_name", name); - undo_redo->add_undo_method(input.ptr(), "set_input_name", prev_name); - - if (type_changed) { - //restore connections if type changed - VisualShader::Type type = get_current_shader_type(); - int id = visual_shader->find_node_id(type, input); - List<VisualShader::Connection> conns; - visual_shader->get_node_connections(type, &conns); - for (List<VisualShader::Connection>::Element *E = conns.front(); E; E = E->next()) { - if (E->get().from_node == id) { - undo_redo->add_undo_method(visual_shader.ptr(), "connect_nodes", type, E->get().from_node, E->get().from_port, E->get().to_node, E->get().to_port); + undo_redo->add_do_method(p_input.ptr(), "set_input_name", p_name); + undo_redo->add_undo_method(p_input.ptr(), "set_input_name", prev_name); + + // update output port + for (int type_id = 0; type_id < VisualShader::TYPE_MAX; type_id++) { + VisualShader::Type type = VisualShader::Type(type_id); + int id = visual_shader->find_node_id(type, p_input); + if (id != VisualShader::NODE_ID_INVALID) { + if (type_changed) { + List<VisualShader::Connection> conns; + visual_shader->get_node_connections(type, &conns); + for (List<VisualShader::Connection>::Element *E = conns.front(); E; E = E->next()) { + if (E->get().from_node == id) { + if (visual_shader->is_port_types_compatible(p_input->get_input_type_by_name(p_name), visual_shader->get_node(type, E->get().to_node)->get_input_port_type(E->get().to_port))) { + undo_redo->add_do_method(visual_shader.ptr(), "connect_nodes", type, E->get().from_node, E->get().from_port, E->get().to_node, E->get().to_port); + undo_redo->add_undo_method(visual_shader.ptr(), "connect_nodes", type, E->get().from_node, E->get().from_port, E->get().to_node, E->get().to_port); + continue; + } + undo_redo->add_do_method(visual_shader.ptr(), "disconnect_nodes", type, E->get().from_node, E->get().from_port, E->get().to_node, E->get().to_port); + undo_redo->add_undo_method(visual_shader.ptr(), "connect_nodes", type, E->get().from_node, E->get().from_port, E->get().to_node, E->get().to_port); + undo_redo->add_do_method(graph_plugin.ptr(), "disconnect_nodes", type, E->get().from_node, E->get().from_port, E->get().to_node, E->get().to_port); + undo_redo->add_undo_method(graph_plugin.ptr(), "connect_nodes", type, E->get().from_node, E->get().from_port, E->get().to_node, E->get().to_port); + } + } } + undo_redo->add_do_method(graph_plugin.ptr(), "update_node", type_id, id); + undo_redo->add_undo_method(graph_plugin.ptr(), "update_node", type_id, id); + break; } } - undo_redo->add_do_method(VisualShaderEditor::get_singleton(), "_update_graph"); - undo_redo->add_undo_method(VisualShaderEditor::get_singleton(), "_update_graph"); - undo_redo->commit_action(); } @@ -2107,23 +2399,32 @@ void VisualShaderEditor::_uniform_select_item(Ref<VisualShaderNodeUniformRef> p_ undo_redo->add_do_method(p_uniform_ref.ptr(), "set_uniform_name", p_name); undo_redo->add_undo_method(p_uniform_ref.ptr(), "set_uniform_name", prev_name); - if (type_changed) { - //restore connections if type changed - VisualShader::Type type = get_current_shader_type(); + // update output port + for (int type_id = 0; type_id < VisualShader::TYPE_MAX; type_id++) { + VisualShader::Type type = VisualShader::Type(type_id); int id = visual_shader->find_node_id(type, p_uniform_ref); - List<VisualShader::Connection> conns; - visual_shader->get_node_connections(type, &conns); - for (List<VisualShader::Connection>::Element *E = conns.front(); E; E = E->next()) { - if (E->get().from_node == id) { - undo_redo->add_do_method(visual_shader.ptr(), "disconnect_nodes", type, E->get().from_node, E->get().from_port, E->get().to_node, E->get().to_port); - undo_redo->add_undo_method(visual_shader.ptr(), "connect_nodes", type, E->get().from_node, E->get().from_port, E->get().to_node, E->get().to_port); + if (id != VisualShader::NODE_ID_INVALID) { + if (type_changed) { + List<VisualShader::Connection> conns; + visual_shader->get_node_connections(type, &conns); + for (List<VisualShader::Connection>::Element *E = conns.front(); E; E = E->next()) { + if (E->get().from_node == id) { + if (visual_shader->is_port_types_compatible(p_uniform_ref->get_uniform_type_by_name(p_name), visual_shader->get_node(type, E->get().to_node)->get_input_port_type(E->get().to_port))) { + continue; + } + undo_redo->add_do_method(visual_shader.ptr(), "disconnect_nodes", type, E->get().from_node, E->get().from_port, E->get().to_node, E->get().to_port); + undo_redo->add_undo_method(visual_shader.ptr(), "connect_nodes", type, E->get().from_node, E->get().from_port, E->get().to_node, E->get().to_port); + undo_redo->add_do_method(graph_plugin.ptr(), "disconnect_nodes", type, E->get().from_node, E->get().from_port, E->get().to_node, E->get().to_port); + undo_redo->add_undo_method(graph_plugin.ptr(), "connect_nodes", type, E->get().from_node, E->get().from_port, E->get().to_node, E->get().to_port); + } + } } + undo_redo->add_do_method(graph_plugin.ptr(), "update_node", type_id, id); + undo_redo->add_undo_method(graph_plugin.ptr(), "update_node", type_id, id); + break; } } - undo_redo->add_do_method(VisualShaderEditor::get_singleton(), "_update_graph"); - undo_redo->add_undo_method(VisualShaderEditor::get_singleton(), "_update_graph"); - undo_redo->commit_action(); } @@ -2355,6 +2656,8 @@ void VisualShaderEditor::_bind_methods() { ClassDB::bind_method("_uniform_select_item", &VisualShaderEditor::_uniform_select_item); ClassDB::bind_method("_set_node_size", &VisualShaderEditor::_set_node_size); ClassDB::bind_method("_clear_buffer", &VisualShaderEditor::_clear_buffer); + ClassDB::bind_method("_update_uniforms", &VisualShaderEditor::_update_uniforms); + ClassDB::bind_method("_set_mode", &VisualShaderEditor::_set_mode); ClassDB::bind_method(D_METHOD("get_drag_data_fw"), &VisualShaderEditor::get_drag_data_fw); ClassDB::bind_method(D_METHOD("can_drop_data_fw"), &VisualShaderEditor::can_drop_data_fw); @@ -2476,14 +2779,14 @@ VisualShaderEditor::VisualShaderEditor() { preview_vbox = memnew(VBoxContainer); preview_vbox->set_visible(preview_showed); main_box->add_child(preview_vbox); - preview_text = memnew(TextEdit); + preview_text = memnew(CodeEdit); syntax_highlighter.instance(); preview_vbox->add_child(preview_text); preview_text->set_h_size_flags(SIZE_EXPAND_FILL); preview_text->set_v_size_flags(SIZE_EXPAND_FILL); preview_text->set_custom_minimum_size(Size2(400 * EDSCALE, 0)); preview_text->set_syntax_highlighter(syntax_highlighter); - preview_text->set_show_line_numbers(true); + preview_text->set_draw_line_numbers(true); preview_text->set_readonly(true); error_text = memnew(Label); @@ -2900,12 +3203,15 @@ VisualShaderEditor::VisualShaderEditor() { add_options.push_back(AddOption("CubeMap", "Textures", "Functions", "VisualShaderNodeCubemap", TTR("Perform the cubic texture lookup."), -1, -1)); texture_node_option_idx = add_options.size(); - add_options.push_back(AddOption("Texture2D", "Textures", "Functions", "VisualShaderNodeTexture", TTR("Perform the texture lookup."), -1, -1)); - add_options.push_back(AddOption("CubeMapUniform", "Textures", "Variables", "VisualShaderNodeCubemapUniform", TTR("Cubic texture uniform lookup."), -1, -1)); + add_options.push_back(AddOption("Texture2D", "Textures", "Functions", "VisualShaderNodeTexture", TTR("Perform the 2D texture lookup."), -1, -1)); add_options.push_back(AddOption("Texture2DArray", "Textures", "Functions", "VisualShaderNodeTexture2DArray", TTR("Perform the 2D-array texture lookup."), -1, -1, -1, -1, -1)); + add_options.push_back(AddOption("Texture3D", "Textures", "Functions", "VisualShaderNodeTexture3D", TTR("Perform the 3D texture lookup."), -1, -1)); + + add_options.push_back(AddOption("CubeMapUniform", "Textures", "Variables", "VisualShaderNodeCubemapUniform", TTR("Cubic texture uniform lookup."), -1, -1)); add_options.push_back(AddOption("TextureUniform", "Textures", "Variables", "VisualShaderNodeTextureUniform", TTR("2D texture uniform lookup."), -1, -1)); add_options.push_back(AddOption("TextureUniformTriplanar", "Textures", "Variables", "VisualShaderNodeTextureUniformTriplanar", TTR("2D texture uniform lookup with triplanar."), -1, -1, TYPE_FLAGS_FRAGMENT | TYPE_FLAGS_LIGHT, Shader::MODE_SPATIAL)); add_options.push_back(AddOption("Texture2DArrayUniform", "Textures", "Variables", "VisualShaderNodeTexture2DArrayUniform", TTR("2D array of textures uniform lookup."), -1, -1, -1, -1, -1)); + add_options.push_back(AddOption("Texture3DUniform", "Textures", "Variables", "VisualShaderNodeTexture3DUniform", TTR("3D texture uniform lookup."), -1, -1, -1, -1, -1)); // TRANSFORM @@ -3030,12 +3336,16 @@ VisualShaderEditor::VisualShaderEditor() { default_plugin.instance(); add_plugin(default_plugin); + graph_plugin.instance(); + property_editor = memnew(CustomPropertyEditor); add_child(property_editor); property_editor->connect("variant_changed", callable_mp(this, &VisualShaderEditor::_port_edited)); } +///////////////// + void VisualShaderEditorPlugin::edit(Object *p_object) { visual_shader_editor->edit(Object::cast_to<VisualShader>(p_object)); } @@ -3170,6 +3480,8 @@ public: class VisualShaderNodePluginDefaultEditor : public VBoxContainer { GDCLASS(VisualShaderNodePluginDefaultEditor, VBoxContainer); Ref<Resource> parent_resource; + int node_id; + VisualShader::Type shader_type; public: void _property_changed(const String &p_property, const Variant &p_value, const String &p_field = "", bool p_changing = false) { @@ -3198,8 +3510,10 @@ public: } else { undo_redo->add_undo_method(this, "_open_inspector", (RES)parent_resource.ptr()); } - undo_redo->add_do_method(this, "_refresh_request"); - undo_redo->add_undo_method(this, "_refresh_request"); + } + if (p_property != "constant") { + undo_redo->add_do_method(VisualShaderEditor::get_singleton()->get_graph_plugin(), "update_node_deferred", shader_type, node_id); + undo_redo->add_undo_method(VisualShaderEditor::get_singleton()->get_graph_plugin(), "update_node_deferred", shader_type, node_id); } undo_redo->commit_action(); @@ -3215,10 +3529,6 @@ public: } } - void _refresh_request() { - VisualShaderEditor::get_singleton()->call_deferred("_update_graph"); - } - void _resource_selected(const String &p_path, RES p_resource) { _open_inspector(p_resource); } @@ -3244,6 +3554,9 @@ public: node = p_node; properties = p_properties; + node_id = (int)p_node->get_meta("id"); + shader_type = VisualShader::Type((int)p_node->get_meta("shader_type")); + for (int i = 0; i < p_properties.size(); i++) { HBoxContainer *hbox = memnew(HBoxContainer); hbox->set_h_size_flags(SIZE_EXPAND_FILL); @@ -3271,11 +3584,9 @@ public: properties[i]->set_name_split_ratio(0); } node->connect("changed", callable_mp(this, &VisualShaderNodePluginDefaultEditor::_node_changed)); - node->connect("editor_refresh_request", callable_mp(this, &VisualShaderNodePluginDefaultEditor::_refresh_request), varray(), CONNECT_DEFERRED); } static void _bind_methods() { - ClassDB::bind_method("_refresh_request", &VisualShaderNodePluginDefaultEditor::_refresh_request); // Used by UndoRedo. ClassDB::bind_method("_open_inspector", &VisualShaderNodePluginDefaultEditor::_open_inspector); // Used by UndoRedo. ClassDB::bind_method("_show_prop_names", &VisualShaderNodePluginDefaultEditor::_show_prop_names); // Used with call_deferred. } @@ -3364,6 +3675,10 @@ void EditorPropertyShaderMode::_option_selected(int p_which) { //do is easy undo_redo->add_do_method(visual_shader.ptr(), "set_mode", p_which); undo_redo->add_undo_method(visual_shader.ptr(), "set_mode", visual_shader->get_mode()); + + undo_redo->add_do_method(VisualShaderEditor::get_singleton(), "_set_mode", p_which); + undo_redo->add_undo_method(VisualShaderEditor::get_singleton(), "_set_mode", visual_shader->get_mode()); + //now undo is hell //1. restore connections to output diff --git a/editor/plugins/visual_shader_editor_plugin.h b/editor/plugins/visual_shader_editor_plugin.h index 6284c9a530..c7443c67f2 100644 --- a/editor/plugins/visual_shader_editor_plugin.h +++ b/editor/plugins/visual_shader_editor_plugin.h @@ -50,8 +50,72 @@ public: virtual Control *create_editor(const Ref<Resource> &p_parent_resource, const Ref<VisualShaderNode> &p_node); }; +class VisualShaderGraphPlugin : public Reference { + GDCLASS(VisualShaderGraphPlugin, Reference); + +private: + struct InputPort { + Button *default_input_button; + }; + + struct Port { + TextureButton *preview_button; + }; + + struct Link { + VisualShader::Type type; + VisualShaderNode *visual_node; + GraphNode *graph_node; + bool preview_visible; + int preview_pos; + Map<int, InputPort> input_ports; + Map<int, Port> output_ports; + VBoxContainer *preview_box; + LineEdit *uniform_name; + }; + + Ref<VisualShader> visual_shader; + Map<int, Link> links; + List<VisualShader::Connection> connections; + bool dirty = false; + +protected: + static void _bind_methods(); + +public: + void register_shader(VisualShader *p_visual_shader); + void set_connections(List<VisualShader::Connection> &p_connections); + void register_link(VisualShader::Type p_type, int p_id, VisualShaderNode *p_visual_node, GraphNode *p_graph_node); + void register_output_port(int p_id, int p_port, TextureButton *p_button); + void register_uniform_name(int p_id, LineEdit *p_uniform_name); + void clear_links(); + void set_shader_type(VisualShader::Type p_type); + bool is_preview_visible(int p_id) const; + bool is_dirty() const; + void make_dirty(bool p_enabled); + void update_node(VisualShader::Type p_type, int p_id); + void update_node_deferred(VisualShader::Type p_type, int p_node_id); + void add_node(VisualShader::Type p_type, int p_id); + void remove_node(VisualShader::Type p_type, int p_id); + void connect_nodes(VisualShader::Type p_type, int p_from_node, int p_from_port, int p_to_node, int p_to_port); + void disconnect_nodes(VisualShader::Type p_type, int p_from_node, int p_from_port, int p_to_node, int p_to_port); + void show_port_preview(VisualShader::Type p_type, int p_node_id, int p_port_id); + void set_node_position(VisualShader::Type p_type, int p_id, const Vector2 &p_position); + void set_node_size(VisualShader::Type p_type, int p_id, const Vector2 &p_size); + void refresh_node_ports(VisualShader::Type p_type, int p_node); + void set_input_port_default_value(VisualShader::Type p_type, int p_node_id, int p_port_id, Variant p_value); + void register_default_input_button(int p_node_id, int p_port_id, Button *p_button); + void update_uniform_refs(); + void set_uniform_name(VisualShader::Type p_type, int p_node_id, const String &p_name); + VisualShader::Type get_shader_type() const; + + VisualShaderGraphPlugin(); + ~VisualShaderGraphPlugin(); +}; + class VisualShaderEditor : public VBoxContainer { GDCLASS(VisualShaderEditor, VBoxContainer); + friend class VisualShaderGraphPlugin; CustomPropertyEditor *property_editor; int editing_node; @@ -73,7 +137,7 @@ class VisualShaderEditor : public VBoxContainer { bool pending_update_preview; bool shader_error; VBoxContainer *preview_vbox; - TextEdit *preview_text; + CodeEdit *preview_text; Ref<CodeHighlighter> syntax_highlighter; Label *error_text; @@ -181,12 +245,15 @@ class VisualShaderEditor : public VBoxContainer { int custom_node_option_idx; List<String> keyword_list; + List<VisualShaderNodeUniformRef> uniform_refs; + void _draw_color_over_button(Object *obj, Color p_color); void _add_custom_node(const String &p_path); void _add_texture_node(const String &p_path); VisualShaderNode *_add_node(int p_idx, int p_op_idx = -1); void _update_options_menu(); + void _set_mode(int p_which); void _show_preview_text(); void _update_preview(); @@ -221,8 +288,8 @@ class VisualShaderEditor : public VBoxContainer { void _connection_to_empty(const String &p_from, int p_from_slot, const Vector2 &p_release_position); void _connection_from_empty(const String &p_to, int p_to_slot, const Vector2 &p_release_position); - void _line_edit_changed(const String &p_text, Object *line_edit, int p_node_id); - void _line_edit_focus_out(Object *line_edit, int p_node_id); + void _uniform_line_edit_changed(const String &p_text, int p_node_id); + void _uniform_line_edit_focus_out(Object *line_edit, int p_node_id); void _port_name_focus_out(Object *line_edit, int p_node_id, int p_port_id, bool p_output); @@ -242,6 +309,7 @@ class VisualShaderEditor : public VBoxContainer { void _paste_nodes(bool p_use_custom_position = false, const Vector2 &p_custom_position = Vector2()); Vector<Ref<VisualShaderNodePlugin>> plugins; + Ref<VisualShaderGraphPlugin> graph_plugin; void _mode_selected(int p_id); void _rebuild(); @@ -261,7 +329,7 @@ class VisualShaderEditor : public VBoxContainer { void _change_output_port_type(int p_type, int p_node, int p_port); void _change_output_port_name(const String &p_text, Object *line_edit, int p_node, int p_port); - void _expression_focus_out(Object *text_edit, int p_node); + void _expression_focus_out(Object *code_edit, int p_node); void _set_node_size(int p_type, int p_node, const Size2 &p_size); void _node_resized(const Vector2 &p_new_size, int p_type, int p_node); @@ -285,6 +353,8 @@ class VisualShaderEditor : public VBoxContainer { bool _is_available(int p_mode); void _update_created_node(GraphNode *node); + void _update_uniforms(bool p_update_refs); + void _update_uniform_refs(Set<String> &p_names); protected: void _notification(int p_what); @@ -296,6 +366,7 @@ public: void remove_plugin(const Ref<VisualShaderNodePlugin> &p_plugin); static VisualShaderEditor *get_singleton() { return singleton; } + VisualShaderGraphPlugin *get_graph_plugin() { return graph_plugin.ptr(); } void clear_custom_types(); void add_custom_type(const String &p_name, const Ref<Script> &p_script, const String &p_description, int p_return_icon_type, const String &p_category, bool p_highend); diff --git a/editor/settings_config_dialog.cpp b/editor/settings_config_dialog.cpp index 35610ef71b..5da682a148 100644 --- a/editor/settings_config_dialog.cpp +++ b/editor/settings_config_dialog.cpp @@ -202,7 +202,7 @@ void EditorSettingsDialog::_update_shortcuts() { Map<String, TreeItem *> sections; for (List<String>::Element *E = slist.front(); E; E = E->next()) { - Ref<ShortCut> sc = EditorSettings::get_singleton()->get_shortcut(E->get()); + Ref<Shortcut> sc = EditorSettings::get_singleton()->get_shortcut(E->get()); if (!sc->has_meta("original")) { continue; } @@ -268,7 +268,7 @@ void EditorSettingsDialog::_shortcut_button_pressed(Object *p_item, int p_column ERR_FAIL_COND(!ti); String item = ti->get_metadata(0); - Ref<ShortCut> sc = EditorSettings::get_singleton()->get_shortcut(item); + Ref<Shortcut> sc = EditorSettings::get_singleton()->get_shortcut(item); if (p_idx == 0) { press_a_key_label->set_text(TTR("Press a Key...")); @@ -335,7 +335,7 @@ void EditorSettingsDialog::_press_a_key_confirm() { ie->set_alt(last_wait_for_key->get_alt()); ie->set_metakey(last_wait_for_key->get_metakey()); - Ref<ShortCut> sc = EditorSettings::get_singleton()->get_shortcut(shortcut_configured); + Ref<Shortcut> sc = EditorSettings::get_singleton()->get_shortcut(shortcut_configured); undo_redo->create_action(TTR("Change Shortcut") + " '" + shortcut_configured + "'"); undo_redo->add_do_method(sc.ptr(), "set_shortcut", ie); diff --git a/editor/translations/extract.py b/editor/translations/extract.py index 0dafd20eed..93124ec30c 100755 --- a/editor/translations/extract.py +++ b/editor/translations/extract.py @@ -70,7 +70,6 @@ def _write_message(msgctx, msg, msg_plural, location): def _add_additional_location(msgctx, msg, location): global main_po # Add additional location to previous occurrence. - msg_pos = -1 if msgctx != "": msg_pos = main_po.find('\nmsgctxt "' + msgctx + '"\nmsgid "' + msg + '"') else: @@ -86,7 +85,6 @@ def _write_translator_comment(msgctx, msg, translator_comment): return global main_po - msg_pos = -1 if msgctx != "": msg_pos = main_po.find('\nmsgctxt "' + msgctx + '"\nmsgid "' + msg + '"') else: diff --git a/glsl_builders.py b/glsl_builders.py index af9afcae70..29971fd4e7 100644 --- a/glsl_builders.py +++ b/glsl_builders.py @@ -96,8 +96,6 @@ def build_rd_header(filename): out_file = filename + ".gen.h" fd = open(out_file, "w") - enum_constants = [] - fd.write("/* WARNING, THIS FILE WAS GENERATED, DO NOT EDIT */\n") out_file_base = out_file @@ -165,7 +163,6 @@ class RAWHeaderStruct: def include_file_in_raw_header(filename, header_data, depth): fs = open(filename, "r") line = fs.readline() - text = "" while line: @@ -192,8 +189,6 @@ def build_raw_header(filename): out_file = filename + ".gen.h" fd = open(out_file, "w") - enum_constants = [] - fd.write("/* WARNING, THIS FILE WAS GENERATED, DO NOT EDIT */\n") out_file_base = out_file.replace(".glsl.gen.h", "_shader_glsl") @@ -211,11 +206,6 @@ def build_raw_header(filename): fd.close() -def build_rd_headers(target, source, env): - for x in source: - build_rd_header(str(x)) - - def build_raw_headers(target, source, env): for x in source: build_raw_header(str(x)) diff --git a/main/main_builders.py b/main/main_builders.py index 2ea774e3b4..aa91201c3e 100644 --- a/main/main_builders.py +++ b/main/main_builders.py @@ -4,7 +4,6 @@ All such functions are invoked in a subprocess on Windows to prevent build flaki """ from platform_methods import subprocess_main -from collections import OrderedDict def make_splash(target, source, env): diff --git a/main/splash_editor.png b/main/splash_editor.png Binary files differindex 29931a71db..855646b346 100644 --- a/main/splash_editor.png +++ b/main/splash_editor.png diff --git a/methods.py b/methods.py index 8f14de6dac..555d9fa594 100644 --- a/methods.py +++ b/methods.py @@ -534,13 +534,28 @@ def generate_vs_project(env, num_jobs): common_build_prefix = [ 'cmd /V /C set "plat=$(PlatformTarget)"', '(if "$(PlatformTarget)"=="x64" (set "plat=x86_amd64"))', - 'set "tools=yes"', + 'set "tools=%s"' % env["tools"], '(if "$(Configuration)"=="release" (set "tools=no"))', - 'set "custom_modules=%s"' % env["custom_modules"], 'call "' + batch_file + '" !plat!', ] - result = " ^& ".join(common_build_prefix + [commands]) + # windows allows us to have spaces in paths, so we need + # to double quote off the directory. However, the path ends + # in a backslash, so we need to remove this, lest it escape the + # last double quote off, confusing MSBuild + common_build_postfix = [ + "--directory=\"$(ProjectDir.TrimEnd('\\'))\"", + "platform=windows", + "target=$(Configuration)", + "progress=no", + "tools=!tools!", + "-j%s" % num_jobs, + ] + + if env["custom_modules"]: + common_build_postfix.append("custom_modules=%s" % env["custom_modules"]) + + result = " ^& ".join(common_build_prefix + [" ".join([commands] + common_build_postfix)]) return result env.AddToVSProject(env.core_sources) @@ -550,22 +565,9 @@ def generate_vs_project(env, num_jobs): env.AddToVSProject(env.servers_sources) env.AddToVSProject(env.editor_sources) - # windows allows us to have spaces in paths, so we need - # to double quote off the directory. However, the path ends - # in a backslash, so we need to remove this, lest it escape the - # last double quote off, confusing MSBuild - env["MSVSBUILDCOM"] = build_commandline( - "scons --directory=\"$(ProjectDir.TrimEnd('\\'))\" platform=windows progress=no target=$(Configuration)" - " tools=!tools! custom_modules=!custom_modules! -j" + str(num_jobs) - ) - env["MSVSREBUILDCOM"] = build_commandline( - "scons --directory=\"$(ProjectDir.TrimEnd('\\'))\" platform=windows progress=no target=$(Configuration)" - " tools=!tools! custom_modules=!custom_modules! vsproj=yes -j" + str(num_jobs) - ) - env["MSVSCLEANCOM"] = build_commandline( - "scons --directory=\"$(ProjectDir.TrimEnd('\\'))\" --clean platform=windows progress=no" - " target=$(Configuration) tools=!tools! custom_modules=!custom_modules! -j" + str(num_jobs) - ) + env["MSVSBUILDCOM"] = build_commandline("scons") + env["MSVSREBUILDCOM"] = build_commandline("scons vsproj=yes") + env["MSVSCLEANCOM"] = build_commandline("scons --clean") # This version information (Win32, x64, Debug, Release, Release_Debug seems to be # required for Visual Studio to understand that it needs to generate an NMAKE diff --git a/misc/dist/html/fixed-size.html b/misc/dist/html/fixed-size.html index a5633115d5..85064b34fd 100644 --- a/misc/dist/html/fixed-size.html +++ b/misc/dist/html/fixed-size.html @@ -236,7 +236,6 @@ $GODOT_HEAD_INCLUDE const DEBUG_ENABLED = $GODOT_DEBUG_ENABLED; const INDETERMINATE_STATUS_STEP_MS = 100; - var container = document.getElementById('container'); var canvas = document.getElementById('canvas'); var statusProgress = document.getElementById('status-progress'); var statusProgressInner = document.getElementById('status-progress-inner'); diff --git a/misc/dist/html/full-size.html b/misc/dist/html/full-size.html index 435013cb5e..be25ce4839 100644 --- a/misc/dist/html/full-size.html +++ b/misc/dist/html/full-size.html @@ -201,7 +201,7 @@ $GODOT_HEAD_INCLUDE throw new Error('Invalid status mode'); } statusMode = mode; - } + }; function animateStatusIndeterminate(ms) { diff --git a/modules/bullet/bullet_types_converter.cpp b/modules/bullet/bullet_types_converter.cpp index c9493d8892..7ecad9b78a 100644 --- a/modules/bullet/bullet_types_converter.cpp +++ b/modules/bullet/bullet_types_converter.cpp @@ -95,12 +95,61 @@ void G_TO_B(Transform const &inVal, btTransform &outVal) { } void UNSCALE_BT_BASIS(btTransform &scaledBasis) { - btMatrix3x3 &m(scaledBasis.getBasis()); - btVector3 column0(m[0][0], m[1][0], m[2][0]); - btVector3 column1(m[0][1], m[1][1], m[2][1]); - btVector3 column2(m[0][2], m[1][2], m[2][2]); + btMatrix3x3 &basis(scaledBasis.getBasis()); + btVector3 column0 = basis.getColumn(0); + btVector3 column1 = basis.getColumn(1); + btVector3 column2 = basis.getColumn(2); + + // Check for zero scaling. + if (btFuzzyZero(column0[0])) { + if (btFuzzyZero(column1[1])) { + if (btFuzzyZero(column2[2])) { + // All dimensions are fuzzy zero. Create a default basis. + column0 = btVector3(1, 0, 0); + column1 = btVector3(0, 1, 0); + column2 = btVector3(0, 0, 1); + } else { // Column 2 scale not fuzzy zero. + // Create two vectors orthogonal to row 2. + // Ensure that a default basis is created if row 2 = <0, 0, 1> + column1 = btVector3(0, column2[2], -column2[1]); + column0 = column1.cross(column2); + } + } else { // Column 1 scale not fuzzy zero. + if (btFuzzyZero(column2[2])) { + // Create two vectors othogonal to column 1. + // Ensure that a default basis is created if column 1 = <0, 1, 0> + column0 = btVector3(column1[1], -column1[0], 0); + column2 = column0.cross(column1); + } else { // Column 1 and column 2 scales not fuzzy zero. + // Create column 0 orthogonal to column 1 and column 2. + column0 = column1.cross(column2); + } + } + } else { // Column 0 scale not fuzzy zero. + if (btFuzzyZero(column1[1])) { + if (btFuzzyZero(column2[2])) { + // Create two vectors orthogonal to column 0. + // Ensure that a default basis is created if column 0 = <1, 0, 0> + column2 = btVector3(-column0[2], 0, column0[0]); + column1 = column2.cross(column0); + } else { // Column 0 and column 2 scales not fuzzy zero. + // Create column 1 orthogonal to column 0 and column 2. + column1 = column2.cross(column0); + } + } else { // Column 0 and column 1 scales not fuzzy zero. + if (btFuzzyZero(column2[2])) { + // Create column 2 orthogonal to column 0 and column 1. + column2 = column0.cross(column1); + } + } + } + + // Normalize column0.normalize(); column1.normalize(); column2.normalize(); - m.setValue(column0[0], column1[0], column2[0], column0[1], column1[1], column2[1], column0[2], column1[2], column2[2]); + + basis.setValue(column0[0], column1[0], column2[0], + column0[1], column1[1], column2[1], + column0[2], column1[2], column2[2]); } diff --git a/modules/bullet/rigid_body_bullet.cpp b/modules/bullet/rigid_body_bullet.cpp index 32c3240a35..f517eecf64 100644 --- a/modules/bullet/rigid_body_bullet.cpp +++ b/modules/bullet/rigid_body_bullet.cpp @@ -849,8 +849,8 @@ void RigidBodyBullet::on_enter_area(AreaBullet *p_area) { } else { if (areasWhereIam[i]->get_spOv_priority() > p_area->get_spOv_priority()) { // The position was found, just shift all elements - for (int j = i; j < areaWhereIamCount; ++j) { - areasWhereIam[j + 1] = areasWhereIam[j]; + for (int j = areaWhereIamCount; j > i; j--) { + areasWhereIam[j] = areasWhereIam[j - 1]; } areasWhereIam[i] = p_area; break; diff --git a/modules/bullet/shape_bullet.cpp b/modules/bullet/shape_bullet.cpp index 274493ed17..74d6e073b3 100644 --- a/modules/bullet/shape_bullet.cpp +++ b/modules/bullet/shape_bullet.cpp @@ -308,7 +308,7 @@ void CapsuleShapeBullet::setup(real_t p_height, real_t p_radius) { } btCollisionShape *CapsuleShapeBullet::internal_create_bt_shape(const btVector3 &p_implicit_scale, real_t p_extra_edge) { - return prepare(ShapeBullet::create_shape_capsule(radius * p_implicit_scale[0] + p_extra_edge, height * p_implicit_scale[1] + p_extra_edge)); + return prepare(ShapeBullet::create_shape_capsule(radius * p_implicit_scale[0] + p_extra_edge, height * p_implicit_scale[1])); } /* Cylinder */ diff --git a/modules/denoise/resource_to_cpp.py b/modules/denoise/resource_to_cpp.py index 4c0b67f701..6c83277355 100644 --- a/modules/denoise/resource_to_cpp.py +++ b/modules/denoise/resource_to_cpp.py @@ -17,8 +17,6 @@ ## ======================================================================== ## import os -import sys -import argparse from array import array # Generates a C++ file from the specified binary resource file diff --git a/tests/test_gdnative_string.h b/modules/gdnative/tests/test_string.h index 76868ba842..aeb855a1c4 100644 --- a/tests/test_gdnative_string.h +++ b/modules/gdnative/tests/test_string.h @@ -1,5 +1,5 @@ /*************************************************************************/ -/* test_gdnative_string.h */ +/* test_string.h */ /*************************************************************************/ /* This file is part of: */ /* GODOT ENGINE */ @@ -28,14 +28,11 @@ /* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ /*************************************************************************/ -#ifndef TEST_GDNATVIE_STRING_H -#define TEST_GDNATVIE_STRING_H +#ifndef TEST_GDNATIVE_STRING_H +#define TEST_GDNATIVE_STRING_H namespace TestGDNativeString { -#include "modules/modules_enabled.gen.h" -#ifdef MODULE_GDNATIVE_ENABLED - #include "gdnative/string.h" #include "tests/test_macros.h" @@ -46,7 +43,7 @@ int u32scmp(const char32_t *l, const char32_t *r) { return *l - *r; } -TEST_CASE("[GDNatvie String] Construct from Latin-1 char string") { +TEST_CASE("[GDNative String] Construct from Latin-1 char string") { godot_string s; godot_string_new_with_latin1_chars(&s, "Hello"); @@ -58,7 +55,7 @@ TEST_CASE("[GDNatvie String] Construct from Latin-1 char string") { godot_string_destroy(&s); } -TEST_CASE("[GDNatvie String] Construct from wchar_t string") { +TEST_CASE("[GDNative String] Construct from wchar_t string") { godot_string s; godot_string_new_with_wide_chars(&s, L"Give me"); @@ -70,7 +67,7 @@ TEST_CASE("[GDNatvie String] Construct from wchar_t string") { godot_string_destroy(&s); } -TEST_CASE("[GDNatvie String] Construct from UTF-8 char string") { +TEST_CASE("[GDNative String] Construct from UTF-8 char string") { static const char32_t u32str[] = { 0x0045, 0x0020, 0x304A, 0x360F, 0x3088, 0x3046, 0x1F3A4, 0 }; static const char32_t u32str_short[] = { 0x0045, 0x0020, 0x304A, 0 }; static const uint8_t u8str[] = { 0x45, 0x20, 0xE3, 0x81, 0x8A, 0xE3, 0x98, 0x8F, 0xE3, 0x82, 0x88, 0xE3, 0x81, 0x86, 0xF0, 0x9F, 0x8E, 0xA4, 0 }; @@ -100,7 +97,7 @@ TEST_CASE("[GDNatvie String] Construct from UTF-8 char string") { godot_char_string_destroy(&cs); } -TEST_CASE("[GDNatvie String] Construct from UTF-8 string with BOM") { +TEST_CASE("[GDNative String] Construct from UTF-8 string with BOM") { static const char32_t u32str[] = { 0x0045, 0x0020, 0x304A, 0x360F, 0x3088, 0x3046, 0x1F3A4, 0 }; static const char32_t u32str_short[] = { 0x0045, 0x0020, 0x304A, 0 }; static const uint8_t u8str[] = { 0xEF, 0xBB, 0xBF, 0x45, 0x20, 0xE3, 0x81, 0x8A, 0xE3, 0x98, 0x8F, 0xE3, 0x82, 0x88, 0xE3, 0x81, 0x86, 0xF0, 0x9F, 0x8E, 0xA4, 0 }; @@ -116,7 +113,7 @@ TEST_CASE("[GDNatvie String] Construct from UTF-8 string with BOM") { godot_string_destroy(&s); } -TEST_CASE("[GDNatvie String] Construct from UTF-16 string") { +TEST_CASE("[GDNative String] Construct from UTF-16 string") { static const char32_t u32str[] = { 0x0045, 0x0020, 0x1F3A4, 0x360F, 0x3088, 0x3046, 0x1F3A4, 0 }; static const char32_t u32str_short[] = { 0x0045, 0x0020, 0x1F3A4, 0 }; static const char16_t u16str[] = { 0x0045, 0x0020, 0xD83C, 0xDFA4, 0x360F, 0x3088, 0x3046, 0xD83C, 0xDFA4, 0 }; @@ -146,7 +143,7 @@ TEST_CASE("[GDNatvie String] Construct from UTF-16 string") { godot_char16_string_destroy(&cs); } -TEST_CASE("[GDNatvie String] Construct from UTF-16 string with BOM ") { +TEST_CASE("[GDNative String] Construct from UTF-16 string with BOM ") { static const char32_t u32str[] = { 0x0045, 0x0020, 0x1F3A4, 0x360F, 0x3088, 0x3046, 0x1F3A4, 0 }; static const char32_t u32str_short[] = { 0x0045, 0x0020, 0x1F3A4, 0 }; static const char16_t u16str[] = { 0xFEFF, 0x0045, 0x0020, 0xD83C, 0xDFA4, 0x360F, 0x3088, 0x3046, 0xD83C, 0xDFA4, 0 }; @@ -171,7 +168,7 @@ TEST_CASE("[GDNatvie String] Construct from UTF-16 string with BOM ") { godot_string_destroy(&s); } -TEST_CASE("[GDNatvie String] Construct string copy") { +TEST_CASE("[GDNative String] Construct string copy") { godot_string s, t; godot_string_new_with_latin1_chars(&s, "Hello"); @@ -181,7 +178,7 @@ TEST_CASE("[GDNatvie String] Construct string copy") { godot_string_destroy(&s); } -TEST_CASE("[GDNatvie String] Construct empty string") { +TEST_CASE("[GDNative String] Construct empty string") { godot_string s; godot_string_new(&s); @@ -189,7 +186,7 @@ TEST_CASE("[GDNatvie String] Construct empty string") { godot_string_destroy(&s); } -TEST_CASE("[GDNatvie String] ASCII/Latin-1") { +TEST_CASE("[GDNative String] ASCII/Latin-1") { godot_string s; godot_string_new_with_utf32_chars(&s, U"Primero Leche"); @@ -204,7 +201,7 @@ TEST_CASE("[GDNatvie String] ASCII/Latin-1") { godot_string_destroy(&s); } -TEST_CASE("[GDNatvie String] Comparisons (equal)") { +TEST_CASE("[GDNative String] Comparisons (equal)") { godot_string s, t; godot_string_new_with_latin1_chars(&s, "Test Compare"); @@ -214,7 +211,7 @@ TEST_CASE("[GDNatvie String] Comparisons (equal)") { godot_string_destroy(&t); } -TEST_CASE("[GDNatvie String] Comparisons (operator <)") { +TEST_CASE("[GDNative String] Comparisons (operator <)") { godot_string s, t; godot_string_new_with_latin1_chars(&s, "Bees"); @@ -234,7 +231,7 @@ TEST_CASE("[GDNatvie String] Comparisons (operator <)") { godot_string_destroy(&s); } -TEST_CASE("[GDNatvie String] Concatenation (operator +)") { +TEST_CASE("[GDNative String] Concatenation (operator +)") { godot_string s, t, x; godot_string_new_with_latin1_chars(&s, "Hel"); @@ -246,7 +243,7 @@ TEST_CASE("[GDNatvie String] Concatenation (operator +)") { godot_string_destroy(&t); } -TEST_CASE("[GDNatvie String] Testing size and length of string") { +TEST_CASE("[GDNative String] Testing size and length of string") { godot_string s; godot_string_new_with_latin1_chars(&s, "Mellon"); @@ -258,7 +255,7 @@ TEST_CASE("[GDNatvie String] Testing size and length of string") { godot_string_destroy(&s); } -TEST_CASE("[GDNatvie String] Testing for empty string") { +TEST_CASE("[GDNative String] Testing for empty string") { godot_string s; godot_string_new_with_latin1_chars(&s, "Mellon"); @@ -270,7 +267,7 @@ TEST_CASE("[GDNatvie String] Testing for empty string") { godot_string_destroy(&s); } -TEST_CASE("[GDNatvie String] Test chr") { +TEST_CASE("[GDNative String] Test chr") { godot_string s; s = godot_string_chr('H'); @@ -292,7 +289,7 @@ TEST_CASE("[GDNatvie String] Test chr") { ERR_PRINT_ON } -TEST_CASE("[GDNatvie String] Operator []") { +TEST_CASE("[GDNative String] Operator []") { godot_string s; godot_string_new_with_latin1_chars(&s, "Hello"); @@ -302,7 +299,7 @@ TEST_CASE("[GDNatvie String] Operator []") { godot_string_destroy(&s); } -TEST_CASE("[GDNatvie String] Case function test") { +TEST_CASE("[GDNative String] Case function test") { godot_string s, t; godot_string_new_with_latin1_chars(&s, "MoMoNgA"); @@ -318,7 +315,7 @@ TEST_CASE("[GDNatvie String] Case function test") { godot_string_destroy(&s); } -TEST_CASE("[GDNatvie String] Case compare function test") { +TEST_CASE("[GDNative String] Case compare function test") { godot_string s, t; godot_string_new_with_latin1_chars(&s, "MoMoNgA"); @@ -331,7 +328,7 @@ TEST_CASE("[GDNatvie String] Case compare function test") { godot_string_destroy(&t); } -TEST_CASE("[GDNatvie String] Natural compare function test") { +TEST_CASE("[GDNative String] Natural compare function test") { godot_string s, t; godot_string_new_with_latin1_chars(&s, "img2.png"); @@ -344,14 +341,14 @@ TEST_CASE("[GDNatvie String] Natural compare function test") { godot_string_destroy(&t); } -TEST_CASE("[GDNatvie String] hex_encode_buffer") { +TEST_CASE("[GDNative String] hex_encode_buffer") { static const uint8_t u8str[] = { 0x45, 0xE3, 0x81, 0x8A, 0x8F, 0xE3 }; godot_string s = godot_string_hex_encode_buffer(u8str, 6); CHECK(u32scmp(godot_string_get_data(&s), U"45e3818a8fe3") == 0); godot_string_destroy(&s); } -TEST_CASE("[GDNatvie String] Substr") { +TEST_CASE("[GDNative String] Substr") { godot_string s, t; godot_string_new_with_latin1_chars(&s, "Killer Baby"); t = godot_string_substr(&s, 3, 4); @@ -360,7 +357,7 @@ TEST_CASE("[GDNatvie String] Substr") { godot_string_destroy(&s); } -TEST_CASE("[GDNatvie String] Find") { +TEST_CASE("[GDNative String] Find") { godot_string s, t; godot_string_new_with_latin1_chars(&s, "Pretty Woman Woman"); @@ -383,7 +380,7 @@ TEST_CASE("[GDNatvie String] Find") { godot_string_destroy(&s); } -TEST_CASE("[GDNatvie String] Find no case") { +TEST_CASE("[GDNative String] Find no case") { godot_string s, t; godot_string_new_with_latin1_chars(&s, "Pretty Whale Whale"); @@ -406,7 +403,7 @@ TEST_CASE("[GDNatvie String] Find no case") { godot_string_destroy(&s); } -TEST_CASE("[GDNatvie String] Find MK") { +TEST_CASE("[GDNative String] Find MK") { godot_packed_string_array keys; godot_packed_string_array_new(&keys); @@ -440,7 +437,7 @@ TEST_CASE("[GDNatvie String] Find MK") { #undef PUSH_KEY } -TEST_CASE("[GDNatvie String] Find and replace") { +TEST_CASE("[GDNative String] Find and replace") { godot_string s, c, w; godot_string_new_with_latin1_chars(&s, "Happy Birthday, Anna!"); godot_string_new_with_latin1_chars(&c, "Birthday"); @@ -462,7 +459,7 @@ TEST_CASE("[GDNatvie String] Find and replace") { godot_string_destroy(&s); } -TEST_CASE("[GDNatvie String] Insertion") { +TEST_CASE("[GDNative String] Insertion") { godot_string s, t, r, u; godot_string_new_with_latin1_chars(&s, "Who is Frederic?"); godot_string_new_with_latin1_chars(&t, "?"); @@ -477,7 +474,7 @@ TEST_CASE("[GDNatvie String] Insertion") { godot_string_destroy(&u); } -TEST_CASE("[GDNatvie String] Number to string") { +TEST_CASE("[GDNative String] Number to string") { godot_string s; s = godot_string_num(3.141593); CHECK(u32scmp(godot_string_get_data(&s), U"3.141593") == 0); @@ -508,7 +505,7 @@ TEST_CASE("[GDNatvie String] Number to string") { godot_string_destroy(&s); } -TEST_CASE("[GDNatvie String] String to integer") { +TEST_CASE("[GDNative String] String to integer") { static const wchar_t *wnums[4] = { L"1237461283", L"- 22", L"0", L" - 1123412" }; static const char *nums[4] = { "1237461283", "- 22", "0", " - 1123412" }; static const int num[4] = { 1237461283, -22, 0, -1123412 }; @@ -524,7 +521,7 @@ TEST_CASE("[GDNatvie String] String to integer") { } } -TEST_CASE("[GDNatvie String] Hex to integer") { +TEST_CASE("[GDNative String] Hex to integer") { static const char *nums[4] = { "0xFFAE", "22", "0", "AADDAD" }; static const int64_t num[4] = { 0xFFAE, 0x22, 0, 0xAADDAD }; static const bool wo_prefix[4] = { false, true, true, true }; @@ -539,7 +536,7 @@ TEST_CASE("[GDNatvie String] Hex to integer") { } } -TEST_CASE("[GDNatvie String] String to float") { +TEST_CASE("[GDNative String] String to float") { static const wchar_t *wnums[4] = { L"-12348298412.2", L"0.05", L"2.0002", L" -0.0001" }; static const char *nums[4] = { "-12348298412.2", "0.05", "2.0002", " -0.0001" }; static const double num[4] = { -12348298412.2, 0.05, 2.0002, -0.0001 }; @@ -555,7 +552,7 @@ TEST_CASE("[GDNatvie String] String to float") { } } -TEST_CASE("[GDNatvie String] CamelCase to underscore") { +TEST_CASE("[GDNative String] CamelCase to underscore") { godot_string s, t; godot_string_new_with_latin1_chars(&s, "TestTestStringGD"); @@ -570,7 +567,7 @@ TEST_CASE("[GDNatvie String] CamelCase to underscore") { godot_string_destroy(&s); } -TEST_CASE("[GDNatvie String] Slicing") { +TEST_CASE("[GDNative String] Slicing") { godot_string s, c; godot_string_new_with_latin1_chars(&s, "Mars,Jupiter,Saturn,Uranus"); godot_string_new_with_latin1_chars(&c, ","); @@ -591,7 +588,7 @@ TEST_CASE("[GDNatvie String] Slicing") { godot_string_destroy(&s); } -TEST_CASE("[GDNatvie String] Splitting") { +TEST_CASE("[GDNative String] Splitting") { godot_string s, c; godot_string_new_with_latin1_chars(&s, "Mars,Jupiter,Saturn,Uranus"); godot_string_new_with_latin1_chars(&c, ","); @@ -682,7 +679,7 @@ TEST_CASE("[GDNatvie String] Splitting") { godot_packed_string_array_destroy(&keys); } -TEST_CASE("[GDNatvie String] Erasing") { +TEST_CASE("[GDNative String] Erasing") { godot_string s, t; godot_string_new_with_latin1_chars(&s, "Josephine is such a cute girl!"); godot_string_new_with_latin1_chars(&t, "cute "); @@ -701,7 +698,7 @@ struct test_27_data { bool expected; }; -TEST_CASE("[GDNatvie String] Begins with") { +TEST_CASE("[GDNative String] Begins with") { test_27_data tc[] = { { "res://foobar", "res://", true }, { "res", "res://", false }, @@ -730,7 +727,7 @@ TEST_CASE("[GDNatvie String] Begins with") { CHECK(state); } -TEST_CASE("[GDNatvie String] Ends with") { +TEST_CASE("[GDNative String] Ends with") { test_27_data tc[] = { { "res://foobar", "foobar", true }, { "res", "res://", false }, @@ -759,7 +756,7 @@ TEST_CASE("[GDNatvie String] Ends with") { CHECK(state); } -TEST_CASE("[GDNatvie String] format") { +TEST_CASE("[GDNative String] format") { godot_string value_format, t; godot_string_new_with_latin1_chars(&value_format, "red=\"$red\" green=\"$green\" blue=\"$blue\" alpha=\"$alpha\""); @@ -813,7 +810,7 @@ TEST_CASE("[GDNatvie String] format") { godot_string_destroy(&value_format); } -TEST_CASE("[GDNatvie String] sprintf") { +TEST_CASE("[GDNative String] sprintf") { //godot_string GDAPI (const godot_string *p_self, const godot_array *p_values, godot_bool *p_error); godot_string format, output; godot_array args; @@ -1215,7 +1212,7 @@ TEST_CASE("[GDNatvie String] sprintf") { #undef ARRAY_PUSH_STRING } -TEST_CASE("[GDNatvie String] is_numeric") { +TEST_CASE("[GDNative String] is_numeric") { #define IS_NUM_TEST(x, r) \ { \ godot_string t; \ @@ -1233,7 +1230,7 @@ TEST_CASE("[GDNatvie String] is_numeric") { #undef IS_NUM_TEST } -TEST_CASE("[GDNatvie String] pad") { +TEST_CASE("[GDNative String] pad") { godot_string s, c; godot_string_new_with_latin1_chars(&s, "test"); godot_string_new_with_latin1_chars(&c, "x"); @@ -1260,7 +1257,7 @@ TEST_CASE("[GDNatvie String] pad") { godot_string_destroy(&s); } -TEST_CASE("[GDNatvie String] is_subsequence_of") { +TEST_CASE("[GDNative String] is_subsequence_of") { godot_string a, t; godot_string_new_with_latin1_chars(&a, "is subsequence of"); @@ -1279,7 +1276,7 @@ TEST_CASE("[GDNatvie String] is_subsequence_of") { godot_string_destroy(&a); } -TEST_CASE("[GDNatvie String] match") { +TEST_CASE("[GDNative String] match") { godot_string s, t; godot_string_new_with_latin1_chars(&s, "*.png"); @@ -1299,7 +1296,7 @@ TEST_CASE("[GDNatvie String] match") { godot_string_destroy(&s); } -TEST_CASE("[GDNatvie String] IPVX address to string") { +TEST_CASE("[GDNative String] IPVX address to string") { godot_string ip; godot_string_new_with_latin1_chars(&ip, "192.168.0.1"); @@ -1331,7 +1328,7 @@ TEST_CASE("[GDNatvie String] IPVX address to string") { godot_string_destroy(&ip); } -TEST_CASE("[GDNatvie String] Capitalize against many strings") { +TEST_CASE("[GDNative String] Capitalize against many strings") { #define CAP_TEST(i, o) \ godot_string_new_with_latin1_chars(&input, i); \ godot_string_new_with_latin1_chars(&output, o); \ @@ -1361,7 +1358,7 @@ TEST_CASE("[GDNatvie String] Capitalize against many strings") { #undef CAP_TEST } -TEST_CASE("[GDNatvie String] lstrip and rstrip") { +TEST_CASE("[GDNative String] lstrip and rstrip") { #define LSTRIP_TEST(x, y, z) \ { \ godot_string xx, yy, zz, rr; \ @@ -1484,7 +1481,7 @@ TEST_CASE("[GDNatvie String] lstrip and rstrip") { #undef RSTRIP_UTF8_TEST } -TEST_CASE("[GDNatvie String] Cyrillic to_lower()") { +TEST_CASE("[GDNative String] Cyrillic to_lower()") { godot_string upper, lower, test; godot_string_new_with_utf8_chars(&upper, "АБВГДЕЁЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯ"); godot_string_new_with_utf8_chars(&lower, "абвгдеёжзийклмнопрстуфхцчшщъыьэюя"); @@ -1498,7 +1495,7 @@ TEST_CASE("[GDNatvie String] Cyrillic to_lower()") { godot_string_destroy(&test); } -TEST_CASE("[GDNatvie String] Count and countn functionality") { +TEST_CASE("[GDNative String] Count and countn functionality") { #define COUNT_TEST(x, y, r) \ { \ godot_string s, t; \ @@ -1570,7 +1567,7 @@ TEST_CASE("[GDNatvie String] Count and countn functionality") { #undef COUNTNR_TEST } -TEST_CASE("[GDNatvie String] Bigrams") { +TEST_CASE("[GDNative String] Bigrams") { godot_string s, t; godot_string_new_with_latin1_chars(&s, "abcd"); godot_packed_string_array bigr = godot_string_bigrams(&s); @@ -1593,7 +1590,7 @@ TEST_CASE("[GDNatvie String] Bigrams") { godot_packed_string_array_destroy(&bigr); } -TEST_CASE("[GDNatvie String] c-escape/unescape") { +TEST_CASE("[GDNative String] c-escape/unescape") { godot_string s; godot_string_new_with_latin1_chars(&s, "\\1\a2\b\f3\n45\r6\t7\v8\'9\?0\""); godot_string t = godot_string_c_escape(&s); @@ -1604,7 +1601,7 @@ TEST_CASE("[GDNatvie String] c-escape/unescape") { godot_string_destroy(&s); } -TEST_CASE("[GDNatvie String] dedent") { +TEST_CASE("[GDNative String] dedent") { godot_string s, t; godot_string_new_with_latin1_chars(&s, " aaa\n bbb"); godot_string_new_with_latin1_chars(&t, "aaa\nbbb"); @@ -1615,7 +1612,7 @@ TEST_CASE("[GDNatvie String] dedent") { godot_string_destroy(&s); } -TEST_CASE("[GDNatvie String] Path functions") { +TEST_CASE("[GDNative String] Path functions") { static const char *path[4] = { "C:\\Godot\\project\\test.tscn", "/Godot/project/test.xscn", "../Godot/project/test.scn", "Godot\\test.doc" }; static const char *base_dir[4] = { "C:\\Godot\\project", "/Godot/project", "../Godot/project", "Godot" }; static const char *base_name[4] = { "C:\\Godot\\project\\test", "/Godot/project/test", "../Godot/project/test", "Godot\\test" }; @@ -1678,7 +1675,7 @@ TEST_CASE("[GDNatvie String] Path functions") { } } -TEST_CASE("[GDNatvie String] hash") { +TEST_CASE("[GDNative String] hash") { godot_string a, b, c; godot_string_new_with_latin1_chars(&a, "Test"); godot_string_new_with_latin1_chars(&b, "Test"); @@ -1694,7 +1691,7 @@ TEST_CASE("[GDNatvie String] hash") { godot_string_destroy(&c); } -TEST_CASE("[GDNatvie String] http_escape/unescape") { +TEST_CASE("[GDNative String] http_escape/unescape") { godot_string s, t, u; godot_string_new_with_latin1_chars(&s, "Godot Engine:'docs'"); godot_string_new_with_latin1_chars(&t, "Godot%20Engine%3A%27docs%27"); @@ -1711,7 +1708,7 @@ TEST_CASE("[GDNatvie String] http_escape/unescape") { godot_string_destroy(&t); } -TEST_CASE("[GDNatvie String] percent_encode/decode") { +TEST_CASE("[GDNative String] percent_encode/decode") { godot_string s, t, u; godot_string_new_with_latin1_chars(&s, "Godot Engine:'docs'"); godot_string_new_with_latin1_chars(&t, "Godot%20Engine%3a%27docs%27"); @@ -1728,7 +1725,7 @@ TEST_CASE("[GDNatvie String] percent_encode/decode") { godot_string_destroy(&t); } -TEST_CASE("[GDNatvie String] xml_escape/unescape") { +TEST_CASE("[GDNative String] xml_escape/unescape") { godot_string s, t, u; godot_string_new_with_latin1_chars(&s, "\"Test\" <test@test&'test'>"); @@ -1747,7 +1744,7 @@ TEST_CASE("[GDNatvie String] xml_escape/unescape") { godot_string_destroy(&s); } -TEST_CASE("[GDNatvie String] Strip escapes") { +TEST_CASE("[GDNative String] Strip escapes") { godot_string s, t, u; godot_string_new_with_latin1_chars(&s, "\t\tTest Test\r\n Test"); godot_string_new_with_latin1_chars(&t, "Test Test Test"); @@ -1760,7 +1757,7 @@ TEST_CASE("[GDNatvie String] Strip escapes") { godot_string_destroy(&s); } -TEST_CASE("[GDNatvie String] Strip edges") { +TEST_CASE("[GDNative String] Strip edges") { godot_string s, t, u; godot_string_new_with_latin1_chars(&s, "\t Test Test "); @@ -1785,7 +1782,7 @@ TEST_CASE("[GDNatvie String] Strip edges") { godot_string_destroy(&s); } -TEST_CASE("[GDNatvie String] Similarity") { +TEST_CASE("[GDNative String] Similarity") { godot_string a, b, c; godot_string_new_with_latin1_chars(&a, "Test"); godot_string_new_with_latin1_chars(&b, "West"); @@ -1798,7 +1795,7 @@ TEST_CASE("[GDNatvie String] Similarity") { godot_string_destroy(&c); } -TEST_CASE("[GDNatvie String] Trim") { +TEST_CASE("[GDNative String] Trim") { godot_string s, t, u, p; godot_string_new_with_latin1_chars(&s, "aaaTestbbb"); @@ -1827,7 +1824,7 @@ TEST_CASE("[GDNatvie String] Trim") { godot_string_destroy(&s); } -TEST_CASE("[GDNatvie String] Right/Left") { +TEST_CASE("[GDNative String] Right/Left") { godot_string s, t, u; godot_string_new_with_latin1_chars(&s, "aaaTestbbb"); // ^ @@ -1847,7 +1844,7 @@ TEST_CASE("[GDNatvie String] Right/Left") { godot_string_destroy(&s); } -TEST_CASE("[GDNatvie String] Repeat") { +TEST_CASE("[GDNative String] Repeat") { godot_string t, u; godot_string_new_with_latin1_chars(&t, "ab"); @@ -1858,7 +1855,7 @@ TEST_CASE("[GDNatvie String] Repeat") { godot_string_destroy(&t); } -TEST_CASE("[GDNatvie String] SHA1/SHA256/MD5") { +TEST_CASE("[GDNative String] SHA1/SHA256/MD5") { godot_string s, t, sha1, sha256, md5; godot_string_new_with_latin1_chars(&s, "Godot"); godot_string_new_with_latin1_chars(&sha1, "a1e91f39b9fce6a9998b14bdbe2aa2b39dc2d201"); @@ -1906,7 +1903,7 @@ TEST_CASE("[GDNatvie String] SHA1/SHA256/MD5") { godot_string_destroy(&md5); } -TEST_CASE("[GDNatvie String] Join") { +TEST_CASE("[GDNative String] Join") { godot_string s, t, u; godot_string_new_with_latin1_chars(&s, ", "); @@ -1932,7 +1929,7 @@ TEST_CASE("[GDNatvie String] Join") { godot_packed_string_array_destroy(&parts); } -TEST_CASE("[GDNatvie String] Is_*") { +TEST_CASE("[GDNative String] Is_*") { static const char *data[12] = { "-30", "100", "10.1", "10,1", "1e2", "1e-2", "1e2e3", "0xAB", "AB", "Test1", "1Test", "Test*1" }; static bool isnum[12] = { true, true, true, false, false, false, false, false, false, false, false, false }; static bool isint[12] = { true, true, false, false, false, false, false, false, false, false, false, false }; @@ -1954,7 +1951,7 @@ TEST_CASE("[GDNatvie String] Is_*") { } } -TEST_CASE("[GDNatvie String] humanize_size") { +TEST_CASE("[GDNative String] humanize_size") { godot_string s; s = godot_string_humanize_size(1000); @@ -1978,8 +1975,6 @@ TEST_CASE("[GDNatvie String] humanize_size") { godot_string_destroy(&s); } -#endif - } // namespace TestGDNativeString -#endif // TEST_GDNATVIE_STRING_H +#endif // TEST_GDNATIVE_STRING_H diff --git a/modules/gdscript/doc_classes/@GDScript.xml b/modules/gdscript/doc_classes/@GDScript.xml index e528fc6623..918581bc9b 100644 --- a/modules/gdscript/doc_classes/@GDScript.xml +++ b/modules/gdscript/doc_classes/@GDScript.xml @@ -369,24 +369,19 @@ <description> Returns the floating-point modulus of [code]a/b[/code] that wraps equally in positive and negative. [codeblock] - var i = -6 - while i < 5: - prints(i, fposmod(i, 3)) - i += 1 + for i in 7: + var x = 0.5 * i - 1.5 + print("%4.1f %4.1f %4.1f" % [x, fmod(x, 1.5), fposmod(x, 1.5)]) [/codeblock] Produces: [codeblock] - -6 0 - -5 1 - -4 2 - -3 0 - -2 1 - -1 2 - 0 0 - 1 1 - 2 2 - 3 0 - 4 1 + -1.5 -0.0 0.0 + -1.0 -1.0 0.5 + -0.5 -0.5 1.0 + 0.0 0.0 0.0 + 0.5 0.5 0.5 + 1.0 1.0 1.0 + 1.5 0.0 0.0 [/codeblock] </description> </method> @@ -771,24 +766,18 @@ <description> Returns the integer modulus of [code]a/b[/code] that wraps equally in positive and negative. [codeblock] - var i = -6 - while i < 5: - prints(i, posmod(i, 3)) - i += 1 + for i in range(-3, 4): + print("%2.0f %2.0f %2.0f" % [i, i % 3, posmod(i, 3)]) [/codeblock] Produces: [codeblock] - -6 0 - -5 1 - -4 2 - -3 0 - -2 1 - -1 2 - 0 0 - 1 1 - 2 2 - 3 0 - 4 1 + -3 0 0 + -2 -2 1 + -1 -1 2 + 0 0 0 + 1 1 1 + 2 2 2 + 3 0 0 [/codeblock] </description> </method> @@ -991,27 +980,15 @@ <description> Returns an array with the given range. Range can be 1 argument N (0 to N-1), two arguments (initial, final-1) or three arguments (initial, final-1, increment). [codeblock] - for i in range(4): - print(i) - for i in range(2, 5): - print(i) - for i in range(0, 6, 2): - print(i) + print(range(4)) + print(range(2, 5)) + print(range(0, 6, 2)) [/codeblock] Output: [codeblock] - 0 - 1 - 2 - 3 - - 2 - 3 - 4 - - 0 - 2 - 4 + [0, 1, 2, 3] + [2, 3, 4] + [0, 2, 4] [/codeblock] </description> </method> diff --git a/modules/gdscript/gdscript.cpp b/modules/gdscript/gdscript.cpp index 0263e32c5b..7f303a966d 100644 --- a/modules/gdscript/gdscript.cpp +++ b/modules/gdscript/gdscript.cpp @@ -1053,7 +1053,9 @@ GDScript::~GDScript() { memdelete(E->get()); } - GDScriptCache::remove_script(get_path()); + if (GDScriptCache::singleton) { // Cache may have been already destroyed at engine shutdown. + GDScriptCache::remove_script(get_path()); + } _save_orphaned_subclasses(); @@ -2035,7 +2037,23 @@ GDScriptLanguage::~GDScriptLanguage() { if (_call_stack) { memdelete_arr(_call_stack); } - singleton = nullptr; + + // Clear dependencies between scripts, to ensure cyclic references are broken (to avoid leaks at exit). + while (script_list.first()) { + GDScript *script = script_list.first()->self(); + for (Map<StringName, GDScriptFunction *>::Element *E = script->member_functions.front(); E; E = E->next()) { + GDScriptFunction *func = E->get(); + for (int i = 0; i < func->argument_types.size(); i++) { + func->argument_types.write[i].script_type_ref = Ref<Script>(); + } + func->return_type.script_type_ref = Ref<Script>(); + } + for (Map<StringName, GDScript::MemberInfo>::Element *E = script->member_indices.front(); E; E = E->next()) { + E->get().data_type.script_type_ref = Ref<Script>(); + } + } + + singleton = NULL; } void GDScriptLanguage::add_orphan_subclass(const String &p_qualified_name, const ObjectID &p_subclass) { diff --git a/modules/gdscript/gdscript_analyzer.cpp b/modules/gdscript/gdscript_analyzer.cpp index b4ede55f0a..943a49060f 100644 --- a/modules/gdscript/gdscript_analyzer.cpp +++ b/modules/gdscript/gdscript_analyzer.cpp @@ -1286,7 +1286,7 @@ void GDScriptAnalyzer::resolve_parameter(GDScriptParser::ParameterNode *p_parame if (p_parameter->default_value != nullptr) { if (!is_type_compatible(result, p_parameter->default_value->get_datatype())) { - push_error(vformat(R"(Type of default value for parameter "%s" (%s) is not compatible with paremeter type (%s).)", p_parameter->identifier->name, p_parameter->default_value->get_datatype().to_string(), p_parameter->datatype_specifier->get_datatype().to_string()), p_parameter->default_value); + push_error(vformat(R"(Type of default value for parameter "%s" (%s) is not compatible with parameter type (%s).)", p_parameter->identifier->name, p_parameter->default_value->get_datatype().to_string(), p_parameter->datatype_specifier->get_datatype().to_string()), p_parameter->default_value); } else if (p_parameter->default_value->get_datatype().is_variant()) { mark_node_unsafe(p_parameter); } diff --git a/modules/gdscript/gdscript_cache.cpp b/modules/gdscript/gdscript_cache.cpp index 992f8f4b58..57b95f5b21 100644 --- a/modules/gdscript/gdscript_cache.cpp +++ b/modules/gdscript/gdscript_cache.cpp @@ -108,20 +108,20 @@ GDScriptParserRef::~GDScriptParserRef() { if (analyzer != nullptr) { memdelete(analyzer); } - MutexLock(GDScriptCache::singleton->lock); + MutexLock lock(GDScriptCache::singleton->lock); GDScriptCache::singleton->parser_map.erase(path); } GDScriptCache *GDScriptCache::singleton = nullptr; void GDScriptCache::remove_script(const String &p_path) { - MutexLock(singleton->lock); + MutexLock lock(singleton->lock); singleton->shallow_gdscript_cache.erase(p_path); singleton->full_gdscript_cache.erase(p_path); } Ref<GDScriptParserRef> GDScriptCache::get_parser(const String &p_path, GDScriptParserRef::Status p_status, Error &r_error, const String &p_owner) { - MutexLock(singleton->lock); + MutexLock lock(singleton->lock); Ref<GDScriptParserRef> ref; if (p_owner != String()) { singleton->dependencies[p_owner].insert(p_path); @@ -168,7 +168,7 @@ String GDScriptCache::get_source_code(const String &p_path) { } Ref<GDScript> GDScriptCache::get_shallow_script(const String &p_path, const String &p_owner) { - MutexLock(singleton->lock); + MutexLock lock(singleton->lock); if (p_owner != String()) { singleton->dependencies[p_owner].insert(p_path); } @@ -190,7 +190,7 @@ Ref<GDScript> GDScriptCache::get_shallow_script(const String &p_path, const Stri } Ref<GDScript> GDScriptCache::get_full_script(const String &p_path, Error &r_error, const String &p_owner) { - MutexLock(singleton->lock); + MutexLock lock(singleton->lock); if (p_owner != String()) { singleton->dependencies[p_owner].insert(p_path); diff --git a/modules/gdscript/gdscript_compiler.cpp b/modules/gdscript/gdscript_compiler.cpp index c3d651ee79..bad450c9f9 100644 --- a/modules/gdscript/gdscript_compiler.cpp +++ b/modules/gdscript/gdscript_compiler.cpp @@ -76,7 +76,7 @@ void GDScriptCompiler::_set_error(const String &p_error, const GDScriptParser::N } } -GDScriptDataType GDScriptCompiler::_gdtype_from_datatype(const GDScriptParser::DataType &p_datatype) const { +GDScriptDataType GDScriptCompiler::_gdtype_from_datatype(const GDScriptParser::DataType &p_datatype, GDScript *p_owner) const { if (!p_datatype.is_set() || !p_datatype.is_hard_type()) { return GDScriptDataType(); } @@ -98,7 +98,7 @@ GDScriptDataType GDScriptCompiler::_gdtype_from_datatype(const GDScriptParser::D } break; case GDScriptParser::DataType::SCRIPT: { result.kind = GDScriptDataType::SCRIPT; - result.script_type = p_datatype.script_type; + result.script_type = Ref<Script>(p_datatype.script_type).ptr(); result.native_type = result.script_type->get_instance_base_type(); } break; case GDScriptParser::DataType::CLASS: { @@ -124,11 +124,11 @@ GDScriptDataType GDScriptCompiler::_gdtype_from_datatype(const GDScriptParser::D names.pop_back(); } result.kind = GDScriptDataType::GDSCRIPT; - result.script_type = script; + result.script_type = script.ptr(); result.native_type = script->get_instance_base_type(); } else { result.kind = GDScriptDataType::GDSCRIPT; - result.script_type = GDScriptCache::get_shallow_script(p_datatype.script_path, main_script->path); + result.script_type = GDScriptCache::get_shallow_script(p_datatype.script_path, main_script->path).ptr(); result.native_type = p_datatype.native_type; } } @@ -149,6 +149,12 @@ GDScriptDataType GDScriptCompiler::_gdtype_from_datatype(const GDScriptParser::D } } + // Only hold strong reference to the script if it's not the owner of the + // element qualified with this type, to avoid cyclic references (leaks). + if (result.script_type && result.script_type != p_owner) { + result.script_type_ref = Ref<Script>(result.script_type); + } + return result; } @@ -1668,7 +1674,7 @@ Error GDScriptCompiler::_parse_function(GDScript *p_script, const GDScriptParser func_name = p_func->identifier->name; is_static = p_func->is_static; rpc_mode = p_func->rpc_mode; - return_type = _gdtype_from_datatype(p_func->get_datatype()); + return_type = _gdtype_from_datatype(p_func->get_datatype(), p_script); } else { if (p_for_ready) { func_name = "_ready"; @@ -1685,7 +1691,7 @@ Error GDScriptCompiler::_parse_function(GDScript *p_script, const GDScriptParser if (p_func) { for (int i = 0; i < p_func->parameters.size(); i++) { const GDScriptParser::ParameterNode *parameter = p_func->parameters[i]; - GDScriptDataType par_type = _gdtype_from_datatype(parameter->get_datatype()); + GDScriptDataType par_type = _gdtype_from_datatype(parameter->get_datatype(), p_script); uint32_t par_addr = codegen.generator->add_parameter(parameter->identifier->name, parameter->default_value != nullptr, par_type); codegen.parameters[parameter->identifier->name] = GDScriptCodeGenerator::Address(GDScriptCodeGenerator::Address::FUNCTION_PARAMETER, par_addr, par_type); @@ -1830,7 +1836,7 @@ Error GDScriptCompiler::_parse_setter_getter(GDScript *p_script, const GDScriptP return_type.kind = GDScriptDataType::BUILTIN; return_type.builtin_type = Variant::NIL; } else { - return_type = _gdtype_from_datatype(p_variable->get_datatype()); + return_type = _gdtype_from_datatype(p_variable->get_datatype(), p_script); } codegen.generator->write_start(p_script, func_name, false, p_variable->rpc_mode, return_type); @@ -1927,7 +1933,7 @@ Error GDScriptCompiler::_parse_class_level(GDScript *p_script, const GDScriptPar p_script->native = native; } break; case GDScriptDataType::GDSCRIPT: { - Ref<GDScript> base = base_type.script_type; + Ref<GDScript> base = Ref<GDScript>(base_type.script_type); p_script->base = base; p_script->_base = base.ptr(); @@ -1994,7 +2000,7 @@ Error GDScriptCompiler::_parse_class_level(GDScript *p_script, const GDScriptPar break; } minfo.rpc_mode = variable->rpc_mode; - minfo.data_type = _gdtype_from_datatype(variable->get_datatype()); + minfo.data_type = _gdtype_from_datatype(variable->get_datatype(), p_script); PropertyInfo prop_info = minfo.data_type; prop_info.name = name; diff --git a/modules/gdscript/gdscript_compiler.h b/modules/gdscript/gdscript_compiler.h index 80fba6a934..db02079d26 100644 --- a/modules/gdscript/gdscript_compiler.h +++ b/modules/gdscript/gdscript_compiler.h @@ -84,7 +84,7 @@ class GDScriptCompiler { Ref<Script> script = obj->get_script(); if (script.is_valid()) { - type.script_type = script; + type.script_type = script.ptr(); Ref<GDScript> gdscript = script; if (gdscript.is_valid()) { type.kind = GDScriptDataType::GDSCRIPT; @@ -125,7 +125,7 @@ class GDScriptCompiler { Error _create_binary_operator(CodeGen &codegen, const GDScriptParser::BinaryOpNode *on, Variant::Operator op, bool p_initializer = false, const GDScriptCodeGenerator::Address &p_index_addr = GDScriptCodeGenerator::Address()); Error _create_binary_operator(CodeGen &codegen, const GDScriptParser::ExpressionNode *p_left_operand, const GDScriptParser::ExpressionNode *p_right_operand, Variant::Operator op, bool p_initializer = false, const GDScriptCodeGenerator::Address &p_index_addr = GDScriptCodeGenerator::Address()); - GDScriptDataType _gdtype_from_datatype(const GDScriptParser::DataType &p_datatype) const; + GDScriptDataType _gdtype_from_datatype(const GDScriptParser::DataType &p_datatype, GDScript *p_owner = nullptr) const; GDScriptCodeGenerator::Address _parse_assign_right_expression(CodeGen &codegen, Error &r_error, const GDScriptParser::AssignmentNode *p_assignmentint, const GDScriptCodeGenerator::Address &p_index_addr = GDScriptCodeGenerator::Address()); GDScriptCodeGenerator::Address _parse_expression(CodeGen &codegen, Error &r_error, const GDScriptParser::ExpressionNode *p_expression, bool p_root = false, bool p_initializer = false, const GDScriptCodeGenerator::Address &p_index_addr = GDScriptCodeGenerator::Address()); diff --git a/modules/gdscript/gdscript_function.h b/modules/gdscript/gdscript_function.h index d1c98a5456..c98ac09310 100644 --- a/modules/gdscript/gdscript_function.h +++ b/modules/gdscript/gdscript_function.h @@ -56,7 +56,8 @@ struct GDScriptDataType { bool has_type = false; Variant::Type builtin_type = Variant::NIL; StringName native_type; - Ref<Script> script_type; + Script *script_type = nullptr; + Ref<Script> script_type_ref; bool is_type(const Variant &p_variant, bool p_allow_implicit_conversion = false) const { if (!has_type) { diff --git a/modules/gdscript/gdscript_parser.cpp b/modules/gdscript/gdscript_parser.cpp index e8dea8180a..2a69db130b 100644 --- a/modules/gdscript/gdscript_parser.cpp +++ b/modules/gdscript/gdscript_parser.cpp @@ -1042,7 +1042,7 @@ GDScriptParser::EnumNode *GDScriptParser::parse_enum() { if (check(GDScriptTokenizer::Token::BRACE_CLOSE)) { break; // Allow trailing comma. } - if (consume(GDScriptTokenizer::Token::IDENTIFIER, R"(Expected identifer for enum key.)")) { + if (consume(GDScriptTokenizer::Token::IDENTIFIER, R"(Expected identifier for enum key.)")) { EnumNode::Value item; item.identifier = parse_identifier(); item.parent_enum = enum_node; @@ -2516,7 +2516,7 @@ GDScriptParser::ExpressionNode *GDScriptParser::parse_call(ExpressionNode *p_pre GDScriptParser::ExpressionNode *GDScriptParser::parse_get_node(ExpressionNode *p_previous_operand, bool p_can_assign) { if (match(GDScriptTokenizer::Token::LITERAL)) { if (previous.literal.get_type() != Variant::STRING) { - push_error(R"(Expect node path as string or identifer after "$".)"); + push_error(R"(Expect node path as string or identifier after "$".)"); return nullptr; } GetNodeNode *get_node = alloc_node<GetNodeNode>(); @@ -2539,7 +2539,7 @@ GDScriptParser::ExpressionNode *GDScriptParser::parse_get_node(ExpressionNode *p } while (match(GDScriptTokenizer::Token::SLASH)); return get_node; } else { - push_error(R"(Expect node path as string or identifer after "$".)"); + push_error(R"(Expect node path as string or identifier after "$".)"); return nullptr; } } diff --git a/modules/gdscript/gdscript_tokenizer.cpp b/modules/gdscript/gdscript_tokenizer.cpp index ed27604aec..9a40aa50ac 100644 --- a/modules/gdscript/gdscript_tokenizer.cpp +++ b/modules/gdscript/gdscript_tokenizer.cpp @@ -578,7 +578,7 @@ GDScriptTokenizer::Token GDScriptTokenizer::potential_identifier() { } void GDScriptTokenizer::newline(bool p_make_token) { - // Don't overwrite previous newline, nor create if we want a line contination. + // Don't overwrite previous newline, nor create if we want a line continuation. if (p_make_token && !pending_newline && !line_continuation) { Token newline(Token::NEWLINE); newline.start_line = line; diff --git a/modules/mbedtls/crypto_mbedtls.cpp b/modules/mbedtls/crypto_mbedtls.cpp index 501bfff075..12a982df6e 100644 --- a/modules/mbedtls/crypto_mbedtls.cpp +++ b/modules/mbedtls/crypto_mbedtls.cpp @@ -43,8 +43,8 @@ #define PEM_BEGIN_CRT "-----BEGIN CERTIFICATE-----\n" #define PEM_END_CRT "-----END CERTIFICATE-----\n" -#include "mbedtls/pem.h" #include <mbedtls/debug.h> +#include <mbedtls/pem.h> CryptoKey *CryptoKeyMbedTLS::create() { return memnew(CryptoKeyMbedTLS); @@ -294,20 +294,15 @@ Ref<X509Certificate> CryptoMbedTLS::generate_self_signed_certificate(Ref<CryptoK unsigned char buf[4096]; memset(buf, 0, 4096); - Ref<X509CertificateMbedTLS> out; - out.instance(); - mbedtls_x509write_crt_pem(&crt, buf, 4096, mbedtls_ctr_drbg_random, &ctr_drbg); - - int err = mbedtls_x509_crt_parse(&(out->cert), buf, 4096); - if (err != 0) { - mbedtls_mpi_free(&serial); - mbedtls_x509write_crt_free(&crt); - ERR_PRINT("Generated invalid certificate: " + itos(err)); - return nullptr; - } - + int ret = mbedtls_x509write_crt_pem(&crt, buf, 4096, mbedtls_ctr_drbg_random, &ctr_drbg); mbedtls_mpi_free(&serial); mbedtls_x509write_crt_free(&crt); + ERR_FAIL_COND_V_MSG(ret != 0, nullptr, "Failed to generate certificate: " + itos(ret)); + buf[4095] = '\0'; // Make sure strlen can't fail. + + Ref<X509CertificateMbedTLS> out; + out.instance(); + out->load_from_memory(buf, strlen((char *)buf) + 1); // Use strlen to find correct output size. return out; } diff --git a/modules/mono/build_scripts/mono_configure.py b/modules/mono/build_scripts/mono_configure.py index 3e771e06f0..e959312393 100644 --- a/modules/mono/build_scripts/mono_configure.py +++ b/modules/mono/build_scripts/mono_configure.py @@ -1,6 +1,5 @@ import os import os.path -import sys import subprocess from SCons.Script import Dir, Environment diff --git a/modules/mono/build_scripts/solution_builder.py b/modules/mono/build_scripts/solution_builder.py index 03f4e57f02..6a621c3c8b 100644 --- a/modules/mono/build_scripts/solution_builder.py +++ b/modules/mono/build_scripts/solution_builder.py @@ -8,9 +8,6 @@ def find_dotnet_cli(): import os.path if os.name == "nt": - windows_exts = os.environ["PATHEXT"] - windows_exts = windows_exts.split(os.pathsep) if windows_exts else [] - for hint_dir in os.environ["PATH"].split(os.pathsep): hint_dir = hint_dir.strip('"') hint_path = os.path.join(hint_dir, "dotnet") diff --git a/modules/mono/editor/Godot.NET.Sdk/Godot.NET.Sdk/Sdk/Sdk.props b/modules/mono/editor/Godot.NET.Sdk/Godot.NET.Sdk/Sdk/Sdk.props index dfc59e6ccb..4f8faffde2 100644 --- a/modules/mono/editor/Godot.NET.Sdk/Godot.NET.Sdk/Sdk/Sdk.props +++ b/modules/mono/editor/Godot.NET.Sdk/Godot.NET.Sdk/Sdk/Sdk.props @@ -88,7 +88,7 @@ <PropertyGroup> <!-- ExportDebug also defines DEBUG like Debug does. --> <DefineConstants Condition=" '$(Configuration)' == 'ExportDebug' ">$(DefineConstants);DEBUG</DefineConstants> - <!-- Debug defines TOOLS to differenciate between Debug and ExportDebug configurations. --> + <!-- Debug defines TOOLS to differentiate between Debug and ExportDebug configurations. --> <DefineConstants Condition=" '$(Configuration)' == 'Debug' ">$(DefineConstants);TOOLS</DefineConstants> <DefineConstants>$(GodotDefineConstants);$(DefineConstants)</DefineConstants> diff --git a/modules/mono/editor/GodotTools/GodotTools.Core/StringExtensions.cs b/modules/mono/editor/GodotTools/GodotTools.Core/StringExtensions.cs index 05e06babd4..b217ae4bf7 100644 --- a/modules/mono/editor/GodotTools/GodotTools.Core/StringExtensions.cs +++ b/modules/mono/editor/GodotTools/GodotTools.Core/StringExtensions.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Runtime.InteropServices; namespace GodotTools.Core { @@ -35,7 +36,17 @@ namespace GodotTools.Core path = string.Join(Path.DirectorySeparatorChar.ToString(), parts).Trim(); - return rooted ? Path.DirectorySeparatorChar + path : path; + if (!rooted) + return path; + + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + string maybeDrive = parts[0]; + if (maybeDrive.Length == 2 && maybeDrive[1] == ':') + return path; // Already has drive letter + } + + return Path.DirectorySeparatorChar + path; } private static readonly string DriveRoot = Path.GetPathRoot(Environment.CurrentDirectory); @@ -49,7 +60,7 @@ namespace GodotTools.Core public static string ToSafeDirName(this string dirName, bool allowDirSeparator = false) { - var invalidChars = new List<string> { ":", "*", "?", "\"", "<", ">", "|" }; + var invalidChars = new List<string> {":", "*", "?", "\"", "<", ">", "|"}; if (allowDirSeparator) { diff --git a/modules/mono/editor/GodotTools/GodotTools/Export/AotBuilder.cs b/modules/mono/editor/GodotTools/GodotTools/Export/AotBuilder.cs index f60e469503..42ede3f3f3 100755 --- a/modules/mono/editor/GodotTools/GodotTools/Export/AotBuilder.cs +++ b/modules/mono/editor/GodotTools/GodotTools/Export/AotBuilder.cs @@ -328,7 +328,7 @@ MONO_AOT_MODE_LAST = 1000, if (lipoExitCode != 0) throw new Exception($"Command 'lipo' exited with code: {lipoExitCode}"); - // TODO: Add the AOT lib and interpreter libs as device only to supress warnings when targeting the simulator + // TODO: Add the AOT lib and interpreter libs as device only to suppress warnings when targeting the simulator // Add the fat AOT static library to the Xcode project exporter.AddIosProjectStaticLib(fatOutputFilePath); diff --git a/modules/mono/glue/GodotSharp/GodotSharp/Core/Array.cs b/modules/mono/glue/GodotSharp/GodotSharp/Core/Array.cs index a963810d63..f77d3052f4 100644 --- a/modules/mono/glue/GodotSharp/GodotSharp/Core/Array.cs +++ b/modules/mono/glue/GodotSharp/GodotSharp/Core/Array.cs @@ -44,6 +44,15 @@ namespace Godot.Collections Add(element); } + public Array(params object[] array) : this() + { + if (array == null) + { + throw new NullReferenceException($"Parameter '{nameof(array)} cannot be null.'"); + } + safeHandle = new ArraySafeHandle(godot_icall_Array_Ctor_MonoArray(array)); + } + internal Array(ArraySafeHandle handle) { safeHandle = handle; @@ -72,6 +81,11 @@ namespace Godot.Collections return godot_icall_Array_Resize(GetPtr(), newSize); } + public static Array operator +(Array left, Array right) + { + return new Array(godot_icall_Array_Concatenate(left.GetPtr(), right.GetPtr())); + } + // IDisposable public void Dispose() @@ -155,6 +169,9 @@ namespace Godot.Collections internal extern static IntPtr godot_icall_Array_Ctor(); [MethodImpl(MethodImplOptions.InternalCall)] + internal extern static IntPtr godot_icall_Array_Ctor_MonoArray(System.Array array); + + [MethodImpl(MethodImplOptions.InternalCall)] internal extern static void godot_icall_Array_Dtor(IntPtr ptr); [MethodImpl(MethodImplOptions.InternalCall)] @@ -176,6 +193,9 @@ namespace Godot.Collections internal extern static void godot_icall_Array_Clear(IntPtr ptr); [MethodImpl(MethodImplOptions.InternalCall)] + internal extern static IntPtr godot_icall_Array_Concatenate(IntPtr left, IntPtr right); + + [MethodImpl(MethodImplOptions.InternalCall)] internal extern static bool godot_icall_Array_Contains(IntPtr ptr, object item); [MethodImpl(MethodImplOptions.InternalCall)] @@ -231,6 +251,15 @@ namespace Godot.Collections objectArray = new Array(collection); } + public Array(params T[] array) : this() + { + if (array == null) + { + throw new NullReferenceException($"Parameter '{nameof(array)} cannot be null.'"); + } + objectArray = new Array(array); + } + public Array(Array array) { objectArray = array; @@ -266,6 +295,11 @@ namespace Godot.Collections return objectArray.Resize(newSize); } + public static Array<T> operator +(Array<T> left, Array<T> right) + { + return new Array<T>(left.objectArray + right.objectArray); + } + // IList<T> public T this[int index] diff --git a/modules/mono/glue/GodotSharp/GodotSharp/Core/StringExtensions.cs b/modules/mono/glue/GodotSharp/GodotSharp/Core/StringExtensions.cs index 41b4e9367f..9421a24b34 100644 --- a/modules/mono/glue/GodotSharp/GodotSharp/Core/StringExtensions.cs +++ b/modules/mono/glue/GodotSharp/GodotSharp/Core/StringExtensions.cs @@ -624,41 +624,46 @@ namespace Godot return instance.Length; } - // <summary> - // Do a simple expression match, where '*' matches zero or more arbitrary characters and '?' matches any single character except '.'. - // </summary> - public static bool ExprMatch(this string instance, string expr, bool caseSensitive) + /// <summary> + /// Do a simple expression match, where '*' matches zero or more arbitrary characters and '?' matches any single character except '.'. + /// </summary> + private static bool ExprMatch(this string instance, string expr, bool caseSensitive) { - if (expr.Length == 0 || instance.Length == 0) - return false; + // case '\0': + if (expr.Length == 0) + return instance.Length == 0; switch (expr[0]) { - case '\0': - return instance[0] == 0; case '*': - return ExprMatch(expr + 1, instance, caseSensitive) || instance[0] != 0 && ExprMatch(expr, instance + 1, caseSensitive); + return ExprMatch(instance, expr.Substring(1), caseSensitive) || (instance.Length > 0 && ExprMatch(instance.Substring(1), expr, caseSensitive)); case '?': - return instance[0] != 0 && instance[0] != '.' && ExprMatch(expr + 1, instance + 1, caseSensitive); + return instance.Length > 0 && instance[0] != '.' && ExprMatch(instance.Substring(1), expr.Substring(1), caseSensitive); default: - return (caseSensitive ? instance[0] == expr[0] : char.ToUpper(instance[0]) == char.ToUpper(expr[0])) && - ExprMatch(expr + 1, instance + 1, caseSensitive); + if (instance.Length == 0) return false; + return (caseSensitive ? instance[0] == expr[0] : char.ToUpper(instance[0]) == char.ToUpper(expr[0])) && ExprMatch(instance.Substring(1), expr.Substring(1), caseSensitive); } } - // <summary> - // Do a simple case sensitive expression match, using ? and * wildcards (see [method expr_match]). - // </summary> + /// <summary> + /// Do a simple case sensitive expression match, using ? and * wildcards (see [method expr_match]). + /// </summary> public static bool Match(this string instance, string expr, bool caseSensitive = true) { + if (instance.Length == 0 || expr.Length == 0) + return false; + return instance.ExprMatch(expr, caseSensitive); } - // <summary> - // Do a simple case insensitive expression match, using ? and * wildcards (see [method expr_match]). - // </summary> + /// <summary> + /// Do a simple case insensitive expression match, using ? and * wildcards (see [method expr_match]). + /// </summary> public static bool MatchN(this string instance, string expr) { + if (instance.Length == 0 || expr.Length == 0) + return false; + return instance.ExprMatch(expr, caseSensitive: false); } diff --git a/modules/mono/glue/collections_glue.cpp b/modules/mono/glue/collections_glue.cpp index 766b00d612..3313e8cb77 100644 --- a/modules/mono/glue/collections_glue.cpp +++ b/modules/mono/glue/collections_glue.cpp @@ -104,10 +104,31 @@ void godot_icall_Array_CopyTo(Array *ptr, MonoArray *array, int array_index) { } } +Array *godot_icall_Array_Ctor_MonoArray(MonoArray *mono_array) { + Array *godot_array = memnew(Array); + unsigned int count = mono_array_length(mono_array); + godot_array->resize(count); + for (unsigned int i = 0; i < count; i++) { + MonoObject *item = mono_array_get(mono_array, MonoObject *, i); + godot_icall_Array_SetAt(godot_array, i, item); + } + return godot_array; +} + Array *godot_icall_Array_Duplicate(Array *ptr, MonoBoolean deep) { return memnew(Array(ptr->duplicate(deep))); } +Array *godot_icall_Array_Concatenate(Array *left, Array *right) { + int count = left->size() + right->size(); + Array *new_array = memnew(Array(left->duplicate(false))); + new_array->resize(count); + for (unsigned int i = 0; i < (unsigned int)right->size(); i++) { + new_array->operator[](i + left->size()) = right->operator[](i); + } + return new_array; +} + int godot_icall_Array_IndexOf(Array *ptr, MonoObject *item) { return ptr->find(GDMonoMarshal::mono_object_to_variant(item)); } @@ -284,6 +305,7 @@ MonoString *godot_icall_Dictionary_ToString(Dictionary *ptr) { void godot_register_collections_icalls() { mono_add_internal_call("Godot.Collections.Array::godot_icall_Array_Ctor", (void *)godot_icall_Array_Ctor); + mono_add_internal_call("Godot.Collections.Array::godot_icall_Array_Ctor_MonoArray", (void *)godot_icall_Array_Ctor_MonoArray); mono_add_internal_call("Godot.Collections.Array::godot_icall_Array_Dtor", (void *)godot_icall_Array_Dtor); mono_add_internal_call("Godot.Collections.Array::godot_icall_Array_At", (void *)godot_icall_Array_At); mono_add_internal_call("Godot.Collections.Array::godot_icall_Array_At_Generic", (void *)godot_icall_Array_At_Generic); @@ -291,6 +313,7 @@ void godot_register_collections_icalls() { mono_add_internal_call("Godot.Collections.Array::godot_icall_Array_Count", (void *)godot_icall_Array_Count); mono_add_internal_call("Godot.Collections.Array::godot_icall_Array_Add", (void *)godot_icall_Array_Add); mono_add_internal_call("Godot.Collections.Array::godot_icall_Array_Clear", (void *)godot_icall_Array_Clear); + mono_add_internal_call("Godot.Collections.Array::godot_icall_Array_Concatenate", (void *)godot_icall_Array_Concatenate); mono_add_internal_call("Godot.Collections.Array::godot_icall_Array_Contains", (void *)godot_icall_Array_Contains); mono_add_internal_call("Godot.Collections.Array::godot_icall_Array_CopyTo", (void *)godot_icall_Array_CopyTo); mono_add_internal_call("Godot.Collections.Array::godot_icall_Array_Duplicate", (void *)godot_icall_Array_Duplicate); diff --git a/modules/mono/mono_gd/gd_mono_log.cpp b/modules/mono/mono_gd/gd_mono_log.cpp index c5a988b8c3..b8ee0204c4 100644 --- a/modules/mono/mono_gd/gd_mono_log.cpp +++ b/modules/mono/mono_gd/gd_mono_log.cpp @@ -64,25 +64,32 @@ static int get_log_level_id(const char *p_log_level) { return -1; } +static String make_text(const char *log_domain, const char *log_level, const char *message) { + String text(message); + text += " (in domain "; + text += log_domain; + if (log_level) { + text += ", "; + text += log_level; + } + text += ")"; + return text; +} + void GDMonoLog::mono_log_callback(const char *log_domain, const char *log_level, const char *message, mono_bool fatal, void *) { FileAccess *f = GDMonoLog::get_singleton()->log_file; if (GDMonoLog::get_singleton()->log_level_id >= get_log_level_id(log_level)) { - String text(message); - text += " (in domain "; - text += log_domain; - if (log_level) { - text += ", "; - text += log_level; - } - text += ")\n"; + String text = make_text(log_domain, log_level, message); + text += "\n"; f->seek_end(); f->store_string(text); } if (fatal) { - ERR_PRINT("Mono: FATAL ERROR, ABORTING! Logfile: '" + GDMonoLog::get_singleton()->log_file_path + "'."); + String text = make_text(log_domain, log_level, message); + ERR_PRINT("Mono: FATAL ERROR '" + text + "', ABORTING! Logfile: '" + GDMonoLog::get_singleton()->log_file_path + "'."); // Make sure to flush before aborting f->flush(); f->close(); diff --git a/modules/stb_vorbis/audio_stream_ogg_vorbis.cpp b/modules/stb_vorbis/audio_stream_ogg_vorbis.cpp index 3aceaf11c5..346833ab9c 100644 --- a/modules/stb_vorbis/audio_stream_ogg_vorbis.cpp +++ b/modules/stb_vorbis/audio_stream_ogg_vorbis.cpp @@ -158,14 +158,16 @@ void AudioStreamOGGVorbis::clear_data() { void AudioStreamOGGVorbis::set_data(const Vector<uint8_t> &p_data) { int src_data_len = p_data.size(); -#define MAX_TEST_MEM (1 << 20) - uint32_t alloc_try = 1024; Vector<char> alloc_mem; char *w; stb_vorbis *ogg_stream = nullptr; stb_vorbis_alloc ogg_alloc; + // Vorbis comments may be up to UINT32_MAX, but that's arguably pretty rare. + // Let's go with 2^30 so we don't risk going out of bounds. + const uint32_t MAX_TEST_MEM = 1 << 30; + while (alloc_try < MAX_TEST_MEM) { alloc_mem.resize(alloc_try); w = alloc_mem.ptrw(); @@ -205,6 +207,8 @@ void AudioStreamOGGVorbis::set_data(const Vector<uint8_t> &p_data) { break; } } + + ERR_FAIL_COND_MSG(alloc_try == MAX_TEST_MEM, vformat("Couldn't set vorbis data even with an alloc buffer of %d bytes, report bug.", MAX_TEST_MEM)); } Vector<uint8_t> AudioStreamOGGVorbis::get_data() const { diff --git a/modules/tinyexr/image_loader_tinyexr.cpp b/modules/tinyexr/image_loader_tinyexr.cpp index 9e7266b95a..5bdcb84244 100644 --- a/modules/tinyexr/image_loader_tinyexr.cpp +++ b/modules/tinyexr/image_loader_tinyexr.cpp @@ -73,8 +73,10 @@ Error ImageLoaderTinyEXR::load_image(Ref<Image> p_image, FileAccess *f, bool p_f } // Read HALF channel as FLOAT. (GH-13490) + bool use_float16 = false; for (int i = 0; i < exr_header.num_channels; i++) { if (exr_header.pixel_types[i] == TINYEXR_PIXELTYPE_HALF) { + use_float16 = true; exr_header.requested_pixel_types[i] = TINYEXR_PIXELTYPE_FLOAT; } } @@ -102,33 +104,10 @@ Error ImageLoaderTinyEXR::load_image(Ref<Image> p_image, FileAccess *f, bool p_f idxB = c; } else if (strcmp(exr_header.channels[c].name, "A") == 0) { idxA = c; - } - } - - if (exr_header.num_channels == 1) { - // Grayscale channel only. - idxR = 0; - idxG = 0; - idxB = 0; - idxA = 0; - } else { - // Assume RGB(A) - if (idxR == -1) { - ERR_PRINT("TinyEXR: R channel not found."); - // @todo { free exr_image } - return ERR_FILE_CORRUPT; - } - - if (idxG == -1) { - ERR_PRINT("TinyEXR: G channel not found."); - // @todo { free exr_image } - return ERR_FILE_CORRUPT; - } - - if (idxB == -1) { - ERR_PRINT("TinyEXR: B channel not found."); - // @todo { free exr_image } - return ERR_FILE_CORRUPT; + } else if (strcmp(exr_header.channels[c].name, "Y") == 0) { + idxR = c; + idxG = c; + idxB = c; } } @@ -138,14 +117,27 @@ Error ImageLoaderTinyEXR::load_image(Ref<Image> p_image, FileAccess *f, bool p_f Image::Format format; int output_channels = 0; + int channel_size = use_float16 ? 2 : 4; if (idxA != -1) { - imgdata.resize(exr_image.width * exr_image.height * 8); //RGBA16 - format = Image::FORMAT_RGBAH; + imgdata.resize(exr_image.width * exr_image.height * 4 * channel_size); //RGBA + format = use_float16 ? Image::FORMAT_RGBAH : Image::FORMAT_RGBAF; output_channels = 4; - } else { - imgdata.resize(exr_image.width * exr_image.height * 6); //RGB16 - format = Image::FORMAT_RGBH; + } else if (idxB != -1) { + ERR_FAIL_COND_V(idxG == -1, ERR_FILE_CORRUPT); + ERR_FAIL_COND_V(idxR == -1, ERR_FILE_CORRUPT); + imgdata.resize(exr_image.width * exr_image.height * 3 * channel_size); //RGB + format = use_float16 ? Image::FORMAT_RGBH : Image::FORMAT_RGBF; output_channels = 3; + } else if (idxG != -1) { + ERR_FAIL_COND_V(idxR == -1, ERR_FILE_CORRUPT); + imgdata.resize(exr_image.width * exr_image.height * 2 * channel_size); //RG + format = use_float16 ? Image::FORMAT_RGH : Image::FORMAT_RGF; + output_channels = 2; + } else { + ERR_FAIL_COND_V(idxR == -1, ERR_FILE_CORRUPT); + imgdata.resize(exr_image.width * exr_image.height * 1 * channel_size); //R + format = use_float16 ? Image::FORMAT_RH : Image::FORMAT_RF; + output_channels = 1; } EXRTile single_image_tile; @@ -175,9 +167,11 @@ Error ImageLoaderTinyEXR::load_image(Ref<Image> p_image, FileAccess *f, bool p_f exr_tiles = exr_image.tiles; } + //print_line("reading format: " + Image::get_format_name(format)); { uint8_t *wd = imgdata.ptrw(); - uint16_t *iw = (uint16_t *)wd; + uint16_t *iw16 = (uint16_t *)wd; + float *iw32 = (float *)wd; // Assume `out_rgba` have enough memory allocated. for (int tile_index = 0; tile_index < num_tiles; tile_index++) { @@ -187,41 +181,99 @@ Error ImageLoaderTinyEXR::load_image(Ref<Image> p_image, FileAccess *f, bool p_f int th = tile.height; const float *r_channel_start = reinterpret_cast<const float *>(tile.images[idxR]); - const float *g_channel_start = reinterpret_cast<const float *>(tile.images[idxG]); - const float *b_channel_start = reinterpret_cast<const float *>(tile.images[idxB]); + const float *g_channel_start = nullptr; + const float *b_channel_start = nullptr; const float *a_channel_start = nullptr; + if (idxG != -1) { + g_channel_start = reinterpret_cast<const float *>(tile.images[idxG]); + } + if (idxB != -1) { + b_channel_start = reinterpret_cast<const float *>(tile.images[idxB]); + } if (idxA != -1) { a_channel_start = reinterpret_cast<const float *>(tile.images[idxA]); } - uint16_t *first_row_w = iw + (tile.offset_y * tile_height * exr_image.width + tile.offset_x * tile_width) * output_channels; + uint16_t *first_row_w16 = iw16 + (tile.offset_y * tile_height * exr_image.width + tile.offset_x * tile_width) * output_channels; + float *first_row_w32 = iw32 + (tile.offset_y * tile_height * exr_image.width + tile.offset_x * tile_width) * output_channels; for (int y = 0; y < th; y++) { const float *r_channel = r_channel_start + y * tile_width; - const float *g_channel = g_channel_start + y * tile_width; - const float *b_channel = b_channel_start + y * tile_width; + const float *g_channel = nullptr; + const float *b_channel = nullptr; const float *a_channel = nullptr; - + if (g_channel_start) { + g_channel = g_channel_start + y * tile_width; + } + if (b_channel_start) { + b_channel = b_channel_start + y * tile_width; + } if (a_channel_start) { a_channel = a_channel_start + y * tile_width; } - uint16_t *row_w = first_row_w + (y * exr_image.width * output_channels); - - for (int x = 0; x < tw; x++) { - Color color(*r_channel++, *g_channel++, *b_channel++); - - if (p_force_linear) { - color = color.to_linear(); + if (use_float16) { + uint16_t *row_w = first_row_w16 + (y * exr_image.width * output_channels); + + for (int x = 0; x < tw; x++) { + Color color; + color.r = *r_channel++; + if (g_channel) { + color.g = *g_channel++; + } + if (b_channel) { + color.b = *b_channel++; + } + if (a_channel) { + color.a = *a_channel++; + } + + if (p_force_linear) { + color = color.to_linear(); + } + + *row_w++ = Math::make_half_float(color.r); + if (g_channel) { + *row_w++ = Math::make_half_float(color.g); + } + if (b_channel) { + *row_w++ = Math::make_half_float(color.b); + } + if (a_channel) { + *row_w++ = Math::make_half_float(color.a); + } } - - *row_w++ = Math::make_half_float(color.r); - *row_w++ = Math::make_half_float(color.g); - *row_w++ = Math::make_half_float(color.b); - - if (idxA != -1) { - *row_w++ = Math::make_half_float(*a_channel++); + } else { + float *row_w = first_row_w32 + (y * exr_image.width * output_channels); + + for (int x = 0; x < tw; x++) { + Color color; + color.r = *r_channel++; + if (g_channel) { + color.g = *g_channel++; + } + if (b_channel) { + color.b = *b_channel++; + } + if (a_channel) { + color.a = *a_channel++; + } + + if (p_force_linear) { + color = color.to_linear(); + } + + *row_w++ = color.r; + if (g_channel) { + *row_w++ = color.g; + } + if (b_channel) { + *row_w++ = color.b; + } + if (a_channel) { + *row_w++ = color.a; + } } } } diff --git a/modules/visual_script/visual_script_editor.cpp b/modules/visual_script/visual_script_editor.cpp index 5581ea9318..b1d8c05d87 100644 --- a/modules/visual_script/visual_script_editor.cpp +++ b/modules/visual_script/visual_script_editor.cpp @@ -1124,8 +1124,8 @@ void VisualScriptEditor::_update_members() { TreeItem *ti = members->create_item(variables); ti->set_text(0, E->get()); - Variant var = script->get_variable_default_value(E->get()); - ti->set_suffix(0, "= " + String(var)); + + ti->set_suffix(0, "= " + _sanitized_variant_text(E->get())); ti->set_icon(0, type_icons[script->get_variable_info(E->get()).type]); ti->set_selectable(0, true); @@ -1167,6 +1167,18 @@ void VisualScriptEditor::_update_members() { updating_members = false; } +String VisualScriptEditor::_sanitized_variant_text(const StringName &property_name) { + Variant var = script->get_variable_default_value(property_name); + + if (script->get_variable_info(property_name).type != Variant::NIL) { + Callable::CallError ce; + const Variant *converted = &var; + var = Variant::construct(script->get_variable_info(property_name).type, &converted, 1, ce, false); + } + + return String(var); +} + void VisualScriptEditor::_member_selected() { if (updating_members) { return; @@ -2682,7 +2694,8 @@ void VisualScriptEditor::reload(bool p_soft) { _update_graph(); } -void VisualScriptEditor::get_breakpoints(List<int> *p_breakpoints) { +Array VisualScriptEditor::get_breakpoints() { + Array breakpoints; List<StringName> functions; script->get_function_list(&functions); for (List<StringName>::Element *E = functions.front(); E; E = E->next()) { @@ -2691,10 +2704,11 @@ void VisualScriptEditor::get_breakpoints(List<int> *p_breakpoints) { for (List<int>::Element *F = nodes.front(); F; F = F->next()) { Ref<VisualScriptNode> vsn = script->get_node(E->get(), F->get()); if (vsn->is_breakpoint()) { - p_breakpoints->push_back(F->get() - 1); //subtract 1 because breakpoints in text start from zero + breakpoints.push_back(F->get() - 1); //subtract 1 because breakpoints in text start from zero } } } + return breakpoints; } void VisualScriptEditor::add_callback(const String &p_function, PackedStringArray p_args) { diff --git a/modules/visual_script/visual_script_editor.h b/modules/visual_script/visual_script_editor.h index 0c5665cee8..66e435741f 100644 --- a/modules/visual_script/visual_script_editor.h +++ b/modules/visual_script/visual_script_editor.h @@ -146,6 +146,7 @@ class VisualScriptEditor : public ScriptEditorBase { bool updating_members; void _update_members(); + String _sanitized_variant_text(const StringName &property_name); StringName selected; @@ -312,7 +313,7 @@ public: virtual void ensure_focus() override; virtual void tag_saved_version() override; virtual void reload(bool p_soft) override; - virtual void get_breakpoints(List<int> *p_breakpoints) override; + virtual Array get_breakpoints() override; virtual void add_callback(const String &p_function, PackedStringArray p_args) override; virtual void update_settings() override; virtual bool show_members_overview() override; diff --git a/platform/iphone/godot_view_gesture_recognizer.h b/platform/iphone/godot_view_gesture_recognizer.h index ca3bd808d1..8d84914712 100644 --- a/platform/iphone/godot_view_gesture_recognizer.h +++ b/platform/iphone/godot_view_gesture_recognizer.h @@ -32,7 +32,7 @@ // emulating UIScrollView's UIScrollViewDelayedTouchesBeganGestureRecognizer. // It catches all gestures incoming to UIView and delays them for 150ms // (the same value used by UIScrollViewDelayedTouchesBeganGestureRecognizer) -// If touch cancelation or end message is fired it fires delayed +// If touch cancellation or end message is fired it fires delayed // begin touch immediately as well as last touch signal #import <UIKit/UIKit.h> diff --git a/platform/javascript/SCsub b/platform/javascript/SCsub index dcf9a46bf9..21456efde5 100644 --- a/platform/javascript/SCsub +++ b/platform/javascript/SCsub @@ -66,5 +66,5 @@ env.Zip( zip_files, ZIPROOT=zip_dir, ZIPSUFFIX="${PROGSUFFIX}${ZIPSUFFIX}", - ZIPCOMSTR="Archving $SOURCES as $TARGET", + ZIPCOMSTR="Archiving $SOURCES as $TARGET", ) diff --git a/platform/javascript/engine/engine.js b/platform/javascript/engine/engine.js index d709422abb..2630812814 100644 --- a/platform/javascript/engine/engine.js +++ b/platform/javascript/engine/engine.js @@ -121,7 +121,7 @@ Function('return this')()['Engine'] = (function() { if (me.onExit) me.onExit(code); me.rtenv = null; - } + }; return new Promise(function(resolve, reject) { preloader.preloadedFiles.forEach(function(file) { me.rtenv['copyToFS'](file.path, file.buffer); @@ -207,18 +207,18 @@ Function('return this')()['Engine'] = (function() { if (this.rtenv) this.rtenv.onExecute = onExecute; this.onExecute = onExecute; - } + }; Engine.prototype.setOnExit = function(onExit) { this.onExit = onExit; - } + }; Engine.prototype.copyToFS = function(path, buffer) { if (this.rtenv == null) { throw new Error("Engine must be inited before copying files"); } this.rtenv['copyToFS'](path, buffer); - } + }; // Closure compiler exported engine methods. /** @export */ diff --git a/platform/linuxbsd/detect.py b/platform/linuxbsd/detect.py index 3eb4c44bc1..f5e2c72bbc 100644 --- a/platform/linuxbsd/detect.py +++ b/platform/linuxbsd/detect.py @@ -35,6 +35,11 @@ def can_build(): print("xinerama not found.. x11 disabled.") return False + x11_error = os.system("pkg-config xext --modversion > /dev/null ") + if x11_error: + print("xext not found.. x11 disabled.") + return False + x11_error = os.system("pkg-config xrandr --modversion > /dev/null ") if x11_error: print("xrandr not found.. x11 disabled.") @@ -194,6 +199,7 @@ def configure(env): env.ParseConfig("pkg-config x11 --cflags --libs") env.ParseConfig("pkg-config xcursor --cflags --libs") env.ParseConfig("pkg-config xinerama --cflags --libs") + env.ParseConfig("pkg-config xext --cflags --libs") env.ParseConfig("pkg-config xrandr --cflags --libs") env.ParseConfig("pkg-config xrender --cflags --libs") env.ParseConfig("pkg-config xi --cflags --libs") diff --git a/platform/linuxbsd/display_server_x11.cpp b/platform/linuxbsd/display_server_x11.cpp index fe9e253cc9..e3cf992302 100644 --- a/platform/linuxbsd/display_server_x11.cpp +++ b/platform/linuxbsd/display_server_x11.cpp @@ -50,6 +50,7 @@ #include <X11/Xatom.h> #include <X11/Xutil.h> #include <X11/extensions/Xinerama.h> +#include <X11/extensions/shape.h> // ICCCM #define WM_NormalState 1L // window normal state @@ -782,6 +783,38 @@ void DisplayServerX11::window_set_title(const String &p_title, WindowID p_window XChangeProperty(x11_display, wd.x11_window, _net_wm_name, utf8_string, 8, PropModeReplace, (unsigned char *)p_title.utf8().get_data(), p_title.utf8().length()); } +void DisplayServerX11::window_set_mouse_passthrough(const Vector<Vector2> &p_region, WindowID p_window) { + _THREAD_SAFE_METHOD_ + + ERR_FAIL_COND(!windows.has(p_window)); + const WindowData &wd = windows[p_window]; + + int event_base, error_base; + const Bool ext_okay = XShapeQueryExtension(x11_display, &event_base, &error_base); + if (ext_okay) { + Region region; + if (p_region.size() == 0) { + region = XCreateRegion(); + XRectangle rect; + rect.x = 0; + rect.y = 0; + rect.width = window_get_real_size(p_window).x; + rect.height = window_get_real_size(p_window).y; + XUnionRectWithRegion(&rect, region, region); + } else { + XPoint *points = (XPoint *)memalloc(sizeof(XPoint) * p_region.size()); + for (int i = 0; i < p_region.size(); i++) { + points[i].x = p_region[i].x; + points[i].y = p_region[i].y; + } + region = XPolygonRegion(points, p_region.size(), EvenOddRule); + memfree(points); + } + XShapeCombineRegion(x11_display, wd.x11_window, ShapeInput, 0, 0, region, ShapeSet); + XDestroyRegion(region); + } +} + void DisplayServerX11::window_set_rect_changed_callback(const Callable &p_callable, WindowID p_window) { _THREAD_SAFE_METHOD_ @@ -3542,7 +3575,12 @@ DisplayServerX11::DisplayServerX11(const String &p_rendering_driver, WindowMode xrandr_handle = dlopen("libXrandr.so.2", RTLD_LAZY); if (!xrandr_handle) { err = dlerror(); - fprintf(stderr, "could not load libXrandr.so.2, Error: %s\n", err); + // For some arcane reason, NetBSD now ships libXrandr.so.3 while the rest of the world has libXrandr.so.2... + // In case this happens for other X11 platforms in the future, let's give it a try too before failing. + xrandr_handle = dlopen("libXrandr.so.3", RTLD_LAZY); + if (!xrandr_handle) { + fprintf(stderr, "could not load libXrandr.so.2, Error: %s\n", err); + } } else { XRRQueryVersion(x11_display, &xrandr_major, &xrandr_minor); if (((xrandr_major << 8) | xrandr_minor) >= 0x0105) { diff --git a/platform/linuxbsd/display_server_x11.h b/platform/linuxbsd/display_server_x11.h index 57cee910a0..e894cf515c 100644 --- a/platform/linuxbsd/display_server_x11.h +++ b/platform/linuxbsd/display_server_x11.h @@ -291,6 +291,8 @@ public: virtual ObjectID window_get_attached_instance_id(WindowID p_window = MAIN_WINDOW_ID) const; virtual void window_set_title(const String &p_title, WindowID p_window = MAIN_WINDOW_ID); + virtual void window_set_mouse_passthrough(const Vector<Vector2> &p_region, WindowID p_window = MAIN_WINDOW_ID); + virtual void window_set_rect_changed_callback(const Callable &p_callable, WindowID p_window = MAIN_WINDOW_ID); virtual void window_set_window_event_callback(const Callable &p_callable, WindowID p_window = MAIN_WINDOW_ID); virtual void window_set_input_event_callback(const Callable &p_callable, WindowID p_window = MAIN_WINDOW_ID); diff --git a/platform/linuxbsd/platform_config.h b/platform/linuxbsd/platform_config.h index 764666681f..571ad03db0 100644 --- a/platform/linuxbsd/platform_config.h +++ b/platform/linuxbsd/platform_config.h @@ -31,7 +31,15 @@ #ifdef __linux__ #include <alloca.h> #endif -#if defined(__FreeBSD__) || defined(__OpenBSD__) -#include <stdlib.h> + +#if defined(__FreeBSD__) || defined(__OpenBSD__) || defined(__NetBSD__) +#include <stdlib.h> // alloca +// FreeBSD and OpenBSD use pthread_set_name_np, while other platforms, +// include NetBSD, use pthread_setname_np. NetBSD's version however requires +// a different format, we handle this directly in thread_posix. +#ifdef __NetBSD__ +#define PTHREAD_NETBSD_SET_NAME +#else #define PTHREAD_BSD_SET_NAME #endif +#endif diff --git a/platform/osx/detect.py b/platform/osx/detect.py index a4a382f3a9..6fc1dc65af 100644 --- a/platform/osx/detect.py +++ b/platform/osx/detect.py @@ -1,6 +1,5 @@ import os import sys -import subprocess from methods import detect_darwin_sdk_path diff --git a/platform/osx/display_server_osx.h b/platform/osx/display_server_osx.h index d8f3f81ff6..073d35008b 100644 --- a/platform/osx/display_server_osx.h +++ b/platform/osx/display_server_osx.h @@ -102,6 +102,8 @@ public: id window_object; id window_view; + Vector<Vector2> mpath; + #if defined(OPENGL_ENABLED) ContextGL_OSX *context_gles2 = nullptr; #endif @@ -240,6 +242,7 @@ public: virtual void window_set_drop_files_callback(const Callable &p_callable, WindowID p_window = MAIN_WINDOW_ID) override; virtual void window_set_title(const String &p_title, WindowID p_window = MAIN_WINDOW_ID) override; + virtual void window_set_mouse_passthrough(const Vector<Vector2> &p_region, WindowID p_window = MAIN_WINDOW_ID) override; virtual int window_get_current_screen(WindowID p_window = MAIN_WINDOW_ID) const override; virtual void window_set_current_screen(int p_screen, WindowID p_window = MAIN_WINDOW_ID) override; diff --git a/platform/osx/display_server_osx.mm b/platform/osx/display_server_osx.mm index adfb47324e..e33af520c9 100644 --- a/platform/osx/display_server_osx.mm +++ b/platform/osx/display_server_osx.mm @@ -33,6 +33,7 @@ #include "os_osx.h" #include "core/io/marshalls.h" +#include "core/math/geometry_2d.h" #include "core/os/keyboard.h" #include "main/main.h" #include "scene/resources/texture.h" @@ -2404,6 +2405,15 @@ void DisplayServerOSX::window_set_title(const String &p_title, WindowID p_window [wd.window_object setTitle:[NSString stringWithUTF8String:p_title.utf8().get_data()]]; } +void DisplayServerOSX::window_set_mouse_passthrough(const Vector<Vector2> &p_region, WindowID p_window) { + _THREAD_SAFE_METHOD_ + + ERR_FAIL_COND(!windows.has(p_window)); + WindowData &wd = windows[p_window]; + + wd.mpath = p_region; +} + void DisplayServerOSX::window_set_rect_changed_callback(const Callable &p_callable, WindowID p_window) { _THREAD_SAFE_METHOD_ @@ -3356,6 +3366,26 @@ void DisplayServerOSX::process_events() { Input::get_singleton()->flush_accumulated_events(); } + for (Map<WindowID, WindowData>::Element *E = windows.front(); E; E = E->next()) { + WindowData &wd = E->get(); + if (wd.mpath.size() > 0) { + const Vector2 mpos = _get_mouse_pos(wd, [wd.window_object mouseLocationOutsideOfEventStream]); + if (Geometry2D::is_point_in_polygon(mpos, wd.mpath)) { + if ([wd.window_object ignoresMouseEvents]) { + [wd.window_object setIgnoresMouseEvents:NO]; + } + } else { + if (![wd.window_object ignoresMouseEvents]) { + [wd.window_object setIgnoresMouseEvents:YES]; + } + } + } else { + if ([wd.window_object ignoresMouseEvents]) { + [wd.window_object setIgnoresMouseEvents:NO]; + } + } + } + [autoreleasePool drain]; autoreleasePool = [[NSAutoreleasePool alloc] init]; } diff --git a/platform/server/platform_config.h b/platform/server/platform_config.h index bdff93f02b..73136ec81b 100644 --- a/platform/server/platform_config.h +++ b/platform/server/platform_config.h @@ -31,10 +31,19 @@ #if defined(__linux__) || defined(__APPLE__) #include <alloca.h> #endif -#if defined(__FreeBSD__) || defined(__OpenBSD__) -#include <stdlib.h> + +#if defined(__FreeBSD__) || defined(__OpenBSD__) || defined(__NetBSD__) +#include <stdlib.h> // alloca +// FreeBSD and OpenBSD use pthread_set_name_np, while other platforms, +// include NetBSD, use pthread_setname_np. NetBSD's version however requires +// a different format, we handle this directly in thread_posix. +#ifdef __NetBSD__ +#define PTHREAD_NETBSD_SET_NAME +#else #define PTHREAD_BSD_SET_NAME #endif +#endif + #ifdef __APPLE__ #define PTHREAD_RENAME_SELF #endif diff --git a/platform/uwp/detect.py b/platform/uwp/detect.py index 04b743f2c8..a7ca26c16c 100644 --- a/platform/uwp/detect.py +++ b/platform/uwp/detect.py @@ -65,12 +65,14 @@ def configure(env): env.Append(CCFLAGS=["/MD"]) env.Append(CPPDEFINES=["DEBUG_ENABLED"]) env.Append(LINKFLAGS=["/SUBSYSTEM:CONSOLE"]) + env.AppendUnique(CPPDEFINES=["WINDOWS_SUBSYSTEM_CONSOLE"]) elif env["target"] == "debug": env.Append(CCFLAGS=["/Zi"]) env.Append(CCFLAGS=["/MDd"]) env.Append(CPPDEFINES=["DEBUG_ENABLED"]) env.Append(LINKFLAGS=["/SUBSYSTEM:CONSOLE"]) + env.AppendUnique(CPPDEFINES=["WINDOWS_SUBSYSTEM_CONSOLE"]) env.Append(LINKFLAGS=["/DEBUG"]) ## Compiler configuration diff --git a/platform/windows/detect.py b/platform/windows/detect.py index 4e1da22bb0..6b503c1561 100644 --- a/platform/windows/detect.py +++ b/platform/windows/detect.py @@ -210,6 +210,7 @@ def configure_msvc(env, manual_msvc_config): env.Append(LINKFLAGS=["/SUBSYSTEM:WINDOWS"]) else: env.Append(LINKFLAGS=["/SUBSYSTEM:CONSOLE"]) + env.AppendUnique(CPPDEFINES=["WINDOWS_SUBSYSTEM_CONSOLE"]) ## Compile/link flags @@ -347,6 +348,7 @@ def configure_mingw(env): env.Append(LINKFLAGS=["-Wl,--subsystem,windows"]) else: env.Append(LINKFLAGS=["-Wl,--subsystem,console"]) + env.AppendUnique(CPPDEFINES=["WINDOWS_SUBSYSTEM_CONSOLE"]) ## Compiler configuration @@ -437,7 +439,7 @@ def configure_mingw(env): else: env.Append(LIBS=["cfgmgr32"]) - ## TODO !!! Reenable when OpenGLES Rendering Device is implemented !!! + ## TODO !!! Re-enable when OpenGLES Rendering Device is implemented !!! # env.Append(CPPDEFINES=['OPENGL_ENABLED']) env.Append(LIBS=["opengl32"]) diff --git a/platform/windows/display_server_windows.cpp b/platform/windows/display_server_windows.cpp index 7f4669b3b2..dfbb734ee4 100644 --- a/platform/windows/display_server_windows.cpp +++ b/platform/windows/display_server_windows.cpp @@ -29,7 +29,9 @@ /*************************************************************************/ #include "display_server_windows.h" + #include "core/io/marshalls.h" +#include "core/math/geometry_2d.h" #include "main/main.h" #include "os_windows.h" #include "scene/resources/texture.h" @@ -597,6 +599,36 @@ void DisplayServerWindows::window_set_title(const String &p_title, WindowID p_wi SetWindowTextW(windows[p_window].hWnd, (LPCWSTR)(p_title.utf16().get_data())); } +void DisplayServerWindows::window_set_mouse_passthrough(const Vector<Vector2> &p_region, WindowID p_window) { + _THREAD_SAFE_METHOD_ + + ERR_FAIL_COND(!windows.has(p_window)); + windows[p_window].mpath = p_region; + _update_window_mouse_passthrough(p_window); +} + +void DisplayServerWindows::_update_window_mouse_passthrough(WindowID p_window) { + if (windows[p_window].mpath.size() == 0) { + SetWindowRgn(windows[p_window].hWnd, nullptr, TRUE); + } else { + POINT *points = (POINT *)memalloc(sizeof(POINT) * windows[p_window].mpath.size()); + for (int i = 0; i < windows[p_window].mpath.size(); i++) { + if (windows[p_window].borderless) { + points[i].x = windows[p_window].mpath[i].x; + points[i].y = windows[p_window].mpath[i].y; + } else { + points[i].x = windows[p_window].mpath[i].x + GetSystemMetrics(SM_CXSIZEFRAME); + points[i].y = windows[p_window].mpath[i].y + GetSystemMetrics(SM_CYSIZEFRAME) + GetSystemMetrics(SM_CYCAPTION); + } + } + + HRGN region = CreatePolygonRgn(points, windows[p_window].mpath.size(), ALTERNATE); + SetWindowRgn(windows[p_window].hWnd, region, TRUE); + DeleteObject(region); + memfree(points); + } +} + int DisplayServerWindows::window_get_current_screen(WindowID p_window) const { _THREAD_SAFE_METHOD_ @@ -1010,6 +1042,7 @@ void DisplayServerWindows::window_set_flag(WindowFlags p_flag, bool p_enabled, W case WINDOW_FLAG_BORDERLESS: { wd.borderless = p_enabled; _update_window_style(p_window); + _update_window_mouse_passthrough(p_window); } break; case WINDOW_FLAG_ALWAYS_ON_TOP: { ERR_FAIL_COND_MSG(wd.transient_parent != INVALID_WINDOW_ID && p_enabled, "Transient windows can't become on top"); diff --git a/platform/windows/display_server_windows.h b/platform/windows/display_server_windows.h index 7bd93a7086..0fca2589ae 100644 --- a/platform/windows/display_server_windows.h +++ b/platform/windows/display_server_windows.h @@ -323,6 +323,8 @@ private: HWND hWnd; //layered window + Vector<Vector2> mpath; + bool preserve_window_size = false; bool pre_fs_valid = false; RECT pre_fs_rect; @@ -416,6 +418,7 @@ private: void _touch_event(WindowID p_window, bool p_pressed, float p_x, float p_y, int idx); void _update_window_style(WindowID p_window, bool p_repaint = true, bool p_maximized = false); + void _update_window_mouse_passthrough(WindowID p_window); void _update_real_mouse_position(WindowID p_window); @@ -477,6 +480,7 @@ public: virtual void window_set_drop_files_callback(const Callable &p_callable, WindowID p_window = MAIN_WINDOW_ID); virtual void window_set_title(const String &p_title, WindowID p_window = MAIN_WINDOW_ID); + virtual void window_set_mouse_passthrough(const Vector<Vector2> &p_region, WindowID p_window = MAIN_WINDOW_ID); virtual int window_get_current_screen(WindowID p_window = MAIN_WINDOW_ID) const; virtual void window_set_current_screen(int p_screen, WindowID p_window = MAIN_WINDOW_ID); diff --git a/scene/2d/ray_cast_2d.cpp b/scene/2d/ray_cast_2d.cpp index 9fd24b5294..a00db36077 100644 --- a/scene/2d/ray_cast_2d.cpp +++ b/scene/2d/ray_cast_2d.cpp @@ -35,15 +35,15 @@ #include "physics_body_2d.h" #include "servers/physics_server_2d.h" -void RayCast2D::set_cast_to(const Vector2 &p_point) { - cast_to = p_point; +void RayCast2D::set_target_position(const Vector2 &p_point) { + target_position = p_point; if (is_inside_tree() && (Engine::get_singleton()->is_editor_hint() || get_tree()->is_debugging_collisions_hint())) { update(); } } -Vector2 RayCast2D::get_cast_to() const { - return cast_to; +Vector2 RayCast2D::get_target_position() const { + return target_position; } void RayCast2D::set_collision_mask(uint32_t p_mask) { @@ -160,8 +160,8 @@ void RayCast2D::_notification(int p_what) { break; } Transform2D xf; - xf.rotate(cast_to.angle()); - xf.translate(Vector2(cast_to.length(), 0)); + xf.rotate(target_position.angle()); + xf.translate(Vector2(target_position.length(), 0)); // Draw an arrow indicating where the RayCast is pointing to Color draw_col = get_tree()->get_debug_collisions_color(); @@ -171,7 +171,7 @@ void RayCast2D::_notification(int p_what) { draw_col.g = g; draw_col.b = g; } - draw_line(Vector2(), cast_to, draw_col, 2); + draw_line(Vector2(), target_position, draw_col, 2); Vector<Vector2> pts; float tsize = 8; pts.push_back(xf.xform(Vector2(tsize, 0))); @@ -206,7 +206,7 @@ void RayCast2D::_update_raycast_state() { Transform2D gt = get_global_transform(); - Vector2 to = cast_to; + Vector2 to = target_position; if (to == Vector2()) { to = Vector2(0, 0.01); } @@ -280,8 +280,8 @@ void RayCast2D::_bind_methods() { ClassDB::bind_method(D_METHOD("set_enabled", "enabled"), &RayCast2D::set_enabled); ClassDB::bind_method(D_METHOD("is_enabled"), &RayCast2D::is_enabled); - ClassDB::bind_method(D_METHOD("set_cast_to", "local_point"), &RayCast2D::set_cast_to); - ClassDB::bind_method(D_METHOD("get_cast_to"), &RayCast2D::get_cast_to); + ClassDB::bind_method(D_METHOD("set_target_position", "local_point"), &RayCast2D::set_target_position); + ClassDB::bind_method(D_METHOD("get_target_position"), &RayCast2D::get_target_position); ClassDB::bind_method(D_METHOD("is_colliding"), &RayCast2D::is_colliding); ClassDB::bind_method(D_METHOD("force_raycast_update"), &RayCast2D::force_raycast_update); @@ -316,7 +316,7 @@ void RayCast2D::_bind_methods() { ADD_PROPERTY(PropertyInfo(Variant::BOOL, "enabled"), "set_enabled", "is_enabled"); ADD_PROPERTY(PropertyInfo(Variant::BOOL, "exclude_parent"), "set_exclude_parent_body", "get_exclude_parent_body"); - ADD_PROPERTY(PropertyInfo(Variant::VECTOR2, "cast_to"), "set_cast_to", "get_cast_to"); + ADD_PROPERTY(PropertyInfo(Variant::VECTOR2, "target_position"), "set_target_position", "get_target_position"); ADD_PROPERTY(PropertyInfo(Variant::INT, "collision_mask", PROPERTY_HINT_LAYERS_2D_PHYSICS), "set_collision_mask", "get_collision_mask"); ADD_GROUP("Collide With", "collide_with"); @@ -329,7 +329,7 @@ RayCast2D::RayCast2D() { collided = false; against_shape = 0; collision_mask = 1; - cast_to = Vector2(0, 50); + target_position = Vector2(0, 50); exclude_parent_body = true; collide_with_bodies = true; collide_with_areas = false; diff --git a/scene/2d/ray_cast_2d.h b/scene/2d/ray_cast_2d.h index 6accc264a0..14932f782b 100644 --- a/scene/2d/ray_cast_2d.h +++ b/scene/2d/ray_cast_2d.h @@ -46,7 +46,7 @@ class RayCast2D : public Node2D { uint32_t collision_mask; bool exclude_parent_body; - Vector2 cast_to; + Vector2 target_position; bool collide_with_areas; bool collide_with_bodies; @@ -66,8 +66,8 @@ public: void set_enabled(bool p_enabled); bool is_enabled() const; - void set_cast_to(const Vector2 &p_point); - Vector2 get_cast_to() const; + void set_target_position(const Vector2 &p_point); + Vector2 get_target_position() const; void set_collision_mask(uint32_t p_mask); uint32_t get_collision_mask() const; diff --git a/scene/2d/tile_map.cpp b/scene/2d/tile_map.cpp index c7a809f6d8..c2951559a4 100644 --- a/scene/2d/tile_map.cpp +++ b/scene/2d/tile_map.cpp @@ -412,6 +412,9 @@ void TileMap::update_dirty_quadrants() { vs->canvas_item_set_light_mask(canvas_item, get_light_mask()); vs->canvas_item_set_z_index(canvas_item, z_index); + vs->canvas_item_set_default_texture_filter(canvas_item, RS::CanvasItemTextureFilter(CanvasItem::get_texture_filter())); + vs->canvas_item_set_default_texture_repeat(canvas_item, RS::CanvasItemTextureRepeat(CanvasItem::get_texture_repeat())); + q.canvas_items.push_back(canvas_item); if (debug_shapes) { @@ -1687,6 +1690,28 @@ bool TileMap::get_clip_uv() const { return clip_uv; } +void TileMap::set_texture_filter(TextureFilter p_texture_filter) { + CanvasItem::set_texture_filter(p_texture_filter); + for (Map<PosKey, Quadrant>::Element *F = quadrant_map.front(); F; F = F->next()) { + Quadrant &q = F->get(); + for (List<RID>::Element *E = q.canvas_items.front(); E; E = E->next()) { + RenderingServer::get_singleton()->canvas_item_set_default_texture_filter(E->get(), RS::CanvasItemTextureFilter(p_texture_filter)); + _make_quadrant_dirty(F); + } + } +} + +void TileMap::set_texture_repeat(CanvasItem::TextureRepeat p_texture_repeat) { + CanvasItem::set_texture_repeat(p_texture_repeat); + for (Map<PosKey, Quadrant>::Element *F = quadrant_map.front(); F; F = F->next()) { + Quadrant &q = F->get(); + for (List<RID>::Element *E = q.canvas_items.front(); E; E = E->next()) { + RenderingServer::get_singleton()->canvas_item_set_default_texture_repeat(E->get(), RS::CanvasItemTextureRepeat(p_texture_repeat)); + _make_quadrant_dirty(F); + } + } +} + String TileMap::get_configuration_warning() const { String warning = Node2D::get_configuration_warning(); diff --git a/scene/2d/tile_map.h b/scene/2d/tile_map.h index 7a2a3e412c..b9dd8f5405 100644 --- a/scene/2d/tile_map.h +++ b/scene/2d/tile_map.h @@ -342,6 +342,10 @@ public: String get_configuration_warning() const override; + virtual void set_texture_filter(CanvasItem::TextureFilter p_texture_filter) override; + + virtual void set_texture_repeat(CanvasItem::TextureRepeat p_texture_repeat) override; + void fix_invalid_tiles(); void clear(); diff --git a/scene/3d/ray_cast_3d.cpp b/scene/3d/ray_cast_3d.cpp index 2a922e3cda..69f09ef569 100644 --- a/scene/3d/ray_cast_3d.cpp +++ b/scene/3d/ray_cast_3d.cpp @@ -35,8 +35,8 @@ #include "mesh_instance_3d.h" #include "servers/physics_server_3d.h" -void RayCast3D::set_cast_to(const Vector3 &p_point) { - cast_to = p_point; +void RayCast3D::set_target_position(const Vector3 &p_point) { + target_position = p_point; if (is_inside_tree() && (Engine::get_singleton()->is_editor_hint() || get_tree()->is_debugging_collisions_hint())) { update_gizmo(); } @@ -45,8 +45,8 @@ void RayCast3D::set_cast_to(const Vector3 &p_point) { } } -Vector3 RayCast3D::get_cast_to() const { - return cast_to; +Vector3 RayCast3D::get_target_position() const { + return target_position; } void RayCast3D::set_collision_mask(uint32_t p_mask) { @@ -202,7 +202,7 @@ void RayCast3D::_update_raycast_state() { Transform gt = get_global_transform(); - Vector3 to = cast_to; + Vector3 to = target_position; if (to == Vector3()) { to = Vector3(0, 0.01, 0); } @@ -276,8 +276,8 @@ void RayCast3D::_bind_methods() { ClassDB::bind_method(D_METHOD("set_enabled", "enabled"), &RayCast3D::set_enabled); ClassDB::bind_method(D_METHOD("is_enabled"), &RayCast3D::is_enabled); - ClassDB::bind_method(D_METHOD("set_cast_to", "local_point"), &RayCast3D::set_cast_to); - ClassDB::bind_method(D_METHOD("get_cast_to"), &RayCast3D::get_cast_to); + ClassDB::bind_method(D_METHOD("set_target_position", "local_point"), &RayCast3D::set_target_position); + ClassDB::bind_method(D_METHOD("get_target_position"), &RayCast3D::get_target_position); ClassDB::bind_method(D_METHOD("is_colliding"), &RayCast3D::is_colliding); ClassDB::bind_method(D_METHOD("force_raycast_update"), &RayCast3D::force_raycast_update); @@ -312,7 +312,7 @@ void RayCast3D::_bind_methods() { ADD_PROPERTY(PropertyInfo(Variant::BOOL, "enabled"), "set_enabled", "is_enabled"); ADD_PROPERTY(PropertyInfo(Variant::BOOL, "exclude_parent"), "set_exclude_parent_body", "get_exclude_parent_body"); - ADD_PROPERTY(PropertyInfo(Variant::VECTOR3, "cast_to"), "set_cast_to", "get_cast_to"); + ADD_PROPERTY(PropertyInfo(Variant::VECTOR3, "target_position"), "set_target_position", "get_target_position"); ADD_PROPERTY(PropertyInfo(Variant::INT, "collision_mask", PROPERTY_HINT_LAYERS_3D_PHYSICS), "set_collision_mask", "get_collision_mask"); ADD_GROUP("Collide With", "collide_with"); @@ -360,7 +360,7 @@ void RayCast3D::_update_debug_shape() { Vector<Vector3> verts; verts.push_back(Vector3()); - verts.push_back(cast_to); + verts.push_back(target_position); a[Mesh::ARRAY_VERTEX] = verts; mesh->add_surface_from_arrays(Mesh::PRIMITIVE_LINES, a); @@ -387,7 +387,7 @@ RayCast3D::RayCast3D() { collided = false; against_shape = 0; collision_mask = 1; - cast_to = Vector3(0, -1, 0); + target_position = Vector3(0, -1, 0); debug_shape = nullptr; exclude_parent_body = true; collide_with_areas = false; diff --git a/scene/3d/ray_cast_3d.h b/scene/3d/ray_cast_3d.h index 8f617e5491..f4fe7ba621 100644 --- a/scene/3d/ray_cast_3d.h +++ b/scene/3d/ray_cast_3d.h @@ -43,7 +43,7 @@ class RayCast3D : public Node3D { Vector3 collision_point; Vector3 collision_normal; - Vector3 cast_to; + Vector3 target_position; Set<RID> exclude; uint32_t collision_mask; @@ -74,8 +74,8 @@ public: void set_enabled(bool p_enabled); bool is_enabled() const; - void set_cast_to(const Vector3 &p_point); - Vector3 get_cast_to() const; + void set_target_position(const Vector3 &p_point); + Vector3 get_target_position() const; void set_collision_mask(uint32_t p_mask); uint32_t get_collision_mask() const; diff --git a/scene/gui/base_button.cpp b/scene/gui/base_button.cpp index aaae03df68..2e77d20d4e 100644 --- a/scene/gui/base_button.cpp +++ b/scene/gui/base_button.cpp @@ -326,12 +326,12 @@ bool BaseButton::is_keep_pressed_outside() const { return keep_pressed_outside; } -void BaseButton::set_shortcut(const Ref<ShortCut> &p_shortcut) { +void BaseButton::set_shortcut(const Ref<Shortcut> &p_shortcut) { shortcut = p_shortcut; set_process_unhandled_input(shortcut.is_valid()); } -Ref<ShortCut> BaseButton::get_shortcut() const { +Ref<Shortcut> BaseButton::get_shortcut() const { return shortcut; } @@ -345,7 +345,7 @@ String BaseButton::get_tooltip(const Point2 &p_pos) const { String tooltip = Control::get_tooltip(p_pos); if (shortcut_in_tooltip && shortcut.is_valid() && shortcut->is_valid()) { String text = shortcut->get_name() + " (" + shortcut->get_as_text() + ")"; - if (shortcut->get_name().nocasecmp_to(tooltip) != 0) { + if (tooltip != String() && shortcut->get_name().nocasecmp_to(tooltip) != 0) { text += "\n" + tooltip; } tooltip = text; @@ -414,7 +414,7 @@ void BaseButton::_bind_methods() { ADD_PROPERTY(PropertyInfo(Variant::INT, "button_mask", PROPERTY_HINT_FLAGS, "Mouse Left, Mouse Right, Mouse Middle"), "set_button_mask", "get_button_mask"); ADD_PROPERTY(PropertyInfo(Variant::INT, "enabled_focus_mode", PROPERTY_HINT_ENUM, "None,Click,All"), "set_enabled_focus_mode", "get_enabled_focus_mode"); ADD_PROPERTY(PropertyInfo(Variant::BOOL, "keep_pressed_outside"), "set_keep_pressed_outside", "is_keep_pressed_outside"); - ADD_PROPERTY(PropertyInfo(Variant::OBJECT, "shortcut", PROPERTY_HINT_RESOURCE_TYPE, "ShortCut"), "set_shortcut", "get_shortcut"); + ADD_PROPERTY(PropertyInfo(Variant::OBJECT, "shortcut", PROPERTY_HINT_RESOURCE_TYPE, "Shortcut"), "set_shortcut", "get_shortcut"); ADD_PROPERTY(PropertyInfo(Variant::OBJECT, "group", PROPERTY_HINT_RESOURCE_TYPE, "ButtonGroup"), "set_button_group", "get_button_group"); BIND_ENUM_CONSTANT(DRAW_NORMAL); diff --git a/scene/gui/base_button.h b/scene/gui/base_button.h index 12272446d5..8e71931f8b 100644 --- a/scene/gui/base_button.h +++ b/scene/gui/base_button.h @@ -50,7 +50,7 @@ private: bool shortcut_in_tooltip; bool keep_pressed_outside; FocusMode enabled_focus_mode; - Ref<ShortCut> shortcut; + Ref<Shortcut> shortcut; ActionMode action_mode; struct Status { @@ -118,8 +118,8 @@ public: void set_enabled_focus_mode(FocusMode p_mode); FocusMode get_enabled_focus_mode() const; - void set_shortcut(const Ref<ShortCut> &p_shortcut); - Ref<ShortCut> get_shortcut() const; + void set_shortcut(const Ref<Shortcut> &p_shortcut); + Ref<Shortcut> get_shortcut() const; virtual String get_tooltip(const Point2 &p_pos) const override; diff --git a/scene/gui/code_edit.cpp b/scene/gui/code_edit.cpp new file mode 100644 index 0000000000..56cf432b38 --- /dev/null +++ b/scene/gui/code_edit.cpp @@ -0,0 +1,444 @@ +/*************************************************************************/ +/* code_edit.cpp */ +/*************************************************************************/ +/* This file is part of: */ +/* GODOT ENGINE */ +/* https://godotengine.org */ +/*************************************************************************/ +/* Copyright (c) 2007-2020 Juan Linietsky, Ariel Manzur. */ +/* Copyright (c) 2014-2020 Godot Engine contributors (cf. AUTHORS.md). */ +/* */ +/* Permission is hereby granted, free of charge, to any person obtaining */ +/* a copy of this software and associated documentation files (the */ +/* "Software"), to deal in the Software without restriction, including */ +/* without limitation the rights to use, copy, modify, merge, publish, */ +/* distribute, sublicense, and/or sell copies of the Software, and to */ +/* permit persons to whom the Software is furnished to do so, subject to */ +/* the following conditions: */ +/* */ +/* The above copyright notice and this permission notice shall be */ +/* included in all copies or substantial portions of the Software. */ +/* */ +/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */ +/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */ +/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*/ +/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */ +/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */ +/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */ +/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +/*************************************************************************/ + +#include "code_edit.h" + +void CodeEdit::_notification(int p_what) { + switch (p_what) { + case NOTIFICATION_THEME_CHANGED: + case NOTIFICATION_ENTER_TREE: { + set_gutter_width(main_gutter, cache.row_height); + set_gutter_width(line_number_gutter, (line_number_digits + 1) * cache.font->get_char_size('0').width); + set_gutter_width(fold_gutter, cache.row_height / 1.2); + + breakpoint_color = get_theme_color("breakpoint_color"); + breakpoint_icon = get_theme_icon("breakpoint"); + + bookmark_color = get_theme_color("bookmark_color"); + bookmark_icon = get_theme_icon("bookmark"); + + executing_line_color = get_theme_color("executing_line_color"); + executing_line_icon = get_theme_icon("executing_line"); + + line_number_color = get_theme_color("line_number_color"); + + folding_color = get_theme_color("code_folding_color"); + can_fold_icon = get_theme_icon("can_fold"); + folded_icon = get_theme_icon("folded"); + } break; + case NOTIFICATION_DRAW: { + } break; + } +} + +/* Main Gutter */ +void CodeEdit::_update_draw_main_gutter() { + set_gutter_draw(main_gutter, draw_breakpoints || draw_bookmarks || draw_executing_lines); +} + +void CodeEdit::set_draw_breakpoints_gutter(bool p_draw) { + draw_breakpoints = p_draw; + set_gutter_clickable(main_gutter, p_draw); + _update_draw_main_gutter(); +} + +bool CodeEdit::is_drawing_breakpoints_gutter() const { + return draw_breakpoints; +} + +void CodeEdit::set_draw_bookmarks_gutter(bool p_draw) { + draw_bookmarks = p_draw; + _update_draw_main_gutter(); +} + +bool CodeEdit::is_drawing_bookmarks_gutter() const { + return draw_bookmarks; +} + +void CodeEdit::set_draw_executing_lines_gutter(bool p_draw) { + draw_executing_lines = p_draw; + _update_draw_main_gutter(); +} + +bool CodeEdit::is_drawing_executing_lines_gutter() const { + return draw_executing_lines; +} + +void CodeEdit::_main_gutter_draw_callback(int p_line, int p_gutter, const Rect2 &p_region) { + if (draw_breakpoints && is_line_breakpointed(p_line)) { + int padding = p_region.size.x / 6; + + Rect2 breakpoint_region = p_region; + breakpoint_region.position += Point2(padding, padding); + breakpoint_region.size -= Point2(padding, padding) * 2; + breakpoint_icon->draw_rect(get_canvas_item(), breakpoint_region, false, breakpoint_color); + } + + if (draw_bookmarks && is_line_bookmarked(p_line)) { + int horizontal_padding = p_region.size.x / 2; + int vertical_padding = p_region.size.y / 4; + + Rect2 bookmark_region = p_region; + bookmark_region.position += Point2(horizontal_padding, 0); + bookmark_region.size -= Point2(horizontal_padding * 1.1, vertical_padding); + bookmark_icon->draw_rect(get_canvas_item(), bookmark_region, false, bookmark_color); + } + + if (draw_executing_lines && is_line_executing(p_line)) { + int horizontal_padding = p_region.size.x / 10; + int vertical_padding = p_region.size.y / 4; + + Rect2 executing_line_region = p_region; + executing_line_region.position += Point2(horizontal_padding, vertical_padding); + executing_line_region.size -= Point2(horizontal_padding, vertical_padding) * 2; + executing_line_icon->draw_rect(get_canvas_item(), executing_line_region, false, executing_line_color); + } +} + +// Breakpoints +void CodeEdit::set_line_as_breakpoint(int p_line, bool p_breakpointed) { + int mask = get_line_gutter_metadata(p_line, main_gutter); + set_line_gutter_metadata(p_line, main_gutter, p_breakpointed ? mask | MAIN_GUTTER_BREAKPOINT : mask & ~MAIN_GUTTER_BREAKPOINT); + if (p_breakpointed) { + breakpointed_lines[p_line] = true; + } else if (breakpointed_lines.has(p_line)) { + breakpointed_lines.erase(p_line); + } + emit_signal("breakpoint_toggled", p_line); +} + +bool CodeEdit::is_line_breakpointed(int p_line) const { + return (int)get_line_gutter_metadata(p_line, main_gutter) & MAIN_GUTTER_BREAKPOINT; +} + +void CodeEdit::clear_breakpointed_lines() { + for (int i = 0; i < get_line_count(); i++) { + if (is_line_breakpointed(i)) { + set_line_as_breakpoint(i, false); + } + } +} + +Array CodeEdit::get_breakpointed_lines() const { + Array ret; + for (int i = 0; i < get_line_count(); i++) { + if (is_line_breakpointed(i)) { + ret.append(i); + } + } + return ret; +} + +// Bookmarks +void CodeEdit::set_line_as_bookmarked(int p_line, bool p_bookmarked) { + int mask = get_line_gutter_metadata(p_line, main_gutter); + set_line_gutter_metadata(p_line, main_gutter, p_bookmarked ? mask | MAIN_GUTTER_BOOKMARK : mask & ~MAIN_GUTTER_BOOKMARK); +} + +bool CodeEdit::is_line_bookmarked(int p_line) const { + return (int)get_line_gutter_metadata(p_line, main_gutter) & MAIN_GUTTER_BOOKMARK; +} + +void CodeEdit::clear_bookmarked_lines() { + for (int i = 0; i < get_line_count(); i++) { + if (is_line_bookmarked(i)) { + set_line_as_bookmarked(i, false); + } + } +} + +Array CodeEdit::get_bookmarked_lines() const { + Array ret; + for (int i = 0; i < get_line_count(); i++) { + if (is_line_bookmarked(i)) { + ret.append(i); + } + } + return ret; +} + +// executing lines +void CodeEdit::set_line_as_executing(int p_line, bool p_executing) { + int mask = get_line_gutter_metadata(p_line, main_gutter); + set_line_gutter_metadata(p_line, main_gutter, p_executing ? mask | MAIN_GUTTER_EXECUTING : mask & ~MAIN_GUTTER_EXECUTING); +} + +bool CodeEdit::is_line_executing(int p_line) const { + return (int)get_line_gutter_metadata(p_line, main_gutter) & MAIN_GUTTER_EXECUTING; +} + +void CodeEdit::clear_executing_lines() { + for (int i = 0; i < get_line_count(); i++) { + if (is_line_executing(i)) { + set_line_as_executing(i, false); + } + } +} + +Array CodeEdit::get_executing_lines() const { + Array ret; + for (int i = 0; i < get_line_count(); i++) { + if (is_line_executing(i)) { + ret.append(i); + } + } + return ret; +} + +/* Line numbers */ +void CodeEdit::set_draw_line_numbers(bool p_draw) { + set_gutter_draw(line_number_gutter, p_draw); +} + +bool CodeEdit::is_draw_line_numbers_enabled() const { + return is_gutter_drawn(line_number_gutter); +} + +void CodeEdit::set_line_numbers_zero_padded(bool p_zero_padded) { + p_zero_padded ? line_number_padding = "0" : line_number_padding = " "; + update(); +} + +bool CodeEdit::is_line_numbers_zero_padded() const { + return line_number_padding == "0"; +} + +void CodeEdit::_line_number_draw_callback(int p_line, int p_gutter, const Rect2 &p_region) { + String fc = String::num(p_line + 1).lpad(line_number_digits, line_number_padding); + + int yofs = p_region.position.y + (cache.row_height - cache.font->get_height()) / 2; + Color number_color = get_line_gutter_item_color(p_line, line_number_gutter); + if (number_color == Color(1, 1, 1)) { + number_color = line_number_color; + } + cache.font->draw(get_canvas_item(), Point2(p_region.position.x, yofs + cache.font->get_ascent()), fc, number_color); +} + +/* Fold Gutter */ +void CodeEdit::set_draw_fold_gutter(bool p_draw) { + set_gutter_draw(fold_gutter, p_draw); +} + +bool CodeEdit::is_drawing_fold_gutter() const { + return is_gutter_drawn(fold_gutter); +} + +void CodeEdit::_fold_gutter_draw_callback(int p_line, int p_gutter, Rect2 p_region) { + if (!can_fold(p_line) && !is_folded(p_line)) { + set_line_gutter_clickable(p_line, fold_gutter, false); + return; + } + set_line_gutter_clickable(p_line, fold_gutter, true); + + int horizontal_padding = p_region.size.x / 10; + int vertical_padding = p_region.size.y / 6; + + p_region.position += Point2(horizontal_padding, vertical_padding); + p_region.size -= Point2(horizontal_padding, vertical_padding) * 2; + + if (can_fold(p_line)) { + can_fold_icon->draw_rect(get_canvas_item(), p_region, false, folding_color); + return; + } + folded_icon->draw_rect(get_canvas_item(), p_region, false, folding_color); +} + +void CodeEdit::_bind_methods() { + /* Main Gutter */ + ClassDB::bind_method(D_METHOD("_main_gutter_draw_callback"), &CodeEdit::_main_gutter_draw_callback); + + ClassDB::bind_method(D_METHOD("set_draw_breakpoints_gutter", "enable"), &CodeEdit::set_draw_breakpoints_gutter); + ClassDB::bind_method(D_METHOD("is_drawing_breakpoints_gutter"), &CodeEdit::is_drawing_breakpoints_gutter); + + ClassDB::bind_method(D_METHOD("set_draw_bookmarks_gutter", "enable"), &CodeEdit::set_draw_bookmarks_gutter); + ClassDB::bind_method(D_METHOD("is_drawing_bookmarks_gutter"), &CodeEdit::is_drawing_bookmarks_gutter); + + ClassDB::bind_method(D_METHOD("set_draw_executing_lines_gutter", "enable"), &CodeEdit::set_draw_executing_lines_gutter); + ClassDB::bind_method(D_METHOD("is_drawing_executing_lines_gutter"), &CodeEdit::is_drawing_executing_lines_gutter); + + // Breakpoints + ClassDB::bind_method(D_METHOD("set_line_as_breakpoint", "line", "breakpointed"), &CodeEdit::set_line_as_breakpoint); + ClassDB::bind_method(D_METHOD("is_line_breakpointed", "line"), &CodeEdit::is_line_breakpointed); + ClassDB::bind_method(D_METHOD("clear_breakpointed_lines"), &CodeEdit::clear_breakpointed_lines); + ClassDB::bind_method(D_METHOD("get_breakpointed_lines"), &CodeEdit::get_breakpointed_lines); + + // Bookmarks + ClassDB::bind_method(D_METHOD("set_line_as_bookmarked", "line", "bookmarked"), &CodeEdit::set_line_as_bookmarked); + ClassDB::bind_method(D_METHOD("is_line_bookmarked", "line"), &CodeEdit::is_line_bookmarked); + ClassDB::bind_method(D_METHOD("clear_bookmarked_lines"), &CodeEdit::clear_bookmarked_lines); + ClassDB::bind_method(D_METHOD("get_bookmarked_lines"), &CodeEdit::get_bookmarked_lines); + + // executing lines + ClassDB::bind_method(D_METHOD("set_line_as_executing", "line", "executing"), &CodeEdit::set_line_as_executing); + ClassDB::bind_method(D_METHOD("is_line_executing", "line"), &CodeEdit::is_line_executing); + ClassDB::bind_method(D_METHOD("clear_executing_lines"), &CodeEdit::clear_executing_lines); + ClassDB::bind_method(D_METHOD("get_executing_lines"), &CodeEdit::get_executing_lines); + + /* Line numbers */ + ClassDB::bind_method(D_METHOD("_line_number_draw_callback"), &CodeEdit::_line_number_draw_callback); + + ClassDB::bind_method(D_METHOD("set_draw_line_numbers", "enable"), &CodeEdit::set_draw_line_numbers); + ClassDB::bind_method(D_METHOD("is_draw_line_numbers_enabled"), &CodeEdit::is_draw_line_numbers_enabled); + ClassDB::bind_method(D_METHOD("set_line_numbers_zero_padded", "enable"), &CodeEdit::set_line_numbers_zero_padded); + ClassDB::bind_method(D_METHOD("is_line_numbers_zero_padded"), &CodeEdit::is_line_numbers_zero_padded); + + /* Fold Gutter */ + ClassDB::bind_method(D_METHOD("_fold_gutter_draw_callback"), &CodeEdit::_fold_gutter_draw_callback); + + ClassDB::bind_method(D_METHOD("set_draw_fold_gutter", "enable"), &CodeEdit::set_draw_fold_gutter); + ClassDB::bind_method(D_METHOD("is_drawing_fold_gutter"), &CodeEdit::is_drawing_fold_gutter); + + ADD_PROPERTY(PropertyInfo(Variant::BOOL, "draw_breakpoints_gutter"), "set_draw_breakpoints_gutter", "is_drawing_breakpoints_gutter"); + + ADD_PROPERTY(PropertyInfo(Variant::BOOL, "draw_bookmarks"), "set_draw_bookmarks_gutter", "is_drawing_bookmarks_gutter"); + + ADD_PROPERTY(PropertyInfo(Variant::BOOL, "draw_executing_lines"), "set_draw_executing_lines_gutter", "is_drawing_executing_lines_gutter"); + + ADD_PROPERTY(PropertyInfo(Variant::BOOL, "draw_line_numbers"), "set_draw_line_numbers", "is_draw_line_numbers_enabled"); + ADD_PROPERTY(PropertyInfo(Variant::BOOL, "zero_pad_line_numbers"), "set_line_numbers_zero_padded", "is_line_numbers_zero_padded"); + + ADD_PROPERTY(PropertyInfo(Variant::BOOL, "draw_fold_gutter"), "set_draw_fold_gutter", "is_drawing_fold_gutter"); + + ADD_SIGNAL(MethodInfo("breakpoint_toggled", PropertyInfo(Variant::INT, "line"))); +} + +void CodeEdit::_gutter_clicked(int p_line, int p_gutter) { + if (p_gutter == main_gutter) { + if (draw_breakpoints) { + set_line_as_breakpoint(p_line, !is_line_breakpointed(p_line)); + } + return; + } + + if (p_gutter == line_number_gutter) { + cursor_set_line(p_line); + return; + } + + if (p_gutter == fold_gutter) { + if (is_folded(p_line)) { + unfold_line(p_line); + } else if (can_fold(p_line)) { + fold_line(p_line); + } + return; + } +} + +void CodeEdit::_lines_edited_from(int p_from_line, int p_to_line) { + if (p_from_line == p_to_line) { + return; + } + + int lc = get_line_count(); + line_number_digits = 1; + while (lc /= 10) { + line_number_digits++; + } + set_gutter_width(line_number_gutter, (line_number_digits + 1) * cache.font->get_char_size('0').width); + + int from_line = MIN(p_from_line, p_to_line); + int line_count = (p_to_line - p_from_line); + List<int> breakpoints; + breakpointed_lines.get_key_list(&breakpoints); + for (const List<int>::Element *E = breakpoints.front(); E; E = E->next()) { + int line = E->get(); + if (line <= from_line) { + continue; + } + breakpointed_lines.erase(line); + + emit_signal("breakpoint_toggled", line); + if (line_count > 0 || line >= p_from_line) { + emit_signal("breakpoint_toggled", line + line_count); + breakpointed_lines[line + line_count] = true; + continue; + } + } +} + +void CodeEdit::_update_gutter_indexes() { + for (int i = 0; i < get_gutter_count(); i++) { + if (get_gutter_name(i) == "main_gutter") { + main_gutter = i; + continue; + } + + if (get_gutter_name(i) == "line_numbers") { + line_number_gutter = i; + continue; + } + + if (get_gutter_name(i) == "fold_gutter") { + fold_gutter = i; + continue; + } + } +} + +CodeEdit::CodeEdit() { + /* Gutters */ + int gutter_idx = 0; + + /* Main Gutter */ + add_gutter(); + set_gutter_name(gutter_idx, "main_gutter"); + set_gutter_draw(gutter_idx, false); + set_gutter_overwritable(gutter_idx, true); + set_gutter_type(gutter_idx, GUTTER_TPYE_CUSTOM); + set_gutter_custom_draw(gutter_idx, this, "_main_gutter_draw_callback"); + gutter_idx++; + + /* Line numbers */ + add_gutter(); + set_gutter_name(gutter_idx, "line_numbers"); + set_gutter_draw(gutter_idx, false); + set_gutter_type(gutter_idx, GUTTER_TPYE_CUSTOM); + set_gutter_custom_draw(gutter_idx, this, "_line_number_draw_callback"); + gutter_idx++; + + /* Fold Gutter */ + add_gutter(); + set_gutter_name(gutter_idx, "fold_gutter"); + set_gutter_draw(gutter_idx, false); + set_gutter_type(gutter_idx, GUTTER_TPYE_CUSTOM); + set_gutter_custom_draw(gutter_idx, this, "_fold_gutter_draw_callback"); + gutter_idx++; + + connect("lines_edited_from", callable_mp(this, &CodeEdit::_lines_edited_from)); + connect("gutter_clicked", callable_mp(this, &CodeEdit::_gutter_clicked)); + + connect("gutter_added", callable_mp(this, &CodeEdit::_update_gutter_indexes)); + connect("gutter_removed", callable_mp(this, &CodeEdit::_update_gutter_indexes)); + _update_gutter_indexes(); +} + +CodeEdit::~CodeEdit() { +} diff --git a/scene/gui/code_edit.h b/scene/gui/code_edit.h new file mode 100644 index 0000000000..c989e5ed79 --- /dev/null +++ b/scene/gui/code_edit.h @@ -0,0 +1,135 @@ +/*************************************************************************/ +/* code_edit.h */ +/*************************************************************************/ +/* This file is part of: */ +/* GODOT ENGINE */ +/* https://godotengine.org */ +/*************************************************************************/ +/* Copyright (c) 2007-2020 Juan Linietsky, Ariel Manzur. */ +/* Copyright (c) 2014-2020 Godot Engine contributors (cf. AUTHORS.md). */ +/* */ +/* Permission is hereby granted, free of charge, to any person obtaining */ +/* a copy of this software and associated documentation files (the */ +/* "Software"), to deal in the Software without restriction, including */ +/* without limitation the rights to use, copy, modify, merge, publish, */ +/* distribute, sublicense, and/or sell copies of the Software, and to */ +/* permit persons to whom the Software is furnished to do so, subject to */ +/* the following conditions: */ +/* */ +/* The above copyright notice and this permission notice shall be */ +/* included in all copies or substantial portions of the Software. */ +/* */ +/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */ +/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */ +/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*/ +/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */ +/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */ +/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */ +/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +/*************************************************************************/ + +#ifndef CODEEDIT_H +#define CODEEDIT_H + +#include "scene/gui/text_edit.h" + +class CodeEdit : public TextEdit { + GDCLASS(CodeEdit, TextEdit) + +private: + /* Main Gutter */ + enum MainGutterType { + MAIN_GUTTER_BREAKPOINT = 0x01, + MAIN_GUTTER_BOOKMARK = 0x02, + MAIN_GUTTER_EXECUTING = 0x04 + }; + + int main_gutter = -1; + void _update_draw_main_gutter(); + void _main_gutter_draw_callback(int p_line, int p_gutter, const Rect2 &p_region); + + // breakpoints + HashMap<int, bool> breakpointed_lines; + bool draw_breakpoints = false; + Color breakpoint_color = Color(1, 1, 1); + Ref<Texture2D> breakpoint_icon = Ref<Texture2D>(); + + // bookmarks + bool draw_bookmarks = false; + Color bookmark_color = Color(1, 1, 1); + Ref<Texture2D> bookmark_icon = Ref<Texture2D>(); + + // executing lines + bool draw_executing_lines = false; + Color executing_line_color = Color(1, 1, 1); + Ref<Texture2D> executing_line_icon = Ref<Texture2D>(); + + /* Line numbers */ + int line_number_gutter = -1; + int line_number_digits = 0; + String line_number_padding = " "; + Color line_number_color = Color(1, 1, 1); + void _line_number_draw_callback(int p_line, int p_gutter, const Rect2 &p_region); + + /* Fold Gutter */ + int fold_gutter = -1; + bool draw_fold_gutter = false; + Color folding_color = Color(1, 1, 1); + Ref<Texture2D> can_fold_icon = Ref<Texture2D>(); + Ref<Texture2D> folded_icon = Ref<Texture2D>(); + void _fold_gutter_draw_callback(int p_line, int p_gutter, Rect2 p_region); + + void _gutter_clicked(int p_line, int p_gutter); + void _lines_edited_from(int p_from_line, int p_to_line); + + void _update_gutter_indexes(); + +protected: + void _notification(int p_what); + + static void _bind_methods(); + +public: + /* Main Gutter */ + void set_draw_breakpoints_gutter(bool p_draw); + bool is_drawing_breakpoints_gutter() const; + + void set_draw_bookmarks_gutter(bool p_draw); + bool is_drawing_bookmarks_gutter() const; + + void set_draw_executing_lines_gutter(bool p_draw); + bool is_drawing_executing_lines_gutter() const; + + // breakpoints + void set_line_as_breakpoint(int p_line, bool p_breakpointed); + bool is_line_breakpointed(int p_line) const; + void clear_breakpointed_lines(); + Array get_breakpointed_lines() const; + + // bookmarks + void set_line_as_bookmarked(int p_line, bool p_bookmarked); + bool is_line_bookmarked(int p_line) const; + void clear_bookmarked_lines(); + Array get_bookmarked_lines() const; + + // executing lines + void set_line_as_executing(int p_line, bool p_executing); + bool is_line_executing(int p_line) const; + void clear_executing_lines(); + Array get_executing_lines() const; + + /* Line numbers */ + void set_draw_line_numbers(bool p_draw); + bool is_draw_line_numbers_enabled() const; + void set_line_numbers_zero_padded(bool p_zero_padded); + bool is_line_numbers_zero_padded() const; + + /* Fold gutter */ + void set_draw_fold_gutter(bool p_draw); + bool is_drawing_fold_gutter() const; + + CodeEdit(); + ~CodeEdit(); +}; + +#endif // CODEEDIT_H diff --git a/scene/gui/color_picker.cpp b/scene/gui/color_picker.cpp index fafbb298b7..cbe0367c7b 100644 --- a/scene/gui/color_picker.cpp +++ b/scene/gui/color_picker.cpp @@ -873,6 +873,7 @@ void ColorPickerButton::_color_changed(const Color &p_color) { void ColorPickerButton::_modal_closed() { emit_signal("popup_closed"); + set_pressed(false); } void ColorPickerButton::pressed() { @@ -976,9 +977,8 @@ void ColorPickerButton::_update_picker() { popup->add_child(picker); add_child(popup); picker->connect("color_changed", callable_mp(this, &ColorPickerButton::_color_changed)); - popup->connect("modal_closed", callable_mp(this, &ColorPickerButton::_modal_closed)); popup->connect("about_to_popup", callable_mp((BaseButton *)this, &BaseButton::set_pressed), varray(true)); - popup->connect("popup_hide", callable_mp((BaseButton *)this, &BaseButton::set_pressed), varray(false)); + popup->connect("popup_hide", callable_mp(this, &ColorPickerButton::_modal_closed)); picker->set_pick_color(color); picker->set_edit_alpha(edit_alpha); emit_signal("picker_created"); diff --git a/scene/gui/graph_edit.cpp b/scene/gui/graph_edit.cpp index 11b4c9e857..467d252c81 100644 --- a/scene/gui/graph_edit.cpp +++ b/scene/gui/graph_edit.cpp @@ -262,6 +262,7 @@ void GraphEdit::remove_child_notify(Node *p_child) { if (gn) { gn->disconnect("offset_changed", callable_mp(this, &GraphEdit::_graph_node_moved)); gn->disconnect("raise_request", callable_mp(this, &GraphEdit::_graph_node_raised)); + gn->disconnect("item_rect_changed", callable_mp((CanvasItem *)connections_layer, &CanvasItem::update)); } } diff --git a/scene/gui/popup_menu.cpp b/scene/gui/popup_menu.cpp index 2fdcf11ca8..bc70809ad5 100644 --- a/scene/gui/popup_menu.cpp +++ b/scene/gui/popup_menu.cpp @@ -717,7 +717,7 @@ void PopupMenu::add_multistate_item(const String &p_label, int p_max_states, int } #define ITEM_SETUP_WITH_SHORTCUT(p_shortcut, p_id, p_global) \ - ERR_FAIL_COND_MSG(p_shortcut.is_null(), "Cannot add item with invalid ShortCut."); \ + ERR_FAIL_COND_MSG(p_shortcut.is_null(), "Cannot add item with invalid Shortcut."); \ _ref_shortcut(p_shortcut); \ item.text = p_shortcut->get_name(); \ item.xl_text = tr(item.text); \ @@ -725,7 +725,7 @@ void PopupMenu::add_multistate_item(const String &p_label, int p_max_states, int item.shortcut = p_shortcut; \ item.shortcut_is_global = p_global; -void PopupMenu::add_shortcut(const Ref<ShortCut> &p_shortcut, int p_id, bool p_global) { +void PopupMenu::add_shortcut(const Ref<Shortcut> &p_shortcut, int p_id, bool p_global) { Item item; ITEM_SETUP_WITH_SHORTCUT(p_shortcut, p_id, p_global); items.push_back(item); @@ -733,7 +733,7 @@ void PopupMenu::add_shortcut(const Ref<ShortCut> &p_shortcut, int p_id, bool p_g child_controls_changed(); } -void PopupMenu::add_icon_shortcut(const Ref<Texture2D> &p_icon, const Ref<ShortCut> &p_shortcut, int p_id, bool p_global) { +void PopupMenu::add_icon_shortcut(const Ref<Texture2D> &p_icon, const Ref<Shortcut> &p_shortcut, int p_id, bool p_global) { Item item; ITEM_SETUP_WITH_SHORTCUT(p_shortcut, p_id, p_global); item.icon = p_icon; @@ -742,7 +742,7 @@ void PopupMenu::add_icon_shortcut(const Ref<Texture2D> &p_icon, const Ref<ShortC child_controls_changed(); } -void PopupMenu::add_check_shortcut(const Ref<ShortCut> &p_shortcut, int p_id, bool p_global) { +void PopupMenu::add_check_shortcut(const Ref<Shortcut> &p_shortcut, int p_id, bool p_global) { Item item; ITEM_SETUP_WITH_SHORTCUT(p_shortcut, p_id, p_global); item.checkable_type = Item::CHECKABLE_TYPE_CHECK_BOX; @@ -751,7 +751,7 @@ void PopupMenu::add_check_shortcut(const Ref<ShortCut> &p_shortcut, int p_id, bo child_controls_changed(); } -void PopupMenu::add_icon_check_shortcut(const Ref<Texture2D> &p_icon, const Ref<ShortCut> &p_shortcut, int p_id, bool p_global) { +void PopupMenu::add_icon_check_shortcut(const Ref<Texture2D> &p_icon, const Ref<Shortcut> &p_shortcut, int p_id, bool p_global) { Item item; ITEM_SETUP_WITH_SHORTCUT(p_shortcut, p_id, p_global); item.icon = p_icon; @@ -761,7 +761,7 @@ void PopupMenu::add_icon_check_shortcut(const Ref<Texture2D> &p_icon, const Ref< child_controls_changed(); } -void PopupMenu::add_radio_check_shortcut(const Ref<ShortCut> &p_shortcut, int p_id, bool p_global) { +void PopupMenu::add_radio_check_shortcut(const Ref<Shortcut> &p_shortcut, int p_id, bool p_global) { Item item; ITEM_SETUP_WITH_SHORTCUT(p_shortcut, p_id, p_global); item.checkable_type = Item::CHECKABLE_TYPE_RADIO_BUTTON; @@ -770,7 +770,7 @@ void PopupMenu::add_radio_check_shortcut(const Ref<ShortCut> &p_shortcut, int p_ child_controls_changed(); } -void PopupMenu::add_icon_radio_check_shortcut(const Ref<Texture2D> &p_icon, const Ref<ShortCut> &p_shortcut, int p_id, bool p_global) { +void PopupMenu::add_icon_radio_check_shortcut(const Ref<Texture2D> &p_icon, const Ref<Shortcut> &p_shortcut, int p_id, bool p_global) { Item item; ITEM_SETUP_WITH_SHORTCUT(p_shortcut, p_id, p_global); item.icon = p_icon; @@ -931,8 +931,8 @@ String PopupMenu::get_item_tooltip(int p_idx) const { return items[p_idx].tooltip; } -Ref<ShortCut> PopupMenu::get_item_shortcut(int p_idx) const { - ERR_FAIL_INDEX_V(p_idx, items.size(), Ref<ShortCut>()); +Ref<Shortcut> PopupMenu::get_item_shortcut(int p_idx) const { + ERR_FAIL_INDEX_V(p_idx, items.size(), Ref<Shortcut>()); return items[p_idx].shortcut; } @@ -970,7 +970,7 @@ void PopupMenu::set_item_tooltip(int p_idx, const String &p_tooltip) { control->update(); } -void PopupMenu::set_item_shortcut(int p_idx, const Ref<ShortCut> &p_shortcut, bool p_global) { +void PopupMenu::set_item_shortcut(int p_idx, const Ref<Shortcut> &p_shortcut, bool p_global) { ERR_FAIL_INDEX(p_idx, items.size()); if (items[p_idx].shortcut.is_valid()) { _unref_shortcut(items[p_idx].shortcut); @@ -1209,7 +1209,7 @@ Array PopupMenu::_get_items() const { return items; } -void PopupMenu::_ref_shortcut(Ref<ShortCut> p_sc) { +void PopupMenu::_ref_shortcut(Ref<Shortcut> p_sc) { if (!shortcut_refcount.has(p_sc)) { shortcut_refcount[p_sc] = 1; p_sc->connect("changed", callable_mp((CanvasItem *)this, &CanvasItem::update)); @@ -1218,7 +1218,7 @@ void PopupMenu::_ref_shortcut(Ref<ShortCut> p_sc) { } } -void PopupMenu::_unref_shortcut(Ref<ShortCut> p_sc) { +void PopupMenu::_unref_shortcut(Ref<Shortcut> p_sc) { ERR_FAIL_COND(!shortcut_refcount.has(p_sc)); shortcut_refcount[p_sc]--; if (shortcut_refcount[p_sc] == 0) { diff --git a/scene/gui/popup_menu.h b/scene/gui/popup_menu.h index 45a3336747..9535fd07d7 100644 --- a/scene/gui/popup_menu.h +++ b/scene/gui/popup_menu.h @@ -61,7 +61,7 @@ class PopupMenu : public Popup { int _ofs_cache; int _height_cache; int h_ofs; - Ref<ShortCut> shortcut; + Ref<Shortcut> shortcut; bool shortcut_is_global; bool shortcut_is_disabled; @@ -114,10 +114,10 @@ class PopupMenu : public Popup { Array _get_items() const; void _set_items(const Array &p_items); - Map<Ref<ShortCut>, int> shortcut_refcount; + Map<Ref<Shortcut>, int> shortcut_refcount; - void _ref_shortcut(Ref<ShortCut> p_sc); - void _unref_shortcut(Ref<ShortCut> p_sc); + void _ref_shortcut(Ref<Shortcut> p_sc); + void _unref_shortcut(Ref<Shortcut> p_sc); bool allow_search; uint64_t search_time_msec; @@ -145,12 +145,12 @@ public: void add_multistate_item(const String &p_label, int p_max_states, int p_default_state = 0, int p_id = -1, uint32_t p_accel = 0); - void add_shortcut(const Ref<ShortCut> &p_shortcut, int p_id = -1, bool p_global = false); - void add_icon_shortcut(const Ref<Texture2D> &p_icon, const Ref<ShortCut> &p_shortcut, int p_id = -1, bool p_global = false); - void add_check_shortcut(const Ref<ShortCut> &p_shortcut, int p_id = -1, bool p_global = false); - void add_icon_check_shortcut(const Ref<Texture2D> &p_icon, const Ref<ShortCut> &p_shortcut, int p_id = -1, bool p_global = false); - void add_radio_check_shortcut(const Ref<ShortCut> &p_shortcut, int p_id = -1, bool p_global = false); - void add_icon_radio_check_shortcut(const Ref<Texture2D> &p_icon, const Ref<ShortCut> &p_shortcut, int p_id = -1, bool p_global = false); + void add_shortcut(const Ref<Shortcut> &p_shortcut, int p_id = -1, bool p_global = false); + void add_icon_shortcut(const Ref<Texture2D> &p_icon, const Ref<Shortcut> &p_shortcut, int p_id = -1, bool p_global = false); + void add_check_shortcut(const Ref<Shortcut> &p_shortcut, int p_id = -1, bool p_global = false); + void add_icon_check_shortcut(const Ref<Texture2D> &p_icon, const Ref<Shortcut> &p_shortcut, int p_id = -1, bool p_global = false); + void add_radio_check_shortcut(const Ref<Shortcut> &p_shortcut, int p_id = -1, bool p_global = false); + void add_icon_radio_check_shortcut(const Ref<Texture2D> &p_icon, const Ref<Shortcut> &p_shortcut, int p_id = -1, bool p_global = false); void add_submenu_item(const String &p_label, const String &p_submenu, int p_id = -1); @@ -166,7 +166,7 @@ public: void set_item_as_checkable(int p_idx, bool p_checkable); void set_item_as_radio_checkable(int p_idx, bool p_radio_checkable); void set_item_tooltip(int p_idx, const String &p_tooltip); - void set_item_shortcut(int p_idx, const Ref<ShortCut> &p_shortcut, bool p_global = false); + void set_item_shortcut(int p_idx, const Ref<Shortcut> &p_shortcut, bool p_global = false); void set_item_h_offset(int p_idx, int p_offset); void set_item_multistate(int p_idx, int p_state); void toggle_item_multistate(int p_idx); @@ -189,7 +189,7 @@ public: bool is_item_radio_checkable(int p_idx) const; bool is_item_shortcut_disabled(int p_idx) const; String get_item_tooltip(int p_idx) const; - Ref<ShortCut> get_item_shortcut(int p_idx) const; + Ref<Shortcut> get_item_shortcut(int p_idx) const; int get_item_state(int p_idx) const; int get_current_index() const; diff --git a/scene/gui/rich_text_label.cpp b/scene/gui/rich_text_label.cpp index c62fd24a5e..fb4931ea91 100644 --- a/scene/gui/rich_text_label.cpp +++ b/scene/gui/rich_text_label.cpp @@ -256,7 +256,7 @@ int RichTextLabel::_process_line(ItemFrame *p_frame, const Vector2 &p_ofs, int & lh = line < l.height_caches.size() ? l.height_caches[line] : 1; \ line_ascent = line < l.ascent_caches.size() ? l.ascent_caches[line] : 1; \ line_descent = line < l.descent_caches.size() ? l.descent_caches[line] : 1; \ - if ((p_mode == PROCESS_DRAW) && (align != ALIGN_FILL)) { \ + if (align != ALIGN_FILL) { \ if (line < l.offset_caches.size()) { \ wofs = l.offset_caches[line]; \ } \ diff --git a/scene/gui/shortcut.cpp b/scene/gui/shortcut.cpp index 9f5b9c40c2..f8c7bc44a7 100644 --- a/scene/gui/shortcut.cpp +++ b/scene/gui/shortcut.cpp @@ -32,20 +32,20 @@ #include "core/os/keyboard.h" -void ShortCut::set_shortcut(const Ref<InputEvent> &p_shortcut) { +void Shortcut::set_shortcut(const Ref<InputEvent> &p_shortcut) { shortcut = p_shortcut; emit_changed(); } -Ref<InputEvent> ShortCut::get_shortcut() const { +Ref<InputEvent> Shortcut::get_shortcut() const { return shortcut; } -bool ShortCut::is_shortcut(const Ref<InputEvent> &p_event) const { +bool Shortcut::is_shortcut(const Ref<InputEvent> &p_event) const { return shortcut.is_valid() && shortcut->shortcut_match(p_event); } -String ShortCut::get_as_text() const { +String Shortcut::get_as_text() const { if (shortcut.is_valid()) { return shortcut->as_text(); } else { @@ -53,21 +53,21 @@ String ShortCut::get_as_text() const { } } -bool ShortCut::is_valid() const { +bool Shortcut::is_valid() const { return shortcut.is_valid(); } -void ShortCut::_bind_methods() { - ClassDB::bind_method(D_METHOD("set_shortcut", "event"), &ShortCut::set_shortcut); - ClassDB::bind_method(D_METHOD("get_shortcut"), &ShortCut::get_shortcut); +void Shortcut::_bind_methods() { + ClassDB::bind_method(D_METHOD("set_shortcut", "event"), &Shortcut::set_shortcut); + ClassDB::bind_method(D_METHOD("get_shortcut"), &Shortcut::get_shortcut); - ClassDB::bind_method(D_METHOD("is_valid"), &ShortCut::is_valid); + ClassDB::bind_method(D_METHOD("is_valid"), &Shortcut::is_valid); - ClassDB::bind_method(D_METHOD("is_shortcut", "event"), &ShortCut::is_shortcut); - ClassDB::bind_method(D_METHOD("get_as_text"), &ShortCut::get_as_text); + ClassDB::bind_method(D_METHOD("is_shortcut", "event"), &Shortcut::is_shortcut); + ClassDB::bind_method(D_METHOD("get_as_text"), &Shortcut::get_as_text); ADD_PROPERTY(PropertyInfo(Variant::OBJECT, "shortcut", PROPERTY_HINT_RESOURCE_TYPE, "InputEvent"), "set_shortcut", "get_shortcut"); } -ShortCut::ShortCut() { +Shortcut::Shortcut() { } diff --git a/scene/gui/shortcut.h b/scene/gui/shortcut.h index 063d4e43dc..0d7809e5cf 100644 --- a/scene/gui/shortcut.h +++ b/scene/gui/shortcut.h @@ -34,8 +34,8 @@ #include "core/input/input_event.h" #include "core/resource.h" -class ShortCut : public Resource { - GDCLASS(ShortCut, Resource); +class Shortcut : public Resource { + GDCLASS(Shortcut, Resource); Ref<InputEvent> shortcut; @@ -50,7 +50,7 @@ public: String get_as_text() const; - ShortCut(); + Shortcut(); }; #endif // SHORTCUT_H diff --git a/scene/gui/tabs.cpp b/scene/gui/tabs.cpp index 8f71aa7cab..d47f771d1d 100644 --- a/scene/gui/tabs.cpp +++ b/scene/gui/tabs.cpp @@ -388,6 +388,7 @@ void Tabs::set_current_tab(int p_current) { } ERR_FAIL_INDEX(p_current, get_tab_count()); + previous = current; current = p_current; _change_notify("current_tab"); @@ -401,6 +402,10 @@ int Tabs::get_current_tab() const { return current; } +int Tabs::get_previous_tab() const { + return previous; +} + int Tabs::get_hovered_tab() const { return hover; } @@ -588,6 +593,7 @@ void Tabs::add_tab(const String &p_str, const Ref<Texture2D> &p_icon) { void Tabs::clear_tabs() { tabs.clear(); current = 0; + previous = 0; call_deferred("_update_hover"); update(); } @@ -605,6 +611,7 @@ void Tabs::remove_tab(int p_idx) { if (current < 0) { current = 0; + previous = 0; } if (current >= tabs.size()) { current = tabs.size() - 1; @@ -917,6 +924,7 @@ void Tabs::_bind_methods() { ClassDB::bind_method(D_METHOD("get_tab_count"), &Tabs::get_tab_count); ClassDB::bind_method(D_METHOD("set_current_tab", "tab_idx"), &Tabs::set_current_tab); ClassDB::bind_method(D_METHOD("get_current_tab"), &Tabs::get_current_tab); + ClassDB::bind_method(D_METHOD("get_previous_tab"), &Tabs::get_previous_tab); ClassDB::bind_method(D_METHOD("set_tab_title", "tab_idx", "title"), &Tabs::set_tab_title); ClassDB::bind_method(D_METHOD("get_tab_title", "tab_idx"), &Tabs::get_tab_title); ClassDB::bind_method(D_METHOD("set_tab_icon", "tab_idx", "icon"), &Tabs::set_tab_icon); @@ -970,6 +978,7 @@ void Tabs::_bind_methods() { Tabs::Tabs() { current = 0; + previous = 0; tab_align = ALIGN_CENTER; rb_hover = -1; rb_pressing = false; diff --git a/scene/gui/tabs.h b/scene/gui/tabs.h index 8d7f1aa37d..b94c4a37a1 100644 --- a/scene/gui/tabs.h +++ b/scene/gui/tabs.h @@ -77,6 +77,7 @@ private: bool missing_right; Vector<Tab> tabs; int current; + int previous; int _get_top_margin() const; TabAlign tab_align; int rb_hover; @@ -138,6 +139,7 @@ public: int get_tab_count() const; void set_current_tab(int p_current); int get_current_tab() const; + int get_previous_tab() const; int get_hovered_tab() const; int get_tab_offset() const; diff --git a/scene/gui/text_edit.cpp b/scene/gui/text_edit.cpp index 1d732bec5b..957e1c11c7 100644 --- a/scene/gui/text_edit.cpp +++ b/scene/gui/text_edit.cpp @@ -107,6 +107,8 @@ static int _find_first_non_whitespace_column_of_line(const String &line) { return left; } +/////////////////////////////////////////////////////////////////////////////// + void TextEdit::Text::set_font(const Ref<Font> &p_font) { font = p_font; } @@ -165,12 +167,6 @@ void TextEdit::Text::clear_wrap_cache() { } } -void TextEdit::Text::clear_info_icons() { - for (int i = 0; i < text.size(); i++) { - text.write[i].has_info = false; - } -} - void TextEdit::Text::clear() { text.clear(); insert(0, ""); @@ -198,12 +194,9 @@ void TextEdit::Text::set(int p_line, const String &p_text) { void TextEdit::Text::insert(int p_at, const String &p_text) { Line line; + line.gutters.resize(gutter_count); line.marked = false; - line.safe = false; - line.breakpoint = false; - line.bookmark = false; line.hidden = false; - line.has_info = false; line.width_cache = -1; line.wrap_amount_cache = -1; line.data = p_text; @@ -231,6 +224,32 @@ int TextEdit::Text::get_char_width(char32_t c, char32_t next_c, int px) const { return w; } +void TextEdit::Text::add_gutter(int p_at) { + for (int i = 0; i < text.size(); i++) { + if (p_at < 0 || p_at > gutter_count) { + text.write[i].gutters.push_back(Gutter()); + } else { + text.write[i].gutters.insert(p_at, Gutter()); + } + } + gutter_count++; +} + +void TextEdit::Text::remove_gutter(int p_gutter) { + for (int i = 0; i < text.size(); i++) { + text.write[i].gutters.remove(p_gutter); + } + gutter_count--; +} + +void TextEdit::Text::move_gutters(int p_from_line, int p_to_line) { + text.write[p_to_line].gutters = text[p_from_line].gutters; + text.write[p_from_line].gutters.clear(); + text.write[p_from_line].gutters.resize(gutter_count); +} + +//////////////////////////////////////////////////////////////////////////////// + void TextEdit::_update_scrollbars() { Size2 size = get_size(); Size2 hmin = h_scroll->get_combined_minimum_size(); @@ -249,23 +268,7 @@ void TextEdit::_update_scrollbars() { } int visible_width = size.width - cache.style_normal->get_minimum_size().width; - int total_width = text.get_max_width(true) + vmin.x; - - if (line_numbers) { - total_width += cache.line_number_w; - } - - if (draw_breakpoint_gutter || draw_bookmark_gutter) { - total_width += cache.breakpoint_gutter_width; - } - - if (draw_info_gutter) { - total_width += cache.info_gutter_width; - } - - if (draw_fold_gutter) { - total_width += cache.fold_gutter_width; - } + int total_width = text.get_max_width(true) + vmin.x + gutters_width + gutter_padding; if (draw_minimap) { total_width += cache.minimap_width; @@ -557,54 +560,16 @@ void TextEdit::_notification(int p_what) { draw_caret = false; } - if (draw_breakpoint_gutter || draw_bookmark_gutter) { - breakpoint_gutter_width = (get_row_height() * 55) / 100; - cache.breakpoint_gutter_width = breakpoint_gutter_width; - } else { - cache.breakpoint_gutter_width = 0; - } - - if (draw_info_gutter) { - info_gutter_width = (get_row_height()); - cache.info_gutter_width = info_gutter_width; - } else { - cache.info_gutter_width = 0; - } - - if (draw_fold_gutter) { - fold_gutter_width = (get_row_height() * 55) / 100; - cache.fold_gutter_width = fold_gutter_width; - } else { - cache.fold_gutter_width = 0; - } - cache.minimap_width = 0; if (draw_minimap) { cache.minimap_width = minimap_width; } - int line_number_char_count = 0; - - { - int lc = text.size(); - cache.line_number_w = 0; - while (lc) { - cache.line_number_w += 1; - lc /= 10; - }; - - if (line_numbers) { - line_number_char_count = cache.line_number_w; - cache.line_number_w = (cache.line_number_w + 1) * cache.font->get_char_size('0').width; - } else { - cache.line_number_w = 0; - } - } _update_scrollbars(); RID ci = get_canvas_item(); RenderingServer::get_singleton()->canvas_item_set_clip(get_canvas_item(), true); - int xmargin_beg = cache.style_normal->get_margin(MARGIN_LEFT) + cache.line_number_w + cache.breakpoint_gutter_width + cache.fold_gutter_width + cache.info_gutter_width; + int xmargin_beg = cache.style_normal->get_margin(MARGIN_LEFT) + gutters_width + gutter_padding; int xmargin_end = size.width - cache.style_normal->get_margin(MARGIN_RIGHT) - cache.minimap_width; // Let's do it easy for now. @@ -795,8 +760,6 @@ void TextEdit::_notification(int p_what) { // Check if highlighted words contains only whitespaces (tabs or spaces). bool only_whitespaces_highlighted = highlighted_text.strip_edges() == String(); - String line_num_padding = line_numbers_zero_padded ? "0" : " "; - int cursor_wrap_index = get_cursor_wrap_index(); FontDrawer drawer(cache.font, Color(1, 1, 1)); @@ -1053,103 +1016,52 @@ void TextEdit::_notification(int p_what) { if (line_wrap_index == 0) { // Only do these if we are on the first wrapped part of a line. - if (text.is_breakpoint(line) && !draw_breakpoint_gutter) { -#ifdef TOOLS_ENABLED - RenderingServer::get_singleton()->canvas_item_add_rect(ci, Rect2(xmargin_beg + ofs_x, ofs_y + get_row_height() - EDSCALE, xmargin_end - xmargin_beg, EDSCALE), cache.breakpoint_color); -#else - RenderingServer::get_singleton()->canvas_item_add_rect(ci, Rect2(xmargin_beg + ofs_x, ofs_y, xmargin_end - xmargin_beg, get_row_height()), cache.breakpoint_color); -#endif - } + int gutter_offset = cache.style_normal->get_margin(MARGIN_LEFT); + for (int g = 0; g < gutters.size(); g++) { + const GutterInfo gutter = gutters[g]; - // Draw bookmark marker. - if (text.is_bookmark(line)) { - if (draw_bookmark_gutter) { - int vertical_gap = (get_row_height() * 40) / 100; - int horizontal_gap = (cache.breakpoint_gutter_width * 30) / 100; - int marker_radius = get_row_height() - (vertical_gap * 2); - RenderingServer::get_singleton()->canvas_item_add_circle(ci, Point2(cache.style_normal->get_margin(MARGIN_LEFT) + horizontal_gap - 2 + marker_radius / 2, ofs_y + vertical_gap + marker_radius / 2), marker_radius, Color(cache.bookmark_color.r, cache.bookmark_color.g, cache.bookmark_color.b)); + if (!gutter.draw || gutter.width <= 0) { + continue; } - } - // Draw breakpoint marker. - if (text.is_breakpoint(line)) { - if (draw_breakpoint_gutter) { - int vertical_gap = (get_row_height() * 40) / 100; - int horizontal_gap = (cache.breakpoint_gutter_width * 30) / 100; - int marker_height = get_row_height() - (vertical_gap * 2); - int marker_width = cache.breakpoint_gutter_width - (horizontal_gap * 2); - // No transparency on marker. - RenderingServer::get_singleton()->canvas_item_add_rect(ci, Rect2(cache.style_normal->get_margin(MARGIN_LEFT) + horizontal_gap - 2, ofs_y + vertical_gap, marker_width, marker_height), Color(cache.breakpoint_color.r, cache.breakpoint_color.g, cache.breakpoint_color.b)); - } - } - - // Draw info icons. - if (draw_info_gutter && text.has_info_icon(line)) { - int vertical_gap = (get_row_height() * 40) / 100; - int horizontal_gap = (cache.info_gutter_width * 30) / 100; - int gutter_left = cache.style_normal->get_margin(MARGIN_LEFT) + cache.breakpoint_gutter_width; - - Ref<Texture2D> info_icon = text.get_info_icon(line); - // Ensure the icon fits the gutter size. - Size2i icon_size = info_icon->get_size(); - if (icon_size.width > cache.info_gutter_width - horizontal_gap) { - icon_size.width = cache.info_gutter_width - horizontal_gap; - } - if (icon_size.height > get_row_height() - horizontal_gap) { - icon_size.height = get_row_height() - horizontal_gap; - } + switch (gutter.type) { + case GUTTER_TYPE_STRING: { + const String &text = get_line_gutter_text(line, g); + if (text == "") { + break; + } - Size2i icon_pos; - int xofs = horizontal_gap - (info_icon->get_width() / 4); - int yofs = vertical_gap - (info_icon->get_height() / 4); - icon_pos.x = gutter_left + xofs + ofs_x; - icon_pos.y = ofs_y + yofs; + int yofs = ofs_y + (get_row_height() - cache.font->get_height()) / 2; + cache.font->draw(ci, Point2(gutter_offset + ofs_x, yofs + cache.font->get_ascent()), text, get_line_gutter_item_color(line, g)); + } break; + case GUTTER_TPYE_ICON: { + const Ref<Texture2D> icon = get_line_gutter_icon(line, g); + if (icon.is_null()) { + break; + } - draw_texture_rect(info_icon, Rect2(icon_pos, icon_size)); - } + Rect2i gutter_rect = Rect2i(Point2i(gutter_offset, ofs_y), Size2i(gutter.width, get_row_height())); - // Draw execution marker. - if (executing_line == line) { - if (draw_breakpoint_gutter) { - int icon_extra_size = 4; - int vertical_gap = (get_row_height() * 40) / 100; - int horizontal_gap = (cache.breakpoint_gutter_width * 30) / 100; - int marker_height = get_row_height() - (vertical_gap * 2) + icon_extra_size; - int marker_width = cache.breakpoint_gutter_width - (horizontal_gap * 2) + icon_extra_size; - cache.executing_icon->draw_rect(ci, Rect2(cache.style_normal->get_margin(MARGIN_LEFT) + horizontal_gap - 2 - icon_extra_size / 2, ofs_y + vertical_gap - icon_extra_size / 2, marker_width, marker_height), false, Color(cache.executing_line_color.r, cache.executing_line_color.g, cache.executing_line_color.b)); - } else { -#ifdef TOOLS_ENABLED - RenderingServer::get_singleton()->canvas_item_add_rect(ci, Rect2(xmargin_beg + ofs_x, ofs_y + get_row_height() - EDSCALE, xmargin_end - xmargin_beg, EDSCALE), cache.executing_line_color); -#else - RenderingServer::get_singleton()->canvas_item_add_rect(ci, Rect2(xmargin_beg + ofs_x, ofs_y, xmargin_end - xmargin_beg, get_row_height()), cache.executing_line_color); -#endif - } - } + int horizontal_padding = gutter_rect.size.x / 6; + int vertical_padding = gutter_rect.size.y / 6; - // Draw fold markers. - if (draw_fold_gutter) { - int horizontal_gap = (cache.fold_gutter_width * 30) / 100; - int gutter_left = cache.style_normal->get_margin(MARGIN_LEFT) + cache.breakpoint_gutter_width + cache.line_number_w + cache.info_gutter_width; - if (is_folded(line)) { - int xofs = horizontal_gap - (cache.can_fold_icon->get_width()) / 2; - int yofs = (get_row_height() - cache.folded_icon->get_height()) / 2; - cache.folded_icon->draw(ci, Point2(gutter_left + xofs + ofs_x, ofs_y + yofs), cache.code_folding_color); - } else if (can_fold(line)) { - int xofs = -cache.can_fold_icon->get_width() / 2 - horizontal_gap + 3; - int yofs = (get_row_height() - cache.can_fold_icon->get_height()) / 2; - cache.can_fold_icon->draw(ci, Point2(gutter_left + xofs + ofs_x, ofs_y + yofs), cache.code_folding_color); - } - } + gutter_rect.position += Point2(horizontal_padding, vertical_padding); + gutter_rect.size -= Point2(horizontal_padding, vertical_padding) * 2; - // Draw line numbers. - if (cache.line_number_w) { - int yofs = ofs_y + (get_row_height() - cache.font->get_height()) / 2; - String fc = String::num(line + 1); - while (fc.length() < line_number_char_count) { - fc = line_num_padding + fc; + icon->draw_rect(ci, gutter_rect, false, get_line_gutter_item_color(line, g)); + } break; + case GUTTER_TPYE_CUSTOM: { + if (gutter.custom_draw_obj.is_valid()) { + Object *cdo = ObjectDB::get_instance(gutter.custom_draw_obj); + if (cdo) { + Rect2i gutter_rect = Rect2i(Point2i(gutter_offset, ofs_y), Size2i(gutter.width, get_row_height())); + cdo->call(gutter.custom_draw_callback, line, g, Rect2(gutter_rect)); + } + } + } break; } - cache.font->draw(ci, Point2(cache.style_normal->get_margin(MARGIN_LEFT) + cache.breakpoint_gutter_width + cache.info_gutter_width + ofs_x, yofs + cache.font->get_ascent()), fc, text.is_safe(line) ? cache.safe_line_number_color : cache.line_number_color); + gutter_offset += gutter.width; } } @@ -1797,18 +1709,34 @@ void TextEdit::backspace_at_cursor() { int prev_line = cursor.column ? cursor.line : cursor.line - 1; int prev_column = cursor.column ? (cursor.column - 1) : (text[cursor.line - 1].length()); - if (is_line_hidden(cursor.line)) { - set_line_as_hidden(prev_line, true); - } - if (is_line_set_as_breakpoint(cursor.line)) { - if (!text.is_breakpoint(prev_line)) { - emit_signal("breakpoint_toggled", prev_line); + if (cursor.line != prev_line) { + for (int i = 0; i < gutters.size(); i++) { + if (!gutters[i].overwritable) { + continue; + } + + if (text.get_line_gutter_text(cursor.line, i) != "") { + text.set_line_gutter_text(prev_line, i, text.get_line_gutter_text(cursor.line, i)); + text.set_line_gutter_item_color(prev_line, i, text.get_line_gutter_item_color(cursor.line, i)); + } + + if (text.get_line_gutter_icon(cursor.line, i).is_valid()) { + text.set_line_gutter_icon(prev_line, i, text.get_line_gutter_icon(cursor.line, i)); + text.set_line_gutter_item_color(prev_line, i, text.get_line_gutter_item_color(cursor.line, i)); + } + + if (text.get_line_gutter_metadata(cursor.line, i) != "") { + text.set_line_gutter_metadata(prev_line, i, text.get_line_gutter_metadata(cursor.line, i)); + } + + if (text.is_line_gutter_clickable(cursor.line, i)) { + text.set_line_gutter_clickable(prev_line, i, true); + } } - set_line_as_breakpoint(prev_line, true); } - if (text.has_info_icon(cursor.line)) { - set_line_info_icon(prev_line, text.get_info_icon(cursor.line), text.get_info(cursor.line)); + if (is_line_hidden(cursor.line)) { + set_line_as_hidden(prev_line, true); } if (auto_brace_completion_enabled && @@ -1998,7 +1926,7 @@ void TextEdit::_get_mouse_pos(const Point2i &p_mouse, int &r_row, int &r_col) co row = text.size() - 1; col = text[row].size(); } else { - int colx = p_mouse.x - (cache.style_normal->get_margin(MARGIN_LEFT) + cache.line_number_w + cache.breakpoint_gutter_width + cache.fold_gutter_width + cache.info_gutter_width); + int colx = p_mouse.x - (cache.style_normal->get_margin(MARGIN_LEFT) + gutters_width + gutter_padding); colx += cursor.x_ofs; col = get_char_pos_for_line(colx, row, wrap_index); if (is_wrap_enabled() && wrap_index < times_line_wraps(row)) { @@ -2043,7 +1971,7 @@ Vector2i TextEdit::_get_cursor_pixel_pos() { // Calculate final pixel position int y = (row - get_v_scroll_offset() + 1 /*Bottom of line*/) * get_row_height(); - int x = cache.style_normal->get_margin(MARGIN_LEFT) + cache.line_number_w + cache.breakpoint_gutter_width + cache.fold_gutter_width + cache.info_gutter_width - cursor.x_ofs; + int x = cache.style_normal->get_margin(MARGIN_LEFT) + gutters_width + gutter_padding - cursor.x_ofs; int ix = 0; while (ix < rows2[0].size() && ix < cursor.column) { if (cache.font != nullptr) { @@ -2178,45 +2106,24 @@ void TextEdit::_gui_input(const Ref<InputEvent> &p_gui_input) { int row, col; _get_mouse_pos(Point2i(mb->get_position().x, mb->get_position().y), row, col); - // Toggle breakpoint on gutter click. - if (draw_breakpoint_gutter) { - int gutter = cache.style_normal->get_margin(MARGIN_LEFT); - if (mb->get_position().x > gutter - 6 && mb->get_position().x <= gutter + cache.breakpoint_gutter_width - 3) { - set_line_as_breakpoint(row, !is_line_set_as_breakpoint(row)); - emit_signal("breakpoint_toggled", row); - return; + int left_margin = cache.style_normal->get_margin(MARGIN_LEFT); + for (int i = 0; i < gutters.size(); i++) { + if (!gutters[i].draw || gutters[i].width <= 0) { + continue; } - } - // Emit info clicked. - if (draw_info_gutter && text.has_info_icon(row)) { - int left_margin = cache.style_normal->get_margin(MARGIN_LEFT); - int gutter_left = left_margin + cache.breakpoint_gutter_width; - if (mb->get_position().x > gutter_left - 6 && mb->get_position().x <= gutter_left + cache.info_gutter_width - 3) { - emit_signal("info_clicked", row, text.get_info(row)); + if (mb->get_position().x > left_margin && mb->get_position().x <= (left_margin + gutters[i].width) - 3) { + emit_signal("gutter_clicked", row, i); return; } - } - // Toggle fold on gutter click if can. - if (draw_fold_gutter) { - int left_margin = cache.style_normal->get_margin(MARGIN_LEFT); - int gutter_left = left_margin + cache.breakpoint_gutter_width + cache.line_number_w + cache.info_gutter_width; - if (mb->get_position().x > gutter_left - 6 && mb->get_position().x <= gutter_left + cache.fold_gutter_width - 3) { - if (is_folded(row)) { - unfold_line(row); - } else if (can_fold(row)) { - fold_line(row); - } - return; - } + left_margin += gutters[i].width; } // Unfold on folded icon click. if (is_folded(row)) { - int line_width = text.get_line_width(row); - line_width += cache.style_normal->get_margin(MARGIN_LEFT) + cache.line_number_w + cache.breakpoint_gutter_width + cache.info_gutter_width + cache.fold_gutter_width - cursor.x_ofs; - if (mb->get_position().x > line_width - 3 && mb->get_position().x <= line_width + cache.folded_eol_icon->get_width() + 3) { + left_margin += gutter_padding + text.get_line_width(row) - cursor.x_ofs; + if (mb->get_position().x > left_margin && mb->get_position().x <= left_margin + cache.folded_eol_icon->get_width() + 3) { unfold_line(row); return; } @@ -3775,30 +3682,15 @@ void TextEdit::_base_insert_text(int p_line, int p_char, const String &p_text, i Vector<String> substrings = p_text.replace("\r", "").split("\n"); - /* STEP 2: Fire breakpoint_toggled signals. */ - // Is this just a new empty line? bool shift_first_line = p_char == 0 && p_text.replace("\r", "") == "\n"; - int i = p_line + !shift_first_line; - int lines = substrings.size() - 1; - for (; i < text.size(); i++) { - if (text.is_breakpoint(i)) { - if ((i - lines < p_line || !text.is_breakpoint(i - lines)) || (i - lines == p_line && !shift_first_line)) { - emit_signal("breakpoint_toggled", i); - } - if (i + lines >= text.size() || !text.is_breakpoint(i + lines)) { - emit_signal("breakpoint_toggled", i + lines); - } - } - } - - /* STEP 3: Add spaces if the char is greater than the end of the line. */ + /* STEP 2: Add spaces if the char is greater than the end of the line. */ while (p_char > text[p_line].length()) { text.set(p_line, text[p_line] + String::chr(' ')); } - /* STEP 4: Separate dest string in pre and post text. */ + /* STEP 3: Separate dest string in pre and post text. */ String preinsert_text = text[p_line].substr(0, p_char); String postinsert_text = text[p_line].substr(p_char, text[p_line].size()); @@ -3818,15 +3710,10 @@ void TextEdit::_base_insert_text(int p_line, int p_char, const String &p_text, i } if (shift_first_line) { - text.set_breakpoint(p_line + 1, text.is_breakpoint(p_line)); + text.move_gutters(p_line, p_line + 1); text.set_hidden(p_line + 1, text.is_hidden(p_line)); - if (text.has_info_icon(p_line)) { - text.set_info_icon(p_line + 1, text.get_info_icon(p_line), text.get_info(p_line)); - } - text.set_breakpoint(p_line, false); text.set_hidden(p_line, false); - text.set_info_icon(p_line, nullptr, ""); } text.set_line_wrap_amount(p_line, -1); @@ -3840,7 +3727,7 @@ void TextEdit::_base_insert_text(int p_line, int p_char, const String &p_text, i } text_changed_dirty = true; } - emit_signal("line_edited_from", p_line); + emit_signal("lines_edited_from", p_line, r_end_line); } String TextEdit::_base_get_text(int p_from_line, int p_from_column, int p_to_line, int p_to_column) const { @@ -3877,19 +3764,6 @@ void TextEdit::_base_remove_text(int p_from_line, int p_from_column, int p_to_li String pre_text = text[p_from_line].substr(0, p_from_column); String post_text = text[p_to_line].substr(p_to_column, text[p_to_line].length()); - int lines = p_to_line - p_from_line; - - for (int i = p_from_line + 1; i < text.size(); i++) { - if (text.is_breakpoint(i)) { - if (i + lines >= text.size() || !text.is_breakpoint(i + lines)) { - emit_signal("breakpoint_toggled", i); - } - if (i > p_to_line && (i - lines < 0 || !text.is_breakpoint(i - lines))) { - emit_signal("breakpoint_toggled", i - lines); - } - } - } - for (int i = p_from_line; i < p_to_line; i++) { text.remove(p_from_line + 1); } @@ -3903,7 +3777,7 @@ void TextEdit::_base_remove_text(int p_from_line, int p_from_column, int p_to_li } text_changed_dirty = true; } - emit_signal("line_edited_from", p_from_line); + emit_signal("lines_edited_from", p_to_line, p_from_line); } void TextEdit::_insert_text(int p_line, int p_char, const String &p_text, int *r_end_line, int *r_end_char) { @@ -4097,7 +3971,7 @@ int TextEdit::get_total_visible_rows() const { } void TextEdit::_update_wrap_at() { - wrap_at = get_size().width - cache.style_normal->get_minimum_size().width - cache.line_number_w - cache.breakpoint_gutter_width - cache.fold_gutter_width - cache.info_gutter_width - cache.minimap_width - wrap_right_offset; + wrap_at = get_size().width - cache.style_normal->get_minimum_size().width - gutters_width - gutter_padding - cache.minimap_width - wrap_right_offset; update_cursor_wrap_offset(); text.clear_wrap_cache(); @@ -4132,7 +4006,7 @@ void TextEdit::adjust_viewport_to_cursor() { set_line_as_last_visible(cur_line, cur_wrap); } - int visible_width = get_size().width - cache.style_normal->get_minimum_size().width - cache.line_number_w - cache.breakpoint_gutter_width - cache.fold_gutter_width - cache.info_gutter_width - cache.minimap_width; + int visible_width = get_size().width - cache.style_normal->get_minimum_size().width - gutters_width - gutter_padding - cache.minimap_width; if (v_scroll->is_visible_in_tree()) { visible_width -= v_scroll->get_combined_minimum_size().width; } @@ -4167,7 +4041,7 @@ void TextEdit::center_viewport_to_cursor() { } set_line_as_center_visible(cursor.line, get_cursor_wrap_index()); - int visible_width = get_size().width - cache.style_normal->get_minimum_size().width - cache.line_number_w - cache.breakpoint_gutter_width - cache.fold_gutter_width - cache.info_gutter_width - cache.minimap_width; + int visible_width = get_size().width - cache.style_normal->get_minimum_size().width - gutters_width - gutter_padding - cache.minimap_width; if (v_scroll->is_visible_in_tree()) { visible_width -= v_scroll->get_combined_minimum_size().width; } @@ -4614,54 +4488,41 @@ Control::CursorShape TextEdit::get_cursor_shape(const Point2 &p_pos) const { return CURSOR_POINTING_HAND; } - int gutter = cache.style_normal->get_margin(MARGIN_LEFT) + cache.line_number_w + cache.breakpoint_gutter_width + cache.fold_gutter_width + cache.info_gutter_width; if ((completion_active && completion_rect.has_point(p_pos))) { return CURSOR_ARROW; } - if (p_pos.x < gutter) { - int row, col; - _get_mouse_pos(p_pos, row, col); - int left_margin = cache.style_normal->get_margin(MARGIN_LEFT); - // Breakpoint icon. - if (draw_breakpoint_gutter && p_pos.x > left_margin - 6 && p_pos.x <= left_margin + cache.breakpoint_gutter_width - 3) { - return CURSOR_POINTING_HAND; - } + int row, col; + _get_mouse_pos(p_pos, row, col); - // Info icons. - int gutter_left = left_margin + cache.breakpoint_gutter_width + cache.info_gutter_width; - if (draw_info_gutter && p_pos.x > left_margin + cache.breakpoint_gutter_width - 6 && p_pos.x <= gutter_left - 3) { - if (text.has_info_icon(row)) { - return CURSOR_POINTING_HAND; + int left_margin = cache.style_normal->get_margin(MARGIN_LEFT); + int gutter = left_margin + gutters_width; + if (p_pos.x < gutter) { + for (int i = 0; i < gutters.size(); i++) { + if (!gutters[i].draw) { + continue; } - return CURSOR_ARROW; - } - // Fold icon. - if (draw_fold_gutter && p_pos.x > gutter_left + cache.line_number_w - 6 && p_pos.x <= gutter_left + cache.line_number_w + cache.fold_gutter_width - 3) { - if (is_folded(row) || can_fold(row)) { - return CURSOR_POINTING_HAND; - } else { - return CURSOR_ARROW; + if (p_pos.x > left_margin && p_pos.x <= (left_margin + gutters[i].width) - 3) { + if (gutters[i].clickable || is_line_gutter_clickable(row, i)) { + return CURSOR_POINTING_HAND; + } } + left_margin += gutters[i].width; } + return CURSOR_ARROW; + } + int xmargin_end = get_size().width - cache.style_normal->get_margin(MARGIN_RIGHT); + if (draw_minimap && p_pos.x > xmargin_end - minimap_width && p_pos.x <= xmargin_end) { return CURSOR_ARROW; - } else { - int xmargin_end = get_size().width - cache.style_normal->get_margin(MARGIN_RIGHT); - if (draw_minimap && p_pos.x > xmargin_end - minimap_width && p_pos.x <= xmargin_end) { - return CURSOR_ARROW; - } - - int row, col; - _get_mouse_pos(p_pos, row, col); - // EOL fold icon. - if (is_folded(row)) { - int line_width = text.get_line_width(row); - line_width += cache.style_normal->get_margin(MARGIN_LEFT) + cache.line_number_w + cache.breakpoint_gutter_width + cache.fold_gutter_width + cache.info_gutter_width - cursor.x_ofs; - if (p_pos.x > line_width - 3 && p_pos.x <= line_width + cache.folded_eol_icon->get_width() + 3) { - return CURSOR_POINTING_HAND; - } + } + + // EOL fold icon. + if (is_folded(row)) { + gutter += gutter_padding + text.get_line_width(row) - cursor.x_ofs; + if (p_pos.x > gutter - 3 && p_pos.x <= gutter + cache.folded_eol_icon->get_width() + 3) { + return CURSOR_POINTING_HAND; } } @@ -4861,8 +4722,6 @@ void TextEdit::_update_caches() { cache.font = get_theme_font("font"); cache.caret_color = get_theme_color("caret_color"); cache.caret_background_color = get_theme_color("caret_background_color"); - cache.line_number_color = get_theme_color("line_number_color"); - cache.safe_line_number_color = get_theme_color("safe_line_number_color"); cache.font_color = get_theme_color("font_color"); cache.font_color_selected = get_theme_color("font_color_selected"); cache.font_color_readonly = get_theme_color("font_color_readonly"); @@ -4870,9 +4729,6 @@ void TextEdit::_update_caches() { cache.mark_color = get_theme_color("mark_color"); cache.current_line_color = get_theme_color("current_line_color"); cache.line_length_guideline_color = get_theme_color("line_length_guideline_color"); - cache.bookmark_color = get_theme_color("bookmark_color"); - cache.breakpoint_color = get_theme_color("breakpoint_color"); - cache.executing_line_color = get_theme_color("executing_line_color"); cache.code_folding_color = get_theme_color("code_folding_color"); cache.brace_mismatch_color = get_theme_color("brace_mismatch_color"); cache.word_highlighted_color = get_theme_color("word_highlighted_color"); @@ -4887,10 +4743,7 @@ void TextEdit::_update_caches() { cache.row_height = cache.font->get_height() + cache.line_spacing; cache.tab_icon = get_theme_icon("tab"); cache.space_icon = get_theme_icon("space"); - cache.folded_icon = get_theme_icon("folded"); - cache.can_fold_icon = get_theme_icon("fold"); cache.folded_eol_icon = get_theme_icon("GuiEllipsis", "EditorIcons"); - cache.executing_icon = get_theme_icon("MainPlay", "EditorIcons"); text.set_font(cache.font); text.clear_width_cache(); @@ -4899,6 +4752,7 @@ void TextEdit::_update_caches() { } } +/* Syntax Highlighting. */ Ref<SyntaxHighlighter> TextEdit::get_syntax_highlighter() { return syntax_highlighter; } @@ -4911,6 +4765,187 @@ void TextEdit::set_syntax_highlighter(Ref<SyntaxHighlighter> p_syntax_highlighte update(); } +/* Gutters. */ +void TextEdit::_update_gutter_width() { + gutters_width = 0; + for (int i = 0; i < gutters.size(); i++) { + if (gutters[i].draw) { + gutters_width += gutters[i].width; + } + } + if (gutters_width > 0) { + gutter_padding = 2; + } + update(); +} + +void TextEdit::add_gutter(int p_at) { + if (p_at < 0 || p_at > gutters.size()) { + gutters.push_back(GutterInfo()); + } else { + gutters.insert(p_at, GutterInfo()); + } + + for (int i = 0; i < text.size() + 1; i++) { + text.add_gutter(p_at); + } + emit_signal("gutter_added"); + update(); +} + +void TextEdit::remove_gutter(int p_gutter) { + ERR_FAIL_INDEX(p_gutter, gutters.size()); + + gutters.remove(p_gutter); + + for (int i = 0; i < text.size() + 1; i++) { + text.remove_gutter(p_gutter); + } + emit_signal("gutter_removed"); + update(); +} + +int TextEdit::get_gutter_count() const { + return gutters.size(); +} + +void TextEdit::set_gutter_name(int p_gutter, const String &p_name) { + ERR_FAIL_INDEX(p_gutter, gutters.size()); + gutters.write[p_gutter].name = p_name; +} + +String TextEdit::get_gutter_name(int p_gutter) const { + ERR_FAIL_INDEX_V(p_gutter, gutters.size(), ""); + return gutters[p_gutter].name; +} + +void TextEdit::set_gutter_type(int p_gutter, GutterType p_type) { + ERR_FAIL_INDEX(p_gutter, gutters.size()); + gutters.write[p_gutter].type = p_type; + update(); +} + +TextEdit::GutterType TextEdit::get_gutter_type(int p_gutter) const { + ERR_FAIL_INDEX_V(p_gutter, gutters.size(), GUTTER_TYPE_STRING); + return gutters[p_gutter].type; +} + +void TextEdit::set_gutter_width(int p_gutter, int p_width) { + ERR_FAIL_INDEX(p_gutter, gutters.size()); + gutters.write[p_gutter].width = p_width; + _update_gutter_width(); +} + +int TextEdit::get_gutter_width(int p_gutter) const { + ERR_FAIL_INDEX_V(p_gutter, gutters.size(), -1); + return gutters[p_gutter].width; +} + +void TextEdit::set_gutter_draw(int p_gutter, bool p_draw) { + ERR_FAIL_INDEX(p_gutter, gutters.size()); + gutters.write[p_gutter].draw = p_draw; + _update_gutter_width(); +} + +bool TextEdit::is_gutter_drawn(int p_gutter) const { + ERR_FAIL_INDEX_V(p_gutter, gutters.size(), false); + return gutters[p_gutter].draw; +} + +void TextEdit::set_gutter_clickable(int p_gutter, bool p_clickable) { + ERR_FAIL_INDEX(p_gutter, gutters.size()); + gutters.write[p_gutter].clickable = p_clickable; + update(); +} + +bool TextEdit::is_gutter_clickable(int p_gutter) const { + ERR_FAIL_INDEX_V(p_gutter, gutters.size(), false); + return gutters[p_gutter].clickable; +} + +void TextEdit::set_gutter_overwritable(int p_gutter, bool p_overwritable) { + ERR_FAIL_INDEX(p_gutter, gutters.size()); + gutters.write[p_gutter].overwritable = p_overwritable; +} + +bool TextEdit::is_gutter_overwritable(int p_gutter) const { + ERR_FAIL_INDEX_V(p_gutter, gutters.size(), false); + return gutters[p_gutter].overwritable; +} + +void TextEdit::set_gutter_custom_draw(int p_gutter, Object *p_object, const StringName &p_callback) { + ERR_FAIL_INDEX(p_gutter, gutters.size()); + ERR_FAIL_NULL(p_object); + + gutters.write[p_gutter].custom_draw_obj = p_object->get_instance_id(); + gutters.write[p_gutter].custom_draw_callback = p_callback; + update(); +} + +// Line gutters. +void TextEdit::set_line_gutter_metadata(int p_line, int p_gutter, const Variant &p_metadata) { + ERR_FAIL_INDEX(p_line, text.size()); + ERR_FAIL_INDEX(p_gutter, gutters.size()); + text.set_line_gutter_metadata(p_line, p_gutter, p_metadata); +} + +Variant TextEdit::get_line_gutter_metadata(int p_line, int p_gutter) const { + ERR_FAIL_INDEX_V(p_line, text.size(), ""); + ERR_FAIL_INDEX_V(p_gutter, gutters.size(), ""); + return text.get_line_gutter_metadata(p_line, p_gutter); +} + +void TextEdit::set_line_gutter_text(int p_line, int p_gutter, const String &p_text) { + ERR_FAIL_INDEX(p_line, text.size()); + ERR_FAIL_INDEX(p_gutter, gutters.size()); + text.set_line_gutter_text(p_line, p_gutter, p_text); + update(); +} + +String TextEdit::get_line_gutter_text(int p_line, int p_gutter) const { + ERR_FAIL_INDEX_V(p_line, text.size(), ""); + ERR_FAIL_INDEX_V(p_gutter, gutters.size(), ""); + return text.get_line_gutter_text(p_line, p_gutter); +} + +void TextEdit::set_line_gutter_icon(int p_line, int p_gutter, Ref<Texture2D> p_icon) { + ERR_FAIL_INDEX(p_line, text.size()); + ERR_FAIL_INDEX(p_gutter, gutters.size()); + text.set_line_gutter_icon(p_line, p_gutter, p_icon); + update(); +} + +Ref<Texture2D> TextEdit::get_line_gutter_icon(int p_line, int p_gutter) const { + ERR_FAIL_INDEX_V(p_line, text.size(), Ref<Texture2D>()); + ERR_FAIL_INDEX_V(p_gutter, gutters.size(), Ref<Texture2D>()); + return text.get_line_gutter_icon(p_line, p_gutter); +} + +void TextEdit::set_line_gutter_item_color(int p_line, int p_gutter, const Color &p_color) { + ERR_FAIL_INDEX(p_line, text.size()); + ERR_FAIL_INDEX(p_gutter, gutters.size()); + text.set_line_gutter_item_color(p_line, p_gutter, p_color); + update(); +} + +Color TextEdit::get_line_gutter_item_color(int p_line, int p_gutter) { + ERR_FAIL_INDEX_V(p_line, text.size(), Color()); + ERR_FAIL_INDEX_V(p_gutter, gutters.size(), Color()); + return text.get_line_gutter_item_color(p_line, p_gutter); +} + +void TextEdit::set_line_gutter_clickable(int p_line, int p_gutter, bool p_clickable) { + ERR_FAIL_INDEX(p_line, text.size()); + ERR_FAIL_INDEX(p_gutter, gutters.size()); + text.set_line_gutter_clickable(p_line, p_gutter, p_clickable); +} + +bool TextEdit::is_line_gutter_clickable(int p_line, int p_gutter) const { + ERR_FAIL_INDEX_V(p_line, text.size(), false); + ERR_FAIL_INDEX_V(p_gutter, gutters.size(), false); + return text.is_line_gutter_clickable(p_line, p_gutter); +} + void TextEdit::add_keyword(const String &p_keyword) { keywords.insert(p_keyword); } @@ -5341,106 +5376,6 @@ void TextEdit::set_line_as_marked(int p_line, bool p_marked) { update(); } -void TextEdit::set_line_as_safe(int p_line, bool p_safe) { - ERR_FAIL_INDEX(p_line, text.size()); - text.set_safe(p_line, p_safe); - update(); -} - -bool TextEdit::is_line_set_as_safe(int p_line) const { - ERR_FAIL_INDEX_V(p_line, text.size(), false); - return text.is_safe(p_line); -} - -void TextEdit::set_executing_line(int p_line) { - ERR_FAIL_INDEX(p_line, text.size()); - executing_line = p_line; - update(); -} - -void TextEdit::clear_executing_line() { - executing_line = -1; - update(); -} - -bool TextEdit::is_line_set_as_bookmark(int p_line) const { - ERR_FAIL_INDEX_V(p_line, text.size(), false); - return text.is_bookmark(p_line); -} - -void TextEdit::set_line_as_bookmark(int p_line, bool p_bookmark) { - ERR_FAIL_INDEX(p_line, text.size()); - text.set_bookmark(p_line, p_bookmark); - update(); -} - -void TextEdit::get_bookmarks(List<int> *p_bookmarks) const { - for (int i = 0; i < text.size(); i++) { - if (text.is_bookmark(i)) { - p_bookmarks->push_back(i); - } - } -} - -Array TextEdit::get_bookmarks_array() const { - Array arr; - for (int i = 0; i < text.size(); i++) { - if (text.is_bookmark(i)) { - arr.append(i); - } - } - return arr; -} - -bool TextEdit::is_line_set_as_breakpoint(int p_line) const { - ERR_FAIL_INDEX_V(p_line, text.size(), false); - return text.is_breakpoint(p_line); -} - -void TextEdit::set_line_as_breakpoint(int p_line, bool p_breakpoint) { - ERR_FAIL_INDEX(p_line, text.size()); - text.set_breakpoint(p_line, p_breakpoint); - update(); -} - -void TextEdit::get_breakpoints(List<int> *p_breakpoints) const { - for (int i = 0; i < text.size(); i++) { - if (text.is_breakpoint(i)) { - p_breakpoints->push_back(i); - } - } -} - -Array TextEdit::get_breakpoints_array() const { - Array arr; - for (int i = 0; i < text.size(); i++) { - if (text.is_breakpoint(i)) { - arr.append(i); - } - } - return arr; -} - -void TextEdit::remove_breakpoints() { - for (int i = 0; i < text.size(); i++) { - if (text.is_breakpoint(i)) { - /* Should "breakpoint_toggled" be fired when breakpoints are removed this way? */ - text.set_breakpoint(i, false); - } - } -} - -void TextEdit::set_line_info_icon(int p_line, Ref<Texture2D> p_icon, String p_info) { - ERR_FAIL_INDEX(p_line, text.size()); - text.set_info_icon(p_line, p_icon, p_info); - update(); -} - -void TextEdit::clear_info_icons() { - text.clear_info_icons(); - update(); -} - void TextEdit::set_line_as_hidden(int p_line, bool p_hidden) { ERR_FAIL_INDEX(p_line, text.size()); if (is_hiding_enabled() || !p_hidden) { @@ -6502,20 +6437,6 @@ void TextEdit::insert_at(const String &p_text, int at) { } } -void TextEdit::set_show_line_numbers(bool p_show) { - line_numbers = p_show; - update(); -} - -void TextEdit::set_line_numbers_zero_padded(bool p_zero_padded) { - line_numbers_zero_padded = p_zero_padded; - update(); -} - -bool TextEdit::is_show_line_numbers_enabled() const { - return line_numbers; -} - void TextEdit::set_show_line_length_guidelines(bool p_show) { line_length_guidelines = p_show; update(); @@ -6531,69 +6452,6 @@ void TextEdit::set_line_length_guideline_hard_column(int p_column) { update(); } -void TextEdit::set_bookmark_gutter_enabled(bool p_draw) { - draw_bookmark_gutter = p_draw; - update(); -} - -bool TextEdit::is_bookmark_gutter_enabled() const { - return draw_bookmark_gutter; -} - -void TextEdit::set_breakpoint_gutter_enabled(bool p_draw) { - draw_breakpoint_gutter = p_draw; - update(); -} - -bool TextEdit::is_breakpoint_gutter_enabled() const { - return draw_breakpoint_gutter; -} - -void TextEdit::set_breakpoint_gutter_width(int p_gutter_width) { - breakpoint_gutter_width = p_gutter_width; - update(); -} - -int TextEdit::get_breakpoint_gutter_width() const { - return cache.breakpoint_gutter_width; -} - -void TextEdit::set_draw_fold_gutter(bool p_draw) { - draw_fold_gutter = p_draw; - update(); -} - -bool TextEdit::is_drawing_fold_gutter() const { - return draw_fold_gutter; -} - -void TextEdit::set_fold_gutter_width(int p_gutter_width) { - fold_gutter_width = p_gutter_width; - update(); -} - -int TextEdit::get_fold_gutter_width() const { - return cache.fold_gutter_width; -} - -void TextEdit::set_draw_info_gutter(bool p_draw) { - draw_info_gutter = p_draw; - update(); -} - -bool TextEdit::is_drawing_info_gutter() const { - return draw_info_gutter; -} - -void TextEdit::set_info_gutter_width(int p_gutter_width) { - info_gutter_width = p_gutter_width; - update(); -} - -int TextEdit::get_info_gutter_width() const { - return info_gutter_width; -} - void TextEdit::set_draw_minimap(bool p_draw) { draw_minimap = p_draw; update(); @@ -6800,16 +6658,10 @@ void TextEdit::_bind_methods() { ClassDB::bind_method(D_METHOD("redo"), &TextEdit::redo); ClassDB::bind_method(D_METHOD("clear_undo_history"), &TextEdit::clear_undo_history); - ClassDB::bind_method(D_METHOD("set_show_line_numbers", "enable"), &TextEdit::set_show_line_numbers); - ClassDB::bind_method(D_METHOD("is_show_line_numbers_enabled"), &TextEdit::is_show_line_numbers_enabled); ClassDB::bind_method(D_METHOD("set_draw_tabs"), &TextEdit::set_draw_tabs); ClassDB::bind_method(D_METHOD("is_drawing_tabs"), &TextEdit::is_drawing_tabs); ClassDB::bind_method(D_METHOD("set_draw_spaces"), &TextEdit::set_draw_spaces); ClassDB::bind_method(D_METHOD("is_drawing_spaces"), &TextEdit::is_drawing_spaces); - ClassDB::bind_method(D_METHOD("set_breakpoint_gutter_enabled", "enable"), &TextEdit::set_breakpoint_gutter_enabled); - ClassDB::bind_method(D_METHOD("is_breakpoint_gutter_enabled"), &TextEdit::is_breakpoint_gutter_enabled); - ClassDB::bind_method(D_METHOD("set_draw_fold_gutter"), &TextEdit::set_draw_fold_gutter); - ClassDB::bind_method(D_METHOD("is_drawing_fold_gutter"), &TextEdit::is_drawing_fold_gutter); ClassDB::bind_method(D_METHOD("set_hiding_enabled", "enable"), &TextEdit::set_hiding_enabled); ClassDB::bind_method(D_METHOD("is_hiding_enabled"), &TextEdit::is_hiding_enabled); @@ -6832,6 +6684,40 @@ void TextEdit::_bind_methods() { ClassDB::bind_method(D_METHOD("set_syntax_highlighter", "syntax_highlighter"), &TextEdit::set_syntax_highlighter); ClassDB::bind_method(D_METHOD("get_syntax_highlighter"), &TextEdit::get_syntax_highlighter); + /* Gutters. */ + BIND_ENUM_CONSTANT(GUTTER_TYPE_STRING); + BIND_ENUM_CONSTANT(GUTTER_TPYE_ICON); + BIND_ENUM_CONSTANT(GUTTER_TPYE_CUSTOM); + + ClassDB::bind_method(D_METHOD("add_gutter", "at"), &TextEdit::add_gutter, DEFVAL(-1)); + ClassDB::bind_method(D_METHOD("remove_gutter", "gutter"), &TextEdit::remove_gutter); + ClassDB::bind_method(D_METHOD("get_gutter_count"), &TextEdit::get_gutter_count); + ClassDB::bind_method(D_METHOD("set_gutter_name", "gutter", "name"), &TextEdit::set_gutter_name); + ClassDB::bind_method(D_METHOD("get_gutter_name", "gutter"), &TextEdit::get_gutter_name); + ClassDB::bind_method(D_METHOD("set_gutter_type", "gutter", "type"), &TextEdit::set_gutter_type); + ClassDB::bind_method(D_METHOD("get_gutter_type", "gutter"), &TextEdit::get_gutter_type); + ClassDB::bind_method(D_METHOD("set_gutter_width", "gutter", "width"), &TextEdit::set_gutter_width); + ClassDB::bind_method(D_METHOD("get_gutter_width", "gutter"), &TextEdit::get_gutter_width); + ClassDB::bind_method(D_METHOD("set_gutter_draw", "gutter", "draw"), &TextEdit::set_gutter_draw); + ClassDB::bind_method(D_METHOD("is_gutter_drawn", "gutter"), &TextEdit::is_gutter_drawn); + ClassDB::bind_method(D_METHOD("set_gutter_clickable", "gutter", "clickable"), &TextEdit::set_gutter_clickable); + ClassDB::bind_method(D_METHOD("is_gutter_clickable", "gutter"), &TextEdit::is_gutter_clickable); + ClassDB::bind_method(D_METHOD("set_gutter_overwritable", "gutter", "overwritable"), &TextEdit::set_gutter_overwritable); + ClassDB::bind_method(D_METHOD("is_gutter_overwritable", "gutter"), &TextEdit::is_gutter_overwritable); + ClassDB::bind_method(D_METHOD("set_gutter_custom_draw", "column", "object", "callback"), &TextEdit::set_gutter_custom_draw); + + // Line gutters. + ClassDB::bind_method(D_METHOD("set_line_gutter_metadata", "line", "gutter", "metadata"), &TextEdit::set_line_gutter_metadata); + ClassDB::bind_method(D_METHOD("get_line_gutter_metadata", "line", "gutter"), &TextEdit::get_line_gutter_metadata); + ClassDB::bind_method(D_METHOD("set_line_gutter_text", "line", "gutter", "text"), &TextEdit::set_line_gutter_text); + ClassDB::bind_method(D_METHOD("get_line_gutter_text", "line", "gutter"), &TextEdit::get_line_gutter_text); + ClassDB::bind_method(D_METHOD("set_line_gutter_icon", "line", "gutter", "icon"), &TextEdit::set_line_gutter_icon); + ClassDB::bind_method(D_METHOD("get_line_gutter_icon", "line", "gutter"), &TextEdit::get_line_gutter_icon); + ClassDB::bind_method(D_METHOD("set_line_gutter_item_color", "line", "gutter", "color"), &TextEdit::set_line_gutter_item_color); + ClassDB::bind_method(D_METHOD("get_line_gutter_item_color", "line", "gutter"), &TextEdit::get_line_gutter_item_color); + ClassDB::bind_method(D_METHOD("set_line_gutter_clickable", "line", "gutter", "clickable"), &TextEdit::set_line_gutter_clickable); + ClassDB::bind_method(D_METHOD("is_line_gutter_clickable", "line", "gutter"), &TextEdit::is_line_gutter_clickable); + ClassDB::bind_method(D_METHOD("set_highlight_current_line", "enabled"), &TextEdit::set_highlight_current_line); ClassDB::bind_method(D_METHOD("is_highlight_current_line_enabled"), &TextEdit::is_highlight_current_line_enabled); @@ -6847,9 +6733,6 @@ void TextEdit::_bind_methods() { ClassDB::bind_method(D_METHOD("menu_option", "option"), &TextEdit::menu_option); ClassDB::bind_method(D_METHOD("get_menu"), &TextEdit::get_menu); - ClassDB::bind_method(D_METHOD("get_breakpoints"), &TextEdit::get_breakpoints_array); - ClassDB::bind_method(D_METHOD("remove_breakpoints"), &TextEdit::remove_breakpoints); - ClassDB::bind_method(D_METHOD("draw_minimap", "draw"), &TextEdit::set_draw_minimap); ClassDB::bind_method(D_METHOD("is_drawing_minimap"), &TextEdit::is_drawing_minimap); ClassDB::bind_method(D_METHOD("set_minimap_width", "width"), &TextEdit::set_minimap_width); @@ -6858,11 +6741,8 @@ void TextEdit::_bind_methods() { ADD_PROPERTY(PropertyInfo(Variant::STRING, "text", PROPERTY_HINT_MULTILINE_TEXT), "set_text", "get_text"); ADD_PROPERTY(PropertyInfo(Variant::BOOL, "readonly"), "set_readonly", "is_readonly"); ADD_PROPERTY(PropertyInfo(Variant::BOOL, "highlight_current_line"), "set_highlight_current_line", "is_highlight_current_line_enabled"); - ADD_PROPERTY(PropertyInfo(Variant::BOOL, "show_line_numbers"), "set_show_line_numbers", "is_show_line_numbers_enabled"); ADD_PROPERTY(PropertyInfo(Variant::BOOL, "draw_tabs"), "set_draw_tabs", "is_drawing_tabs"); ADD_PROPERTY(PropertyInfo(Variant::BOOL, "draw_spaces"), "set_draw_spaces", "is_drawing_spaces"); - ADD_PROPERTY(PropertyInfo(Variant::BOOL, "breakpoint_gutter"), "set_breakpoint_gutter_enabled", "is_breakpoint_gutter_enabled"); - ADD_PROPERTY(PropertyInfo(Variant::BOOL, "fold_gutter"), "set_draw_fold_gutter", "is_drawing_fold_gutter"); ADD_PROPERTY(PropertyInfo(Variant::BOOL, "highlight_all_occurrences"), "set_highlight_all_occurrences", "is_highlight_all_occurrences_enabled"); ADD_PROPERTY(PropertyInfo(Variant::BOOL, "override_selected_font_color"), "set_override_selected_font_color", "is_overriding_selected_font_color"); ADD_PROPERTY(PropertyInfo(Variant::BOOL, "context_menu_enabled"), "set_context_menu_enabled", "is_context_menu_enabled"); @@ -6890,11 +6770,12 @@ void TextEdit::_bind_methods() { ADD_SIGNAL(MethodInfo("cursor_changed")); ADD_SIGNAL(MethodInfo("text_changed")); - ADD_SIGNAL(MethodInfo("line_edited_from", PropertyInfo(Variant::INT, "line"))); + ADD_SIGNAL(MethodInfo("lines_edited_from", PropertyInfo(Variant::INT, "from_line"), PropertyInfo(Variant::INT, "to_line"))); ADD_SIGNAL(MethodInfo("request_completion")); - ADD_SIGNAL(MethodInfo("breakpoint_toggled", PropertyInfo(Variant::INT, "row"))); + ADD_SIGNAL(MethodInfo("gutter_clicked", PropertyInfo(Variant::INT, "line"), PropertyInfo(Variant::INT, "gutter"))); + ADD_SIGNAL(MethodInfo("gutter_added")); + ADD_SIGNAL(MethodInfo("gutter_removed")); ADD_SIGNAL(MethodInfo("symbol_lookup", PropertyInfo(Variant::STRING, "symbol"), PropertyInfo(Variant::INT, "row"), PropertyInfo(Variant::INT, "column"))); - ADD_SIGNAL(MethodInfo("info_clicked", PropertyInfo(Variant::INT, "row"), PropertyInfo(Variant::STRING, "info"))); ADD_SIGNAL(MethodInfo("symbol_validate", PropertyInfo(Variant::STRING, "symbol"))); BIND_ENUM_CONSTANT(MENU_CUT); @@ -6927,13 +6808,6 @@ TextEdit::TextEdit() { _update_caches(); cache.row_height = 1; cache.line_spacing = 1; - cache.line_number_w = 1; - cache.breakpoint_gutter_width = 0; - breakpoint_gutter_width = 0; - cache.fold_gutter_width = 0; - fold_gutter_width = 0; - info_gutter_width = 0; - cache.info_gutter_width = 0; set_default_cursor_shape(CURSOR_IBEAM); indent_size = 4; @@ -6997,15 +6871,9 @@ TextEdit::TextEdit() { completion_active = false; completion_line_ofs = 0; tooltip_obj = nullptr; - line_numbers = false; - line_numbers_zero_padded = false; line_length_guidelines = false; line_length_guideline_soft_col = 80; line_length_guideline_hard_col = 100; - draw_bookmark_gutter = false; - draw_breakpoint_gutter = false; - draw_fold_gutter = false; - draw_info_gutter = false; hiding_enabled = false; next_operation_is_complex = false; scroll_past_end_of_file_enabled = false; @@ -7043,8 +6911,6 @@ TextEdit::TextEdit() { set_readonly(false); menu->connect("id_pressed", callable_mp(this, &TextEdit::menu_option)); first_draw = true; - - executing_line = -1; } TextEdit::~TextEdit() { diff --git a/scene/gui/text_edit.h b/scene/gui/text_edit.h index 70d7365d71..562f4768ae 100644 --- a/scene/gui/text_edit.h +++ b/scene/gui/text_edit.h @@ -41,28 +41,53 @@ class TextEdit : public Control { GDCLASS(TextEdit, Control); public: + enum GutterType { + GUTTER_TYPE_STRING, + GUTTER_TPYE_ICON, + GUTTER_TPYE_CUSTOM + }; + +private: + struct GutterInfo { + GutterType type = GutterType::GUTTER_TYPE_STRING; + String name = ""; + int width = 24; + bool draw = true; + bool clickable = false; + bool overwritable = false; + + ObjectID custom_draw_obj = ObjectID(); + StringName custom_draw_callback; + }; + Vector<GutterInfo> gutters; + int gutters_width = 0; + int gutter_padding = 0; + + void _update_gutter_width(); + class Text { public: + struct Gutter { + Variant metadata; + bool clickable = false; + + Ref<Texture2D> icon = Ref<Texture2D>(); + String text = ""; + Color color = Color(1, 1, 1); + }; + struct Line { - int width_cache : 24; - bool marked : 1; - bool breakpoint : 1; - bool bookmark : 1; - bool hidden : 1; - bool safe : 1; - bool has_info : 1; - int wrap_amount_cache : 24; - Ref<Texture2D> info_icon; - String info; + Vector<Gutter> gutters; + + int32_t width_cache; + bool marked; + bool hidden; + int32_t wrap_amount_cache; String data; Line() { width_cache = 0; marked = false; - breakpoint = false; - bookmark = false; hidden = false; - safe = false; - has_info = false; wrap_amount_cache = 0; } }; @@ -70,7 +95,8 @@ public: private: mutable Vector<Line> text; Ref<Font> font; - int indent_size; + int indent_size = 4; + int gutter_count = 0; void _update_line_cache(int p_line) const; @@ -85,38 +111,37 @@ public: void set(int p_line, const String &p_text); void set_marked(int p_line, bool p_marked) { text.write[p_line].marked = p_marked; } bool is_marked(int p_line) const { return text[p_line].marked; } - void set_bookmark(int p_line, bool p_bookmark) { text.write[p_line].bookmark = p_bookmark; } - bool is_bookmark(int p_line) const { return text[p_line].bookmark; } - void set_breakpoint(int p_line, bool p_breakpoint) { text.write[p_line].breakpoint = p_breakpoint; } - bool is_breakpoint(int p_line) const { return text[p_line].breakpoint; } void set_hidden(int p_line, bool p_hidden) { text.write[p_line].hidden = p_hidden; } bool is_hidden(int p_line) const { return text[p_line].hidden; } - void set_safe(int p_line, bool p_safe) { text.write[p_line].safe = p_safe; } - bool is_safe(int p_line) const { return text[p_line].safe; } - void set_info_icon(int p_line, Ref<Texture2D> p_icon, String p_info) { - if (p_icon.is_null()) { - text.write[p_line].has_info = false; - return; - } - text.write[p_line].info_icon = p_icon; - text.write[p_line].info = p_info; - text.write[p_line].has_info = true; - } - bool has_info_icon(int p_line) const { return text[p_line].has_info; } - const Ref<Texture2D> &get_info_icon(int p_line) const { return text[p_line].info_icon; } - const String &get_info(int p_line) const { return text[p_line].info; } void insert(int p_at, const String &p_text); void remove(int p_at); int size() const { return text.size(); } void clear(); void clear_width_cache(); void clear_wrap_cache(); - void clear_info_icons(); _FORCE_INLINE_ const String &operator[](int p_line) const { return text[p_line].data; } - Text() { indent_size = 4; } + + /* Gutters. */ + void add_gutter(int p_at); + void remove_gutter(int p_gutter); + void move_gutters(int p_from_line, int p_to_line); + + void set_line_gutter_metadata(int p_line, int p_gutter, const Variant &p_metadata) { text.write[p_line].gutters.write[p_gutter].metadata = p_metadata; } + const Variant &get_line_gutter_metadata(int p_line, int p_gutter) const { return text[p_line].gutters[p_gutter].metadata; } + + void set_line_gutter_text(int p_line, int p_gutter, const String &p_text) { text.write[p_line].gutters.write[p_gutter].text = p_text; } + const String &get_line_gutter_text(int p_line, int p_gutter) const { return text[p_line].gutters[p_gutter].text; } + + void set_line_gutter_icon(int p_line, int p_gutter, Ref<Texture2D> p_icon) { text.write[p_line].gutters.write[p_gutter].icon = p_icon; } + const Ref<Texture2D> &get_line_gutter_icon(int p_line, int p_gutter) const { return text[p_line].gutters[p_gutter].icon; } + + void set_line_gutter_item_color(int p_line, int p_gutter, const Color &p_color) { text.write[p_line].gutters.write[p_gutter].color = p_color; } + const Color &get_line_gutter_item_color(int p_line, int p_gutter) const { return text[p_line].gutters[p_gutter].color; } + + void set_line_gutter_clickable(int p_line, int p_gutter, bool p_clickable) { text.write[p_line].gutters.write[p_gutter].clickable = p_clickable; } + bool is_line_gutter_clickable(int p_line, int p_gutter) const { return text[p_line].gutters[p_gutter].clickable; } }; -private: struct Cursor { int last_fit_x; int line, column; ///< cursor @@ -169,60 +194,6 @@ private: } } selection; - struct Cache { - Ref<Texture2D> tab_icon; - Ref<Texture2D> space_icon; - Ref<Texture2D> can_fold_icon; - Ref<Texture2D> folded_icon; - Ref<Texture2D> folded_eol_icon; - Ref<Texture2D> executing_icon; - Ref<StyleBox> style_normal; - Ref<StyleBox> style_focus; - Ref<StyleBox> style_readonly; - Ref<Font> font; - Color completion_background_color; - Color completion_selected_color; - Color completion_existing_color; - Color completion_font_color; - Color caret_color; - Color caret_background_color; - Color line_number_color; - Color safe_line_number_color; - Color font_color; - Color font_color_selected; - Color font_color_readonly; - Color selection_color; - Color mark_color; - Color bookmark_color; - Color breakpoint_color; - Color executing_line_color; - Color code_folding_color; - Color current_line_color; - Color line_length_guideline_color; - Color brace_mismatch_color; - Color word_highlighted_color; - Color search_result_color; - Color search_result_border_color; - Color background_color; - - int row_height; - int line_spacing; - int line_number_w; - int breakpoint_gutter_width; - int fold_gutter_width; - int info_gutter_width; - int minimap_width; - Cache() { - row_height = 0; - line_spacing = 0; - line_number_w = 0; - breakpoint_gutter_width = 0; - fold_gutter_width = 0; - info_gutter_width = 0; - minimap_width = 0; - } - } cache; - Map<int, Dictionary> syntax_highlighting_cache; struct TextOperation { @@ -318,19 +289,10 @@ private: bool cursor_changed_dirty; bool text_changed_dirty; bool undo_enabled; - bool line_numbers; - bool line_numbers_zero_padded; bool line_length_guidelines; int line_length_guideline_soft_col; int line_length_guideline_hard_col; - bool draw_bookmark_gutter; - bool draw_breakpoint_gutter; - int breakpoint_gutter_width; - bool draw_fold_gutter; - int fold_gutter_width; bool hiding_enabled; - bool draw_info_gutter; - int info_gutter_width; bool draw_minimap; int minimap_width; Point2 minimap_char_size; @@ -385,11 +347,8 @@ private: bool context_menu_enabled; bool shortcut_keys_enabled; - bool virtual_keyboard_enabled = true; - int executing_line; - void _generate_context_menu(); int get_visible_rows() const; @@ -447,8 +406,6 @@ private: Size2 get_minimum_size() const override; int _get_control_height() const; - int get_row_height() const; - void _reset_caret_blink_timer(); void _toggle_draw_caret(); @@ -480,6 +437,44 @@ private: int _calculate_spaces_till_next_right_indent(int column); protected: + struct Cache { + Ref<Texture2D> tab_icon; + Ref<Texture2D> space_icon; + Ref<Texture2D> folded_eol_icon; + Ref<StyleBox> style_normal; + Ref<StyleBox> style_focus; + Ref<StyleBox> style_readonly; + Ref<Font> font; + Color completion_background_color; + Color completion_selected_color; + Color completion_existing_color; + Color completion_font_color; + Color caret_color; + Color caret_background_color; + Color font_color; + Color font_color_selected; + Color font_color_readonly; + Color selection_color; + Color mark_color; + Color code_folding_color; + Color current_line_color; + Color line_length_guideline_color; + Color brace_mismatch_color; + Color word_highlighted_color; + Color search_result_color; + Color search_result_border_color; + Color background_color; + + int row_height; + int line_spacing; + int minimap_width; + Cache() { + row_height = 0; + line_spacing = 0; + minimap_width = 0; + } + } cache; + virtual String get_tooltip(const Point2 &p_pos) const override; void _insert_text(int p_line, int p_char, const String &p_text, int *r_end_line = nullptr, int *r_end_char = nullptr); @@ -494,9 +489,51 @@ protected: static void _bind_methods(); public: + /* Syntax Highlighting. */ Ref<SyntaxHighlighter> get_syntax_highlighter(); void set_syntax_highlighter(Ref<SyntaxHighlighter> p_syntax_highlighter); + /* Gutters. */ + void add_gutter(int p_at = -1); + void remove_gutter(int p_gutter); + int get_gutter_count() const; + + void set_gutter_name(int p_gutter, const String &p_name); + String get_gutter_name(int p_gutter) const; + + void set_gutter_type(int p_gutter, GutterType p_type); + GutterType get_gutter_type(int p_gutter) const; + + void set_gutter_width(int p_gutter, int p_width); + int get_gutter_width(int p_gutter) const; + + void set_gutter_draw(int p_gutter, bool p_draw); + bool is_gutter_drawn(int p_gutter) const; + + void set_gutter_clickable(int p_gutter, bool p_clickable); + bool is_gutter_clickable(int p_gutter) const; + + void set_gutter_overwritable(int p_gutter, bool p_overwritable); + bool is_gutter_overwritable(int p_gutter) const; + + void set_gutter_custom_draw(int p_gutter, Object *p_object, const StringName &p_callback); + + // Line gutters. + void set_line_gutter_metadata(int p_line, int p_gutter, const Variant &p_metadata); + Variant get_line_gutter_metadata(int p_line, int p_gutter) const; + + void set_line_gutter_text(int p_line, int p_gutter, const String &p_text); + String get_line_gutter_text(int p_line, int p_gutter) const; + + void set_line_gutter_icon(int p_line, int p_gutter, Ref<Texture2D> p_icon); + Ref<Texture2D> get_line_gutter_icon(int p_line, int p_gutter) const; + + void set_line_gutter_item_color(int p_line, int p_gutter, const Color &p_color); + Color get_line_gutter_item_color(int p_line, int p_gutter); + + void set_line_gutter_clickable(int p_line, int p_gutter, bool p_clickable); + bool is_line_gutter_clickable(int p_line, int p_gutter) const; + enum MenuItems { MENU_CUT, MENU_COPY, @@ -534,22 +571,6 @@ public: void insert_at(const String &p_text, int at); int get_line_count() const; void set_line_as_marked(int p_line, bool p_marked); - void set_line_as_bookmark(int p_line, bool p_bookmark); - bool is_line_set_as_bookmark(int p_line) const; - void get_bookmarks(List<int> *p_bookmarks) const; - Array get_bookmarks_array() const; - void set_line_as_breakpoint(int p_line, bool p_breakpoint); - bool is_line_set_as_breakpoint(int p_line) const; - void set_executing_line(int p_line); - void clear_executing_line(); - void set_line_as_safe(int p_line, bool p_safe); - bool is_line_set_as_safe(int p_line) const; - void get_breakpoints(List<int> *p_breakpoints) const; - Array get_breakpoints_array() const; - void remove_breakpoints(); - - void set_line_info_icon(int p_line, Ref<Texture2D> p_icon, String p_info = ""); - void clear_info_icons(); void set_line_as_hidden(int p_line, bool p_hidden); bool is_line_hidden(int p_line) const; @@ -569,6 +590,7 @@ public: String get_text(); String get_line(int line) const; void set_line(int line, String new_text); + int get_row_height() const; void backspace_at_cursor(); void indent_left(); @@ -690,39 +712,13 @@ public: void menu_option(int p_option); - void set_show_line_numbers(bool p_show); - bool is_show_line_numbers_enabled() const; - void set_highlight_current_line(bool p_enabled); bool is_highlight_current_line_enabled() const; - void set_line_numbers_zero_padded(bool p_zero_padded); - void set_show_line_length_guidelines(bool p_show); void set_line_length_guideline_soft_column(int p_column); void set_line_length_guideline_hard_column(int p_column); - void set_bookmark_gutter_enabled(bool p_draw); - bool is_bookmark_gutter_enabled() const; - - void set_breakpoint_gutter_enabled(bool p_draw); - bool is_breakpoint_gutter_enabled() const; - - void set_breakpoint_gutter_width(int p_gutter_width); - int get_breakpoint_gutter_width() const; - - void set_draw_fold_gutter(bool p_draw); - bool is_drawing_fold_gutter() const; - - void set_fold_gutter_width(int p_gutter_width); - int get_fold_gutter_width() const; - - void set_draw_info_gutter(bool p_draw); - bool is_drawing_info_gutter() const; - - void set_info_gutter_width(int p_gutter_width); - int get_info_gutter_width() const; - void set_draw_minimap(bool p_draw); bool is_drawing_minimap() const; @@ -764,6 +760,7 @@ public: ~TextEdit(); }; +VARIANT_ENUM_CAST(TextEdit::GutterType); VARIANT_ENUM_CAST(TextEdit::MenuItems); VARIANT_ENUM_CAST(TextEdit::SearchFlags); diff --git a/scene/main/canvas_item.cpp b/scene/main/canvas_item.cpp index 5cd45ea408..796408a38d 100644 --- a/scene/main/canvas_item.cpp +++ b/scene/main/canvas_item.cpp @@ -1369,6 +1369,7 @@ void CanvasItem::set_texture_filter(TextureFilter p_texture_filter) { } texture_filter = p_texture_filter; _update_texture_filter_changed(true); + _change_notify(); } CanvasItem::TextureFilter CanvasItem::get_texture_filter() const { @@ -1421,6 +1422,7 @@ void CanvasItem::set_texture_repeat(TextureRepeat p_texture_repeat) { } texture_repeat = p_texture_repeat; _update_texture_repeat_changed(true); + _change_notify(); } CanvasItem::TextureRepeat CanvasItem::get_texture_repeat() const { diff --git a/scene/main/canvas_item.h b/scene/main/canvas_item.h index d9ffe770ff..3f32df87a7 100644 --- a/scene/main/canvas_item.h +++ b/scene/main/canvas_item.h @@ -405,10 +405,10 @@ public: void force_update_transform(); - void set_texture_filter(TextureFilter p_texture_filter); + virtual void set_texture_filter(TextureFilter p_texture_filter); TextureFilter get_texture_filter() const; - void set_texture_repeat(TextureRepeat p_texture_repeat); + virtual void set_texture_repeat(TextureRepeat p_texture_repeat); TextureRepeat get_texture_repeat() const; // Used by control nodes to retrieve the parent's anchorable area diff --git a/scene/main/http_request.cpp b/scene/main/http_request.cpp index 2b506b686b..da0169b60b 100644 --- a/scene/main/http_request.cpp +++ b/scene/main/http_request.cpp @@ -631,6 +631,7 @@ void HTTPRequest::_bind_methods() { BIND_ENUM_CONSTANT(RESULT_SSL_HANDSHAKE_ERROR); BIND_ENUM_CONSTANT(RESULT_NO_RESPONSE); BIND_ENUM_CONSTANT(RESULT_BODY_SIZE_LIMIT_EXCEEDED); + BIND_ENUM_CONSTANT(RESULT_BODY_DECOMPRESS_FAILED); BIND_ENUM_CONSTANT(RESULT_REQUEST_FAILED); BIND_ENUM_CONSTANT(RESULT_DOWNLOAD_FILE_CANT_OPEN); BIND_ENUM_CONSTANT(RESULT_DOWNLOAD_FILE_WRITE_ERROR); diff --git a/scene/main/scene_tree.h b/scene/main/scene_tree.h index 0f74f2e973..e5ab4f9958 100644 --- a/scene/main/scene_tree.h +++ b/scene/main/scene_tree.h @@ -28,8 +28,8 @@ /* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ /*************************************************************************/ -#ifndef SCENE_MAIN_LOOP_H -#define SCENE_MAIN_LOOP_H +#ifndef SCENE_TREE_H +#define SCENE_TREE_H #include "core/io/multiplayer_api.h" #include "core/os/main_loop.h" @@ -354,4 +354,4 @@ public: VARIANT_ENUM_CAST(SceneTree::GroupCallFlags); -#endif +#endif // SCENE_TREE_H diff --git a/scene/main/viewport.cpp b/scene/main/viewport.cpp index d962171555..b29b40ea5f 100644 --- a/scene/main/viewport.cpp +++ b/scene/main/viewport.cpp @@ -1509,7 +1509,7 @@ void Viewport::_gui_show_tooltip() { } Control *which = nullptr; - String tooltip = _gui_get_tooltip(gui.tooltip, gui.tooltip->get_global_transform().xform_inv(gui.tooltip_pos), &which); + String tooltip = _gui_get_tooltip(gui.tooltip, gui.tooltip->get_global_transform().xform_inv(gui.last_mouse_pos), &which); tooltip = tooltip.strip_edges(); if (tooltip.length() == 0) { return; // bye diff --git a/scene/register_scene_types.cpp b/scene/register_scene_types.cpp index 0f6475cf0d..820513c53d 100644 --- a/scene/register_scene_types.cpp +++ b/scene/register_scene_types.cpp @@ -81,6 +81,7 @@ #include "scene/gui/center_container.h" #include "scene/gui/check_box.h" #include "scene/gui/check_button.h" +#include "scene/gui/code_edit.h" #include "scene/gui/color_picker.h" #include "scene/gui/color_rect.h" #include "scene/gui/control.h" @@ -230,6 +231,7 @@ static Ref<ResourceFormatLoaderDynamicFont> resource_loader_dynamic_font; static Ref<ResourceFormatLoaderStreamTexture2D> resource_loader_stream_texture; static Ref<ResourceFormatLoaderStreamTextureLayered> resource_loader_texture_layered; +static Ref<ResourceFormatLoaderStreamTexture3D> resource_loader_texture_3d; static Ref<ResourceFormatLoaderBMFont> resource_loader_bmfont; @@ -252,6 +254,9 @@ void register_scene_types() { resource_loader_texture_layered.instance(); ResourceLoader::add_resource_format_loader(resource_loader_texture_layered); + resource_loader_texture_3d.instance(); + ResourceLoader::add_resource_format_loader(resource_loader_texture_3d); + resource_saver_text.instance(); ResourceSaver::add_resource_format_saver(resource_saver_text, true); @@ -291,7 +296,7 @@ void register_scene_types() { OS::get_singleton()->yield(); //may take time to init - ClassDB::register_class<ShortCut>(); + ClassDB::register_class<Shortcut>(); ClassDB::register_class<Control>(); ClassDB::register_class<Button>(); ClassDB::register_class<Label>(); @@ -347,6 +352,7 @@ void register_scene_types() { ClassDB::register_class<Tree>(); ClassDB::register_class<TextEdit>(); + ClassDB::register_class<CodeEdit>(); ClassDB::register_class<SyntaxHighlighter>(); ClassDB::register_class<CodeHighlighter>(); @@ -549,6 +555,7 @@ void register_scene_types() { ClassDB::register_class<VisualShaderNodeTexture>(); ClassDB::register_virtual_class<VisualShaderNodeSample3D>(); ClassDB::register_class<VisualShaderNodeTexture2DArray>(); + ClassDB::register_class<VisualShaderNodeTexture3D>(); ClassDB::register_class<VisualShaderNodeCubemap>(); ClassDB::register_virtual_class<VisualShaderNodeUniform>(); ClassDB::register_class<VisualShaderNodeUniformRef>(); @@ -561,6 +568,7 @@ void register_scene_types() { ClassDB::register_class<VisualShaderNodeTextureUniform>(); ClassDB::register_class<VisualShaderNodeTextureUniformTriplanar>(); ClassDB::register_class<VisualShaderNodeTexture2DArrayUniform>(); + ClassDB::register_class<VisualShaderNodeTexture3DUniform>(); ClassDB::register_class<VisualShaderNodeCubemapUniform>(); ClassDB::register_class<VisualShaderNodeIf>(); ClassDB::register_class<VisualShaderNodeSwitch>(); @@ -701,6 +709,9 @@ void register_scene_types() { ClassDB::register_class<CameraTexture>(); ClassDB::register_virtual_class<TextureLayered>(); ClassDB::register_virtual_class<ImageTextureLayered>(); + ClassDB::register_virtual_class<Texture3D>(); + ClassDB::register_class<ImageTexture3D>(); + ClassDB::register_class<StreamTexture3D>(); ClassDB::register_class<Cubemap>(); ClassDB::register_class<CubemapArray>(); ClassDB::register_class<Texture2DArray>(); @@ -864,6 +875,7 @@ void register_scene_types() { ClassDB::add_compatibility_class("RemoteTransform", "RemoteTransform3D"); ClassDB::add_compatibility_class("RigidBody", "RigidBody3D"); ClassDB::add_compatibility_class("Shape", "Shape3D"); + ClassDB::add_compatibility_class("ShortCut", "Shortcut"); ClassDB::add_compatibility_class("Skeleton", "Skeleton3D"); ClassDB::add_compatibility_class("SkeletonIK", "SkeletonIK3D"); ClassDB::add_compatibility_class("SliderJoint", "SliderJoint3D"); @@ -946,6 +958,9 @@ void unregister_scene_types() { ResourceLoader::remove_resource_format_loader(resource_loader_texture_layered); resource_loader_texture_layered.unref(); + ResourceLoader::remove_resource_format_loader(resource_loader_texture_3d); + resource_loader_texture_3d.unref(); + ResourceLoader::remove_resource_format_loader(resource_loader_stream_texture); resource_loader_stream_texture.unref(); diff --git a/scene/resources/default_theme/bookmark.png b/scene/resources/default_theme/bookmark.png Binary files differnew file mode 100644 index 0000000000..9718cf53b6 --- /dev/null +++ b/scene/resources/default_theme/bookmark.png diff --git a/scene/resources/default_theme/default_theme.cpp b/scene/resources/default_theme/default_theme.cpp index 9008f6d5b9..f65f78b332 100644 --- a/scene/resources/default_theme/default_theme.cpp +++ b/scene/resources/default_theme/default_theme.cpp @@ -382,8 +382,6 @@ void fill_default_theme(Ref<Theme> &theme, const Ref<Font> &default_font, const theme->set_icon("tab", "TextEdit", make_icon(tab_png)); theme->set_icon("space", "TextEdit", make_icon(space_png)); - theme->set_icon("folded", "TextEdit", make_icon(arrow_right_png)); - theme->set_icon("fold", "TextEdit", make_icon(arrow_down_png)); theme->set_font("font", "TextEdit", Ref<Font>()); @@ -398,16 +396,11 @@ void fill_default_theme(Ref<Theme> &theme, const Ref<Font> &default_font, const theme->set_color("font_color_readonly", "TextEdit", Color(control_font_color.r, control_font_color.g, control_font_color.b, 0.5f)); theme->set_color("selection_color", "TextEdit", font_color_selection); theme->set_color("mark_color", "TextEdit", Color(1.0, 0.4, 0.4, 0.4)); - theme->set_color("bookmark_color", "TextEdit", Color(0.08, 0.49, 0.98)); - theme->set_color("breakpoint_color", "TextEdit", Color(0.8, 0.8, 0.4, 0.2)); - theme->set_color("executing_line_color", "TextEdit", Color(0.2, 0.8, 0.2, 0.4)); theme->set_color("code_folding_color", "TextEdit", Color(0.8, 0.8, 0.8, 0.8)); theme->set_color("current_line_color", "TextEdit", Color(0.25, 0.25, 0.26, 0.8)); theme->set_color("caret_color", "TextEdit", control_font_color); theme->set_color("caret_background_color", "TextEdit", Color(0, 0, 0)); theme->set_color("brace_mismatch_color", "TextEdit", Color(1, 0.2, 0.2)); - theme->set_color("line_number_color", "TextEdit", Color(0.67, 0.67, 0.67, 0.4)); - theme->set_color("safe_line_number_color", "TextEdit", Color(0.67, 0.78, 0.67, 0.6)); theme->set_color("word_highlighted_color", "TextEdit", Color(0.8, 0.9, 0.9, 0.15)); theme->set_constant("completion_lines", "TextEdit", 7); @@ -415,6 +408,50 @@ void fill_default_theme(Ref<Theme> &theme, const Ref<Font> &default_font, const theme->set_constant("completion_scroll_width", "TextEdit", 3); theme->set_constant("line_spacing", "TextEdit", 4 * scale); + // CodeEdit + theme->set_stylebox("normal", "CodeEdit", make_stylebox(tree_bg_png, 3, 3, 3, 3, 0, 0, 0, 0)); + theme->set_stylebox("focus", "CodeEdit", focus); + theme->set_stylebox("read_only", "CodeEdit", make_stylebox(tree_bg_disabled_png, 4, 4, 4, 4, 0, 0, 0, 0)); + theme->set_stylebox("completion", "CodeEdit", make_stylebox(tree_bg_png, 3, 3, 3, 3, 0, 0, 0, 0)); + + theme->set_icon("tab", "CodeEdit", make_icon(tab_png)); + theme->set_icon("space", "CodeEdit", make_icon(space_png)); + theme->set_icon("breakpoint", "CodeEdit", make_icon(graph_port_png)); + theme->set_icon("bookmark", "CodeEdit", make_icon(bookmark_png)); + theme->set_icon("executing_line", "CodeEdit", make_icon(arrow_right_png)); + theme->set_icon("can_fold", "CodeEdit", make_icon(arrow_down_png)); + theme->set_icon("folded", "CodeEdit", make_icon(arrow_right_png)); + + theme->set_font("font", "CodeEdit", Ref<Font>()); + + theme->set_color("background_color", "CodeEdit", Color(0, 0, 0, 0)); + theme->set_color("completion_background_color", "CodeEdit", Color(0.17, 0.16, 0.2)); + theme->set_color("completion_selected_color", "CodeEdit", Color(0.26, 0.26, 0.27)); + theme->set_color("completion_existing_color", "CodeEdit", Color(0.87, 0.87, 0.87, 0.13)); + theme->set_color("completion_scroll_color", "CodeEdit", control_font_color_pressed); + theme->set_color("completion_font_color", "CodeEdit", Color(0.67, 0.67, 0.67)); + theme->set_color("font_color", "CodeEdit", control_font_color); + theme->set_color("font_color_selected", "CodeEdit", Color(0, 0, 0)); + theme->set_color("font_color_readonly", "CodeEdit", Color(control_font_color.r, control_font_color.g, control_font_color.b, 0.5f)); + theme->set_color("selection_color", "CodeEdit", font_color_selection); + theme->set_color("mark_color", "CodeEdit", Color(1.0, 0.4, 0.4, 0.4)); + theme->set_color("bookmark_color", "CodeEdit", Color(0.5, 0.64, 1, 0.8)); + theme->set_color("breakpoint_color", "CodeEdit", Color(0.9, 0.29, 0.3)); + theme->set_color("executing_line_color", "CodeEdit", Color(0.98, 0.89, 0.27)); + theme->set_color("code_folding_color", "CodeEdit", Color(0.8, 0.8, 0.8, 0.8)); + theme->set_color("current_line_color", "CodeEdit", Color(0.25, 0.25, 0.26, 0.8)); + theme->set_color("caret_color", "CodeEdit", control_font_color); + theme->set_color("caret_background_color", "CodeEdit", Color(0, 0, 0)); + theme->set_color("brace_mismatch_color", "CodeEdit", Color(1, 0.2, 0.2)); + theme->set_color("line_number_color", "CodeEdit", Color(0.67, 0.67, 0.67, 0.4)); + theme->set_color("safe_line_number_color", "CodeEdit", Color(0.67, 0.78, 0.67, 0.6)); + theme->set_color("word_highlighted_color", "CodeEdit", Color(0.8, 0.9, 0.9, 0.15)); + + theme->set_constant("completion_lines", "CodeEdit", 7); + theme->set_constant("completion_max_width", "CodeEdit", 50); + theme->set_constant("completion_scroll_width", "CodeEdit", 3); + theme->set_constant("line_spacing", "CodeEdit", 4 * scale); + Ref<Texture2D> empty_icon = memnew(ImageTexture); // HScrollBar diff --git a/scene/resources/default_theme/theme_data.h b/scene/resources/default_theme/theme_data.h index edcdb90db9..a15efb593a 100644 --- a/scene/resources/default_theme/theme_data.h +++ b/scene/resources/default_theme/theme_data.h @@ -10,6 +10,10 @@ static const unsigned char arrow_right_png[] = { 0x89, 0x50, 0x4e, 0x47, 0xd, 0xa, 0x1a, 0xa, 0x0, 0x0, 0x0, 0xd, 0x49, 0x48, 0x44, 0x52, 0x0, 0x0, 0x0, 0xc, 0x0, 0x0, 0x0, 0xc, 0x8, 0x4, 0x0, 0x0, 0x0, 0xfc, 0x7c, 0x94, 0x6c, 0x0, 0x0, 0x0, 0x2e, 0x49, 0x44, 0x41, 0x54, 0x78, 0xda, 0x63, 0x20, 0x17, 0x3c, 0xf8, 0xf, 0x82, 0xf7, 0x13, 0x70, 0x48, 0x3c, 0xf8, 0xf2, 0x50, 0x1b, 0x43, 0x2, 0xa, 0xaf, 0xbe, 0xe0, 0xc6, 0x2e, 0xf1, 0xff, 0xe1, 0x7c, 0x12, 0x24, 0x10, 0x46, 0x11, 0xb6, 0x1c, 0xe1, 0x5c, 0xa, 0x0, 0x0, 0xe0, 0x14, 0x48, 0xb1, 0x3d, 0x1b, 0x7a, 0xf0, 0x0, 0x0, 0x0, 0x0, 0x49, 0x45, 0x4e, 0x44, 0xae, 0x42, 0x60, 0x82 }; +static const unsigned char bookmark_png[] = { + 0x89, 0x50, 0x4e, 0x47, 0xd, 0xa, 0x1a, 0xa, 0x0, 0x0, 0x0, 0xd, 0x49, 0x48, 0x44, 0x52, 0x0, 0x0, 0x0, 0x10, 0x0, 0x0, 0x0, 0x10, 0x8, 0x6, 0x0, 0x0, 0x0, 0x1f, 0xf3, 0xff, 0x61, 0x0, 0x0, 0x0, 0x4, 0x73, 0x42, 0x49, 0x54, 0x8, 0x8, 0x8, 0x8, 0x7c, 0x8, 0x64, 0x88, 0x0, 0x0, 0x0, 0x57, 0x49, 0x44, 0x41, 0x54, 0x38, 0x8d, 0xed, 0x93, 0x31, 0xa, 0xc0, 0x30, 0xc, 0x3, 0xa5, 0xd0, 0xff, 0x7f, 0x59, 0x1d, 0x8a, 0x42, 0x8, 0x9, 0x95, 0xc9, 0xd2, 0xa1, 0x9a, 0x8c, 0xf1, 0xdd, 0x62, 0x1b, 0x38, 0xc, 0x87, 0x5a, 0x5, 0xae, 0x79, 0xde, 0x2, 0x1, 0x80, 0x94, 0x39, 0x48, 0x76, 0x49, 0x17, 0xa4, 0xf0, 0x24, 0x61, 0x2b, 0x51, 0x8b, 0xfc, 0x82, 0xcf, 0xb, 0x48, 0x7a, 0xdf, 0x75, 0x81, 0xf, 0xe5, 0x29, 0xf7, 0x92, 0x6b, 0x3, 0x1a, 0x1e, 0xda, 0x7c, 0x3d, 0x77, 0x21, 0x7b, 0xa8, 0x74, 0x2e, 0xcb, 0xd, 0xc8, 0x75, 0x13, 0x28, 0x9, 0xed, 0xc2, 0xc8, 0x0, 0x0, 0x0, 0x0, 0x49, 0x45, 0x4e, 0x44, 0xae, 0x42, 0x60, 0x82 +}; + static const unsigned char button_disabled_png[] = { 0x89, 0x50, 0x4e, 0x47, 0xd, 0xa, 0x1a, 0xa, 0x0, 0x0, 0x0, 0xd, 0x49, 0x48, 0x44, 0x52, 0x0, 0x0, 0x0, 0x10, 0x0, 0x0, 0x0, 0x10, 0x8, 0x4, 0x0, 0x0, 0x0, 0xb5, 0xfa, 0x37, 0xea, 0x0, 0x0, 0x0, 0xc7, 0x49, 0x44, 0x41, 0x54, 0x78, 0xda, 0x6c, 0xd0, 0x81, 0x66, 0x43, 0x31, 0x14, 0x87, 0xf1, 0xf, 0x5, 0x17, 0xb8, 0x28, 0x2e, 0x8, 0x71, 0xf3, 0x6, 0x19, 0xb6, 0xb9, 0xcb, 0xac, 0x95, 0xa4, 0xb7, 0xad, 0x6a, 0xd5, 0x68, 0x5f, 0xe4, 0x3e, 0x76, 0x1e, 0xe1, 0xbf, 0x21, 0xa6, 0xab, 0xf8, 0x1, 0x7c, 0x9c, 0x73, 0xe, 0xac, 0xe8, 0xe8, 0x19, 0x30, 0x58, 0xc6, 0xca, 0x62, 0x18, 0xe8, 0xe9, 0x58, 0x41, 0xc7, 0x1a, 0x87, 0x27, 0x10, 0x49, 0xe4, 0x5f, 0x89, 0x48, 0xc0, 0xe3, 0x58, 0xd3, 0x41, 0x8f, 0xb, 0xcb, 0xbd, 0x7c, 0xeb, 0xbf, 0x7b, 0x9, 0xb, 0x8e, 0x1e, 0x6, 0xfc, 0xad, 0x64, 0x6d, 0xb5, 0x79, 0xb0, 0x55, 0xd6, 0xad, 0xe0, 0x19, 0xc0, 0x10, 0xae, 0xda, 0x34, 0x5c, 0x45, 0xc0, 0x80, 0x25, 0x5e, 0xf4, 0xd5, 0x70, 0x11, 0x11, 0xb, 0x23, 0xe9, 0xac, 0xcf, 0x86, 0xb3, 0x48, 0x8c, 0x30, 0x92, 0x4f, 0xa, 0xd, 0x27, 0x91, 0x6b, 0x70, 0xd4, 0x47, 0xc3, 0xf1, 0x2f, 0x48, 0x7, 0x4d, 0xd, 0x87, 0x3a, 0xc2, 0x12, 0x67, 0xbd, 0x37, 0xcc, 0x75, 0x49, 0x43, 0xd8, 0xe9, 0xad, 0x61, 0x57, 0xcf, 0x1c, 0xf0, 0xfb, 0x32, 0xe9, 0xf5, 0xc9, 0xa4, 0x7d, 0x7d, 0x54, 0x8f, 0x7b, 0x59, 0xe6, 0x92, 0x14, 0x1f, 0x24, 0xcd, 0x3f, 0x7b, 0x6b, 0xa, 0xe, 0x6a, 0x82, 0x91, 0x45, 0x30, 0xba, 0x1, 0x4a, 0x51, 0xc4, 0x35, 0x1f, 0xe5, 0xa1, 0xc0, 0x0, 0x0, 0x0, 0x0, 0x49, 0x45, 0x4e, 0x44, 0xae, 0x42, 0x60, 0x82 }; diff --git a/scene/resources/packed_scene.cpp b/scene/resources/packed_scene.cpp index cb201bc539..5e8bfd9387 100644 --- a/scene/resources/packed_scene.cpp +++ b/scene/resources/packed_scene.cpp @@ -848,7 +848,7 @@ Error SceneState::pack(Node *p_scene) { Map<Node *, int> node_map; Map<Node *, int> nodepath_map; - //if using scene inheritance, pack the scene it inherits from + // If using scene inheritance, pack the scene it inherits from. if (scene->get_scene_inherited_state().is_valid()) { String path = scene->get_scene_inherited_state()->get_path(); Ref<PackedScene> instance = ResourceLoader::load(path); @@ -856,8 +856,8 @@ Error SceneState::pack(Node *p_scene) { base_scene_idx = _vm_get_variant(instance, variant_map); } } - //instanced, only direct sub-scnes are supported of course + // Instanced, only direct sub-scenes are supported of course. Error err = _parse_node(scene, scene, -1, name_map, variant_map, node_map, nodepath_map); if (err) { clear(); diff --git a/scene/resources/particles_material.cpp b/scene/resources/particles_material.cpp index 49b8cbe7e0..4bbfa8965a 100644 --- a/scene/resources/particles_material.cpp +++ b/scene/resources/particles_material.cpp @@ -1246,6 +1246,12 @@ void ParticlesMaterial::_bind_methods() { BIND_ENUM_CONSTANT(EMISSION_SHAPE_POINTS); BIND_ENUM_CONSTANT(EMISSION_SHAPE_DIRECTED_POINTS); BIND_ENUM_CONSTANT(EMISSION_SHAPE_MAX); + + BIND_ENUM_CONSTANT(SUB_EMITTER_DISABLED); + BIND_ENUM_CONSTANT(SUB_EMITTER_CONSTANT); + BIND_ENUM_CONSTANT(SUB_EMITTER_AT_END); + BIND_ENUM_CONSTANT(SUB_EMITTER_AT_COLLISION); + BIND_ENUM_CONSTANT(SUB_EMITTER_MAX); } ParticlesMaterial::ParticlesMaterial() : diff --git a/scene/resources/particles_material.h b/scene/resources/particles_material.h index 9d41c91937..fa8858f67f 100644 --- a/scene/resources/particles_material.h +++ b/scene/resources/particles_material.h @@ -50,7 +50,6 @@ class ParticlesMaterial : public Material { public: enum Parameter { - PARAM_INITIAL_LINEAR_VELOCITY, PARAM_ANGULAR_VELOCITY, PARAM_ORBIT_VELOCITY, diff --git a/scene/resources/syntax_highlighter.cpp b/scene/resources/syntax_highlighter.cpp index d7a08d9eb6..5d58e71fc5 100644 --- a/scene/resources/syntax_highlighter.cpp +++ b/scene/resources/syntax_highlighter.cpp @@ -53,13 +53,13 @@ Dictionary SyntaxHighlighter::get_line_syntax_highlighting(int p_line) { return color_map; } -void SyntaxHighlighter::_line_edited_from(int p_line) { +void SyntaxHighlighter::_lines_edited_from(int p_from_line, int p_to_line) { if (highlighting_cache.size() < 1) { return; } int cache_size = highlighting_cache.back()->key(); - for (int i = p_line - 1; i <= cache_size; i++) { + for (int i = MIN(p_from_line, p_to_line) - 1; i <= cache_size; i++) { if (highlighting_cache.has(i)) { highlighting_cache.erase(i); } @@ -93,7 +93,7 @@ void SyntaxHighlighter::update_cache() { void SyntaxHighlighter::set_text_edit(TextEdit *p_text_edit) { if (text_edit && ObjectDB::get_instance(text_edit_instance_id)) { - text_edit->disconnect("line_edited_from", callable_mp(this, &SyntaxHighlighter::_line_edited_from)); + text_edit->disconnect("lines_edited_from", callable_mp(this, &SyntaxHighlighter::_lines_edited_from)); } text_edit = p_text_edit; @@ -101,7 +101,7 @@ void SyntaxHighlighter::set_text_edit(TextEdit *p_text_edit) { return; } text_edit_instance_id = text_edit->get_instance_id(); - text_edit->connect("line_edited_from", callable_mp(this, &SyntaxHighlighter::_line_edited_from)); + text_edit->connect("lines_edited_from", callable_mp(this, &SyntaxHighlighter::_lines_edited_from)); update_cache(); } diff --git a/scene/resources/syntax_highlighter.h b/scene/resources/syntax_highlighter.h index 40a8870b45..720227a256 100644 --- a/scene/resources/syntax_highlighter.h +++ b/scene/resources/syntax_highlighter.h @@ -40,7 +40,7 @@ class SyntaxHighlighter : public Resource { private: Map<int, Dictionary> highlighting_cache; - void _line_edited_from(int p_line); + void _lines_edited_from(int p_from_line, int p_to_line); protected: ObjectID text_edit_instance_id; // For validity check diff --git a/scene/resources/texture.cpp b/scene/resources/texture.cpp index 5681613c04..39237e1a33 100644 --- a/scene/resources/texture.cpp +++ b/scene/resources/texture.cpp @@ -791,8 +791,312 @@ String ResourceFormatLoaderStreamTexture2D::get_resource_type(const String &p_pa return ""; } +//////////////////////////////////// + +TypedArray<Image> Texture3D::_get_data() const { + Vector<Ref<Image>> data = get_data(); + + TypedArray<Image> ret; + ret.resize(data.size()); + for (int i = 0; i < data.size(); i++) { + ret[i] = data[i]; + } + return ret; +} + +void Texture3D::_bind_methods() { + ClassDB::bind_method(D_METHOD("get_format"), &Texture3D::get_format); + ClassDB::bind_method(D_METHOD("get_width"), &Texture3D::get_width); + ClassDB::bind_method(D_METHOD("get_height"), &Texture3D::get_height); + ClassDB::bind_method(D_METHOD("get_depth"), &Texture3D::get_depth); + ClassDB::bind_method(D_METHOD("has_mipmaps"), &Texture3D::has_mipmaps); + ClassDB::bind_method(D_METHOD("get_data"), &Texture3D::_get_data); +} ////////////////////////////////////////// +Image::Format ImageTexture3D::get_format() const { + return format; +} +int ImageTexture3D::get_width() const { + return width; +} +int ImageTexture3D::get_height() const { + return height; +} +int ImageTexture3D::get_depth() const { + return depth; +} +bool ImageTexture3D::has_mipmaps() const { + return mipmaps; +} + +Error ImageTexture3D::_create(Image::Format p_format, int p_width, int p_height, int p_depth, bool p_mipmaps, const TypedArray<Image> &p_data) { + Vector<Ref<Image>> images; + images.resize(p_data.size()); + for (int i = 0; i < images.size(); i++) { + images.write[i] = p_data[i]; + } + return create(p_format, p_width, p_height, p_depth, p_mipmaps, images); +} + +void ImageTexture3D::_update(const TypedArray<Image> &p_data) { + Vector<Ref<Image>> images; + images.resize(p_data.size()); + for (int i = 0; i < images.size(); i++) { + images.write[i] = p_data[i]; + } + return update(images); +} + +Error ImageTexture3D::create(Image::Format p_format, int p_width, int p_height, int p_depth, bool p_mipmaps, const Vector<Ref<Image>> &p_data) { + RID tex = RenderingServer::get_singleton()->texture_3d_create(p_format, p_width, p_height, p_depth, p_mipmaps, p_data); + ERR_FAIL_COND_V(tex.is_null(), ERR_CANT_CREATE); + + if (texture.is_valid()) { + RenderingServer::get_singleton()->texture_replace(texture, tex); + } + + return OK; +} + +void ImageTexture3D::update(const Vector<Ref<Image>> &p_data) { + ERR_FAIL_COND(!texture.is_valid()); + RenderingServer::get_singleton()->texture_3d_update(texture, p_data); +} + +Vector<Ref<Image>> ImageTexture3D::get_data() const { + ERR_FAIL_COND_V(!texture.is_valid(), Vector<Ref<Image>>()); + return RS::get_singleton()->texture_3d_get(texture); +} + +RID ImageTexture3D::get_rid() const { + if (!texture.is_valid()) { + texture = RS::get_singleton()->texture_3d_placeholder_create(); + } + return texture; +} +void ImageTexture3D::set_path(const String &p_path, bool p_take_over) { + if (texture.is_valid()) { + RenderingServer::get_singleton()->texture_set_path(texture, p_path); + } + + Resource::set_path(p_path, p_take_over); +} + +void ImageTexture3D::_bind_methods() { + ClassDB::bind_method(D_METHOD("create", "format", "width", "height", "depth", "use_mipmaps", "data"), &ImageTexture3D::_create); + ClassDB::bind_method(D_METHOD("update", "data"), &ImageTexture3D::_update); +} + +ImageTexture3D::ImageTexture3D() { +} + +ImageTexture3D::~ImageTexture3D() { + if (texture.is_valid()) { + RS::get_singleton()->free(texture); + } +} + +//////////////////////////////////////////// + +void StreamTexture3D::set_path(const String &p_path, bool p_take_over) { + if (texture.is_valid()) { + RenderingServer::get_singleton()->texture_set_path(texture, p_path); + } + + Resource::set_path(p_path, p_take_over); +} + +Image::Format StreamTexture3D::get_format() const { + return format; +} + +Error StreamTexture3D::_load_data(const String &p_path, Vector<Ref<Image>> &r_data, Image::Format &r_format, int &r_width, int &r_height, int &r_depth, bool &r_mipmaps) { + FileAccessRef f = FileAccess::open(p_path, FileAccess::READ); + ERR_FAIL_COND_V(!f, ERR_CANT_OPEN); + + uint8_t header[4]; + f->get_buffer(header, 4); + ERR_FAIL_COND_V(header[0] != 'G' || header[1] != 'S' || header[2] != 'T' || header[3] != 'L', ERR_FILE_UNRECOGNIZED); + + //stored as stream textures (used for lossless and lossy compression) + uint32_t version = f->get_32(); + + if (version > FORMAT_VERSION) { + ERR_FAIL_V_MSG(ERR_FILE_CORRUPT, "Stream texture file is too new."); + } + + r_depth = f->get_32(); //depth + f->get_32(); //ignored (mode) + f->get_32(); // ignored (data format) + + f->get_32(); //ignored + int mipmaps = f->get_32(); + f->get_32(); //ignored + f->get_32(); //ignored + + r_mipmaps = mipmaps != 0; + + r_data.clear(); + + for (int i = 0; i < (r_depth + mipmaps); i++) { + Ref<Image> image = StreamTexture2D::load_image_from_file(f, 0); + ERR_FAIL_COND_V(image.is_null() || image->empty(), ERR_CANT_OPEN); + if (i == 0) { + r_format = image->get_format(); + r_width = image->get_width(); + r_height = image->get_height(); + } + r_data.push_back(image); + } + + return OK; +} + +Error StreamTexture3D::load(const String &p_path) { + Vector<Ref<Image>> data; + + int tw, th, td; + Image::Format tfmt; + bool tmm; + + Error err = _load_data(p_path, data, tfmt, tw, th, td, tmm); + if (err) { + return err; + } + + if (texture.is_valid()) { + RID new_texture = RS::get_singleton()->texture_3d_create(tfmt, tw, th, td, tmm, data); + RS::get_singleton()->texture_replace(texture, new_texture); + } else { + texture = RS::get_singleton()->texture_3d_create(tfmt, tw, th, td, tmm, data); + } + + w = tw; + h = th; + d = td; + mipmaps = tmm; + format = tfmt; + + path_to_file = p_path; + + if (get_path() == String()) { + //temporarily set path if no path set for resource, helps find errors + RenderingServer::get_singleton()->texture_set_path(texture, p_path); + } + + _change_notify(); + emit_changed(); + return OK; +} + +String StreamTexture3D::get_load_path() const { + return path_to_file; +} + +int StreamTexture3D::get_width() const { + return w; +} + +int StreamTexture3D::get_height() const { + return h; +} + +int StreamTexture3D::get_depth() const { + return d; +} + +bool StreamTexture3D::has_mipmaps() const { + return mipmaps; +} + +RID StreamTexture3D::get_rid() const { + if (!texture.is_valid()) { + texture = RS::get_singleton()->texture_3d_placeholder_create(); + } + return texture; +} + +Vector<Ref<Image>> StreamTexture3D::get_data() const { + if (texture.is_valid()) { + return RS::get_singleton()->texture_3d_get(texture); + } else { + return Vector<Ref<Image>>(); + } +} + +void StreamTexture3D::reload_from_file() { + String path = get_path(); + if (!path.is_resource_file()) { + return; + } + + path = ResourceLoader::path_remap(path); //remap for translation + path = ResourceLoader::import_remap(path); //remap for import + if (!path.is_resource_file()) { + return; + } + + load(path); +} + +void StreamTexture3D::_validate_property(PropertyInfo &property) const { +} + +void StreamTexture3D::_bind_methods() { + ClassDB::bind_method(D_METHOD("load", "path"), &StreamTexture3D::load); + ClassDB::bind_method(D_METHOD("get_load_path"), &StreamTexture3D::get_load_path); + + ADD_PROPERTY(PropertyInfo(Variant::STRING, "load_path", PROPERTY_HINT_FILE, "*.stex"), "load", "get_load_path"); +} + +StreamTexture3D::StreamTexture3D() { + format = Image::FORMAT_MAX; + w = 0; + h = 0; + d = 0; + mipmaps = false; +} + +StreamTexture3D::~StreamTexture3D() { + if (texture.is_valid()) { + RS::get_singleton()->free(texture); + } +} + +///////////////////////////// + +RES ResourceFormatLoaderStreamTexture3D::load(const String &p_path, const String &p_original_path, Error *r_error, bool p_use_sub_threads, float *r_progress, bool p_no_cache) { + Ref<StreamTexture3D> st; + st.instance(); + Error err = st->load(p_path); + if (r_error) { + *r_error = err; + } + if (err != OK) { + return RES(); + } + + return st; +} + +void ResourceFormatLoaderStreamTexture3D::get_recognized_extensions(List<String> *p_extensions) const { + p_extensions->push_back("stex3d"); +} + +bool ResourceFormatLoaderStreamTexture3D::handles_type(const String &p_type) const { + return p_type == "StreamTexture3D"; +} + +String ResourceFormatLoaderStreamTexture3D::get_resource_type(const String &p_path) const { + if (p_path.get_extension().to_lower() == "stex3d") { + return "StreamTexture3D"; + } + return ""; +} + +//////////////////////////////////////////// + int AtlasTexture::get_width() const { if (region.size.width == 0) { if (atlas.is_valid()) { diff --git a/scene/resources/texture.h b/scene/resources/texture.h index fd213859b7..eebbf4f233 100644 --- a/scene/resources/texture.h +++ b/scene/resources/texture.h @@ -515,6 +515,122 @@ public: virtual String get_resource_type(const String &p_path) const; }; +class Texture3D : public Texture { + GDCLASS(Texture3D, Texture); + +protected: + static void _bind_methods(); + + TypedArray<Image> _get_data() const; + +public: + virtual Image::Format get_format() const = 0; + virtual int get_width() const = 0; + virtual int get_height() const = 0; + virtual int get_depth() const = 0; + virtual bool has_mipmaps() const = 0; + virtual Vector<Ref<Image>> get_data() const = 0; +}; + +class ImageTexture3D : public Texture3D { + GDCLASS(ImageTexture3D, Texture3D); + + mutable RID texture; + + Image::Format format = Image::FORMAT_MAX; + int width = 1; + int height = 1; + int depth = 1; + bool mipmaps = false; + +protected: + static void _bind_methods(); + + Error _create(Image::Format p_format, int p_width, int p_height, int p_depth, bool p_mipmaps, const TypedArray<Image> &p_data); + void _update(const TypedArray<Image> &p_data); + +public: + virtual Image::Format get_format() const override; + virtual int get_width() const override; + virtual int get_height() const override; + virtual int get_depth() const override; + virtual bool has_mipmaps() const override; + + Error create(Image::Format p_format, int p_width, int p_height, int p_depth, bool p_mipmaps, const Vector<Ref<Image>> &p_data); + void update(const Vector<Ref<Image>> &p_data); + virtual Vector<Ref<Image>> get_data() const override; + + virtual RID get_rid() const override; + virtual void set_path(const String &p_path, bool p_take_over = false) override; + + ImageTexture3D(); + ~ImageTexture3D(); +}; + +class StreamTexture3D : public Texture3D { + GDCLASS(StreamTexture3D, Texture3D); + +public: + enum DataFormat { + DATA_FORMAT_IMAGE, + DATA_FORMAT_LOSSLESS, + DATA_FORMAT_LOSSY, + DATA_FORMAT_BASIS_UNIVERSAL, + }; + + enum { + FORMAT_VERSION = 1 + }; + + enum FormatBits { + FORMAT_MASK_IMAGE_FORMAT = (1 << 20) - 1, + FORMAT_BIT_LOSSLESS = 1 << 20, + FORMAT_BIT_LOSSY = 1 << 21, + FORMAT_BIT_STREAM = 1 << 22, + FORMAT_BIT_HAS_MIPMAPS = 1 << 23, + }; + +private: + Error _load_data(const String &p_path, Vector<Ref<Image>> &r_data, Image::Format &r_format, int &r_width, int &r_height, int &r_depth, bool &r_mipmaps); + String path_to_file; + mutable RID texture; + Image::Format format; + int w, h, d; + bool mipmaps; + + virtual void reload_from_file() override; + +protected: + static void _bind_methods(); + void _validate_property(PropertyInfo &property) const override; + +public: + Image::Format get_format() const override; + Error load(const String &p_path); + String get_load_path() const; + + int get_width() const override; + int get_height() const override; + int get_depth() const override; + virtual bool has_mipmaps() const override; + virtual RID get_rid() const override; + + virtual void set_path(const String &p_path, bool p_take_over) override; + + virtual Vector<Ref<Image>> get_data() const override; + + StreamTexture3D(); + ~StreamTexture3D(); +}; + +class ResourceFormatLoaderStreamTexture3D : public ResourceFormatLoader { +public: + virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = nullptr, bool p_use_sub_threads = false, float *r_progress = nullptr, bool p_no_cache = false); + virtual void get_recognized_extensions(List<String> *p_extensions) const; + virtual bool handles_type(const String &p_type) const; + virtual String get_resource_type(const String &p_path) const; +}; + class CurveTexture : public Texture2D { GDCLASS(CurveTexture, Texture2D); RES_BASE_EXTENSION("curvetex") diff --git a/scene/resources/visual_shader.cpp b/scene/resources/visual_shader.cpp index 394133dd47..f15e43c141 100644 --- a/scene/resources/visual_shader.cpp +++ b/scene/resources/visual_shader.cpp @@ -101,6 +101,14 @@ bool VisualShaderNode::is_code_generated() const { return true; } +bool VisualShaderNode::is_show_prop_names() const { + return false; +} + +bool VisualShaderNode::is_use_prop_slots() const { + return false; +} + Vector<VisualShader::DefaultTextureParam> VisualShaderNode::get_default_texture_parameters(VisualShader::Type p_type, int p_id) const { return Vector<VisualShader::DefaultTextureParam>(); } @@ -317,17 +325,14 @@ VisualShaderNodeCustom::VisualShaderNodeCustom() { ///////////////////////////////////////////////////////// -void VisualShader::set_graph_node(Type p_type, int p_id, GraphNode *p_graph_node) { - ERR_FAIL_INDEX(p_type, TYPE_MAX); - Graph *g = &graph[p_type]; - ERR_FAIL_COND(!g->nodes.has(p_id)); - g->nodes[p_id].graph_node = p_graph_node; -} - void VisualShader::set_shader_type(Type p_type) { current_type = p_type; } +VisualShader::Type VisualShader::get_shader_type() const { + return current_type; +} + void VisualShader::set_version(const String &p_version) { version = p_version; } @@ -411,11 +416,6 @@ void VisualShader::set_node_position(Type p_type, int p_id, const Vector2 &p_pos Graph *g = &graph[p_type]; ERR_FAIL_COND(!g->nodes.has(p_id)); g->nodes[p_id].position = p_position; - if (current_type == p_type) { - if (g->nodes[p_id].graph_node != nullptr) { - g->nodes[p_id].graph_node->set_offset(p_position); - } - } } Vector2 VisualShader::get_node_position(Type p_type, int p_id) const { @@ -1517,11 +1517,11 @@ void VisualShader::_update_shader() const { if (shader_mode != Shader::MODE_PARTICLES) { func_code += "\nvoid " + String(func_name[i]) + "() {\n"; } + insertion_pos.insert(i, code.get_string_length() + func_code.get_string_length()); Set<int> processed; Error err = _write_node(Type(i), global_code, global_code_per_node, global_code_per_func, func_code, default_tex_params, input_connections, output_connections, NODE_ID_OUTPUT, processed, false, classes); ERR_FAIL_COND(err != OK); - insertion_pos.insert(i, code.get_string_length()); if (shader_mode == Shader::MODE_PARTICLES) { code_map.insert(i, func_code); @@ -1632,6 +1632,9 @@ void VisualShader::_bind_methods() { BIND_ENUM_CONSTANT(TYPE_VERTEX); BIND_ENUM_CONSTANT(TYPE_FRAGMENT); BIND_ENUM_CONSTANT(TYPE_LIGHT); + BIND_ENUM_CONSTANT(TYPE_EMIT); + BIND_ENUM_CONSTANT(TYPE_PROCESS); + BIND_ENUM_CONSTANT(TYPE_END); BIND_ENUM_CONSTANT(TYPE_MAX); BIND_CONSTANT(NODE_ID_INVALID); @@ -2128,6 +2131,15 @@ void VisualShaderNodeUniformRef::clear_uniforms() { uniforms.clear(); } +bool VisualShaderNodeUniformRef::has_uniform(const String &p_name) { + for (List<VisualShaderNodeUniformRef::Uniform>::Element *E = uniforms.front(); E; E = E->next()) { + if (E->get().name == p_name) { + return true; + } + } + return false; +} + String VisualShaderNodeUniformRef::get_caption() const { return "UniformRef"; } @@ -2145,10 +2157,6 @@ String VisualShaderNodeUniformRef::get_input_port_name(int p_port) const { } int VisualShaderNodeUniformRef::get_output_port_count() const { - if (uniform_name == "[None]") { - return 0; - } - switch (uniform_type) { case UniformType::UNIFORM_TYPE_FLOAT: return 1; @@ -2167,7 +2175,7 @@ int VisualShaderNodeUniformRef::get_output_port_count() const { default: break; } - return 0; + return 1; } VisualShaderNodeUniformRef::PortType VisualShaderNodeUniformRef::get_output_port_type(int p_port) const { @@ -2227,8 +2235,8 @@ String VisualShaderNodeUniformRef::get_output_port_name(int p_port) const { void VisualShaderNodeUniformRef::set_uniform_name(const String &p_name) { uniform_name = p_name; - if (p_name != "[None]") { - uniform_type = get_uniform_type_by_name(p_name); + if (uniform_name != "[None]") { + uniform_type = get_uniform_type_by_name(uniform_name); } else { uniform_type = UniformType::UNIFORM_TYPE_FLOAT; } @@ -2269,6 +2277,9 @@ VisualShaderNodeUniformRef::UniformType VisualShaderNodeUniformRef::get_uniform_ String VisualShaderNodeUniformRef::generate_code(Shader::Mode p_mode, VisualShader::Type p_type, int p_id, const String *p_input_vars, const String *p_output_vars, bool p_for_preview) const { switch (uniform_type) { case UniformType::UNIFORM_TYPE_FLOAT: + if (uniform_name == "[None]") { + return "\t" + p_output_vars[0] + " = 0.0;\n"; + } return "\t" + p_output_vars[0] + " = " + get_uniform_name() + ";\n"; case UniformType::UNIFORM_TYPE_INT: return "\t" + p_output_vars[0] + " = " + get_uniform_name() + ";\n"; @@ -2291,16 +2302,29 @@ String VisualShaderNodeUniformRef::generate_code(Shader::Mode p_mode, VisualShad return ""; } +void VisualShaderNodeUniformRef::_set_uniform_type(int p_uniform_type) { + uniform_type = (UniformType)p_uniform_type; +} + +int VisualShaderNodeUniformRef::_get_uniform_type() const { + return (int)uniform_type; +} + void VisualShaderNodeUniformRef::_bind_methods() { ClassDB::bind_method(D_METHOD("set_uniform_name", "name"), &VisualShaderNodeUniformRef::set_uniform_name); ClassDB::bind_method(D_METHOD("get_uniform_name"), &VisualShaderNodeUniformRef::get_uniform_name); + ClassDB::bind_method(D_METHOD("_set_uniform_type", "type"), &VisualShaderNodeUniformRef::_set_uniform_type); + ClassDB::bind_method(D_METHOD("_get_uniform_type"), &VisualShaderNodeUniformRef::_get_uniform_type); + ADD_PROPERTY(PropertyInfo(Variant::STRING_NAME, "uniform_name", PROPERTY_HINT_ENUM, ""), "set_uniform_name", "get_uniform_name"); + ADD_PROPERTY(PropertyInfo(Variant::INT, "uniform_type", PROPERTY_HINT_NONE, "", PROPERTY_USAGE_NOEDITOR | PROPERTY_USAGE_INTERNAL), "_set_uniform_type", "_get_uniform_type"); } Vector<StringName> VisualShaderNodeUniformRef::get_editable_properties() const { Vector<StringName> props; props.push_back("uniform_name"); + props.push_back("uniform_type"); return props; } diff --git a/scene/resources/visual_shader.h b/scene/resources/visual_shader.h index d45029ec03..e7d74b6c17 100644 --- a/scene/resources/visual_shader.h +++ b/scene/resources/visual_shader.h @@ -33,7 +33,6 @@ #include "core/string_builder.h" #include "scene/gui/control.h" -#include "scene/gui/graph_edit.h" #include "scene/resources/shader.h" class VisualShaderNodeUniform; @@ -76,7 +75,6 @@ private: Ref<VisualShaderNode> node; Vector2 position; List<int> prev_connected_nodes; - GraphNode *graph_node; }; struct Graph { @@ -129,8 +127,8 @@ protected: void _get_property_list(List<PropertyInfo> *p_list) const; public: // internal methods - void set_graph_node(Type p_type, int p_id, GraphNode *p_graph_node); void set_shader_type(Type p_type); + Type get_shader_type() const; public: void set_version(const String &p_version); @@ -244,6 +242,8 @@ public: virtual bool is_generate_input_var(int p_port) const; virtual bool is_code_generated() const; + virtual bool is_show_prop_names() const; + virtual bool is_use_prop_slots() const; virtual Vector<StringName> get_editable_properties() const; @@ -456,6 +456,7 @@ protected: public: static void add_uniform(const String &p_name, UniformType p_type); static void clear_uniforms(); + static bool has_uniform(const String &p_name); public: virtual String get_caption() const override; @@ -471,6 +472,9 @@ public: void set_uniform_name(const String &p_name); String get_uniform_name() const; + void _set_uniform_type(int p_uniform_type); + int _get_uniform_type() const; + int get_uniforms_count() const; String get_uniform_name_by_index(int p_idx) const; UniformType get_uniform_type_by_name(const String &p_name) const; diff --git a/scene/resources/visual_shader_nodes.cpp b/scene/resources/visual_shader_nodes.cpp index 182aeae5d4..cffe0bb5cd 100644 --- a/scene/resources/visual_shader_nodes.cpp +++ b/scene/resources/visual_shader_nodes.cpp @@ -951,6 +951,64 @@ void VisualShaderNodeTexture2DArray::_bind_methods() { VisualShaderNodeTexture2DArray::VisualShaderNodeTexture2DArray() { } + +////////////// Texture3D + +String VisualShaderNodeTexture3D::get_caption() const { + return "Texture3D"; +} + +String VisualShaderNodeTexture3D::get_input_port_name(int p_port) const { + if (p_port == 2) { + return "sampler3D"; + } + return VisualShaderNodeSample3D::get_input_port_name(p_port); +} + +Vector<VisualShader::DefaultTextureParam> VisualShaderNodeTexture3D::get_default_texture_parameters(VisualShader::Type p_type, int p_id) const { + VisualShader::DefaultTextureParam dtp; + dtp.name = make_unique_id(p_type, p_id, "tex3d"); + dtp.param = texture; + Vector<VisualShader::DefaultTextureParam> ret; + ret.push_back(dtp); + return ret; +} + +String VisualShaderNodeTexture3D::generate_global(Shader::Mode p_mode, VisualShader::Type p_type, int p_id) const { + if (source == SOURCE_TEXTURE) { + return "uniform sampler3D " + make_unique_id(p_type, p_id, "tex3d") + ";\n"; + } + return String(); +} + +void VisualShaderNodeTexture3D::set_texture(Ref<Texture3D> p_value) { + texture = p_value; + emit_changed(); +} + +Ref<Texture3D> VisualShaderNodeTexture3D::get_texture() const { + return texture; +} + +Vector<StringName> VisualShaderNodeTexture3D::get_editable_properties() const { + Vector<StringName> props; + props.push_back("source"); + if (source == SOURCE_TEXTURE) { + props.push_back("texture"); + } + return props; +} + +void VisualShaderNodeTexture3D::_bind_methods() { + ClassDB::bind_method(D_METHOD("set_texture", "value"), &VisualShaderNodeTexture3D::set_texture); + ClassDB::bind_method(D_METHOD("get_texture"), &VisualShaderNodeTexture3D::get_texture); + + ADD_PROPERTY(PropertyInfo(Variant::OBJECT, "texture", PROPERTY_HINT_RESOURCE_TYPE, "Texture3D"), "set_texture", "get_texture"); +} + +VisualShaderNodeTexture3D::VisualShaderNodeTexture3D() { +} + ////////////// Cubemap String VisualShaderNodeCubemap::get_caption() const { @@ -3432,6 +3490,14 @@ String VisualShaderNodeFloatUniform::generate_code(Shader::Mode p_mode, VisualSh return "\t" + p_output_vars[0] + " = " + get_uniform_name() + ";\n"; } +bool VisualShaderNodeFloatUniform::is_show_prop_names() const { + return true; +} + +bool VisualShaderNodeFloatUniform::is_use_prop_slots() const { + return true; +} + void VisualShaderNodeFloatUniform::set_hint(Hint p_hint) { hint = p_hint; emit_changed(); @@ -3591,6 +3657,14 @@ String VisualShaderNodeIntUniform::generate_code(Shader::Mode p_mode, VisualShad return "\t" + p_output_vars[0] + " = " + get_uniform_name() + ";\n"; } +bool VisualShaderNodeIntUniform::is_show_prop_names() const { + return true; +} + +bool VisualShaderNodeIntUniform::is_use_prop_slots() const { + return true; +} + void VisualShaderNodeIntUniform::set_hint(Hint p_hint) { hint = p_hint; emit_changed(); @@ -3765,6 +3839,14 @@ String VisualShaderNodeBooleanUniform::generate_code(Shader::Mode p_mode, Visual return "\t" + p_output_vars[0] + " = " + get_uniform_name() + ";\n"; } +bool VisualShaderNodeBooleanUniform::is_show_prop_names() const { + return true; +} + +bool VisualShaderNodeBooleanUniform::is_use_prop_slots() const { + return true; +} + void VisualShaderNodeBooleanUniform::_bind_methods() { ClassDB::bind_method(D_METHOD("set_default_value_enabled", "enabled"), &VisualShaderNodeBooleanUniform::set_default_value_enabled); ClassDB::bind_method(D_METHOD("is_default_value_enabled"), &VisualShaderNodeBooleanUniform::is_default_value_enabled); @@ -3855,6 +3937,10 @@ String VisualShaderNodeColorUniform::generate_code(Shader::Mode p_mode, VisualSh return code; } +bool VisualShaderNodeColorUniform::is_show_prop_names() const { + return true; +} + void VisualShaderNodeColorUniform::_bind_methods() { ClassDB::bind_method(D_METHOD("set_default_value_enabled", "enabled"), &VisualShaderNodeColorUniform::set_default_value_enabled); ClassDB::bind_method(D_METHOD("is_default_value_enabled"), &VisualShaderNodeColorUniform::is_default_value_enabled); @@ -3954,6 +4040,14 @@ void VisualShaderNodeVec3Uniform::_bind_methods() { ADD_PROPERTY(PropertyInfo(Variant::VECTOR3, "default_value"), "set_default_value", "get_default_value"); } +bool VisualShaderNodeVec3Uniform::is_show_prop_names() const { + return true; +} + +bool VisualShaderNodeVec3Uniform::is_use_prop_slots() const { + return true; +} + bool VisualShaderNodeVec3Uniform::is_qualifier_supported(Qualifier p_qual) const { return true; // all qualifiers are supported } @@ -4046,6 +4140,14 @@ void VisualShaderNodeTransformUniform::_bind_methods() { ADD_PROPERTY(PropertyInfo(Variant::TRANSFORM, "default_value"), "set_default_value", "get_default_value"); } +bool VisualShaderNodeTransformUniform::is_show_prop_names() const { + return true; +} + +bool VisualShaderNodeTransformUniform::is_use_prop_slots() const { + return true; +} + bool VisualShaderNodeTransformUniform::is_qualifier_supported(Qualifier p_qual) const { return true; // all qualifiers are supported } @@ -4301,13 +4403,13 @@ String VisualShaderNodeTextureUniformTriplanar::generate_code(Shader::Mode p_mod String code = "\t{\n"; if (p_input_vars[0] == String() && p_input_vars[1] == String()) { - code += "\t\tvec4 n_tex_read = triplanar_texture( " + id + ", triplanar_power_normal, triplanar_pos );\n"; + code += "\t\tvec4 n_tex_read = triplanar_texture(" + id + ", triplanar_power_normal, triplanar_pos);\n"; } else if (p_input_vars[0] != String() && p_input_vars[1] == String()) { - code += "\t\tvec4 n_tex_read = triplanar_texture( " + id + ", " + p_input_vars[0] + ", triplanar_pos );\n"; + code += "\t\tvec4 n_tex_read = triplanar_texture(" + id + ", " + p_input_vars[0] + ", triplanar_pos);\n"; } else if (p_input_vars[0] == String() && p_input_vars[1] != String()) { - code += "\t\tvec4 n_tex_read = triplanar_texture( " + id + ", triplanar_power_normal," + p_input_vars[1] + " );\n"; + code += "\t\tvec4 n_tex_read = triplanar_texture(" + id + ", triplanar_power_normal, " + p_input_vars[1] + ");\n"; } else { - code += "\t\tvec4 n_tex_read = triplanar_texture( " + id + ", " + p_input_vars[0] + ", " + p_input_vars[1] + " );\n"; + code += "\t\tvec4 n_tex_read = triplanar_texture(" + id + ", " + p_input_vars[0] + ", " + p_input_vars[1] + ");\n"; } code += "\t\t" + p_output_vars[0] + " = n_tex_read.rgb;\n"; @@ -4397,6 +4499,74 @@ String VisualShaderNodeTexture2DArrayUniform::generate_code(Shader::Mode p_mode, VisualShaderNodeTexture2DArrayUniform::VisualShaderNodeTexture2DArrayUniform() { } +////////////// Texture3D Uniform + +String VisualShaderNodeTexture3DUniform::get_caption() const { + return "Texture3DUniform"; +} + +int VisualShaderNodeTexture3DUniform::get_output_port_count() const { + return 1; +} + +VisualShaderNodeTexture3DUniform::PortType VisualShaderNodeTexture3DUniform::get_output_port_type(int p_port) const { + return PORT_TYPE_SAMPLER; +} + +String VisualShaderNodeTexture3DUniform::get_output_port_name(int p_port) const { + return "sampler3D"; +} + +int VisualShaderNodeTexture3DUniform::get_input_port_count() const { + return 0; +} + +VisualShaderNodeTexture3DUniform::PortType VisualShaderNodeTexture3DUniform::get_input_port_type(int p_port) const { + return PORT_TYPE_SCALAR; +} + +String VisualShaderNodeTexture3DUniform::get_input_port_name(int p_port) const { + return ""; +} + +String VisualShaderNodeTexture3DUniform::get_input_port_default_hint(int p_port) const { + return ""; +} + +String VisualShaderNodeTexture3DUniform::generate_global(Shader::Mode p_mode, VisualShader::Type p_type, int p_id) const { + String code = _get_qual_str() + "uniform sampler3D " + get_uniform_name(); + + switch (texture_type) { + case TYPE_DATA: + if (color_default == COLOR_DEFAULT_BLACK) + code += " : hint_black;\n"; + else + code += ";\n"; + break; + case TYPE_COLOR: + if (color_default == COLOR_DEFAULT_BLACK) + code += " : hint_black_albedo;\n"; + else + code += " : hint_albedo;\n"; + break; + case TYPE_NORMALMAP: + code += " : hint_normal;\n"; + break; + case TYPE_ANISO: + code += " : hint_aniso;\n"; + break; + } + + return code; +} + +String VisualShaderNodeTexture3DUniform::generate_code(Shader::Mode p_mode, VisualShader::Type p_type, int p_id, const String *p_input_vars, const String *p_output_vars, bool p_for_preview) const { + return String(); +} + +VisualShaderNodeTexture3DUniform::VisualShaderNodeTexture3DUniform() { +} + ////////////// Cubemap Uniform String VisualShaderNodeCubemapUniform::get_caption() const { diff --git a/scene/resources/visual_shader_nodes.h b/scene/resources/visual_shader_nodes.h index 245435591b..1c986d1ef4 100644 --- a/scene/resources/visual_shader_nodes.h +++ b/scene/resources/visual_shader_nodes.h @@ -343,6 +343,29 @@ public: VisualShaderNodeTexture2DArray(); }; +class VisualShaderNodeTexture3D : public VisualShaderNodeSample3D { + GDCLASS(VisualShaderNodeTexture3D, VisualShaderNodeSample3D); + Ref<Texture3D> texture; + +protected: + static void _bind_methods(); + +public: + virtual String get_caption() const override; + + virtual String get_input_port_name(int p_port) const override; + + virtual Vector<VisualShader::DefaultTextureParam> get_default_texture_parameters(VisualShader::Type p_type, int p_id) const override; + virtual String generate_global(Shader::Mode p_mode, VisualShader::Type p_type, int p_id) const override; + + void set_texture(Ref<Texture3D> p_value); + Ref<Texture3D> get_texture() const; + + virtual Vector<StringName> get_editable_properties() const override; + + VisualShaderNodeTexture3D(); +}; + class VisualShaderNodeCubemap : public VisualShaderNode { GDCLASS(VisualShaderNodeCubemap, VisualShaderNode); Ref<Cubemap> cube_map; @@ -1506,6 +1529,9 @@ public: virtual String generate_global(Shader::Mode p_mode, VisualShader::Type p_type, int p_id) const override; virtual String generate_code(Shader::Mode p_mode, VisualShader::Type p_type, int p_id, const String *p_input_vars, const String *p_output_vars, bool p_for_preview = false) const override; //if no output is connected, the output var passed will be empty. if no input is connected and input is NIL, the input var passed will be empty + virtual bool is_show_prop_names() const override; + virtual bool is_use_prop_slots() const override; + void set_hint(Hint p_hint); Hint get_hint() const; @@ -1568,6 +1594,9 @@ public: virtual String generate_global(Shader::Mode p_mode, VisualShader::Type p_type, int p_id) const override; virtual String generate_code(Shader::Mode p_mode, VisualShader::Type p_type, int p_id, const String *p_input_vars, const String *p_output_vars, bool p_for_preview = false) const override; //if no output is connected, the output var passed will be empty. if no input is connected and input is NIL, the input var passed will be empty + virtual bool is_show_prop_names() const override; + virtual bool is_use_prop_slots() const override; + void set_hint(Hint p_hint); Hint get_hint() const; @@ -1621,6 +1650,9 @@ public: virtual String generate_global(Shader::Mode p_mode, VisualShader::Type p_type, int p_id) const override; virtual String generate_code(Shader::Mode p_mode, VisualShader::Type p_type, int p_id, const String *p_input_vars, const String *p_output_vars, bool p_for_preview = false) const override; //if no output is connected, the output var passed will be empty. if no input is connected and input is NIL, the input var passed will be empty + virtual bool is_show_prop_names() const override; + virtual bool is_use_prop_slots() const override; + void set_default_value_enabled(bool p_enabled); bool is_default_value_enabled() const; @@ -1660,6 +1692,8 @@ public: virtual String generate_global(Shader::Mode p_mode, VisualShader::Type p_type, int p_id) const override; virtual String generate_code(Shader::Mode p_mode, VisualShader::Type p_type, int p_id, const String *p_input_vars, const String *p_output_vars, bool p_for_preview = false) const override; //if no output is connected, the output var passed will be empty. if no input is connected and input is NIL, the input var passed will be empty + virtual bool is_show_prop_names() const override; + void set_default_value_enabled(bool p_enabled); bool is_default_value_enabled() const; @@ -1699,6 +1733,9 @@ public: virtual String generate_global(Shader::Mode p_mode, VisualShader::Type p_type, int p_id) const override; virtual String generate_code(Shader::Mode p_mode, VisualShader::Type p_type, int p_id, const String *p_input_vars, const String *p_output_vars, bool p_for_preview = false) const override; //if no output is connected, the output var passed will be empty. if no input is connected and input is NIL, the input var passed will be empty + virtual bool is_show_prop_names() const override; + virtual bool is_use_prop_slots() const override; + void set_default_value_enabled(bool p_enabled); bool is_default_value_enabled() const; @@ -1738,6 +1775,9 @@ public: virtual String generate_global(Shader::Mode p_mode, VisualShader::Type p_type, int p_id) const override; virtual String generate_code(Shader::Mode p_mode, VisualShader::Type p_type, int p_id, const String *p_input_vars, const String *p_output_vars, bool p_for_preview = false) const override; //if no output is connected, the output var passed will be empty. if no input is connected and input is NIL, the input var passed will be empty + virtual bool is_show_prop_names() const override; + virtual bool is_use_prop_slots() const override; + void set_default_value_enabled(bool p_enabled); bool is_default_value_enabled() const; @@ -1855,6 +1895,29 @@ public: /////////////////////////////////////// +class VisualShaderNodeTexture3DUniform : public VisualShaderNodeTextureUniform { + GDCLASS(VisualShaderNodeTexture3DUniform, VisualShaderNodeTextureUniform); + +public: + virtual String get_caption() const override; + + virtual int get_input_port_count() const override; + virtual PortType get_input_port_type(int p_port) const override; + virtual String get_input_port_name(int p_port) const override; + + virtual int get_output_port_count() const override; + virtual PortType get_output_port_type(int p_port) const override; + virtual String get_output_port_name(int p_port) const override; + + virtual String get_input_port_default_hint(int p_port) const override; + virtual String generate_global(Shader::Mode p_mode, VisualShader::Type p_type, int p_id) const override; + virtual String generate_code(Shader::Mode p_mode, VisualShader::Type p_type, int p_id, const String *p_input_vars, const String *p_output_vars, bool p_for_preview = false) const override; //if no output is connected, the output var passed will be empty. if no input is connected and input is NIL, the input var passed will be empty + + VisualShaderNodeTexture3DUniform(); +}; + +/////////////////////////////////////// + class VisualShaderNodeCubemapUniform : public VisualShaderNodeTextureUniform { GDCLASS(VisualShaderNodeCubemapUniform, VisualShaderNodeTextureUniform); diff --git a/servers/display_server.cpp b/servers/display_server.cpp index 8f6d6d3b99..356f4b884a 100644 --- a/servers/display_server.cpp +++ b/servers/display_server.cpp @@ -194,6 +194,10 @@ void DisplayServer::delete_sub_window(WindowID p_id) { ERR_FAIL_MSG("Sub-windows not supported by this display server."); } +void DisplayServer::window_set_mouse_passthrough(const Vector<Vector2> &p_region, WindowID p_window) { + ERR_FAIL_MSG("Mouse passthrough not supported by this display server."); +} + void DisplayServer::window_set_ime_active(const bool p_active, WindowID p_window) { WARN_PRINT("IME not supported by this display server."); } @@ -412,6 +416,7 @@ void DisplayServer::_bind_methods() { ClassDB::bind_method(D_METHOD("delete_sub_window", "window_id"), &DisplayServer::delete_sub_window); ClassDB::bind_method(D_METHOD("window_set_title", "title", "window_id"), &DisplayServer::window_set_title, DEFVAL(MAIN_WINDOW_ID)); + ClassDB::bind_method(D_METHOD("window_set_mouse_passthrough", "region", "window_id"), &DisplayServer::window_set_mouse_passthrough, DEFVAL(MAIN_WINDOW_ID)); ClassDB::bind_method(D_METHOD("window_get_current_screen", "window_id"), &DisplayServer::window_get_current_screen, DEFVAL(MAIN_WINDOW_ID)); ClassDB::bind_method(D_METHOD("window_set_current_screen", "screen", "window_id"), &DisplayServer::window_set_current_screen, DEFVAL(MAIN_WINDOW_ID)); diff --git a/servers/display_server.h b/servers/display_server.h index b652418244..3ee0da709b 100644 --- a/servers/display_server.h +++ b/servers/display_server.h @@ -247,6 +247,8 @@ public: virtual void window_set_title(const String &p_title, WindowID p_window = MAIN_WINDOW_ID) = 0; + virtual void window_set_mouse_passthrough(const Vector<Vector2> &p_region, WindowID p_window = MAIN_WINDOW_ID); + virtual int window_get_current_screen(WindowID p_window = MAIN_WINDOW_ID) const = 0; virtual void window_set_current_screen(int p_screen, WindowID p_window = MAIN_WINDOW_ID) = 0; diff --git a/servers/physics_2d/body_2d_sw.cpp b/servers/physics_2d/body_2d_sw.cpp index 856bba78f7..75c9a95739 100644 --- a/servers/physics_2d/body_2d_sw.cpp +++ b/servers/physics_2d/body_2d_sw.cpp @@ -468,7 +468,7 @@ void Body2DSW::integrate_forces(real_t p_step) { linear_velocity = motion / p_step; real_t rot = new_transform.get_rotation() - get_transform().get_rotation(); - angular_velocity = rot / p_step; + angular_velocity = remainder(rot, 2.0 * Math_PI) / p_step; do_motion = true; diff --git a/servers/rendering/rasterizer.h b/servers/rendering/rasterizer.h index 9cb611f3b4..a24189bdd7 100644 --- a/servers/rendering/rasterizer.h +++ b/servers/rendering/rasterizer.h @@ -331,12 +331,12 @@ public: virtual RID texture_2d_create(const Ref<Image> &p_image) = 0; virtual RID texture_2d_layered_create(const Vector<Ref<Image>> &p_layers, RS::TextureLayeredType p_layered_type) = 0; - virtual RID texture_3d_create(const Vector<Ref<Image>> &p_slices) = 0; //all slices, then all the mipmaps, must be coherent + virtual RID texture_3d_create(Image::Format, int p_width, int p_height, int p_depth, bool p_mipmaps, const Vector<Ref<Image>> &p_data) = 0; virtual RID texture_proxy_create(RID p_base) = 0; //all slices, then all the mipmaps, must be coherent virtual void texture_2d_update_immediate(RID p_texture, const Ref<Image> &p_image, int p_layer = 0) = 0; //mostly used for video and streaming virtual void texture_2d_update(RID p_texture, const Ref<Image> &p_image, int p_layer = 0) = 0; - virtual void texture_3d_update(RID p_texture, const Ref<Image> &p_image, int p_depth, int p_mipmap) = 0; + virtual void texture_3d_update(RID p_texture, const Vector<Ref<Image>> &p_data) = 0; virtual void texture_proxy_update(RID p_proxy, RID p_base) = 0; //these two APIs can be used together or in combination with the others. @@ -346,7 +346,7 @@ public: virtual Ref<Image> texture_2d_get(RID p_texture) const = 0; virtual Ref<Image> texture_2d_layer_get(RID p_texture, int p_layer) const = 0; - virtual Ref<Image> texture_3d_slice_get(RID p_texture, int p_depth, int p_mipmap) const = 0; + virtual Vector<Ref<Image>> texture_3d_get(RID p_texture) const = 0; virtual void texture_replace(RID p_texture, RID p_by_texture) = 0; virtual void texture_set_size_override(RID p_texture, int p_width, int p_height) = 0; diff --git a/servers/rendering/rasterizer_rd/rasterizer_scene_high_end_rd.cpp b/servers/rendering/rasterizer_rd/rasterizer_scene_high_end_rd.cpp index efa16628e4..c56c208098 100644 --- a/servers/rendering/rasterizer_rd/rasterizer_scene_high_end_rd.cpp +++ b/servers/rendering/rasterizer_rd/rasterizer_scene_high_end_rd.cpp @@ -2734,6 +2734,7 @@ RasterizerSceneHighEndRD::RasterizerSceneHighEndRD(RasterizerStorageRD *p_storag actions.renames["LIGHT_COLOR"] = "light_color"; actions.renames["LIGHT"] = "light"; actions.renames["ATTENUATION"] = "attenuation"; + actions.renames["SHADOW_ATTENUATION"] = "shadow_attenuation"; actions.renames["DIFFUSE_LIGHT"] = "diffuse_light"; actions.renames["SPECULAR_LIGHT"] = "specular_light"; diff --git a/servers/rendering/rasterizer_rd/rasterizer_storage_rd.cpp b/servers/rendering/rasterizer_rd/rasterizer_storage_rd.cpp index d751f474cd..a13e7d786b 100644 --- a/servers/rendering/rasterizer_rd/rasterizer_storage_rd.cpp +++ b/servers/rendering/rasterizer_rd/rasterizer_storage_rd.cpp @@ -715,8 +715,120 @@ RID RasterizerStorageRD::texture_2d_layered_create(const Vector<Ref<Image>> &p_l return texture_owner.make_rid(texture); } -RID RasterizerStorageRD::texture_3d_create(const Vector<Ref<Image>> &p_slices) { - return RID(); +RID RasterizerStorageRD::texture_3d_create(Image::Format p_format, int p_width, int p_height, int p_depth, bool p_mipmaps, const Vector<Ref<Image>> &p_data) { + ERR_FAIL_COND_V(p_data.size() == 0, RID()); + Image::Image3DValidateError verr = Image::validate_3d_image(p_format, p_width, p_height, p_depth, p_mipmaps, p_data); + if (verr != Image::VALIDATE_3D_OK) { + ERR_FAIL_V_MSG(RID(), Image::get_3d_image_validation_error_text(verr)); + } + + TextureToRDFormat ret_format; + Image::Format validated_format = Image::FORMAT_MAX; + Vector<uint8_t> all_data; + uint32_t mipmap_count = 0; + Vector<Texture::BufferSlice3D> slices; + { + Vector<Ref<Image>> images; + uint32_t all_data_size = 0; + images.resize(p_data.size()); + for (int i = 0; i < p_data.size(); i++) { + TextureToRDFormat f; + images.write[i] = _validate_texture_format(p_data[i], f); + if (i == 0) { + ret_format = f; + validated_format = images[0]->get_format(); + } + + all_data_size += images[i]->get_data().size(); + } + + all_data.resize(all_data_size); //consolidate all data here + uint32_t offset = 0; + Size2i prev_size; + for (int i = 0; i < p_data.size(); i++) { + uint32_t s = images[i]->get_data().size(); + + copymem(&all_data.write[offset], images[i]->get_data().ptr(), s); + { + Texture::BufferSlice3D slice; + slice.size.width = images[i]->get_width(); + slice.size.height = images[i]->get_height(); + slice.offset = offset; + slice.buffer_size = s; + slices.push_back(slice); + } + offset += s; + + Size2i img_size(images[i]->get_width(), images[i]->get_height()); + if (img_size != prev_size) { + mipmap_count++; + } + prev_size = img_size; + } + } + + Texture texture; + + texture.type = Texture::TYPE_3D; + texture.width = p_width; + texture.height = p_height; + texture.depth = p_depth; + texture.mipmaps = mipmap_count; + texture.format = p_data[0]->get_format(); + texture.validated_format = validated_format; + + texture.buffer_size_3d = all_data.size(); + texture.buffer_slices_3d = slices; + + texture.rd_type = RD::TEXTURE_TYPE_3D; + texture.rd_format = ret_format.format; + texture.rd_format_srgb = ret_format.format_srgb; + + RD::TextureFormat rd_format; + RD::TextureView rd_view; + { //attempt register + rd_format.format = texture.rd_format; + rd_format.width = texture.width; + rd_format.height = texture.height; + rd_format.depth = texture.depth; + rd_format.array_layers = 1; + rd_format.mipmaps = texture.mipmaps; + rd_format.type = texture.rd_type; + rd_format.samples = RD::TEXTURE_SAMPLES_1; + rd_format.usage_bits = RD::TEXTURE_USAGE_SAMPLING_BIT | RD::TEXTURE_USAGE_CAN_UPDATE_BIT | RD::TEXTURE_USAGE_CAN_COPY_FROM_BIT; + if (texture.rd_format_srgb != RD::DATA_FORMAT_MAX) { + rd_format.shareable_formats.push_back(texture.rd_format); + rd_format.shareable_formats.push_back(texture.rd_format_srgb); + } + } + { + rd_view.swizzle_r = ret_format.swizzle_r; + rd_view.swizzle_g = ret_format.swizzle_g; + rd_view.swizzle_b = ret_format.swizzle_b; + rd_view.swizzle_a = ret_format.swizzle_a; + } + Vector<Vector<uint8_t>> data_slices; + data_slices.push_back(all_data); //one slice + + texture.rd_texture = RD::get_singleton()->texture_create(rd_format, rd_view, data_slices); + ERR_FAIL_COND_V(texture.rd_texture.is_null(), RID()); + if (texture.rd_format_srgb != RD::DATA_FORMAT_MAX) { + rd_view.format_override = texture.rd_format_srgb; + texture.rd_texture_srgb = RD::get_singleton()->texture_create_shared(rd_view, texture.rd_texture); + if (texture.rd_texture_srgb.is_null()) { + RD::get_singleton()->free(texture.rd_texture); + ERR_FAIL_COND_V(texture.rd_texture_srgb.is_null(), RID()); + } + } + + //used for 2D, overridable + texture.width_2d = texture.width; + texture.height_2d = texture.height; + texture.is_render_target = false; + texture.rd_view = rd_view; + texture.is_proxy = false; + + return texture_owner.make_rid(texture); } RID RasterizerStorageRD::texture_proxy_create(RID p_base) { @@ -772,7 +884,41 @@ void RasterizerStorageRD::texture_2d_update(RID p_texture, const Ref<Image> &p_i _texture_2d_update(p_texture, p_image, p_layer, false); } -void RasterizerStorageRD::texture_3d_update(RID p_texture, const Ref<Image> &p_image, int p_depth, int p_mipmap) { +void RasterizerStorageRD::texture_3d_update(RID p_texture, const Vector<Ref<Image>> &p_data) { + Texture *tex = texture_owner.getornull(p_texture); + ERR_FAIL_COND(!tex); + ERR_FAIL_COND(tex->type != Texture::TYPE_3D); + Image::Image3DValidateError verr = Image::validate_3d_image(tex->format, tex->width, tex->height, tex->depth, tex->mipmaps > 1, p_data); + if (verr != Image::VALIDATE_3D_OK) { + ERR_FAIL_MSG(Image::get_3d_image_validation_error_text(verr)); + } + + Vector<uint8_t> all_data; + { + Vector<Ref<Image>> images; + uint32_t all_data_size = 0; + images.resize(p_data.size()); + for (int i = 0; i < p_data.size(); i++) { + Ref<Image> image = p_data[i]; + if (image->get_format() != tex->validated_format) { + image = image->duplicate(); + image->convert(tex->validated_format); + } + all_data_size += images[i]->get_data().size(); + images.push_back(image); + } + + all_data.resize(all_data_size); //consolidate all data here + uint32_t offset = 0; + + for (int i = 0; i < p_data.size(); i++) { + uint32_t s = images[i]->get_data().size(); + copymem(&all_data.write[offset], images[i]->get_data().ptr(), s); + offset += s; + } + } + + RD::get_singleton()->texture_update(tex->rd_texture, 0, all_data, true); } void RasterizerStorageRD::texture_proxy_update(RID p_texture, RID p_proxy_to) { @@ -858,7 +1004,25 @@ RID RasterizerStorageRD::texture_2d_layered_placeholder_create(RS::TextureLayere } RID RasterizerStorageRD::texture_3d_placeholder_create() { - return RID(); + //this could be better optimized to reuse an existing image , done this way + //for now to get it working + Ref<Image> image; + image.instance(); + image->create(4, 4, false, Image::FORMAT_RGBA8); + + for (int i = 0; i < 4; i++) { + for (int j = 0; j < 4; j++) { + image->set_pixel(i, j, Color(1, 0, 1, 1)); + } + } + + Vector<Ref<Image>> images; + //cube + for (int i = 0; i < 4; i++) { + images.push_back(image); + } + + return texture_3d_create(Image::FORMAT_RGBA8, 4, 4, 4, false, images); } Ref<Image> RasterizerStorageRD::texture_2d_get(RID p_texture) const { @@ -890,11 +1054,51 @@ Ref<Image> RasterizerStorageRD::texture_2d_get(RID p_texture) const { } Ref<Image> RasterizerStorageRD::texture_2d_layer_get(RID p_texture, int p_layer) const { - return Ref<Image>(); + Texture *tex = texture_owner.getornull(p_texture); + ERR_FAIL_COND_V(!tex, Ref<Image>()); + + Vector<uint8_t> data = RD::get_singleton()->texture_get_data(tex->rd_texture, p_layer); + ERR_FAIL_COND_V(data.size() == 0, Ref<Image>()); + Ref<Image> image; + image.instance(); + image->create(tex->width, tex->height, tex->mipmaps > 1, tex->validated_format, data); + ERR_FAIL_COND_V(image->empty(), Ref<Image>()); + if (tex->format != tex->validated_format) { + image->convert(tex->format); + } + + return image; } -Ref<Image> RasterizerStorageRD::texture_3d_slice_get(RID p_texture, int p_depth, int p_mipmap) const { - return Ref<Image>(); +Vector<Ref<Image>> RasterizerStorageRD::texture_3d_get(RID p_texture) const { + Texture *tex = texture_owner.getornull(p_texture); + ERR_FAIL_COND_V(!tex, Vector<Ref<Image>>()); + ERR_FAIL_COND_V(tex->type != Texture::TYPE_3D, Vector<Ref<Image>>()); + + Vector<uint8_t> all_data = RD::get_singleton()->texture_get_data(tex->rd_texture, 0); + + ERR_FAIL_COND_V(all_data.size() != (int)tex->buffer_size_3d, Vector<Ref<Image>>()); + + Vector<Ref<Image>> ret; + + for (int i = 0; i < tex->buffer_slices_3d.size(); i++) { + const Texture::BufferSlice3D &bs = tex->buffer_slices_3d[i]; + ERR_FAIL_COND_V(bs.offset >= (uint32_t)all_data.size(), Vector<Ref<Image>>()); + ERR_FAIL_COND_V(bs.offset + bs.buffer_size > (uint32_t)all_data.size(), Vector<Ref<Image>>()); + Vector<uint8_t> sub_region = all_data.subarray(bs.offset, bs.offset + bs.buffer_size - 1); + + Ref<Image> img; + img.instance(); + img->create(bs.size.width, bs.size.height, false, tex->validated_format, sub_region); + ERR_FAIL_COND_V(img->empty(), Vector<Ref<Image>>()); + if (tex->format != tex->validated_format) { + img->convert(tex->format); + } + + ret.push_back(img); + } + + return ret; } void RasterizerStorageRD::texture_replace(RID p_texture, RID p_by_texture) { @@ -3004,9 +3208,9 @@ Vector<float> RasterizerStorageRD::multimesh_get_buffer(RID p_multimesh) const { Vector<uint8_t> buffer = RD::get_singleton()->buffer_get_data(multimesh->buffer); Vector<float> ret; - ret.resize(multimesh->instances); + ret.resize(multimesh->instances * multimesh->stride_cache); { - float *w = multimesh->data_cache.ptrw(); + float *w = ret.ptrw(); const uint8_t *r = buffer.ptr(); copymem(w, r, buffer.size()); } @@ -6986,15 +7190,18 @@ RasterizerStorageRD::RasterizerStorageRD() { case RS::CANVAS_ITEM_TEXTURE_REPEAT_DISABLED: { sampler_state.repeat_u = RD::SAMPLER_REPEAT_MODE_CLAMP_TO_EDGE; sampler_state.repeat_v = RD::SAMPLER_REPEAT_MODE_CLAMP_TO_EDGE; + sampler_state.repeat_w = RD::SAMPLER_REPEAT_MODE_CLAMP_TO_EDGE; } break; case RS::CANVAS_ITEM_TEXTURE_REPEAT_ENABLED: { sampler_state.repeat_u = RD::SAMPLER_REPEAT_MODE_REPEAT; sampler_state.repeat_v = RD::SAMPLER_REPEAT_MODE_REPEAT; + sampler_state.repeat_w = RD::SAMPLER_REPEAT_MODE_REPEAT; } break; case RS::CANVAS_ITEM_TEXTURE_REPEAT_MIRROR: { sampler_state.repeat_u = RD::SAMPLER_REPEAT_MODE_MIRRORED_REPEAT; sampler_state.repeat_v = RD::SAMPLER_REPEAT_MODE_MIRRORED_REPEAT; + sampler_state.repeat_w = RD::SAMPLER_REPEAT_MODE_MIRRORED_REPEAT; } break; default: { } diff --git a/servers/rendering/rasterizer_rd/rasterizer_storage_rd.h b/servers/rendering/rasterizer_rd/rasterizer_storage_rd.h index cecae6bbb2..e14b9528cf 100644 --- a/servers/rendering/rasterizer_rd/rasterizer_storage_rd.h +++ b/servers/rendering/rasterizer_rd/rasterizer_storage_rd.h @@ -205,6 +205,14 @@ private: int height_2d; int width_2d; + struct BufferSlice3D { + Size2i size; + uint32_t offset = 0; + uint32_t buffer_size = 0; + }; + Vector<BufferSlice3D> buffer_slices_3d; + uint32_t buffer_size_3d = 0; + bool is_render_target; bool is_proxy; @@ -980,14 +988,14 @@ public: virtual RID texture_2d_create(const Ref<Image> &p_image); virtual RID texture_2d_layered_create(const Vector<Ref<Image>> &p_layers, RS::TextureLayeredType p_layered_type); - virtual RID texture_3d_create(const Vector<Ref<Image>> &p_slices); //all slices, then all the mipmaps, must be coherent + virtual RID texture_3d_create(Image::Format p_format, int p_width, int p_height, int p_depth, bool p_mipmaps, const Vector<Ref<Image>> &p_data); //all slices, then all the mipmaps, must be coherent virtual RID texture_proxy_create(RID p_base); virtual void _texture_2d_update(RID p_texture, const Ref<Image> &p_image, int p_layer, bool p_immediate); virtual void texture_2d_update_immediate(RID p_texture, const Ref<Image> &p_image, int p_layer = 0); //mostly used for video and streaming virtual void texture_2d_update(RID p_texture, const Ref<Image> &p_image, int p_layer = 0); - virtual void texture_3d_update(RID p_texture, const Ref<Image> &p_image, int p_depth, int p_mipmap); + virtual void texture_3d_update(RID p_texture, const Vector<Ref<Image>> &p_data); virtual void texture_proxy_update(RID p_texture, RID p_proxy_to); //these two APIs can be used together or in combination with the others. @@ -997,7 +1005,7 @@ public: virtual Ref<Image> texture_2d_get(RID p_texture) const; virtual Ref<Image> texture_2d_layer_get(RID p_texture, int p_layer) const; - virtual Ref<Image> texture_3d_slice_get(RID p_texture, int p_depth, int p_mipmap) const; + virtual Vector<Ref<Image>> texture_3d_get(RID p_texture) const; virtual void texture_replace(RID p_texture, RID p_by_texture); virtual void texture_set_size_override(RID p_texture, int p_width, int p_height); diff --git a/servers/rendering/rasterizer_rd/shaders/canvas.glsl b/servers/rendering/rasterizer_rd/shaders/canvas.glsl index e33b3face9..4a40584e16 100644 --- a/servers/rendering/rasterizer_rd/shaders/canvas.glsl +++ b/servers/rendering/rasterizer_rd/shaders/canvas.glsl @@ -101,7 +101,7 @@ void main() { offset += 1; } else { instance_color = vec4(texelFetch(instancing_buffer, offset + 0), texelFetch(instancing_buffer, offset + 1), texelFetch(instancing_buffer, offset + 2), texelFetch(instancing_buffer, offset + 3)); - offser += 4; + offset += 4; } color *= instance_color; diff --git a/servers/rendering/rasterizer_rd/shaders/particles.glsl b/servers/rendering/rasterizer_rd/shaders/particles.glsl index 3de807b57c..a924509771 100644 --- a/servers/rendering/rasterizer_rd/shaders/particles.glsl +++ b/servers/rendering/rasterizer_rd/shaders/particles.glsl @@ -147,7 +147,7 @@ bool emit_particle(mat4 p_xform, vec3 p_velocity, vec4 p_color, vec4 p_custom, u while(attempts-- > 0) { dst_index = dst_particles.particle_count; if (dst_index == dst_particles.particle_max) { - return false; //cant emit anymore + return false; //can't emit anymore } if (atomicCompSwap(dst_particles.particle_count, dst_index, dst_index +1 ) != dst_index) { diff --git a/servers/rendering/rasterizer_rd/shaders/scene_high_end.glsl b/servers/rendering/rasterizer_rd/shaders/scene_high_end.glsl index 2a7b73d9aa..e11f3983c5 100644 --- a/servers/rendering/rasterizer_rd/shaders/scene_high_end.glsl +++ b/servers/rendering/rasterizer_rd/shaders/scene_high_end.glsl @@ -681,7 +681,7 @@ LIGHT_SHADER_CODE #ifndef USE_NO_SHADOWS -// Produces cheap white noise, optmized for window-space +// Produces cheap white noise, optimized for window-space // Comes from: https://www.shadertoy.com/view/4djSRW // Copyright: Dave Hoskins, MIT License float quick_hash(vec2 pos) { diff --git a/servers/rendering/rasterizer_rd/shaders/scene_high_end_inc.glsl b/servers/rendering/rasterizer_rd/shaders/scene_high_end_inc.glsl index 66bfefbe89..0cc2b90c53 100644 --- a/servers/rendering/rasterizer_rd/shaders/scene_high_end_inc.glsl +++ b/servers/rendering/rasterizer_rd/shaders/scene_high_end_inc.glsl @@ -252,7 +252,7 @@ layout(set = 1, binding = 0) uniform textureCube radiance_cubemap; #endif -/* Set 2, Reflection and Shadow Atlases (view dependant) */ +/* Set 2, Reflection and Shadow Atlases (view dependent) */ layout(set = 2, binding = 0) uniform textureCubeArray reflection_atlas; diff --git a/servers/rendering/rasterizer_rd/shaders/sdfgi_direct_light.glsl b/servers/rendering/rasterizer_rd/shaders/sdfgi_direct_light.glsl index c4b29216d5..61e4bf5e18 100644 --- a/servers/rendering/rasterizer_rd/shaders/sdfgi_direct_light.glsl +++ b/servers/rendering/rasterizer_rd/shaders/sdfgi_direct_light.glsl @@ -22,7 +22,7 @@ dispatch_data; struct ProcessVoxel { uint position; //xyz 7 bit packed, extra 11 bits for neigbours uint albedo; //rgb bits 0-15 albedo, bits 16-21 are normal bits (set if geometry exists toward that side), extra 11 bits for neibhbours - uint light; //rgbe8985 encoded total saved light, extra 2 bits for neighbous + uint light; //rgbe8985 encoded total saved light, extra 2 bits for neighbours uint light_aniso; //55555 light anisotropy, extra 2 bits for neighbours //total neighbours: 26 }; diff --git a/servers/rendering/rasterizer_rd/shaders/sdfgi_integrate.glsl b/servers/rendering/rasterizer_rd/shaders/sdfgi_integrate.glsl index 1ec471d204..d516ab22c3 100644 --- a/servers/rendering/rasterizer_rd/shaders/sdfgi_integrate.glsl +++ b/servers/rendering/rasterizer_rd/shaders/sdfgi_integrate.glsl @@ -336,7 +336,7 @@ void main() { #ifdef MODE_STORE - // converting to octahedral in this step is requiered because + // converting to octahedral in this step is required because // octahedral is much faster to read from the screen than spherical harmonics, // despite the very slight quality loss @@ -512,7 +512,7 @@ void main() { imageStore(lightprobe_average_scroll_texture, dst_pos, value); } } else if (params.cascade < params.max_cascades - 1) { - //cant scroll, must look for position in parent cascade + //can't scroll, must look for position in parent cascade //to global coords float probe_cell_size = float(params.grid_size.x / float(params.probe_axis_size - 1)) / cascades.data[params.cascade].to_cell; diff --git a/servers/rendering/rasterizer_rd/shaders/sdfgi_preprocess.glsl b/servers/rendering/rasterizer_rd/shaders/sdfgi_preprocess.glsl index dd0ca5c506..916c60ac89 100644 --- a/servers/rendering/rasterizer_rd/shaders/sdfgi_preprocess.glsl +++ b/servers/rendering/rasterizer_rd/shaders/sdfgi_preprocess.glsl @@ -103,7 +103,7 @@ dispatch_data; struct ProcessVoxel { uint position; //xyz 7 bit packed, extra 11 bits for neigbours uint albedo; //rgb bits 0-15 albedo, bits 16-21 are normal bits (set if geometry exists toward that side), extra 11 bits for neibhbours - uint light; //rgbe8985 encoded total saved light, extra 2 bits for neighbous + uint light; //rgbe8985 encoded total saved light, extra 2 bits for neighbours uint light_aniso; //55555 light anisotropy, extra 2 bits for neighbours //total neighbours: 26 }; @@ -136,7 +136,7 @@ dispatch_data; struct ProcessVoxel { uint position; //xyz 7 bit packed, extra 11 bits for neigbours uint albedo; //rgb bits 0-15 albedo, bits 16-21 are normal bits (set if geometry exists toward that side), extra 11 bits for neibhbours - uint light; //rgbe8985 encoded total saved light, extra 2 bits for neighbous + uint light; //rgbe8985 encoded total saved light, extra 2 bits for neighbours uint light_aniso; //55555 light anisotropy, extra 2 bits for neighbours //total neighbours: 26 }; @@ -274,7 +274,7 @@ void main() { #ifdef MODE_JUMPFLOOD - //regular jumpflood, efficent for large steps, inefficient for small steps + //regular jumpflood, efficient for large steps, inefficient for small steps ivec3 pos = ivec3(gl_GlobalInvocationID.xyz); vec3 posf = vec3(pos); diff --git a/servers/rendering/rasterizer_rd/shaders/volumetric_fog.glsl b/servers/rendering/rasterizer_rd/shaders/volumetric_fog.glsl index cb19fb0b69..13b162f0c9 100644 --- a/servers/rendering/rasterizer_rd/shaders/volumetric_fog.glsl +++ b/servers/rendering/rasterizer_rd/shaders/volumetric_fog.glsl @@ -485,7 +485,7 @@ void main() { //get depth at cell pos float z = get_depth_at_pos(fog_cell_size.z, i); - //get distance from previos pos + //get distance from previous pos float d = abs(prev_z - z); //compute exinction based on beer's float extinction = t * exp(-d * fog.a); diff --git a/servers/rendering/rendering_server_raster.h b/servers/rendering/rendering_server_raster.h index ceefcfa1fc..b554425bef 100644 --- a/servers/rendering/rendering_server_raster.h +++ b/servers/rendering/rendering_server_raster.h @@ -114,6 +114,14 @@ public: m_r m_name(m_type1 arg1, m_type2 arg2, m_type3 arg3, m_type4 arg4) { return BINDBASE->m_name(arg1, arg2, arg3, arg4); } #define BIND4RC(m_r, m_name, m_type1, m_type2, m_type3, m_type4) \ m_r m_name(m_type1 arg1, m_type2 arg2, m_type3 arg3, m_type4 arg4) const { return BINDBASE->m_name(arg1, arg2, arg3, arg4); } +#define BIND5R(m_r, m_name, m_type1, m_type2, m_type3, m_type4, m_type5) \ + m_r m_name(m_type1 arg1, m_type2 arg2, m_type3 arg3, m_type4 arg4, m_type5 arg5) { return BINDBASE->m_name(arg1, arg2, arg3, arg4, arg5); } +#define BIND5RC(m_r, m_name, m_type1, m_type2, m_type3, m_type4, m_type5) \ + m_r m_name(m_type1 arg1, m_type2 arg2, m_type3 arg3, m_type4 arg4, m_type5 arg5) const { return BINDBASE->m_name(arg1, arg2, arg3, arg4, arg5); } +#define BIND6R(m_r, m_name, m_type1, m_type2, m_type3, m_type4, m_type5, m_type6) \ + m_r m_name(m_type1 arg1, m_type2 arg2, m_type3 arg3, m_type4 arg4, m_type5 arg5, m_type6 arg6) { return BINDBASE->m_name(arg1, arg2, arg3, arg4, arg5, arg6); } +#define BIND6RC(m_r, m_name, m_type1, m_type2, m_type3, m_type4, m_type5, m_type6) \ + m_r m_name(m_type1 arg1, m_type2 arg2, m_type3 arg3, m_type4 arg4, m_type5 arg5, m_type6 arg6) const { return BINDBASE->m_name(arg1, arg2, arg3, arg4, arg5, arg6); } #define BIND0(m_name) \ void m_name() { DISPLAY_CHANGED BINDBASE->m_name(); } @@ -160,14 +168,14 @@ public: //these go pass-through, as they can be called from any thread BIND1R(RID, texture_2d_create, const Ref<Image> &) BIND2R(RID, texture_2d_layered_create, const Vector<Ref<Image>> &, TextureLayeredType) - BIND1R(RID, texture_3d_create, const Vector<Ref<Image>> &) + BIND6R(RID, texture_3d_create, Image::Format, int, int, int, bool, const Vector<Ref<Image>> &) BIND1R(RID, texture_proxy_create, RID) //goes pass-through BIND3(texture_2d_update_immediate, RID, const Ref<Image> &, int) //these go through command queue if they are in another thread BIND3(texture_2d_update, RID, const Ref<Image> &, int) - BIND4(texture_3d_update, RID, const Ref<Image> &, int, int) + BIND2(texture_3d_update, RID, const Vector<Ref<Image>> &) BIND2(texture_proxy_update, RID, RID) //these also go pass-through @@ -177,7 +185,7 @@ public: BIND1RC(Ref<Image>, texture_2d_get, RID) BIND2RC(Ref<Image>, texture_2d_layer_get, RID, int) - BIND3RC(Ref<Image>, texture_3d_slice_get, RID, int, int) + BIND1RC(Vector<Ref<Image>>, texture_3d_get, RID) BIND2(texture_replace, RID, RID) diff --git a/servers/rendering/rendering_server_wrap_mt.h b/servers/rendering/rendering_server_wrap_mt.h index a8a56e7d56..372a7269dc 100644 --- a/servers/rendering/rendering_server_wrap_mt.h +++ b/servers/rendering/rendering_server_wrap_mt.h @@ -79,14 +79,14 @@ public: //these go pass-through, as they can be called from any thread virtual RID texture_2d_create(const Ref<Image> &p_image) { return rendering_server->texture_2d_create(p_image); } virtual RID texture_2d_layered_create(const Vector<Ref<Image>> &p_layers, TextureLayeredType p_layered_type) { return rendering_server->texture_2d_layered_create(p_layers, p_layered_type); } - virtual RID texture_3d_create(const Vector<Ref<Image>> &p_slices) { return rendering_server->texture_3d_create(p_slices); } + virtual RID texture_3d_create(Image::Format p_format, int p_width, int p_height, int p_depth, bool p_mipmaps, const Vector<Ref<Image>> &p_data) { return rendering_server->texture_3d_create(p_format, p_width, p_height, p_depth, p_mipmaps, p_data); } virtual RID texture_proxy_create(RID p_base) { return rendering_server->texture_proxy_create(p_base); } //goes pass-through virtual void texture_2d_update_immediate(RID p_texture, const Ref<Image> &p_image, int p_layer = 0) { rendering_server->texture_2d_update_immediate(p_texture, p_image, p_layer); } //these go through command queue if they are in another thread FUNC3(texture_2d_update, RID, const Ref<Image> &, int) - FUNC4(texture_3d_update, RID, const Ref<Image> &, int, int) + FUNC2(texture_3d_update, RID, const Vector<Ref<Image>> &) FUNC2(texture_proxy_update, RID, RID) //these also go pass-through @@ -96,7 +96,7 @@ public: FUNC1RC(Ref<Image>, texture_2d_get, RID) FUNC2RC(Ref<Image>, texture_2d_layer_get, RID, int) - FUNC3RC(Ref<Image>, texture_3d_slice_get, RID, int, int) + FUNC1RC(Vector<Ref<Image>>, texture_3d_get, RID) FUNC2(texture_replace, RID, RID) diff --git a/servers/rendering/shader_language.cpp b/servers/rendering/shader_language.cpp index 28c41fb2dc..6c835fcadf 100644 --- a/servers/rendering/shader_language.cpp +++ b/servers/rendering/shader_language.cpp @@ -6099,6 +6099,14 @@ Error ShaderLanguage::_parse_shader(const Map<StringName, FunctionInfo> &p_funct case TK_UNIFORM: case TK_VARYING: { bool uniform = tk.type == TK_UNIFORM; + + if (!uniform) { + if (shader_type_identifier == "particles" || shader_type_identifier == "sky") { + _set_error(vformat("Varyings cannot be used in '%s' shaders!", shader_type_identifier)); + return ERR_PARSE_ERROR; + } + } + DataPrecision precision = PRECISION_DEFAULT; DataInterpolation interpolation = INTERPOLATION_SMOOTH; DataType type; diff --git a/servers/rendering/shader_types.cpp b/servers/rendering/shader_types.cpp index f3ef322ad7..ad5cbc9e51 100644 --- a/servers/rendering/shader_types.cpp +++ b/servers/rendering/shader_types.cpp @@ -145,7 +145,8 @@ ShaderTypes::ShaderTypes() { shader_modes[RS::SHADER_SPATIAL].functions["light"].built_ins["VIEW"] = constt(ShaderLanguage::TYPE_VEC3); shader_modes[RS::SHADER_SPATIAL].functions["light"].built_ins["LIGHT"] = constt(ShaderLanguage::TYPE_VEC3); shader_modes[RS::SHADER_SPATIAL].functions["light"].built_ins["LIGHT_COLOR"] = constt(ShaderLanguage::TYPE_VEC3); - shader_modes[RS::SHADER_SPATIAL].functions["light"].built_ins["ATTENUATION"] = constt(ShaderLanguage::TYPE_VEC3); + shader_modes[RS::SHADER_SPATIAL].functions["light"].built_ins["ATTENUATION"] = constt(ShaderLanguage::TYPE_FLOAT); + shader_modes[RS::SHADER_SPATIAL].functions["light"].built_ins["SHADOW_ATTENUATION"] = constt(ShaderLanguage::TYPE_VEC3); shader_modes[RS::SHADER_SPATIAL].functions["light"].built_ins["ALBEDO"] = constt(ShaderLanguage::TYPE_VEC3); shader_modes[RS::SHADER_SPATIAL].functions["light"].built_ins["BACKLIGHT"] = constt(ShaderLanguage::TYPE_VEC3); shader_modes[RS::SHADER_SPATIAL].functions["light"].built_ins["ROUGHNESS"] = constt(ShaderLanguage::TYPE_FLOAT); diff --git a/servers/rendering_server.h b/servers/rendering_server.h index 64fa06ae72..49f840948f 100644 --- a/servers/rendering_server.h +++ b/servers/rendering_server.h @@ -97,12 +97,12 @@ public: virtual RID texture_2d_create(const Ref<Image> &p_image) = 0; virtual RID texture_2d_layered_create(const Vector<Ref<Image>> &p_layers, TextureLayeredType p_layered_type) = 0; - virtual RID texture_3d_create(const Vector<Ref<Image>> &p_slices) = 0; //all slices, then all the mipmaps, must be coherent + virtual RID texture_3d_create(Image::Format, int p_width, int p_height, int p_depth, bool p_mipmaps, const Vector<Ref<Image>> &p_data) = 0; //all slices, then all the mipmaps, must be coherent virtual RID texture_proxy_create(RID p_base) = 0; virtual void texture_2d_update_immediate(RID p_texture, const Ref<Image> &p_image, int p_layer = 0) = 0; //mostly used for video and streaming virtual void texture_2d_update(RID p_texture, const Ref<Image> &p_image, int p_layer = 0) = 0; - virtual void texture_3d_update(RID p_texture, const Ref<Image> &p_image, int p_depth, int p_mipmap) = 0; + virtual void texture_3d_update(RID p_texture, const Vector<Ref<Image>> &p_data) = 0; virtual void texture_proxy_update(RID p_texture, RID p_proxy_to) = 0; //these two APIs can be used together or in combination with the others. @@ -112,7 +112,7 @@ public: virtual Ref<Image> texture_2d_get(RID p_texture) const = 0; virtual Ref<Image> texture_2d_layer_get(RID p_texture, int p_layer) const = 0; - virtual Ref<Image> texture_3d_slice_get(RID p_texture, int p_depth, int p_mipmap) const = 0; + virtual Vector<Ref<Image>> texture_3d_get(RID p_texture) const = 0; virtual void texture_replace(RID p_texture, RID p_by_texture) = 0; virtual void texture_set_size_override(RID p_texture, int p_width, int p_height) = 0; diff --git a/tests/SCsub b/tests/SCsub index 44b4cdc4b0..7aab28531d 100644 --- a/tests/SCsub +++ b/tests/SCsub @@ -7,7 +7,8 @@ env.tests_sources = [] env_tests = env.Clone() # Include GDNative headers. -env_tests.Append(CPPPATH=["#modules/gdnative/include"]) +if env["module_gdnative_enabled"]: + env_tests.Append(CPPPATH=["#modules/gdnative/include"]) # We must disable the THREAD_LOCAL entirely in doctest to prevent crashes on debugging # Since we link with /MT thread_local is always expired when the header is used diff --git a/tests/test_expression.h b/tests/test_expression.h index 85d37d1460..a3d4877d52 100644 --- a/tests/test_expression.h +++ b/tests/test_expression.h @@ -66,7 +66,7 @@ TEST_CASE("[Expression] Integer arithmetic") { "Integer / integer division should parse successfully."); CHECK_MESSAGE( int(expression.execute()) == 2, - "Integer / integer divsion should return the expected result."); + "Integer / integer division should return the expected result."); CHECK_MESSAGE( expression.parse("2 * (6 + 14) / 2 - 5") == OK, @@ -98,21 +98,21 @@ TEST_CASE("[Expression] Floating-point arithmetic") { "Float / integer division should parse successfully."); CHECK_MESSAGE( Math::is_equal_approx(float(expression.execute()), 0.3), - "Float / integer divsion should return the expected result."); + "Float / integer division should return the expected result."); CHECK_MESSAGE( expression.parse("3 / 10.0") == OK, "Basic integer / float division should parse successfully."); CHECK_MESSAGE( Math::is_equal_approx(float(expression.execute()), 0.3), - "Basic integer / float divsion should return the expected result."); + "Basic integer / float division should return the expected result."); CHECK_MESSAGE( expression.parse("3.0 / 10.0") == OK, "Float / float division should parse successfully."); CHECK_MESSAGE( Math::is_equal_approx(float(expression.execute()), 0.3), - "Float / float divsion should return the expected result."); + "Float / float division should return the expected result."); CHECK_MESSAGE( expression.parse("2.5 * (6.0 + 14.25) / 2.0 - 5.12345") == OK, @@ -148,6 +148,20 @@ TEST_CASE("[Expression] Scientific notation") { "The expression should return the expected result."); } +TEST_CASE("[Expression] Underscored numeric literals") { + Expression expression; + + CHECK_MESSAGE( + expression.parse("1_000_000") == OK, + "The expression should parse successfully."); + CHECK_MESSAGE( + expression.parse("1_000.000") == OK, + "The expression should parse successfully."); + CHECK_MESSAGE( + expression.parse("0xff_99_00") == OK, + "The expression should parse successfully."); +} + TEST_CASE("[Expression] Built-in functions") { Expression expression; diff --git a/tests/test_list.h b/tests/test_list.h new file mode 100644 index 0000000000..7412fbc3c8 --- /dev/null +++ b/tests/test_list.h @@ -0,0 +1,279 @@ +/*************************************************************************/ +/* test_list.h */ +/*************************************************************************/ +/* This file is part of: */ +/* GODOT ENGINE */ +/* https://godotengine.org */ +/*************************************************************************/ +/* Copyright (c) 2007-2020 Juan Linietsky, Ariel Manzur. */ +/* Copyright (c) 2014-2020 Godot Engine contributors (cf. AUTHORS.md). */ +/* */ +/* Permission is hereby granted, free of charge, to any person obtaining */ +/* a copy of this software and associated documentation files (the */ +/* "Software"), to deal in the Software without restriction, including */ +/* without limitation the rights to use, copy, modify, merge, publish, */ +/* distribute, sublicense, and/or sell copies of the Software, and to */ +/* permit persons to whom the Software is furnished to do so, subject to */ +/* the following conditions: */ +/* */ +/* The above copyright notice and this permission notice shall be */ +/* included in all copies or substantial portions of the Software. */ +/* */ +/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */ +/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */ +/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*/ +/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */ +/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */ +/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */ +/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +/*************************************************************************/ + +#ifndef TEST_LIST_H +#define TEST_LIST_H + +#include "core/list.h" + +#include "tests/test_macros.h" + +namespace TestList { + +static void populate_integers(List<int> &p_list, List<int>::Element *r_elements[], int num_elements) { + p_list.clear(); + for (int i = 0; i < num_elements; ++i) { + List<int>::Element *n = p_list.push_back(i); + r_elements[i] = n; + } +} + +TEST_CASE("[List] Swap adjacent front and back") { + List<int> list; + List<int>::Element *n[2]; + populate_integers(list, n, 2); + + list.swap(list.front(), list.back()); + + CHECK(list.front()->prev() == nullptr); + CHECK(list.front() != list.front()->next()); + + CHECK(list.front() == n[1]); + CHECK(list.back() == n[0]); + + CHECK(list.back()->next() == nullptr); + CHECK(list.back() != list.back()->prev()); +} + +TEST_CASE("[List] Swap first adjacent pair") { + List<int> list; + List<int>::Element *n[4]; + populate_integers(list, n, 4); + + list.swap(n[0], n[1]); + + CHECK(list.front()->prev() == nullptr); + CHECK(list.front() != list.front()->next()); + + CHECK(list.front() == n[1]); + CHECK(list.front()->next() == n[0]); + CHECK(list.back()->prev() == n[2]); + CHECK(list.back() == n[3]); + + CHECK(list.back()->next() == nullptr); + CHECK(list.back() != list.back()->prev()); +} + +TEST_CASE("[List] Swap middle adjacent pair") { + List<int> list; + List<int>::Element *n[4]; + populate_integers(list, n, 4); + + list.swap(n[1], n[2]); + + CHECK(list.front()->prev() == nullptr); + + CHECK(list.front() == n[0]); + CHECK(list.front()->next() == n[2]); + CHECK(list.back()->prev() == n[1]); + CHECK(list.back() == n[3]); + + CHECK(list.back()->next() == nullptr); +} + +TEST_CASE("[List] Swap last adjacent pair") { + List<int> list; + List<int>::Element *n[4]; + populate_integers(list, n, 4); + + list.swap(n[2], n[3]); + + CHECK(list.front()->prev() == nullptr); + + CHECK(list.front() == n[0]); + CHECK(list.front()->next() == n[1]); + CHECK(list.back()->prev() == n[3]); + CHECK(list.back() == n[2]); + + CHECK(list.back()->next() == nullptr); +} + +TEST_CASE("[List] Swap first cross pair") { + List<int> list; + List<int>::Element *n[4]; + populate_integers(list, n, 4); + + list.swap(n[0], n[2]); + + CHECK(list.front()->prev() == nullptr); + + CHECK(list.front() == n[2]); + CHECK(list.front()->next() == n[1]); + CHECK(list.back()->prev() == n[0]); + CHECK(list.back() == n[3]); + + CHECK(list.back()->next() == nullptr); +} + +TEST_CASE("[List] Swap last cross pair") { + List<int> list; + List<int>::Element *n[4]; + populate_integers(list, n, 4); + + list.swap(n[1], n[3]); + + CHECK(list.front()->prev() == nullptr); + + CHECK(list.front() == n[0]); + CHECK(list.front()->next() == n[3]); + CHECK(list.back()->prev() == n[2]); + CHECK(list.back() == n[1]); + + CHECK(list.back()->next() == nullptr); +} + +TEST_CASE("[List] Swap edges") { + List<int> list; + List<int>::Element *n[4]; + populate_integers(list, n, 4); + + list.swap(n[1], n[3]); + + CHECK(list.front()->prev() == nullptr); + + CHECK(list.front() == n[0]); + CHECK(list.front()->next() == n[3]); + CHECK(list.back()->prev() == n[2]); + CHECK(list.back() == n[1]); + + CHECK(list.back()->next() == nullptr); +} + +TEST_CASE("[List] Swap middle (values check)") { + List<String> list; + List<String>::Element *n_str1 = list.push_back("Still"); + List<String>::Element *n_str2 = list.push_back("waiting"); + List<String>::Element *n_str3 = list.push_back("for"); + List<String>::Element *n_str4 = list.push_back("Godot."); + + CHECK(n_str1->get() == "Still"); + CHECK(n_str4->get() == "Godot."); + + CHECK(list.front()->get() == "Still"); + CHECK(list.front()->next()->get() == "waiting"); + CHECK(list.back()->prev()->get() == "for"); + CHECK(list.back()->get() == "Godot."); + + list.swap(n_str2, n_str3); + + CHECK(list.front()->next()->get() == "for"); + CHECK(list.back()->prev()->get() == "waiting"); +} + +TEST_CASE("[List] Swap front and back (values check)") { + List<Variant> list; + Variant str = "Godot"; + List<Variant>::Element *n_str = list.push_back(str); + Variant color = Color(0, 0, 1); + List<Variant>::Element *n_color = list.push_back(color); + + CHECK(list.front()->get() == "Godot"); + CHECK(list.back()->get() == Color(0, 0, 1)); + + list.swap(n_str, n_color); + + CHECK(list.front()->get() == Color(0, 0, 1)); + CHECK(list.back()->get() == "Godot"); +} + +TEST_CASE("[List] Swap adjacent back and front (reverse order of elements)") { + List<int> list; + List<int>::Element *n[2]; + populate_integers(list, n, 2); + + list.swap(n[1], n[0]); + + List<int>::Element *it = list.front(); + while (it) { + List<int>::Element *prev_it = it; + it = it->next(); + if (it == prev_it) { + FAIL_CHECK("Infinite loop detected."); + break; + } + } +} + +static void swap_random(List<int> &p_list, List<int>::Element *r_elements[], size_t p_size, size_t p_iterations) { + Math::seed(0); + + for (size_t test_i = 0; test_i < p_iterations; ++test_i) { + // A and B elements have corresponding indices as values. + const int a_idx = static_cast<int>(Math::rand() % p_size); + const int b_idx = static_cast<int>(Math::rand() % p_size); + List<int>::Element *a = p_list.find(a_idx); // via find. + List<int>::Element *b = r_elements[b_idx]; // via pointer. + + int va = a->get(); + int vb = b->get(); + + p_list.swap(a, b); + + CHECK(va == a->get()); + CHECK(vb == b->get()); + + size_t element_count = 0; + + // Fully traversable after swap? + List<int>::Element *it = p_list.front(); + while (it) { + element_count += 1; + List<int>::Element *prev_it = it; + it = it->next(); + if (it == prev_it) { + FAIL_CHECK("Infinite loop detected."); + break; + } + } + // We should not lose anything in the process. + if (element_count != p_size) { + FAIL_CHECK("Element count mismatch."); + break; + } + } +} + +TEST_CASE("[Stress][List] Swap random 100 elements, 500 iterations.") { + List<int> list; + List<int>::Element *n[100]; + populate_integers(list, n, 100); + swap_random(list, n, 100, 500); +} + +TEST_CASE("[Stress][List] Swap random 10 elements, 1000 iterations.") { + List<int> list; + List<int>::Element *n[10]; + populate_integers(list, n, 10); + swap_random(list, n, 10, 1000); +} + +} // namespace TestList + +#endif // TEST_LIST_H diff --git a/tests/test_main.cpp b/tests/test_main.cpp index d41581883a..f0dec22738 100644 --- a/tests/test_main.cpp +++ b/tests/test_main.cpp @@ -37,9 +37,9 @@ #include "test_class_db.h" #include "test_color.h" #include "test_expression.h" -#include "test_gdnative_string.h" #include "test_gradient.h" #include "test_gui.h" +#include "test_list.h" #include "test_math.h" #include "test_oa_hash_map.h" #include "test_ordered_hash_map.h" diff --git a/thirdparty/README.md b/thirdparty/README.md index 881590ddf1..46a75d94b0 100644 --- a/thirdparty/README.md +++ b/thirdparty/README.md @@ -564,7 +564,7 @@ comments and a patch is provided in the squish/ folder. ## tinyexr - Upstream: https://github.com/syoyo/tinyexr -- Version: git (4dbd05a22f51a2d7462311569b8b0cba0bbe2ac5, 2020) +- Version: 1.0.0 (e4b7840d9448b7d57a88384ce26143004f3c0c71, 2020) - License: BSD-3-Clause Files extracted from upstream source: @@ -658,7 +658,7 @@ Files extracted from upstream source: ## zstd - Upstream: https://github.com/facebook/zstd -- Version: 1.4.4 (2019) +- Version: 1.4.5 (2020) - License: BSD-3-Clause Files extracted from upstream source: diff --git a/thirdparty/tinyexr/tinyexr.cc b/thirdparty/tinyexr/tinyexr.cc index 969a6d505d..fef8f66c98 100644 --- a/thirdparty/tinyexr/tinyexr.cc +++ b/thirdparty/tinyexr/tinyexr.cc @@ -1,2 +1,8 @@ +#if defined(_WIN32) +#ifndef NOMINMAX +#define NOMINMAX +#endif +#endif + #define TINYEXR_IMPLEMENTATION #include "tinyexr.h" diff --git a/thirdparty/tinyexr/tinyexr.h b/thirdparty/tinyexr/tinyexr.h index 7e8956f7d3..a3e7b23161 100644 --- a/thirdparty/tinyexr/tinyexr.h +++ b/thirdparty/tinyexr/tinyexr.h @@ -1,5 +1,7 @@ +#ifndef TINYEXR_H_ +#define TINYEXR_H_ /* -Copyright (c) 2014 - 2019, Syoyo Fujita and many contributors. +Copyright (c) 2014 - 2020, Syoyo Fujita and many contributors. All rights reserved. Redistribution and use in source and binary forms, with or without @@ -63,9 +65,6 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // End of OpenEXR license ------------------------------------------------- -#ifndef TINYEXR_H_ -#define TINYEXR_H_ - // // // Do this: @@ -198,11 +197,18 @@ typedef struct _EXRTile { unsigned char **images; // image[channels][pixels] } EXRTile; +typedef struct _EXRBox2i { + int min_x; + int min_y; + int max_x; + int max_y; +} EXRBox2i; + typedef struct _EXRHeader { float pixel_aspect_ratio; int line_order; - int data_window[4]; - int display_window[4]; + EXRBox2i data_window; + EXRBox2i display_window; float screen_window_center[2]; float screen_window_width; @@ -287,26 +293,29 @@ typedef struct _DeepImage { extern int LoadEXR(float **out_rgba, int *width, int *height, const char *filename, const char **err); -// Loads single-frame OpenEXR image by specifing layer name. Assume EXR image contains A(single channel -// alpha) or RGB(A) channels. -// Application must free image data as returned by `out_rgba` -// Result image format is: float x RGBA x width x hight -// Returns negative value and may set error string in `err` when there's an -// error -// When the specified layer name is not found in the EXR file, the function will return `TINYEXR_ERROR_LAYER_NOT_FOUND`. +// Loads single-frame OpenEXR image by specifying layer name. Assume EXR image +// contains A(single channel alpha) or RGB(A) channels. Application must free +// image data as returned by `out_rgba` Result image format is: float x RGBA x +// width x hight Returns negative value and may set error string in `err` when +// there's an error When the specified layer name is not found in the EXR file, +// the function will return `TINYEXR_ERROR_LAYER_NOT_FOUND`. extern int LoadEXRWithLayer(float **out_rgba, int *width, int *height, - const char *filename, const char *layer_name, const char **err); + const char *filename, const char *layer_name, + const char **err); // // Get layer infos from EXR file. // -// @param[out] layer_names List of layer names. Application must free memory after using this. +// @param[out] layer_names List of layer names. Application must free memory +// after using this. // @param[out] num_layers The number of layers -// @param[out] err Error string(wll be filled when the function returns error code). Free it using FreeEXRErrorMessage after using this value. +// @param[out] err Error string(will be filled when the function returns error +// code). Free it using FreeEXRErrorMessage after using this value. // // @return TINYEXR_SUCCEES upon success. // -extern int EXRLayers(const char *filename, const char **layer_names[], int *num_layers, const char **err); +extern int EXRLayers(const char *filename, const char **layer_names[], + int *num_layers, const char **err); // @deprecated { to be removed. } // Simple wrapper API for ParseEXRHeaderFromFile. @@ -336,13 +345,13 @@ extern void InitEXRHeader(EXRHeader *exr_header); // Initialize EXRImage struct extern void InitEXRImage(EXRImage *exr_image); -// Free's internal data of EXRHeader struct +// Frees internal data of EXRHeader struct extern int FreeEXRHeader(EXRHeader *exr_header); -// Free's internal data of EXRImage struct +// Frees internal data of EXRImage struct extern int FreeEXRImage(EXRImage *exr_image); -// Free's error message +// Frees error message extern void FreeEXRErrorMessage(const char *msg); // Parse EXR version header of a file. @@ -497,8 +506,17 @@ extern int LoadEXRFromMemory(float **out_rgba, int *width, int *height, #endif // TINYEXR_H_ #ifdef TINYEXR_IMPLEMENTATION -#ifndef TINYEXR_IMPLEMENTATION_DEIFNED -#define TINYEXR_IMPLEMENTATION_DEIFNED +#ifndef TINYEXR_IMPLEMENTATION_DEFINED +#define TINYEXR_IMPLEMENTATION_DEFINED + +#ifdef _WIN32 + +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#include <windows.h> // for UTF-8 + +#endif #include <algorithm> #include <cassert> @@ -536,7 +554,18 @@ extern int LoadEXRFromMemory(float **out_rgba, int *width, int *height, #endif #if TINYEXR_USE_ZFP + +#ifdef __clang__ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Weverything" +#endif + #include "zfp.h" + +#ifdef __clang__ +#pragma clang diagnostic pop +#endif + #endif namespace tinyexr { @@ -619,7 +648,7 @@ namespace miniz { - Critical fix for the MZ_ZIP_FLAG_DO_NOT_SORT_CENTRAL_DIRECTORY bug (thanks kahmyong.moon@hp.com) which could cause locate files to not find files. This bug - would only have occured in earlier versions if you explicitly used this + would only have occurred in earlier versions if you explicitly used this flag, OR if you used mz_zip_extract_archive_file_to_heap() or mz_zip_add_mem_to_archive_file_in_place() (which used this flag). If you can't switch to v1.15 but want to fix @@ -7002,6 +7031,13 @@ void *mz_zip_extract_archive_file_to_heap(const char *pZip_filename, // Reuse MINIZ_LITTE_ENDIAN macro +#if defined(_M_IX86) || defined(_M_X64) || defined(__i386__) || \ + defined(__i386) || defined(__i486__) || defined(__i486) || \ + defined(i386) || defined(__ia64__) || defined(__x86_64__) +// MINIZ_X86_OR_X64_CPU is only used to help set the below macros. +#define MINIZ_X86_OR_X64_CPU 1 +#endif + #if defined(__sparcv9) // Big endian #else @@ -7116,6 +7152,36 @@ static void swap4(unsigned int *val) { #endif } +static void swap4(int *val) { +#ifdef MINIZ_LITTLE_ENDIAN + (void)val; +#else + int tmp = *val; + unsigned char *dst = reinterpret_cast<unsigned char *>(val); + unsigned char *src = reinterpret_cast<unsigned char *>(&tmp); + + dst[0] = src[3]; + dst[1] = src[2]; + dst[2] = src[1]; + dst[3] = src[0]; +#endif +} + +static void swap4(float *val) { +#ifdef MINIZ_LITTLE_ENDIAN + (void)val; +#else + float tmp = *val; + unsigned char *dst = reinterpret_cast<unsigned char *>(val); + unsigned char *src = reinterpret_cast<unsigned char *>(&tmp); + + dst[0] = src[3]; + dst[1] = src[2]; + dst[2] = src[1]; + dst[3] = src[0]; +#endif +} + #if 0 static void cpy8(tinyexr::tinyexr_uint64 *dst_val, const tinyexr::tinyexr_uint64 *src_val) { unsigned char *dst = reinterpret_cast<unsigned char *>(dst_val); @@ -7363,7 +7429,7 @@ static void WriteAttributeToMemory(std::vector<unsigned char> *out, out->insert(out->end(), type, type + strlen(type) + 1); int outLen = len; - tinyexr::swap4(reinterpret_cast<unsigned int *>(&outLen)); + tinyexr::swap4(&outLen); out->insert(out->end(), reinterpret_cast<unsigned char *>(&outLen), reinterpret_cast<unsigned char *>(&outLen) + sizeof(int)); out->insert(out->end(), data, data + len); @@ -7379,12 +7445,19 @@ typedef struct { } ChannelInfo; typedef struct { + int min_x; + int min_y; + int max_x; + int max_y; +} Box2iInfo; + +struct HeaderInfo { std::vector<tinyexr::ChannelInfo> channels; std::vector<EXRAttribute> attributes; - int data_window[4]; + Box2iInfo data_window; int line_order; - int display_window[4]; + Box2iInfo display_window; float screen_window_center[2]; float screen_window_width; float pixel_aspect_ratio; @@ -7405,15 +7478,15 @@ typedef struct { channels.clear(); attributes.clear(); - data_window[0] = 0; - data_window[1] = 0; - data_window[2] = 0; - data_window[3] = 0; + data_window.min_x = 0; + data_window.min_y = 0; + data_window.max_x = 0; + data_window.max_y = 0; line_order = 0; - display_window[0] = 0; - display_window[1] = 0; - display_window[2] = 0; - display_window[3] = 0; + display_window.min_x = 0; + display_window.min_y = 0; + display_window.max_x = 0; + display_window.max_y = 0; screen_window_center[0] = 0.0f; screen_window_center[1] = 0.0f; screen_window_width = 0.0f; @@ -7430,7 +7503,7 @@ typedef struct { header_len = 0; compression_type = 0; } -} HeaderInfo; +}; static bool ReadChannelInfo(std::vector<ChannelInfo> &channels, const std::vector<unsigned char> &data) { @@ -7469,9 +7542,9 @@ static bool ReadChannelInfo(std::vector<ChannelInfo> &channels, memcpy(&info.y_sampling, p, sizeof(int)); // int p += 4; - tinyexr::swap4(reinterpret_cast<unsigned int *>(&info.pixel_type)); - tinyexr::swap4(reinterpret_cast<unsigned int *>(&info.x_sampling)); - tinyexr::swap4(reinterpret_cast<unsigned int *>(&info.y_sampling)); + tinyexr::swap4(&info.pixel_type); + tinyexr::swap4(&info.x_sampling); + tinyexr::swap4(&info.y_sampling); channels.push_back(info); } @@ -7501,9 +7574,9 @@ static void WriteChannelInfo(std::vector<unsigned char> &data, int pixel_type = channels[c].pixel_type; int x_sampling = channels[c].x_sampling; int y_sampling = channels[c].y_sampling; - tinyexr::swap4(reinterpret_cast<unsigned int *>(&pixel_type)); - tinyexr::swap4(reinterpret_cast<unsigned int *>(&x_sampling)); - tinyexr::swap4(reinterpret_cast<unsigned int *>(&y_sampling)); + tinyexr::swap4(&pixel_type); + tinyexr::swap4(&x_sampling); + tinyexr::swap4(&y_sampling); memcpy(p, &pixel_type, sizeof(int)); p += sizeof(int); @@ -7712,7 +7785,7 @@ static int rleCompress(int inLength, const char in[], signed char out[]) { if (runEnd - runStart >= MIN_RUN_LENGTH) { // - // Compressable run + // Compressible run // *outWrite++ = static_cast<char>(runEnd - runStart) - 1; @@ -8056,7 +8129,7 @@ static void wav2Encode( int p2 = 2; // == 1 << (level+1) // - // Hierachical loop on smaller dimension n + // Hierarchical loop on smaller dimension n // while (p2 <= n) { @@ -8287,9 +8360,9 @@ const int HUF_DECMASK = HUF_DECSIZE - 1; struct HufDec { // short code long code //------------------------------- - int len : 8; // code length 0 - int lit : 24; // lit p size - int *p; // 0 lits + unsigned int len : 8; // code length 0 + unsigned int lit : 24; // lit p size + unsigned int *p; // 0 lits }; inline long long hufLength(long long code) { return code & 63; } @@ -8745,14 +8818,14 @@ static bool hufBuildDecTable(const long long *hcode, // i : encoding table pl->lit++; if (pl->p) { - int *p = pl->p; - pl->p = new int[pl->lit]; + unsigned int *p = pl->p; + pl->p = new unsigned int[pl->lit]; for (int i = 0; i < pl->lit - 1; ++i) pl->p[i] = p[i]; delete[] p; } else { - pl->p = new int[1]; + pl->p = new unsigned int[1]; } pl->p[pl->lit - 1] = im; @@ -9491,35 +9564,48 @@ static bool DecompressPiz(unsigned char *outPtr, const unsigned char *inPtr, #endif // TINYEXR_USE_PIZ #if TINYEXR_USE_ZFP + struct ZFPCompressionParam { double rate; - int precision; + unsigned int precision; + unsigned int __pad0; double tolerance; int type; // TINYEXR_ZFP_COMPRESSIONTYPE_* + unsigned int __pad1; ZFPCompressionParam() { type = TINYEXR_ZFP_COMPRESSIONTYPE_RATE; rate = 2.0; precision = 0; - tolerance = 0.0f; + tolerance = 0.0; } }; -bool FindZFPCompressionParam(ZFPCompressionParam *param, - const EXRAttribute *attributes, - int num_attributes) { +static bool FindZFPCompressionParam(ZFPCompressionParam *param, + const EXRAttribute *attributes, + int num_attributes, std::string *err) { bool foundType = false; for (int i = 0; i < num_attributes; i++) { - if ((strcmp(attributes[i].name, "zfpCompressionType") == 0) && - (attributes[i].size == 1)) { - param->type = static_cast<int>(attributes[i].value[0]); - - foundType = true; + if ((strcmp(attributes[i].name, "zfpCompressionType") == 0)) { + if (attributes[i].size == 1) { + param->type = static_cast<int>(attributes[i].value[0]); + foundType = true; + break; + } else { + if (err) { + (*err) += + "zfpCompressionType attribute must be uchar(1 byte) type.\n"; + } + return false; + } } } if (!foundType) { + if (err) { + (*err) += "`zfpCompressionType` attribute not found.\n"; + } return false; } @@ -9531,6 +9617,11 @@ bool FindZFPCompressionParam(ZFPCompressionParam *param, return true; } } + + if (err) { + (*err) += "`zfpCompressionRate` attribute not found.\n"; + } + } else if (param->type == TINYEXR_ZFP_COMPRESSIONTYPE_PRECISION) { for (int i = 0; i < num_attributes; i++) { if ((strcmp(attributes[i].name, "zfpCompressionPrecision") == 0) && @@ -9539,6 +9630,11 @@ bool FindZFPCompressionParam(ZFPCompressionParam *param, return true; } } + + if (err) { + (*err) += "`zfpCompressionPrecision` attribute not found.\n"; + } + } else if (param->type == TINYEXR_ZFP_COMPRESSIONTYPE_ACCURACY) { for (int i = 0; i < num_attributes; i++) { if ((strcmp(attributes[i].name, "zfpCompressionTolerance") == 0) && @@ -9547,8 +9643,14 @@ bool FindZFPCompressionParam(ZFPCompressionParam *param, return true; } } + + if (err) { + (*err) += "`zfpCompressionTolerance` attribute not found.\n"; + } } else { - assert(0); + if (err) { + (*err) += "Unknown value specified for `zfpCompressionType`.\n"; + } } return false; @@ -9556,10 +9658,11 @@ bool FindZFPCompressionParam(ZFPCompressionParam *param, // Assume pixel format is FLOAT for all channels. static bool DecompressZfp(float *dst, int dst_width, int dst_num_lines, - int num_channels, const unsigned char *src, + size_t num_channels, const unsigned char *src, unsigned long src_size, const ZFPCompressionParam ¶m) { - size_t uncompressed_size = dst_width * dst_num_lines * num_channels; + size_t uncompressed_size = + size_t(dst_width) * size_t(dst_num_lines) * num_channels; if (uncompressed_size == src_size) { // Data is not compressed(Issue 40). @@ -9572,22 +9675,24 @@ static bool DecompressZfp(float *dst, int dst_width, int dst_num_lines, assert((dst_width % 4) == 0); assert((dst_num_lines % 4) == 0); - if ((dst_width & 3U) || (dst_num_lines & 3U)) { + if ((size_t(dst_width) & 3U) || (size_t(dst_num_lines) & 3U)) { return false; } field = zfp_field_2d(reinterpret_cast<void *>(const_cast<unsigned char *>(src)), - zfp_type_float, dst_width, dst_num_lines * num_channels); + zfp_type_float, static_cast<unsigned int>(dst_width), + static_cast<unsigned int>(dst_num_lines) * + static_cast<unsigned int>(num_channels)); zfp = zfp_stream_open(NULL); if (param.type == TINYEXR_ZFP_COMPRESSIONTYPE_RATE) { - zfp_stream_set_rate(zfp, param.rate, zfp_type_float, /* dimention */ 2, + zfp_stream_set_rate(zfp, param.rate, zfp_type_float, /* dimension */ 2, /* write random access */ 0); } else if (param.type == TINYEXR_ZFP_COMPRESSIONTYPE_PRECISION) { - zfp_stream_set_precision(zfp, param.precision, zfp_type_float); + zfp_stream_set_precision(zfp, param.precision); } else if (param.type == TINYEXR_ZFP_COMPRESSIONTYPE_ACCURACY) { - zfp_stream_set_accuracy(zfp, param.tolerance, zfp_type_float); + zfp_stream_set_accuracy(zfp, param.tolerance); } else { assert(0); } @@ -9600,17 +9705,17 @@ static bool DecompressZfp(float *dst, int dst_width, int dst_num_lines, zfp_stream_set_bit_stream(zfp, stream); zfp_stream_rewind(zfp); - size_t image_size = dst_width * dst_num_lines; + size_t image_size = size_t(dst_width) * size_t(dst_num_lines); - for (int c = 0; c < num_channels; c++) { + for (size_t c = 0; c < size_t(num_channels); c++) { // decompress 4x4 pixel block. - for (int y = 0; y < dst_num_lines; y += 4) { - for (int x = 0; x < dst_width; x += 4) { + for (size_t y = 0; y < size_t(dst_num_lines); y += 4) { + for (size_t x = 0; x < size_t(dst_width); x += 4) { float fblock[16]; zfp_decode_block_float_2(zfp, fblock); - for (int j = 0; j < 4; j++) { - for (int i = 0; i < 4; i++) { - dst[c * image_size + ((y + j) * dst_width + (x + i))] = + for (size_t j = 0; j < 4; j++) { + for (size_t i = 0; i < 4; i++) { + dst[c * image_size + ((y + j) * size_t(dst_width) + (x + i))] = fblock[j * 4 + i]; } } @@ -9626,31 +9731,33 @@ static bool DecompressZfp(float *dst, int dst_width, int dst_num_lines, } // Assume pixel format is FLOAT for all channels. -bool CompressZfp(std::vector<unsigned char> *outBuf, unsigned int *outSize, - const float *inPtr, int width, int num_lines, int num_channels, - const ZFPCompressionParam ¶m) { +static bool CompressZfp(std::vector<unsigned char> *outBuf, + unsigned int *outSize, const float *inPtr, int width, + int num_lines, int num_channels, + const ZFPCompressionParam ¶m) { zfp_stream *zfp = NULL; zfp_field *field = NULL; assert((width % 4) == 0); assert((num_lines % 4) == 0); - if ((width & 3U) || (num_lines & 3U)) { + if ((size_t(width) & 3U) || (size_t(num_lines) & 3U)) { return false; } // create input array. field = zfp_field_2d(reinterpret_cast<void *>(const_cast<float *>(inPtr)), - zfp_type_float, width, num_lines * num_channels); + zfp_type_float, static_cast<unsigned int>(width), + static_cast<unsigned int>(num_lines * num_channels)); zfp = zfp_stream_open(NULL); if (param.type == TINYEXR_ZFP_COMPRESSIONTYPE_RATE) { zfp_stream_set_rate(zfp, param.rate, zfp_type_float, 2, 0); } else if (param.type == TINYEXR_ZFP_COMPRESSIONTYPE_PRECISION) { - zfp_stream_set_precision(zfp, param.precision, zfp_type_float); + zfp_stream_set_precision(zfp, param.precision); } else if (param.type == TINYEXR_ZFP_COMPRESSIONTYPE_ACCURACY) { - zfp_stream_set_accuracy(zfp, param.tolerance, zfp_type_float); + zfp_stream_set_accuracy(zfp, param.tolerance); } else { assert(0); } @@ -9663,17 +9770,17 @@ bool CompressZfp(std::vector<unsigned char> *outBuf, unsigned int *outSize, zfp_stream_set_bit_stream(zfp, stream); zfp_field_free(field); - size_t image_size = width * num_lines; + size_t image_size = size_t(width) * size_t(num_lines); - for (int c = 0; c < num_channels; c++) { + for (size_t c = 0; c < size_t(num_channels); c++) { // compress 4x4 pixel block. - for (int y = 0; y < num_lines; y += 4) { - for (int x = 0; x < width; x += 4) { + for (size_t y = 0; y < size_t(num_lines); y += 4) { + for (size_t x = 0; x < size_t(width); x += 4) { float fblock[16]; - for (int j = 0; j < 4; j++) { - for (int i = 0; i < 4; i++) { + for (size_t j = 0; j < 4; j++) { + for (size_t i = 0; i < 4; i++) { fblock[j * 4 + i] = - inPtr[c * image_size + ((y + j) * width + (x + i))]; + inPtr[c * image_size + ((y + j) * size_t(width) + (x + i))]; } } zfp_encode_block_float_2(zfp, fblock); @@ -9682,7 +9789,7 @@ bool CompressZfp(std::vector<unsigned char> *outBuf, unsigned int *outSize, } zfp_stream_flush(zfp); - (*outSize) = zfp_stream_compressed_size(zfp); + (*outSize) = static_cast<unsigned int>(zfp_stream_compressed_size(zfp)); zfp_stream_close(zfp); @@ -10122,8 +10229,10 @@ static bool DecodePixelData(/* out */ unsigned char **out_images, } else if (compression_type == TINYEXR_COMPRESSIONTYPE_ZFP) { #if TINYEXR_USE_ZFP tinyexr::ZFPCompressionParam zfp_compression_param; - if (!FindZFPCompressionParam(&zfp_compression_param, attributes, - num_attributes)) { + std::string e; + if (!tinyexr::FindZFPCompressionParam(&zfp_compression_param, attributes, + int(num_attributes), &e)) { + // This code path should not be reachable. assert(0); return false; } @@ -10323,8 +10432,11 @@ static bool DecodeTiledPixelData( const EXRAttribute *attributes, size_t num_channels, const EXRChannelInfo *channels, const std::vector<size_t> &channel_offset_list) { - assert(tile_offset_x * tile_size_x < data_width); - assert(tile_offset_y * tile_size_y < data_height); + if (tile_size_x > data_width || tile_size_y > data_height || + tile_size_x * tile_offset_x > data_width || + tile_size_y * tile_offset_y > data_height) { + return false; + } // Compute actual image size in a tile. if ((tile_offset_x + 1) * tile_size_x >= data_width) { @@ -10418,6 +10530,17 @@ static unsigned char **AllocateImage(int num_channels, return images; } +#ifdef _WIN32 +static inline std::wstring UTF8ToWchar(const std::string &str) { + int wstr_size = + MultiByteToWideChar(CP_UTF8, 0, str.data(), (int)str.size(), NULL, 0); + std::wstring wstr(wstr_size, 0); + MultiByteToWideChar(CP_UTF8, 0, str.data(), (int)str.size(), &wstr[0], + (int)wstr.size()); + return wstr; +} +#endif + static int ParseEXRHeader(HeaderInfo *info, bool *empty_header, const EXRVersion *version, std::string *err, const unsigned char *buf, size_t size) { @@ -10457,15 +10580,15 @@ static int ParseEXRHeader(HeaderInfo *info, bool *empty_header, bool has_screen_window_center = false; bool has_screen_window_width = false; - info->data_window[0] = 0; - info->data_window[1] = 0; - info->data_window[2] = 0; - info->data_window[3] = 0; + info->data_window.min_x = 0; + info->data_window.min_y = 0; + info->data_window.max_x = 0; + info->data_window.max_y = 0; info->line_order = 0; // @fixme - info->display_window[0] = 0; - info->display_window[1] = 0; - info->display_window[2] = 0; - info->display_window[3] = 0; + info->display_window.min_x = 0; + info->display_window.min_y = 0; + info->display_window.max_x = 0; + info->display_window.max_y = 0; info->screen_window_center[0] = 0.0f; info->screen_window_center[1] = 0.0f; info->screen_window_width = -1.0f; @@ -10515,6 +10638,14 @@ static int ParseEXRHeader(HeaderInfo *info, bool *empty_header, tinyexr::swap4(&x_size); tinyexr::swap4(&y_size); + if (x_size > static_cast<unsigned int>(std::numeric_limits<int>::max()) || + y_size > static_cast<unsigned int>(std::numeric_limits<int>::max())) { + if (err) { + (*err) = "Tile sizes were invalid."; + } + return TINYEXR_ERROR_UNSUPPORTED_FORMAT; + } + info->tile_size_x = static_cast<int>(x_size); info->tile_size_y = static_cast<int>(y_size); @@ -10586,30 +10717,26 @@ static int ParseEXRHeader(HeaderInfo *info, bool *empty_header, } else if (attr_name.compare("dataWindow") == 0) { if (data.size() >= 16) { - memcpy(&info->data_window[0], &data.at(0), sizeof(int)); - memcpy(&info->data_window[1], &data.at(4), sizeof(int)); - memcpy(&info->data_window[2], &data.at(8), sizeof(int)); - memcpy(&info->data_window[3], &data.at(12), sizeof(int)); - tinyexr::swap4(reinterpret_cast<unsigned int *>(&info->data_window[0])); - tinyexr::swap4(reinterpret_cast<unsigned int *>(&info->data_window[1])); - tinyexr::swap4(reinterpret_cast<unsigned int *>(&info->data_window[2])); - tinyexr::swap4(reinterpret_cast<unsigned int *>(&info->data_window[3])); + memcpy(&info->data_window.min_x, &data.at(0), sizeof(int)); + memcpy(&info->data_window.min_y, &data.at(4), sizeof(int)); + memcpy(&info->data_window.max_x, &data.at(8), sizeof(int)); + memcpy(&info->data_window.max_y, &data.at(12), sizeof(int)); + tinyexr::swap4(&info->data_window.min_x); + tinyexr::swap4(&info->data_window.min_y); + tinyexr::swap4(&info->data_window.max_x); + tinyexr::swap4(&info->data_window.max_y); has_data_window = true; } } else if (attr_name.compare("displayWindow") == 0) { if (data.size() >= 16) { - memcpy(&info->display_window[0], &data.at(0), sizeof(int)); - memcpy(&info->display_window[1], &data.at(4), sizeof(int)); - memcpy(&info->display_window[2], &data.at(8), sizeof(int)); - memcpy(&info->display_window[3], &data.at(12), sizeof(int)); - tinyexr::swap4( - reinterpret_cast<unsigned int *>(&info->display_window[0])); - tinyexr::swap4( - reinterpret_cast<unsigned int *>(&info->display_window[1])); - tinyexr::swap4( - reinterpret_cast<unsigned int *>(&info->display_window[2])); - tinyexr::swap4( - reinterpret_cast<unsigned int *>(&info->display_window[3])); + memcpy(&info->display_window.min_x, &data.at(0), sizeof(int)); + memcpy(&info->display_window.min_y, &data.at(4), sizeof(int)); + memcpy(&info->display_window.max_x, &data.at(8), sizeof(int)); + memcpy(&info->display_window.max_y, &data.at(12), sizeof(int)); + tinyexr::swap4(&info->display_window.min_x); + tinyexr::swap4(&info->display_window.min_y); + tinyexr::swap4(&info->display_window.max_x); + tinyexr::swap4(&info->display_window.max_y); has_display_window = true; } @@ -10621,32 +10748,28 @@ static int ParseEXRHeader(HeaderInfo *info, bool *empty_header, } else if (attr_name.compare("pixelAspectRatio") == 0) { if (data.size() >= sizeof(float)) { memcpy(&info->pixel_aspect_ratio, &data.at(0), sizeof(float)); - tinyexr::swap4( - reinterpret_cast<unsigned int *>(&info->pixel_aspect_ratio)); + tinyexr::swap4(&info->pixel_aspect_ratio); has_pixel_aspect_ratio = true; } } else if (attr_name.compare("screenWindowCenter") == 0) { if (data.size() >= 8) { memcpy(&info->screen_window_center[0], &data.at(0), sizeof(float)); memcpy(&info->screen_window_center[1], &data.at(4), sizeof(float)); - tinyexr::swap4( - reinterpret_cast<unsigned int *>(&info->screen_window_center[0])); - tinyexr::swap4( - reinterpret_cast<unsigned int *>(&info->screen_window_center[1])); + tinyexr::swap4(&info->screen_window_center[0]); + tinyexr::swap4(&info->screen_window_center[1]); has_screen_window_center = true; } } else if (attr_name.compare("screenWindowWidth") == 0) { if (data.size() >= sizeof(float)) { memcpy(&info->screen_window_width, &data.at(0), sizeof(float)); - tinyexr::swap4( - reinterpret_cast<unsigned int *>(&info->screen_window_width)); + tinyexr::swap4(&info->screen_window_width); has_screen_window_width = true; } } else if (attr_name.compare("chunkCount") == 0) { if (data.size() >= sizeof(int)) { memcpy(&info->chunk_count, &data.at(0), sizeof(int)); - tinyexr::swap4(reinterpret_cast<unsigned int *>(&info->chunk_count)); + tinyexr::swap4(&info->chunk_count); } } else { // Custom attribute(up to TINYEXR_MAX_CUSTOM_ATTRIBUTES) @@ -10732,14 +10855,14 @@ static void ConvertHeader(EXRHeader *exr_header, const HeaderInfo &info) { exr_header->screen_window_center[1] = info.screen_window_center[1]; exr_header->screen_window_width = info.screen_window_width; exr_header->chunk_count = info.chunk_count; - exr_header->display_window[0] = info.display_window[0]; - exr_header->display_window[1] = info.display_window[1]; - exr_header->display_window[2] = info.display_window[2]; - exr_header->display_window[3] = info.display_window[3]; - exr_header->data_window[0] = info.data_window[0]; - exr_header->data_window[1] = info.data_window[1]; - exr_header->data_window[2] = info.data_window[2]; - exr_header->data_window[3] = info.data_window[3]; + exr_header->display_window.min_x = info.display_window.min_x; + exr_header->display_window.min_y = info.display_window.min_y; + exr_header->display_window.max_x = info.display_window.max_x; + exr_header->display_window.max_y = info.display_window.max_y; + exr_header->data_window.min_x = info.data_window.min_x; + exr_header->data_window.min_y = info.data_window.min_y; + exr_header->data_window.max_x = info.data_window.max_x; + exr_header->data_window.max_y = info.data_window.max_y; exr_header->line_order = info.line_order; exr_header->compression_type = info.compression_type; @@ -10798,7 +10921,7 @@ static void ConvertHeader(EXRHeader *exr_header, const HeaderInfo &info) { memcpy(exr_header->custom_attributes[i].type, info.attributes[i].type, 256); exr_header->custom_attributes[i].size = info.attributes[i].size; - // Just copy poiner + // Just copy pointer exr_header->custom_attributes[i].value = info.attributes[i].value; } @@ -10822,21 +10945,30 @@ static int DecodeChunk(EXRImage *exr_image, const EXRHeader *exr_header, num_scanline_blocks = 32; } else if (exr_header->compression_type == TINYEXR_COMPRESSIONTYPE_ZFP) { num_scanline_blocks = 16; - } - int data_width = exr_header->data_window[2] - exr_header->data_window[0] + 1; - int data_height = exr_header->data_window[3] - exr_header->data_window[1] + 1; +#if TINYEXR_USE_ZFP + tinyexr::ZFPCompressionParam zfp_compression_param; + if (!FindZFPCompressionParam(&zfp_compression_param, + exr_header->custom_attributes, + int(exr_header->num_custom_attributes), err)) { + return TINYEXR_ERROR_INVALID_HEADER; + } +#endif + } - if ((data_width < 0) || (data_height < 0)) { + if (exr_header->data_window.max_x < exr_header->data_window.min_x || + exr_header->data_window.max_y < exr_header->data_window.min_y) { if (err) { - std::stringstream ss; - ss << "Invalid data width or data height: " << data_width << ", " - << data_height << std::endl; - (*err) += ss.str(); + (*err) += "Invalid data window.\n"; } return TINYEXR_ERROR_INVALID_DATA; } + int data_width = + exr_header->data_window.max_x - exr_header->data_window.min_x + 1; + int data_height = + exr_header->data_window.max_y - exr_header->data_window.min_y + 1; + // Do not allow too large data_width and data_height. header invalid? { const int threshold = 1024 * 8192; // heuristics @@ -10938,14 +11070,10 @@ static int DecodeChunk(EXRImage *exr_image, const EXRHeader *exr_header, int tile_coordinates[4]; memcpy(tile_coordinates, data_ptr, sizeof(int) * 4); - tinyexr::swap4( - reinterpret_cast<unsigned int *>(&tile_coordinates[0])); - tinyexr::swap4( - reinterpret_cast<unsigned int *>(&tile_coordinates[1])); - tinyexr::swap4( - reinterpret_cast<unsigned int *>(&tile_coordinates[2])); - tinyexr::swap4( - reinterpret_cast<unsigned int *>(&tile_coordinates[3])); + tinyexr::swap4(&tile_coordinates[0]); + tinyexr::swap4(&tile_coordinates[1]); + tinyexr::swap4(&tile_coordinates[2]); + tinyexr::swap4(&tile_coordinates[3]); // @todo{ LoD } if (tile_coordinates[2] != 0) { @@ -10960,7 +11088,7 @@ static int DecodeChunk(EXRImage *exr_image, const EXRHeader *exr_header, int data_len; memcpy(&data_len, data_ptr + 16, sizeof(int)); // 16 = sizeof(tile_coordinates) - tinyexr::swap4(reinterpret_cast<unsigned int *>(&data_len)); + tinyexr::swap4(&data_len); if (data_len < 4 || size_t(data_len) > data_size) { // TODO(LTE): atomic @@ -11081,8 +11209,8 @@ static int DecodeChunk(EXRImage *exr_image, const EXRHeader *exr_header, memcpy(&line_no, data_ptr, sizeof(int)); int data_len; memcpy(&data_len, data_ptr + 4, sizeof(int)); - tinyexr::swap4(reinterpret_cast<unsigned int *>(&line_no)); - tinyexr::swap4(reinterpret_cast<unsigned int *>(&data_len)); + tinyexr::swap4(&line_no); + tinyexr::swap4(&data_len); if (size_t(data_len) > data_size) { invalid_data = true; @@ -11098,7 +11226,7 @@ static int DecodeChunk(EXRImage *exr_image, const EXRHeader *exr_header, } else { // line_no may be negative. int end_line_no = (std::min)(line_no + num_scanline_blocks, - (exr_header->data_window[3] + 1)); + (exr_header->data_window.max_y + 1)); int num_lines = end_line_no - line_no; @@ -11113,13 +11241,13 @@ static int DecodeChunk(EXRImage *exr_image, const EXRHeader *exr_header, // overflow check tinyexr_int64 lno = static_cast<tinyexr_int64>(line_no) - - static_cast<tinyexr_int64>(exr_header->data_window[1]); + static_cast<tinyexr_int64>(exr_header->data_window.min_y); if (lno > std::numeric_limits<int>::max()) { line_no = -1; // invalid } else if (lno < -std::numeric_limits<int>::max()) { line_no = -1; // invalid } else { - line_no -= exr_header->data_window[1]; + line_no -= exr_header->data_window.min_y; } if (line_no < 0) { @@ -11204,8 +11332,8 @@ static bool ReconstructLineOffsets( return false; } - tinyexr::swap4(reinterpret_cast<unsigned int *>(&y)); - tinyexr::swap4(reinterpret_cast<unsigned int *>(&data_len)); + tinyexr::swap4(&y); + tinyexr::swap4(&data_len); (*offsets)[i] = offset; @@ -11234,25 +11362,24 @@ static int DecodeEXRImage(EXRImage *exr_image, const EXRHeader *exr_header, num_scanline_blocks = 16; } - int data_width = exr_header->data_window[2] - exr_header->data_window[0]; - if (data_width >= std::numeric_limits<int>::max()) { + if (exr_header->data_window.max_x < exr_header->data_window.min_x || + exr_header->data_window.max_x - exr_header->data_window.min_x == + std::numeric_limits<int>::max()) { // Issue 63 tinyexr::SetErrorMessage("Invalid data width value", err); return TINYEXR_ERROR_INVALID_DATA; } - data_width++; + int data_width = + exr_header->data_window.max_x - exr_header->data_window.min_x + 1; - int data_height = exr_header->data_window[3] - exr_header->data_window[1]; - if (data_height >= std::numeric_limits<int>::max()) { + if (exr_header->data_window.max_y < exr_header->data_window.min_y || + exr_header->data_window.max_y - exr_header->data_window.min_y == + std::numeric_limits<int>::max()) { tinyexr::SetErrorMessage("Invalid data height value", err); return TINYEXR_ERROR_INVALID_DATA; } - data_height++; - - if ((data_width < 0) || (data_height < 0)) { - tinyexr::SetErrorMessage("data width or data height is negative.", err); - return TINYEXR_ERROR_INVALID_DATA; - } + int data_height = + exr_header->data_window.max_y - exr_header->data_window.min_y + 1; // Do not allow too large data_width and data_height. header invalid? { @@ -11275,6 +11402,12 @@ static int DecodeEXRImage(EXRImage *exr_image, const EXRHeader *exr_header, num_blocks = static_cast<size_t>(exr_header->chunk_count); } else if (exr_header->tiled) { // @todo { LoD } + if (exr_header->tile_size_x > data_width || exr_header->tile_size_x < 1 || + exr_header->tile_size_y > data_height || exr_header->tile_size_y < 1) { + tinyexr::SetErrorMessage("tile sizes are invalid.", err); + return TINYEXR_ERROR_INVALID_DATA; + } + size_t num_x_tiles = static_cast<size_t>(data_width) / static_cast<size_t>(exr_header->tile_size_x); if (num_x_tiles * static_cast<size_t>(exr_header->tile_size_x) < @@ -11371,7 +11504,8 @@ static int DecodeEXRImage(EXRImage *exr_image, const EXRHeader *exr_header, } } -static void GetLayers(const EXRHeader& exr_header, std::vector<std::string>& layer_names) { +static void GetLayers(const EXRHeader &exr_header, + std::vector<std::string> &layer_names) { // Naive implementation // Group channels by layers // go over all channel names, split by periods @@ -11382,22 +11516,22 @@ static void GetLayers(const EXRHeader& exr_header, std::vector<std::string>& lay const size_t pos = full_name.find_last_of('.'); if (pos != std::string::npos && pos != 0 && pos + 1 < full_name.size()) { full_name.erase(pos); - if (std::find(layer_names.begin(), layer_names.end(), full_name) == layer_names.end()) + if (std::find(layer_names.begin(), layer_names.end(), full_name) == + layer_names.end()) layer_names.push_back(full_name); } } } struct LayerChannel { - explicit LayerChannel (size_t i, std::string n) - : index(i) - , name(n) - {} + explicit LayerChannel(size_t i, std::string n) : index(i), name(n) {} size_t index; std::string name; }; -static void ChannelsInLayer(const EXRHeader& exr_header, const std::string layer_name, std::vector<LayerChannel>& channels) { +static void ChannelsInLayer(const EXRHeader &exr_header, + const std::string layer_name, + std::vector<LayerChannel> &channels) { channels.clear(); for (int c = 0; c < exr_header.num_channels; c++) { std::string ch_name(exr_header.channels[c].name); @@ -11408,8 +11542,7 @@ static void ChannelsInLayer(const EXRHeader& exr_header, const std::string layer } } else { const size_t pos = ch_name.find(layer_name + '.'); - if (pos == std::string::npos) - continue; + if (pos == std::string::npos) continue; if (pos == 0) { ch_name = ch_name.substr(layer_name.size() + 1); } @@ -11421,7 +11554,8 @@ static void ChannelsInLayer(const EXRHeader& exr_header, const std::string layer } // namespace tinyexr -int EXRLayers(const char *filename, const char **layer_names[], int *num_layers, const char **err) { +int EXRLayers(const char *filename, const char **layer_names[], int *num_layers, + const char **err) { EXRVersion exr_version; EXRHeader exr_header; InitEXRHeader(&exr_header); @@ -11435,8 +11569,8 @@ int EXRLayers(const char *filename, const char **layer_names[], int *num_layers, if (exr_version.multipart || exr_version.non_image) { tinyexr::SetErrorMessage( - "Loading multipart or DeepImage is not supported in LoadEXR() API", - err); + "Loading multipart or DeepImage is not supported in LoadEXR() API", + err); return TINYEXR_ERROR_INVALID_DATA; // @fixme. } } @@ -11452,7 +11586,7 @@ int EXRLayers(const char *filename, const char **layer_names[], int *num_layers, (*num_layers) = int(layer_vec.size()); (*layer_names) = static_cast<const char **>( - malloc(sizeof(const char *) * static_cast<size_t>(layer_vec.size()))); + malloc(sizeof(const char *) * static_cast<size_t>(layer_vec.size()))); for (size_t c = 0; c < static_cast<size_t>(layer_vec.size()); c++) { #ifdef _MSC_VER (*layer_names)[c] = _strdup(layer_vec[c].c_str()); @@ -11467,11 +11601,13 @@ int EXRLayers(const char *filename, const char **layer_names[], int *num_layers, int LoadEXR(float **out_rgba, int *width, int *height, const char *filename, const char **err) { - return LoadEXRWithLayer(out_rgba, width, height, filename, /* layername */NULL, err); + return LoadEXRWithLayer(out_rgba, width, height, filename, + /* layername */ NULL, err); } -int LoadEXRWithLayer(float **out_rgba, int *width, int *height, const char *filename, const char *layername, - const char **err) { +int LoadEXRWithLayer(float **out_rgba, int *width, int *height, + const char *filename, const char *layername, + const char **err) { if (out_rgba == NULL) { tinyexr::SetErrorMessage("Invalid argument for LoadEXR()", err); return TINYEXR_ERROR_INVALID_ARGUMENT; @@ -11487,7 +11623,8 @@ int LoadEXRWithLayer(float **out_rgba, int *width, int *height, const char *file int ret = ParseEXRVersionFromFile(&exr_version, filename); if (ret != TINYEXR_SUCCESS) { std::stringstream ss; - ss << "Failed to open EXR file or read version info from EXR file. code(" << ret << ")"; + ss << "Failed to open EXR file or read version info from EXR file. code(" + << ret << ")"; tinyexr::SetErrorMessage(ss.str(), err); return ret; } @@ -11534,7 +11671,8 @@ int LoadEXRWithLayer(float **out_rgba, int *width, int *height, const char *file tinyexr::GetLayers(exr_header, layer_names); std::vector<tinyexr::LayerChannel> channels; - tinyexr::ChannelsInLayer(exr_header, layername == NULL ? "" : std::string(layername), channels); + tinyexr::ChannelsInLayer( + exr_header, layername == NULL ? "" : std::string(layername), channels); if (channels.size() < 1) { tinyexr::SetErrorMessage("Layer Not Found", err); @@ -11549,14 +11687,11 @@ int LoadEXRWithLayer(float **out_rgba, int *width, int *height, const char *file if (ch.name == "R") { idxR = int(ch.index); - } - else if (ch.name == "G") { + } else if (ch.name == "G") { idxG = int(ch.index); - } - else if (ch.name == "B") { + } else if (ch.name == "B") { idxB = int(ch.index); - } - else if (ch.name == "A") { + } else if (ch.name == "A") { idxA = int(ch.index); } } @@ -11573,11 +11708,13 @@ int LoadEXRWithLayer(float **out_rgba, int *width, int *height, const char *file for (int it = 0; it < exr_image.num_tiles; it++) { for (int j = 0; j < exr_header.tile_size_y; j++) { for (int i = 0; i < exr_header.tile_size_x; i++) { - const int ii = - exr_image.tiles[it].offset_x * exr_header.tile_size_x + i; - const int jj = - exr_image.tiles[it].offset_y * exr_header.tile_size_y + j; - const int idx = ii + jj * exr_image.width; + const int ii = exr_image.tiles[it].offset_x * + static_cast<int>(exr_header.tile_size_x) + + i; + const int jj = exr_image.tiles[it].offset_y * + static_cast<int>(exr_header.tile_size_y) + + j; + const int idx = ii + jj * static_cast<int>(exr_image.width); // out of region check. if (ii >= exr_image.width) { @@ -11601,7 +11738,8 @@ int LoadEXRWithLayer(float **out_rgba, int *width, int *height, const char *file } } else { for (int i = 0; i < exr_image.width * exr_image.height; i++) { - const float val = reinterpret_cast<float **>(exr_image.images)[chIdx][i]; + const float val = + reinterpret_cast<float **>(exr_image.images)[chIdx][i]; (*out_rgba)[4 * i + 0] = val; (*out_rgba)[4 * i + 1] = val; (*out_rgba)[4 * i + 2] = val; @@ -11947,11 +12085,22 @@ int LoadEXRImageFromFile(EXRImage *exr_image, const EXRHeader *exr_header, return TINYEXR_ERROR_INVALID_ARGUMENT; } -#ifdef _WIN32 FILE *fp = NULL; - fopen_s(&fp, filename, "rb"); +#ifdef _WIN32 +#if defined(_MSC_VER) || defined(__MINGW32__) // MSVC, MinGW gcc or clang + errno_t errcode = + _wfopen_s(&fp, tinyexr::UTF8ToWchar(filename).c_str(), L"rb"); + if (errcode != 0) { + tinyexr::SetErrorMessage("Cannot read file " + std::string(filename), err); + // TODO(syoyo): return wfopen_s erro code + return TINYEXR_ERROR_CANT_OPEN_FILE; + } #else - FILE *fp = fopen(filename, "rb"); + // Unknown compiler + fp = fopen(filename, "rb"); +#endif +#else + fp = fopen(filename, "rb"); #endif if (!fp) { tinyexr::SetErrorMessage("Cannot read file " + std::string(filename), err); @@ -12101,7 +12250,7 @@ size_t SaveEXRImageToMemory(const EXRImage *exr_image, { int comp = exr_header->compression_type; - tinyexr::swap4(reinterpret_cast<unsigned int *>(&comp)); + tinyexr::swap4(&comp); tinyexr::WriteAttributeToMemory( &memory, "compression", "compression", reinterpret_cast<const unsigned char *>(&comp), 1); @@ -12109,10 +12258,10 @@ size_t SaveEXRImageToMemory(const EXRImage *exr_image, { int data[4] = {0, 0, exr_image->width - 1, exr_image->height - 1}; - tinyexr::swap4(reinterpret_cast<unsigned int *>(&data[0])); - tinyexr::swap4(reinterpret_cast<unsigned int *>(&data[1])); - tinyexr::swap4(reinterpret_cast<unsigned int *>(&data[2])); - tinyexr::swap4(reinterpret_cast<unsigned int *>(&data[3])); + tinyexr::swap4(&data[0]); + tinyexr::swap4(&data[1]); + tinyexr::swap4(&data[2]); + tinyexr::swap4(&data[3]); tinyexr::WriteAttributeToMemory( &memory, "dataWindow", "box2i", reinterpret_cast<const unsigned char *>(data), sizeof(int) * 4); @@ -12129,7 +12278,7 @@ size_t SaveEXRImageToMemory(const EXRImage *exr_image, { float aspectRatio = 1.0f; - tinyexr::swap4(reinterpret_cast<unsigned int *>(&aspectRatio)); + tinyexr::swap4(&aspectRatio); tinyexr::WriteAttributeToMemory( &memory, "pixelAspectRatio", "float", reinterpret_cast<const unsigned char *>(&aspectRatio), sizeof(float)); @@ -12137,8 +12286,8 @@ size_t SaveEXRImageToMemory(const EXRImage *exr_image, { float center[2] = {0.0f, 0.0f}; - tinyexr::swap4(reinterpret_cast<unsigned int *>(¢er[0])); - tinyexr::swap4(reinterpret_cast<unsigned int *>(¢er[1])); + tinyexr::swap4(¢er[0]); + tinyexr::swap4(¢er[1]); tinyexr::WriteAttributeToMemory( &memory, "screenWindowCenter", "v2f", reinterpret_cast<const unsigned char *>(center), 2 * sizeof(float)); @@ -12146,7 +12295,7 @@ size_t SaveEXRImageToMemory(const EXRImage *exr_image, { float w = static_cast<float>(exr_image->width); - tinyexr::swap4(reinterpret_cast<unsigned int *>(&w)); + tinyexr::swap4(&w); tinyexr::WriteAttributeToMemory(&memory, "screenWindowWidth", "float", reinterpret_cast<const unsigned char *>(&w), sizeof(float)); @@ -12213,9 +12362,10 @@ size_t SaveEXRImageToMemory(const EXRImage *exr_image, // Use ZFP compression parameter from custom attributes(if such a parameter // exists) { + std::string e; bool ret = tinyexr::FindZFPCompressionParam( &zfp_compression_param, exr_header->custom_attributes, - exr_header->num_custom_attributes); + exr_header->num_custom_attributes, &e); if (!ret) { // Use predefined compression parameter. @@ -12225,7 +12375,7 @@ size_t SaveEXRImageToMemory(const EXRImage *exr_image, } #endif - // TOOD(LTE): C++11 thread + // TODO(LTE): C++11 thread // Use signed int since some OpenMP compiler doesn't allow unsigned type for // `parallel for` @@ -12257,7 +12407,7 @@ size_t SaveEXRImageToMemory(const EXRImage *exr_image, tinyexr::FP32 f32 = half_to_float(h16); - tinyexr::swap4(reinterpret_cast<unsigned int *>(&f32.f)); + tinyexr::swap4(&f32.f); // line_ptr[x] = f32.f; tinyexr::cpy4(line_ptr + x, &(f32.f)); @@ -12321,7 +12471,7 @@ size_t SaveEXRImageToMemory(const EXRImage *exr_image, float val = reinterpret_cast<float **>( exr_image->images)[c][(y + start_y) * exr_image->width + x]; - tinyexr::swap4(reinterpret_cast<unsigned int *>(&val)); + tinyexr::swap4(&val); // line_ptr[x] = val; tinyexr::cpy4(line_ptr + x, &val); @@ -12538,14 +12688,26 @@ int SaveEXRImageToFile(const EXRImage *exr_image, const EXRHeader *exr_header, } #endif -#ifdef _WIN32 FILE *fp = NULL; - fopen_s(&fp, filename, "wb"); +#ifdef _WIN32 +#if defined(_MSC_VER) || defined(__MINGW32__) // MSVC, MinGW gcc or clang + errno_t errcode = + _wfopen_s(&fp, tinyexr::UTF8ToWchar(filename).c_str(), L"wb"); + if (errcode != 0) { + tinyexr::SetErrorMessage("Cannot write a file: " + std::string(filename), + err); + return TINYEXR_ERROR_CANT_WRITE_FILE; + } +#else + // Unknown compiler + fp = fopen(filename, "wb"); +#endif #else - FILE *fp = fopen(filename, "wb"); + fp = fopen(filename, "wb"); #endif if (!fp) { - tinyexr::SetErrorMessage("Cannot write a file", err); + tinyexr::SetErrorMessage("Cannot write a file: " + std::string(filename), + err); return TINYEXR_ERROR_CANT_WRITE_FILE; } @@ -12577,10 +12739,21 @@ int LoadDeepEXR(DeepImage *deep_image, const char *filename, const char **err) { return TINYEXR_ERROR_INVALID_ARGUMENT; } -#ifdef _MSC_VER +#ifdef _WIN32 FILE *fp = NULL; - errno_t errcode = fopen_s(&fp, filename, "rb"); - if ((0 != errcode) || (!fp)) { +#if defined(_MSC_VER) || defined(__MINGW32__) // MSVC, MinGW gcc or clang + errno_t errcode = + _wfopen_s(&fp, tinyexr::UTF8ToWchar(filename).c_str(), L"rb"); + if (errcode != 0) { + tinyexr::SetErrorMessage("Cannot read a file " + std::string(filename), + err); + return TINYEXR_ERROR_CANT_OPEN_FILE; + } +#else + // Unknown compiler + fp = fopen(filename, "rb"); +#endif + if (!fp) { tinyexr::SetErrorMessage("Cannot read a file " + std::string(filename), err); return TINYEXR_ERROR_CANT_OPEN_FILE; @@ -12714,10 +12887,10 @@ int LoadDeepEXR(DeepImage *deep_image, const char *filename, const char **err) { memcpy(&dy, &data.at(4), sizeof(int)); memcpy(&dw, &data.at(8), sizeof(int)); memcpy(&dh, &data.at(12), sizeof(int)); - tinyexr::swap4(reinterpret_cast<unsigned int *>(&dx)); - tinyexr::swap4(reinterpret_cast<unsigned int *>(&dy)); - tinyexr::swap4(reinterpret_cast<unsigned int *>(&dw)); - tinyexr::swap4(reinterpret_cast<unsigned int *>(&dh)); + tinyexr::swap4(&dx); + tinyexr::swap4(&dy); + tinyexr::swap4(&dw); + tinyexr::swap4(&dh); } else if (attr_name.compare("displayWindow") == 0) { int x; @@ -12728,10 +12901,10 @@ int LoadDeepEXR(DeepImage *deep_image, const char *filename, const char **err) { memcpy(&y, &data.at(4), sizeof(int)); memcpy(&w, &data.at(8), sizeof(int)); memcpy(&h, &data.at(12), sizeof(int)); - tinyexr::swap4(reinterpret_cast<unsigned int *>(&x)); - tinyexr::swap4(reinterpret_cast<unsigned int *>(&y)); - tinyexr::swap4(reinterpret_cast<unsigned int *>(&w)); - tinyexr::swap4(reinterpret_cast<unsigned int *>(&h)); + tinyexr::swap4(&x); + tinyexr::swap4(&y); + tinyexr::swap4(&w); + tinyexr::swap4(&h); } } @@ -12819,7 +12992,7 @@ int LoadDeepEXR(DeepImage *deep_image, const char *filename, const char **err) { memcpy(&unpackedSampleDataSize, data_ptr + 20, sizeof(tinyexr::tinyexr_int64)); - tinyexr::swap4(reinterpret_cast<unsigned int *>(&line_no)); + tinyexr::swap4(&line_no); tinyexr::swap8( reinterpret_cast<tinyexr::tinyexr_uint64 *>(&packedOffsetTableSize)); tinyexr::swap8( @@ -13054,11 +13227,21 @@ int ParseEXRHeaderFromFile(EXRHeader *exr_header, const EXRVersion *exr_version, return TINYEXR_ERROR_INVALID_ARGUMENT; } -#ifdef _WIN32 FILE *fp = NULL; - fopen_s(&fp, filename, "rb"); +#ifdef _WIN32 +#if defined(_MSC_VER) || defined(__MINGW32__) // MSVC, MinGW gcc or clang + errno_t errcode = + _wfopen_s(&fp, tinyexr::UTF8ToWchar(filename).c_str(), L"rb"); + if (errcode != 0) { + tinyexr::SetErrorMessage("Cannot read file " + std::string(filename), err); + return TINYEXR_ERROR_INVALID_FILE; + } #else - FILE *fp = fopen(filename, "rb"); + // Unknown compiler + fp = fopen(filename, "rb"); +#endif +#else + fp = fopen(filename, "rb"); #endif if (!fp) { tinyexr::SetErrorMessage("Cannot read file " + std::string(filename), err); @@ -13174,11 +13357,21 @@ int ParseEXRMultipartHeaderFromFile(EXRHeader ***exr_headers, int *num_headers, return TINYEXR_ERROR_INVALID_ARGUMENT; } -#ifdef _WIN32 FILE *fp = NULL; - fopen_s(&fp, filename, "rb"); +#ifdef _WIN32 +#if defined(_MSC_VER) || defined(__MINGW32__) // MSVC, MinGW gcc or clang + errno_t errcode = + _wfopen_s(&fp, tinyexr::UTF8ToWchar(filename).c_str(), L"rb"); + if (errcode != 0) { + tinyexr::SetErrorMessage("Cannot read file " + std::string(filename), err); + return TINYEXR_ERROR_INVALID_FILE; + } #else - FILE *fp = fopen(filename, "rb"); + // Unknown compiler + fp = fopen(filename, "rb"); +#endif +#else + fp = fopen(filename, "rb"); #endif if (!fp) { tinyexr::SetErrorMessage("Cannot read file " + std::string(filename), err); @@ -13270,11 +13463,20 @@ int ParseEXRVersionFromFile(EXRVersion *version, const char *filename) { return TINYEXR_ERROR_INVALID_ARGUMENT; } -#ifdef _WIN32 FILE *fp = NULL; - fopen_s(&fp, filename, "rb"); +#ifdef _WIN32 +#if defined(_MSC_VER) || defined(__MINGW32__) // MSVC, MinGW gcc or clang + errno_t err = _wfopen_s(&fp, tinyexr::UTF8ToWchar(filename).c_str(), L"rb"); + if (err != 0) { + // TODO(syoyo): return wfopen_s erro code + return TINYEXR_ERROR_CANT_OPEN_FILE; + } #else - FILE *fp = fopen(filename, "rb"); + // Unknown compiler + fp = fopen(filename, "rb"); +#endif +#else + fp = fopen(filename, "rb"); #endif if (!fp) { return TINYEXR_ERROR_CANT_OPEN_FILE; @@ -13408,11 +13610,21 @@ int LoadEXRMultipartImageFromFile(EXRImage *exr_images, return TINYEXR_ERROR_INVALID_ARGUMENT; } -#ifdef _WIN32 FILE *fp = NULL; - fopen_s(&fp, filename, "rb"); +#ifdef _WIN32 +#if defined(_MSC_VER) || defined(__MINGW32__) // MSVC, MinGW gcc or clang + errno_t errcode = + _wfopen_s(&fp, tinyexr::UTF8ToWchar(filename).c_str(), L"rb"); + if (errcode != 0) { + tinyexr::SetErrorMessage("Cannot read file " + std::string(filename), err); + return TINYEXR_ERROR_CANT_OPEN_FILE; + } #else - FILE *fp = fopen(filename, "rb"); + // Unknown compiler + fp = fopen(filename, "rb"); +#endif +#else + fp = fopen(filename, "rb"); #endif if (!fp) { tinyexr::SetErrorMessage("Cannot read file " + std::string(filename), err); @@ -13582,5 +13794,5 @@ int SaveEXR(const float *data, int width, int height, int components, #pragma clang diagnostic pop #endif -#endif // TINYEXR_IMPLEMENTATION_DEIFNED +#endif // TINYEXR_IMPLEMENTATION_DEFINED #endif // TINYEXR_IMPLEMENTATION diff --git a/thirdparty/zstd/common/bitstream.h b/thirdparty/zstd/common/bitstream.h index 1c294b80d1..37b99c01ee 100644 --- a/thirdparty/zstd/common/bitstream.h +++ b/thirdparty/zstd/common/bitstream.h @@ -1,35 +1,15 @@ /* ****************************************************************** - bitstream - Part of FSE library - Copyright (C) 2013-present, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - Source repository : https://github.com/Cyan4973/FiniteStateEntropy + * bitstream + * Part of FSE library + * Copyright (c) 2013-2020, Yann Collet, Facebook, Inc. + * + * You can contact the author at : + * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. ****************************************************************** */ #ifndef BITSTREAM_H_MODULE #define BITSTREAM_H_MODULE @@ -48,6 +28,7 @@ extern "C" { * Dependencies ******************************************/ #include "mem.h" /* unaligned access routines */ +#include "compiler.h" /* UNLIKELY() */ #include "debug.h" /* assert(), DEBUGLOG(), RAWLOG() */ #include "error_private.h" /* error codes and messages */ @@ -161,8 +142,7 @@ MEM_STATIC unsigned BIT_highbit32 (U32 val) { # if defined(_MSC_VER) /* Visual */ unsigned long r=0; - _BitScanReverse ( &r, val ); - return (unsigned) r; + return _BitScanReverse ( &r, val ) ? (unsigned)r : 0; # elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */ return __builtin_clz (val) ^ 31; # elif defined(__ICCARM__) /* IAR Intrinsic */ @@ -411,6 +391,23 @@ MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits) return value; } +/*! BIT_reloadDStreamFast() : + * Similar to BIT_reloadDStream(), but with two differences: + * 1. bitsConsumed <= sizeof(bitD->bitContainer)*8 must hold! + * 2. Returns BIT_DStream_overflow when bitD->ptr < bitD->limitPtr, at this + * point you must use BIT_reloadDStream() to reload. + */ +MEM_STATIC BIT_DStream_status BIT_reloadDStreamFast(BIT_DStream_t* bitD) +{ + if (UNLIKELY(bitD->ptr < bitD->limitPtr)) + return BIT_DStream_overflow; + assert(bitD->bitsConsumed <= sizeof(bitD->bitContainer)*8); + bitD->ptr -= bitD->bitsConsumed >> 3; + bitD->bitsConsumed &= 7; + bitD->bitContainer = MEM_readLEST(bitD->ptr); + return BIT_DStream_unfinished; +} + /*! BIT_reloadDStream() : * Refill `bitD` from buffer previously set in BIT_initDStream() . * This function is safe, it guarantees it will not read beyond src buffer. @@ -422,10 +419,7 @@ MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD) return BIT_DStream_overflow; if (bitD->ptr >= bitD->limitPtr) { - bitD->ptr -= bitD->bitsConsumed >> 3; - bitD->bitsConsumed &= 7; - bitD->bitContainer = MEM_readLEST(bitD->ptr); - return BIT_DStream_unfinished; + return BIT_reloadDStreamFast(bitD); } if (bitD->ptr == bitD->start) { if (bitD->bitsConsumed < sizeof(bitD->bitContainer)*8) return BIT_DStream_endOfBuffer; diff --git a/thirdparty/zstd/common/compiler.h b/thirdparty/zstd/common/compiler.h index 1877a0c1d9..95e9483521 100644 --- a/thirdparty/zstd/common/compiler.h +++ b/thirdparty/zstd/common/compiler.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -17,7 +17,7 @@ /* force inlining */ #if !defined(ZSTD_NO_INLINE) -#if defined (__GNUC__) || defined(__cplusplus) || defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ +#if (defined(__GNUC__) && !defined(__STRICT_ANSI__)) || defined(__cplusplus) || defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ # define INLINE_KEYWORD inline #else # define INLINE_KEYWORD @@ -114,6 +114,9 @@ # include <mmintrin.h> /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */ # define PREFETCH_L1(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T0) # define PREFETCH_L2(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T1) +# elif defined(__aarch64__) +# define PREFETCH_L1(ptr) __asm__ __volatile__("prfm pldl1keep, %0" ::"Q"(*(ptr))) +# define PREFETCH_L2(ptr) __asm__ __volatile__("prfm pldl2keep, %0" ::"Q"(*(ptr))) # elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) ) # define PREFETCH_L1(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */) # define PREFETCH_L2(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 2 /* locality */) @@ -136,7 +139,7 @@ /* vectorization * older GCC (pre gcc-4.3 picked as the cutoff) uses a different syntax */ -#if !defined(__clang__) && defined(__GNUC__) +#if !defined(__INTEL_COMPILER) && !defined(__clang__) && defined(__GNUC__) # if (__GNUC__ == 4 && __GNUC_MINOR__ > 3) || (__GNUC__ >= 5) # define DONT_VECTORIZE __attribute__((optimize("no-tree-vectorize"))) # else @@ -146,6 +149,19 @@ # define DONT_VECTORIZE #endif +/* Tell the compiler that a branch is likely or unlikely. + * Only use these macros if it causes the compiler to generate better code. + * If you can remove a LIKELY/UNLIKELY annotation without speed changes in gcc + * and clang, please do. + */ +#if defined(__GNUC__) +#define LIKELY(x) (__builtin_expect((x), 1)) +#define UNLIKELY(x) (__builtin_expect((x), 0)) +#else +#define LIKELY(x) (x) +#define UNLIKELY(x) (x) +#endif + /* disable warnings */ #ifdef _MSC_VER /* Visual Studio */ # include <intrin.h> /* For Visual 2005 */ diff --git a/thirdparty/zstd/common/cpu.h b/thirdparty/zstd/common/cpu.h index 5f0923fc92..6e8a974f62 100644 --- a/thirdparty/zstd/common/cpu.h +++ b/thirdparty/zstd/common/cpu.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-present, Facebook, Inc. + * Copyright (c) 2018-2020, Facebook, Inc. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/thirdparty/zstd/common/debug.c b/thirdparty/zstd/common/debug.c index 3ebdd1cb15..f303f4a2e5 100644 --- a/thirdparty/zstd/common/debug.c +++ b/thirdparty/zstd/common/debug.c @@ -1,35 +1,15 @@ /* ****************************************************************** - debug - Part of FSE library - Copyright (C) 2013-present, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - Source repository : https://github.com/Cyan4973/FiniteStateEntropy + * debug + * Part of FSE library + * Copyright (c) 2013-2020, Yann Collet, Facebook, Inc. + * + * You can contact the author at : + * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. ****************************************************************** */ diff --git a/thirdparty/zstd/common/debug.h b/thirdparty/zstd/common/debug.h index b4fc89d497..ac6224888d 100644 --- a/thirdparty/zstd/common/debug.h +++ b/thirdparty/zstd/common/debug.h @@ -1,35 +1,15 @@ /* ****************************************************************** - debug - Part of FSE library - Copyright (C) 2013-present, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - Source repository : https://github.com/Cyan4973/FiniteStateEntropy + * debug + * Part of FSE library + * Copyright (c) 2013-2020, Yann Collet, Facebook, Inc. + * + * You can contact the author at : + * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. ****************************************************************** */ diff --git a/thirdparty/zstd/common/entropy_common.c b/thirdparty/zstd/common/entropy_common.c index b12944e1de..9d3e4e8e36 100644 --- a/thirdparty/zstd/common/entropy_common.c +++ b/thirdparty/zstd/common/entropy_common.c @@ -1,36 +1,16 @@ -/* - Common functions of New Generation Entropy library - Copyright (C) 2016, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy - - Public forum : https://groups.google.com/forum/#!forum/lz4c -*************************************************************************** */ +/* ****************************************************************** + * Common functions of New Generation Entropy library + * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc. + * + * You can contact the author at : + * - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy + * - Public forum : https://groups.google.com/forum/#!forum/lz4c + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ /* ************************************* * Dependencies diff --git a/thirdparty/zstd/common/error_private.c b/thirdparty/zstd/common/error_private.c index 7c1bb67a23..cd437529c1 100644 --- a/thirdparty/zstd/common/error_private.c +++ b/thirdparty/zstd/common/error_private.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -47,6 +47,7 @@ const char* ERR_getErrorString(ERR_enum code) /* following error codes are not stable and may be removed or changed in a future version */ case PREFIX(frameIndex_tooLarge): return "Frame index is too large"; case PREFIX(seekableIO): return "An I/O error occurred when reading/seeking"; + case PREFIX(dstBuffer_wrong): return "Destination buffer is wrong"; case PREFIX(maxCode): default: return notErrorCode; } diff --git a/thirdparty/zstd/common/error_private.h b/thirdparty/zstd/common/error_private.h index 0d2fa7e34b..982cf8e9fe 100644 --- a/thirdparty/zstd/common/error_private.h +++ b/thirdparty/zstd/common/error_private.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -49,7 +49,7 @@ typedef ZSTD_ErrorCode ERR_enum; /*-**************************************** * Error codes handling ******************************************/ -#undef ERROR /* reported already defined on VS 2015 (Rich Geldreich) */ +#undef ERROR /* already defined on Visual Studio */ #define ERROR(name) ZSTD_ERROR(name) #define ZSTD_ERROR(name) ((size_t)-PREFIX(name)) @@ -57,6 +57,10 @@ ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); } ERR_STATIC ERR_enum ERR_getErrorCode(size_t code) { if (!ERR_isError(code)) return (ERR_enum)0; return (ERR_enum) (0-code); } +/* check and forward error code */ +#define CHECK_V_F(e, f) size_t const e = f; if (ERR_isError(e)) return e +#define CHECK_F(f) { CHECK_V_F(_var_err__, f); } + /*-**************************************** * Error Strings diff --git a/thirdparty/zstd/common/fse.h b/thirdparty/zstd/common/fse.h index a7553e3721..ff54e70ea7 100644 --- a/thirdparty/zstd/common/fse.h +++ b/thirdparty/zstd/common/fse.h @@ -1,35 +1,15 @@ /* ****************************************************************** - FSE : Finite State Entropy codec - Public Prototypes declaration - Copyright (C) 2013-2016, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - Source repository : https://github.com/Cyan4973/FiniteStateEntropy + * FSE : Finite State Entropy codec + * Public Prototypes declaration + * Copyright (c) 2013-2020, Yann Collet, Facebook, Inc. + * + * You can contact the author at : + * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. ****************************************************************** */ #if defined (__cplusplus) diff --git a/thirdparty/zstd/common/fse_decompress.c b/thirdparty/zstd/common/fse_decompress.c index 4f07378982..bcc2223ccc 100644 --- a/thirdparty/zstd/common/fse_decompress.c +++ b/thirdparty/zstd/common/fse_decompress.c @@ -1,35 +1,15 @@ /* ****************************************************************** - FSE : Finite State Entropy decoder - Copyright (C) 2013-2015, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy - - Public forum : https://groups.google.com/forum/#!forum/lz4c + * FSE : Finite State Entropy decoder + * Copyright (c) 2013-2020, Yann Collet, Facebook, Inc. + * + * You can contact the author at : + * - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy + * - Public forum : https://groups.google.com/forum/#!forum/lz4c + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. ****************************************************************** */ @@ -51,11 +31,6 @@ #define FSE_isError ERR_isError #define FSE_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c) /* use only *after* variable declarations */ -/* check and forward error code */ -#ifndef CHECK_F -#define CHECK_F(f) { size_t const e = f; if (FSE_isError(e)) return e; } -#endif - /* ************************************************************** * Templates @@ -287,7 +262,7 @@ size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size /* normal FSE decoding mode */ size_t const NCountLength = FSE_readNCount (counting, &maxSymbolValue, &tableLog, istart, cSrcSize); if (FSE_isError(NCountLength)) return NCountLength; - //if (NCountLength >= cSrcSize) return ERROR(srcSize_wrong); /* too small input size; supposed to be already checked in NCountLength, only remaining case : NCountLength==cSrcSize */ + /* if (NCountLength >= cSrcSize) return ERROR(srcSize_wrong); */ /* too small input size; supposed to be already checked in NCountLength, only remaining case : NCountLength==cSrcSize */ if (tableLog > maxLog) return ERROR(tableLog_tooLarge); ip += NCountLength; cSrcSize -= NCountLength; diff --git a/thirdparty/zstd/common/huf.h b/thirdparty/zstd/common/huf.h index 6b572c448d..ef432685da 100644 --- a/thirdparty/zstd/common/huf.h +++ b/thirdparty/zstd/common/huf.h @@ -1,35 +1,15 @@ /* ****************************************************************** - huff0 huffman codec, - part of Finite State Entropy library - Copyright (C) 2013-present, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - Source repository : https://github.com/Cyan4973/FiniteStateEntropy + * huff0 huffman codec, + * part of Finite State Entropy library + * Copyright (c) 2013-2020, Yann Collet, Facebook, Inc. + * + * You can contact the author at : + * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. ****************************************************************** */ #if defined (__cplusplus) @@ -110,7 +90,7 @@ HUF_PUBLIC_API size_t HUF_compress2 (void* dst, size_t dstCapacity, /** HUF_compress4X_wksp() : * Same as HUF_compress2(), but uses externally allocated `workSpace`. * `workspace` must have minimum alignment of 4, and be at least as large as HUF_WORKSPACE_SIZE */ -#define HUF_WORKSPACE_SIZE (6 << 10) +#define HUF_WORKSPACE_SIZE ((6 << 10) + 256) #define HUF_WORKSPACE_SIZE_U32 (HUF_WORKSPACE_SIZE / sizeof(U32)) HUF_PUBLIC_API size_t HUF_compress4X_wksp (void* dst, size_t dstCapacity, const void* src, size_t srcSize, @@ -208,6 +188,8 @@ typedef struct HUF_CElt_s HUF_CElt; /* incomplete type */ size_t HUF_buildCTable (HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits); /* @return : maxNbBits; CTable and count can overlap. In which case, CTable will overwrite count content */ size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog); size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable); +size_t HUF_estimateCompressedSize(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue); +int HUF_validateCTable(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue); typedef enum { HUF_repeat_none, /**< Cannot use the previous table */ @@ -246,7 +228,7 @@ size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, /** HUF_readCTable() : * Loading a CTable saved with HUF_writeCTable() */ -size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize); +size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize, unsigned *hasZeroWeights); /** HUF_getNbBits() : * Read nbBits from CTable symbolTable, for symbol `symbolValue` presumed <= HUF_SYMBOLVALUE_MAX diff --git a/thirdparty/zstd/common/mem.h b/thirdparty/zstd/common/mem.h index 530d30c8f7..89c8aea7d2 100644 --- a/thirdparty/zstd/common/mem.h +++ b/thirdparty/zstd/common/mem.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/thirdparty/zstd/common/pool.c b/thirdparty/zstd/common/pool.c index f575935076..aa4b4de0d3 100644 --- a/thirdparty/zstd/common/pool.c +++ b/thirdparty/zstd/common/pool.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/thirdparty/zstd/common/pool.h b/thirdparty/zstd/common/pool.h index 458d37f13c..259bafc975 100644 --- a/thirdparty/zstd/common/pool.h +++ b/thirdparty/zstd/common/pool.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -18,7 +18,7 @@ extern "C" { #include <stddef.h> /* size_t */ #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_customMem */ -#include "zstd.h" +#include "../zstd.h" typedef struct POOL_ctx_s POOL_ctx; diff --git a/thirdparty/zstd/common/threading.c b/thirdparty/zstd/common/threading.c index 482664bd9a..e2edb313eb 100644 --- a/thirdparty/zstd/common/threading.c +++ b/thirdparty/zstd/common/threading.c @@ -2,12 +2,13 @@ * Copyright (c) 2016 Tino Reichardt * All rights reserved. * + * You can contact the author at: + * - zstdmt source repository: https://github.com/mcmilk/zstdmt + * * This source code is licensed under both the BSD-style license (found in the * LICENSE file in the root directory of this source tree) and the GPLv2 (found * in the COPYING file in the root directory of this source tree). - * - * You can contact the author at: - * - zstdmt source repository: https://github.com/mcmilk/zstdmt + * You may select, at your option, one of the above-listed licenses. */ /** diff --git a/thirdparty/zstd/common/threading.h b/thirdparty/zstd/common/threading.h index 3193ca7db8..fd0060d5aa 100644 --- a/thirdparty/zstd/common/threading.h +++ b/thirdparty/zstd/common/threading.h @@ -2,12 +2,13 @@ * Copyright (c) 2016 Tino Reichardt * All rights reserved. * + * You can contact the author at: + * - zstdmt source repository: https://github.com/mcmilk/zstdmt + * * This source code is licensed under both the BSD-style license (found in the * LICENSE file in the root directory of this source tree) and the GPLv2 (found * in the COPYING file in the root directory of this source tree). - * - * You can contact the author at: - * - zstdmt source repository: https://github.com/mcmilk/zstdmt + * You may select, at your option, one of the above-listed licenses. */ #ifndef THREADING_H_938743 diff --git a/thirdparty/zstd/common/xxhash.c b/thirdparty/zstd/common/xxhash.c index 99d2459621..597de18fc8 100644 --- a/thirdparty/zstd/common/xxhash.c +++ b/thirdparty/zstd/common/xxhash.c @@ -1,35 +1,15 @@ /* -* xxHash - Fast Hash algorithm -* Copyright (C) 2012-2016, Yann Collet -* -* BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) -* -* Redistribution and use in source and binary forms, with or without -* modification, are permitted provided that the following conditions are -* met: -* -* * Redistributions of source code must retain the above copyright -* notice, this list of conditions and the following disclaimer. -* * Redistributions in binary form must reproduce the above -* copyright notice, this list of conditions and the following disclaimer -* in the documentation and/or other materials provided with the -* distribution. -* -* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -* -* You can contact the author at : -* - xxHash homepage: http://www.xxhash.com -* - xxHash source repository : https://github.com/Cyan4973/xxHash + * xxHash - Fast Hash algorithm + * Copyright (c) 2012-2020, Yann Collet, Facebook, Inc. + * + * You can contact the author at : + * - xxHash homepage: http://www.xxhash.com + * - xxHash source repository : https://github.com/Cyan4973/xxHash + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. */ @@ -115,7 +95,7 @@ static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcp /* ************************************* * Compiler Specific Options ***************************************/ -#if defined (__GNUC__) || defined(__cplusplus) || defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ +#if (defined(__GNUC__) && !defined(__STRICT_ANSI__)) || defined(__cplusplus) || defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ # define INLINE_KEYWORD inline #else # define INLINE_KEYWORD @@ -729,7 +709,9 @@ FORCE_INLINE_TEMPLATE XXH_errorcode XXH64_update_endian (XXH64_state_t* state, c state->total_len += len; if (state->memsize + len < 32) { /* fill in tmp buffer */ - XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, len); + if (input != NULL) { + XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, len); + } state->memsize += (U32)len; return XXH_OK; } diff --git a/thirdparty/zstd/common/xxhash.h b/thirdparty/zstd/common/xxhash.h index 9bad1f59f6..4207eba832 100644 --- a/thirdparty/zstd/common/xxhash.h +++ b/thirdparty/zstd/common/xxhash.h @@ -1,35 +1,15 @@ /* - xxHash - Extremely Fast Hash algorithm - Header File - Copyright (C) 2012-2016, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - xxHash source repository : https://github.com/Cyan4973/xxHash + * xxHash - Extremely Fast Hash algorithm + * Header File + * Copyright (c) 2012-2020, Yann Collet, Facebook, Inc. + * + * You can contact the author at : + * - xxHash source repository : https://github.com/Cyan4973/xxHash + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. */ /* Notice extracted from xxHash homepage : diff --git a/thirdparty/zstd/common/zstd_common.c b/thirdparty/zstd/common/zstd_common.c index 667f4a27fc..91fe3323a5 100644 --- a/thirdparty/zstd/common/zstd_common.c +++ b/thirdparty/zstd/common/zstd_common.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/thirdparty/zstd/common/zstd_errors.h b/thirdparty/zstd/common/zstd_errors.h index 92a3433896..998398e7e5 100644 --- a/thirdparty/zstd/common/zstd_errors.h +++ b/thirdparty/zstd/common/zstd_errors.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -76,6 +76,7 @@ typedef enum { /* following error codes are __NOT STABLE__, they can be removed or changed in future versions */ ZSTD_error_frameIndex_tooLarge = 100, ZSTD_error_seekableIO = 102, + ZSTD_error_dstBuffer_wrong = 104, ZSTD_error_maxCode = 120 /* never EVER use this value directly, it can change in future versions! Use ZSTD_isError() instead */ } ZSTD_ErrorCode; diff --git a/thirdparty/zstd/common/zstd_internal.h b/thirdparty/zstd/common/zstd_internal.h index dcdcbdb81c..3bc7e55a0a 100644 --- a/thirdparty/zstd/common/zstd_internal.h +++ b/thirdparty/zstd/common/zstd_internal.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -19,12 +19,15 @@ /*-************************************* * Dependencies ***************************************/ +#ifdef __aarch64__ +#include <arm_neon.h> +#endif #include "compiler.h" #include "mem.h" #include "debug.h" /* assert, DEBUGLOG, RAWLOG, g_debuglevel */ #include "error_private.h" #define ZSTD_STATIC_LINKING_ONLY -#include "zstd.h" +#include "../zstd.h" #define FSE_STATIC_LINKING_ONLY #include "fse.h" #define HUF_STATIC_LINKING_ONLY @@ -54,6 +57,31 @@ extern "C" { #define MAX(a,b) ((a)>(b) ? (a) : (b)) /** + * Ignore: this is an internal helper. + * + * This is a helper function to help force C99-correctness during compilation. + * Under strict compilation modes, variadic macro arguments can't be empty. + * However, variadic function arguments can be. Using a function therefore lets + * us statically check that at least one (string) argument was passed, + * independent of the compilation flags. + */ +static INLINE_KEYWORD UNUSED_ATTR +void _force_has_format_string(const char *format, ...) { + (void)format; +} + +/** + * Ignore: this is an internal helper. + * + * We want to force this function invocation to be syntactically correct, but + * we don't want to force runtime evaluation of its arguments. + */ +#define _FORCE_HAS_FORMAT_STRING(...) \ + if (0) { \ + _force_has_format_string(__VA_ARGS__); \ + } + +/** * Return the specified error if the condition evaluates to true. * * In debug modes, prints additional information. @@ -62,7 +90,9 @@ extern "C" { */ #define RETURN_ERROR_IF(cond, err, ...) \ if (cond) { \ - RAWLOG(3, "%s:%d: ERROR!: check %s failed, returning %s", __FILE__, __LINE__, ZSTD_QUOTE(cond), ZSTD_QUOTE(ERROR(err))); \ + RAWLOG(3, "%s:%d: ERROR!: check %s failed, returning %s", \ + __FILE__, __LINE__, ZSTD_QUOTE(cond), ZSTD_QUOTE(ERROR(err))); \ + _FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \ RAWLOG(3, ": " __VA_ARGS__); \ RAWLOG(3, "\n"); \ return ERROR(err); \ @@ -75,7 +105,9 @@ extern "C" { */ #define RETURN_ERROR(err, ...) \ do { \ - RAWLOG(3, "%s:%d: ERROR!: unconditional check failed, returning %s", __FILE__, __LINE__, ZSTD_QUOTE(ERROR(err))); \ + RAWLOG(3, "%s:%d: ERROR!: unconditional check failed, returning %s", \ + __FILE__, __LINE__, ZSTD_QUOTE(ERROR(err))); \ + _FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \ RAWLOG(3, ": " __VA_ARGS__); \ RAWLOG(3, "\n"); \ return ERROR(err); \ @@ -90,7 +122,9 @@ extern "C" { do { \ size_t const err_code = (err); \ if (ERR_isError(err_code)) { \ - RAWLOG(3, "%s:%d: ERROR!: forwarding error in %s: %s", __FILE__, __LINE__, ZSTD_QUOTE(err), ERR_getErrorName(err_code)); \ + RAWLOG(3, "%s:%d: ERROR!: forwarding error in %s: %s", \ + __FILE__, __LINE__, ZSTD_QUOTE(err), ERR_getErrorName(err_code)); \ + _FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \ RAWLOG(3, ": " __VA_ARGS__); \ RAWLOG(3, "\n"); \ return err_code; \ @@ -128,6 +162,8 @@ static const size_t ZSTD_did_fieldSize[4] = { 0, 1, 2, 4 }; static const size_t ZSTD_blockHeaderSize = ZSTD_BLOCKHEADERSIZE; typedef enum { bt_raw, bt_rle, bt_compressed, bt_reserved } blockType_e; +#define ZSTD_FRAMECHECKSUMSIZE 4 + #define MIN_SEQUENCES_SIZE 1 /* nbSeq==0 */ #define MIN_CBLOCK_SIZE (1 /*litCSize*/ + 1 /* RLE or RAW */ + MIN_SEQUENCES_SIZE /* nbSeq==0 */) /* for a non-null block */ @@ -191,10 +227,22 @@ static const U32 OF_defaultNormLog = OF_DEFAULTNORMLOG; /*-******************************************* * Shared functions to include for inlining *********************************************/ -static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); } +static void ZSTD_copy8(void* dst, const void* src) { +#ifdef __aarch64__ + vst1_u8((uint8_t*)dst, vld1_u8((const uint8_t*)src)); +#else + memcpy(dst, src, 8); +#endif +} #define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; } -static void ZSTD_copy16(void* dst, const void* src) { memcpy(dst, src, 16); } +static void ZSTD_copy16(void* dst, const void* src) { +#ifdef __aarch64__ + vst1q_u8((uint8_t*)dst, vld1q_u8((const uint8_t*)src)); +#else + memcpy(dst, src, 16); +#endif +} #define COPY16(d,s) { ZSTD_copy16(d,s); d+=16; s+=16; } #define WILDCOPY_OVERLENGTH 32 @@ -213,7 +261,7 @@ typedef enum { * - ZSTD_overlap_src_before_dst: The src and dst may overlap, but they MUST be at least 8 bytes apart. * The src buffer must be before the dst buffer. */ -MEM_STATIC FORCE_INLINE_ATTR DONT_VECTORIZE +MEM_STATIC FORCE_INLINE_ATTR void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length, ZSTD_overlap_e const ovtype) { ptrdiff_t diff = (BYTE*)dst - (const BYTE*)src; @@ -230,13 +278,18 @@ void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length, ZSTD_overlap_e } while (op < oend); } else { assert(diff >= WILDCOPY_VECLEN || diff <= -WILDCOPY_VECLEN); - /* Separate out the first two COPY16() calls because the copy length is + /* Separate out the first COPY16() call because the copy length is * almost certain to be short, so the branches have different - * probabilities. - * On gcc-9 unrolling once is +1.6%, twice is +2%, thrice is +1.8%. - * On clang-8 unrolling once is +1.4%, twice is +3.3%, thrice is +3%. + * probabilities. Since it is almost certain to be short, only do + * one COPY16() in the first call. Then, do two calls per loop since + * at that point it is more likely to have a high trip count. */ - COPY16(op, ip); +#ifndef __aarch64__ + do { + COPY16(op, ip); + } + while (op < oend); +#else COPY16(op, ip); if (op >= oend) return; do { @@ -244,9 +297,29 @@ void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length, ZSTD_overlap_e COPY16(op, ip); } while (op < oend); +#endif + } +} + +MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + size_t const length = MIN(dstCapacity, srcSize); + if (length > 0) { + memcpy(dst, src, length); } + return length; } +/* define "workspace is too large" as this number of times larger than needed */ +#define ZSTD_WORKSPACETOOLARGE_FACTOR 3 + +/* when workspace is continuously too large + * during at least this number of times, + * context's memory usage is considered wasteful, + * because it's sized to handle a worst case scenario which rarely happens. + * In which case, resize it down to free some memory */ +#define ZSTD_WORKSPACETOOLARGE_MAXDURATION 128 + /*-******************************************* * Private declarations @@ -271,6 +344,31 @@ typedef struct { U32 longLengthPos; } seqStore_t; +typedef struct { + U32 litLength; + U32 matchLength; +} ZSTD_sequenceLength; + +/** + * Returns the ZSTD_sequenceLength for the given sequences. It handles the decoding of long sequences + * indicated by longLengthPos and longLengthID, and adds MINMATCH back to matchLength. + */ +MEM_STATIC ZSTD_sequenceLength ZSTD_getSequenceLength(seqStore_t const* seqStore, seqDef const* seq) +{ + ZSTD_sequenceLength seqLen; + seqLen.litLength = seq->litLength; + seqLen.matchLength = seq->matchLength + MINMATCH; + if (seqStore->longLengthPos == (U32)(seq - seqStore->sequencesStart)) { + if (seqStore->longLengthID == 1) { + seqLen.litLength += 0xFFFF; + } + if (seqStore->longLengthID == 2) { + seqLen.matchLength += 0xFFFF; + } + } + return seqLen; +} + /** * Contains the compressed frame size and an upper-bound for the decompressed frame size. * Note: before using `compressedSize`, check for errors using ZSTD_isError(). @@ -297,8 +395,7 @@ MEM_STATIC U32 ZSTD_highbit32(U32 val) /* compress, dictBuilder, decodeCorpus { # if defined(_MSC_VER) /* Visual */ unsigned long r=0; - _BitScanReverse(&r, val); - return (unsigned)r; + return _BitScanReverse(&r, val) ? (unsigned)r : 0; # elif defined(__GNUC__) && (__GNUC__ >= 3) /* GCC Intrinsic */ return __builtin_clz (val) ^ 31; # elif defined(__ICCARM__) /* IAR Intrinsic */ diff --git a/thirdparty/zstd/compress/fse_compress.c b/thirdparty/zstd/compress/fse_compress.c index 68b47e1093..a42759814f 100644 --- a/thirdparty/zstd/compress/fse_compress.c +++ b/thirdparty/zstd/compress/fse_compress.c @@ -1,35 +1,15 @@ /* ****************************************************************** - FSE : Finite State Entropy encoder - Copyright (C) 2013-present, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy - - Public forum : https://groups.google.com/forum/#!forum/lz4c + * FSE : Finite State Entropy encoder + * Copyright (c) 2013-2020, Yann Collet, Facebook, Inc. + * + * You can contact the author at : + * - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy + * - Public forum : https://groups.google.com/forum/#!forum/lz4c + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. ****************************************************************** */ /* ************************************************************** @@ -37,14 +17,14 @@ ****************************************************************/ #include <stdlib.h> /* malloc, free, qsort */ #include <string.h> /* memcpy, memset */ -#include "compiler.h" -#include "mem.h" /* U32, U16, etc. */ -#include "debug.h" /* assert, DEBUGLOG */ +#include "../common/compiler.h" +#include "../common/mem.h" /* U32, U16, etc. */ +#include "../common/debug.h" /* assert, DEBUGLOG */ #include "hist.h" /* HIST_count_wksp */ -#include "bitstream.h" +#include "../common/bitstream.h" #define FSE_STATIC_LINKING_ONLY -#include "fse.h" -#include "error_private.h" +#include "../common/fse.h" +#include "../common/error_private.h" /* ************************************************************** @@ -645,9 +625,6 @@ size_t FSE_compress_usingCTable (void* dst, size_t dstSize, size_t FSE_compressBound(size_t size) { return FSE_COMPRESSBOUND(size); } -#define CHECK_V_F(e, f) size_t const e = f; if (ERR_isError(e)) return e -#define CHECK_F(f) { CHECK_V_F(_var_err__, f); } - /* FSE_compress_wksp() : * Same as FSE_compress2(), but using an externally allocated scratch buffer (`workSpace`). * `wkspSize` size must be `(1<<tableLog)`. diff --git a/thirdparty/zstd/compress/hist.c b/thirdparty/zstd/compress/hist.c index 45b7babc1e..61e08c7968 100644 --- a/thirdparty/zstd/compress/hist.c +++ b/thirdparty/zstd/compress/hist.c @@ -1,42 +1,22 @@ /* ****************************************************************** - hist : Histogram functions - part of Finite State Entropy project - Copyright (C) 2013-present, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy - - Public forum : https://groups.google.com/forum/#!forum/lz4c + * hist : Histogram functions + * part of Finite State Entropy project + * Copyright (c) 2013-2020, Yann Collet, Facebook, Inc. + * + * You can contact the author at : + * - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy + * - Public forum : https://groups.google.com/forum/#!forum/lz4c + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. ****************************************************************** */ /* --- dependencies --- */ -#include "mem.h" /* U32, BYTE, etc. */ -#include "debug.h" /* assert, DEBUGLOG */ -#include "error_private.h" /* ERROR */ +#include "../common/mem.h" /* U32, BYTE, etc. */ +#include "../common/debug.h" /* assert, DEBUGLOG */ +#include "../common/error_private.h" /* ERROR */ #include "hist.h" diff --git a/thirdparty/zstd/compress/hist.h b/thirdparty/zstd/compress/hist.h index 8b389358dc..77e3ec4fb1 100644 --- a/thirdparty/zstd/compress/hist.h +++ b/thirdparty/zstd/compress/hist.h @@ -1,36 +1,16 @@ /* ****************************************************************** - hist : Histogram functions - part of Finite State Entropy project - Copyright (C) 2013-present, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy - - Public forum : https://groups.google.com/forum/#!forum/lz4c + * hist : Histogram functions + * part of Finite State Entropy project + * Copyright (c) 2013-2020, Yann Collet, Facebook, Inc. + * + * You can contact the author at : + * - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy + * - Public forum : https://groups.google.com/forum/#!forum/lz4c + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. ****************************************************************** */ /* --- dependencies --- */ diff --git a/thirdparty/zstd/compress/huf_compress.c b/thirdparty/zstd/compress/huf_compress.c index f074f1e0a9..546879868a 100644 --- a/thirdparty/zstd/compress/huf_compress.c +++ b/thirdparty/zstd/compress/huf_compress.c @@ -1,35 +1,15 @@ /* ****************************************************************** - Huffman encoder, part of New Generation Entropy library - Copyright (C) 2013-2016, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy - - Public forum : https://groups.google.com/forum/#!forum/lz4c + * Huffman encoder, part of New Generation Entropy library + * Copyright (c) 2013-2020, Yann Collet, Facebook, Inc. + * + * You can contact the author at : + * - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy + * - Public forum : https://groups.google.com/forum/#!forum/lz4c + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. ****************************************************************** */ /* ************************************************************** @@ -45,14 +25,14 @@ ****************************************************************/ #include <string.h> /* memcpy, memset */ #include <stdio.h> /* printf (debug) */ -#include "compiler.h" -#include "bitstream.h" +#include "../common/compiler.h" +#include "../common/bitstream.h" #include "hist.h" #define FSE_STATIC_LINKING_ONLY /* FSE_optimalTableLog_internal */ -#include "fse.h" /* header compression */ +#include "../common/fse.h" /* header compression */ #define HUF_STATIC_LINKING_ONLY -#include "huf.h" -#include "error_private.h" +#include "../common/huf.h" +#include "../common/error_private.h" /* ************************************************************** @@ -60,8 +40,6 @@ ****************************************************************/ #define HUF_isError ERR_isError #define HUF_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c) /* use only *after* variable declarations */ -#define CHECK_V_F(e, f) size_t const e = f; if (ERR_isError(e)) return e -#define CHECK_F(f) { CHECK_V_F(_var_err__, f); } /* ************************************************************** @@ -110,18 +88,18 @@ static size_t HUF_compressWeights (void* dst, size_t dstSize, const void* weight CHECK_F( FSE_normalizeCount(norm, tableLog, count, wtSize, maxSymbolValue) ); /* Write table description header */ - { CHECK_V_F(hSize, FSE_writeNCount(op, oend-op, norm, maxSymbolValue, tableLog) ); + { CHECK_V_F(hSize, FSE_writeNCount(op, (size_t)(oend-op), norm, maxSymbolValue, tableLog) ); op += hSize; } /* Compress */ CHECK_F( FSE_buildCTable_wksp(CTable, norm, maxSymbolValue, tableLog, scratchBuffer, sizeof(scratchBuffer)) ); - { CHECK_V_F(cSize, FSE_compress_usingCTable(op, oend - op, weightTable, wtSize, CTable) ); + { CHECK_V_F(cSize, FSE_compress_usingCTable(op, (size_t)(oend - op), weightTable, wtSize, CTable) ); if (cSize == 0) return 0; /* not enough space for compressed data */ op += cSize; } - return op-ostart; + return (size_t)(op-ostart); } @@ -169,7 +147,7 @@ size_t HUF_writeCTable (void* dst, size_t maxDstSize, } -size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize) +size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize, unsigned* hasZeroWeights) { BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1]; /* init not required, even though some static analyzer may complain */ U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; /* large enough for values from 0 to 16 */ @@ -192,9 +170,11 @@ size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void } } /* fill nbBits */ + *hasZeroWeights = 0; { U32 n; for (n=0; n<nbSymbols; n++) { const U32 w = huffWeight[n]; - CTable[n].nbBits = (BYTE)(tableLog + 1 - w); + *hasZeroWeights |= (w == 0); + CTable[n].nbBits = (BYTE)(tableLog + 1 - w) & -(w != 0); } } /* fill val */ @@ -240,7 +220,7 @@ static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits) /* there are several too large elements (at least >= 2) */ { int totalCost = 0; const U32 baseCost = 1 << (largestBits - maxNbBits); - U32 n = lastNonNull; + int n = (int)lastNonNull; while (huffNode[n].nbBits > maxNbBits) { totalCost += baseCost - (1 << (largestBits - huffNode[n].nbBits)); @@ -255,22 +235,22 @@ static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits) /* repay normalized cost */ { U32 const noSymbol = 0xF0F0F0F0; U32 rankLast[HUF_TABLELOG_MAX+2]; - int pos; /* Get pos of last (smallest) symbol per rank */ memset(rankLast, 0xF0, sizeof(rankLast)); { U32 currentNbBits = maxNbBits; + int pos; for (pos=n ; pos >= 0; pos--) { if (huffNode[pos].nbBits >= currentNbBits) continue; currentNbBits = huffNode[pos].nbBits; /* < maxNbBits */ - rankLast[maxNbBits-currentNbBits] = pos; + rankLast[maxNbBits-currentNbBits] = (U32)pos; } } while (totalCost > 0) { - U32 nBitsToDecrease = BIT_highbit32(totalCost) + 1; + U32 nBitsToDecrease = BIT_highbit32((U32)totalCost) + 1; for ( ; nBitsToDecrease > 1; nBitsToDecrease--) { - U32 highPos = rankLast[nBitsToDecrease]; - U32 lowPos = rankLast[nBitsToDecrease-1]; + U32 const highPos = rankLast[nBitsToDecrease]; + U32 const lowPos = rankLast[nBitsToDecrease-1]; if (highPos == noSymbol) continue; if (lowPos == noSymbol) break; { U32 const highTotal = huffNode[highPos].count; @@ -297,7 +277,8 @@ static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits) if (rankLast[1] == noSymbol) { /* special case : no rank 1 symbol (using maxNbBits-1); let's create one from largest rank 0 (using maxNbBits) */ while (huffNode[n].nbBits == maxNbBits) n--; huffNode[n+1].nbBits--; - rankLast[1] = n+1; + assert(n >= 0); + rankLast[1] = (U32)(n+1); totalCost++; continue; } @@ -309,29 +290,36 @@ static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits) return maxNbBits; } - typedef struct { U32 base; U32 current; } rankPos; -static void HUF_sort(nodeElt* huffNode, const unsigned* count, U32 maxSymbolValue) +typedef nodeElt huffNodeTable[HUF_CTABLE_WORKSPACE_SIZE_U32]; + +#define RANK_POSITION_TABLE_SIZE 32 + +typedef struct { + huffNodeTable huffNodeTbl; + rankPos rankPosition[RANK_POSITION_TABLE_SIZE]; +} HUF_buildCTable_wksp_tables; + +static void HUF_sort(nodeElt* huffNode, const unsigned* count, U32 maxSymbolValue, rankPos* rankPosition) { - rankPos rank[32]; U32 n; - memset(rank, 0, sizeof(rank)); + memset(rankPosition, 0, sizeof(*rankPosition) * RANK_POSITION_TABLE_SIZE); for (n=0; n<=maxSymbolValue; n++) { U32 r = BIT_highbit32(count[n] + 1); - rank[r].base ++; + rankPosition[r].base ++; } - for (n=30; n>0; n--) rank[n-1].base += rank[n].base; - for (n=0; n<32; n++) rank[n].current = rank[n].base; + for (n=30; n>0; n--) rankPosition[n-1].base += rankPosition[n].base; + for (n=0; n<32; n++) rankPosition[n].current = rankPosition[n].base; for (n=0; n<=maxSymbolValue; n++) { U32 const c = count[n]; U32 const r = BIT_highbit32(c+1) + 1; - U32 pos = rank[r].current++; - while ((pos > rank[r].base) && (c > huffNode[pos-1].count)) { + U32 pos = rankPosition[r].current++; + while ((pos > rankPosition[r].base) && (c > huffNode[pos-1].count)) { huffNode[pos] = huffNode[pos-1]; pos--; } @@ -343,45 +331,48 @@ static void HUF_sort(nodeElt* huffNode, const unsigned* count, U32 maxSymbolValu /** HUF_buildCTable_wksp() : * Same as HUF_buildCTable(), but using externally allocated scratch buffer. - * `workSpace` must be aligned on 4-bytes boundaries, and be at least as large as a table of HUF_CTABLE_WORKSPACE_SIZE_U32 unsigned. + * `workSpace` must be aligned on 4-bytes boundaries, and be at least as large as sizeof(HUF_buildCTable_wksp_tables). */ #define STARTNODE (HUF_SYMBOLVALUE_MAX+1) -typedef nodeElt huffNodeTable[HUF_CTABLE_WORKSPACE_SIZE_U32]; + size_t HUF_buildCTable_wksp (HUF_CElt* tree, const unsigned* count, U32 maxSymbolValue, U32 maxNbBits, void* workSpace, size_t wkspSize) { - nodeElt* const huffNode0 = (nodeElt*)workSpace; + HUF_buildCTable_wksp_tables* const wksp_tables = (HUF_buildCTable_wksp_tables*)workSpace; + nodeElt* const huffNode0 = wksp_tables->huffNodeTbl; nodeElt* const huffNode = huffNode0+1; - U32 n, nonNullRank; + int nonNullRank; int lowS, lowN; - U16 nodeNb = STARTNODE; - U32 nodeRoot; + int nodeNb = STARTNODE; + int n, nodeRoot; /* safety checks */ if (((size_t)workSpace & 3) != 0) return ERROR(GENERIC); /* must be aligned on 4-bytes boundaries */ - if (wkspSize < sizeof(huffNodeTable)) return ERROR(workSpace_tooSmall); + if (wkspSize < sizeof(HUF_buildCTable_wksp_tables)) + return ERROR(workSpace_tooSmall); if (maxNbBits == 0) maxNbBits = HUF_TABLELOG_DEFAULT; - if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(maxSymbolValue_tooLarge); + if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) + return ERROR(maxSymbolValue_tooLarge); memset(huffNode0, 0, sizeof(huffNodeTable)); /* sort, decreasing order */ - HUF_sort(huffNode, count, maxSymbolValue); + HUF_sort(huffNode, count, maxSymbolValue, wksp_tables->rankPosition); /* init for parents */ - nonNullRank = maxSymbolValue; + nonNullRank = (int)maxSymbolValue; while(huffNode[nonNullRank].count == 0) nonNullRank--; lowS = nonNullRank; nodeRoot = nodeNb + lowS - 1; lowN = nodeNb; huffNode[nodeNb].count = huffNode[lowS].count + huffNode[lowS-1].count; - huffNode[lowS].parent = huffNode[lowS-1].parent = nodeNb; + huffNode[lowS].parent = huffNode[lowS-1].parent = (U16)nodeNb; nodeNb++; lowS-=2; for (n=nodeNb; n<=nodeRoot; n++) huffNode[n].count = (U32)(1U<<30); huffNode0[0].count = (U32)(1U<<31); /* fake entry, strong barrier */ /* create parents */ while (nodeNb <= nodeRoot) { - U32 n1 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++; - U32 n2 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++; + int const n1 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++; + int const n2 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++; huffNode[nodeNb].count = huffNode[n1].count + huffNode[n2].count; - huffNode[n1].parent = huffNode[n2].parent = nodeNb; + huffNode[n1].parent = huffNode[n2].parent = (U16)nodeNb; nodeNb++; } @@ -393,24 +384,25 @@ size_t HUF_buildCTable_wksp (HUF_CElt* tree, const unsigned* count, U32 maxSymbo huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1; /* enforce maxTableLog */ - maxNbBits = HUF_setMaxHeight(huffNode, nonNullRank, maxNbBits); + maxNbBits = HUF_setMaxHeight(huffNode, (U32)nonNullRank, maxNbBits); /* fill result into tree (val, nbBits) */ { U16 nbPerRank[HUF_TABLELOG_MAX+1] = {0}; U16 valPerRank[HUF_TABLELOG_MAX+1] = {0}; + int const alphabetSize = (int)(maxSymbolValue + 1); if (maxNbBits > HUF_TABLELOG_MAX) return ERROR(GENERIC); /* check fit into table */ for (n=0; n<=nonNullRank; n++) nbPerRank[huffNode[n].nbBits]++; /* determine stating value per rank */ { U16 min = 0; - for (n=maxNbBits; n>0; n--) { + for (n=(int)maxNbBits; n>0; n--) { valPerRank[n] = min; /* get starting value within each rank */ min += nbPerRank[n]; min >>= 1; } } - for (n=0; n<=maxSymbolValue; n++) + for (n=0; n<alphabetSize; n++) tree[huffNode[n].byte].nbBits = huffNode[n].nbBits; /* push nbBits per symbol, symbol order */ - for (n=0; n<=maxSymbolValue; n++) + for (n=0; n<alphabetSize; n++) tree[n].val = valPerRank[tree[n].nbBits]++; /* assign value within rank, symbol order */ } @@ -423,11 +415,11 @@ size_t HUF_buildCTable_wksp (HUF_CElt* tree, const unsigned* count, U32 maxSymbo */ size_t HUF_buildCTable (HUF_CElt* tree, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits) { - huffNodeTable nodeTable; - return HUF_buildCTable_wksp(tree, count, maxSymbolValue, maxNbBits, nodeTable, sizeof(nodeTable)); + HUF_buildCTable_wksp_tables workspace; + return HUF_buildCTable_wksp(tree, count, maxSymbolValue, maxNbBits, &workspace, sizeof(workspace)); } -static size_t HUF_estimateCompressedSize(HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue) +size_t HUF_estimateCompressedSize(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue) { size_t nbBits = 0; int s; @@ -437,7 +429,7 @@ static size_t HUF_estimateCompressedSize(HUF_CElt* CTable, const unsigned* count return nbBits >> 3; } -static int HUF_validateCTable(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue) { +int HUF_validateCTable(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue) { int bad = 0; int s; for (s = 0; s <= (int)maxSymbolValue; ++s) { @@ -476,7 +468,7 @@ HUF_compress1X_usingCTable_internal_body(void* dst, size_t dstSize, /* init */ if (dstSize < 8) return 0; /* not enough space to compress */ - { size_t const initErr = BIT_initCStream(&bitC, op, oend-op); + { size_t const initErr = BIT_initCStream(&bitC, op, (size_t)(oend-op)); if (HUF_isError(initErr)) return 0; } n = srcSize & ~3; /* join to mod 4 */ @@ -573,7 +565,8 @@ HUF_compress4X_usingCTable_internal(void* dst, size_t dstSize, if (srcSize < 12) return 0; /* no saving possible : too small input */ op += 6; /* jumpTable */ - { CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, oend-op, ip, segmentSize, CTable, bmi2) ); + assert(op <= oend); + { CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, bmi2) ); if (cSize==0) return 0; assert(cSize <= 65535); MEM_writeLE16(ostart, (U16)cSize); @@ -581,7 +574,8 @@ HUF_compress4X_usingCTable_internal(void* dst, size_t dstSize, } ip += segmentSize; - { CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, oend-op, ip, segmentSize, CTable, bmi2) ); + assert(op <= oend); + { CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, bmi2) ); if (cSize==0) return 0; assert(cSize <= 65535); MEM_writeLE16(ostart+2, (U16)cSize); @@ -589,7 +583,8 @@ HUF_compress4X_usingCTable_internal(void* dst, size_t dstSize, } ip += segmentSize; - { CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, oend-op, ip, segmentSize, CTable, bmi2) ); + assert(op <= oend); + { CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, bmi2) ); if (cSize==0) return 0; assert(cSize <= 65535); MEM_writeLE16(ostart+4, (U16)cSize); @@ -597,12 +592,14 @@ HUF_compress4X_usingCTable_internal(void* dst, size_t dstSize, } ip += segmentSize; - { CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, oend-op, ip, iend-ip, CTable, bmi2) ); + assert(op <= oend); + assert(ip <= iend); + { CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, (size_t)(iend-ip), CTable, bmi2) ); if (cSize==0) return 0; op += cSize; } - return op-ostart; + return (size_t)(op-ostart); } size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable) @@ -618,20 +615,21 @@ static size_t HUF_compressCTable_internal( HUF_nbStreams_e nbStreams, const HUF_CElt* CTable, const int bmi2) { size_t const cSize = (nbStreams==HUF_singleStream) ? - HUF_compress1X_usingCTable_internal(op, oend - op, src, srcSize, CTable, bmi2) : - HUF_compress4X_usingCTable_internal(op, oend - op, src, srcSize, CTable, bmi2); + HUF_compress1X_usingCTable_internal(op, (size_t)(oend - op), src, srcSize, CTable, bmi2) : + HUF_compress4X_usingCTable_internal(op, (size_t)(oend - op), src, srcSize, CTable, bmi2); if (HUF_isError(cSize)) { return cSize; } if (cSize==0) { return 0; } /* uncompressible */ op += cSize; /* check compressibility */ + assert(op >= ostart); if ((size_t)(op-ostart) >= srcSize-1) { return 0; } - return op-ostart; + return (size_t)(op-ostart); } typedef struct { unsigned count[HUF_SYMBOLVALUE_MAX + 1]; HUF_CElt CTable[HUF_SYMBOLVALUE_MAX + 1]; - huffNodeTable nodeTable; + HUF_buildCTable_wksp_tables buildCTable_wksp; } HUF_compress_tables_t; /* HUF_compress_internal() : @@ -650,6 +648,8 @@ HUF_compress_internal (void* dst, size_t dstSize, BYTE* const oend = ostart + dstSize; BYTE* op = ostart; + HUF_STATIC_ASSERT(sizeof(*table) <= HUF_WORKSPACE_SIZE); + /* checks & inits */ if (((size_t)workSpace & 3) != 0) return ERROR(GENERIC); /* must be aligned on 4-bytes boundaries */ if (wkspSize < HUF_WORKSPACE_SIZE) return ERROR(workSpace_tooSmall); @@ -691,7 +691,7 @@ HUF_compress_internal (void* dst, size_t dstSize, huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue); { size_t const maxBits = HUF_buildCTable_wksp(table->CTable, table->count, maxSymbolValue, huffLog, - table->nodeTable, sizeof(table->nodeTable)); + &table->buildCTable_wksp, sizeof(table->buildCTable_wksp)); CHECK_F(maxBits); huffLog = (U32)maxBits; /* Zero unused symbols in CTable, so we can check it for validity */ diff --git a/thirdparty/zstd/compress/zstd_compress.c b/thirdparty/zstd/compress/zstd_compress.c index 35346b92cb..3f963b1cff 100644 --- a/thirdparty/zstd/compress/zstd_compress.c +++ b/thirdparty/zstd/compress/zstd_compress.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -13,13 +13,13 @@ ***************************************/ #include <limits.h> /* INT_MAX */ #include <string.h> /* memset */ -#include "cpu.h" -#include "mem.h" +#include "../common/cpu.h" +#include "../common/mem.h" #include "hist.h" /* HIST_countFast_wksp */ #define FSE_STATIC_LINKING_ONLY /* FSE_encodeSymbol */ -#include "fse.h" +#include "../common/fse.h" #define HUF_STATIC_LINKING_ONLY -#include "huf.h" +#include "../common/huf.h" #include "zstd_compress_internal.h" #include "zstd_compress_sequences.h" #include "zstd_compress_literals.h" @@ -28,11 +28,19 @@ #include "zstd_lazy.h" #include "zstd_opt.h" #include "zstd_ldm.h" +#include "zstd_compress_superblock.h" /*-************************************* * Helper functions ***************************************/ +/* ZSTD_compressBound() + * Note that the result from this function is only compatible with the "normal" + * full-block strategy. + * When there are a lot of small blocks due to frequent flush in streaming mode + * the overhead of headers can make the compressed data to be larger than the + * return value of ZSTD_compressBound(). + */ size_t ZSTD_compressBound(size_t srcSize) { return ZSTD_COMPRESSBOUND(srcSize); } @@ -82,7 +90,7 @@ ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem) } } -ZSTD_CCtx* ZSTD_initStaticCCtx(void *workspace, size_t workspaceSize) +ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize) { ZSTD_cwksp ws; ZSTD_CCtx* cctx; @@ -91,9 +99,8 @@ ZSTD_CCtx* ZSTD_initStaticCCtx(void *workspace, size_t workspaceSize) ZSTD_cwksp_init(&ws, workspace, workspaceSize); cctx = (ZSTD_CCtx*)ZSTD_cwksp_reserve_object(&ws, sizeof(ZSTD_CCtx)); - if (cctx == NULL) { - return NULL; - } + if (cctx == NULL) return NULL; + memset(cctx, 0, sizeof(ZSTD_CCtx)); ZSTD_cwksp_move(&cctx->workspace, &ws); cctx->staticSize = workspaceSize; @@ -102,8 +109,7 @@ ZSTD_CCtx* ZSTD_initStaticCCtx(void *workspace, size_t workspaceSize) if (!ZSTD_cwksp_check_available(&cctx->workspace, HUF_WORKSPACE_SIZE + 2 * sizeof(ZSTD_compressedBlockState_t))) return NULL; cctx->blockState.prevCBlock = (ZSTD_compressedBlockState_t*)ZSTD_cwksp_reserve_object(&cctx->workspace, sizeof(ZSTD_compressedBlockState_t)); cctx->blockState.nextCBlock = (ZSTD_compressedBlockState_t*)ZSTD_cwksp_reserve_object(&cctx->workspace, sizeof(ZSTD_compressedBlockState_t)); - cctx->entropyWorkspace = (U32*)ZSTD_cwksp_reserve_object( - &cctx->workspace, HUF_WORKSPACE_SIZE); + cctx->entropyWorkspace = (U32*)ZSTD_cwksp_reserve_object(&cctx->workspace, HUF_WORKSPACE_SIZE); cctx->bmi2 = ZSTD_cpuid_bmi2(ZSTD_cpuid()); return cctx; } @@ -227,7 +233,7 @@ size_t ZSTD_CCtxParams_reset(ZSTD_CCtx_params* params) } size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel) { - RETURN_ERROR_IF(!cctxParams, GENERIC); + RETURN_ERROR_IF(!cctxParams, GENERIC, "NULL pointer!"); memset(cctxParams, 0, sizeof(*cctxParams)); cctxParams->compressionLevel = compressionLevel; cctxParams->fParams.contentSizeFlag = 1; @@ -236,8 +242,8 @@ size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel) size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params) { - RETURN_ERROR_IF(!cctxParams, GENERIC); - FORWARD_IF_ERROR( ZSTD_checkCParams(params.cParams) ); + RETURN_ERROR_IF(!cctxParams, GENERIC, "NULL pointer!"); + FORWARD_IF_ERROR( ZSTD_checkCParams(params.cParams) , ""); memset(cctxParams, 0, sizeof(*cctxParams)); assert(!ZSTD_checkCParams(params.cParams)); cctxParams->cParams = params.cParams; @@ -249,12 +255,12 @@ size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_paramete /* ZSTD_assignParamsToCCtxParams() : * params is presumed valid at this stage */ static ZSTD_CCtx_params ZSTD_assignParamsToCCtxParams( - const ZSTD_CCtx_params* cctxParams, ZSTD_parameters params) + const ZSTD_CCtx_params* cctxParams, const ZSTD_parameters* params) { ZSTD_CCtx_params ret = *cctxParams; - assert(!ZSTD_checkCParams(params.cParams)); - ret.cParams = params.cParams; - ret.fParams = params.fParams; + assert(!ZSTD_checkCParams(params->cParams)); + ret.cParams = params->cParams; + ret.fParams = params->fParams; ret.compressionLevel = ZSTD_CLEVEL_DEFAULT; /* should not matter, as all cParams are presumed properly defined */ return ret; } @@ -339,8 +345,13 @@ ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param) return bounds; case ZSTD_c_overlapLog: +#ifdef ZSTD_MULTITHREAD bounds.lowerBound = ZSTD_OVERLAPLOG_MIN; bounds.upperBound = ZSTD_OVERLAPLOG_MAX; +#else + bounds.lowerBound = 0; + bounds.upperBound = 0; +#endif return bounds; case ZSTD_c_enableLongDistanceMatching: @@ -408,9 +419,8 @@ ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param) return bounds; default: - { ZSTD_bounds const boundError = { ERROR(parameter_unsupported), 0, 0 }; - return boundError; - } + bounds.error = ERROR(parameter_unsupported); + return bounds; } } @@ -428,7 +438,7 @@ static size_t ZSTD_cParam_clampBounds(ZSTD_cParameter cParam, int* value) #define BOUNDCHECK(cParam, val) { \ RETURN_ERROR_IF(!ZSTD_cParam_withinBounds(cParam,val), \ - parameter_outOfBound); \ + parameter_outOfBound, "Param out of bounds"); \ } @@ -476,7 +486,7 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value) if (ZSTD_isUpdateAuthorized(param)) { cctx->cParamsChanged = 1; } else { - RETURN_ERROR(stage_wrong); + RETURN_ERROR(stage_wrong, "can only set params in ctx init stage"); } } switch(param) @@ -513,7 +523,7 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value) case ZSTD_c_srcSizeHint: break; - default: RETURN_ERROR(parameter_unsupported); + default: RETURN_ERROR(parameter_unsupported, "unknown parameter"); } return ZSTD_CCtxParams_setParameter(&cctx->requestedParams, param, value); } @@ -530,7 +540,7 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams, return (size_t)CCtxParams->format; case ZSTD_c_compressionLevel : { - FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(param, &value)); + FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(param, &value), ""); if (value) { /* 0 : does not change current level */ CCtxParams->compressionLevel = value; } @@ -618,7 +628,7 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams, RETURN_ERROR_IF(value!=0, parameter_unsupported, "not compiled with multithreading"); return 0; #else - FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(param, &value)); + FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(param, &value), ""); CCtxParams->nbWorkers = value; return CCtxParams->nbWorkers; #endif @@ -631,7 +641,7 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams, /* Adjust to the minimum non-default value. */ if (value != 0 && value < ZSTDMT_JOBSIZE_MIN) value = ZSTDMT_JOBSIZE_MIN; - FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(param, &value)); + FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(param, &value), ""); assert(value >= 0); CCtxParams->jobSize = value; return CCtxParams->jobSize; @@ -642,7 +652,7 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams, RETURN_ERROR_IF(value!=0, parameter_unsupported, "not compiled with multithreading"); return 0; #else - FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(ZSTD_c_overlapLog, &value)); + FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(ZSTD_c_overlapLog, &value), ""); CCtxParams->overlapLog = value; return CCtxParams->overlapLog; #endif @@ -652,7 +662,7 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams, RETURN_ERROR_IF(value!=0, parameter_unsupported, "not compiled with multithreading"); return 0; #else - FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(ZSTD_c_overlapLog, &value)); + FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(ZSTD_c_overlapLog, &value), ""); CCtxParams->rsyncable = value; return CCtxParams->rsyncable; #endif @@ -681,7 +691,7 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams, case ZSTD_c_ldmHashRateLog : RETURN_ERROR_IF(value > ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN, - parameter_outOfBound); + parameter_outOfBound, "Param out of bounds!"); CCtxParams->ldmParams.hashRateLog = value; return CCtxParams->ldmParams.hashRateLog; @@ -821,8 +831,11 @@ size_t ZSTD_CCtx_setParametersUsingCCtxParams( ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params) { DEBUGLOG(4, "ZSTD_CCtx_setParametersUsingCCtxParams"); - RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong); - RETURN_ERROR_IF(cctx->cdict, stage_wrong); + RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, + "The context is in the wrong stage!"); + RETURN_ERROR_IF(cctx->cdict, stage_wrong, + "Can't override parameters with cdict attached (some must " + "be inherited from the cdict)."); cctx->requestedParams = *params; return 0; @@ -831,7 +844,8 @@ size_t ZSTD_CCtx_setParametersUsingCCtxParams( ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize) { DEBUGLOG(4, "ZSTD_CCtx_setPledgedSrcSize to %u bytes", (U32)pledgedSrcSize); - RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong); + RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, + "Can't set pledgedSrcSize when not in init stage."); cctx->pledgedSrcSizePlusOne = pledgedSrcSize+1; return 0; } @@ -845,7 +859,7 @@ static size_t ZSTD_initLocalDict(ZSTD_CCtx* cctx) { ZSTD_localDict* const dl = &cctx->localDict; ZSTD_compressionParameters const cParams = ZSTD_getCParamsFromCCtxParams( - &cctx->requestedParams, 0, dl->dictSize); + &cctx->requestedParams, ZSTD_CONTENTSIZE_UNKNOWN, dl->dictSize); if (dl->dict == NULL) { /* No local dictionary. */ assert(dl->dictBuffer == NULL); @@ -869,7 +883,7 @@ static size_t ZSTD_initLocalDict(ZSTD_CCtx* cctx) dl->dictContentType, cParams, cctx->customMem); - RETURN_ERROR_IF(!dl->cdict, memory_allocation); + RETURN_ERROR_IF(!dl->cdict, memory_allocation, "ZSTD_createCDict_advanced failed"); cctx->cdict = dl->cdict; return 0; } @@ -878,7 +892,8 @@ size_t ZSTD_CCtx_loadDictionary_advanced( ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType) { - RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong); + RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, + "Can't load a dictionary when ctx is not in init stage."); RETURN_ERROR_IF(cctx->staticSize, memory_allocation, "no malloc for static CCtx"); DEBUGLOG(4, "ZSTD_CCtx_loadDictionary_advanced (size: %u)", (U32)dictSize); @@ -889,7 +904,7 @@ size_t ZSTD_CCtx_loadDictionary_advanced( cctx->localDict.dict = dict; } else { void* dictBuffer = ZSTD_malloc(dictSize, cctx->customMem); - RETURN_ERROR_IF(!dictBuffer, memory_allocation); + RETURN_ERROR_IF(!dictBuffer, memory_allocation, "NULL pointer!"); memcpy(dictBuffer, dict, dictSize); cctx->localDict.dictBuffer = dictBuffer; cctx->localDict.dict = dictBuffer; @@ -915,7 +930,8 @@ ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, s size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict) { - RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong); + RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, + "Can't ref a dict when ctx not in init stage."); /* Free the existing local cdict (if any) to save memory. */ ZSTD_clearAllDicts(cctx); cctx->cdict = cdict; @@ -930,11 +946,14 @@ size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSiz size_t ZSTD_CCtx_refPrefix_advanced( ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType) { - RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong); + RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, + "Can't ref a prefix when ctx not in init stage."); ZSTD_clearAllDicts(cctx); - cctx->prefixDict.dict = prefix; - cctx->prefixDict.dictSize = prefixSize; - cctx->prefixDict.dictContentType = dictContentType; + if (prefix != NULL && prefixSize > 0) { + cctx->prefixDict.dict = prefix; + cctx->prefixDict.dictSize = prefixSize; + cctx->prefixDict.dictContentType = dictContentType; + } return 0; } @@ -949,7 +968,8 @@ size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset) } if ( (reset == ZSTD_reset_parameters) || (reset == ZSTD_reset_session_and_parameters) ) { - RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong); + RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, + "Can't reset parameters only when not in init stage."); ZSTD_clearAllDicts(cctx); return ZSTD_CCtxParams_reset(&cctx->requestedParams); } @@ -996,7 +1016,7 @@ ZSTD_clampCParams(ZSTD_compressionParameters cParams) /** ZSTD_cycleLog() : * condition for correct operation : hashLog > 1 */ -static U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat) +U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat) { U32 const btScale = ((U32)strat >= (U32)ZSTD_btlazy2); return hashLog - btScale; @@ -1006,7 +1026,7 @@ static U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat) * optimize `cPar` for a specified input (`srcSize` and `dictSize`). * mostly downsize to reduce memory consumption and initialization latency. * `srcSize` can be ZSTD_CONTENTSIZE_UNKNOWN when not known. - * note : for the time being, `srcSize==0` means "unknown" too, for compatibility with older convention. + * note : `srcSize==0` means 0! * condition : cPar is presumed validated (can be checked using ZSTD_checkCParams()). */ static ZSTD_compressionParameters ZSTD_adjustCParams_internal(ZSTD_compressionParameters cPar, @@ -1017,10 +1037,8 @@ ZSTD_adjustCParams_internal(ZSTD_compressionParameters cPar, static const U64 maxWindowResize = 1ULL << (ZSTD_WINDOWLOG_MAX-1); assert(ZSTD_checkCParams(cPar)==0); - if (dictSize && (srcSize+1<2) /* ZSTD_CONTENTSIZE_UNKNOWN and 0 mean "unknown" */ ) - srcSize = minSrcSize; /* presumed small when there is a dictionary */ - else if (srcSize == 0) - srcSize = ZSTD_CONTENTSIZE_UNKNOWN; /* 0 == unknown : presumed large */ + if (dictSize && srcSize == ZSTD_CONTENTSIZE_UNKNOWN) + srcSize = minSrcSize; /* resize windowLog if input is small enough, to use less memory */ if ( (srcSize < maxWindowResize) @@ -1049,9 +1067,13 @@ ZSTD_adjustCParams(ZSTD_compressionParameters cPar, size_t dictSize) { cPar = ZSTD_clampCParams(cPar); /* resulting cPar is necessarily valid (all parameters within range) */ + if (srcSize == 0) srcSize = ZSTD_CONTENTSIZE_UNKNOWN; return ZSTD_adjustCParams_internal(cPar, srcSize, dictSize); } +static ZSTD_compressionParameters ZSTD_getCParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize); +static ZSTD_parameters ZSTD_getParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize); + ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams( const ZSTD_CCtx_params* CCtxParams, U64 srcSizeHint, size_t dictSize) { @@ -1059,7 +1081,7 @@ ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams( if (srcSizeHint == ZSTD_CONTENTSIZE_UNKNOWN && CCtxParams->srcSizeHint > 0) { srcSizeHint = CCtxParams->srcSizeHint; } - cParams = ZSTD_getCParams(CCtxParams->compressionLevel, srcSizeHint, dictSize); + cParams = ZSTD_getCParams_internal(CCtxParams->compressionLevel, srcSizeHint, dictSize); if (CCtxParams->ldmParams.enableLdm) cParams.windowLog = ZSTD_LDM_DEFAULT_WINDOW_LOG; if (CCtxParams->cParams.windowLog) cParams.windowLog = CCtxParams->cParams.windowLog; if (CCtxParams->cParams.hashLog) cParams.hashLog = CCtxParams->cParams.hashLog; @@ -1069,6 +1091,7 @@ ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams( if (CCtxParams->cParams.targetLength) cParams.targetLength = CCtxParams->cParams.targetLength; if (CCtxParams->cParams.strategy) cParams.strategy = CCtxParams->cParams.strategy; assert(!ZSTD_checkCParams(cParams)); + /* srcSizeHint == 0 means 0 */ return ZSTD_adjustCParams_internal(cParams, srcSizeHint, dictSize); } @@ -1104,7 +1127,7 @@ size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params) { RETURN_ERROR_IF(params->nbWorkers > 0, GENERIC, "Estimate CCtx size is supported for single-threaded compression only."); { ZSTD_compressionParameters const cParams = - ZSTD_getCParamsFromCCtxParams(params, 0, 0); + ZSTD_getCParamsFromCCtxParams(params, ZSTD_CONTENTSIZE_UNKNOWN, 0); size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << cParams.windowLog); U32 const divider = (cParams.minMatch==3) ? 3 : 4; size_t const maxNbSeq = blockSize / divider; @@ -1118,13 +1141,26 @@ size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params) size_t const ldmSpace = ZSTD_ldm_getTableSize(params->ldmParams); size_t const ldmSeqSpace = ZSTD_cwksp_alloc_size(ZSTD_ldm_getMaxNbSeq(params->ldmParams, blockSize) * sizeof(rawSeq)); - size_t const neededSpace = entropySpace + blockStateSpace + tokenSpace + - matchStateSize + ldmSpace + ldmSeqSpace; + /* estimateCCtxSize is for one-shot compression. So no buffers should + * be needed. However, we still allocate two 0-sized buffers, which can + * take space under ASAN. */ + size_t const bufferSpace = ZSTD_cwksp_alloc_size(0) + + ZSTD_cwksp_alloc_size(0); + size_t const cctxSpace = ZSTD_cwksp_alloc_size(sizeof(ZSTD_CCtx)); - DEBUGLOG(5, "sizeof(ZSTD_CCtx) : %u", (U32)cctxSpace); + size_t const neededSpace = + cctxSpace + + entropySpace + + blockStateSpace + + ldmSpace + + ldmSeqSpace + + matchStateSize + + tokenSpace + + bufferSpace; + DEBUGLOG(5, "estimate workspace : %u", (U32)neededSpace); - return cctxSpace + neededSpace; + return neededSpace; } } @@ -1136,7 +1172,7 @@ size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams) static size_t ZSTD_estimateCCtxSize_internal(int compressionLevel) { - ZSTD_compressionParameters const cParams = ZSTD_getCParams(compressionLevel, 0, 0); + ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, 0); return ZSTD_estimateCCtxSize_usingCParams(cParams); } @@ -1155,7 +1191,7 @@ size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params) { RETURN_ERROR_IF(params->nbWorkers > 0, GENERIC, "Estimate CCtx size is supported for single-threaded compression only."); { ZSTD_compressionParameters const cParams = - ZSTD_getCParamsFromCCtxParams(params, 0, 0); + ZSTD_getCParamsFromCCtxParams(params, ZSTD_CONTENTSIZE_UNKNOWN, 0); size_t const CCtxSize = ZSTD_estimateCCtxSize_usingCCtxParams(params); size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << cParams.windowLog); size_t const inBuffSize = ((size_t)1 << cParams.windowLog) + blockSize; @@ -1175,7 +1211,7 @@ size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams) static size_t ZSTD_estimateCStreamSize_internal(int compressionLevel) { - ZSTD_compressionParameters const cParams = ZSTD_getCParams(compressionLevel, 0, 0); + ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, 0); return ZSTD_estimateCStreamSize_usingCParams(cParams); } @@ -1243,7 +1279,7 @@ static void ZSTD_assertEqualCParams(ZSTD_compressionParameters cParams1, assert(cParams1.strategy == cParams2.strategy); } -static void ZSTD_reset_compressedBlockState(ZSTD_compressedBlockState_t* bs) +void ZSTD_reset_compressedBlockState(ZSTD_compressedBlockState_t* bs) { int i; for (i = 0; i < ZSTD_REP_NUM; ++i) @@ -1320,10 +1356,7 @@ ZSTD_reset_matchState(ZSTD_matchState_t* ms, DEBUGLOG(4, "reset indices : %u", forceResetIndex == ZSTDirp_reset); if (forceResetIndex == ZSTDirp_reset) { - memset(&ms->window, 0, sizeof(ms->window)); - ms->window.dictLimit = 1; /* start from 1, so that 1st position is valid */ - ms->window.lowLimit = 1; /* it ensures first and later CCtx usages compress the same */ - ms->window.nextSrc = ms->window.base + 1; /* see issue #1241 */ + ZSTD_window_init(&ms->window); ZSTD_cwksp_mark_tables_dirty(ws); } @@ -1416,13 +1449,13 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, size_t const matchStateSize = ZSTD_sizeof_matchState(¶ms.cParams, /* forCCtx */ 1); size_t const maxNbLdmSeq = ZSTD_ldm_getMaxNbSeq(params.ldmParams, blockSize); - ZSTD_indexResetPolicy_e needsIndexReset = ZSTDirp_continue; + ZSTD_indexResetPolicy_e needsIndexReset = zc->initialized ? ZSTDirp_continue : ZSTDirp_reset; if (ZSTD_indexTooCloseToMax(zc->blockState.matchState.window)) { needsIndexReset = ZSTDirp_reset; } - ZSTD_cwksp_bump_oversized_duration(ws, 0); + if (!zc->staticSize) ZSTD_cwksp_bump_oversized_duration(ws, 0); /* Check if workspace is large enough, alloc a new one if needed */ { size_t const cctxSpace = zc->staticSize ? ZSTD_cwksp_alloc_size(sizeof(ZSTD_CCtx)) : 0; @@ -1459,7 +1492,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, needsIndexReset = ZSTDirp_reset; ZSTD_cwksp_free(ws, zc->customMem); - FORWARD_IF_ERROR(ZSTD_cwksp_create(ws, neededSpace, zc->customMem)); + FORWARD_IF_ERROR(ZSTD_cwksp_create(ws, neededSpace, zc->customMem), ""); DEBUGLOG(5, "reserving object space"); /* Statically sized space. @@ -1530,7 +1563,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, ¶ms.cParams, crp, needsIndexReset, - ZSTD_resetTarget_CCtx)); + ZSTD_resetTarget_CCtx), ""); /* ldm hash table */ if (params.ldmParams.enableLdm) { @@ -1541,11 +1574,13 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, zc->ldmSequences = (rawSeq*)ZSTD_cwksp_reserve_aligned(ws, maxNbLdmSeq * sizeof(rawSeq)); zc->maxNbLdmSequences = maxNbLdmSeq; - memset(&zc->ldmState.window, 0, sizeof(zc->ldmState.window)); + ZSTD_window_init(&zc->ldmState.window); ZSTD_window_clear(&zc->ldmState.window); + zc->ldmState.loadedDictEnd = 0; } DEBUGLOG(3, "wksp: finished allocating, %zd bytes remain available", ZSTD_cwksp_available_space(ws)); + zc->initialized = 1; return 0; } @@ -1603,10 +1638,11 @@ ZSTD_resetCCtx_byAttachingCDict(ZSTD_CCtx* cctx, assert(windowLog != 0); /* Resize working context table params for input only, since the dict * has its own tables. */ + /* pledgeSrcSize == 0 means 0! */ params.cParams = ZSTD_adjustCParams_internal(*cdict_cParams, pledgedSrcSize, 0); params.cParams.windowLog = windowLog; FORWARD_IF_ERROR(ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize, - ZSTDcrp_makeClean, zbuff)); + ZSTDcrp_makeClean, zbuff), ""); assert(cctx->appliedParams.cParams.strategy == cdict_cParams->strategy); } @@ -1655,7 +1691,7 @@ static size_t ZSTD_resetCCtx_byCopyingCDict(ZSTD_CCtx* cctx, params.cParams = *cdict_cParams; params.cParams.windowLog = windowLog; FORWARD_IF_ERROR(ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize, - ZSTDcrp_leaveDirty, zbuff)); + ZSTDcrp_leaveDirty, zbuff), ""); assert(cctx->appliedParams.cParams.strategy == cdict_cParams->strategy); assert(cctx->appliedParams.cParams.hashLog == cdict_cParams->hashLog); assert(cctx->appliedParams.cParams.chainLog == cdict_cParams->chainLog); @@ -1736,7 +1772,8 @@ static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx, ZSTD_buffered_policy_e zbuff) { DEBUGLOG(5, "ZSTD_copyCCtx_internal"); - RETURN_ERROR_IF(srcCCtx->stage!=ZSTDcs_init, stage_wrong); + RETURN_ERROR_IF(srcCCtx->stage!=ZSTDcs_init, stage_wrong, + "Can't copy a ctx that's not in init stage."); memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem)); { ZSTD_CCtx_params params = dstCCtx->requestedParams; @@ -1889,16 +1926,6 @@ static void ZSTD_reduceIndex (ZSTD_matchState_t* ms, ZSTD_CCtx_params const* par /* See doc/zstd_compression_format.md for detailed format description */ -static size_t ZSTD_noCompressBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize, U32 lastBlock) -{ - U32 const cBlockHeader24 = lastBlock + (((U32)bt_raw)<<1) + (U32)(srcSize << 3); - RETURN_ERROR_IF(srcSize + ZSTD_blockHeaderSize > dstCapacity, - dstSize_tooSmall); - MEM_writeLE24(dst, cBlockHeader24); - memcpy((BYTE*)dst + ZSTD_blockHeaderSize, src, srcSize); - return ZSTD_blockHeaderSize + srcSize; -} - void ZSTD_seqToCodes(const seqStore_t* seqStorePtr) { const seqDef* const sequences = seqStorePtr->sequencesStart; @@ -1921,19 +1948,14 @@ void ZSTD_seqToCodes(const seqStore_t* seqStorePtr) mlCodeTable[seqStorePtr->longLengthPos] = MaxML; } -static int ZSTD_disableLiteralsCompression(const ZSTD_CCtx_params* cctxParams) +/* ZSTD_useTargetCBlockSize(): + * Returns if target compressed block size param is being used. + * If used, compression will do best effort to make a compressed block size to be around targetCBlockSize. + * Returns 1 if true, 0 otherwise. */ +static int ZSTD_useTargetCBlockSize(const ZSTD_CCtx_params* cctxParams) { - switch (cctxParams->literalCompressionMode) { - case ZSTD_lcm_huffman: - return 0; - case ZSTD_lcm_uncompressed: - return 1; - default: - assert(0 /* impossible: pre-validated */); - /* fall-through */ - case ZSTD_lcm_auto: - return (cctxParams->cParams.strategy == ZSTD_fast) && (cctxParams->cParams.targetLength > 0); - } + DEBUGLOG(5, "ZSTD_useTargetCBlockSize (targetCBlockSize=%zu)", cctxParams->targetCBlockSize); + return (cctxParams->targetCBlockSize != 0); } /* ZSTD_compressSequences_internal(): @@ -1979,14 +2001,14 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, literals, litSize, entropyWorkspace, entropyWkspSize, bmi2); - FORWARD_IF_ERROR(cSize); + FORWARD_IF_ERROR(cSize, "ZSTD_compressLiterals failed"); assert(cSize <= dstCapacity); op += cSize; } /* Sequences Header */ RETURN_ERROR_IF((oend-op) < 3 /*max nbSeq Size*/ + 1 /*seqHead*/, - dstSize_tooSmall); + dstSize_tooSmall, "Can't fit seq hdr in output buf!"); if (nbSeq < 128) { *op++ = (BYTE)nbSeq; } else if (nbSeq < LONGNBSEQ) { @@ -2031,7 +2053,7 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, prevEntropy->fse.litlengthCTable, sizeof(prevEntropy->fse.litlengthCTable), entropyWorkspace, entropyWkspSize); - FORWARD_IF_ERROR(countSize); + FORWARD_IF_ERROR(countSize, "ZSTD_buildCTable for LitLens failed"); if (LLtype == set_compressed) lastNCount = op; op += countSize; @@ -2059,7 +2081,7 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, prevEntropy->fse.offcodeCTable, sizeof(prevEntropy->fse.offcodeCTable), entropyWorkspace, entropyWkspSize); - FORWARD_IF_ERROR(countSize); + FORWARD_IF_ERROR(countSize, "ZSTD_buildCTable for Offsets failed"); if (Offtype == set_compressed) lastNCount = op; op += countSize; @@ -2085,7 +2107,7 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, prevEntropy->fse.matchlengthCTable, sizeof(prevEntropy->fse.matchlengthCTable), entropyWorkspace, entropyWkspSize); - FORWARD_IF_ERROR(countSize); + FORWARD_IF_ERROR(countSize, "ZSTD_buildCTable for MatchLengths failed"); if (MLtype == set_compressed) lastNCount = op; op += countSize; @@ -2101,7 +2123,7 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, CTable_LitLength, llCodeTable, sequences, nbSeq, longOffsets, bmi2); - FORWARD_IF_ERROR(bitstreamSize); + FORWARD_IF_ERROR(bitstreamSize, "ZSTD_encodeSequences failed"); op += bitstreamSize; assert(op <= oend); /* zstd versions <= 1.3.4 mistakenly report corruption when @@ -2145,7 +2167,7 @@ ZSTD_compressSequences(seqStore_t* seqStorePtr, */ if ((cSize == ERROR(dstSize_tooSmall)) & (srcSize <= dstCapacity)) return 0; /* block not compressed */ - FORWARD_IF_ERROR(cSize); + FORWARD_IF_ERROR(cSize, "ZSTD_compressSequences_internal failed"); /* Check compressibility */ { size_t const maxCSize = srcSize - ZSTD_minGain(srcSize, cctxParams->cParams.strategy); @@ -2271,7 +2293,7 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize) /* Updates ldmSeqStore.size */ FORWARD_IF_ERROR(ZSTD_ldm_generateSequences(&zc->ldmState, &ldmSeqStore, &zc->appliedParams.ldmParams, - src, srcSize)); + src, srcSize), ""); /* Updates ldmSeqStore.pos */ lastLLSize = ZSTD_ldm_blockCompress(&ldmSeqStore, @@ -2347,7 +2369,7 @@ size_t ZSTD_getSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs, void* dst = ZSTD_malloc(dstCapacity, ZSTD_defaultCMem); SeqCollector seqCollector; - RETURN_ERROR_IF(dst == NULL, memory_allocation); + RETURN_ERROR_IF(dst == NULL, memory_allocation, "NULL pointer!"); seqCollector.collectSequences = 1; seqCollector.seqStart = outSeqs; @@ -2370,6 +2392,25 @@ static int ZSTD_isRLE(const BYTE *ip, size_t length) { return 1; } +/* Returns true if the given block may be RLE. + * This is just a heuristic based on the compressibility. + * It may return both false positives and false negatives. + */ +static int ZSTD_maybeRLE(seqStore_t const* seqStore) +{ + size_t const nbSeqs = (size_t)(seqStore->sequences - seqStore->sequencesStart); + size_t const nbLits = (size_t)(seqStore->lit - seqStore->litStart); + + return nbSeqs < 4 && nbLits < 10; +} + +static void ZSTD_confirmRepcodesAndEntropyTables(ZSTD_CCtx* zc) +{ + ZSTD_compressedBlockState_t* const tmp = zc->blockState.prevCBlock; + zc->blockState.prevCBlock = zc->blockState.nextCBlock; + zc->blockState.nextCBlock = tmp; +} + static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCapacity, const void* src, size_t srcSize, U32 frame) @@ -2387,7 +2428,7 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, (unsigned)zc->blockState.matchState.nextToUpdate); { const size_t bss = ZSTD_buildSeqStore(zc, src, srcSize); - FORWARD_IF_ERROR(bss); + FORWARD_IF_ERROR(bss, "ZSTD_buildSeqStore failed"); if (bss == ZSTDbss_noCompress) { cSize = 0; goto out; } } @@ -2420,10 +2461,7 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, out: if (!ZSTD_isError(cSize) && cSize > 1) { - /* confirm repcodes and entropy tables when emitting a compressed block */ - ZSTD_compressedBlockState_t* const tmp = zc->blockState.prevCBlock; - zc->blockState.prevCBlock = zc->blockState.nextCBlock; - zc->blockState.nextCBlock = tmp; + ZSTD_confirmRepcodesAndEntropyTables(zc); } /* We check that dictionaries have offset codes available for the first * block. After the first block, the offcode table might not have large @@ -2435,6 +2473,80 @@ out: return cSize; } +static size_t ZSTD_compressBlock_targetCBlockSize_body(ZSTD_CCtx* zc, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const size_t bss, U32 lastBlock) +{ + DEBUGLOG(6, "Attempting ZSTD_compressSuperBlock()"); + if (bss == ZSTDbss_compress) { + if (/* We don't want to emit our first block as a RLE even if it qualifies because + * doing so will cause the decoder (cli only) to throw a "should consume all input error." + * This is only an issue for zstd <= v1.4.3 + */ + !zc->isFirstBlock && + ZSTD_maybeRLE(&zc->seqStore) && + ZSTD_isRLE((BYTE const*)src, srcSize)) + { + return ZSTD_rleCompressBlock(dst, dstCapacity, *(BYTE const*)src, srcSize, lastBlock); + } + /* Attempt superblock compression. + * + * Note that compressed size of ZSTD_compressSuperBlock() is not bound by the + * standard ZSTD_compressBound(). This is a problem, because even if we have + * space now, taking an extra byte now could cause us to run out of space later + * and violate ZSTD_compressBound(). + * + * Define blockBound(blockSize) = blockSize + ZSTD_blockHeaderSize. + * + * In order to respect ZSTD_compressBound() we must attempt to emit a raw + * uncompressed block in these cases: + * * cSize == 0: Return code for an uncompressed block. + * * cSize == dstSize_tooSmall: We may have expanded beyond blockBound(srcSize). + * ZSTD_noCompressBlock() will return dstSize_tooSmall if we are really out of + * output space. + * * cSize >= blockBound(srcSize): We have expanded the block too much so + * emit an uncompressed block. + */ + { + size_t const cSize = ZSTD_compressSuperBlock(zc, dst, dstCapacity, src, srcSize, lastBlock); + if (cSize != ERROR(dstSize_tooSmall)) { + size_t const maxCSize = srcSize - ZSTD_minGain(srcSize, zc->appliedParams.cParams.strategy); + FORWARD_IF_ERROR(cSize, "ZSTD_compressSuperBlock failed"); + if (cSize != 0 && cSize < maxCSize + ZSTD_blockHeaderSize) { + ZSTD_confirmRepcodesAndEntropyTables(zc); + return cSize; + } + } + } + } + + DEBUGLOG(6, "Resorting to ZSTD_noCompressBlock()"); + /* Superblock compression failed, attempt to emit a single no compress block. + * The decoder will be able to stream this block since it is uncompressed. + */ + return ZSTD_noCompressBlock(dst, dstCapacity, src, srcSize, lastBlock); +} + +static size_t ZSTD_compressBlock_targetCBlockSize(ZSTD_CCtx* zc, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + U32 lastBlock) +{ + size_t cSize = 0; + const size_t bss = ZSTD_buildSeqStore(zc, src, srcSize); + DEBUGLOG(5, "ZSTD_compressBlock_targetCBlockSize (dstCapacity=%u, dictLimit=%u, nextToUpdate=%u, srcSize=%zu)", + (unsigned)dstCapacity, (unsigned)zc->blockState.matchState.window.dictLimit, (unsigned)zc->blockState.matchState.nextToUpdate, srcSize); + FORWARD_IF_ERROR(bss, "ZSTD_buildSeqStore failed"); + + cSize = ZSTD_compressBlock_targetCBlockSize_body(zc, dst, dstCapacity, src, srcSize, bss, lastBlock); + FORWARD_IF_ERROR(cSize, "ZSTD_compressBlock_targetCBlockSize_body failed"); + + if (zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid) + zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check; + + return cSize; +} static void ZSTD_overflowCorrectIfNeeded(ZSTD_matchState_t* ms, ZSTD_cwksp* ws, @@ -2478,6 +2590,7 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx, BYTE* const ostart = (BYTE*)dst; BYTE* op = ostart; U32 const maxDist = (U32)1 << cctx->appliedParams.cParams.windowLog; + assert(cctx->appliedParams.cParams.windowLog <= ZSTD_WINDOWLOG_MAX); DEBUGLOG(5, "ZSTD_compress_frameChunk (blockSize=%u)", (unsigned)blockSize); @@ -2500,21 +2613,31 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx, /* Ensure hash/chain table insertion resumes no sooner than lowlimit */ if (ms->nextToUpdate < ms->window.lowLimit) ms->nextToUpdate = ms->window.lowLimit; - { size_t cSize = ZSTD_compressBlock_internal(cctx, - op+ZSTD_blockHeaderSize, dstCapacity-ZSTD_blockHeaderSize, - ip, blockSize, 1 /* frame */); - FORWARD_IF_ERROR(cSize); - if (cSize == 0) { /* block is not compressible */ - cSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock); - FORWARD_IF_ERROR(cSize); + { size_t cSize; + if (ZSTD_useTargetCBlockSize(&cctx->appliedParams)) { + cSize = ZSTD_compressBlock_targetCBlockSize(cctx, op, dstCapacity, ip, blockSize, lastBlock); + FORWARD_IF_ERROR(cSize, "ZSTD_compressBlock_targetCBlockSize failed"); + assert(cSize > 0); + assert(cSize <= blockSize + ZSTD_blockHeaderSize); } else { - const U32 cBlockHeader = cSize == 1 ? - lastBlock + (((U32)bt_rle)<<1) + (U32)(blockSize << 3) : - lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3); - MEM_writeLE24(op, cBlockHeader); - cSize += ZSTD_blockHeaderSize; + cSize = ZSTD_compressBlock_internal(cctx, + op+ZSTD_blockHeaderSize, dstCapacity-ZSTD_blockHeaderSize, + ip, blockSize, 1 /* frame */); + FORWARD_IF_ERROR(cSize, "ZSTD_compressBlock_internal failed"); + + if (cSize == 0) { /* block is not compressible */ + cSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock); + FORWARD_IF_ERROR(cSize, "ZSTD_noCompressBlock failed"); + } else { + U32 const cBlockHeader = cSize == 1 ? + lastBlock + (((U32)bt_rle)<<1) + (U32)(blockSize << 3) : + lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3); + MEM_writeLE24(op, cBlockHeader); + cSize += ZSTD_blockHeaderSize; + } } + ip += blockSize; assert(remaining >= blockSize); remaining -= blockSize; @@ -2546,7 +2669,8 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity, size_t pos=0; assert(!(params->fParams.contentSizeFlag && pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN)); - RETURN_ERROR_IF(dstCapacity < ZSTD_FRAMEHEADERSIZE_MAX, dstSize_tooSmall); + RETURN_ERROR_IF(dstCapacity < ZSTD_FRAMEHEADERSIZE_MAX, dstSize_tooSmall, + "dst buf is too small to fit worst-case frame header size."); DEBUGLOG(4, "ZSTD_writeFrameHeader : dictIDFlag : %u ; dictID : %u ; dictIDSizeCode : %u", !params->fParams.noDictIDFlag, (unsigned)dictID, (unsigned)dictIDSizeCode); @@ -2582,7 +2706,8 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity, */ size_t ZSTD_writeLastEmptyBlock(void* dst, size_t dstCapacity) { - RETURN_ERROR_IF(dstCapacity < ZSTD_blockHeaderSize, dstSize_tooSmall); + RETURN_ERROR_IF(dstCapacity < ZSTD_blockHeaderSize, dstSize_tooSmall, + "dst buf is too small to write frame trailer empty block."); { U32 const cBlockHeader24 = 1 /*lastBlock*/ + (((U32)bt_raw)<<1); /* 0 size */ MEM_writeLE24(dst, cBlockHeader24); return ZSTD_blockHeaderSize; @@ -2591,9 +2716,11 @@ size_t ZSTD_writeLastEmptyBlock(void* dst, size_t dstCapacity) size_t ZSTD_referenceExternalSequences(ZSTD_CCtx* cctx, rawSeq* seq, size_t nbSeq) { - RETURN_ERROR_IF(cctx->stage != ZSTDcs_init, stage_wrong); + RETURN_ERROR_IF(cctx->stage != ZSTDcs_init, stage_wrong, + "wrong cctx stage"); RETURN_ERROR_IF(cctx->appliedParams.ldmParams.enableLdm, - parameter_unsupported); + parameter_unsupported, + "incompatible with ldm"); cctx->externSeqStore.seq = seq; cctx->externSeqStore.size = nbSeq; cctx->externSeqStore.capacity = nbSeq; @@ -2618,7 +2745,7 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx, if (frame && (cctx->stage==ZSTDcs_init)) { fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, &cctx->appliedParams, cctx->pledgedSrcSizePlusOne-1, cctx->dictID); - FORWARD_IF_ERROR(fhSize); + FORWARD_IF_ERROR(fhSize, "ZSTD_writeFrameHeader failed"); assert(fhSize <= dstCapacity); dstCapacity -= fhSize; dst = (char*)dst + fhSize; @@ -2645,7 +2772,7 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx, { size_t const cSize = frame ? ZSTD_compress_frameChunk (cctx, dst, dstCapacity, src, srcSize, lastFrameChunk) : ZSTD_compressBlock_internal (cctx, dst, dstCapacity, src, srcSize, 0 /* frame */); - FORWARD_IF_ERROR(cSize); + FORWARD_IF_ERROR(cSize, "%s", frame ? "ZSTD_compress_frameChunk failed" : "ZSTD_compressBlock_internal failed"); cctx->consumedSrcSize += srcSize; cctx->producedCSize += (cSize + fhSize); assert(!(cctx->appliedParams.fParams.contentSizeFlag && cctx->pledgedSrcSizePlusOne == 0)); @@ -2682,7 +2809,7 @@ size_t ZSTD_compressBlock(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const { DEBUGLOG(5, "ZSTD_compressBlock: srcSize = %u", (unsigned)srcSize); { size_t const blockSizeMax = ZSTD_getBlockSize(cctx); - RETURN_ERROR_IF(srcSize > blockSizeMax, srcSize_wrong); } + RETURN_ERROR_IF(srcSize > blockSizeMax, srcSize_wrong, "input is larger than a block"); } return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 0 /* frame mode */, 0 /* last chunk */); } @@ -2691,6 +2818,7 @@ size_t ZSTD_compressBlock(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const * @return : 0, or an error code */ static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms, + ldmState_t* ls, ZSTD_cwksp* ws, ZSTD_CCtx_params const* params, const void* src, size_t srcSize, @@ -2702,6 +2830,11 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms, ZSTD_window_update(&ms->window, src, srcSize); ms->loadedDictEnd = params->forceWindow ? 0 : (U32)(iend - ms->window.base); + if (params->ldmParams.enableLdm && ls != NULL) { + ZSTD_window_update(&ls->window, src, srcSize); + ls->loadedDictEnd = params->forceWindow ? 0 : (U32)(iend - ls->window.base); + } + /* Assert that we the ms params match the params we're being given */ ZSTD_assertEqualCParams(params->cParams, ms->cParams); @@ -2714,6 +2847,9 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms, ZSTD_overflowCorrectIfNeeded(ms, ws, params, ip, ichunk); + if (params->ldmParams.enableLdm && ls != NULL) + ZSTD_ldm_fillHashTable(ls, (const BYTE*)src, (const BYTE*)src + srcSize, ¶ms->ldmParams); + switch(params->cParams.strategy) { case ZSTD_fast: @@ -2756,102 +2892,123 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms, NOTE: This behavior is not standard and could be improved in the future. */ static size_t ZSTD_checkDictNCount(short* normalizedCounter, unsigned dictMaxSymbolValue, unsigned maxSymbolValue) { U32 s; - RETURN_ERROR_IF(dictMaxSymbolValue < maxSymbolValue, dictionary_corrupted); + RETURN_ERROR_IF(dictMaxSymbolValue < maxSymbolValue, dictionary_corrupted, "dict fse tables don't have all symbols"); for (s = 0; s <= maxSymbolValue; ++s) { - RETURN_ERROR_IF(normalizedCounter[s] == 0, dictionary_corrupted); + RETURN_ERROR_IF(normalizedCounter[s] == 0, dictionary_corrupted, "dict fse tables don't have all symbols"); } return 0; } - -/* Dictionary format : - * See : - * https://github.com/facebook/zstd/blob/master/doc/zstd_compression_format.md#dictionary-format - */ -/*! ZSTD_loadZstdDictionary() : - * @return : dictID, or an error code - * assumptions : magic number supposed already checked - * dictSize supposed >= 8 - */ -static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs, - ZSTD_matchState_t* ms, - ZSTD_cwksp* ws, - ZSTD_CCtx_params const* params, - const void* dict, size_t dictSize, - ZSTD_dictTableLoadMethod_e dtlm, - void* workspace) +size_t ZSTD_loadCEntropy(ZSTD_compressedBlockState_t* bs, void* workspace, + short* offcodeNCount, unsigned* offcodeMaxValue, + const void* const dict, size_t dictSize) { - const BYTE* dictPtr = (const BYTE*)dict; + const BYTE* dictPtr = (const BYTE*)dict; /* skip magic num and dict ID */ const BYTE* const dictEnd = dictPtr + dictSize; - short offcodeNCount[MaxOff+1]; - unsigned offcodeMaxValue = MaxOff; - size_t dictID; + dictPtr += 8; + bs->entropy.huf.repeatMode = HUF_repeat_check; - ZSTD_STATIC_ASSERT(HUF_WORKSPACE_SIZE >= (1<<MAX(MLFSELog,LLFSELog))); - assert(dictSize >= 8); - assert(MEM_readLE32(dictPtr) == ZSTD_MAGIC_DICTIONARY); + { unsigned maxSymbolValue = 255; + unsigned hasZeroWeights = 1; + size_t const hufHeaderSize = HUF_readCTable((HUF_CElt*)bs->entropy.huf.CTable, &maxSymbolValue, dictPtr, + dictEnd-dictPtr, &hasZeroWeights); - dictPtr += 4; /* skip magic number */ - dictID = params->fParams.noDictIDFlag ? 0 : MEM_readLE32(dictPtr); - dictPtr += 4; + /* We only set the loaded table as valid if it contains all non-zero + * weights. Otherwise, we set it to check */ + if (!hasZeroWeights) + bs->entropy.huf.repeatMode = HUF_repeat_valid; - { unsigned maxSymbolValue = 255; - size_t const hufHeaderSize = HUF_readCTable((HUF_CElt*)bs->entropy.huf.CTable, &maxSymbolValue, dictPtr, dictEnd-dictPtr); - RETURN_ERROR_IF(HUF_isError(hufHeaderSize), dictionary_corrupted); - RETURN_ERROR_IF(maxSymbolValue < 255, dictionary_corrupted); + RETURN_ERROR_IF(HUF_isError(hufHeaderSize), dictionary_corrupted, ""); + RETURN_ERROR_IF(maxSymbolValue < 255, dictionary_corrupted, ""); dictPtr += hufHeaderSize; } { unsigned offcodeLog; - size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr); - RETURN_ERROR_IF(FSE_isError(offcodeHeaderSize), dictionary_corrupted); - RETURN_ERROR_IF(offcodeLog > OffFSELog, dictionary_corrupted); + size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr); + RETURN_ERROR_IF(FSE_isError(offcodeHeaderSize), dictionary_corrupted, ""); + RETURN_ERROR_IF(offcodeLog > OffFSELog, dictionary_corrupted, ""); /* Defer checking offcodeMaxValue because we need to know the size of the dictionary content */ /* fill all offset symbols to avoid garbage at end of table */ RETURN_ERROR_IF(FSE_isError(FSE_buildCTable_wksp( bs->entropy.fse.offcodeCTable, offcodeNCount, MaxOff, offcodeLog, workspace, HUF_WORKSPACE_SIZE)), - dictionary_corrupted); + dictionary_corrupted, ""); dictPtr += offcodeHeaderSize; } { short matchlengthNCount[MaxML+1]; unsigned matchlengthMaxValue = MaxML, matchlengthLog; size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr); - RETURN_ERROR_IF(FSE_isError(matchlengthHeaderSize), dictionary_corrupted); - RETURN_ERROR_IF(matchlengthLog > MLFSELog, dictionary_corrupted); + RETURN_ERROR_IF(FSE_isError(matchlengthHeaderSize), dictionary_corrupted, ""); + RETURN_ERROR_IF(matchlengthLog > MLFSELog, dictionary_corrupted, ""); /* Every match length code must have non-zero probability */ - FORWARD_IF_ERROR( ZSTD_checkDictNCount(matchlengthNCount, matchlengthMaxValue, MaxML)); + FORWARD_IF_ERROR( ZSTD_checkDictNCount(matchlengthNCount, matchlengthMaxValue, MaxML), ""); RETURN_ERROR_IF(FSE_isError(FSE_buildCTable_wksp( bs->entropy.fse.matchlengthCTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog, workspace, HUF_WORKSPACE_SIZE)), - dictionary_corrupted); + dictionary_corrupted, ""); dictPtr += matchlengthHeaderSize; } { short litlengthNCount[MaxLL+1]; unsigned litlengthMaxValue = MaxLL, litlengthLog; size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr); - RETURN_ERROR_IF(FSE_isError(litlengthHeaderSize), dictionary_corrupted); - RETURN_ERROR_IF(litlengthLog > LLFSELog, dictionary_corrupted); + RETURN_ERROR_IF(FSE_isError(litlengthHeaderSize), dictionary_corrupted, ""); + RETURN_ERROR_IF(litlengthLog > LLFSELog, dictionary_corrupted, ""); /* Every literal length code must have non-zero probability */ - FORWARD_IF_ERROR( ZSTD_checkDictNCount(litlengthNCount, litlengthMaxValue, MaxLL)); + FORWARD_IF_ERROR( ZSTD_checkDictNCount(litlengthNCount, litlengthMaxValue, MaxLL), ""); RETURN_ERROR_IF(FSE_isError(FSE_buildCTable_wksp( bs->entropy.fse.litlengthCTable, litlengthNCount, litlengthMaxValue, litlengthLog, workspace, HUF_WORKSPACE_SIZE)), - dictionary_corrupted); + dictionary_corrupted, ""); dictPtr += litlengthHeaderSize; } - RETURN_ERROR_IF(dictPtr+12 > dictEnd, dictionary_corrupted); + RETURN_ERROR_IF(dictPtr+12 > dictEnd, dictionary_corrupted, ""); bs->rep[0] = MEM_readLE32(dictPtr+0); bs->rep[1] = MEM_readLE32(dictPtr+4); bs->rep[2] = MEM_readLE32(dictPtr+8); dictPtr += 12; + return dictPtr - (const BYTE*)dict; +} + +/* Dictionary format : + * See : + * https://github.com/facebook/zstd/blob/master/doc/zstd_compression_format.md#dictionary-format + */ +/*! ZSTD_loadZstdDictionary() : + * @return : dictID, or an error code + * assumptions : magic number supposed already checked + * dictSize supposed >= 8 + */ +static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs, + ZSTD_matchState_t* ms, + ZSTD_cwksp* ws, + ZSTD_CCtx_params const* params, + const void* dict, size_t dictSize, + ZSTD_dictTableLoadMethod_e dtlm, + void* workspace) +{ + const BYTE* dictPtr = (const BYTE*)dict; + const BYTE* const dictEnd = dictPtr + dictSize; + short offcodeNCount[MaxOff+1]; + unsigned offcodeMaxValue = MaxOff; + size_t dictID; + size_t eSize; + + ZSTD_STATIC_ASSERT(HUF_WORKSPACE_SIZE >= (1<<MAX(MLFSELog,LLFSELog))); + assert(dictSize >= 8); + assert(MEM_readLE32(dictPtr) == ZSTD_MAGIC_DICTIONARY); + + dictID = params->fParams.noDictIDFlag ? 0 : MEM_readLE32(dictPtr + 4 /* skip magic number */ ); + eSize = ZSTD_loadCEntropy(bs, workspace, offcodeNCount, &offcodeMaxValue, dict, dictSize); + FORWARD_IF_ERROR(eSize, "ZSTD_loadCEntropy failed"); + dictPtr += eSize; + { size_t const dictContentSize = (size_t)(dictEnd - dictPtr); U32 offcodeMax = MaxOff; if (dictContentSize <= ((U32)-1) - 128 KB) { @@ -2859,20 +3016,19 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs, offcodeMax = ZSTD_highbit32(maxOffset); /* Calculate minimum offset code required to represent maxOffset */ } /* All offset values <= dictContentSize + 128 KB must be representable */ - FORWARD_IF_ERROR(ZSTD_checkDictNCount(offcodeNCount, offcodeMaxValue, MIN(offcodeMax, MaxOff))); + FORWARD_IF_ERROR(ZSTD_checkDictNCount(offcodeNCount, offcodeMaxValue, MIN(offcodeMax, MaxOff)), ""); /* All repCodes must be <= dictContentSize and != 0*/ { U32 u; for (u=0; u<3; u++) { - RETURN_ERROR_IF(bs->rep[u] == 0, dictionary_corrupted); - RETURN_ERROR_IF(bs->rep[u] > dictContentSize, dictionary_corrupted); + RETURN_ERROR_IF(bs->rep[u] == 0, dictionary_corrupted, ""); + RETURN_ERROR_IF(bs->rep[u] > dictContentSize, dictionary_corrupted, ""); } } - bs->entropy.huf.repeatMode = HUF_repeat_valid; bs->entropy.fse.offcode_repeatMode = FSE_repeat_valid; bs->entropy.fse.matchlength_repeatMode = FSE_repeat_valid; bs->entropy.fse.litlength_repeatMode = FSE_repeat_valid; FORWARD_IF_ERROR(ZSTD_loadDictionaryContent( - ms, ws, params, dictPtr, dictContentSize, dtlm)); + ms, NULL, ws, params, dictPtr, dictContentSize, dtlm), ""); return dictID; } } @@ -2882,6 +3038,7 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs, static size_t ZSTD_compress_insertDictionary(ZSTD_compressedBlockState_t* bs, ZSTD_matchState_t* ms, + ldmState_t* ls, ZSTD_cwksp* ws, const ZSTD_CCtx_params* params, const void* dict, size_t dictSize, @@ -2891,7 +3048,7 @@ ZSTD_compress_insertDictionary(ZSTD_compressedBlockState_t* bs, { DEBUGLOG(4, "ZSTD_compress_insertDictionary (dictSize=%u)", (U32)dictSize); if ((dict==NULL) || (dictSize<8)) { - RETURN_ERROR_IF(dictContentType == ZSTD_dct_fullDict, dictionary_wrong); + RETURN_ERROR_IF(dictContentType == ZSTD_dct_fullDict, dictionary_wrong, ""); return 0; } @@ -2899,15 +3056,15 @@ ZSTD_compress_insertDictionary(ZSTD_compressedBlockState_t* bs, /* dict restricted modes */ if (dictContentType == ZSTD_dct_rawContent) - return ZSTD_loadDictionaryContent(ms, ws, params, dict, dictSize, dtlm); + return ZSTD_loadDictionaryContent(ms, ls, ws, params, dict, dictSize, dtlm); if (MEM_readLE32(dict) != ZSTD_MAGIC_DICTIONARY) { if (dictContentType == ZSTD_dct_auto) { DEBUGLOG(4, "raw content dictionary detected"); return ZSTD_loadDictionaryContent( - ms, ws, params, dict, dictSize, dtlm); + ms, ls, ws, params, dict, dictSize, dtlm); } - RETURN_ERROR_IF(dictContentType == ZSTD_dct_fullDict, dictionary_wrong); + RETURN_ERROR_IF(dictContentType == ZSTD_dct_fullDict, dictionary_wrong, ""); assert(0); /* impossible */ } @@ -2944,17 +3101,18 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, } FORWARD_IF_ERROR( ZSTD_resetCCtx_internal(cctx, *params, pledgedSrcSize, - ZSTDcrp_makeClean, zbuff) ); + ZSTDcrp_makeClean, zbuff) , ""); { size_t const dictID = cdict ? ZSTD_compress_insertDictionary( cctx->blockState.prevCBlock, &cctx->blockState.matchState, - &cctx->workspace, params, cdict->dictContent, cdict->dictContentSize, - dictContentType, dtlm, cctx->entropyWorkspace) + &cctx->ldmState, &cctx->workspace, &cctx->appliedParams, cdict->dictContent, + cdict->dictContentSize, dictContentType, dtlm, + cctx->entropyWorkspace) : ZSTD_compress_insertDictionary( cctx->blockState.prevCBlock, &cctx->blockState.matchState, - &cctx->workspace, params, dict, dictSize, + &cctx->ldmState, &cctx->workspace, &cctx->appliedParams, dict, dictSize, dictContentType, dtlm, cctx->entropyWorkspace); - FORWARD_IF_ERROR(dictID); + FORWARD_IF_ERROR(dictID, "ZSTD_compress_insertDictionary failed"); assert(dictID <= UINT_MAX); cctx->dictID = (U32)dictID; } @@ -2971,7 +3129,7 @@ size_t ZSTD_compressBegin_advanced_internal(ZSTD_CCtx* cctx, { DEBUGLOG(4, "ZSTD_compressBegin_advanced_internal: wlog=%u", params->cParams.windowLog); /* compression parameters verification and optimization */ - FORWARD_IF_ERROR( ZSTD_checkCParams(params->cParams) ); + FORWARD_IF_ERROR( ZSTD_checkCParams(params->cParams) , ""); return ZSTD_compressBegin_internal(cctx, dict, dictSize, dictContentType, dtlm, cdict, @@ -2986,7 +3144,7 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, ZSTD_parameters params, unsigned long long pledgedSrcSize) { ZSTD_CCtx_params const cctxParams = - ZSTD_assignParamsToCCtxParams(&cctx->requestedParams, params); + ZSTD_assignParamsToCCtxParams(&cctx->requestedParams, ¶ms); return ZSTD_compressBegin_advanced_internal(cctx, dict, dictSize, ZSTD_dct_auto, ZSTD_dtlm_fast, NULL /*cdict*/, @@ -2995,9 +3153,9 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel) { - ZSTD_parameters const params = ZSTD_getParams(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize); + ZSTD_parameters const params = ZSTD_getParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize); ZSTD_CCtx_params const cctxParams = - ZSTD_assignParamsToCCtxParams(&cctx->requestedParams, params); + ZSTD_assignParamsToCCtxParams(&cctx->requestedParams, ¶ms); DEBUGLOG(4, "ZSTD_compressBegin_usingDict (dictSize=%u)", (unsigned)dictSize); return ZSTD_compressBegin_internal(cctx, dict, dictSize, ZSTD_dct_auto, ZSTD_dtlm_fast, NULL, &cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, ZSTDb_not_buffered); @@ -3024,7 +3182,7 @@ static size_t ZSTD_writeEpilogue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity) /* special case : empty frame */ if (cctx->stage == ZSTDcs_init) { fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, &cctx->appliedParams, 0, 0); - FORWARD_IF_ERROR(fhSize); + FORWARD_IF_ERROR(fhSize, "ZSTD_writeFrameHeader failed"); dstCapacity -= fhSize; op += fhSize; cctx->stage = ZSTDcs_ongoing; @@ -3033,7 +3191,7 @@ static size_t ZSTD_writeEpilogue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity) if (cctx->stage != ZSTDcs_ending) { /* write one last empty block, make it the "last" block */ U32 const cBlockHeader24 = 1 /* last block */ + (((U32)bt_raw)<<1) + 0; - RETURN_ERROR_IF(dstCapacity<4, dstSize_tooSmall); + RETURN_ERROR_IF(dstCapacity<4, dstSize_tooSmall, "no room for epilogue"); MEM_writeLE32(op, cBlockHeader24); op += ZSTD_blockHeaderSize; dstCapacity -= ZSTD_blockHeaderSize; @@ -3041,7 +3199,7 @@ static size_t ZSTD_writeEpilogue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity) if (cctx->appliedParams.fParams.checksumFlag) { U32 const checksum = (U32) XXH64_digest(&cctx->xxhState); - RETURN_ERROR_IF(dstCapacity<4, dstSize_tooSmall); + RETURN_ERROR_IF(dstCapacity<4, dstSize_tooSmall, "no room for checksum"); DEBUGLOG(4, "ZSTD_writeEpilogue: write checksum : %08X", (unsigned)checksum); MEM_writeLE32(op, checksum); op += 4; @@ -3059,9 +3217,9 @@ size_t ZSTD_compressEnd (ZSTD_CCtx* cctx, size_t const cSize = ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 1 /* frame mode */, 1 /* last chunk */); - FORWARD_IF_ERROR(cSize); + FORWARD_IF_ERROR(cSize, "ZSTD_compressContinue_internal failed"); endResult = ZSTD_writeEpilogue(cctx, (char*)dst + cSize, dstCapacity-cSize); - FORWARD_IF_ERROR(endResult); + FORWARD_IF_ERROR(endResult, "ZSTD_writeEpilogue failed"); assert(!(cctx->appliedParams.fParams.contentSizeFlag && cctx->pledgedSrcSizePlusOne == 0)); if (cctx->pledgedSrcSizePlusOne != 0) { /* control src size */ ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_UNKNOWN == (unsigned long long)-1); @@ -3081,7 +3239,7 @@ static size_t ZSTD_compress_internal (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, const void* dict,size_t dictSize, - ZSTD_parameters params) + const ZSTD_parameters* params) { ZSTD_CCtx_params const cctxParams = ZSTD_assignParamsToCCtxParams(&cctx->requestedParams, params); @@ -3100,12 +3258,12 @@ size_t ZSTD_compress_advanced (ZSTD_CCtx* cctx, ZSTD_parameters params) { DEBUGLOG(4, "ZSTD_compress_advanced"); - FORWARD_IF_ERROR(ZSTD_checkCParams(params.cParams)); + FORWARD_IF_ERROR(ZSTD_checkCParams(params.cParams), ""); return ZSTD_compress_internal(cctx, dst, dstCapacity, src, srcSize, dict, dictSize, - params); + ¶ms); } /* Internal */ @@ -3119,7 +3277,7 @@ size_t ZSTD_compress_advanced_internal( DEBUGLOG(4, "ZSTD_compress_advanced_internal (srcSize:%u)", (unsigned)srcSize); FORWARD_IF_ERROR( ZSTD_compressBegin_internal(cctx, dict, dictSize, ZSTD_dct_auto, ZSTD_dtlm_fast, NULL, - params, srcSize, ZSTDb_not_buffered) ); + params, srcSize, ZSTDb_not_buffered) , ""); return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize); } @@ -3129,8 +3287,9 @@ size_t ZSTD_compress_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel) { - ZSTD_parameters const params = ZSTD_getParams(compressionLevel, srcSize + (!srcSize), dict ? dictSize : 0); - ZSTD_CCtx_params cctxParams = ZSTD_assignParamsToCCtxParams(&cctx->requestedParams, params); + ZSTD_parameters const params = ZSTD_getParams_internal(compressionLevel, srcSize, dict ? dictSize : 0); + ZSTD_CCtx_params cctxParams = ZSTD_assignParamsToCCtxParams(&cctx->requestedParams, ¶ms); + DEBUGLOG(4, "ZSTD_compress_usingDict (srcSize=%u)", (unsigned)srcSize); assert(params.fParams.contentSizeFlag == 1); return ZSTD_compress_advanced_internal(cctx, dst, dstCapacity, src, srcSize, dict, dictSize, &cctxParams); } @@ -3176,7 +3335,7 @@ size_t ZSTD_estimateCDictSize_advanced( size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel) { - ZSTD_compressionParameters const cParams = ZSTD_getCParams(compressionLevel, 0, dictSize); + ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize); return ZSTD_estimateCDictSize_advanced(dictSize, cParams, ZSTD_dlm_byCopy); } @@ -3203,7 +3362,7 @@ static size_t ZSTD_initCDict_internal( cdict->dictContent = dictBuffer; } else { void *internalBuffer = ZSTD_cwksp_reserve_object(&cdict->workspace, ZSTD_cwksp_align(dictSize, sizeof(void*))); - RETURN_ERROR_IF(!internalBuffer, memory_allocation); + RETURN_ERROR_IF(!internalBuffer, memory_allocation, "NULL pointer!"); cdict->dictContent = internalBuffer; memcpy(internalBuffer, dictBuffer, dictSize); } @@ -3220,7 +3379,7 @@ static size_t ZSTD_initCDict_internal( &cParams, ZSTDcrp_makeClean, ZSTDirp_reset, - ZSTD_resetTarget_CDict)); + ZSTD_resetTarget_CDict), ""); /* (Maybe) load the dictionary * Skips loading the dictionary if it is < 8 bytes. */ @@ -3230,10 +3389,10 @@ static size_t ZSTD_initCDict_internal( params.fParams.contentSizeFlag = 1; params.cParams = cParams; { size_t const dictID = ZSTD_compress_insertDictionary( - &cdict->cBlockState, &cdict->matchState, &cdict->workspace, + &cdict->cBlockState, &cdict->matchState, NULL, &cdict->workspace, ¶ms, cdict->dictContent, cdict->dictContentSize, dictContentType, ZSTD_dtlm_full, cdict->entropyWorkspace); - FORWARD_IF_ERROR(dictID); + FORWARD_IF_ERROR(dictID, "ZSTD_compress_insertDictionary failed"); assert(dictID <= (size_t)(U32)-1); cdict->dictID = (U32)dictID; } @@ -3287,7 +3446,7 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel) { - ZSTD_compressionParameters cParams = ZSTD_getCParams(compressionLevel, 0, dictSize); + ZSTD_compressionParameters cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize); ZSTD_CDict* cdict = ZSTD_createCDict_advanced(dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto, cParams, ZSTD_defaultCMem); @@ -3298,7 +3457,7 @@ ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionL ZSTD_CDict* ZSTD_createCDict_byReference(const void* dict, size_t dictSize, int compressionLevel) { - ZSTD_compressionParameters cParams = ZSTD_getCParams(compressionLevel, 0, dictSize); + ZSTD_compressionParameters cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize); return ZSTD_createCDict_advanced(dict, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cParams, ZSTD_defaultCMem); @@ -3381,7 +3540,7 @@ size_t ZSTD_compressBegin_usingCDict_advanced( ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize) { DEBUGLOG(4, "ZSTD_compressBegin_usingCDict_advanced"); - RETURN_ERROR_IF(cdict==NULL, dictionary_wrong); + RETURN_ERROR_IF(cdict==NULL, dictionary_wrong, "NULL pointer!"); { ZSTD_CCtx_params params = cctx->requestedParams; params.cParams = ( pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF || pledgedSrcSize < cdict->dictContentSize * ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER @@ -3425,7 +3584,7 @@ size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx, const void* src, size_t srcSize, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams) { - FORWARD_IF_ERROR(ZSTD_compressBegin_usingCDict_advanced(cctx, cdict, fParams, srcSize)); /* will check if cdict != NULL */ + FORWARD_IF_ERROR(ZSTD_compressBegin_usingCDict_advanced(cctx, cdict, fParams, srcSize), ""); /* will check if cdict != NULL */ return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize); } @@ -3497,7 +3656,7 @@ static size_t ZSTD_resetCStream_internal(ZSTD_CStream* cctx, dict, dictSize, dictContentType, ZSTD_dtlm_fast, cdict, ¶ms, pledgedSrcSize, - ZSTDb_buffered) ); + ZSTDb_buffered) , ""); cctx->inToCompress = 0; cctx->inBuffPos = 0; @@ -3519,8 +3678,8 @@ size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pss) */ U64 const pledgedSrcSize = (pss==0) ? ZSTD_CONTENTSIZE_UNKNOWN : pss; DEBUGLOG(4, "ZSTD_resetCStream: pledgedSrcSize = %u", (unsigned)pledgedSrcSize); - FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) ); - FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) ); + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) , ""); return 0; } @@ -3534,16 +3693,16 @@ size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize) { DEBUGLOG(4, "ZSTD_initCStream_internal"); - FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) ); - FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) ); + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) , ""); assert(!ZSTD_isError(ZSTD_checkCParams(params->cParams))); zcs->requestedParams = *params; assert(!((dict) && (cdict))); /* either dict or cdict, not both */ if (dict) { - FORWARD_IF_ERROR( ZSTD_CCtx_loadDictionary(zcs, dict, dictSize) ); + FORWARD_IF_ERROR( ZSTD_CCtx_loadDictionary(zcs, dict, dictSize) , ""); } else { /* Dictionary is cleared if !cdict */ - FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, cdict) ); + FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, cdict) , ""); } return 0; } @@ -3556,10 +3715,10 @@ size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize) { DEBUGLOG(4, "ZSTD_initCStream_usingCDict_advanced"); - FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) ); - FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) ); + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) , ""); zcs->requestedParams.fParams = fParams; - FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, cdict) ); + FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, cdict) , ""); return 0; } @@ -3567,8 +3726,8 @@ size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict) { DEBUGLOG(4, "ZSTD_initCStream_usingCDict"); - FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) ); - FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, cdict) ); + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, cdict) , ""); return 0; } @@ -3587,20 +3746,20 @@ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, */ U64 const pledgedSrcSize = (pss==0 && params.fParams.contentSizeFlag==0) ? ZSTD_CONTENTSIZE_UNKNOWN : pss; DEBUGLOG(4, "ZSTD_initCStream_advanced"); - FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) ); - FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) ); - FORWARD_IF_ERROR( ZSTD_checkCParams(params.cParams) ); - zcs->requestedParams = ZSTD_assignParamsToCCtxParams(&zcs->requestedParams, params); - FORWARD_IF_ERROR( ZSTD_CCtx_loadDictionary(zcs, dict, dictSize) ); + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) , ""); + FORWARD_IF_ERROR( ZSTD_checkCParams(params.cParams) , ""); + zcs->requestedParams = ZSTD_assignParamsToCCtxParams(&zcs->requestedParams, ¶ms); + FORWARD_IF_ERROR( ZSTD_CCtx_loadDictionary(zcs, dict, dictSize) , ""); return 0; } size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel) { DEBUGLOG(4, "ZSTD_initCStream_usingDict"); - FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) ); - FORWARD_IF_ERROR( ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel) ); - FORWARD_IF_ERROR( ZSTD_CCtx_loadDictionary(zcs, dict, dictSize) ); + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_loadDictionary(zcs, dict, dictSize) , ""); return 0; } @@ -3612,19 +3771,19 @@ size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigne */ U64 const pledgedSrcSize = (pss==0) ? ZSTD_CONTENTSIZE_UNKNOWN : pss; DEBUGLOG(4, "ZSTD_initCStream_srcSize"); - FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) ); - FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, NULL) ); - FORWARD_IF_ERROR( ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel) ); - FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) ); + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, NULL) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) , ""); return 0; } size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel) { DEBUGLOG(4, "ZSTD_initCStream"); - FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) ); - FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, NULL) ); - FORWARD_IF_ERROR( ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel) ); + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, NULL) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel) , ""); return 0; } @@ -3637,14 +3796,6 @@ static size_t ZSTD_nextInputSizeHint(const ZSTD_CCtx* cctx) return hintInSize; } -static size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, - const void* src, size_t srcSize) -{ - size_t const length = MIN(dstCapacity, srcSize); - if (length) memcpy(dst, src, length); - return length; -} - /** ZSTD_compressStream_generic(): * internal function for all *compressStream*() variants * non-static, because can be called from zstdmt_compress.c @@ -3655,11 +3806,11 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, ZSTD_EndDirective const flushMode) { const char* const istart = (const char*)input->src; - const char* const iend = istart + input->size; - const char* ip = istart + input->pos; + const char* const iend = input->size != 0 ? istart + input->size : istart; + const char* ip = input->pos != 0 ? istart + input->pos : istart; char* const ostart = (char*)output->dst; - char* const oend = ostart + output->size; - char* op = ostart + output->pos; + char* const oend = output->size != 0 ? ostart + output->size : ostart; + char* op = output->pos != 0 ? ostart + output->pos : ostart; U32 someMoreWork = 1; /* check expectations */ @@ -3685,7 +3836,7 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, size_t const cSize = ZSTD_compressEnd(zcs, op, oend-op, ip, iend-ip); DEBUGLOG(4, "ZSTD_compressEnd : cSize=%u", (unsigned)cSize); - FORWARD_IF_ERROR(cSize); + FORWARD_IF_ERROR(cSize, "ZSTD_compressEnd failed"); ip = iend; op += cSize; zcs->frameEnded = 1; @@ -3698,7 +3849,8 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, zcs->inBuff + zcs->inBuffPos, toLoad, ip, iend-ip); zcs->inBuffPos += loaded; - ip += loaded; + if (loaded != 0) + ip += loaded; if ( (flushMode == ZSTD_e_continue) && (zcs->inBuffPos < zcs->inBuffTarget) ) { /* not enough input to fill full block : stop here */ @@ -3726,7 +3878,7 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, zcs->inBuff + zcs->inToCompress, iSize) : ZSTD_compressContinue(zcs, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize); - FORWARD_IF_ERROR(cSize); + FORWARD_IF_ERROR(cSize, "%s", lastBlock ? "ZSTD_compressEnd failed" : "ZSTD_compressContinue failed"); zcs->frameEnded = lastBlock; /* prepare next block */ zcs->inBuffTarget = zcs->inBuffPos + zcs->blockSize; @@ -3758,7 +3910,8 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, zcs->outBuff + zcs->outBuffFlushedSize, toFlush); DEBUGLOG(5, "toFlush: %u into %u ==> flushed: %u", (unsigned)toFlush, (unsigned)(oend-op), (unsigned)flushed); - op += flushed; + if (flushed) + op += flushed; zcs->outBuffFlushedSize += flushed; if (toFlush!=flushed) { /* flush not fully completed, presumably because dst is too small */ @@ -3802,7 +3955,7 @@ static size_t ZSTD_nextInputSizeHint_MTorST(const ZSTD_CCtx* cctx) size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input) { - FORWARD_IF_ERROR( ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue) ); + FORWARD_IF_ERROR( ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue) , ""); return ZSTD_nextInputSizeHint_MTorST(zcs); } @@ -3814,15 +3967,15 @@ size_t ZSTD_compressStream2( ZSTD_CCtx* cctx, { DEBUGLOG(5, "ZSTD_compressStream2, endOp=%u ", (unsigned)endOp); /* check conditions */ - RETURN_ERROR_IF(output->pos > output->size, GENERIC); - RETURN_ERROR_IF(input->pos > input->size, GENERIC); + RETURN_ERROR_IF(output->pos > output->size, GENERIC, "invalid buffer"); + RETURN_ERROR_IF(input->pos > input->size, GENERIC, "invalid buffer"); assert(cctx!=NULL); /* transparent initialization stage */ if (cctx->streamStage == zcss_init) { ZSTD_CCtx_params params = cctx->requestedParams; ZSTD_prefixDict const prefixDict = cctx->prefixDict; - FORWARD_IF_ERROR( ZSTD_initLocalDict(cctx) ); /* Init the local dict if present. */ + FORWARD_IF_ERROR( ZSTD_initLocalDict(cctx) , ""); /* Init the local dict if present. */ memset(&cctx->prefixDict, 0, sizeof(cctx->prefixDict)); /* single usage */ assert(prefixDict.dict==NULL || cctx->cdict==NULL); /* only one can be set */ DEBUGLOG(4, "ZSTD_compressStream2 : transparent init stage"); @@ -3841,14 +3994,14 @@ size_t ZSTD_compressStream2( ZSTD_CCtx* cctx, DEBUGLOG(4, "ZSTD_compressStream2: creating new mtctx for nbWorkers=%u", params.nbWorkers); cctx->mtctx = ZSTDMT_createCCtx_advanced((U32)params.nbWorkers, cctx->customMem); - RETURN_ERROR_IF(cctx->mtctx == NULL, memory_allocation); + RETURN_ERROR_IF(cctx->mtctx == NULL, memory_allocation, "NULL pointer!"); } /* mt compression */ DEBUGLOG(4, "call ZSTDMT_initCStream_internal as nbWorkers=%u", params.nbWorkers); FORWARD_IF_ERROR( ZSTDMT_initCStream_internal( cctx->mtctx, - prefixDict.dict, prefixDict.dictSize, ZSTD_dct_rawContent, - cctx->cdict, params, cctx->pledgedSrcSizePlusOne-1) ); + prefixDict.dict, prefixDict.dictSize, prefixDict.dictContentType, + cctx->cdict, params, cctx->pledgedSrcSizePlusOne-1) , ""); cctx->streamStage = zcss_load; cctx->appliedParams.nbWorkers = params.nbWorkers; } else @@ -3856,7 +4009,7 @@ size_t ZSTD_compressStream2( ZSTD_CCtx* cctx, { FORWARD_IF_ERROR( ZSTD_resetCStream_internal(cctx, prefixDict.dict, prefixDict.dictSize, prefixDict.dictContentType, cctx->cdict, - params, cctx->pledgedSrcSizePlusOne-1) ); + params, cctx->pledgedSrcSizePlusOne-1) , ""); assert(cctx->streamStage == zcss_load); assert(cctx->appliedParams.nbWorkers == 0); } } @@ -3878,7 +4031,7 @@ size_t ZSTD_compressStream2( ZSTD_CCtx* cctx, || (endOp == ZSTD_e_end && flushMin == 0) ) { /* compression completed */ ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only); } - FORWARD_IF_ERROR(flushMin); + FORWARD_IF_ERROR(flushMin, "ZSTDMT_compressStream_generic failed"); } while (forceMaxProgress && flushMin != 0 && output->pos < output->size); DEBUGLOG(5, "completed ZSTD_compressStream2 delegating to ZSTDMT_compressStream_generic"); /* Either we don't require maximum forward progress, we've finished the @@ -3888,7 +4041,7 @@ size_t ZSTD_compressStream2( ZSTD_CCtx* cctx, return flushMin; } #endif - FORWARD_IF_ERROR( ZSTD_compressStream_generic(cctx, output, input, endOp) ); + FORWARD_IF_ERROR( ZSTD_compressStream_generic(cctx, output, input, endOp) , ""); DEBUGLOG(5, "completed ZSTD_compressStream2"); return cctx->outBuffContentSize - cctx->outBuffFlushedSize; /* remaining to flush */ } @@ -3912,6 +4065,7 @@ size_t ZSTD_compress2(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) { + DEBUGLOG(4, "ZSTD_compress2 (srcSize=%u)", (unsigned)srcSize); ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only); { size_t oPos = 0; size_t iPos = 0; @@ -3919,10 +4073,10 @@ size_t ZSTD_compress2(ZSTD_CCtx* cctx, dst, dstCapacity, &oPos, src, srcSize, &iPos, ZSTD_e_end); - FORWARD_IF_ERROR(result); + FORWARD_IF_ERROR(result, "ZSTD_compressStream2_simpleArgs failed"); if (result != 0) { /* compression not completed, due to lack of output space */ assert(oPos == dstCapacity); - RETURN_ERROR(dstSize_tooSmall); + RETURN_ERROR(dstSize_tooSmall, ""); } assert(iPos == srcSize); /* all input is expected consumed */ return oPos; @@ -3944,7 +4098,7 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output) { ZSTD_inBuffer input = { NULL, 0, 0 }; size_t const remainingToFlush = ZSTD_compressStream2(zcs, output, &input, ZSTD_e_end); - FORWARD_IF_ERROR( remainingToFlush ); + FORWARD_IF_ERROR( remainingToFlush , "ZSTD_compressStream2 failed"); if (zcs->appliedParams.nbWorkers > 0) return remainingToFlush; /* minimal estimation */ /* single thread mode : attempt to calculate remaining to flush more precisely */ { size_t const lastBlockSize = zcs->frameEnded ? 0 : ZSTD_BLOCKHEADERSIZE; @@ -4069,35 +4223,56 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV }, }; -/*! ZSTD_getCParams() : +/*! ZSTD_getCParams_internal() : * @return ZSTD_compressionParameters structure for a selected compression level, srcSize and dictSize. - * Size values are optional, provide 0 if not known or unused */ -ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize) + * Note: srcSizeHint 0 means 0, use ZSTD_CONTENTSIZE_UNKNOWN for unknown. + * Use dictSize == 0 for unknown or unused. */ +static ZSTD_compressionParameters ZSTD_getCParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize) { - size_t const addedSize = srcSizeHint ? 0 : 500; - U64 const rSize = srcSizeHint+dictSize ? srcSizeHint+dictSize+addedSize : ZSTD_CONTENTSIZE_UNKNOWN; /* intentional overflow for srcSizeHint == ZSTD_CONTENTSIZE_UNKNOWN */ + int const unknown = srcSizeHint == ZSTD_CONTENTSIZE_UNKNOWN; + size_t const addedSize = unknown && dictSize > 0 ? 500 : 0; + U64 const rSize = unknown && dictSize == 0 ? ZSTD_CONTENTSIZE_UNKNOWN : srcSizeHint+dictSize+addedSize; U32 const tableID = (rSize <= 256 KB) + (rSize <= 128 KB) + (rSize <= 16 KB); int row = compressionLevel; - DEBUGLOG(5, "ZSTD_getCParams (cLevel=%i)", compressionLevel); + DEBUGLOG(5, "ZSTD_getCParams_internal (cLevel=%i)", compressionLevel); if (compressionLevel == 0) row = ZSTD_CLEVEL_DEFAULT; /* 0 == default */ if (compressionLevel < 0) row = 0; /* entry 0 is baseline for fast mode */ if (compressionLevel > ZSTD_MAX_CLEVEL) row = ZSTD_MAX_CLEVEL; { ZSTD_compressionParameters cp = ZSTD_defaultCParameters[tableID][row]; if (compressionLevel < 0) cp.targetLength = (unsigned)(-compressionLevel); /* acceleration factor */ - return ZSTD_adjustCParams_internal(cp, srcSizeHint, dictSize); /* refine parameters based on srcSize & dictSize */ + /* refine parameters based on srcSize & dictSize */ + return ZSTD_adjustCParams_internal(cp, srcSizeHint, dictSize); } } +/*! ZSTD_getCParams() : + * @return ZSTD_compressionParameters structure for a selected compression level, srcSize and dictSize. + * Size values are optional, provide 0 if not known or unused */ +ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize) +{ + if (srcSizeHint == 0) srcSizeHint = ZSTD_CONTENTSIZE_UNKNOWN; + return ZSTD_getCParams_internal(compressionLevel, srcSizeHint, dictSize); +} + /*! ZSTD_getParams() : * same idea as ZSTD_getCParams() * @return a `ZSTD_parameters` structure (instead of `ZSTD_compressionParameters`). * Fields of `ZSTD_frameParameters` are set to default values */ -ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize) { +static ZSTD_parameters ZSTD_getParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize) { ZSTD_parameters params; - ZSTD_compressionParameters const cParams = ZSTD_getCParams(compressionLevel, srcSizeHint, dictSize); + ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, srcSizeHint, dictSize); DEBUGLOG(5, "ZSTD_getParams (cLevel=%i)", compressionLevel); memset(¶ms, 0, sizeof(params)); params.cParams = cParams; params.fParams.contentSizeFlag = 1; return params; } + +/*! ZSTD_getParams() : + * same idea as ZSTD_getCParams() + * @return a `ZSTD_parameters` structure (instead of `ZSTD_compressionParameters`). + * Fields of `ZSTD_frameParameters` are set to default values */ +ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize) { + if (srcSizeHint == 0) srcSizeHint = ZSTD_CONTENTSIZE_UNKNOWN; + return ZSTD_getParams_internal(compressionLevel, srcSizeHint, dictSize); +} diff --git a/thirdparty/zstd/compress/zstd_compress_internal.h b/thirdparty/zstd/compress/zstd_compress_internal.h index 14036f873f..db73f6ce21 100644 --- a/thirdparty/zstd/compress/zstd_compress_internal.h +++ b/thirdparty/zstd/compress/zstd_compress_internal.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -18,7 +18,7 @@ /*-************************************* * Dependencies ***************************************/ -#include "zstd_internal.h" +#include "../common/zstd_internal.h" #include "zstd_cwksp.h" #ifdef ZSTD_MULTITHREAD # include "zstdmt_compress.h" @@ -166,6 +166,7 @@ typedef struct { typedef struct { ZSTD_window_t window; /* State for the window round buffer management */ ldmEntry_t* hashTable; + U32 loadedDictEnd; BYTE* bucketOffsets; /* Next position in bucket to insert entry */ U64 hashPower; /* Used to compute the rolling hash. * Depends on ldmParams.minMatchLength */ @@ -249,6 +250,7 @@ struct ZSTD_CCtx_s { size_t staticSize; SeqCollector seqCollector; int isFirstBlock; + int initialized; seqStore_t seqStore; /* sequences storage ptrs */ ldmState_t ldmState; /* long distance matching state */ @@ -324,6 +326,31 @@ MEM_STATIC U32 ZSTD_MLcode(U32 mlBase) return (mlBase > 127) ? ZSTD_highbit32(mlBase) + ML_deltaCode : ML_Code[mlBase]; } +typedef struct repcodes_s { + U32 rep[3]; +} repcodes_t; + +MEM_STATIC repcodes_t ZSTD_updateRep(U32 const rep[3], U32 const offset, U32 const ll0) +{ + repcodes_t newReps; + if (offset >= ZSTD_REP_NUM) { /* full offset */ + newReps.rep[2] = rep[1]; + newReps.rep[1] = rep[0]; + newReps.rep[0] = offset - ZSTD_REP_MOVE; + } else { /* repcode */ + U32 const repCode = offset + ll0; + if (repCode > 0) { /* note : if repCode==0, no change */ + U32 const currentOffset = (repCode==ZSTD_REP_NUM) ? (rep[0] - 1) : rep[repCode]; + newReps.rep[2] = (repCode >= 2) ? rep[1] : rep[2]; + newReps.rep[1] = rep[0]; + newReps.rep[0] = currentOffset; + } else { /* repCode == 0 */ + memcpy(&newReps, rep, sizeof(newReps)); + } + } + return newReps; +} + /* ZSTD_cParam_withinBounds: * @return 1 if value is within cParam bounds, * 0 otherwise */ @@ -336,6 +363,30 @@ MEM_STATIC int ZSTD_cParam_withinBounds(ZSTD_cParameter cParam, int value) return 1; } +/* ZSTD_noCompressBlock() : + * Writes uncompressed block to dst buffer from given src. + * Returns the size of the block */ +MEM_STATIC size_t ZSTD_noCompressBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize, U32 lastBlock) +{ + U32 const cBlockHeader24 = lastBlock + (((U32)bt_raw)<<1) + (U32)(srcSize << 3); + RETURN_ERROR_IF(srcSize + ZSTD_blockHeaderSize > dstCapacity, + dstSize_tooSmall, "dst buf too small for uncompressed block"); + MEM_writeLE24(dst, cBlockHeader24); + memcpy((BYTE*)dst + ZSTD_blockHeaderSize, src, srcSize); + return ZSTD_blockHeaderSize + srcSize; +} + +MEM_STATIC size_t ZSTD_rleCompressBlock (void* dst, size_t dstCapacity, BYTE src, size_t srcSize, U32 lastBlock) +{ + BYTE* const op = (BYTE*)dst; + U32 const cBlockHeader = lastBlock + (((U32)bt_rle)<<1) + (U32)(srcSize << 3); + RETURN_ERROR_IF(dstCapacity < 4, dstSize_tooSmall, ""); + MEM_writeLE24(op, cBlockHeader); + op[3] = src; + return 4; +} + + /* ZSTD_minGain() : * minimum compression required * to generate a compress block or a compressed literals section. @@ -348,6 +399,21 @@ MEM_STATIC size_t ZSTD_minGain(size_t srcSize, ZSTD_strategy strat) return (srcSize >> minlog) + 2; } +MEM_STATIC int ZSTD_disableLiteralsCompression(const ZSTD_CCtx_params* cctxParams) +{ + switch (cctxParams->literalCompressionMode) { + case ZSTD_lcm_huffman: + return 0; + case ZSTD_lcm_uncompressed: + return 1; + default: + assert(0 /* impossible: pre-validated */); + /* fall-through */ + case ZSTD_lcm_auto: + return (cctxParams->cParams.strategy == ZSTD_fast) && (cctxParams->cParams.targetLength > 0); + } +} + /*! ZSTD_safecopyLiterals() : * memcpy() function that won't read beyond more than WILDCOPY_OVERLENGTH bytes past ilimit_w. * Only called when the sequence ends past ilimit_w, so it only needs to be optimized for single @@ -433,8 +499,7 @@ static unsigned ZSTD_NbCommonBytes (size_t val) if (MEM_64bits()) { # if defined(_MSC_VER) && defined(_WIN64) unsigned long r = 0; - _BitScanForward64( &r, (U64)val ); - return (unsigned)(r>>3); + return _BitScanForward64( &r, (U64)val ) ? (unsigned)(r >> 3) : 0; # elif defined(__GNUC__) && (__GNUC__ >= 4) return (__builtin_ctzll((U64)val) >> 3); # else @@ -451,8 +516,7 @@ static unsigned ZSTD_NbCommonBytes (size_t val) } else { /* 32 bits */ # if defined(_MSC_VER) unsigned long r=0; - _BitScanForward( &r, (U32)val ); - return (unsigned)(r>>3); + return _BitScanForward( &r, (U32)val ) ? (unsigned)(r >> 3) : 0; # elif defined(__GNUC__) && (__GNUC__ >= 3) return (__builtin_ctz((U32)val) >> 3); # else @@ -467,8 +531,7 @@ static unsigned ZSTD_NbCommonBytes (size_t val) if (MEM_64bits()) { # if defined(_MSC_VER) && defined(_WIN64) unsigned long r = 0; - _BitScanReverse64( &r, val ); - return (unsigned)(r>>3); + return _BitScanReverse64( &r, val ) ? (unsigned)(r >> 3) : 0; # elif defined(__GNUC__) && (__GNUC__ >= 4) return (__builtin_clzll(val) >> 3); # else @@ -482,8 +545,7 @@ static unsigned ZSTD_NbCommonBytes (size_t val) } else { /* 32 bits */ # if defined(_MSC_VER) unsigned long r = 0; - _BitScanReverse( &r, (unsigned long)val ); - return (unsigned)(r>>3); + return _BitScanReverse( &r, (unsigned long)val ) ? (unsigned)(r >> 3) : 0; # elif defined(__GNUC__) && (__GNUC__ >= 3) return (__builtin_clz((U32)val) >> 3); # else @@ -730,7 +792,10 @@ MEM_STATIC U32 ZSTD_window_correctOverflow(ZSTD_window_t* window, U32 cycleLog, */ U32 const cycleMask = (1U << cycleLog) - 1; U32 const current = (U32)((BYTE const*)src - window->base); - U32 const newCurrent = (current & cycleMask) + maxDist; + U32 const currentCycle0 = current & cycleMask; + /* Exclude zero so that newCurrent - maxDist >= 1. */ + U32 const currentCycle1 = currentCycle0 == 0 ? (1U << cycleLog) : currentCycle0; + U32 const newCurrent = currentCycle1 + maxDist; U32 const correction = current - newCurrent; assert((maxDist & cycleMask) == 0); assert(current > newCurrent); @@ -739,8 +804,17 @@ MEM_STATIC U32 ZSTD_window_correctOverflow(ZSTD_window_t* window, U32 cycleLog, window->base += correction; window->dictBase += correction; - window->lowLimit -= correction; - window->dictLimit -= correction; + if (window->lowLimit <= correction) window->lowLimit = 1; + else window->lowLimit -= correction; + if (window->dictLimit <= correction) window->dictLimit = 1; + else window->dictLimit -= correction; + + /* Ensure we can still reference the full window. */ + assert(newCurrent >= maxDist); + assert(newCurrent - maxDist >= 1); + /* Ensure that lowLimit and dictLimit didn't underflow. */ + assert(window->lowLimit <= newCurrent); + assert(window->dictLimit <= newCurrent); DEBUGLOG(4, "Correction of 0x%x bytes to lowLimit=0x%x", correction, window->lowLimit); @@ -844,6 +918,15 @@ ZSTD_checkDictValidity(const ZSTD_window_t* window, } } } } +MEM_STATIC void ZSTD_window_init(ZSTD_window_t* window) { + memset(window, 0, sizeof(*window)); + window->base = (BYTE const*)""; + window->dictBase = (BYTE const*)""; + window->dictLimit = 1; /* start from 1, so that 1st position is valid */ + window->lowLimit = 1; /* it ensures first and later CCtx usages compress the same */ + window->nextSrc = window->base + 1; /* see issue #1241 */ +} + /** * ZSTD_window_update(): * Updates the window by appending [src, src + srcSize) to the window. @@ -857,6 +940,10 @@ MEM_STATIC U32 ZSTD_window_update(ZSTD_window_t* window, BYTE const* const ip = (BYTE const*)src; U32 contiguous = 1; DEBUGLOG(5, "ZSTD_window_update"); + if (srcSize == 0) + return contiguous; + assert(window->base != NULL); + assert(window->dictBase != NULL); /* Check if blocks follow each other */ if (src != window->nextSrc) { /* not contiguous */ @@ -867,7 +954,7 @@ MEM_STATIC U32 ZSTD_window_update(ZSTD_window_t* window, window->dictLimit = (U32)distanceFromBase; window->dictBase = window->base; window->base = ip - distanceFromBase; - // ms->nextToUpdate = window->dictLimit; + /* ms->nextToUpdate = window->dictLimit; */ if (window->dictLimit - window->lowLimit < HASH_READ_SIZE) window->lowLimit = window->dictLimit; /* too small extDict */ contiguous = 0; } @@ -883,6 +970,9 @@ MEM_STATIC U32 ZSTD_window_update(ZSTD_window_t* window, return contiguous; } +/** + * Returns the lowest allowed match index. It may either be in the ext-dict or the prefix. + */ MEM_STATIC U32 ZSTD_getLowestMatchIndex(const ZSTD_matchState_t* ms, U32 current, unsigned windowLog) { U32 const maxDistance = 1U << windowLog; @@ -893,6 +983,19 @@ MEM_STATIC U32 ZSTD_getLowestMatchIndex(const ZSTD_matchState_t* ms, U32 current return matchLowest; } +/** + * Returns the lowest allowed match index in the prefix. + */ +MEM_STATIC U32 ZSTD_getLowestPrefixIndex(const ZSTD_matchState_t* ms, U32 current, unsigned windowLog) +{ + U32 const maxDistance = 1U << windowLog; + U32 const lowestValid = ms->window.dictLimit; + U32 const withinWindow = (current - lowestValid > maxDistance) ? current - maxDistance : lowestValid; + U32 const isDictionary = (ms->loadedDictEnd != 0); + U32 const matchLowest = isDictionary ? lowestValid : withinWindow; + return matchLowest; +} + /* debug functions */ @@ -931,6 +1034,21 @@ MEM_STATIC void ZSTD_debugTable(const U32* table, U32 max) } #endif +/* =============================================================== + * Shared internal declarations + * These prototypes may be called from sources not in lib/compress + * =============================================================== */ + +/* ZSTD_loadCEntropy() : + * dict : must point at beginning of a valid zstd dictionary. + * return : size of dictionary header (size of magic number + dict ID + entropy tables) + * assumptions : magic number supposed already checked + * and dictSize >= 8 */ +size_t ZSTD_loadCEntropy(ZSTD_compressedBlockState_t* bs, void* workspace, + short* offcodeNCount, unsigned* offcodeMaxValue, + const void* const dict, size_t dictSize); + +void ZSTD_reset_compressedBlockState(ZSTD_compressedBlockState_t* bs); /* ============================================================== * Private declarations @@ -940,6 +1058,7 @@ MEM_STATIC void ZSTD_debugTable(const U32* table, U32 max) /* ZSTD_getCParamsFromCCtxParams() : * cParams are built depending on compressionLevel, src size hints, * LDM and manually set compression parameters. + * Note: srcSizeHint == 0 means 0! */ ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams( const ZSTD_CCtx_params* CCtxParams, U64 srcSizeHint, size_t dictSize); @@ -999,5 +1118,8 @@ size_t ZSTD_writeLastEmptyBlock(void* dst, size_t dstCapacity); */ size_t ZSTD_referenceExternalSequences(ZSTD_CCtx* cctx, rawSeq* seq, size_t nbSeq); +/** ZSTD_cycleLog() : + * condition for correct operation : hashLog > 1 */ +U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat); #endif /* ZSTD_COMPRESS_H */ diff --git a/thirdparty/zstd/compress/zstd_compress_literals.c b/thirdparty/zstd/compress/zstd_compress_literals.c index 6c13331182..17e7168d89 100644 --- a/thirdparty/zstd/compress/zstd_compress_literals.c +++ b/thirdparty/zstd/compress/zstd_compress_literals.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -18,7 +18,7 @@ size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src, BYTE* const ostart = (BYTE* const)dst; U32 const flSize = 1 + (srcSize>31) + (srcSize>4095); - RETURN_ERROR_IF(srcSize + flSize > dstCapacity, dstSize_tooSmall); + RETURN_ERROR_IF(srcSize + flSize > dstCapacity, dstSize_tooSmall, ""); switch(flSize) { @@ -36,6 +36,7 @@ size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src, } memcpy(ostart + flSize, src, srcSize); + DEBUGLOG(5, "Raw literals: %u -> %u", (U32)srcSize, (U32)(srcSize + flSize)); return srcSize + flSize; } @@ -62,6 +63,7 @@ size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, const void* } ostart[flSize] = *(const BYTE*)src; + DEBUGLOG(5, "RLE literals: %u -> %u", (U32)srcSize, (U32)flSize + 1); return flSize+1; } @@ -80,8 +82,8 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf, symbolEncodingType_e hType = set_compressed; size_t cLitSize; - DEBUGLOG(5,"ZSTD_compressLiterals (disableLiteralCompression=%i)", - disableLiteralCompression); + DEBUGLOG(5,"ZSTD_compressLiterals (disableLiteralCompression=%i srcSize=%u)", + disableLiteralCompression, (U32)srcSize); /* Prepare nextEntropy assuming reusing the existing table */ memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); @@ -102,14 +104,15 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf, cLitSize = singleStream ? HUF_compress1X_repeat( ostart+lhSize, dstCapacity-lhSize, src, srcSize, - 255, 11, entropyWorkspace, entropyWorkspaceSize, + HUF_SYMBOLVALUE_MAX, HUF_TABLELOG_DEFAULT, entropyWorkspace, entropyWorkspaceSize, (HUF_CElt*)nextHuf->CTable, &repeat, preferRepeat, bmi2) : HUF_compress4X_repeat( ostart+lhSize, dstCapacity-lhSize, src, srcSize, - 255, 11, entropyWorkspace, entropyWorkspaceSize, + HUF_SYMBOLVALUE_MAX, HUF_TABLELOG_DEFAULT, entropyWorkspace, entropyWorkspaceSize, (HUF_CElt*)nextHuf->CTable, &repeat, preferRepeat, bmi2); if (repeat != HUF_repeat_none) { /* reused the existing table */ + DEBUGLOG(5, "Reusing previous huffman table"); hType = set_repeat; } } @@ -150,5 +153,6 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf, default: /* not possible : lhSize is {3,4,5} */ assert(0); } + DEBUGLOG(5, "Compressed literals: %u -> %u", (U32)srcSize, (U32)(lhSize+cLitSize)); return lhSize+cLitSize; } diff --git a/thirdparty/zstd/compress/zstd_compress_literals.h b/thirdparty/zstd/compress/zstd_compress_literals.h index 97273d7cfd..8b08705743 100644 --- a/thirdparty/zstd/compress/zstd_compress_literals.h +++ b/thirdparty/zstd/compress/zstd_compress_literals.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/thirdparty/zstd/compress/zstd_compress_sequences.c b/thirdparty/zstd/compress/zstd_compress_sequences.c index 0ff7a26823..f9f8097c83 100644 --- a/thirdparty/zstd/compress/zstd_compress_sequences.c +++ b/thirdparty/zstd/compress/zstd_compress_sequences.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -60,7 +60,7 @@ static size_t ZSTD_NCountCost(unsigned const* count, unsigned const max, BYTE wksp[FSE_NCOUNTBOUND]; S16 norm[MaxSeq + 1]; const U32 tableLog = FSE_optimalTableLog(FSELog, nbSeq, max); - FORWARD_IF_ERROR(FSE_normalizeCount(norm, tableLog, count, nbSeq, max)); + FORWARD_IF_ERROR(FSE_normalizeCount(norm, tableLog, count, nbSeq, max), ""); return FSE_writeNCount(wksp, sizeof(wksp), norm, max, tableLog); } @@ -86,7 +86,7 @@ static size_t ZSTD_entropyCost(unsigned const* count, unsigned const max, size_t * Returns the cost in bits of encoding the distribution in count using ctable. * Returns an error if ctable cannot represent all the symbols in count. */ -static size_t ZSTD_fseBitCost( +size_t ZSTD_fseBitCost( FSE_CTable const* ctable, unsigned const* count, unsigned const max) @@ -96,18 +96,22 @@ static size_t ZSTD_fseBitCost( unsigned s; FSE_CState_t cstate; FSE_initCState(&cstate, ctable); - RETURN_ERROR_IF(ZSTD_getFSEMaxSymbolValue(ctable) < max, GENERIC, - "Repeat FSE_CTable has maxSymbolValue %u < %u", + if (ZSTD_getFSEMaxSymbolValue(ctable) < max) { + DEBUGLOG(5, "Repeat FSE_CTable has maxSymbolValue %u < %u", ZSTD_getFSEMaxSymbolValue(ctable), max); + return ERROR(GENERIC); + } for (s = 0; s <= max; ++s) { unsigned const tableLog = cstate.stateLog; unsigned const badCost = (tableLog + 1) << kAccuracyLog; unsigned const bitCost = FSE_bitCost(cstate.symbolTT, tableLog, s, kAccuracyLog); if (count[s] == 0) continue; - RETURN_ERROR_IF(bitCost >= badCost, GENERIC, - "Repeat FSE_CTable has Prob[%u] == 0", s); - cost += count[s] * bitCost; + if (bitCost >= badCost) { + DEBUGLOG(5, "Repeat FSE_CTable has Prob[%u] == 0", s); + return ERROR(GENERIC); + } + cost += (size_t)count[s] * bitCost; } return cost >> kAccuracyLog; } @@ -117,15 +121,15 @@ static size_t ZSTD_fseBitCost( * table described by norm. The max symbol support by norm is assumed >= max. * norm must be valid for every symbol with non-zero probability in count. */ -static size_t ZSTD_crossEntropyCost(short const* norm, unsigned accuracyLog, - unsigned const* count, unsigned const max) +size_t ZSTD_crossEntropyCost(short const* norm, unsigned accuracyLog, + unsigned const* count, unsigned const max) { unsigned const shift = 8 - accuracyLog; size_t cost = 0; unsigned s; assert(accuracyLog <= 8); for (s = 0; s <= max; ++s) { - unsigned const normAcc = norm[s] != -1 ? norm[s] : 1; + unsigned const normAcc = (norm[s] != -1) ? (unsigned)norm[s] : 1; unsigned const norm256 = normAcc << shift; assert(norm256 > 0); assert(norm256 < 256); @@ -230,15 +234,15 @@ ZSTD_buildCTable(void* dst, size_t dstCapacity, switch (type) { case set_rle: - FORWARD_IF_ERROR(FSE_buildCTable_rle(nextCTable, (BYTE)max)); - RETURN_ERROR_IF(dstCapacity==0, dstSize_tooSmall); + FORWARD_IF_ERROR(FSE_buildCTable_rle(nextCTable, (BYTE)max), ""); + RETURN_ERROR_IF(dstCapacity==0, dstSize_tooSmall, "not enough space"); *op = codeTable[0]; return 1; case set_repeat: memcpy(nextCTable, prevCTable, prevCTableSize); return 0; case set_basic: - FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, defaultNorm, defaultMax, defaultNormLog, entropyWorkspace, entropyWorkspaceSize)); /* note : could be pre-calculated */ + FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, defaultNorm, defaultMax, defaultNormLog, entropyWorkspace, entropyWorkspaceSize), ""); /* note : could be pre-calculated */ return 0; case set_compressed: { S16 norm[MaxSeq + 1]; @@ -249,14 +253,14 @@ ZSTD_buildCTable(void* dst, size_t dstCapacity, nbSeq_1--; } assert(nbSeq_1 > 1); - FORWARD_IF_ERROR(FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max)); + FORWARD_IF_ERROR(FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max), ""); { size_t const NCountSize = FSE_writeNCount(op, oend - op, norm, max, tableLog); /* overflow protected */ - FORWARD_IF_ERROR(NCountSize); - FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, norm, max, tableLog, entropyWorkspace, entropyWorkspaceSize)); + FORWARD_IF_ERROR(NCountSize, "FSE_writeNCount failed"); + FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, norm, max, tableLog, entropyWorkspace, entropyWorkspaceSize), ""); return NCountSize; } } - default: assert(0); RETURN_ERROR(GENERIC); + default: assert(0); RETURN_ERROR(GENERIC, "impossible to reach"); } } @@ -290,7 +294,7 @@ ZSTD_encodeSequences_body( if (MEM_32bits()) BIT_flushBits(&blockStream); if (longOffsets) { U32 const ofBits = ofCodeTable[nbSeq-1]; - int const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1); + unsigned const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1); if (extraBits) { BIT_addBits(&blockStream, sequences[nbSeq-1].offset, extraBits); BIT_flushBits(&blockStream); @@ -327,7 +331,7 @@ ZSTD_encodeSequences_body( BIT_addBits(&blockStream, sequences[n].matchLength, mlBits); if (MEM_32bits() || (ofBits+mlBits+llBits > 56)) BIT_flushBits(&blockStream); if (longOffsets) { - int const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1); + unsigned const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1); if (extraBits) { BIT_addBits(&blockStream, sequences[n].offset, extraBits); BIT_flushBits(&blockStream); /* (7)*/ diff --git a/thirdparty/zstd/compress/zstd_compress_sequences.h b/thirdparty/zstd/compress/zstd_compress_sequences.h index 57e8e367b0..68c6f9a5ac 100644 --- a/thirdparty/zstd/compress/zstd_compress_sequences.h +++ b/thirdparty/zstd/compress/zstd_compress_sequences.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -11,8 +11,8 @@ #ifndef ZSTD_COMPRESS_SEQUENCES_H #define ZSTD_COMPRESS_SEQUENCES_H -#include "fse.h" /* FSE_repeat, FSE_CTable */ -#include "zstd_internal.h" /* symbolEncodingType_e, ZSTD_strategy */ +#include "../common/fse.h" /* FSE_repeat, FSE_CTable */ +#include "../common/zstd_internal.h" /* symbolEncodingType_e, ZSTD_strategy */ typedef enum { ZSTD_defaultDisallowed = 0, @@ -44,4 +44,11 @@ size_t ZSTD_encodeSequences( FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, seqDef const* sequences, size_t nbSeq, int longOffsets, int bmi2); +size_t ZSTD_fseBitCost( + FSE_CTable const* ctable, + unsigned const* count, + unsigned const max); + +size_t ZSTD_crossEntropyCost(short const* norm, unsigned accuracyLog, + unsigned const* count, unsigned const max); #endif /* ZSTD_COMPRESS_SEQUENCES_H */ diff --git a/thirdparty/zstd/compress/zstd_compress_superblock.c b/thirdparty/zstd/compress/zstd_compress_superblock.c new file mode 100644 index 0000000000..b693866c0a --- /dev/null +++ b/thirdparty/zstd/compress/zstd_compress_superblock.c @@ -0,0 +1,845 @@ +/* + * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + /*-************************************* + * Dependencies + ***************************************/ +#include "zstd_compress_superblock.h" + +#include "../common/zstd_internal.h" /* ZSTD_getSequenceLength */ +#include "hist.h" /* HIST_countFast_wksp */ +#include "zstd_compress_internal.h" +#include "zstd_compress_sequences.h" +#include "zstd_compress_literals.h" + +/*-************************************* +* Superblock entropy buffer structs +***************************************/ +/** ZSTD_hufCTablesMetadata_t : + * Stores Literals Block Type for a super-block in hType, and + * huffman tree description in hufDesBuffer. + * hufDesSize refers to the size of huffman tree description in bytes. + * This metadata is populated in ZSTD_buildSuperBlockEntropy_literal() */ +typedef struct { + symbolEncodingType_e hType; + BYTE hufDesBuffer[500]; /* TODO give name to this value */ + size_t hufDesSize; +} ZSTD_hufCTablesMetadata_t; + +/** ZSTD_fseCTablesMetadata_t : + * Stores symbol compression modes for a super-block in {ll, ol, ml}Type, and + * fse tables in fseTablesBuffer. + * fseTablesSize refers to the size of fse tables in bytes. + * This metadata is populated in ZSTD_buildSuperBlockEntropy_sequences() */ +typedef struct { + symbolEncodingType_e llType; + symbolEncodingType_e ofType; + symbolEncodingType_e mlType; + BYTE fseTablesBuffer[500]; /* TODO give name to this value */ + size_t fseTablesSize; + size_t lastCountSize; /* This is to account for bug in 1.3.4. More detail in ZSTD_compressSubBlock_sequences() */ +} ZSTD_fseCTablesMetadata_t; + +typedef struct { + ZSTD_hufCTablesMetadata_t hufMetadata; + ZSTD_fseCTablesMetadata_t fseMetadata; +} ZSTD_entropyCTablesMetadata_t; + + +/** ZSTD_buildSuperBlockEntropy_literal() : + * Builds entropy for the super-block literals. + * Stores literals block type (raw, rle, compressed, repeat) and + * huffman description table to hufMetadata. + * @return : size of huffman description table or error code */ +static size_t ZSTD_buildSuperBlockEntropy_literal(void* const src, size_t srcSize, + const ZSTD_hufCTables_t* prevHuf, + ZSTD_hufCTables_t* nextHuf, + ZSTD_hufCTablesMetadata_t* hufMetadata, + const int disableLiteralsCompression, + void* workspace, size_t wkspSize) +{ + BYTE* const wkspStart = (BYTE*)workspace; + BYTE* const wkspEnd = wkspStart + wkspSize; + BYTE* const countWkspStart = wkspStart; + unsigned* const countWksp = (unsigned*)workspace; + const size_t countWkspSize = (HUF_SYMBOLVALUE_MAX + 1) * sizeof(unsigned); + BYTE* const nodeWksp = countWkspStart + countWkspSize; + const size_t nodeWkspSize = wkspEnd-nodeWksp; + unsigned maxSymbolValue = 255; + unsigned huffLog = HUF_TABLELOG_DEFAULT; + HUF_repeat repeat = prevHuf->repeatMode; + + DEBUGLOG(5, "ZSTD_buildSuperBlockEntropy_literal (srcSize=%zu)", srcSize); + + /* Prepare nextEntropy assuming reusing the existing table */ + memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); + + if (disableLiteralsCompression) { + DEBUGLOG(5, "set_basic - disabled"); + hufMetadata->hType = set_basic; + return 0; + } + + /* small ? don't even attempt compression (speed opt) */ +# define COMPRESS_LITERALS_SIZE_MIN 63 + { size_t const minLitSize = (prevHuf->repeatMode == HUF_repeat_valid) ? 6 : COMPRESS_LITERALS_SIZE_MIN; + if (srcSize <= minLitSize) { + DEBUGLOG(5, "set_basic - too small"); + hufMetadata->hType = set_basic; + return 0; + } + } + + /* Scan input and build symbol stats */ + { size_t const largest = HIST_count_wksp (countWksp, &maxSymbolValue, (const BYTE*)src, srcSize, workspace, wkspSize); + FORWARD_IF_ERROR(largest, "HIST_count_wksp failed"); + if (largest == srcSize) { + DEBUGLOG(5, "set_rle"); + hufMetadata->hType = set_rle; + return 0; + } + if (largest <= (srcSize >> 7)+4) { + DEBUGLOG(5, "set_basic - no gain"); + hufMetadata->hType = set_basic; + return 0; + } + } + + /* Validate the previous Huffman table */ + if (repeat == HUF_repeat_check && !HUF_validateCTable((HUF_CElt const*)prevHuf->CTable, countWksp, maxSymbolValue)) { + repeat = HUF_repeat_none; + } + + /* Build Huffman Tree */ + memset(nextHuf->CTable, 0, sizeof(nextHuf->CTable)); + huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue); + { size_t const maxBits = HUF_buildCTable_wksp((HUF_CElt*)nextHuf->CTable, countWksp, + maxSymbolValue, huffLog, + nodeWksp, nodeWkspSize); + FORWARD_IF_ERROR(maxBits, "HUF_buildCTable_wksp"); + huffLog = (U32)maxBits; + { /* Build and write the CTable */ + size_t const newCSize = HUF_estimateCompressedSize( + (HUF_CElt*)nextHuf->CTable, countWksp, maxSymbolValue); + size_t const hSize = HUF_writeCTable( + hufMetadata->hufDesBuffer, sizeof(hufMetadata->hufDesBuffer), + (HUF_CElt*)nextHuf->CTable, maxSymbolValue, huffLog); + /* Check against repeating the previous CTable */ + if (repeat != HUF_repeat_none) { + size_t const oldCSize = HUF_estimateCompressedSize( + (HUF_CElt const*)prevHuf->CTable, countWksp, maxSymbolValue); + if (oldCSize < srcSize && (oldCSize <= hSize + newCSize || hSize + 12 >= srcSize)) { + DEBUGLOG(5, "set_repeat - smaller"); + memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); + hufMetadata->hType = set_repeat; + return 0; + } + } + if (newCSize + hSize >= srcSize) { + DEBUGLOG(5, "set_basic - no gains"); + memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); + hufMetadata->hType = set_basic; + return 0; + } + DEBUGLOG(5, "set_compressed (hSize=%u)", (U32)hSize); + hufMetadata->hType = set_compressed; + nextHuf->repeatMode = HUF_repeat_check; + return hSize; + } + } +} + +/** ZSTD_buildSuperBlockEntropy_sequences() : + * Builds entropy for the super-block sequences. + * Stores symbol compression modes and fse table to fseMetadata. + * @return : size of fse tables or error code */ +static size_t ZSTD_buildSuperBlockEntropy_sequences(seqStore_t* seqStorePtr, + const ZSTD_fseCTables_t* prevEntropy, + ZSTD_fseCTables_t* nextEntropy, + const ZSTD_CCtx_params* cctxParams, + ZSTD_fseCTablesMetadata_t* fseMetadata, + void* workspace, size_t wkspSize) +{ + BYTE* const wkspStart = (BYTE*)workspace; + BYTE* const wkspEnd = wkspStart + wkspSize; + BYTE* const countWkspStart = wkspStart; + unsigned* const countWksp = (unsigned*)workspace; + const size_t countWkspSize = (MaxSeq + 1) * sizeof(unsigned); + BYTE* const cTableWksp = countWkspStart + countWkspSize; + const size_t cTableWkspSize = wkspEnd-cTableWksp; + ZSTD_strategy const strategy = cctxParams->cParams.strategy; + FSE_CTable* CTable_LitLength = nextEntropy->litlengthCTable; + FSE_CTable* CTable_OffsetBits = nextEntropy->offcodeCTable; + FSE_CTable* CTable_MatchLength = nextEntropy->matchlengthCTable; + const BYTE* const ofCodeTable = seqStorePtr->ofCode; + const BYTE* const llCodeTable = seqStorePtr->llCode; + const BYTE* const mlCodeTable = seqStorePtr->mlCode; + size_t const nbSeq = seqStorePtr->sequences - seqStorePtr->sequencesStart; + BYTE* const ostart = fseMetadata->fseTablesBuffer; + BYTE* const oend = ostart + sizeof(fseMetadata->fseTablesBuffer); + BYTE* op = ostart; + + assert(cTableWkspSize >= (1 << MaxFSELog) * sizeof(FSE_FUNCTION_TYPE)); + DEBUGLOG(5, "ZSTD_buildSuperBlockEntropy_sequences (nbSeq=%zu)", nbSeq); + memset(workspace, 0, wkspSize); + + fseMetadata->lastCountSize = 0; + /* convert length/distances into codes */ + ZSTD_seqToCodes(seqStorePtr); + /* build CTable for Literal Lengths */ + { U32 LLtype; + unsigned max = MaxLL; + size_t const mostFrequent = HIST_countFast_wksp(countWksp, &max, llCodeTable, nbSeq, workspace, wkspSize); /* can't fail */ + DEBUGLOG(5, "Building LL table"); + nextEntropy->litlength_repeatMode = prevEntropy->litlength_repeatMode; + LLtype = ZSTD_selectEncodingType(&nextEntropy->litlength_repeatMode, + countWksp, max, mostFrequent, nbSeq, + LLFSELog, prevEntropy->litlengthCTable, + LL_defaultNorm, LL_defaultNormLog, + ZSTD_defaultAllowed, strategy); + assert(set_basic < set_compressed && set_rle < set_compressed); + assert(!(LLtype < set_compressed && nextEntropy->litlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */ + { size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_LitLength, LLFSELog, (symbolEncodingType_e)LLtype, + countWksp, max, llCodeTable, nbSeq, LL_defaultNorm, LL_defaultNormLog, MaxLL, + prevEntropy->litlengthCTable, sizeof(prevEntropy->litlengthCTable), + cTableWksp, cTableWkspSize); + FORWARD_IF_ERROR(countSize, "ZSTD_buildCTable for LitLens failed"); + if (LLtype == set_compressed) + fseMetadata->lastCountSize = countSize; + op += countSize; + fseMetadata->llType = (symbolEncodingType_e) LLtype; + } } + /* build CTable for Offsets */ + { U32 Offtype; + unsigned max = MaxOff; + size_t const mostFrequent = HIST_countFast_wksp(countWksp, &max, ofCodeTable, nbSeq, workspace, wkspSize); /* can't fail */ + /* We can only use the basic table if max <= DefaultMaxOff, otherwise the offsets are too large */ + ZSTD_defaultPolicy_e const defaultPolicy = (max <= DefaultMaxOff) ? ZSTD_defaultAllowed : ZSTD_defaultDisallowed; + DEBUGLOG(5, "Building OF table"); + nextEntropy->offcode_repeatMode = prevEntropy->offcode_repeatMode; + Offtype = ZSTD_selectEncodingType(&nextEntropy->offcode_repeatMode, + countWksp, max, mostFrequent, nbSeq, + OffFSELog, prevEntropy->offcodeCTable, + OF_defaultNorm, OF_defaultNormLog, + defaultPolicy, strategy); + assert(!(Offtype < set_compressed && nextEntropy->offcode_repeatMode != FSE_repeat_none)); /* We don't copy tables */ + { size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_OffsetBits, OffFSELog, (symbolEncodingType_e)Offtype, + countWksp, max, ofCodeTable, nbSeq, OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff, + prevEntropy->offcodeCTable, sizeof(prevEntropy->offcodeCTable), + cTableWksp, cTableWkspSize); + FORWARD_IF_ERROR(countSize, "ZSTD_buildCTable for Offsets failed"); + if (Offtype == set_compressed) + fseMetadata->lastCountSize = countSize; + op += countSize; + fseMetadata->ofType = (symbolEncodingType_e) Offtype; + } } + /* build CTable for MatchLengths */ + { U32 MLtype; + unsigned max = MaxML; + size_t const mostFrequent = HIST_countFast_wksp(countWksp, &max, mlCodeTable, nbSeq, workspace, wkspSize); /* can't fail */ + DEBUGLOG(5, "Building ML table (remaining space : %i)", (int)(oend-op)); + nextEntropy->matchlength_repeatMode = prevEntropy->matchlength_repeatMode; + MLtype = ZSTD_selectEncodingType(&nextEntropy->matchlength_repeatMode, + countWksp, max, mostFrequent, nbSeq, + MLFSELog, prevEntropy->matchlengthCTable, + ML_defaultNorm, ML_defaultNormLog, + ZSTD_defaultAllowed, strategy); + assert(!(MLtype < set_compressed && nextEntropy->matchlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */ + { size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_MatchLength, MLFSELog, (symbolEncodingType_e)MLtype, + countWksp, max, mlCodeTable, nbSeq, ML_defaultNorm, ML_defaultNormLog, MaxML, + prevEntropy->matchlengthCTable, sizeof(prevEntropy->matchlengthCTable), + cTableWksp, cTableWkspSize); + FORWARD_IF_ERROR(countSize, "ZSTD_buildCTable for MatchLengths failed"); + if (MLtype == set_compressed) + fseMetadata->lastCountSize = countSize; + op += countSize; + fseMetadata->mlType = (symbolEncodingType_e) MLtype; + } } + assert((size_t) (op-ostart) <= sizeof(fseMetadata->fseTablesBuffer)); + return op-ostart; +} + + +/** ZSTD_buildSuperBlockEntropy() : + * Builds entropy for the super-block. + * @return : 0 on success or error code */ +static size_t +ZSTD_buildSuperBlockEntropy(seqStore_t* seqStorePtr, + const ZSTD_entropyCTables_t* prevEntropy, + ZSTD_entropyCTables_t* nextEntropy, + const ZSTD_CCtx_params* cctxParams, + ZSTD_entropyCTablesMetadata_t* entropyMetadata, + void* workspace, size_t wkspSize) +{ + size_t const litSize = seqStorePtr->lit - seqStorePtr->litStart; + DEBUGLOG(5, "ZSTD_buildSuperBlockEntropy"); + entropyMetadata->hufMetadata.hufDesSize = + ZSTD_buildSuperBlockEntropy_literal(seqStorePtr->litStart, litSize, + &prevEntropy->huf, &nextEntropy->huf, + &entropyMetadata->hufMetadata, + ZSTD_disableLiteralsCompression(cctxParams), + workspace, wkspSize); + FORWARD_IF_ERROR(entropyMetadata->hufMetadata.hufDesSize, "ZSTD_buildSuperBlockEntropy_literal failed"); + entropyMetadata->fseMetadata.fseTablesSize = + ZSTD_buildSuperBlockEntropy_sequences(seqStorePtr, + &prevEntropy->fse, &nextEntropy->fse, + cctxParams, + &entropyMetadata->fseMetadata, + workspace, wkspSize); + FORWARD_IF_ERROR(entropyMetadata->fseMetadata.fseTablesSize, "ZSTD_buildSuperBlockEntropy_sequences failed"); + return 0; +} + +/** ZSTD_compressSubBlock_literal() : + * Compresses literals section for a sub-block. + * When we have to write the Huffman table we will sometimes choose a header + * size larger than necessary. This is because we have to pick the header size + * before we know the table size + compressed size, so we have a bound on the + * table size. If we guessed incorrectly, we fall back to uncompressed literals. + * + * We write the header when writeEntropy=1 and set entropyWrriten=1 when we succeeded + * in writing the header, otherwise it is set to 0. + * + * hufMetadata->hType has literals block type info. + * If it is set_basic, all sub-blocks literals section will be Raw_Literals_Block. + * If it is set_rle, all sub-blocks literals section will be RLE_Literals_Block. + * If it is set_compressed, first sub-block's literals section will be Compressed_Literals_Block + * If it is set_compressed, first sub-block's literals section will be Treeless_Literals_Block + * and the following sub-blocks' literals sections will be Treeless_Literals_Block. + * @return : compressed size of literals section of a sub-block + * Or 0 if it unable to compress. + * Or error code */ +static size_t ZSTD_compressSubBlock_literal(const HUF_CElt* hufTable, + const ZSTD_hufCTablesMetadata_t* hufMetadata, + const BYTE* literals, size_t litSize, + void* dst, size_t dstSize, + const int bmi2, int writeEntropy, int* entropyWritten) +{ + size_t const header = writeEntropy ? 200 : 0; + size_t const lhSize = 3 + (litSize >= (1 KB - header)) + (litSize >= (16 KB - header)); + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + dstSize; + BYTE* op = ostart + lhSize; + U32 const singleStream = lhSize == 3; + symbolEncodingType_e hType = writeEntropy ? hufMetadata->hType : set_repeat; + size_t cLitSize = 0; + + (void)bmi2; /* TODO bmi2... */ + + DEBUGLOG(5, "ZSTD_compressSubBlock_literal (litSize=%zu, lhSize=%zu, writeEntropy=%d)", litSize, lhSize, writeEntropy); + + *entropyWritten = 0; + if (litSize == 0 || hufMetadata->hType == set_basic) { + DEBUGLOG(5, "ZSTD_compressSubBlock_literal using raw literal"); + return ZSTD_noCompressLiterals(dst, dstSize, literals, litSize); + } else if (hufMetadata->hType == set_rle) { + DEBUGLOG(5, "ZSTD_compressSubBlock_literal using rle literal"); + return ZSTD_compressRleLiteralsBlock(dst, dstSize, literals, litSize); + } + + assert(litSize > 0); + assert(hufMetadata->hType == set_compressed || hufMetadata->hType == set_repeat); + + if (writeEntropy && hufMetadata->hType == set_compressed) { + memcpy(op, hufMetadata->hufDesBuffer, hufMetadata->hufDesSize); + op += hufMetadata->hufDesSize; + cLitSize += hufMetadata->hufDesSize; + DEBUGLOG(5, "ZSTD_compressSubBlock_literal (hSize=%zu)", hufMetadata->hufDesSize); + } + + /* TODO bmi2 */ + { const size_t cSize = singleStream ? HUF_compress1X_usingCTable(op, oend-op, literals, litSize, hufTable) + : HUF_compress4X_usingCTable(op, oend-op, literals, litSize, hufTable); + op += cSize; + cLitSize += cSize; + if (cSize == 0 || ERR_isError(cSize)) { + DEBUGLOG(5, "Failed to write entropy tables %s", ZSTD_getErrorName(cSize)); + return 0; + } + /* If we expand and we aren't writing a header then emit uncompressed */ + if (!writeEntropy && cLitSize >= litSize) { + DEBUGLOG(5, "ZSTD_compressSubBlock_literal using raw literal because uncompressible"); + return ZSTD_noCompressLiterals(dst, dstSize, literals, litSize); + } + /* If we are writing headers then allow expansion that doesn't change our header size. */ + if (lhSize < (size_t)(3 + (cLitSize >= 1 KB) + (cLitSize >= 16 KB))) { + assert(cLitSize > litSize); + DEBUGLOG(5, "Literals expanded beyond allowed header size"); + return ZSTD_noCompressLiterals(dst, dstSize, literals, litSize); + } + DEBUGLOG(5, "ZSTD_compressSubBlock_literal (cSize=%zu)", cSize); + } + + /* Build header */ + switch(lhSize) + { + case 3: /* 2 - 2 - 10 - 10 */ + { U32 const lhc = hType + ((!singleStream) << 2) + ((U32)litSize<<4) + ((U32)cLitSize<<14); + MEM_writeLE24(ostart, lhc); + break; + } + case 4: /* 2 - 2 - 14 - 14 */ + { U32 const lhc = hType + (2 << 2) + ((U32)litSize<<4) + ((U32)cLitSize<<18); + MEM_writeLE32(ostart, lhc); + break; + } + case 5: /* 2 - 2 - 18 - 18 */ + { U32 const lhc = hType + (3 << 2) + ((U32)litSize<<4) + ((U32)cLitSize<<22); + MEM_writeLE32(ostart, lhc); + ostart[4] = (BYTE)(cLitSize >> 10); + break; + } + default: /* not possible : lhSize is {3,4,5} */ + assert(0); + } + *entropyWritten = 1; + DEBUGLOG(5, "Compressed literals: %u -> %u", (U32)litSize, (U32)(op-ostart)); + return op-ostart; +} + +static size_t ZSTD_seqDecompressedSize(seqStore_t const* seqStore, const seqDef* sequences, size_t nbSeq, size_t litSize, int lastSequence) { + const seqDef* const sstart = sequences; + const seqDef* const send = sequences + nbSeq; + const seqDef* sp = sstart; + size_t matchLengthSum = 0; + size_t litLengthSum = 0; + while (send-sp > 0) { + ZSTD_sequenceLength const seqLen = ZSTD_getSequenceLength(seqStore, sp); + litLengthSum += seqLen.litLength; + matchLengthSum += seqLen.matchLength; + sp++; + } + assert(litLengthSum <= litSize); + if (!lastSequence) { + assert(litLengthSum == litSize); + } + return matchLengthSum + litSize; +} + +/** ZSTD_compressSubBlock_sequences() : + * Compresses sequences section for a sub-block. + * fseMetadata->llType, fseMetadata->ofType, and fseMetadata->mlType have + * symbol compression modes for the super-block. + * The first successfully compressed block will have these in its header. + * We set entropyWritten=1 when we succeed in compressing the sequences. + * The following sub-blocks will always have repeat mode. + * @return : compressed size of sequences section of a sub-block + * Or 0 if it is unable to compress + * Or error code. */ +static size_t ZSTD_compressSubBlock_sequences(const ZSTD_fseCTables_t* fseTables, + const ZSTD_fseCTablesMetadata_t* fseMetadata, + const seqDef* sequences, size_t nbSeq, + const BYTE* llCode, const BYTE* mlCode, const BYTE* ofCode, + const ZSTD_CCtx_params* cctxParams, + void* dst, size_t dstCapacity, + const int bmi2, int writeEntropy, int* entropyWritten) +{ + const int longOffsets = cctxParams->cParams.windowLog > STREAM_ACCUMULATOR_MIN; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + dstCapacity; + BYTE* op = ostart; + BYTE* seqHead; + + DEBUGLOG(5, "ZSTD_compressSubBlock_sequences (nbSeq=%zu, writeEntropy=%d, longOffsets=%d)", nbSeq, writeEntropy, longOffsets); + + *entropyWritten = 0; + /* Sequences Header */ + RETURN_ERROR_IF((oend-op) < 3 /*max nbSeq Size*/ + 1 /*seqHead*/, + dstSize_tooSmall, ""); + if (nbSeq < 0x7F) + *op++ = (BYTE)nbSeq; + else if (nbSeq < LONGNBSEQ) + op[0] = (BYTE)((nbSeq>>8) + 0x80), op[1] = (BYTE)nbSeq, op+=2; + else + op[0]=0xFF, MEM_writeLE16(op+1, (U16)(nbSeq - LONGNBSEQ)), op+=3; + if (nbSeq==0) { + return op - ostart; + } + + /* seqHead : flags for FSE encoding type */ + seqHead = op++; + + DEBUGLOG(5, "ZSTD_compressSubBlock_sequences (seqHeadSize=%u)", (unsigned)(op-ostart)); + + if (writeEntropy) { + const U32 LLtype = fseMetadata->llType; + const U32 Offtype = fseMetadata->ofType; + const U32 MLtype = fseMetadata->mlType; + DEBUGLOG(5, "ZSTD_compressSubBlock_sequences (fseTablesSize=%zu)", fseMetadata->fseTablesSize); + *seqHead = (BYTE)((LLtype<<6) + (Offtype<<4) + (MLtype<<2)); + memcpy(op, fseMetadata->fseTablesBuffer, fseMetadata->fseTablesSize); + op += fseMetadata->fseTablesSize; + } else { + const U32 repeat = set_repeat; + *seqHead = (BYTE)((repeat<<6) + (repeat<<4) + (repeat<<2)); + } + + { size_t const bitstreamSize = ZSTD_encodeSequences( + op, oend - op, + fseTables->matchlengthCTable, mlCode, + fseTables->offcodeCTable, ofCode, + fseTables->litlengthCTable, llCode, + sequences, nbSeq, + longOffsets, bmi2); + FORWARD_IF_ERROR(bitstreamSize, "ZSTD_encodeSequences failed"); + op += bitstreamSize; + /* zstd versions <= 1.3.4 mistakenly report corruption when + * FSE_readNCount() receives a buffer < 4 bytes. + * Fixed by https://github.com/facebook/zstd/pull/1146. + * This can happen when the last set_compressed table present is 2 + * bytes and the bitstream is only one byte. + * In this exceedingly rare case, we will simply emit an uncompressed + * block, since it isn't worth optimizing. + */ +#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + if (writeEntropy && fseMetadata->lastCountSize && fseMetadata->lastCountSize + bitstreamSize < 4) { + /* NCountSize >= 2 && bitstreamSize > 0 ==> lastCountSize == 3 */ + assert(fseMetadata->lastCountSize + bitstreamSize == 3); + DEBUGLOG(5, "Avoiding bug in zstd decoder in versions <= 1.3.4 by " + "emitting an uncompressed block."); + return 0; + } +#endif + DEBUGLOG(5, "ZSTD_compressSubBlock_sequences (bitstreamSize=%zu)", bitstreamSize); + } + + /* zstd versions <= 1.4.0 mistakenly report error when + * sequences section body size is less than 3 bytes. + * Fixed by https://github.com/facebook/zstd/pull/1664. + * This can happen when the previous sequences section block is compressed + * with rle mode and the current block's sequences section is compressed + * with repeat mode where sequences section body size can be 1 byte. + */ +#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + if (op-seqHead < 4) { + DEBUGLOG(5, "Avoiding bug in zstd decoder in versions <= 1.4.0 by emitting " + "an uncompressed block when sequences are < 4 bytes"); + return 0; + } +#endif + + *entropyWritten = 1; + return op - ostart; +} + +/** ZSTD_compressSubBlock() : + * Compresses a single sub-block. + * @return : compressed size of the sub-block + * Or 0 if it failed to compress. */ +static size_t ZSTD_compressSubBlock(const ZSTD_entropyCTables_t* entropy, + const ZSTD_entropyCTablesMetadata_t* entropyMetadata, + const seqDef* sequences, size_t nbSeq, + const BYTE* literals, size_t litSize, + const BYTE* llCode, const BYTE* mlCode, const BYTE* ofCode, + const ZSTD_CCtx_params* cctxParams, + void* dst, size_t dstCapacity, + const int bmi2, + int writeLitEntropy, int writeSeqEntropy, + int* litEntropyWritten, int* seqEntropyWritten, + U32 lastBlock) +{ + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + dstCapacity; + BYTE* op = ostart + ZSTD_blockHeaderSize; + DEBUGLOG(5, "ZSTD_compressSubBlock (litSize=%zu, nbSeq=%zu, writeLitEntropy=%d, writeSeqEntropy=%d, lastBlock=%d)", + litSize, nbSeq, writeLitEntropy, writeSeqEntropy, lastBlock); + { size_t cLitSize = ZSTD_compressSubBlock_literal((const HUF_CElt*)entropy->huf.CTable, + &entropyMetadata->hufMetadata, literals, litSize, + op, oend-op, bmi2, writeLitEntropy, litEntropyWritten); + FORWARD_IF_ERROR(cLitSize, "ZSTD_compressSubBlock_literal failed"); + if (cLitSize == 0) return 0; + op += cLitSize; + } + { size_t cSeqSize = ZSTD_compressSubBlock_sequences(&entropy->fse, + &entropyMetadata->fseMetadata, + sequences, nbSeq, + llCode, mlCode, ofCode, + cctxParams, + op, oend-op, + bmi2, writeSeqEntropy, seqEntropyWritten); + FORWARD_IF_ERROR(cSeqSize, "ZSTD_compressSubBlock_sequences failed"); + if (cSeqSize == 0) return 0; + op += cSeqSize; + } + /* Write block header */ + { size_t cSize = (op-ostart)-ZSTD_blockHeaderSize; + U32 const cBlockHeader24 = lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3); + MEM_writeLE24(ostart, cBlockHeader24); + } + return op-ostart; +} + +static size_t ZSTD_estimateSubBlockSize_literal(const BYTE* literals, size_t litSize, + const ZSTD_hufCTables_t* huf, + const ZSTD_hufCTablesMetadata_t* hufMetadata, + void* workspace, size_t wkspSize, + int writeEntropy) +{ + unsigned* const countWksp = (unsigned*)workspace; + unsigned maxSymbolValue = 255; + size_t literalSectionHeaderSize = 3; /* Use hard coded size of 3 bytes */ + + if (hufMetadata->hType == set_basic) return litSize; + else if (hufMetadata->hType == set_rle) return 1; + else if (hufMetadata->hType == set_compressed || hufMetadata->hType == set_repeat) { + size_t const largest = HIST_count_wksp (countWksp, &maxSymbolValue, (const BYTE*)literals, litSize, workspace, wkspSize); + if (ZSTD_isError(largest)) return litSize; + { size_t cLitSizeEstimate = HUF_estimateCompressedSize((const HUF_CElt*)huf->CTable, countWksp, maxSymbolValue); + if (writeEntropy) cLitSizeEstimate += hufMetadata->hufDesSize; + return cLitSizeEstimate + literalSectionHeaderSize; + } } + assert(0); /* impossible */ + return 0; +} + +static size_t ZSTD_estimateSubBlockSize_symbolType(symbolEncodingType_e type, + const BYTE* codeTable, unsigned maxCode, + size_t nbSeq, const FSE_CTable* fseCTable, + const U32* additionalBits, + short const* defaultNorm, U32 defaultNormLog, + void* workspace, size_t wkspSize) +{ + unsigned* const countWksp = (unsigned*)workspace; + const BYTE* ctp = codeTable; + const BYTE* const ctStart = ctp; + const BYTE* const ctEnd = ctStart + nbSeq; + size_t cSymbolTypeSizeEstimateInBits = 0; + unsigned max = maxCode; + + HIST_countFast_wksp(countWksp, &max, codeTable, nbSeq, workspace, wkspSize); /* can't fail */ + if (type == set_basic) { + cSymbolTypeSizeEstimateInBits = ZSTD_crossEntropyCost(defaultNorm, defaultNormLog, countWksp, max); + } else if (type == set_rle) { + cSymbolTypeSizeEstimateInBits = 0; + } else if (type == set_compressed || type == set_repeat) { + cSymbolTypeSizeEstimateInBits = ZSTD_fseBitCost(fseCTable, countWksp, max); + } + if (ZSTD_isError(cSymbolTypeSizeEstimateInBits)) return nbSeq * 10; + while (ctp < ctEnd) { + if (additionalBits) cSymbolTypeSizeEstimateInBits += additionalBits[*ctp]; + else cSymbolTypeSizeEstimateInBits += *ctp; /* for offset, offset code is also the number of additional bits */ + ctp++; + } + return cSymbolTypeSizeEstimateInBits / 8; +} + +static size_t ZSTD_estimateSubBlockSize_sequences(const BYTE* ofCodeTable, + const BYTE* llCodeTable, + const BYTE* mlCodeTable, + size_t nbSeq, + const ZSTD_fseCTables_t* fseTables, + const ZSTD_fseCTablesMetadata_t* fseMetadata, + void* workspace, size_t wkspSize, + int writeEntropy) +{ + size_t sequencesSectionHeaderSize = 3; /* Use hard coded size of 3 bytes */ + size_t cSeqSizeEstimate = 0; + cSeqSizeEstimate += ZSTD_estimateSubBlockSize_symbolType(fseMetadata->ofType, ofCodeTable, MaxOff, + nbSeq, fseTables->offcodeCTable, NULL, + OF_defaultNorm, OF_defaultNormLog, + workspace, wkspSize); + cSeqSizeEstimate += ZSTD_estimateSubBlockSize_symbolType(fseMetadata->llType, llCodeTable, MaxLL, + nbSeq, fseTables->litlengthCTable, LL_bits, + LL_defaultNorm, LL_defaultNormLog, + workspace, wkspSize); + cSeqSizeEstimate += ZSTD_estimateSubBlockSize_symbolType(fseMetadata->mlType, mlCodeTable, MaxML, + nbSeq, fseTables->matchlengthCTable, ML_bits, + ML_defaultNorm, ML_defaultNormLog, + workspace, wkspSize); + if (writeEntropy) cSeqSizeEstimate += fseMetadata->fseTablesSize; + return cSeqSizeEstimate + sequencesSectionHeaderSize; +} + +static size_t ZSTD_estimateSubBlockSize(const BYTE* literals, size_t litSize, + const BYTE* ofCodeTable, + const BYTE* llCodeTable, + const BYTE* mlCodeTable, + size_t nbSeq, + const ZSTD_entropyCTables_t* entropy, + const ZSTD_entropyCTablesMetadata_t* entropyMetadata, + void* workspace, size_t wkspSize, + int writeLitEntropy, int writeSeqEntropy) { + size_t cSizeEstimate = 0; + cSizeEstimate += ZSTD_estimateSubBlockSize_literal(literals, litSize, + &entropy->huf, &entropyMetadata->hufMetadata, + workspace, wkspSize, writeLitEntropy); + cSizeEstimate += ZSTD_estimateSubBlockSize_sequences(ofCodeTable, llCodeTable, mlCodeTable, + nbSeq, &entropy->fse, &entropyMetadata->fseMetadata, + workspace, wkspSize, writeSeqEntropy); + return cSizeEstimate + ZSTD_blockHeaderSize; +} + +static int ZSTD_needSequenceEntropyTables(ZSTD_fseCTablesMetadata_t const* fseMetadata) +{ + if (fseMetadata->llType == set_compressed || fseMetadata->llType == set_rle) + return 1; + if (fseMetadata->mlType == set_compressed || fseMetadata->mlType == set_rle) + return 1; + if (fseMetadata->ofType == set_compressed || fseMetadata->ofType == set_rle) + return 1; + return 0; +} + +/** ZSTD_compressSubBlock_multi() : + * Breaks super-block into multiple sub-blocks and compresses them. + * Entropy will be written to the first block. + * The following blocks will use repeat mode to compress. + * All sub-blocks are compressed blocks (no raw or rle blocks). + * @return : compressed size of the super block (which is multiple ZSTD blocks) + * Or 0 if it failed to compress. */ +static size_t ZSTD_compressSubBlock_multi(const seqStore_t* seqStorePtr, + const ZSTD_compressedBlockState_t* prevCBlock, + ZSTD_compressedBlockState_t* nextCBlock, + const ZSTD_entropyCTablesMetadata_t* entropyMetadata, + const ZSTD_CCtx_params* cctxParams, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const int bmi2, U32 lastBlock, + void* workspace, size_t wkspSize) +{ + const seqDef* const sstart = seqStorePtr->sequencesStart; + const seqDef* const send = seqStorePtr->sequences; + const seqDef* sp = sstart; + const BYTE* const lstart = seqStorePtr->litStart; + const BYTE* const lend = seqStorePtr->lit; + const BYTE* lp = lstart; + BYTE const* ip = (BYTE const*)src; + BYTE const* const iend = ip + srcSize; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + dstCapacity; + BYTE* op = ostart; + const BYTE* llCodePtr = seqStorePtr->llCode; + const BYTE* mlCodePtr = seqStorePtr->mlCode; + const BYTE* ofCodePtr = seqStorePtr->ofCode; + size_t targetCBlockSize = cctxParams->targetCBlockSize; + size_t litSize, seqCount; + int writeLitEntropy = entropyMetadata->hufMetadata.hType == set_compressed; + int writeSeqEntropy = 1; + int lastSequence = 0; + + DEBUGLOG(5, "ZSTD_compressSubBlock_multi (litSize=%u, nbSeq=%u)", + (unsigned)(lend-lp), (unsigned)(send-sstart)); + + litSize = 0; + seqCount = 0; + do { + size_t cBlockSizeEstimate = 0; + if (sstart == send) { + lastSequence = 1; + } else { + const seqDef* const sequence = sp + seqCount; + lastSequence = sequence == send - 1; + litSize += ZSTD_getSequenceLength(seqStorePtr, sequence).litLength; + seqCount++; + } + if (lastSequence) { + assert(lp <= lend); + assert(litSize <= (size_t)(lend - lp)); + litSize = (size_t)(lend - lp); + } + /* I think there is an optimization opportunity here. + * Calling ZSTD_estimateSubBlockSize for every sequence can be wasteful + * since it recalculates estimate from scratch. + * For example, it would recount literal distribution and symbol codes everytime. + */ + cBlockSizeEstimate = ZSTD_estimateSubBlockSize(lp, litSize, ofCodePtr, llCodePtr, mlCodePtr, seqCount, + &nextCBlock->entropy, entropyMetadata, + workspace, wkspSize, writeLitEntropy, writeSeqEntropy); + if (cBlockSizeEstimate > targetCBlockSize || lastSequence) { + int litEntropyWritten = 0; + int seqEntropyWritten = 0; + const size_t decompressedSize = ZSTD_seqDecompressedSize(seqStorePtr, sp, seqCount, litSize, lastSequence); + const size_t cSize = ZSTD_compressSubBlock(&nextCBlock->entropy, entropyMetadata, + sp, seqCount, + lp, litSize, + llCodePtr, mlCodePtr, ofCodePtr, + cctxParams, + op, oend-op, + bmi2, writeLitEntropy, writeSeqEntropy, + &litEntropyWritten, &seqEntropyWritten, + lastBlock && lastSequence); + FORWARD_IF_ERROR(cSize, "ZSTD_compressSubBlock failed"); + if (cSize > 0 && cSize < decompressedSize) { + DEBUGLOG(5, "Committed the sub-block"); + assert(ip + decompressedSize <= iend); + ip += decompressedSize; + sp += seqCount; + lp += litSize; + op += cSize; + llCodePtr += seqCount; + mlCodePtr += seqCount; + ofCodePtr += seqCount; + litSize = 0; + seqCount = 0; + /* Entropy only needs to be written once */ + if (litEntropyWritten) { + writeLitEntropy = 0; + } + if (seqEntropyWritten) { + writeSeqEntropy = 0; + } + } + } + } while (!lastSequence); + if (writeLitEntropy) { + DEBUGLOG(5, "ZSTD_compressSubBlock_multi has literal entropy tables unwritten"); + memcpy(&nextCBlock->entropy.huf, &prevCBlock->entropy.huf, sizeof(prevCBlock->entropy.huf)); + } + if (writeSeqEntropy && ZSTD_needSequenceEntropyTables(&entropyMetadata->fseMetadata)) { + /* If we haven't written our entropy tables, then we've violated our contract and + * must emit an uncompressed block. + */ + DEBUGLOG(5, "ZSTD_compressSubBlock_multi has sequence entropy tables unwritten"); + return 0; + } + if (ip < iend) { + size_t const cSize = ZSTD_noCompressBlock(op, oend - op, ip, iend - ip, lastBlock); + DEBUGLOG(5, "ZSTD_compressSubBlock_multi last sub-block uncompressed, %zu bytes", (size_t)(iend - ip)); + FORWARD_IF_ERROR(cSize, "ZSTD_noCompressBlock failed"); + assert(cSize != 0); + op += cSize; + /* We have to regenerate the repcodes because we've skipped some sequences */ + if (sp < send) { + seqDef const* seq; + repcodes_t rep; + memcpy(&rep, prevCBlock->rep, sizeof(rep)); + for (seq = sstart; seq < sp; ++seq) { + rep = ZSTD_updateRep(rep.rep, seq->offset - 1, ZSTD_getSequenceLength(seqStorePtr, seq).litLength == 0); + } + memcpy(nextCBlock->rep, &rep, sizeof(rep)); + } + } + DEBUGLOG(5, "ZSTD_compressSubBlock_multi compressed"); + return op-ostart; +} + +size_t ZSTD_compressSuperBlock(ZSTD_CCtx* zc, + void* dst, size_t dstCapacity, + void const* src, size_t srcSize, + unsigned lastBlock) { + ZSTD_entropyCTablesMetadata_t entropyMetadata; + + FORWARD_IF_ERROR(ZSTD_buildSuperBlockEntropy(&zc->seqStore, + &zc->blockState.prevCBlock->entropy, + &zc->blockState.nextCBlock->entropy, + &zc->appliedParams, + &entropyMetadata, + zc->entropyWorkspace, HUF_WORKSPACE_SIZE /* statically allocated in resetCCtx */), ""); + + return ZSTD_compressSubBlock_multi(&zc->seqStore, + zc->blockState.prevCBlock, + zc->blockState.nextCBlock, + &entropyMetadata, + &zc->appliedParams, + dst, dstCapacity, + src, srcSize, + zc->bmi2, lastBlock, + zc->entropyWorkspace, HUF_WORKSPACE_SIZE /* statically allocated in resetCCtx */); +} diff --git a/thirdparty/zstd/compress/zstd_compress_superblock.h b/thirdparty/zstd/compress/zstd_compress_superblock.h new file mode 100644 index 0000000000..07f4cb1dc6 --- /dev/null +++ b/thirdparty/zstd/compress/zstd_compress_superblock.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_COMPRESS_ADVANCED_H +#define ZSTD_COMPRESS_ADVANCED_H + +/*-************************************* +* Dependencies +***************************************/ + +#include "../zstd.h" /* ZSTD_CCtx */ + +/*-************************************* +* Target Compressed Block Size +***************************************/ + +/* ZSTD_compressSuperBlock() : + * Used to compress a super block when targetCBlockSize is being used. + * The given block will be compressed into multiple sub blocks that are around targetCBlockSize. */ +size_t ZSTD_compressSuperBlock(ZSTD_CCtx* zc, + void* dst, size_t dstCapacity, + void const* src, size_t srcSize, + unsigned lastBlock); + +#endif /* ZSTD_COMPRESS_ADVANCED_H */ diff --git a/thirdparty/zstd/compress/zstd_cwksp.h b/thirdparty/zstd/compress/zstd_cwksp.h index fc9765bd3f..a25c9263b7 100644 --- a/thirdparty/zstd/compress/zstd_cwksp.h +++ b/thirdparty/zstd/compress/zstd_cwksp.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -14,7 +14,7 @@ /*-************************************* * Dependencies ***************************************/ -#include "zstd_internal.h" +#include "../common/zstd_internal.h" #if defined (__cplusplus) extern "C" { @@ -24,16 +24,6 @@ extern "C" { * Constants ***************************************/ -/* define "workspace is too large" as this number of times larger than needed */ -#define ZSTD_WORKSPACETOOLARGE_FACTOR 3 - -/* when workspace is continuously too large - * during at least this number of times, - * context's memory usage is considered wasteful, - * because it's sized to handle a worst case scenario which rarely happens. - * In which case, resize it down to free some memory */ -#define ZSTD_WORKSPACETOOLARGE_MAXDURATION 128 - /* Since the workspace is effectively its own little malloc implementation / * arena, when we run under ASAN, we should similarly insert redzones between * each internal element of the workspace, so ASAN will catch overruns that @@ -468,7 +458,7 @@ MEM_STATIC void ZSTD_cwksp_init(ZSTD_cwksp* ws, void* start, size_t size) { MEM_STATIC size_t ZSTD_cwksp_create(ZSTD_cwksp* ws, size_t size, ZSTD_customMem customMem) { void* workspace = ZSTD_malloc(size, customMem); DEBUGLOG(4, "cwksp: creating new workspace with %zd bytes", size); - RETURN_ERROR_IF(workspace == NULL, memory_allocation); + RETURN_ERROR_IF(workspace == NULL, memory_allocation, "NULL pointer!"); ZSTD_cwksp_init(ws, workspace, size); return 0; } diff --git a/thirdparty/zstd/compress/zstd_double_fast.c b/thirdparty/zstd/compress/zstd_double_fast.c index a661a48534..27eed66cfe 100644 --- a/thirdparty/zstd/compress/zstd_double_fast.c +++ b/thirdparty/zstd/compress/zstd_double_fast.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -63,10 +63,8 @@ size_t ZSTD_compressBlock_doubleFast_generic( const BYTE* ip = istart; const BYTE* anchor = istart; const U32 endIndex = (U32)((size_t)(istart - base) + srcSize); - const U32 lowestValid = ms->window.dictLimit; - const U32 maxDistance = 1U << cParams->windowLog; /* presumes that, if there is a dictionary, it must be using Attach mode */ - const U32 prefixLowestIndex = (endIndex - lowestValid > maxDistance) ? endIndex - maxDistance : lowestValid; + const U32 prefixLowestIndex = ZSTD_getLowestPrefixIndex(ms, endIndex, cParams->windowLog); const BYTE* const prefixLowest = base + prefixLowestIndex; const BYTE* const iend = istart + srcSize; const BYTE* const ilimit = iend - HASH_READ_SIZE; @@ -96,7 +94,7 @@ size_t ZSTD_compressBlock_doubleFast_generic( dictCParams->hashLog : hBitsL; const U32 dictHBitsS = dictMode == ZSTD_dictMatchState ? dictCParams->chainLog : hBitsS; - const U32 dictAndPrefixLength = (U32)(ip - prefixLowest + dictEnd - dictStart); + const U32 dictAndPrefixLength = (U32)((ip - prefixLowest) + (dictEnd - dictStart)); DEBUGLOG(5, "ZSTD_compressBlock_doubleFast_generic"); @@ -104,13 +102,15 @@ size_t ZSTD_compressBlock_doubleFast_generic( /* if a dictionary is attached, it must be within window range */ if (dictMode == ZSTD_dictMatchState) { - assert(lowestValid + maxDistance >= endIndex); + assert(ms->window.dictLimit + (1U << cParams->windowLog) >= endIndex); } /* init */ ip += (dictAndPrefixLength == 0); if (dictMode == ZSTD_noDict) { - U32 const maxRep = (U32)(ip - prefixLowest); + U32 const current = (U32)(ip - base); + U32 const windowLow = ZSTD_getLowestPrefixIndex(ms, current, cParams->windowLog); + U32 const maxRep = current - windowLow; if (offset_2 > maxRep) offsetSaved = offset_2, offset_2 = 0; if (offset_1 > maxRep) offsetSaved = offset_1, offset_1 = 0; } @@ -198,6 +198,9 @@ size_t ZSTD_compressBlock_doubleFast_generic( } } ip += ((ip-anchor) >> kSearchStrength) + 1; +#if defined(__aarch64__) + PREFETCH_L1(ip+256); +#endif continue; _search_next_long: @@ -271,7 +274,7 @@ _match_stored: U32 const repIndex2 = current2 - offset_2; const BYTE* repMatch2 = dictMode == ZSTD_dictMatchState && repIndex2 < prefixLowestIndex ? - dictBase - dictIndexDelta + repIndex2 : + dictBase + repIndex2 - dictIndexDelta : base + repIndex2; if ( ((U32)((prefixLowestIndex-1) - (U32)repIndex2) >= 3 /* intentional overflow */) && (MEM_read32(repMatch2) == MEM_read32(ip)) ) { diff --git a/thirdparty/zstd/compress/zstd_double_fast.h b/thirdparty/zstd/compress/zstd_double_fast.h index 4fa31acfc0..14d944d69b 100644 --- a/thirdparty/zstd/compress/zstd_double_fast.h +++ b/thirdparty/zstd/compress/zstd_double_fast.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -15,7 +15,7 @@ extern "C" { #endif -#include "mem.h" /* U32 */ +#include "../common/mem.h" /* U32 */ #include "zstd_compress_internal.h" /* ZSTD_CCtx, size_t */ void ZSTD_fillDoubleHashTable(ZSTD_matchState_t* ms, diff --git a/thirdparty/zstd/compress/zstd_fast.c b/thirdparty/zstd/compress/zstd_fast.c index 6dbefee6b7..85a3a7a91e 100644 --- a/thirdparty/zstd/compress/zstd_fast.c +++ b/thirdparty/zstd/compress/zstd_fast.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -61,9 +61,7 @@ ZSTD_compressBlock_fast_generic( const BYTE* ip1; const BYTE* anchor = istart; const U32 endIndex = (U32)((size_t)(istart - base) + srcSize); - const U32 maxDistance = 1U << cParams->windowLog; - const U32 validStartIndex = ms->window.dictLimit; - const U32 prefixStartIndex = (endIndex - validStartIndex > maxDistance) ? endIndex - maxDistance : validStartIndex; + const U32 prefixStartIndex = ZSTD_getLowestPrefixIndex(ms, endIndex, cParams->windowLog); const BYTE* const prefixStart = base + prefixStartIndex; const BYTE* const iend = istart + srcSize; const BYTE* const ilimit = iend - HASH_READ_SIZE; @@ -74,12 +72,21 @@ ZSTD_compressBlock_fast_generic( DEBUGLOG(5, "ZSTD_compressBlock_fast_generic"); ip0 += (ip0 == prefixStart); ip1 = ip0 + 1; - { U32 const maxRep = (U32)(ip0 - prefixStart); + { U32 const current = (U32)(ip0 - base); + U32 const windowLow = ZSTD_getLowestPrefixIndex(ms, current, cParams->windowLog); + U32 const maxRep = current - windowLow; if (offset_2 > maxRep) offsetSaved = offset_2, offset_2 = 0; if (offset_1 > maxRep) offsetSaved = offset_1, offset_1 = 0; } /* Main Search Loop */ +#ifdef __INTEL_COMPILER + /* From intel 'The vector pragma indicates that the loop should be + * vectorized if it is legal to do so'. Can be used together with + * #pragma ivdep (but have opted to exclude that because intel + * warns against using it).*/ + #pragma vector always +#endif while (ip1 < ilimit) { /* < instead of <=, because check at ip0+2 */ size_t mLength; BYTE const* ip2 = ip0 + 2; @@ -91,19 +98,25 @@ ZSTD_compressBlock_fast_generic( U32 const current1 = (U32)(ip1-base); U32 const matchIndex0 = hashTable[h0]; U32 const matchIndex1 = hashTable[h1]; - BYTE const* repMatch = ip2-offset_1; + BYTE const* repMatch = ip2 - offset_1; const BYTE* match0 = base + matchIndex0; const BYTE* match1 = base + matchIndex1; U32 offcode; + +#if defined(__aarch64__) + PREFETCH_L1(ip0+256); +#endif + hashTable[h0] = current0; /* update hash table */ hashTable[h1] = current1; /* update hash table */ assert(ip0 + 1 == ip1); if ((offset_1 > 0) & (MEM_read32(repMatch) == MEM_read32(ip2))) { - mLength = ip2[-1] == repMatch[-1] ? 1 : 0; + mLength = (ip2[-1] == repMatch[-1]) ? 1 : 0; ip0 = ip2 - mLength; match0 = repMatch - mLength; + mLength += 4; offcode = 0; goto _match; } @@ -128,19 +141,18 @@ _offset: /* Requires: ip0, match0 */ offset_2 = offset_1; offset_1 = (U32)(ip0-match0); offcode = offset_1 + ZSTD_REP_MOVE; - mLength = 0; + mLength = 4; /* Count the backwards match length */ while (((ip0>anchor) & (match0>prefixStart)) && (ip0[-1] == match0[-1])) { ip0--; match0--; mLength++; } /* catch up */ _match: /* Requires: ip0, match0, offcode */ /* Count the forward length */ - mLength += ZSTD_count(ip0+mLength+4, match0+mLength+4, iend) + 4; + mLength += ZSTD_count(ip0+mLength, match0+mLength, iend); ZSTD_storeSeq(seqStore, (size_t)(ip0-anchor), anchor, iend, offcode, mLength-MINMATCH); /* match found */ ip0 += mLength; anchor = ip0; - ip1 = ip0 + 1; if (ip0 <= ilimit) { /* Fill Table */ @@ -148,19 +160,18 @@ _match: /* Requires: ip0, match0, offcode */ hashTable[ZSTD_hashPtr(base+current0+2, hlog, mls)] = current0+2; /* here because current+2 could be > iend-8 */ hashTable[ZSTD_hashPtr(ip0-2, hlog, mls)] = (U32)(ip0-2-base); - while ( ((ip0 <= ilimit) & (offset_2>0)) /* offset_2==0 means offset_2 is invalidated */ - && (MEM_read32(ip0) == MEM_read32(ip0 - offset_2)) ) { - /* store sequence */ - size_t const rLength = ZSTD_count(ip0+4, ip0+4-offset_2, iend) + 4; - { U32 const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff; } /* swap offset_2 <=> offset_1 */ - hashTable[ZSTD_hashPtr(ip0, hlog, mls)] = (U32)(ip0-base); - ip0 += rLength; - ip1 = ip0 + 1; - ZSTD_storeSeq(seqStore, 0 /*litLen*/, anchor, iend, 0 /*offCode*/, rLength-MINMATCH); - anchor = ip0; - continue; /* faster when present (confirmed on gcc-8) ... (?) */ - } - } + if (offset_2 > 0) { /* offset_2==0 means offset_2 is invalidated */ + while ( (ip0 <= ilimit) && (MEM_read32(ip0) == MEM_read32(ip0 - offset_2)) ) { + /* store sequence */ + size_t const rLength = ZSTD_count(ip0+4, ip0+4-offset_2, iend) + 4; + { U32 const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff; } /* swap offset_2 <=> offset_1 */ + hashTable[ZSTD_hashPtr(ip0, hlog, mls)] = (U32)(ip0-base); + ip0 += rLength; + ZSTD_storeSeq(seqStore, 0 /*litLen*/, anchor, iend, 0 /*offCode*/, rLength-MINMATCH); + anchor = ip0; + continue; /* faster when present (confirmed on gcc-8) ... (?) */ + } } } + ip1 = ip0 + 1; } /* save reps for next block */ @@ -387,7 +398,7 @@ static size_t ZSTD_compressBlock_fast_extDict_generic( const BYTE* const ilimit = iend - 8; U32 offset_1=rep[0], offset_2=rep[1]; - DEBUGLOG(5, "ZSTD_compressBlock_fast_extDict_generic"); + DEBUGLOG(5, "ZSTD_compressBlock_fast_extDict_generic (offset_1=%u)", offset_1); /* switch to "regular" variant if extDict is invalidated due to maxDistance */ if (prefixStartIndex == dictStartIndex) @@ -404,6 +415,7 @@ static size_t ZSTD_compressBlock_fast_extDict_generic( const BYTE* const repBase = repIndex < prefixStartIndex ? dictBase : base; const BYTE* const repMatch = repBase + repIndex; hashTable[h] = current; /* update hash table */ + DEBUGLOG(7, "offset_1 = %u , current = %u", offset_1, current); assert(offset_1 <= current +1); /* check repIndex */ if ( (((U32)((prefixStartIndex-1) - repIndex) >= 3) /* intentional underflow */ & (repIndex > dictStartIndex)) diff --git a/thirdparty/zstd/compress/zstd_fast.h b/thirdparty/zstd/compress/zstd_fast.h index b74a88c57c..cf6aaa8e67 100644 --- a/thirdparty/zstd/compress/zstd_fast.h +++ b/thirdparty/zstd/compress/zstd_fast.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -15,7 +15,7 @@ extern "C" { #endif -#include "mem.h" /* U32 */ +#include "../common/mem.h" /* U32 */ #include "zstd_compress_internal.h" void ZSTD_fillHashTable(ZSTD_matchState_t* ms, diff --git a/thirdparty/zstd/compress/zstd_lazy.c b/thirdparty/zstd/compress/zstd_lazy.c index 9ad7e03b54..4cf5c88b53 100644 --- a/thirdparty/zstd/compress/zstd_lazy.c +++ b/thirdparty/zstd/compress/zstd_lazy.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -660,12 +660,16 @@ ZSTD_compressBlock_lazy_generic( const U32 dictIndexDelta = dictMode == ZSTD_dictMatchState ? prefixLowestIndex - (U32)(dictEnd - dictBase) : 0; - const U32 dictAndPrefixLength = (U32)(ip - prefixLowest + dictEnd - dictLowest); + const U32 dictAndPrefixLength = (U32)((ip - prefixLowest) + (dictEnd - dictLowest)); + + DEBUGLOG(5, "ZSTD_compressBlock_lazy_generic (dictMode=%u)", (U32)dictMode); /* init */ ip += (dictAndPrefixLength == 0); if (dictMode == ZSTD_noDict) { - U32 const maxRep = (U32)(ip - prefixLowest); + U32 const current = (U32)(ip - base); + U32 const windowLow = ZSTD_getLowestPrefixIndex(ms, current, ms->cParams.windowLog); + U32 const maxRep = current - windowLow; if (offset_2 > maxRep) savedOffset = offset_2, offset_2 = 0; if (offset_1 > maxRep) savedOffset = offset_1, offset_1 = 0; } @@ -677,6 +681,12 @@ ZSTD_compressBlock_lazy_generic( } /* Match Loop */ +#if defined(__GNUC__) && defined(__x86_64__) + /* I've measured random a 5% speed loss on levels 5 & 6 (greedy) when the + * code alignment is perturbed. To fix the instability align the loop on 32-bytes. + */ + __asm__(".p2align 5"); +#endif while (ip < ilimit) { size_t matchLength=0; size_t offset=0; @@ -929,11 +939,11 @@ size_t ZSTD_compressBlock_lazy_extDict_generic( const BYTE* const ilimit = iend - 8; const BYTE* const base = ms->window.base; const U32 dictLimit = ms->window.dictLimit; - const U32 lowestIndex = ms->window.lowLimit; const BYTE* const prefixStart = base + dictLimit; const BYTE* const dictBase = ms->window.dictBase; const BYTE* const dictEnd = dictBase + dictLimit; - const BYTE* const dictStart = dictBase + lowestIndex; + const BYTE* const dictStart = dictBase + ms->window.lowLimit; + const U32 windowLog = ms->cParams.windowLog; typedef size_t (*searchMax_f)( ZSTD_matchState_t* ms, @@ -942,10 +952,18 @@ size_t ZSTD_compressBlock_lazy_extDict_generic( U32 offset_1 = rep[0], offset_2 = rep[1]; + DEBUGLOG(5, "ZSTD_compressBlock_lazy_extDict_generic"); + /* init */ ip += (ip == prefixStart); /* Match Loop */ +#if defined(__GNUC__) && defined(__x86_64__) + /* I've measured random a 5% speed loss on levels 5 & 6 (greedy) when the + * code alignment is perturbed. To fix the instability align the loop on 32-bytes. + */ + __asm__(".p2align 5"); +#endif while (ip < ilimit) { size_t matchLength=0; size_t offset=0; @@ -953,10 +971,11 @@ size_t ZSTD_compressBlock_lazy_extDict_generic( U32 current = (U32)(ip-base); /* check repCode */ - { const U32 repIndex = (U32)(current+1 - offset_1); + { const U32 windowLow = ZSTD_getLowestMatchIndex(ms, current+1, windowLog); + const U32 repIndex = (U32)(current+1 - offset_1); const BYTE* const repBase = repIndex < dictLimit ? dictBase : base; const BYTE* const repMatch = repBase + repIndex; - if (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex > lowestIndex)) /* intentional overflow */ + if (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex > windowLow)) /* intentional overflow */ if (MEM_read32(ip+1) == MEM_read32(repMatch)) { /* repcode detected we should take it */ const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; @@ -983,10 +1002,11 @@ size_t ZSTD_compressBlock_lazy_extDict_generic( current++; /* check repCode */ if (offset) { + const U32 windowLow = ZSTD_getLowestMatchIndex(ms, current, windowLog); const U32 repIndex = (U32)(current - offset_1); const BYTE* const repBase = repIndex < dictLimit ? dictBase : base; const BYTE* const repMatch = repBase + repIndex; - if (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex > lowestIndex)) /* intentional overflow */ + if (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex > windowLow)) /* intentional overflow */ if (MEM_read32(ip) == MEM_read32(repMatch)) { /* repcode detected */ const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; @@ -1013,10 +1033,11 @@ size_t ZSTD_compressBlock_lazy_extDict_generic( current++; /* check repCode */ if (offset) { + const U32 windowLow = ZSTD_getLowestMatchIndex(ms, current, windowLog); const U32 repIndex = (U32)(current - offset_1); const BYTE* const repBase = repIndex < dictLimit ? dictBase : base; const BYTE* const repMatch = repBase + repIndex; - if (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex > lowestIndex)) /* intentional overflow */ + if (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex > windowLow)) /* intentional overflow */ if (MEM_read32(ip) == MEM_read32(repMatch)) { /* repcode detected */ const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; @@ -1057,10 +1078,12 @@ _storeSequence: /* check immediate repcode */ while (ip <= ilimit) { - const U32 repIndex = (U32)((ip-base) - offset_2); + const U32 repCurrent = (U32)(ip-base); + const U32 windowLow = ZSTD_getLowestMatchIndex(ms, repCurrent, windowLog); + const U32 repIndex = repCurrent - offset_2; const BYTE* const repBase = repIndex < dictLimit ? dictBase : base; const BYTE* const repMatch = repBase + repIndex; - if (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex > lowestIndex)) /* intentional overflow */ + if (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex > windowLow)) /* intentional overflow */ if (MEM_read32(ip) == MEM_read32(repMatch)) { /* repcode detected we should take it */ const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; diff --git a/thirdparty/zstd/compress/zstd_lazy.h b/thirdparty/zstd/compress/zstd_lazy.h index bb1763069f..581936f03b 100644 --- a/thirdparty/zstd/compress/zstd_lazy.h +++ b/thirdparty/zstd/compress/zstd_lazy.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/thirdparty/zstd/compress/zstd_ldm.c b/thirdparty/zstd/compress/zstd_ldm.c index c3312ad3e3..8c47948358 100644 --- a/thirdparty/zstd/compress/zstd_ldm.c +++ b/thirdparty/zstd/compress/zstd_ldm.c @@ -1,15 +1,16 @@ /* - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the * LICENSE file in the root directory of this source tree) and the GPLv2 (found * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. */ #include "zstd_ldm.h" -#include "debug.h" +#include "../common/debug.h" #include "zstd_fast.h" /* ZSTD_fillHashTable() */ #include "zstd_double_fast.h" /* ZSTD_fillDoubleHashTable() */ @@ -223,6 +224,20 @@ static U64 ZSTD_ldm_fillLdmHashTable(ldmState_t* state, return rollingHash; } +void ZSTD_ldm_fillHashTable( + ldmState_t* state, const BYTE* ip, + const BYTE* iend, ldmParams_t const* params) +{ + DEBUGLOG(5, "ZSTD_ldm_fillHashTable"); + if ((size_t)(iend - ip) >= params->minMatchLength) { + U64 startingHash = ZSTD_rollingHash_compute(ip, params->minMatchLength); + ZSTD_ldm_fillLdmHashTable( + state, startingHash, ip, iend - params->minMatchLength, state->window.base, + params->hashLog - params->bucketSizeLog, + *params); + } +} + /** ZSTD_ldm_limitTableUpdate() : * @@ -449,6 +464,8 @@ size_t ZSTD_ldm_generateSequences( U32 const correction = ZSTD_window_correctOverflow( &ldmState->window, /* cycleLog */ 0, maxDist, chunkStart); ZSTD_ldm_reduceTable(ldmState->hashTable, ldmHSize, correction); + /* invalidate dictionaries on overflow correction */ + ldmState->loadedDictEnd = 0; } /* 2. We enforce the maximum offset allowed. * @@ -457,8 +474,14 @@ size_t ZSTD_ldm_generateSequences( * TODO: * Test the chunk size. * * Try invalidation after the sequence generation and test the * the offset against maxDist directly. + * + * NOTE: Because of dictionaries + sequence splitting we MUST make sure + * that any offset used is valid at the END of the sequence, since it may + * be split into two sequences. This condition holds when using + * ZSTD_window_enforceMaxDist(), but if we move to checking offsets + * against maxDist directly, we'll have to carefully handle that case. */ - ZSTD_window_enforceMaxDist(&ldmState->window, chunkEnd, maxDist, NULL, NULL); + ZSTD_window_enforceMaxDist(&ldmState->window, chunkEnd, maxDist, &ldmState->loadedDictEnd, NULL); /* 3. Generate the sequences for the chunk, and get newLeftoverSize. */ newLeftoverSize = ZSTD_ldm_generateSequences_internal( ldmState, sequences, params, chunkStart, chunkSize); @@ -566,14 +589,13 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore, if (sequence.offset == 0) break; - assert(sequence.offset <= (1U << cParams->windowLog)); assert(ip + sequence.litLength + sequence.matchLength <= iend); /* Fill tables for block compressor */ ZSTD_ldm_limitTableUpdate(ms, ip); ZSTD_ldm_fillFastTables(ms, ip); /* Run the block compressor */ - DEBUGLOG(5, "calling block compressor on segment of size %u", sequence.litLength); + DEBUGLOG(5, "pos %u : calling block compressor on segment of size %u", (unsigned)(ip-istart), sequence.litLength); { size_t const newLitLength = blockCompressor(ms, seqStore, rep, ip, sequence.litLength); diff --git a/thirdparty/zstd/compress/zstd_ldm.h b/thirdparty/zstd/compress/zstd_ldm.h index a47846128b..229ea05a9e 100644 --- a/thirdparty/zstd/compress/zstd_ldm.h +++ b/thirdparty/zstd/compress/zstd_ldm.h @@ -1,10 +1,11 @@ /* - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the * LICENSE file in the root directory of this source tree) and the GPLv2 (found * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. */ #ifndef ZSTD_LDM_H @@ -15,7 +16,7 @@ extern "C" { #endif #include "zstd_compress_internal.h" /* ldmParams_t, U32 */ -#include "zstd.h" /* ZSTD_CCtx, size_t */ +#include "../zstd.h" /* ZSTD_CCtx, size_t */ /*-************************************* * Long distance matching @@ -23,6 +24,10 @@ extern "C" { #define ZSTD_LDM_DEFAULT_WINDOW_LOG ZSTD_WINDOWLOG_LIMIT_DEFAULT +void ZSTD_ldm_fillHashTable( + ldmState_t* state, const BYTE* ip, + const BYTE* iend, ldmParams_t const* params); + /** * ZSTD_ldm_generateSequences(): * diff --git a/thirdparty/zstd/compress/zstd_opt.c b/thirdparty/zstd/compress/zstd_opt.c index 2e50fca6ff..36fff050cf 100644 --- a/thirdparty/zstd/compress/zstd_opt.c +++ b/thirdparty/zstd/compress/zstd_opt.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Facebook, Inc. + * Copyright (c) 2016-2020, Przemyslaw Skibinski, Yann Collet, Facebook, Inc. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -249,40 +249,6 @@ static U32 ZSTD_litLengthPrice(U32 const litLength, const optState_t* const optP } } -/* ZSTD_litLengthContribution() : - * @return ( cost(litlength) - cost(0) ) - * this value can then be added to rawLiteralsCost() - * to provide a cost which is directly comparable to a match ending at same position */ -static int ZSTD_litLengthContribution(U32 const litLength, const optState_t* const optPtr, int optLevel) -{ - if (optPtr->priceType >= zop_predef) return (int)WEIGHT(litLength, optLevel); - - /* dynamic statistics */ - { U32 const llCode = ZSTD_LLcode(litLength); - int const contribution = (int)(LL_bits[llCode] * BITCOST_MULTIPLIER) - + (int)WEIGHT(optPtr->litLengthFreq[0], optLevel) /* note: log2litLengthSum cancel out */ - - (int)WEIGHT(optPtr->litLengthFreq[llCode], optLevel); -#if 1 - return contribution; -#else - return MAX(0, contribution); /* sometimes better, sometimes not ... */ -#endif - } -} - -/* ZSTD_literalsContribution() : - * creates a fake cost for the literals part of a sequence - * which can be compared to the ending cost of a match - * should a new match start at this position */ -static int ZSTD_literalsContribution(const BYTE* const literals, U32 const litLength, - const optState_t* const optPtr, - int optLevel) -{ - int const contribution = (int)ZSTD_rawLiteralsCost(literals, litLength, optPtr, optLevel) - + ZSTD_litLengthContribution(litLength, optPtr, optLevel); - return contribution; -} - /* ZSTD_getMatchPrice() : * Provides the cost of the match part (offset + matchLength) of a sequence * Must be combined with ZSTD_fullLiteralsCost() to get the full cost of a sequence. @@ -603,7 +569,10 @@ U32 ZSTD_insertBtAndGetAllMatches ( U32 repLen = 0; assert(current >= dictLimit); if (repOffset-1 /* intentional overflow, discards 0 and -1 */ < current-dictLimit) { /* equivalent to `current > repIndex >= dictLimit` */ - if (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(ip - repOffset, minMatch)) { + /* We must validate the repcode offset because when we're using a dictionary the + * valid offset range shrinks when the dictionary goes out of bounds. + */ + if ((repIndex >= windowLow) & (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(ip - repOffset, minMatch))) { repLen = (U32)ZSTD_count(ip+minMatch, ip+minMatch-repOffset, iLimit) + minMatch; } } else { /* repIndex < dictLimit || repIndex >= current */ @@ -799,30 +768,6 @@ FORCE_INLINE_TEMPLATE U32 ZSTD_BtGetAllMatches ( /*-******************************* * Optimal parser *********************************/ -typedef struct repcodes_s { - U32 rep[3]; -} repcodes_t; - -static repcodes_t ZSTD_updateRep(U32 const rep[3], U32 const offset, U32 const ll0) -{ - repcodes_t newReps; - if (offset >= ZSTD_REP_NUM) { /* full offset */ - newReps.rep[2] = rep[1]; - newReps.rep[1] = rep[0]; - newReps.rep[0] = offset - ZSTD_REP_MOVE; - } else { /* repcode */ - U32 const repCode = offset + ll0; - if (repCode > 0) { /* note : if repCode==0, no change */ - U32 const currentOffset = (repCode==ZSTD_REP_NUM) ? (rep[0] - 1) : rep[repCode]; - newReps.rep[2] = (repCode >= 2) ? rep[1] : rep[2]; - newReps.rep[1] = rep[0]; - newReps.rep[0] = currentOffset; - } else { /* repCode == 0 */ - memcpy(&newReps, rep, sizeof(newReps)); - } - } - return newReps; -} static U32 ZSTD_totalLen(ZSTD_optimal_t sol) @@ -839,7 +784,7 @@ listStats(const U32* table, int lastEltID) int enb; for (enb=0; enb < nbElts; enb++) { (void)table; - //RAWLOG(2, "%3i:%3i, ", enb, table[enb]); + /* RAWLOG(2, "%3i:%3i, ", enb, table[enb]); */ RAWLOG(2, "%4i,", table[enb]); } RAWLOG(2, " \n"); @@ -894,7 +839,12 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, { U32 i ; for (i=0; i<ZSTD_REP_NUM; i++) opt[0].rep[i] = rep[i]; } opt[0].mlen = 0; /* means is_a_literal */ opt[0].litlen = litlen; - opt[0].price = ZSTD_literalsContribution(anchor, litlen, optStatePtr, optLevel); + /* We don't need to include the actual price of the literals because + * it is static for the duration of the forward pass, and is included + * in every price. We include the literal length to avoid negative + * prices when we subtract the previous literal length. + */ + opt[0].price = ZSTD_litLengthPrice(litlen, optStatePtr, optLevel); /* large match -> immediate encoding */ { U32 const maxML = matches[nbMatches-1].len; @@ -923,7 +873,6 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, for (matchNb = 0; matchNb < nbMatches; matchNb++) { U32 const offset = matches[matchNb].off; U32 const end = matches[matchNb].len; - repcodes_t const repHistory = ZSTD_updateRep(rep, offset, ll0); for ( ; pos <= end ; pos++ ) { U32 const matchPrice = ZSTD_getMatchPrice(offset, pos, optStatePtr, optLevel); U32 const sequencePrice = literalsPrice + matchPrice; @@ -933,8 +882,6 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, opt[pos].off = offset; opt[pos].litlen = litlen; opt[pos].price = sequencePrice; - ZSTD_STATIC_ASSERT(sizeof(opt[pos].rep) == sizeof(repHistory)); - memcpy(opt[pos].rep, &repHistory, sizeof(repHistory)); } } last_pos = pos-1; } @@ -961,7 +908,6 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, opt[cur].off = 0; opt[cur].litlen = litlen; opt[cur].price = price; - memcpy(opt[cur].rep, opt[cur-1].rep, sizeof(opt[cur].rep)); } else { DEBUGLOG(7, "cPos:%zi==rPos:%u : literal would cost more (%.2f>%.2f) (hist:%u,%u,%u)", inr-istart, cur, ZSTD_fCost(price), ZSTD_fCost(opt[cur].price), @@ -969,6 +915,21 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, } } + /* Set the repcodes of the current position. We must do it here + * because we rely on the repcodes of the 2nd to last sequence being + * correct to set the next chunks repcodes during the backward + * traversal. + */ + ZSTD_STATIC_ASSERT(sizeof(opt[cur].rep) == sizeof(repcodes_t)); + assert(cur >= opt[cur].mlen); + if (opt[cur].mlen != 0) { + U32 const prev = cur - opt[cur].mlen; + repcodes_t newReps = ZSTD_updateRep(opt[prev].rep, opt[cur].off, opt[cur].litlen==0); + memcpy(opt[cur].rep, &newReps, sizeof(repcodes_t)); + } else { + memcpy(opt[cur].rep, opt[cur - 1].rep, sizeof(repcodes_t)); + } + /* last match must start at a minimum distance of 8 from oend */ if (inr > ilimit) continue; @@ -1009,7 +970,6 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, /* set prices using matches found at position == cur */ for (matchNb = 0; matchNb < nbMatches; matchNb++) { U32 const offset = matches[matchNb].off; - repcodes_t const repHistory = ZSTD_updateRep(opt[cur].rep, offset, ll0); U32 const lastML = matches[matchNb].len; U32 const startML = (matchNb>0) ? matches[matchNb-1].len+1 : minMatch; U32 mlen; @@ -1029,8 +989,6 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, opt[pos].off = offset; opt[pos].litlen = litlen; opt[pos].price = price; - ZSTD_STATIC_ASSERT(sizeof(opt[pos].rep) == sizeof(repHistory)); - memcpy(opt[pos].rep, &repHistory, sizeof(repHistory)); } else { DEBUGLOG(7, "rPos:%u (ml=%2u) => new price is worse (%.2f>=%.2f)", pos, mlen, ZSTD_fCost(price), ZSTD_fCost(opt[pos].price)); @@ -1046,6 +1004,17 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, _shortestPath: /* cur, last_pos, best_mlen, best_off have to be set */ assert(opt[0].mlen == 0); + /* Set the next chunk's repcodes based on the repcodes of the beginning + * of the last match, and the last sequence. This avoids us having to + * update them while traversing the sequences. + */ + if (lastSequence.mlen != 0) { + repcodes_t reps = ZSTD_updateRep(opt[cur].rep, lastSequence.off, lastSequence.litlen==0); + memcpy(rep, &reps, sizeof(reps)); + } else { + memcpy(rep, opt[cur].rep, sizeof(repcodes_t)); + } + { U32 const storeEnd = cur + 1; U32 storeStart = storeEnd; U32 seqPos = cur; @@ -1082,20 +1051,6 @@ _shortestPath: /* cur, last_pos, best_mlen, best_off have to be set */ continue; /* will finish */ } - /* repcodes update : like ZSTD_updateRep(), but update in place */ - if (offCode >= ZSTD_REP_NUM) { /* full offset */ - rep[2] = rep[1]; - rep[1] = rep[0]; - rep[0] = offCode - ZSTD_REP_MOVE; - } else { /* repcode */ - U32 const repCode = offCode + (llen==0); - if (repCode) { /* note : if repCode==0, no change */ - U32 const currentOffset = (repCode==ZSTD_REP_NUM) ? (rep[0] - 1) : rep[repCode]; - if (repCode >= 2) rep[2] = rep[1]; - rep[1] = rep[0]; - rep[0] = currentOffset; - } } - assert(anchor + llen <= iend); ZSTD_updateStats(optStatePtr, llen, anchor, offCode, mlen); ZSTD_storeSeq(seqStore, llen, anchor, iend, offCode, mlen-MINMATCH); @@ -1104,7 +1059,6 @@ _shortestPath: /* cur, last_pos, best_mlen, best_off have to be set */ } } ZSTD_setBasePrices(optStatePtr, optLevel); } - } /* while (ip < ilimit) */ /* Return the last literals size */ diff --git a/thirdparty/zstd/compress/zstd_opt.h b/thirdparty/zstd/compress/zstd_opt.h index 094f747665..9aba8a9018 100644 --- a/thirdparty/zstd/compress/zstd_opt.h +++ b/thirdparty/zstd/compress/zstd_opt.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/thirdparty/zstd/compress/zstdmt_compress.c b/thirdparty/zstd/compress/zstdmt_compress.c index bc3062b530..1e3c8fdbee 100644 --- a/thirdparty/zstd/compress/zstdmt_compress.c +++ b/thirdparty/zstd/compress/zstdmt_compress.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -22,9 +22,9 @@ /* ====== Dependencies ====== */ #include <string.h> /* memcpy, memset */ #include <limits.h> /* INT_MAX, UINT_MAX */ -#include "mem.h" /* MEM_STATIC */ -#include "pool.h" /* threadpool */ -#include "threading.h" /* mutex */ +#include "../common/mem.h" /* MEM_STATIC */ +#include "../common/pool.h" /* threadpool */ +#include "../common/threading.h" /* mutex */ #include "zstd_compress_internal.h" /* MIN, ERROR, ZSTD_*, ZSTD_highbit32 */ #include "zstd_ldm.h" #include "zstdmt_compress.h" @@ -461,7 +461,13 @@ typedef struct { ZSTD_window_t ldmWindow; /* A thread-safe copy of ldmState.window */ } serialState_t; -static int ZSTDMT_serialState_reset(serialState_t* serialState, ZSTDMT_seqPool* seqPool, ZSTD_CCtx_params params, size_t jobSize) +static int +ZSTDMT_serialState_reset(serialState_t* serialState, + ZSTDMT_seqPool* seqPool, + ZSTD_CCtx_params params, + size_t jobSize, + const void* dict, size_t const dictSize, + ZSTD_dictContentType_e dictContentType) { /* Adjust parameters */ if (params.ldmParams.enableLdm) { @@ -490,8 +496,7 @@ static int ZSTDMT_serialState_reset(serialState_t* serialState, ZSTDMT_seqPool* /* Size the seq pool tables */ ZSTDMT_setNbSeq(seqPool, ZSTD_ldm_getMaxNbSeq(params.ldmParams, jobSize)); /* Reset the window */ - ZSTD_window_clear(&serialState->ldmState.window); - serialState->ldmWindow = serialState->ldmState.window; + ZSTD_window_init(&serialState->ldmState.window); /* Resize tables and output space if necessary. */ if (serialState->ldmState.hashTable == NULL || serialState->params.ldmParams.hashLog < hashLog) { ZSTD_free(serialState->ldmState.hashTable, cMem); @@ -506,7 +511,24 @@ static int ZSTDMT_serialState_reset(serialState_t* serialState, ZSTDMT_seqPool* /* Zero the tables */ memset(serialState->ldmState.hashTable, 0, hashSize); memset(serialState->ldmState.bucketOffsets, 0, bucketSize); + + /* Update window state and fill hash table with dict */ + serialState->ldmState.loadedDictEnd = 0; + if (dictSize > 0) { + if (dictContentType == ZSTD_dct_rawContent) { + BYTE const* const dictEnd = (const BYTE*)dict + dictSize; + ZSTD_window_update(&serialState->ldmState.window, dict, dictSize); + ZSTD_ldm_fillHashTable(&serialState->ldmState, (const BYTE*)dict, dictEnd, ¶ms.ldmParams); + serialState->ldmState.loadedDictEnd = params.forceWindow ? 0 : (U32)(dictEnd - serialState->ldmState.window.base); + } else { + /* don't even load anything */ + } + } + + /* Initialize serialState's copy of ldmWindow. */ + serialState->ldmWindow = serialState->ldmState.window; } + serialState->params = params; serialState->params.jobSize = (U32)jobSize; return 0; @@ -1054,7 +1076,7 @@ static ZSTD_CCtx_params ZSTDMT_initJobCCtxParams(const ZSTD_CCtx_params* params) static size_t ZSTDMT_resize(ZSTDMT_CCtx* mtctx, unsigned nbWorkers) { if (POOL_resize(mtctx->factory, nbWorkers)) return ERROR(memory_allocation); - FORWARD_IF_ERROR( ZSTDMT_expandJobsTable(mtctx, nbWorkers) ); + FORWARD_IF_ERROR( ZSTDMT_expandJobsTable(mtctx, nbWorkers) , ""); mtctx->bufPool = ZSTDMT_expandBufferPool(mtctx->bufPool, nbWorkers); if (mtctx->bufPool == NULL) return ERROR(memory_allocation); mtctx->cctxPool = ZSTDMT_expandCCtxPool(mtctx->cctxPool, nbWorkers); @@ -1076,7 +1098,7 @@ void ZSTDMT_updateCParams_whileCompressing(ZSTDMT_CCtx* mtctx, const ZSTD_CCtx_p DEBUGLOG(5, "ZSTDMT_updateCParams_whileCompressing (level:%i)", compressionLevel); mtctx->params.compressionLevel = compressionLevel; - { ZSTD_compressionParameters cParams = ZSTD_getCParamsFromCCtxParams(cctxParams, 0, 0); + { ZSTD_compressionParameters cParams = ZSTD_getCParamsFromCCtxParams(cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, 0); cParams.windowLog = saved_wlog; mtctx->params.cParams = cParams; } @@ -1235,7 +1257,8 @@ ZSTDMT_computeNbJobs(const ZSTD_CCtx_params* params, size_t srcSize, unsigned nb /* ZSTDMT_compress_advanced_internal() : * This is a blocking function : it will only give back control to caller after finishing its compression job. */ -static size_t ZSTDMT_compress_advanced_internal( +static size_t +ZSTDMT_compress_advanced_internal( ZSTDMT_CCtx* mtctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, @@ -1267,10 +1290,11 @@ static size_t ZSTDMT_compress_advanced_internal( assert(avgJobSize >= 256 KB); /* condition for ZSTD_compressBound(A) + ZSTD_compressBound(B) <= ZSTD_compressBound(A+B), required to compress directly into Dst (no additional buffer) */ ZSTDMT_setBufferSize(mtctx->bufPool, ZSTD_compressBound(avgJobSize) ); - if (ZSTDMT_serialState_reset(&mtctx->serial, mtctx->seqPool, params, avgJobSize)) + /* LDM doesn't even try to load the dictionary in single-ingestion mode */ + if (ZSTDMT_serialState_reset(&mtctx->serial, mtctx->seqPool, params, avgJobSize, NULL, 0, ZSTD_dct_auto)) return ERROR(memory_allocation); - FORWARD_IF_ERROR( ZSTDMT_expandJobsTable(mtctx, nbJobs) ); /* only expands if necessary */ + FORWARD_IF_ERROR( ZSTDMT_expandJobsTable(mtctx, nbJobs) , ""); /* only expands if necessary */ { unsigned u; for (u=0; u<nbJobs; u++) { @@ -1403,7 +1427,7 @@ size_t ZSTDMT_initCStream_internal( /* init */ if (params.nbWorkers != mtctx->params.nbWorkers) - FORWARD_IF_ERROR( ZSTDMT_resize(mtctx, params.nbWorkers) ); + FORWARD_IF_ERROR( ZSTDMT_resize(mtctx, params.nbWorkers) , ""); if (params.jobSize != 0 && params.jobSize < ZSTDMT_JOBSIZE_MIN) params.jobSize = ZSTDMT_JOBSIZE_MIN; if (params.jobSize > (size_t)ZSTDMT_JOBSIZE_MAX) params.jobSize = (size_t)ZSTDMT_JOBSIZE_MAX; @@ -1500,7 +1524,8 @@ size_t ZSTDMT_initCStream_internal( mtctx->allJobsCompleted = 0; mtctx->consumed = 0; mtctx->produced = 0; - if (ZSTDMT_serialState_reset(&mtctx->serial, mtctx->seqPool, params, mtctx->targetSectionSize)) + if (ZSTDMT_serialState_reset(&mtctx->serial, mtctx->seqPool, params, mtctx->targetSectionSize, + dict, dictSize, dictContentType)) return ERROR(memory_allocation); return 0; } @@ -1714,9 +1739,11 @@ static size_t ZSTDMT_flushProduced(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, u assert(mtctx->doneJobID < mtctx->nextJobID); assert(cSize >= mtctx->jobs[wJobID].dstFlushed); assert(mtctx->jobs[wJobID].dstBuff.start != NULL); - memcpy((char*)output->dst + output->pos, - (const char*)mtctx->jobs[wJobID].dstBuff.start + mtctx->jobs[wJobID].dstFlushed, - toFlush); + if (toFlush > 0) { + memcpy((char*)output->dst + output->pos, + (const char*)mtctx->jobs[wJobID].dstBuff.start + mtctx->jobs[wJobID].dstFlushed, + toFlush); + } output->pos += toFlush; mtctx->jobs[wJobID].dstFlushed += toFlush; /* can write : this value is only used by mtctx */ @@ -1786,7 +1813,7 @@ static int ZSTDMT_isOverlapped(buffer_t buffer, range_t range) BYTE const* const bufferStart = (BYTE const*)buffer.start; BYTE const* const bufferEnd = bufferStart + buffer.capacity; BYTE const* const rangeStart = (BYTE const*)range.start; - BYTE const* const rangeEnd = rangeStart + range.size; + BYTE const* const rangeEnd = range.size != 0 ? rangeStart + range.size : rangeStart; if (rangeStart == NULL || bufferStart == NULL) return 0; @@ -2060,7 +2087,7 @@ size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx, || ((endOp == ZSTD_e_end) && (!mtctx->frameEnded)) ) { /* must finish the frame with a zero-size block */ size_t const jobSize = mtctx->inBuff.filled; assert(mtctx->inBuff.filled <= mtctx->targetSectionSize); - FORWARD_IF_ERROR( ZSTDMT_createCompressionJob(mtctx, jobSize, endOp) ); + FORWARD_IF_ERROR( ZSTDMT_createCompressionJob(mtctx, jobSize, endOp) , ""); } /* check for potential compressed data ready to be flushed */ @@ -2074,7 +2101,7 @@ size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx, size_t ZSTDMT_compressStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, ZSTD_inBuffer* input) { - FORWARD_IF_ERROR( ZSTDMT_compressStream_generic(mtctx, output, input, ZSTD_e_continue) ); + FORWARD_IF_ERROR( ZSTDMT_compressStream_generic(mtctx, output, input, ZSTD_e_continue) , ""); /* recommended next input size : fill current input buffer */ return mtctx->targetSectionSize - mtctx->inBuff.filled; /* note : could be zero when input buffer is fully filled and no more availability to create new job */ @@ -2091,7 +2118,7 @@ static size_t ZSTDMT_flushStream_internal(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* ou || ((endFrame==ZSTD_e_end) && !mtctx->frameEnded)) { /* need a last 0-size block to end frame */ DEBUGLOG(5, "ZSTDMT_flushStream_internal : create a new job (%u bytes, end:%u)", (U32)srcSize, (U32)endFrame); - FORWARD_IF_ERROR( ZSTDMT_createCompressionJob(mtctx, srcSize, endFrame) ); + FORWARD_IF_ERROR( ZSTDMT_createCompressionJob(mtctx, srcSize, endFrame) , ""); } /* check if there is any data available to flush */ diff --git a/thirdparty/zstd/compress/zstdmt_compress.h b/thirdparty/zstd/compress/zstdmt_compress.h index 12a526087d..89914eb7f8 100644 --- a/thirdparty/zstd/compress/zstdmt_compress.h +++ b/thirdparty/zstd/compress/zstdmt_compress.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -40,7 +40,7 @@ /* === Dependencies === */ #include <stddef.h> /* size_t */ #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_parameters */ -#include "zstd.h" /* ZSTD_inBuffer, ZSTD_outBuffer, ZSTDLIB_API */ +#include "../zstd.h" /* ZSTD_inBuffer, ZSTD_outBuffer, ZSTDLIB_API */ /* === Constants === */ diff --git a/thirdparty/zstd/decompress/huf_decompress.c b/thirdparty/zstd/decompress/huf_decompress.c index bb2d0a96bc..68293a1309 100644 --- a/thirdparty/zstd/decompress/huf_decompress.c +++ b/thirdparty/zstd/decompress/huf_decompress.c @@ -1,47 +1,27 @@ /* ****************************************************************** - huff0 huffman decoder, - part of Finite State Entropy library - Copyright (C) 2013-present, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy + * huff0 huffman decoder, + * part of Finite State Entropy library + * Copyright (c) 2013-2020, Yann Collet, Facebook, Inc. + * + * You can contact the author at : + * - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. ****************************************************************** */ /* ************************************************************** * Dependencies ****************************************************************/ #include <string.h> /* memcpy, memset */ -#include "compiler.h" -#include "bitstream.h" /* BIT_* */ -#include "fse.h" /* to compress headers */ +#include "../common/compiler.h" +#include "../common/bitstream.h" /* BIT_* */ +#include "../common/fse.h" /* to compress headers */ #define HUF_STATIC_LINKING_ONLY -#include "huf.h" -#include "error_private.h" +#include "../common/huf.h" +#include "../common/error_private.h" /* ************************************************************** * Macros @@ -61,9 +41,6 @@ * Error Management ****************************************************************/ #define HUF_isError ERR_isError -#ifndef CHECK_F -#define CHECK_F(f) { size_t const err_ = (f); if (HUF_isError(err_)) return err_; } -#endif /* ************************************************************** @@ -181,17 +158,29 @@ size_t HUF_readDTableX1_wksp(HUF_DTable* DTable, const void* src, size_t srcSize /* fill DTable */ { U32 n; - for (n=0; n<nbSymbols; n++) { - U32 const w = huffWeight[n]; - U32 const length = (1 << w) >> 1; - U32 u; + size_t const nEnd = nbSymbols; + for (n=0; n<nEnd; n++) { + size_t const w = huffWeight[n]; + size_t const length = (1 << w) >> 1; + size_t const uStart = rankVal[w]; + size_t const uEnd = uStart + length; + size_t u; HUF_DEltX1 D; - D.byte = (BYTE)n; D.nbBits = (BYTE)(tableLog + 1 - w); - for (u = rankVal[w]; u < rankVal[w] + length; u++) - dt[u] = D; - rankVal[w] += length; - } } - + D.byte = (BYTE)n; + D.nbBits = (BYTE)(tableLog + 1 - w); + rankVal[w] = (U32)uEnd; + if (length < 4) { + /* Use length in the loop bound so the compiler knows it is short. */ + for (u = 0; u < length; ++u) + dt[uStart + u] = D; + } else { + /* Unroll the loop 4 times, we know it is a power of 2. */ + for (u = uStart; u < uEnd; u += 4) { + dt[u + 0] = D; + dt[u + 1] = D; + dt[u + 2] = D; + dt[u + 3] = D; + } } } } return iSize; } @@ -282,6 +271,7 @@ HUF_decompress4X1_usingDTable_internal_body( { const BYTE* const istart = (const BYTE*) cSrc; BYTE* const ostart = (BYTE*) dst; BYTE* const oend = ostart + dstSize; + BYTE* const olimit = oend - 3; const void* const dtPtr = DTable + 1; const HUF_DEltX1* const dt = (const HUF_DEltX1*)dtPtr; @@ -306,9 +296,9 @@ HUF_decompress4X1_usingDTable_internal_body( BYTE* op2 = opStart2; BYTE* op3 = opStart3; BYTE* op4 = opStart4; - U32 endSignal = BIT_DStream_unfinished; DTableDesc const dtd = HUF_getDTableDesc(DTable); U32 const dtLog = dtd.tableLog; + U32 endSignal = 1; if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ CHECK_F( BIT_initDStream(&bitD1, istart1, length1) ); @@ -317,8 +307,7 @@ HUF_decompress4X1_usingDTable_internal_body( CHECK_F( BIT_initDStream(&bitD4, istart4, length4) ); /* up to 16 symbols per loop (4 symbols per stream) in 64-bit mode */ - endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); - while ( (endSignal==BIT_DStream_unfinished) && (op4<(oend-3)) ) { + for ( ; (endSignal) & (op4 < olimit) ; ) { HUF_DECODE_SYMBOLX1_2(op1, &bitD1); HUF_DECODE_SYMBOLX1_2(op2, &bitD2); HUF_DECODE_SYMBOLX1_2(op3, &bitD3); @@ -335,10 +324,10 @@ HUF_decompress4X1_usingDTable_internal_body( HUF_DECODE_SYMBOLX1_0(op2, &bitD2); HUF_DECODE_SYMBOLX1_0(op3, &bitD3); HUF_DECODE_SYMBOLX1_0(op4, &bitD4); - BIT_reloadDStream(&bitD1); - BIT_reloadDStream(&bitD2); - BIT_reloadDStream(&bitD3); - BIT_reloadDStream(&bitD4); + endSignal &= BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished; + endSignal &= BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished; + endSignal &= BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished; + endSignal &= BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished; } /* check corruption */ @@ -757,7 +746,6 @@ HUF_decompress1X2_usingDTable_internal_body( return dstSize; } - FORCE_INLINE_TEMPLATE size_t HUF_decompress4X2_usingDTable_internal_body( void* dst, size_t dstSize, @@ -769,6 +757,7 @@ HUF_decompress4X2_usingDTable_internal_body( { const BYTE* const istart = (const BYTE*) cSrc; BYTE* const ostart = (BYTE*) dst; BYTE* const oend = ostart + dstSize; + BYTE* const olimit = oend - (sizeof(size_t)-1); const void* const dtPtr = DTable+1; const HUF_DEltX2* const dt = (const HUF_DEltX2*)dtPtr; @@ -793,7 +782,7 @@ HUF_decompress4X2_usingDTable_internal_body( BYTE* op2 = opStart2; BYTE* op3 = opStart3; BYTE* op4 = opStart4; - U32 endSignal; + U32 endSignal = 1; DTableDesc const dtd = HUF_getDTableDesc(DTable); U32 const dtLog = dtd.tableLog; @@ -804,8 +793,29 @@ HUF_decompress4X2_usingDTable_internal_body( CHECK_F( BIT_initDStream(&bitD4, istart4, length4) ); /* 16-32 symbols per loop (4-8 symbols per stream) */ - endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); - for ( ; (endSignal==BIT_DStream_unfinished) & (op4<(oend-(sizeof(bitD4.bitContainer)-1))) ; ) { + for ( ; (endSignal) & (op4 < olimit); ) { +#if defined(__clang__) && (defined(__x86_64__) || defined(__i386__)) + HUF_DECODE_SYMBOLX2_2(op1, &bitD1); + HUF_DECODE_SYMBOLX2_1(op1, &bitD1); + HUF_DECODE_SYMBOLX2_2(op1, &bitD1); + HUF_DECODE_SYMBOLX2_0(op1, &bitD1); + HUF_DECODE_SYMBOLX2_2(op2, &bitD2); + HUF_DECODE_SYMBOLX2_1(op2, &bitD2); + HUF_DECODE_SYMBOLX2_2(op2, &bitD2); + HUF_DECODE_SYMBOLX2_0(op2, &bitD2); + endSignal &= BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished; + endSignal &= BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished; + HUF_DECODE_SYMBOLX2_2(op3, &bitD3); + HUF_DECODE_SYMBOLX2_1(op3, &bitD3); + HUF_DECODE_SYMBOLX2_2(op3, &bitD3); + HUF_DECODE_SYMBOLX2_0(op3, &bitD3); + HUF_DECODE_SYMBOLX2_2(op4, &bitD4); + HUF_DECODE_SYMBOLX2_1(op4, &bitD4); + HUF_DECODE_SYMBOLX2_2(op4, &bitD4); + HUF_DECODE_SYMBOLX2_0(op4, &bitD4); + endSignal &= BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished; + endSignal &= BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished; +#else HUF_DECODE_SYMBOLX2_2(op1, &bitD1); HUF_DECODE_SYMBOLX2_2(op2, &bitD2); HUF_DECODE_SYMBOLX2_2(op3, &bitD3); @@ -822,8 +832,12 @@ HUF_decompress4X2_usingDTable_internal_body( HUF_DECODE_SYMBOLX2_0(op2, &bitD2); HUF_DECODE_SYMBOLX2_0(op3, &bitD3); HUF_DECODE_SYMBOLX2_0(op4, &bitD4); - - endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); + endSignal = (U32)LIKELY( + (BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished) + & (BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished) + & (BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished) + & (BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished)); +#endif } /* check corruption */ diff --git a/thirdparty/zstd/decompress/zstd_ddict.c b/thirdparty/zstd/decompress/zstd_ddict.c index 0af3d23bfe..c8cb8ecc95 100644 --- a/thirdparty/zstd/decompress/zstd_ddict.c +++ b/thirdparty/zstd/decompress/zstd_ddict.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -15,17 +15,17 @@ * Dependencies *********************************************************/ #include <string.h> /* memcpy, memmove, memset */ -#include "cpu.h" /* bmi2 */ -#include "mem.h" /* low level memory routines */ +#include "../common/cpu.h" /* bmi2 */ +#include "../common/mem.h" /* low level memory routines */ #define FSE_STATIC_LINKING_ONLY -#include "fse.h" +#include "../common/fse.h" #define HUF_STATIC_LINKING_ONLY -#include "huf.h" +#include "../common/huf.h" #include "zstd_decompress_internal.h" #include "zstd_ddict.h" #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) -# include "zstd_legacy.h" +# include "../legacy/zstd_legacy.h" #endif @@ -65,6 +65,10 @@ void ZSTD_copyDDictParameters(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict) dctx->virtualStart = ddict->dictContent; dctx->dictEnd = (const BYTE*)ddict->dictContent + ddict->dictSize; dctx->previousDstEnd = dctx->dictEnd; +#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + dctx->dictContentBeginForFuzzing = dctx->prefixStart; + dctx->dictContentEndForFuzzing = dctx->previousDstEnd; +#endif if (ddict->entropyPresent) { dctx->litEntropy = 1; dctx->fseEntropy = 1; @@ -107,7 +111,7 @@ ZSTD_loadEntropy_intoDDict(ZSTD_DDict* ddict, /* load entropy tables */ RETURN_ERROR_IF(ZSTD_isError(ZSTD_loadDEntropy( &ddict->entropy, ddict->dictContent, ddict->dictSize)), - dictionary_corrupted); + dictionary_corrupted, ""); ddict->entropyPresent = 1; return 0; } @@ -133,7 +137,7 @@ static size_t ZSTD_initDDict_internal(ZSTD_DDict* ddict, ddict->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */ /* parse dictionary content */ - FORWARD_IF_ERROR( ZSTD_loadEntropy_intoDDict(ddict, dictContentType) ); + FORWARD_IF_ERROR( ZSTD_loadEntropy_intoDDict(ddict, dictContentType) , ""); return 0; } diff --git a/thirdparty/zstd/decompress/zstd_ddict.h b/thirdparty/zstd/decompress/zstd_ddict.h index 0479d11bb0..af307efd3d 100644 --- a/thirdparty/zstd/decompress/zstd_ddict.h +++ b/thirdparty/zstd/decompress/zstd_ddict.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -16,7 +16,7 @@ * Dependencies *********************************************************/ #include <stddef.h> /* size_t */ -#include "zstd.h" /* ZSTD_DDict, and several public functions */ +#include "../zstd.h" /* ZSTD_DDict, and several public functions */ /*-******************************************************* diff --git a/thirdparty/zstd/decompress/zstd_decompress.c b/thirdparty/zstd/decompress/zstd_decompress.c index dd4591b7be..be5c7cfc33 100644 --- a/thirdparty/zstd/decompress/zstd_decompress.c +++ b/thirdparty/zstd/decompress/zstd_decompress.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -56,19 +56,19 @@ * Dependencies *********************************************************/ #include <string.h> /* memcpy, memmove, memset */ -#include "cpu.h" /* bmi2 */ -#include "mem.h" /* low level memory routines */ +#include "../common/cpu.h" /* bmi2 */ +#include "../common/mem.h" /* low level memory routines */ #define FSE_STATIC_LINKING_ONLY -#include "fse.h" +#include "../common/fse.h" #define HUF_STATIC_LINKING_ONLY -#include "huf.h" -#include "zstd_internal.h" /* blockProperties_t */ +#include "../common/huf.h" +#include "../common/zstd_internal.h" /* blockProperties_t */ #include "zstd_decompress_internal.h" /* ZSTD_DCtx */ #include "zstd_ddict.h" /* ZSTD_DDictDictContent */ #include "zstd_decompress_block.h" /* ZSTD_decompressBlock_internal */ #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) -# include "zstd_legacy.h" +# include "../legacy/zstd_legacy.h" #endif @@ -111,7 +111,12 @@ static void ZSTD_initDCtx_internal(ZSTD_DCtx* dctx) dctx->legacyContext = NULL; dctx->previousLegacyVersion = 0; dctx->noForwardProgress = 0; + dctx->oversizedDuration = 0; dctx->bmi2 = ZSTD_cpuid_bmi2(ZSTD_cpuid()); + dctx->outBufferMode = ZSTD_obm_buffered; +#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + dctx->dictContentEndForFuzzing = NULL; +#endif } ZSTD_DCtx* ZSTD_initStaticDCtx(void *workspace, size_t workspaceSize) @@ -208,7 +213,7 @@ unsigned ZSTD_isFrame(const void* buffer, size_t size) static size_t ZSTD_frameHeaderSize_internal(const void* src, size_t srcSize, ZSTD_format_e format) { size_t const minInputSize = ZSTD_startingInputLength(format); - RETURN_ERROR_IF(srcSize < minInputSize, srcSize_wrong); + RETURN_ERROR_IF(srcSize < minInputSize, srcSize_wrong, ""); { BYTE const fhd = ((const BYTE*)src)[minInputSize-1]; U32 const dictID= fhd & 3; @@ -256,7 +261,7 @@ size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, s zfhPtr->frameType = ZSTD_skippableFrame; return 0; } - RETURN_ERROR(prefix_unknown); + RETURN_ERROR(prefix_unknown, ""); } /* ensure there is enough `srcSize` to fully read/decode frame header */ @@ -280,7 +285,7 @@ size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, s if (!singleSegment) { BYTE const wlByte = ip[pos++]; U32 const windowLog = (wlByte >> 3) + ZSTD_WINDOWLOG_ABSOLUTEMIN; - RETURN_ERROR_IF(windowLog > ZSTD_WINDOWLOG_MAX, frameParameter_windowTooLarge); + RETURN_ERROR_IF(windowLog > ZSTD_WINDOWLOG_MAX, frameParameter_windowTooLarge, ""); windowSize = (1ULL << windowLog); windowSize += (windowSize >> 3) * (wlByte&7); } @@ -352,14 +357,14 @@ static size_t readSkippableFrameSize(void const* src, size_t srcSize) size_t const skippableHeaderSize = ZSTD_SKIPPABLEHEADERSIZE; U32 sizeU32; - RETURN_ERROR_IF(srcSize < ZSTD_SKIPPABLEHEADERSIZE, srcSize_wrong); + RETURN_ERROR_IF(srcSize < ZSTD_SKIPPABLEHEADERSIZE, srcSize_wrong, ""); sizeU32 = MEM_readLE32((BYTE const*)src + ZSTD_FRAMEIDSIZE); RETURN_ERROR_IF((U32)(sizeU32 + ZSTD_SKIPPABLEHEADERSIZE) < sizeU32, - frameParameter_unsupported); + frameParameter_unsupported, ""); { size_t const skippableSize = skippableHeaderSize + sizeU32; - RETURN_ERROR_IF(skippableSize > srcSize, srcSize_wrong); + RETURN_ERROR_IF(skippableSize > srcSize, srcSize_wrong, ""); return skippableSize; } } @@ -439,7 +444,7 @@ static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t he * harder. */ RETURN_ERROR_IF(dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID), - dictionary_wrong); + dictionary_wrong, ""); #endif if (dctx->fParams.checksumFlag) XXH64_reset(&dctx->xxhState, 0); return 0; @@ -559,17 +564,6 @@ unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize) * Frame decoding ***************************************************************/ - -void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst) -{ - if (dst != dctx->previousDstEnd) { /* not contiguous */ - dctx->dictEnd = dctx->previousDstEnd; - dctx->virtualStart = (const char*)dst - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->prefixStart)); - dctx->prefixStart = dst; - dctx->previousDstEnd = dst; - } -} - /** ZSTD_insertBlock() : * insert `src` block into `dctx` history. Useful to track uncompressed blocks. */ size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize) @@ -587,9 +581,9 @@ static size_t ZSTD_copyRawBlock(void* dst, size_t dstCapacity, DEBUGLOG(5, "ZSTD_copyRawBlock"); if (dst == NULL) { if (srcSize == 0) return 0; - RETURN_ERROR(dstBuffer_null); + RETURN_ERROR(dstBuffer_null, ""); } - RETURN_ERROR_IF(srcSize > dstCapacity, dstSize_tooSmall); + RETURN_ERROR_IF(srcSize > dstCapacity, dstSize_tooSmall, ""); memcpy(dst, src, srcSize); return srcSize; } @@ -600,9 +594,9 @@ static size_t ZSTD_setRleBlock(void* dst, size_t dstCapacity, { if (dst == NULL) { if (regenSize == 0) return 0; - RETURN_ERROR(dstBuffer_null); + RETURN_ERROR(dstBuffer_null, ""); } - RETURN_ERROR_IF(regenSize > dstCapacity, dstSize_tooSmall); + RETURN_ERROR_IF(regenSize > dstCapacity, dstSize_tooSmall, ""); memset(dst, b, regenSize); return regenSize; } @@ -618,7 +612,7 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, { const BYTE* ip = (const BYTE*)(*srcPtr); BYTE* const ostart = (BYTE* const)dst; - BYTE* const oend = ostart + dstCapacity; + BYTE* const oend = dstCapacity != 0 ? ostart + dstCapacity : ostart; BYTE* op = ostart; size_t remainingSrcSize = *srcSizePtr; @@ -627,15 +621,15 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, /* check */ RETURN_ERROR_IF( remainingSrcSize < ZSTD_FRAMEHEADERSIZE_MIN(dctx->format)+ZSTD_blockHeaderSize, - srcSize_wrong); + srcSize_wrong, ""); /* Frame Header */ { size_t const frameHeaderSize = ZSTD_frameHeaderSize_internal( ip, ZSTD_FRAMEHEADERSIZE_PREFIX(dctx->format), dctx->format); if (ZSTD_isError(frameHeaderSize)) return frameHeaderSize; RETURN_ERROR_IF(remainingSrcSize < frameHeaderSize+ZSTD_blockHeaderSize, - srcSize_wrong); - FORWARD_IF_ERROR( ZSTD_decodeFrameHeader(dctx, ip, frameHeaderSize) ); + srcSize_wrong, ""); + FORWARD_IF_ERROR( ZSTD_decodeFrameHeader(dctx, ip, frameHeaderSize) , ""); ip += frameHeaderSize; remainingSrcSize -= frameHeaderSize; } @@ -648,7 +642,7 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, ip += ZSTD_blockHeaderSize; remainingSrcSize -= ZSTD_blockHeaderSize; - RETURN_ERROR_IF(cBlockSize > remainingSrcSize, srcSize_wrong); + RETURN_ERROR_IF(cBlockSize > remainingSrcSize, srcSize_wrong, ""); switch(blockProperties.blockType) { @@ -663,13 +657,15 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, break; case bt_reserved : default: - RETURN_ERROR(corruption_detected); + RETURN_ERROR(corruption_detected, "invalid block type"); } if (ZSTD_isError(decodedSize)) return decodedSize; if (dctx->fParams.checksumFlag) XXH64_update(&dctx->xxhState, op, decodedSize); - op += decodedSize; + if (decodedSize != 0) + op += decodedSize; + assert(ip != NULL); ip += cBlockSize; remainingSrcSize -= cBlockSize; if (blockProperties.lastBlock) break; @@ -677,14 +673,14 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, if (dctx->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN) { RETURN_ERROR_IF((U64)(op-ostart) != dctx->fParams.frameContentSize, - corruption_detected); + corruption_detected, ""); } if (dctx->fParams.checksumFlag) { /* Frame content checksum verification */ U32 const checkCalc = (U32)XXH64_digest(&dctx->xxhState); U32 checkRead; - RETURN_ERROR_IF(remainingSrcSize<4, checksum_wrong); + RETURN_ERROR_IF(remainingSrcSize<4, checksum_wrong, ""); checkRead = MEM_readLE32(ip); - RETURN_ERROR_IF(checkRead != checkCalc, checksum_wrong); + RETURN_ERROR_IF(checkRead != checkCalc, checksum_wrong, ""); ip += 4; remainingSrcSize -= 4; } @@ -741,7 +737,7 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx, (unsigned)magicNumber, ZSTD_MAGICNUMBER); if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { size_t const skippableSize = readSkippableFrameSize(src, srcSize); - FORWARD_IF_ERROR(skippableSize); + FORWARD_IF_ERROR(skippableSize, "readSkippableFrameSize failed"); assert(skippableSize <= srcSize); src = (const BYTE *)src + skippableSize; @@ -751,11 +747,11 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx, if (ddict) { /* we were called from ZSTD_decompress_usingDDict */ - FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDDict(dctx, ddict)); + FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDDict(dctx, ddict), ""); } else { /* this will initialize correctly with no dict if dict == NULL, so * use this in all cases but ddict */ - FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDict(dctx, dict, dictSize)); + FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDict(dctx, dict, dictSize), ""); } ZSTD_checkContinuity(dctx, dst); @@ -776,7 +772,8 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx, "error."); if (ZSTD_isError(res)) return res; assert(res <= dstCapacity); - dst = (BYTE*)dst + res; + if (res != 0) + dst = (BYTE*)dst + res; dstCapacity -= res; } moreThan1Frame = 1; @@ -824,7 +821,7 @@ size_t ZSTD_decompress(void* dst, size_t dstCapacity, const void* src, size_t sr #if defined(ZSTD_HEAPMODE) && (ZSTD_HEAPMODE>=1) size_t regenSize; ZSTD_DCtx* const dctx = ZSTD_createDCtx(); - RETURN_ERROR_IF(dctx==NULL, memory_allocation); + RETURN_ERROR_IF(dctx==NULL, memory_allocation, "NULL pointer!"); regenSize = ZSTD_decompressDCtx(dctx, dst, dstCapacity, src, srcSize); ZSTD_freeDCtx(dctx); return regenSize; @@ -842,6 +839,24 @@ size_t ZSTD_decompress(void* dst, size_t dstCapacity, const void* src, size_t sr ****************************************/ size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx) { return dctx->expected; } +/** + * Similar to ZSTD_nextSrcSizeToDecompress(), but when when a block input can be streamed, + * we allow taking a partial block as the input. Currently only raw uncompressed blocks can + * be streamed. + * + * For blocks that can be streamed, this allows us to reduce the latency until we produce + * output, and avoid copying the input. + * + * @param inputSize - The total amount of input that the caller currently has. + */ +static size_t ZSTD_nextSrcSizeToDecompressWithInputSize(ZSTD_DCtx* dctx, size_t inputSize) { + if (!(dctx->stage == ZSTDds_decompressBlock || dctx->stage == ZSTDds_decompressLastBlock)) + return dctx->expected; + if (dctx->bType != bt_raw) + return dctx->expected; + return MIN(MAX(inputSize, 1), dctx->expected); +} + ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx) { switch(dctx->stage) { @@ -874,7 +889,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c { DEBUGLOG(5, "ZSTD_decompressContinue (srcSize:%u)", (unsigned)srcSize); /* Sanity check */ - RETURN_ERROR_IF(srcSize != dctx->expected, srcSize_wrong, "not allowed"); + RETURN_ERROR_IF(srcSize != ZSTD_nextSrcSizeToDecompressWithInputSize(dctx, srcSize), srcSize_wrong, "not allowed"); if (dstCapacity) ZSTD_checkContinuity(dctx, dst); switch (dctx->stage) @@ -899,7 +914,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c case ZSTDds_decodeFrameHeader: assert(src != NULL); memcpy(dctx->headerBuffer + (dctx->headerSize - srcSize), src, srcSize); - FORWARD_IF_ERROR(ZSTD_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize)); + FORWARD_IF_ERROR(ZSTD_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize), ""); dctx->expected = ZSTD_blockHeaderSize; dctx->stage = ZSTDds_decodeBlockHeader; return 0; @@ -941,29 +956,41 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c case bt_compressed: DEBUGLOG(5, "ZSTD_decompressContinue: case bt_compressed"); rSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, /* frame */ 1); + dctx->expected = 0; /* Streaming not supported */ break; case bt_raw : + assert(srcSize <= dctx->expected); rSize = ZSTD_copyRawBlock(dst, dstCapacity, src, srcSize); + FORWARD_IF_ERROR(rSize, "ZSTD_copyRawBlock failed"); + assert(rSize == srcSize); + dctx->expected -= rSize; break; case bt_rle : rSize = ZSTD_setRleBlock(dst, dstCapacity, *(const BYTE*)src, dctx->rleSize); + dctx->expected = 0; /* Streaming not supported */ break; case bt_reserved : /* should never happen */ default: - RETURN_ERROR(corruption_detected); + RETURN_ERROR(corruption_detected, "invalid block type"); } - if (ZSTD_isError(rSize)) return rSize; + FORWARD_IF_ERROR(rSize, ""); RETURN_ERROR_IF(rSize > dctx->fParams.blockSizeMax, corruption_detected, "Decompressed Block Size Exceeds Maximum"); DEBUGLOG(5, "ZSTD_decompressContinue: decoded size from block : %u", (unsigned)rSize); dctx->decodedSize += rSize; if (dctx->fParams.checksumFlag) XXH64_update(&dctx->xxhState, dst, rSize); + dctx->previousDstEnd = (char*)dst + rSize; + + /* Stay on the same stage until we are finished streaming the block. */ + if (dctx->expected > 0) { + return rSize; + } if (dctx->stage == ZSTDds_decompressLastBlock) { /* end of frame */ DEBUGLOG(4, "ZSTD_decompressContinue: decoded size from frame : %u", (unsigned)dctx->decodedSize); RETURN_ERROR_IF( dctx->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN && dctx->decodedSize != dctx->fParams.frameContentSize, - corruption_detected); + corruption_detected, ""); if (dctx->fParams.checksumFlag) { /* another round for frame checksum */ dctx->expected = 4; dctx->stage = ZSTDds_checkChecksum; @@ -974,7 +1001,6 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c } else { dctx->stage = ZSTDds_decodeBlockHeader; dctx->expected = ZSTD_blockHeaderSize; - dctx->previousDstEnd = (char*)dst + rSize; } return rSize; } @@ -984,7 +1010,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c { U32 const h32 = (U32)XXH64_digest(&dctx->xxhState); U32 const check32 = MEM_readLE32(src); DEBUGLOG(4, "ZSTD_decompressContinue: checksum : calculated %08X :: %08X read", (unsigned)h32, (unsigned)check32); - RETURN_ERROR_IF(check32 != h32, checksum_wrong); + RETURN_ERROR_IF(check32 != h32, checksum_wrong, ""); dctx->expected = 0; dctx->stage = ZSTDds_getFrameHeaderSize; return 0; @@ -1005,7 +1031,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c default: assert(0); /* impossible */ - RETURN_ERROR(GENERIC); /* some compiler require default to do something */ + RETURN_ERROR(GENERIC, "impossible to reach"); /* some compiler require default to do something */ } } @@ -1016,6 +1042,10 @@ static size_t ZSTD_refDictContent(ZSTD_DCtx* dctx, const void* dict, size_t dict dctx->virtualStart = (const char*)dict - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->prefixStart)); dctx->prefixStart = dict; dctx->previousDstEnd = (const char*)dict + dictSize; +#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + dctx->dictContentBeginForFuzzing = dctx->prefixStart; + dctx->dictContentEndForFuzzing = dctx->previousDstEnd; +#endif return 0; } @@ -1029,7 +1059,7 @@ ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy, const BYTE* dictPtr = (const BYTE*)dict; const BYTE* const dictEnd = dictPtr + dictSize; - RETURN_ERROR_IF(dictSize <= 8, dictionary_corrupted); + RETURN_ERROR_IF(dictSize <= 8, dictionary_corrupted, "dict is too small"); assert(MEM_readLE32(dict) == ZSTD_MAGIC_DICTIONARY); /* dict must be valid */ dictPtr += 8; /* skip header = magic + dictID */ @@ -1048,16 +1078,16 @@ ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy, dictPtr, dictEnd - dictPtr, workspace, workspaceSize); #endif - RETURN_ERROR_IF(HUF_isError(hSize), dictionary_corrupted); + RETURN_ERROR_IF(HUF_isError(hSize), dictionary_corrupted, ""); dictPtr += hSize; } { short offcodeNCount[MaxOff+1]; unsigned offcodeMaxValue = MaxOff, offcodeLog; size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr); - RETURN_ERROR_IF(FSE_isError(offcodeHeaderSize), dictionary_corrupted); - RETURN_ERROR_IF(offcodeMaxValue > MaxOff, dictionary_corrupted); - RETURN_ERROR_IF(offcodeLog > OffFSELog, dictionary_corrupted); + RETURN_ERROR_IF(FSE_isError(offcodeHeaderSize), dictionary_corrupted, ""); + RETURN_ERROR_IF(offcodeMaxValue > MaxOff, dictionary_corrupted, ""); + RETURN_ERROR_IF(offcodeLog > OffFSELog, dictionary_corrupted, ""); ZSTD_buildFSETable( entropy->OFTable, offcodeNCount, offcodeMaxValue, OF_base, OF_bits, @@ -1068,9 +1098,9 @@ ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy, { short matchlengthNCount[MaxML+1]; unsigned matchlengthMaxValue = MaxML, matchlengthLog; size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr); - RETURN_ERROR_IF(FSE_isError(matchlengthHeaderSize), dictionary_corrupted); - RETURN_ERROR_IF(matchlengthMaxValue > MaxML, dictionary_corrupted); - RETURN_ERROR_IF(matchlengthLog > MLFSELog, dictionary_corrupted); + RETURN_ERROR_IF(FSE_isError(matchlengthHeaderSize), dictionary_corrupted, ""); + RETURN_ERROR_IF(matchlengthMaxValue > MaxML, dictionary_corrupted, ""); + RETURN_ERROR_IF(matchlengthLog > MLFSELog, dictionary_corrupted, ""); ZSTD_buildFSETable( entropy->MLTable, matchlengthNCount, matchlengthMaxValue, ML_base, ML_bits, @@ -1081,9 +1111,9 @@ ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy, { short litlengthNCount[MaxLL+1]; unsigned litlengthMaxValue = MaxLL, litlengthLog; size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr); - RETURN_ERROR_IF(FSE_isError(litlengthHeaderSize), dictionary_corrupted); - RETURN_ERROR_IF(litlengthMaxValue > MaxLL, dictionary_corrupted); - RETURN_ERROR_IF(litlengthLog > LLFSELog, dictionary_corrupted); + RETURN_ERROR_IF(FSE_isError(litlengthHeaderSize), dictionary_corrupted, ""); + RETURN_ERROR_IF(litlengthMaxValue > MaxLL, dictionary_corrupted, ""); + RETURN_ERROR_IF(litlengthLog > LLFSELog, dictionary_corrupted, ""); ZSTD_buildFSETable( entropy->LLTable, litlengthNCount, litlengthMaxValue, LL_base, LL_bits, @@ -1091,13 +1121,13 @@ ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy, dictPtr += litlengthHeaderSize; } - RETURN_ERROR_IF(dictPtr+12 > dictEnd, dictionary_corrupted); + RETURN_ERROR_IF(dictPtr+12 > dictEnd, dictionary_corrupted, ""); { int i; size_t const dictContentSize = (size_t)(dictEnd - (dictPtr+12)); for (i=0; i<3; i++) { U32 const rep = MEM_readLE32(dictPtr); dictPtr += 4; RETURN_ERROR_IF(rep==0 || rep > dictContentSize, - dictionary_corrupted); + dictionary_corrupted, ""); entropy->rep[i] = rep; } } @@ -1115,7 +1145,7 @@ static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict /* load entropy tables */ { size_t const eSize = ZSTD_loadDEntropy(&dctx->entropy, dict, dictSize); - RETURN_ERROR_IF(ZSTD_isError(eSize), dictionary_corrupted); + RETURN_ERROR_IF(ZSTD_isError(eSize), dictionary_corrupted, ""); dict = (const char*)dict + eSize; dictSize -= eSize; } @@ -1138,6 +1168,7 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx) dctx->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */ dctx->litEntropy = dctx->fseEntropy = 0; dctx->dictID = 0; + dctx->bType = bt_reserved; ZSTD_STATIC_ASSERT(sizeof(dctx->entropy.rep) == sizeof(repStartValue)); memcpy(dctx->entropy.rep, repStartValue, sizeof(repStartValue)); /* initial repcodes */ dctx->LLTptr = dctx->entropy.LLTable; @@ -1149,11 +1180,11 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx) size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) { - FORWARD_IF_ERROR( ZSTD_decompressBegin(dctx) ); + FORWARD_IF_ERROR( ZSTD_decompressBegin(dctx) , ""); if (dict && dictSize) RETURN_ERROR_IF( ZSTD_isError(ZSTD_decompress_insertDictionary(dctx, dict, dictSize)), - dictionary_corrupted); + dictionary_corrupted, ""); return 0; } @@ -1172,7 +1203,7 @@ size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict) DEBUGLOG(4, "DDict is %s", dctx->ddictIsCold ? "~cold~" : "hot!"); } - FORWARD_IF_ERROR( ZSTD_decompressBegin(dctx) ); + FORWARD_IF_ERROR( ZSTD_decompressBegin(dctx) , ""); if (ddict) { /* NULL ddict is equivalent to no dictionary */ ZSTD_copyDDictParameters(dctx, ddict); } @@ -1263,11 +1294,11 @@ size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType) { - RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong); + RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, ""); ZSTD_clearDict(dctx); if (dict && dictSize != 0) { dctx->ddictLocal = ZSTD_createDDict_advanced(dict, dictSize, dictLoadMethod, dictContentType, dctx->customMem); - RETURN_ERROR_IF(dctx->ddictLocal == NULL, memory_allocation); + RETURN_ERROR_IF(dctx->ddictLocal == NULL, memory_allocation, "NULL pointer!"); dctx->ddict = dctx->ddictLocal; dctx->dictUses = ZSTD_use_indefinitely; } @@ -1286,7 +1317,7 @@ size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSi size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType) { - FORWARD_IF_ERROR(ZSTD_DCtx_loadDictionary_advanced(dctx, prefix, prefixSize, ZSTD_dlm_byRef, dictContentType)); + FORWARD_IF_ERROR(ZSTD_DCtx_loadDictionary_advanced(dctx, prefix, prefixSize, ZSTD_dlm_byRef, dictContentType), ""); dctx->dictUses = ZSTD_use_once; return 0; } @@ -1303,8 +1334,8 @@ size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSiz size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize) { DEBUGLOG(4, "ZSTD_initDStream_usingDict"); - FORWARD_IF_ERROR( ZSTD_DCtx_reset(zds, ZSTD_reset_session_only) ); - FORWARD_IF_ERROR( ZSTD_DCtx_loadDictionary(zds, dict, dictSize) ); + FORWARD_IF_ERROR( ZSTD_DCtx_reset(zds, ZSTD_reset_session_only) , ""); + FORWARD_IF_ERROR( ZSTD_DCtx_loadDictionary(zds, dict, dictSize) , ""); return ZSTD_startingInputLength(zds->format); } @@ -1320,8 +1351,8 @@ size_t ZSTD_initDStream(ZSTD_DStream* zds) * this function cannot fail */ size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* dctx, const ZSTD_DDict* ddict) { - FORWARD_IF_ERROR( ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only) ); - FORWARD_IF_ERROR( ZSTD_DCtx_refDDict(dctx, ddict) ); + FORWARD_IF_ERROR( ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only) , ""); + FORWARD_IF_ERROR( ZSTD_DCtx_refDDict(dctx, ddict) , ""); return ZSTD_startingInputLength(dctx->format); } @@ -1330,14 +1361,14 @@ size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* dctx, const ZSTD_DDict* ddict) * this function cannot fail */ size_t ZSTD_resetDStream(ZSTD_DStream* dctx) { - FORWARD_IF_ERROR(ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only)); + FORWARD_IF_ERROR(ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only), ""); return ZSTD_startingInputLength(dctx->format); } size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict) { - RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong); + RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, ""); ZSTD_clearDict(dctx); if (ddict) { dctx->ddict = ddict; @@ -1354,9 +1385,9 @@ size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize) ZSTD_bounds const bounds = ZSTD_dParam_getBounds(ZSTD_d_windowLogMax); size_t const min = (size_t)1 << bounds.lowerBound; size_t const max = (size_t)1 << bounds.upperBound; - RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong); - RETURN_ERROR_IF(maxWindowSize < min, parameter_outOfBound); - RETURN_ERROR_IF(maxWindowSize > max, parameter_outOfBound); + RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, ""); + RETURN_ERROR_IF(maxWindowSize < min, parameter_outOfBound, ""); + RETURN_ERROR_IF(maxWindowSize > max, parameter_outOfBound, ""); dctx->maxWindowSize = maxWindowSize; return 0; } @@ -1379,6 +1410,10 @@ ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam) bounds.upperBound = (int)ZSTD_f_zstd1_magicless; ZSTD_STATIC_ASSERT(ZSTD_f_zstd1 < ZSTD_f_zstd1_magicless); return bounds; + case ZSTD_d_stableOutBuffer: + bounds.lowerBound = (int)ZSTD_obm_buffered; + bounds.upperBound = (int)ZSTD_obm_stable; + return bounds; default:; } bounds.error = ERROR(parameter_unsupported); @@ -1398,12 +1433,12 @@ static int ZSTD_dParam_withinBounds(ZSTD_dParameter dParam, int value) } #define CHECK_DBOUNDS(p,v) { \ - RETURN_ERROR_IF(!ZSTD_dParam_withinBounds(p, v), parameter_outOfBound); \ + RETURN_ERROR_IF(!ZSTD_dParam_withinBounds(p, v), parameter_outOfBound, ""); \ } size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter dParam, int value) { - RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong); + RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, ""); switch(dParam) { case ZSTD_d_windowLogMax: if (value == 0) value = ZSTD_WINDOWLOG_LIMIT_DEFAULT; @@ -1414,9 +1449,13 @@ size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter dParam, int value CHECK_DBOUNDS(ZSTD_d_format, value); dctx->format = (ZSTD_format_e)value; return 0; + case ZSTD_d_stableOutBuffer: + CHECK_DBOUNDS(ZSTD_d_stableOutBuffer, value); + dctx->outBufferMode = (ZSTD_outBufferMode_e)value; + return 0; default:; } - RETURN_ERROR(parameter_unsupported); + RETURN_ERROR(parameter_unsupported, ""); } size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset) @@ -1428,7 +1467,7 @@ size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset) } if ( (reset == ZSTD_reset_parameters) || (reset == ZSTD_reset_session_and_parameters) ) { - RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong); + RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, ""); ZSTD_clearDict(dctx); dctx->format = ZSTD_f_zstd1; dctx->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT; @@ -1449,7 +1488,7 @@ size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long unsigned long long const neededSize = MIN(frameContentSize, neededRBSize); size_t const minRBSize = (size_t) neededSize; RETURN_ERROR_IF((unsigned long long)minRBSize != neededSize, - frameParameter_windowTooLarge); + frameParameter_windowTooLarge, ""); return minRBSize; } @@ -1467,30 +1506,94 @@ size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize) ZSTD_frameHeader zfh; size_t const err = ZSTD_getFrameHeader(&zfh, src, srcSize); if (ZSTD_isError(err)) return err; - RETURN_ERROR_IF(err>0, srcSize_wrong); + RETURN_ERROR_IF(err>0, srcSize_wrong, ""); RETURN_ERROR_IF(zfh.windowSize > windowSizeMax, - frameParameter_windowTooLarge); + frameParameter_windowTooLarge, ""); return ZSTD_estimateDStreamSize((size_t)zfh.windowSize); } /* ***** Decompression ***** */ -MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize) +static int ZSTD_DCtx_isOverflow(ZSTD_DStream* zds, size_t const neededInBuffSize, size_t const neededOutBuffSize) { - size_t const length = MIN(dstCapacity, srcSize); - memcpy(dst, src, length); - return length; + return (zds->inBuffSize + zds->outBuffSize) >= (neededInBuffSize + neededOutBuffSize) * ZSTD_WORKSPACETOOLARGE_FACTOR; +} + +static void ZSTD_DCtx_updateOversizedDuration(ZSTD_DStream* zds, size_t const neededInBuffSize, size_t const neededOutBuffSize) +{ + if (ZSTD_DCtx_isOverflow(zds, neededInBuffSize, neededOutBuffSize)) + zds->oversizedDuration++; + else + zds->oversizedDuration = 0; +} + +static int ZSTD_DCtx_isOversizedTooLong(ZSTD_DStream* zds) +{ + return zds->oversizedDuration >= ZSTD_WORKSPACETOOLARGE_MAXDURATION; +} + +/* Checks that the output buffer hasn't changed if ZSTD_obm_stable is used. */ +static size_t ZSTD_checkOutBuffer(ZSTD_DStream const* zds, ZSTD_outBuffer const* output) +{ + ZSTD_outBuffer const expect = zds->expectedOutBuffer; + /* No requirement when ZSTD_obm_stable is not enabled. */ + if (zds->outBufferMode != ZSTD_obm_stable) + return 0; + /* Any buffer is allowed in zdss_init, this must be the same for every other call until + * the context is reset. + */ + if (zds->streamStage == zdss_init) + return 0; + /* The buffer must match our expectation exactly. */ + if (expect.dst == output->dst && expect.pos == output->pos && expect.size == output->size) + return 0; + RETURN_ERROR(dstBuffer_wrong, "ZSTD_obm_stable enabled but output differs!"); } +/* Calls ZSTD_decompressContinue() with the right parameters for ZSTD_decompressStream() + * and updates the stage and the output buffer state. This call is extracted so it can be + * used both when reading directly from the ZSTD_inBuffer, and in buffered input mode. + * NOTE: You must break after calling this function since the streamStage is modified. + */ +static size_t ZSTD_decompressContinueStream( + ZSTD_DStream* zds, char** op, char* oend, + void const* src, size_t srcSize) { + int const isSkipFrame = ZSTD_isSkipFrame(zds); + if (zds->outBufferMode == ZSTD_obm_buffered) { + size_t const dstSize = isSkipFrame ? 0 : zds->outBuffSize - zds->outStart; + size_t const decodedSize = ZSTD_decompressContinue(zds, + zds->outBuff + zds->outStart, dstSize, src, srcSize); + FORWARD_IF_ERROR(decodedSize, ""); + if (!decodedSize && !isSkipFrame) { + zds->streamStage = zdss_read; + } else { + zds->outEnd = zds->outStart + decodedSize; + zds->streamStage = zdss_flush; + } + } else { + /* Write directly into the output buffer */ + size_t const dstSize = isSkipFrame ? 0 : oend - *op; + size_t const decodedSize = ZSTD_decompressContinue(zds, *op, dstSize, src, srcSize); + FORWARD_IF_ERROR(decodedSize, ""); + *op += decodedSize; + /* Flushing is not needed. */ + zds->streamStage = zdss_read; + assert(*op <= oend); + assert(zds->outBufferMode == ZSTD_obm_stable); + } + return 0; +} size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input) { - const char* const istart = (const char*)(input->src) + input->pos; - const char* const iend = (const char*)(input->src) + input->size; + const char* const src = (const char*)input->src; + const char* const istart = input->pos != 0 ? src + input->pos : src; + const char* const iend = input->size != 0 ? src + input->size : src; const char* ip = istart; - char* const ostart = (char*)(output->dst) + output->pos; - char* const oend = (char*)(output->dst) + output->size; + char* const dst = (char*)output->dst; + char* const ostart = output->pos != 0 ? dst + output->pos : dst; + char* const oend = output->size != 0 ? dst + output->size : dst; char* op = ostart; U32 someMoreWork = 1; @@ -1506,6 +1609,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB "forbidden. out: pos: %u vs size: %u", (U32)output->pos, (U32)output->size); DEBUGLOG(5, "input size : %u", (U32)(input->size - input->pos)); + FORWARD_IF_ERROR(ZSTD_checkOutBuffer(zds, output), ""); while (someMoreWork) { switch(zds->streamStage) @@ -1516,6 +1620,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0; zds->legacyVersion = 0; zds->hostageByte = 0; + zds->expectedOutBuffer = *output; /* fall-through */ case zdss_loadHeader : @@ -1543,7 +1648,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB "legacy support is incompatible with static dctx"); FORWARD_IF_ERROR(ZSTD_initLegacyStream(&zds->legacyContext, zds->previousLegacyVersion, legacyVersion, - dict, dictSize)); + dict, dictSize), ""); zds->legacyVersion = zds->previousLegacyVersion = legacyVersion; { size_t const hint = ZSTD_decompressLegacyStream(zds->legacyContext, legacyVersion, output, input); if (hint==0) zds->streamStage = zdss_init; /* or stay in stage zdss_loadHeader */ @@ -1570,7 +1675,8 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB } } /* check for single-pass mode opportunity */ - if (zds->fParams.frameContentSize && zds->fParams.windowSize /* skippable frame if == 0 */ + if (zds->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN + && zds->fParams.frameType != ZSTD_skippableFrame && (U64)(size_t)(oend-op) >= zds->fParams.frameContentSize) { size_t const cSize = ZSTD_findFrameCompressedSize(istart, iend-istart); if (cSize <= (size_t)(iend-istart)) { @@ -1586,15 +1692,23 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB break; } } + /* Check output buffer is large enough for ZSTD_odm_stable. */ + if (zds->outBufferMode == ZSTD_obm_stable + && zds->fParams.frameType != ZSTD_skippableFrame + && zds->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN + && (U64)(size_t)(oend-op) < zds->fParams.frameContentSize) { + RETURN_ERROR(dstSize_tooSmall, "ZSTD_obm_stable passed but ZSTD_outBuffer is too small"); + } + /* Consume header (see ZSTDds_decodeFrameHeader) */ DEBUGLOG(4, "Consume header"); - FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDDict(zds, ZSTD_getDDict(zds))); + FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDDict(zds, ZSTD_getDDict(zds)), ""); if ((MEM_readLE32(zds->headerBuffer) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */ zds->expected = MEM_readLE32(zds->headerBuffer + ZSTD_FRAMEIDSIZE); zds->stage = ZSTDds_skipFrame; } else { - FORWARD_IF_ERROR(ZSTD_decodeFrameHeader(zds, zds->headerBuffer, zds->lhSize)); + FORWARD_IF_ERROR(ZSTD_decodeFrameHeader(zds, zds->headerBuffer, zds->lhSize), ""); zds->expected = ZSTD_blockHeaderSize; zds->stage = ZSTDds_decodeBlockHeader; } @@ -1605,40 +1719,48 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB (U32)(zds->maxWindowSize >> 10) ); zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN); RETURN_ERROR_IF(zds->fParams.windowSize > zds->maxWindowSize, - frameParameter_windowTooLarge); + frameParameter_windowTooLarge, ""); /* Adapt buffer sizes to frame header instructions */ { size_t const neededInBuffSize = MAX(zds->fParams.blockSizeMax, 4 /* frame checksum */); - size_t const neededOutBuffSize = ZSTD_decodingBufferSize_min(zds->fParams.windowSize, zds->fParams.frameContentSize); - if ((zds->inBuffSize < neededInBuffSize) || (zds->outBuffSize < neededOutBuffSize)) { - size_t const bufferSize = neededInBuffSize + neededOutBuffSize; - DEBUGLOG(4, "inBuff : from %u to %u", - (U32)zds->inBuffSize, (U32)neededInBuffSize); - DEBUGLOG(4, "outBuff : from %u to %u", - (U32)zds->outBuffSize, (U32)neededOutBuffSize); - if (zds->staticSize) { /* static DCtx */ - DEBUGLOG(4, "staticSize : %u", (U32)zds->staticSize); - assert(zds->staticSize >= sizeof(ZSTD_DCtx)); /* controlled at init */ - RETURN_ERROR_IF( - bufferSize > zds->staticSize - sizeof(ZSTD_DCtx), - memory_allocation); - } else { - ZSTD_free(zds->inBuff, zds->customMem); - zds->inBuffSize = 0; - zds->outBuffSize = 0; - zds->inBuff = (char*)ZSTD_malloc(bufferSize, zds->customMem); - RETURN_ERROR_IF(zds->inBuff == NULL, memory_allocation); - } - zds->inBuffSize = neededInBuffSize; - zds->outBuff = zds->inBuff + zds->inBuffSize; - zds->outBuffSize = neededOutBuffSize; - } } + size_t const neededOutBuffSize = zds->outBufferMode == ZSTD_obm_buffered + ? ZSTD_decodingBufferSize_min(zds->fParams.windowSize, zds->fParams.frameContentSize) + : 0; + + ZSTD_DCtx_updateOversizedDuration(zds, neededInBuffSize, neededOutBuffSize); + + { int const tooSmall = (zds->inBuffSize < neededInBuffSize) || (zds->outBuffSize < neededOutBuffSize); + int const tooLarge = ZSTD_DCtx_isOversizedTooLong(zds); + + if (tooSmall || tooLarge) { + size_t const bufferSize = neededInBuffSize + neededOutBuffSize; + DEBUGLOG(4, "inBuff : from %u to %u", + (U32)zds->inBuffSize, (U32)neededInBuffSize); + DEBUGLOG(4, "outBuff : from %u to %u", + (U32)zds->outBuffSize, (U32)neededOutBuffSize); + if (zds->staticSize) { /* static DCtx */ + DEBUGLOG(4, "staticSize : %u", (U32)zds->staticSize); + assert(zds->staticSize >= sizeof(ZSTD_DCtx)); /* controlled at init */ + RETURN_ERROR_IF( + bufferSize > zds->staticSize - sizeof(ZSTD_DCtx), + memory_allocation, ""); + } else { + ZSTD_free(zds->inBuff, zds->customMem); + zds->inBuffSize = 0; + zds->outBuffSize = 0; + zds->inBuff = (char*)ZSTD_malloc(bufferSize, zds->customMem); + RETURN_ERROR_IF(zds->inBuff == NULL, memory_allocation, ""); + } + zds->inBuffSize = neededInBuffSize; + zds->outBuff = zds->inBuff + zds->inBuffSize; + zds->outBuffSize = neededOutBuffSize; + } } } zds->streamStage = zdss_read; /* fall-through */ case zdss_read: DEBUGLOG(5, "stage zdss_read"); - { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds); + { size_t const neededInSize = ZSTD_nextSrcSizeToDecompressWithInputSize(zds, iend - ip); DEBUGLOG(5, "neededInSize = %u", (U32)neededInSize); if (neededInSize==0) { /* end of frame */ zds->streamStage = zdss_init; @@ -1646,15 +1768,9 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB break; } if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */ - int const isSkipFrame = ZSTD_isSkipFrame(zds); - size_t const decodedSize = ZSTD_decompressContinue(zds, - zds->outBuff + zds->outStart, (isSkipFrame ? 0 : zds->outBuffSize - zds->outStart), - ip, neededInSize); - if (ZSTD_isError(decodedSize)) return decodedSize; + FORWARD_IF_ERROR(ZSTD_decompressContinueStream(zds, &op, oend, ip, neededInSize), ""); ip += neededInSize; - if (!decodedSize && !isSkipFrame) break; /* this was just a header */ - zds->outEnd = zds->outStart + decodedSize; - zds->streamStage = zdss_flush; + /* Function modifies the stage so we must break */ break; } } if (ip==iend) { someMoreWork = 0; break; } /* no more input */ @@ -1666,6 +1782,8 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB size_t const toLoad = neededInSize - zds->inPos; int const isSkipFrame = ZSTD_isSkipFrame(zds); size_t loadedSize; + /* At this point we shouldn't be decompressing a block that we can stream. */ + assert(neededInSize == ZSTD_nextSrcSizeToDecompressWithInputSize(zds, iend - ip)); if (isSkipFrame) { loadedSize = MIN(toLoad, (size_t)(iend-ip)); } else { @@ -1679,17 +1797,11 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB if (loadedSize < toLoad) { someMoreWork = 0; break; } /* not enough input, wait for more */ /* decode loaded input */ - { size_t const decodedSize = ZSTD_decompressContinue(zds, - zds->outBuff + zds->outStart, zds->outBuffSize - zds->outStart, - zds->inBuff, neededInSize); - if (ZSTD_isError(decodedSize)) return decodedSize; - zds->inPos = 0; /* input is consumed */ - if (!decodedSize && !isSkipFrame) { zds->streamStage = zdss_read; break; } /* this was just a header */ - zds->outEnd = zds->outStart + decodedSize; - } } - zds->streamStage = zdss_flush; - /* fall-through */ - + zds->inPos = 0; /* input is consumed */ + FORWARD_IF_ERROR(ZSTD_decompressContinueStream(zds, &op, oend, zds->inBuff, neededInSize), ""); + /* Function modifies the stage so we must break */ + break; + } case zdss_flush: { size_t const toFlushSize = zds->outEnd - zds->outStart; size_t const flushedSize = ZSTD_limitCopy(op, oend-op, zds->outBuff + zds->outStart, toFlushSize); @@ -1712,17 +1824,21 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB default: assert(0); /* impossible */ - RETURN_ERROR(GENERIC); /* some compiler require default to do something */ + RETURN_ERROR(GENERIC, "impossible to reach"); /* some compiler require default to do something */ } } /* result */ input->pos = (size_t)(ip - (const char*)(input->src)); output->pos = (size_t)(op - (char*)(output->dst)); + + /* Update the expected output buffer for ZSTD_obm_stable. */ + zds->expectedOutBuffer = *output; + if ((ip==istart) && (op==ostart)) { /* no forward progress */ zds->noForwardProgress ++; if (zds->noForwardProgress >= ZSTD_NO_FORWARD_PROGRESS_MAX) { - RETURN_ERROR_IF(op==oend, dstSize_tooSmall); - RETURN_ERROR_IF(ip==iend, srcSize_wrong); + RETURN_ERROR_IF(op==oend, dstSize_tooSmall, ""); + RETURN_ERROR_IF(ip==iend, srcSize_wrong, ""); assert(0); } } else { diff --git a/thirdparty/zstd/decompress/zstd_decompress_block.c b/thirdparty/zstd/decompress/zstd_decompress_block.c index 767e5f9a0b..ad3b3d8dbd 100644 --- a/thirdparty/zstd/decompress/zstd_decompress_block.c +++ b/thirdparty/zstd/decompress/zstd_decompress_block.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -15,14 +15,14 @@ * Dependencies *********************************************************/ #include <string.h> /* memcpy, memmove, memset */ -#include "compiler.h" /* prefetch */ -#include "cpu.h" /* bmi2 */ -#include "mem.h" /* low level memory routines */ +#include "../common/compiler.h" /* prefetch */ +#include "../common/cpu.h" /* bmi2 */ +#include "../common/mem.h" /* low level memory routines */ #define FSE_STATIC_LINKING_ONLY -#include "fse.h" +#include "../common/fse.h" #define HUF_STATIC_LINKING_ONLY -#include "huf.h" -#include "zstd_internal.h" +#include "../common/huf.h" +#include "../common/zstd_internal.h" #include "zstd_decompress_internal.h" /* ZSTD_DCtx */ #include "zstd_ddict.h" /* ZSTD_DDictDictContent */ #include "zstd_decompress_block.h" @@ -56,7 +56,7 @@ static void ZSTD_copy4(void* dst, const void* src) { memcpy(dst, src, 4); } size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr) { - RETURN_ERROR_IF(srcSize < ZSTD_blockHeaderSize, srcSize_wrong); + RETURN_ERROR_IF(srcSize < ZSTD_blockHeaderSize, srcSize_wrong, ""); { U32 const cBlockHeader = MEM_readLE24(src); U32 const cSize = cBlockHeader >> 3; @@ -64,7 +64,7 @@ size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, bpPtr->blockType = (blockType_e)((cBlockHeader >> 1) & 3); bpPtr->origSize = cSize; /* only useful for RLE */ if (bpPtr->blockType == bt_rle) return 1; - RETURN_ERROR_IF(bpPtr->blockType == bt_reserved, corruption_detected); + RETURN_ERROR_IF(bpPtr->blockType == bt_reserved, corruption_detected, ""); return cSize; } } @@ -80,7 +80,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, const void* src, size_t srcSize) /* note : srcSize < BLOCKSIZE */ { DEBUGLOG(5, "ZSTD_decodeLiteralsBlock"); - RETURN_ERROR_IF(srcSize < MIN_CBLOCK_SIZE, corruption_detected); + RETURN_ERROR_IF(srcSize < MIN_CBLOCK_SIZE, corruption_detected, ""); { const BYTE* const istart = (const BYTE*) src; symbolEncodingType_e const litEncType = (symbolEncodingType_e)(istart[0] & 3); @@ -89,7 +89,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, { case set_repeat: DEBUGLOG(5, "set_repeat flag : re-using stats from previous compressed literals block"); - RETURN_ERROR_IF(dctx->litEntropy==0, dictionary_corrupted); + RETURN_ERROR_IF(dctx->litEntropy==0, dictionary_corrupted, ""); /* fall-through */ case set_compressed: @@ -121,8 +121,8 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, litCSize = (lhc >> 22) + ((size_t)istart[4] << 10); break; } - RETURN_ERROR_IF(litSize > ZSTD_BLOCKSIZE_MAX, corruption_detected); - RETURN_ERROR_IF(litCSize + lhSize > srcSize, corruption_detected); + RETURN_ERROR_IF(litSize > ZSTD_BLOCKSIZE_MAX, corruption_detected, ""); + RETURN_ERROR_IF(litCSize + lhSize > srcSize, corruption_detected, ""); /* prefetch huffman table if cold */ if (dctx->ddictIsCold && (litSize > 768 /* heuristic */)) { @@ -160,7 +160,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, } } - RETURN_ERROR_IF(HUF_isError(hufSuccess), corruption_detected); + RETURN_ERROR_IF(HUF_isError(hufSuccess), corruption_detected, ""); dctx->litPtr = dctx->litBuffer; dctx->litSize = litSize; @@ -190,7 +190,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, } if (lhSize+litSize+WILDCOPY_OVERLENGTH > srcSize) { /* risk reading beyond src buffer with wildcopy */ - RETURN_ERROR_IF(litSize+lhSize > srcSize, corruption_detected); + RETURN_ERROR_IF(litSize+lhSize > srcSize, corruption_detected, ""); memcpy(dctx->litBuffer, istart+lhSize, litSize); dctx->litPtr = dctx->litBuffer; dctx->litSize = litSize; @@ -222,7 +222,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, RETURN_ERROR_IF(srcSize<4, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 3; here we need lhSize+1 = 4"); break; } - RETURN_ERROR_IF(litSize > ZSTD_BLOCKSIZE_MAX, corruption_detected); + RETURN_ERROR_IF(litSize > ZSTD_BLOCKSIZE_MAX, corruption_detected, ""); memset(dctx->litBuffer, istart[lhSize], litSize + WILDCOPY_OVERLENGTH); dctx->litPtr = dctx->litBuffer; dctx->litSize = litSize; @@ -440,8 +440,8 @@ static size_t ZSTD_buildSeqTable(ZSTD_seqSymbol* DTableSpace, const ZSTD_seqSymb switch(type) { case set_rle : - RETURN_ERROR_IF(!srcSize, srcSize_wrong); - RETURN_ERROR_IF((*(const BYTE*)src) > max, corruption_detected); + RETURN_ERROR_IF(!srcSize, srcSize_wrong, ""); + RETURN_ERROR_IF((*(const BYTE*)src) > max, corruption_detected, ""); { U32 const symbol = *(const BYTE*)src; U32 const baseline = baseValue[symbol]; U32 const nbBits = nbAdditionalBits[symbol]; @@ -453,7 +453,7 @@ static size_t ZSTD_buildSeqTable(ZSTD_seqSymbol* DTableSpace, const ZSTD_seqSymb *DTablePtr = defaultTable; return 0; case set_repeat: - RETURN_ERROR_IF(!flagRepeatTable, corruption_detected); + RETURN_ERROR_IF(!flagRepeatTable, corruption_detected, ""); /* prefetch FSE table if used */ if (ddictIsCold && (nbSeq > 24 /* heuristic */)) { const void* const pStart = *DTablePtr; @@ -465,8 +465,8 @@ static size_t ZSTD_buildSeqTable(ZSTD_seqSymbol* DTableSpace, const ZSTD_seqSymb { unsigned tableLog; S16 norm[MaxSeq+1]; size_t const headerSize = FSE_readNCount(norm, &max, &tableLog, src, srcSize); - RETURN_ERROR_IF(FSE_isError(headerSize), corruption_detected); - RETURN_ERROR_IF(tableLog > maxLog, corruption_detected); + RETURN_ERROR_IF(FSE_isError(headerSize), corruption_detected, ""); + RETURN_ERROR_IF(tableLog > maxLog, corruption_detected, ""); ZSTD_buildFSETable(DTableSpace, norm, max, baseValue, nbAdditionalBits, tableLog); *DTablePtr = DTableSpace; return headerSize; @@ -487,28 +487,28 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, DEBUGLOG(5, "ZSTD_decodeSeqHeaders"); /* check */ - RETURN_ERROR_IF(srcSize < MIN_SEQUENCES_SIZE, srcSize_wrong); + RETURN_ERROR_IF(srcSize < MIN_SEQUENCES_SIZE, srcSize_wrong, ""); /* SeqHead */ nbSeq = *ip++; if (!nbSeq) { *nbSeqPtr=0; - RETURN_ERROR_IF(srcSize != 1, srcSize_wrong); + RETURN_ERROR_IF(srcSize != 1, srcSize_wrong, ""); return 1; } if (nbSeq > 0x7F) { if (nbSeq == 0xFF) { - RETURN_ERROR_IF(ip+2 > iend, srcSize_wrong); + RETURN_ERROR_IF(ip+2 > iend, srcSize_wrong, ""); nbSeq = MEM_readLE16(ip) + LONGNBSEQ, ip+=2; } else { - RETURN_ERROR_IF(ip >= iend, srcSize_wrong); + RETURN_ERROR_IF(ip >= iend, srcSize_wrong, ""); nbSeq = ((nbSeq-0x80)<<8) + *ip++; } } *nbSeqPtr = nbSeq; /* FSE table descriptors */ - RETURN_ERROR_IF(ip+1 > iend, srcSize_wrong); /* minimum possible size: 1 byte for symbol encoding types */ + RETURN_ERROR_IF(ip+1 > iend, srcSize_wrong, ""); /* minimum possible size: 1 byte for symbol encoding types */ { symbolEncodingType_e const LLtype = (symbolEncodingType_e)(*ip >> 6); symbolEncodingType_e const OFtype = (symbolEncodingType_e)((*ip >> 4) & 3); symbolEncodingType_e const MLtype = (symbolEncodingType_e)((*ip >> 2) & 3); @@ -521,7 +521,7 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, LL_base, LL_bits, LL_defaultDTable, dctx->fseEntropy, dctx->ddictIsCold, nbSeq); - RETURN_ERROR_IF(ZSTD_isError(llhSize), corruption_detected); + RETURN_ERROR_IF(ZSTD_isError(llhSize), corruption_detected, "ZSTD_buildSeqTable failed"); ip += llhSize; } @@ -531,7 +531,7 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, OF_base, OF_bits, OF_defaultDTable, dctx->fseEntropy, dctx->ddictIsCold, nbSeq); - RETURN_ERROR_IF(ZSTD_isError(ofhSize), corruption_detected); + RETURN_ERROR_IF(ZSTD_isError(ofhSize), corruption_detected, "ZSTD_buildSeqTable failed"); ip += ofhSize; } @@ -541,7 +541,7 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, ML_base, ML_bits, ML_defaultDTable, dctx->fseEntropy, dctx->ddictIsCold, nbSeq); - RETURN_ERROR_IF(ZSTD_isError(mlhSize), corruption_detected); + RETURN_ERROR_IF(ZSTD_isError(mlhSize), corruption_detected, "ZSTD_buildSeqTable failed"); ip += mlhSize; } } @@ -580,7 +580,7 @@ typedef struct { * Precondition: *ip <= *op * Postcondition: *op - *op >= 8 */ -static void ZSTD_overlapCopy8(BYTE** op, BYTE const** ip, size_t offset) { +HINT_INLINE void ZSTD_overlapCopy8(BYTE** op, BYTE const** ip, size_t offset) { assert(*ip <= *op); if (offset < 8) { /* close range match, overlap */ @@ -665,15 +665,15 @@ size_t ZSTD_execSequenceEnd(BYTE* op, { BYTE* const oLitEnd = op + sequence.litLength; size_t const sequenceLength = sequence.litLength + sequence.matchLength; - BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */ const BYTE* const iLitEnd = *litPtr + sequence.litLength; const BYTE* match = oLitEnd - sequence.offset; BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH; - /* bounds checks */ - assert(oLitEnd < oMatchEnd); - RETURN_ERROR_IF(oMatchEnd > oend, dstSize_tooSmall, "last match must fit within dstBuffer"); - RETURN_ERROR_IF(iLitEnd > litLimit, corruption_detected, "try to read beyond literal buffer"); + /* bounds checks : careful of address space overflow in 32-bit mode */ + RETURN_ERROR_IF(sequenceLength > (size_t)(oend - op), dstSize_tooSmall, "last match must fit within dstBuffer"); + RETURN_ERROR_IF(sequence.litLength > (size_t)(litLimit - *litPtr), corruption_detected, "try to read beyond literal buffer"); + assert(op < op + sequenceLength); + assert(oLitEnd < op + sequenceLength); /* copy literals */ ZSTD_safecopy(op, oend_w, *litPtr, sequence.litLength, ZSTD_no_overlap); @@ -683,7 +683,7 @@ size_t ZSTD_execSequenceEnd(BYTE* op, /* copy Match */ if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { /* offset beyond prefix */ - RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - virtualStart), corruption_detected); + RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - virtualStart), corruption_detected, ""); match = dictEnd - (prefixStart-match); if (match + sequence.matchLength <= dictEnd) { memmove(oLitEnd, match, sequence.matchLength); @@ -709,16 +709,27 @@ size_t ZSTD_execSequence(BYTE* op, BYTE* const oLitEnd = op + sequence.litLength; size_t const sequenceLength = sequence.litLength + sequence.matchLength; BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */ - BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH; + BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH; /* risk : address space underflow on oend=NULL */ const BYTE* const iLitEnd = *litPtr + sequence.litLength; const BYTE* match = oLitEnd - sequence.offset; - /* Errors and uncommon cases handled here. */ - assert(oLitEnd < oMatchEnd); - if (iLitEnd > litLimit || oMatchEnd > oend_w) + assert(op != NULL /* Precondition */); + assert(oend_w < oend /* No underflow */); + /* Handle edge cases in a slow path: + * - Read beyond end of literals + * - Match end is within WILDCOPY_OVERLIMIT of oend + * - 32-bit mode and the match length overflows + */ + if (UNLIKELY( + iLitEnd > litLimit || + oMatchEnd > oend_w || + (MEM_32bits() && (size_t)(oend - op) < sequenceLength + WILDCOPY_OVERLENGTH))) return ZSTD_execSequenceEnd(op, oend, sequence, litPtr, litLimit, prefixStart, virtualStart, dictEnd); /* Assumptions (everything else goes into ZSTD_execSequenceEnd()) */ + assert(op <= oLitEnd /* No overflow */); + assert(oLitEnd < oMatchEnd /* Non-zero match & no overflow */); + assert(oMatchEnd <= oend /* No underflow */); assert(iLitEnd <= litLimit /* Literal length is in bounds */); assert(oLitEnd <= oend_w /* Can wildcopy literals */); assert(oMatchEnd <= oend_w /* Can wildcopy matches */); @@ -729,7 +740,7 @@ size_t ZSTD_execSequence(BYTE* op, */ assert(WILDCOPY_OVERLENGTH >= 16); ZSTD_copy16(op, (*litPtr)); - if (sequence.litLength > 16) { + if (UNLIKELY(sequence.litLength > 16)) { ZSTD_wildcopy(op+16, (*litPtr)+16, sequence.litLength-16, ZSTD_no_overlap); } op = oLitEnd; @@ -738,7 +749,7 @@ size_t ZSTD_execSequence(BYTE* op, /* Copy Match */ if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { /* offset beyond prefix -> go into extDict */ - RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - virtualStart), corruption_detected); + RETURN_ERROR_IF(UNLIKELY(sequence.offset > (size_t)(oLitEnd - virtualStart)), corruption_detected, ""); match = dictEnd + (match - prefixStart); if (match + sequence.matchLength <= dictEnd) { memmove(oLitEnd, match, sequence.matchLength); @@ -760,7 +771,7 @@ size_t ZSTD_execSequence(BYTE* op, /* Nearly all offsets are >= WILDCOPY_VECLEN bytes, which means we can use wildcopy * without overlap checking. */ - if (sequence.offset >= WILDCOPY_VECLEN) { + if (LIKELY(sequence.offset >= WILDCOPY_VECLEN)) { /* We bet on a full wildcopy for matches, since we expect matches to be * longer than literals (in general). In silesia, ~10% of matches are longer * than 16 bytes. @@ -802,6 +813,14 @@ ZSTD_updateFseState(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD) DStatePtr->state = DInfo.nextState + lowBits; } +FORCE_INLINE_TEMPLATE void +ZSTD_updateFseStateWithDInfo(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD, ZSTD_seqSymbol const DInfo) +{ + U32 const nbBits = DInfo.nbBits; + size_t const lowBits = BIT_readBits(bitD, nbBits); + DStatePtr->state = DInfo.nextState + lowBits; +} + /* We need to add at most (ZSTD_WINDOWLOG_MAX_32 - 1) bits to read the maximum * offset bits. But we can only read at most (STREAM_ACCUMULATOR_MIN_32 - 1) * bits before reloading. This value is the maximum number of bytes we read @@ -813,25 +832,26 @@ ZSTD_updateFseState(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD) : 0) typedef enum { ZSTD_lo_isRegularOffset, ZSTD_lo_isLongOffset=1 } ZSTD_longOffset_e; +typedef enum { ZSTD_p_noPrefetch=0, ZSTD_p_prefetch=1 } ZSTD_prefetch_e; -#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG FORCE_INLINE_TEMPLATE seq_t -ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets) +ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets, const ZSTD_prefetch_e prefetch) { seq_t seq; - U32 const llBits = seqState->stateLL.table[seqState->stateLL.state].nbAdditionalBits; - U32 const mlBits = seqState->stateML.table[seqState->stateML.state].nbAdditionalBits; - U32 const ofBits = seqState->stateOffb.table[seqState->stateOffb.state].nbAdditionalBits; - U32 const totalBits = llBits+mlBits+ofBits; - U32 const llBase = seqState->stateLL.table[seqState->stateLL.state].baseValue; - U32 const mlBase = seqState->stateML.table[seqState->stateML.state].baseValue; - U32 const ofBase = seqState->stateOffb.table[seqState->stateOffb.state].baseValue; + ZSTD_seqSymbol const llDInfo = seqState->stateLL.table[seqState->stateLL.state]; + ZSTD_seqSymbol const mlDInfo = seqState->stateML.table[seqState->stateML.state]; + ZSTD_seqSymbol const ofDInfo = seqState->stateOffb.table[seqState->stateOffb.state]; + U32 const llBase = llDInfo.baseValue; + U32 const mlBase = mlDInfo.baseValue; + U32 const ofBase = ofDInfo.baseValue; + BYTE const llBits = llDInfo.nbAdditionalBits; + BYTE const mlBits = mlDInfo.nbAdditionalBits; + BYTE const ofBits = ofDInfo.nbAdditionalBits; + BYTE const totalBits = llBits+mlBits+ofBits; /* sequence */ { size_t offset; - if (!ofBits) - offset = 0; - else { + if (ofBits > 1) { ZSTD_STATIC_ASSERT(ZSTD_lo_isLongOffset == 1); ZSTD_STATIC_ASSERT(LONG_OFFSETS_MAX_EXTRA_BITS_32 == 5); assert(ofBits <= MaxOff); @@ -845,59 +865,138 @@ ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets) offset = ofBase + BIT_readBitsFast(&seqState->DStream, ofBits/*>0*/); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */ if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); } - } - - if (ofBits <= 1) { - offset += (llBase==0); - if (offset) { - size_t temp = (offset==3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset]; - temp += !temp; /* 0 is not valid; input is corrupted; force offset to 1 */ - if (offset != 1) seqState->prevOffset[2] = seqState->prevOffset[1]; - seqState->prevOffset[1] = seqState->prevOffset[0]; - seqState->prevOffset[0] = offset = temp; - } else { /* offset == 0 */ - offset = seqState->prevOffset[0]; - } - } else { seqState->prevOffset[2] = seqState->prevOffset[1]; seqState->prevOffset[1] = seqState->prevOffset[0]; seqState->prevOffset[0] = offset; - } + } else { + U32 const ll0 = (llBase == 0); + if (LIKELY((ofBits == 0))) { + if (LIKELY(!ll0)) + offset = seqState->prevOffset[0]; + else { + offset = seqState->prevOffset[1]; + seqState->prevOffset[1] = seqState->prevOffset[0]; + seqState->prevOffset[0] = offset; + } + } else { + offset = ofBase + ll0 + BIT_readBitsFast(&seqState->DStream, 1); + { size_t temp = (offset==3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset]; + temp += !temp; /* 0 is not valid; input is corrupted; force offset to 1 */ + if (offset != 1) seqState->prevOffset[2] = seqState->prevOffset[1]; + seqState->prevOffset[1] = seqState->prevOffset[0]; + seqState->prevOffset[0] = offset = temp; + } } } seq.offset = offset; } - seq.matchLength = mlBase - + ((mlBits>0) ? BIT_readBitsFast(&seqState->DStream, mlBits/*>0*/) : 0); /* <= 16 bits */ + seq.matchLength = mlBase; + if (mlBits > 0) + seq.matchLength += BIT_readBitsFast(&seqState->DStream, mlBits/*>0*/); + if (MEM_32bits() && (mlBits+llBits >= STREAM_ACCUMULATOR_MIN_32-LONG_OFFSETS_MAX_EXTRA_BITS_32)) BIT_reloadDStream(&seqState->DStream); - if (MEM_64bits() && (totalBits >= STREAM_ACCUMULATOR_MIN_64-(LLFSELog+MLFSELog+OffFSELog))) + if (MEM_64bits() && UNLIKELY(totalBits >= STREAM_ACCUMULATOR_MIN_64-(LLFSELog+MLFSELog+OffFSELog))) BIT_reloadDStream(&seqState->DStream); /* Ensure there are enough bits to read the rest of data in 64-bit mode. */ ZSTD_STATIC_ASSERT(16+LLFSELog+MLFSELog+OffFSELog < STREAM_ACCUMULATOR_MIN_64); - seq.litLength = llBase - + ((llBits>0) ? BIT_readBitsFast(&seqState->DStream, llBits/*>0*/) : 0); /* <= 16 bits */ + seq.litLength = llBase; + if (llBits > 0) + seq.litLength += BIT_readBitsFast(&seqState->DStream, llBits/*>0*/); + if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); DEBUGLOG(6, "seq: litL=%u, matchL=%u, offset=%u", (U32)seq.litLength, (U32)seq.matchLength, (U32)seq.offset); - /* ANS state update */ - ZSTD_updateFseState(&seqState->stateLL, &seqState->DStream); /* <= 9 bits */ - ZSTD_updateFseState(&seqState->stateML, &seqState->DStream); /* <= 9 bits */ - if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */ - ZSTD_updateFseState(&seqState->stateOffb, &seqState->DStream); /* <= 8 bits */ + if (prefetch == ZSTD_p_prefetch) { + size_t const pos = seqState->pos + seq.litLength; + const BYTE* const matchBase = (seq.offset > pos) ? seqState->dictEnd : seqState->prefixStart; + seq.match = matchBase + pos - seq.offset; /* note : this operation can overflow when seq.offset is really too large, which can only happen when input is corrupted. + * No consequence though : no memory access will occur, offset is only used for prefetching */ + seqState->pos = pos + seq.matchLength; + } + + /* ANS state update + * gcc-9.0.0 does 2.5% worse with ZSTD_updateFseStateWithDInfo(). + * clang-9.2.0 does 7% worse with ZSTD_updateFseState(). + * Naturally it seems like ZSTD_updateFseStateWithDInfo() should be the + * better option, so it is the default for other compilers. But, if you + * measure that it is worse, please put up a pull request. + */ + { +#if defined(__GNUC__) && !defined(__clang__) + const int kUseUpdateFseState = 1; +#else + const int kUseUpdateFseState = 0; +#endif + if (kUseUpdateFseState) { + ZSTD_updateFseState(&seqState->stateLL, &seqState->DStream); /* <= 9 bits */ + ZSTD_updateFseState(&seqState->stateML, &seqState->DStream); /* <= 9 bits */ + if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */ + ZSTD_updateFseState(&seqState->stateOffb, &seqState->DStream); /* <= 8 bits */ + } else { + ZSTD_updateFseStateWithDInfo(&seqState->stateLL, &seqState->DStream, llDInfo); /* <= 9 bits */ + ZSTD_updateFseStateWithDInfo(&seqState->stateML, &seqState->DStream, mlDInfo); /* <= 9 bits */ + if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */ + ZSTD_updateFseStateWithDInfo(&seqState->stateOffb, &seqState->DStream, ofDInfo); /* <= 8 bits */ + } + } return seq; } +#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION +static int ZSTD_dictionaryIsActive(ZSTD_DCtx const* dctx, BYTE const* prefixStart, BYTE const* oLitEnd) +{ + size_t const windowSize = dctx->fParams.windowSize; + /* No dictionary used. */ + if (dctx->dictContentEndForFuzzing == NULL) return 0; + /* Dictionary is our prefix. */ + if (prefixStart == dctx->dictContentBeginForFuzzing) return 1; + /* Dictionary is not our ext-dict. */ + if (dctx->dictEnd != dctx->dictContentEndForFuzzing) return 0; + /* Dictionary is not within our window size. */ + if ((size_t)(oLitEnd - prefixStart) >= windowSize) return 0; + /* Dictionary is active. */ + return 1; +} + +MEM_STATIC void ZSTD_assertValidSequence( + ZSTD_DCtx const* dctx, + BYTE const* op, BYTE const* oend, + seq_t const seq, + BYTE const* prefixStart, BYTE const* virtualStart) +{ + size_t const windowSize = dctx->fParams.windowSize; + size_t const sequenceSize = seq.litLength + seq.matchLength; + BYTE const* const oLitEnd = op + seq.litLength; + DEBUGLOG(6, "Checking sequence: litL=%u matchL=%u offset=%u", + (U32)seq.litLength, (U32)seq.matchLength, (U32)seq.offset); + assert(op <= oend); + assert((size_t)(oend - op) >= sequenceSize); + assert(sequenceSize <= ZSTD_BLOCKSIZE_MAX); + if (ZSTD_dictionaryIsActive(dctx, prefixStart, oLitEnd)) { + size_t const dictSize = (size_t)((char const*)dctx->dictContentEndForFuzzing - (char const*)dctx->dictContentBeginForFuzzing); + /* Offset must be within the dictionary. */ + assert(seq.offset <= (size_t)(oLitEnd - virtualStart)); + assert(seq.offset <= windowSize + dictSize); + } else { + /* Offset must be within our window. */ + assert(seq.offset <= windowSize); + } +} +#endif + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG FORCE_INLINE_TEMPLATE size_t DONT_VECTORIZE ZSTD_decompressSequences_body( ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset) + const ZSTD_longOffset_e isLongOffset, + const int frame) { const BYTE* ip = (const BYTE*)seqStart; const BYTE* const iend = ip + seqSize; @@ -910,46 +1009,104 @@ ZSTD_decompressSequences_body( ZSTD_DCtx* dctx, const BYTE* const vBase = (const BYTE*) (dctx->virtualStart); const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd); DEBUGLOG(5, "ZSTD_decompressSequences_body"); + (void)frame; /* Regen sequences */ if (nbSeq) { seqState_t seqState; + size_t error = 0; dctx->fseEntropy = 1; { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->entropy.rep[i]; } RETURN_ERROR_IF( ERR_isError(BIT_initDStream(&seqState.DStream, ip, iend-ip)), - corruption_detected); + corruption_detected, ""); ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr); ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr); ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); + assert(dst != NULL); ZSTD_STATIC_ASSERT( BIT_DStream_unfinished < BIT_DStream_completed && BIT_DStream_endOfBuffer < BIT_DStream_completed && BIT_DStream_completed < BIT_DStream_overflow); - for ( ; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && nbSeq ; ) { - nbSeq--; - { seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset); - size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litEnd, prefixStart, vBase, dictEnd); - DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize); - if (ZSTD_isError(oneSeqSize)) return oneSeqSize; - op += oneSeqSize; - } } +#if defined(__GNUC__) && defined(__x86_64__) + /* Align the decompression loop to 32 + 16 bytes. + * + * zstd compiled with gcc-9 on an Intel i9-9900k shows 10% decompression + * speed swings based on the alignment of the decompression loop. This + * performance swing is caused by parts of the decompression loop falling + * out of the DSB. The entire decompression loop should fit in the DSB, + * when it can't we get much worse performance. You can measure if you've + * hit the good case or the bad case with this perf command for some + * compressed file test.zst: + * + * perf stat -e cycles -e instructions -e idq.all_dsb_cycles_any_uops \ + * -e idq.all_mite_cycles_any_uops -- ./zstd -tq test.zst + * + * If you see most cycles served out of the MITE you've hit the bad case. + * If you see most cycles served out of the DSB you've hit the good case. + * If it is pretty even then you may be in an okay case. + * + * I've been able to reproduce this issue on the following CPUs: + * - Kabylake: Macbook Pro (15-inch, 2019) 2.4 GHz Intel Core i9 + * Use Instruments->Counters to get DSB/MITE cycles. + * I never got performance swings, but I was able to + * go from the good case of mostly DSB to half of the + * cycles served from MITE. + * - Coffeelake: Intel i9-9900k + * + * I haven't been able to reproduce the instability or DSB misses on any + * of the following CPUS: + * - Haswell + * - Broadwell: Intel(R) Xeon(R) CPU E5-2680 v4 @ 2.40GH + * - Skylake + * + * If you are seeing performance stability this script can help test. + * It tests on 4 commits in zstd where I saw performance change. + * + * https://gist.github.com/terrelln/9889fc06a423fd5ca6e99351564473f4 + */ + __asm__(".p2align 5"); + __asm__("nop"); + __asm__(".p2align 4"); +#endif + for ( ; ; ) { + seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset, ZSTD_p_noPrefetch); + size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litEnd, prefixStart, vBase, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase); +#endif + DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize); + BIT_reloadDStream(&(seqState.DStream)); + /* gcc and clang both don't like early returns in this loop. + * gcc doesn't like early breaks either. + * Instead save an error and report it at the end. + * When there is an error, don't increment op, so we don't + * overwrite. + */ + if (UNLIKELY(ZSTD_isError(oneSeqSize))) error = oneSeqSize; + else op += oneSeqSize; + if (UNLIKELY(!--nbSeq)) break; + } /* check if reached exact end */ DEBUGLOG(5, "ZSTD_decompressSequences_body: after decode loop, remaining nbSeq : %i", nbSeq); - RETURN_ERROR_IF(nbSeq, corruption_detected); - RETURN_ERROR_IF(BIT_reloadDStream(&seqState.DStream) < BIT_DStream_completed, corruption_detected); + if (ZSTD_isError(error)) return error; + RETURN_ERROR_IF(nbSeq, corruption_detected, ""); + RETURN_ERROR_IF(BIT_reloadDStream(&seqState.DStream) < BIT_DStream_completed, corruption_detected, ""); /* save reps for next block */ { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) dctx->entropy.rep[i] = (U32)(seqState.prevOffset[i]); } } /* last literal segment */ { size_t const lastLLSize = litEnd - litPtr; - RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall); - memcpy(op, litPtr, lastLLSize); - op += lastLLSize; + RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, ""); + if (op != NULL) { + memcpy(op, litPtr, lastLLSize); + op += lastLLSize; + } } return op-ostart; @@ -959,99 +1116,21 @@ static size_t ZSTD_decompressSequences_default(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset) + const ZSTD_longOffset_e isLongOffset, + const int frame) { - return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); + return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); } #endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */ - - #ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT -FORCE_INLINE_TEMPLATE seq_t -ZSTD_decodeSequenceLong(seqState_t* seqState, ZSTD_longOffset_e const longOffsets) -{ - seq_t seq; - U32 const llBits = seqState->stateLL.table[seqState->stateLL.state].nbAdditionalBits; - U32 const mlBits = seqState->stateML.table[seqState->stateML.state].nbAdditionalBits; - U32 const ofBits = seqState->stateOffb.table[seqState->stateOffb.state].nbAdditionalBits; - U32 const totalBits = llBits+mlBits+ofBits; - U32 const llBase = seqState->stateLL.table[seqState->stateLL.state].baseValue; - U32 const mlBase = seqState->stateML.table[seqState->stateML.state].baseValue; - U32 const ofBase = seqState->stateOffb.table[seqState->stateOffb.state].baseValue; - - /* sequence */ - { size_t offset; - if (!ofBits) - offset = 0; - else { - ZSTD_STATIC_ASSERT(ZSTD_lo_isLongOffset == 1); - ZSTD_STATIC_ASSERT(LONG_OFFSETS_MAX_EXTRA_BITS_32 == 5); - assert(ofBits <= MaxOff); - if (MEM_32bits() && longOffsets) { - U32 const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN_32-1); - offset = ofBase + (BIT_readBitsFast(&seqState->DStream, ofBits - extraBits) << extraBits); - if (MEM_32bits() || extraBits) BIT_reloadDStream(&seqState->DStream); - if (extraBits) offset += BIT_readBitsFast(&seqState->DStream, extraBits); - } else { - offset = ofBase + BIT_readBitsFast(&seqState->DStream, ofBits); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */ - if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); - } - } - - if (ofBits <= 1) { - offset += (llBase==0); - if (offset) { - size_t temp = (offset==3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset]; - temp += !temp; /* 0 is not valid; input is corrupted; force offset to 1 */ - if (offset != 1) seqState->prevOffset[2] = seqState->prevOffset[1]; - seqState->prevOffset[1] = seqState->prevOffset[0]; - seqState->prevOffset[0] = offset = temp; - } else { - offset = seqState->prevOffset[0]; - } - } else { - seqState->prevOffset[2] = seqState->prevOffset[1]; - seqState->prevOffset[1] = seqState->prevOffset[0]; - seqState->prevOffset[0] = offset; - } - seq.offset = offset; - } - - seq.matchLength = mlBase + ((mlBits>0) ? BIT_readBitsFast(&seqState->DStream, mlBits) : 0); /* <= 16 bits */ - if (MEM_32bits() && (mlBits+llBits >= STREAM_ACCUMULATOR_MIN_32-LONG_OFFSETS_MAX_EXTRA_BITS_32)) - BIT_reloadDStream(&seqState->DStream); - if (MEM_64bits() && (totalBits >= STREAM_ACCUMULATOR_MIN_64-(LLFSELog+MLFSELog+OffFSELog))) - BIT_reloadDStream(&seqState->DStream); - /* Verify that there is enough bits to read the rest of the data in 64-bit mode. */ - ZSTD_STATIC_ASSERT(16+LLFSELog+MLFSELog+OffFSELog < STREAM_ACCUMULATOR_MIN_64); - - seq.litLength = llBase + ((llBits>0) ? BIT_readBitsFast(&seqState->DStream, llBits) : 0); /* <= 16 bits */ - if (MEM_32bits()) - BIT_reloadDStream(&seqState->DStream); - - { size_t const pos = seqState->pos + seq.litLength; - const BYTE* const matchBase = (seq.offset > pos) ? seqState->dictEnd : seqState->prefixStart; - seq.match = matchBase + pos - seq.offset; /* note : this operation can overflow when seq.offset is really too large, which can only happen when input is corrupted. - * No consequence though : no memory access will occur, overly large offset will be detected in ZSTD_execSequenceLong() */ - seqState->pos = pos + seq.matchLength; - } - - /* ANS state update */ - ZSTD_updateFseState(&seqState->stateLL, &seqState->DStream); /* <= 9 bits */ - ZSTD_updateFseState(&seqState->stateML, &seqState->DStream); /* <= 9 bits */ - if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */ - ZSTD_updateFseState(&seqState->stateOffb, &seqState->DStream); /* <= 8 bits */ - - return seq; -} - FORCE_INLINE_TEMPLATE size_t ZSTD_decompressSequencesLong_body( ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset) + const ZSTD_longOffset_e isLongOffset, + const int frame) { const BYTE* ip = (const BYTE*)seqStart; const BYTE* const iend = ip + seqSize; @@ -1063,6 +1142,7 @@ ZSTD_decompressSequencesLong_body( const BYTE* const prefixStart = (const BYTE*) (dctx->prefixStart); const BYTE* const dictStart = (const BYTE*) (dctx->virtualStart); const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd); + (void)frame; /* Regen sequences */ if (nbSeq) { @@ -1078,36 +1158,45 @@ ZSTD_decompressSequencesLong_body( seqState.prefixStart = prefixStart; seqState.pos = (size_t)(op-prefixStart); seqState.dictEnd = dictEnd; + assert(dst != NULL); assert(iend >= ip); RETURN_ERROR_IF( ERR_isError(BIT_initDStream(&seqState.DStream, ip, iend-ip)), - corruption_detected); + corruption_detected, ""); ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr); ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr); ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); /* prepare in advance */ for (seqNb=0; (BIT_reloadDStream(&seqState.DStream) <= BIT_DStream_completed) && (seqNb<seqAdvance); seqNb++) { - sequences[seqNb] = ZSTD_decodeSequenceLong(&seqState, isLongOffset); + sequences[seqNb] = ZSTD_decodeSequence(&seqState, isLongOffset, ZSTD_p_prefetch); PREFETCH_L1(sequences[seqNb].match); PREFETCH_L1(sequences[seqNb].match + sequences[seqNb].matchLength - 1); /* note : it's safe to invoke PREFETCH() on any memory address, including invalid ones */ } - RETURN_ERROR_IF(seqNb<seqAdvance, corruption_detected); + RETURN_ERROR_IF(seqNb<seqAdvance, corruption_detected, ""); /* decode and decompress */ for ( ; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && (seqNb<nbSeq) ; seqNb++) { - seq_t const sequence = ZSTD_decodeSequenceLong(&seqState, isLongOffset); + seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset, ZSTD_p_prefetch); size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequences[(seqNb-ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litEnd, prefixStart, dictStart, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequences[(seqNb-ADVANCED_SEQS) & STORED_SEQS_MASK], prefixStart, dictStart); +#endif if (ZSTD_isError(oneSeqSize)) return oneSeqSize; PREFETCH_L1(sequence.match); PREFETCH_L1(sequence.match + sequence.matchLength - 1); /* note : it's safe to invoke PREFETCH() on any memory address, including invalid ones */ sequences[seqNb & STORED_SEQS_MASK] = sequence; op += oneSeqSize; } - RETURN_ERROR_IF(seqNb<nbSeq, corruption_detected); + RETURN_ERROR_IF(seqNb<nbSeq, corruption_detected, ""); /* finish queue */ seqNb -= seqAdvance; for ( ; seqNb<nbSeq ; seqNb++) { size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequences[seqNb&STORED_SEQS_MASK], &litPtr, litEnd, prefixStart, dictStart, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequences[seqNb&STORED_SEQS_MASK], prefixStart, dictStart); +#endif if (ZSTD_isError(oneSeqSize)) return oneSeqSize; op += oneSeqSize; } @@ -1118,9 +1207,11 @@ ZSTD_decompressSequencesLong_body( /* last literal segment */ { size_t const lastLLSize = litEnd - litPtr; - RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall); - memcpy(op, litPtr, lastLLSize); - op += lastLLSize; + RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, ""); + if (op != NULL) { + memcpy(op, litPtr, lastLLSize); + op += lastLLSize; + } } return op-ostart; @@ -1130,9 +1221,10 @@ static size_t ZSTD_decompressSequencesLong_default(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset) + const ZSTD_longOffset_e isLongOffset, + const int frame) { - return ZSTD_decompressSequencesLong_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); + return ZSTD_decompressSequencesLong_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); } #endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */ @@ -1146,9 +1238,10 @@ DONT_VECTORIZE ZSTD_decompressSequences_bmi2(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset) + const ZSTD_longOffset_e isLongOffset, + const int frame) { - return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); + return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); } #endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */ @@ -1157,9 +1250,10 @@ static TARGET_ATTRIBUTE("bmi2") size_t ZSTD_decompressSequencesLong_bmi2(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset) + const ZSTD_longOffset_e isLongOffset, + const int frame) { - return ZSTD_decompressSequencesLong_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); + return ZSTD_decompressSequencesLong_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); } #endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */ @@ -1169,21 +1263,23 @@ typedef size_t (*ZSTD_decompressSequences_t)( ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset); + const ZSTD_longOffset_e isLongOffset, + const int frame); #ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG static size_t ZSTD_decompressSequences(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset) + const ZSTD_longOffset_e isLongOffset, + const int frame) { DEBUGLOG(5, "ZSTD_decompressSequences"); #if DYNAMIC_BMI2 if (dctx->bmi2) { - return ZSTD_decompressSequences_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); + return ZSTD_decompressSequences_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); } #endif - return ZSTD_decompressSequences_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); + return ZSTD_decompressSequences_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); } #endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */ @@ -1198,15 +1294,16 @@ static size_t ZSTD_decompressSequencesLong(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset) + const ZSTD_longOffset_e isLongOffset, + const int frame) { DEBUGLOG(5, "ZSTD_decompressSequencesLong"); #if DYNAMIC_BMI2 if (dctx->bmi2) { - return ZSTD_decompressSequencesLong_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); + return ZSTD_decompressSequencesLong_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); } #endif - return ZSTD_decompressSequencesLong_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); + return ZSTD_decompressSequencesLong_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); } #endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */ @@ -1240,7 +1337,6 @@ ZSTD_getLongOffsetsShare(const ZSTD_seqSymbol* offTable) } #endif - size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, @@ -1256,7 +1352,7 @@ ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, ZSTD_longOffset_e const isLongOffset = (ZSTD_longOffset_e)(MEM_32bits() && (!frame || (dctx->fParams.windowSize > (1ULL << STREAM_ACCUMULATOR_MIN)))); DEBUGLOG(5, "ZSTD_decompressBlock_internal (size : %u)", (U32)srcSize); - RETURN_ERROR_IF(srcSize >= ZSTD_BLOCKSIZE_MAX, srcSize_wrong); + RETURN_ERROR_IF(srcSize >= ZSTD_BLOCKSIZE_MAX, srcSize_wrong, ""); /* Decode literals section */ { size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize); @@ -1282,6 +1378,8 @@ ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, ip += seqHSize; srcSize -= seqHSize; + RETURN_ERROR_IF(dst == NULL && nbSeq > 0, dstSize_tooSmall, "NULL not handled"); + #if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \ !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG) if ( !usePrefetchDecoder @@ -1300,17 +1398,28 @@ ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, if (usePrefetchDecoder) #endif #ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT - return ZSTD_decompressSequencesLong(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset); + return ZSTD_decompressSequencesLong(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset, frame); #endif #ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG /* else */ - return ZSTD_decompressSequences(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset); + return ZSTD_decompressSequences(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset, frame); #endif } } +void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst) +{ + if (dst != dctx->previousDstEnd) { /* not contiguous */ + dctx->dictEnd = dctx->previousDstEnd; + dctx->virtualStart = (const char*)dst - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->prefixStart)); + dctx->prefixStart = dst; + dctx->previousDstEnd = dst; + } +} + + size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) diff --git a/thirdparty/zstd/decompress/zstd_decompress_block.h b/thirdparty/zstd/decompress/zstd_decompress_block.h index 7e92960410..bf39b7350c 100644 --- a/thirdparty/zstd/decompress/zstd_decompress_block.h +++ b/thirdparty/zstd/decompress/zstd_decompress_block.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -16,8 +16,8 @@ * Dependencies *********************************************************/ #include <stddef.h> /* size_t */ -#include "zstd.h" /* DCtx, and some public functions */ -#include "zstd_internal.h" /* blockProperties_t, and some public functions */ +#include "../zstd.h" /* DCtx, and some public functions */ +#include "../common/zstd_internal.h" /* blockProperties_t, and some public functions */ #include "zstd_decompress_internal.h" /* ZSTD_seqSymbol */ diff --git a/thirdparty/zstd/decompress/zstd_decompress_internal.h b/thirdparty/zstd/decompress/zstd_decompress_internal.h index ccbdfa090f..9ad96c5548 100644 --- a/thirdparty/zstd/decompress/zstd_decompress_internal.h +++ b/thirdparty/zstd/decompress/zstd_decompress_internal.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -19,8 +19,8 @@ /*-******************************************************* * Dependencies *********************************************************/ -#include "mem.h" /* BYTE, U16, U32 */ -#include "zstd_internal.h" /* ZSTD_seqSymbol */ +#include "../common/mem.h" /* BYTE, U16, U32 */ +#include "../common/zstd_internal.h" /* ZSTD_seqSymbol */ @@ -95,6 +95,11 @@ typedef enum { ZSTD_use_once = 1 /* Use the dictionary once and set to ZSTD_dont_use */ } ZSTD_dictUses_e; +typedef enum { + ZSTD_obm_buffered = 0, /* Buffer the output */ + ZSTD_obm_stable = 1 /* ZSTD_outBuffer is stable */ +} ZSTD_outBufferMode_e; + struct ZSTD_DCtx_s { const ZSTD_seqSymbol* LLTptr; @@ -147,10 +152,19 @@ struct ZSTD_DCtx_s U32 legacyVersion; U32 hostageByte; int noForwardProgress; + ZSTD_outBufferMode_e outBufferMode; + ZSTD_outBuffer expectedOutBuffer; /* workspace */ BYTE litBuffer[ZSTD_BLOCKSIZE_MAX + WILDCOPY_OVERLENGTH]; BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; + + size_t oversizedDuration; + +#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + void const* dictContentBeginForFuzzing; + void const* dictContentEndForFuzzing; +#endif }; /* typedef'd to ZSTD_DCtx within "zstd.h" */ @@ -160,7 +174,7 @@ struct ZSTD_DCtx_s /*! ZSTD_loadDEntropy() : * dict : must point at beginning of a valid zstd dictionary. - * @return : size of entropy tables read */ + * @return : size of dictionary header (size of magic number + dict ID + entropy tables) */ size_t ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy, const void* const dict, size_t const dictSize); diff --git a/thirdparty/zstd/zstd.h b/thirdparty/zstd/zstd.h index 72080ea87e..8c6fc6ae90 100644 --- a/thirdparty/zstd/zstd.h +++ b/thirdparty/zstd/zstd.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -72,7 +72,7 @@ extern "C" { /*------ Version ------*/ #define ZSTD_VERSION_MAJOR 1 #define ZSTD_VERSION_MINOR 4 -#define ZSTD_VERSION_RELEASE 4 +#define ZSTD_VERSION_RELEASE 5 #define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE) ZSTDLIB_API unsigned ZSTD_versionNumber(void); /**< to check runtime library version */ @@ -274,7 +274,10 @@ typedef enum { * Default level is ZSTD_CLEVEL_DEFAULT==3. * Special: value 0 means default, which is controlled by ZSTD_CLEVEL_DEFAULT. * Note 1 : it's possible to pass a negative compression level. - * Note 2 : setting a level resets all other compression parameters to default */ + * Note 2 : setting a level does not automatically set all other compression parameters + * to default. Setting this will however eventually dynamically impact the compression + * parameters which have not been manually set. The manually set + * ones will 'stick'. */ /* Advanced compression parameters : * It's possible to pin down compression parameters to some specific values. * In which case, these values are no longer dynamically selected by the compressor */ @@ -519,11 +522,13 @@ typedef enum { /* note : additional experimental parameters are also available * within the experimental section of the API. * At the time of this writing, they include : - * ZSTD_c_format + * ZSTD_d_format + * ZSTD_d_stableOutBuffer * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. * note : never ever use experimentalParam? names directly */ - ZSTD_d_experimentalParam1=1000 + ZSTD_d_experimentalParam1=1000, + ZSTD_d_experimentalParam2=1001 } ZSTD_dParameter; @@ -763,7 +768,7 @@ ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds); /* This function is redundant with the advanced API and equivalent to: * - * ZSTD_DCtx_reset(zds); + * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); * ZSTD_DCtx_refDDict(zds, NULL); */ ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds); @@ -1263,23 +1268,28 @@ ZSTDLIB_API size_t ZSTD_getSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs, ***************************************/ /*! ZSTD_estimate*() : - * These functions make it possible to estimate memory usage of a future - * {D,C}Ctx, before its creation. + * These functions make it possible to estimate memory usage + * of a future {D,C}Ctx, before its creation. + * + * ZSTD_estimateCCtxSize() will provide a memory budget large enough + * for any compression level up to selected one. + * Note : Unlike ZSTD_estimateCStreamSize*(), this estimate + * does not include space for a window buffer. + * Therefore, the estimation is only guaranteed for single-shot compressions, not streaming. + * The estimate will assume the input may be arbitrarily large, + * which is the worst case. * - * ZSTD_estimateCCtxSize() will provide a budget large enough for any - * compression level up to selected one. Unlike ZSTD_estimateCStreamSize*(), - * this estimate does not include space for a window buffer, so this estimate - * is guaranteed to be enough for single-shot compressions, but not streaming - * compressions. It will however assume the input may be arbitrarily large, - * which is the worst case. If srcSize is known to always be small, - * ZSTD_estimateCCtxSize_usingCParams() can provide a tighter estimation. - * ZSTD_estimateCCtxSize_usingCParams() can be used in tandem with - * ZSTD_getCParams() to create cParams from compressionLevel. - * ZSTD_estimateCCtxSize_usingCCtxParams() can be used in tandem with - * ZSTD_CCtxParams_setParameter(). + * When srcSize can be bound by a known and rather "small" value, + * this fact can be used to provide a tighter estimation + * because the CCtx compression context will need less memory. + * This tighter estimation can be provided by more advanced functions + * ZSTD_estimateCCtxSize_usingCParams(), which can be used in tandem with ZSTD_getCParams(), + * and ZSTD_estimateCCtxSize_usingCCtxParams(), which can be used in tandem with ZSTD_CCtxParams_setParameter(). + * Both can be used to estimate memory using custom compression parameters and arbitrary srcSize limits. * - * Note: only single-threaded compression is supported. This function will - * return an error code if ZSTD_c_nbWorkers is >= 1. */ + * Note 2 : only single-threaded compression is supported. + * ZSTD_estimateCCtxSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1. + */ ZSTDLIB_API size_t ZSTD_estimateCCtxSize(int compressionLevel); ZSTDLIB_API size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams); ZSTDLIB_API size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params); @@ -1642,6 +1652,37 @@ ZSTDLIB_API size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowS * allowing selection between ZSTD_format_e input compression formats */ #define ZSTD_d_format ZSTD_d_experimentalParam1 +/* ZSTD_d_stableOutBuffer + * Experimental parameter. + * Default is 0 == disabled. Set to 1 to enable. + * + * Tells the decompressor that the ZSTD_outBuffer will ALWAYS be the same + * between calls, except for the modifications that zstd makes to pos (the + * caller must not modify pos). This is checked by the decompressor, and + * decompression will fail if it ever changes. Therefore the ZSTD_outBuffer + * MUST be large enough to fit the entire decompressed frame. This will be + * checked when the frame content size is known. The data in the ZSTD_outBuffer + * in the range [dst, dst + pos) MUST not be modified during decompression + * or you will get data corruption. + * + * When this flags is enabled zstd won't allocate an output buffer, because + * it can write directly to the ZSTD_outBuffer, but it will still allocate + * an input buffer large enough to fit any compressed block. This will also + * avoid the memcpy() from the internal output buffer to the ZSTD_outBuffer. + * If you need to avoid the input buffer allocation use the buffer-less + * streaming API. + * + * NOTE: So long as the ZSTD_outBuffer always points to valid memory, using + * this flag is ALWAYS memory safe, and will never access out-of-bounds + * memory. However, decompression WILL fail if you violate the preconditions. + * + * WARNING: The data in the ZSTD_outBuffer in the range [dst, dst + pos) MUST + * not be modified during decompression or you will get data corruption. This + * is because zstd needs to reference data in the ZSTD_outBuffer to regenerate + * matches. Normally zstd maintains its own buffer for this purpose, but passing + * this flag tells zstd to use the user provided buffer. + */ +#define ZSTD_d_stableOutBuffer ZSTD_d_experimentalParam2 /*! ZSTD_DCtx_setFormat() : * Instruct the decoder context about what kind of data to decode next. |
