diff options
158 files changed, 4143 insertions, 1408 deletions
diff --git a/core/io/compression.cpp b/core/io/compression.cpp index a113f3b61b..b51e50150e 100644 --- a/core/io/compression.cpp +++ b/core/io/compression.cpp @@ -175,7 +175,9 @@ int Compression::decompress(uint8_t *p_dst, int p_dst_max_size, const uint8_t *p } break; case MODE_ZSTD: { ZSTD_DCtx *dctx = ZSTD_createDCtx(); - if (zstd_long_distance_matching) ZSTD_DCtx_setMaxWindowSize(dctx, (size_t)1 << zstd_window_log_size); + if (zstd_long_distance_matching) { + ZSTD_DCtx_setParameter(dctx, ZSTD_d_windowLogMax, zstd_window_log_size); + } int ret = ZSTD_decompressDCtx(dctx, p_dst, p_dst_max_size, p_src, p_src_size); ZSTD_freeDCtx(dctx); return ret; @@ -189,4 +191,4 @@ int Compression::zlib_level = Z_DEFAULT_COMPRESSION; int Compression::gzip_level = Z_DEFAULT_COMPRESSION; int Compression::zstd_level = 3; bool Compression::zstd_long_distance_matching = false; -int Compression::zstd_window_log_size = 27; +int Compression::zstd_window_log_size = 27; // ZSTD_WINDOWLOG_LIMIT_DEFAULT diff --git a/core/os/input.cpp b/core/os/input.cpp index caa9fb1493..63bf1db499 100644 --- a/core/os/input.cpp +++ b/core/os/input.cpp @@ -89,6 +89,7 @@ void Input::_bind_methods() { ClassDB::bind_method(D_METHOD("action_press", "action", "strength"), &Input::action_press, DEFVAL(1.f)); ClassDB::bind_method(D_METHOD("action_release", "action"), &Input::action_release); ClassDB::bind_method(D_METHOD("set_default_cursor_shape", "shape"), &Input::set_default_cursor_shape, DEFVAL(CURSOR_ARROW)); + ClassDB::bind_method(D_METHOD("get_current_cursor_shape"), &Input::get_current_cursor_shape); ClassDB::bind_method(D_METHOD("set_custom_mouse_cursor", "image", "shape", "hotspot"), &Input::set_custom_mouse_cursor, DEFVAL(CURSOR_ARROW), DEFVAL(Vector2())); ClassDB::bind_method(D_METHOD("parse_input_event", "event"), &Input::parse_input_event); ClassDB::bind_method(D_METHOD("set_use_accumulated_input", "enable"), &Input::set_use_accumulated_input); diff --git a/core/os/input.h b/core/os/input.h index c8b80b28d0..de04f239e6 100644 --- a/core/os/input.h +++ b/core/os/input.h @@ -121,10 +121,10 @@ public: virtual bool is_emulating_touch_from_mouse() const = 0; virtual bool is_emulating_mouse_from_touch() const = 0; - virtual CursorShape get_default_cursor_shape() = 0; + virtual CursorShape get_default_cursor_shape() const = 0; virtual void set_default_cursor_shape(CursorShape p_shape) = 0; + virtual CursorShape get_current_cursor_shape() const = 0; virtual void set_custom_mouse_cursor(const RES &p_cursor, CursorShape p_shape = CURSOR_ARROW, const Vector2 &p_hotspot = Vector2()) = 0; - virtual void set_mouse_in_window(bool p_in_window) = 0; virtual String get_joy_button_string(int p_button) = 0; virtual String get_joy_axis_string(int p_axis) = 0; diff --git a/core/os/os.cpp b/core/os/os.cpp index 03e63f636e..ea378c9e83 100644 --- a/core/os/os.cpp +++ b/core/os/os.cpp @@ -220,6 +220,16 @@ int OS::get_virtual_keyboard_height() const { return 0; } +void OS::set_cursor_shape(CursorShape p_shape) { +} + +OS::CursorShape OS::get_cursor_shape() const { + return CURSOR_ARROW; +} + +void OS::set_custom_mouse_cursor(const RES &p_cursor, CursorShape p_shape, const Vector2 &p_hotspot) { +} + void OS::print_all_resources(String p_to_file) { ERR_FAIL_COND(p_to_file != "" && _OSPRF); diff --git a/core/os/os.h b/core/os/os.h index d02d5a2c84..12012fba80 100644 --- a/core/os/os.h +++ b/core/os/os.h @@ -377,8 +377,9 @@ public: // returns height of the currently shown virtual keyboard (0 if keyboard is hidden) virtual int get_virtual_keyboard_height() const; - virtual void set_cursor_shape(CursorShape p_shape) = 0; - virtual void set_custom_mouse_cursor(const RES &p_cursor, CursorShape p_shape, const Vector2 &p_hotspot) = 0; + virtual void set_cursor_shape(CursorShape p_shape); + virtual CursorShape get_cursor_shape() const; + virtual void set_custom_mouse_cursor(const RES &p_cursor, CursorShape p_shape, const Vector2 &p_hotspot); virtual bool get_swap_ok_cancel() { return false; } virtual void dump_memory_to_file(const char *p_file); diff --git a/core/project_settings.cpp b/core/project_settings.cpp index 5752cdca2b..c86d1567dd 100644 --- a/core/project_settings.cpp +++ b/core/project_settings.cpp @@ -501,7 +501,7 @@ Error ProjectSettings::_load_settings_binary(const String p_path) { d.resize(vlen); f->get_buffer(d.ptrw(), vlen); Variant value; - err = decode_variant(value, d.ptr(), d.size(), NULL, false); + err = decode_variant(value, d.ptr(), d.size(), NULL, true); ERR_EXPLAIN("Error decoding property: " + key); ERR_CONTINUE(err != OK); set(key, value); @@ -694,7 +694,7 @@ Error ProjectSettings::_save_settings_binary(const String &p_file, const Map<Str file->store_string(key); int len; - err = encode_variant(value, NULL, len, false); + err = encode_variant(value, NULL, len, true); if (err != OK) memdelete(file); ERR_FAIL_COND_V(err != OK, ERR_INVALID_DATA); @@ -702,7 +702,7 @@ Error ProjectSettings::_save_settings_binary(const String &p_file, const Map<Str Vector<uint8_t> buff; buff.resize(len); - err = encode_variant(value, buff.ptrw(), len, false); + err = encode_variant(value, buff.ptrw(), len, true); if (err != OK) memdelete(file); ERR_FAIL_COND_V(err != OK, ERR_INVALID_DATA); diff --git a/doc/classes/AnimatedSprite.xml b/doc/classes/AnimatedSprite.xml index 0ee1c75348..555169545e 100644 --- a/doc/classes/AnimatedSprite.xml +++ b/doc/classes/AnimatedSprite.xml @@ -23,6 +23,8 @@ </return> <argument index="0" name="anim" type="String" default=""""> </argument> + <argument index="1" name="backwards" type="bool" default="false"> + </argument> <description> Play the animation set in parameter. If no parameter is provided, the current animation is played. Property [code]backwards[/code] plays the animation in reverse if set to [code]true[/code]. </description> diff --git a/doc/classes/AnimationNodeBlendSpace2D.xml b/doc/classes/AnimationNodeBlendSpace2D.xml index ad7d9dd380..449c177924 100644 --- a/doc/classes/AnimationNodeBlendSpace2D.xml +++ b/doc/classes/AnimationNodeBlendSpace2D.xml @@ -136,8 +136,10 @@ <constant name="BLEND_MODE_INTERPOLATED" value="0" enum="BlendMode"> </constant> <constant name="BLEND_MODE_DISCRETE" value="1" enum="BlendMode"> + Useful for frame-by-frame 2D animations. </constant> <constant name="BLEND_MODE_DISCRETE_CARRY" value="2" enum="BlendMode"> + Keep the current play position when switching between discrete animations. </constant> </constants> </class> diff --git a/doc/classes/AnimationNodeStateMachine.xml b/doc/classes/AnimationNodeStateMachine.xml index 6de544c9ac..348bd9eda1 100644 --- a/doc/classes/AnimationNodeStateMachine.xml +++ b/doc/classes/AnimationNodeStateMachine.xml @@ -3,6 +3,11 @@ <brief_description> </brief_description> <description> + Contains multiple root nodes as children in a graph. Each node is used as a state, and provides multiple functions to alternate between states. Retrieve the AnimationNodeStateMachinePlayback object from the [AnimationTree] node to control it programatically. + [codeblock] + var state_machine = anim_tree["parameters/StateMachine/playback"] + state_machine.travel("SomeState") + [codeblock] </description> <tutorials> </tutorials> @@ -170,7 +175,7 @@ <method name="set_graph_offset"> <return type="void"> </return> - <argument index="0" name="name" type="Vector2"> + <argument index="0" name="offset" type="Vector2"> </argument> <description> </description> diff --git a/doc/classes/AnimationNodeStateMachinePlayback.xml b/doc/classes/AnimationNodeStateMachinePlayback.xml index 01bf982281..70a9fed787 100644 --- a/doc/classes/AnimationNodeStateMachinePlayback.xml +++ b/doc/classes/AnimationNodeStateMachinePlayback.xml @@ -47,6 +47,7 @@ <argument index="0" name="to_node" type="String"> </argument> <description> + Transition from the current state to another one, while visiting all the intermediate ones. This is done via the A* algorithm. </description> </method> </methods> diff --git a/doc/classes/AnimationNodeStateMachineTransition.xml b/doc/classes/AnimationNodeStateMachineTransition.xml index a1ee381882..d18d5ee630 100644 --- a/doc/classes/AnimationNodeStateMachineTransition.xml +++ b/doc/classes/AnimationNodeStateMachineTransition.xml @@ -12,16 +12,22 @@ </methods> <members> <member name="advance_condition" type="String" setter="set_advance_condition" getter="get_advance_condition"> + Turn on auto advance when this condition is set. This is a custom text field that can be filled with a variable name. The variable can be modified from code. </member> <member name="auto_advance" type="bool" setter="set_auto_advance" getter="has_auto_advance"> + Turn on the transition automatically when this state is reached. This works best with [code]SWITCH_MODE_AT_END[/code]. </member> <member name="disabled" type="bool" setter="set_disabled" getter="is_disabled"> + Don't use this transition during [method AnimationNodeStateMachinePlayback.travel] or [member auto_advance]. </member> <member name="priority" type="int" setter="set_priority" getter="get_priority"> + Lower priority transitions are preferred when travelling through the tree via [method AnimationNodeStateMachinePlayback.travel] or [member auto_advance]. </member> <member name="switch_mode" type="int" setter="set_switch_mode" getter="get_switch_mode" enum="AnimationNodeStateMachineTransition.SwitchMode"> + The transition type. </member> <member name="xfade_time" type="float" setter="set_xfade_time" getter="get_xfade_time"> + The time to cross-fade between this state and the next. </member> </members> <signals> @@ -32,10 +38,13 @@ </signals> <constants> <constant name="SWITCH_MODE_IMMEDIATE" value="0" enum="SwitchMode"> + Switch to the next state immediately. The current state will end and blend into the beginning of the new one. </constant> <constant name="SWITCH_MODE_SYNC" value="1" enum="SwitchMode"> + Switch to the next state immediately, but will seek the new state to the playback position of the old state. </constant> <constant name="SWITCH_MODE_AT_END" value="2" enum="SwitchMode"> + Wait for the current state playback to end, then switch to the beginning of the next state animation. </constant> </constants> </class> diff --git a/doc/classes/AnimationTree.xml b/doc/classes/AnimationTree.xml index 1101c0ee62..51812c6b02 100644 --- a/doc/classes/AnimationTree.xml +++ b/doc/classes/AnimationTree.xml @@ -5,8 +5,10 @@ <description> </description> <tutorials> + <link>https://docs.godotengine.org/en/latest/tutorials/animation/animation_tree.html</link> </tutorials> <demos> + <link>https://github.com/godotengine/tps-demo</link> </demos> <methods> <method name="advance"> diff --git a/doc/classes/AudioEffectInstance.xml b/doc/classes/AudioEffectInstance.xml new file mode 100644 index 0000000000..ed42928eb1 --- /dev/null +++ b/doc/classes/AudioEffectInstance.xml @@ -0,0 +1,15 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<class name="AudioEffectInstance" inherits="Reference" category="Core" version="3.2"> + <brief_description> + </brief_description> + <description> + </description> + <tutorials> + </tutorials> + <demos> + </demos> + <methods> + </methods> + <constants> + </constants> +</class> diff --git a/doc/classes/AudioEffectSpectrumAnalyzer.xml b/doc/classes/AudioEffectSpectrumAnalyzer.xml new file mode 100644 index 0000000000..5732039467 --- /dev/null +++ b/doc/classes/AudioEffectSpectrumAnalyzer.xml @@ -0,0 +1,23 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<class name="AudioEffectSpectrumAnalyzer" inherits="AudioEffect" category="Core" version="3.2"> + <brief_description> + </brief_description> + <description> + </description> + <tutorials> + </tutorials> + <demos> + </demos> + <methods> + </methods> + <members> + <member name="buffer_length" type="float" setter="set_buffer_length" getter="get_buffer_length"> + </member> + <member name="fft_size" type="int" setter="set_fft_size" getter="get_fft_size" enum="AudioEffectSpectrumAnalyzer.FFT_Size"> + </member> + <member name="tap_back_pos" type="float" setter="set_tap_back_pos" getter="get_tap_back_pos"> + </member> + </members> + <constants> + </constants> +</class> diff --git a/doc/classes/AudioEffectSpectrumAnalyzerInstance.xml b/doc/classes/AudioEffectSpectrumAnalyzerInstance.xml new file mode 100644 index 0000000000..0c0061beb4 --- /dev/null +++ b/doc/classes/AudioEffectSpectrumAnalyzerInstance.xml @@ -0,0 +1,31 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<class name="AudioEffectSpectrumAnalyzerInstance" inherits="AudioEffectInstance" category="Core" version="3.2"> + <brief_description> + </brief_description> + <description> + </description> + <tutorials> + </tutorials> + <demos> + </demos> + <methods> + <method name="get_magnitude_for_frequency_range" qualifiers="const"> + <return type="Vector2"> + </return> + <argument index="0" name="from_hz" type="float"> + </argument> + <argument index="1" name="to_hz" type="float"> + </argument> + <argument index="2" name="mode" type="int" enum="AudioEffectSpectrumAnalyzerInstance.MagnitudeMode" default="1"> + </argument> + <description> + </description> + </method> + </methods> + <constants> + <constant name="MAGNITUDE_AVERAGE" value="0" enum="MagnitudeMode"> + </constant> + <constant name="MAGNITUDE_MAX" value="1" enum="MagnitudeMode"> + </constant> + </constants> +</class> diff --git a/doc/classes/AudioServer.xml b/doc/classes/AudioServer.xml index 82e17e4957..32eed54882 100644 --- a/doc/classes/AudioServer.xml +++ b/doc/classes/AudioServer.xml @@ -97,6 +97,18 @@ Returns the number of effects on the bus at [code]bus_idx[/code]. </description> </method> + <method name="get_bus_effect_instance"> + <return type="AudioEffectInstance"> + </return> + <argument index="0" name="bus_idx" type="int"> + </argument> + <argument index="1" name="effect_idx" type="int"> + </argument> + <argument index="2" name="channel" type="int" default="0"> + </argument> + <description> + </description> + </method> <method name="get_bus_index" qualifiers="const"> <return type="int"> </return> diff --git a/doc/classes/AudioStreamGenerator.xml b/doc/classes/AudioStreamGenerator.xml new file mode 100644 index 0000000000..e41b5827b1 --- /dev/null +++ b/doc/classes/AudioStreamGenerator.xml @@ -0,0 +1,21 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<class name="AudioStreamGenerator" inherits="AudioStream" category="Core" version="3.2"> + <brief_description> + </brief_description> + <description> + </description> + <tutorials> + </tutorials> + <demos> + </demos> + <methods> + </methods> + <members> + <member name="buffer_length" type="float" setter="set_buffer_length" getter="get_buffer_length"> + </member> + <member name="mix_rate" type="float" setter="set_mix_rate" getter="get_mix_rate"> + </member> + </members> + <constants> + </constants> +</class> diff --git a/doc/classes/AudioStreamGeneratorPlayback.xml b/doc/classes/AudioStreamGeneratorPlayback.xml new file mode 100644 index 0000000000..4b4d622741 --- /dev/null +++ b/doc/classes/AudioStreamGeneratorPlayback.xml @@ -0,0 +1,57 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<class name="AudioStreamGeneratorPlayback" inherits="AudioStreamPlaybackResampled" category="Core" version="3.2"> + <brief_description> + </brief_description> + <description> + </description> + <tutorials> + </tutorials> + <demos> + </demos> + <methods> + <method name="can_push_buffer" qualifiers="const"> + <return type="bool"> + </return> + <argument index="0" name="amount" type="int"> + </argument> + <description> + </description> + </method> + <method name="clear_buffer"> + <return type="void"> + </return> + <description> + </description> + </method> + <method name="get_frames_available" qualifiers="const"> + <return type="int"> + </return> + <description> + </description> + </method> + <method name="get_skips" qualifiers="const"> + <return type="int"> + </return> + <description> + </description> + </method> + <method name="push_buffer"> + <return type="bool"> + </return> + <argument index="0" name="frames" type="PoolVector2Array"> + </argument> + <description> + </description> + </method> + <method name="push_frame"> + <return type="bool"> + </return> + <argument index="0" name="frame" type="Vector2"> + </argument> + <description> + </description> + </method> + </methods> + <constants> + </constants> +</class> diff --git a/doc/classes/AudioStreamPlaybackResampled.xml b/doc/classes/AudioStreamPlaybackResampled.xml new file mode 100644 index 0000000000..764f966167 --- /dev/null +++ b/doc/classes/AudioStreamPlaybackResampled.xml @@ -0,0 +1,15 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<class name="AudioStreamPlaybackResampled" inherits="AudioStreamPlayback" category="Core" version="3.2"> + <brief_description> + </brief_description> + <description> + </description> + <tutorials> + </tutorials> + <demos> + </demos> + <methods> + </methods> + <constants> + </constants> +</class> diff --git a/doc/classes/AudioStreamPlayer.xml b/doc/classes/AudioStreamPlayer.xml index d8b87a3c19..1681d796ea 100644 --- a/doc/classes/AudioStreamPlayer.xml +++ b/doc/classes/AudioStreamPlayer.xml @@ -19,6 +19,12 @@ Returns the position in the [AudioStream] in seconds. </description> </method> + <method name="get_stream_playback"> + <return type="AudioStreamPlayback"> + </return> + <description> + </description> + </method> <method name="play"> <return type="void"> </return> diff --git a/doc/classes/AudioStreamPlayer2D.xml b/doc/classes/AudioStreamPlayer2D.xml index 1cd2bf7645..e001d11727 100644 --- a/doc/classes/AudioStreamPlayer2D.xml +++ b/doc/classes/AudioStreamPlayer2D.xml @@ -19,6 +19,12 @@ Returns the position in the [AudioStream]. </description> </method> + <method name="get_stream_playback"> + <return type="AudioStreamPlayback"> + </return> + <description> + </description> + </method> <method name="play"> <return type="void"> </return> diff --git a/doc/classes/AudioStreamPlayer3D.xml b/doc/classes/AudioStreamPlayer3D.xml index 8ff5223e4f..0d973a761a 100644 --- a/doc/classes/AudioStreamPlayer3D.xml +++ b/doc/classes/AudioStreamPlayer3D.xml @@ -19,6 +19,12 @@ Returns the position in the [AudioStream]. </description> </method> + <method name="get_stream_playback"> + <return type="AudioStreamPlayback"> + </return> + <description> + </description> + </method> <method name="play"> <return type="void"> </return> diff --git a/doc/classes/Basis.xml b/doc/classes/Basis.xml index 7b43d58c71..3710bc3de6 100644 --- a/doc/classes/Basis.xml +++ b/doc/classes/Basis.xml @@ -76,6 +76,12 @@ This function considers a discretization of rotations into 24 points on unit sphere, lying along the vectors (x,y,z) with each component being either -1,0 or 1, and returns the index of the point best representing the orientation of the object. It is mainly used by the grid map editor. For further details, refer to Godot source code. </description> </method> + <method name="get_rotation_quat"> + <return type="Quat"> + </return> + <description> + </description> + </method> <method name="get_scale"> <return type="Vector3"> </return> @@ -90,6 +96,16 @@ Return the inverse of the matrix. </description> </method> + <method name="is_equal_approx"> + <return type="bool"> + </return> + <argument index="0" name="b" type="Basis"> + </argument> + <argument index="1" name="epsilon" type="float" default="0.00001"> + </argument> + <description> + </description> + </method> <method name="orthonormalized"> <return type="Basis"> </return> diff --git a/doc/classes/CanvasItem.xml b/doc/classes/CanvasItem.xml index c1cf7dcabe..61ccc167a5 100644 --- a/doc/classes/CanvasItem.xml +++ b/doc/classes/CanvasItem.xml @@ -575,7 +575,7 @@ <constant name="BLEND_MODE_DISABLED" value="5" enum="BlendMode"> Disable blending mode. Colors including alpha are written as is. Only applicable for render targets with a transparent background. No lighting will be applied. </constant> - <constant name="NOTIFICATION_TRANSFORM_CHANGED" value="29"> + <constant name="NOTIFICATION_TRANSFORM_CHANGED" value="2000"> Canvas item transform has changed. Notification is only received if enabled by [method set_notify_transform] or [method set_notify_local_transform]. </constant> <constant name="NOTIFICATION_DRAW" value="30"> diff --git a/doc/classes/CanvasLayer.xml b/doc/classes/CanvasLayer.xml index 3d69c92ea3..04aa7fbbf2 100644 --- a/doc/classes/CanvasLayer.xml +++ b/doc/classes/CanvasLayer.xml @@ -25,6 +25,10 @@ <member name="custom_viewport" type="Node" setter="set_custom_viewport" getter="get_custom_viewport"> The custom [Viewport] node assigned to the [CanvasLayer]. If null, uses the default viewport instead. </member> + <member name="follow_viewport_enable" type="bool" setter="set_follow_viewport" getter="is_following_viewport"> + </member> + <member name="follow_viewport_scale" type="float" setter="set_follow_viewport_scale" getter="get_follow_viewport_scale"> + </member> <member name="layer" type="int" setter="set_layer" getter="get_layer"> Layer index for draw order. Lower values are drawn first. Default value: [code]1[/code]. </member> diff --git a/doc/classes/EditorFeatureProfile.xml b/doc/classes/EditorFeatureProfile.xml new file mode 100644 index 0000000000..e0e56dc009 --- /dev/null +++ b/doc/classes/EditorFeatureProfile.xml @@ -0,0 +1,131 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<class name="EditorFeatureProfile" inherits="Reference" category="Core" version="3.2"> + <brief_description> + </brief_description> + <description> + </description> + <tutorials> + </tutorials> + <demos> + </demos> + <methods> + <method name="get_feature_name"> + <return type="String"> + </return> + <argument index="0" name="feature" type="int" enum="EditorFeatureProfile.Feature"> + </argument> + <description> + </description> + </method> + <method name="is_class_disabled" qualifiers="const"> + <return type="bool"> + </return> + <argument index="0" name="class_name" type="String"> + </argument> + <description> + </description> + </method> + <method name="is_class_editor_disabled" qualifiers="const"> + <return type="bool"> + </return> + <argument index="0" name="class_name" type="String"> + </argument> + <description> + </description> + </method> + <method name="is_class_property_disabled" qualifiers="const"> + <return type="bool"> + </return> + <argument index="0" name="class_name" type="String"> + </argument> + <argument index="1" name="arg1" type="String"> + </argument> + <description> + </description> + </method> + <method name="is_feature_disabled" qualifiers="const"> + <return type="bool"> + </return> + <argument index="0" name="feature" type="int" enum="EditorFeatureProfile.Feature"> + </argument> + <description> + </description> + </method> + <method name="load_from_file"> + <return type="int" enum="Error"> + </return> + <argument index="0" name="path" type="String"> + </argument> + <description> + </description> + </method> + <method name="save_to_file"> + <return type="int" enum="Error"> + </return> + <argument index="0" name="path" type="String"> + </argument> + <description> + </description> + </method> + <method name="set_disable_class"> + <return type="void"> + </return> + <argument index="0" name="class_name" type="String"> + </argument> + <argument index="1" name="disable" type="bool"> + </argument> + <description> + </description> + </method> + <method name="set_disable_class_editor"> + <return type="void"> + </return> + <argument index="0" name="class_name" type="String"> + </argument> + <argument index="1" name="disable" type="bool"> + </argument> + <description> + </description> + </method> + <method name="set_disable_class_property"> + <return type="void"> + </return> + <argument index="0" name="class_name" type="String"> + </argument> + <argument index="1" name="property" type="String"> + </argument> + <argument index="2" name="arg2" type="bool"> + </argument> + <description> + </description> + </method> + <method name="set_disable_feature"> + <return type="void"> + </return> + <argument index="0" name="feature" type="int" enum="EditorFeatureProfile.Feature"> + </argument> + <argument index="1" name="disable" type="bool"> + </argument> + <description> + </description> + </method> + </methods> + <constants> + <constant name="FEATURE_3D" value="0" enum="Feature"> + </constant> + <constant name="FEATURE_SCRIPT" value="1" enum="Feature"> + </constant> + <constant name="FEATURE_ASSET_LIB" value="2" enum="Feature"> + </constant> + <constant name="FEATURE_SCENE_TREE" value="3" enum="Feature"> + </constant> + <constant name="FEATURE_IMPORT_DOCK" value="4" enum="Feature"> + </constant> + <constant name="FEATURE_NODE_DOCK" value="5" enum="Feature"> + </constant> + <constant name="FEATURE_FILESYSTEM_DOCK" value="6" enum="Feature"> + </constant> + <constant name="FEATURE_MAX" value="7" enum="Feature"> + </constant> + </constants> +</class> diff --git a/doc/classes/EditorInterface.xml b/doc/classes/EditorInterface.xml index 30ff609a81..2dc8d716a7 100644 --- a/doc/classes/EditorInterface.xml +++ b/doc/classes/EditorInterface.xml @@ -48,6 +48,12 @@ Returns the editor [Viewport]. </description> </method> + <method name="get_inspector" qualifiers="const"> + <return type="EditorInspector"> + </return> + <description> + </description> + </method> <method name="get_open_scenes" qualifiers="const"> <return type="Array"> </return> diff --git a/doc/classes/File.xml b/doc/classes/File.xml index 8a785ab263..79462816e3 100644 --- a/doc/classes/File.xml +++ b/doc/classes/File.xml @@ -227,7 +227,7 @@ </return> <argument index="0" name="path" type="String"> </argument> - <argument index="1" name="flags" type="int"> + <argument index="1" name="flags" type="int" enum="File.ModeFlags"> </argument> <description> Opens the file for writing or reading, depending on the flags. @@ -238,9 +238,9 @@ </return> <argument index="0" name="path" type="String"> </argument> - <argument index="1" name="mode_flags" type="int"> + <argument index="1" name="mode_flags" type="int" enum="File.ModeFlags"> </argument> - <argument index="2" name="compression_mode" type="int" default="0"> + <argument index="2" name="compression_mode" type="int" enum="File.CompressionMode" default="0"> </argument> <description> Opens a compressed file for reading or writing. Use COMPRESSION_* constants to set [code]compression_mode[/code]. @@ -251,7 +251,7 @@ </return> <argument index="0" name="path" type="String"> </argument> - <argument index="1" name="mode_flags" type="int"> + <argument index="1" name="mode_flags" type="int" enum="File.ModeFlags"> </argument> <argument index="2" name="key" type="PoolByteArray"> </argument> @@ -264,7 +264,7 @@ </return> <argument index="0" name="path" type="String"> </argument> - <argument index="1" name="mode_flags" type="int"> + <argument index="1" name="mode_flags" type="int" enum="File.ModeFlags"> </argument> <argument index="2" name="pass" type="String"> </argument> diff --git a/doc/classes/MainLoop.xml b/doc/classes/MainLoop.xml index 8d8d4c46cc..6b7d9cc834 100644 --- a/doc/classes/MainLoop.xml +++ b/doc/classes/MainLoop.xml @@ -114,29 +114,29 @@ </method> </methods> <constants> - <constant name="NOTIFICATION_WM_MOUSE_ENTER" value="2"> + <constant name="NOTIFICATION_WM_MOUSE_ENTER" value="1002"> </constant> - <constant name="NOTIFICATION_WM_MOUSE_EXIT" value="3"> + <constant name="NOTIFICATION_WM_MOUSE_EXIT" value="1003"> </constant> - <constant name="NOTIFICATION_WM_FOCUS_IN" value="4"> + <constant name="NOTIFICATION_WM_FOCUS_IN" value="1004"> </constant> - <constant name="NOTIFICATION_WM_FOCUS_OUT" value="5"> + <constant name="NOTIFICATION_WM_FOCUS_OUT" value="1005"> </constant> - <constant name="NOTIFICATION_WM_QUIT_REQUEST" value="6"> + <constant name="NOTIFICATION_WM_QUIT_REQUEST" value="1006"> </constant> - <constant name="NOTIFICATION_WM_GO_BACK_REQUEST" value="7"> + <constant name="NOTIFICATION_WM_GO_BACK_REQUEST" value="1007"> </constant> - <constant name="NOTIFICATION_WM_UNFOCUS_REQUEST" value="8"> + <constant name="NOTIFICATION_WM_UNFOCUS_REQUEST" value="1008"> </constant> - <constant name="NOTIFICATION_OS_MEMORY_WARNING" value="9"> + <constant name="NOTIFICATION_OS_MEMORY_WARNING" value="1009"> </constant> - <constant name="NOTIFICATION_TRANSLATION_CHANGED" value="90"> + <constant name="NOTIFICATION_TRANSLATION_CHANGED" value="1010"> </constant> - <constant name="NOTIFICATION_WM_ABOUT" value="91"> + <constant name="NOTIFICATION_WM_ABOUT" value="1011"> </constant> - <constant name="NOTIFICATION_CRASH" value="92"> + <constant name="NOTIFICATION_CRASH" value="1012"> </constant> - <constant name="NOTIFICATION_OS_IME_UPDATE" value="93"> + <constant name="NOTIFICATION_OS_IME_UPDATE" value="1013"> </constant> </constants> </class> diff --git a/doc/classes/MeshLibrary.xml b/doc/classes/MeshLibrary.xml index 8fb10696f2..02d87a9c48 100644 --- a/doc/classes/MeshLibrary.xml +++ b/doc/classes/MeshLibrary.xml @@ -68,6 +68,14 @@ <description> </description> </method> + <method name="get_item_navmesh_transform" qualifiers="const"> + <return type="Transform"> + </return> + <argument index="0" name="id" type="int"> + </argument> + <description> + </description> + </method> <method name="get_item_preview" qualifiers="const"> <return type="Texture"> </return> @@ -132,6 +140,16 @@ <description> </description> </method> + <method name="set_item_navmesh_transform"> + <return type="void"> + </return> + <argument index="0" name="id" type="int"> + </argument> + <argument index="1" name="navmesh" type="Transform"> + </argument> + <description> + </description> + </method> <method name="set_item_preview"> <return type="void"> </return> diff --git a/doc/classes/Node.xml b/doc/classes/Node.xml index bf32d4041a..646b921743 100644 --- a/doc/classes/Node.xml +++ b/doc/classes/Node.xml @@ -886,15 +886,37 @@ <constant name="NOTIFICATION_PATH_CHANGED" value="23"> Notification received when the node's [NodePath] changed. </constant> - <constant name="NOTIFICATION_TRANSLATION_CHANGED" value="24"> - Notification received when translations may have changed. Can be triggered by the user changing the locale. Can be used to respond to language changes, for example to change the UI strings on the fly. Useful when working with the built-in translation support, like [method Object.tr]. - </constant> <constant name="NOTIFICATION_INTERNAL_PROCESS" value="25"> Notification received every frame when the internal process flag is set (see [method set_process_internal]). </constant> <constant name="NOTIFICATION_INTERNAL_PHYSICS_PROCESS" value="26"> Notification received every frame when the internal physics process flag is set (see [method set_physics_process_internal]). </constant> + <constant name="NOTIFICATION_WM_MOUSE_ENTER" value="1002"> + </constant> + <constant name="NOTIFICATION_WM_MOUSE_EXIT" value="1003"> + </constant> + <constant name="NOTIFICATION_WM_FOCUS_IN" value="1004"> + </constant> + <constant name="NOTIFICATION_WM_FOCUS_OUT" value="1005"> + </constant> + <constant name="NOTIFICATION_WM_QUIT_REQUEST" value="1006"> + </constant> + <constant name="NOTIFICATION_WM_GO_BACK_REQUEST" value="1007"> + </constant> + <constant name="NOTIFICATION_WM_UNFOCUS_REQUEST" value="1008"> + </constant> + <constant name="NOTIFICATION_OS_MEMORY_WARNING" value="1009"> + </constant> + <constant name="NOTIFICATION_TRANSLATION_CHANGED" value="1010"> + Notification received when translations may have changed. Can be triggered by the user changing the locale. Can be used to respond to language changes, for example to change the UI strings on the fly. Useful when working with the built-in translation support, like [method Object.tr]. + </constant> + <constant name="NOTIFICATION_WM_ABOUT" value="1011"> + </constant> + <constant name="NOTIFICATION_CRASH" value="1012"> + </constant> + <constant name="NOTIFICATION_OS_IME_UPDATE" value="1013"> + </constant> <constant name="PAUSE_MODE_INHERIT" value="0" enum="PauseMode"> Inherits pause mode from the node's parent. For the root node, it is equivalent to PAUSE_MODE_STOP. Default. </constant> diff --git a/doc/classes/ProjectSettings.xml b/doc/classes/ProjectSettings.xml index e67e3c9a03..ee3f3260d7 100644 --- a/doc/classes/ProjectSettings.xml +++ b/doc/classes/ProjectSettings.xml @@ -215,6 +215,8 @@ <member name="audio/channel_disable_time" type="float" setter="" getter=""> Audio buses will disable automatically when sound goes below a given DB threshold for a given time. This saves CPU as effects assigned to that bus will no longer do any processing. </member> + <member name="audio/default_bus_layout" type="String" setter="" getter=""> + </member> <member name="audio/driver" type="String" setter="" getter=""> </member> <member name="audio/enable_audio_input" type="bool" setter="" getter=""> diff --git a/doc/classes/ResourceSaver.xml b/doc/classes/ResourceSaver.xml index 165ab0dc95..079d1388f0 100644 --- a/doc/classes/ResourceSaver.xml +++ b/doc/classes/ResourceSaver.xml @@ -27,7 +27,7 @@ </argument> <argument index="1" name="resource" type="Resource"> </argument> - <argument index="2" name="flags" type="int" default="0"> + <argument index="2" name="flags" type="int" enum="ResourceSaver.SaverFlags" default="0"> </argument> <description> Saves a resource to disk. diff --git a/doc/classes/Slider.xml b/doc/classes/Slider.xml index 1e665f0236..98be77641d 100644 --- a/doc/classes/Slider.xml +++ b/doc/classes/Slider.xml @@ -14,14 +14,18 @@ </methods> <members> <member name="editable" type="bool" setter="set_editable" getter="is_editable"> + If [code]true[/code], the slider can be interacted with. If [code]false[/code], the value can be changed only by code. </member> <member name="focus_mode" type="int" setter="set_focus_mode" getter="get_focus_mode" enum="Control.FocusMode"> </member> <member name="scrollable" type="bool" setter="set_scrollable" getter="is_scrollable"> + If [code]true[/code], the value can be changed using the mouse wheel. </member> <member name="tick_count" type="int" setter="set_ticks" getter="get_ticks"> + Number of ticks displayed on the slider, including border ticks. Ticks are uniformly-distributed value markers. </member> <member name="ticks_on_borders" type="bool" setter="set_ticks_on_borders" getter="get_ticks_on_borders"> + If [code]true[/code], the slider will display ticks for minimum and maximum values. </member> </members> <constants> diff --git a/doc/classes/Spatial.xml b/doc/classes/Spatial.xml index 7e42e06960..f28ddcf351 100644 --- a/doc/classes/Spatial.xml +++ b/doc/classes/Spatial.xml @@ -329,7 +329,7 @@ </signal> </signals> <constants> - <constant name="NOTIFICATION_TRANSFORM_CHANGED" value="29"> + <constant name="NOTIFICATION_TRANSFORM_CHANGED" value="2000"> Spatial nodes receives this notification when their global transform changes. This means that either the current or a parent node changed its transform. In order for [code]NOTIFICATION_TRANSFORM_CHANGED[/code] to work, users first need to ask for it, with [method set_notify_transform]. </constant> diff --git a/doc/classes/String.xml b/doc/classes/String.xml index 0cfa246275..67a9b80602 100644 --- a/doc/classes/String.xml +++ b/doc/classes/String.xml @@ -437,7 +437,7 @@ </return> <description> Returns [code]true[/code] if this string is free from characters that aren't allowed in file names, those being: - [code]: / \ ? * " | % < >[/code] + [code]: / \ ? * " | % < >[/code] </description> </method> <method name="is_valid_float"> diff --git a/doc/classes/StyleBoxFlat.xml b/doc/classes/StyleBoxFlat.xml index d74f481f55..34f99755a4 100644 --- a/doc/classes/StyleBoxFlat.xml +++ b/doc/classes/StyleBoxFlat.xml @@ -148,12 +148,12 @@ <member name="shadow_color" type="Color" setter="set_shadow_color" getter="get_shadow_color"> The color of the shadow. (This has no effect when shadow_size < 1) </member> - <member name="shadow_size" type="int" setter="set_shadow_size" getter="get_shadow_size"> - The shadow size in pixels. - </member> <member name="shadow_offset" type="Vector2" setter="set_shadow_offset" getter="get_shadow_offset"> The shadow offset in pixels. Adjusts the position of the shadow relatively to the stylebox. </member> + <member name="shadow_size" type="int" setter="set_shadow_size" getter="get_shadow_size"> + The shadow size in pixels. + </member> </members> <constants> </constants> diff --git a/doc/classes/TextEdit.xml b/doc/classes/TextEdit.xml index 0d52245538..c85738609f 100644 --- a/doc/classes/TextEdit.xml +++ b/doc/classes/TextEdit.xml @@ -401,9 +401,10 @@ String value of the [TextEdit]. </member> <member name="v_scroll_speed" type="float" setter="set_v_scroll_speed" getter="get_v_scroll_speed"> - If [code]true[/code], enables text wrapping when it goes beyond he edge of what is visible. + Vertical scroll sensitivity. </member> <member name="wrap_enabled" type="bool" setter="set_wrap_enabled" getter="is_wrap_enabled"> + If [code]true[/code], enables text wrapping when it goes beyond the edge of what is visible. </member> </members> <signals> diff --git a/doc/classes/TextureRect.xml b/doc/classes/TextureRect.xml index 7f26fe18e5..eb8014bab8 100644 --- a/doc/classes/TextureRect.xml +++ b/doc/classes/TextureRect.xml @@ -16,18 +16,18 @@ <member name="expand" type="bool" setter="set_expand" getter="has_expand"> If [code]true[/code], the texture scales to fit its bounding rectangle. Default value: [code]false[/code]. </member> - <member name="stretch_mode" type="int" setter="set_stretch_mode" getter="get_stretch_mode" enum="TextureRect.StretchMode"> - Controls the texture's behavior when resizing the node's bounding rectangle. See [enum StretchMode]. - </member> - <member name="texture" type="Texture" setter="set_texture" getter="get_texture"> - The node's [Texture] resource. - </member> <member name="flip_h" type="bool" setter="set_flip_h" getter="is_flipped_h"> If [code]true[/code], texture is flipped horizontally. Default value: [code]false[/code]. </member> <member name="flip_v" type="bool" setter="set_flip_v" getter="is_flipped_v"> If [code]true[/code], texture is flipped vertically. Default value: [code]false[/code]. </member> + <member name="stretch_mode" type="int" setter="set_stretch_mode" getter="get_stretch_mode" enum="TextureRect.StretchMode"> + Controls the texture's behavior when resizing the node's bounding rectangle. See [enum StretchMode]. + </member> + <member name="texture" type="Texture" setter="set_texture" getter="get_texture"> + The node's [Texture] resource. + </member> </members> <constants> <constant name="STRETCH_SCALE_ON_EXPAND" value="0" enum="StretchMode"> diff --git a/doc/classes/Thread.xml b/doc/classes/Thread.xml index d2aa1a3f03..ea558735aa 100644 --- a/doc/classes/Thread.xml +++ b/doc/classes/Thread.xml @@ -34,7 +34,7 @@ </argument> <argument index="2" name="userdata" type="Variant" default="null"> </argument> - <argument index="3" name="priority" type="int" default="1"> + <argument index="3" name="priority" type="int" enum="Thread.Priority" default="1"> </argument> <description> Starts a new [Thread] that runs "method" on object "instance" with "userdata" passed as an argument. The "priority" of the [Thread] can be changed by passing a PRIORITY_* enum. diff --git a/doc/classes/TileMap.xml b/doc/classes/TileMap.xml index 3c047487e0..54d9e05180 100644 --- a/doc/classes/TileMap.xml +++ b/doc/classes/TileMap.xml @@ -171,7 +171,7 @@ # Write your custom logic here. # To call the default method: .set_cell(x, y, tile, flip_x, flip_y, transpose, autotile_coord) - [/codeblock] + [/codeblock] </description> </method> <method name="set_cellv"> diff --git a/doc/classes/Transform.xml b/doc/classes/Transform.xml index e110fa7b40..8c9a1a36b3 100644 --- a/doc/classes/Transform.xml +++ b/doc/classes/Transform.xml @@ -172,9 +172,9 @@ </constant> <constant name="FLIP_X" value="Transform( -1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0 )"> </constant> - <constant name="FLIP_Y" value="Transform( 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0 )"> + <constant name="FLIP_Y" value="Transform( 1, 0, 0, 0, -1, 0, 0, 0, 1, 0, 0, 0 )"> </constant> - <constant name="FLIP_Z" value="Transform( 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0 )"> + <constant name="FLIP_Z" value="Transform( 1, 0, 0, 0, 1, 0, 0, 0, -1, 0, 0, 0 )"> </constant> </constants> </class> diff --git a/doc/classes/VSlider.xml b/doc/classes/VSlider.xml index 477c618064..847e5e5272 100644 --- a/doc/classes/VSlider.xml +++ b/doc/classes/VSlider.xml @@ -4,7 +4,7 @@ Vertical slider. </brief_description> <description> - Vertical slider. See [Slider]. This one goes from left (min) to right (max). + Vertical slider. See [Slider]. This one goes from bottom (min) to top (max). </description> <tutorials> </tutorials> diff --git a/doc/classes/VSplitContainer.xml b/doc/classes/VSplitContainer.xml index 83b2dca23c..13bbfb6385 100644 --- a/doc/classes/VSplitContainer.xml +++ b/doc/classes/VSplitContainer.xml @@ -4,7 +4,7 @@ Vertical split container. </brief_description> <description> - Vertical split container. See [SplitContainer]. This goes from left to right. + Vertical split container. See [SplitContainer]. This goes from top to bottom. </description> <tutorials> </tutorials> diff --git a/doc/classes/VisibilityNotifier.xml b/doc/classes/VisibilityNotifier.xml index e30078d82f..062371ba29 100644 --- a/doc/classes/VisibilityNotifier.xml +++ b/doc/classes/VisibilityNotifier.xml @@ -16,6 +16,7 @@ </return> <description> If [code]true[/code], the bounding box is on the screen. + Note: It takes one frame for the node's visibility to be assessed once added to the scene tree, so this method will return [code]false[/code] right after it is instantiated, even if it will be on screen in the draw pass. </description> </method> </methods> diff --git a/doc/classes/VisibilityNotifier2D.xml b/doc/classes/VisibilityNotifier2D.xml index 5689ced4bf..7b553c7518 100644 --- a/doc/classes/VisibilityNotifier2D.xml +++ b/doc/classes/VisibilityNotifier2D.xml @@ -16,6 +16,7 @@ </return> <description> If [code]true[/code], the bounding rectangle is on the screen. + Note: It takes one frame for the node's visibility to be assessed once added to the scene tree, so this method will return [code]false[/code] right after it is instantiated, even if it will be on screen in the draw pass. </description> </method> </methods> diff --git a/doc/classes/VisualShaderNodeBooleanConstant.xml b/doc/classes/VisualShaderNodeBooleanConstant.xml new file mode 100644 index 0000000000..dc2abfff25 --- /dev/null +++ b/doc/classes/VisualShaderNodeBooleanConstant.xml @@ -0,0 +1,19 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<class name="VisualShaderNodeBooleanConstant" inherits="VisualShaderNode" category="Core" version="3.2"> + <brief_description> + </brief_description> + <description> + </description> + <tutorials> + </tutorials> + <demos> + </demos> + <methods> + </methods> + <members> + <member name="constant" type="bool" setter="set_constant" getter="get_constant"> + </member> + </members> + <constants> + </constants> +</class> diff --git a/doc/classes/VisualShaderNodeBooleanUniform.xml b/doc/classes/VisualShaderNodeBooleanUniform.xml new file mode 100644 index 0000000000..06dca13426 --- /dev/null +++ b/doc/classes/VisualShaderNodeBooleanUniform.xml @@ -0,0 +1,15 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<class name="VisualShaderNodeBooleanUniform" inherits="VisualShaderNodeUniform" category="Core" version="3.2"> + <brief_description> + </brief_description> + <description> + </description> + <tutorials> + </tutorials> + <demos> + </demos> + <methods> + </methods> + <constants> + </constants> +</class> diff --git a/doc/classes/VisualShaderNodeColorFunc.xml b/doc/classes/VisualShaderNodeColorFunc.xml new file mode 100644 index 0000000000..e906049cae --- /dev/null +++ b/doc/classes/VisualShaderNodeColorFunc.xml @@ -0,0 +1,23 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<class name="VisualShaderNodeColorFunc" inherits="VisualShaderNode" category="Core" version="3.2"> + <brief_description> + </brief_description> + <description> + </description> + <tutorials> + </tutorials> + <demos> + </demos> + <methods> + </methods> + <members> + <member name="function" type="int" setter="set_function" getter="get_function" enum="VisualShaderNodeColorFunc.Function"> + </member> + </members> + <constants> + <constant name="FUNC_GRAYSCALE" value="0" enum="Function"> + </constant> + <constant name="FUNC_SEPIA" value="1" enum="Function"> + </constant> + </constants> +</class> diff --git a/doc/classes/VisualShaderNodeDeterminant.xml b/doc/classes/VisualShaderNodeDeterminant.xml new file mode 100644 index 0000000000..661894e7b6 --- /dev/null +++ b/doc/classes/VisualShaderNodeDeterminant.xml @@ -0,0 +1,15 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<class name="VisualShaderNodeDeterminant" inherits="VisualShaderNode" category="Core" version="3.2"> + <brief_description> + </brief_description> + <description> + </description> + <tutorials> + </tutorials> + <demos> + </demos> + <methods> + </methods> + <constants> + </constants> +</class> diff --git a/doc/classes/VisualShaderNodeFaceForward.xml b/doc/classes/VisualShaderNodeFaceForward.xml new file mode 100644 index 0000000000..823e251203 --- /dev/null +++ b/doc/classes/VisualShaderNodeFaceForward.xml @@ -0,0 +1,15 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<class name="VisualShaderNodeFaceForward" inherits="VisualShaderNode" category="Core" version="3.2"> + <brief_description> + </brief_description> + <description> + </description> + <tutorials> + </tutorials> + <demos> + </demos> + <methods> + </methods> + <constants> + </constants> +</class> diff --git a/doc/classes/VisualShaderNodeIf.xml b/doc/classes/VisualShaderNodeIf.xml new file mode 100644 index 0000000000..4e606b95aa --- /dev/null +++ b/doc/classes/VisualShaderNodeIf.xml @@ -0,0 +1,15 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<class name="VisualShaderNodeIf" inherits="VisualShaderNode" category="Core" version="3.2"> + <brief_description> + </brief_description> + <description> + </description> + <tutorials> + </tutorials> + <demos> + </demos> + <methods> + </methods> + <constants> + </constants> +</class> diff --git a/doc/classes/VisualShaderNodeOuterProduct.xml b/doc/classes/VisualShaderNodeOuterProduct.xml new file mode 100644 index 0000000000..4ea4713428 --- /dev/null +++ b/doc/classes/VisualShaderNodeOuterProduct.xml @@ -0,0 +1,15 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<class name="VisualShaderNodeOuterProduct" inherits="VisualShaderNode" category="Core" version="3.2"> + <brief_description> + </brief_description> + <description> + </description> + <tutorials> + </tutorials> + <demos> + </demos> + <methods> + </methods> + <constants> + </constants> +</class> diff --git a/doc/classes/VisualShaderNodeScalarClamp.xml b/doc/classes/VisualShaderNodeScalarClamp.xml new file mode 100644 index 0000000000..66401c6bf6 --- /dev/null +++ b/doc/classes/VisualShaderNodeScalarClamp.xml @@ -0,0 +1,15 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<class name="VisualShaderNodeScalarClamp" inherits="VisualShaderNode" category="Core" version="3.2"> + <brief_description> + </brief_description> + <description> + </description> + <tutorials> + </tutorials> + <demos> + </demos> + <methods> + </methods> + <constants> + </constants> +</class> diff --git a/doc/classes/VisualShaderNodeScalarDerivativeFunc.xml b/doc/classes/VisualShaderNodeScalarDerivativeFunc.xml new file mode 100644 index 0000000000..c78be81034 --- /dev/null +++ b/doc/classes/VisualShaderNodeScalarDerivativeFunc.xml @@ -0,0 +1,25 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<class name="VisualShaderNodeScalarDerivativeFunc" inherits="VisualShaderNode" category="Core" version="3.2"> + <brief_description> + </brief_description> + <description> + </description> + <tutorials> + </tutorials> + <demos> + </demos> + <methods> + </methods> + <members> + <member name="function" type="int" setter="set_function" getter="get_function" enum="VisualShaderNodeScalarDerivativeFunc.Function"> + </member> + </members> + <constants> + <constant name="FUNC_SUM" value="0" enum="Function"> + </constant> + <constant name="FUNC_X" value="1" enum="Function"> + </constant> + <constant name="FUNC_Y" value="2" enum="Function"> + </constant> + </constants> +</class> diff --git a/doc/classes/VisualShaderNodeScalarFunc.xml b/doc/classes/VisualShaderNodeScalarFunc.xml index 67a5d3f5c4..66bdfab5a6 100644 --- a/doc/classes/VisualShaderNodeScalarFunc.xml +++ b/doc/classes/VisualShaderNodeScalarFunc.xml @@ -55,5 +55,27 @@ </constant> <constant name="FUNC_NEGATE" value="19" enum="Function"> </constant> + <constant name="FUNC_ACOSH" value="20" enum="Function"> + </constant> + <constant name="FUNC_ASINH" value="21" enum="Function"> + </constant> + <constant name="FUNC_ATANH" value="22" enum="Function"> + </constant> + <constant name="FUNC_DEGREES" value="23" enum="Function"> + </constant> + <constant name="FUNC_EXP2" value="24" enum="Function"> + </constant> + <constant name="FUNC_INVERSE_SQRT" value="25" enum="Function"> + </constant> + <constant name="FUNC_LOG2" value="26" enum="Function"> + </constant> + <constant name="FUNC_RADIANS" value="27" enum="Function"> + </constant> + <constant name="FUNC_RECIPROCAL" value="28" enum="Function"> + </constant> + <constant name="FUNC_ROUNDEVEN" value="29" enum="Function"> + </constant> + <constant name="FUNC_TRUNC" value="30" enum="Function"> + </constant> </constants> </class> diff --git a/doc/classes/VisualShaderNodeScalarOp.xml b/doc/classes/VisualShaderNodeScalarOp.xml index 34ec89f8be..c5a331679f 100644 --- a/doc/classes/VisualShaderNodeScalarOp.xml +++ b/doc/classes/VisualShaderNodeScalarOp.xml @@ -33,5 +33,7 @@ </constant> <constant name="OP_ATAN2" value="8" enum="Operator"> </constant> + <constant name="OP_STEP" value="9" enum="Operator"> + </constant> </constants> </class> diff --git a/doc/classes/VisualShaderNodeScalarSmoothStep.xml b/doc/classes/VisualShaderNodeScalarSmoothStep.xml new file mode 100644 index 0000000000..5be54d0b4f --- /dev/null +++ b/doc/classes/VisualShaderNodeScalarSmoothStep.xml @@ -0,0 +1,15 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<class name="VisualShaderNodeScalarSmoothStep" inherits="VisualShaderNode" category="Core" version="3.2"> + <brief_description> + </brief_description> + <description> + </description> + <tutorials> + </tutorials> + <demos> + </demos> + <methods> + </methods> + <constants> + </constants> +</class> diff --git a/doc/classes/VisualShaderNodeSwitch.xml b/doc/classes/VisualShaderNodeSwitch.xml new file mode 100644 index 0000000000..1c346e7a1f --- /dev/null +++ b/doc/classes/VisualShaderNodeSwitch.xml @@ -0,0 +1,15 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<class name="VisualShaderNodeSwitch" inherits="VisualShaderNode" category="Core" version="3.2"> + <brief_description> + </brief_description> + <description> + </description> + <tutorials> + </tutorials> + <demos> + </demos> + <methods> + </methods> + <constants> + </constants> +</class> diff --git a/doc/classes/VisualShaderNodeTransformFunc.xml b/doc/classes/VisualShaderNodeTransformFunc.xml new file mode 100644 index 0000000000..601e66ca16 --- /dev/null +++ b/doc/classes/VisualShaderNodeTransformFunc.xml @@ -0,0 +1,23 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<class name="VisualShaderNodeTransformFunc" inherits="VisualShaderNode" category="Core" version="3.2"> + <brief_description> + </brief_description> + <description> + </description> + <tutorials> + </tutorials> + <demos> + </demos> + <methods> + </methods> + <members> + <member name="function" type="int" setter="set_function" getter="get_function" enum="VisualShaderNodeTransformFunc.Function"> + </member> + </members> + <constants> + <constant name="FUNC_INVERSE" value="0" enum="Function"> + </constant> + <constant name="FUNC_TRANSPOSE" value="1" enum="Function"> + </constant> + </constants> +</class> diff --git a/doc/classes/VisualShaderNodeTransformMult.xml b/doc/classes/VisualShaderNodeTransformMult.xml index 47026b400e..e9c445526d 100644 --- a/doc/classes/VisualShaderNodeTransformMult.xml +++ b/doc/classes/VisualShaderNodeTransformMult.xml @@ -19,5 +19,9 @@ </constant> <constant name="OP_BxA" value="1" enum="Operator"> </constant> + <constant name="OP_AxB_COMP" value="2" enum="Operator"> + </constant> + <constant name="OP_BxA_COMP" value="3" enum="Operator"> + </constant> </constants> </class> diff --git a/doc/classes/VisualShaderNodeVectorClamp.xml b/doc/classes/VisualShaderNodeVectorClamp.xml new file mode 100644 index 0000000000..677625ebce --- /dev/null +++ b/doc/classes/VisualShaderNodeVectorClamp.xml @@ -0,0 +1,15 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<class name="VisualShaderNodeVectorClamp" inherits="VisualShaderNode" category="Core" version="3.2"> + <brief_description> + </brief_description> + <description> + </description> + <tutorials> + </tutorials> + <demos> + </demos> + <methods> + </methods> + <constants> + </constants> +</class> diff --git a/doc/classes/VisualShaderNodeVectorDerivativeFunc.xml b/doc/classes/VisualShaderNodeVectorDerivativeFunc.xml new file mode 100644 index 0000000000..e382cb0b58 --- /dev/null +++ b/doc/classes/VisualShaderNodeVectorDerivativeFunc.xml @@ -0,0 +1,25 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<class name="VisualShaderNodeVectorDerivativeFunc" inherits="VisualShaderNode" category="Core" version="3.2"> + <brief_description> + </brief_description> + <description> + </description> + <tutorials> + </tutorials> + <demos> + </demos> + <methods> + </methods> + <members> + <member name="function" type="int" setter="set_function" getter="get_function" enum="VisualShaderNodeVectorDerivativeFunc.Function"> + </member> + </members> + <constants> + <constant name="FUNC_SUM" value="0" enum="Function"> + </constant> + <constant name="FUNC_X" value="1" enum="Function"> + </constant> + <constant name="FUNC_Y" value="2" enum="Function"> + </constant> + </constants> +</class> diff --git a/doc/classes/VisualShaderNodeVectorDistance.xml b/doc/classes/VisualShaderNodeVectorDistance.xml new file mode 100644 index 0000000000..d83a520a46 --- /dev/null +++ b/doc/classes/VisualShaderNodeVectorDistance.xml @@ -0,0 +1,15 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<class name="VisualShaderNodeVectorDistance" inherits="VisualShaderNode" category="Core" version="3.2"> + <brief_description> + </brief_description> + <description> + </description> + <tutorials> + </tutorials> + <demos> + </demos> + <methods> + </methods> + <constants> + </constants> +</class> diff --git a/doc/classes/VisualShaderNodeVectorFunc.xml b/doc/classes/VisualShaderNodeVectorFunc.xml index 968d73d4e1..5e57d85b4c 100644 --- a/doc/classes/VisualShaderNodeVectorFunc.xml +++ b/doc/classes/VisualShaderNodeVectorFunc.xml @@ -27,5 +27,61 @@ </constant> <constant name="FUNC_HSV2RGB" value="5" enum="Function"> </constant> + <constant name="FUNC_ABS" value="6" enum="Function"> + </constant> + <constant name="FUNC_ACOS" value="7" enum="Function"> + </constant> + <constant name="FUNC_ACOSH" value="8" enum="Function"> + </constant> + <constant name="FUNC_ASIN" value="9" enum="Function"> + </constant> + <constant name="FUNC_ASINH" value="10" enum="Function"> + </constant> + <constant name="FUNC_ATAN" value="11" enum="Function"> + </constant> + <constant name="FUNC_ATANH" value="12" enum="Function"> + </constant> + <constant name="FUNC_CEIL" value="13" enum="Function"> + </constant> + <constant name="FUNC_COS" value="14" enum="Function"> + </constant> + <constant name="FUNC_COSH" value="15" enum="Function"> + </constant> + <constant name="FUNC_DEGREES" value="16" enum="Function"> + </constant> + <constant name="FUNC_EXP" value="17" enum="Function"> + </constant> + <constant name="FUNC_EXP2" value="18" enum="Function"> + </constant> + <constant name="FUNC_FLOOR" value="19" enum="Function"> + </constant> + <constant name="FUNC_FRAC" value="20" enum="Function"> + </constant> + <constant name="FUNC_INVERSE_SQRT" value="21" enum="Function"> + </constant> + <constant name="FUNC_LOG" value="22" enum="Function"> + </constant> + <constant name="FUNC_LOG2" value="23" enum="Function"> + </constant> + <constant name="FUNC_RADIANS" value="24" enum="Function"> + </constant> + <constant name="FUNC_ROUND" value="25" enum="Function"> + </constant> + <constant name="FUNC_ROUNDEVEN" value="26" enum="Function"> + </constant> + <constant name="FUNC_SIGN" value="27" enum="Function"> + </constant> + <constant name="FUNC_SIN" value="28" enum="Function"> + </constant> + <constant name="FUNC_SINH" value="29" enum="Function"> + </constant> + <constant name="FUNC_SQRT" value="30" enum="Function"> + </constant> + <constant name="FUNC_TAN" value="31" enum="Function"> + </constant> + <constant name="FUNC_TANH" value="32" enum="Function"> + </constant> + <constant name="FUNC_TRUNC" value="33" enum="Function"> + </constant> </constants> </class> diff --git a/doc/classes/VisualShaderNodeVectorOp.xml b/doc/classes/VisualShaderNodeVectorOp.xml index 16c1f8f25d..6524f129da 100644 --- a/doc/classes/VisualShaderNodeVectorOp.xml +++ b/doc/classes/VisualShaderNodeVectorOp.xml @@ -33,5 +33,11 @@ </constant> <constant name="OP_CROSS" value="8" enum="Operator"> </constant> + <constant name="OP_ATAN2" value="9" enum="Operator"> + </constant> + <constant name="OP_REFLECT" value="10" enum="Operator"> + </constant> + <constant name="OP_STEP" value="11" enum="Operator"> + </constant> </constants> </class> diff --git a/doc/classes/VisualShaderNodeVectorRefract.xml b/doc/classes/VisualShaderNodeVectorRefract.xml new file mode 100644 index 0000000000..32bb0692d4 --- /dev/null +++ b/doc/classes/VisualShaderNodeVectorRefract.xml @@ -0,0 +1,15 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<class name="VisualShaderNodeVectorRefract" inherits="VisualShaderNode" category="Core" version="3.2"> + <brief_description> + </brief_description> + <description> + </description> + <tutorials> + </tutorials> + <demos> + </demos> + <methods> + </methods> + <constants> + </constants> +</class> diff --git a/doc/classes/VisualShaderNodeVectorScalarSmoothStep.xml b/doc/classes/VisualShaderNodeVectorScalarSmoothStep.xml new file mode 100644 index 0000000000..a158b4731d --- /dev/null +++ b/doc/classes/VisualShaderNodeVectorScalarSmoothStep.xml @@ -0,0 +1,15 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<class name="VisualShaderNodeVectorScalarSmoothStep" inherits="VisualShaderNode" category="Core" version="3.2"> + <brief_description> + </brief_description> + <description> + </description> + <tutorials> + </tutorials> + <demos> + </demos> + <methods> + </methods> + <constants> + </constants> +</class> diff --git a/doc/classes/VisualShaderNodeVectorScalarStep.xml b/doc/classes/VisualShaderNodeVectorScalarStep.xml new file mode 100644 index 0000000000..88c9b006fc --- /dev/null +++ b/doc/classes/VisualShaderNodeVectorScalarStep.xml @@ -0,0 +1,15 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<class name="VisualShaderNodeVectorScalarStep" inherits="VisualShaderNode" category="Core" version="3.2"> + <brief_description> + </brief_description> + <description> + </description> + <tutorials> + </tutorials> + <demos> + </demos> + <methods> + </methods> + <constants> + </constants> +</class> diff --git a/doc/classes/VisualShaderNodeVectorSmoothStep.xml b/doc/classes/VisualShaderNodeVectorSmoothStep.xml new file mode 100644 index 0000000000..1ddd5f26ce --- /dev/null +++ b/doc/classes/VisualShaderNodeVectorSmoothStep.xml @@ -0,0 +1,15 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<class name="VisualShaderNodeVectorSmoothStep" inherits="VisualShaderNode" category="Core" version="3.2"> + <brief_description> + </brief_description> + <description> + </description> + <tutorials> + </tutorials> + <demos> + </demos> + <methods> + </methods> + <constants> + </constants> +</class> diff --git a/editor/animation_track_editor.cpp b/editor/animation_track_editor.cpp index 2307d340d8..9f6fa74dd3 100644 --- a/editor/animation_track_editor.cpp +++ b/editor/animation_track_editor.cpp @@ -936,7 +936,7 @@ void AnimationTimelineEdit::_notification(int p_what) { if (frame != prev_frame && i >= prev_frame_ofs) { - draw_line(Point2(get_name_limit() + i, 0), Point2(get_name_limit() + i, h), linecolor); + draw_line(Point2(get_name_limit() + i, 0), Point2(get_name_limit() + i, h), linecolor, Math::round(EDSCALE)); draw_string(font, Point2(get_name_limit() + i + 3 * EDSCALE, (h - font->get_height()) / 2 + font->get_ascent()).floor(), itos(frame), sub ? color_time_dec : color_time_sec, zoomw - i); prev_frame_ofs = i + font->get_string_size(itos(frame)).x + 5 * EDSCALE; @@ -957,13 +957,13 @@ void AnimationTimelineEdit::_notification(int p_what) { if ((sc / step) != (prev_sc / step) || (prev_sc < 0 && sc >= 0)) { int scd = sc < 0 ? prev_sc : sc; - draw_line(Point2(get_name_limit() + i, 0), Point2(get_name_limit() + i, h), linecolor); + draw_line(Point2(get_name_limit() + i, 0), Point2(get_name_limit() + i, h), linecolor, Math::round(EDSCALE)); draw_string(font, Point2(get_name_limit() + i + 3, (h - font->get_height()) / 2 + font->get_ascent()).floor(), String::num((scd - (scd % step)) / double(SC_ADJ), decimals), sub ? color_time_dec : color_time_sec, zoomw - i); } } } - draw_line(Vector2(0, get_size().height), get_size(), linecolor); + draw_line(Vector2(0, get_size().height), get_size(), linecolor, Math::round(EDSCALE)); } } @@ -1041,7 +1041,7 @@ void AnimationTimelineEdit::_play_position_draw() { if (px >= get_name_limit() && px < (play_position->get_size().width - get_buttons_width())) { Color color = get_color("accent_color", "Editor"); - play_position->draw_line(Point2(px, 0), Point2(px, h), color); + play_position->draw_line(Point2(px, 0), Point2(px, h), color, Math::round(EDSCALE)); } } @@ -1310,7 +1310,7 @@ void AnimationTrackEdit::_notification(int p_what) { string_pos = string_pos.floor(); draw_string(font, string_pos, text, text_color, limit - ofs - hsep); - draw_line(Point2(limit, 0), Point2(limit, get_size().height), linecolor); + draw_line(Point2(limit, 0), Point2(limit, get_size().height), linecolor, Math::round(EDSCALE)); } // KEYFAMES // @@ -1370,7 +1370,7 @@ void AnimationTrackEdit::_notification(int p_what) { Ref<Texture> down_icon = get_icon("select_arrow", "Tree"); - draw_line(Point2(ofs, 0), Point2(ofs, get_size().height), linecolor); + draw_line(Point2(ofs, 0), Point2(ofs, get_size().height), linecolor, Math::round(EDSCALE)); ofs += hsep; { @@ -1417,7 +1417,7 @@ void AnimationTrackEdit::_notification(int p_what) { } ofs += down_icon->get_width(); - draw_line(Point2(ofs + hsep * 0.5, 0), Point2(ofs + hsep * 0.5, get_size().height), linecolor); + draw_line(Point2(ofs + hsep * 0.5, 0), Point2(ofs + hsep * 0.5, get_size().height), linecolor, Math::round(EDSCALE)); ofs += hsep; } @@ -1450,7 +1450,7 @@ void AnimationTrackEdit::_notification(int p_what) { } ofs += down_icon->get_width(); - draw_line(Point2(ofs + hsep * 0.5, 0), Point2(ofs + hsep * 0.5, get_size().height), linecolor); + draw_line(Point2(ofs + hsep * 0.5, 0), Point2(ofs + hsep * 0.5, get_size().height), linecolor, Math::round(EDSCALE)); ofs += hsep; } @@ -1483,7 +1483,7 @@ void AnimationTrackEdit::_notification(int p_what) { } ofs += down_icon->get_width(); - draw_line(Point2(ofs + hsep * 0.5, 0), Point2(ofs + hsep * 0.5, get_size().height), linecolor); + draw_line(Point2(ofs + hsep * 0.5, 0), Point2(ofs + hsep * 0.5, get_size().height), linecolor, Math::round(EDSCALE)); ofs += hsep; } @@ -1501,17 +1501,17 @@ void AnimationTrackEdit::_notification(int p_what) { } if (in_group) { - draw_line(Vector2(timeline->get_name_limit(), get_size().height), get_size(), linecolor); + draw_line(Vector2(timeline->get_name_limit(), get_size().height), get_size(), linecolor, Math::round(EDSCALE)); } else { - draw_line(Vector2(0, get_size().height), get_size(), linecolor); + draw_line(Vector2(0, get_size().height), get_size(), linecolor, Math::round(EDSCALE)); } if (dropping_at != 0) { Color drop_color = get_color("accent_color", "Editor"); if (dropping_at < 0) { - draw_line(Vector2(0, 0), Vector2(get_size().width, 0), drop_color); + draw_line(Vector2(0, 0), Vector2(get_size().width, 0), drop_color, Math::round(EDSCALE)); } else { - draw_line(Vector2(0, get_size().height), get_size(), drop_color); + draw_line(Vector2(0, get_size().height), get_size(), drop_color, Math::round(EDSCALE)); } } } @@ -1560,7 +1560,7 @@ void AnimationTrackEdit::draw_key_link(int p_index, float p_pixels_sec, int p_x, int from_x = MAX(p_x, p_clip_left); int to_x = MIN(p_next_x, p_clip_right); - draw_line(Point2(from_x + 1, get_size().height / 2), Point2(to_x, get_size().height / 2), color, 2); + draw_line(Point2(from_x + 1, get_size().height / 2), Point2(to_x, get_size().height / 2), color, Math::round(2 * EDSCALE)); } void AnimationTrackEdit::draw_key(int p_index, float p_pixels_sec, int p_x, bool p_selected, int p_clip_left, int p_clip_right) { @@ -1740,7 +1740,7 @@ void AnimationTrackEdit::_play_position_draw() { if (px >= timeline->get_name_limit() && px < (get_size().width - timeline->get_buttons_width())) { Color color = get_color("accent_color", "Editor"); - play_position->draw_line(Point2(px, 0), Point2(px, h), color); + play_position->draw_line(Point2(px, 0), Point2(px, h), color, Math::round(EDSCALE)); } } @@ -2443,9 +2443,9 @@ void AnimationTrackEditGroup::_notification(int p_what) { Color linecolor = color; linecolor.a = 0.2; - draw_line(Point2(), Point2(get_size().width, 0), linecolor); - draw_line(Point2(timeline->get_name_limit(), 0), Point2(timeline->get_name_limit(), get_size().height), linecolor); - draw_line(Point2(get_size().width - timeline->get_buttons_width(), 0), Point2(get_size().width - timeline->get_buttons_width(), get_size().height), linecolor); + draw_line(Point2(), Point2(get_size().width, 0), linecolor, Math::round(EDSCALE)); + draw_line(Point2(timeline->get_name_limit(), 0), Point2(timeline->get_name_limit(), get_size().height), linecolor, Math::round(EDSCALE)); + draw_line(Point2(get_size().width - timeline->get_buttons_width(), 0), Point2(get_size().width - timeline->get_buttons_width(), get_size().height), linecolor, Math::round(EDSCALE)); int ofs = 0; draw_texture(icon, Point2(ofs, int(get_size().height - icon->get_height()) / 2)); @@ -2456,7 +2456,7 @@ void AnimationTrackEditGroup::_notification(int p_what) { if (px >= timeline->get_name_limit() && px < (get_size().width - timeline->get_buttons_width())) { Color accent = get_color("accent_color", "Editor"); - draw_line(Point2(px, 0), Point2(px, get_size().height), accent); + draw_line(Point2(px, 0), Point2(px, get_size().height), accent, Math::round(EDSCALE)); } } } @@ -2540,7 +2540,7 @@ void AnimationTrackEditor::set_animation(const Ref<Animation> &p_anim) { step->set_block_signals(false); step->set_read_only(false); snap->set_disabled(false); - snap_mode->set_disabled(true); + snap_mode->set_disabled(false); } else { hscroll->hide(); edit->set_disabled(true); @@ -2549,7 +2549,7 @@ void AnimationTrackEditor::set_animation(const Ref<Animation> &p_anim) { step->set_block_signals(false); step->set_read_only(true); snap->set_disabled(true); - snap_mode->set_disabled(false); + snap_mode->set_disabled(true); } } diff --git a/editor/editor_node.cpp b/editor/editor_node.cpp index 8a67002503..851d6a0aa6 100644 --- a/editor/editor_node.cpp +++ b/editor/editor_node.cpp @@ -1306,6 +1306,7 @@ void EditorNode::_dialog_action(String p_file) { _save_default_environment(); _save_scene_with_preview(p_file, scene_idx); _add_to_recent_scenes(p_file); + save_layout(); if (scene_idx != -1) _discard_changes(); @@ -4069,6 +4070,7 @@ void EditorNode::_load_open_scenes_from_config(Ref<ConfigFile> p_layout, const S for (int i = 0; i < scenes.size(); i++) { load_scene(scenes[i]); } + save_layout(); restoring_scenes = false; } diff --git a/editor/filesystem_dock.cpp b/editor/filesystem_dock.cpp index 5a1383be6d..c32cd1de50 100644 --- a/editor/filesystem_dock.cpp +++ b/editor/filesystem_dock.cpp @@ -1021,6 +1021,7 @@ void FileSystemDock::_try_move_item(const FileOrFolder &p_item, const String &p_ for (int j = 0; j < ed->get_edited_scene_count(); j++) { if (ed->get_scene_path(j) == file_changed_paths[i]) { ed->get_edited_scene_root(j)->set_filename(new_item_path); + editor->save_layout(); break; } } diff --git a/editor/plugins/item_list_editor_plugin.h b/editor/plugins/item_list_editor_plugin.h index 679235e316..701632e576 100644 --- a/editor/plugins/item_list_editor_plugin.h +++ b/editor/plugins/item_list_editor_plugin.h @@ -157,7 +157,7 @@ public: virtual void set_item_enabled(int p_idx, int p_enabled) { pp->set_item_disabled(p_idx, !p_enabled); } virtual bool is_item_enabled(int p_idx) const { return !pp->is_item_disabled(p_idx); } - virtual void set_item_id(int p_idx, int p_id) { pp->set_item_id(p_idx, p_idx); } + virtual void set_item_id(int p_idx, int p_id) { pp->set_item_id(p_idx, p_id); } virtual int get_item_id(int p_idx) const { return pp->get_item_id(p_idx); } virtual void set_item_separator(int p_idx, bool p_separator) { pp->set_item_as_separator(p_idx, p_separator); } diff --git a/editor/plugins/script_text_editor.cpp b/editor/plugins/script_text_editor.cpp index 543771ad1c..c586985957 100644 --- a/editor/plugins/script_text_editor.cpp +++ b/editor/plugins/script_text_editor.cpp @@ -81,6 +81,8 @@ void ScriptTextEditor::set_edited_resource(const RES &p_res) { emit_signal("name_changed"); code_editor->update_line_and_column(); + + _validate_script(); } void ScriptTextEditor::_update_member_keywords() { diff --git a/editor/plugins/tile_set_editor_plugin.cpp b/editor/plugins/tile_set_editor_plugin.cpp index 03277159fc..8cf00cf67d 100644 --- a/editor/plugins/tile_set_editor_plugin.cpp +++ b/editor/plugins/tile_set_editor_plugin.cpp @@ -1038,25 +1038,26 @@ void TileSetEditor::_on_workspace_overlay_draw() { tileset->get_tile_list(tiles); for (List<int>::Element *E = tiles->front(); E; E = E->next()) { int t_id = E->get(); - if (tileset->tile_get_texture(t_id)->get_rid() == current_texture_rid) { - Rect2i region = tileset->tile_get_region(t_id); - region.position += WORKSPACE_MARGIN; - region.position *= workspace->get_scale().x; - Color c; - if (tileset->tile_get_tile_mode(t_id) == TileSet::SINGLE_TILE) - c = COLOR_SINGLE; - else if (tileset->tile_get_tile_mode(t_id) == TileSet::AUTO_TILE) - c = COLOR_AUTOTILE; - else if (tileset->tile_get_tile_mode(t_id) == TileSet::ATLAS_TILE) - c = COLOR_ATLAS; - String tile_id_name = String::num(t_id, 0) + ": " + tileset->tile_get_name(t_id); - Ref<Font> font = get_font("font", "Label"); - region.set_size(font->get_string_size(tile_id_name)); - workspace_overlay->draw_rect(region, c); - region.position.y += region.size.y - 2; - c = Color(0.1, 0.1, 0.1); - workspace_overlay->draw_string(font, region.position, tile_id_name, c); - } + if (tileset->tile_get_texture(t_id)->get_rid() != current_texture_rid) + continue; + + Rect2 region = tileset->tile_get_region(t_id); + region.position += WORKSPACE_MARGIN; + region.position *= workspace->get_scale().x; + Color c; + if (tileset->tile_get_tile_mode(t_id) == TileSet::SINGLE_TILE) + c = COLOR_SINGLE; + else if (tileset->tile_get_tile_mode(t_id) == TileSet::AUTO_TILE) + c = COLOR_AUTOTILE; + else if (tileset->tile_get_tile_mode(t_id) == TileSet::ATLAS_TILE) + c = COLOR_ATLAS; + String tile_id_name = String::num(t_id, 0) + ": " + tileset->tile_get_name(t_id); + Ref<Font> font = get_font("font", "Label"); + region.set_size(font->get_string_size(tile_id_name)); + workspace_overlay->draw_rect(region, c); + region.position.y += region.size.y - 2; + c = Color(0.1, 0.1, 0.1); + workspace_overlay->draw_string(font, region.position, tile_id_name, c); } } diff --git a/editor/scene_tree_editor.cpp b/editor/scene_tree_editor.cpp index 42272d0e6e..f2d11c2753 100644 --- a/editor/scene_tree_editor.cpp +++ b/editor/scene_tree_editor.cpp @@ -1121,22 +1121,17 @@ SceneTreeEditor::~SceneTreeEditor() { void SceneTreeDialog::_notification(int p_what) { - if (p_what == NOTIFICATION_ENTER_TREE) { - connect("confirmed", this, "_select"); - } - - if (p_what == NOTIFICATION_EXIT_TREE) { - disconnect("confirmed", this, "_select"); - } - if (p_what == NOTIFICATION_DRAW) { - - RID ci = get_canvas_item(); - get_stylebox("panel", "PopupMenu")->draw(ci, Rect2(Point2(), get_size())); - } - - if (p_what == NOTIFICATION_VISIBILITY_CHANGED && is_visible_in_tree()) { - - tree->update_tree(); + switch (p_what) { + case NOTIFICATION_ENTER_TREE: { + connect("confirmed", this, "_select"); + } break; + case NOTIFICATION_EXIT_TREE: { + disconnect("confirmed", this, "_select"); + } break; + case NOTIFICATION_VISIBILITY_CHANGED: { + if (is_visible_in_tree()) + tree->update_tree(); + } break; } } @@ -1165,8 +1160,6 @@ SceneTreeDialog::SceneTreeDialog() { tree = memnew(SceneTreeEditor(false, false, true)); add_child(tree); - //set_child_rect(tree); - tree->get_scene_tree()->connect("item_activated", this, "_select"); } diff --git a/main/input_default.cpp b/main/input_default.cpp index e9f1eeff6a..6c3252ad1a 100644 --- a/main/input_default.cpp +++ b/main/input_default.cpp @@ -623,7 +623,8 @@ bool InputDefault::is_emulating_mouse_from_touch() const { return emulate_mouse_from_touch; } -Input::CursorShape InputDefault::get_default_cursor_shape() { +Input::CursorShape InputDefault::get_default_cursor_shape() const { + return default_shape; } @@ -638,6 +639,11 @@ void InputDefault::set_default_cursor_shape(CursorShape p_shape) { parse_input_event(mm); } +Input::CursorShape InputDefault::get_current_cursor_shape() const { + + return (Input::CursorShape)OS::get_singleton()->get_cursor_shape(); +} + void InputDefault::set_custom_mouse_cursor(const RES &p_cursor, CursorShape p_shape, const Vector2 &p_hotspot) { if (Engine::get_singleton()->is_editor_hint()) return; @@ -645,21 +651,6 @@ void InputDefault::set_custom_mouse_cursor(const RES &p_cursor, CursorShape p_sh OS::get_singleton()->set_custom_mouse_cursor(p_cursor, (OS::CursorShape)p_shape, p_hotspot); } -void InputDefault::set_mouse_in_window(bool p_in_window) { - /* no longer supported, leaving this for reference to anyone who might want to implement hardware cursors - if (custom_cursor.is_valid()) { - - if (p_in_window) { - set_mouse_mode(MOUSE_MODE_HIDDEN); - VisualServer::get_singleton()->cursor_set_visible(true); - } else { - set_mouse_mode(MOUSE_MODE_VISIBLE); - VisualServer::get_singleton()->cursor_set_visible(false); - } - } - */ -} - void InputDefault::accumulate_input_event(const Ref<InputEvent> &p_event) { ERR_FAIL_COND(p_event.is_null()); diff --git a/main/input_default.h b/main/input_default.h index 79a90cc4a4..f9b4cbbd57 100644 --- a/main/input_default.h +++ b/main/input_default.h @@ -242,10 +242,10 @@ public: void set_emulate_mouse_from_touch(bool p_emulate); virtual bool is_emulating_mouse_from_touch() const; - virtual CursorShape get_default_cursor_shape(); + virtual CursorShape get_default_cursor_shape() const; virtual void set_default_cursor_shape(CursorShape p_shape); + virtual CursorShape get_current_cursor_shape() const; virtual void set_custom_mouse_cursor(const RES &p_cursor, CursorShape p_shape = Input::CURSOR_ARROW, const Vector2 &p_hotspot = Vector2()); - virtual void set_mouse_in_window(bool p_in_window); void parse_mapping(String p_mapping); void joy_button(int p_device, int p_button, bool p_pressed); diff --git a/modules/gdnative/gdnative_api.json b/modules/gdnative/gdnative_api.json index 71e7eecd1d..9882a89794 100644 --- a/modules/gdnative/gdnative_api.json +++ b/modules/gdnative/gdnative_api.json @@ -6427,7 +6427,24 @@ "major": 3, "minor": 1 }, - "next": null, + "next": { + "type": "NET", + "version": { + "major": 3, + "minor": 2 + }, + "next": null, + "api": [ + { + "name": "godot_net_bind_webrtc_peer", + "return_type": "void", + "arguments": [ + ["godot_object *", "p_obj"], + ["const godot_net_webrtc_peer *", "p_interface"] + ] + } + ] + }, "api": [ { "name": "godot_net_bind_stream_peer", diff --git a/modules/gdnative/include/net/godot_net.h b/modules/gdnative/include/net/godot_net.h index d7de04e725..c1bc9daab5 100644 --- a/modules/gdnative/include/net/godot_net.h +++ b/modules/gdnative/include/net/godot_net.h @@ -111,6 +111,35 @@ typedef struct { /* Binds a MultiplayerPeerGDNative to the provided interface */ void GDAPI godot_net_bind_multiplayer_peer(godot_object *p_obj, const godot_net_multiplayer_peer *); +typedef struct { + godot_gdnative_api_version version; /* version of our API */ + + godot_object *data; /* User reference */ + + /* This is PacketPeer */ + godot_error (*get_packet)(void *, const uint8_t **, int *); + godot_error (*put_packet)(void *, const uint8_t *, int); + godot_int (*get_available_packet_count)(const void *); + godot_int (*get_max_packet_size)(const void *); + + /* This is WebRTCPeer */ + void (*set_write_mode)(void *, godot_int); + godot_int (*get_write_mode)(const void *); + bool (*was_string_packet)(const void *); + godot_int (*get_connection_state)(const void *); + + godot_error (*create_offer)(void *); + godot_error (*set_remote_description)(void *, const char *, const char *); + godot_error (*set_local_description)(void *, const char *, const char *); + godot_error (*add_ice_candidate)(void *, const char *, int, const char *); + godot_error (*poll)(void *); + + void *next; /* For extension? */ +} godot_net_webrtc_peer; + +/* Binds a PacketPeerGDNative to the provided interface */ +void GDAPI godot_net_bind_webrtc_peer(godot_object *p_obj, const godot_net_webrtc_peer *); + #ifdef __cplusplus } #endif diff --git a/modules/gdnative/net/SCsub b/modules/gdnative/net/SCsub index e915703935..18ab9986b0 100644 --- a/modules/gdnative/net/SCsub +++ b/modules/gdnative/net/SCsub @@ -3,5 +3,11 @@ Import('env') Import('env_gdnative') -env_gdnative.add_source_files(env.modules_sources, '*.cpp') +env_net = env_gdnative.Clone() + +has_webrtc = env_net["module_webrtc_enabled"] +if has_webrtc: + env_net.Append(CPPDEFINES=['WEBRTC_GDNATIVE_ENABLED']) + +env_net.add_source_files(env.modules_sources, '*.cpp') diff --git a/modules/gdnative/net/webrtc_peer_gdnative.cpp b/modules/gdnative/net/webrtc_peer_gdnative.cpp new file mode 100644 index 0000000000..60b1ed4fe4 --- /dev/null +++ b/modules/gdnative/net/webrtc_peer_gdnative.cpp @@ -0,0 +1,45 @@ +/*************************************************************************/ +/* packet_peer_gdnative.cpp */ +/*************************************************************************/ +/* This file is part of: */ +/* GODOT ENGINE */ +/* https://godotengine.org */ +/*************************************************************************/ +/* Copyright (c) 2007-2019 Juan Linietsky, Ariel Manzur. */ +/* Copyright (c) 2014-2019 Godot Engine contributors (cf. AUTHORS.md) */ +/* */ +/* Permission is hereby granted, free of charge, to any person obtaining */ +/* a copy of this software and associated documentation files (the */ +/* "Software"), to deal in the Software without restriction, including */ +/* without limitation the rights to use, copy, modify, merge, publish, */ +/* distribute, sublicense, and/or sell copies of the Software, and to */ +/* permit persons to whom the Software is furnished to do so, subject to */ +/* the following conditions: */ +/* */ +/* The above copyright notice and this permission notice shall be */ +/* included in all copies or substantial portions of the Software. */ +/* */ +/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */ +/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */ +/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*/ +/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */ +/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */ +/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */ +/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +/*************************************************************************/ + +#include "modules/gdnative/gdnative.h" +#include "modules/gdnative/include/net/godot_net.h" + +#ifdef WEBRTC_GDNATIVE_ENABLED +#include "modules/webrtc/webrtc_peer_gdnative.h" +#endif + +extern "C" { + +void GDAPI godot_net_bind_webrtc_peer(godot_object *p_obj, const godot_net_webrtc_peer *p_impl) { +#ifdef WEBRTC_GDNATIVE_ENABLED + ((WebRTCPeerGDNative *)p_obj)->set_native_webrtc_peer(p_impl); +#endif +} +} diff --git a/modules/gdscript/gdscript.cpp b/modules/gdscript/gdscript.cpp index 0676317f6e..eada389c51 100644 --- a/modules/gdscript/gdscript.cpp +++ b/modules/gdscript/gdscript.cpp @@ -1945,6 +1945,10 @@ String GDScriptWarning::get_message() const { CHECK_SYMBOLS(1); return "The local variable '" + symbols[0] + "' is declared but never used in the block."; } break; + case SHADOWED_VARIABLE: { + CHECK_SYMBOLS(2); + return "The local variable '" + symbols[0] + "' is shadowing an already defined variable at line " + symbols[1] + "."; + } break; case UNUSED_CLASS_VARIABLE: { CHECK_SYMBOLS(1); return "The class variable '" + symbols[0] + "' is declared but never used in the script."; @@ -2048,6 +2052,7 @@ String GDScriptWarning::get_name_from_code(Code p_code) { "UNASSIGNED_VARIABLE", "UNASSIGNED_VARIABLE_OP_ASSIGN", "UNUSED_VARIABLE", + "SHADOWED_VARIABLE", "UNUSED_CLASS_VARIABLE", "UNUSED_ARGUMENT", "UNREACHABLE_CODE", diff --git a/modules/gdscript/gdscript.h b/modules/gdscript/gdscript.h index c67e390e32..1d75d9e2fe 100644 --- a/modules/gdscript/gdscript.h +++ b/modules/gdscript/gdscript.h @@ -273,6 +273,7 @@ struct GDScriptWarning { UNASSIGNED_VARIABLE, // Variable used but never assigned UNASSIGNED_VARIABLE_OP_ASSIGN, // Variable never assigned but used in an assignment operation (+=, *=, etc) UNUSED_VARIABLE, // Local variable is declared but never used + SHADOWED_VARIABLE, // Variable name shadowed by other variable UNUSED_CLASS_VARIABLE, // Class variable is declared but never used in the file UNUSED_ARGUMENT, // Function argument is never used UNREACHABLE_CODE, // Code after a return statement diff --git a/modules/gdscript/gdscript_parser.cpp b/modules/gdscript/gdscript_parser.cpp index 8a9eacd835..80af094c2c 100644 --- a/modules/gdscript/gdscript_parser.cpp +++ b/modules/gdscript/gdscript_parser.cpp @@ -7658,6 +7658,11 @@ void GDScriptParser::_check_function_types(FunctionNode *p_function) { if (p_function->arguments_usage[i] == 0 && !p_function->arguments[i].operator String().begins_with("_")) { _add_warning(GDScriptWarning::UNUSED_ARGUMENT, p_function->line, p_function->name, p_function->arguments[i].operator String()); } + for (int j = 0; j < current_class->variables.size(); j++) { + if (current_class->variables[j].identifier == p_function->arguments[i]) { + _add_warning(GDScriptWarning::SHADOWED_VARIABLE, p_function->line, p_function->arguments[i], itos(current_class->variables[j].line)); + } + } #endif // DEBUG_ENABLED } @@ -7731,6 +7736,17 @@ void GDScriptParser::_check_function_types(FunctionNode *p_function) { p_function->return_type.has_type = false; p_function->return_type.may_yield = true; } + +#ifdef DEBUG_ENABLED + for (Map<StringName, LocalVarNode *>::Element *E = p_function->body->variables.front(); E; E = E->next()) { + LocalVarNode *lv = E->get(); + for (int i = 0; i < current_class->variables.size(); i++) { + if (current_class->variables[i].identifier == lv->name) { + _add_warning(GDScriptWarning::SHADOWED_VARIABLE, lv->line, lv->name, itos(current_class->variables[i].line)); + } + } + } +#endif // DEBUG_ENABLED } void GDScriptParser::_check_class_blocks_types(ClassNode *p_class) { diff --git a/modules/mono/glue/Managed/Files/Array.cs b/modules/mono/glue/Managed/Files/Array.cs index 2277c7bf18..0e7b0362e0 100644 --- a/modules/mono/glue/Managed/Files/Array.cs +++ b/modules/mono/glue/Managed/Files/Array.cs @@ -143,6 +143,11 @@ namespace Godot.Collections } } + public override string ToString() + { + return godot_icall_Array_ToString(GetPtr()); + } + [MethodImpl(MethodImplOptions.InternalCall)] internal extern static IntPtr godot_icall_Array_Ctor(); @@ -190,6 +195,9 @@ namespace Godot.Collections [MethodImpl(MethodImplOptions.InternalCall)] internal extern static void godot_icall_Array_Generic_GetElementTypeInfo(Type elemType, out int elemTypeEncoding, out IntPtr elemTypeClass); + + [MethodImpl(MethodImplOptions.InternalCall)] + internal extern static string godot_icall_Array_ToString(IntPtr ptr); } public class Array<T> : IList<T>, ICollection<T>, IEnumerable<T> @@ -353,5 +361,7 @@ namespace Godot.Collections { return GetEnumerator(); } + + public override string ToString() => objectArray.ToString(); } } diff --git a/modules/mono/glue/Managed/Files/Dictionary.cs b/modules/mono/glue/Managed/Files/Dictionary.cs index af1782b79a..6ab8549a01 100644 --- a/modules/mono/glue/Managed/Files/Dictionary.cs +++ b/modules/mono/glue/Managed/Files/Dictionary.cs @@ -193,6 +193,11 @@ namespace Godot.Collections } } + public override string ToString() + { + return godot_icall_Dictionary_ToString(GetPtr()); + } + [MethodImpl(MethodImplOptions.InternalCall)] internal extern static IntPtr godot_icall_Dictionary_Ctor(); @@ -243,6 +248,9 @@ namespace Godot.Collections [MethodImpl(MethodImplOptions.InternalCall)] internal extern static void godot_icall_Dictionary_Generic_GetValueTypeInfo(Type valueType, out int valTypeEncoding, out IntPtr valTypeClass); + + [MethodImpl(MethodImplOptions.InternalCall)] + internal extern static string godot_icall_Dictionary_ToString(IntPtr ptr); } public class Dictionary<TKey, TValue> : @@ -442,5 +450,7 @@ namespace Godot.Collections { return GetEnumerator(); } + + public override string ToString() => objectDict.ToString(); } } diff --git a/modules/mono/glue/Managed/Files/DynamicObject.cs b/modules/mono/glue/Managed/Files/DynamicObject.cs index 9504415664..9860feafdd 100644 --- a/modules/mono/glue/Managed/Files/DynamicObject.cs +++ b/modules/mono/glue/Managed/Files/DynamicObject.cs @@ -26,7 +26,7 @@ namespace Godot /// dynamic sprite = GetNode("Sprite").DynamicGodotObject; /// sprite.add_child(this); /// - /// if ((sprite.hframes * sprite.vframes) > 0) + /// if ((sprite.hframes * sprite.vframes) > 0) /// sprite.frame = 0; /// </code> /// </example> diff --git a/modules/mono/glue/Managed/Files/Object.base.cs b/modules/mono/glue/Managed/Files/Object.base.cs index e152d56871..de80f7fddc 100644 --- a/modules/mono/glue/Managed/Files/Object.base.cs +++ b/modules/mono/glue/Managed/Files/Object.base.cs @@ -73,6 +73,11 @@ namespace Godot disposed = true; } + public override string ToString() + { + return godot_icall_Object_ToString(GetPtr(this)); + } + /// <summary> /// Returns a new <see cref="Godot.SignalAwaiter"/> awaiter configured to complete when the instance /// <paramref name="source"/> emits the signal specified by the <paramref name="signal"/> parameter. @@ -88,7 +93,7 @@ namespace Godot /// <code> /// public override void _Ready() /// { - /// for (int i = 0; i < 100; i++) + /// for (int i = 0; i < 100; i++) /// { /// await ToSignal(GetTree(), "idle_frame"); /// GD.Print($"Frame {i}"); @@ -115,6 +120,9 @@ namespace Godot [MethodImpl(MethodImplOptions.InternalCall)] internal extern static void godot_icall_Reference_Disposed(Object obj, IntPtr ptr, bool isFinalizer); + [MethodImpl(MethodImplOptions.InternalCall)] + internal extern static string godot_icall_Object_ToString(IntPtr ptr); + // Used by the generated API [MethodImpl(MethodImplOptions.InternalCall)] internal extern static IntPtr godot_icall_Object_ClassDB_get_method(string type, string method); diff --git a/modules/mono/glue/Managed/Files/RID.cs b/modules/mono/glue/Managed/Files/RID.cs index f1268c8518..12064beca2 100644 --- a/modules/mono/glue/Managed/Files/RID.cs +++ b/modules/mono/glue/Managed/Files/RID.cs @@ -70,6 +70,8 @@ namespace Godot return godot_icall_RID_get_id(RID.GetPtr(this)); } + public override string ToString() => "[RID]"; + [MethodImpl(MethodImplOptions.InternalCall)] internal extern static IntPtr godot_icall_RID_Ctor(IntPtr from); diff --git a/modules/mono/glue/base_object_glue.cpp b/modules/mono/glue/base_object_glue.cpp index 7f367fa095..7385014a53 100644 --- a/modules/mono/glue/base_object_glue.cpp +++ b/modules/mono/glue/base_object_glue.cpp @@ -218,11 +218,16 @@ MonoBoolean godot_icall_DynamicGodotObject_SetMember(Object *p_ptr, MonoString * return valid; } +MonoString *godot_icall_Object_ToString(Object *p_ptr) { + return GDMonoMarshal::mono_string_from_godot(Variant(p_ptr).operator String()); +} + void godot_register_object_icalls() { mono_add_internal_call("Godot.Object::godot_icall_Object_Ctor", (void *)godot_icall_Object_Ctor); mono_add_internal_call("Godot.Object::godot_icall_Object_Disposed", (void *)godot_icall_Object_Disposed); mono_add_internal_call("Godot.Object::godot_icall_Reference_Disposed", (void *)godot_icall_Reference_Disposed); mono_add_internal_call("Godot.Object::godot_icall_Object_ClassDB_get_method", (void *)godot_icall_Object_ClassDB_get_method); + mono_add_internal_call("Godot.Object::godot_icall_Object_ToString", (void *)godot_icall_Object_ToString); mono_add_internal_call("Godot.Object::godot_icall_Object_weakref", (void *)godot_icall_Object_weakref); mono_add_internal_call("Godot.SignalAwaiter::godot_icall_SignalAwaiter_connect", (void *)godot_icall_SignalAwaiter_connect); mono_add_internal_call("Godot.DynamicGodotObject::godot_icall_DynamicGodotObject_SetMemberList", (void *)godot_icall_DynamicGodotObject_SetMemberList); diff --git a/modules/mono/glue/base_object_glue.h b/modules/mono/glue/base_object_glue.h index 9b5224a347..e4ac9fb99d 100644 --- a/modules/mono/glue/base_object_glue.h +++ b/modules/mono/glue/base_object_glue.h @@ -60,6 +60,8 @@ MonoBoolean godot_icall_DynamicGodotObject_GetMember(Object *p_ptr, MonoString * MonoBoolean godot_icall_DynamicGodotObject_SetMember(Object *p_ptr, MonoString *p_name, MonoObject *p_value); +MonoString *godot_icall_Object_ToString(Object *p_ptr); + // Register internal calls void godot_register_object_icalls(); diff --git a/modules/mono/glue/collections_glue.cpp b/modules/mono/glue/collections_glue.cpp index 84c3f354d8..4aef5684fd 100644 --- a/modules/mono/glue/collections_glue.cpp +++ b/modules/mono/glue/collections_glue.cpp @@ -143,6 +143,10 @@ void godot_icall_Array_Generic_GetElementTypeInfo(MonoReflectionType *refltype, *type_class = GDMono::get_singleton()->get_class(type_class_raw); } +MonoString *godot_icall_Array_ToString(Array *ptr) { + return GDMonoMarshal::mono_string_from_godot(Variant(*ptr).operator String()); +} + Dictionary *godot_icall_Dictionary_Ctor() { return memnew(Dictionary); } @@ -264,6 +268,10 @@ void godot_icall_Dictionary_Generic_GetValueTypeInfo(MonoReflectionType *refltyp *type_class = GDMono::get_singleton()->get_class(type_class_raw); } +MonoString *godot_icall_Dictionary_ToString(Dictionary *ptr) { + return GDMonoMarshal::mono_string_from_godot(Variant(*ptr).operator String()); +} + void godot_register_collections_icalls() { mono_add_internal_call("Godot.Collections.Array::godot_icall_Array_Ctor", (void *)godot_icall_Array_Ctor); mono_add_internal_call("Godot.Collections.Array::godot_icall_Array_Dtor", (void *)godot_icall_Array_Dtor); @@ -281,6 +289,7 @@ void godot_register_collections_icalls() { mono_add_internal_call("Godot.Collections.Array::godot_icall_Array_RemoveAt", (void *)godot_icall_Array_RemoveAt); mono_add_internal_call("Godot.Collections.Array::godot_icall_Array_Resize", (void *)godot_icall_Array_Resize); mono_add_internal_call("Godot.Collections.Array::godot_icall_Array_Generic_GetElementTypeInfo", (void *)godot_icall_Array_Generic_GetElementTypeInfo); + mono_add_internal_call("Godot.Collections.Array::godot_icall_Array_ToString", (void *)godot_icall_Array_ToString); mono_add_internal_call("Godot.Collections.Dictionary::godot_icall_Dictionary_Ctor", (void *)godot_icall_Dictionary_Ctor); mono_add_internal_call("Godot.Collections.Dictionary::godot_icall_Dictionary_Dtor", (void *)godot_icall_Dictionary_Dtor); @@ -299,6 +308,7 @@ void godot_register_collections_icalls() { mono_add_internal_call("Godot.Collections.Dictionary::godot_icall_Dictionary_TryGetValue", (void *)godot_icall_Dictionary_TryGetValue); mono_add_internal_call("Godot.Collections.Dictionary::godot_icall_Dictionary_TryGetValue_Generic", (void *)godot_icall_Dictionary_TryGetValue_Generic); mono_add_internal_call("Godot.Collections.Dictionary::godot_icall_Dictionary_Generic_GetValueTypeInfo", (void *)godot_icall_Dictionary_Generic_GetValueTypeInfo); + mono_add_internal_call("Godot.Collections.Dictionary::godot_icall_Dictionary_ToString", (void *)godot_icall_Dictionary_ToString); } #endif // MONO_GLUE_ENABLED diff --git a/modules/mono/glue/collections_glue.h b/modules/mono/glue/collections_glue.h index 69768500d8..54ed42c3b6 100644 --- a/modules/mono/glue/collections_glue.h +++ b/modules/mono/glue/collections_glue.h @@ -71,6 +71,8 @@ Error godot_icall_Array_Resize(Array *ptr, int new_size); void godot_icall_Array_Generic_GetElementTypeInfo(MonoReflectionType *refltype, uint32_t *type_encoding, GDMonoClass **type_class); +MonoString *godot_icall_Array_ToString(Array *ptr); + // Dictionary Dictionary *godot_icall_Dictionary_Ctor(); @@ -107,6 +109,8 @@ MonoBoolean godot_icall_Dictionary_TryGetValue_Generic(Dictionary *ptr, MonoObje void godot_icall_Dictionary_Generic_GetValueTypeInfo(MonoReflectionType *refltype, uint32_t *type_encoding, GDMonoClass **type_class); +MonoString *godot_icall_Dictionary_ToString(Dictionary *ptr); + // Register internal calls void godot_register_collections_icalls(); diff --git a/modules/mono/mono_gd/gd_mono.cpp b/modules/mono/mono_gd/gd_mono.cpp index 48bea4addf..bfb6c13224 100644 --- a/modules/mono/mono_gd/gd_mono.cpp +++ b/modules/mono/mono_gd/gd_mono.cpp @@ -30,6 +30,7 @@ #include "gd_mono.h" +#include <mono/metadata/environment.h> #include <mono/metadata/exception.h> #include <mono/metadata/mono-config.h> #include <mono/metadata/mono-debug.h> @@ -1008,7 +1009,9 @@ void GDMono::unhandled_exception_hook(MonoObject *p_exc, void *) { if (ScriptDebugger::get_singleton()) ScriptDebugger::get_singleton()->idle_poll(); #endif - abort(); + + exit(mono_environment_exitcode_get()); + GD_UNREACHABLE(); } diff --git a/modules/mono/mono_gd/gd_mono_internals.cpp b/modules/mono/mono_gd/gd_mono_internals.cpp index c8cba5cf1b..63bcfe053c 100644 --- a/modules/mono/mono_gd/gd_mono_internals.cpp +++ b/modules/mono/mono_gd/gd_mono_internals.cpp @@ -111,7 +111,8 @@ void tie_managed_to_unmanaged(MonoObject *managed, Object *unmanaged) { void unhandled_exception(MonoException *p_exc) { mono_unhandled_exception((MonoObject *)p_exc); // prints the exception as well - abort(); + // Too bad 'mono_invoke_unhandled_exception_hook' is not exposed to embedders + GDMono::unhandled_exception_hook((MonoObject *)p_exc, NULL); GD_UNREACHABLE(); } diff --git a/modules/mono/mono_gd/gd_mono_log.cpp b/modules/mono/mono_gd/gd_mono_log.cpp index ec89df1959..3191cdbd53 100644 --- a/modules/mono/mono_gd/gd_mono_log.cpp +++ b/modules/mono/mono_gd/gd_mono_log.cpp @@ -72,8 +72,11 @@ static void mono_log_callback(const char *log_domain, const char *log_level, con if (fatal) { ERR_PRINTS("Mono: FATAL ERROR, ABORTING! Logfile: " + GDMonoLog::get_singleton()->get_log_file_path() + "\n"); - // If we were to abort without flushing, the log wouldn't get written. + // Make sure to flush before aborting f->flush(); + f->close(); + memdelete(f); + abort(); } } @@ -93,17 +96,9 @@ bool GDMonoLog::_try_create_logs_dir(const String &p_logs_dir) { return true; } -void GDMonoLog::_open_log_file(const String &p_file_path) { - - log_file = FileAccess::open(p_file_path, FileAccess::WRITE); - - ERR_EXPLAIN("Failed to create log file"); - ERR_FAIL_COND(!log_file); -} - void GDMonoLog::_delete_old_log_files(const String &p_logs_dir) { - static const uint64_t MAX_SECS = 5 * 86400; + static const uint64_t MAX_SECS = 5 * 86400; // 5 days DirAccessRef da = DirAccess::create(DirAccess::ACCESS_FILESYSTEM); ERR_FAIL_COND(!da); @@ -120,10 +115,9 @@ void GDMonoLog::_delete_old_log_files(const String &p_logs_dir) { if (!current.ends_with(".txt")) continue; - String name = current.get_basename(); - uint64_t unixtime = (uint64_t)name.to_int64(); + uint64_t modified_time = FileAccess::get_modified_time(da->get_current_dir().plus_file(current)); - if (OS::get_singleton()->get_unix_time() - unixtime > MAX_SECS) { + if (OS::get_singleton()->get_unix_time() - modified_time > MAX_SECS) { da->remove(current); } } @@ -131,28 +125,70 @@ void GDMonoLog::_delete_old_log_files(const String &p_logs_dir) { da->list_dir_end(); } +static String format(const char *p_fmt, ...) { + va_list args; + + va_start(args, p_fmt); + int len = vsnprintf(NULL, 0, p_fmt, args); + va_end(args); + + len += 1; // for the trailing '/0' + + char *buffer(memnew_arr(char, len)); + + va_start(args, p_fmt); + vsnprintf(buffer, len, p_fmt, args); + va_end(args); + + String res(buffer); + memdelete_arr(buffer); + + return res; +} + void GDMonoLog::initialize() { + CharString log_level = OS::get_singleton()->get_environment("GODOT_MONO_LOG_LEVEL").utf8(); + + if (log_level.length() != 0 && log_level_get_id(log_level.get_data()) == -1) { + ERR_PRINTS(String() + "Mono: Ignoring invalid log level (GODOT_MONO_LOG_LEVEL): " + log_level.get_data()); + log_level = CharString(); + } + + if (log_level.length() == 0) { #ifdef DEBUG_ENABLED - const char *log_level = "debug"; + log_level = String("info").utf8(); #else - const char *log_level = "warning"; + log_level = String("warning").utf8(); #endif + } String logs_dir = GodotSharpDirs::get_mono_logs_dir(); if (_try_create_logs_dir(logs_dir)) { _delete_old_log_files(logs_dir); - log_file_path = logs_dir.plus_file(String::num_int64(OS::get_singleton()->get_unix_time()) + ".txt"); - _open_log_file(log_file_path); + OS::Date date_now = OS::get_singleton()->get_date(); + OS::Time time_now = OS::get_singleton()->get_time(); + int pid = OS::get_singleton()->get_process_id(); + + String log_file_name = format("%d-%02d-%02d %02d:%02d:%02d (%d).txt", + date_now.year, date_now.month, date_now.day, + time_now.hour, time_now.min, time_now.sec, pid); + + log_file_path = logs_dir.plus_file(log_file_name); + + log_file = FileAccess::open(log_file_path, FileAccess::WRITE); + if (!log_file) { + ERR_PRINT("Mono: Cannot create log file"); + } } - mono_trace_set_level_string(log_level); - log_level_id = log_level_get_id(log_level); + mono_trace_set_level_string(log_level.get_data()); + log_level_id = log_level_get_id(log_level.get_data()); if (log_file) { - print_verbose("Mono: Logfile is " + log_file_path); + OS::get_singleton()->print("Mono: Logfile is: %s\n", log_file_path.utf8().get_data()); mono_trace_set_log_handler(mono_log_callback, this); } else { OS::get_singleton()->printerr("Mono: No log file, using default log handler\n"); diff --git a/modules/mono/mono_gd/gd_mono_log.h b/modules/mono/mono_gd/gd_mono_log.h index a477e5edee..91d0557aa3 100644 --- a/modules/mono/mono_gd/gd_mono_log.h +++ b/modules/mono/mono_gd/gd_mono_log.h @@ -41,7 +41,6 @@ class GDMonoLog { String log_file_path; bool _try_create_logs_dir(const String &p_logs_dir); - void _open_log_file(const String &p_file_path); void _delete_old_log_files(const String &p_logs_dir); static GDMonoLog *singleton; diff --git a/modules/mono/mono_gd/gd_mono_utils.cpp b/modules/mono/mono_gd/gd_mono_utils.cpp index 429d4f68d9..bcf5712d16 100644 --- a/modules/mono/mono_gd/gd_mono_utils.cpp +++ b/modules/mono/mono_gd/gd_mono_utils.cpp @@ -632,11 +632,6 @@ void debug_send_unhandled_exception_error(MonoException *p_exc) { } void debug_unhandled_exception(MonoException *p_exc) { -#ifdef DEBUG_ENABLED - GDMonoUtils::debug_send_unhandled_exception_error(p_exc); - if (ScriptDebugger::get_singleton()) - ScriptDebugger::get_singleton()->idle_poll(); -#endif GDMonoInternals::unhandled_exception(p_exc); // prints the exception as well GD_UNREACHABLE(); } diff --git a/modules/opensimplex/doc_classes/OpenSimplexNoise.xml b/modules/opensimplex/doc_classes/OpenSimplexNoise.xml index b5bc35df69..06d22a2d65 100644 --- a/modules/opensimplex/doc_classes/OpenSimplexNoise.xml +++ b/modules/opensimplex/doc_classes/OpenSimplexNoise.xml @@ -37,6 +37,14 @@ Generate a noise image with the requested [code]width[/code] and [code]height[/code], based on the current noise parameters. </description> </method> + <method name="get_noise_1d"> + <return type="float"> + </return> + <argument index="0" name="x" type="float"> + </argument> + <description> + </description> + </method> <method name="get_noise_2d"> <return type="float"> </return> diff --git a/modules/webrtc/SCsub b/modules/webrtc/SCsub new file mode 100644 index 0000000000..446bd530c2 --- /dev/null +++ b/modules/webrtc/SCsub @@ -0,0 +1,16 @@ +#!/usr/bin/env python + +Import('env') +Import('env_modules') + +# Thirdparty source files + +env_webrtc = env_modules.Clone() +use_gdnative = env_webrtc["module_gdnative_enabled"] + +if use_gdnative: # GDNative is retained in Javascript for export compatibility + env_webrtc.Append(CPPDEFINES=['WEBRTC_GDNATIVE_ENABLED']) + gdnative_includes = ["#modules/gdnative/include/"] + env_webrtc.Append(CPPPATH=gdnative_includes) + +env_webrtc.add_source_files(env.modules_sources, "*.cpp") diff --git a/modules/webrtc/config.py b/modules/webrtc/config.py new file mode 100644 index 0000000000..5ed245bad2 --- /dev/null +++ b/modules/webrtc/config.py @@ -0,0 +1,13 @@ +def can_build(env, platform): + return True + +def configure(env): + pass + +def get_doc_classes(): + return [ + "WebRTCPeer" + ] + +def get_doc_path(): + return "doc_classes" diff --git a/modules/webrtc/register_types.cpp b/modules/webrtc/register_types.cpp new file mode 100644 index 0000000000..ee7a766bd9 --- /dev/null +++ b/modules/webrtc/register_types.cpp @@ -0,0 +1,55 @@ +/*************************************************************************/ +/* register_types.cpp */ +/*************************************************************************/ +/* This file is part of: */ +/* GODOT ENGINE */ +/* https://godotengine.org */ +/*************************************************************************/ +/* Copyright (c) 2007-2018 Juan Linietsky, Ariel Manzur. */ +/* Copyright (c) 2014-2018 Godot Engine contributors (cf. AUTHORS.md) */ +/* */ +/* Permission is hereby granted, free of charge, to any person obtaining */ +/* a copy of this software and associated documentation files (the */ +/* "Software"), to deal in the Software without restriction, including */ +/* without limitation the rights to use, copy, modify, merge, publish, */ +/* distribute, sublicense, and/or sell copies of the Software, and to */ +/* permit persons to whom the Software is furnished to do so, subject to */ +/* the following conditions: */ +/* */ +/* The above copyright notice and this permission notice shall be */ +/* included in all copies or substantial portions of the Software. */ +/* */ +/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */ +/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */ +/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*/ +/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */ +/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */ +/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */ +/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +/*************************************************************************/ + +#include "register_types.h" +#include "webrtc_peer.h" + +#ifdef JAVASCRIPT_ENABLED +#include "emscripten.h" +#include "webrtc_peer_js.h" +#endif +#ifdef WEBRTC_GDNATIVE_ENABLED +#include "webrtc_peer_gdnative.h" +#endif + +void register_webrtc_types() { +#ifdef JAVASCRIPT_ENABLED + WebRTCPeerJS::make_default(); +#elif defined(WEBRTC_GDNATIVE_ENABLED) + WebRTCPeerGDNative::make_default(); +#endif + + ClassDB::register_custom_instance_class<WebRTCPeer>(); +#ifdef WEBRTC_GDNATIVE_ENABLED + ClassDB::register_class<WebRTCPeerGDNative>(); +#endif +} + +void unregister_webrtc_types() {} diff --git a/modules/webrtc/register_types.h b/modules/webrtc/register_types.h new file mode 100644 index 0000000000..18a5dcc5aa --- /dev/null +++ b/modules/webrtc/register_types.h @@ -0,0 +1,32 @@ +/*************************************************************************/ +/* register_types.h */ +/*************************************************************************/ +/* This file is part of: */ +/* GODOT ENGINE */ +/* https://godotengine.org */ +/*************************************************************************/ +/* Copyright (c) 2007-2018 Juan Linietsky, Ariel Manzur. */ +/* Copyright (c) 2014-2018 Godot Engine contributors (cf. AUTHORS.md) */ +/* */ +/* Permission is hereby granted, free of charge, to any person obtaining */ +/* a copy of this software and associated documentation files (the */ +/* "Software"), to deal in the Software without restriction, including */ +/* without limitation the rights to use, copy, modify, merge, publish, */ +/* distribute, sublicense, and/or sell copies of the Software, and to */ +/* permit persons to whom the Software is furnished to do so, subject to */ +/* the following conditions: */ +/* */ +/* The above copyright notice and this permission notice shall be */ +/* included in all copies or substantial portions of the Software. */ +/* */ +/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */ +/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */ +/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*/ +/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */ +/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */ +/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */ +/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +/*************************************************************************/ + +void register_webrtc_types(); +void unregister_webrtc_types(); diff --git a/modules/webrtc/webrtc_peer.cpp b/modules/webrtc/webrtc_peer.cpp new file mode 100644 index 0000000000..30c4505df9 --- /dev/null +++ b/modules/webrtc/webrtc_peer.cpp @@ -0,0 +1,81 @@ +/*************************************************************************/ +/* webrtc_peer.cpp */ +/*************************************************************************/ +/* This file is part of: */ +/* GODOT ENGINE */ +/* https://godotengine.org */ +/*************************************************************************/ +/* Copyright (c) 2007-2018 Juan Linietsky, Ariel Manzur. */ +/* Copyright (c) 2014-2018 Godot Engine contributors (cf. AUTHORS.md) */ +/* */ +/* Permission is hereby granted, free of charge, to any person obtaining */ +/* a copy of this software and associated documentation files (the */ +/* "Software"), to deal in the Software without restriction, including */ +/* without limitation the rights to use, copy, modify, merge, publish, */ +/* distribute, sublicense, and/or sell copies of the Software, and to */ +/* permit persons to whom the Software is furnished to do so, subject to */ +/* the following conditions: */ +/* */ +/* The above copyright notice and this permission notice shall be */ +/* included in all copies or substantial portions of the Software. */ +/* */ +/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */ +/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */ +/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*/ +/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */ +/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */ +/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */ +/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +/*************************************************************************/ + +#include "webrtc_peer.h" + +WebRTCPeer *(*WebRTCPeer::_create)() = NULL; + +Ref<WebRTCPeer> WebRTCPeer::create_ref() { + + if (!_create) + return Ref<WebRTCPeer>(); + return Ref<WebRTCPeer>(_create()); +} + +WebRTCPeer *WebRTCPeer::create() { + + if (!_create) + return NULL; + return _create(); +} + +void WebRTCPeer::_bind_methods() { + ClassDB::bind_method(D_METHOD("create_offer"), &WebRTCPeer::create_offer); + ClassDB::bind_method(D_METHOD("set_local_description", "type", "sdp"), &WebRTCPeer::set_local_description); + ClassDB::bind_method(D_METHOD("set_remote_description", "type", "sdp"), &WebRTCPeer::set_remote_description); + ClassDB::bind_method(D_METHOD("poll"), &WebRTCPeer::poll); + ClassDB::bind_method(D_METHOD("add_ice_candidate", "media", "index", "name"), &WebRTCPeer::add_ice_candidate); + + ClassDB::bind_method(D_METHOD("was_string_packet"), &WebRTCPeer::was_string_packet); + ClassDB::bind_method(D_METHOD("set_write_mode", "write_mode"), &WebRTCPeer::set_write_mode); + ClassDB::bind_method(D_METHOD("get_write_mode"), &WebRTCPeer::get_write_mode); + ClassDB::bind_method(D_METHOD("get_connection_state"), &WebRTCPeer::get_connection_state); + + ADD_SIGNAL(MethodInfo("offer_created", PropertyInfo(Variant::STRING, "type"), PropertyInfo(Variant::STRING, "sdp"))); + ADD_SIGNAL(MethodInfo("new_ice_candidate", PropertyInfo(Variant::STRING, "media"), PropertyInfo(Variant::INT, "index"), PropertyInfo(Variant::STRING, "name"))); + + ADD_PROPERTY(PropertyInfo(Variant::INT, "write_mode", PROPERTY_HINT_ENUM), "set_write_mode", "get_write_mode"); + + BIND_ENUM_CONSTANT(WRITE_MODE_TEXT); + BIND_ENUM_CONSTANT(WRITE_MODE_BINARY); + + BIND_ENUM_CONSTANT(STATE_NEW); + BIND_ENUM_CONSTANT(STATE_CONNECTING); + BIND_ENUM_CONSTANT(STATE_CONNECTED); + BIND_ENUM_CONSTANT(STATE_DISCONNECTED); + BIND_ENUM_CONSTANT(STATE_FAILED); + BIND_ENUM_CONSTANT(STATE_CLOSED); +} + +WebRTCPeer::WebRTCPeer() { +} + +WebRTCPeer::~WebRTCPeer() { +} diff --git a/modules/webrtc/webrtc_peer.h b/modules/webrtc/webrtc_peer.h new file mode 100644 index 0000000000..e141c14655 --- /dev/null +++ b/modules/webrtc/webrtc_peer.h @@ -0,0 +1,86 @@ +/*************************************************************************/ +/* webrtc_peer.h */ +/*************************************************************************/ +/* This file is part of: */ +/* GODOT ENGINE */ +/* https://godotengine.org */ +/*************************************************************************/ +/* Copyright (c) 2007-2018 Juan Linietsky, Ariel Manzur. */ +/* Copyright (c) 2014-2018 Godot Engine contributors (cf. AUTHORS.md) */ +/* */ +/* Permission is hereby granted, free of charge, to any person obtaining */ +/* a copy of this software and associated documentation files (the */ +/* "Software"), to deal in the Software without restriction, including */ +/* without limitation the rights to use, copy, modify, merge, publish, */ +/* distribute, sublicense, and/or sell copies of the Software, and to */ +/* permit persons to whom the Software is furnished to do so, subject to */ +/* the following conditions: */ +/* */ +/* The above copyright notice and this permission notice shall be */ +/* included in all copies or substantial portions of the Software. */ +/* */ +/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */ +/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */ +/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*/ +/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */ +/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */ +/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */ +/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +/*************************************************************************/ + +#ifndef WEBRTC_PEER_H +#define WEBRTC_PEER_H + +#include "core/io/packet_peer.h" + +class WebRTCPeer : public PacketPeer { + GDCLASS(WebRTCPeer, PacketPeer); + +public: + enum WriteMode { + WRITE_MODE_TEXT, + WRITE_MODE_BINARY, + }; + + enum ConnectionState { + STATE_NEW, + STATE_CONNECTING, + STATE_CONNECTED, + STATE_DISCONNECTED, + STATE_FAILED, + STATE_CLOSED + }; + +protected: + static void _bind_methods(); + static WebRTCPeer *(*_create)(); + +public: + virtual void set_write_mode(WriteMode mode) = 0; + virtual WriteMode get_write_mode() const = 0; + virtual bool was_string_packet() const = 0; + virtual ConnectionState get_connection_state() const = 0; + + virtual Error create_offer() = 0; + virtual Error set_remote_description(String type, String sdp) = 0; + virtual Error set_local_description(String type, String sdp) = 0; + virtual Error add_ice_candidate(String sdpMidName, int sdpMlineIndexName, String sdpName) = 0; + virtual Error poll() = 0; + + /** Inherited from PacketPeer: **/ + virtual int get_available_packet_count() const = 0; + virtual Error get_packet(const uint8_t **r_buffer, int &r_buffer_size) = 0; ///< buffer is GONE after next get_packet + virtual Error put_packet(const uint8_t *p_buffer, int p_buffer_size) = 0; + + virtual int get_max_packet_size() const = 0; + + static Ref<WebRTCPeer> create_ref(); + static WebRTCPeer *create(); + + WebRTCPeer(); + ~WebRTCPeer(); +}; + +VARIANT_ENUM_CAST(WebRTCPeer::WriteMode); +VARIANT_ENUM_CAST(WebRTCPeer::ConnectionState); +#endif // WEBRTC_PEER_H diff --git a/modules/webrtc/webrtc_peer_gdnative.cpp b/modules/webrtc/webrtc_peer_gdnative.cpp new file mode 100644 index 0000000000..f782944980 --- /dev/null +++ b/modules/webrtc/webrtc_peer_gdnative.cpp @@ -0,0 +1,114 @@ +/*************************************************************************/ +/* webrtc_peer_gdnative.cpp */ +/*************************************************************************/ +/* This file is part of: */ +/* GODOT ENGINE */ +/* https://godotengine.org */ +/*************************************************************************/ +/* Copyright (c) 2007-2018 Juan Linietsky, Ariel Manzur. */ +/* Copyright (c) 2014-2018 Godot Engine contributors (cf. AUTHORS.md) */ +/* */ +/* Permission is hereby granted, free of charge, to any person obtaining */ +/* a copy of this software and associated documentation files (the */ +/* "Software"), to deal in the Software without restriction, including */ +/* without limitation the rights to use, copy, modify, merge, publish, */ +/* distribute, sublicense, and/or sell copies of the Software, and to */ +/* permit persons to whom the Software is furnished to do so, subject to */ +/* the following conditions: */ +/* */ +/* The above copyright notice and this permission notice shall be */ +/* included in all copies or substantial portions of the Software. */ +/* */ +/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */ +/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */ +/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*/ +/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */ +/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */ +/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */ +/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +/*************************************************************************/ + +#ifdef WEBRTC_GDNATIVE_ENABLED + +#include "webrtc_peer_gdnative.h" + +void WebRTCPeerGDNative::_bind_methods() { +} + +WebRTCPeerGDNative::WebRTCPeerGDNative() { + interface = NULL; +} + +WebRTCPeerGDNative::~WebRTCPeerGDNative() { +} + +Error WebRTCPeerGDNative::create_offer() { + ERR_FAIL_COND_V(interface == NULL, ERR_UNCONFIGURED); + return (Error)interface->create_offer(interface->data); +} + +Error WebRTCPeerGDNative::set_local_description(String p_type, String p_sdp) { + ERR_FAIL_COND_V(interface == NULL, ERR_UNCONFIGURED); + return (Error)interface->set_local_description(interface->data, p_type.utf8().get_data(), p_sdp.utf8().get_data()); +} + +Error WebRTCPeerGDNative::set_remote_description(String p_type, String p_sdp) { + ERR_FAIL_COND_V(interface == NULL, ERR_UNCONFIGURED); + return (Error)interface->set_remote_description(interface->data, p_type.utf8().get_data(), p_sdp.utf8().get_data()); +} + +Error WebRTCPeerGDNative::add_ice_candidate(String sdpMidName, int sdpMlineIndexName, String sdpName) { + ERR_FAIL_COND_V(interface == NULL, ERR_UNCONFIGURED); + return (Error)interface->add_ice_candidate(interface->data, sdpMidName.utf8().get_data(), sdpMlineIndexName, sdpName.utf8().get_data()); +} + +Error WebRTCPeerGDNative::poll() { + ERR_FAIL_COND_V(interface == NULL, ERR_UNCONFIGURED); + return (Error)interface->poll(interface->data); +} + +void WebRTCPeerGDNative::set_write_mode(WriteMode p_mode) { + ERR_FAIL_COND(interface == NULL); + interface->set_write_mode(interface->data, p_mode); +} + +WebRTCPeer::WriteMode WebRTCPeerGDNative::get_write_mode() const { + ERR_FAIL_COND_V(interface == NULL, WRITE_MODE_BINARY); + return (WriteMode)interface->get_write_mode(interface->data); +} + +bool WebRTCPeerGDNative::was_string_packet() const { + ERR_FAIL_COND_V(interface == NULL, false); + return interface->was_string_packet(interface->data); +} + +WebRTCPeer::ConnectionState WebRTCPeerGDNative::get_connection_state() const { + ERR_FAIL_COND_V(interface == NULL, STATE_DISCONNECTED); + return STATE_DISCONNECTED; +} + +Error WebRTCPeerGDNative::get_packet(const uint8_t **r_buffer, int &r_buffer_size) { + ERR_FAIL_COND_V(interface == NULL, ERR_UNCONFIGURED); + return (Error)interface->get_packet(interface->data, r_buffer, &r_buffer_size); +} + +Error WebRTCPeerGDNative::put_packet(const uint8_t *p_buffer, int p_buffer_size) { + ERR_FAIL_COND_V(interface == NULL, ERR_UNCONFIGURED); + return (Error)interface->put_packet(interface->data, p_buffer, p_buffer_size); +} + +int WebRTCPeerGDNative::get_max_packet_size() const { + ERR_FAIL_COND_V(interface == NULL, 0); + return interface->get_max_packet_size(interface->data); +} + +int WebRTCPeerGDNative::get_available_packet_count() const { + ERR_FAIL_COND_V(interface == NULL, 0); + return interface->get_available_packet_count(interface->data); +} + +void WebRTCPeerGDNative::set_native_webrtc_peer(const godot_net_webrtc_peer *p_impl) { + interface = p_impl; +} + +#endif // WEBRTC_GDNATIVE_ENABLED diff --git a/modules/webrtc/webrtc_peer_gdnative.h b/modules/webrtc/webrtc_peer_gdnative.h new file mode 100644 index 0000000000..6786cec8ea --- /dev/null +++ b/modules/webrtc/webrtc_peer_gdnative.h @@ -0,0 +1,78 @@ +/*************************************************************************/ +/* webrtc_peer_gdnative.h */ +/*************************************************************************/ +/* This file is part of: */ +/* GODOT ENGINE */ +/* https://godotengine.org */ +/*************************************************************************/ +/* Copyright (c) 2007-2018 Juan Linietsky, Ariel Manzur. */ +/* Copyright (c) 2014-2018 Godot Engine contributors (cf. AUTHORS.md) */ +/* */ +/* Permission is hereby granted, free of charge, to any person obtaining */ +/* a copy of this software and associated documentation files (the */ +/* "Software"), to deal in the Software without restriction, including */ +/* without limitation the rights to use, copy, modify, merge, publish, */ +/* distribute, sublicense, and/or sell copies of the Software, and to */ +/* permit persons to whom the Software is furnished to do so, subject to */ +/* the following conditions: */ +/* */ +/* The above copyright notice and this permission notice shall be */ +/* included in all copies or substantial portions of the Software. */ +/* */ +/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */ +/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */ +/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*/ +/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */ +/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */ +/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */ +/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +/*************************************************************************/ + +#ifdef WEBRTC_GDNATIVE_ENABLED + +#ifndef WEBRTC_PEER_GDNATIVE_H +#define WEBRTC_PEER_GDNATIVE_H + +#include "modules/gdnative/include/net/godot_net.h" +#include "webrtc_peer.h" + +class WebRTCPeerGDNative : public WebRTCPeer { + GDCLASS(WebRTCPeerGDNative, WebRTCPeer); + +protected: + static void _bind_methods(); + +private: + const godot_net_webrtc_peer *interface; + +public: + static WebRTCPeer *_create() { return memnew(WebRTCPeerGDNative); } + static void make_default() { WebRTCPeer::_create = WebRTCPeerGDNative::_create; } + + void set_native_webrtc_peer(const godot_net_webrtc_peer *p_impl); + + virtual void set_write_mode(WriteMode mode); + virtual WriteMode get_write_mode() const; + virtual bool was_string_packet() const; + virtual ConnectionState get_connection_state() const; + + virtual Error create_offer(); + virtual Error set_remote_description(String type, String sdp); + virtual Error set_local_description(String type, String sdp); + virtual Error add_ice_candidate(String sdpMidName, int sdpMlineIndexName, String sdpName); + virtual Error poll(); + + /** Inherited from PacketPeer: **/ + virtual int get_available_packet_count() const; + virtual Error get_packet(const uint8_t **r_buffer, int &r_buffer_size); ///< buffer is GONE after next get_packet + virtual Error put_packet(const uint8_t *p_buffer, int p_buffer_size); + + virtual int get_max_packet_size() const; + + WebRTCPeerGDNative(); + ~WebRTCPeerGDNative(); +}; + +#endif // WEBRTC_PEER_GDNATIVE_H + +#endif // WEBRTC_GDNATIVE_ENABLED diff --git a/modules/webrtc/webrtc_peer_js.cpp b/modules/webrtc/webrtc_peer_js.cpp new file mode 100644 index 0000000000..1282e075ab --- /dev/null +++ b/modules/webrtc/webrtc_peer_js.cpp @@ -0,0 +1,455 @@ +/*************************************************************************/ +/* webrtc_peer_js.cpp */ +/*************************************************************************/ +/* This file is part of: */ +/* GODOT ENGINE */ +/* https://godotengine.org */ +/*************************************************************************/ +/* Copyright (c) 2007-2018 Juan Linietsky, Ariel Manzur. */ +/* Copyright (c) 2014-2018 Godot Engine contributors (cf. AUTHORS.md) */ +/* */ +/* Permission is hereby granted, free of charge, to any person obtaining */ +/* a copy of this software and associated documentation files (the */ +/* "Software"), to deal in the Software without restriction, including */ +/* without limitation the rights to use, copy, modify, merge, publish, */ +/* distribute, sublicense, and/or sell copies of the Software, and to */ +/* permit persons to whom the Software is furnished to do so, subject to */ +/* the following conditions: */ +/* */ +/* The above copyright notice and this permission notice shall be */ +/* included in all copies or substantial portions of the Software. */ +/* */ +/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */ +/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */ +/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*/ +/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */ +/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */ +/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */ +/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +/*************************************************************************/ + +#ifdef JAVASCRIPT_ENABLED + +#include "webrtc_peer_js.h" +#include "emscripten.h" + +extern "C" { +EMSCRIPTEN_KEEPALIVE void _emrtc_on_ice_candidate(void *obj, char *p_MidName, int p_MlineIndexName, char *p_sdpName) { + WebRTCPeerJS *peer = static_cast<WebRTCPeerJS *>(obj); + peer->emit_signal("new_ice_candidate", String(p_MidName), p_MlineIndexName, String(p_sdpName)); +} + +EMSCRIPTEN_KEEPALIVE void _emrtc_offer_created(void *obj, char *p_type, char *p_offer) { + WebRTCPeerJS *peer = static_cast<WebRTCPeerJS *>(obj); + peer->emit_signal("offer_created", String(p_type), String(p_offer)); +} + +EMSCRIPTEN_KEEPALIVE void _emrtc_on_error(void *obj) { + WebRTCPeerJS *peer = static_cast<WebRTCPeerJS *>(obj); + peer->_on_error(); +} + +EMSCRIPTEN_KEEPALIVE void _emrtc_on_open(void *obj) { + WebRTCPeerJS *peer = static_cast<WebRTCPeerJS *>(obj); + peer->_on_open(); +} + +EMSCRIPTEN_KEEPALIVE void _emrtc_on_close(void *obj) { + WebRTCPeerJS *peer = static_cast<WebRTCPeerJS *>(obj); + peer->_on_close(); +} + +EMSCRIPTEN_KEEPALIVE void _emrtc_on_message(void *obj, uint8_t *p_data, uint32_t p_size, bool p_is_string) { + WebRTCPeerJS *peer = static_cast<WebRTCPeerJS *>(obj); + peer->_on_message(p_data, p_size, p_is_string); +} + +EMSCRIPTEN_KEEPALIVE void _emrtc_bind_channel(int p_id) { + /* clang-format off */ + EM_ASM({ + if (!Module.IDHandler.has($0)) { + return; // Godot Object is gone! + } + var dict = Module.IDHandler.get($0); + var channel = dict["channel"]; + var c_ptr = dict["ptr"]; + + channel.onopen = function (evt) { + ccall("_emrtc_on_open", + "void", + ["number"], + [c_ptr] + ); + }; + channel.onclose = function (evt) { + ccall("_emrtc_on_close", + "void", + ["number"], + [c_ptr] + ); + }; + channel.onerror = function (evt) { + ccall("_emrtc_on_error", + "void", + ["number"], + [c_ptr] + ); + }; + + channel.binaryType = "arraybuffer"; + channel.onmessage = function(event) { + var buffer; + var is_string = 0; + if (event.data instanceof ArrayBuffer) { + + buffer = new Uint8Array(event.data); + + } else if (event.data instanceof Blob) { + + alert("Blob type not supported"); + return; + + } else if (typeof event.data === "string") { + + is_string = 1; + var enc = new TextEncoder("utf-8"); + buffer = new Uint8Array(enc.encode(event.data)); + + } else { + + alert("Unknown message type"); + return; + + } + var len = buffer.length*buffer.BYTES_PER_ELEMENT; + var out = Module._malloc(len); + Module.HEAPU8.set(buffer, out); + ccall("_emrtc_on_message", + "void", + ["number", "number", "number", "number"], + [c_ptr, out, len, is_string] + ); + Module._free(out); + } + }, p_id); + /* clang-format on */ +} +} + +void _emrtc_create_pc(int p_id) { + /* clang-format off */ + EM_ASM({ + var dict = Module.IDHandler.get($0); + var c_ptr = dict["ptr"]; + // Setup local connaction + var conn = new RTCPeerConnection(); + conn.onicecandidate = function(event) { + if (!Module.IDHandler.get($0)) return; + if (!event.candidate) return; + + var c = event.candidate; + // should emit on ice candidate + ccall("_emrtc_on_ice_candidate", + "void", + ["number", "string", "number", "string"], + [c_ptr, c.sdpMid, c.sdpMLineIndex, c.candidate] + ); + }; + conn.ondatachannel = function (evt) { + var dict = Module.IDHandler.get($0); + if (!dict || dict["channel"]) { + return; + } + var channel = evt.channel; + dict["channel"] = channel; + ccall("_emrtc_bind_channel", + "void", + ["number"], + [$0] + ); + }; + dict["conn"] = conn; + }, p_id); + /* clang-format on */ +} + +void WebRTCPeerJS::_on_open() { + in_buffer.resize(16); + _conn_state = STATE_CONNECTED; +} + +void WebRTCPeerJS::_on_close() { + close(); +} + +void WebRTCPeerJS::_on_error() { + close(); + _conn_state = STATE_FAILED; +} + +void WebRTCPeerJS::_on_message(uint8_t *p_data, uint32_t p_size, bool p_is_string) { + if (in_buffer.space_left() < p_size + 5) { + ERR_EXPLAIN("Buffer full! Dropping data"); + ERR_FAIL(); + } + + uint8_t is_string = p_is_string ? 1 : 0; + in_buffer.write((uint8_t *)&p_size, 4); + in_buffer.write((uint8_t *)&is_string, 1); + in_buffer.write(p_data, p_size); + queue_count++; +} + +void WebRTCPeerJS::close() { + in_buffer.resize(0); + queue_count = 0; + _was_string = false; + /* clang-format off */ + EM_ASM({ + var dict = Module.IDHandler.get($0); + if (!dict) return; + if (dict["channel"]) { + dict["channel"].close(); + dict["channel"] = null; + } + if (dict["conn"]) { + dict["conn"].close(); + } + }, _js_id); + /* clang-format on */ + _conn_state = STATE_CLOSED; +} + +Error WebRTCPeerJS::create_offer() { + ERR_FAIL_COND_V(_conn_state != STATE_NEW, FAILED); + + _conn_state = STATE_CONNECTING; + /* clang-format off */ + EM_ASM({ + var dict = Module.IDHandler.get($0); + var conn = dict["conn"]; + var c_ptr = dict["ptr"]; + var onError = function(error) { + console.log(error); + ccall("_emrtc_on_error", + "void", + ["number"], + [c_ptr] + ); + }; + var onCreated = function(offer) { + ccall("_emrtc_offer_created", + "void", + ["number", "string", "string"], + [c_ptr, offer.type, offer.sdp] + ); + }; + + var channel = conn.createDataChannel("default"); + dict["channel"] = channel; + ccall("_emrtc_bind_channel", + "void", + ["number"], + [$0] + ); + conn.createOffer().then(onCreated).catch(onError); + }, _js_id); + /* clang-format on */ + return OK; +} + +Error WebRTCPeerJS::set_local_description(String type, String sdp) { + /* clang-format off */ + EM_ASM({ + var dict = Module.IDHandler.get($0); + var conn = dict["conn"]; + var c_ptr = dict["ptr"]; + var type = UTF8ToString($1); + var sdp = UTF8ToString($2); + var onError = function(error) { + console.log(error); + ccall("_emrtc_on_error", + "void", + ["number"], + [c_ptr] + ); + }; + conn.setLocalDescription({ + "sdp": sdp, + "type": type + }).catch(onError); + }, _js_id, type.utf8().get_data(), sdp.utf8().get_data()); + /* clang-format on */ + return OK; +} + +Error WebRTCPeerJS::set_remote_description(String type, String sdp) { + if (type == "offer") { + ERR_FAIL_COND_V(_conn_state != STATE_NEW, FAILED); + _conn_state = STATE_CONNECTING; + } + /* clang-format off */ + EM_ASM({ + var dict = Module.IDHandler.get($0); + var conn = dict["conn"]; + var c_ptr = dict["ptr"]; + var type = UTF8ToString($1); + var sdp = UTF8ToString($2); + + var onError = function(error) { + console.log(error); + ccall("_emrtc_on_error", + "void", + ["number"], + [c_ptr] + ); + }; + var onCreated = function(offer) { + ccall("_emrtc_offer_created", + "void", + ["number", "string", "string"], + [c_ptr, offer.type, offer.sdp] + ); + }; + var onSet = function() { + if (type != "offer") { + return; + } + conn.createAnswer().then(onCreated); + }; + conn.setRemoteDescription({ + "sdp": sdp, + "type": type + }).then(onSet).catch(onError); + }, _js_id, type.utf8().get_data(), sdp.utf8().get_data()); + /* clang-format on */ + return OK; +} + +Error WebRTCPeerJS::add_ice_candidate(String sdpMidName, int sdpMlineIndexName, String sdpName) { + /* clang-format off */ + EM_ASM({ + var dict = Module.IDHandler.get($0); + var conn = dict["conn"]; + var c_ptr = dict["ptr"]; + var sdpMidName = UTF8ToString($1); + var sdpMlineIndexName = UTF8ToString($2); + var sdpName = UTF8ToString($3); + conn.addIceCandidate(new RTCIceCandidate({ + "candidate": sdpName, + "sdpMid": sdpMidName, + "sdpMlineIndex": sdpMlineIndexName + })); + }, _js_id, sdpMidName.utf8().get_data(), sdpMlineIndexName, sdpName.utf8().get_data()); + /* clang-format on */ + return OK; +} + +Error WebRTCPeerJS::poll() { + return OK; +} + +WebRTCPeer::ConnectionState WebRTCPeerJS::get_connection_state() const { + return _conn_state; +} + +int WebRTCPeerJS::get_available_packet_count() const { + return queue_count; +} + +Error WebRTCPeerJS::get_packet(const uint8_t **r_buffer, int &r_buffer_size) { + ERR_FAIL_COND_V(_conn_state != STATE_CONNECTED, ERR_UNCONFIGURED); + + if (queue_count == 0) + return ERR_UNAVAILABLE; + + uint32_t to_read = 0; + uint32_t left = 0; + uint8_t is_string = 0; + r_buffer_size = 0; + + in_buffer.read((uint8_t *)&to_read, 4); + --queue_count; + left = in_buffer.data_left(); + + if (left < to_read + 1) { + in_buffer.advance_read(left); + return FAILED; + } + + in_buffer.read(&is_string, 1); + _was_string = is_string == 1; + in_buffer.read(packet_buffer, to_read); + *r_buffer = packet_buffer; + r_buffer_size = to_read; + + return OK; +} + +Error WebRTCPeerJS::put_packet(const uint8_t *p_buffer, int p_buffer_size) { + ERR_FAIL_COND_V(_conn_state != STATE_CONNECTED, ERR_UNCONFIGURED); + + int is_bin = _write_mode == WebRTCPeer::WRITE_MODE_BINARY ? 1 : 0; + + /* clang-format off */ + EM_ASM({ + var dict = Module.IDHandler.get($0); + var channel = dict["channel"]; + var bytes_array = new Uint8Array($2); + var i = 0; + + for(i=0; i<$2; i++) { + bytes_array[i] = getValue($1+i, 'i8'); + } + + if ($3) { + channel.send(bytes_array.buffer); + } else { + var string = new TextDecoder("utf-8").decode(bytes_array); + channel.send(string); + } + }, _js_id, p_buffer, p_buffer_size, is_bin); + /* clang-format on */ + + return OK; +} + +int WebRTCPeerJS::get_max_packet_size() const { + return 1200; +} + +void WebRTCPeerJS::set_write_mode(WriteMode p_mode) { + _write_mode = p_mode; +} + +WebRTCPeer::WriteMode WebRTCPeerJS::get_write_mode() const { + return _write_mode; +} + +bool WebRTCPeerJS::was_string_packet() const { + return _was_string; +} + +WebRTCPeerJS::WebRTCPeerJS() { + queue_count = 0; + _was_string = false; + _write_mode = WRITE_MODE_BINARY; + _conn_state = STATE_NEW; + + /* clang-format off */ + _js_id = EM_ASM_INT({ + return Module.IDHandler.add({"conn": null, "ptr": $0, "channel": null}); + }, this); + /* clang-format on */ + _emrtc_create_pc(_js_id); +} + +WebRTCPeerJS::~WebRTCPeerJS() { + close(); + /* clang-format off */ + EM_ASM({ + Module.IDHandler.remove($0); + }, _js_id); + /* clang-format on */ +}; +#endif diff --git a/modules/webrtc/webrtc_peer_js.h b/modules/webrtc/webrtc_peer_js.h new file mode 100644 index 0000000000..02f0c9b55d --- /dev/null +++ b/modules/webrtc/webrtc_peer_js.h @@ -0,0 +1,88 @@ +/*************************************************************************/ +/* webrtc_peer_js.h */ +/*************************************************************************/ +/* This file is part of: */ +/* GODOT ENGINE */ +/* https://godotengine.org */ +/*************************************************************************/ +/* Copyright (c) 2007-2018 Juan Linietsky, Ariel Manzur. */ +/* Copyright (c) 2014-2018 Godot Engine contributors (cf. AUTHORS.md) */ +/* */ +/* Permission is hereby granted, free of charge, to any person obtaining */ +/* a copy of this software and associated documentation files (the */ +/* "Software"), to deal in the Software without restriction, including */ +/* without limitation the rights to use, copy, modify, merge, publish, */ +/* distribute, sublicense, and/or sell copies of the Software, and to */ +/* permit persons to whom the Software is furnished to do so, subject to */ +/* the following conditions: */ +/* */ +/* The above copyright notice and this permission notice shall be */ +/* included in all copies or substantial portions of the Software. */ +/* */ +/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */ +/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */ +/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*/ +/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */ +/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */ +/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */ +/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +/*************************************************************************/ + +#ifndef WEBRTC_PEER_JS_H +#define WEBRTC_PEER_JS_H + +#ifdef JAVASCRIPT_ENABLED + +#include "webrtc_peer.h" + +class WebRTCPeerJS : public WebRTCPeer { + +private: + enum { + PACKET_BUFFER_SIZE = 65536 - 5 // 4 bytes for the size, 1 for for type + }; + + bool _was_string; + WriteMode _write_mode; + + int _js_id; + RingBuffer<uint8_t> in_buffer; + int queue_count; + uint8_t packet_buffer[PACKET_BUFFER_SIZE]; + ConnectionState _conn_state; + +public: + static WebRTCPeer *_create() { return memnew(WebRTCPeerJS); } + static void make_default() { WebRTCPeer::_create = WebRTCPeerJS::_create; } + + virtual void set_write_mode(WriteMode mode); + virtual WriteMode get_write_mode() const; + virtual bool was_string_packet() const; + virtual ConnectionState get_connection_state() const; + + virtual Error create_offer(); + virtual Error set_remote_description(String type, String sdp); + virtual Error set_local_description(String type, String sdp); + virtual Error add_ice_candidate(String sdpMidName, int sdpMlineIndexName, String sdpName); + virtual Error poll(); + + /** Inherited from PacketPeer: **/ + virtual int get_available_packet_count() const; + virtual Error get_packet(const uint8_t **r_buffer, int &r_buffer_size); ///< buffer is GONE after next get_packet + virtual Error put_packet(const uint8_t *p_buffer, int p_buffer_size); + + virtual int get_max_packet_size() const; + + void close(); + void _on_open(); + void _on_close(); + void _on_error(); + void _on_message(uint8_t *p_data, uint32_t p_size, bool p_is_string); + + WebRTCPeerJS(); + ~WebRTCPeerJS(); +}; + +#endif + +#endif // WEBRTC_PEER_JS_H diff --git a/modules/websocket/register_types.cpp b/modules/websocket/register_types.cpp index ed6cc5638e..39bf3de982 100644 --- a/modules/websocket/register_types.cpp +++ b/modules/websocket/register_types.cpp @@ -60,25 +60,6 @@ void register_websocket_types() { _SET_HINT(WSS_OUT_PKT, 1024, 16384); #ifdef JAVASCRIPT_ENABLED - EM_ASM({ - var IDHandler = {}; - IDHandler["ids"] = {}; - IDHandler["has"] = function(id) { - return IDHandler.ids.hasOwnProperty(id); - }; - IDHandler["add"] = function(obj) { - var id = crypto.getRandomValues(new Int32Array(32))[0]; - IDHandler.ids[id] = obj; - return id; - }; - IDHandler["get"] = function(id) { - return IDHandler.ids[id]; - }; - IDHandler["remove"] = function(id) { - delete IDHandler.ids[id]; - }; - Module["IDHandler"] = IDHandler; - }); EMWSPeer::make_default(); EMWSClient::make_default(); EMWSServer::make_default(); diff --git a/platform/android/java/src/org/godotengine/godot/Godot.java b/platform/android/java/src/org/godotengine/godot/Godot.java index a9edc2f2f4..5ef72bab6c 100644 --- a/platform/android/java/src/org/godotengine/godot/Godot.java +++ b/platform/android/java/src/org/godotengine/godot/Godot.java @@ -425,7 +425,7 @@ public class Godot extends Activity implements SensorEventListener, IDownloaderC } io = new GodotIO(this); - io.unique_id = Secure.ANDROID_ID; + io.unique_id = Secure.getString(getContentResolver(), Secure.ANDROID_ID); GodotLib.io = io; mSensorManager = (SensorManager)getSystemService(Context.SENSOR_SERVICE); mAccelerometer = mSensorManager.getDefaultSensor(Sensor.TYPE_ACCELEROMETER); diff --git a/platform/android/os_android.cpp b/platform/android/os_android.cpp index d65cf2f31e..93d39859f2 100644 --- a/platform/android/os_android.cpp +++ b/platform/android/os_android.cpp @@ -292,14 +292,6 @@ bool OS_Android::can_draw() const { return true; //always? } -void OS_Android::set_cursor_shape(CursorShape p_shape) { - - //android really really really has no mouse.. how amazing.. -} - -void OS_Android::set_custom_mouse_cursor(const RES &p_cursor, CursorShape p_shape, const Vector2 &p_hotspot) { -} - void OS_Android::main_loop_begin() { if (main_loop) diff --git a/platform/android/os_android.h b/platform/android/os_android.h index 3a5404124a..d2198b0579 100644 --- a/platform/android/os_android.h +++ b/platform/android/os_android.h @@ -144,9 +144,6 @@ public: virtual bool can_draw() const; - virtual void set_cursor_shape(CursorShape p_shape); - virtual void set_custom_mouse_cursor(const RES &p_cursor, CursorShape p_shape, const Vector2 &p_hotspot); - void main_loop_begin(); bool main_loop_iterate(); void main_loop_request_go_back(); diff --git a/platform/haiku/os_haiku.cpp b/platform/haiku/os_haiku.cpp index a6d5a00852..f3fed6669b 100644 --- a/platform/haiku/os_haiku.cpp +++ b/platform/haiku/os_haiku.cpp @@ -203,6 +203,10 @@ void OS_Haiku::set_cursor_shape(CursorShape p_shape) { //ERR_PRINT("set_cursor_shape() NOT IMPLEMENTED"); } +OS::CursorShape OS_Haiku::get_cursor_shape() const { + // TODO: implement get_cursor_shape +} + void OS_Haiku::set_custom_mouse_cursor(const RES &p_cursor, CursorShape p_shape, const Vector2 &p_hotspot) { // TODO } diff --git a/platform/haiku/os_haiku.h b/platform/haiku/os_haiku.h index d7eac10635..6ab006843a 100644 --- a/platform/haiku/os_haiku.h +++ b/platform/haiku/os_haiku.h @@ -86,6 +86,7 @@ public: virtual Point2 get_mouse_position() const; virtual int get_mouse_button_state() const; virtual void set_cursor_shape(CursorShape p_shape); + virtual CursorShape get_cursor_shape() const; virtual void set_custom_mouse_cursor(const RES &p_cursor, CursorShape p_shape, const Vector2 &p_hotspot); virtual int get_screen_count() const; diff --git a/platform/iphone/os_iphone.cpp b/platform/iphone/os_iphone.cpp index 7d0fdd2078..e25efc813b 100644 --- a/platform/iphone/os_iphone.cpp +++ b/platform/iphone/os_iphone.cpp @@ -490,17 +490,11 @@ void OSIPhone::set_keep_screen_on(bool p_enabled) { _set_keep_screen_on(p_enabled); }; -void OSIPhone::set_cursor_shape(CursorShape p_shape){ - -}; - String OSIPhone::get_user_data_dir() const { return data_dir; }; -void OSIPhone::set_custom_mouse_cursor(const RES &p_cursor, CursorShape p_shape, const Vector2 &p_hotspot){}; - String OSIPhone::get_name() { return "iOS"; diff --git a/platform/iphone/os_iphone.h b/platform/iphone/os_iphone.h index 30d7a1ba41..49c6475cf9 100644 --- a/platform/iphone/os_iphone.h +++ b/platform/iphone/os_iphone.h @@ -167,9 +167,6 @@ public: virtual void hide_virtual_keyboard(); virtual int get_virtual_keyboard_height() const; - virtual void set_cursor_shape(CursorShape p_shape); - virtual void set_custom_mouse_cursor(const RES &p_cursor, CursorShape p_shape, const Vector2 &p_hotspot); - virtual Size2 get_window_size() const; virtual Rect2 get_window_safe_area() const; diff --git a/platform/javascript/SCsub b/platform/javascript/SCsub index a93c98a89f..85a633442e 100644 --- a/platform/javascript/SCsub +++ b/platform/javascript/SCsub @@ -20,6 +20,13 @@ for lib in js_libraries: env.Append(LINKFLAGS=['--js-library', env.File(lib).path]) env.Depends(build, js_libraries) +js_modules = [ + 'id_handler.js', +] +for module in js_modules: + env.Append(LINKFLAGS=['--pre-js', env.File(module).path]) +env.Depends(build, js_modules) + wrapper_start = env.File('pre.js') wrapper_end = env.File('engine.js') js_wrapped = env.Textfile('#bin/godot', [wrapper_start, js, wrapper_end], TEXTFILESUFFIX='${PROGSUFFIX}.wrapped.js') diff --git a/platform/javascript/id_handler.js b/platform/javascript/id_handler.js new file mode 100644 index 0000000000..36ef5aa8ef --- /dev/null +++ b/platform/javascript/id_handler.js @@ -0,0 +1,62 @@ +/*************************************************************************/ +/* id_handler.js */ +/*************************************************************************/ +/* This file is part of: */ +/* GODOT ENGINE */ +/* http://www.godotengine.org */ +/*************************************************************************/ +/* Copyright (c) 2007-2018 Juan Linietsky, Ariel Manzur. */ +/* */ +/* Permission is hereby granted, free of charge, to any person obtaining */ +/* a copy of this software and associated documentation files (the */ +/* "Software"), to deal in the Software without restriction, including */ +/* without limitation the rights to use, copy, modify, merge, publish, */ +/* distribute, sublicense, and/or sell copies of the Software, and to */ +/* permit persons to whom the Software is furnished to do so, subject to */ +/* the following conditions: */ +/* */ +/* The above copyright notice and this permission notice shall be */ +/* included in all copies or substantial portions of the Software. */ +/* */ +/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */ +/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */ +/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*/ +/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */ +/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */ +/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */ +/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +/*************************************************************************/ + +var IDHandler = function() { + + var ids = {}; + var size = 0; + + this.has = function(id) { + return ids.hasOwnProperty(id); + } + + this.add = function(obj) { + size += 1; + var id = crypto.getRandomValues(new Int32Array(32))[0]; + ids[id] = obj; + return id; + } + + this.get = function(id) { + return ids[id]; + } + + this.remove = function(id) { + size -= 1; + delete ids[id]; + } + + this.size = function() { + return size; + } + + this.ids = ids; +}; + +Module.IDHandler = new IDHandler; diff --git a/platform/osx/os_osx.h b/platform/osx/os_osx.h index dfe7b27bd0..125a88ab6d 100644 --- a/platform/osx/os_osx.h +++ b/platform/osx/os_osx.h @@ -172,6 +172,7 @@ public: virtual Error open_dynamic_library(const String p_path, void *&p_library_handle, bool p_also_set_library_path = false); virtual void set_cursor_shape(CursorShape p_shape); + virtual CursorShape get_cursor_shape() const; virtual void set_custom_mouse_cursor(const RES &p_cursor, CursorShape p_shape, const Vector2 &p_hotspot); virtual void set_mouse_show(bool p_show); diff --git a/platform/osx/os_osx.mm b/platform/osx/os_osx.mm index 4bbe30a7d1..5a4c0593bc 100644 --- a/platform/osx/os_osx.mm +++ b/platform/osx/os_osx.mm @@ -712,8 +712,6 @@ static void _mouseDownEvent(NSEvent *event, int index, int mask, bool pressed) { if (OS_OSX::singleton->main_loop && OS_OSX::singleton->mouse_mode != OS::MOUSE_MODE_CAPTURED) OS_OSX::singleton->main_loop->notification(MainLoop::NOTIFICATION_WM_MOUSE_EXIT); - if (OS_OSX::singleton->input) - OS_OSX::singleton->input->set_mouse_in_window(false); } - (void)mouseEntered:(NSEvent *)event { @@ -721,8 +719,6 @@ static void _mouseDownEvent(NSEvent *event, int index, int mask, bool pressed) { return; if (OS_OSX::singleton->main_loop && OS_OSX::singleton->mouse_mode != OS::MOUSE_MODE_CAPTURED) OS_OSX::singleton->main_loop->notification(MainLoop::NOTIFICATION_WM_MOUSE_ENTER); - if (OS_OSX::singleton->input) - OS_OSX::singleton->input->set_mouse_in_window(true); OS::CursorShape p_shape = OS_OSX::singleton->cursor_shape; OS_OSX::singleton->cursor_shape = OS::CURSOR_MAX; @@ -1696,6 +1692,11 @@ void OS_OSX::set_cursor_shape(CursorShape p_shape) { cursor_shape = p_shape; } +OS::CursorShape OS_OSX::get_cursor_shape() const { + + return cursor_shape; +} + void OS_OSX::set_custom_mouse_cursor(const RES &p_cursor, CursorShape p_shape, const Vector2 &p_hotspot) { if (p_cursor.is_valid()) { Ref<Texture> texture = p_cursor; diff --git a/platform/server/os_server.cpp b/platform/server/os_server.cpp index e643d3e8bb..53f2a65c8e 100644 --- a/platform/server/os_server.cpp +++ b/platform/server/os_server.cpp @@ -198,12 +198,6 @@ String OS_Server::get_name() { void OS_Server::move_window_to_foreground() { } -void OS_Server::set_cursor_shape(CursorShape p_shape) { -} - -void OS_Server::set_custom_mouse_cursor(const RES &p_cursor, CursorShape p_shape, const Vector2 &p_hotspot) { -} - OS::PowerState OS_Server::get_power_state() { return power_manager->get_power_state(); } diff --git a/platform/server/os_server.h b/platform/server/os_server.h index eebe8ae777..7441064790 100644 --- a/platform/server/os_server.h +++ b/platform/server/os_server.h @@ -95,9 +95,6 @@ protected: public: virtual String get_name(); - virtual void set_cursor_shape(CursorShape p_shape); - virtual void set_custom_mouse_cursor(const RES &p_cursor, CursorShape p_shape, const Vector2 &p_hotspot); - virtual void set_mouse_show(bool p_show); virtual void set_mouse_grab(bool p_grab); virtual bool is_mouse_grab_enabled() const; diff --git a/platform/uwp/os_uwp.cpp b/platform/uwp/os_uwp.cpp index bc74da8a1b..82f09032f5 100644 --- a/platform/uwp/os_uwp.cpp +++ b/platform/uwp/os_uwp.cpp @@ -704,6 +704,11 @@ void OS_UWP::set_cursor_shape(CursorShape p_shape) { cursor_shape = p_shape; } +OS::CursorShape OS_UWP::get_cursor_shape() const { + + return cursor_shape; +} + void OS_UWP::set_custom_mouse_cursor(const RES &p_cursor, CursorShape p_shape, const Vector2 &p_hotspot) { // TODO } diff --git a/platform/uwp/os_uwp.h b/platform/uwp/os_uwp.h index fd78b3cdf7..00f79efb04 100644 --- a/platform/uwp/os_uwp.h +++ b/platform/uwp/os_uwp.h @@ -219,6 +219,7 @@ public: virtual String get_clipboard() const; void set_cursor_shape(CursorShape p_shape); + CursorShape get_cursor_shape() const; virtual void set_custom_mouse_cursor(const RES &p_cursor, CursorShape p_shape, const Vector2 &p_hotspot); void set_icon(const Ref<Image> &p_icon); diff --git a/platform/windows/os_windows.cpp b/platform/windows/os_windows.cpp index 093fb05bae..1a5050a15a 100644 --- a/platform/windows/os_windows.cpp +++ b/platform/windows/os_windows.cpp @@ -384,8 +384,6 @@ LRESULT OS_Windows::WndProc(HWND hWnd, UINT uMsg, WPARAM wParam, LPARAM lParam) outside = true; if (main_loop && mouse_mode != MOUSE_MODE_CAPTURED) main_loop->notification(MainLoop::NOTIFICATION_WM_MOUSE_EXIT); - if (input) - input->set_mouse_in_window(false); } break; case WM_INPUT: { @@ -480,8 +478,6 @@ LRESULT OS_Windows::WndProc(HWND hWnd, UINT uMsg, WPARAM wParam, LPARAM lParam) if (main_loop && mouse_mode != MOUSE_MODE_CAPTURED) main_loop->notification(MainLoop::NOTIFICATION_WM_MOUSE_ENTER); - if (input) - input->set_mouse_in_window(true); CursorShape c = cursor_shape; cursor_shape = CURSOR_MAX; @@ -2299,6 +2295,11 @@ void OS_Windows::set_cursor_shape(CursorShape p_shape) { cursor_shape = p_shape; } +OS::CursorShape OS_Windows::get_cursor_shape() const { + + return cursor_shape; +} + void OS_Windows::set_custom_mouse_cursor(const RES &p_cursor, CursorShape p_shape, const Vector2 &p_hotspot) { if (p_cursor.is_valid()) { Ref<Texture> texture = p_cursor; diff --git a/platform/windows/os_windows.h b/platform/windows/os_windows.h index 2d03532c69..c15e1cabc3 100644 --- a/platform/windows/os_windows.h +++ b/platform/windows/os_windows.h @@ -273,6 +273,7 @@ public: virtual String get_clipboard() const; void set_cursor_shape(CursorShape p_shape); + CursorShape get_cursor_shape() const; virtual void set_custom_mouse_cursor(const RES &p_cursor, CursorShape p_shape, const Vector2 &p_hotspot); void GetMaskBitmaps(HBITMAP hSourceBitmap, COLORREF clrTransparent, OUT HBITMAP &hAndMaskBitmap, OUT HBITMAP &hXorMaskBitmap); void set_icon(const Ref<Image> &p_icon); diff --git a/platform/x11/os_x11.cpp b/platform/x11/os_x11.cpp index ba74ba2437..c96047dfbd 100644 --- a/platform/x11/os_x11.cpp +++ b/platform/x11/os_x11.cpp @@ -2041,15 +2041,11 @@ void OS_X11::process_xevents() { case LeaveNotify: { if (main_loop && !mouse_mode_grab) main_loop->notification(MainLoop::NOTIFICATION_WM_MOUSE_EXIT); - if (input) - input->set_mouse_in_window(false); } break; case EnterNotify: { if (main_loop && !mouse_mode_grab) main_loop->notification(MainLoop::NOTIFICATION_WM_MOUSE_ENTER); - if (input) - input->set_mouse_in_window(true); } break; case FocusIn: minimized = false; @@ -2722,6 +2718,11 @@ void OS_X11::set_cursor_shape(CursorShape p_shape) { current_cursor = p_shape; } +OS::CursorShape OS_X11::get_cursor_shape() const { + + return current_cursor; +} + void OS_X11::set_custom_mouse_cursor(const RES &p_cursor, CursorShape p_shape, const Vector2 &p_hotspot) { if (p_cursor.is_valid()) { Ref<Texture> texture = p_cursor; diff --git a/platform/x11/os_x11.h b/platform/x11/os_x11.h index 6d1a66af84..a54851d4e7 100644 --- a/platform/x11/os_x11.h +++ b/platform/x11/os_x11.h @@ -219,6 +219,7 @@ public: virtual String get_name(); virtual void set_cursor_shape(CursorShape p_shape); + virtual CursorShape get_cursor_shape() const; virtual void set_custom_mouse_cursor(const RES &p_cursor, CursorShape p_shape, const Vector2 &p_hotspot); void set_mouse_mode(MouseMode p_mode); diff --git a/scene/2d/area_2d.cpp b/scene/2d/area_2d.cpp index 2a225e5797..b322cfe8f1 100644 --- a/scene/2d/area_2d.cpp +++ b/scene/2d/area_2d.cpp @@ -261,8 +261,9 @@ void Area2D::_area_inout(int p_status, const RID &p_area, int p_instance, int p_ Map<ObjectID, AreaState>::Element *E = area_map.find(objid); - ERR_FAIL_COND(!area_in && !E); - + if (!area_in && !E) { + return; //likely removed from the tree + } locked = true; if (area_in) { diff --git a/scene/2d/light_occluder_2d.cpp b/scene/2d/light_occluder_2d.cpp index 5334caed67..3a3f90ac4b 100644 --- a/scene/2d/light_occluder_2d.cpp +++ b/scene/2d/light_occluder_2d.cpp @@ -32,9 +32,59 @@ #include "core/engine.h" +#define LINE_GRAB_WIDTH 8 +Rect2 OccluderPolygon2D::_edit_get_rect() const { + + if (rect_cache_dirty) { + if (closed) { + PoolVector<Vector2>::Read r = polygon.read(); + item_rect = Rect2(); + for (int i = 0; i < polygon.size(); i++) { + Vector2 pos = r[i]; + if (i == 0) + item_rect.position = pos; + else + item_rect.expand_to(pos); + } + rect_cache_dirty = false; + } else { + if (polygon.size() == 0) { + item_rect = Rect2(); + } else { + Vector2 d = Vector2(LINE_GRAB_WIDTH, LINE_GRAB_WIDTH); + item_rect = Rect2(polygon[0] - d, 2 * d); + for (int i = 1; i < polygon.size(); i++) { + item_rect.expand_to(polygon[i] - d); + item_rect.expand_to(polygon[i] + d); + } + } + } + } + + return item_rect; +} + +bool OccluderPolygon2D::_edit_is_selected_on_click(const Point2 &p_point, double p_tolerance) const { + + if (closed) { + return Geometry::is_point_in_polygon(p_point, Variant(polygon)); + } else { + const real_t d = LINE_GRAB_WIDTH / 2 + p_tolerance; + PoolVector<Vector2>::Read points = polygon.read(); + for (int i = 0; i < polygon.size() - 1; i++) { + Vector2 p = Geometry::get_closest_point_to_segment_2d(p_point, &points[i]); + if (p.distance_to(p_point) <= d) + return true; + } + + return false; + } +} + void OccluderPolygon2D::set_polygon(const PoolVector<Vector2> &p_polygon) { polygon = p_polygon; + rect_cache_dirty = true; VS::get_singleton()->canvas_occluder_polygon_set_shape(occ_polygon, p_polygon, closed); emit_changed(); } @@ -100,6 +150,7 @@ OccluderPolygon2D::OccluderPolygon2D() { occ_polygon = VS::get_singleton()->canvas_occluder_polygon_create(); closed = true; cull = CULL_DISABLED; + rect_cache_dirty = true; } OccluderPolygon2D::~OccluderPolygon2D() { @@ -164,6 +215,16 @@ void LightOccluder2D::_notification(int p_what) { } } +Rect2 LightOccluder2D::_edit_get_rect() const { + + return occluder_polygon.is_valid() ? occluder_polygon->_edit_get_rect() : Rect2(); +} + +bool LightOccluder2D::_edit_is_selected_on_click(const Point2 &p_point, double p_tolerance) const { + + return occluder_polygon.is_valid() ? occluder_polygon->_edit_is_selected_on_click(p_point, p_tolerance) : false; +} + void LightOccluder2D::set_occluder_polygon(const Ref<OccluderPolygon2D> &p_polygon) { #ifdef DEBUG_ENABLED diff --git a/scene/2d/light_occluder_2d.h b/scene/2d/light_occluder_2d.h index 498d65d764..7931cd0b30 100644 --- a/scene/2d/light_occluder_2d.h +++ b/scene/2d/light_occluder_2d.h @@ -50,10 +50,16 @@ private: bool closed; CullMode cull; + mutable Rect2 item_rect; + mutable bool rect_cache_dirty; + protected: static void _bind_methods(); public: + virtual Rect2 _edit_get_rect() const; + virtual bool _edit_is_selected_on_click(const Point2 &p_point, double p_tolerance) const; + void set_polygon(const PoolVector<Vector2> &p_polygon); PoolVector<Vector2> get_polygon() const; @@ -85,6 +91,9 @@ protected: static void _bind_methods(); public: + virtual Rect2 _edit_get_rect() const; + virtual bool _edit_is_selected_on_click(const Point2 &p_point, double p_tolerance) const; + void set_occluder_polygon(const Ref<OccluderPolygon2D> &p_polygon); Ref<OccluderPolygon2D> get_occluder_polygon() const; diff --git a/scene/3d/area.cpp b/scene/3d/area.cpp index 13d9181082..3557f0425c 100644 --- a/scene/3d/area.cpp +++ b/scene/3d/area.cpp @@ -356,7 +356,9 @@ void Area::_area_inout(int p_status, const RID &p_area, int p_instance, int p_ar Map<ObjectID, AreaState>::Element *E = area_map.find(objid); - ERR_FAIL_COND(!area_in && !E); + if (!area_in && !E) { + return; //likely removed from the tree + } locked = true; diff --git a/scene/animation/animation_node_state_machine.cpp b/scene/animation/animation_node_state_machine.cpp index 68ad71d01c..1e3470cd90 100644 --- a/scene/animation/animation_node_state_machine.cpp +++ b/scene/animation/animation_node_state_machine.cpp @@ -960,7 +960,7 @@ void AnimationNodeStateMachine::_bind_methods() { ClassDB::bind_method(D_METHOD("set_end_node", "name"), &AnimationNodeStateMachine::set_end_node); ClassDB::bind_method(D_METHOD("get_end_node"), &AnimationNodeStateMachine::get_end_node); - ClassDB::bind_method(D_METHOD("set_graph_offset", "name"), &AnimationNodeStateMachine::set_graph_offset); + ClassDB::bind_method(D_METHOD("set_graph_offset", "offset"), &AnimationNodeStateMachine::set_graph_offset); ClassDB::bind_method(D_METHOD("get_graph_offset"), &AnimationNodeStateMachine::get_graph_offset); ClassDB::bind_method(D_METHOD("_tree_changed"), &AnimationNodeStateMachine::_tree_changed); diff --git a/scene/gui/slider.cpp b/scene/gui/slider.cpp index eb04b85931..028ca41cbf 100644 --- a/scene/gui/slider.cpp +++ b/scene/gui/slider.cpp @@ -34,8 +34,15 @@ Size2 Slider::get_minimum_size() const { Ref<StyleBox> style = get_stylebox("slider"); - Size2i ms = style->get_minimum_size() + style->get_center_size(); - return ms; + Size2i ss = style->get_minimum_size() + style->get_center_size(); + + Ref<Texture> grabber = get_icon("grabber"); + Size2i rs = grabber->get_size(); + + if (orientation == HORIZONTAL) + return Size2i(ss.width, MAX(ss.height, rs.height)); + else + return Size2i(MAX(ss.width, rs.width), ss.height); } void Slider::_gui_input(Ref<InputEvent> p_event) { @@ -134,7 +141,11 @@ void Slider::_gui_input(Ref<InputEvent> p_event) { void Slider::_notification(int p_what) { switch (p_what) { + case NOTIFICATION_THEME_CHANGED: { + minimum_size_changed(); + update(); + } break; case NOTIFICATION_MOUSE_ENTER: { mouse_inside = true; diff --git a/servers/audio/effects/audio_effect_spectrum_analyzer.cpp b/servers/audio/effects/audio_effect_spectrum_analyzer.cpp index 71a8904472..01a52aa01f 100644 --- a/servers/audio/effects/audio_effect_spectrum_analyzer.cpp +++ b/servers/audio/effects/audio_effect_spectrum_analyzer.cpp @@ -240,6 +240,13 @@ void AudioEffectSpectrumAnalyzer::_bind_methods() { ADD_PROPERTY(PropertyInfo(Variant::REAL, "buffer_length", PROPERTY_HINT_RANGE, "0.1,4,0.1"), "set_buffer_length", "get_buffer_length"); ADD_PROPERTY(PropertyInfo(Variant::REAL, "tap_back_pos", PROPERTY_HINT_RANGE, "0.1,4,0.1"), "set_tap_back_pos", "get_tap_back_pos"); ADD_PROPERTY(PropertyInfo(Variant::INT, "fft_size", PROPERTY_HINT_ENUM, "256,512,1024,2048,4096"), "set_fft_size", "get_fft_size"); + + BIND_ENUM_CONSTANT(FFT_SIZE_256); + BIND_ENUM_CONSTANT(FFT_SIZE_512); + BIND_ENUM_CONSTANT(FFT_SIZE_1024); + BIND_ENUM_CONSTANT(FFT_SIZE_2048); + BIND_ENUM_CONSTANT(FFT_SIZE_4096); + BIND_ENUM_CONSTANT(FFT_SIZE_MAX); } AudioEffectSpectrumAnalyzer::AudioEffectSpectrumAnalyzer() { diff --git a/thirdparty/README.md b/thirdparty/README.md index 6997a9ba96..ebdd0b5d04 100644 --- a/thirdparty/README.md +++ b/thirdparty/README.md @@ -559,7 +559,7 @@ Files extracted from upstream source: ## zstd - Upstream: https://github.com/facebook/zstd -- Version: 1.3.8 +- Version: 1.4.0 - License: BSD-3-Clause Files extracted from upstream source: diff --git a/thirdparty/zstd/common/compiler.h b/thirdparty/zstd/common/compiler.h index 7f561282ca..0836e3ed27 100644 --- a/thirdparty/zstd/common/compiler.h +++ b/thirdparty/zstd/common/compiler.h @@ -40,7 +40,7 @@ /** * FORCE_INLINE_TEMPLATE is used to define C "templates", which take constant - * parameters. They must be inlined for the compiler to elimininate the constant + * parameters. They must be inlined for the compiler to eliminate the constant * branches. */ #define FORCE_INLINE_TEMPLATE static INLINE_KEYWORD FORCE_INLINE_ATTR diff --git a/thirdparty/zstd/common/fse.h b/thirdparty/zstd/common/fse.h index f72c519b25..811c670bdd 100644 --- a/thirdparty/zstd/common/fse.h +++ b/thirdparty/zstd/common/fse.h @@ -358,7 +358,7 @@ size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size typedef enum { FSE_repeat_none, /**< Cannot use the previous table */ FSE_repeat_check, /**< Can use the previous table but it must be checked */ - FSE_repeat_valid /**< Can use the previous table and it is asumed to be valid */ + FSE_repeat_valid /**< Can use the previous table and it is assumed to be valid */ } FSE_repeat; /* ***************************************** diff --git a/thirdparty/zstd/common/threading.c b/thirdparty/zstd/common/threading.c index 8be8c8da94..f3d4fa8418 100644 --- a/thirdparty/zstd/common/threading.c +++ b/thirdparty/zstd/common/threading.c @@ -14,8 +14,8 @@ * This file will hold wrapper for systems, which do not support pthreads */ -/* create fake symbol to avoid empty trnaslation unit warning */ -int g_ZSTD_threading_useles_symbol; +/* create fake symbol to avoid empty translation unit warning */ +int g_ZSTD_threading_useless_symbol; #if defined(ZSTD_MULTITHREAD) && defined(_WIN32) diff --git a/thirdparty/zstd/common/xxhash.c b/thirdparty/zstd/common/xxhash.c index 532b816192..30599aaae4 100644 --- a/thirdparty/zstd/common/xxhash.c +++ b/thirdparty/zstd/common/xxhash.c @@ -66,10 +66,10 @@ /* #define XXH_ACCEPT_NULL_INPUT_POINTER 1 */ /*!XXH_FORCE_NATIVE_FORMAT : - * By default, xxHash library provides endian-independant Hash values, based on little-endian convention. + * By default, xxHash library provides endian-independent Hash values, based on little-endian convention. * Results are therefore identical for little-endian and big-endian CPU. * This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format. - * Should endian-independance be of no importance for your application, you may set the #define below to 1, + * Should endian-independence be of no importance for your application, you may set the #define below to 1, * to improve speed for Big-endian CPU. * This option has no impact on Little_Endian CPU. */ diff --git a/thirdparty/zstd/common/zstd_internal.h b/thirdparty/zstd/common/zstd_internal.h index edeb74b9c3..31f756ab58 100644 --- a/thirdparty/zstd/common/zstd_internal.h +++ b/thirdparty/zstd/common/zstd_internal.h @@ -53,8 +53,50 @@ extern "C" { #undef MAX #define MIN(a,b) ((a)<(b) ? (a) : (b)) #define MAX(a,b) ((a)>(b) ? (a) : (b)) -#define CHECK_F(f) { size_t const errcod = f; if (ERR_isError(errcod)) return errcod; } /* check and Forward error code */ -#define CHECK_E(f, e) { size_t const errcod = f; if (ERR_isError(errcod)) return ERROR(e); } /* check and send Error code */ + +/** + * Return the specified error if the condition evaluates to true. + * + * In debug modes, prints additional information. In order to do that + * (particularly, printing the conditional that failed), this can't just wrap + * RETURN_ERROR(). + */ +#define RETURN_ERROR_IF(cond, err, ...) \ + if (cond) { \ + RAWLOG(3, "%s:%d: ERROR!: check %s failed, returning %s", __FILE__, __LINE__, ZSTD_QUOTE(cond), ZSTD_QUOTE(ERROR(err))); \ + RAWLOG(3, ": " __VA_ARGS__); \ + RAWLOG(3, "\n"); \ + return ERROR(err); \ + } + +/** + * Unconditionally return the specified error. + * + * In debug modes, prints additional information. + */ +#define RETURN_ERROR(err, ...) \ + do { \ + RAWLOG(3, "%s:%d: ERROR!: unconditional check failed, returning %s", __FILE__, __LINE__, ZSTD_QUOTE(ERROR(err))); \ + RAWLOG(3, ": " __VA_ARGS__); \ + RAWLOG(3, "\n"); \ + return ERROR(err); \ + } while(0); + +/** + * If the provided expression evaluates to an error code, returns that error code. + * + * In debug modes, prints additional information. + */ +#define FORWARD_IF_ERROR(err, ...) \ + do { \ + size_t const err_code = (err); \ + if (ERR_isError(err_code)) { \ + RAWLOG(3, "%s:%d: ERROR!: forwarding error in %s: %s", __FILE__, __LINE__, ZSTD_QUOTE(err), ERR_getErrorName(err_code)); \ + RAWLOG(3, ": " __VA_ARGS__); \ + RAWLOG(3, "\n"); \ + return err_code; \ + } \ + } while(0); /*-************************************* @@ -200,6 +242,17 @@ typedef struct { U32 longLengthPos; } seqStore_t; +/** + * Contains the compressed frame size and an upper-bound for the decompressed frame size. + * Note: before using `compressedSize`, check for errors using ZSTD_isError(). + * similarly, before using `decompressedBound`, check for errors using: + * `decompressedBound != ZSTD_CONTENTSIZE_ERROR` + */ +typedef struct { + size_t compressedSize; + unsigned long long decompressedBound; +} ZSTD_frameSizeInfo; /* decompress & legacy */ + const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx); /* compress & dictBuilder */ void ZSTD_seqToCodes(const seqStore_t* seqStorePtr); /* compress, dictBuilder, decodeCorpus (shouldn't get its definition from here) */ diff --git a/thirdparty/zstd/compress/fse_compress.c b/thirdparty/zstd/compress/fse_compress.c index 60f357bbd2..68b47e1093 100644 --- a/thirdparty/zstd/compress/fse_compress.c +++ b/thirdparty/zstd/compress/fse_compress.c @@ -129,9 +129,9 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct, { U32 position = 0; U32 symbol; for (symbol=0; symbol<=maxSymbolValue; symbol++) { - int nbOccurences; + int nbOccurrences; int const freq = normalizedCounter[symbol]; - for (nbOccurences=0; nbOccurences<freq; nbOccurences++) { + for (nbOccurrences=0; nbOccurrences<freq; nbOccurrences++) { tableSymbol[position] = (FSE_FUNCTION_TYPE)symbol; position = (position + step) & tableMask; while (position > highThreshold) diff --git a/thirdparty/zstd/compress/zstd_compress.c b/thirdparty/zstd/compress/zstd_compress.c index c2c9d3bc55..2e163c8bf3 100644 --- a/thirdparty/zstd/compress/zstd_compress.c +++ b/thirdparty/zstd/compress/zstd_compress.c @@ -103,12 +103,31 @@ ZSTD_CCtx* ZSTD_initStaticCCtx(void *workspace, size_t workspaceSize) return cctx; } +/** + * Clears and frees all of the dictionaries in the CCtx. + */ +static void ZSTD_clearAllDicts(ZSTD_CCtx* cctx) +{ + ZSTD_free(cctx->localDict.dictBuffer, cctx->customMem); + ZSTD_freeCDict(cctx->localDict.cdict); + memset(&cctx->localDict, 0, sizeof(cctx->localDict)); + memset(&cctx->prefixDict, 0, sizeof(cctx->prefixDict)); + cctx->cdict = NULL; +} + +static size_t ZSTD_sizeof_localDict(ZSTD_localDict dict) +{ + size_t const bufferSize = dict.dictBuffer != NULL ? dict.dictSize : 0; + size_t const cdictSize = ZSTD_sizeof_CDict(dict.cdict); + return bufferSize + cdictSize; +} + static void ZSTD_freeCCtxContent(ZSTD_CCtx* cctx) { assert(cctx != NULL); assert(cctx->staticSize == 0); ZSTD_free(cctx->workSpace, cctx->customMem); cctx->workSpace = NULL; - ZSTD_freeCDict(cctx->cdictLocal); cctx->cdictLocal = NULL; + ZSTD_clearAllDicts(cctx); #ifdef ZSTD_MULTITHREAD ZSTDMT_freeCCtx(cctx->mtctx); cctx->mtctx = NULL; #endif @@ -117,7 +136,8 @@ static void ZSTD_freeCCtxContent(ZSTD_CCtx* cctx) size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx) { if (cctx==NULL) return 0; /* support free on NULL */ - if (cctx->staticSize) return ERROR(memory_allocation); /* not compatible with static CCtx */ + RETURN_ERROR_IF(cctx->staticSize, memory_allocation, + "not compatible with static CCtx"); ZSTD_freeCCtxContent(cctx); ZSTD_free(cctx, cctx->customMem); return 0; @@ -139,7 +159,7 @@ size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx) { if (cctx==NULL) return 0; /* support sizeof on NULL */ return sizeof(*cctx) + cctx->workSpaceSize - + ZSTD_sizeof_CDict(cctx->cdictLocal) + + ZSTD_sizeof_localDict(cctx->localDict) + ZSTD_sizeof_mtctx(cctx); } @@ -195,7 +215,7 @@ size_t ZSTD_CCtxParams_reset(ZSTD_CCtx_params* params) } size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel) { - if (!cctxParams) { return ERROR(GENERIC); } + RETURN_ERROR_IF(!cctxParams, GENERIC); memset(cctxParams, 0, sizeof(*cctxParams)); cctxParams->compressionLevel = compressionLevel; cctxParams->fParams.contentSizeFlag = 1; @@ -204,8 +224,8 @@ size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel) size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params) { - if (!cctxParams) { return ERROR(GENERIC); } - CHECK_F( ZSTD_checkCParams(params.cParams) ); + RETURN_ERROR_IF(!cctxParams, GENERIC); + FORWARD_IF_ERROR( ZSTD_checkCParams(params.cParams) ); memset(cctxParams, 0, sizeof(*cctxParams)); cctxParams->cParams = params.cParams; cctxParams->fParams = params.fParams; @@ -359,6 +379,12 @@ ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param) bounds.upperBound = ZSTD_dictForceCopy; /* note : how to ensure at compile time that this is the highest value enum ? */ return bounds; + case ZSTD_c_literalCompressionMode: + ZSTD_STATIC_ASSERT(ZSTD_lcm_auto < ZSTD_lcm_huffman && ZSTD_lcm_huffman < ZSTD_lcm_uncompressed); + bounds.lowerBound = ZSTD_lcm_auto; + bounds.upperBound = ZSTD_lcm_uncompressed; + return bounds; + default: { ZSTD_bounds const boundError = { ERROR(parameter_unsupported), 0, 0 }; return boundError; @@ -378,10 +404,22 @@ static int ZSTD_cParam_withinBounds(ZSTD_cParameter cParam, int value) return 1; } -#define BOUNDCHECK(cParam, val) { \ - if (!ZSTD_cParam_withinBounds(cParam,val)) { \ - return ERROR(parameter_outOfBound); \ -} } +/* ZSTD_cParam_clampBounds: + * Clamps the value into the bounded range. + */ +static size_t ZSTD_cParam_clampBounds(ZSTD_cParameter cParam, int* value) +{ + ZSTD_bounds const bounds = ZSTD_cParam_getBounds(cParam); + if (ZSTD_isError(bounds.error)) return bounds.error; + if (*value < bounds.lowerBound) *value = bounds.lowerBound; + if (*value > bounds.upperBound) *value = bounds.upperBound; + return 0; +} + +#define BOUNDCHECK(cParam, val) { \ + RETURN_ERROR_IF(!ZSTD_cParam_withinBounds(cParam,val), \ + parameter_outOfBound); \ +} static int ZSTD_isUpdateAuthorized(ZSTD_cParameter param) @@ -413,6 +451,7 @@ static int ZSTD_isUpdateAuthorized(ZSTD_cParameter param) case ZSTD_c_ldmBucketSizeLog: case ZSTD_c_ldmHashRateLog: case ZSTD_c_forceAttachDict: + case ZSTD_c_literalCompressionMode: default: return 0; } @@ -425,18 +464,17 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value) if (ZSTD_isUpdateAuthorized(param)) { cctx->cParamsChanged = 1; } else { - return ERROR(stage_wrong); + RETURN_ERROR(stage_wrong); } } switch(param) { - case ZSTD_c_format : - return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); + case ZSTD_c_nbWorkers: + RETURN_ERROR_IF((value!=0) && cctx->staticSize, parameter_unsupported, + "MT not compatible with static alloc"); + break; case ZSTD_c_compressionLevel: - if (cctx->cdict) return ERROR(stage_wrong); - return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); - case ZSTD_c_windowLog: case ZSTD_c_hashLog: case ZSTD_c_chainLog: @@ -444,49 +482,32 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value) case ZSTD_c_minMatch: case ZSTD_c_targetLength: case ZSTD_c_strategy: - if (cctx->cdict) return ERROR(stage_wrong); - return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); - + case ZSTD_c_ldmHashRateLog: + case ZSTD_c_format: case ZSTD_c_contentSizeFlag: case ZSTD_c_checksumFlag: case ZSTD_c_dictIDFlag: - return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); - - case ZSTD_c_forceMaxWindow : /* Force back-references to remain < windowSize, - * even when referencing into Dictionary content. - * default : 0 when using a CDict, 1 when using a Prefix */ - return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); - + case ZSTD_c_forceMaxWindow: case ZSTD_c_forceAttachDict: - return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); - - case ZSTD_c_nbWorkers: - if ((value!=0) && cctx->staticSize) { - return ERROR(parameter_unsupported); /* MT not compatible with static alloc */ - } - return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); - + case ZSTD_c_literalCompressionMode: case ZSTD_c_jobSize: case ZSTD_c_overlapLog: case ZSTD_c_rsyncable: - return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); - case ZSTD_c_enableLongDistanceMatching: case ZSTD_c_ldmHashLog: case ZSTD_c_ldmMinMatch: case ZSTD_c_ldmBucketSizeLog: - case ZSTD_c_ldmHashRateLog: - if (cctx->cdict) return ERROR(stage_wrong); - return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); + break; - default: return ERROR(parameter_unsupported); + default: RETURN_ERROR(parameter_unsupported); } + return ZSTD_CCtxParams_setParameter(&cctx->requestedParams, param, value); } -size_t ZSTD_CCtxParam_setParameter(ZSTD_CCtx_params* CCtxParams, - ZSTD_cParameter param, int value) +size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams, + ZSTD_cParameter param, int value) { - DEBUGLOG(4, "ZSTD_CCtxParam_setParameter (%i, %i)", (int)param, value); + DEBUGLOG(4, "ZSTD_CCtxParams_setParameter (%i, %i)", (int)param, value); switch(param) { case ZSTD_c_format : @@ -495,11 +516,9 @@ size_t ZSTD_CCtxParam_setParameter(ZSTD_CCtx_params* CCtxParams, return (size_t)CCtxParams->format; case ZSTD_c_compressionLevel : { - int cLevel = value; - if (cLevel > ZSTD_maxCLevel()) cLevel = ZSTD_maxCLevel(); - if (cLevel < ZSTD_minCLevel()) cLevel = ZSTD_minCLevel(); - if (cLevel) { /* 0 : does not change current level */ - CCtxParams->compressionLevel = cLevel; + FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(param, &value)); + if (value) { /* 0 : does not change current level */ + CCtxParams->compressionLevel = value; } if (CCtxParams->compressionLevel >= 0) return CCtxParams->compressionLevel; return 0; /* return type (size_t) cannot represent negative values */ @@ -573,33 +592,55 @@ size_t ZSTD_CCtxParam_setParameter(ZSTD_CCtx_params* CCtxParams, return CCtxParams->attachDictPref; } + case ZSTD_c_literalCompressionMode : { + const ZSTD_literalCompressionMode_e lcm = (ZSTD_literalCompressionMode_e)value; + BOUNDCHECK(ZSTD_c_literalCompressionMode, lcm); + CCtxParams->literalCompressionMode = lcm; + return CCtxParams->literalCompressionMode; + } + case ZSTD_c_nbWorkers : #ifndef ZSTD_MULTITHREAD - if (value!=0) return ERROR(parameter_unsupported); + RETURN_ERROR_IF(value!=0, parameter_unsupported, "not compiled with multithreading"); return 0; #else - return ZSTDMT_CCtxParam_setNbWorkers(CCtxParams, value); + FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(param, &value)); + CCtxParams->nbWorkers = value; + return CCtxParams->nbWorkers; #endif case ZSTD_c_jobSize : #ifndef ZSTD_MULTITHREAD - return ERROR(parameter_unsupported); + RETURN_ERROR_IF(value!=0, parameter_unsupported, "not compiled with multithreading"); + return 0; #else - return ZSTDMT_CCtxParam_setMTCtxParameter(CCtxParams, ZSTDMT_p_jobSize, value); + /* Adjust to the minimum non-default value. */ + if (value != 0 && value < ZSTDMT_JOBSIZE_MIN) + value = ZSTDMT_JOBSIZE_MIN; + FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(param, &value)); + assert(value >= 0); + CCtxParams->jobSize = value; + return CCtxParams->jobSize; #endif case ZSTD_c_overlapLog : #ifndef ZSTD_MULTITHREAD - return ERROR(parameter_unsupported); + RETURN_ERROR_IF(value!=0, parameter_unsupported, "not compiled with multithreading"); + return 0; #else - return ZSTDMT_CCtxParam_setMTCtxParameter(CCtxParams, ZSTDMT_p_overlapLog, value); + FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(ZSTD_c_overlapLog, &value)); + CCtxParams->overlapLog = value; + return CCtxParams->overlapLog; #endif case ZSTD_c_rsyncable : #ifndef ZSTD_MULTITHREAD - return ERROR(parameter_unsupported); + RETURN_ERROR_IF(value!=0, parameter_unsupported, "not compiled with multithreading"); + return 0; #else - return ZSTDMT_CCtxParam_setMTCtxParameter(CCtxParams, ZSTDMT_p_rsyncable, value); + FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(ZSTD_c_overlapLog, &value)); + CCtxParams->rsyncable = value; + return CCtxParams->rsyncable; #endif case ZSTD_c_enableLongDistanceMatching : @@ -625,21 +666,21 @@ size_t ZSTD_CCtxParam_setParameter(ZSTD_CCtx_params* CCtxParams, return CCtxParams->ldmParams.bucketSizeLog; case ZSTD_c_ldmHashRateLog : - if (value > ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN) - return ERROR(parameter_outOfBound); + RETURN_ERROR_IF(value > ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN, + parameter_outOfBound); CCtxParams->ldmParams.hashRateLog = value; return CCtxParams->ldmParams.hashRateLog; - default: return ERROR(parameter_unsupported); + default: RETURN_ERROR(parameter_unsupported, "unknown parameter"); } } size_t ZSTD_CCtx_getParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int* value) { - return ZSTD_CCtxParam_getParameter(&cctx->requestedParams, param, value); + return ZSTD_CCtxParams_getParameter(&cctx->requestedParams, param, value); } -size_t ZSTD_CCtxParam_getParameter( +size_t ZSTD_CCtxParams_getParameter( ZSTD_CCtx_params* CCtxParams, ZSTD_cParameter param, int* value) { switch(param) @@ -686,6 +727,9 @@ size_t ZSTD_CCtxParam_getParameter( case ZSTD_c_forceAttachDict : *value = CCtxParams->attachDictPref; break; + case ZSTD_c_literalCompressionMode : + *value = CCtxParams->literalCompressionMode; + break; case ZSTD_c_nbWorkers : #ifndef ZSTD_MULTITHREAD assert(CCtxParams->nbWorkers == 0); @@ -694,7 +738,7 @@ size_t ZSTD_CCtxParam_getParameter( break; case ZSTD_c_jobSize : #ifndef ZSTD_MULTITHREAD - return ERROR(parameter_unsupported); + RETURN_ERROR(parameter_unsupported, "not compiled with multithreading"); #else assert(CCtxParams->jobSize <= INT_MAX); *value = (int)CCtxParams->jobSize; @@ -702,14 +746,14 @@ size_t ZSTD_CCtxParam_getParameter( #endif case ZSTD_c_overlapLog : #ifndef ZSTD_MULTITHREAD - return ERROR(parameter_unsupported); + RETURN_ERROR(parameter_unsupported, "not compiled with multithreading"); #else *value = CCtxParams->overlapLog; break; #endif case ZSTD_c_rsyncable : #ifndef ZSTD_MULTITHREAD - return ERROR(parameter_unsupported); + RETURN_ERROR(parameter_unsupported, "not compiled with multithreading"); #else *value = CCtxParams->rsyncable; break; @@ -729,7 +773,7 @@ size_t ZSTD_CCtxParam_getParameter( case ZSTD_c_ldmHashRateLog : *value = CCtxParams->ldmParams.hashRateLog; break; - default: return ERROR(parameter_unsupported); + default: RETURN_ERROR(parameter_unsupported, "unknown parameter"); } return 0; } @@ -745,8 +789,8 @@ size_t ZSTD_CCtx_setParametersUsingCCtxParams( ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params) { DEBUGLOG(4, "ZSTD_CCtx_setParametersUsingCCtxParams"); - if (cctx->streamStage != zcss_init) return ERROR(stage_wrong); - if (cctx->cdict) return ERROR(stage_wrong); + RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong); + RETURN_ERROR_IF(cctx->cdict, stage_wrong); cctx->requestedParams = *params; return 0; @@ -755,33 +799,71 @@ size_t ZSTD_CCtx_setParametersUsingCCtxParams( ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize) { DEBUGLOG(4, "ZSTD_CCtx_setPledgedSrcSize to %u bytes", (U32)pledgedSrcSize); - if (cctx->streamStage != zcss_init) return ERROR(stage_wrong); + RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong); cctx->pledgedSrcSizePlusOne = pledgedSrcSize+1; return 0; } +/** + * Initializes the local dict using the requested parameters. + * NOTE: This does not use the pledged src size, because it may be used for more + * than one compression. + */ +static size_t ZSTD_initLocalDict(ZSTD_CCtx* cctx) +{ + ZSTD_localDict* const dl = &cctx->localDict; + ZSTD_compressionParameters const cParams = ZSTD_getCParamsFromCCtxParams( + &cctx->requestedParams, 0, dl->dictSize); + if (dl->dict == NULL) { + /* No local dictionary. */ + assert(dl->dictBuffer == NULL); + assert(dl->cdict == NULL); + assert(dl->dictSize == 0); + return 0; + } + if (dl->cdict != NULL) { + assert(cctx->cdict == dl->cdict); + /* Local dictionary already initialized. */ + return 0; + } + assert(dl->dictSize > 0); + assert(cctx->cdict == NULL); + assert(cctx->prefixDict.dict == NULL); + + dl->cdict = ZSTD_createCDict_advanced( + dl->dict, + dl->dictSize, + ZSTD_dlm_byRef, + dl->dictContentType, + cParams, + cctx->customMem); + RETURN_ERROR_IF(!dl->cdict, memory_allocation); + cctx->cdict = dl->cdict; + return 0; +} + size_t ZSTD_CCtx_loadDictionary_advanced( ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType) { - if (cctx->streamStage != zcss_init) return ERROR(stage_wrong); - if (cctx->staticSize) return ERROR(memory_allocation); /* no malloc for static CCtx */ + RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong); + RETURN_ERROR_IF(cctx->staticSize, memory_allocation, + "no malloc for static CCtx"); DEBUGLOG(4, "ZSTD_CCtx_loadDictionary_advanced (size: %u)", (U32)dictSize); - ZSTD_freeCDict(cctx->cdictLocal); /* in case one already exists */ - if (dict==NULL || dictSize==0) { /* no dictionary mode */ - cctx->cdictLocal = NULL; - cctx->cdict = NULL; + ZSTD_clearAllDicts(cctx); /* in case one already exists */ + if (dict == NULL || dictSize == 0) /* no dictionary mode */ + return 0; + if (dictLoadMethod == ZSTD_dlm_byRef) { + cctx->localDict.dict = dict; } else { - ZSTD_compressionParameters const cParams = - ZSTD_getCParamsFromCCtxParams(&cctx->requestedParams, cctx->pledgedSrcSizePlusOne-1, dictSize); - cctx->cdictLocal = ZSTD_createCDict_advanced( - dict, dictSize, - dictLoadMethod, dictContentType, - cParams, cctx->customMem); - cctx->cdict = cctx->cdictLocal; - if (cctx->cdictLocal == NULL) - return ERROR(memory_allocation); + void* dictBuffer = ZSTD_malloc(dictSize, cctx->customMem); + RETURN_ERROR_IF(!dictBuffer, memory_allocation); + memcpy(dictBuffer, dict, dictSize); + cctx->localDict.dictBuffer = dictBuffer; + cctx->localDict.dict = dictBuffer; } + cctx->localDict.dictSize = dictSize; + cctx->localDict.dictContentType = dictContentType; return 0; } @@ -801,9 +883,10 @@ ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, s size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict) { - if (cctx->streamStage != zcss_init) return ERROR(stage_wrong); + RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong); + /* Free the existing local cdict (if any) to save memory. */ + ZSTD_clearAllDicts(cctx); cctx->cdict = cdict; - memset(&cctx->prefixDict, 0, sizeof(cctx->prefixDict)); /* exclusive */ return 0; } @@ -815,8 +898,8 @@ size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSiz size_t ZSTD_CCtx_refPrefix_advanced( ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType) { - if (cctx->streamStage != zcss_init) return ERROR(stage_wrong); - cctx->cdict = NULL; /* prefix discards any prior cdict */ + RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong); + ZSTD_clearAllDicts(cctx); cctx->prefixDict.dict = prefix; cctx->prefixDict.dictSize = prefixSize; cctx->prefixDict.dictContentType = dictContentType; @@ -834,8 +917,8 @@ size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset) } if ( (reset == ZSTD_reset_parameters) || (reset == ZSTD_reset_session_and_parameters) ) { - if (cctx->streamStage != zcss_init) return ERROR(stage_wrong); - cctx->cdict = NULL; + RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong); + ZSTD_clearAllDicts(cctx); return ZSTD_CCtxParams_reset(&cctx->requestedParams); } return 0; @@ -888,10 +971,11 @@ static U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat) } /** ZSTD_adjustCParams_internal() : - optimize `cPar` for a given input (`srcSize` and `dictSize`). - mostly downsizing to reduce memory consumption and initialization latency. - Both `srcSize` and `dictSize` are optional (use 0 if unknown). - Note : cPar is assumed validated. Use ZSTD_checkCParams() to ensure this condition. */ + * optimize `cPar` for a specified input (`srcSize` and `dictSize`). + * mostly downsize to reduce memory consumption and initialization latency. + * `srcSize` can be ZSTD_CONTENTSIZE_UNKNOWN when not known. + * note : for the time being, `srcSize==0` means "unknown" too, for compatibility with older convention. + * condition : cPar is presumed validated (can be checked using ZSTD_checkCParams()). */ static ZSTD_compressionParameters ZSTD_adjustCParams_internal(ZSTD_compressionParameters cPar, unsigned long long srcSize, @@ -901,7 +985,7 @@ ZSTD_adjustCParams_internal(ZSTD_compressionParameters cPar, static const U64 maxWindowResize = 1ULL << (ZSTD_WINDOWLOG_MAX-1); assert(ZSTD_checkCParams(cPar)==0); - if (dictSize && (srcSize+1<2) /* srcSize unknown */ ) + if (dictSize && (srcSize+1<2) /* ZSTD_CONTENTSIZE_UNKNOWN and 0 mean "unknown" */ ) srcSize = minSrcSize; /* presumed small when there is a dictionary */ else if (srcSize == 0) srcSize = ZSTD_CONTENTSIZE_UNKNOWN; /* 0 == unknown : presumed large */ @@ -922,7 +1006,7 @@ ZSTD_adjustCParams_internal(ZSTD_compressionParameters cPar, } if (cPar.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) - cPar.windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* required for frame header */ + cPar.windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* minimum wlog required for valid frame header */ return cPar; } @@ -932,7 +1016,7 @@ ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize) { - cPar = ZSTD_clampCParams(cPar); + cPar = ZSTD_clampCParams(cPar); /* resulting cPar is necessarily valid (all parameters within range) */ return ZSTD_adjustCParams_internal(cPar, srcSize, dictSize); } @@ -973,8 +1057,7 @@ ZSTD_sizeof_matchState(const ZSTD_compressionParameters* const cParams, size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params) { - /* Estimate CCtx size is supported for single-threaded compression only. */ - if (params->nbWorkers > 0) { return ERROR(GENERIC); } + RETURN_ERROR_IF(params->nbWorkers > 0, GENERIC, "Estimate CCtx size is supported for single-threaded compression only."); { ZSTD_compressionParameters const cParams = ZSTD_getCParamsFromCCtxParams(params, 0, 0); size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << cParams.windowLog); @@ -1022,10 +1105,12 @@ size_t ZSTD_estimateCCtxSize(int compressionLevel) size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params) { - if (params->nbWorkers > 0) { return ERROR(GENERIC); } - { size_t const CCtxSize = ZSTD_estimateCCtxSize_usingCCtxParams(params); - size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << params->cParams.windowLog); - size_t const inBuffSize = ((size_t)1 << params->cParams.windowLog) + blockSize; + RETURN_ERROR_IF(params->nbWorkers > 0, GENERIC, "Estimate CCtx size is supported for single-threaded compression only."); + { ZSTD_compressionParameters const cParams = + ZSTD_getCParamsFromCCtxParams(params, 0, 0); + size_t const CCtxSize = ZSTD_estimateCCtxSize_usingCCtxParams(params); + size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << cParams.windowLog); + size_t const inBuffSize = ((size_t)1 << cParams.windowLog) + blockSize; size_t const outBuffSize = ZSTD_compressBound(blockSize) + 1; size_t const streamingSize = inBuffSize + outBuffSize; @@ -1367,13 +1452,13 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, DEBUGLOG(4, "Need to resize workSpaceSize from %zuKB to %zuKB", zc->workSpaceSize >> 10, neededSpace >> 10); - /* static cctx : no resize, error out */ - if (zc->staticSize) return ERROR(memory_allocation); + + RETURN_ERROR_IF(zc->staticSize, memory_allocation, "static cctx : no resize"); zc->workSpaceSize = 0; ZSTD_free(zc->workSpace, zc->customMem); zc->workSpace = ZSTD_malloc(neededSpace, zc->customMem); - if (zc->workSpace == NULL) return ERROR(memory_allocation); + RETURN_ERROR_IF(zc->workSpace == NULL, memory_allocation); zc->workSpaceSize = neededSpace; zc->workSpaceOversizedDuration = 0; @@ -1644,7 +1729,7 @@ static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx, ZSTD_buffered_policy_e zbuff) { DEBUGLOG(5, "ZSTD_copyCCtx_internal"); - if (srcCCtx->stage!=ZSTDcs_init) return ERROR(stage_wrong); + RETURN_ERROR_IF(srcCCtx->stage!=ZSTDcs_init, stage_wrong); memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem)); { ZSTD_CCtx_params params = dstCCtx->requestedParams; @@ -1777,7 +1862,8 @@ static void ZSTD_reduceIndex (ZSTD_CCtx* zc, const U32 reducerValue) static size_t ZSTD_noCompressBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize, U32 lastBlock) { U32 const cBlockHeader24 = lastBlock + (((U32)bt_raw)<<1) + (U32)(srcSize << 3); - if (srcSize + ZSTD_blockHeaderSize > dstCapacity) return ERROR(dstSize_tooSmall); + RETURN_ERROR_IF(srcSize + ZSTD_blockHeaderSize > dstCapacity, + dstSize_tooSmall); MEM_writeLE24(dst, cBlockHeader24); memcpy((BYTE*)dst + ZSTD_blockHeaderSize, src, srcSize); return ZSTD_blockHeaderSize + srcSize; @@ -1788,7 +1874,7 @@ static size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void BYTE* const ostart = (BYTE* const)dst; U32 const flSize = 1 + (srcSize>31) + (srcSize>4095); - if (srcSize + flSize > dstCapacity) return ERROR(dstSize_tooSmall); + RETURN_ERROR_IF(srcSize + flSize > dstCapacity, dstSize_tooSmall); switch(flSize) { @@ -1878,7 +1964,7 @@ static size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf, if (srcSize <= minLitSize) return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); } - if (dstCapacity < lhSize+1) return ERROR(dstSize_tooSmall); /* not enough space for compression */ + RETURN_ERROR_IF(dstCapacity < lhSize+1, dstSize_tooSmall, "not enough space for compression"); { HUF_repeat repeat = prevHuf->repeatMode; int const preferRepeat = strategy < ZSTD_lazy ? srcSize <= 1024 : 0; if (repeat == HUF_repeat_valid && lhSize == 3) singleStream = 1; @@ -1960,7 +2046,7 @@ void ZSTD_seqToCodes(const seqStore_t* seqStorePtr) * If x == 0: Return 0 * Else: Return floor(-log2(x / 256) * 256) */ -static unsigned const kInverseProbabiltyLog256[256] = { +static unsigned const kInverseProbabilityLog256[256] = { 0, 2048, 1792, 1642, 1536, 1453, 1386, 1329, 1280, 1236, 1197, 1162, 1130, 1100, 1073, 1047, 1024, 1001, 980, 960, 941, 923, 906, 889, 874, 859, 844, 830, 817, 804, 791, 779, 768, 756, 745, 734, @@ -1999,7 +2085,7 @@ static size_t ZSTD_entropyCost(unsigned const* count, unsigned const max, size_t if (count[s] != 0 && norm == 0) norm = 1; assert(count[s] < total); - cost += count[s] * kInverseProbabiltyLog256[norm]; + cost += count[s] * kInverseProbabilityLog256[norm]; } return cost >> 8; } @@ -2022,7 +2108,7 @@ static size_t ZSTD_crossEntropyCost(short const* norm, unsigned accuracyLog, unsigned const norm256 = normAcc << shift; assert(norm256 > 0); assert(norm256 < 256); - cost += count[s] * kInverseProbabiltyLog256[norm256]; + cost += count[s] * kInverseProbabilityLog256[norm256]; } return cost >> 8; } @@ -2050,21 +2136,17 @@ static size_t ZSTD_fseBitCost( unsigned s; FSE_CState_t cstate; FSE_initCState(&cstate, ctable); - if (ZSTD_getFSEMaxSymbolValue(ctable) < max) { - DEBUGLOG(5, "Repeat FSE_CTable has maxSymbolValue %u < %u", + RETURN_ERROR_IF(ZSTD_getFSEMaxSymbolValue(ctable) < max, GENERIC, + "Repeat FSE_CTable has maxSymbolValue %u < %u", ZSTD_getFSEMaxSymbolValue(ctable), max); - return ERROR(GENERIC); - } for (s = 0; s <= max; ++s) { unsigned const tableLog = cstate.stateLog; unsigned const badCost = (tableLog + 1) << kAccuracyLog; unsigned const bitCost = FSE_bitCost(cstate.symbolTT, tableLog, s, kAccuracyLog); if (count[s] == 0) continue; - if (bitCost >= badCost) { - DEBUGLOG(5, "Repeat FSE_CTable has Prob[%u] == 0", s); - return ERROR(GENERIC); - } + RETURN_ERROR_IF(bitCost >= badCost, GENERIC, + "Repeat FSE_CTable has Prob[%u] == 0", s); cost += count[s] * bitCost; } return cost >> kAccuracyLog; @@ -2080,7 +2162,7 @@ static size_t ZSTD_NCountCost(unsigned const* count, unsigned const max, BYTE wksp[FSE_NCOUNTBOUND]; S16 norm[MaxSeq + 1]; const U32 tableLog = FSE_optimalTableLog(FSELog, nbSeq, max); - CHECK_F(FSE_normalizeCount(norm, tableLog, count, nbSeq, max)); + FORWARD_IF_ERROR(FSE_normalizeCount(norm, tableLog, count, nbSeq, max)); return FSE_writeNCount(wksp, sizeof(wksp), norm, max, tableLog); } @@ -2186,15 +2268,15 @@ ZSTD_buildCTable(void* dst, size_t dstCapacity, switch (type) { case set_rle: - CHECK_F(FSE_buildCTable_rle(nextCTable, (BYTE)max)); - if (dstCapacity==0) return ERROR(dstSize_tooSmall); + FORWARD_IF_ERROR(FSE_buildCTable_rle(nextCTable, (BYTE)max)); + RETURN_ERROR_IF(dstCapacity==0, dstSize_tooSmall); *op = codeTable[0]; return 1; case set_repeat: memcpy(nextCTable, prevCTable, prevCTableSize); return 0; case set_basic: - CHECK_F(FSE_buildCTable_wksp(nextCTable, defaultNorm, defaultMax, defaultNormLog, workspace, workspaceSize)); /* note : could be pre-calculated */ + FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, defaultNorm, defaultMax, defaultNormLog, workspace, workspaceSize)); /* note : could be pre-calculated */ return 0; case set_compressed: { S16 norm[MaxSeq + 1]; @@ -2205,14 +2287,14 @@ ZSTD_buildCTable(void* dst, size_t dstCapacity, nbSeq_1--; } assert(nbSeq_1 > 1); - CHECK_F(FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max)); + FORWARD_IF_ERROR(FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max)); { size_t const NCountSize = FSE_writeNCount(op, oend - op, norm, max, tableLog); /* overflow protected */ - if (FSE_isError(NCountSize)) return NCountSize; - CHECK_F(FSE_buildCTable_wksp(nextCTable, norm, max, tableLog, workspace, workspaceSize)); + FORWARD_IF_ERROR(NCountSize); + FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, norm, max, tableLog, workspace, workspaceSize)); return NCountSize; } } - default: return assert(0), ERROR(GENERIC); + default: assert(0); RETURN_ERROR(GENERIC); } } @@ -2229,7 +2311,9 @@ ZSTD_encodeSequences_body( FSE_CState_t stateOffsetBits; FSE_CState_t stateLitLength; - CHECK_E(BIT_initCStream(&blockStream, dst, dstCapacity), dstSize_tooSmall); /* not enough space remaining */ + RETURN_ERROR_IF( + ERR_isError(BIT_initCStream(&blockStream, dst, dstCapacity)), + dstSize_tooSmall, "not enough space remaining"); DEBUGLOG(6, "available space for bitstream : %i (dstCapacity=%u)", (int)(blockStream.endPtr - blockStream.startPtr), (unsigned)dstCapacity); @@ -2303,7 +2387,7 @@ ZSTD_encodeSequences_body( FSE_flushCState(&blockStream, &stateLitLength); { size_t const streamSize = BIT_closeCStream(&blockStream); - if (streamSize==0) return ERROR(dstSize_tooSmall); /* not enough space */ + RETURN_ERROR_IF(streamSize==0, dstSize_tooSmall, "not enough space"); return streamSize; } } @@ -2368,6 +2452,21 @@ static size_t ZSTD_encodeSequences( sequences, nbSeq, longOffsets); } +static int ZSTD_disableLiteralsCompression(const ZSTD_CCtx_params* cctxParams) +{ + switch (cctxParams->literalCompressionMode) { + case ZSTD_lcm_huffman: + return 0; + case ZSTD_lcm_uncompressed: + return 1; + default: + assert(0 /* impossible: pre-validated */); + /* fall-through */ + case ZSTD_lcm_auto: + return (cctxParams->cParams.strategy == ZSTD_fast) && (cctxParams->cParams.targetLength > 0); + } +} + /* ZSTD_compressSequences_internal(): * actually compresses both literals and sequences */ MEM_STATIC size_t @@ -2403,22 +2502,22 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, /* Compress literals */ { const BYTE* const literals = seqStorePtr->litStart; size_t const litSize = seqStorePtr->lit - literals; - int const disableLiteralCompression = (cctxParams->cParams.strategy == ZSTD_fast) && (cctxParams->cParams.targetLength > 0); size_t const cSize = ZSTD_compressLiterals( &prevEntropy->huf, &nextEntropy->huf, - cctxParams->cParams.strategy, disableLiteralCompression, + cctxParams->cParams.strategy, + ZSTD_disableLiteralsCompression(cctxParams), op, dstCapacity, literals, litSize, workspace, wkspSize, bmi2); - if (ZSTD_isError(cSize)) - return cSize; + FORWARD_IF_ERROR(cSize); assert(cSize <= dstCapacity); op += cSize; } /* Sequences Header */ - if ((oend-op) < 3 /*max nbSeq Size*/ + 1 /*seqHead*/) return ERROR(dstSize_tooSmall); + RETURN_ERROR_IF((oend-op) < 3 /*max nbSeq Size*/ + 1 /*seqHead*/, + dstSize_tooSmall); if (nbSeq < 0x7F) *op++ = (BYTE)nbSeq; else if (nbSeq < LONGNBSEQ) @@ -2452,7 +2551,7 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, count, max, llCodeTable, nbSeq, LL_defaultNorm, LL_defaultNormLog, MaxLL, prevEntropy->fse.litlengthCTable, sizeof(prevEntropy->fse.litlengthCTable), workspace, wkspSize); - if (ZSTD_isError(countSize)) return countSize; + FORWARD_IF_ERROR(countSize); if (LLtype == set_compressed) lastNCount = op; op += countSize; @@ -2474,7 +2573,7 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, count, max, ofCodeTable, nbSeq, OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff, prevEntropy->fse.offcodeCTable, sizeof(prevEntropy->fse.offcodeCTable), workspace, wkspSize); - if (ZSTD_isError(countSize)) return countSize; + FORWARD_IF_ERROR(countSize); if (Offtype == set_compressed) lastNCount = op; op += countSize; @@ -2494,7 +2593,7 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, count, max, mlCodeTable, nbSeq, ML_defaultNorm, ML_defaultNormLog, MaxML, prevEntropy->fse.matchlengthCTable, sizeof(prevEntropy->fse.matchlengthCTable), workspace, wkspSize); - if (ZSTD_isError(countSize)) return countSize; + FORWARD_IF_ERROR(countSize); if (MLtype == set_compressed) lastNCount = op; op += countSize; @@ -2509,10 +2608,10 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, CTable_LitLength, llCodeTable, sequences, nbSeq, longOffsets, bmi2); - if (ZSTD_isError(bitstreamSize)) return bitstreamSize; + FORWARD_IF_ERROR(bitstreamSize); op += bitstreamSize; /* zstd versions <= 1.3.4 mistakenly report corruption when - * FSE_readNCount() recieves a buffer < 4 bytes. + * FSE_readNCount() receives a buffer < 4 bytes. * Fixed by https://github.com/facebook/zstd/pull/1146. * This can happen when the last set_compressed table present is 2 * bytes and the bitstream is only one byte. @@ -2552,7 +2651,7 @@ ZSTD_compressSequences(seqStore_t* seqStorePtr, */ if ((cSize == ERROR(dstSize_tooSmall)) & (srcSize <= dstCapacity)) return 0; /* block not compressed */ - if (ZSTD_isError(cSize)) return cSize; + FORWARD_IF_ERROR(cSize); /* Check compressibility */ { size_t const maxCSize = srcSize - ZSTD_minGain(srcSize, cctxParams->cParams.strategy); @@ -2641,7 +2740,10 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, goto out; /* don't even attempt compression below a certain srcSize */ } ZSTD_resetSeqStore(&(zc->seqStore)); - ms->opt.symbolCosts = &zc->blockState.prevCBlock->entropy; /* required for optimal parser to read stats from dictionary */ + /* required for optimal parser to read stats from dictionary */ + ms->opt.symbolCosts = &zc->blockState.prevCBlock->entropy; + /* tell the optimal parser how we expect to compress literals */ + ms->opt.literalCompressionMode = zc->appliedParams.literalCompressionMode; /* a gap between an attached dict and the current window is not safe, * they must remain adjacent, @@ -2679,7 +2781,7 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, ldmSeqStore.seq = zc->ldmSequences; ldmSeqStore.capacity = zc->maxNbLdmSequences; /* Updates ldmSeqStore.size */ - CHECK_F(ZSTD_ldm_generateSequences(&zc->ldmState, &ldmSeqStore, + FORWARD_IF_ERROR(ZSTD_ldm_generateSequences(&zc->ldmState, &ldmSeqStore, &zc->appliedParams.ldmParams, src, srcSize)); /* Updates ldmSeqStore.pos */ @@ -2752,8 +2854,9 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx, ZSTD_matchState_t* const ms = &cctx->blockState.matchState; U32 const lastBlock = lastFrameChunk & (blockSize >= remaining); - if (dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE) - return ERROR(dstSize_tooSmall); /* not enough space to store compressed block */ + RETURN_ERROR_IF(dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE, + dstSize_tooSmall, + "not enough space to store compressed block"); if (remaining < blockSize) blockSize = remaining; if (ZSTD_window_needOverflowCorrection(ms->window, ip + blockSize)) { @@ -2774,11 +2877,11 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx, { size_t cSize = ZSTD_compressBlock_internal(cctx, op+ZSTD_blockHeaderSize, dstCapacity-ZSTD_blockHeaderSize, ip, blockSize); - if (ZSTD_isError(cSize)) return cSize; + FORWARD_IF_ERROR(cSize); if (cSize == 0) { /* block is not compressible */ cSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock); - if (ZSTD_isError(cSize)) return cSize; + FORWARD_IF_ERROR(cSize); } else { U32 const cBlockHeader24 = lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3); MEM_writeLE24(op, cBlockHeader24); @@ -2811,11 +2914,11 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity, BYTE const windowLogByte = (BYTE)((params.cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) << 3); U32 const fcsCode = params.fParams.contentSizeFlag ? (pledgedSrcSize>=256) + (pledgedSrcSize>=65536+256) + (pledgedSrcSize>=0xFFFFFFFFU) : 0; /* 0-3 */ - BYTE const frameHeaderDecriptionByte = (BYTE)(dictIDSizeCode + (checksumFlag<<2) + (singleSegment<<5) + (fcsCode<<6) ); + BYTE const frameHeaderDescriptionByte = (BYTE)(dictIDSizeCode + (checksumFlag<<2) + (singleSegment<<5) + (fcsCode<<6) ); size_t pos=0; assert(!(params.fParams.contentSizeFlag && pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN)); - if (dstCapacity < ZSTD_FRAMEHEADERSIZE_MAX) return ERROR(dstSize_tooSmall); + RETURN_ERROR_IF(dstCapacity < ZSTD_FRAMEHEADERSIZE_MAX, dstSize_tooSmall); DEBUGLOG(4, "ZSTD_writeFrameHeader : dictIDFlag : %u ; dictID : %u ; dictIDSizeCode : %u", !params.fParams.noDictIDFlag, (unsigned)dictID, (unsigned)dictIDSizeCode); @@ -2823,7 +2926,7 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity, MEM_writeLE32(dst, ZSTD_MAGICNUMBER); pos = 4; } - op[pos++] = frameHeaderDecriptionByte; + op[pos++] = frameHeaderDescriptionByte; if (!singleSegment) op[pos++] = windowLogByte; switch(dictIDSizeCode) { @@ -2847,11 +2950,11 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity, /* ZSTD_writeLastEmptyBlock() : * output an empty Block with end-of-frame mark to complete a frame * @return : size of data written into `dst` (== ZSTD_blockHeaderSize (defined in zstd_internal.h)) - * or an error code if `dstCapcity` is too small (<ZSTD_blockHeaderSize) + * or an error code if `dstCapacity` is too small (<ZSTD_blockHeaderSize) */ size_t ZSTD_writeLastEmptyBlock(void* dst, size_t dstCapacity) { - if (dstCapacity < ZSTD_blockHeaderSize) return ERROR(dstSize_tooSmall); + RETURN_ERROR_IF(dstCapacity < ZSTD_blockHeaderSize, dstSize_tooSmall); { U32 const cBlockHeader24 = 1 /*lastBlock*/ + (((U32)bt_raw)<<1); /* 0 size */ MEM_writeLE24(dst, cBlockHeader24); return ZSTD_blockHeaderSize; @@ -2860,10 +2963,9 @@ size_t ZSTD_writeLastEmptyBlock(void* dst, size_t dstCapacity) size_t ZSTD_referenceExternalSequences(ZSTD_CCtx* cctx, rawSeq* seq, size_t nbSeq) { - if (cctx->stage != ZSTDcs_init) - return ERROR(stage_wrong); - if (cctx->appliedParams.ldmParams.enableLdm) - return ERROR(parameter_unsupported); + RETURN_ERROR_IF(cctx->stage != ZSTDcs_init, stage_wrong); + RETURN_ERROR_IF(cctx->appliedParams.ldmParams.enableLdm, + parameter_unsupported); cctx->externSeqStore.seq = seq; cctx->externSeqStore.size = nbSeq; cctx->externSeqStore.capacity = nbSeq; @@ -2882,12 +2984,13 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx, DEBUGLOG(5, "ZSTD_compressContinue_internal, stage: %u, srcSize: %u", cctx->stage, (unsigned)srcSize); - if (cctx->stage==ZSTDcs_created) return ERROR(stage_wrong); /* missing init (ZSTD_compressBegin) */ + RETURN_ERROR_IF(cctx->stage==ZSTDcs_created, stage_wrong, + "missing init (ZSTD_compressBegin)"); if (frame && (cctx->stage==ZSTDcs_init)) { fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, cctx->appliedParams, cctx->pledgedSrcSizePlusOne-1, cctx->dictID); - if (ZSTD_isError(fhSize)) return fhSize; + FORWARD_IF_ERROR(fhSize); dstCapacity -= fhSize; dst = (char*)dst + fhSize; cctx->stage = ZSTDcs_ongoing; @@ -2922,17 +3025,18 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx, { size_t const cSize = frame ? ZSTD_compress_frameChunk (cctx, dst, dstCapacity, src, srcSize, lastFrameChunk) : ZSTD_compressBlock_internal (cctx, dst, dstCapacity, src, srcSize); - if (ZSTD_isError(cSize)) return cSize; + FORWARD_IF_ERROR(cSize); cctx->consumedSrcSize += srcSize; cctx->producedCSize += (cSize + fhSize); assert(!(cctx->appliedParams.fParams.contentSizeFlag && cctx->pledgedSrcSizePlusOne == 0)); if (cctx->pledgedSrcSizePlusOne != 0) { /* control src size */ ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_UNKNOWN == (unsigned long long)-1); - if (cctx->consumedSrcSize+1 > cctx->pledgedSrcSizePlusOne) { - DEBUGLOG(4, "error : pledgedSrcSize = %u, while realSrcSize >= %u", - (unsigned)cctx->pledgedSrcSizePlusOne-1, (unsigned)cctx->consumedSrcSize); - return ERROR(srcSize_wrong); - } + RETURN_ERROR_IF( + cctx->consumedSrcSize+1 > cctx->pledgedSrcSizePlusOne, + srcSize_wrong, + "error : pledgedSrcSize = %u, while realSrcSize >= %u", + (unsigned)cctx->pledgedSrcSizePlusOne-1, + (unsigned)cctx->consumedSrcSize); } return cSize + fhSize; } @@ -2957,7 +3061,7 @@ size_t ZSTD_getBlockSize(const ZSTD_CCtx* cctx) size_t ZSTD_compressBlock(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) { size_t const blockSizeMax = ZSTD_getBlockSize(cctx); - if (srcSize > blockSizeMax) return ERROR(srcSize_wrong); + RETURN_ERROR_IF(srcSize > blockSizeMax, srcSize_wrong); return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 0 /* frame mode */, 0 /* last chunk */); } @@ -3020,9 +3124,9 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms, NOTE: This behavior is not standard and could be improved in the future. */ static size_t ZSTD_checkDictNCount(short* normalizedCounter, unsigned dictMaxSymbolValue, unsigned maxSymbolValue) { U32 s; - if (dictMaxSymbolValue < maxSymbolValue) return ERROR(dictionary_corrupted); + RETURN_ERROR_IF(dictMaxSymbolValue < maxSymbolValue, dictionary_corrupted); for (s = 0; s <= maxSymbolValue; ++s) { - if (normalizedCounter[s] == 0) return ERROR(dictionary_corrupted); + RETURN_ERROR_IF(normalizedCounter[s] == 0, dictionary_corrupted); } return 0; } @@ -3060,53 +3164,56 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs, { unsigned maxSymbolValue = 255; size_t const hufHeaderSize = HUF_readCTable((HUF_CElt*)bs->entropy.huf.CTable, &maxSymbolValue, dictPtr, dictEnd-dictPtr); - if (HUF_isError(hufHeaderSize)) return ERROR(dictionary_corrupted); - if (maxSymbolValue < 255) return ERROR(dictionary_corrupted); + RETURN_ERROR_IF(HUF_isError(hufHeaderSize), dictionary_corrupted); + RETURN_ERROR_IF(maxSymbolValue < 255, dictionary_corrupted); dictPtr += hufHeaderSize; } { unsigned offcodeLog; size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr); - if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted); - if (offcodeLog > OffFSELog) return ERROR(dictionary_corrupted); + RETURN_ERROR_IF(FSE_isError(offcodeHeaderSize), dictionary_corrupted); + RETURN_ERROR_IF(offcodeLog > OffFSELog, dictionary_corrupted); /* Defer checking offcodeMaxValue because we need to know the size of the dictionary content */ /* fill all offset symbols to avoid garbage at end of table */ - CHECK_E( FSE_buildCTable_wksp(bs->entropy.fse.offcodeCTable, - offcodeNCount, MaxOff, offcodeLog, - workspace, HUF_WORKSPACE_SIZE), - dictionary_corrupted); + RETURN_ERROR_IF(FSE_isError(FSE_buildCTable_wksp( + bs->entropy.fse.offcodeCTable, + offcodeNCount, MaxOff, offcodeLog, + workspace, HUF_WORKSPACE_SIZE)), + dictionary_corrupted); dictPtr += offcodeHeaderSize; } { short matchlengthNCount[MaxML+1]; unsigned matchlengthMaxValue = MaxML, matchlengthLog; size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr); - if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted); - if (matchlengthLog > MLFSELog) return ERROR(dictionary_corrupted); + RETURN_ERROR_IF(FSE_isError(matchlengthHeaderSize), dictionary_corrupted); + RETURN_ERROR_IF(matchlengthLog > MLFSELog, dictionary_corrupted); /* Every match length code must have non-zero probability */ - CHECK_F( ZSTD_checkDictNCount(matchlengthNCount, matchlengthMaxValue, MaxML)); - CHECK_E( FSE_buildCTable_wksp(bs->entropy.fse.matchlengthCTable, - matchlengthNCount, matchlengthMaxValue, matchlengthLog, - workspace, HUF_WORKSPACE_SIZE), - dictionary_corrupted); + FORWARD_IF_ERROR( ZSTD_checkDictNCount(matchlengthNCount, matchlengthMaxValue, MaxML)); + RETURN_ERROR_IF(FSE_isError(FSE_buildCTable_wksp( + bs->entropy.fse.matchlengthCTable, + matchlengthNCount, matchlengthMaxValue, matchlengthLog, + workspace, HUF_WORKSPACE_SIZE)), + dictionary_corrupted); dictPtr += matchlengthHeaderSize; } { short litlengthNCount[MaxLL+1]; unsigned litlengthMaxValue = MaxLL, litlengthLog; size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr); - if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted); - if (litlengthLog > LLFSELog) return ERROR(dictionary_corrupted); + RETURN_ERROR_IF(FSE_isError(litlengthHeaderSize), dictionary_corrupted); + RETURN_ERROR_IF(litlengthLog > LLFSELog, dictionary_corrupted); /* Every literal length code must have non-zero probability */ - CHECK_F( ZSTD_checkDictNCount(litlengthNCount, litlengthMaxValue, MaxLL)); - CHECK_E( FSE_buildCTable_wksp(bs->entropy.fse.litlengthCTable, - litlengthNCount, litlengthMaxValue, litlengthLog, - workspace, HUF_WORKSPACE_SIZE), - dictionary_corrupted); + FORWARD_IF_ERROR( ZSTD_checkDictNCount(litlengthNCount, litlengthMaxValue, MaxLL)); + RETURN_ERROR_IF(FSE_isError(FSE_buildCTable_wksp( + bs->entropy.fse.litlengthCTable, + litlengthNCount, litlengthMaxValue, litlengthLog, + workspace, HUF_WORKSPACE_SIZE)), + dictionary_corrupted); dictPtr += litlengthHeaderSize; } - if (dictPtr+12 > dictEnd) return ERROR(dictionary_corrupted); + RETURN_ERROR_IF(dictPtr+12 > dictEnd, dictionary_corrupted); bs->rep[0] = MEM_readLE32(dictPtr+0); bs->rep[1] = MEM_readLE32(dictPtr+4); bs->rep[2] = MEM_readLE32(dictPtr+8); @@ -3119,19 +3226,19 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs, offcodeMax = ZSTD_highbit32(maxOffset); /* Calculate minimum offset code required to represent maxOffset */ } /* All offset values <= dictContentSize + 128 KB must be representable */ - CHECK_F (ZSTD_checkDictNCount(offcodeNCount, offcodeMaxValue, MIN(offcodeMax, MaxOff))); + FORWARD_IF_ERROR(ZSTD_checkDictNCount(offcodeNCount, offcodeMaxValue, MIN(offcodeMax, MaxOff))); /* All repCodes must be <= dictContentSize and != 0*/ { U32 u; for (u=0; u<3; u++) { - if (bs->rep[u] == 0) return ERROR(dictionary_corrupted); - if (bs->rep[u] > dictContentSize) return ERROR(dictionary_corrupted); + RETURN_ERROR_IF(bs->rep[u] == 0, dictionary_corrupted); + RETURN_ERROR_IF(bs->rep[u] > dictContentSize, dictionary_corrupted); } } bs->entropy.huf.repeatMode = HUF_repeat_valid; bs->entropy.fse.offcode_repeatMode = FSE_repeat_valid; bs->entropy.fse.matchlength_repeatMode = FSE_repeat_valid; bs->entropy.fse.litlength_repeatMode = FSE_repeat_valid; - CHECK_F(ZSTD_loadDictionaryContent(ms, params, dictPtr, dictContentSize, dtlm)); + FORWARD_IF_ERROR(ZSTD_loadDictionaryContent(ms, params, dictPtr, dictContentSize, dtlm)); return dictID; } } @@ -3161,8 +3268,7 @@ ZSTD_compress_insertDictionary(ZSTD_compressedBlockState_t* bs, DEBUGLOG(4, "raw content dictionary detected"); return ZSTD_loadDictionaryContent(ms, params, dict, dictSize, dtlm); } - if (dictContentType == ZSTD_dct_fullDict) - return ERROR(dictionary_wrong); + RETURN_ERROR_IF(dictContentType == ZSTD_dct_fullDict, dictionary_wrong); assert(0); /* impossible */ } @@ -3189,13 +3295,13 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, return ZSTD_resetCCtx_usingCDict(cctx, cdict, params, pledgedSrcSize, zbuff); } - CHECK_F( ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize, + FORWARD_IF_ERROR( ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize, ZSTDcrp_continue, zbuff) ); { size_t const dictID = ZSTD_compress_insertDictionary( cctx->blockState.prevCBlock, &cctx->blockState.matchState, ¶ms, dict, dictSize, dictContentType, dtlm, cctx->entropyWorkspace); - if (ZSTD_isError(dictID)) return dictID; + FORWARD_IF_ERROR(dictID); assert(dictID <= (size_t)(U32)-1); cctx->dictID = (U32)dictID; } @@ -3212,7 +3318,7 @@ size_t ZSTD_compressBegin_advanced_internal(ZSTD_CCtx* cctx, { DEBUGLOG(4, "ZSTD_compressBegin_advanced_internal: wlog=%u", params.cParams.windowLog); /* compression parameters verification and optimization */ - CHECK_F( ZSTD_checkCParams(params.cParams) ); + FORWARD_IF_ERROR( ZSTD_checkCParams(params.cParams) ); return ZSTD_compressBegin_internal(cctx, dict, dictSize, dictContentType, dtlm, cdict, @@ -3260,12 +3366,12 @@ static size_t ZSTD_writeEpilogue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity) size_t fhSize = 0; DEBUGLOG(4, "ZSTD_writeEpilogue"); - if (cctx->stage == ZSTDcs_created) return ERROR(stage_wrong); /* init missing */ + RETURN_ERROR_IF(cctx->stage == ZSTDcs_created, stage_wrong, "init missing"); /* special case : empty frame */ if (cctx->stage == ZSTDcs_init) { fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, cctx->appliedParams, 0, 0); - if (ZSTD_isError(fhSize)) return fhSize; + FORWARD_IF_ERROR(fhSize); dstCapacity -= fhSize; op += fhSize; cctx->stage = ZSTDcs_ongoing; @@ -3274,7 +3380,7 @@ static size_t ZSTD_writeEpilogue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity) if (cctx->stage != ZSTDcs_ending) { /* write one last empty block, make it the "last" block */ U32 const cBlockHeader24 = 1 /* last block */ + (((U32)bt_raw)<<1) + 0; - if (dstCapacity<4) return ERROR(dstSize_tooSmall); + RETURN_ERROR_IF(dstCapacity<4, dstSize_tooSmall); MEM_writeLE32(op, cBlockHeader24); op += ZSTD_blockHeaderSize; dstCapacity -= ZSTD_blockHeaderSize; @@ -3282,7 +3388,7 @@ static size_t ZSTD_writeEpilogue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity) if (cctx->appliedParams.fParams.checksumFlag) { U32 const checksum = (U32) XXH64_digest(&cctx->xxhState); - if (dstCapacity<4) return ERROR(dstSize_tooSmall); + RETURN_ERROR_IF(dstCapacity<4, dstSize_tooSmall); DEBUGLOG(4, "ZSTD_writeEpilogue: write checksum : %08X", (unsigned)checksum); MEM_writeLE32(op, checksum); op += 4; @@ -3300,18 +3406,20 @@ size_t ZSTD_compressEnd (ZSTD_CCtx* cctx, size_t const cSize = ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 1 /* frame mode */, 1 /* last chunk */); - if (ZSTD_isError(cSize)) return cSize; + FORWARD_IF_ERROR(cSize); endResult = ZSTD_writeEpilogue(cctx, (char*)dst + cSize, dstCapacity-cSize); - if (ZSTD_isError(endResult)) return endResult; + FORWARD_IF_ERROR(endResult); assert(!(cctx->appliedParams.fParams.contentSizeFlag && cctx->pledgedSrcSizePlusOne == 0)); if (cctx->pledgedSrcSizePlusOne != 0) { /* control src size */ ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_UNKNOWN == (unsigned long long)-1); DEBUGLOG(4, "end of frame : controlling src size"); - if (cctx->pledgedSrcSizePlusOne != cctx->consumedSrcSize+1) { - DEBUGLOG(4, "error : pledgedSrcSize = %u, while realSrcSize = %u", - (unsigned)cctx->pledgedSrcSizePlusOne-1, (unsigned)cctx->consumedSrcSize); - return ERROR(srcSize_wrong); - } } + RETURN_ERROR_IF( + cctx->pledgedSrcSizePlusOne != cctx->consumedSrcSize+1, + srcSize_wrong, + "error : pledgedSrcSize = %u, while realSrcSize = %u", + (unsigned)cctx->pledgedSrcSizePlusOne-1, + (unsigned)cctx->consumedSrcSize); + } return cSize + endResult; } @@ -3339,7 +3447,7 @@ size_t ZSTD_compress_advanced (ZSTD_CCtx* cctx, ZSTD_parameters params) { DEBUGLOG(4, "ZSTD_compress_advanced"); - CHECK_F(ZSTD_checkCParams(params.cParams)); + FORWARD_IF_ERROR(ZSTD_checkCParams(params.cParams)); return ZSTD_compress_internal(cctx, dst, dstCapacity, src, srcSize, @@ -3356,7 +3464,7 @@ size_t ZSTD_compress_advanced_internal( ZSTD_CCtx_params params) { DEBUGLOG(4, "ZSTD_compress_advanced_internal (srcSize:%u)", (unsigned)srcSize); - CHECK_F( ZSTD_compressBegin_internal(cctx, + FORWARD_IF_ERROR( ZSTD_compressBegin_internal(cctx, dict, dictSize, ZSTD_dct_auto, ZSTD_dtlm_fast, NULL, params, srcSize, ZSTDb_not_buffered) ); return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize); @@ -3440,7 +3548,7 @@ static size_t ZSTD_initCDict_internal( void* const internalBuffer = ZSTD_malloc(dictSize, cdict->customMem); cdict->dictBuffer = internalBuffer; cdict->dictContent = internalBuffer; - if (!internalBuffer) return ERROR(memory_allocation); + RETURN_ERROR_IF(!internalBuffer, memory_allocation); memcpy(internalBuffer, dictBuffer, dictSize); } cdict->dictContentSize = dictSize; @@ -3466,7 +3574,7 @@ static size_t ZSTD_initCDict_internal( &cdict->cBlockState, &cdict->matchState, ¶ms, cdict->dictContent, cdict->dictContentSize, dictContentType, ZSTD_dtlm_full, cdict->workspace); - if (ZSTD_isError(dictID)) return dictID; + FORWARD_IF_ERROR(dictID); assert(dictID <= (size_t)(U32)-1); cdict->dictID = (U32)dictID; } @@ -3596,7 +3704,7 @@ size_t ZSTD_compressBegin_usingCDict_advanced( ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize) { DEBUGLOG(4, "ZSTD_compressBegin_usingCDict_advanced"); - if (cdict==NULL) return ERROR(dictionary_wrong); + RETURN_ERROR_IF(cdict==NULL, dictionary_wrong); { ZSTD_CCtx_params params = cctx->requestedParams; params.cParams = ZSTD_getCParamsFromCDict(cdict); /* Increase window log to fit the entire dictionary and source if the @@ -3632,7 +3740,7 @@ size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx, const void* src, size_t srcSize, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams) { - CHECK_F (ZSTD_compressBegin_usingCDict_advanced(cctx, cdict, fParams, srcSize)); /* will check if cdict != NULL */ + FORWARD_IF_ERROR(ZSTD_compressBegin_usingCDict_advanced(cctx, cdict, fParams, srcSize)); /* will check if cdict != NULL */ return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize); } @@ -3700,7 +3808,7 @@ static size_t ZSTD_resetCStream_internal(ZSTD_CStream* cctx, assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); assert(!((dict) && (cdict))); /* either dict or cdict, not both */ - CHECK_F( ZSTD_compressBegin_internal(cctx, + FORWARD_IF_ERROR( ZSTD_compressBegin_internal(cctx, dict, dictSize, dictContentType, ZSTD_dtlm_fast, cdict, params, pledgedSrcSize, @@ -3718,13 +3826,17 @@ static size_t ZSTD_resetCStream_internal(ZSTD_CStream* cctx, /* ZSTD_resetCStream(): * pledgedSrcSize == 0 means "unknown" */ -size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize) +size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pss) { - ZSTD_CCtx_params params = zcs->requestedParams; + /* temporary : 0 interpreted as "unknown" during transition period. + * Users willing to specify "unknown" **must** use ZSTD_CONTENTSIZE_UNKNOWN. + * 0 will be interpreted as "empty" in the future. + */ + U64 const pledgedSrcSize = (pss==0) ? ZSTD_CONTENTSIZE_UNKNOWN : pss; DEBUGLOG(4, "ZSTD_resetCStream: pledgedSrcSize = %u", (unsigned)pledgedSrcSize); - if (pledgedSrcSize==0) pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN; - params.fParams.contentSizeFlag = 1; - return ZSTD_resetCStream_internal(zcs, NULL, 0, ZSTD_dct_auto, zcs->cdict, params, pledgedSrcSize); + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) ); + FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) ); + return 0; } /*! ZSTD_initCStream_internal() : @@ -3736,32 +3848,18 @@ size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs, ZSTD_CCtx_params params, unsigned long long pledgedSrcSize) { DEBUGLOG(4, "ZSTD_initCStream_internal"); - params.cParams = ZSTD_getCParamsFromCCtxParams(¶ms, pledgedSrcSize, dictSize); + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) ); + FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) ); assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); + zcs->requestedParams = params; assert(!((dict) && (cdict))); /* either dict or cdict, not both */ - - if (dict && dictSize >= 8) { - DEBUGLOG(4, "loading dictionary of size %u", (unsigned)dictSize); - if (zcs->staticSize) { /* static CCtx : never uses malloc */ - /* incompatible with internal cdict creation */ - return ERROR(memory_allocation); - } - ZSTD_freeCDict(zcs->cdictLocal); - zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, - ZSTD_dlm_byCopy, ZSTD_dct_auto, - params.cParams, zcs->customMem); - zcs->cdict = zcs->cdictLocal; - if (zcs->cdictLocal == NULL) return ERROR(memory_allocation); + if (dict) { + FORWARD_IF_ERROR( ZSTD_CCtx_loadDictionary(zcs, dict, dictSize) ); } else { - if (cdict) { - params.cParams = ZSTD_getCParamsFromCDict(cdict); /* cParams are enforced from cdict; it includes windowLog */ - } - ZSTD_freeCDict(zcs->cdictLocal); - zcs->cdictLocal = NULL; - zcs->cdict = cdict; + /* Dictionary is cleared if !cdict */ + FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, cdict) ); } - - return ZSTD_resetCStream_internal(zcs, NULL, 0, ZSTD_dct_auto, zcs->cdict, params, pledgedSrcSize); + return 0; } /* ZSTD_initCStream_usingCDict_advanced() : @@ -3772,22 +3870,20 @@ size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize) { DEBUGLOG(4, "ZSTD_initCStream_usingCDict_advanced"); - if (!cdict) return ERROR(dictionary_wrong); /* cannot handle NULL cdict (does not know what to do) */ - { ZSTD_CCtx_params params = zcs->requestedParams; - params.cParams = ZSTD_getCParamsFromCDict(cdict); - params.fParams = fParams; - return ZSTD_initCStream_internal(zcs, - NULL, 0, cdict, - params, pledgedSrcSize); - } + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) ); + FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) ); + zcs->requestedParams.fParams = fParams; + FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, cdict) ); + return 0; } /* note : cdict must outlive compression session */ size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict) { - ZSTD_frameParameters const fParams = { 0 /* contentSizeFlag */, 0 /* checksum */, 0 /* hideDictID */ }; DEBUGLOG(4, "ZSTD_initCStream_usingCDict"); - return ZSTD_initCStream_usingCDict_advanced(zcs, cdict, fParams, ZSTD_CONTENTSIZE_UNKNOWN); /* note : will check that cdict != NULL */ + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) ); + FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, cdict) ); + return 0; } @@ -3797,33 +3893,53 @@ size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict) * dict is loaded with default parameters ZSTD_dm_auto and ZSTD_dlm_byCopy. */ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize, - ZSTD_parameters params, unsigned long long pledgedSrcSize) + ZSTD_parameters params, unsigned long long pss) { - DEBUGLOG(4, "ZSTD_initCStream_advanced: pledgedSrcSize=%u, flag=%u", - (unsigned)pledgedSrcSize, params.fParams.contentSizeFlag); - CHECK_F( ZSTD_checkCParams(params.cParams) ); - if ((pledgedSrcSize==0) && (params.fParams.contentSizeFlag==0)) pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN; /* for compatibility with older programs relying on this behavior. Users should now specify ZSTD_CONTENTSIZE_UNKNOWN. This line will be removed in the future. */ + /* for compatibility with older programs relying on this behavior. + * Users should now specify ZSTD_CONTENTSIZE_UNKNOWN. + * This line will be removed in the future. + */ + U64 const pledgedSrcSize = (pss==0 && params.fParams.contentSizeFlag==0) ? ZSTD_CONTENTSIZE_UNKNOWN : pss; + DEBUGLOG(4, "ZSTD_initCStream_advanced"); + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) ); + FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) ); + FORWARD_IF_ERROR( ZSTD_checkCParams(params.cParams) ); zcs->requestedParams = ZSTD_assignParamsToCCtxParams(zcs->requestedParams, params); - return ZSTD_initCStream_internal(zcs, dict, dictSize, NULL /*cdict*/, zcs->requestedParams, pledgedSrcSize); + FORWARD_IF_ERROR( ZSTD_CCtx_loadDictionary(zcs, dict, dictSize) ); + return 0; } size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel) { - ZSTD_CCtxParams_init(&zcs->requestedParams, compressionLevel); - return ZSTD_initCStream_internal(zcs, dict, dictSize, NULL, zcs->requestedParams, ZSTD_CONTENTSIZE_UNKNOWN); + DEBUGLOG(4, "ZSTD_initCStream_usingDict"); + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) ); + FORWARD_IF_ERROR( ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel) ); + FORWARD_IF_ERROR( ZSTD_CCtx_loadDictionary(zcs, dict, dictSize) ); + return 0; } size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pss) { - U64 const pledgedSrcSize = (pss==0) ? ZSTD_CONTENTSIZE_UNKNOWN : pss; /* temporary : 0 interpreted as "unknown" during transition period. Users willing to specify "unknown" **must** use ZSTD_CONTENTSIZE_UNKNOWN. `0` will be interpreted as "empty" in the future */ - ZSTD_CCtxParams_init(&zcs->requestedParams, compressionLevel); - return ZSTD_initCStream_internal(zcs, NULL, 0, NULL, zcs->requestedParams, pledgedSrcSize); + /* temporary : 0 interpreted as "unknown" during transition period. + * Users willing to specify "unknown" **must** use ZSTD_CONTENTSIZE_UNKNOWN. + * 0 will be interpreted as "empty" in the future. + */ + U64 const pledgedSrcSize = (pss==0) ? ZSTD_CONTENTSIZE_UNKNOWN : pss; + DEBUGLOG(4, "ZSTD_initCStream_srcSize"); + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) ); + FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, NULL) ); + FORWARD_IF_ERROR( ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel) ); + FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) ); + return 0; } size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel) { DEBUGLOG(4, "ZSTD_initCStream"); - return ZSTD_initCStream_srcSize(zcs, compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN); + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) ); + FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, NULL) ); + FORWARD_IF_ERROR( ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel) ); + return 0; } /*====== Compression ======*/ @@ -3847,10 +3963,10 @@ static size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, * internal function for all *compressStream*() variants * non-static, because can be called from zstdmt_compress.c * @return : hint size for next input */ -size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, - ZSTD_outBuffer* output, - ZSTD_inBuffer* input, - ZSTD_EndDirective const flushMode) +static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, + ZSTD_outBuffer* output, + ZSTD_inBuffer* input, + ZSTD_EndDirective const flushMode) { const char* const istart = (const char*)input->src; const char* const iend = istart + input->size; @@ -3873,8 +3989,7 @@ size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, switch(zcs->streamStage) { case zcss_init: - /* call ZSTD_initCStream() first ! */ - return ERROR(init_missing); + RETURN_ERROR(init_missing, "call ZSTD_initCStream() first!"); case zcss_load: if ( (flushMode == ZSTD_e_end) @@ -3884,7 +3999,7 @@ size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, size_t const cSize = ZSTD_compressEnd(zcs, op, oend-op, ip, iend-ip); DEBUGLOG(4, "ZSTD_compressEnd : cSize=%u", (unsigned)cSize); - if (ZSTD_isError(cSize)) return cSize; + FORWARD_IF_ERROR(cSize); ip = iend; op += cSize; zcs->frameEnded = 1; @@ -3925,7 +4040,7 @@ size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, zcs->inBuff + zcs->inToCompress, iSize) : ZSTD_compressContinue(zcs, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize); - if (ZSTD_isError(cSize)) return cSize; + FORWARD_IF_ERROR(cSize); zcs->frameEnded = lastBlock; /* prepare next block */ zcs->inBuffTarget = zcs->inBuffPos + zcs->blockSize; @@ -4001,7 +4116,7 @@ static size_t ZSTD_nextInputSizeHint_MTorST(const ZSTD_CCtx* cctx) size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input) { - CHECK_F( ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue) ); + FORWARD_IF_ERROR( ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue) ); return ZSTD_nextInputSizeHint_MTorST(zcs); } @@ -4013,14 +4128,15 @@ size_t ZSTD_compressStream2( ZSTD_CCtx* cctx, { DEBUGLOG(5, "ZSTD_compressStream2, endOp=%u ", (unsigned)endOp); /* check conditions */ - if (output->pos > output->size) return ERROR(GENERIC); - if (input->pos > input->size) return ERROR(GENERIC); + RETURN_ERROR_IF(output->pos > output->size, GENERIC); + RETURN_ERROR_IF(input->pos > input->size, GENERIC); assert(cctx!=NULL); /* transparent initialization stage */ if (cctx->streamStage == zcss_init) { ZSTD_CCtx_params params = cctx->requestedParams; ZSTD_prefixDict const prefixDict = cctx->prefixDict; + FORWARD_IF_ERROR( ZSTD_initLocalDict(cctx) ); /* Init the local dict if present. */ memset(&cctx->prefixDict, 0, sizeof(cctx->prefixDict)); /* single usage */ assert(prefixDict.dict==NULL || cctx->cdict==NULL); /* only one can be set */ DEBUGLOG(4, "ZSTD_compressStream2 : transparent init stage"); @@ -4039,11 +4155,11 @@ size_t ZSTD_compressStream2( ZSTD_CCtx* cctx, DEBUGLOG(4, "ZSTD_compressStream2: creating new mtctx for nbWorkers=%u", params.nbWorkers); cctx->mtctx = ZSTDMT_createCCtx_advanced(params.nbWorkers, cctx->customMem); - if (cctx->mtctx == NULL) return ERROR(memory_allocation); + RETURN_ERROR_IF(cctx->mtctx == NULL, memory_allocation); } /* mt compression */ DEBUGLOG(4, "call ZSTDMT_initCStream_internal as nbWorkers=%u", params.nbWorkers); - CHECK_F( ZSTDMT_initCStream_internal( + FORWARD_IF_ERROR( ZSTDMT_initCStream_internal( cctx->mtctx, prefixDict.dict, prefixDict.dictSize, ZSTD_dct_rawContent, cctx->cdict, params, cctx->pledgedSrcSizePlusOne-1) ); @@ -4051,7 +4167,7 @@ size_t ZSTD_compressStream2( ZSTD_CCtx* cctx, cctx->appliedParams.nbWorkers = params.nbWorkers; } else #endif - { CHECK_F( ZSTD_resetCStream_internal(cctx, + { FORWARD_IF_ERROR( ZSTD_resetCStream_internal(cctx, prefixDict.dict, prefixDict.dictSize, prefixDict.dictContentType, cctx->cdict, params, cctx->pledgedSrcSizePlusOne-1) ); @@ -4063,20 +4179,30 @@ size_t ZSTD_compressStream2( ZSTD_CCtx* cctx, /* compression stage */ #ifdef ZSTD_MULTITHREAD if (cctx->appliedParams.nbWorkers > 0) { + int const forceMaxProgress = (endOp == ZSTD_e_flush || endOp == ZSTD_e_end); + size_t flushMin; + assert(forceMaxProgress || endOp == ZSTD_e_continue /* Protection for a new flush type */); if (cctx->cParamsChanged) { ZSTDMT_updateCParams_whileCompressing(cctx->mtctx, &cctx->requestedParams); cctx->cParamsChanged = 0; } - { size_t const flushMin = ZSTDMT_compressStream_generic(cctx->mtctx, output, input, endOp); + do { + flushMin = ZSTDMT_compressStream_generic(cctx->mtctx, output, input, endOp); if ( ZSTD_isError(flushMin) || (endOp == ZSTD_e_end && flushMin == 0) ) { /* compression completed */ ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only); } - DEBUGLOG(5, "completed ZSTD_compressStream2 delegating to ZSTDMT_compressStream_generic"); - return flushMin; - } } + FORWARD_IF_ERROR(flushMin); + } while (forceMaxProgress && flushMin != 0 && output->pos < output->size); + DEBUGLOG(5, "completed ZSTD_compressStream2 delegating to ZSTDMT_compressStream_generic"); + /* Either we don't require maximum forward progress, we've finished the + * flush, or we are out of output space. + */ + assert(!forceMaxProgress || flushMin == 0 || output->pos == output->size); + return flushMin; + } #endif - CHECK_F( ZSTD_compressStream_generic(cctx, output, input, endOp) ); + FORWARD_IF_ERROR( ZSTD_compressStream_generic(cctx, output, input, endOp) ); DEBUGLOG(5, "completed ZSTD_compressStream2"); return cctx->outBuffContentSize - cctx->outBuffFlushedSize; /* remaining to flush */ } @@ -4107,10 +4233,10 @@ size_t ZSTD_compress2(ZSTD_CCtx* cctx, dst, dstCapacity, &oPos, src, srcSize, &iPos, ZSTD_e_end); - if (ZSTD_isError(result)) return result; + FORWARD_IF_ERROR(result); if (result != 0) { /* compression not completed, due to lack of output space */ assert(oPos == dstCapacity); - return ERROR(dstSize_tooSmall); + RETURN_ERROR(dstSize_tooSmall); } assert(iPos == srcSize); /* all input is expected consumed */ return oPos; @@ -4132,7 +4258,7 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output) { ZSTD_inBuffer input = { NULL, 0, 0 }; size_t const remainingToFlush = ZSTD_compressStream2(zcs, output, &input, ZSTD_e_end); - CHECK_F( remainingToFlush ); + FORWARD_IF_ERROR( remainingToFlush ); if (zcs->appliedParams.nbWorkers > 0) return remainingToFlush; /* minimal estimation */ /* single thread mode : attempt to calculate remaining to flush more precisely */ { size_t const lastBlockSize = zcs->frameEnded ? 0 : ZSTD_BLOCKHEADERSIZE; @@ -4151,7 +4277,7 @@ int ZSTD_maxCLevel(void) { return ZSTD_MAX_CLEVEL; } int ZSTD_minCLevel(void) { return (int)-ZSTD_TARGETLENGTH_MAX; } static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEVEL+1] = { -{ /* "default" - guarantees a monotonically increasing memory budget */ +{ /* "default" - for any srcSize > 256 KB */ /* W, C, H, S, L, TL, strat */ { 19, 12, 13, 1, 6, 1, ZSTD_fast }, /* base for negative levels */ { 19, 13, 14, 1, 7, 0, ZSTD_fast }, /* level 1 */ @@ -4258,13 +4384,13 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV }; /*! ZSTD_getCParams() : -* @return ZSTD_compressionParameters structure for a selected compression level, srcSize and dictSize. -* Size values are optional, provide 0 if not known or unused */ + * @return ZSTD_compressionParameters structure for a selected compression level, srcSize and dictSize. + * Size values are optional, provide 0 if not known or unused */ ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize) { size_t const addedSize = srcSizeHint ? 0 : 500; - U64 const rSize = srcSizeHint+dictSize ? srcSizeHint+dictSize+addedSize : (U64)-1; - U32 const tableID = (rSize <= 256 KB) + (rSize <= 128 KB) + (rSize <= 16 KB); /* intentional underflow for srcSizeHint == 0 */ + U64 const rSize = srcSizeHint+dictSize ? srcSizeHint+dictSize+addedSize : ZSTD_CONTENTSIZE_UNKNOWN; /* intentional overflow for srcSizeHint == ZSTD_CONTENTSIZE_UNKNOWN */ + U32 const tableID = (rSize <= 256 KB) + (rSize <= 128 KB) + (rSize <= 16 KB); int row = compressionLevel; DEBUGLOG(5, "ZSTD_getCParams (cLevel=%i)", compressionLevel); if (compressionLevel == 0) row = ZSTD_CLEVEL_DEFAULT; /* 0 == default */ @@ -4272,13 +4398,14 @@ ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long l if (compressionLevel > ZSTD_MAX_CLEVEL) row = ZSTD_MAX_CLEVEL; { ZSTD_compressionParameters cp = ZSTD_defaultCParameters[tableID][row]; if (compressionLevel < 0) cp.targetLength = (unsigned)(-compressionLevel); /* acceleration factor */ - return ZSTD_adjustCParams_internal(cp, srcSizeHint, dictSize); + return ZSTD_adjustCParams_internal(cp, srcSizeHint, dictSize); /* refine parameters based on srcSize & dictSize */ } } /*! ZSTD_getParams() : -* same as ZSTD_getCParams(), but @return a `ZSTD_parameters` object (instead of `ZSTD_compressionParameters`). -* All fields of `ZSTD_frameParameters` are set to default (0) */ + * same idea as ZSTD_getCParams() + * @return a `ZSTD_parameters` structure (instead of `ZSTD_compressionParameters`). + * Fields of `ZSTD_frameParameters` are set to default values */ ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize) { ZSTD_parameters params; ZSTD_compressionParameters const cParams = ZSTD_getCParams(compressionLevel, srcSizeHint, dictSize); diff --git a/thirdparty/zstd/compress/zstd_compress_internal.h b/thirdparty/zstd/compress/zstd_compress_internal.h index 29bca59859..cc3cbb9da9 100644 --- a/thirdparty/zstd/compress/zstd_compress_internal.h +++ b/thirdparty/zstd/compress/zstd_compress_internal.h @@ -36,9 +36,9 @@ extern "C" { #define ZSTD_DUBT_UNSORTED_MARK 1 /* For btlazy2 strategy, index 1 now means "unsorted". It could be confused for a real successor at index "1", if sorted as larger than its predecessor. It's not a big deal though : candidate will just be sorted again. - Additionnally, candidate position 1 will be lost. + Additionally, candidate position 1 will be lost. But candidate 1 cannot hide a large tree of candidates, so it's a minimal loss. - The benefit is that ZSTD_DUBT_UNSORTED_MARK cannot be misdhandled after table re-use with a different strategy + The benefit is that ZSTD_DUBT_UNSORTED_MARK cannot be mishandled after table re-use with a different strategy Constant required by ZSTD_compressBlock_btlazy2() and ZSTD_reduceTable_internal() */ @@ -55,6 +55,14 @@ typedef struct ZSTD_prefixDict_s { } ZSTD_prefixDict; typedef struct { + void* dictBuffer; + void const* dict; + size_t dictSize; + ZSTD_dictContentType_e dictContentType; + ZSTD_CDict* cdict; +} ZSTD_localDict; + +typedef struct { U32 CTable[HUF_CTABLE_SIZE_U32(255)]; HUF_repeat repeatMode; } ZSTD_hufCTables_t; @@ -107,6 +115,7 @@ typedef struct { U32 offCodeSumBasePrice; /* to compare to log2(offreq) */ ZSTD_OptPrice_e priceType; /* prices can be determined dynamically, or follow a pre-defined cost structure */ const ZSTD_entropyCTables_t* symbolCosts; /* pre-calculated dictionary statistics */ + ZSTD_literalCompressionMode_e literalCompressionMode; } optState_t; typedef struct { @@ -188,6 +197,7 @@ struct ZSTD_CCtx_params_s { * 1<<wLog, even for dictionary */ ZSTD_dictAttachPref_e attachDictPref; + ZSTD_literalCompressionMode_e literalCompressionMode; /* Multithreading: used to pass parameters to mtctx */ int nbWorkers; @@ -243,7 +253,7 @@ struct ZSTD_CCtx_s { U32 frameEnded; /* Dictionary */ - ZSTD_CDict* cdictLocal; + ZSTD_localDict localDict; const ZSTD_CDict* cdict; ZSTD_prefixDict prefixDict; /* single-usage dictionary */ @@ -806,13 +816,6 @@ size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs, void ZSTD_resetSeqStore(seqStore_t* ssPtr); -/*! ZSTD_compressStream_generic() : - * Private use only. To be called from zstdmt_compress.c in single-thread mode. */ -size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, - ZSTD_outBuffer* output, - ZSTD_inBuffer* input, - ZSTD_EndDirective const flushMode); - /*! ZSTD_getCParamsFromCDict() : * as the name implies */ ZSTD_compressionParameters ZSTD_getCParamsFromCDict(const ZSTD_CDict* cdict); @@ -839,7 +842,7 @@ size_t ZSTD_compress_advanced_internal(ZSTD_CCtx* cctx, /* ZSTD_writeLastEmptyBlock() : * output an empty Block with end-of-frame mark to complete a frame * @return : size of data written into `dst` (== ZSTD_blockHeaderSize (defined in zstd_internal.h)) - * or an error code if `dstCapcity` is too small (<ZSTD_blockHeaderSize) + * or an error code if `dstCapacity` is too small (<ZSTD_blockHeaderSize) */ size_t ZSTD_writeLastEmptyBlock(void* dst, size_t dstCapacity); diff --git a/thirdparty/zstd/compress/zstd_fast.c b/thirdparty/zstd/compress/zstd_fast.c index 40ba0f73e6..ed997b441c 100644 --- a/thirdparty/zstd/compress/zstd_fast.c +++ b/thirdparty/zstd/compress/zstd_fast.c @@ -45,7 +45,155 @@ FORCE_INLINE_TEMPLATE size_t ZSTD_compressBlock_fast_generic( ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize, - U32 const mls, ZSTD_dictMode_e const dictMode) + U32 const mls) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const hashTable = ms->hashTable; + U32 const hlog = cParams->hashLog; + /* support stepSize of 0 */ + size_t const stepSize = cParams->targetLength + !(cParams->targetLength) + 1; + const BYTE* const base = ms->window.base; + const BYTE* const istart = (const BYTE*)src; + /* We check ip0 (ip + 0) and ip1 (ip + 1) each loop */ + const BYTE* ip0 = istart; + const BYTE* ip1; + const BYTE* anchor = istart; + const U32 prefixStartIndex = ms->window.dictLimit; + const BYTE* const prefixStart = base + prefixStartIndex; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - HASH_READ_SIZE; + U32 offset_1=rep[0], offset_2=rep[1]; + U32 offsetSaved = 0; + + /* init */ + ip0 += (ip0 == prefixStart); + ip1 = ip0 + 1; + { + U32 const maxRep = (U32)(ip0 - prefixStart); + if (offset_2 > maxRep) offsetSaved = offset_2, offset_2 = 0; + if (offset_1 > maxRep) offsetSaved = offset_1, offset_1 = 0; + } + + /* Main Search Loop */ + while (ip1 < ilimit) { /* < instead of <=, because check at ip0+2 */ + size_t mLength; + BYTE const* ip2 = ip0 + 2; + size_t const h0 = ZSTD_hashPtr(ip0, hlog, mls); + U32 const val0 = MEM_read32(ip0); + size_t const h1 = ZSTD_hashPtr(ip1, hlog, mls); + U32 const val1 = MEM_read32(ip1); + U32 const current0 = (U32)(ip0-base); + U32 const current1 = (U32)(ip1-base); + U32 const matchIndex0 = hashTable[h0]; + U32 const matchIndex1 = hashTable[h1]; + BYTE const* repMatch = ip2-offset_1; + const BYTE* match0 = base + matchIndex0; + const BYTE* match1 = base + matchIndex1; + U32 offcode; + hashTable[h0] = current0; /* update hash table */ + hashTable[h1] = current1; /* update hash table */ + + assert(ip0 + 1 == ip1); + + if ((offset_1 > 0) & (MEM_read32(repMatch) == MEM_read32(ip2))) { + mLength = ip2[-1] == repMatch[-1] ? 1 : 0; + ip0 = ip2 - mLength; + match0 = repMatch - mLength; + offcode = 0; + goto _match; + } + if ((matchIndex0 > prefixStartIndex) && MEM_read32(match0) == val0) { + /* found a regular match */ + goto _offset; + } + if ((matchIndex1 > prefixStartIndex) && MEM_read32(match1) == val1) { + /* found a regular match after one literal */ + ip0 = ip1; + match0 = match1; + goto _offset; + } + { + size_t const step = ((ip0-anchor) >> (kSearchStrength - 1)) + stepSize; + assert(step >= 2); + ip0 += step; + ip1 += step; + continue; + } +_offset: /* Requires: ip0, match0 */ + /* Compute the offset code */ + offset_2 = offset_1; + offset_1 = (U32)(ip0-match0); + offcode = offset_1 + ZSTD_REP_MOVE; + mLength = 0; + /* Count the backwards match length */ + while (((ip0>anchor) & (match0>prefixStart)) + && (ip0[-1] == match0[-1])) { ip0--; match0--; mLength++; } /* catch up */ + +_match: /* Requires: ip0, match0, offcode */ + /* Count the forward length */ + mLength += ZSTD_count(ip0+mLength+4, match0+mLength+4, iend) + 4; + ZSTD_storeSeq(seqStore, ip0-anchor, anchor, offcode, mLength-MINMATCH); + /* match found */ + ip0 += mLength; + anchor = ip0; + ip1 = ip0 + 1; + + if (ip0 <= ilimit) { + /* Fill Table */ + assert(base+current0+2 > istart); /* check base overflow */ + hashTable[ZSTD_hashPtr(base+current0+2, hlog, mls)] = current0+2; /* here because current+2 could be > iend-8 */ + hashTable[ZSTD_hashPtr(ip0-2, hlog, mls)] = (U32)(ip0-2-base); + + while ( (ip0 <= ilimit) + && ( (offset_2>0) + & (MEM_read32(ip0) == MEM_read32(ip0 - offset_2)) )) { + /* store sequence */ + size_t const rLength = ZSTD_count(ip0+4, ip0+4-offset_2, iend) + 4; + U32 const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff; /* swap offset_2 <=> offset_1 */ + hashTable[ZSTD_hashPtr(ip0, hlog, mls)] = (U32)(ip0-base); + ip0 += rLength; + ip1 = ip0 + 1; + ZSTD_storeSeq(seqStore, 0, anchor, 0, rLength-MINMATCH); + anchor = ip0; + continue; /* faster when present (confirmed on gcc-8) ... (?) */ + } + } + } + + /* save reps for next block */ + rep[0] = offset_1 ? offset_1 : offsetSaved; + rep[1] = offset_2 ? offset_2 : offsetSaved; + + /* Return the last literals size */ + return iend - anchor; +} + + +size_t ZSTD_compressBlock_fast( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + ZSTD_compressionParameters const* cParams = &ms->cParams; + U32 const mls = cParams->minMatch; + assert(ms->dictMatchState == NULL); + switch(mls) + { + default: /* includes case 3 */ + case 4 : + return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 4); + case 5 : + return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 5); + case 6 : + return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 6); + case 7 : + return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 7); + } +} + +FORCE_INLINE_TEMPLATE +size_t ZSTD_compressBlock_fast_dictMatchState_generic( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize, U32 const mls) { const ZSTD_compressionParameters* const cParams = &ms->cParams; U32* const hashTable = ms->hashTable; @@ -64,46 +212,26 @@ size_t ZSTD_compressBlock_fast_generic( U32 offsetSaved = 0; const ZSTD_matchState_t* const dms = ms->dictMatchState; - const ZSTD_compressionParameters* const dictCParams = - dictMode == ZSTD_dictMatchState ? - &dms->cParams : NULL; - const U32* const dictHashTable = dictMode == ZSTD_dictMatchState ? - dms->hashTable : NULL; - const U32 dictStartIndex = dictMode == ZSTD_dictMatchState ? - dms->window.dictLimit : 0; - const BYTE* const dictBase = dictMode == ZSTD_dictMatchState ? - dms->window.base : NULL; - const BYTE* const dictStart = dictMode == ZSTD_dictMatchState ? - dictBase + dictStartIndex : NULL; - const BYTE* const dictEnd = dictMode == ZSTD_dictMatchState ? - dms->window.nextSrc : NULL; - const U32 dictIndexDelta = dictMode == ZSTD_dictMatchState ? - prefixStartIndex - (U32)(dictEnd - dictBase) : - 0; + const ZSTD_compressionParameters* const dictCParams = &dms->cParams ; + const U32* const dictHashTable = dms->hashTable; + const U32 dictStartIndex = dms->window.dictLimit; + const BYTE* const dictBase = dms->window.base; + const BYTE* const dictStart = dictBase + dictStartIndex; + const BYTE* const dictEnd = dms->window.nextSrc; + const U32 dictIndexDelta = prefixStartIndex - (U32)(dictEnd - dictBase); const U32 dictAndPrefixLength = (U32)(ip - prefixStart + dictEnd - dictStart); - const U32 dictHLog = dictMode == ZSTD_dictMatchState ? - dictCParams->hashLog : hlog; - - assert(dictMode == ZSTD_noDict || dictMode == ZSTD_dictMatchState); + const U32 dictHLog = dictCParams->hashLog; /* otherwise, we would get index underflow when translating a dict index * into a local index */ - assert(dictMode != ZSTD_dictMatchState - || prefixStartIndex >= (U32)(dictEnd - dictBase)); + assert(prefixStartIndex >= (U32)(dictEnd - dictBase)); /* init */ ip += (dictAndPrefixLength == 0); - if (dictMode == ZSTD_noDict) { - U32 const maxRep = (U32)(ip - prefixStart); - if (offset_2 > maxRep) offsetSaved = offset_2, offset_2 = 0; - if (offset_1 > maxRep) offsetSaved = offset_1, offset_1 = 0; - } - if (dictMode == ZSTD_dictMatchState) { - /* dictMatchState repCode checks don't currently handle repCode == 0 - * disabling. */ - assert(offset_1 <= dictAndPrefixLength); - assert(offset_2 <= dictAndPrefixLength); - } + /* dictMatchState repCode checks don't currently handle repCode == 0 + * disabling. */ + assert(offset_1 <= dictAndPrefixLength); + assert(offset_2 <= dictAndPrefixLength); /* Main Search Loop */ while (ip < ilimit) { /* < instead of <=, because repcode check at (ip+1) */ @@ -113,50 +241,37 @@ size_t ZSTD_compressBlock_fast_generic( U32 const matchIndex = hashTable[h]; const BYTE* match = base + matchIndex; const U32 repIndex = current + 1 - offset_1; - const BYTE* repMatch = (dictMode == ZSTD_dictMatchState - && repIndex < prefixStartIndex) ? + const BYTE* repMatch = (repIndex < prefixStartIndex) ? dictBase + (repIndex - dictIndexDelta) : base + repIndex; hashTable[h] = current; /* update hash table */ - if ( (dictMode == ZSTD_dictMatchState) - && ((U32)((prefixStartIndex-1) - repIndex) >= 3) /* intentional underflow : ensure repIndex isn't overlapping dict + prefix */ + if ( ((U32)((prefixStartIndex-1) - repIndex) >= 3) /* intentional underflow : ensure repIndex isn't overlapping dict + prefix */ && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) { const BYTE* const repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend; mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixStart) + 4; ip++; ZSTD_storeSeq(seqStore, ip-anchor, anchor, 0, mLength-MINMATCH); - } else if ( dictMode == ZSTD_noDict - && ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1)))) { - mLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4; - ip++; - ZSTD_storeSeq(seqStore, ip-anchor, anchor, 0, mLength-MINMATCH); } else if ( (matchIndex <= prefixStartIndex) ) { - if (dictMode == ZSTD_dictMatchState) { - size_t const dictHash = ZSTD_hashPtr(ip, dictHLog, mls); - U32 const dictMatchIndex = dictHashTable[dictHash]; - const BYTE* dictMatch = dictBase + dictMatchIndex; - if (dictMatchIndex <= dictStartIndex || - MEM_read32(dictMatch) != MEM_read32(ip)) { - assert(stepSize >= 1); - ip += ((ip-anchor) >> kSearchStrength) + stepSize; - continue; - } else { - /* found a dict match */ - U32 const offset = (U32)(current-dictMatchIndex-dictIndexDelta); - mLength = ZSTD_count_2segments(ip+4, dictMatch+4, iend, dictEnd, prefixStart) + 4; - while (((ip>anchor) & (dictMatch>dictStart)) - && (ip[-1] == dictMatch[-1])) { - ip--; dictMatch--; mLength++; - } /* catch up */ - offset_2 = offset_1; - offset_1 = offset; - ZSTD_storeSeq(seqStore, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); - } - } else { + size_t const dictHash = ZSTD_hashPtr(ip, dictHLog, mls); + U32 const dictMatchIndex = dictHashTable[dictHash]; + const BYTE* dictMatch = dictBase + dictMatchIndex; + if (dictMatchIndex <= dictStartIndex || + MEM_read32(dictMatch) != MEM_read32(ip)) { assert(stepSize >= 1); ip += ((ip-anchor) >> kSearchStrength) + stepSize; continue; + } else { + /* found a dict match */ + U32 const offset = (U32)(current-dictMatchIndex-dictIndexDelta); + mLength = ZSTD_count_2segments(ip+4, dictMatch+4, iend, dictEnd, prefixStart) + 4; + while (((ip>anchor) & (dictMatch>dictStart)) + && (ip[-1] == dictMatch[-1])) { + ip--; dictMatch--; mLength++; + } /* catch up */ + offset_2 = offset_1; + offset_1 = offset; + ZSTD_storeSeq(seqStore, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); } } else if (MEM_read32(match) != MEM_read32(ip)) { /* it's not a match, and we're not going to check the dictionary */ @@ -185,41 +300,27 @@ size_t ZSTD_compressBlock_fast_generic( hashTable[ZSTD_hashPtr(ip-2, hlog, mls)] = (U32)(ip-2-base); /* check immediate repcode */ - if (dictMode == ZSTD_dictMatchState) { - while (ip <= ilimit) { - U32 const current2 = (U32)(ip-base); - U32 const repIndex2 = current2 - offset_2; - const BYTE* repMatch2 = repIndex2 < prefixStartIndex ? - dictBase - dictIndexDelta + repIndex2 : - base + repIndex2; - if ( ((U32)((prefixStartIndex-1) - (U32)repIndex2) >= 3 /* intentional overflow */) - && (MEM_read32(repMatch2) == MEM_read32(ip)) ) { - const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend; - size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixStart) + 4; - U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ - ZSTD_storeSeq(seqStore, 0, anchor, 0, repLength2-MINMATCH); - hashTable[ZSTD_hashPtr(ip, hlog, mls)] = current2; - ip += repLength2; - anchor = ip; - continue; - } - break; + while (ip <= ilimit) { + U32 const current2 = (U32)(ip-base); + U32 const repIndex2 = current2 - offset_2; + const BYTE* repMatch2 = repIndex2 < prefixStartIndex ? + dictBase - dictIndexDelta + repIndex2 : + base + repIndex2; + if ( ((U32)((prefixStartIndex-1) - (U32)repIndex2) >= 3 /* intentional overflow */) + && (MEM_read32(repMatch2) == MEM_read32(ip)) ) { + const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend; + size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixStart) + 4; + U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ + ZSTD_storeSeq(seqStore, 0, anchor, 0, repLength2-MINMATCH); + hashTable[ZSTD_hashPtr(ip, hlog, mls)] = current2; + ip += repLength2; + anchor = ip; + continue; } + break; } - - if (dictMode == ZSTD_noDict) { - while ( (ip <= ilimit) - && ( (offset_2>0) - & (MEM_read32(ip) == MEM_read32(ip - offset_2)) )) { - /* store sequence */ - size_t const rLength = ZSTD_count(ip+4, ip+4-offset_2, iend) + 4; - U32 const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff; /* swap offset_2 <=> offset_1 */ - hashTable[ZSTD_hashPtr(ip, hlog, mls)] = (U32)(ip-base); - ZSTD_storeSeq(seqStore, 0, anchor, 0, rLength-MINMATCH); - ip += rLength; - anchor = ip; - continue; /* faster when present ... (?) */ - } } } } + } + } /* save reps for next block */ rep[0] = offset_1 ? offset_1 : offsetSaved; @@ -229,28 +330,6 @@ size_t ZSTD_compressBlock_fast_generic( return iend - anchor; } - -size_t ZSTD_compressBlock_fast( - ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize) -{ - ZSTD_compressionParameters const* cParams = &ms->cParams; - U32 const mls = cParams->minMatch; - assert(ms->dictMatchState == NULL); - switch(mls) - { - default: /* includes case 3 */ - case 4 : - return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 4, ZSTD_noDict); - case 5 : - return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 5, ZSTD_noDict); - case 6 : - return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 6, ZSTD_noDict); - case 7 : - return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 7, ZSTD_noDict); - } -} - size_t ZSTD_compressBlock_fast_dictMatchState( ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) @@ -262,13 +341,13 @@ size_t ZSTD_compressBlock_fast_dictMatchState( { default: /* includes case 3 */ case 4 : - return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 4, ZSTD_dictMatchState); + return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 4); case 5 : - return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 5, ZSTD_dictMatchState); + return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 5); case 6 : - return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 6, ZSTD_dictMatchState); + return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 6); case 7 : - return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 7, ZSTD_dictMatchState); + return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 7); } } diff --git a/thirdparty/zstd/compress/zstd_lazy.h b/thirdparty/zstd/compress/zstd_lazy.h index ef85a6df9c..bb1763069f 100644 --- a/thirdparty/zstd/compress/zstd_lazy.h +++ b/thirdparty/zstd/compress/zstd_lazy.h @@ -19,7 +19,7 @@ extern "C" { U32 ZSTD_insertAndFindFirstIndex(ZSTD_matchState_t* ms, const BYTE* ip); -void ZSTD_preserveUnsortedMark (U32* const table, U32 const size, U32 const reducerValue); /*! used in ZSTD_reduceIndex(). pre-emptively increase value of ZSTD_DUBT_UNSORTED_MARK */ +void ZSTD_preserveUnsortedMark (U32* const table, U32 const size, U32 const reducerValue); /*! used in ZSTD_reduceIndex(). preemptively increase value of ZSTD_DUBT_UNSORTED_MARK */ size_t ZSTD_compressBlock_btlazy2( ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], diff --git a/thirdparty/zstd/compress/zstd_ldm.c b/thirdparty/zstd/compress/zstd_ldm.c index 58eb2ffe4d..784d20f3ab 100644 --- a/thirdparty/zstd/compress/zstd_ldm.c +++ b/thirdparty/zstd/compress/zstd_ldm.c @@ -429,7 +429,7 @@ size_t ZSTD_ldm_generateSequences( */ assert(ldmState->window.nextSrc >= (BYTE const*)src + srcSize); /* The input could be very large (in zstdmt), so it must be broken up into - * chunks to enforce the maximmum distance and handle overflow correction. + * chunks to enforce the maximum distance and handle overflow correction. */ assert(sequences->pos <= sequences->size); assert(sequences->size <= sequences->capacity); diff --git a/thirdparty/zstd/compress/zstd_opt.c b/thirdparty/zstd/compress/zstd_opt.c index 44de6e97fd..efb69d3267 100644 --- a/thirdparty/zstd/compress/zstd_opt.c +++ b/thirdparty/zstd/compress/zstd_opt.c @@ -64,9 +64,15 @@ MEM_STATIC double ZSTD_fCost(U32 price) } #endif +static int ZSTD_compressedLiterals(optState_t const* const optPtr) +{ + return optPtr->literalCompressionMode != ZSTD_lcm_uncompressed; +} + static void ZSTD_setBasePrices(optState_t* optPtr, int optLevel) { - optPtr->litSumBasePrice = WEIGHT(optPtr->litSum, optLevel); + if (ZSTD_compressedLiterals(optPtr)) + optPtr->litSumBasePrice = WEIGHT(optPtr->litSum, optLevel); optPtr->litLengthSumBasePrice = WEIGHT(optPtr->litLengthSum, optLevel); optPtr->matchLengthSumBasePrice = WEIGHT(optPtr->matchLengthSum, optLevel); optPtr->offCodeSumBasePrice = WEIGHT(optPtr->offCodeSum, optLevel); @@ -99,6 +105,7 @@ ZSTD_rescaleFreqs(optState_t* const optPtr, const BYTE* const src, size_t const srcSize, int const optLevel) { + int const compressedLiterals = ZSTD_compressedLiterals(optPtr); DEBUGLOG(5, "ZSTD_rescaleFreqs (srcSize=%u)", (unsigned)srcSize); optPtr->priceType = zop_dynamic; @@ -113,9 +120,10 @@ ZSTD_rescaleFreqs(optState_t* const optPtr, /* huffman table presumed generated by dictionary */ optPtr->priceType = zop_dynamic; - assert(optPtr->litFreq != NULL); - optPtr->litSum = 0; - { unsigned lit; + if (compressedLiterals) { + unsigned lit; + assert(optPtr->litFreq != NULL); + optPtr->litSum = 0; for (lit=0; lit<=MaxLit; lit++) { U32 const scaleLog = 11; /* scale to 2K */ U32 const bitCost = HUF_getNbBits(optPtr->symbolCosts->huf.CTable, lit); @@ -163,10 +171,11 @@ ZSTD_rescaleFreqs(optState_t* const optPtr, } else { /* not a dictionary */ assert(optPtr->litFreq != NULL); - { unsigned lit = MaxLit; + if (compressedLiterals) { + unsigned lit = MaxLit; HIST_count_simple(optPtr->litFreq, &lit, src, srcSize); /* use raw first block to init statistics */ + optPtr->litSum = ZSTD_downscaleStat(optPtr->litFreq, MaxLit, 1); } - optPtr->litSum = ZSTD_downscaleStat(optPtr->litFreq, MaxLit, 1); { unsigned ll; for (ll=0; ll<=MaxLL; ll++) @@ -190,7 +199,8 @@ ZSTD_rescaleFreqs(optState_t* const optPtr, } else { /* new block : re-use previous statistics, scaled down */ - optPtr->litSum = ZSTD_downscaleStat(optPtr->litFreq, MaxLit, 1); + if (compressedLiterals) + optPtr->litSum = ZSTD_downscaleStat(optPtr->litFreq, MaxLit, 1); optPtr->litLengthSum = ZSTD_downscaleStat(optPtr->litLengthFreq, MaxLL, 0); optPtr->matchLengthSum = ZSTD_downscaleStat(optPtr->matchLengthFreq, MaxML, 0); optPtr->offCodeSum = ZSTD_downscaleStat(optPtr->offCodeFreq, MaxOff, 0); @@ -207,6 +217,10 @@ static U32 ZSTD_rawLiteralsCost(const BYTE* const literals, U32 const litLength, int optLevel) { if (litLength == 0) return 0; + + if (!ZSTD_compressedLiterals(optPtr)) + return (litLength << 3) * BITCOST_MULTIPLIER; /* Uncompressed - 8 bytes per literal. */ + if (optPtr->priceType == zop_predef) return (litLength*6) * BITCOST_MULTIPLIER; /* 6 bit per literal - no statistic used */ @@ -310,7 +324,8 @@ static void ZSTD_updateStats(optState_t* const optPtr, U32 offsetCode, U32 matchLength) { /* literals */ - { U32 u; + if (ZSTD_compressedLiterals(optPtr)) { + U32 u; for (u=0; u < litLength; u++) optPtr->litFreq[literals[u]] += ZSTD_LITFREQ_ADD; optPtr->litSum += litLength*ZSTD_LITFREQ_ADD; @@ -870,7 +885,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, /* large match -> immediate encoding */ { U32 const maxML = matches[nbMatches-1].len; U32 const maxOffset = matches[nbMatches-1].off; - DEBUGLOG(6, "found %u matches of maxLength=%u and maxOffCode=%u at cPos=%u => start new serie", + DEBUGLOG(6, "found %u matches of maxLength=%u and maxOffCode=%u at cPos=%u => start new series", nbMatches, maxML, maxOffset, (U32)(ip-prefixStart)); if (maxML > sufficient_len) { @@ -1108,7 +1123,8 @@ static U32 ZSTD_upscaleStat(unsigned* table, U32 lastEltIndex, int bonus) /* used in 2-pass strategy */ MEM_STATIC void ZSTD_upscaleStats(optState_t* optPtr) { - optPtr->litSum = ZSTD_upscaleStat(optPtr->litFreq, MaxLit, 0); + if (ZSTD_compressedLiterals(optPtr)) + optPtr->litSum = ZSTD_upscaleStat(optPtr->litFreq, MaxLit, 0); optPtr->litLengthSum = ZSTD_upscaleStat(optPtr->litLengthFreq, MaxLL, 0); optPtr->matchLengthSum = ZSTD_upscaleStat(optPtr->matchLengthFreq, MaxML, 0); optPtr->offCodeSum = ZSTD_upscaleStat(optPtr->offCodeFreq, MaxOff, 0); @@ -1117,7 +1133,7 @@ MEM_STATIC void ZSTD_upscaleStats(optState_t* optPtr) /* ZSTD_initStats_ultra(): * make a first compression pass, just to seed stats with more accurate starting values. * only works on first block, with no dictionary and no ldm. - * this function cannot error, hence its constract must be respected. + * this function cannot error, hence its contract must be respected. */ static void ZSTD_initStats_ultra(ZSTD_matchState_t* ms, diff --git a/thirdparty/zstd/compress/zstdmt_compress.c b/thirdparty/zstd/compress/zstdmt_compress.c index 2cbd6ffade..38fbb90768 100644 --- a/thirdparty/zstd/compress/zstdmt_compress.c +++ b/thirdparty/zstd/compress/zstdmt_compress.c @@ -22,6 +22,7 @@ /* ====== Dependencies ====== */ #include <string.h> /* memcpy, memset */ #include <limits.h> /* INT_MAX, UINT_MAX */ +#include "mem.h" /* MEM_STATIC */ #include "pool.h" /* threadpool */ #include "threading.h" /* mutex */ #include "zstd_compress_internal.h" /* MIN, ERROR, ZSTD_*, ZSTD_highbit32 */ @@ -456,7 +457,7 @@ typedef struct { * Must be acquired after the main mutex when acquiring both. */ ZSTD_pthread_mutex_t ldmWindowMutex; - ZSTD_pthread_cond_t ldmWindowCond; /* Signaled when ldmWindow is udpated */ + ZSTD_pthread_cond_t ldmWindowCond; /* Signaled when ldmWindow is updated */ ZSTD_window_t ldmWindow; /* A thread-safe copy of ldmState.window */ } serialState_t; @@ -647,7 +648,7 @@ static void ZSTDMT_compressionJob(void* jobDescription) buffer_t dstBuff = job->dstBuff; size_t lastCBlockSize = 0; - /* ressources */ + /* resources */ if (cctx==NULL) JOB_ERROR(ERROR(memory_allocation)); if (dstBuff.start == NULL) { /* streaming job : doesn't provide a dstBuffer */ dstBuff = ZSTDMT_getBuffer(job->bufPool); @@ -672,7 +673,7 @@ static void ZSTDMT_compressionJob(void* jobDescription) if (ZSTD_isError(initError)) JOB_ERROR(initError); } else { /* srcStart points at reloaded section */ U64 const pledgedSrcSize = job->firstJob ? job->fullFrameSize : job->src.size; - { size_t const forceWindowError = ZSTD_CCtxParam_setParameter(&jobParams, ZSTD_c_forceMaxWindow, !job->firstJob); + { size_t const forceWindowError = ZSTD_CCtxParams_setParameter(&jobParams, ZSTD_c_forceMaxWindow, !job->firstJob); if (ZSTD_isError(forceWindowError)) JOB_ERROR(forceWindowError); } { size_t const initError = ZSTD_compressBegin_advanced_internal(cctx, @@ -864,14 +865,10 @@ static size_t ZSTDMT_expandJobsTable (ZSTDMT_CCtx* mtctx, U32 nbWorkers) { * Internal use only */ size_t ZSTDMT_CCtxParam_setNbWorkers(ZSTD_CCtx_params* params, unsigned nbWorkers) { - if (nbWorkers > ZSTDMT_NBWORKERS_MAX) nbWorkers = ZSTDMT_NBWORKERS_MAX; - params->nbWorkers = nbWorkers; - params->overlapLog = ZSTDMT_OVERLAPLOG_DEFAULT; - params->jobSize = 0; - return nbWorkers; + return ZSTD_CCtxParams_setParameter(params, ZSTD_c_nbWorkers, (int)nbWorkers); } -ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbWorkers, ZSTD_customMem cMem) +MEM_STATIC ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced_internal(unsigned nbWorkers, ZSTD_customMem cMem) { ZSTDMT_CCtx* mtctx; U32 nbJobs = nbWorkers + 2; @@ -906,6 +903,17 @@ ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbWorkers, ZSTD_customMem cMem) return mtctx; } +ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbWorkers, ZSTD_customMem cMem) +{ +#ifdef ZSTD_MULTITHREAD + return ZSTDMT_createCCtx_advanced_internal(nbWorkers, cMem); +#else + (void)nbWorkers; + (void)cMem; + return NULL; +#endif +} + ZSTDMT_CCtx* ZSTDMT_createCCtx(unsigned nbWorkers) { return ZSTDMT_createCCtx_advanced(nbWorkers, ZSTD_defaultCMem); @@ -986,26 +994,13 @@ ZSTDMT_CCtxParam_setMTCtxParameter(ZSTD_CCtx_params* params, { case ZSTDMT_p_jobSize : DEBUGLOG(4, "ZSTDMT_CCtxParam_setMTCtxParameter : set jobSize to %i", value); - if ( value != 0 /* default */ - && value < ZSTDMT_JOBSIZE_MIN) - value = ZSTDMT_JOBSIZE_MIN; - assert(value >= 0); - if (value > ZSTDMT_JOBSIZE_MAX) value = ZSTDMT_JOBSIZE_MAX; - params->jobSize = value; - return value; - + return ZSTD_CCtxParams_setParameter(params, ZSTD_c_jobSize, value); case ZSTDMT_p_overlapLog : DEBUGLOG(4, "ZSTDMT_p_overlapLog : %i", value); - if (value < ZSTD_OVERLAPLOG_MIN) value = ZSTD_OVERLAPLOG_MIN; - if (value > ZSTD_OVERLAPLOG_MAX) value = ZSTD_OVERLAPLOG_MAX; - params->overlapLog = value; - return value; - + return ZSTD_CCtxParams_setParameter(params, ZSTD_c_overlapLog, value); case ZSTDMT_p_rsyncable : - value = (value != 0); - params->rsyncable = value; - return value; - + DEBUGLOG(4, "ZSTD_p_rsyncable : %i", value); + return ZSTD_CCtxParams_setParameter(params, ZSTD_c_rsyncable, value); default : return ERROR(parameter_unsupported); } @@ -1021,32 +1016,29 @@ size_t ZSTDMT_getMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter, { switch (parameter) { case ZSTDMT_p_jobSize: - assert(mtctx->params.jobSize <= INT_MAX); - *value = (int)(mtctx->params.jobSize); - break; + return ZSTD_CCtxParams_getParameter(&mtctx->params, ZSTD_c_jobSize, value); case ZSTDMT_p_overlapLog: - *value = mtctx->params.overlapLog; - break; + return ZSTD_CCtxParams_getParameter(&mtctx->params, ZSTD_c_overlapLog, value); case ZSTDMT_p_rsyncable: - *value = mtctx->params.rsyncable; - break; + return ZSTD_CCtxParams_getParameter(&mtctx->params, ZSTD_c_rsyncable, value); default: return ERROR(parameter_unsupported); } - return 0; } /* Sets parameters relevant to the compression job, * initializing others to default values. */ static ZSTD_CCtx_params ZSTDMT_initJobCCtxParams(ZSTD_CCtx_params const params) { - ZSTD_CCtx_params jobParams; - memset(&jobParams, 0, sizeof(jobParams)); - - jobParams.cParams = params.cParams; - jobParams.fParams = params.fParams; - jobParams.compressionLevel = params.compressionLevel; - + ZSTD_CCtx_params jobParams = params; + /* Clear parameters related to multithreading */ + jobParams.forceWindow = 0; + jobParams.nbWorkers = 0; + jobParams.jobSize = 0; + jobParams.overlapLog = 0; + jobParams.rsyncable = 0; + memset(&jobParams.ldmParams, 0, sizeof(ldmParams_t)); + memset(&jobParams.customMem, 0, sizeof(ZSTD_customMem)); return jobParams; } @@ -1056,7 +1048,7 @@ static ZSTD_CCtx_params ZSTDMT_initJobCCtxParams(ZSTD_CCtx_params const params) static size_t ZSTDMT_resize(ZSTDMT_CCtx* mtctx, unsigned nbWorkers) { if (POOL_resize(mtctx->factory, nbWorkers)) return ERROR(memory_allocation); - CHECK_F( ZSTDMT_expandJobsTable(mtctx, nbWorkers) ); + FORWARD_IF_ERROR( ZSTDMT_expandJobsTable(mtctx, nbWorkers) ); mtctx->bufPool = ZSTDMT_expandBufferPool(mtctx->bufPool, nbWorkers); if (mtctx->bufPool == NULL) return ERROR(memory_allocation); mtctx->cctxPool = ZSTDMT_expandCCtxPool(mtctx->cctxPool, nbWorkers); @@ -1263,7 +1255,7 @@ static size_t ZSTDMT_compress_advanced_internal( if (ZSTDMT_serialState_reset(&mtctx->serial, mtctx->seqPool, params, avgJobSize)) return ERROR(memory_allocation); - CHECK_F( ZSTDMT_expandJobsTable(mtctx, nbJobs) ); /* only expands if necessary */ + FORWARD_IF_ERROR( ZSTDMT_expandJobsTable(mtctx, nbJobs) ); /* only expands if necessary */ { unsigned u; for (u=0; u<nbJobs; u++) { @@ -1396,7 +1388,7 @@ size_t ZSTDMT_initCStream_internal( /* init */ if (params.nbWorkers != mtctx->params.nbWorkers) - CHECK_F( ZSTDMT_resize(mtctx, params.nbWorkers) ); + FORWARD_IF_ERROR( ZSTDMT_resize(mtctx, params.nbWorkers) ); if (params.jobSize != 0 && params.jobSize < ZSTDMT_JOBSIZE_MIN) params.jobSize = ZSTDMT_JOBSIZE_MIN; if (params.jobSize > (size_t)ZSTDMT_JOBSIZE_MAX) params.jobSize = ZSTDMT_JOBSIZE_MAX; @@ -1547,7 +1539,7 @@ size_t ZSTDMT_initCStream(ZSTDMT_CCtx* mtctx, int compressionLevel) { /* ZSTDMT_writeLastEmptyBlock() * Write a single empty block with an end-of-frame to finish a frame. * Job must be created from streaming variant. - * This function is always successfull if expected conditions are fulfilled. + * This function is always successful if expected conditions are fulfilled. */ static void ZSTDMT_writeLastEmptyBlock(ZSTDMT_jobDescription* job) { @@ -1987,7 +1979,7 @@ size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx, assert(input->pos <= input->size); if (mtctx->singleBlockingThread) { /* delegate to single-thread (synchronous) */ - return ZSTD_compressStream_generic(mtctx->cctxPool->cctx[0], output, input, endOp); + return ZSTD_compressStream2(mtctx->cctxPool->cctx[0], output, input, endOp); } if ((mtctx->frameEnded) && (endOp==ZSTD_e_continue)) { @@ -2051,7 +2043,7 @@ size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx, || ((endOp == ZSTD_e_end) && (!mtctx->frameEnded)) ) { /* must finish the frame with a zero-size block */ size_t const jobSize = mtctx->inBuff.filled; assert(mtctx->inBuff.filled <= mtctx->targetSectionSize); - CHECK_F( ZSTDMT_createCompressionJob(mtctx, jobSize, endOp) ); + FORWARD_IF_ERROR( ZSTDMT_createCompressionJob(mtctx, jobSize, endOp) ); } /* check for potential compressed data ready to be flushed */ @@ -2065,7 +2057,7 @@ size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx, size_t ZSTDMT_compressStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, ZSTD_inBuffer* input) { - CHECK_F( ZSTDMT_compressStream_generic(mtctx, output, input, ZSTD_e_continue) ); + FORWARD_IF_ERROR( ZSTDMT_compressStream_generic(mtctx, output, input, ZSTD_e_continue) ); /* recommended next input size : fill current input buffer */ return mtctx->targetSectionSize - mtctx->inBuff.filled; /* note : could be zero when input buffer is fully filled and no more availability to create new job */ @@ -2082,7 +2074,7 @@ static size_t ZSTDMT_flushStream_internal(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* ou || ((endFrame==ZSTD_e_end) && !mtctx->frameEnded)) { /* need a last 0-size block to end frame */ DEBUGLOG(5, "ZSTDMT_flushStream_internal : create a new job (%u bytes, end:%u)", (U32)srcSize, (U32)endFrame); - CHECK_F( ZSTDMT_createCompressionJob(mtctx, srcSize, endFrame) ); + FORWARD_IF_ERROR( ZSTDMT_createCompressionJob(mtctx, srcSize, endFrame) ); } /* check if there is any data available to flush */ diff --git a/thirdparty/zstd/compress/zstdmt_compress.h b/thirdparty/zstd/compress/zstdmt_compress.h index ee771681fa..12e6bcb3a3 100644 --- a/thirdparty/zstd/compress/zstdmt_compress.h +++ b/thirdparty/zstd/compress/zstdmt_compress.h @@ -17,10 +17,25 @@ /* Note : This is an internal API. - * Some methods are still exposed (ZSTDLIB_API), + * These APIs used to be exposed with ZSTDLIB_API, * because it used to be the only way to invoke MT compression. - * Now, it's recommended to use ZSTD_compress_generic() instead. - * These methods will stop being exposed in a future version */ + * Now, it's recommended to use ZSTD_compress2 and ZSTD_compressStream2() + * instead. + * + * If you depend on these APIs and can't switch, then define + * ZSTD_LEGACY_MULTITHREADED_API when making the dynamic library. + * However, we may completely remove these functions in a future + * release, so please switch soon. + * + * This API requires ZSTD_MULTITHREAD to be defined during compilation, + * otherwise ZSTDMT_createCCtx*() will fail. + */ + +#ifdef ZSTD_LEGACY_MULTITHREADED_API +# define ZSTDMT_API ZSTDLIB_API +#else +# define ZSTDMT_API +#endif /* === Dependencies === */ #include <stddef.h> /* size_t */ @@ -40,17 +55,19 @@ /* === Memory management === */ typedef struct ZSTDMT_CCtx_s ZSTDMT_CCtx; -ZSTDLIB_API ZSTDMT_CCtx* ZSTDMT_createCCtx(unsigned nbWorkers); -ZSTDLIB_API ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbWorkers, +/* Requires ZSTD_MULTITHREAD to be defined during compilation, otherwise it will return NULL. */ +ZSTDMT_API ZSTDMT_CCtx* ZSTDMT_createCCtx(unsigned nbWorkers); +/* Requires ZSTD_MULTITHREAD to be defined during compilation, otherwise it will return NULL. */ +ZSTDMT_API ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbWorkers, ZSTD_customMem cMem); -ZSTDLIB_API size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* mtctx); +ZSTDMT_API size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* mtctx); -ZSTDLIB_API size_t ZSTDMT_sizeof_CCtx(ZSTDMT_CCtx* mtctx); +ZSTDMT_API size_t ZSTDMT_sizeof_CCtx(ZSTDMT_CCtx* mtctx); /* === Simple one-pass compression function === */ -ZSTDLIB_API size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx, +ZSTDMT_API size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel); @@ -59,31 +76,31 @@ ZSTDLIB_API size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx, /* === Streaming functions === */ -ZSTDLIB_API size_t ZSTDMT_initCStream(ZSTDMT_CCtx* mtctx, int compressionLevel); -ZSTDLIB_API size_t ZSTDMT_resetCStream(ZSTDMT_CCtx* mtctx, unsigned long long pledgedSrcSize); /**< if srcSize is not known at reset time, use ZSTD_CONTENTSIZE_UNKNOWN. Note: for compatibility with older programs, 0 means the same as ZSTD_CONTENTSIZE_UNKNOWN, but it will change in the future to mean "empty" */ +ZSTDMT_API size_t ZSTDMT_initCStream(ZSTDMT_CCtx* mtctx, int compressionLevel); +ZSTDMT_API size_t ZSTDMT_resetCStream(ZSTDMT_CCtx* mtctx, unsigned long long pledgedSrcSize); /**< if srcSize is not known at reset time, use ZSTD_CONTENTSIZE_UNKNOWN. Note: for compatibility with older programs, 0 means the same as ZSTD_CONTENTSIZE_UNKNOWN, but it will change in the future to mean "empty" */ -ZSTDLIB_API size_t ZSTDMT_nextInputSizeHint(const ZSTDMT_CCtx* mtctx); -ZSTDLIB_API size_t ZSTDMT_compressStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, ZSTD_inBuffer* input); +ZSTDMT_API size_t ZSTDMT_nextInputSizeHint(const ZSTDMT_CCtx* mtctx); +ZSTDMT_API size_t ZSTDMT_compressStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, ZSTD_inBuffer* input); -ZSTDLIB_API size_t ZSTDMT_flushStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output); /**< @return : 0 == all flushed; >0 : still some data to be flushed; or an error code (ZSTD_isError()) */ -ZSTDLIB_API size_t ZSTDMT_endStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output); /**< @return : 0 == all flushed; >0 : still some data to be flushed; or an error code (ZSTD_isError()) */ +ZSTDMT_API size_t ZSTDMT_flushStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output); /**< @return : 0 == all flushed; >0 : still some data to be flushed; or an error code (ZSTD_isError()) */ +ZSTDMT_API size_t ZSTDMT_endStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output); /**< @return : 0 == all flushed; >0 : still some data to be flushed; or an error code (ZSTD_isError()) */ /* === Advanced functions and parameters === */ -ZSTDLIB_API size_t ZSTDMT_compress_advanced(ZSTDMT_CCtx* mtctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const ZSTD_CDict* cdict, - ZSTD_parameters params, - int overlapLog); +ZSTDMT_API size_t ZSTDMT_compress_advanced(ZSTDMT_CCtx* mtctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_CDict* cdict, + ZSTD_parameters params, + int overlapLog); -ZSTDLIB_API size_t ZSTDMT_initCStream_advanced(ZSTDMT_CCtx* mtctx, +ZSTDMT_API size_t ZSTDMT_initCStream_advanced(ZSTDMT_CCtx* mtctx, const void* dict, size_t dictSize, /* dict can be released after init, a local copy is preserved within zcs */ ZSTD_parameters params, unsigned long long pledgedSrcSize); /* pledgedSrcSize is optional and can be zero == unknown */ -ZSTDLIB_API size_t ZSTDMT_initCStream_usingCDict(ZSTDMT_CCtx* mtctx, +ZSTDMT_API size_t ZSTDMT_initCStream_usingCDict(ZSTDMT_CCtx* mtctx, const ZSTD_CDict* cdict, ZSTD_frameParameters fparams, unsigned long long pledgedSrcSize); /* note : zero means empty */ @@ -92,7 +109,7 @@ ZSTDLIB_API size_t ZSTDMT_initCStream_usingCDict(ZSTDMT_CCtx* mtctx, * List of parameters that can be set using ZSTDMT_setMTCtxParameter() */ typedef enum { ZSTDMT_p_jobSize, /* Each job is compressed in parallel. By default, this value is dynamically determined depending on compression parameters. Can be set explicitly here. */ - ZSTDMT_p_overlapLog, /* Each job may reload a part of previous job to enhance compressionr ratio; 0 == no overlap, 6(default) == use 1/8th of window, >=9 == use full window. This is a "sticky" parameter : its value will be re-used on next compression job */ + ZSTDMT_p_overlapLog, /* Each job may reload a part of previous job to enhance compression ratio; 0 == no overlap, 6(default) == use 1/8th of window, >=9 == use full window. This is a "sticky" parameter : its value will be re-used on next compression job */ ZSTDMT_p_rsyncable /* Enables rsyncable mode. */ } ZSTDMT_parameter; @@ -101,12 +118,12 @@ typedef enum { * The function must be called typically after ZSTD_createCCtx() but __before ZSTDMT_init*() !__ * Parameters not explicitly reset by ZSTDMT_init*() remain the same in consecutive compression sessions. * @return : 0, or an error code (which can be tested using ZSTD_isError()) */ -ZSTDLIB_API size_t ZSTDMT_setMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter, int value); +ZSTDMT_API size_t ZSTDMT_setMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter, int value); /* ZSTDMT_getMTCtxParameter() : * Query the ZSTDMT_CCtx for a parameter value. * @return : 0, or an error code (which can be tested using ZSTD_isError()) */ -ZSTDLIB_API size_t ZSTDMT_getMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter, int* value); +ZSTDMT_API size_t ZSTDMT_getMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter, int* value); /*! ZSTDMT_compressStream_generic() : @@ -116,7 +133,7 @@ ZSTDLIB_API size_t ZSTDMT_getMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter * 0 if fully flushed * or an error code * note : needs to be init using any ZSTD_initCStream*() variant */ -ZSTDLIB_API size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx, +ZSTDMT_API size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, ZSTD_inBuffer* input, ZSTD_EndDirective endOp); diff --git a/thirdparty/zstd/decompress/zstd_ddict.c b/thirdparty/zstd/decompress/zstd_ddict.c index 2ad0440684..0af3d23bfe 100644 --- a/thirdparty/zstd/decompress/zstd_ddict.c +++ b/thirdparty/zstd/decompress/zstd_ddict.c @@ -105,9 +105,9 @@ ZSTD_loadEntropy_intoDDict(ZSTD_DDict* ddict, ddict->dictID = MEM_readLE32((const char*)ddict->dictContent + ZSTD_FRAMEIDSIZE); /* load entropy tables */ - CHECK_E( ZSTD_loadDEntropy(&ddict->entropy, - ddict->dictContent, ddict->dictSize), - dictionary_corrupted ); + RETURN_ERROR_IF(ZSTD_isError(ZSTD_loadDEntropy( + &ddict->entropy, ddict->dictContent, ddict->dictSize)), + dictionary_corrupted); ddict->entropyPresent = 1; return 0; } @@ -133,7 +133,7 @@ static size_t ZSTD_initDDict_internal(ZSTD_DDict* ddict, ddict->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */ /* parse dictionary content */ - CHECK_F( ZSTD_loadEntropy_intoDDict(ddict, dictContentType) ); + FORWARD_IF_ERROR( ZSTD_loadEntropy_intoDDict(ddict, dictContentType) ); return 0; } diff --git a/thirdparty/zstd/decompress/zstd_decompress.c b/thirdparty/zstd/decompress/zstd_decompress.c index feef1ef67a..675596f5aa 100644 --- a/thirdparty/zstd/decompress/zstd_decompress.c +++ b/thirdparty/zstd/decompress/zstd_decompress.c @@ -106,6 +106,7 @@ static void ZSTD_initDCtx_internal(ZSTD_DCtx* dctx) dctx->ddictLocal = NULL; dctx->dictEnd = NULL; dctx->ddictIsCold = 0; + dctx->dictUses = ZSTD_dont_use; dctx->inBuff = NULL; dctx->inBuffSize = 0; dctx->outBuffSize = 0; @@ -147,13 +148,20 @@ ZSTD_DCtx* ZSTD_createDCtx(void) return ZSTD_createDCtx_advanced(ZSTD_defaultCMem); } +static void ZSTD_clearDict(ZSTD_DCtx* dctx) +{ + ZSTD_freeDDict(dctx->ddictLocal); + dctx->ddictLocal = NULL; + dctx->ddict = NULL; + dctx->dictUses = ZSTD_dont_use; +} + size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx) { if (dctx==NULL) return 0; /* support free on NULL */ - if (dctx->staticSize) return ERROR(memory_allocation); /* not compatible with static DCtx */ + RETURN_ERROR_IF(dctx->staticSize, memory_allocation, "not compatible with static DCtx"); { ZSTD_customMem const cMem = dctx->customMem; - ZSTD_freeDDict(dctx->ddictLocal); - dctx->ddictLocal = NULL; + ZSTD_clearDict(dctx); ZSTD_free(dctx->inBuff, cMem); dctx->inBuff = NULL; #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) @@ -203,7 +211,7 @@ unsigned ZSTD_isFrame(const void* buffer, size_t size) static size_t ZSTD_frameHeaderSize_internal(const void* src, size_t srcSize, ZSTD_format_e format) { size_t const minInputSize = ZSTD_startingInputLength(format); - if (srcSize < minInputSize) return ERROR(srcSize_wrong); + RETURN_ERROR_IF(srcSize < minInputSize, srcSize_wrong); { BYTE const fhd = ((const BYTE*)src)[minInputSize-1]; U32 const dictID= fhd & 3; @@ -238,7 +246,7 @@ size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, s memset(zfhPtr, 0, sizeof(*zfhPtr)); /* not strictly necessary, but static analyzer do not understand that zfhPtr is only going to be read only if return value is zero, since they are 2 different signals */ if (srcSize < minInputSize) return minInputSize; - if (src==NULL) return ERROR(GENERIC); /* invalid parameter */ + RETURN_ERROR_IF(src==NULL, GENERIC, "invalid parameter"); if ( (format != ZSTD_f_zstd1_magicless) && (MEM_readLE32(src) != ZSTD_MAGICNUMBER) ) { @@ -251,7 +259,7 @@ size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, s zfhPtr->frameType = ZSTD_skippableFrame; return 0; } - return ERROR(prefix_unknown); + RETURN_ERROR(prefix_unknown); } /* ensure there is enough `srcSize` to fully read/decode frame header */ @@ -269,14 +277,13 @@ size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, s U64 windowSize = 0; U32 dictID = 0; U64 frameContentSize = ZSTD_CONTENTSIZE_UNKNOWN; - if ((fhdByte & 0x08) != 0) - return ERROR(frameParameter_unsupported); /* reserved bits, must be zero */ + RETURN_ERROR_IF((fhdByte & 0x08) != 0, frameParameter_unsupported, + "reserved bits, must be zero"); if (!singleSegment) { BYTE const wlByte = ip[pos++]; U32 const windowLog = (wlByte >> 3) + ZSTD_WINDOWLOG_ABSOLUTEMIN; - if (windowLog > ZSTD_WINDOWLOG_MAX) - return ERROR(frameParameter_windowTooLarge); + RETURN_ERROR_IF(windowLog > ZSTD_WINDOWLOG_MAX, frameParameter_windowTooLarge); windowSize = (1ULL << windowLog); windowSize += (windowSize >> 3) * (wlByte&7); } @@ -348,12 +355,11 @@ static size_t readSkippableFrameSize(void const* src, size_t srcSize) size_t const skippableHeaderSize = ZSTD_SKIPPABLEHEADERSIZE; U32 sizeU32; - if (srcSize < ZSTD_SKIPPABLEHEADERSIZE) - return ERROR(srcSize_wrong); + RETURN_ERROR_IF(srcSize < ZSTD_SKIPPABLEHEADERSIZE, srcSize_wrong); sizeU32 = MEM_readLE32((BYTE const*)src + ZSTD_FRAMEIDSIZE); - if ((U32)(sizeU32 + ZSTD_SKIPPABLEHEADERSIZE) < sizeU32) - return ERROR(frameParameter_unsupported); + RETURN_ERROR_IF((U32)(sizeU32 + ZSTD_SKIPPABLEHEADERSIZE) < sizeU32, + frameParameter_unsupported); return skippableHeaderSize + sizeU32; } @@ -428,67 +434,124 @@ static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t he { size_t const result = ZSTD_getFrameHeader_advanced(&(dctx->fParams), src, headerSize, dctx->format); if (ZSTD_isError(result)) return result; /* invalid header */ - if (result>0) return ERROR(srcSize_wrong); /* headerSize too small */ - if (dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID)) - return ERROR(dictionary_wrong); + RETURN_ERROR_IF(result>0, srcSize_wrong, "headerSize too small"); +#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + /* Skip the dictID check in fuzzing mode, because it makes the search + * harder. + */ + RETURN_ERROR_IF(dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID), + dictionary_wrong); +#endif if (dctx->fParams.checksumFlag) XXH64_reset(&dctx->xxhState, 0); return 0; } +static ZSTD_frameSizeInfo ZSTD_errorFrameSizeInfo(size_t ret) +{ + ZSTD_frameSizeInfo frameSizeInfo; + frameSizeInfo.compressedSize = ret; + frameSizeInfo.decompressedBound = ZSTD_CONTENTSIZE_ERROR; + return frameSizeInfo; +} -/** ZSTD_findFrameCompressedSize() : - * compatible with legacy mode - * `src` must point to the start of a ZSTD frame, ZSTD legacy frame, or skippable frame - * `srcSize` must be at least as large as the frame contained - * @return : the compressed size of the frame starting at `src` */ -size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize) +static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize) { + ZSTD_frameSizeInfo frameSizeInfo; + memset(&frameSizeInfo, 0, sizeof(ZSTD_frameSizeInfo)); + #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) if (ZSTD_isLegacy(src, srcSize)) - return ZSTD_findFrameCompressedSizeLegacy(src, srcSize); + return ZSTD_findFrameSizeInfoLegacy(src, srcSize); #endif - if ( (srcSize >= ZSTD_SKIPPABLEHEADERSIZE) - && (MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START ) { - return readSkippableFrameSize(src, srcSize); + + if ((srcSize >= ZSTD_SKIPPABLEHEADERSIZE) + && (MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { + frameSizeInfo.compressedSize = readSkippableFrameSize(src, srcSize); + return frameSizeInfo; } else { const BYTE* ip = (const BYTE*)src; const BYTE* const ipstart = ip; size_t remainingSize = srcSize; + size_t nbBlocks = 0; ZSTD_frameHeader zfh; /* Extract Frame Header */ { size_t const ret = ZSTD_getFrameHeader(&zfh, src, srcSize); - if (ZSTD_isError(ret)) return ret; - if (ret > 0) return ERROR(srcSize_wrong); + if (ZSTD_isError(ret)) + return ZSTD_errorFrameSizeInfo(ret); + if (ret > 0) + return ZSTD_errorFrameSizeInfo(ERROR(srcSize_wrong)); } ip += zfh.headerSize; remainingSize -= zfh.headerSize; - /* Loop on each block */ + /* Iterate over each block */ while (1) { blockProperties_t blockProperties; size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties); - if (ZSTD_isError(cBlockSize)) return cBlockSize; + if (ZSTD_isError(cBlockSize)) + return ZSTD_errorFrameSizeInfo(cBlockSize); if (ZSTD_blockHeaderSize + cBlockSize > remainingSize) - return ERROR(srcSize_wrong); + return ZSTD_errorFrameSizeInfo(ERROR(srcSize_wrong)); ip += ZSTD_blockHeaderSize + cBlockSize; remainingSize -= ZSTD_blockHeaderSize + cBlockSize; + nbBlocks++; if (blockProperties.lastBlock) break; } - if (zfh.checksumFlag) { /* Final frame content checksum */ - if (remainingSize < 4) return ERROR(srcSize_wrong); + /* Final frame content checksum */ + if (zfh.checksumFlag) { + if (remainingSize < 4) + return ZSTD_errorFrameSizeInfo(ERROR(srcSize_wrong)); ip += 4; } - return ip - ipstart; + frameSizeInfo.compressedSize = ip - ipstart; + frameSizeInfo.decompressedBound = (zfh.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN) + ? zfh.frameContentSize + : nbBlocks * zfh.blockSizeMax; + return frameSizeInfo; } } +/** ZSTD_findFrameCompressedSize() : + * compatible with legacy mode + * `src` must point to the start of a ZSTD frame, ZSTD legacy frame, or skippable frame + * `srcSize` must be at least as large as the frame contained + * @return : the compressed size of the frame starting at `src` */ +size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize) +{ + ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize); + return frameSizeInfo.compressedSize; +} + + +/** ZSTD_decompressBound() : + * compatible with legacy mode + * `src` must point to the start of a ZSTD frame or a skippeable frame + * `srcSize` must be at least as large as the frame contained + * @return : the maximum decompressed size of the compressed source + */ +unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize) +{ + unsigned long long bound = 0; + /* Iterate over each frame */ + while (srcSize > 0) { + ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize); + size_t const compressedSize = frameSizeInfo.compressedSize; + unsigned long long const decompressedBound = frameSizeInfo.decompressedBound; + if (ZSTD_isError(compressedSize) || decompressedBound == ZSTD_CONTENTSIZE_ERROR) + return ZSTD_CONTENTSIZE_ERROR; + src = (const BYTE*)src + compressedSize; + srcSize -= compressedSize; + bound += decompressedBound; + } + return bound; +} /*-************************************************************* @@ -522,9 +585,9 @@ static size_t ZSTD_copyRawBlock(void* dst, size_t dstCapacity, DEBUGLOG(5, "ZSTD_copyRawBlock"); if (dst == NULL) { if (srcSize == 0) return 0; - return ERROR(dstBuffer_null); + RETURN_ERROR(dstBuffer_null); } - if (srcSize > dstCapacity) return ERROR(dstSize_tooSmall); + RETURN_ERROR_IF(srcSize > dstCapacity, dstSize_tooSmall); memcpy(dst, src, srcSize); return srcSize; } @@ -535,9 +598,9 @@ static size_t ZSTD_setRleBlock(void* dst, size_t dstCapacity, { if (dst == NULL) { if (regenSize == 0) return 0; - return ERROR(dstBuffer_null); + RETURN_ERROR(dstBuffer_null); } - if (regenSize > dstCapacity) return ERROR(dstSize_tooSmall); + RETURN_ERROR_IF(regenSize > dstCapacity, dstSize_tooSmall); memset(dst, b, regenSize); return regenSize; } @@ -560,15 +623,16 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, DEBUGLOG(4, "ZSTD_decompressFrame (srcSize:%i)", (int)*srcSizePtr); /* check */ - if (remainingSrcSize < ZSTD_FRAMEHEADERSIZE_MIN+ZSTD_blockHeaderSize) - return ERROR(srcSize_wrong); + RETURN_ERROR_IF( + remainingSrcSize < ZSTD_FRAMEHEADERSIZE_MIN+ZSTD_blockHeaderSize, + srcSize_wrong); /* Frame Header */ { size_t const frameHeaderSize = ZSTD_frameHeaderSize(ip, ZSTD_FRAMEHEADERSIZE_PREFIX); if (ZSTD_isError(frameHeaderSize)) return frameHeaderSize; - if (remainingSrcSize < frameHeaderSize+ZSTD_blockHeaderSize) - return ERROR(srcSize_wrong); - CHECK_F( ZSTD_decodeFrameHeader(dctx, ip, frameHeaderSize) ); + RETURN_ERROR_IF(remainingSrcSize < frameHeaderSize+ZSTD_blockHeaderSize, + srcSize_wrong); + FORWARD_IF_ERROR( ZSTD_decodeFrameHeader(dctx, ip, frameHeaderSize) ); ip += frameHeaderSize; remainingSrcSize -= frameHeaderSize; } @@ -581,7 +645,7 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, ip += ZSTD_blockHeaderSize; remainingSrcSize -= ZSTD_blockHeaderSize; - if (cBlockSize > remainingSrcSize) return ERROR(srcSize_wrong); + RETURN_ERROR_IF(cBlockSize > remainingSrcSize, srcSize_wrong); switch(blockProperties.blockType) { @@ -596,7 +660,7 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, break; case bt_reserved : default: - return ERROR(corruption_detected); + RETURN_ERROR(corruption_detected); } if (ZSTD_isError(decodedSize)) return decodedSize; @@ -609,15 +673,15 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, } if (dctx->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN) { - if ((U64)(op-ostart) != dctx->fParams.frameContentSize) { - return ERROR(corruption_detected); - } } + RETURN_ERROR_IF((U64)(op-ostart) != dctx->fParams.frameContentSize, + corruption_detected); + } if (dctx->fParams.checksumFlag) { /* Frame content checksum verification */ U32 const checkCalc = (U32)XXH64_digest(&dctx->xxhState); U32 checkRead; - if (remainingSrcSize<4) return ERROR(checksum_wrong); + RETURN_ERROR_IF(remainingSrcSize<4, checksum_wrong); checkRead = MEM_readLE32(ip); - if (checkRead != checkCalc) return ERROR(checksum_wrong); + RETURN_ERROR_IF(checkRead != checkCalc, checksum_wrong); ip += 4; remainingSrcSize -= 4; } @@ -652,8 +716,8 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx, size_t decodedSize; size_t const frameSize = ZSTD_findFrameCompressedSizeLegacy(src, srcSize); if (ZSTD_isError(frameSize)) return frameSize; - /* legacy support is not compatible with static dctx */ - if (dctx->staticSize) return ERROR(memory_allocation); + RETURN_ERROR_IF(dctx->staticSize, memory_allocation, + "legacy support is not compatible with static dctx"); decodedSize = ZSTD_decompressLegacy(dst, dstCapacity, src, frameSize, dict, dictSize); if (ZSTD_isError(decodedSize)) return decodedSize; @@ -676,7 +740,7 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx, size_t const skippableSize = readSkippableFrameSize(src, srcSize); if (ZSTD_isError(skippableSize)) return skippableSize; - if (srcSize < skippableSize) return ERROR(srcSize_wrong); + RETURN_ERROR_IF(srcSize < skippableSize, srcSize_wrong); src = (const BYTE *)src + skippableSize; srcSize -= skippableSize; @@ -685,29 +749,29 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx, if (ddict) { /* we were called from ZSTD_decompress_usingDDict */ - CHECK_F(ZSTD_decompressBegin_usingDDict(dctx, ddict)); + FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDDict(dctx, ddict)); } else { /* this will initialize correctly with no dict if dict == NULL, so * use this in all cases but ddict */ - CHECK_F(ZSTD_decompressBegin_usingDict(dctx, dict, dictSize)); + FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDict(dctx, dict, dictSize)); } ZSTD_checkContinuity(dctx, dst); { const size_t res = ZSTD_decompressFrame(dctx, dst, dstCapacity, &src, &srcSize); - if ( (ZSTD_getErrorCode(res) == ZSTD_error_prefix_unknown) - && (moreThan1Frame==1) ) { - /* at least one frame successfully completed, - * but following bytes are garbage : - * it's more likely to be a srcSize error, - * specifying more bytes than compressed size of frame(s). - * This error message replaces ERROR(prefix_unknown), - * which would be confusing, as the first header is actually correct. - * Note that one could be unlucky, it might be a corruption error instead, - * happening right at the place where we expect zstd magic bytes. - * But this is _much_ less likely than a srcSize field error. */ - return ERROR(srcSize_wrong); - } + RETURN_ERROR_IF( + (ZSTD_getErrorCode(res) == ZSTD_error_prefix_unknown) + && (moreThan1Frame==1), + srcSize_wrong, + "at least one frame successfully completed, but following " + "bytes are garbage: it's more likely to be a srcSize error, " + "specifying more bytes than compressed size of frame(s). This " + "error message replaces ERROR(prefix_unknown), which would be " + "confusing, as the first header is actually correct. Note that " + "one could be unlucky, it might be a corruption error instead, " + "happening right at the place where we expect zstd magic " + "bytes. But this is _much_ less likely than a srcSize field " + "error."); if (ZSTD_isError(res)) return res; assert(res <= dstCapacity); dst = (BYTE*)dst + res; @@ -716,7 +780,7 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx, moreThan1Frame = 1; } /* while (srcSize >= ZSTD_frameHeaderSize_prefix) */ - if (srcSize) return ERROR(srcSize_wrong); /* input not entirely consumed */ + RETURN_ERROR_IF(srcSize, srcSize_wrong, "input not entirely consumed"); return (BYTE*)dst - (BYTE*)dststart; } @@ -730,9 +794,26 @@ size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, } +static ZSTD_DDict const* ZSTD_getDDict(ZSTD_DCtx* dctx) +{ + switch (dctx->dictUses) { + default: + assert(0 /* Impossible */); + /* fall-through */ + case ZSTD_dont_use: + ZSTD_clearDict(dctx); + return NULL; + case ZSTD_use_indefinitely: + return dctx->ddict; + case ZSTD_use_once: + dctx->dictUses = ZSTD_dont_use; + return dctx->ddict; + } +} + size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) { - return ZSTD_decompress_usingDict(dctx, dst, dstCapacity, src, srcSize, NULL, 0); + return ZSTD_decompress_usingDDict(dctx, dst, dstCapacity, src, srcSize, ZSTD_getDDict(dctx)); } @@ -741,7 +822,7 @@ size_t ZSTD_decompress(void* dst, size_t dstCapacity, const void* src, size_t sr #if defined(ZSTD_HEAPMODE) && (ZSTD_HEAPMODE>=1) size_t regenSize; ZSTD_DCtx* const dctx = ZSTD_createDCtx(); - if (dctx==NULL) return ERROR(memory_allocation); + RETURN_ERROR_IF(dctx==NULL, memory_allocation); regenSize = ZSTD_decompressDCtx(dctx, dst, dstCapacity, src, srcSize); ZSTD_freeDCtx(dctx); return regenSize; @@ -791,8 +872,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c { DEBUGLOG(5, "ZSTD_decompressContinue (srcSize:%u)", (unsigned)srcSize); /* Sanity check */ - if (srcSize != dctx->expected) - return ERROR(srcSize_wrong); /* not allowed */ + RETURN_ERROR_IF(srcSize != dctx->expected, srcSize_wrong, "not allowed"); if (dstCapacity) ZSTD_checkContinuity(dctx, dst); switch (dctx->stage) @@ -817,7 +897,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c case ZSTDds_decodeFrameHeader: assert(src != NULL); memcpy(dctx->headerBuffer + (dctx->headerSize - srcSize), src, srcSize); - CHECK_F(ZSTD_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize)); + FORWARD_IF_ERROR(ZSTD_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize)); dctx->expected = ZSTD_blockHeaderSize; dctx->stage = ZSTDds_decodeBlockHeader; return 0; @@ -867,7 +947,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c break; case bt_reserved : /* should never happen */ default: - return ERROR(corruption_detected); + RETURN_ERROR(corruption_detected); } if (ZSTD_isError(rSize)) return rSize; DEBUGLOG(5, "ZSTD_decompressContinue: decoded size from block : %u", (unsigned)rSize); @@ -876,10 +956,10 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c if (dctx->stage == ZSTDds_decompressLastBlock) { /* end of frame */ DEBUGLOG(4, "ZSTD_decompressContinue: decoded size from frame : %u", (unsigned)dctx->decodedSize); - if (dctx->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN) { - if (dctx->decodedSize != dctx->fParams.frameContentSize) { - return ERROR(corruption_detected); - } } + RETURN_ERROR_IF( + dctx->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN + && dctx->decodedSize != dctx->fParams.frameContentSize, + corruption_detected); if (dctx->fParams.checksumFlag) { /* another round for frame checksum */ dctx->expected = 4; dctx->stage = ZSTDds_checkChecksum; @@ -900,7 +980,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c { U32 const h32 = (U32)XXH64_digest(&dctx->xxhState); U32 const check32 = MEM_readLE32(src); DEBUGLOG(4, "ZSTD_decompressContinue: checksum : calculated %08X :: %08X read", (unsigned)h32, (unsigned)check32); - if (check32 != h32) return ERROR(checksum_wrong); + RETURN_ERROR_IF(check32 != h32, checksum_wrong); dctx->expected = 0; dctx->stage = ZSTDds_getFrameHeaderSize; return 0; @@ -921,7 +1001,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c default: assert(0); /* impossible */ - return ERROR(GENERIC); /* some compiler require default to do something */ + RETURN_ERROR(GENERIC); /* some compiler require default to do something */ } } @@ -945,7 +1025,7 @@ ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy, const BYTE* dictPtr = (const BYTE*)dict; const BYTE* const dictEnd = dictPtr + dictSize; - if (dictSize <= 8) return ERROR(dictionary_corrupted); + RETURN_ERROR_IF(dictSize <= 8, dictionary_corrupted); assert(MEM_readLE32(dict) == ZSTD_MAGIC_DICTIONARY); /* dict must be valid */ dictPtr += 8; /* skip header = magic + dictID */ @@ -964,16 +1044,16 @@ ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy, dictPtr, dictEnd - dictPtr, workspace, workspaceSize); #endif - if (HUF_isError(hSize)) return ERROR(dictionary_corrupted); + RETURN_ERROR_IF(HUF_isError(hSize), dictionary_corrupted); dictPtr += hSize; } { short offcodeNCount[MaxOff+1]; unsigned offcodeMaxValue = MaxOff, offcodeLog; size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr); - if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted); - if (offcodeMaxValue > MaxOff) return ERROR(dictionary_corrupted); - if (offcodeLog > OffFSELog) return ERROR(dictionary_corrupted); + RETURN_ERROR_IF(FSE_isError(offcodeHeaderSize), dictionary_corrupted); + RETURN_ERROR_IF(offcodeMaxValue > MaxOff, dictionary_corrupted); + RETURN_ERROR_IF(offcodeLog > OffFSELog, dictionary_corrupted); ZSTD_buildFSETable( entropy->OFTable, offcodeNCount, offcodeMaxValue, OF_base, OF_bits, @@ -984,9 +1064,9 @@ ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy, { short matchlengthNCount[MaxML+1]; unsigned matchlengthMaxValue = MaxML, matchlengthLog; size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr); - if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted); - if (matchlengthMaxValue > MaxML) return ERROR(dictionary_corrupted); - if (matchlengthLog > MLFSELog) return ERROR(dictionary_corrupted); + RETURN_ERROR_IF(FSE_isError(matchlengthHeaderSize), dictionary_corrupted); + RETURN_ERROR_IF(matchlengthMaxValue > MaxML, dictionary_corrupted); + RETURN_ERROR_IF(matchlengthLog > MLFSELog, dictionary_corrupted); ZSTD_buildFSETable( entropy->MLTable, matchlengthNCount, matchlengthMaxValue, ML_base, ML_bits, @@ -997,9 +1077,9 @@ ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy, { short litlengthNCount[MaxLL+1]; unsigned litlengthMaxValue = MaxLL, litlengthLog; size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr); - if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted); - if (litlengthMaxValue > MaxLL) return ERROR(dictionary_corrupted); - if (litlengthLog > LLFSELog) return ERROR(dictionary_corrupted); + RETURN_ERROR_IF(FSE_isError(litlengthHeaderSize), dictionary_corrupted); + RETURN_ERROR_IF(litlengthMaxValue > MaxLL, dictionary_corrupted); + RETURN_ERROR_IF(litlengthLog > LLFSELog, dictionary_corrupted); ZSTD_buildFSETable( entropy->LLTable, litlengthNCount, litlengthMaxValue, LL_base, LL_bits, @@ -1007,12 +1087,13 @@ ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy, dictPtr += litlengthHeaderSize; } - if (dictPtr+12 > dictEnd) return ERROR(dictionary_corrupted); + RETURN_ERROR_IF(dictPtr+12 > dictEnd, dictionary_corrupted); { int i; size_t const dictContentSize = (size_t)(dictEnd - (dictPtr+12)); for (i=0; i<3; i++) { U32 const rep = MEM_readLE32(dictPtr); dictPtr += 4; - if (rep==0 || rep >= dictContentSize) return ERROR(dictionary_corrupted); + RETURN_ERROR_IF(rep==0 || rep >= dictContentSize, + dictionary_corrupted); entropy->rep[i] = rep; } } @@ -1030,7 +1111,7 @@ static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict /* load entropy tables */ { size_t const eSize = ZSTD_loadDEntropy(&dctx->entropy, dict, dictSize); - if (ZSTD_isError(eSize)) return ERROR(dictionary_corrupted); + RETURN_ERROR_IF(ZSTD_isError(eSize), dictionary_corrupted); dict = (const char*)dict + eSize; dictSize -= eSize; } @@ -1064,9 +1145,11 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx) size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) { - CHECK_F( ZSTD_decompressBegin(dctx) ); + FORWARD_IF_ERROR( ZSTD_decompressBegin(dctx) ); if (dict && dictSize) - CHECK_E(ZSTD_decompress_insertDictionary(dctx, dict, dictSize), dictionary_corrupted); + RETURN_ERROR_IF( + ZSTD_isError(ZSTD_decompress_insertDictionary(dctx, dict, dictSize)), + dictionary_corrupted); return 0; } @@ -1085,7 +1168,7 @@ size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict) DEBUGLOG(4, "DDict is %s", dctx->ddictIsCold ? "~cold~" : "hot!"); } - CHECK_F( ZSTD_decompressBegin(dctx) ); + FORWARD_IF_ERROR( ZSTD_decompressBegin(dctx) ); if (ddict) { /* NULL ddict is equivalent to no dictionary */ ZSTD_copyDDictParameters(dctx, ddict); } @@ -1104,7 +1187,7 @@ unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize) } /*! ZSTD_getDictID_fromFrame() : - * Provides the dictID required to decompresse frame stored within `src`. + * Provides the dictID required to decompress frame stored within `src`. * If @return == 0, the dictID could not be decoded. * This could for one of the following reasons : * - The frame does not require a dictionary (most common case). @@ -1176,15 +1259,14 @@ size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType) { - if (dctx->streamStage != zdss_init) return ERROR(stage_wrong); - ZSTD_freeDDict(dctx->ddictLocal); + RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong); + ZSTD_clearDict(dctx); if (dict && dictSize >= 8) { dctx->ddictLocal = ZSTD_createDDict_advanced(dict, dictSize, dictLoadMethod, dictContentType, dctx->customMem); - if (dctx->ddictLocal == NULL) return ERROR(memory_allocation); - } else { - dctx->ddictLocal = NULL; + RETURN_ERROR_IF(dctx->ddictLocal == NULL, memory_allocation); + dctx->ddict = dctx->ddictLocal; + dctx->dictUses = ZSTD_use_indefinitely; } - dctx->ddict = dctx->ddictLocal; return 0; } @@ -1200,7 +1282,9 @@ size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSi size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType) { - return ZSTD_DCtx_loadDictionary_advanced(dctx, prefix, prefixSize, ZSTD_dlm_byRef, dictContentType); + FORWARD_IF_ERROR(ZSTD_DCtx_loadDictionary_advanced(dctx, prefix, prefixSize, ZSTD_dlm_byRef, dictContentType)); + dctx->dictUses = ZSTD_use_once; + return 0; } size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize) @@ -1215,9 +1299,8 @@ size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSiz size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize) { DEBUGLOG(4, "ZSTD_initDStream_usingDict"); - zds->streamStage = zdss_init; - zds->noForwardProgress = 0; - CHECK_F( ZSTD_DCtx_loadDictionary(zds, dict, dictSize) ); + FORWARD_IF_ERROR( ZSTD_DCtx_reset(zds, ZSTD_reset_session_only) ); + FORWARD_IF_ERROR( ZSTD_DCtx_loadDictionary(zds, dict, dictSize) ); return ZSTD_FRAMEHEADERSIZE_PREFIX; } @@ -1225,7 +1308,7 @@ size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t di size_t ZSTD_initDStream(ZSTD_DStream* zds) { DEBUGLOG(4, "ZSTD_initDStream"); - return ZSTD_initDStream_usingDict(zds, NULL, 0); + return ZSTD_initDStream_usingDDict(zds, NULL); } /* ZSTD_initDStream_usingDDict() : @@ -1233,9 +1316,9 @@ size_t ZSTD_initDStream(ZSTD_DStream* zds) * this function cannot fail */ size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* dctx, const ZSTD_DDict* ddict) { - size_t const initResult = ZSTD_initDStream(dctx); - dctx->ddict = ddict; - return initResult; + FORWARD_IF_ERROR( ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only) ); + FORWARD_IF_ERROR( ZSTD_DCtx_refDDict(dctx, ddict) ); + return ZSTD_FRAMEHEADERSIZE_PREFIX; } /* ZSTD_resetDStream() : @@ -1243,19 +1326,19 @@ size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* dctx, const ZSTD_DDict* ddict) * this function cannot fail */ size_t ZSTD_resetDStream(ZSTD_DStream* dctx) { - DEBUGLOG(4, "ZSTD_resetDStream"); - dctx->streamStage = zdss_loadHeader; - dctx->lhSize = dctx->inPos = dctx->outStart = dctx->outEnd = 0; - dctx->legacyVersion = 0; - dctx->hostageByte = 0; + FORWARD_IF_ERROR(ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only)); return ZSTD_FRAMEHEADERSIZE_PREFIX; } size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict) { - if (dctx->streamStage != zdss_init) return ERROR(stage_wrong); - dctx->ddict = ddict; + RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong); + ZSTD_clearDict(dctx); + if (ddict) { + dctx->ddict = ddict; + dctx->dictUses = ZSTD_use_indefinitely; + } return 0; } @@ -1267,9 +1350,9 @@ size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize) ZSTD_bounds const bounds = ZSTD_dParam_getBounds(ZSTD_d_windowLogMax); size_t const min = (size_t)1 << bounds.lowerBound; size_t const max = (size_t)1 << bounds.upperBound; - if (dctx->streamStage != zdss_init) return ERROR(stage_wrong); - if (maxWindowSize < min) return ERROR(parameter_outOfBound); - if (maxWindowSize > max) return ERROR(parameter_outOfBound); + RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong); + RETURN_ERROR_IF(maxWindowSize < min, parameter_outOfBound); + RETURN_ERROR_IF(maxWindowSize > max, parameter_outOfBound); dctx->maxWindowSize = maxWindowSize; return 0; } @@ -1311,15 +1394,15 @@ static int ZSTD_dParam_withinBounds(ZSTD_dParameter dParam, int value) } #define CHECK_DBOUNDS(p,v) { \ - if (!ZSTD_dParam_withinBounds(p, v)) \ - return ERROR(parameter_outOfBound); \ + RETURN_ERROR_IF(!ZSTD_dParam_withinBounds(p, v), parameter_outOfBound); \ } size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter dParam, int value) { - if (dctx->streamStage != zdss_init) return ERROR(stage_wrong); + RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong); switch(dParam) { case ZSTD_d_windowLogMax: + if (value == 0) value = ZSTD_WINDOWLOG_LIMIT_DEFAULT; CHECK_DBOUNDS(ZSTD_d_windowLogMax, value); dctx->maxWindowSize = ((size_t)1) << value; return 0; @@ -1329,19 +1412,20 @@ size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter dParam, int value return 0; default:; } - return ERROR(parameter_unsupported); + RETURN_ERROR(parameter_unsupported); } size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset) { if ( (reset == ZSTD_reset_session_only) || (reset == ZSTD_reset_session_and_parameters) ) { - (void)ZSTD_initDStream(dctx); + dctx->streamStage = zdss_init; + dctx->noForwardProgress = 0; } if ( (reset == ZSTD_reset_parameters) || (reset == ZSTD_reset_session_and_parameters) ) { - if (dctx->streamStage != zdss_init) - return ERROR(stage_wrong); + RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong); + ZSTD_clearDict(dctx); dctx->format = ZSTD_f_zstd1; dctx->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT; } @@ -1360,7 +1444,8 @@ size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long unsigned long long const neededRBSize = windowSize + blockSize + (WILDCOPY_OVERLENGTH * 2); unsigned long long const neededSize = MIN(frameContentSize, neededRBSize); size_t const minRBSize = (size_t) neededSize; - if ((unsigned long long)minRBSize != neededSize) return ERROR(frameParameter_windowTooLarge); + RETURN_ERROR_IF((unsigned long long)minRBSize != neededSize, + frameParameter_windowTooLarge); return minRBSize; } @@ -1378,9 +1463,9 @@ size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize) ZSTD_frameHeader zfh; size_t const err = ZSTD_getFrameHeader(&zfh, src, srcSize); if (ZSTD_isError(err)) return err; - if (err>0) return ERROR(srcSize_wrong); - if (zfh.windowSize > windowSizeMax) - return ERROR(frameParameter_windowTooLarge); + RETURN_ERROR_IF(err>0, srcSize_wrong); + RETURN_ERROR_IF(zfh.windowSize > windowSizeMax, + frameParameter_windowTooLarge); return ZSTD_estimateDStreamSize((size_t)zfh.windowSize); } @@ -1406,16 +1491,16 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB U32 someMoreWork = 1; DEBUGLOG(5, "ZSTD_decompressStream"); - if (input->pos > input->size) { /* forbidden */ - DEBUGLOG(5, "in: pos: %u vs size: %u", - (U32)input->pos, (U32)input->size); - return ERROR(srcSize_wrong); - } - if (output->pos > output->size) { /* forbidden */ - DEBUGLOG(5, "out: pos: %u vs size: %u", - (U32)output->pos, (U32)output->size); - return ERROR(dstSize_tooSmall); - } + RETURN_ERROR_IF( + input->pos > input->size, + srcSize_wrong, + "forbidden. in: pos: %u vs size: %u", + (U32)input->pos, (U32)input->size); + RETURN_ERROR_IF( + output->pos > output->size, + dstSize_tooSmall, + "forbidden. out: pos: %u vs size: %u", + (U32)output->pos, (U32)output->size); DEBUGLOG(5, "input size : %u", (U32)(input->size - input->pos)); while (someMoreWork) { @@ -1423,15 +1508,18 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB { case zdss_init : DEBUGLOG(5, "stage zdss_init => transparent reset "); - ZSTD_resetDStream(zds); /* transparent reset on starting decoding a new frame */ + zds->streamStage = zdss_loadHeader; + zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0; + zds->legacyVersion = 0; + zds->hostageByte = 0; /* fall-through */ case zdss_loadHeader : DEBUGLOG(5, "stage zdss_loadHeader (srcSize : %u)", (U32)(iend - ip)); #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) if (zds->legacyVersion) { - /* legacy support is incompatible with static dctx */ - if (zds->staticSize) return ERROR(memory_allocation); + RETURN_ERROR_IF(zds->staticSize, memory_allocation, + "legacy support is incompatible with static dctx"); { size_t const hint = ZSTD_decompressLegacyStream(zds->legacyContext, zds->legacyVersion, output, input); if (hint==0) zds->streamStage = zdss_init; return hint; @@ -1443,12 +1531,13 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) U32 const legacyVersion = ZSTD_isLegacy(istart, iend-istart); if (legacyVersion) { - const void* const dict = zds->ddict ? ZSTD_DDict_dictContent(zds->ddict) : NULL; - size_t const dictSize = zds->ddict ? ZSTD_DDict_dictSize(zds->ddict) : 0; + ZSTD_DDict const* const ddict = ZSTD_getDDict(zds); + const void* const dict = ddict ? ZSTD_DDict_dictContent(ddict) : NULL; + size_t const dictSize = ddict ? ZSTD_DDict_dictSize(ddict) : 0; DEBUGLOG(5, "ZSTD_decompressStream: detected legacy version v0.%u", legacyVersion); - /* legacy support is incompatible with static dctx */ - if (zds->staticSize) return ERROR(memory_allocation); - CHECK_F(ZSTD_initLegacyStream(&zds->legacyContext, + RETURN_ERROR_IF(zds->staticSize, memory_allocation, + "legacy support is incompatible with static dctx"); + FORWARD_IF_ERROR(ZSTD_initLegacyStream(&zds->legacyContext, zds->previousLegacyVersion, legacyVersion, dict, dictSize)); zds->legacyVersion = zds->previousLegacyVersion = legacyVersion; @@ -1482,7 +1571,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB size_t const cSize = ZSTD_findFrameCompressedSize(istart, iend-istart); if (cSize <= (size_t)(iend-istart)) { /* shortcut : using single-pass mode */ - size_t const decompressedSize = ZSTD_decompress_usingDDict(zds, op, oend-op, istart, cSize, zds->ddict); + size_t const decompressedSize = ZSTD_decompress_usingDDict(zds, op, oend-op, istart, cSize, ZSTD_getDDict(zds)); if (ZSTD_isError(decompressedSize)) return decompressedSize; DEBUGLOG(4, "shortcut to single-pass ZSTD_decompress_usingDDict()") ip = istart + cSize; @@ -1495,13 +1584,13 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB /* Consume header (see ZSTDds_decodeFrameHeader) */ DEBUGLOG(4, "Consume header"); - CHECK_F(ZSTD_decompressBegin_usingDDict(zds, zds->ddict)); + FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDDict(zds, ZSTD_getDDict(zds))); if ((MEM_readLE32(zds->headerBuffer) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */ zds->expected = MEM_readLE32(zds->headerBuffer + ZSTD_FRAMEIDSIZE); zds->stage = ZSTDds_skipFrame; } else { - CHECK_F(ZSTD_decodeFrameHeader(zds, zds->headerBuffer, zds->lhSize)); + FORWARD_IF_ERROR(ZSTD_decodeFrameHeader(zds, zds->headerBuffer, zds->lhSize)); zds->expected = ZSTD_blockHeaderSize; zds->stage = ZSTDds_decodeBlockHeader; } @@ -1511,7 +1600,8 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB (U32)(zds->fParams.windowSize >>10), (U32)(zds->maxWindowSize >> 10) ); zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN); - if (zds->fParams.windowSize > zds->maxWindowSize) return ERROR(frameParameter_windowTooLarge); + RETURN_ERROR_IF(zds->fParams.windowSize > zds->maxWindowSize, + frameParameter_windowTooLarge); /* Adapt buffer sizes to frame header instructions */ { size_t const neededInBuffSize = MAX(zds->fParams.blockSizeMax, 4 /* frame checksum */); @@ -1525,14 +1615,15 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB if (zds->staticSize) { /* static DCtx */ DEBUGLOG(4, "staticSize : %u", (U32)zds->staticSize); assert(zds->staticSize >= sizeof(ZSTD_DCtx)); /* controlled at init */ - if (bufferSize > zds->staticSize - sizeof(ZSTD_DCtx)) - return ERROR(memory_allocation); + RETURN_ERROR_IF( + bufferSize > zds->staticSize - sizeof(ZSTD_DCtx), + memory_allocation); } else { ZSTD_free(zds->inBuff, zds->customMem); zds->inBuffSize = 0; zds->outBuffSize = 0; zds->inBuff = (char*)ZSTD_malloc(bufferSize, zds->customMem); - if (zds->inBuff == NULL) return ERROR(memory_allocation); + RETURN_ERROR_IF(zds->inBuff == NULL, memory_allocation); } zds->inBuffSize = neededInBuffSize; zds->outBuff = zds->inBuff + zds->inBuffSize; @@ -1574,7 +1665,9 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB if (isSkipFrame) { loadedSize = MIN(toLoad, (size_t)(iend-ip)); } else { - if (toLoad > zds->inBuffSize - zds->inPos) return ERROR(corruption_detected); /* should never happen */ + RETURN_ERROR_IF(toLoad > zds->inBuffSize - zds->inPos, + corruption_detected, + "should never happen"); loadedSize = ZSTD_limitCopy(zds->inBuff + zds->inPos, toLoad, ip, iend-ip); } ip += loadedSize; @@ -1615,7 +1708,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB default: assert(0); /* impossible */ - return ERROR(GENERIC); /* some compiler require default to do something */ + RETURN_ERROR(GENERIC); /* some compiler require default to do something */ } } /* result */ @@ -1624,8 +1717,8 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB if ((ip==istart) && (op==ostart)) { /* no forward progress */ zds->noForwardProgress ++; if (zds->noForwardProgress >= ZSTD_NO_FORWARD_PROGRESS_MAX) { - if (op==oend) return ERROR(dstSize_tooSmall); - if (ip==iend) return ERROR(srcSize_wrong); + RETURN_ERROR_IF(op==oend, dstSize_tooSmall); + RETURN_ERROR_IF(ip==iend, srcSize_wrong); assert(0); } } else { diff --git a/thirdparty/zstd/decompress/zstd_decompress_block.c b/thirdparty/zstd/decompress/zstd_decompress_block.c index 32baad9fbb..a2a7eedcf2 100644 --- a/thirdparty/zstd/decompress/zstd_decompress_block.c +++ b/thirdparty/zstd/decompress/zstd_decompress_block.c @@ -56,14 +56,15 @@ static void ZSTD_copy4(void* dst, const void* src) { memcpy(dst, src, 4); } size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr) { - if (srcSize < ZSTD_blockHeaderSize) return ERROR(srcSize_wrong); + RETURN_ERROR_IF(srcSize < ZSTD_blockHeaderSize, srcSize_wrong); + { U32 const cBlockHeader = MEM_readLE24(src); U32 const cSize = cBlockHeader >> 3; bpPtr->lastBlock = cBlockHeader & 1; bpPtr->blockType = (blockType_e)((cBlockHeader >> 1) & 3); bpPtr->origSize = cSize; /* only useful for RLE */ if (bpPtr->blockType == bt_rle) return 1; - if (bpPtr->blockType == bt_reserved) return ERROR(corruption_detected); + RETURN_ERROR_IF(bpPtr->blockType == bt_reserved, corruption_detected); return cSize; } } @@ -78,7 +79,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, const void* src, size_t srcSize) /* note : srcSize < BLOCKSIZE */ { - if (srcSize < MIN_CBLOCK_SIZE) return ERROR(corruption_detected); + RETURN_ERROR_IF(srcSize < MIN_CBLOCK_SIZE, corruption_detected); { const BYTE* const istart = (const BYTE*) src; symbolEncodingType_e const litEncType = (symbolEncodingType_e)(istart[0] & 3); @@ -86,11 +87,11 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, switch(litEncType) { case set_repeat: - if (dctx->litEntropy==0) return ERROR(dictionary_corrupted); + RETURN_ERROR_IF(dctx->litEntropy==0, dictionary_corrupted); /* fall-through */ case set_compressed: - if (srcSize < 5) return ERROR(corruption_detected); /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need up to 5 for case 3 */ + RETURN_ERROR_IF(srcSize < 5, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 3; here we need up to 5 for case 3"); { size_t lhSize, litSize, litCSize; U32 singleStream=0; U32 const lhlCode = (istart[0] >> 2) & 3; @@ -118,8 +119,8 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, litCSize = (lhc >> 22) + (istart[4] << 10); break; } - if (litSize > ZSTD_BLOCKSIZE_MAX) return ERROR(corruption_detected); - if (litCSize + lhSize > srcSize) return ERROR(corruption_detected); + RETURN_ERROR_IF(litSize > ZSTD_BLOCKSIZE_MAX, corruption_detected); + RETURN_ERROR_IF(litCSize + lhSize > srcSize, corruption_detected); /* prefetch huffman table if cold */ if (dctx->ddictIsCold && (litSize > 768 /* heuristic */)) { @@ -157,7 +158,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, } } - if (HUF_isError(hufSuccess)) return ERROR(corruption_detected); + RETURN_ERROR_IF(HUF_isError(hufSuccess), corruption_detected); dctx->litPtr = dctx->litBuffer; dctx->litSize = litSize; @@ -187,7 +188,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, } if (lhSize+litSize+WILDCOPY_OVERLENGTH > srcSize) { /* risk reading beyond src buffer with wildcopy */ - if (litSize+lhSize > srcSize) return ERROR(corruption_detected); + RETURN_ERROR_IF(litSize+lhSize > srcSize, corruption_detected); memcpy(dctx->litBuffer, istart+lhSize, litSize); dctx->litPtr = dctx->litBuffer; dctx->litSize = litSize; @@ -216,17 +217,17 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, case 3: lhSize = 3; litSize = MEM_readLE24(istart) >> 4; - if (srcSize<4) return ERROR(corruption_detected); /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need lhSize+1 = 4 */ + RETURN_ERROR_IF(srcSize<4, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 3; here we need lhSize+1 = 4"); break; } - if (litSize > ZSTD_BLOCKSIZE_MAX) return ERROR(corruption_detected); + RETURN_ERROR_IF(litSize > ZSTD_BLOCKSIZE_MAX, corruption_detected); memset(dctx->litBuffer, istart[lhSize], litSize + WILDCOPY_OVERLENGTH); dctx->litPtr = dctx->litBuffer; dctx->litSize = litSize; return lhSize+1; } default: - return ERROR(corruption_detected); /* impossible */ + RETURN_ERROR(corruption_detected, "impossible"); } } } @@ -436,8 +437,8 @@ static size_t ZSTD_buildSeqTable(ZSTD_seqSymbol* DTableSpace, const ZSTD_seqSymb switch(type) { case set_rle : - if (!srcSize) return ERROR(srcSize_wrong); - if ( (*(const BYTE*)src) > max) return ERROR(corruption_detected); + RETURN_ERROR_IF(!srcSize, srcSize_wrong); + RETURN_ERROR_IF((*(const BYTE*)src) > max, corruption_detected); { U32 const symbol = *(const BYTE*)src; U32 const baseline = baseValue[symbol]; U32 const nbBits = nbAdditionalBits[symbol]; @@ -449,7 +450,7 @@ static size_t ZSTD_buildSeqTable(ZSTD_seqSymbol* DTableSpace, const ZSTD_seqSymb *DTablePtr = defaultTable; return 0; case set_repeat: - if (!flagRepeatTable) return ERROR(corruption_detected); + RETURN_ERROR_IF(!flagRepeatTable, corruption_detected); /* prefetch FSE table if used */ if (ddictIsCold && (nbSeq > 24 /* heuristic */)) { const void* const pStart = *DTablePtr; @@ -461,15 +462,15 @@ static size_t ZSTD_buildSeqTable(ZSTD_seqSymbol* DTableSpace, const ZSTD_seqSymb { unsigned tableLog; S16 norm[MaxSeq+1]; size_t const headerSize = FSE_readNCount(norm, &max, &tableLog, src, srcSize); - if (FSE_isError(headerSize)) return ERROR(corruption_detected); - if (tableLog > maxLog) return ERROR(corruption_detected); + RETURN_ERROR_IF(FSE_isError(headerSize), corruption_detected); + RETURN_ERROR_IF(tableLog > maxLog, corruption_detected); ZSTD_buildFSETable(DTableSpace, norm, max, baseValue, nbAdditionalBits, tableLog); *DTablePtr = DTableSpace; return headerSize; } - default : /* impossible */ + default : assert(0); - return ERROR(GENERIC); + RETURN_ERROR(GENERIC, "impossible"); } } @@ -483,28 +484,28 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, DEBUGLOG(5, "ZSTD_decodeSeqHeaders"); /* check */ - if (srcSize < MIN_SEQUENCES_SIZE) return ERROR(srcSize_wrong); + RETURN_ERROR_IF(srcSize < MIN_SEQUENCES_SIZE, srcSize_wrong); /* SeqHead */ nbSeq = *ip++; if (!nbSeq) { *nbSeqPtr=0; - if (srcSize != 1) return ERROR(srcSize_wrong); + RETURN_ERROR_IF(srcSize != 1, srcSize_wrong); return 1; } if (nbSeq > 0x7F) { if (nbSeq == 0xFF) { - if (ip+2 > iend) return ERROR(srcSize_wrong); + RETURN_ERROR_IF(ip+2 > iend, srcSize_wrong); nbSeq = MEM_readLE16(ip) + LONGNBSEQ, ip+=2; } else { - if (ip >= iend) return ERROR(srcSize_wrong); + RETURN_ERROR_IF(ip >= iend, srcSize_wrong); nbSeq = ((nbSeq-0x80)<<8) + *ip++; } } *nbSeqPtr = nbSeq; /* FSE table descriptors */ - if (ip+4 > iend) return ERROR(srcSize_wrong); /* minimum possible size */ + RETURN_ERROR_IF(ip+4 > iend, srcSize_wrong); /* minimum possible size */ { symbolEncodingType_e const LLtype = (symbolEncodingType_e)(*ip >> 6); symbolEncodingType_e const OFtype = (symbolEncodingType_e)((*ip >> 4) & 3); symbolEncodingType_e const MLtype = (symbolEncodingType_e)((*ip >> 2) & 3); @@ -517,7 +518,7 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, LL_base, LL_bits, LL_defaultDTable, dctx->fseEntropy, dctx->ddictIsCold, nbSeq); - if (ZSTD_isError(llhSize)) return ERROR(corruption_detected); + RETURN_ERROR_IF(ZSTD_isError(llhSize), corruption_detected); ip += llhSize; } @@ -527,7 +528,7 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, OF_base, OF_bits, OF_defaultDTable, dctx->fseEntropy, dctx->ddictIsCold, nbSeq); - if (ZSTD_isError(ofhSize)) return ERROR(corruption_detected); + RETURN_ERROR_IF(ZSTD_isError(ofhSize), corruption_detected); ip += ofhSize; } @@ -537,7 +538,7 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, ML_base, ML_bits, ML_defaultDTable, dctx->fseEntropy, dctx->ddictIsCold, nbSeq); - if (ZSTD_isError(mlhSize)) return ERROR(corruption_detected); + RETURN_ERROR_IF(ZSTD_isError(mlhSize), corruption_detected); ip += mlhSize; } } @@ -590,8 +591,8 @@ size_t ZSTD_execSequenceLast7(BYTE* op, const BYTE* match = oLitEnd - sequence.offset; /* check */ - if (oMatchEnd>oend) return ERROR(dstSize_tooSmall); /* last match must fit within dstBuffer */ - if (iLitEnd > litLimit) return ERROR(corruption_detected); /* try to read beyond literal buffer */ + RETURN_ERROR_IF(oMatchEnd>oend, dstSize_tooSmall, "last match must fit within dstBuffer"); + RETURN_ERROR_IF(iLitEnd > litLimit, corruption_detected, "try to read beyond literal buffer"); /* copy literals */ while (op < oLitEnd) *op++ = *(*litPtr)++; @@ -599,7 +600,7 @@ size_t ZSTD_execSequenceLast7(BYTE* op, /* copy Match */ if (sequence.offset > (size_t)(oLitEnd - base)) { /* offset beyond prefix */ - if (sequence.offset > (size_t)(oLitEnd - vBase)) return ERROR(corruption_detected); + RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - vBase),corruption_detected); match = dictEnd - (base-match); if (match + sequence.matchLength <= dictEnd) { memmove(oLitEnd, match, sequence.matchLength); @@ -631,8 +632,8 @@ size_t ZSTD_execSequence(BYTE* op, const BYTE* match = oLitEnd - sequence.offset; /* check */ - if (oMatchEnd>oend) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */ - if (iLitEnd > litLimit) return ERROR(corruption_detected); /* over-read beyond lit buffer */ + RETURN_ERROR_IF(oMatchEnd>oend, dstSize_tooSmall, "last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend"); + RETURN_ERROR_IF(iLitEnd > litLimit, corruption_detected, "over-read beyond lit buffer"); if (oLitEnd>oend_w) return ZSTD_execSequenceLast7(op, oend, sequence, litPtr, litLimit, prefixStart, virtualStart, dictEnd); /* copy Literals */ @@ -645,8 +646,7 @@ size_t ZSTD_execSequence(BYTE* op, /* copy Match */ if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { /* offset beyond prefix -> go into extDict */ - if (sequence.offset > (size_t)(oLitEnd - virtualStart)) - return ERROR(corruption_detected); + RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - virtualStart), corruption_detected); match = dictEnd + (match - prefixStart); if (match + sequence.matchLength <= dictEnd) { memmove(oLitEnd, match, sequence.matchLength); @@ -712,8 +712,8 @@ size_t ZSTD_execSequenceLong(BYTE* op, const BYTE* match = sequence.match; /* check */ - if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */ - if (iLitEnd > litLimit) return ERROR(corruption_detected); /* over-read beyond lit buffer */ + RETURN_ERROR_IF(oMatchEnd > oend, dstSize_tooSmall, "last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend"); + RETURN_ERROR_IF(iLitEnd > litLimit, corruption_detected, "over-read beyond lit buffer"); if (oLitEnd > oend_w) return ZSTD_execSequenceLast7(op, oend, sequence, litPtr, litLimit, prefixStart, dictStart, dictEnd); /* copy Literals */ @@ -726,7 +726,7 @@ size_t ZSTD_execSequenceLong(BYTE* op, /* copy Match */ if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { /* offset beyond prefix */ - if (sequence.offset > (size_t)(oLitEnd - dictStart)) return ERROR(corruption_detected); + RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - dictStart), corruption_detected); if (match + sequence.matchLength <= dictEnd) { memmove(oLitEnd, match, sequence.matchLength); return sequenceLength; @@ -801,7 +801,7 @@ ZSTD_updateFseState(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD) /* We need to add at most (ZSTD_WINDOWLOG_MAX_32 - 1) bits to read the maximum * offset bits. But we can only read at most (STREAM_ACCUMULATOR_MIN_32 - 1) * bits before reloading. This value is the maximum number of bytes we read - * after reloading when we are decoding long offets. + * after reloading when we are decoding long offsets. */ #define LONG_OFFSETS_MAX_EXTRA_BITS_32 \ (ZSTD_WINDOWLOG_MAX_32 > STREAM_ACCUMULATOR_MIN_32 \ @@ -911,7 +911,9 @@ ZSTD_decompressSequences_body( ZSTD_DCtx* dctx, seqState_t seqState; dctx->fseEntropy = 1; { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->entropy.rep[i]; } - CHECK_E(BIT_initDStream(&seqState.DStream, ip, iend-ip), corruption_detected); + RETURN_ERROR_IF( + ERR_isError(BIT_initDStream(&seqState.DStream, ip, iend-ip)), + corruption_detected); ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr); ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr); ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); @@ -927,14 +929,14 @@ ZSTD_decompressSequences_body( ZSTD_DCtx* dctx, /* check if reached exact end */ DEBUGLOG(5, "ZSTD_decompressSequences_body: after decode loop, remaining nbSeq : %i", nbSeq); - if (nbSeq) return ERROR(corruption_detected); + RETURN_ERROR_IF(nbSeq, corruption_detected); /* save reps for next block */ { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) dctx->entropy.rep[i] = (U32)(seqState.prevOffset[i]); } } /* last literal segment */ { size_t const lastLLSize = litEnd - litPtr; - if (lastLLSize > (size_t)(oend-op)) return ERROR(dstSize_tooSmall); + RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall); memcpy(op, litPtr, lastLLSize); op += lastLLSize; } @@ -1066,7 +1068,9 @@ ZSTD_decompressSequencesLong_body( seqState.pos = (size_t)(op-prefixStart); seqState.dictEnd = dictEnd; assert(iend >= ip); - CHECK_E(BIT_initDStream(&seqState.DStream, ip, iend-ip), corruption_detected); + RETURN_ERROR_IF( + ERR_isError(BIT_initDStream(&seqState.DStream, ip, iend-ip)), + corruption_detected); ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr); ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr); ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); @@ -1076,7 +1080,7 @@ ZSTD_decompressSequencesLong_body( sequences[seqNb] = ZSTD_decodeSequenceLong(&seqState, isLongOffset); PREFETCH_L1(sequences[seqNb].match); PREFETCH_L1(sequences[seqNb].match + sequences[seqNb].matchLength - 1); /* note : it's safe to invoke PREFETCH() on any memory address, including invalid ones */ } - if (seqNb<seqAdvance) return ERROR(corruption_detected); + RETURN_ERROR_IF(seqNb<seqAdvance, corruption_detected); /* decode and decompress */ for ( ; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && (seqNb<nbSeq) ; seqNb++) { @@ -1087,7 +1091,7 @@ ZSTD_decompressSequencesLong_body( sequences[seqNb & STORED_SEQS_MASK] = sequence; op += oneSeqSize; } - if (seqNb<nbSeq) return ERROR(corruption_detected); + RETURN_ERROR_IF(seqNb<nbSeq, corruption_detected); /* finish queue */ seqNb -= seqAdvance; @@ -1103,7 +1107,7 @@ ZSTD_decompressSequencesLong_body( /* last literal segment */ { size_t const lastLLSize = litEnd - litPtr; - if (lastLLSize > (size_t)(oend-op)) return ERROR(dstSize_tooSmall); + RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall); memcpy(op, litPtr, lastLLSize); op += lastLLSize; } @@ -1176,7 +1180,7 @@ ZSTD_decompressSequences(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, /* ZSTD_decompressSequencesLong() : * decompression function triggered when a minimum share of offsets is considered "long", * aka out of cache. - * note : "long" definition seems overloaded here, sometimes meaning "wider than bitstream register", and sometimes mearning "farther than memory cache distance". + * note : "long" definition seems overloaded here, sometimes meaning "wider than bitstream register", and sometimes meaning "farther than memory cache distance". * This function will try to mitigate main memory latency through the use of prefetching */ static size_t ZSTD_decompressSequencesLong(ZSTD_DCtx* dctx, @@ -1240,7 +1244,7 @@ ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, ZSTD_longOffset_e const isLongOffset = (ZSTD_longOffset_e)(MEM_32bits() && (!frame || (dctx->fParams.windowSize > (1ULL << STREAM_ACCUMULATOR_MIN)))); DEBUGLOG(5, "ZSTD_decompressBlock_internal (size : %u)", (U32)srcSize); - if (srcSize >= ZSTD_BLOCKSIZE_MAX) return ERROR(srcSize_wrong); + RETURN_ERROR_IF(srcSize >= ZSTD_BLOCKSIZE_MAX, srcSize_wrong); /* Decode literals section */ { size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize); diff --git a/thirdparty/zstd/decompress/zstd_decompress_internal.h b/thirdparty/zstd/decompress/zstd_decompress_internal.h index abd0030519..ccbdfa090f 100644 --- a/thirdparty/zstd/decompress/zstd_decompress_internal.h +++ b/thirdparty/zstd/decompress/zstd_decompress_internal.h @@ -89,6 +89,12 @@ typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader, typedef enum { zdss_init=0, zdss_loadHeader, zdss_read, zdss_load, zdss_flush } ZSTD_dStreamStage; +typedef enum { + ZSTD_use_indefinitely = -1, /* Use the dictionary indefinitely */ + ZSTD_dont_use = 0, /* Do not use the dictionary (if one exists free it) */ + ZSTD_use_once = 1 /* Use the dictionary once and set to ZSTD_dont_use */ +} ZSTD_dictUses_e; + struct ZSTD_DCtx_s { const ZSTD_seqSymbol* LLTptr; @@ -123,6 +129,7 @@ struct ZSTD_DCtx_s const ZSTD_DDict* ddict; /* set by ZSTD_initDStream_usingDDict(), or ZSTD_DCtx_refDDict() */ U32 dictID; int ddictIsCold; /* if == 1 : dictionary is "new" for working context, and presumed "cold" (not in cpu cache) */ + ZSTD_dictUses_e dictUses; /* streaming */ ZSTD_dStreamStage streamStage; diff --git a/thirdparty/zstd/zstd.h b/thirdparty/zstd/zstd.h index b18fc8a44b..53470c18f3 100644 --- a/thirdparty/zstd/zstd.h +++ b/thirdparty/zstd/zstd.h @@ -70,8 +70,8 @@ extern "C" { /*------ Version ------*/ #define ZSTD_VERSION_MAJOR 1 -#define ZSTD_VERSION_MINOR 3 -#define ZSTD_VERSION_RELEASE 8 +#define ZSTD_VERSION_MINOR 4 +#define ZSTD_VERSION_RELEASE 0 #define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE) ZSTDLIB_API unsigned ZSTD_versionNumber(void); /**< to check runtime library version */ @@ -90,6 +90,21 @@ ZSTDLIB_API const char* ZSTD_versionString(void); /* requires v1.3.0+ */ #endif /*************************************** +* Constants +***************************************/ + +/* All magic numbers are supposed read/written to/from files/memory using little-endian convention */ +#define ZSTD_MAGICNUMBER 0xFD2FB528 /* valid since v0.8.0 */ +#define ZSTD_MAGIC_DICTIONARY 0xEC30A437 /* valid since v0.7.0 */ +#define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50 /* all 16 values, from 0x184D2A50 to 0x184D2A5F, signal the beginning of a skippable frame */ +#define ZSTD_MAGIC_SKIPPABLE_MASK 0xFFFFFFF0 + +#define ZSTD_BLOCKSIZELOG_MAX 17 +#define ZSTD_BLOCKSIZE_MAX (1<<ZSTD_BLOCKSIZELOG_MAX) + + + +/*************************************** * Simple API ***************************************/ /*! ZSTD_compress() : @@ -145,12 +160,21 @@ ZSTDLIB_API unsigned long long ZSTD_getFrameContentSize(const void *src, size_t * @return : decompressed size of `src` frame content _if known and not empty_, 0 otherwise. */ ZSTDLIB_API unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize); +/*! ZSTD_findFrameCompressedSize() : + * `src` should point to the start of a ZSTD frame or skippable frame. + * `srcSize` must be >= first frame size + * @return : the compressed size of the first frame starting at `src`, + * suitable to pass as `srcSize` to `ZSTD_decompress` or similar, + * or an error code if input is invalid */ +ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize); + /*====== Helper functions ======*/ #define ZSTD_COMPRESSBOUND(srcSize) ((srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0)) /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */ ZSTDLIB_API size_t ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case single-pass scenario */ ZSTDLIB_API unsigned ZSTD_isError(size_t code); /*!< tells if a `size_t` function result is an error code */ ZSTDLIB_API const char* ZSTD_getErrorName(size_t code); /*!< provides readable string from an error code */ +ZSTDLIB_API int ZSTD_minCLevel(void); /*!< minimum negative compression level allowed */ ZSTDLIB_API int ZSTD_maxCLevel(void); /*!< maximum compression level available */ @@ -195,279 +219,6 @@ ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx, const void* src, size_t srcSize); -/************************** -* Simple dictionary API -***************************/ -/*! ZSTD_compress_usingDict() : - * Compression at an explicit compression level using a Dictionary. - * A dictionary can be any arbitrary data segment (also called a prefix), - * or a buffer with specified information (see dictBuilder/zdict.h). - * Note : This function loads the dictionary, resulting in significant startup delay. - * It's intended for a dictionary used only once. - * Note 2 : When `dict == NULL || dictSize < 8` no dictionary is used. */ -ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const void* dict,size_t dictSize, - int compressionLevel); - -/*! ZSTD_decompress_usingDict() : - * Decompression using a known Dictionary. - * Dictionary must be identical to the one used during compression. - * Note : This function loads the dictionary, resulting in significant startup delay. - * It's intended for a dictionary used only once. - * Note : When `dict == NULL || dictSize < 8` no dictionary is used. */ -ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const void* dict,size_t dictSize); - - -/*********************************** - * Bulk processing dictionary API - **********************************/ -typedef struct ZSTD_CDict_s ZSTD_CDict; - -/*! ZSTD_createCDict() : - * When compressing multiple messages / blocks using the same dictionary, it's recommended to load it only once. - * ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup cost. - * ZSTD_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only. - * `dictBuffer` can be released after ZSTD_CDict creation, because its content is copied within CDict. - * Consider experimental function `ZSTD_createCDict_byReference()` if you prefer to not duplicate `dictBuffer` content. - * Note : A ZSTD_CDict can be created from an empty dictBuffer, but it is inefficient when used to compress small data. */ -ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize, - int compressionLevel); - -/*! ZSTD_freeCDict() : - * Function frees memory allocated by ZSTD_createCDict(). */ -ZSTDLIB_API size_t ZSTD_freeCDict(ZSTD_CDict* CDict); - -/*! ZSTD_compress_usingCDict() : - * Compression using a digested Dictionary. - * Recommended when same dictionary is used multiple times. - * Note : compression level is _decided at dictionary creation time_, - * and frame parameters are hardcoded (dictID=yes, contentSize=yes, checksum=no) */ -ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const ZSTD_CDict* cdict); - - -typedef struct ZSTD_DDict_s ZSTD_DDict; - -/*! ZSTD_createDDict() : - * Create a digested dictionary, ready to start decompression operation without startup delay. - * dictBuffer can be released after DDict creation, as its content is copied inside DDict. */ -ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dictBuffer, size_t dictSize); - -/*! ZSTD_freeDDict() : - * Function frees memory allocated with ZSTD_createDDict() */ -ZSTDLIB_API size_t ZSTD_freeDDict(ZSTD_DDict* ddict); - -/*! ZSTD_decompress_usingDDict() : - * Decompression using a digested Dictionary. - * Recommended when same dictionary is used multiple times. */ -ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const ZSTD_DDict* ddict); - - -/**************************** -* Streaming -****************************/ - -typedef struct ZSTD_inBuffer_s { - const void* src; /**< start of input buffer */ - size_t size; /**< size of input buffer */ - size_t pos; /**< position where reading stopped. Will be updated. Necessarily 0 <= pos <= size */ -} ZSTD_inBuffer; - -typedef struct ZSTD_outBuffer_s { - void* dst; /**< start of output buffer */ - size_t size; /**< size of output buffer */ - size_t pos; /**< position where writing stopped. Will be updated. Necessarily 0 <= pos <= size */ -} ZSTD_outBuffer; - - - -/*-*********************************************************************** -* Streaming compression - HowTo -* -* A ZSTD_CStream object is required to track streaming operation. -* Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources. -* ZSTD_CStream objects can be reused multiple times on consecutive compression operations. -* It is recommended to re-use ZSTD_CStream since it will play nicer with system's memory, by re-using already allocated memory. -* -* For parallel execution, use one separate ZSTD_CStream per thread. -* -* note : since v1.3.0, ZSTD_CStream and ZSTD_CCtx are the same thing. -* -* Parameters are sticky : when starting a new compression on the same context, -* it will re-use the same sticky parameters as previous compression session. -* When in doubt, it's recommended to fully initialize the context before usage. -* Use ZSTD_initCStream() to set the parameter to a selected compression level. -* Use advanced API (ZSTD_CCtx_setParameter(), etc.) to set more specific parameters. -* -* Use ZSTD_compressStream() as many times as necessary to consume input stream. -* The function will automatically update both `pos` fields within `input` and `output`. -* Note that the function may not consume the entire input, -* for example, because the output buffer is already full, -* in which case `input.pos < input.size`. -* The caller must check if input has been entirely consumed. -* If not, the caller must make some room to receive more compressed data, -* and then present again remaining input data. -* @return : a size hint, preferred nb of bytes to use as input for next function call -* or an error code, which can be tested using ZSTD_isError(). -* Note 1 : it's just a hint, to help latency a little, any value will work fine. -* Note 2 : size hint is guaranteed to be <= ZSTD_CStreamInSize() -* -* At any moment, it's possible to flush whatever data might remain stuck within internal buffer, -* using ZSTD_flushStream(). `output->pos` will be updated. -* Note that, if `output->size` is too small, a single invocation of ZSTD_flushStream() might not be enough (return code > 0). -* In which case, make some room to receive more compressed data, and call again ZSTD_flushStream(). -* @return : 0 if internal buffers are entirely flushed, -* >0 if some data still present within internal buffer (the value is minimal estimation of remaining size), -* or an error code, which can be tested using ZSTD_isError(). -* -* ZSTD_endStream() instructs to finish a frame. -* It will perform a flush and write frame epilogue. -* The epilogue is required for decoders to consider a frame completed. -* flush() operation is the same, and follows same rules as ZSTD_flushStream(). -* @return : 0 if frame fully completed and fully flushed, -* >0 if some data still present within internal buffer (the value is minimal estimation of remaining size), -* or an error code, which can be tested using ZSTD_isError(). -* -* *******************************************************************/ - -typedef ZSTD_CCtx ZSTD_CStream; /**< CCtx and CStream are now effectively same object (>= v1.3.0) */ - /* Continue to distinguish them for compatibility with older versions <= v1.2.0 */ -/*===== ZSTD_CStream management functions =====*/ -ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void); -ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs); - -/*===== Streaming compression functions =====*/ -ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel); -ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input); -ZSTDLIB_API size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); -ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); - -ZSTDLIB_API size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */ -ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block in all circumstances. */ - - - -/*-*************************************************************************** -* Streaming decompression - HowTo -* -* A ZSTD_DStream object is required to track streaming operations. -* Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources. -* ZSTD_DStream objects can be re-used multiple times. -* -* Use ZSTD_initDStream() to start a new decompression operation. -* @return : recommended first input size -* Alternatively, use advanced API to set specific properties. -* -* Use ZSTD_decompressStream() repetitively to consume your input. -* The function will update both `pos` fields. -* If `input.pos < input.size`, some input has not been consumed. -* It's up to the caller to present again remaining data. -* The function tries to flush all data decoded immediately, respecting output buffer size. -* If `output.pos < output.size`, decoder has flushed everything it could. -* But if `output.pos == output.size`, there might be some data left within internal buffers., -* In which case, call ZSTD_decompressStream() again to flush whatever remains in the buffer. -* Note : with no additional input provided, amount of data flushed is necessarily <= ZSTD_BLOCKSIZE_MAX. -* @return : 0 when a frame is completely decoded and fully flushed, -* or an error code, which can be tested using ZSTD_isError(), -* or any other value > 0, which means there is still some decoding or flushing to do to complete current frame : -* the return value is a suggested next input size (just a hint for better latency) -* that will never request more than the remaining frame size. -* *******************************************************************************/ - -typedef ZSTD_DCtx ZSTD_DStream; /**< DCtx and DStream are now effectively same object (>= v1.3.0) */ - /* For compatibility with versions <= v1.2.0, prefer differentiating them. */ -/*===== ZSTD_DStream management functions =====*/ -ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void); -ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds); - -/*===== Streaming decompression functions =====*/ -ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds); -ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input); - -ZSTDLIB_API size_t ZSTD_DStreamInSize(void); /*!< recommended size for input buffer */ -ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output buffer. Guarantee to successfully flush at least one complete block in all circumstances. */ - -#endif /* ZSTD_H_235446 */ - - - - -/**************************************************************************************** - * ADVANCED AND EXPERIMENTAL FUNCTIONS - **************************************************************************************** - * The definitions in the following section are considered experimental. - * They are provided for advanced scenarios. - * They should never be used with a dynamic library, as prototypes may change in the future. - * Use them only in association with static linking. - * ***************************************************************************************/ - -#if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY) -#define ZSTD_H_ZSTD_STATIC_LINKING_ONLY - - -/**************************************************************************************** - * Candidate API for promotion to stable status - **************************************************************************************** - * The following symbols and constants form the "staging area" : - * they are considered to join "stable API" by v1.4.0. - * The proposal is written so that it can be made stable "as is", - * though it's still possible to suggest improvements. - * Staging is in fact last chance for changes, - * the API is locked once reaching "stable" status. - * ***************************************************************************************/ - - -/* === Constants === */ - -/* all magic numbers are supposed read/written to/from files/memory using little-endian convention */ -#define ZSTD_MAGICNUMBER 0xFD2FB528 /* valid since v0.8.0 */ -#define ZSTD_MAGIC_DICTIONARY 0xEC30A437 /* valid since v0.7.0 */ -#define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50 /* all 16 values, from 0x184D2A50 to 0x184D2A5F, signal the beginning of a skippable frame */ -#define ZSTD_MAGIC_SKIPPABLE_MASK 0xFFFFFFF0 - -#define ZSTD_BLOCKSIZELOG_MAX 17 -#define ZSTD_BLOCKSIZE_MAX (1<<ZSTD_BLOCKSIZELOG_MAX) - - -/* === query limits === */ - -ZSTDLIB_API int ZSTD_minCLevel(void); /*!< minimum negative compression level allowed */ - - -/* === frame size === */ - -/*! ZSTD_findFrameCompressedSize() : - * `src` should point to the start of a ZSTD frame or skippable frame. - * `srcSize` must be >= first frame size - * @return : the compressed size of the first frame starting at `src`, - * suitable to pass as `srcSize` to `ZSTD_decompress` or similar, - * or an error code if input is invalid */ -ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize); - - -/* === Memory management === */ - -/*! ZSTD_sizeof_*() : - * These functions give the _current_ memory usage of selected object. - * Note that object memory usage can evolve (increase or decrease) over time. */ -ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx); -ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx); -ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs); -ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds); -ZSTDLIB_API size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict); -ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); - - /*************************************** * Advanced compression API ***************************************/ @@ -503,7 +254,10 @@ typedef enum { ZSTD_fast=1, typedef enum { - /* compression parameters */ + /* compression parameters + * Note: When compressing with a ZSTD_CDict these parameters are superseded + * by the parameters used to construct the ZSTD_CDict. See ZSTD_CCtx_refCDict() + * for more info (superseded-by-cdict). */ ZSTD_c_compressionLevel=100, /* Update all compression parameters according to pre-defined cLevel table * Default level is ZSTD_CLEVEL_DEFAULT==3. * Special: value 0 means default, which is controlled by ZSTD_CLEVEL_DEFAULT. @@ -625,6 +379,7 @@ typedef enum { * ZSTD_c_format * ZSTD_c_forceMaxWindow * ZSTD_c_forceAttachDict + * ZSTD_c_literalCompressionMode * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. * note : never ever use experimentalParam? names directly; * also, the enums values themselves are unstable and can still change. @@ -632,10 +387,10 @@ typedef enum { ZSTD_c_experimentalParam1=500, ZSTD_c_experimentalParam2=10, ZSTD_c_experimentalParam3=1000, - ZSTD_c_experimentalParam4=1001 + ZSTD_c_experimentalParam4=1001, + ZSTD_c_experimentalParam5=1002, } ZSTD_cParameter; - typedef struct { size_t error; int lowerBound; @@ -677,63 +432,10 @@ ZSTDLIB_API size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param * Note 3 : Whenever all input data is provided and consumed in a single round, * for example with ZSTD_compress2(), * or invoking immediately ZSTD_compressStream2(,,,ZSTD_e_end), - * this value is automatically overriden by srcSize instead. + * this value is automatically overridden by srcSize instead. */ ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize); -/*! ZSTD_CCtx_loadDictionary() : - * Create an internal CDict from `dict` buffer. - * Decompression will have to use same dictionary. - * @result : 0, or an error code (which can be tested with ZSTD_isError()). - * Special: Loading a NULL (or 0-size) dictionary invalidates previous dictionary, - * meaning "return to no-dictionary mode". - * Note 1 : Dictionary is sticky, it will be used for all future compressed frames. - * To return to "no-dictionary" situation, load a NULL dictionary (or reset parameters). - * Note 2 : Loading a dictionary involves building tables. - * It's also a CPU consuming operation, with non-negligible impact on latency. - * Tables are dependent on compression parameters, and for this reason, - * compression parameters can no longer be changed after loading a dictionary. - * Note 3 :`dict` content will be copied internally. - * Use experimental ZSTD_CCtx_loadDictionary_byReference() to reference content instead. - * In such a case, dictionary buffer must outlive its users. - * Note 4 : Use ZSTD_CCtx_loadDictionary_advanced() - * to precisely select how dictionary content must be interpreted. */ -ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize); - -/*! ZSTD_CCtx_refCDict() : - * Reference a prepared dictionary, to be used for all next compressed frames. - * Note that compression parameters are enforced from within CDict, - * and supercede any compression parameter previously set within CCtx. - * The dictionary will remain valid for future compressed frames using same CCtx. - * @result : 0, or an error code (which can be tested with ZSTD_isError()). - * Special : Referencing a NULL CDict means "return to no-dictionary mode". - * Note 1 : Currently, only one dictionary can be managed. - * Referencing a new dictionary effectively "discards" any previous one. - * Note 2 : CDict is just referenced, its lifetime must outlive its usage within CCtx. */ -ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); - -/*! ZSTD_CCtx_refPrefix() : - * Reference a prefix (single-usage dictionary) for next compressed frame. - * A prefix is **only used once**. Tables are discarded at end of frame (ZSTD_e_end). - * Decompression will need same prefix to properly regenerate data. - * Compressing with a prefix is similar in outcome as performing a diff and compressing it, - * but performs much faster, especially during decompression (compression speed is tunable with compression level). - * @result : 0, or an error code (which can be tested with ZSTD_isError()). - * Special: Adding any prefix (including NULL) invalidates any previous prefix or dictionary - * Note 1 : Prefix buffer is referenced. It **must** outlive compression. - * Its content must remain unmodified during compression. - * Note 2 : If the intention is to diff some large src data blob with some prior version of itself, - * ensure that the window size is large enough to contain the entire source. - * See ZSTD_c_windowLog. - * Note 3 : Referencing a prefix involves building tables, which are dependent on compression parameters. - * It's a CPU consuming operation, with non-negligible impact on latency. - * If there is a need to use the same prefix multiple times, consider loadDictionary instead. - * Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dm_rawContent). - * Use experimental ZSTD_CCtx_refPrefix_advanced() to alter dictionary interpretation. */ -ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, - const void* prefix, size_t prefixSize); - - typedef enum { ZSTD_reset_session_only = 1, ZSTD_reset_parameters = 2, @@ -756,8 +458,6 @@ typedef enum { */ ZSTDLIB_API size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset); - - /*! ZSTD_compress2() : * Behave the same as ZSTD_compressCCtx(), but compression parameters are set using the advanced API. * ZSTD_compress2() always starts a new frame. @@ -772,48 +472,10 @@ ZSTDLIB_API size_t ZSTD_compress2( ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); -typedef enum { - ZSTD_e_continue=0, /* collect more data, encoder decides when to output compressed result, for optimal compression ratio */ - ZSTD_e_flush=1, /* flush any data provided so far, - * it creates (at least) one new block, that can be decoded immediately on reception; - * frame will continue: any future data can still reference previously compressed data, improving compression. */ - ZSTD_e_end=2 /* flush any remaining data _and_ close current frame. - * note that frame is only closed after compressed data is fully flushed (return value == 0). - * After that point, any additional data starts a new frame. - * note : each frame is independent (does not reference any content from previous frame). */ -} ZSTD_EndDirective; - -/*! ZSTD_compressStream2() : - * Behaves about the same as ZSTD_compressStream, with additional control on end directive. - * - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*() - * - Compression parameters cannot be changed once compression is started (save a list of exceptions in multi-threading mode) - * - outpot->pos must be <= dstCapacity, input->pos must be <= srcSize - * - outpot->pos and input->pos will be updated. They are guaranteed to remain below their respective limit. - * - When nbWorkers==0 (default), function is blocking : it completes its job before returning to caller. - * - When nbWorkers>=1, function is non-blocking : it just acquires a copy of input, and distributes jobs to internal worker threads, flush whatever is available, - * and then immediately returns, just indicating that there is some data remaining to be flushed. - * The function nonetheless guarantees forward progress : it will return only after it reads or write at least 1+ byte. - * - Exception : if the first call requests a ZSTD_e_end directive and provides enough dstCapacity, the function delegates to ZSTD_compress2() which is always blocking. - * - @return provides a minimum amount of data remaining to be flushed from internal buffers - * or an error code, which can be tested using ZSTD_isError(). - * if @return != 0, flush is not fully completed, there is still some data left within internal buffers. - * This is useful for ZSTD_e_flush, since in this case more flushes are necessary to empty all buffers. - * For ZSTD_e_end, @return == 0 when internal buffers are fully flushed and frame is completed. - * - after a ZSTD_e_end directive, if internal buffer is not fully flushed (@return != 0), - * only ZSTD_e_end or ZSTD_e_flush operations are allowed. - * Before starting a new compression job, or changing compression parameters, - * it is required to fully flush internal buffers. - */ -ZSTDLIB_API size_t ZSTD_compressStream2( ZSTD_CCtx* cctx, - ZSTD_outBuffer* output, - ZSTD_inBuffer* input, - ZSTD_EndDirective endOp); - - -/* ============================== */ -/* Advanced decompression API */ -/* ============================== */ +/*************************************** +* Advanced decompression API +***************************************/ /* The advanced API pushes parameters one by one into an existing DCtx context. * Parameters are sticky, and remain valid for all following frames @@ -823,14 +485,14 @@ ZSTDLIB_API size_t ZSTD_compressStream2( ZSTD_CCtx* cctx, * Therefore, no new decompression function is necessary. */ - typedef enum { ZSTD_d_windowLogMax=100, /* Select a size limit (in power of 2) beyond which * the streaming API will refuse to allocate memory buffer * in order to protect the host from unreasonable memory requirements. * This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode. - * By default, a decompression context accepts window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT) */ + * By default, a decompression context accepts window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT). + * Special: value 0 means "use default maximum windowLog". */ /* note : additional experimental parameters are also available * within the experimental section of the API. @@ -843,7 +505,6 @@ typedef enum { } ZSTD_dParameter; - /*! ZSTD_dParam_getBounds() : * All parameters must belong to an interval with lower and upper bounds, * otherwise they will either trigger an error or be automatically clamped. @@ -862,6 +523,389 @@ ZSTDLIB_API ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam); */ ZSTDLIB_API size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int value); +/*! ZSTD_DCtx_reset() : + * Return a DCtx to clean state. + * Session and parameters can be reset jointly or separately. + * Parameters can only be reset when no active frame is being decompressed. + * @return : 0, or an error code, which can be tested with ZSTD_isError() + */ +ZSTDLIB_API size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset); + + +/**************************** +* Streaming +****************************/ + +typedef struct ZSTD_inBuffer_s { + const void* src; /**< start of input buffer */ + size_t size; /**< size of input buffer */ + size_t pos; /**< position where reading stopped. Will be updated. Necessarily 0 <= pos <= size */ +} ZSTD_inBuffer; + +typedef struct ZSTD_outBuffer_s { + void* dst; /**< start of output buffer */ + size_t size; /**< size of output buffer */ + size_t pos; /**< position where writing stopped. Will be updated. Necessarily 0 <= pos <= size */ +} ZSTD_outBuffer; + + + +/*-*********************************************************************** +* Streaming compression - HowTo +* +* A ZSTD_CStream object is required to track streaming operation. +* Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources. +* ZSTD_CStream objects can be reused multiple times on consecutive compression operations. +* It is recommended to re-use ZSTD_CStream since it will play nicer with system's memory, by re-using already allocated memory. +* +* For parallel execution, use one separate ZSTD_CStream per thread. +* +* note : since v1.3.0, ZSTD_CStream and ZSTD_CCtx are the same thing. +* +* Parameters are sticky : when starting a new compression on the same context, +* it will re-use the same sticky parameters as previous compression session. +* When in doubt, it's recommended to fully initialize the context before usage. +* Use ZSTD_CCtx_reset() to reset the context and ZSTD_CCtx_setParameter(), +* ZSTD_CCtx_setPledgedSrcSize(), or ZSTD_CCtx_loadDictionary() and friends to +* set more specific parameters, the pledged source size, or load a dictionary. +* +* Use ZSTD_compressStream2() with ZSTD_e_continue as many times as necessary to +* consume input stream. The function will automatically update both `pos` +* fields within `input` and `output`. +* Note that the function may not consume the entire input, for example, because +* the output buffer is already full, in which case `input.pos < input.size`. +* The caller must check if input has been entirely consumed. +* If not, the caller must make some room to receive more compressed data, +* and then present again remaining input data. +* note: ZSTD_e_continue is guaranteed to make some forward progress when called, +* but doesn't guarantee maximal forward progress. This is especially relevant +* when compressing with multiple threads. The call won't block if it can +* consume some input, but if it can't it will wait for some, but not all, +* output to be flushed. +* @return : provides a minimum amount of data remaining to be flushed from internal buffers +* or an error code, which can be tested using ZSTD_isError(). +* +* At any moment, it's possible to flush whatever data might remain stuck within internal buffer, +* using ZSTD_compressStream2() with ZSTD_e_flush. `output->pos` will be updated. +* Note that, if `output->size` is too small, a single invocation with ZSTD_e_flush might not be enough (return code > 0). +* In which case, make some room to receive more compressed data, and call again ZSTD_compressStream2() with ZSTD_e_flush. +* You must continue calling ZSTD_compressStream2() with ZSTD_e_flush until it returns 0, at which point you can change the +* operation. +* note: ZSTD_e_flush will flush as much output as possible, meaning when compressing with multiple threads, it will +* block until the flush is complete or the output buffer is full. +* @return : 0 if internal buffers are entirely flushed, +* >0 if some data still present within internal buffer (the value is minimal estimation of remaining size), +* or an error code, which can be tested using ZSTD_isError(). +* +* Calling ZSTD_compressStream2() with ZSTD_e_end instructs to finish a frame. +* It will perform a flush and write frame epilogue. +* The epilogue is required for decoders to consider a frame completed. +* flush operation is the same, and follows same rules as calling ZSTD_compressStream2() with ZSTD_e_flush. +* You must continue calling ZSTD_compressStream2() with ZSTD_e_end until it returns 0, at which point you are free to +* start a new frame. +* note: ZSTD_e_end will flush as much output as possible, meaning when compressing with multiple threads, it will +* block until the flush is complete or the output buffer is full. +* @return : 0 if frame fully completed and fully flushed, +* >0 if some data still present within internal buffer (the value is minimal estimation of remaining size), +* or an error code, which can be tested using ZSTD_isError(). +* +* *******************************************************************/ + +typedef ZSTD_CCtx ZSTD_CStream; /**< CCtx and CStream are now effectively same object (>= v1.3.0) */ + /* Continue to distinguish them for compatibility with older versions <= v1.2.0 */ +/*===== ZSTD_CStream management functions =====*/ +ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void); +ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs); + +/*===== Streaming compression functions =====*/ +typedef enum { + ZSTD_e_continue=0, /* collect more data, encoder decides when to output compressed result, for optimal compression ratio */ + ZSTD_e_flush=1, /* flush any data provided so far, + * it creates (at least) one new block, that can be decoded immediately on reception; + * frame will continue: any future data can still reference previously compressed data, improving compression. + * note : multithreaded compression will block to flush as much output as possible. */ + ZSTD_e_end=2 /* flush any remaining data _and_ close current frame. + * note that frame is only closed after compressed data is fully flushed (return value == 0). + * After that point, any additional data starts a new frame. + * note : each frame is independent (does not reference any content from previous frame). + : note : multithreaded compression will block to flush as much output as possible. */ +} ZSTD_EndDirective; + +/*! ZSTD_compressStream2() : + * Behaves about the same as ZSTD_compressStream, with additional control on end directive. + * - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*() + * - Compression parameters cannot be changed once compression is started (save a list of exceptions in multi-threading mode) + * - output->pos must be <= dstCapacity, input->pos must be <= srcSize + * - output->pos and input->pos will be updated. They are guaranteed to remain below their respective limit. + * - When nbWorkers==0 (default), function is blocking : it completes its job before returning to caller. + * - When nbWorkers>=1, function is non-blocking : it just acquires a copy of input, and distributes jobs to internal worker threads, flush whatever is available, + * and then immediately returns, just indicating that there is some data remaining to be flushed. + * The function nonetheless guarantees forward progress : it will return only after it reads or write at least 1+ byte. + * - Exception : if the first call requests a ZSTD_e_end directive and provides enough dstCapacity, the function delegates to ZSTD_compress2() which is always blocking. + * - @return provides a minimum amount of data remaining to be flushed from internal buffers + * or an error code, which can be tested using ZSTD_isError(). + * if @return != 0, flush is not fully completed, there is still some data left within internal buffers. + * This is useful for ZSTD_e_flush, since in this case more flushes are necessary to empty all buffers. + * For ZSTD_e_end, @return == 0 when internal buffers are fully flushed and frame is completed. + * - after a ZSTD_e_end directive, if internal buffer is not fully flushed (@return != 0), + * only ZSTD_e_end or ZSTD_e_flush operations are allowed. + * Before starting a new compression job, or changing compression parameters, + * it is required to fully flush internal buffers. + */ +ZSTDLIB_API size_t ZSTD_compressStream2( ZSTD_CCtx* cctx, + ZSTD_outBuffer* output, + ZSTD_inBuffer* input, + ZSTD_EndDirective endOp); + +ZSTDLIB_API size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */ +ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block in all circumstances. */ + +/******************************************************************************* + * This is a legacy streaming API, and can be replaced by ZSTD_CCtx_reset() and + * ZSTD_compressStream2(). It is redundant, but is still fully supported. + * Advanced parameters and dictionary compression can only be used through the + * new API. + ******************************************************************************/ + +/** + * Equivalent to: + * + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any) + * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); + */ +ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel); +/** + * Alternative for ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue). + * NOTE: The return value is different. ZSTD_compressStream() returns a hint for + * the next read size (if non-zero and not an error). ZSTD_compressStream2() + * returns the number of bytes left to flush (if non-zero and not an error). + */ +ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input); +/** Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_flush). */ +ZSTDLIB_API size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); +/** Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_end). */ +ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); + + +/*-*************************************************************************** +* Streaming decompression - HowTo +* +* A ZSTD_DStream object is required to track streaming operations. +* Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources. +* ZSTD_DStream objects can be re-used multiple times. +* +* Use ZSTD_initDStream() to start a new decompression operation. +* @return : recommended first input size +* Alternatively, use advanced API to set specific properties. +* +* Use ZSTD_decompressStream() repetitively to consume your input. +* The function will update both `pos` fields. +* If `input.pos < input.size`, some input has not been consumed. +* It's up to the caller to present again remaining data. +* The function tries to flush all data decoded immediately, respecting output buffer size. +* If `output.pos < output.size`, decoder has flushed everything it could. +* But if `output.pos == output.size`, there might be some data left within internal buffers., +* In which case, call ZSTD_decompressStream() again to flush whatever remains in the buffer. +* Note : with no additional input provided, amount of data flushed is necessarily <= ZSTD_BLOCKSIZE_MAX. +* @return : 0 when a frame is completely decoded and fully flushed, +* or an error code, which can be tested using ZSTD_isError(), +* or any other value > 0, which means there is still some decoding or flushing to do to complete current frame : +* the return value is a suggested next input size (just a hint for better latency) +* that will never request more than the remaining frame size. +* *******************************************************************************/ + +typedef ZSTD_DCtx ZSTD_DStream; /**< DCtx and DStream are now effectively same object (>= v1.3.0) */ + /* For compatibility with versions <= v1.2.0, prefer differentiating them. */ +/*===== ZSTD_DStream management functions =====*/ +ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void); +ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds); + +/*===== Streaming decompression functions =====*/ + +/* This function is redundant with the advanced API and equivalent to: + * + * ZSTD_DCtx_reset(zds); + * ZSTD_DCtx_refDDict(zds, NULL); + */ +ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds); + +ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input); + +ZSTDLIB_API size_t ZSTD_DStreamInSize(void); /*!< recommended size for input buffer */ +ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output buffer. Guarantee to successfully flush at least one complete block in all circumstances. */ + + +/************************** +* Simple dictionary API +***************************/ +/*! ZSTD_compress_usingDict() : + * Compression at an explicit compression level using a Dictionary. + * A dictionary can be any arbitrary data segment (also called a prefix), + * or a buffer with specified information (see dictBuilder/zdict.h). + * Note : This function loads the dictionary, resulting in significant startup delay. + * It's intended for a dictionary used only once. + * Note 2 : When `dict == NULL || dictSize < 8` no dictionary is used. */ +ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize, + int compressionLevel); + +/*! ZSTD_decompress_usingDict() : + * Decompression using a known Dictionary. + * Dictionary must be identical to the one used during compression. + * Note : This function loads the dictionary, resulting in significant startup delay. + * It's intended for a dictionary used only once. + * Note : When `dict == NULL || dictSize < 8` no dictionary is used. */ +ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize); + + +/*********************************** + * Bulk processing dictionary API + **********************************/ +typedef struct ZSTD_CDict_s ZSTD_CDict; + +/*! ZSTD_createCDict() : + * When compressing multiple messages / blocks using the same dictionary, it's recommended to load it only once. + * ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup cost. + * ZSTD_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only. + * `dictBuffer` can be released after ZSTD_CDict creation, because its content is copied within CDict. + * Consider experimental function `ZSTD_createCDict_byReference()` if you prefer to not duplicate `dictBuffer` content. + * Note : A ZSTD_CDict can be created from an empty dictBuffer, but it is inefficient when used to compress small data. */ +ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize, + int compressionLevel); + +/*! ZSTD_freeCDict() : + * Function frees memory allocated by ZSTD_createCDict(). */ +ZSTDLIB_API size_t ZSTD_freeCDict(ZSTD_CDict* CDict); + +/*! ZSTD_compress_usingCDict() : + * Compression using a digested Dictionary. + * Recommended when same dictionary is used multiple times. + * Note : compression level is _decided at dictionary creation time_, + * and frame parameters are hardcoded (dictID=yes, contentSize=yes, checksum=no) */ +ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_CDict* cdict); + + +typedef struct ZSTD_DDict_s ZSTD_DDict; + +/*! ZSTD_createDDict() : + * Create a digested dictionary, ready to start decompression operation without startup delay. + * dictBuffer can be released after DDict creation, as its content is copied inside DDict. */ +ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dictBuffer, size_t dictSize); + +/*! ZSTD_freeDDict() : + * Function frees memory allocated with ZSTD_createDDict() */ +ZSTDLIB_API size_t ZSTD_freeDDict(ZSTD_DDict* ddict); + +/*! ZSTD_decompress_usingDDict() : + * Decompression using a digested Dictionary. + * Recommended when same dictionary is used multiple times. */ +ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_DDict* ddict); + + +/******************************** + * Dictionary helper functions + *******************************/ + +/*! ZSTD_getDictID_fromDict() : + * Provides the dictID stored within dictionary. + * if @return == 0, the dictionary is not conformant with Zstandard specification. + * It can still be loaded, but as a content-only dictionary. */ +ZSTDLIB_API unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize); + +/*! ZSTD_getDictID_fromDDict() : + * Provides the dictID of the dictionary loaded into `ddict`. + * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. + * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ +ZSTDLIB_API unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict); + +/*! ZSTD_getDictID_fromFrame() : + * Provides the dictID required to decompressed the frame stored within `src`. + * If @return == 0, the dictID could not be decoded. + * This could for one of the following reasons : + * - The frame does not require a dictionary to be decoded (most common case). + * - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden information. + * Note : this use case also happens when using a non-conformant dictionary. + * - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`). + * - This is not a Zstandard frame. + * When identifying the exact failure cause, it's possible to use ZSTD_getFrameHeader(), which will provide a more precise error code. */ +ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize); + + +/******************************************************************************* + * Advanced dictionary and prefix API + * + * This API allows dictionaries to be used with ZSTD_compress2(), + * ZSTD_compressStream2(), and ZSTD_decompress(). Dictionaries are sticky, and + * only reset with the context is reset with ZSTD_reset_parameters or + * ZSTD_reset_session_and_parameters. Prefixes are single-use. + ******************************************************************************/ + + +/*! ZSTD_CCtx_loadDictionary() : + * Create an internal CDict from `dict` buffer. + * Decompression will have to use same dictionary. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Special: Loading a NULL (or 0-size) dictionary invalidates previous dictionary, + * meaning "return to no-dictionary mode". + * Note 1 : Dictionary is sticky, it will be used for all future compressed frames. + * To return to "no-dictionary" situation, load a NULL dictionary (or reset parameters). + * Note 2 : Loading a dictionary involves building tables. + * It's also a CPU consuming operation, with non-negligible impact on latency. + * Tables are dependent on compression parameters, and for this reason, + * compression parameters can no longer be changed after loading a dictionary. + * Note 3 :`dict` content will be copied internally. + * Use experimental ZSTD_CCtx_loadDictionary_byReference() to reference content instead. + * In such a case, dictionary buffer must outlive its users. + * Note 4 : Use ZSTD_CCtx_loadDictionary_advanced() + * to precisely select how dictionary content must be interpreted. */ +ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize); + +/*! ZSTD_CCtx_refCDict() : + * Reference a prepared dictionary, to be used for all next compressed frames. + * Note that compression parameters are enforced from within CDict, + * and supersede any compression parameter previously set within CCtx. + * The parameters ignored are labled as "superseded-by-cdict" in the ZSTD_cParameter enum docs. + * The ignored parameters will be used again if the CCtx is returned to no-dictionary mode. + * The dictionary will remain valid for future compressed frames using same CCtx. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Special : Referencing a NULL CDict means "return to no-dictionary mode". + * Note 1 : Currently, only one dictionary can be managed. + * Referencing a new dictionary effectively "discards" any previous one. + * Note 2 : CDict is just referenced, its lifetime must outlive its usage within CCtx. */ +ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); + +/*! ZSTD_CCtx_refPrefix() : + * Reference a prefix (single-usage dictionary) for next compressed frame. + * A prefix is **only used once**. Tables are discarded at end of frame (ZSTD_e_end). + * Decompression will need same prefix to properly regenerate data. + * Compressing with a prefix is similar in outcome as performing a diff and compressing it, + * but performs much faster, especially during decompression (compression speed is tunable with compression level). + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Special: Adding any prefix (including NULL) invalidates any previous prefix or dictionary + * Note 1 : Prefix buffer is referenced. It **must** outlive compression. + * Its content must remain unmodified during compression. + * Note 2 : If the intention is to diff some large src data blob with some prior version of itself, + * ensure that the window size is large enough to contain the entire source. + * See ZSTD_c_windowLog. + * Note 3 : Referencing a prefix involves building tables, which are dependent on compression parameters. + * It's a CPU consuming operation, with non-negligible impact on latency. + * If there is a need to use the same prefix multiple times, consider loadDictionary instead. + * Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dm_rawContent). + * Use experimental ZSTD_CCtx_refPrefix_advanced() to alter dictionary interpretation. */ +ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, + const void* prefix, size_t prefixSize); /*! ZSTD_DCtx_loadDictionary() : * Create an internal DDict from dict buffer, @@ -910,15 +954,32 @@ ZSTDLIB_API size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); ZSTDLIB_API size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize); -/*! ZSTD_DCtx_reset() : - * Return a DCtx to clean state. - * Session and parameters can be reset jointly or separately. - * Parameters can only be reset when no active frame is being decompressed. - * @return : 0, or an error code, which can be tested with ZSTD_isError() - */ -ZSTDLIB_API size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset); +/* === Memory management === */ + +/*! ZSTD_sizeof_*() : + * These functions give the _current_ memory usage of selected object. + * Note that object memory usage can evolve (increase or decrease) over time. */ +ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx); +ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx); +ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs); +ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds); +ZSTDLIB_API size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict); +ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); + +#endif /* ZSTD_H_235446 */ + +/**************************************************************************************** + * ADVANCED AND EXPERIMENTAL FUNCTIONS + **************************************************************************************** + * The definitions in the following section are considered experimental. + * They are provided for advanced scenarios. + * They should never be used with a dynamic library, as prototypes may change in the future. + * Use them only in association with static linking. + * ***************************************************************************************/ +#if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY) +#define ZSTD_H_ZSTD_STATIC_LINKING_ONLY /**************************************************************************************** * experimental API (static linking only) @@ -962,7 +1023,7 @@ ZSTDLIB_API size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset); #define ZSTD_WINDOWLOG_LIMIT_DEFAULT 27 /* by default, the streaming decoder will refuse any frame * requiring larger than (1<<ZSTD_WINDOWLOG_LIMIT_DEFAULT) window size, * to preserve host's memory from unreasonable requirements. - * This limit can be overriden using ZSTD_DCtx_setParameter(,ZSTD_d_windowLogMax,). + * This limit can be overridden using ZSTD_DCtx_setParameter(,ZSTD_d_windowLogMax,). * The limit does not apply for one-pass decoders (such as ZSTD_decompress()), since no additional memory is allocated */ @@ -1064,15 +1125,24 @@ typedef enum { ZSTD_dictForceCopy = 2, /* Always copy the dictionary. */ } ZSTD_dictAttachPref_e; +typedef enum { + ZSTD_lcm_auto = 0, /**< Automatically determine the compression mode based on the compression level. + * Negative compression levels will be uncompressed, and positive compression + * levels will be compressed. */ + ZSTD_lcm_huffman = 1, /**< Always attempt Huffman compression. Uncompressed literals will still be + * emitted if Huffman compression is not profitable. */ + ZSTD_lcm_uncompressed = 2, /**< Always emit uncompressed literals. */ +} ZSTD_literalCompressionMode_e; + /*************************************** * Frame size functions ***************************************/ /*! ZSTD_findDecompressedSize() : - * `src` should point the start of a series of ZSTD encoded and/or skippable frames + * `src` should point to the start of a series of ZSTD encoded and/or skippable frames * `srcSize` must be the _exact_ size of this series - * (i.e. there should be a frame boundary exactly at `srcSize` bytes after `src`) + * (i.e. there should be a frame boundary at `src + srcSize`) * @return : - decompressed size of all data in all successive frames * - if the decompressed size cannot be determined: ZSTD_CONTENTSIZE_UNKNOWN * - if an error occurred: ZSTD_CONTENTSIZE_ERROR @@ -1092,6 +1162,21 @@ typedef enum { * however it does mean that all frame data must be present and valid. */ ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize); +/** ZSTD_decompressBound() : + * `src` should point to the start of a series of ZSTD encoded and/or skippable frames + * `srcSize` must be the _exact_ size of this series + * (i.e. there should be a frame boundary at `src + srcSize`) + * @return : - upper-bound for the decompressed size of all data in all successive frames + * - if an error occured: ZSTD_CONTENTSIZE_ERROR + * + * note 1 : an error can occur if `src` contains an invalid or incorrectly formatted frame. + * note 2 : the upper-bound is exact when the decompressed size field is available in every ZSTD encoded frame of `src`. + * in this case, `ZSTD_findDecompressedSize` and `ZSTD_decompressBound` return the same value. + * note 3 : when the decompressed size field isn't available, the upper-bound for that frame is calculated by: + * upper-bound = # blocks * min(128 KB, Window_Size) + */ +ZSTDLIB_API unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize); + /*! ZSTD_frameHeaderSize() : * srcSize must be >= ZSTD_FRAMEHEADERSIZE_PREFIX. * @return : size of the Frame Header, @@ -1110,7 +1195,7 @@ ZSTDLIB_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize); * It will also consider src size to be arbitrarily "large", which is worst case. * If srcSize is known to always be small, ZSTD_estimateCCtxSize_usingCParams() can provide a tighter estimation. * ZSTD_estimateCCtxSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel. - * ZSTD_estimateCCtxSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParam_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_c_nbWorkers is >= 1. + * ZSTD_estimateCCtxSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParams_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_c_nbWorkers is >= 1. * Note : CCtx size estimation is only correct for single-threaded compression. */ ZSTDLIB_API size_t ZSTD_estimateCCtxSize(int compressionLevel); ZSTDLIB_API size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams); @@ -1122,7 +1207,7 @@ ZSTDLIB_API size_t ZSTD_estimateDCtxSize(void); * It will also consider src size to be arbitrarily "large", which is worst case. * If srcSize is known to always be small, ZSTD_estimateCStreamSize_usingCParams() can provide a tighter estimation. * ZSTD_estimateCStreamSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel. - * ZSTD_estimateCStreamSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParam_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_c_nbWorkers is >= 1. + * ZSTD_estimateCStreamSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParams_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_c_nbWorkers is >= 1. * Note : CStream size estimation is only correct for single-threaded compression. * ZSTD_DStream memory budget depends on window Size. * This information can be passed manually, using ZSTD_estimateDStreamSize, @@ -1226,22 +1311,26 @@ ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictS ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel); /*! ZSTD_getCParams() : -* @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize. -* `estimatedSrcSize` value is optional, select 0 if not known */ + * @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize. + * `estimatedSrcSize` value is optional, select 0 if not known */ ZSTDLIB_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize); /*! ZSTD_getParams() : -* same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of sub-component `ZSTD_compressionParameters`. -* All fields of `ZSTD_frameParameters` are set to default : contentSize=1, checksum=0, noDictID=0 */ + * same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of sub-component `ZSTD_compressionParameters`. + * All fields of `ZSTD_frameParameters` are set to default : contentSize=1, checksum=0, noDictID=0 */ ZSTDLIB_API ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize); /*! ZSTD_checkCParams() : -* Ensure param values remain within authorized range */ + * Ensure param values remain within authorized range. + * @return 0 on success, or an error code (can be checked with ZSTD_isError()) */ ZSTDLIB_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params); /*! ZSTD_adjustCParams() : * optimize params for a given `srcSize` and `dictSize`. - * both values are optional, select `0` if unknown. */ + * `srcSize` can be unknown, in which case use ZSTD_CONTENTSIZE_UNKNOWN. + * `dictSize` must be `0` when there is no dictionary. + * cPar can be invalid : all parameters will be clamped within valid range in the @return struct. + * This function never fails (wide contract) */ ZSTDLIB_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize); /*! ZSTD_compress_advanced() : @@ -1314,6 +1403,12 @@ ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* pre * See the comments on that enum for an explanation of the feature. */ #define ZSTD_c_forceAttachDict ZSTD_c_experimentalParam4 +/* Controls how the literals are compressed (default is auto). + * The value must be of type ZSTD_literalCompressionMode_e. + * See ZSTD_literalCompressionMode_t enum definition for details. + */ +#define ZSTD_c_literalCompressionMode ZSTD_c_experimentalParam5 + /*! ZSTD_CCtx_getParameter() : * Get the requested compression parameter value, selected by enum ZSTD_cParameter, * and store it into int* value. @@ -1325,10 +1420,10 @@ ZSTDLIB_API size_t ZSTD_CCtx_getParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param /*! ZSTD_CCtx_params : * Quick howto : * - ZSTD_createCCtxParams() : Create a ZSTD_CCtx_params structure - * - ZSTD_CCtxParam_setParameter() : Push parameters one by one into - * an existing ZSTD_CCtx_params structure. - * This is similar to - * ZSTD_CCtx_setParameter(). + * - ZSTD_CCtxParams_setParameter() : Push parameters one by one into + * an existing ZSTD_CCtx_params structure. + * This is similar to + * ZSTD_CCtx_setParameter(). * - ZSTD_CCtx_setParametersUsingCCtxParams() : Apply parameters to * an existing CCtx. * These parameters will be applied to @@ -1359,20 +1454,20 @@ ZSTDLIB_API size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compre */ ZSTDLIB_API size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params); -/*! ZSTD_CCtxParam_setParameter() : +/*! ZSTD_CCtxParams_setParameter() : * Similar to ZSTD_CCtx_setParameter. * Set one compression parameter, selected by enum ZSTD_cParameter. * Parameters must be applied to a ZSTD_CCtx using ZSTD_CCtx_setParametersUsingCCtxParams(). * @result : 0, or an error code (which can be tested with ZSTD_isError()). */ -ZSTDLIB_API size_t ZSTD_CCtxParam_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int value); +ZSTDLIB_API size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int value); -/*! ZSTD_CCtxParam_getParameter() : +/*! ZSTD_CCtxParams_getParameter() : * Similar to ZSTD_CCtx_getParameter. * Get the requested value of one compression parameter, selected by enum ZSTD_cParameter. * @result : 0, or an error code (which can be tested with ZSTD_isError()). */ -ZSTDLIB_API size_t ZSTD_CCtxParam_getParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int* value); +ZSTDLIB_API size_t ZSTD_CCtxParams_getParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int* value); /*! ZSTD_CCtx_setParametersUsingCCtxParams() : * Apply a set of ZSTD_CCtx_params to the compression context. @@ -1415,31 +1510,6 @@ ZSTDLIB_API unsigned ZSTD_isFrame(const void* buffer, size_t size); * it must remain read accessible throughout the lifetime of DDict */ ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize); - -/*! ZSTD_getDictID_fromDict() : - * Provides the dictID stored within dictionary. - * if @return == 0, the dictionary is not conformant with Zstandard specification. - * It can still be loaded, but as a content-only dictionary. */ -ZSTDLIB_API unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize); - -/*! ZSTD_getDictID_fromDDict() : - * Provides the dictID of the dictionary loaded into `ddict`. - * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. - * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ -ZSTDLIB_API unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict); - -/*! ZSTD_getDictID_fromFrame() : - * Provides the dictID required to decompressed the frame stored within `src`. - * If @return == 0, the dictID could not be decoded. - * This could for one of the following reasons : - * - The frame does not require a dictionary to be decoded (most common case). - * - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden information. - * Note : this use case also happens when using a non-conformant dictionary. - * - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`). - * - This is not a Zstandard frame. - * When identifying the exact failure cause, it's possible to use ZSTD_getFrameHeader(), which will provide a more precise error code. */ -ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize); - /*! ZSTD_DCtx_loadDictionary_byReference() : * Same as ZSTD_DCtx_loadDictionary(), * but references `dict` content instead of copying it into `dctx`. @@ -1501,14 +1571,68 @@ ZSTDLIB_API size_t ZSTD_decompressStream_simpleArgs ( ********************************************************************/ /*===== Advanced Streaming compression functions =====*/ -ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); /**< pledgedSrcSize must be correct. If it is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs, "0" also disables frame content size field. It may be enabled in the future. */ -ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< creates of an internal CDict (incompatible with static CCtx), except if dict == NULL or dictSize < 8, in which case no dict is used. Note: dict is loaded with ZSTD_dm_auto (treated as a full zstd dictionary if it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.*/ +/**! ZSTD_initCStream_srcSize() : + * This function is deprecated, and equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any) + * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); + * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); + * + * pledgedSrcSize must be correct. If it is not known at init time, use + * ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs, + * "0" also disables frame content size field. It may be enabled in the future. + */ +ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); +/**! ZSTD_initCStream_usingDict() : + * This function is deprecated, and is equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); + * ZSTD_CCtx_loadDictionary(zcs, dict, dictSize); + * + * Creates of an internal CDict (incompatible with static CCtx), except if + * dict == NULL or dictSize < 8, in which case no dict is used. + * Note: dict is loaded with ZSTD_dm_auto (treated as a full zstd dictionary if + * it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy. + */ +ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); +/**! ZSTD_initCStream_advanced() : + * This function is deprecated, and is approximately equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_setZstdParams(zcs, params); // Set the zstd params and leave the rest as-is + * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); + * ZSTD_CCtx_loadDictionary(zcs, dict, dictSize); + * + * pledgedSrcSize must be correct. If srcSize is not known at init time, use + * value ZSTD_CONTENTSIZE_UNKNOWN. dict is loaded with ZSTD_dm_auto and ZSTD_dlm_byCopy. + */ ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize, - ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize must be correct. If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. dict is loaded with ZSTD_dm_auto and ZSTD_dlm_byCopy. */ -ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); /**< note : cdict will just be referenced, and must outlive compression session */ -ZSTDLIB_API size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize); /**< same as ZSTD_initCStream_usingCDict(), with control over frame parameters. pledgedSrcSize must be correct. If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. */ + ZSTD_parameters params, unsigned long long pledgedSrcSize); +/**! ZSTD_initCStream_usingCDict() : + * This function is deprecated, and equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_refCDict(zcs, cdict); + * + * note : cdict will just be referenced, and must outlive compression session + */ +ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); +/**! ZSTD_initCStream_usingCDict_advanced() : + * This function is deprecated, and is approximately equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_setZstdFrameParams(zcs, fParams); // Set the zstd frame params and leave the rest as-is + * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); + * ZSTD_CCtx_refCDict(zcs, cdict); + * + * same as ZSTD_initCStream_usingCDict(), with control over frame parameters. + * pledgedSrcSize must be correct. If srcSize is not known at init time, use + * value ZSTD_CONTENTSIZE_UNKNOWN. + */ +ZSTDLIB_API size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize); /*! ZSTD_resetCStream() : + * This function is deprecated, and is equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); + * * start a new frame, using same parameters from previous frame. * This is typically useful to skip dictionary loading stage, since it will re-use it in-place. * Note that zcs must be init at least once before using ZSTD_resetCStream(). @@ -1555,9 +1679,32 @@ ZSTDLIB_API size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx); /*===== Advanced Streaming decompression functions =====*/ -ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); /**< note: no dictionary will be used if dict == NULL or dictSize < 8 */ -ZSTDLIB_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); /**< note : ddict is referenced, it must outlive decompression session */ -ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); /**< re-use decompression parameters from previous init; saves dictionary loading */ +/** + * This function is deprecated, and is equivalent to: + * + * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); + * ZSTD_DCtx_loadDictionary(zds, dict, dictSize); + * + * note: no dictionary will be used if dict == NULL or dictSize < 8 + */ +ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); +/** + * This function is deprecated, and is equivalent to: + * + * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); + * ZSTD_DCtx_refDDict(zds, ddict); + * + * note : ddict is referenced, it must outlive decompression session + */ +ZSTDLIB_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); +/** + * This function is deprecated, and is equivalent to: + * + * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); + * + * re-use decompression parameters from previous init; saves dictionary loading + */ +ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); /********************************************************************* |