diff options
74 files changed, 1870 insertions, 1105 deletions
diff --git a/SConstruct b/SConstruct index 92dc4d9da2..aa5e3a98c8 100644 --- a/SConstruct +++ b/SConstruct @@ -311,6 +311,10 @@ if selected_platform in platform_list: # must happen after the flags, so when flags are used by configure, stuff happens (ie, ssl on x11) detect.configure(env) + # Enable C++11 support + if not env.msvc: + env.Append(CXXFLAGS=['-std=c++11']) + # Configure compiler warnings if env.msvc: # Truncations, narrowing conversions, signed/unsigned comparisons... diff --git a/core/math/expression.cpp b/core/math/expression.cpp index 25a40c70bf..15eea1d308 100644 --- a/core/math/expression.cpp +++ b/core/math/expression.cpp @@ -68,6 +68,7 @@ const char *Expression::func_name[Expression::FUNC_MAX] = { "step_decimals", "stepify", "lerp", + "lerp_angle", "inverse_lerp", "range_lerp", "smoothstep", @@ -190,6 +191,7 @@ int Expression::get_func_argument_count(BuiltinFunc p_func) { case COLORN: return 2; case MATH_LERP: + case MATH_LERP_ANGLE: case MATH_INVERSE_LERP: case MATH_SMOOTHSTEP: case MATH_MOVE_TOWARD: @@ -395,6 +397,13 @@ void Expression::exec_func(BuiltinFunc p_func, const Variant **p_inputs, Variant VALIDATE_ARG_NUM(2); *r_return = Math::lerp((double)*p_inputs[0], (double)*p_inputs[1], (double)*p_inputs[2]); } break; + case MATH_LERP_ANGLE: { + + VALIDATE_ARG_NUM(0); + VALIDATE_ARG_NUM(1); + VALIDATE_ARG_NUM(2); + *r_return = Math::lerp_angle((double)*p_inputs[0], (double)*p_inputs[1], (double)*p_inputs[2]); + } break; case MATH_INVERSE_LERP: { VALIDATE_ARG_NUM(0); diff --git a/core/math/expression.h b/core/math/expression.h index 03a2bb70e6..833220592c 100644 --- a/core/math/expression.h +++ b/core/math/expression.h @@ -67,6 +67,7 @@ public: MATH_STEP_DECIMALS, MATH_STEPIFY, MATH_LERP, + MATH_LERP_ANGLE, MATH_INVERSE_LERP, MATH_RANGE_LERP, MATH_SMOOTHSTEP, diff --git a/core/math/math_funcs.h b/core/math/math_funcs.h index b6398712e4..b8b5151802 100644 --- a/core/math/math_funcs.h +++ b/core/math/math_funcs.h @@ -215,6 +215,17 @@ public: static _ALWAYS_INLINE_ double lerp(double p_from, double p_to, double p_weight) { return p_from + (p_to - p_from) * p_weight; } static _ALWAYS_INLINE_ float lerp(float p_from, float p_to, float p_weight) { return p_from + (p_to - p_from) * p_weight; } + static _ALWAYS_INLINE_ double lerp_angle(double p_from, double p_to, double p_weight) { + double difference = fmod(p_to - p_from, Math_TAU); + double distance = fmod(2.0 * difference, Math_TAU) - difference; + return p_from + distance * p_weight; + } + static _ALWAYS_INLINE_ float lerp_angle(float p_from, float p_to, float p_weight) { + float difference = fmod(p_to - p_from, (float)Math_TAU); + float distance = fmod(2.0f * difference, (float)Math_TAU) - difference; + return p_from + distance * p_weight; + } + static _ALWAYS_INLINE_ double inverse_lerp(double p_from, double p_to, double p_value) { return (p_value - p_from) / (p_to - p_from); } static _ALWAYS_INLINE_ float inverse_lerp(float p_from, float p_to, float p_value) { return (p_value - p_from) / (p_to - p_from); } diff --git a/core/object.cpp b/core/object.cpp index 67ab27c190..3367d6b6c3 100644 --- a/core/object.cpp +++ b/core/object.cpp @@ -635,12 +635,9 @@ void Object::get_property_list(List<PropertyInfo> *p_list, bool p_reversed) cons #endif p_list->push_back(PropertyInfo(Variant::OBJECT, "script", PROPERTY_HINT_RESOURCE_TYPE, "Script", PROPERTY_USAGE_DEFAULT)); } - -#ifdef TOOLS_ENABLED - p_list->push_back(PropertyInfo(Variant::NIL, "Metadata", PROPERTY_HINT_NONE, "", PROPERTY_USAGE_GROUP)); -#endif - p_list->push_back(PropertyInfo(Variant::DICTIONARY, "__meta__", PROPERTY_HINT_NONE, "", PROPERTY_USAGE_DEFAULT)); - + if (!metadata.empty()) { + p_list->push_back(PropertyInfo(Variant::DICTIONARY, "__meta__", PROPERTY_HINT_NONE, "", PROPERTY_USAGE_NOEDITOR | PROPERTY_USAGE_INTERNAL)); + } if (script_instance && !p_reversed) { p_list->push_back(PropertyInfo(Variant::NIL, "Script Variables", PROPERTY_HINT_NONE, String(), PROPERTY_USAGE_CATEGORY)); script_instance->get_property_list(p_list); diff --git a/core/type_info.h b/core/type_info.h index d85a63ee42..61ec7b2f20 100644 --- a/core/type_info.h +++ b/core/type_info.h @@ -83,15 +83,13 @@ enum Metadata { }; } +// If the compiler fails because it's trying to instantiate the primary 'GetTypeInfo' template +// instead of one of the specializations, it's most likely because the type 'T' is not supported. +// If 'T' is a class that inherits 'Object', make sure it can see the actual class declaration +// instead of a forward declaration. You can always forward declare 'T' in a header file, and then +// include the actual declaration of 'T' in the source file where 'GetTypeInfo<T>' is instantiated. template <class T, typename = void> -struct GetTypeInfo { - static const Variant::Type VARIANT_TYPE = Variant::NIL; - static const GodotTypeInfo::Metadata METADATA = GodotTypeInfo::METADATA_NONE; - static inline PropertyInfo get_class_info() { - ERR_PRINT("GetTypeInfo fallback. Bug!"); - return PropertyInfo(); // Not "Nil", this is an error - } -}; +struct GetTypeInfo; #define MAKE_TYPE_INFO(m_type, m_var_type) \ template <> \ @@ -283,10 +281,7 @@ inline StringName __constant_get_enum_name(T param, const String &p_constant) { return GetTypeInfo<T>::get_class_info().class_name; } -#define CLASS_INFO(m_type) \ - (GetTypeInfo<m_type *>::VARIANT_TYPE != Variant::NIL ? \ - GetTypeInfo<m_type *>::get_class_info() : \ - GetTypeInfo<m_type>::get_class_info()) +#define CLASS_INFO(m_type) (GetTypeInfo<m_type *>::get_class_info()) #else diff --git a/doc/classes/CollisionObject2D.xml b/doc/classes/CollisionObject2D.xml index eb69a4aed4..b9ec9480cf 100644 --- a/doc/classes/CollisionObject2D.xml +++ b/doc/classes/CollisionObject2D.xml @@ -19,7 +19,7 @@ <argument index="2" name="shape_idx" type="int"> </argument> <description> - Accepts unhandled [InputEvent]s. [code]shape_idx[/code] is the child index of the clicked [Shape2D]. Connect to the [code]input_event[/code] signal to easily pick up these events. + Accepts unhandled [InputEvent]s. Requires [member input_pickable] to be [code]true[/code]. [code]shape_idx[/code] is the child index of the clicked [Shape2D]. Connect to the [code]input_event[/code] signal to easily pick up these events. </description> </method> <method name="create_shape_owner"> @@ -227,17 +227,17 @@ <argument index="2" name="shape_idx" type="int"> </argument> <description> - Emitted when an input event occurs. Requires [code]input_pickable[/code] to be [code]true[/code] and at least one [code]collision_layer[/code] bit to be set. See [method _input_event] for details. + Emitted when an input event occurs. Requires [member input_pickable] to be [code]true[/code] and at least one [code]collision_layer[/code] bit to be set. See [method _input_event] for details. </description> </signal> <signal name="mouse_entered"> <description> - Emitted when the mouse pointer enters any of this object's shapes. Requires [code]input_pickable[/code] to be [code]true[/code] and at least one [code]collision_layer[/code] bit to be set. + Emitted when the mouse pointer enters any of this object's shapes. Requires [member input_pickable] to be [code]true[/code] and at least one [code]collision_layer[/code] bit to be set. </description> </signal> <signal name="mouse_exited"> <description> - Emitted when the mouse pointer exits all this object's shapes. Requires [code]input_pickable[/code] to be [code]true[/code] and at least one [code]collision_layer[/code] bit to be set. + Emitted when the mouse pointer exits all this object's shapes. Requires [member input_pickable] to be [code]true[/code] and at least one [code]collision_layer[/code] bit to be set. </description> </signal> </signals> diff --git a/doc/classes/DirectionalLight.xml b/doc/classes/DirectionalLight.xml index 4d0ff7f13b..687e7519b2 100644 --- a/doc/classes/DirectionalLight.xml +++ b/doc/classes/DirectionalLight.xml @@ -21,7 +21,7 @@ <member name="directional_shadow_depth_range" type="int" setter="set_shadow_depth_range" getter="get_shadow_depth_range" enum="DirectionalLight.ShadowDepthRange" default="0"> Optimizes shadow rendering for detail versus movement. See [enum ShadowDepthRange]. </member> - <member name="directional_shadow_max_distance" type="float" setter="set_param" getter="get_param" default="200.0"> + <member name="directional_shadow_max_distance" type="float" setter="set_param" getter="get_param" default="100.0"> The maximum distance for shadow splits. </member> <member name="directional_shadow_mode" type="int" setter="set_shadow_mode" getter="get_shadow_mode" enum="DirectionalLight.ShadowMode" default="2"> diff --git a/doc/classes/EditorInterface.xml b/doc/classes/EditorInterface.xml index 6f07682b04..d55e810c9e 100644 --- a/doc/classes/EditorInterface.xml +++ b/doc/classes/EditorInterface.xml @@ -166,6 +166,7 @@ <argument index="0" name="file" type="String"> </argument> <description> + Selects the file, with the path provided by [code]file[/code], in the FileSystem dock. </description> </method> <method name="set_plugin_enabled"> diff --git a/doc/classes/EditorPlugin.xml b/doc/classes/EditorPlugin.xml index bd9a100267..fddc5e9d36 100644 --- a/doc/classes/EditorPlugin.xml +++ b/doc/classes/EditorPlugin.xml @@ -29,7 +29,7 @@ <argument index="1" name="title" type="String"> </argument> <description> - Adds a control to the bottom panel (together with Output, Debug, Animation, etc). Returns a reference to the button added. It's up to you to hide/show the button when needed. When your plugin is deactivated, make sure to remove your custom control with [method remove_control_from_bottom_panel] and free it with [code]queue_free()[/code]. + Adds a control to the bottom panel (together with Output, Debug, Animation, etc). Returns a reference to the button added. It's up to you to hide/show the button when needed. When your plugin is deactivated, make sure to remove your custom control with [method remove_control_from_bottom_panel] and free it with [method Node.queue_free]. </description> </method> <method name="add_control_to_container"> @@ -40,9 +40,9 @@ <argument index="1" name="control" type="Control"> </argument> <description> - Adds a custom control to a container (see [code]CONTAINER_*[/code] enum). There are many locations where custom controls can be added in the editor UI. + Adds a custom control to a container (see [enum CustomControlContainer]). There are many locations where custom controls can be added in the editor UI. Please remember that you have to manage the visibility of your custom controls yourself (and likely hide it after adding it). - When your plugin is deactivated, make sure to remove your custom control with [method remove_control_from_container] and free it with [code]queue_free()[/code]. + When your plugin is deactivated, make sure to remove your custom control with [method remove_control_from_container] and free it with [method Node.queue_free]. </description> </method> <method name="add_control_to_dock"> @@ -53,9 +53,9 @@ <argument index="1" name="control" type="Control"> </argument> <description> - Adds the control to a specific dock slot (see [code]DOCK_*[/code] enum for options). + Adds the control to a specific dock slot (see [enum DockSlot] for options). If the dock is repositioned and as long as the plugin is active, the editor will save the dock position on further sessions. - When your plugin is deactivated, make sure to remove your custom control with [method remove_control_from_docks] and free it with [code]queue_free()[/code]. + When your plugin is deactivated, make sure to remove your custom control with [method remove_control_from_docks] and free it with [method Node.queue_free]. </description> </method> <method name="add_custom_type"> @@ -167,6 +167,7 @@ <return type="void"> </return> <description> + Called by the engine when the user disables the [EditorPlugin] in the Plugin tab of the project settings window. </description> </method> <method name="edit" qualifiers="virtual"> @@ -182,6 +183,7 @@ <return type="void"> </return> <description> + Called by the engine when the user enables the [EditorPlugin] in the Plugin tab of the project settings window. </description> </method> <method name="forward_canvas_draw_over_viewport" qualifiers="virtual"> @@ -190,12 +192,6 @@ <argument index="0" name="overlay" type="Control"> </argument> <description> - This method is called when there is an input event in the 2D viewport, e.g. the user clicks with the mouse in the 2D space (canvas GUI). Keep in mind that for this method to be called you have to first declare the virtual method [method handles] so the editor knows that you want to work with the workspace: - [codeblock] - func handles(object): - return true - [/codeblock] - Also note that the edited scene must have a root node. </description> </method> <method name="forward_canvas_force_draw_over_viewport" qualifiers="virtual"> @@ -212,6 +208,22 @@ <argument index="0" name="event" type="InputEvent"> </argument> <description> + Called when there is a root node in the current edited scene, [method handles] is implemented and an [InputEvent] happens in the 2D viewport. Intercepts the [InputEvent], if [code]return true[/code] [EditorPlugin] consumes the [code]event[/code], otherwise forwards [code]event[/code] to other Editor classes. Example: + [codeblock] + # Prevents the InputEvent to reach other Editor classes + func forward_canvas_gui_input(event): + var forward = true + return forward + [/codeblock] + Must [code]return false[/code] in order to forward the [InputEvent] to other Editor classes. Example: + [codeblock] + # Consumes InputEventMouseMotion and forwards other InputEvent types + func forward_canvas_gui_input(event): + var forward = false + if event is InputEventMouseMotion: + forward = true + return forward + [/codeblock] </description> </method> <method name="forward_spatial_gui_input" qualifiers="virtual"> @@ -222,12 +234,22 @@ <argument index="1" name="event" type="InputEvent"> </argument> <description> - This method is called when there is an input event in the 3D viewport, e.g. the user clicks with the mouse in the 3D space (spatial GUI). Keep in mind that for this method to be called you have to first declare the virtual method [method handles] so the editor knows that you want to work with the workspace: + Called when there is a root node in the current edited scene, [method handles] is implemented and an [InputEvent] happens in the 3D viewport. Intercepts the [InputEvent], if [code]return true[/code] [EditorPlugin] consumes the [code]event[/code], otherwise forwards [code]event[/code] to other Editor classes. Example: + [codeblock] + # Prevents the InputEvent to reach other Editor classes + func forward_spatial_gui_input(camera, event): + var forward = true + return forward + [/codeblock] + Must [code]return false[/code] in order to forward the [InputEvent] to other Editor classes. Example: [codeblock] - func handles(object): - return true + # Consumes InputEventMouseMotion and forwards other InputEvent types + func forward_spatial_gui_input(camera, event): + var forward = false + if event is InputEventMouseMotion: + forward = true + return forward [/codeblock] - Also note that the edited scene must have a root node. </description> </method> <method name="get_breakpoints" qualifiers="virtual"> @@ -349,7 +371,7 @@ <argument index="0" name="control" type="Control"> </argument> <description> - Removes the control from the bottom panel. You have to manually [code]queue_free()[/code] the control. + Removes the control from the bottom panel. You have to manually [method Node.queue_free] the control. </description> </method> <method name="remove_control_from_container"> @@ -360,7 +382,7 @@ <argument index="1" name="control" type="Control"> </argument> <description> - Removes the control from the specified container. You have to manually [code]queue_free()[/code] the control. + Removes the control from the specified container. You have to manually [method Node.queue_free] the control. </description> </method> <method name="remove_control_from_docks"> @@ -369,7 +391,7 @@ <argument index="0" name="control" type="Control"> </argument> <description> - Removes the control from the dock. You have to manually [code]queue_free()[/code] the control. + Removes the control from the dock. You have to manually [method Node.queue_free] the control. </description> </method> <method name="remove_custom_type"> diff --git a/doc/classes/EditorSceneImporter.xml b/doc/classes/EditorSceneImporter.xml index 4707543c91..96d8ce698d 100644 --- a/doc/classes/EditorSceneImporter.xml +++ b/doc/classes/EditorSceneImporter.xml @@ -1,6 +1,7 @@ <?xml version="1.0" encoding="UTF-8" ?> <class name="EditorSceneImporter" inherits="Reference" category="Core" version="3.2"> <brief_description> + Imports scenes from third-parties' 3D files. </brief_description> <description> </description> diff --git a/doc/classes/ProjectSettings.xml b/doc/classes/ProjectSettings.xml index 7ab29e67ae..2d6ab4f72c 100644 --- a/doc/classes/ProjectSettings.xml +++ b/doc/classes/ProjectSettings.xml @@ -825,7 +825,7 @@ <member name="rendering/quality/subsurface_scattering/weight_samples" type="bool" setter="" getter="" default="true"> Weight subsurface scattering samples. Helps to avoid reading samples from unrelated parts of the screen. </member> - <member name="rendering/quality/voxel_cone_tracing/high_quality" type="bool" setter="" getter="" default="true"> + <member name="rendering/quality/voxel_cone_tracing/high_quality" type="bool" setter="" getter="" default="false"> Use high-quality voxel cone tracing. This results in better-looking reflections, but is much more expensive on the GPU. </member> <member name="rendering/threads/thread_model" type="int" setter="" getter="" default="1"> diff --git a/doc/classes/SpatialMaterial.xml b/doc/classes/SpatialMaterial.xml index f684f09263..df315d7430 100644 --- a/doc/classes/SpatialMaterial.xml +++ b/doc/classes/SpatialMaterial.xml @@ -192,7 +192,7 @@ </member> <member name="metallic_texture" type="Texture" setter="set_texture" getter="get_texture"> </member> - <member name="metallic_texture_channel" type="int" setter="set_metallic_texture_channel" getter="get_metallic_texture_channel" enum="SpatialMaterial.TextureChannel" default="2"> + <member name="metallic_texture_channel" type="int" setter="set_metallic_texture_channel" getter="get_metallic_texture_channel" enum="SpatialMaterial.TextureChannel" default="0"> </member> <member name="normal_enabled" type="bool" setter="set_feature" getter="get_feature" default="false"> If [code]true[/code], normal mapping is enabled. @@ -277,7 +277,7 @@ </member> <member name="roughness_texture" type="Texture" setter="set_texture" getter="get_texture"> </member> - <member name="roughness_texture_channel" type="int" setter="set_roughness_texture_channel" getter="get_roughness_texture_channel" enum="SpatialMaterial.TextureChannel" default="1"> + <member name="roughness_texture_channel" type="int" setter="set_roughness_texture_channel" getter="get_roughness_texture_channel" enum="SpatialMaterial.TextureChannel" default="0"> </member> <member name="subsurf_scatter_enabled" type="bool" setter="set_feature" getter="get_feature" default="false"> If [code]true[/code], subsurface scattering is enabled. Emulates light that penetrates an object's surface, is scattered, and then emerges. diff --git a/doc/classes/Variant.xml b/doc/classes/Variant.xml index eb07c70cc6..522e131b45 100644 --- a/doc/classes/Variant.xml +++ b/doc/classes/Variant.xml @@ -5,6 +5,19 @@ </brief_description> <description> A Variant takes up only 20 bytes and can store almost any engine datatype inside of it. Variants are rarely used to hold information for long periods of time. Instead, they are used mainly for communication, editing, serialization and moving data around. + A Variant: + - Can store almost any datatype. + - Can perform operations between many variants. GDScript uses Variant as its atomic/native datatype. + - Can be hashed, so it can be compared quickly to other variants. + - Can be used to convert safely between datatypes. + - Can be used to abstract calling methods and their arguments. Godot exports all its functions through variants. + - Can be used to defer calls or move data between threads. + - Can be serialized as binary and stored to disk, or transferred via network. + - Can be serialized to text and use it for printing values and editable settings. + - Can work as an exported property, so the editor can edit it universally. + - Can be used for dictionaries, arrays, parsers, etc. + [b]Containers ([Array] and [Dictionary]):[/b] Both are implemented using variants. A [Dictionary] can match any datatype used as key to any other datatype. An [Array] just holds an array of Variants. Of course, a Variant can also hold a [Dictionary] and an [Array] inside, making it even more flexible. + Modifications to a container will modify all references to it. A [Mutex] should be created to lock it if multi-threaded access is desired. </description> <tutorials> </tutorials> diff --git a/drivers/gles2/rasterizer_scene_gles2.cpp b/drivers/gles2/rasterizer_scene_gles2.cpp index aea89ddaa1..23b01b4e09 100644 --- a/drivers/gles2/rasterizer_scene_gles2.cpp +++ b/drivers/gles2/rasterizer_scene_gles2.cpp @@ -2287,19 +2287,6 @@ void RasterizerSceneGLES2::_render_render_list(RenderList::Element **p_elements, prev_unshaded = unshaded; } - bool depth_prepass = false; - - if (!p_alpha_pass && material->shader->spatial.depth_draw_mode == RasterizerStorageGLES2::Shader::Spatial::DEPTH_DRAW_ALPHA_PREPASS) { - depth_prepass = true; - } - - if (depth_prepass != prev_depth_prepass) { - - state.scene_shader.set_conditional(SceneShaderGLES2::USE_DEPTH_PREPASS, depth_prepass); - prev_depth_prepass = depth_prepass; - rebind = true; - } - bool base_pass = !accum_pass && !unshaded; //conditions for a base pass if (base_pass != prev_base_pass) { @@ -2434,6 +2421,19 @@ void RasterizerSceneGLES2::_render_render_list(RenderList::Element **p_elements, } } + bool depth_prepass = false; + + if (!p_alpha_pass && material->shader->spatial.depth_draw_mode == RasterizerStorageGLES2::Shader::Spatial::DEPTH_DRAW_ALPHA_PREPASS) { + depth_prepass = true; + } + + if (depth_prepass != prev_depth_prepass) { + + state.scene_shader.set_conditional(SceneShaderGLES2::USE_DEPTH_PREPASS, depth_prepass); + prev_depth_prepass = depth_prepass; + rebind = true; + } + bool instancing = e->instance->base_type == VS::INSTANCE_MULTIMESH; if (instancing != prev_instancing) { diff --git a/drivers/gles2/shaders/scene.glsl b/drivers/gles2/shaders/scene.glsl index ca222362e7..b7f8ec3ce9 100644 --- a/drivers/gles2/shaders/scene.glsl +++ b/drivers/gles2/shaders/scene.glsl @@ -1549,7 +1549,7 @@ FRAGMENT_SHADER_CODE #endif // ALPHA_SCISSOR_USED #ifdef USE_DEPTH_PREPASS - if (alpha < 0.99) { + if (alpha < 0.1) { discard; } #endif // USE_DEPTH_PREPASS @@ -2112,7 +2112,7 @@ FRAGMENT_SHADER_CODE #endif // ALPHA_SCISSOR_USED #ifdef USE_DEPTH_PREPASS - if (alpha < 0.99) { + if (alpha < 0.1) { discard; } #endif // USE_DEPTH_PREPASS diff --git a/drivers/gles3/rasterizer_scene_gles3.cpp b/drivers/gles3/rasterizer_scene_gles3.cpp index 812e3711c5..f3ba29a0e7 100644 --- a/drivers/gles3/rasterizer_scene_gles3.cpp +++ b/drivers/gles3/rasterizer_scene_gles3.cpp @@ -2365,7 +2365,7 @@ void RasterizerSceneGLES3::_add_geometry_with_material(RasterizerStorageGLES3::G if (p_depth_pass) { - if (has_blend_alpha || p_material->shader->spatial.uses_depth_texture || (has_base_alpha && p_material->shader->spatial.depth_draw_mode != RasterizerStorageGLES3::Shader::Spatial::DEPTH_DRAW_ALPHA_PREPASS) || p_material->shader->spatial.depth_draw_mode == RasterizerStorageGLES3::Shader::Spatial::DEPTH_DRAW_NEVER || p_material->shader->spatial.no_depth_test) + if (has_blend_alpha || p_material->shader->spatial.uses_depth_texture || (has_base_alpha && p_material->shader->spatial.depth_draw_mode != RasterizerStorageGLES3::Shader::Spatial::DEPTH_DRAW_ALPHA_PREPASS) || p_material->shader->spatial.depth_draw_mode == RasterizerStorageGLES3::Shader::Spatial::DEPTH_DRAW_NEVER || p_material->shader->spatial.no_depth_test || p_instance->cast_shadows == VS::SHADOW_CASTING_SETTING_OFF) return; //bye if (!p_material->shader->spatial.uses_alpha_scissor && !p_material->shader->spatial.writes_modelview_or_projection && !p_material->shader->spatial.uses_vertex && !p_material->shader->spatial.uses_discard && p_material->shader->spatial.depth_draw_mode != RasterizerStorageGLES3::Shader::Spatial::DEPTH_DRAW_ALPHA_PREPASS) { @@ -5296,7 +5296,7 @@ void RasterizerSceneGLES3::initialize() { GLOBAL_DEF("rendering/quality/subsurface_scattering/follow_surface", false); GLOBAL_DEF("rendering/quality/subsurface_scattering/weight_samples", true); - GLOBAL_DEF("rendering/quality/voxel_cone_tracing/high_quality", true); + GLOBAL_DEF("rendering/quality/voxel_cone_tracing/high_quality", false); } exposure_shrink_size = 243; diff --git a/drivers/unix/dir_access_unix.cpp b/drivers/unix/dir_access_unix.cpp index d5582d00ed..251bab5783 100644 --- a/drivers/unix/dir_access_unix.cpp +++ b/drivers/unix/dir_access_unix.cpp @@ -136,31 +136,27 @@ String DirAccessUnix::get_next() { return ""; } - String fname = fix_unicode_name(entry->d_name); - - if (entry->d_type == DT_UNKNOWN) { - //typedef struct stat Stat; - struct stat flags; - - String f = current_dir.plus_file(fname); + //typedef struct stat Stat; + struct stat flags; - if (stat(f.utf8().get_data(), &flags) == 0) { + String fname = fix_unicode_name(entry->d_name); - if (S_ISDIR(flags.st_mode)) { + String f = current_dir.plus_file(fname); - _cisdir = true; + if (stat(f.utf8().get_data(), &flags) == 0) { - } else { + if (S_ISDIR(flags.st_mode)) { - _cisdir = false; - } + _cisdir = true; } else { _cisdir = false; } + } else { - _cisdir = (entry->d_type == DT_DIR); + + _cisdir = false; } _cishidden = (fname != "." && fname != ".." && fname.begins_with(".")); diff --git a/editor/SCsub b/editor/SCsub index 7d48e47c9f..2b560f68e8 100644 --- a/editor/SCsub +++ b/editor/SCsub @@ -31,7 +31,7 @@ if env['tools']: reg_exporters_inc = '#include "register_exporters.h"\n' reg_exporters = 'void register_exporters() {\n' for e in env.platform_exporters: - env.editor_sources.append("#platform/" + e + "/export/export.cpp") + env.add_source_files(env.editor_sources, "#platform/" + e + "/export/export.cpp") reg_exporters += '\tregister_' + e + '_exporter();\n' reg_exporters_inc += '#include "platform/' + e + '/export/export.h"\n' reg_exporters += '}\n' diff --git a/editor/animation_track_editor.cpp b/editor/animation_track_editor.cpp index 0f86a32974..d1ac69c8d8 100644 --- a/editor/animation_track_editor.cpp +++ b/editor/animation_track_editor.cpp @@ -46,7 +46,7 @@ class AnimationTrackKeyEdit : public Object { public: bool setting; - bool _hide_object_properties_from_inspector() { + bool _hide_script_from_inspector() { return true; } @@ -58,7 +58,7 @@ public: ClassDB::bind_method("_update_obj", &AnimationTrackKeyEdit::_update_obj); ClassDB::bind_method("_key_ofs_changed", &AnimationTrackKeyEdit::_key_ofs_changed); - ClassDB::bind_method("_hide_object_properties_from_inspector", &AnimationTrackKeyEdit::_hide_object_properties_from_inspector); + ClassDB::bind_method("_hide_script_from_inspector", &AnimationTrackKeyEdit::_hide_script_from_inspector); ClassDB::bind_method("get_root_path", &AnimationTrackKeyEdit::get_root_path); ClassDB::bind_method("_dont_undo_redo", &AnimationTrackKeyEdit::_dont_undo_redo); } diff --git a/editor/editor_inspector.cpp b/editor/editor_inspector.cpp index 0f76aeb818..e4ddf44bc4 100644 --- a/editor/editor_inspector.cpp +++ b/editor/editor_inspector.cpp @@ -1550,7 +1550,7 @@ void EditorInspector::update_tree() { if (p.usage & PROPERTY_USAGE_HIGH_END_GFX && VS::get_singleton()->is_low_end()) continue; //do not show this property in low end gfx - if ((hide_object_properties || bool(object->call("_hide_object_properties_from_inspector"))) && (p.name == "script" || p.name == "__meta__")) { + if (p.name == "script" && (hide_script || bool(object->call("_hide_script_from_inspector")))) { continue; } @@ -1877,8 +1877,8 @@ void EditorInspector::set_use_doc_hints(bool p_enable) { use_doc_hints = p_enable; update_tree(); } -void EditorInspector::set_hide_object_properties(bool p_hide) { - hide_object_properties = p_hide; +void EditorInspector::set_hide_script(bool p_hide) { + hide_script = p_hide; update_tree(); } void EditorInspector::set_use_filter(bool p_use) { @@ -2318,7 +2318,7 @@ EditorInspector::EditorInspector() { set_enable_v_scroll(true); show_categories = false; - hide_object_properties = true; + hide_script = true; use_doc_hints = false; capitalize_paths = true; use_filter = false; diff --git a/editor/editor_inspector.h b/editor/editor_inspector.h index 028e1e4111..117a63699e 100644 --- a/editor/editor_inspector.h +++ b/editor/editor_inspector.h @@ -272,7 +272,7 @@ class EditorInspector : public ScrollContainer { LineEdit *search_box; bool show_categories; - bool hide_object_properties; + bool hide_script; bool use_doc_hints; bool capitalize_paths; bool use_filter; @@ -360,7 +360,7 @@ public: void set_show_categories(bool p_show); void set_use_doc_hints(bool p_enable); - void set_hide_object_properties(bool p_hide); + void set_hide_script(bool p_hide); void set_use_filter(bool p_use); void register_text_enter(Node *p_line_edit); diff --git a/editor/editor_node.cpp b/editor/editor_node.cpp index b59d42a5f7..f1944bf63f 100644 --- a/editor/editor_node.cpp +++ b/editor/editor_node.cpp @@ -2488,12 +2488,6 @@ void EditorNode::_menu_option_confirm(int p_option, bool p_confirmed) { screenshot_timer->start(); } break; - case EDITOR_OPEN_SCREENSHOT: { - - bool is_checked = settings_menu->get_popup()->is_item_checked(settings_menu->get_popup()->get_item_index(EDITOR_OPEN_SCREENSHOT)); - settings_menu->get_popup()->set_item_checked(settings_menu->get_popup()->get_item_index(EDITOR_OPEN_SCREENSHOT), !is_checked); - EditorSettings::get_singleton()->set_project_metadata("screenshot_options", "open_screenshot", !is_checked); - } break; case SETTINGS_PICK_MAIN_SCENE: { file->set_mode(EditorFileDialog::MODE_OPEN_FILE); @@ -2553,7 +2547,7 @@ void EditorNode::_screenshot(bool p_use_utc) { String name = "editor_screenshot_" + OS::get_singleton()->get_iso_date_time(p_use_utc).replace(":", "") + ".png"; NodePath path = String("user://") + name; _save_screenshot(path); - if (EditorSettings::get_singleton()->get_project_metadata("screenshot_options", "open_screenshot", true)) { + if (EditorSettings::get_singleton()->get("interface/editor/automatically_open_screenshots")) { OS::get_singleton()->shell_open(String("file://") + ProjectSettings::get_singleton()->globalize_path(path)); } } @@ -6000,16 +5994,13 @@ EditorNode::EditorNode() { p->add_child(editor_layouts); editor_layouts->connect("id_pressed", this, "_layout_menu_option"); p->add_submenu_item(TTR("Editor Layout"), "Layouts"); + p->add_separator(); #ifdef OSX_ENABLED p->add_shortcut(ED_SHORTCUT("editor/take_screenshot", TTR("Take Screenshot"), KEY_MASK_CMD | KEY_F12), EDITOR_SCREENSHOT); #else p->add_shortcut(ED_SHORTCUT("editor/take_screenshot", TTR("Take Screenshot"), KEY_MASK_CTRL | KEY_F12), EDITOR_SCREENSHOT); #endif p->set_item_tooltip(p->get_item_count() - 1, TTR("Screenshots are stored in the Editor Data/Settings Folder.")); - p->add_check_shortcut(ED_SHORTCUT("editor/open_screenshot", TTR("Automatically Open Screenshots")), EDITOR_OPEN_SCREENSHOT); - bool is_open_screenshot = EditorSettings::get_singleton()->get_project_metadata("screenshot_options", "open_screenshot", true); - p->set_item_checked(p->get_item_count() - 1, is_open_screenshot); - p->set_item_tooltip(p->get_item_count() - 1, TTR("Open in an external image editor.")); #ifdef OSX_ENABLED p->add_shortcut(ED_SHORTCUT("editor/fullscreen_mode", TTR("Toggle Fullscreen"), KEY_MASK_CMD | KEY_MASK_CTRL | KEY_F), SETTINGS_TOGGLE_FULLSCREEN); #else diff --git a/editor/editor_settings.cpp b/editor/editor_settings.cpp index 223ca7a108..b4c1fc3463 100644 --- a/editor/editor_settings.cpp +++ b/editor/editor_settings.cpp @@ -345,6 +345,7 @@ void EditorSettings::_load_defaults(Ref<ConfigFile> p_extra_config) { _initial_set("interface/editor/unfocused_low_processor_mode_sleep_usec", 50000); // 20 FPS hints["interface/editor/unfocused_low_processor_mode_sleep_usec"] = PropertyInfo(Variant::REAL, "interface/editor/unfocused_low_processor_mode_sleep_usec", PROPERTY_HINT_RANGE, "1,100000,1", PROPERTY_USAGE_DEFAULT); _initial_set("interface/editor/separate_distraction_mode", false); + _initial_set("interface/editor/automatically_open_screenshots", true); _initial_set("interface/editor/hide_console_window", false); _initial_set("interface/editor/save_each_scene_on_quit", true); // Regression _initial_set("interface/editor/quit_confirmation", true); diff --git a/editor/import/editor_scene_importer_gltf.cpp b/editor/import/editor_scene_importer_gltf.cpp index bc507e91b2..8246e74814 100644 --- a/editor/import/editor_scene_importer_gltf.cpp +++ b/editor/import/editor_scene_importer_gltf.cpp @@ -1327,7 +1327,9 @@ Error EditorSceneImporterGLTF::_parse_materials(GLTFState &state) { if (bct.has("index")) { Ref<Texture> t = _get_texture(state, bct["index"]); material->set_texture(SpatialMaterial::TEXTURE_METALLIC, t); + material->set_metallic_texture_channel(SpatialMaterial::TEXTURE_CHANNEL_BLUE); material->set_texture(SpatialMaterial::TEXTURE_ROUGHNESS, t); + material->set_roughness_texture_channel(SpatialMaterial::TEXTURE_CHANNEL_GREEN); if (!mr.has("metallicFactor")) { material->set_metallic(1); } @@ -1352,6 +1354,7 @@ Error EditorSceneImporterGLTF::_parse_materials(GLTFState &state) { Dictionary bct = d["occlusionTexture"]; if (bct.has("index")) { material->set_texture(SpatialMaterial::TEXTURE_AMBIENT_OCCLUSION, _get_texture(state, bct["index"])); + material->set_ao_texture_channel(SpatialMaterial::TEXTURE_CHANNEL_RED); material->set_feature(SpatialMaterial::FEATURE_AMBIENT_OCCLUSION, true); } } diff --git a/editor/inspector_dock.cpp b/editor/inspector_dock.cpp index 08df780232..8a0812973f 100644 --- a/editor/inspector_dock.cpp +++ b/editor/inspector_dock.cpp @@ -597,7 +597,7 @@ InspectorDock::InspectorDock(EditorNode *p_editor, EditorData &p_editor_data) { inspector->set_show_categories(true); inspector->set_v_size_flags(Control::SIZE_EXPAND_FILL); inspector->set_use_doc_hints(true); - inspector->set_hide_object_properties(false); + inspector->set_hide_script(false); inspector->set_enable_capitalize_paths(bool(EDITOR_GET("interface/inspector/capitalize_properties"))); inspector->set_use_folding(!bool(EDITOR_GET("interface/inspector/disable_folding"))); inspector->register_text_enter(search); diff --git a/editor/plugins/canvas_item_editor_plugin.cpp b/editor/plugins/canvas_item_editor_plugin.cpp index 2078a8b053..19199f37ef 100644 --- a/editor/plugins/canvas_item_editor_plugin.cpp +++ b/editor/plugins/canvas_item_editor_plugin.cpp @@ -1061,9 +1061,11 @@ bool CanvasItemEditor::_gui_input_rulers_and_guides(const Ref<InputEvent> &p_eve bool CanvasItemEditor::_gui_input_zoom_or_pan(const Ref<InputEvent> &p_event) { Ref<InputEventMouseButton> b = p_event; if (b.is_valid()) { + bool pan_on_scroll = bool(EditorSettings::get_singleton()->get("editors/2d/scroll_to_pan")) && !b->get_control(); + if (b->is_pressed() && b->get_button_index() == BUTTON_WHEEL_DOWN) { // Scroll or pan down - if (bool(EditorSettings::get_singleton()->get("editors/2d/scroll_to_pan"))) { + if (pan_on_scroll) { view_offset.y += int(EditorSettings::get_singleton()->get("editors/2d/pan_speed")) / zoom * b->get_factor(); update_viewport(); } else { @@ -1074,7 +1076,7 @@ bool CanvasItemEditor::_gui_input_zoom_or_pan(const Ref<InputEvent> &p_event) { if (b->is_pressed() && b->get_button_index() == BUTTON_WHEEL_UP) { // Scroll or pan up - if (bool(EditorSettings::get_singleton()->get("editors/2d/scroll_to_pan"))) { + if (pan_on_scroll) { view_offset.y -= int(EditorSettings::get_singleton()->get("editors/2d/pan_speed")) / zoom * b->get_factor(); update_viewport(); } else { @@ -1085,7 +1087,7 @@ bool CanvasItemEditor::_gui_input_zoom_or_pan(const Ref<InputEvent> &p_event) { if (b->is_pressed() && b->get_button_index() == BUTTON_WHEEL_LEFT) { // Pan left - if (bool(EditorSettings::get_singleton()->get("editors/2d/scroll_to_pan"))) { + if (pan_on_scroll) { view_offset.x -= int(EditorSettings::get_singleton()->get("editors/2d/pan_speed")) / zoom * b->get_factor(); update_viewport(); return true; @@ -1094,7 +1096,7 @@ bool CanvasItemEditor::_gui_input_zoom_or_pan(const Ref<InputEvent> &p_event) { if (b->is_pressed() && b->get_button_index() == BUTTON_WHEEL_RIGHT) { // Pan right - if (bool(EditorSettings::get_singleton()->get("editors/2d/scroll_to_pan"))) { + if (pan_on_scroll) { view_offset.x += int(EditorSettings::get_singleton()->get("editors/2d/pan_speed")) / zoom * b->get_factor(); update_viewport(); return true; @@ -1104,6 +1106,7 @@ bool CanvasItemEditor::_gui_input_zoom_or_pan(const Ref<InputEvent> &p_event) { if (!panning) { if (b->is_pressed() && (b->get_button_index() == BUTTON_MIDDLE || + b->get_button_index() == BUTTON_RIGHT || (b->get_button_index() == BUTTON_LEFT && tool == TOOL_PAN) || (b->get_button_index() == BUTTON_LEFT && !EditorSettings::get_singleton()->get("editors/2d/simple_panning") && pan_pressed))) { // Pan the viewport diff --git a/editor/plugins/tile_set_editor_plugin.cpp b/editor/plugins/tile_set_editor_plugin.cpp index 5e87016171..f135becf5f 100644 --- a/editor/plugins/tile_set_editor_plugin.cpp +++ b/editor/plugins/tile_set_editor_plugin.cpp @@ -3369,7 +3369,7 @@ void TilesetEditorContext::_get_property_list(List<PropertyInfo> *p_list) const void TilesetEditorContext::_bind_methods() { - ClassDB::bind_method("_hide_object_properties_from_inspector", &TilesetEditorContext::_hide_object_properties_from_inspector); + ClassDB::bind_method("_hide_script_from_inspector", &TilesetEditorContext::_hide_script_from_inspector); } TilesetEditorContext::TilesetEditorContext(TileSetEditor *p_tileset_editor) { diff --git a/editor/plugins/tile_set_editor_plugin.h b/editor/plugins/tile_set_editor_plugin.h index 946c042b02..69ad8205a4 100644 --- a/editor/plugins/tile_set_editor_plugin.h +++ b/editor/plugins/tile_set_editor_plugin.h @@ -252,7 +252,7 @@ class TilesetEditorContext : public Object { bool snap_options_visible; public: - bool _hide_object_properties_from_inspector() { return true; } + bool _hide_script_from_inspector() { return true; } void set_tileset(const Ref<TileSet> &p_tileset); private: diff --git a/editor/project_manager.cpp b/editor/project_manager.cpp index e013aae164..48a587d4db 100644 --- a/editor/project_manager.cpp +++ b/editor/project_manager.cpp @@ -52,6 +52,10 @@ #include "scene/gui/texture_rect.h" #include "scene/gui/tool_button.h" +static inline String get_project_key_from_path(const String &dir) { + return dir.replace("/", "::"); +} + class ProjectDialog : public ConfirmationDialog { GDCLASS(ProjectDialog, ConfirmationDialog); @@ -606,7 +610,7 @@ private: dir = dir.replace("\\", "/"); if (dir.ends_with("/")) dir = dir.substr(0, dir.length() - 1); - String proj = dir.replace("/", "::"); + String proj = get_project_key_from_path(dir); EditorSettings::get_singleton()->set("projects/" + proj, dir); EditorSettings::get_singleton()->save(); @@ -918,596 +922,960 @@ public: } }; -struct ProjectItem { - String project; - String project_name; - String path; - String conf; - String icon; - String main_scene; - uint64_t last_modified; - bool favorite; - bool grayed; - ProjectListFilter::FilterOption filter_order_option; - ProjectItem() {} - ProjectItem(const String &p_project, const String &p_name, const String &p_path, const String &p_conf, const String &p_icon, const String &p_main_scene, uint64_t p_last_modified, bool p_favorite = false, bool p_grayed = false, const ProjectListFilter::FilterOption p_filter_order_option = ProjectListFilter::FILTER_NAME) { - project = p_project; - project_name = p_name; - path = p_path; - conf = p_conf; - icon = p_icon; - main_scene = p_main_scene; - last_modified = p_last_modified; - favorite = p_favorite; - grayed = p_grayed; - filter_order_option = p_filter_order_option; - } - _FORCE_INLINE_ bool operator<(const ProjectItem &l) const { - switch (filter_order_option) { +class ProjectListItemControl : public HBoxContainer { + GDCLASS(ProjectListItemControl, HBoxContainer) +public: + TextureButton *favorite_button; + TextureRect *icon; + bool icon_needs_reload; + + ProjectListItemControl() { + favorite_button = NULL; + icon = NULL; + icon_needs_reload = true; + } + + void set_is_favorite(bool fav) { + favorite_button->set_modulate(fav ? Color(1, 1, 1, 1) : Color(1, 1, 1, 0.2)); + } +}; + +class ProjectList : public ScrollContainer { + GDCLASS(ProjectList, ScrollContainer) +public: + static const char *SIGNAL_SELECTION_CHANGED; + static const char *SIGNAL_PROJECT_ASK_OPEN; + + // Can often be passed by copy + struct Item { + String project_key; + String project_name; + String path; + String icon; + String main_scene; + uint64_t last_modified; + bool favorite; + bool grayed; + bool missing; + int version; + + ProjectListItemControl *control; + + Item() {} + + Item(const String &p_project, + const String &p_name, + const String &p_path, + const String &p_icon, + const String &p_main_scene, + uint64_t p_last_modified, + bool p_favorite, + bool p_grayed, + bool p_missing, + int p_version) { + + project_key = p_project; + project_name = p_name; + path = p_path; + icon = p_icon; + main_scene = p_main_scene; + last_modified = p_last_modified; + favorite = p_favorite; + grayed = p_grayed; + missing = p_missing; + version = p_version; + control = NULL; + } + + _FORCE_INLINE_ bool operator==(const Item &l) const { + return project_key == l.project_key; + } + }; + + ProjectList(); + ~ProjectList(); + + void load_projects(); + void set_search_term(String p_search_term); + void set_filter_option(ProjectListFilter::FilterOption p_option); + void set_order_option(ProjectListFilter::FilterOption p_option); + void sort_projects(); + int get_project_count() const; + void select_project(int p_index); + void erase_selected_projects(); + Vector<Item> get_selected_projects() const; + const Set<String> &get_selected_project_keys() const; + void ensure_project_visible(int p_index); + int get_single_selected_index() const; + bool is_any_project_missing() const; + void erase_missing_projects(); + int refresh_project(const String &dir_path); + +private: + static void _bind_methods(); + void _notification(int p_what); + + void _panel_draw(Node *p_hb); + void _panel_input(const Ref<InputEvent> &p_ev, Node *p_hb); + void _favorite_pressed(Node *p_hb); + void _show_project(const String &p_path); + + void select_range(int p_begin, int p_end); + void toggle_select(int p_index); + void create_project_item_control(int p_index); + void remove_project(int p_index, bool p_update_settings); + void update_icons_async(); + void load_project_icon(int p_index); + + static void load_project_data(const String &p_property_key, Item &p_item, bool p_favorite); + + String _search_term; + ProjectListFilter::FilterOption _filter_option; + ProjectListFilter::FilterOption _order_option; + Set<String> _selected_project_keys; + String _last_clicked; // Project key + VBoxContainer *_scroll_children; + int _icon_load_index; + + Vector<Item> _projects; +}; + +struct ProjectListComparator { + ProjectListFilter::FilterOption order_option; + + // operator< + _FORCE_INLINE_ bool operator()(const ProjectList::Item &a, const ProjectList::Item &b) const { + if (a.favorite && !b.favorite) { + return true; + } + if (b.favorite && !a.favorite) { + return false; + } + switch (order_option) { case ProjectListFilter::FILTER_PATH: - return project < l.project; + return a.project_key < b.project_key; case ProjectListFilter::FILTER_MODIFIED: - return last_modified > l.last_modified; + return a.last_modified > b.last_modified; default: - return project_name < l.project_name; + return a.project_name < b.project_name; } } - _FORCE_INLINE_ bool operator==(const ProjectItem &l) const { return project == l.project; } }; -void ProjectManager::_notification(int p_what) { +ProjectList::ProjectList() { + _filter_option = ProjectListFilter::FILTER_NAME; + _order_option = ProjectListFilter::FILTER_MODIFIED; - switch (p_what) { - case NOTIFICATION_ENTER_TREE: { + _scroll_children = memnew(VBoxContainer); + _scroll_children->set_h_size_flags(SIZE_EXPAND_FILL); + add_child(_scroll_children); - Engine::get_singleton()->set_editor_hint(false); - } break; - case NOTIFICATION_READY: { + _icon_load_index = 0; +} - if (scroll_children->get_child_count() == 0 && StreamPeerSSL::is_available()) - open_templates->popup_centered_minsize(); - } break; - case NOTIFICATION_VISIBILITY_CHANGED: { +ProjectList::~ProjectList() { +} - set_process_unhandled_input(is_visible_in_tree()); - } break; - case NOTIFICATION_WM_QUIT_REQUEST: { +void ProjectList::update_icons_async() { + _icon_load_index = 0; + set_process(true); +} - _dim_window(); - } break; +void ProjectList::_notification(int p_what) { + if (p_what == NOTIFICATION_PROCESS) { + + // Load icons as a coroutine to speed up launch when you have hundreds of projects + if (_icon_load_index < _projects.size()) { + Item &item = _projects.write[_icon_load_index]; + if (item.control->icon_needs_reload) { + load_project_icon(_icon_load_index); + } + _icon_load_index++; + + } else { + set_process(false); + } } } -void ProjectManager::_dim_window() { - - // This method must be called before calling `get_tree()->quit()`. - // Otherwise, its effect won't be visible +void ProjectList::load_project_icon(int p_index) { + Item &item = _projects.write[p_index]; + + Ref<Texture> default_icon = get_icon("DefaultProjectIcon", "EditorIcons"); + Ref<Texture> icon; + if (item.icon != "") { + Ref<Image> img; + img.instance(); + Error err = img->load(item.icon.replace_first("res://", item.path + "/")); + if (err == OK) { + + img->resize(default_icon->get_width(), default_icon->get_height()); + Ref<ImageTexture> it = memnew(ImageTexture); + it->create_from_image(img); + icon = it; + } + } + if (icon.is_null()) { + icon = default_icon; + } - // Dim the project manager window while it's quitting to make it clearer that it's busy. - // No transition is applied, as the effect needs to be visible immediately - float c = 1.0f - float(EDITOR_GET("interface/editor/dim_amount")); - Color dim_color = Color(c, c, c); - gui_base->set_modulate(dim_color); + item.control->icon->set_texture(icon); + item.control->icon_needs_reload = false; } -void ProjectManager::_panel_draw(Node *p_hb) { +void ProjectList::load_project_data(const String &p_property_key, Item &p_item, bool p_favorite) { - HBoxContainer *hb = Object::cast_to<HBoxContainer>(p_hb); + String path = EditorSettings::get_singleton()->get(p_property_key); + String conf = path.plus_file("project.godot"); + bool grayed = false; + bool missing = false; - hb->draw_line(Point2(0, hb->get_size().y + 1), Point2(hb->get_size().x - 10, hb->get_size().y + 1), get_color("guide_color", "Tree")); + Ref<ConfigFile> cf = memnew(ConfigFile); + Error cf_err = cf->load(conf); - if (selected_list.has(hb->get_meta("name"))) { - hb->draw_style_box(gui_base->get_stylebox("selected", "Tree"), Rect2(Point2(), hb->get_size() - Size2(10, 0) * EDSCALE)); + int config_version = 0; + String project_name = TTR("Unnamed Project"); + if (cf_err == OK) { + String cf_project_name = static_cast<String>(cf->get_value("application", "config/name", "")); + if (cf_project_name != "") + project_name = cf_project_name.xml_unescape(); + config_version = (int)cf->get_value("", "config_version", 0); } + + if (config_version > ProjectSettings::CONFIG_VERSION) { + // Comes from an incompatible (more recent) Godot version, grey it out + grayed = true; + } + + String icon = cf->get_value("application", "config/icon", ""); + String main_scene = cf->get_value("application", "run/main_scene", ""); + + uint64_t last_modified = 0; + if (FileAccess::exists(conf)) { + last_modified = FileAccess::get_modified_time(conf); + + String fscache = path.plus_file(".fscache"); + if (FileAccess::exists(fscache)) { + uint64_t cache_modified = FileAccess::get_modified_time(fscache); + if (cache_modified > last_modified) + last_modified = cache_modified; + } + } else { + grayed = true; + missing = true; + print_line("Project is missing: " + conf); + } + + String project_key = p_property_key.get_slice("/", 1); + + p_item = Item(project_key, project_name, path, icon, main_scene, last_modified, p_favorite, grayed, missing, config_version); } -void ProjectManager::_update_project_buttons() { - for (int i = 0; i < scroll_children->get_child_count(); i++) { +void ProjectList::load_projects() { + // This is a full, hard reload of the list. Don't call this unless really required, it's expensive. + // If you have 150 projects, it may read through 150 files on your disk at once + load 150 icons. - CanvasItem *item = Object::cast_to<CanvasItem>(scroll_children->get_child(i)); - item->update(); + // Clear whole list + for (int i = 0; i < _projects.size(); ++i) { + Item &project = _projects.write[i]; + CRASH_COND(project.control == NULL); + memdelete(project.control); // Why not queue_free()? } + _projects.clear(); + _last_clicked = ""; + _selected_project_keys.clear(); - bool empty_selection = selected_list.empty(); - erase_btn->set_disabled(empty_selection); - open_btn->set_disabled(empty_selection); - rename_btn->set_disabled(empty_selection); - run_btn->set_disabled(empty_selection); + // Load data + // TODO Would be nice to change how projects and favourites are stored... it complicates things a bit. + // Use a dictionary associating project path to metadata (like is_favorite). + + List<PropertyInfo> properties; + EditorSettings::get_singleton()->get_property_list(&properties); - bool missing_projects = false; - Map<String, String> list_all_projects; - for (int i = 0; i < scroll_children->get_child_count(); i++) { - HBoxContainer *hb = Object::cast_to<HBoxContainer>(scroll_children->get_child(i)); - if (hb) { - list_all_projects.insert(hb->get_meta("name"), hb->get_meta("main_scene")); + Set<String> favorites; + // Find favourites... + for (List<PropertyInfo>::Element *E = properties.front(); E; E = E->next()) { + String property_key = E->get().name; + if (property_key.begins_with("favorite_projects/")) { + favorites.insert(property_key); } } - for (Map<String, String>::Element *E = list_all_projects.front(); E; E = E->next()) { - String project_name = E->key().replace(":::", ":/").replace("::", "/") + "/project.godot"; - if (!FileAccess::exists(project_name)) { - missing_projects = true; - break; - } + + for (List<PropertyInfo>::Element *E = properties.front(); E; E = E->next()) { + // This is actually something like "projects/C:::Documents::Godot::Projects::MyGame" + String property_key = E->get().name; + if (!property_key.begins_with("projects/")) + continue; + + String project_key = property_key.get_slice("/", 1); + bool favorite = favorites.has("favorite_projects/" + project_key); + + Item item; + load_project_data(property_key, item, favorite); + + _projects.push_back(item); + } + + // Create controls + for (int i = 0; i < _projects.size(); ++i) { + create_project_item_control(i); } - erase_missing_btn->set_visible(missing_projects); + sort_projects(); + + set_v_scroll(0); + + update_icons_async(); } -void ProjectManager::_panel_input(const Ref<InputEvent> &p_ev, Node *p_hb) { +void ProjectList::create_project_item_control(int p_index) { - Ref<InputEventMouseButton> mb = p_ev; + // Will be added last in the list, so make sure indexes match + ERR_FAIL_COND(p_index != _scroll_children->get_child_count()); - if (mb.is_valid() && mb->is_pressed() && mb->get_button_index() == BUTTON_LEFT) { + Item &item = _projects.write[p_index]; + ERR_FAIL_COND(item.control != NULL); // Already created - String clicked = p_hb->get_meta("name"); - String clicked_main_scene = p_hb->get_meta("main_scene"); + Ref<Texture> favorite_icon = get_icon("Favorites", "EditorIcons"); + Color font_color = get_color("font_color", "Tree"); + + ProjectListItemControl *hb = memnew(ProjectListItemControl); + hb->connect("draw", this, "_panel_draw", varray(hb)); + hb->connect("gui_input", this, "_panel_input", varray(hb)); + hb->add_constant_override("separation", 10 * EDSCALE); + + VBoxContainer *favorite_box = memnew(VBoxContainer); + favorite_box->set_name("FavoriteBox"); + TextureButton *favorite = memnew(TextureButton); + favorite->set_name("FavoriteButton"); + favorite->set_normal_texture(favorite_icon); + favorite->connect("pressed", this, "_favorite_pressed", varray(hb)); + favorite_box->add_child(favorite); + favorite_box->set_alignment(BoxContainer::ALIGN_CENTER); + hb->add_child(favorite_box); + hb->favorite_button = favorite; + hb->set_is_favorite(item.favorite); + + TextureRect *tf = memnew(TextureRect); + tf->set_texture(get_icon("DefaultProjectIcon", "EditorIcons")); + hb->add_child(tf); + hb->icon = tf; - if (mb->get_shift() && selected_list.size() > 0 && last_clicked != "" && clicked != last_clicked) { + VBoxContainer *vb = memnew(VBoxContainer); + if (item.grayed) + vb->set_modulate(Color(0.5, 0.5, 0.5)); + vb->set_h_size_flags(SIZE_EXPAND_FILL); + hb->add_child(vb); + Control *ec = memnew(Control); + ec->set_custom_minimum_size(Size2(0, 1)); + ec->set_mouse_filter(MOUSE_FILTER_PASS); + vb->add_child(ec); + Label *title = memnew(Label(item.project_name)); + title->add_font_override("font", get_font("title", "EditorFonts")); + title->add_color_override("font_color", font_color); + title->set_clip_text(true); + vb->add_child(title); + + HBoxContainer *path_hb = memnew(HBoxContainer); + path_hb->set_h_size_flags(SIZE_EXPAND_FILL); + vb->add_child(path_hb); + + Button *show = memnew(Button); + show->set_icon(get_icon("Load", "EditorIcons")); // Folder icon + show->set_flat(true); + show->set_modulate(Color(1, 1, 1, 0.5)); + path_hb->add_child(show); + show->connect("pressed", this, "_show_project", varray(item.path)); + show->set_tooltip(TTR("Show in File Manager")); + + Label *fpath = memnew(Label(item.path)); + path_hb->add_child(fpath); + fpath->set_h_size_flags(SIZE_EXPAND_FILL); + fpath->set_modulate(Color(1, 1, 1, 0.5)); + fpath->add_color_override("font_color", font_color); + fpath->set_clip_text(true); + + _scroll_children->add_child(hb); + item.control = hb; +} - int clicked_id = -1; - int last_clicked_id = -1; - for (int i = 0; i < scroll_children->get_child_count(); i++) { - HBoxContainer *hb = Object::cast_to<HBoxContainer>(scroll_children->get_child(i)); - if (!hb) continue; - if (hb->get_meta("name") == clicked) clicked_id = i; - if (hb->get_meta("name") == last_clicked) last_clicked_id = i; - } +void ProjectList::set_search_term(String p_search_term) { + _search_term = p_search_term; +} - if (last_clicked_id != -1 && clicked_id != -1) { - int min = clicked_id < last_clicked_id ? clicked_id : last_clicked_id; - int max = clicked_id > last_clicked_id ? clicked_id : last_clicked_id; - for (int i = 0; i < scroll_children->get_child_count(); ++i) { - HBoxContainer *hb = Object::cast_to<HBoxContainer>(scroll_children->get_child(i)); - if (!hb) continue; - if (i != clicked_id && (i < min || i > max) && !mb->get_control()) { - selected_list.erase(hb->get_meta("name")); - } else if (i >= min && i <= max) { - selected_list.insert(hb->get_meta("name"), hb->get_meta("main_scene")); - } - } - } +void ProjectList::set_filter_option(ProjectListFilter::FilterOption p_option) { + if (_filter_option != p_option) { + _filter_option = p_option; + } +} + +void ProjectList::set_order_option(ProjectListFilter::FilterOption p_option) { + if (_order_option != p_option) { + _order_option = p_option; + EditorSettings::get_singleton()->set("project_manager/sorting_order", (int)_filter_option); + EditorSettings::get_singleton()->save(); + } +} - } else if (selected_list.has(clicked) && mb->get_control()) { +void ProjectList::sort_projects() { - selected_list.erase(clicked); + SortArray<Item, ProjectListComparator> sorter; + sorter.compare.order_option = _order_option; + sorter.sort(_projects.ptrw(), _projects.size()); - } else { + for (int i = 0; i < _projects.size(); ++i) { + Item &item = _projects.write[i]; - last_clicked = clicked; - if (mb->get_control() || selected_list.size() == 0) { - selected_list.insert(clicked, clicked_main_scene); - } else { - selected_list.clear(); - selected_list.insert(clicked, clicked_main_scene); + bool visible = true; + if (_search_term != "") { + if (_filter_option == ProjectListFilter::FILTER_PATH) { + visible = item.path.findn(_search_term) != -1; + } else if (_filter_option == ProjectListFilter::FILTER_NAME) { + visible = item.project_name.findn(_search_term) != -1; } } - _update_project_buttons(); + item.control->set_visible(visible); + } - if (mb->is_doubleclick()) - _open_selected_projects_ask(); //open if doubleclicked + for (int i = 0; i < _projects.size(); ++i) { + Item &item = _projects.write[i]; + if (item.control->is_visible()) { + item.control->get_parent()->move_child(item.control, i); + } } + + // Rewind the coroutine because order of projects changed + update_icons_async(); } -void ProjectManager::_unhandled_input(const Ref<InputEvent> &p_ev) { +const Set<String> &ProjectList::get_selected_project_keys() const { + // Faster if that's all you need + return _selected_project_keys; +} - Ref<InputEventKey> k = p_ev; +Vector<ProjectList::Item> ProjectList::get_selected_projects() const { + Vector<Item> items; + if (_selected_project_keys.size() == 0) { + return items; + } + items.resize(_selected_project_keys.size()); + int j = 0; + for (int i = 0; i < _projects.size(); ++i) { + const Item &item = _projects[i]; + if (_selected_project_keys.has(item.project_key)) { + items.write[j++] = item; + } + } + ERR_FAIL_COND_V(j != items.size(), items); + return items; +} - if (k.is_valid()) { +void ProjectList::ensure_project_visible(int p_index) { + const Item &item = _projects[p_index]; - if (!k->is_pressed()) - return; + int item_top = item.control->get_position().y; + int item_bottom = item.control->get_position().y + item.control->get_size().y; - if (tabs->get_current_tab() != 0) - return; + if (item_top < get_v_scroll()) { + set_v_scroll(item_top); - bool scancode_handled = true; + } else if (item_bottom > get_v_scroll() + get_size().y) { + set_v_scroll(item_bottom - get_size().y); + } +} - switch (k->get_scancode()) { +int ProjectList::get_single_selected_index() const { + if (_selected_project_keys.size() == 0) { + // Default selection + return 0; + } + String key; + if (_selected_project_keys.size() == 1) { + // Only one selected + key = _selected_project_keys.front()->get(); + } else { + // Multiple selected, consider the last clicked one as "main" + key = _last_clicked; + } + for (int i = 0; i < _projects.size(); ++i) { + if (_projects[i].project_key == key) { + return i; + } + } + return 0; +} - case KEY_ENTER: { +void ProjectList::remove_project(int p_index, bool p_update_settings) { + const Item item = _projects[p_index]; // Take a copy - _open_selected_projects_ask(); - } break; - case KEY_DELETE: { + _selected_project_keys.erase(item.project_key); - _erase_project(); - } break; - case KEY_HOME: { + if (_last_clicked == item.project_key) { + _last_clicked = ""; + } - for (int i = 0; i < scroll_children->get_child_count(); i++) { + memdelete(item.control); + _projects.remove(p_index); - HBoxContainer *hb = Object::cast_to<HBoxContainer>(scroll_children->get_child(i)); - if (hb) { - selected_list.clear(); - selected_list.insert(hb->get_meta("name"), hb->get_meta("main_scene")); - scroll->set_v_scroll(0); - _update_project_buttons(); - break; - } - } + if (p_update_settings) { + EditorSettings::get_singleton()->erase("projects/" + item.project_key); + EditorSettings::get_singleton()->erase("favorite_projects/" + item.project_key); + // Not actually saving the file, in case you are doing more changes to settings + } +} - } break; - case KEY_END: { +bool ProjectList::is_any_project_missing() const { + for (int i = 0; i < _projects.size(); ++i) { + if (_projects[i].missing) { + return true; + } + } + return false; +} - for (int i = scroll_children->get_child_count() - 1; i >= 0; i--) { +void ProjectList::erase_missing_projects() { - HBoxContainer *hb = Object::cast_to<HBoxContainer>(scroll_children->get_child(i)); - if (hb) { - selected_list.clear(); - selected_list.insert(hb->get_meta("name"), hb->get_meta("main_scene")); - scroll->set_v_scroll(scroll_children->get_size().y); - _update_project_buttons(); - break; - } - } + if (_projects.empty()) { + return; + } - } break; - case KEY_UP: { + int deleted_count = 0; + int remaining_count = 0; - if (k->get_shift()) - break; + for (int i = 0; i < _projects.size(); ++i) { + const Item &item = _projects[i]; + + if (item.missing) { + remove_project(i, true); + --i; + ++deleted_count; - if (selected_list.size()) { + } else { + ++remaining_count; + } + } - bool found = false; + print_line("Removed " + itos(deleted_count) + " projects from the list, remaining " + itos(remaining_count) + " projects"); - for (int i = scroll_children->get_child_count() - 1; i >= 0; i--) { + EditorSettings::get_singleton()->save(); +} - HBoxContainer *hb = Object::cast_to<HBoxContainer>(scroll_children->get_child(i)); - if (!hb) continue; +int ProjectList::refresh_project(const String &dir_path) { + // Reads editor settings and reloads information about a specific project. + // If it wasn't loaded and should be in the list, it is added (i.e new project). + // If it isn't in the list anymore, it is removed. + // If it is in the list but doesn't exist anymore, it is marked as missing. - String current = hb->get_meta("name"); + String project_key = get_project_key_from_path(dir_path); - if (found) { - selected_list.clear(); - selected_list.insert(current, hb->get_meta("main_scene")); + // Read project manager settings + bool is_favourite = false; + bool should_be_in_list = false; + String property_key = "projects/" + project_key; + { + List<PropertyInfo> properties; + EditorSettings::get_singleton()->get_property_list(&properties); + String favorite_property_key = "favorite_projects/" + project_key; + + bool found = false; + for (List<PropertyInfo>::Element *E = properties.front(); E; E = E->next()) { + String prop = E->get().name; + if (!found && prop == property_key) { + found = true; + } else if (!is_favourite && prop == favorite_property_key) { + is_favourite = true; + } + } - int offset_diff = scroll->get_v_scroll() - hb->get_position().y; + should_be_in_list = found; + } - if (offset_diff > 0) - scroll->set_v_scroll(scroll->get_v_scroll() - offset_diff); + bool was_selected = _selected_project_keys.has(project_key); - _update_project_buttons(); + // Remove item in any case + for (int i = 0; i < _projects.size(); ++i) { + const Item &existing_item = _projects[i]; + if (existing_item.path == dir_path) { + remove_project(i, false); + break; + } + } - break; + int index = -1; + if (should_be_in_list) { + // Recreate it with updated info - } else if (current == selected_list.back()->key()) { + Item item; + load_project_data(property_key, item, is_favourite); - found = true; - } - } + _projects.push_back(item); + create_project_item_control(_projects.size() - 1); - break; + sort_projects(); + + for (int i = 0; i < _projects.size(); ++i) { + if (_projects[i].project_key == project_key) { + if (was_selected) { + select_project(i); + ensure_project_visible(i); } - FALLTHROUGH; + load_project_icon(i); + index = i; + break; } - case KEY_DOWN: { + } + } - if (k->get_shift()) - break; + return index; +} - bool found = selected_list.empty(); +int ProjectList::get_project_count() const { + return _projects.size(); +} - for (int i = 0; i < scroll_children->get_child_count(); i++) { +void ProjectList::select_project(int p_index) { - HBoxContainer *hb = Object::cast_to<HBoxContainer>(scroll_children->get_child(i)); - if (!hb) continue; + Vector<Item> previous_selected_items = get_selected_projects(); + _selected_project_keys.clear(); - String current = hb->get_meta("name"); + for (int i = 0; i < previous_selected_items.size(); ++i) { + previous_selected_items[i].control->update(); + } - if (found) { - selected_list.clear(); - selected_list.insert(current, hb->get_meta("main_scene")); + toggle_select(p_index); +} - int last_y_visible = scroll->get_v_scroll() + scroll->get_size().y; - int offset_diff = (hb->get_position().y + hb->get_size().y) - last_y_visible; +inline void sort(int &a, int &b) { + if (a > b) { + int temp = a; + a = b; + b = temp; + } +} - if (offset_diff > 0) - scroll->set_v_scroll(scroll->get_v_scroll() + offset_diff); +void ProjectList::select_range(int p_begin, int p_end) { + sort(p_begin, p_end); + select_project(p_begin); + for (int i = p_begin + 1; i <= p_end; ++i) { + toggle_select(i); + } +} - _update_project_buttons(); +void ProjectList::toggle_select(int p_index) { + Item &item = _projects.write[p_index]; + if (_selected_project_keys.has(item.project_key)) { + _selected_project_keys.erase(item.project_key); + } else { + _selected_project_keys.insert(item.project_key); + } + item.control->update(); +} - break; +void ProjectList::erase_selected_projects() { - } else if (current == selected_list.back()->key()) { + if (_selected_project_keys.size() == 0) { + return; + } - found = true; - } + for (int i = 0; i < _projects.size(); ++i) { + Item &item = _projects.write[i]; + if (_selected_project_keys.has(item.project_key) && item.control->is_visible()) { + + EditorSettings::get_singleton()->erase("projects/" + item.project_key); + EditorSettings::get_singleton()->erase("favorite_projects/" + item.project_key); + + memdelete(item.control); + _projects.remove(i); + --i; + } + } + + EditorSettings::get_singleton()->save(); + + _selected_project_keys.clear(); + _last_clicked = ""; +} + +// Draws selected project highlight +void ProjectList::_panel_draw(Node *p_hb) { + Control *hb = Object::cast_to<Control>(p_hb); + + hb->draw_line(Point2(0, hb->get_size().y + 1), Point2(hb->get_size().x - 10, hb->get_size().y + 1), get_color("guide_color", "Tree")); + + String key = _projects[p_hb->get_index()].project_key; + + if (_selected_project_keys.has(key)) { + hb->draw_style_box(get_stylebox("selected", "Tree"), Rect2(Point2(), hb->get_size() - Size2(10, 0) * EDSCALE)); + } +} + +// Input for each item in the list +void ProjectList::_panel_input(const Ref<InputEvent> &p_ev, Node *p_hb) { + + Ref<InputEventMouseButton> mb = p_ev; + int clicked_index = p_hb->get_index(); + const Item &clicked_project = _projects[clicked_index]; + + if (mb.is_valid() && mb->is_pressed() && mb->get_button_index() == BUTTON_LEFT) { + + if (mb->get_shift() && _selected_project_keys.size() > 0 && _last_clicked != "" && clicked_project.project_key != _last_clicked) { + + int anchor_index = -1; + for (int i = 0; i < _projects.size(); ++i) { + const Item &p = _projects[i]; + if (p.project_key == _last_clicked) { + anchor_index = p.control->get_index(); + break; } + } + CRASH_COND(anchor_index == -1); + select_range(anchor_index, clicked_index); - } break; - case KEY_F: { - if (k->get_command()) - this->project_filter->search_box->grab_focus(); - else - scancode_handled = false; - } break; - default: { - scancode_handled = false; - } break; + } else if (mb->get_control()) { + toggle_select(clicked_index); + + } else { + _last_clicked = clicked_project.project_key; + select_project(clicked_index); } - if (scancode_handled) { - accept_event(); + emit_signal(SIGNAL_SELECTION_CHANGED); + + if (mb->is_doubleclick()) { + emit_signal(SIGNAL_PROJECT_ASK_OPEN); } } } -void ProjectManager::_favorite_pressed(Node *p_hb) { +void ProjectList::_favorite_pressed(Node *p_hb) { + + ProjectListItemControl *control = Object::cast_to<ProjectListItemControl>(p_hb); + + int index = control->get_index(); + Item item = _projects.write[index]; // Take copy - String clicked = p_hb->get_meta("name"); - bool favorite = !p_hb->get_meta("favorite"); - String proj = clicked.replace(":::", ":/"); - proj = proj.replace("::", "/"); + item.favorite = !item.favorite; - if (favorite) { - EditorSettings::get_singleton()->set("favorite_projects/" + clicked, proj); + if (item.favorite) { + EditorSettings::get_singleton()->set("favorite_projects/" + item.project_key, item.path); } else { - EditorSettings::get_singleton()->erase("favorite_projects/" + clicked); + EditorSettings::get_singleton()->erase("favorite_projects/" + item.project_key); } EditorSettings::get_singleton()->save(); - call_deferred("_load_recent_projects"); -} -void ProjectManager::_load_recent_projects() { + _projects.write[index] = item; + + control->set_is_favorite(item.favorite); - ProjectListFilter::FilterOption filter_option = project_filter->get_filter_option(); - String search_term = project_filter->get_search_term(); + sort_projects(); - while (scroll_children->get_child_count() > 0) { - memdelete(scroll_children->get_child(0)); + if (item.favorite) { + for (int i = 0; i < _projects.size(); ++i) { + if (_projects[i].project_key == item.project_key) { + ensure_project_visible(i); + break; + } + } } +} - Map<String, String> selected_list_copy = selected_list; +void ProjectList::_show_project(const String &p_path) { - List<PropertyInfo> properties; - EditorSettings::get_singleton()->get_property_list(&properties); + OS::get_singleton()->shell_open(String("file://") + p_path); +} - Color font_color = gui_base->get_color("font_color", "Tree"); +const char *ProjectList::SIGNAL_SELECTION_CHANGED = "selection_changed"; +const char *ProjectList::SIGNAL_PROJECT_ASK_OPEN = "project_ask_open"; - ProjectListFilter::FilterOption filter_order_option = project_order_filter->get_filter_option(); - EditorSettings::get_singleton()->set("project_manager/sorting_order", (int)filter_order_option); +void ProjectList::_bind_methods() { - List<ProjectItem> projects; - List<ProjectItem> favorite_projects; + ClassDB::bind_method("_panel_draw", &ProjectList::_panel_draw); + ClassDB::bind_method("_panel_input", &ProjectList::_panel_input); + ClassDB::bind_method("_favorite_pressed", &ProjectList::_favorite_pressed); + ClassDB::bind_method("_show_project", &ProjectList::_show_project); + //ClassDB::bind_method("_unhandled_input", &ProjectList::_unhandled_input); - for (List<PropertyInfo>::Element *E = properties.front(); E; E = E->next()) { + ADD_SIGNAL(MethodInfo(SIGNAL_SELECTION_CHANGED)); + ADD_SIGNAL(MethodInfo(SIGNAL_PROJECT_ASK_OPEN)); +} - String _name = E->get().name; - if (!_name.begins_with("projects/") && !_name.begins_with("favorite_projects/")) - continue; +void ProjectManager::_notification(int p_what) { - String path = EditorSettings::get_singleton()->get(_name); - if (filter_option == ProjectListFilter::FILTER_PATH && search_term != "" && path.findn(search_term) == -1) - continue; + switch (p_what) { + case NOTIFICATION_ENTER_TREE: { - String project = _name.get_slice("/", 1); - String conf = path.plus_file("project.godot"); - bool favorite = (_name.begins_with("favorite_projects/")) ? true : false; - bool grayed = false; + Engine::get_singleton()->set_editor_hint(false); + } break; + case NOTIFICATION_READY: { - Ref<ConfigFile> cf = memnew(ConfigFile); - Error cf_err = cf->load(conf); + if (_project_list->get_project_count() == 0 && StreamPeerSSL::is_available()) + open_templates->popup_centered_minsize(); + } break; + case NOTIFICATION_VISIBILITY_CHANGED: { - int config_version = 0; - String project_name = TTR("Unnamed Project"); - if (cf_err == OK) { + set_process_unhandled_input(is_visible_in_tree()); + } break; + case NOTIFICATION_WM_QUIT_REQUEST: { - String cf_project_name = static_cast<String>(cf->get_value("application", "config/name", "")); - if (cf_project_name != "") - project_name = cf_project_name.xml_unescape(); - config_version = (int)cf->get_value("", "config_version", 0); - } + _dim_window(); + } break; + } +} - if (config_version > ProjectSettings::CONFIG_VERSION) { - // Comes from an incompatible (more recent) Godot version, grey it out - grayed = true; - } +void ProjectManager::_dim_window() { - String icon = cf->get_value("application", "config/icon", ""); - String main_scene = cf->get_value("application", "run/main_scene", ""); + // This method must be called before calling `get_tree()->quit()`. + // Otherwise, its effect won't be visible - uint64_t last_modified = 0; - if (FileAccess::exists(conf)) { - last_modified = FileAccess::get_modified_time(conf); + // Dim the project manager window while it's quitting to make it clearer that it's busy. + // No transition is applied, as the effect needs to be visible immediately + float c = 1.0f - float(EDITOR_GET("interface/editor/dim_amount")); + Color dim_color = Color(c, c, c); + gui_base->set_modulate(dim_color); +} - String fscache = path.plus_file(".fscache"); - if (FileAccess::exists(fscache)) { - uint64_t cache_modified = FileAccess::get_modified_time(fscache); - if (cache_modified > last_modified) - last_modified = cache_modified; - } - } else { - grayed = true; - } +void ProjectManager::_update_project_buttons() { - ProjectItem item(project, project_name, path, conf, icon, main_scene, last_modified, favorite, grayed, filter_order_option); - if (favorite) - favorite_projects.push_back(item); - else - projects.push_back(item); - } - projects.sort(); - favorite_projects.sort(); + Vector<ProjectList::Item> selected_projects = _project_list->get_selected_projects(); + bool empty_selection = selected_projects.empty(); - for (List<ProjectItem>::Element *E = projects.front(); E;) { - List<ProjectItem>::Element *next = E->next(); - if (favorite_projects.find(E->get()) != NULL) - projects.erase(E->get()); - E = next; - } - for (List<ProjectItem>::Element *E = favorite_projects.back(); E; E = E->prev()) { - projects.push_front(E->get()); - } + erase_btn->set_disabled(empty_selection); + open_btn->set_disabled(empty_selection); + rename_btn->set_disabled(empty_selection); + run_btn->set_disabled(empty_selection); - Ref<Texture> favorite_icon = get_icon("Favorites", "EditorIcons"); + erase_missing_btn->set_visible(_project_list->is_any_project_missing()); +} - for (List<ProjectItem>::Element *E = projects.front(); E; E = E->next()) { +void ProjectManager::_unhandled_input(const Ref<InputEvent> &p_ev) { - ProjectItem &item = E->get(); - String project = item.project; - String path = item.path; - String conf = item.conf; + Ref<InputEventKey> k = p_ev; - if (filter_option == ProjectListFilter::FILTER_NAME && search_term != "" && item.project_name.findn(search_term) == -1) - continue; + if (k.is_valid()) { + + if (!k->is_pressed()) + return; + + if (tabs->get_current_tab() != 0) + return; + + bool scancode_handled = true; + + switch (k->get_scancode()) { - Ref<Texture> icon; + case KEY_ENTER: { - if (item.icon != "") { - Ref<Image> img; - img.instance(); - Error err = img->load(item.icon.replace_first("res://", path + "/")); - if (err == OK) { + _open_selected_projects_ask(); + } break; + case KEY_DELETE: { + + _erase_project(); + } break; + case KEY_HOME: { + + if (_project_list->get_project_count() > 0) { + _project_list->select_project(0); + _update_project_buttons(); + } + + } break; + case KEY_END: { - Ref<Texture> default_icon = get_icon("DefaultProjectIcon", "EditorIcons"); - img->resize(default_icon->get_width(), default_icon->get_height()); - Ref<ImageTexture> it = memnew(ImageTexture); - it->create_from_image(img); - icon = it; + if (_project_list->get_project_count() > 0) { + _project_list->select_project(_project_list->get_project_count() - 1); + _update_project_buttons(); + } + + } break; + case KEY_UP: { + + if (k->get_shift()) + break; + + int index = _project_list->get_single_selected_index(); + if (index - 1 > 0) { + _project_list->select_project(index - 1); + _project_list->ensure_project_visible(index - 1); + _update_project_buttons(); + } + + break; } + case KEY_DOWN: { + + if (k->get_shift()) + break; + + int index = _project_list->get_single_selected_index(); + if (index + 1 < _project_list->get_project_count()) { + _project_list->select_project(index + 1); + _project_list->ensure_project_visible(index + 1); + _update_project_buttons(); + } + + } break; + case KEY_F: { + if (k->get_command()) + this->project_filter->search_box->grab_focus(); + else + scancode_handled = false; + } break; + default: { + scancode_handled = false; + } break; } - if (icon.is_null()) { - icon = get_icon("DefaultProjectIcon", "EditorIcons"); + if (scancode_handled) { + accept_event(); } + } +} - selected_list_copy.erase(project); - - bool is_favorite = item.favorite; - bool is_grayed = item.grayed; - - HBoxContainer *hb = memnew(HBoxContainer); - hb->set_meta("name", project); - hb->set_meta("main_scene", item.main_scene); - hb->set_meta("favorite", is_favorite); - hb->connect("draw", this, "_panel_draw", varray(hb)); - hb->connect("gui_input", this, "_panel_input", varray(hb)); - hb->add_constant_override("separation", 10 * EDSCALE); - - VBoxContainer *favorite_box = memnew(VBoxContainer); - TextureButton *favorite = memnew(TextureButton); - favorite->set_normal_texture(favorite_icon); - if (!is_favorite) - favorite->set_modulate(Color(1, 1, 1, 0.2)); - favorite->connect("pressed", this, "_favorite_pressed", varray(hb)); - favorite_box->add_child(favorite); - favorite_box->set_alignment(BoxContainer::ALIGN_CENTER); - hb->add_child(favorite_box); - - TextureRect *tf = memnew(TextureRect); - tf->set_texture(icon); - hb->add_child(tf); +void ProjectManager::_load_recent_projects() { - VBoxContainer *vb = memnew(VBoxContainer); - if (is_grayed) - vb->set_modulate(Color(0.5, 0.5, 0.5)); - vb->set_name("project"); - vb->set_h_size_flags(SIZE_EXPAND_FILL); - hb->add_child(vb); - Control *ec = memnew(Control); - ec->set_custom_minimum_size(Size2(0, 1)); - ec->set_mouse_filter(MOUSE_FILTER_PASS); - vb->add_child(ec); - Label *title = memnew(Label(item.project_name)); - title->add_font_override("font", gui_base->get_font("title", "EditorFonts")); - title->add_color_override("font_color", font_color); - title->set_clip_text(true); - vb->add_child(title); - - HBoxContainer *path_hb = memnew(HBoxContainer); - path_hb->set_name("path_box"); - path_hb->set_h_size_flags(SIZE_EXPAND_FILL); - vb->add_child(path_hb); - - Button *show = memnew(Button); - show->set_name("show"); - show->set_icon(get_icon("Filesystem", "EditorIcons")); - show->set_flat(true); - show->set_modulate(Color(1, 1, 1, 0.5)); - path_hb->add_child(show); - show->connect("pressed", this, "_show_project", varray(path)); - show->set_tooltip(TTR("Show in File Manager")); - - Label *fpath = memnew(Label(path)); - fpath->set_name("path"); - path_hb->add_child(fpath); - fpath->set_h_size_flags(SIZE_EXPAND_FILL); - fpath->set_modulate(Color(1, 1, 1, 0.5)); - fpath->add_color_override("font_color", font_color); - fpath->set_clip_text(true); - - scroll_children->add_child(hb); - } - - for (Map<String, String>::Element *E = selected_list_copy.front(); E; E = E->next()) { - String key = E->key(); - selected_list.erase(key); - } - - scroll->set_v_scroll(0); + _project_list->set_filter_option(project_filter->get_filter_option()); + _project_list->set_order_option(project_order_filter->get_filter_option()); + _project_list->set_search_term(project_filter->get_search_term()); + _project_list->load_projects(); _update_project_buttons(); - EditorSettings::get_singleton()->save(); - tabs->set_current_tab(0); } void ProjectManager::_on_projects_updated() { - _load_recent_projects(); + Vector<ProjectList::Item> selected_projects = _project_list->get_selected_projects(); + int index = 0; + for (int i = 0; i < selected_projects.size(); ++i) { + index = _project_list->refresh_project(selected_projects[i].path); + } + if (index != -1) { + _project_list->ensure_project_visible(index); + } } void ProjectManager::_on_project_created(const String &dir) { project_filter->clear(); - bool has_already = false; - for (int i = 0; i < scroll_children->get_child_count(); i++) { - HBoxContainer *hb = Object::cast_to<HBoxContainer>(scroll_children->get_child(i)); - Label *fpath = Object::cast_to<Label>(hb->get_node(NodePath("project/path_box/path"))); - if (fpath->get_text() == dir) { - has_already = true; - break; - } - } - if (has_already) { - _update_scroll_position(dir); - } else { - _load_recent_projects(); - _update_scroll_position(dir); - } + int i = _project_list->refresh_project(dir); + _project_list->select_project(i); + _project_list->ensure_project_visible(i); _open_selected_projects_ask(); } -void ProjectManager::_update_scroll_position(const String &dir) { - for (int i = 0; i < scroll_children->get_child_count(); i++) { - HBoxContainer *hb = Object::cast_to<HBoxContainer>(scroll_children->get_child(i)); - Label *fpath = Object::cast_to<Label>(hb->get_node(NodePath("project/path_box/path"))); - if (fpath->get_text() == dir) { - last_clicked = hb->get_meta("name"); - selected_list.clear(); - selected_list.insert(hb->get_meta("name"), hb->get_meta("main_scene")); - _update_project_buttons(); - int last_y_visible = scroll->get_v_scroll() + scroll->get_size().y; - int offset_diff = (hb->get_position().y + hb->get_size().y) - last_y_visible; - - if (offset_diff > 0) - scroll->set_v_scroll(scroll->get_v_scroll() + offset_diff); - break; - } - } -} - void ProjectManager::_confirm_update_settings() { _open_selected_projects(); } void ProjectManager::_open_selected_projects() { - for (const Map<String, String>::Element *E = selected_list.front(); E; E = E->next()) { - const String &selected = E->key(); + const Set<String> &selected_list = _project_list->get_selected_project_keys(); + + for (const Set<String>::Element *E = selected_list.front(); E; E = E->next()) { + const String &selected = E->get(); String path = EditorSettings::get_singleton()->get("projects/" + selected); String conf = path.plus_file("project.godot"); + if (!FileAccess::exists(conf)) { dialog_error->set_text(vformat(TTR("Can't open project at '%s'."), path)); dialog_error->popup_centered_minsize(); @@ -1540,6 +1908,8 @@ void ProjectManager::_open_selected_projects() { void ProjectManager::_open_selected_projects_ask() { + const Set<String> &selected_list = _project_list->get_selected_project_keys(); + if (selected_list.size() < 1) { return; } @@ -1550,22 +1920,11 @@ void ProjectManager::_open_selected_projects_ask() { return; } - // Update the project settings or don't open - String path = EditorSettings::get_singleton()->get("projects/" + selected_list.front()->key()); - String conf = path.plus_file("project.godot"); + ProjectList::Item project = _project_list->get_selected_projects()[0]; - // FIXME: We already parse those in _load_recent_projects, we could instead make - // its `projects` list global and reuse its parsed metadata here. - Ref<ConfigFile> cf = memnew(ConfigFile); - Error cf_err = cf->load(conf); - - if (cf_err != OK) { - dialog_error->set_text(vformat(TTR("Can't open project at '%s'."), path)); - dialog_error->popup_centered_minsize(); - return; - } - - int config_version = (int)cf->get_value("", "config_version", 0); + // Update the project settings or don't open + String conf = project.path.plus_file("project.godot"); + int config_version = project.version; // Check if the config_version property was empty or 0 if (config_version == 0) { @@ -1581,7 +1940,7 @@ void ProjectManager::_open_selected_projects_ask() { } // Check if the file was generated by a newer, incompatible engine version if (config_version > ProjectSettings::CONFIG_VERSION) { - dialog_error->set_text(vformat(TTR("Can't open project at '%s'.") + "\n" + TTR("The project settings were created by a newer engine version, whose settings are not compatible with this version."), path)); + dialog_error->set_text(vformat(TTR("Can't open project at '%s'.") + "\n" + TTR("The project settings were created by a newer engine version, whose settings are not compatible with this version."), project.path)); dialog_error->popup_centered_minsize(); return; } @@ -1592,16 +1951,18 @@ void ProjectManager::_open_selected_projects_ask() { void ProjectManager::_run_project_confirm() { - for (Map<String, String>::Element *E = selected_list.front(); E; E = E->next()) { + Vector<ProjectList::Item> selected_list = _project_list->get_selected_projects(); + + for (int i = 0; i < selected_list.size(); ++i) { - const String &selected_main = E->get(); + const String &selected_main = selected_list[i].main_scene; if (selected_main == "") { run_error_diag->set_text(TTR("Can't run project: no main scene defined.\nPlease edit the project and set the main scene in the Project Settings under the \"Application\" category.")); run_error_diag->popup_centered(); return; } - const String &selected = E->key(); + const String &selected = selected_list[i].project_key; String path = EditorSettings::get_singleton()->get("projects/" + selected); if (!DirAccess::exists(path + "/.import")) { @@ -1629,8 +1990,11 @@ void ProjectManager::_run_project_confirm() { } } +// When you press the "Run" button void ProjectManager::_run_project() { + const Set<String> &selected_list = _project_list->get_selected_project_keys(); + if (selected_list.size() < 1) { return; } @@ -1643,11 +2007,6 @@ void ProjectManager::_run_project() { } } -void ProjectManager::_show_project(const String &p_path) { - - OS::get_singleton()->shell_open(String("file://") + p_path); -} - void ProjectManager::_scan_dir(const String &path, List<String> *r_projects) { DirAccess *da = DirAccess::create(DirAccess::ACCESS_FILESYSTEM); da->change_dir(path); @@ -1673,7 +2032,7 @@ void ProjectManager::_scan_begin(const String &p_base) { print_line("Found " + itos(projects.size()) + " projects."); for (List<String>::Element *E = projects.front(); E; E = E->next()) { - String proj = E->get().replace("/", "::"); + String proj = get_project_key_from_path(E->get()); EditorSettings::get_singleton()->set("projects/" + proj, E->get()); } EditorSettings::get_singleton()->save(); @@ -1699,12 +2058,14 @@ void ProjectManager::_import_project() { void ProjectManager::_rename_project() { + const Set<String> &selected_list = _project_list->get_selected_project_keys(); + if (selected_list.size() == 0) { return; } - for (Map<String, String>::Element *E = selected_list.front(); E; E = E->next()) { - const String &selected = E->key(); + for (Set<String>::Element *E = selected_list.front(); E; E = E->next()) { + const String &selected = E->get(); String path = EditorSettings::get_singleton()->get("projects/" + selected); npdialog->set_project_path(path); npdialog->set_mode(ProjectDialog::MODE_RENAME); @@ -1713,55 +2074,17 @@ void ProjectManager::_rename_project() { } void ProjectManager::_erase_project_confirm() { - - if (selected_list.size() == 0) { - return; - } - for (Map<String, String>::Element *E = selected_list.front(); E; E = E->next()) { - EditorSettings::get_singleton()->erase("projects/" + E->key()); - EditorSettings::get_singleton()->erase("favorite_projects/" + E->key()); - } - EditorSettings::get_singleton()->save(); - selected_list.clear(); - last_clicked = ""; - _load_recent_projects(); + _project_list->erase_selected_projects(); } void ProjectManager::_erase_missing_projects_confirm() { - - Map<String, String> list_all_projects; - for (int i = 0; i < scroll_children->get_child_count(); i++) { - HBoxContainer *hb = Object::cast_to<HBoxContainer>(scroll_children->get_child(i)); - if (hb) { - list_all_projects.insert(hb->get_meta("name"), hb->get_meta("main_scene")); - } - } - - if (list_all_projects.size() == 0) { - return; - } - - int deleted_projects = 0; - int remaining_projects = 0; - for (Map<String, String>::Element *E = list_all_projects.front(); E; E = E->next()) { - String project_name = E->key().replace(":::", ":/").replace("::", "/") + "/project.godot"; - if (!FileAccess::exists(project_name)) { - deleted_projects++; - EditorSettings::get_singleton()->erase("projects/" + E->key()); - EditorSettings::get_singleton()->erase("favorite_projects/" + E->key()); - } else { - remaining_projects++; - } - } - print_line("Deleted " + itos(deleted_projects) + " projects, remaining " + itos(remaining_projects) + " projects"); - EditorSettings::get_singleton()->save(); - selected_list.clear(); - last_clicked = ""; - _load_recent_projects(); + _project_list->erase_missing_projects(); } void ProjectManager::_erase_project() { + const Set<String> &selected_list = _project_list->get_selected_project_keys(); + if (selected_list.size() == 0) return; @@ -1867,13 +2190,23 @@ void ProjectManager::_scan_multiple_folders(PoolStringArray p_files) { } } +void ProjectManager::_on_order_option_changed() { + _project_list->set_order_option(project_order_filter->get_filter_option()); + _project_list->sort_projects(); +} + +void ProjectManager::_on_filter_option_changed() { + _project_list->set_filter_option(project_filter->get_filter_option()); + _project_list->set_search_term(project_filter->get_search_term()); + _project_list->sort_projects(); +} + void ProjectManager::_bind_methods() { ClassDB::bind_method("_open_selected_projects_ask", &ProjectManager::_open_selected_projects_ask); ClassDB::bind_method("_open_selected_projects", &ProjectManager::_open_selected_projects); ClassDB::bind_method("_run_project", &ProjectManager::_run_project); ClassDB::bind_method("_run_project_confirm", &ProjectManager::_run_project_confirm); - ClassDB::bind_method("_show_project", &ProjectManager::_show_project); ClassDB::bind_method("_scan_projects", &ProjectManager::_scan_projects); ClassDB::bind_method("_scan_begin", &ProjectManager::_scan_begin); ClassDB::bind_method("_import_project", &ProjectManager::_import_project); @@ -1886,18 +2219,16 @@ void ProjectManager::_bind_methods() { ClassDB::bind_method("_language_selected", &ProjectManager::_language_selected); ClassDB::bind_method("_restart_confirm", &ProjectManager::_restart_confirm); ClassDB::bind_method("_exit_dialog", &ProjectManager::_exit_dialog); - ClassDB::bind_method("_load_recent_projects", &ProjectManager::_load_recent_projects); + ClassDB::bind_method("_on_order_option_changed", &ProjectManager::_on_order_option_changed); + ClassDB::bind_method("_on_filter_option_changed", &ProjectManager::_on_filter_option_changed); ClassDB::bind_method("_on_projects_updated", &ProjectManager::_on_projects_updated); ClassDB::bind_method("_on_project_created", &ProjectManager::_on_project_created); - ClassDB::bind_method("_update_scroll_position", &ProjectManager::_update_scroll_position); - ClassDB::bind_method("_panel_draw", &ProjectManager::_panel_draw); - ClassDB::bind_method("_panel_input", &ProjectManager::_panel_input); ClassDB::bind_method("_unhandled_input", &ProjectManager::_unhandled_input); - ClassDB::bind_method("_favorite_pressed", &ProjectManager::_favorite_pressed); ClassDB::bind_method("_install_project", &ProjectManager::_install_project); ClassDB::bind_method("_files_dropped", &ProjectManager::_files_dropped); ClassDB::bind_method("_open_asset_library", &ProjectManager::_open_asset_library); ClassDB::bind_method("_confirm_update_settings", &ProjectManager::_confirm_update_settings); + ClassDB::bind_method("_update_project_buttons", &ProjectManager::_update_project_buttons); ClassDB::bind_method(D_METHOD("_scan_multiple_folders", "files"), &ProjectManager::_scan_multiple_folders); } @@ -1925,29 +2256,12 @@ ProjectManager::ProjectManager() { editor_set_scale(OS::get_singleton()->get_screen_dpi(screen) >= 192 && OS::get_singleton()->get_screen_size(screen).x > 2000 ? 2.0 : 1.0); } break; - case 1: { - editor_set_scale(0.75); - } break; - - case 2: { - editor_set_scale(1.0); - } break; - - case 3: { - editor_set_scale(1.25); - } break; - - case 4: { - editor_set_scale(1.5); - } break; - - case 5: { - editor_set_scale(1.75); - } break; - - case 6: { - editor_set_scale(2.0); - } break; + case 1: editor_set_scale(0.75); break; + case 2: editor_set_scale(1.0); break; + case 3: editor_set_scale(1.25); break; + case 4: editor_set_scale(1.5); break; + case 5: editor_set_scale(1.75); break; + case 6: editor_set_scale(2.0); break; default: { editor_set_scale(custom_display_scale); @@ -2030,7 +2344,7 @@ ProjectManager::ProjectManager() { project_order_filter->_setup_filters(sort_filter_titles); project_order_filter->set_filter_size(150); sort_filters->add_child(project_order_filter); - project_order_filter->connect("filter_changed", this, "_load_recent_projects"); + project_order_filter->connect("filter_changed", this, "_on_order_option_changed"); project_order_filter->set_custom_minimum_size(Size2(180, 10) * EDSCALE); int projects_sorting_order = (int)EditorSettings::get_singleton()->get("project_manager/sorting_order"); @@ -2049,7 +2363,7 @@ ProjectManager::ProjectManager() { project_filter->_setup_filters(vec2); project_filter->add_search_box(); search_filters->add_child(project_filter); - project_filter->connect("filter_changed", this, "_load_recent_projects"); + project_filter->connect("filter_changed", this, "_on_filter_option_changed"); project_filter->set_custom_minimum_size(Size2(280, 10) * EDSCALE); sort_filters->add_child(search_filters); @@ -2060,15 +2374,14 @@ ProjectManager::ProjectManager() { search_tree_vb->add_child(pc); pc->set_v_size_flags(SIZE_EXPAND_FILL); - scroll = memnew(ScrollContainer); - pc->add_child(scroll); - scroll->set_enable_h_scroll(false); + _project_list = memnew(ProjectList); + _project_list->connect(ProjectList::SIGNAL_SELECTION_CHANGED, this, "_update_project_buttons"); + _project_list->connect(ProjectList::SIGNAL_PROJECT_ASK_OPEN, this, "_open_selected_projects_ask"); + pc->add_child(_project_list); + _project_list->set_enable_h_scroll(false); VBoxContainer *tree_vb = memnew(VBoxContainer); tree_hb->add_child(tree_vb); - scroll_children = memnew(VBoxContainer); - scroll_children->set_h_size_flags(SIZE_EXPAND_FILL); - scroll->add_child(scroll_children); Button *open = memnew(Button); open->set_text(TTR("Edit")); @@ -2238,14 +2551,13 @@ ProjectManager::ProjectManager() { npdialog->connect("projects_updated", this, "_on_projects_updated"); npdialog->connect("project_created", this, "_on_project_created"); + _load_recent_projects(); if (EditorSettings::get_singleton()->get("filesystem/directories/autoscan_project_path")) { _scan_begin(EditorSettings::get_singleton()->get("filesystem/directories/autoscan_project_path")); } - last_clicked = ""; - SceneTree::get_singleton()->connect("files_dropped", this, "_files_dropped"); run_error_diag = memnew(AcceptDialog); diff --git a/editor/project_manager.h b/editor/project_manager.h index d75d7164cc..4ccb99d6bd 100644 --- a/editor/project_manager.h +++ b/editor/project_manager.h @@ -39,6 +39,7 @@ #include "scene/gui/tree.h" class ProjectDialog; +class ProjectList; class ProjectListFilter; class ProjectManager : public Control { @@ -68,16 +69,13 @@ class ProjectManager : public Control { AcceptDialog *dialog_error; ProjectDialog *npdialog; - ScrollContainer *scroll; - VBoxContainer *scroll_children; HBoxContainer *projects_hb; TabContainer *tabs; + ProjectList *_project_list; OptionButton *language_btn; Control *gui_base; - Map<String, String> selected_list; // name -> main_scene - String last_clicked; bool importing; void _open_asset_library(); @@ -86,7 +84,6 @@ class ProjectManager : public Control { void _run_project_confirm(); void _open_selected_projects(); void _open_selected_projects_ask(); - void _show_project(const String &p_path); void _import_project(); void _new_project(); void _rename_project(); @@ -111,13 +108,13 @@ class ProjectManager : public Control { void _install_project(const String &p_zip_path, const String &p_title); void _dim_window(); - void _panel_draw(Node *p_hb); - void _panel_input(const Ref<InputEvent> &p_ev, Node *p_hb); void _unhandled_input(const Ref<InputEvent> &p_ev); - void _favorite_pressed(Node *p_hb); void _files_dropped(PoolStringArray p_files, int p_screen); void _scan_multiple_folders(PoolStringArray p_files); + void _on_order_option_changed(); + void _on_filter_option_changed(); + protected: void _notification(int p_what); static void _bind_methods(); diff --git a/main/SCsub b/main/SCsub index e7fe6ab4e1..62bc155c67 100644 --- a/main/SCsub +++ b/main/SCsub @@ -6,6 +6,7 @@ from platform_methods import run_in_subprocess import main_builders env.main_sources = [] + env.add_source_files(env.main_sources, "*.cpp") # order matters here. higher index controller database files write on top of lower index database files @@ -14,7 +15,9 @@ controller_databases = ["#main/gamecontrollerdb.txt", "#main/gamecontrollerdb_20 env.Depends("#main/default_controller_mappings.gen.cpp", controller_databases) env.CommandNoCache("#main/default_controller_mappings.gen.cpp", controller_databases, run_in_subprocess(main_builders.make_default_controller_mappings)) -env.main_sources.append("#main/default_controller_mappings.gen.cpp") +# Don't warn about duplicate entry here, we need it registered manually for first build, +# even if later builds will pick it up twice due to above *.cpp globbing. +env.add_source_files(env.main_sources, "#main/default_controller_mappings.gen.cpp", warn_duplicates=False) env.Depends("#main/splash.gen.h", "#main/splash.png") env.CommandNoCache("#main/splash.gen.h", "#main/splash.png", run_in_subprocess(main_builders.make_splash)) diff --git a/methods.py b/methods.py index bb4adfb70b..7840fb1b64 100644 --- a/methods.py +++ b/methods.py @@ -8,14 +8,28 @@ import subprocess from compat import iteritems, isbasestring, decode_utf8 -def add_source_files(self, sources, filetype, lib_env=None, shared=False): - - if isbasestring(filetype): - dir_path = self.Dir('.').abspath - filetype = sorted(glob.glob(dir_path + "/" + filetype)) - - for path in filetype: - sources.append(self.Object(path)) +def add_source_files(self, sources, files, warn_duplicates=True): + # Convert string to list of absolute paths (including expanding wildcard) + if isbasestring(files): + # Keep SCons project-absolute path as they are (no wildcard support) + if files.startswith('#'): + if '*' in files: + print("ERROR: Wildcards can't be expanded in SCons project-absolute path: '{}'".format(files)) + return + files = [files] + else: + dir_path = self.Dir('.').abspath + files = sorted(glob.glob(dir_path + "/" + files)) + + # Add each path as compiled Object following environment (self) configuration + for path in files: + obj = self.Object(path) + if obj in sources: + if warn_duplicates: + print("WARNING: Object \"{}\" already included in environment sources.".format(obj)) + else: + continue + sources.append(obj) def disable_warnings(self): diff --git a/modules/SCsub b/modules/SCsub index 36c2472c42..42d89d6ce2 100644 --- a/modules/SCsub +++ b/modules/SCsub @@ -6,9 +6,9 @@ env_modules = env.Clone() Export('env_modules') -env.modules_sources = [ - "register_module_types.gen.cpp", -] +env.modules_sources = [] + +env_modules.add_source_files(env.modules_sources, "register_module_types.gen.cpp") for x in env.module_list: if (x in env.disabled_modules): diff --git a/modules/arkit/arkit_interface.mm b/modules/arkit/arkit_interface.mm index de58f93276..68844c54c2 100644 --- a/modules/arkit/arkit_interface.mm +++ b/modules/arkit/arkit_interface.mm @@ -430,7 +430,7 @@ void ARKitInterface::process() { // get some info about our screen and orientation Size2 screen_size = OS::get_singleton()->get_window_size(); - UIDeviceOrientation orientation = [[UIApplication sharedApplication] statusBarOrientation]; + UIInterfaceOrientation orientation = [[UIApplication sharedApplication] statusBarOrientation]; // Grab our camera image for our backbuffer CVPixelBufferRef pixelBuffer = current_frame.capturedImage; @@ -531,7 +531,7 @@ void ARKitInterface::process() { // we need to invert this, probably row v.s. column notation affine_transform = CGAffineTransformInvert(affine_transform); - if (orientation != UIDeviceOrientationPortrait) { + if (orientation != UIInterfaceOrientationPortrait) { affine_transform.b = -affine_transform.b; affine_transform.d = -affine_transform.d; affine_transform.ty = 1.0 - affine_transform.ty; @@ -582,28 +582,28 @@ void ARKitInterface::process() { // copy our current frame transform matrix_float4x4 m44 = camera.transform; - if (orientation == UIDeviceOrientationLandscapeLeft) { + if (orientation == UIInterfaceOrientationLandscapeLeft) { transform.basis.elements[0].x = m44.columns[0][0]; transform.basis.elements[1].x = m44.columns[0][1]; transform.basis.elements[2].x = m44.columns[0][2]; transform.basis.elements[0].y = m44.columns[1][0]; transform.basis.elements[1].y = m44.columns[1][1]; transform.basis.elements[2].y = m44.columns[1][2]; - } else if (orientation == UIDeviceOrientationPortrait) { + } else if (orientation == UIInterfaceOrientationPortrait) { transform.basis.elements[0].x = m44.columns[1][0]; transform.basis.elements[1].x = m44.columns[1][1]; transform.basis.elements[2].x = m44.columns[1][2]; transform.basis.elements[0].y = -m44.columns[0][0]; transform.basis.elements[1].y = -m44.columns[0][1]; transform.basis.elements[2].y = -m44.columns[0][2]; - } else if (orientation == UIDeviceOrientationLandscapeRight) { + } else if (orientation == UIInterfaceOrientationLandscapeRight) { transform.basis.elements[0].x = -m44.columns[0][0]; transform.basis.elements[1].x = -m44.columns[0][1]; transform.basis.elements[2].x = -m44.columns[0][2]; transform.basis.elements[0].y = -m44.columns[1][0]; transform.basis.elements[1].y = -m44.columns[1][1]; transform.basis.elements[2].y = -m44.columns[1][2]; - } else if (orientation == UIDeviceOrientationPortraitUpsideDown) { + } else if (orientation == UIInterfaceOrientationPortraitUpsideDown) { // this may not be correct transform.basis.elements[0].x = m44.columns[1][0]; transform.basis.elements[1].x = m44.columns[1][1]; diff --git a/modules/assimp/SCsub b/modules/assimp/SCsub index 8a77e4f803..d8ef866bec 100644 --- a/modules/assimp/SCsub +++ b/modules/assimp/SCsub @@ -72,11 +72,6 @@ env_assimp.Append(CPPDEFINES=['ASSIMP_BUILD_NO_X3D_IMPORTER']) env_assimp.Append(CPPDEFINES=['ASSIMP_BUILD_SINGLETHREADED']) -if (not env.msvc): - env_assimp.Append(CXXFLAGS=['-std=c++11']) -elif (env.msvc == False and env['platform'] == 'windows'): - env_assimp.Append(LDFLAGS=['-pthread']) - if(env['platform'] == 'windows'): env_assimp.Append(CPPDEFINES=['PLATFORM_WINDOWS']) env_assimp.Append(CPPDEFINES=[('PLATFORM', 'WINDOWS')]) diff --git a/modules/etc/SCsub b/modules/etc/SCsub index 532b97b006..1742d3534f 100644 --- a/modules/etc/SCsub +++ b/modules/etc/SCsub @@ -29,10 +29,6 @@ thirdparty_sources = [thirdparty_dir + file for file in thirdparty_sources] env_etc.Prepend(CPPPATH=[thirdparty_dir]) -# upstream uses c++11 -if not env.msvc: - env_etc.Append(CXXFLAGS="-std=c++11") - env_thirdparty = env_etc.Clone() env_thirdparty.disable_warnings() env_thirdparty.add_source_files(env.modules_sources, thirdparty_sources) diff --git a/modules/gdscript/doc_classes/@GDScript.xml b/modules/gdscript/doc_classes/@GDScript.xml index a21b6a2907..62a3794c6d 100644 --- a/modules/gdscript/doc_classes/@GDScript.xml +++ b/modules/gdscript/doc_classes/@GDScript.xml @@ -389,37 +389,6 @@ [/codeblock] </description> </method> - <method name="posmod"> - <return type="int"> - </return> - <argument index="0" name="a" type="int"> - </argument> - <argument index="1" name="b" type="int"> - </argument> - <description> - Returns the integer modulus of [code]a/b[/code] that wraps equally in positive and negative. - [codeblock] - var i = -6 - while i < 5: - prints(i, posmod(i, 3)) - i += 1 - [/codeblock] - Produces: - [codeblock] - -6 0 - -5 1 - -4 2 - -3 0 - -2 1 - -1 2 - 0 0 - 1 1 - 2 2 - 3 0 - 4 1 - [/codeblock] - </description> - </method> <method name="funcref"> <return type="FuncRef"> </return> @@ -604,6 +573,29 @@ [/codeblock] </description> </method> + <method name="lerp_angle"> + <return type="float"> + </return> + <argument index="0" name="from" type="float"> + </argument> + <argument index="1" name="to" type="float"> + </argument> + <argument index="2" name="weight" type="float"> + </argument> + <description> + Linearly interpolates between two angles (in radians) by a normalized value. + Similar to [method lerp] but interpolate correctly when the angles wrap around [constant @GDScript.TAU]. + [codeblock] + extends Sprite + var elapsed = 0.0 + func _process(delta): + var min_angle = deg2rad(0.0) + var max_angle = deg2rad(90.0) + rotation = lerp_angle(min_angle, max_angle, elapsed) + elapsed += delta + [/codeblock] + </description> + </method> <method name="linear2db"> <return type="float"> </return> @@ -730,12 +722,43 @@ Converts a 2D point expressed in the polar coordinate system (a distance from the origin [code]r[/code] and an angle [code]th[/code]) to the cartesian coordinate system (X and Y axis). </description> </method> + <method name="posmod"> + <return type="int"> + </return> + <argument index="0" name="a" type="int"> + </argument> + <argument index="1" name="b" type="int"> + </argument> + <description> + Returns the integer modulus of [code]a/b[/code] that wraps equally in positive and negative. + [codeblock] + var i = -6 + while i < 5: + prints(i, posmod(i, 3)) + i += 1 + [/codeblock] + Produces: + [codeblock] + -6 0 + -5 1 + -4 2 + -3 0 + -2 1 + -1 2 + 0 0 + 1 1 + 2 2 + 3 0 + 4 1 + [/codeblock] + </description> + </method> <method name="pow"> <return type="float"> </return> - <argument index="0" name="x" type="float"> + <argument index="0" name="base" type="float"> </argument> - <argument index="1" name="y" type="float"> + <argument index="1" name="exp" type="float"> </argument> <description> Returns the result of [code]x[/code] raised to the power of [code]y[/code]. diff --git a/modules/gdscript/gdscript_functions.cpp b/modules/gdscript/gdscript_functions.cpp index 8c33d6613c..f5f245b25f 100644 --- a/modules/gdscript/gdscript_functions.cpp +++ b/modules/gdscript/gdscript_functions.cpp @@ -76,6 +76,7 @@ const char *GDScriptFunctions::get_func_name(Function p_func) { "step_decimals", "stepify", "lerp", + "lerp_angle", "inverse_lerp", "range_lerp", "smoothstep", @@ -383,6 +384,13 @@ void GDScriptFunctions::call(Function p_func, const Variant **p_args, int p_arg_ } break; } } break; + case MATH_LERP_ANGLE: { + VALIDATE_ARG_COUNT(3); + VALIDATE_ARG_NUM(0); + VALIDATE_ARG_NUM(1); + VALIDATE_ARG_NUM(2); + r_ret = Math::lerp_angle((double)*p_args[0], (double)*p_args[1], (double)*p_args[2]); + } break; case MATH_INVERSE_LERP: { VALIDATE_ARG_COUNT(3); VALIDATE_ARG_NUM(0); @@ -1676,6 +1684,11 @@ MethodInfo GDScriptFunctions::get_info(Function p_func) { mi.return_val.usage |= PROPERTY_USAGE_NIL_IS_VARIANT; return mi; } break; + case MATH_LERP_ANGLE: { + MethodInfo mi("lerp_angle", PropertyInfo(Variant::REAL, "from"), PropertyInfo(Variant::REAL, "to"), PropertyInfo(Variant::REAL, "weight")); + mi.return_val.type = Variant::REAL; + return mi; + } break; case MATH_INVERSE_LERP: { MethodInfo mi("inverse_lerp", PropertyInfo(Variant::REAL, "from"), PropertyInfo(Variant::REAL, "to"), PropertyInfo(Variant::REAL, "weight")); mi.return_val.type = Variant::REAL; diff --git a/modules/gdscript/gdscript_functions.h b/modules/gdscript/gdscript_functions.h index d52b9118d3..8f7ba76d2c 100644 --- a/modules/gdscript/gdscript_functions.h +++ b/modules/gdscript/gdscript_functions.h @@ -67,6 +67,7 @@ public: MATH_STEP_DECIMALS, MATH_STEPIFY, MATH_LERP, + MATH_LERP_ANGLE, MATH_INVERSE_LERP, MATH_RANGE_LERP, MATH_SMOOTHSTEP, diff --git a/modules/mono/class_db_api_json.cpp b/modules/mono/class_db_api_json.cpp index 71ccdb7aab..4a6637434a 100644 --- a/modules/mono/class_db_api_json.cpp +++ b/modules/mono/class_db_api_json.cpp @@ -30,6 +30,8 @@ #include "class_db_api_json.h" +#ifdef DEBUG_METHODS_ENABLED + #include "core/io/json.h" #include "core/os/file_access.h" #include "core/project_settings.h" @@ -240,3 +242,5 @@ void class_db_api_to_json(const String &p_output_file, ClassDB::APIType p_api) { print_line(String() + "ClassDB API JSON written to: " + ProjectSettings::get_singleton()->globalize_path(p_output_file)); } + +#endif // DEBUG_METHODS_ENABLED diff --git a/modules/mono/class_db_api_json.h b/modules/mono/class_db_api_json.h index 0aa9c20930..ddfe2debea 100644 --- a/modules/mono/class_db_api_json.h +++ b/modules/mono/class_db_api_json.h @@ -31,9 +31,16 @@ #ifndef CLASS_DB_API_JSON_H #define CLASS_DB_API_JSON_H +// 'core/method_bind.h' defines DEBUG_METHODS_ENABLED, but it looks like we +// cannot include it here. That's why we include it through 'core/class_db.h'. #include "core/class_db.h" + +#ifdef DEBUG_METHODS_ENABLED + #include "core/ustring.h" void class_db_api_to_json(const String &p_output_file, ClassDB::APIType p_api); +#endif // DEBUG_METHODS_ENABLED + #endif // CLASS_DB_API_JSON_H diff --git a/modules/mono/glue/Managed/Files/Mathf.cs b/modules/mono/glue/Managed/Files/Mathf.cs index 2d8c63fe7f..6c1a51fcf9 100644 --- a/modules/mono/glue/Managed/Files/Mathf.cs +++ b/modules/mono/glue/Managed/Files/Mathf.cs @@ -185,6 +185,12 @@ namespace Godot return from + (to - from) * weight; } + public static real_t LerpAngle(real_t from, real_t to, real_t weight) { + real_t difference = (to - from) % Mathf.Tau; + real_t distance = ((2 * difference) % Mathf.Tau) - difference; + return from + distance * weight; + } + public static real_t Log(real_t s) { return (real_t)Math.Log(s); diff --git a/modules/mono/mono_gd/gd_mono.cpp b/modules/mono/mono_gd/gd_mono.cpp index 096ad0f5e3..571bf598f3 100644 --- a/modules/mono/mono_gd/gd_mono.cpp +++ b/modules/mono/mono_gd/gd_mono.cpp @@ -602,12 +602,12 @@ bool GDMono::copy_prebuilt_api_assembly(APIAssembly::Type p_api_type) { String GDMono::update_api_assemblies_from_prebuilt() { -#define FAIL_REASON(m_out_of_sync, m_prebuilt_exist) \ +#define FAIL_REASON(m_out_of_sync, m_prebuilt_exists) \ ( \ (m_out_of_sync ? \ String("The assembly is invalidated") : \ String("The assembly was not found")) + \ - (m_prebuilt_exist ? \ + (m_prebuilt_exists ? \ String(" and the prebuilt assemblies are missing") : \ String(" and we failed to copy the prebuilt assemblies"))) @@ -619,16 +619,18 @@ String GDMono::update_api_assemblies_from_prebuilt() { if (!api_assembly_out_of_sync && FileAccess::exists(core_assembly_path) && FileAccess::exists(editor_assembly_path)) return String(); // No update needed + print_verbose("Updating API assemblies"); + String prebuilt_api_dir = GodotSharpDirs::get_data_editor_prebuilt_api_dir().plus_file("Debug"); String prebuilt_core_dll_path = prebuilt_api_dir.plus_file(CORE_API_ASSEMBLY_NAME ".dll"); String prebuilt_editor_dll_path = prebuilt_api_dir.plus_file(EDITOR_API_ASSEMBLY_NAME ".dll"); if (!FileAccess::exists(prebuilt_core_dll_path) || !FileAccess::exists(prebuilt_editor_dll_path)) - return FAIL_REASON(api_assembly_out_of_sync, /* prebuilt_exist: */ false); + return FAIL_REASON(api_assembly_out_of_sync, /* prebuilt_exists: */ false); // Copy the prebuilt Api if (!copy_prebuilt_api_assembly(APIAssembly::API_CORE) || !copy_prebuilt_api_assembly(APIAssembly::API_EDITOR)) - return FAIL_REASON(api_assembly_out_of_sync, /* prebuilt_exist: */ true); + return FAIL_REASON(api_assembly_out_of_sync, /* prebuilt_exists: */ true); return String(); // Updated successfully @@ -699,9 +701,6 @@ bool GDMono::_try_load_api_assemblies() { return false; } - if (core_api_assembly_out_of_sync || !GDMonoUtils::mono_cache.godot_api_cache_updated) - return false; - #ifdef TOOLS_ENABLED if (!_load_editor_api_assembly()) { if (OS::get_singleton()->is_stdout_verbose()) @@ -713,12 +712,19 @@ bool GDMono::_try_load_api_assemblies() { return false; #endif + // Check if the core API assembly is out of sync only after trying to load the + // editor API assembly. Otherwise, if both assemblies are out of sync, we would + // only update the former as we won't know the latter also needs to be updated. + if (core_api_assembly_out_of_sync || !GDMonoUtils::mono_cache.godot_api_cache_updated) + return false; + return true; } void GDMono::_load_api_assemblies() { if (!_try_load_api_assemblies()) { +#ifdef TOOLS_ENABLED // The API assemblies are out of sync. Fine, try one more time, but this time // update them from the prebuilt assemblies directory before trying to load them. @@ -752,11 +758,9 @@ void GDMono::_load_api_assemblies() { ERR_PRINT("The loaded assembly '" CORE_API_ASSEMBLY_NAME "' is in sync, but the cache update failed"); } -#ifdef TOOLS_ENABLED if (editor_api_assembly_out_of_sync) { ERR_PRINT("The assembly '" EDITOR_API_ASSEMBLY_NAME "' is out of sync"); } -#endif CRASH_NOW(); } else { @@ -764,6 +768,10 @@ void GDMono::_load_api_assemblies() { CRASH_NOW(); } } +#else + ERR_EXPLAIN("Failed to load one of the API assemblies"); + CRASH_NOW(); +#endif } } diff --git a/modules/vhacd/SCsub b/modules/vhacd/SCsub index e581fb7bb2..685976dc33 100644 --- a/modules/vhacd/SCsub +++ b/modules/vhacd/SCsub @@ -26,10 +26,6 @@ thirdparty_sources = [thirdparty_dir + file for file in thirdparty_sources] env_vhacd.Prepend(CPPPATH=[thirdparty_dir + "/inc"]) -# upstream uses c++11 -if not env.msvc: - env_vhacd.Append(CXXFLAGS="-std=c++11") - env_thirdparty = env_vhacd.Clone() env_thirdparty.disable_warnings() env_thirdparty.add_source_files(env.modules_sources, thirdparty_sources) diff --git a/modules/visual_script/doc_classes/VisualScriptBuiltinFunc.xml b/modules/visual_script/doc_classes/VisualScriptBuiltinFunc.xml index 470a3a5e35..9e3670ec35 100644 --- a/modules/visual_script/doc_classes/VisualScriptBuiltinFunc.xml +++ b/modules/visual_script/doc_classes/VisualScriptBuiltinFunc.xml @@ -213,7 +213,11 @@ return t * t * (3.0 - 2.0 * t) [/codeblock] </constant> - <constant name="FUNC_MAX" value="65" enum="BuiltinFunc"> + <constant name="MATH_POSMOD" value="65" enum="BuiltinFunc"> + </constant> + <constant name="MATH_LERP_ANGLE" value="66" enum="BuiltinFunc"> + </constant> + <constant name="FUNC_MAX" value="67" enum="BuiltinFunc"> Represents the size of the [enum BuiltinFunc] enum. </constant> </constants> diff --git a/modules/visual_script/visual_script_builtin_funcs.cpp b/modules/visual_script/visual_script_builtin_funcs.cpp index 6c9c6b0fc8..8088a71198 100644 --- a/modules/visual_script/visual_script_builtin_funcs.cpp +++ b/modules/visual_script/visual_script_builtin_funcs.cpp @@ -105,6 +105,7 @@ const char *VisualScriptBuiltinFunc::func_name[VisualScriptBuiltinFunc::FUNC_MAX "color_named", "smoothstep", "posmod", + "lerp_angle", }; VisualScriptBuiltinFunc::BuiltinFunc VisualScriptBuiltinFunc::find_function(const String &p_string) { @@ -207,6 +208,7 @@ int VisualScriptBuiltinFunc::get_func_argument_count(BuiltinFunc p_func) { case COLORN: return 2; case MATH_LERP: + case MATH_LERP_ANGLE: case MATH_INVERSE_LERP: case MATH_SMOOTHSTEP: case MATH_MOVE_TOWARD: @@ -318,7 +320,9 @@ PropertyInfo VisualScriptBuiltinFunc::get_input_value_port_info(int p_idx) const return PropertyInfo(Variant::REAL, "steps"); } break; case MATH_LERP: - case MATH_INVERSE_LERP: { + case MATH_LERP_ANGLE: + case MATH_INVERSE_LERP: + case MATH_SMOOTHSTEP: { if (p_idx == 0) return PropertyInfo(Variant::REAL, "from"); else if (p_idx == 1) @@ -338,14 +342,6 @@ PropertyInfo VisualScriptBuiltinFunc::get_input_value_port_info(int p_idx) const else return PropertyInfo(Variant::REAL, "ostop"); } break; - case MATH_SMOOTHSTEP: { - if (p_idx == 0) - return PropertyInfo(Variant::REAL, "from"); - else if (p_idx == 1) - return PropertyInfo(Variant::REAL, "to"); - else - return PropertyInfo(Variant::REAL, "weight"); - } break; case MATH_MOVE_TOWARD: { if (p_idx == 0) return PropertyInfo(Variant::REAL, "from"); @@ -532,6 +528,7 @@ PropertyInfo VisualScriptBuiltinFunc::get_output_value_port_info(int p_idx) cons } break; case MATH_STEPIFY: case MATH_LERP: + case MATH_LERP_ANGLE: case MATH_INVERSE_LERP: case MATH_RANGE_LERP: case MATH_SMOOTHSTEP: @@ -856,6 +853,13 @@ void VisualScriptBuiltinFunc::exec_func(BuiltinFunc p_func, const Variant **p_in VALIDATE_ARG_NUM(2); *r_return = Math::lerp((double)*p_inputs[0], (double)*p_inputs[1], (double)*p_inputs[2]); } break; + case VisualScriptBuiltinFunc::MATH_LERP_ANGLE: { + + VALIDATE_ARG_NUM(0); + VALIDATE_ARG_NUM(1); + VALIDATE_ARG_NUM(2); + *r_return = Math::lerp_angle((double)*p_inputs[0], (double)*p_inputs[1], (double)*p_inputs[2]); + } break; case VisualScriptBuiltinFunc::MATH_INVERSE_LERP: { VALIDATE_ARG_NUM(0); @@ -1316,7 +1320,6 @@ void VisualScriptBuiltinFunc::_bind_methods() { BIND_ENUM_CONSTANT(MATH_SQRT); BIND_ENUM_CONSTANT(MATH_FMOD); BIND_ENUM_CONSTANT(MATH_FPOSMOD); - BIND_ENUM_CONSTANT(MATH_POSMOD); BIND_ENUM_CONSTANT(MATH_FLOOR); BIND_ENUM_CONSTANT(MATH_CEIL); BIND_ENUM_CONSTANT(MATH_ROUND); @@ -1369,6 +1372,8 @@ void VisualScriptBuiltinFunc::_bind_methods() { BIND_ENUM_CONSTANT(BYTES_TO_VAR); BIND_ENUM_CONSTANT(COLORN); BIND_ENUM_CONSTANT(MATH_SMOOTHSTEP); + BIND_ENUM_CONSTANT(MATH_POSMOD); + BIND_ENUM_CONSTANT(MATH_LERP_ANGLE); BIND_ENUM_CONSTANT(FUNC_MAX); } @@ -1422,6 +1427,7 @@ void register_visual_script_builtin_func_node() { VisualScriptLanguage::singleton->add_register_func("functions/built_in/decimals", create_builtin_func_node<VisualScriptBuiltinFunc::MATH_DECIMALS>); VisualScriptLanguage::singleton->add_register_func("functions/built_in/stepify", create_builtin_func_node<VisualScriptBuiltinFunc::MATH_STEPIFY>); VisualScriptLanguage::singleton->add_register_func("functions/built_in/lerp", create_builtin_func_node<VisualScriptBuiltinFunc::MATH_LERP>); + VisualScriptLanguage::singleton->add_register_func("functions/built_in/lerp_angle", create_builtin_func_node<VisualScriptBuiltinFunc::MATH_LERP_ANGLE>); VisualScriptLanguage::singleton->add_register_func("functions/built_in/inverse_lerp", create_builtin_func_node<VisualScriptBuiltinFunc::MATH_INVERSE_LERP>); VisualScriptLanguage::singleton->add_register_func("functions/built_in/range_lerp", create_builtin_func_node<VisualScriptBuiltinFunc::MATH_RANGE_LERP>); VisualScriptLanguage::singleton->add_register_func("functions/built_in/smoothstep", create_builtin_func_node<VisualScriptBuiltinFunc::MATH_SMOOTHSTEP>); diff --git a/modules/visual_script/visual_script_builtin_funcs.h b/modules/visual_script/visual_script_builtin_funcs.h index 7821e0eb2d..cf475d675d 100644 --- a/modules/visual_script/visual_script_builtin_funcs.h +++ b/modules/visual_script/visual_script_builtin_funcs.h @@ -105,6 +105,7 @@ public: COLORN, MATH_SMOOTHSTEP, MATH_POSMOD, + MATH_LERP_ANGLE, FUNC_MAX }; diff --git a/modules/webm/SCsub b/modules/webm/SCsub index e57437229f..32e6727656 100644 --- a/modules/webm/SCsub +++ b/modules/webm/SCsub @@ -17,10 +17,6 @@ thirdparty_sources = [thirdparty_dir + file for file in thirdparty_sources] env_webm.Prepend(CPPPATH=[thirdparty_dir, thirdparty_dir + "libwebm/"]) -# upstream uses c++11 -if (not env_webm.msvc): - env_webm.Append(CXXFLAGS="-std=c++11") - # also requires libogg, libvorbis and libopus if env['builtin_libogg']: env_webm.Prepend(CPPPATH=["#thirdparty/libogg"]) diff --git a/modules/xatlas_unwrap/SCsub b/modules/xatlas_unwrap/SCsub index 50e3cb1551..b242fd4673 100644 --- a/modules/xatlas_unwrap/SCsub +++ b/modules/xatlas_unwrap/SCsub @@ -15,10 +15,6 @@ if env['builtin_xatlas']: env_xatlas_unwrap.Prepend(CPPPATH=[thirdparty_dir]) - # upstream uses c++11 - if (not env.msvc): - env_xatlas_unwrap.Append(CXXFLAGS="-std=c++11") - env_thirdparty = env_xatlas_unwrap.Clone() env_thirdparty.disable_warnings() env_thirdparty.add_source_files(env.modules_sources, thirdparty_sources) diff --git a/platform/SCsub b/platform/SCsub index aa83154ee0..20c89ae8c6 100644 --- a/platform/SCsub +++ b/platform/SCsub @@ -3,7 +3,8 @@ from compat import open_utf8 Import('env') -platform_sources = [] + +env.platform_sources = [] # Register platform-exclusive APIs reg_apis_inc = '#include "register_platform_apis.h"\n' @@ -11,7 +12,7 @@ reg_apis = 'void register_platform_apis() {\n' unreg_apis = 'void unregister_platform_apis() {\n' for platform in env.platform_apis: platform_dir = env.Dir(platform) - platform_sources.append(platform_dir.File('api/api.cpp')) + env.add_source_files(env.platform_sources, platform + '/api/api.cpp') reg_apis += '\tregister_' + platform + '_api();\n' unreg_apis += '\tunregister_' + platform + '_api();\n' reg_apis_inc += '#include "' + platform + '/api/api.h"\n' @@ -25,7 +26,7 @@ with open_utf8('register_platform_apis.gen.cpp', 'w') as f: f.write(reg_apis) f.write(unreg_apis) -platform_sources.append('register_platform_apis.gen.cpp') +env.add_source_files(env.platform_sources, 'register_platform_apis.gen.cpp') -lib = env.add_library('platform', platform_sources) +lib = env.add_library('platform', env.platform_sources) env.Prepend(LIBS=lib) diff --git a/platform/iphone/camera_ios.mm b/platform/iphone/camera_ios.mm index 029ce6debf..ff84df66ff 100644 --- a/platform/iphone/camera_ios.mm +++ b/platform/iphone/camera_ios.mm @@ -158,7 +158,7 @@ } else if (dataCbCr == NULL) { print_line("Couldn't access CbCr pixel buffer data"); } else { - UIDeviceOrientation orientation = [[UIApplication sharedApplication] statusBarOrientation]; + UIInterfaceOrientation orientation = [[UIApplication sharedApplication] statusBarOrientation]; Ref<Image> img[2]; { diff --git a/platform/javascript/detect.py b/platform/javascript/detect.py index 10680ad1f5..ac43392700 100644 --- a/platform/javascript/detect.py +++ b/platform/javascript/detect.py @@ -128,7 +128,6 @@ def configure(env): ## Link flags env.Append(LINKFLAGS=['-s', 'BINARYEN=1']) - env.Append(LINKFLAGS=['-s', 'BINARYEN_TRAP_MODE=\'clamp\'']) # Allow increasing memory buffer size during runtime. This is efficient # when using WebAssembly (in comparison to asm.js) and works well for diff --git a/scene/3d/SCsub b/scene/3d/SCsub index 200cf4316f..31a443bad1 100644 --- a/scene/3d/SCsub +++ b/scene/3d/SCsub @@ -3,10 +3,10 @@ Import('env') if env['disable_3d']: - env.scene_sources.append("3d/spatial.cpp") - env.scene_sources.append("3d/skeleton.cpp") - env.scene_sources.append("3d/particles.cpp") - env.scene_sources.append("3d/visual_instance.cpp") - env.scene_sources.append("3d/world_environment.cpp") + env.add_source_files(env.scene_sources, "spatial.cpp") + env.add_source_files(env.scene_sources, "skeleton.cpp") + env.add_source_files(env.scene_sources, "particles.cpp") + env.add_source_files(env.scene_sources, "visual_instance.cpp") + env.add_source_files(env.scene_sources, "world_environment.cpp") else: env.add_source_files(env.scene_sources, "*.cpp") diff --git a/scene/3d/audio_stream_player_3d.cpp b/scene/3d/audio_stream_player_3d.cpp index 054c211d23..27f16f7601 100644 --- a/scene/3d/audio_stream_player_3d.cpp +++ b/scene/3d/audio_stream_player_3d.cpp @@ -35,6 +35,99 @@ #include "scene/3d/listener.h" #include "scene/main/viewport.h" +// Based on "A Novel Multichannel Panning Method for Standard and Arbitrary Loudspeaker Configurations" by Ramy Sadek and Chris Kyriakakis (2004) +// Speaker-Placement Correction Amplitude Panning (SPCAP) +class Spcap { +private: + struct Speaker { + Vector3 direction; + real_t effective_number_of_speakers; // precalculated + mutable real_t squared_gain; // temporary + }; + + PoolVector<Speaker> speakers; + +public: + Spcap(unsigned int speaker_count, const Vector3 *speaker_directions) { + this->speakers.resize(speaker_count); + PoolVector<Speaker>::Write w = this->speakers.write(); + for (unsigned int speaker_num = 0; speaker_num < speaker_count; speaker_num++) { + w[speaker_num].direction = speaker_directions[speaker_num]; + w[speaker_num].squared_gain = 0.0; + w[speaker_num].effective_number_of_speakers = 0.0; + for (unsigned int other_speaker_num = 0; other_speaker_num < speaker_count; other_speaker_num++) { + w[speaker_num].effective_number_of_speakers += 0.5 * (1.0 + w[speaker_num].direction.dot(w[other_speaker_num].direction)); + } + } + } + + unsigned int get_speaker_count() const { + return (unsigned int)this->speakers.size(); + } + + Vector3 get_speaker_direction(unsigned int index) const { + return this->speakers.read()[index].direction; + } + + void calculate(const Vector3 &source_direction, real_t tightness, unsigned int volume_count, real_t *volumes) const { + PoolVector<Speaker>::Read r = this->speakers.read(); + real_t sum_squared_gains = 0.0; + for (unsigned int speaker_num = 0; speaker_num < (unsigned int)this->speakers.size(); speaker_num++) { + real_t initial_gain = 0.5 * powf(1.0 + r[speaker_num].direction.dot(source_direction), tightness) / r[speaker_num].effective_number_of_speakers; + r[speaker_num].squared_gain = initial_gain * initial_gain; + sum_squared_gains += r[speaker_num].squared_gain; + } + + for (unsigned int speaker_num = 0; speaker_num < MIN(volume_count, (unsigned int)this->speakers.size()); speaker_num++) { + volumes[speaker_num] = sqrtf(r[speaker_num].squared_gain / sum_squared_gains); + } + } +}; + +//TODO: hardcoded main speaker directions for 2, 3.1, 5.1 and 7.1 setups - these are simplified and could also be made configurable +static const Vector3 speaker_directions[7] = { + Vector3(-1.0, 0.0, -1.0).normalized(), // front-left + Vector3(1.0, 0.0, -1.0).normalized(), // front-right + Vector3(0.0, 0.0, -1.0).normalized(), // center + Vector3(-1.0, 0.0, 1.0).normalized(), // rear-left + Vector3(1.0, 0.0, 1.0).normalized(), // rear-right + Vector3(-1.0, 0.0, 0.0).normalized(), // side-left + Vector3(1.0, 0.0, 0.0).normalized(), // side-right +}; + +void AudioStreamPlayer3D::_calc_output_vol(const Vector3 &source_dir, real_t tightness, AudioStreamPlayer3D::Output &output) { + unsigned int speaker_count; // only main speakers (no LFE) + switch (AudioServer::get_singleton()->get_speaker_mode()) { + default: //fallthrough + case AudioServer::SPEAKER_MODE_STEREO: speaker_count = 2; break; + case AudioServer::SPEAKER_SURROUND_31: speaker_count = 3; break; + case AudioServer::SPEAKER_SURROUND_51: speaker_count = 5; break; + case AudioServer::SPEAKER_SURROUND_71: speaker_count = 7; break; + } + + Spcap spcap(speaker_count, speaker_directions); //TODO: should only be created/recreated once the speaker mode / speaker positions changes + real_t volumes[7]; + spcap.calculate(source_dir, tightness, speaker_count, volumes); + + switch (AudioServer::get_singleton()->get_speaker_mode()) { + case AudioServer::SPEAKER_SURROUND_71: + output.vol[3].l = volumes[5]; // side-left + output.vol[3].r = volumes[6]; // side-right + //fallthrough + case AudioServer::SPEAKER_SURROUND_51: + output.vol[2].l = volumes[3]; // rear-left + output.vol[2].r = volumes[4]; // rear-right + //fallthrough + case AudioServer::SPEAKER_SURROUND_31: + output.vol[1].r = 1.0; // LFE - always full power + output.vol[1].l = volumes[2]; // center + //fallthrough + case AudioServer::SPEAKER_MODE_STEREO: + output.vol[0].r = volumes[1]; // front-right + output.vol[0].l = volumes[0]; // front-left + } +} + void AudioStreamPlayer3D::_mix_audio() { if (!stream_playback.is_valid() || !active || @@ -381,59 +474,11 @@ void AudioStreamPlayer3D::_notification(int p_what) { output.filter_gain = Math::db2linear(db_att); - Vector3 flat_pos = local_pos; - flat_pos.y = 0; - flat_pos.normalize(); + //TODO: The lower the second parameter (tightness) the more the sound will "enclose" the listener (more undirected / playing from + // speakers not facing the source) - this could be made distance dependent. + _calc_output_vol(local_pos.normalized(), 4.0, output); unsigned int cc = AudioServer::get_singleton()->get_channel_count(); - if (cc == 1) { - // Stereo pair - float c = flat_pos.x * 0.5 + 0.5; - - output.vol[0].l = 1.0 - c; - output.vol[0].r = c; - } else { - Vector3 listenertopos = global_pos - listener_node->get_global_transform().origin; - float c = listenertopos.normalized().dot(get_global_transform().basis.get_axis(2).normalized()); //it's z negative - float angle = Math::rad2deg(Math::acos(c)); - float av = angle * (flat_pos.x < 0 ? -1 : 1) / 180.0; - - if (cc >= 1) { - // Stereo pair - float fl = Math::abs(1.0 - Math::abs(-0.8 - av)); - float fr = Math::abs(1.0 - Math::abs(0.8 - av)); - - output.vol[0].l = fl; - output.vol[0].r = fr; - } - - if (cc >= 2) { - // Center pair - float center = 1.0 - Math::sin(Math::acos(c)); - - output.vol[1].l = center; - output.vol[1].r = center; - } - - if (cc >= 3) { - // Side pair - float sleft = Math::abs(1.0 - Math::abs(-0.4 - av)); - float sright = Math::abs(1.0 - Math::abs(0.4 - av)); - - output.vol[2].l = sleft; - output.vol[2].r = sright; - } - - if (cc >= 4) { - // Rear pair - float rleft = Math::abs(1.0 - Math::abs(-0.2 - av)); - float rright = Math::abs(1.0 - Math::abs(0.2 - av)); - - output.vol[3].l = rleft; - output.vol[3].r = rright; - } - } - for (unsigned int k = 0; k < cc; k++) { output.vol[k] *= multiplier; } diff --git a/scene/3d/audio_stream_player_3d.h b/scene/3d/audio_stream_player_3d.h index 93954e758a..494aa70097 100644 --- a/scene/3d/audio_stream_player_3d.h +++ b/scene/3d/audio_stream_player_3d.h @@ -115,6 +115,7 @@ private: bool stream_paused_fade_out; StringName bus; + static void _calc_output_vol(const Vector3 &source_dir, real_t tightness, Output &output); void _mix_audio(); static void _mix_audios(void *self) { reinterpret_cast<AudioStreamPlayer3D *>(self)->_mix_audio(); } diff --git a/scene/resources/default_theme/make_header.py b/scene/resources/default_theme/make_header.py index bd5a723b23..cf0ccf1c3a 100755 --- a/scene/resources/default_theme/make_header.py +++ b/scene/resources/default_theme/make_header.py @@ -1,9 +1,14 @@ #!/usr/bin/env python import glob +import os enc = "utf-8" +# Change to the directory where the script is located, +# so that the script can be run from any location +os.chdir(os.path.dirname(os.path.realpath(__file__))) + # Generate include files f = open("theme_data.h", "wb") diff --git a/scene/resources/material.cpp b/scene/resources/material.cpp index 5372a69079..44bc862198 100644 --- a/scene/resources/material.cpp +++ b/scene/resources/material.cpp @@ -2353,8 +2353,8 @@ SpatialMaterial::SpatialMaterial() : set_ao_light_affect(0.0); - set_metallic_texture_channel(TEXTURE_CHANNEL_BLUE); - set_roughness_texture_channel(TEXTURE_CHANNEL_GREEN); + set_metallic_texture_channel(TEXTURE_CHANNEL_RED); + set_roughness_texture_channel(TEXTURE_CHANNEL_RED); set_ao_texture_channel(TEXTURE_CHANNEL_RED); set_refraction_texture_channel(TEXTURE_CHANNEL_RED); diff --git a/servers/visual/visual_server_viewport.cpp b/servers/visual/visual_server_viewport.cpp index f8ed035766..86c5227f30 100644 --- a/servers/visual/visual_server_viewport.cpp +++ b/servers/visual/visual_server_viewport.cpp @@ -63,7 +63,10 @@ static Transform2D _canvas_get_transform(VisualServerViewport::Viewport *p_viewp } void VisualServerViewport::_draw_3d(Viewport *p_viewport, ARVRInterface::Eyes p_eye) { - Ref<ARVRInterface> arvr_interface = ARVRServer::get_singleton()->get_primary_interface(); + Ref<ARVRInterface> arvr_interface; + if (ARVRServer::get_singleton() != NULL) { + arvr_interface = ARVRServer::get_singleton()->get_primary_interface(); + } if (p_viewport->use_arvr && arvr_interface.is_valid()) { VSG::scene->render_camera(arvr_interface, p_eye, p_viewport->camera, p_viewport->scenario, p_viewport->size, p_viewport->shadow_atlas); @@ -260,11 +263,16 @@ void VisualServerViewport::_draw_viewport(Viewport *p_viewport, ARVRInterface::E } void VisualServerViewport::draw_viewports() { + // get our arvr interface in case we need it - Ref<ARVRInterface> arvr_interface = ARVRServer::get_singleton()->get_primary_interface(); + Ref<ARVRInterface> arvr_interface; - // process all our active interfaces - ARVRServer::get_singleton()->_process(); + if (ARVRServer::get_singleton() != NULL) { + arvr_interface = ARVRServer::get_singleton()->get_primary_interface(); + + // process all our active interfaces + ARVRServer::get_singleton()->_process(); + } if (Engine::get_singleton()->is_editor_hint()) { clear_color = GLOBAL_GET("rendering/environment/default_clear_color"); diff --git a/thirdparty/README.md b/thirdparty/README.md index c6817e2389..81e30c3f6b 100644 --- a/thirdparty/README.md +++ b/thirdparty/README.md @@ -200,6 +200,7 @@ Important: Some files have Godot-made changes. They are marked with `// -- GODOT start --` and `// -- GODOT end --` comments. + ## libtheora - Upstream: https://www.theora.org @@ -262,18 +263,6 @@ changes to ensure they build for Javascript/HTML5. Those changes are marked with `// -- GODOT --` comments. -## wslay - -- Upstream: https://github.com/tatsuhiro-t/wslay -- Version: 1.1.0 -- License: MIT - -File extracted from upstream release tarball: - -- All `*.c` and `*.h` in `lib/` and `lib/includes/` -- `wslay.h` has a small Godot addition to fix MSVC build. - See `thirdparty/wslay/msvcfix.diff` - ## mbedtls - Upstream: https://tls.mbed.org/ @@ -508,6 +497,19 @@ They can be reapplied using the patches included in the `vhacd` folder. +## wslay + +- Upstream: https://github.com/tatsuhiro-t/wslay +- Version: 1.1.0 +- License: MIT + +File extracted from upstream release tarball: + +- All `*.c` and `*.h` in `lib/` and `lib/includes/` +- `wslay.h` has a small Godot addition to fix MSVC build. + See `thirdparty/wslay/msvcfix.diff` + + ## xatlas - Upstream: https://github.com/jpcy/xatlas @@ -536,7 +538,7 @@ Files extracted from upstream source: ## zstd - Upstream: https://github.com/facebook/zstd -- Version: 1.4.0 +- Version: 1.4.1 - License: BSD-3-Clause Files extracted from upstream source: diff --git a/thirdparty/zstd/common/compiler.h b/thirdparty/zstd/common/compiler.h index 0836e3ed27..87bf51ae8c 100644 --- a/thirdparty/zstd/common/compiler.h +++ b/thirdparty/zstd/common/compiler.h @@ -127,6 +127,13 @@ } \ } +/* vectorization */ +#if !defined(__clang__) && defined(__GNUC__) +# define DONT_VECTORIZE __attribute__((optimize("no-tree-vectorize"))) +#else +# define DONT_VECTORIZE +#endif + /* disable warnings */ #ifdef _MSC_VER /* Visual Studio */ # include <intrin.h> /* For Visual 2005 */ diff --git a/thirdparty/zstd/common/zstd_internal.h b/thirdparty/zstd/common/zstd_internal.h index 31f756ab58..81b16eac2e 100644 --- a/thirdparty/zstd/common/zstd_internal.h +++ b/thirdparty/zstd/common/zstd_internal.h @@ -34,7 +34,6 @@ #endif #include "xxhash.h" /* XXH_reset, update, digest */ - #if defined (__cplusplus) extern "C" { #endif @@ -193,19 +192,72 @@ static const U32 OF_defaultNormLog = OF_DEFAULTNORMLOG; * Shared functions to include for inlining *********************************************/ static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); } + #define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; } +static void ZSTD_copy16(void* dst, const void* src) { memcpy(dst, src, 16); } +#define COPY16(d,s) { ZSTD_copy16(d,s); d+=16; s+=16; } + +#define WILDCOPY_OVERLENGTH 8 +#define VECLEN 16 + +typedef enum { + ZSTD_no_overlap, + ZSTD_overlap_src_before_dst, + /* ZSTD_overlap_dst_before_src, */ +} ZSTD_overlap_e; /*! ZSTD_wildcopy() : * custom version of memcpy(), can overwrite up to WILDCOPY_OVERLENGTH bytes (if length==0) */ -#define WILDCOPY_OVERLENGTH 8 -MEM_STATIC void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length) +MEM_STATIC FORCE_INLINE_ATTR DONT_VECTORIZE +void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length, ZSTD_overlap_e ovtype) { + ptrdiff_t diff = (BYTE*)dst - (const BYTE*)src; const BYTE* ip = (const BYTE*)src; BYTE* op = (BYTE*)dst; BYTE* const oend = op + length; - do - COPY8(op, ip) - while (op < oend); + + assert(diff >= 8 || (ovtype == ZSTD_no_overlap && diff < -8)); + if (length < VECLEN || (ovtype == ZSTD_overlap_src_before_dst && diff < VECLEN)) { + do + COPY8(op, ip) + while (op < oend); + } + else { + if ((length & 8) == 0) + COPY8(op, ip); + do { + COPY16(op, ip); + } + while (op < oend); + } +} + +/*! ZSTD_wildcopy_16min() : + * same semantics as ZSTD_wilcopy() except guaranteed to be able to copy 16 bytes at the start */ +MEM_STATIC FORCE_INLINE_ATTR DONT_VECTORIZE +void ZSTD_wildcopy_16min(void* dst, const void* src, ptrdiff_t length, ZSTD_overlap_e ovtype) +{ + ptrdiff_t diff = (BYTE*)dst - (const BYTE*)src; + const BYTE* ip = (const BYTE*)src; + BYTE* op = (BYTE*)dst; + BYTE* const oend = op + length; + + assert(length >= 8); + assert(diff >= 8 || (ovtype == ZSTD_no_overlap && diff < -8)); + + if (ovtype == ZSTD_overlap_src_before_dst && diff < VECLEN) { + do + COPY8(op, ip) + while (op < oend); + } + else { + if ((length & 8) == 0) + COPY8(op, ip); + do { + COPY16(op, ip); + } + while (op < oend); + } } MEM_STATIC void ZSTD_wildcopy_e(void* dst, const void* src, void* dstEnd) /* should be faster for decoding, but strangely, not verified on all platform */ diff --git a/thirdparty/zstd/compress/zstd_compress.c b/thirdparty/zstd/compress/zstd_compress.c index 2e163c8bf3..1476512580 100644 --- a/thirdparty/zstd/compress/zstd_compress.c +++ b/thirdparty/zstd/compress/zstd_compress.c @@ -385,6 +385,11 @@ ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param) bounds.upperBound = ZSTD_lcm_uncompressed; return bounds; + case ZSTD_c_targetCBlockSize: + bounds.lowerBound = ZSTD_TARGETCBLOCKSIZE_MIN; + bounds.upperBound = ZSTD_TARGETCBLOCKSIZE_MAX; + return bounds; + default: { ZSTD_bounds const boundError = { ERROR(parameter_unsupported), 0, 0 }; return boundError; @@ -452,6 +457,7 @@ static int ZSTD_isUpdateAuthorized(ZSTD_cParameter param) case ZSTD_c_ldmHashRateLog: case ZSTD_c_forceAttachDict: case ZSTD_c_literalCompressionMode: + case ZSTD_c_targetCBlockSize: default: return 0; } @@ -497,6 +503,7 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value) case ZSTD_c_ldmHashLog: case ZSTD_c_ldmMinMatch: case ZSTD_c_ldmBucketSizeLog: + case ZSTD_c_targetCBlockSize: break; default: RETURN_ERROR(parameter_unsupported); @@ -671,6 +678,12 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams, CCtxParams->ldmParams.hashRateLog = value; return CCtxParams->ldmParams.hashRateLog; + case ZSTD_c_targetCBlockSize : + if (value!=0) /* 0 ==> default */ + BOUNDCHECK(ZSTD_c_targetCBlockSize, value); + CCtxParams->targetCBlockSize = value; + return CCtxParams->targetCBlockSize; + default: RETURN_ERROR(parameter_unsupported, "unknown parameter"); } } @@ -692,13 +705,13 @@ size_t ZSTD_CCtxParams_getParameter( *value = CCtxParams->compressionLevel; break; case ZSTD_c_windowLog : - *value = CCtxParams->cParams.windowLog; + *value = (int)CCtxParams->cParams.windowLog; break; case ZSTD_c_hashLog : - *value = CCtxParams->cParams.hashLog; + *value = (int)CCtxParams->cParams.hashLog; break; case ZSTD_c_chainLog : - *value = CCtxParams->cParams.chainLog; + *value = (int)CCtxParams->cParams.chainLog; break; case ZSTD_c_searchLog : *value = CCtxParams->cParams.searchLog; @@ -773,6 +786,9 @@ size_t ZSTD_CCtxParams_getParameter( case ZSTD_c_ldmHashRateLog : *value = CCtxParams->ldmParams.hashRateLog; break; + case ZSTD_c_targetCBlockSize : + *value = (int)CCtxParams->targetCBlockSize; + break; default: RETURN_ERROR(parameter_unsupported, "unknown parameter"); } return 0; @@ -930,12 +946,12 @@ size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset) @return : 0, or an error code if one value is beyond authorized range */ size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams) { - BOUNDCHECK(ZSTD_c_windowLog, cParams.windowLog); - BOUNDCHECK(ZSTD_c_chainLog, cParams.chainLog); - BOUNDCHECK(ZSTD_c_hashLog, cParams.hashLog); - BOUNDCHECK(ZSTD_c_searchLog, cParams.searchLog); - BOUNDCHECK(ZSTD_c_minMatch, cParams.minMatch); - BOUNDCHECK(ZSTD_c_targetLength,cParams.targetLength); + BOUNDCHECK(ZSTD_c_windowLog, (int)cParams.windowLog); + BOUNDCHECK(ZSTD_c_chainLog, (int)cParams.chainLog); + BOUNDCHECK(ZSTD_c_hashLog, (int)cParams.hashLog); + BOUNDCHECK(ZSTD_c_searchLog, (int)cParams.searchLog); + BOUNDCHECK(ZSTD_c_minMatch, (int)cParams.minMatch); + BOUNDCHECK(ZSTD_c_targetLength,(int)cParams.targetLength); BOUNDCHECK(ZSTD_c_strategy, cParams.strategy); return 0; } @@ -951,7 +967,7 @@ ZSTD_clampCParams(ZSTD_compressionParameters cParams) if ((int)val<bounds.lowerBound) val=(type)bounds.lowerBound; \ else if ((int)val>bounds.upperBound) val=(type)bounds.upperBound; \ } -# define CLAMP(cParam, val) CLAMP_TYPE(cParam, val, int) +# define CLAMP(cParam, val) CLAMP_TYPE(cParam, val, unsigned) CLAMP(ZSTD_c_windowLog, cParams.windowLog); CLAMP(ZSTD_c_chainLog, cParams.chainLog); CLAMP(ZSTD_c_hashLog, cParams.hashLog); @@ -1282,15 +1298,14 @@ static void ZSTD_reset_compressedBlockState(ZSTD_compressedBlockState_t* bs) } /*! ZSTD_invalidateMatchState() - * Invalidate all the matches in the match finder tables. - * Requires nextSrc and base to be set (can be NULL). + * Invalidate all the matches in the match finder tables. + * Requires nextSrc and base to be set (can be NULL). */ static void ZSTD_invalidateMatchState(ZSTD_matchState_t* ms) { ZSTD_window_clear(&ms->window); ms->nextToUpdate = ms->window.dictLimit; - ms->nextToUpdate3 = ms->window.dictLimit; ms->loadedDictEnd = 0; ms->opt.litLengthSum = 0; /* force reset of btopt stats */ ms->dictMatchState = NULL; @@ -1327,15 +1342,17 @@ static size_t ZSTD_continueCCtx(ZSTD_CCtx* cctx, ZSTD_CCtx_params params, U64 pl typedef enum { ZSTDcrp_continue, ZSTDcrp_noMemset } ZSTD_compResetPolicy_e; +typedef enum { ZSTD_resetTarget_CDict, ZSTD_resetTarget_CCtx } ZSTD_resetTarget_e; + static void* ZSTD_reset_matchState(ZSTD_matchState_t* ms, void* ptr, const ZSTD_compressionParameters* cParams, - ZSTD_compResetPolicy_e const crp, U32 const forCCtx) + ZSTD_compResetPolicy_e const crp, ZSTD_resetTarget_e const forWho) { size_t const chainSize = (cParams->strategy == ZSTD_fast) ? 0 : ((size_t)1 << cParams->chainLog); size_t const hSize = ((size_t)1) << cParams->hashLog; - U32 const hashLog3 = (forCCtx && cParams->minMatch==3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0; + U32 const hashLog3 = ((forWho == ZSTD_resetTarget_CCtx) && cParams->minMatch==3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0; size_t const h3Size = ((size_t)1) << hashLog3; size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); @@ -1349,7 +1366,7 @@ ZSTD_reset_matchState(ZSTD_matchState_t* ms, ZSTD_invalidateMatchState(ms); /* opt parser space */ - if (forCCtx && (cParams->strategy >= ZSTD_btopt)) { + if ((forWho == ZSTD_resetTarget_CCtx) && (cParams->strategy >= ZSTD_btopt)) { DEBUGLOG(4, "reserving optimal parser space"); ms->opt.litFreq = (unsigned*)ptr; ms->opt.litLengthFreq = ms->opt.litFreq + (1<<Litbits); @@ -1377,6 +1394,19 @@ ZSTD_reset_matchState(ZSTD_matchState_t* ms, return ptr; } +/* ZSTD_indexTooCloseToMax() : + * minor optimization : prefer memset() rather than reduceIndex() + * which is measurably slow in some circumstances (reported for Visual Studio). + * Works when re-using a context for a lot of smallish inputs : + * if all inputs are smaller than ZSTD_INDEXOVERFLOW_MARGIN, + * memset() will be triggered before reduceIndex(). + */ +#define ZSTD_INDEXOVERFLOW_MARGIN (16 MB) +static int ZSTD_indexTooCloseToMax(ZSTD_window_t w) +{ + return (size_t)(w.nextSrc - w.base) > (ZSTD_CURRENT_MAX - ZSTD_INDEXOVERFLOW_MARGIN); +} + #define ZSTD_WORKSPACETOOLARGE_FACTOR 3 /* define "workspace is too large" as this number of times larger than needed */ #define ZSTD_WORKSPACETOOLARGE_MAXDURATION 128 /* when workspace is continuously too large * during at least this number of times, @@ -1388,7 +1418,7 @@ ZSTD_reset_matchState(ZSTD_matchState_t* ms, note : `params` are assumed fully validated at this stage */ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, ZSTD_CCtx_params params, - U64 pledgedSrcSize, + U64 const pledgedSrcSize, ZSTD_compResetPolicy_e const crp, ZSTD_buffered_policy_e const zbuff) { @@ -1400,13 +1430,21 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, if (ZSTD_equivalentParams(zc->appliedParams, params, zc->inBuffSize, zc->seqStore.maxNbSeq, zc->seqStore.maxNbLit, - zbuff, pledgedSrcSize)) { - DEBUGLOG(4, "ZSTD_equivalentParams()==1 -> continue mode (wLog1=%u, blockSize1=%zu)", - zc->appliedParams.cParams.windowLog, zc->blockSize); + zbuff, pledgedSrcSize) ) { + DEBUGLOG(4, "ZSTD_equivalentParams()==1 -> consider continue mode"); zc->workSpaceOversizedDuration += (zc->workSpaceOversizedDuration > 0); /* if it was too large, it still is */ - if (zc->workSpaceOversizedDuration <= ZSTD_WORKSPACETOOLARGE_MAXDURATION) + if (zc->workSpaceOversizedDuration <= ZSTD_WORKSPACETOOLARGE_MAXDURATION) { + DEBUGLOG(4, "continue mode confirmed (wLog1=%u, blockSize1=%zu)", + zc->appliedParams.cParams.windowLog, zc->blockSize); + if (ZSTD_indexTooCloseToMax(zc->blockState.matchState.window)) { + /* prefer a reset, faster than a rescale */ + ZSTD_reset_matchState(&zc->blockState.matchState, + zc->entropyWorkspace + HUF_WORKSPACE_SIZE_U32, + ¶ms.cParams, + crp, ZSTD_resetTarget_CCtx); + } return ZSTD_continueCCtx(zc, params, pledgedSrcSize); - } } + } } } DEBUGLOG(4, "ZSTD_equivalentParams()==0 -> reset CCtx"); if (params.ldmParams.enableLdm) { @@ -1449,7 +1487,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, DEBUGLOG(4, "windowSize: %zu - blockSize: %zu", windowSize, blockSize); if (workSpaceTooSmall || workSpaceWasteful) { - DEBUGLOG(4, "Need to resize workSpaceSize from %zuKB to %zuKB", + DEBUGLOG(4, "Resize workSpaceSize from %zuKB to %zuKB", zc->workSpaceSize >> 10, neededSpace >> 10); @@ -1491,7 +1529,10 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, ZSTD_reset_compressedBlockState(zc->blockState.prevCBlock); - ptr = zc->entropyWorkspace + HUF_WORKSPACE_SIZE_U32; + ptr = ZSTD_reset_matchState(&zc->blockState.matchState, + zc->entropyWorkspace + HUF_WORKSPACE_SIZE_U32, + ¶ms.cParams, + crp, ZSTD_resetTarget_CCtx); /* ldm hash table */ /* initialize bucketOffsets table later for pointer alignment */ @@ -1509,8 +1550,6 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, } assert(((size_t)ptr & 3) == 0); /* ensure ptr is properly aligned */ - ptr = ZSTD_reset_matchState(&zc->blockState.matchState, ptr, ¶ms.cParams, crp, /* forCCtx */ 1); - /* sequences storage */ zc->seqStore.maxNbSeq = maxNbSeq; zc->seqStore.sequencesStart = (seqDef*)ptr; @@ -1587,15 +1626,14 @@ static int ZSTD_shouldAttachDict(const ZSTD_CDict* cdict, * handled in _enforceMaxDist */ } -static size_t ZSTD_resetCCtx_byAttachingCDict( - ZSTD_CCtx* cctx, - const ZSTD_CDict* cdict, - ZSTD_CCtx_params params, - U64 pledgedSrcSize, - ZSTD_buffered_policy_e zbuff) +static size_t +ZSTD_resetCCtx_byAttachingCDict(ZSTD_CCtx* cctx, + const ZSTD_CDict* cdict, + ZSTD_CCtx_params params, + U64 pledgedSrcSize, + ZSTD_buffered_policy_e zbuff) { - { - const ZSTD_compressionParameters *cdict_cParams = &cdict->matchState.cParams; + { const ZSTD_compressionParameters* const cdict_cParams = &cdict->matchState.cParams; unsigned const windowLog = params.cParams.windowLog; assert(windowLog != 0); /* Resize working context table params for input only, since the dict @@ -1607,8 +1645,7 @@ static size_t ZSTD_resetCCtx_byAttachingCDict( assert(cctx->appliedParams.cParams.strategy == cdict_cParams->strategy); } - { - const U32 cdictEnd = (U32)( cdict->matchState.window.nextSrc + { const U32 cdictEnd = (U32)( cdict->matchState.window.nextSrc - cdict->matchState.window.base); const U32 cdictLen = cdictEnd - cdict->matchState.window.dictLimit; if (cdictLen == 0) { @@ -1625,9 +1662,9 @@ static size_t ZSTD_resetCCtx_byAttachingCDict( cctx->blockState.matchState.window.base + cdictEnd; ZSTD_window_clear(&cctx->blockState.matchState.window); } + /* loadedDictEnd is expressed within the referential of the active context */ cctx->blockState.matchState.loadedDictEnd = cctx->blockState.matchState.window.dictLimit; - } - } + } } cctx->dictID = cdict->dictID; @@ -1681,7 +1718,6 @@ static size_t ZSTD_resetCCtx_byCopyingCDict(ZSTD_CCtx* cctx, ZSTD_matchState_t* dstMatchState = &cctx->blockState.matchState; dstMatchState->window = srcMatchState->window; dstMatchState->nextToUpdate = srcMatchState->nextToUpdate; - dstMatchState->nextToUpdate3= srcMatchState->nextToUpdate3; dstMatchState->loadedDictEnd= srcMatchState->loadedDictEnd; } @@ -1761,7 +1797,6 @@ static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx, ZSTD_matchState_t* dstMatchState = &dstCCtx->blockState.matchState; dstMatchState->window = srcMatchState->window; dstMatchState->nextToUpdate = srcMatchState->nextToUpdate; - dstMatchState->nextToUpdate3= srcMatchState->nextToUpdate3; dstMatchState->loadedDictEnd= srcMatchState->loadedDictEnd; } dstCCtx->dictID = srcCCtx->dictID; @@ -1831,16 +1866,15 @@ static void ZSTD_reduceTable_btlazy2(U32* const table, U32 const size, U32 const /*! ZSTD_reduceIndex() : * rescale all indexes to avoid future overflow (indexes are U32) */ -static void ZSTD_reduceIndex (ZSTD_CCtx* zc, const U32 reducerValue) +static void ZSTD_reduceIndex (ZSTD_matchState_t* ms, ZSTD_CCtx_params const* params, const U32 reducerValue) { - ZSTD_matchState_t* const ms = &zc->blockState.matchState; - { U32 const hSize = (U32)1 << zc->appliedParams.cParams.hashLog; + { U32 const hSize = (U32)1 << params->cParams.hashLog; ZSTD_reduceTable(ms->hashTable, hSize, reducerValue); } - if (zc->appliedParams.cParams.strategy != ZSTD_fast) { - U32 const chainSize = (U32)1 << zc->appliedParams.cParams.chainLog; - if (zc->appliedParams.cParams.strategy == ZSTD_btlazy2) + if (params->cParams.strategy != ZSTD_fast) { + U32 const chainSize = (U32)1 << params->cParams.chainLog; + if (params->cParams.strategy == ZSTD_btlazy2) ZSTD_reduceTable_btlazy2(ms->chainTable, chainSize, reducerValue); else ZSTD_reduceTable(ms->chainTable, chainSize, reducerValue); @@ -2524,6 +2558,7 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, op[0] = (BYTE)((nbSeq>>8) + 0x80), op[1] = (BYTE)nbSeq, op+=2; else op[0]=0xFF, MEM_writeLE16(op+1, (U16)(nbSeq - LONGNBSEQ)), op+=3; + assert(op <= oend); if (nbSeq==0) { /* Copy the old tables over as if we repeated them */ memcpy(&nextEntropy->fse, &prevEntropy->fse, sizeof(prevEntropy->fse)); @@ -2532,6 +2567,7 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, /* seqHead : flags for FSE encoding type */ seqHead = op++; + assert(op <= oend); /* convert length/distances into codes */ ZSTD_seqToCodes(seqStorePtr); @@ -2555,6 +2591,7 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, if (LLtype == set_compressed) lastNCount = op; op += countSize; + assert(op <= oend); } } /* build CTable for Offsets */ { unsigned max = MaxOff; @@ -2577,6 +2614,7 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, if (Offtype == set_compressed) lastNCount = op; op += countSize; + assert(op <= oend); } } /* build CTable for MatchLengths */ { unsigned max = MaxML; @@ -2597,6 +2635,7 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, if (MLtype == set_compressed) lastNCount = op; op += countSize; + assert(op <= oend); } } *seqHead = (BYTE)((LLtype<<6) + (Offtype<<4) + (MLtype<<2)); @@ -2610,6 +2649,7 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, longOffsets, bmi2); FORWARD_IF_ERROR(bitstreamSize); op += bitstreamSize; + assert(op <= oend); /* zstd versions <= 1.3.4 mistakenly report corruption when * FSE_readNCount() receives a buffer < 4 bytes. * Fixed by https://github.com/facebook/zstd/pull/1146. @@ -2721,30 +2761,24 @@ void ZSTD_resetSeqStore(seqStore_t* ssPtr) ssPtr->longLengthID = 0; } -static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize) +typedef enum { ZSTDbss_compress, ZSTDbss_noCompress } ZSTD_buildSeqStore_e; + +static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize) { ZSTD_matchState_t* const ms = &zc->blockState.matchState; - size_t cSize; - DEBUGLOG(5, "ZSTD_compressBlock_internal (dstCapacity=%u, dictLimit=%u, nextToUpdate=%u)", - (unsigned)dstCapacity, (unsigned)ms->window.dictLimit, (unsigned)ms->nextToUpdate); + DEBUGLOG(5, "ZSTD_buildSeqStore (srcSize=%zu)", srcSize); assert(srcSize <= ZSTD_BLOCKSIZE_MAX); - /* Assert that we have correctly flushed the ctx params into the ms's copy */ ZSTD_assertEqualCParams(zc->appliedParams.cParams, ms->cParams); - if (srcSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1) { ZSTD_ldm_skipSequences(&zc->externSeqStore, srcSize, zc->appliedParams.cParams.minMatch); - cSize = 0; - goto out; /* don't even attempt compression below a certain srcSize */ + return ZSTDbss_noCompress; /* don't even attempt compression below a certain srcSize */ } ZSTD_resetSeqStore(&(zc->seqStore)); /* required for optimal parser to read stats from dictionary */ ms->opt.symbolCosts = &zc->blockState.prevCBlock->entropy; /* tell the optimal parser how we expect to compress literals */ ms->opt.literalCompressionMode = zc->appliedParams.literalCompressionMode; - /* a gap between an attached dict and the current window is not safe, * they must remain adjacent, * and when that stops being the case, the dict must be unset */ @@ -2798,6 +2832,21 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, { const BYTE* const lastLiterals = (const BYTE*)src + srcSize - lastLLSize; ZSTD_storeLastLiterals(&zc->seqStore, lastLiterals, lastLLSize); } } + return ZSTDbss_compress; +} + +static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + size_t cSize; + DEBUGLOG(5, "ZSTD_compressBlock_internal (dstCapacity=%u, dictLimit=%u, nextToUpdate=%u)", + (unsigned)dstCapacity, (unsigned)zc->blockState.matchState.window.dictLimit, (unsigned)zc->blockState.matchState.nextToUpdate); + + { const size_t bss = ZSTD_buildSeqStore(zc, src, srcSize); + FORWARD_IF_ERROR(bss); + if (bss == ZSTDbss_noCompress) { cSize = 0; goto out; } + } /* encode sequences and literals */ cSize = ZSTD_compressSequences(&zc->seqStore, @@ -2826,6 +2875,25 @@ out: } +static void ZSTD_overflowCorrectIfNeeded(ZSTD_matchState_t* ms, ZSTD_CCtx_params const* params, void const* ip, void const* iend) +{ + if (ZSTD_window_needOverflowCorrection(ms->window, iend)) { + U32 const maxDist = (U32)1 << params->cParams.windowLog; + U32 const cycleLog = ZSTD_cycleLog(params->cParams.chainLog, params->cParams.strategy); + U32 const correction = ZSTD_window_correctOverflow(&ms->window, cycleLog, maxDist, ip); + ZSTD_STATIC_ASSERT(ZSTD_CHAINLOG_MAX <= 30); + ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX_32 <= 30); + ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX <= 31); + ZSTD_reduceIndex(ms, params, correction); + if (ms->nextToUpdate < correction) ms->nextToUpdate = 0; + else ms->nextToUpdate -= correction; + /* invalidate dictionaries on overflow correction */ + ms->loadedDictEnd = 0; + ms->dictMatchState = NULL; + } +} + + /*! ZSTD_compress_frameChunk() : * Compress a chunk of data into one or multiple blocks. * All blocks will be terminated, all input will be consumed. @@ -2844,7 +2912,7 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx, BYTE* const ostart = (BYTE*)dst; BYTE* op = ostart; U32 const maxDist = (U32)1 << cctx->appliedParams.cParams.windowLog; - assert(cctx->appliedParams.cParams.windowLog <= 31); + assert(cctx->appliedParams.cParams.windowLog <= ZSTD_WINDOWLOG_MAX); DEBUGLOG(5, "ZSTD_compress_frameChunk (blockSize=%u)", (unsigned)blockSize); if (cctx->appliedParams.fParams.checksumFlag && srcSize) @@ -2859,19 +2927,10 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx, "not enough space to store compressed block"); if (remaining < blockSize) blockSize = remaining; - if (ZSTD_window_needOverflowCorrection(ms->window, ip + blockSize)) { - U32 const cycleLog = ZSTD_cycleLog(cctx->appliedParams.cParams.chainLog, cctx->appliedParams.cParams.strategy); - U32 const correction = ZSTD_window_correctOverflow(&ms->window, cycleLog, maxDist, ip); - ZSTD_STATIC_ASSERT(ZSTD_CHAINLOG_MAX <= 30); - ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX_32 <= 30); - ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX <= 31); - ZSTD_reduceIndex(cctx, correction); - if (ms->nextToUpdate < correction) ms->nextToUpdate = 0; - else ms->nextToUpdate -= correction; - ms->loadedDictEnd = 0; - ms->dictMatchState = NULL; - } - ZSTD_window_enforceMaxDist(&ms->window, ip + blockSize, maxDist, &ms->loadedDictEnd, &ms->dictMatchState); + ZSTD_overflowCorrectIfNeeded(ms, &cctx->appliedParams, ip, ip + blockSize); + ZSTD_checkDictValidity(&ms->window, ip + blockSize, maxDist, &ms->loadedDictEnd, &ms->dictMatchState); + + /* Ensure hash/chain table insertion resumes no sooner than lowlimit */ if (ms->nextToUpdate < ms->window.lowLimit) ms->nextToUpdate = ms->window.lowLimit; { size_t cSize = ZSTD_compressBlock_internal(cctx, @@ -2899,7 +2958,7 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx, } } if (lastFrameChunk && (op>ostart)) cctx->stage = ZSTDcs_ending; - return op-ostart; + return (size_t)(op-ostart); } @@ -2991,6 +3050,7 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx, fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, cctx->appliedParams, cctx->pledgedSrcSizePlusOne-1, cctx->dictID); FORWARD_IF_ERROR(fhSize); + assert(fhSize <= dstCapacity); dstCapacity -= fhSize; dst = (char*)dst + fhSize; cctx->stage = ZSTDcs_ongoing; @@ -3007,18 +3067,7 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx, if (!frame) { /* overflow check and correction for block mode */ - if (ZSTD_window_needOverflowCorrection(ms->window, (const char*)src + srcSize)) { - U32 const cycleLog = ZSTD_cycleLog(cctx->appliedParams.cParams.chainLog, cctx->appliedParams.cParams.strategy); - U32 const correction = ZSTD_window_correctOverflow(&ms->window, cycleLog, 1 << cctx->appliedParams.cParams.windowLog, src); - ZSTD_STATIC_ASSERT(ZSTD_CHAINLOG_MAX <= 30); - ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX_32 <= 30); - ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX <= 31); - ZSTD_reduceIndex(cctx, correction); - if (ms->nextToUpdate < correction) ms->nextToUpdate = 0; - else ms->nextToUpdate -= correction; - ms->loadedDictEnd = 0; - ms->dictMatchState = NULL; - } + ZSTD_overflowCorrectIfNeeded(ms, &cctx->appliedParams, src, (BYTE const*)src + srcSize); } DEBUGLOG(5, "ZSTD_compressContinue_internal (blockSize=%u)", (unsigned)cctx->blockSize); @@ -3074,7 +3123,7 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms, const void* src, size_t srcSize, ZSTD_dictTableLoadMethod_e dtlm) { - const BYTE* const ip = (const BYTE*) src; + const BYTE* ip = (const BYTE*) src; const BYTE* const iend = ip + srcSize; ZSTD_window_update(&ms->window, src, srcSize); @@ -3085,32 +3134,42 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms, if (srcSize <= HASH_READ_SIZE) return 0; - switch(params->cParams.strategy) - { - case ZSTD_fast: - ZSTD_fillHashTable(ms, iend, dtlm); - break; - case ZSTD_dfast: - ZSTD_fillDoubleHashTable(ms, iend, dtlm); - break; + while (iend - ip > HASH_READ_SIZE) { + size_t const remaining = iend - ip; + size_t const chunk = MIN(remaining, ZSTD_CHUNKSIZE_MAX); + const BYTE* const ichunk = ip + chunk; - case ZSTD_greedy: - case ZSTD_lazy: - case ZSTD_lazy2: - if (srcSize >= HASH_READ_SIZE) - ZSTD_insertAndFindFirstIndex(ms, iend-HASH_READ_SIZE); - break; + ZSTD_overflowCorrectIfNeeded(ms, params, ip, ichunk); - case ZSTD_btlazy2: /* we want the dictionary table fully sorted */ - case ZSTD_btopt: - case ZSTD_btultra: - case ZSTD_btultra2: - if (srcSize >= HASH_READ_SIZE) - ZSTD_updateTree(ms, iend-HASH_READ_SIZE, iend); - break; + switch(params->cParams.strategy) + { + case ZSTD_fast: + ZSTD_fillHashTable(ms, ichunk, dtlm); + break; + case ZSTD_dfast: + ZSTD_fillDoubleHashTable(ms, ichunk, dtlm); + break; - default: - assert(0); /* not possible : not a valid strategy id */ + case ZSTD_greedy: + case ZSTD_lazy: + case ZSTD_lazy2: + if (chunk >= HASH_READ_SIZE) + ZSTD_insertAndFindFirstIndex(ms, ichunk-HASH_READ_SIZE); + break; + + case ZSTD_btlazy2: /* we want the dictionary table fully sorted */ + case ZSTD_btopt: + case ZSTD_btultra: + case ZSTD_btultra2: + if (chunk >= HASH_READ_SIZE) + ZSTD_updateTree(ms, ichunk-HASH_READ_SIZE, ichunk); + break; + + default: + assert(0); /* not possible : not a valid strategy id */ + } + + ip = ichunk; } ms->nextToUpdate = (U32)(iend - ms->window.base); @@ -3297,12 +3356,11 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, FORWARD_IF_ERROR( ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize, ZSTDcrp_continue, zbuff) ); - { - size_t const dictID = ZSTD_compress_insertDictionary( + { size_t const dictID = ZSTD_compress_insertDictionary( cctx->blockState.prevCBlock, &cctx->blockState.matchState, ¶ms, dict, dictSize, dictContentType, dtlm, cctx->entropyWorkspace); FORWARD_IF_ERROR(dictID); - assert(dictID <= (size_t)(U32)-1); + assert(dictID <= UINT_MAX); cctx->dictID = (U32)dictID; } return 0; @@ -3555,10 +3613,10 @@ static size_t ZSTD_initCDict_internal( /* Reset the state to no dictionary */ ZSTD_reset_compressedBlockState(&cdict->cBlockState); - { void* const end = ZSTD_reset_matchState( - &cdict->matchState, - (U32*)cdict->workspace + HUF_WORKSPACE_SIZE_U32, - &cParams, ZSTDcrp_continue, /* forCCtx */ 0); + { void* const end = ZSTD_reset_matchState(&cdict->matchState, + (U32*)cdict->workspace + HUF_WORKSPACE_SIZE_U32, + &cParams, + ZSTDcrp_continue, ZSTD_resetTarget_CDict); assert(end == (char*)cdict->workspace + cdict->workspaceSize); (void)end; } @@ -4068,7 +4126,7 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, case zcss_flush: DEBUGLOG(5, "flush stage"); { size_t const toFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize; - size_t const flushed = ZSTD_limitCopy(op, oend-op, + size_t const flushed = ZSTD_limitCopy(op, (size_t)(oend-op), zcs->outBuff + zcs->outBuffFlushedSize, toFlush); DEBUGLOG(5, "toFlush: %u into %u ==> flushed: %u", (unsigned)toFlush, (unsigned)(oend-op), (unsigned)flushed); @@ -4262,7 +4320,7 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output) if (zcs->appliedParams.nbWorkers > 0) return remainingToFlush; /* minimal estimation */ /* single thread mode : attempt to calculate remaining to flush more precisely */ { size_t const lastBlockSize = zcs->frameEnded ? 0 : ZSTD_BLOCKHEADERSIZE; - size_t const checksumSize = zcs->frameEnded ? 0 : zcs->appliedParams.fParams.checksumFlag * 4; + size_t const checksumSize = (size_t)(zcs->frameEnded ? 0 : zcs->appliedParams.fParams.checksumFlag * 4); size_t const toFlush = remainingToFlush + lastBlockSize + checksumSize; DEBUGLOG(4, "ZSTD_endStream : remaining to flush : %u", (unsigned)toFlush); return toFlush; diff --git a/thirdparty/zstd/compress/zstd_compress_internal.h b/thirdparty/zstd/compress/zstd_compress_internal.h index cc3cbb9da9..5495899be3 100644 --- a/thirdparty/zstd/compress/zstd_compress_internal.h +++ b/thirdparty/zstd/compress/zstd_compress_internal.h @@ -33,13 +33,13 @@ extern "C" { ***************************************/ #define kSearchStrength 8 #define HASH_READ_SIZE 8 -#define ZSTD_DUBT_UNSORTED_MARK 1 /* For btlazy2 strategy, index 1 now means "unsorted". +#define ZSTD_DUBT_UNSORTED_MARK 1 /* For btlazy2 strategy, index ZSTD_DUBT_UNSORTED_MARK==1 means "unsorted". It could be confused for a real successor at index "1", if sorted as larger than its predecessor. It's not a big deal though : candidate will just be sorted again. Additionally, candidate position 1 will be lost. But candidate 1 cannot hide a large tree of candidates, so it's a minimal loss. - The benefit is that ZSTD_DUBT_UNSORTED_MARK cannot be mishandled after table re-use with a different strategy - Constant required by ZSTD_compressBlock_btlazy2() and ZSTD_reduceTable_internal() */ + The benefit is that ZSTD_DUBT_UNSORTED_MARK cannot be mishandled after table re-use with a different strategy. + This constant is required by ZSTD_compressBlock_btlazy2() and ZSTD_reduceTable_internal() */ /*-************************************* @@ -128,21 +128,20 @@ typedef struct { BYTE const* base; /* All regular indexes relative to this position */ BYTE const* dictBase; /* extDict indexes relative to this position */ U32 dictLimit; /* below that point, need extDict */ - U32 lowLimit; /* below that point, no more data */ + U32 lowLimit; /* below that point, no more valid data */ } ZSTD_window_t; typedef struct ZSTD_matchState_t ZSTD_matchState_t; struct ZSTD_matchState_t { ZSTD_window_t window; /* State for window round buffer management */ - U32 loadedDictEnd; /* index of end of dictionary */ + U32 loadedDictEnd; /* index of end of dictionary, within context's referential. When dict referential is copied into active context (i.e. not attached), effectively same value as dictSize, since referential starts from zero */ U32 nextToUpdate; /* index from which to continue table update */ - U32 nextToUpdate3; /* index from which to continue table update */ U32 hashLog3; /* dispatch table : larger == faster, more memory */ U32* hashTable; U32* hashTable3; U32* chainTable; optState_t opt; /* optimal parser state */ - const ZSTD_matchState_t * dictMatchState; + const ZSTD_matchState_t* dictMatchState; ZSTD_compressionParameters cParams; }; @@ -195,6 +194,9 @@ struct ZSTD_CCtx_params_s { int compressionLevel; int forceWindow; /* force back-references to respect limit of * 1<<wLog, even for dictionary */ + size_t targetCBlockSize; /* Tries to fit compressed block size to be around targetCBlockSize. + * No target when targetCBlockSize == 0. + * There is no guarantee on compressed block size */ ZSTD_dictAttachPref_e attachDictPref; ZSTD_literalCompressionMode_e literalCompressionMode; @@ -324,7 +326,7 @@ MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const v /* copy Literals */ assert(seqStorePtr->maxNbLit <= 128 KB); assert(seqStorePtr->lit + litLength <= seqStorePtr->litStart + seqStorePtr->maxNbLit); - ZSTD_wildcopy(seqStorePtr->lit, literals, litLength); + ZSTD_wildcopy(seqStorePtr->lit, literals, litLength, ZSTD_no_overlap); seqStorePtr->lit += litLength; /* literal Length */ @@ -564,6 +566,9 @@ MEM_STATIC U64 ZSTD_rollingHash_rotate(U64 hash, BYTE toRemove, BYTE toAdd, U64 /*-************************************* * Round buffer management ***************************************/ +#if (ZSTD_WINDOWLOG_MAX_64 > 31) +# error "ZSTD_WINDOWLOG_MAX is too large : would overflow ZSTD_CURRENT_MAX" +#endif /* Max current allowed */ #define ZSTD_CURRENT_MAX ((3U << 29) + (1U << ZSTD_WINDOWLOG_MAX)) /* Maximum chunk size before overflow correction needs to be called again */ @@ -675,31 +680,49 @@ MEM_STATIC U32 ZSTD_window_correctOverflow(ZSTD_window_t* window, U32 cycleLog, * Updates lowLimit so that: * (srcEnd - base) - lowLimit == maxDist + loadedDictEnd * - * This allows a simple check that index >= lowLimit to see if index is valid. - * This must be called before a block compression call, with srcEnd as the block - * source end. + * It ensures index is valid as long as index >= lowLimit. + * This must be called before a block compression call. + * + * loadedDictEnd is only defined if a dictionary is in use for current compression. + * As the name implies, loadedDictEnd represents the index at end of dictionary. + * The value lies within context's referential, it can be directly compared to blockEndIdx. * - * If loadedDictEndPtr is not NULL, we set it to zero once we update lowLimit. - * This is because dictionaries are allowed to be referenced as long as the last - * byte of the dictionary is in the window, but once they are out of range, - * they cannot be referenced. If loadedDictEndPtr is NULL, we use - * loadedDictEnd == 0. + * If loadedDictEndPtr is NULL, no dictionary is in use, and we use loadedDictEnd == 0. + * If loadedDictEndPtr is not NULL, we set it to zero after updating lowLimit. + * This is because dictionaries are allowed to be referenced fully + * as long as the last byte of the dictionary is in the window. + * Once input has progressed beyond window size, dictionary cannot be referenced anymore. * - * In normal dict mode, the dict is between lowLimit and dictLimit. In - * dictMatchState mode, lowLimit and dictLimit are the same, and the dictionary - * is below them. forceWindow and dictMatchState are therefore incompatible. + * In normal dict mode, the dictionary lies between lowLimit and dictLimit. + * In dictMatchState mode, lowLimit and dictLimit are the same, + * and the dictionary is below them. + * forceWindow and dictMatchState are therefore incompatible. */ MEM_STATIC void ZSTD_window_enforceMaxDist(ZSTD_window_t* window, - void const* srcEnd, - U32 maxDist, - U32* loadedDictEndPtr, + const void* blockEnd, + U32 maxDist, + U32* loadedDictEndPtr, const ZSTD_matchState_t** dictMatchStatePtr) { - U32 const blockEndIdx = (U32)((BYTE const*)srcEnd - window->base); - U32 loadedDictEnd = (loadedDictEndPtr != NULL) ? *loadedDictEndPtr : 0; - DEBUGLOG(5, "ZSTD_window_enforceMaxDist: blockEndIdx=%u, maxDist=%u", - (unsigned)blockEndIdx, (unsigned)maxDist); + U32 const blockEndIdx = (U32)((BYTE const*)blockEnd - window->base); + U32 const loadedDictEnd = (loadedDictEndPtr != NULL) ? *loadedDictEndPtr : 0; + DEBUGLOG(5, "ZSTD_window_enforceMaxDist: blockEndIdx=%u, maxDist=%u, loadedDictEnd=%u", + (unsigned)blockEndIdx, (unsigned)maxDist, (unsigned)loadedDictEnd); + + /* - When there is no dictionary : loadedDictEnd == 0. + In which case, the test (blockEndIdx > maxDist) is merely to avoid + overflowing next operation `newLowLimit = blockEndIdx - maxDist`. + - When there is a standard dictionary : + Index referential is copied from the dictionary, + which means it starts from 0. + In which case, loadedDictEnd == dictSize, + and it makes sense to compare `blockEndIdx > maxDist + dictSize` + since `blockEndIdx` also starts from zero. + - When there is an attached dictionary : + loadedDictEnd is expressed within the referential of the context, + so it can be directly compared against blockEndIdx. + */ if (blockEndIdx > maxDist + loadedDictEnd) { U32 const newLowLimit = blockEndIdx - maxDist; if (window->lowLimit < newLowLimit) window->lowLimit = newLowLimit; @@ -708,10 +731,31 @@ ZSTD_window_enforceMaxDist(ZSTD_window_t* window, (unsigned)window->dictLimit, (unsigned)window->lowLimit); window->dictLimit = window->lowLimit; } - if (loadedDictEndPtr) - *loadedDictEndPtr = 0; - if (dictMatchStatePtr) - *dictMatchStatePtr = NULL; + /* On reaching window size, dictionaries are invalidated */ + if (loadedDictEndPtr) *loadedDictEndPtr = 0; + if (dictMatchStatePtr) *dictMatchStatePtr = NULL; + } +} + +/* Similar to ZSTD_window_enforceMaxDist(), + * but only invalidates dictionary + * when input progresses beyond window size. */ +MEM_STATIC void +ZSTD_checkDictValidity(ZSTD_window_t* window, + const void* blockEnd, + U32 maxDist, + U32* loadedDictEndPtr, + const ZSTD_matchState_t** dictMatchStatePtr) +{ + U32 const blockEndIdx = (U32)((BYTE const*)blockEnd - window->base); + U32 const loadedDictEnd = (loadedDictEndPtr != NULL) ? *loadedDictEndPtr : 0; + DEBUGLOG(5, "ZSTD_checkDictValidity: blockEndIdx=%u, maxDist=%u, loadedDictEnd=%u", + (unsigned)blockEndIdx, (unsigned)maxDist, (unsigned)loadedDictEnd); + + if (loadedDictEnd && (blockEndIdx > maxDist + loadedDictEnd)) { + /* On reaching window size, dictionaries are invalidated */ + if (loadedDictEndPtr) *loadedDictEndPtr = 0; + if (dictMatchStatePtr) *dictMatchStatePtr = NULL; } } diff --git a/thirdparty/zstd/compress/zstd_double_fast.c b/thirdparty/zstd/compress/zstd_double_fast.c index 47faf6d641..5957255d90 100644 --- a/thirdparty/zstd/compress/zstd_double_fast.c +++ b/thirdparty/zstd/compress/zstd_double_fast.c @@ -43,8 +43,7 @@ void ZSTD_fillDoubleHashTable(ZSTD_matchState_t* ms, /* Only load extra positions for ZSTD_dtlm_full */ if (dtlm == ZSTD_dtlm_fast) break; - } - } + } } } @@ -63,7 +62,10 @@ size_t ZSTD_compressBlock_doubleFast_generic( const BYTE* const istart = (const BYTE*)src; const BYTE* ip = istart; const BYTE* anchor = istart; - const U32 prefixLowestIndex = ms->window.dictLimit; + const U32 endIndex = (U32)((size_t)(istart - base) + srcSize); + const U32 lowestValid = ms->window.dictLimit; + const U32 maxDistance = 1U << cParams->windowLog; + const U32 prefixLowestIndex = (endIndex - lowestValid > maxDistance) ? endIndex - maxDistance : lowestValid; const BYTE* const prefixLowest = base + prefixLowestIndex; const BYTE* const iend = istart + srcSize; const BYTE* const ilimit = iend - HASH_READ_SIZE; @@ -95,8 +97,15 @@ size_t ZSTD_compressBlock_doubleFast_generic( dictCParams->chainLog : hBitsS; const U32 dictAndPrefixLength = (U32)(ip - prefixLowest + dictEnd - dictStart); + DEBUGLOG(5, "ZSTD_compressBlock_doubleFast_generic"); + assert(dictMode == ZSTD_noDict || dictMode == ZSTD_dictMatchState); + /* if a dictionary is attached, it must be within window range */ + if (dictMode == ZSTD_dictMatchState) { + assert(lowestValid + maxDistance >= endIndex); + } + /* init */ ip += (dictAndPrefixLength == 0); if (dictMode == ZSTD_noDict) { @@ -138,7 +147,7 @@ size_t ZSTD_compressBlock_doubleFast_generic( const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend; mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4; ip++; - ZSTD_storeSeq(seqStore, ip-anchor, anchor, 0, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, 0, mLength-MINMATCH); goto _match_stored; } @@ -147,7 +156,7 @@ size_t ZSTD_compressBlock_doubleFast_generic( && ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1)))) { mLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4; ip++; - ZSTD_storeSeq(seqStore, ip-anchor, anchor, 0, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, 0, mLength-MINMATCH); goto _match_stored; } @@ -170,8 +179,7 @@ size_t ZSTD_compressBlock_doubleFast_generic( offset = (U32)(current - dictMatchIndexL - dictIndexDelta); while (((ip>anchor) & (dictMatchL>dictStart)) && (ip[-1] == dictMatchL[-1])) { ip--; dictMatchL--; mLength++; } /* catch up */ goto _match_found; - } - } + } } if (matchIndexS > prefixLowestIndex) { /* check prefix short match */ @@ -186,16 +194,14 @@ size_t ZSTD_compressBlock_doubleFast_generic( if (match > dictStart && MEM_read32(match) == MEM_read32(ip)) { goto _search_next_long; - } - } + } } ip += ((ip-anchor) >> kSearchStrength) + 1; continue; _search_next_long: - { - size_t const hl3 = ZSTD_hashPtr(ip+1, hBitsL, 8); + { size_t const hl3 = ZSTD_hashPtr(ip+1, hBitsL, 8); size_t const dictHLNext = ZSTD_hashPtr(ip+1, dictHBitsL, 8); U32 const matchIndexL3 = hashLong[hl3]; const BYTE* matchL3 = base + matchIndexL3; @@ -221,9 +227,7 @@ _search_next_long: offset = (U32)(current + 1 - dictMatchIndexL3 - dictIndexDelta); while (((ip>anchor) & (dictMatchL3>dictStart)) && (ip[-1] == dictMatchL3[-1])) { ip--; dictMatchL3--; mLength++; } /* catch up */ goto _match_found; - } - } - } + } } } /* if no long +1 match, explore the short match we found */ if (dictMode == ZSTD_dictMatchState && matchIndexS < prefixLowestIndex) { @@ -242,7 +246,7 @@ _match_found: offset_2 = offset_1; offset_1 = offset; - ZSTD_storeSeq(seqStore, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); _match_stored: /* match found */ @@ -250,11 +254,14 @@ _match_stored: anchor = ip; if (ip <= ilimit) { - /* Fill Table */ - hashLong[ZSTD_hashPtr(base+current+2, hBitsL, 8)] = - hashSmall[ZSTD_hashPtr(base+current+2, hBitsS, mls)] = current+2; /* here because current+2 could be > iend-8 */ - hashLong[ZSTD_hashPtr(ip-2, hBitsL, 8)] = - hashSmall[ZSTD_hashPtr(ip-2, hBitsS, mls)] = (U32)(ip-2-base); + /* Complementary insertion */ + /* done after iLimit test, as candidates could be > iend-8 */ + { U32 const indexToInsert = current+2; + hashLong[ZSTD_hashPtr(base+indexToInsert, hBitsL, 8)] = indexToInsert; + hashLong[ZSTD_hashPtr(ip-2, hBitsL, 8)] = (U32)(ip-2-base); + hashSmall[ZSTD_hashPtr(base+indexToInsert, hBitsS, mls)] = indexToInsert; + hashSmall[ZSTD_hashPtr(ip-1, hBitsS, mls)] = (U32)(ip-1-base); + } /* check immediate repcode */ if (dictMode == ZSTD_dictMatchState) { @@ -278,8 +285,7 @@ _match_stored: continue; } break; - } - } + } } if (dictMode == ZSTD_noDict) { while ( (ip <= ilimit) @@ -294,14 +300,15 @@ _match_stored: ip += rLength; anchor = ip; continue; /* faster when present ... (?) */ - } } } } + } } } + } /* while (ip < ilimit) */ /* save reps for next block */ rep[0] = offset_1 ? offset_1 : offsetSaved; rep[1] = offset_2 ? offset_2 : offsetSaved; /* Return the last literals size */ - return iend - anchor; + return (size_t)(iend - anchor); } @@ -360,10 +367,15 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic( const BYTE* anchor = istart; const BYTE* const iend = istart + srcSize; const BYTE* const ilimit = iend - 8; - const U32 prefixStartIndex = ms->window.dictLimit; const BYTE* const base = ms->window.base; + const U32 endIndex = (U32)((size_t)(istart - base) + srcSize); + const U32 maxDistance = 1U << cParams->windowLog; + const U32 lowestValid = ms->window.lowLimit; + const U32 lowLimit = (endIndex - lowestValid > maxDistance) ? endIndex - maxDistance : lowestValid; + const U32 dictStartIndex = lowLimit; + const U32 dictLimit = ms->window.dictLimit; + const U32 prefixStartIndex = (dictLimit > lowLimit) ? dictLimit : lowLimit; const BYTE* const prefixStart = base + prefixStartIndex; - const U32 dictStartIndex = ms->window.lowLimit; const BYTE* const dictBase = ms->window.dictBase; const BYTE* const dictStart = dictBase + dictStartIndex; const BYTE* const dictEnd = dictBase + prefixStartIndex; @@ -371,6 +383,10 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic( DEBUGLOG(5, "ZSTD_compressBlock_doubleFast_extDict_generic (srcSize=%zu)", srcSize); + /* if extDict is invalidated due to maxDistance, switch to "regular" variant */ + if (prefixStartIndex == dictStartIndex) + return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, mls, ZSTD_noDict); + /* Search Loop */ while (ip < ilimit) { /* < instead of <=, because (ip+1) */ const size_t hSmall = ZSTD_hashPtr(ip, hBitsS, mls); @@ -396,7 +412,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic( const BYTE* repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend; mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixStart) + 4; ip++; - ZSTD_storeSeq(seqStore, ip-anchor, anchor, 0, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, 0, mLength-MINMATCH); } else { if ((matchLongIndex > dictStartIndex) && (MEM_read64(matchLong) == MEM_read64(ip))) { const BYTE* const matchEnd = matchLongIndex < prefixStartIndex ? dictEnd : iend; @@ -407,7 +423,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic( while (((ip>anchor) & (matchLong>lowMatchPtr)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */ offset_2 = offset_1; offset_1 = offset; - ZSTD_storeSeq(seqStore, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); } else if ((matchIndex > dictStartIndex) && (MEM_read32(match) == MEM_read32(ip))) { size_t const h3 = ZSTD_hashPtr(ip+1, hBitsL, 8); @@ -432,23 +448,27 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic( } offset_2 = offset_1; offset_1 = offset; - ZSTD_storeSeq(seqStore, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); } else { ip += ((ip-anchor) >> kSearchStrength) + 1; continue; } } - /* found a match : store it */ + /* move to next sequence start */ ip += mLength; anchor = ip; if (ip <= ilimit) { - /* Fill Table */ - hashSmall[ZSTD_hashPtr(base+current+2, hBitsS, mls)] = current+2; - hashLong[ZSTD_hashPtr(base+current+2, hBitsL, 8)] = current+2; - hashSmall[ZSTD_hashPtr(ip-2, hBitsS, mls)] = (U32)(ip-2-base); - hashLong[ZSTD_hashPtr(ip-2, hBitsL, 8)] = (U32)(ip-2-base); + /* Complementary insertion */ + /* done after iLimit test, as candidates could be > iend-8 */ + { U32 const indexToInsert = current+2; + hashLong[ZSTD_hashPtr(base+indexToInsert, hBitsL, 8)] = indexToInsert; + hashLong[ZSTD_hashPtr(ip-2, hBitsL, 8)] = (U32)(ip-2-base); + hashSmall[ZSTD_hashPtr(base+indexToInsert, hBitsS, mls)] = indexToInsert; + hashSmall[ZSTD_hashPtr(ip-1, hBitsS, mls)] = (U32)(ip-1-base); + } + /* check immediate repcode */ while (ip <= ilimit) { U32 const current2 = (U32)(ip-base); @@ -475,7 +495,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic( rep[1] = offset_2; /* Return the last literals size */ - return iend - anchor; + return (size_t)(iend - anchor); } diff --git a/thirdparty/zstd/compress/zstd_fast.c b/thirdparty/zstd/compress/zstd_fast.c index ed997b441c..a05b8a47f1 100644 --- a/thirdparty/zstd/compress/zstd_fast.c +++ b/thirdparty/zstd/compress/zstd_fast.c @@ -13,7 +13,8 @@ void ZSTD_fillHashTable(ZSTD_matchState_t* ms, - void const* end, ZSTD_dictTableLoadMethod_e dtlm) + const void* const end, + ZSTD_dictTableLoadMethod_e dtlm) { const ZSTD_compressionParameters* const cParams = &ms->cParams; U32* const hashTable = ms->hashTable; @@ -41,6 +42,7 @@ void ZSTD_fillHashTable(ZSTD_matchState_t* ms, } } } } } + FORCE_INLINE_TEMPLATE size_t ZSTD_compressBlock_fast_generic( ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], @@ -58,7 +60,10 @@ size_t ZSTD_compressBlock_fast_generic( const BYTE* ip0 = istart; const BYTE* ip1; const BYTE* anchor = istart; - const U32 prefixStartIndex = ms->window.dictLimit; + const U32 endIndex = (U32)((size_t)(istart - base) + srcSize); + const U32 maxDistance = 1U << cParams->windowLog; + const U32 validStartIndex = ms->window.dictLimit; + const U32 prefixStartIndex = (endIndex - validStartIndex > maxDistance) ? endIndex - maxDistance : validStartIndex; const BYTE* const prefixStart = base + prefixStartIndex; const BYTE* const iend = istart + srcSize; const BYTE* const ilimit = iend - HASH_READ_SIZE; @@ -165,7 +170,7 @@ _match: /* Requires: ip0, match0, offcode */ rep[1] = offset_2 ? offset_2 : offsetSaved; /* Return the last literals size */ - return iend - anchor; + return (size_t)(iend - anchor); } @@ -222,8 +227,15 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic( const U32 dictAndPrefixLength = (U32)(ip - prefixStart + dictEnd - dictStart); const U32 dictHLog = dictCParams->hashLog; - /* otherwise, we would get index underflow when translating a dict index - * into a local index */ + /* if a dictionary is still attached, it necessarily means that + * it is within window size. So we just check it. */ + const U32 maxDistance = 1U << cParams->windowLog; + const U32 endIndex = (U32)((size_t)(ip - base) + srcSize); + assert(endIndex - prefixStartIndex <= maxDistance); + (void)maxDistance; (void)endIndex; /* these variables are not used when assert() is disabled */ + + /* ensure there will be no no underflow + * when translating a dict index into a local index */ assert(prefixStartIndex >= (U32)(dictEnd - dictBase)); /* init */ @@ -251,7 +263,7 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic( const BYTE* const repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend; mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixStart) + 4; ip++; - ZSTD_storeSeq(seqStore, ip-anchor, anchor, 0, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, 0, mLength-MINMATCH); } else if ( (matchIndex <= prefixStartIndex) ) { size_t const dictHash = ZSTD_hashPtr(ip, dictHLog, mls); U32 const dictMatchIndex = dictHashTable[dictHash]; @@ -271,7 +283,7 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic( } /* catch up */ offset_2 = offset_1; offset_1 = offset; - ZSTD_storeSeq(seqStore, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); } } else if (MEM_read32(match) != MEM_read32(ip)) { /* it's not a match, and we're not going to check the dictionary */ @@ -286,7 +298,7 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic( && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ offset_2 = offset_1; offset_1 = offset; - ZSTD_storeSeq(seqStore, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); } /* match found */ @@ -327,7 +339,7 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic( rep[1] = offset_2 ? offset_2 : offsetSaved; /* Return the last literals size */ - return iend - anchor; + return (size_t)(iend - anchor); } size_t ZSTD_compressBlock_fast_dictMatchState( @@ -366,15 +378,24 @@ static size_t ZSTD_compressBlock_fast_extDict_generic( const BYTE* const istart = (const BYTE*)src; const BYTE* ip = istart; const BYTE* anchor = istart; - const U32 dictStartIndex = ms->window.lowLimit; + const U32 endIndex = (U32)((size_t)(istart - base) + srcSize); + const U32 maxDistance = 1U << cParams->windowLog; + const U32 validLow = ms->window.lowLimit; + const U32 lowLimit = (endIndex - validLow > maxDistance) ? endIndex - maxDistance : validLow; + const U32 dictStartIndex = lowLimit; const BYTE* const dictStart = dictBase + dictStartIndex; - const U32 prefixStartIndex = ms->window.dictLimit; + const U32 dictLimit = ms->window.dictLimit; + const U32 prefixStartIndex = dictLimit < lowLimit ? lowLimit : dictLimit; const BYTE* const prefixStart = base + prefixStartIndex; const BYTE* const dictEnd = dictBase + prefixStartIndex; const BYTE* const iend = istart + srcSize; const BYTE* const ilimit = iend - 8; U32 offset_1=rep[0], offset_2=rep[1]; + /* switch to "regular" variant if extDict is invalidated due to maxDistance */ + if (prefixStartIndex == dictStartIndex) + return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, mls); + /* Search Loop */ while (ip < ilimit) { /* < instead of <=, because (ip+1) */ const size_t h = ZSTD_hashPtr(ip, hlog, mls); @@ -394,7 +415,7 @@ static size_t ZSTD_compressBlock_fast_extDict_generic( const BYTE* repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend; mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixStart) + 4; ip++; - ZSTD_storeSeq(seqStore, ip-anchor, anchor, 0, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, 0, mLength-MINMATCH); } else { if ( (matchIndex < dictStartIndex) || (MEM_read32(match) != MEM_read32(ip)) ) { @@ -410,7 +431,7 @@ static size_t ZSTD_compressBlock_fast_extDict_generic( offset = current - matchIndex; offset_2 = offset_1; offset_1 = offset; - ZSTD_storeSeq(seqStore, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); } } /* found a match : store it */ @@ -445,7 +466,7 @@ static size_t ZSTD_compressBlock_fast_extDict_generic( rep[1] = offset_2; /* Return the last literals size */ - return iend - anchor; + return (size_t)(iend - anchor); } diff --git a/thirdparty/zstd/compress/zstd_lazy.c b/thirdparty/zstd/compress/zstd_lazy.c index 53f998a437..94d906c01f 100644 --- a/thirdparty/zstd/compress/zstd_lazy.c +++ b/thirdparty/zstd/compress/zstd_lazy.c @@ -83,7 +83,10 @@ ZSTD_insertDUBT1(ZSTD_matchState_t* ms, U32* largerPtr = smallerPtr + 1; U32 matchIndex = *smallerPtr; /* this candidate is unsorted : next sorted candidate is reached through *smallerPtr, while *largerPtr contains previous unsorted candidate (which is already saved and can be overwritten) */ U32 dummy32; /* to be nullified at the end */ - U32 const windowLow = ms->window.lowLimit; + U32 const windowValid = ms->window.lowLimit; + U32 const maxDistance = 1U << cParams->windowLog; + U32 const windowLow = (current - windowValid > maxDistance) ? current - maxDistance : windowValid; + DEBUGLOG(8, "ZSTD_insertDUBT1(%u) (dictLimit=%u, lowLimit=%u)", current, dictLimit, windowLow); @@ -239,7 +242,9 @@ ZSTD_DUBT_findBestMatch(ZSTD_matchState_t* ms, const BYTE* const base = ms->window.base; U32 const current = (U32)(ip-base); - U32 const windowLow = ms->window.lowLimit; + U32 const maxDistance = 1U << cParams->windowLog; + U32 const windowValid = ms->window.lowLimit; + U32 const windowLow = (current - windowValid > maxDistance) ? current - maxDistance : windowValid; U32* const bt = ms->chainTable; U32 const btLog = cParams->chainLog - 1; @@ -490,8 +495,10 @@ size_t ZSTD_HcFindBestMatch_generic ( const U32 dictLimit = ms->window.dictLimit; const BYTE* const prefixStart = base + dictLimit; const BYTE* const dictEnd = dictBase + dictLimit; - const U32 lowLimit = ms->window.lowLimit; const U32 current = (U32)(ip-base); + const U32 maxDistance = 1U << cParams->windowLog; + const U32 lowValid = ms->window.lowLimit; + const U32 lowLimit = (current - lowValid > maxDistance) ? current - maxDistance : lowValid; const U32 minChain = current > chainSize ? current - chainSize : 0; U32 nbAttempts = 1U << cParams->searchLog; size_t ml=4-1; @@ -653,7 +660,6 @@ size_t ZSTD_compressBlock_lazy_generic( /* init */ ip += (dictAndPrefixLength == 0); - ms->nextToUpdate3 = ms->nextToUpdate; if (dictMode == ZSTD_noDict) { U32 const maxRep = (U32)(ip - prefixLowest); if (offset_2 > maxRep) savedOffset = offset_2, offset_2 = 0; @@ -933,7 +939,6 @@ size_t ZSTD_compressBlock_lazy_extDict_generic( U32 offset_1 = rep[0], offset_2 = rep[1]; /* init */ - ms->nextToUpdate3 = ms->nextToUpdate; ip += (ip == prefixStart); /* Match Loop */ diff --git a/thirdparty/zstd/compress/zstd_ldm.c b/thirdparty/zstd/compress/zstd_ldm.c index 784d20f3ab..3dcf86e6e8 100644 --- a/thirdparty/zstd/compress/zstd_ldm.c +++ b/thirdparty/zstd/compress/zstd_ldm.c @@ -447,7 +447,7 @@ size_t ZSTD_ldm_generateSequences( if (ZSTD_window_needOverflowCorrection(ldmState->window, chunkEnd)) { U32 const ldmHSize = 1U << params->hashLog; U32 const correction = ZSTD_window_correctOverflow( - &ldmState->window, /* cycleLog */ 0, maxDist, src); + &ldmState->window, /* cycleLog */ 0, maxDist, chunkStart); ZSTD_ldm_reduceTable(ldmState->hashTable, ldmHSize, correction); } /* 2. We enforce the maximum offset allowed. diff --git a/thirdparty/zstd/compress/zstd_opt.c b/thirdparty/zstd/compress/zstd_opt.c index efb69d3267..e32e542e02 100644 --- a/thirdparty/zstd/compress/zstd_opt.c +++ b/thirdparty/zstd/compress/zstd_opt.c @@ -255,13 +255,13 @@ static U32 ZSTD_litLengthPrice(U32 const litLength, const optState_t* const optP * to provide a cost which is directly comparable to a match ending at same position */ static int ZSTD_litLengthContribution(U32 const litLength, const optState_t* const optPtr, int optLevel) { - if (optPtr->priceType >= zop_predef) return WEIGHT(litLength, optLevel); + if (optPtr->priceType >= zop_predef) return (int)WEIGHT(litLength, optLevel); /* dynamic statistics */ { U32 const llCode = ZSTD_LLcode(litLength); - int const contribution = (LL_bits[llCode] * BITCOST_MULTIPLIER) - + WEIGHT(optPtr->litLengthFreq[0], optLevel) /* note: log2litLengthSum cancel out */ - - WEIGHT(optPtr->litLengthFreq[llCode], optLevel); + int const contribution = (int)(LL_bits[llCode] * BITCOST_MULTIPLIER) + + (int)WEIGHT(optPtr->litLengthFreq[0], optLevel) /* note: log2litLengthSum cancel out */ + - (int)WEIGHT(optPtr->litLengthFreq[llCode], optLevel); #if 1 return contribution; #else @@ -278,7 +278,7 @@ static int ZSTD_literalsContribution(const BYTE* const literals, U32 const litLe const optState_t* const optPtr, int optLevel) { - int const contribution = ZSTD_rawLiteralsCost(literals, litLength, optPtr, optLevel) + int const contribution = (int)ZSTD_rawLiteralsCost(literals, litLength, optPtr, optLevel) + ZSTD_litLengthContribution(litLength, optPtr, optLevel); return contribution; } @@ -372,13 +372,15 @@ MEM_STATIC U32 ZSTD_readMINMATCH(const void* memPtr, U32 length) /* Update hashTable3 up to ip (excluded) Assumption : always within prefix (i.e. not within extDict) */ -static U32 ZSTD_insertAndFindFirstIndexHash3 (ZSTD_matchState_t* ms, const BYTE* const ip) +static U32 ZSTD_insertAndFindFirstIndexHash3 (ZSTD_matchState_t* ms, + U32* nextToUpdate3, + const BYTE* const ip) { U32* const hashTable3 = ms->hashTable3; U32 const hashLog3 = ms->hashLog3; const BYTE* const base = ms->window.base; - U32 idx = ms->nextToUpdate3; - U32 const target = ms->nextToUpdate3 = (U32)(ip - base); + U32 idx = *nextToUpdate3; + U32 const target = (U32)(ip - base); size_t const hash3 = ZSTD_hash3Ptr(ip, hashLog3); assert(hashLog3 > 0); @@ -387,6 +389,7 @@ static U32 ZSTD_insertAndFindFirstIndexHash3 (ZSTD_matchState_t* ms, const BYTE* idx++; } + *nextToUpdate3 = target; return hashTable3[hash3]; } @@ -503,9 +506,11 @@ static U32 ZSTD_insertBt1( } } *smallerPtr = *largerPtr = 0; - if (bestLength > 384) return MIN(192, (U32)(bestLength - 384)); /* speed optimization */ - assert(matchEndIdx > current + 8); - return matchEndIdx - (current + 8); + { U32 positions = 0; + if (bestLength > 384) positions = MIN(192, (U32)(bestLength - 384)); /* speed optimization */ + assert(matchEndIdx > current + 8); + return MAX(positions, matchEndIdx - (current + 8)); + } } FORCE_INLINE_TEMPLATE @@ -520,8 +525,13 @@ void ZSTD_updateTree_internal( DEBUGLOG(6, "ZSTD_updateTree_internal, from %u to %u (dictMode:%u)", idx, target, dictMode); - while(idx < target) - idx += ZSTD_insertBt1(ms, base+idx, iend, mls, dictMode == ZSTD_extDict); + while(idx < target) { + U32 const forward = ZSTD_insertBt1(ms, base+idx, iend, mls, dictMode == ZSTD_extDict); + assert(idx < (U32)(idx + forward)); + idx += forward; + } + assert((size_t)(ip - base) <= (size_t)(U32)(-1)); + assert((size_t)(iend - base) <= (size_t)(U32)(-1)); ms->nextToUpdate = target; } @@ -531,16 +541,18 @@ void ZSTD_updateTree(ZSTD_matchState_t* ms, const BYTE* ip, const BYTE* iend) { FORCE_INLINE_TEMPLATE U32 ZSTD_insertBtAndGetAllMatches ( + ZSTD_match_t* matches, /* store result (found matches) in this table (presumed large enough) */ ZSTD_matchState_t* ms, + U32* nextToUpdate3, const BYTE* const ip, const BYTE* const iLimit, const ZSTD_dictMode_e dictMode, - U32 rep[ZSTD_REP_NUM], + const U32 rep[ZSTD_REP_NUM], U32 const ll0, /* tells if associated literal length is 0 or not. This value must be 0 or 1 */ - ZSTD_match_t* matches, const U32 lengthToBeat, U32 const mls /* template */) { const ZSTD_compressionParameters* const cParams = &ms->cParams; U32 const sufficient_len = MIN(cParams->targetLength, ZSTD_OPT_NUM -1); + U32 const maxDistance = 1U << cParams->windowLog; const BYTE* const base = ms->window.base; U32 const current = (U32)(ip-base); U32 const hashLog = cParams->hashLog; @@ -556,8 +568,9 @@ U32 ZSTD_insertBtAndGetAllMatches ( U32 const dictLimit = ms->window.dictLimit; const BYTE* const dictEnd = dictBase + dictLimit; const BYTE* const prefixStart = base + dictLimit; - U32 const btLow = btMask >= current ? 0 : current - btMask; - U32 const windowLow = ms->window.lowLimit; + U32 const btLow = (btMask >= current) ? 0 : current - btMask; + U32 const windowValid = ms->window.lowLimit; + U32 const windowLow = ((current - windowValid) > maxDistance) ? current - maxDistance : windowValid; U32 const matchLow = windowLow ? windowLow : 1; U32* smallerPtr = bt + 2*(current&btMask); U32* largerPtr = bt + 2*(current&btMask) + 1; @@ -627,7 +640,7 @@ U32 ZSTD_insertBtAndGetAllMatches ( /* HC3 match finder */ if ((mls == 3) /*static*/ && (bestLength < mls)) { - U32 const matchIndex3 = ZSTD_insertAndFindFirstIndexHash3(ms, ip); + U32 const matchIndex3 = ZSTD_insertAndFindFirstIndexHash3(ms, nextToUpdate3, ip); if ((matchIndex3 >= matchLow) & (current - matchIndex3 < (1<<18)) /*heuristic : longer distance likely too expensive*/ ) { size_t mlen; @@ -653,9 +666,7 @@ U32 ZSTD_insertBtAndGetAllMatches ( (ip+mlen == iLimit) ) { /* best possible length */ ms->nextToUpdate = current+1; /* skip insertion */ return 1; - } - } - } + } } } /* no dictMatchState lookup: dicts don't have a populated HC3 table */ } @@ -760,10 +771,13 @@ U32 ZSTD_insertBtAndGetAllMatches ( FORCE_INLINE_TEMPLATE U32 ZSTD_BtGetAllMatches ( + ZSTD_match_t* matches, /* store result (match found, increasing size) in this table */ ZSTD_matchState_t* ms, + U32* nextToUpdate3, const BYTE* ip, const BYTE* const iHighLimit, const ZSTD_dictMode_e dictMode, - U32 rep[ZSTD_REP_NUM], U32 const ll0, - ZSTD_match_t* matches, U32 const lengthToBeat) + const U32 rep[ZSTD_REP_NUM], + U32 const ll0, + U32 const lengthToBeat) { const ZSTD_compressionParameters* const cParams = &ms->cParams; U32 const matchLengthSearch = cParams->minMatch; @@ -772,12 +786,12 @@ FORCE_INLINE_TEMPLATE U32 ZSTD_BtGetAllMatches ( ZSTD_updateTree_internal(ms, ip, iHighLimit, matchLengthSearch, dictMode); switch(matchLengthSearch) { - case 3 : return ZSTD_insertBtAndGetAllMatches(ms, ip, iHighLimit, dictMode, rep, ll0, matches, lengthToBeat, 3); + case 3 : return ZSTD_insertBtAndGetAllMatches(matches, ms, nextToUpdate3, ip, iHighLimit, dictMode, rep, ll0, lengthToBeat, 3); default : - case 4 : return ZSTD_insertBtAndGetAllMatches(ms, ip, iHighLimit, dictMode, rep, ll0, matches, lengthToBeat, 4); - case 5 : return ZSTD_insertBtAndGetAllMatches(ms, ip, iHighLimit, dictMode, rep, ll0, matches, lengthToBeat, 5); + case 4 : return ZSTD_insertBtAndGetAllMatches(matches, ms, nextToUpdate3, ip, iHighLimit, dictMode, rep, ll0, lengthToBeat, 4); + case 5 : return ZSTD_insertBtAndGetAllMatches(matches, ms, nextToUpdate3, ip, iHighLimit, dictMode, rep, ll0, lengthToBeat, 5); case 7 : - case 6 : return ZSTD_insertBtAndGetAllMatches(ms, ip, iHighLimit, dictMode, rep, ll0, matches, lengthToBeat, 6); + case 6 : return ZSTD_insertBtAndGetAllMatches(matches, ms, nextToUpdate3, ip, iHighLimit, dictMode, rep, ll0, lengthToBeat, 6); } } @@ -853,6 +867,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, U32 const sufficient_len = MIN(cParams->targetLength, ZSTD_OPT_NUM -1); U32 const minMatch = (cParams->minMatch == 3) ? 3 : 4; + U32 nextToUpdate3 = ms->nextToUpdate; ZSTD_optimal_t* const opt = optStatePtr->priceTable; ZSTD_match_t* const matches = optStatePtr->matchTable; @@ -862,7 +877,6 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, DEBUGLOG(5, "ZSTD_compressBlock_opt_generic: current=%u, prefix=%u, nextToUpdate=%u", (U32)(ip - base), ms->window.dictLimit, ms->nextToUpdate); assert(optLevel <= 2); - ms->nextToUpdate3 = ms->nextToUpdate; ZSTD_rescaleFreqs(optStatePtr, (const BYTE*)src, srcSize, optLevel); ip += (ip==prefixStart); @@ -873,7 +887,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, /* find first match */ { U32 const litlen = (U32)(ip - anchor); U32 const ll0 = !litlen; - U32 const nbMatches = ZSTD_BtGetAllMatches(ms, ip, iend, dictMode, rep, ll0, matches, minMatch); + U32 const nbMatches = ZSTD_BtGetAllMatches(matches, ms, &nextToUpdate3, ip, iend, dictMode, rep, ll0, minMatch); if (!nbMatches) { ip++; continue; } /* initialize opt[0] */ @@ -970,7 +984,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, U32 const litlen = (opt[cur].mlen == 0) ? opt[cur].litlen : 0; U32 const previousPrice = opt[cur].price; U32 const basePrice = previousPrice + ZSTD_litLengthPrice(0, optStatePtr, optLevel); - U32 const nbMatches = ZSTD_BtGetAllMatches(ms, inr, iend, dictMode, opt[cur].rep, ll0, matches, minMatch); + U32 const nbMatches = ZSTD_BtGetAllMatches(matches, ms, &nextToUpdate3, inr, iend, dictMode, opt[cur].rep, ll0, minMatch); U32 matchNb; if (!nbMatches) { DEBUGLOG(7, "rPos:%u : no match found", cur); @@ -1094,7 +1108,7 @@ _shortestPath: /* cur, last_pos, best_mlen, best_off have to be set */ } /* while (ip < ilimit) */ /* Return the last literals size */ - return iend - anchor; + return (size_t)(iend - anchor); } @@ -1158,7 +1172,6 @@ ZSTD_initStats_ultra(ZSTD_matchState_t* ms, ms->window.dictLimit += (U32)srcSize; ms->window.lowLimit = ms->window.dictLimit; ms->nextToUpdate = ms->window.dictLimit; - ms->nextToUpdate3 = ms->window.dictLimit; /* re-inforce weight of collected statistics */ ZSTD_upscaleStats(&ms->opt); diff --git a/thirdparty/zstd/compress/zstdmt_compress.c b/thirdparty/zstd/compress/zstdmt_compress.c index 38fbb90768..9e537b8848 100644 --- a/thirdparty/zstd/compress/zstdmt_compress.c +++ b/thirdparty/zstd/compress/zstdmt_compress.c @@ -1129,9 +1129,14 @@ size_t ZSTDMT_toFlushNow(ZSTDMT_CCtx* mtctx) size_t const produced = ZSTD_isError(cResult) ? 0 : cResult; size_t const flushed = ZSTD_isError(cResult) ? 0 : jobPtr->dstFlushed; assert(flushed <= produced); + assert(jobPtr->consumed <= jobPtr->src.size); toFlush = produced - flushed; - if (toFlush==0 && (jobPtr->consumed >= jobPtr->src.size)) { - /* doneJobID is not-fully-flushed, but toFlush==0 : doneJobID should be compressing some more data */ + /* if toFlush==0, nothing is available to flush. + * However, jobID is expected to still be active: + * if jobID was already completed and fully flushed, + * ZSTDMT_flushProduced() should have already moved onto next job. + * Therefore, some input has not yet been consumed. */ + if (toFlush==0) { assert(jobPtr->consumed < jobPtr->src.size); } } @@ -1148,12 +1153,16 @@ size_t ZSTDMT_toFlushNow(ZSTDMT_CCtx* mtctx) static unsigned ZSTDMT_computeTargetJobLog(ZSTD_CCtx_params const params) { - if (params.ldmParams.enableLdm) + unsigned jobLog; + if (params.ldmParams.enableLdm) { /* In Long Range Mode, the windowLog is typically oversized. * In which case, it's preferable to determine the jobSize * based on chainLog instead. */ - return MAX(21, params.cParams.chainLog + 4); - return MAX(20, params.cParams.windowLog + 2); + jobLog = MAX(21, params.cParams.chainLog + 4); + } else { + jobLog = MAX(20, params.cParams.windowLog + 2); + } + return MIN(jobLog, (unsigned)ZSTDMT_JOBLOG_MAX); } static int ZSTDMT_overlapLog_default(ZSTD_strategy strat) @@ -1197,7 +1206,7 @@ static size_t ZSTDMT_computeOverlapSize(ZSTD_CCtx_params const params) ovLog = MIN(params.cParams.windowLog, ZSTDMT_computeTargetJobLog(params) - 2) - overlapRLog; } - assert(0 <= ovLog && ovLog <= 30); + assert(0 <= ovLog && ovLog <= ZSTD_WINDOWLOG_MAX); DEBUGLOG(4, "overlapLog : %i", params.overlapLog); DEBUGLOG(4, "overlap size : %i", 1 << ovLog); return (ovLog==0) ? 0 : (size_t)1 << ovLog; @@ -1391,7 +1400,7 @@ size_t ZSTDMT_initCStream_internal( FORWARD_IF_ERROR( ZSTDMT_resize(mtctx, params.nbWorkers) ); if (params.jobSize != 0 && params.jobSize < ZSTDMT_JOBSIZE_MIN) params.jobSize = ZSTDMT_JOBSIZE_MIN; - if (params.jobSize > (size_t)ZSTDMT_JOBSIZE_MAX) params.jobSize = ZSTDMT_JOBSIZE_MAX; + if (params.jobSize > (size_t)ZSTDMT_JOBSIZE_MAX) params.jobSize = (size_t)ZSTDMT_JOBSIZE_MAX; mtctx->singleBlockingThread = (pledgedSrcSize <= ZSTDMT_JOBSIZE_MIN); /* do not trigger multi-threading when srcSize is too small */ if (mtctx->singleBlockingThread) { @@ -1432,6 +1441,8 @@ size_t ZSTDMT_initCStream_internal( if (mtctx->targetSectionSize == 0) { mtctx->targetSectionSize = 1ULL << ZSTDMT_computeTargetJobLog(params); } + assert(mtctx->targetSectionSize <= (size_t)ZSTDMT_JOBSIZE_MAX); + if (params.rsyncable) { /* Aim for the targetsectionSize as the average job size. */ U32 const jobSizeMB = (U32)(mtctx->targetSectionSize >> 20); diff --git a/thirdparty/zstd/compress/zstdmt_compress.h b/thirdparty/zstd/compress/zstdmt_compress.h index 12e6bcb3a3..12a526087d 100644 --- a/thirdparty/zstd/compress/zstdmt_compress.h +++ b/thirdparty/zstd/compress/zstdmt_compress.h @@ -50,6 +50,7 @@ #ifndef ZSTDMT_JOBSIZE_MIN # define ZSTDMT_JOBSIZE_MIN (1 MB) #endif +#define ZSTDMT_JOBLOG_MAX (MEM_32bits() ? 29 : 30) #define ZSTDMT_JOBSIZE_MAX (MEM_32bits() ? (512 MB) : (1024 MB)) diff --git a/thirdparty/zstd/decompress/zstd_decompress.c b/thirdparty/zstd/decompress/zstd_decompress.c index 675596f5aa..e42872ad96 100644 --- a/thirdparty/zstd/decompress/zstd_decompress.c +++ b/thirdparty/zstd/decompress/zstd_decompress.c @@ -360,8 +360,11 @@ static size_t readSkippableFrameSize(void const* src, size_t srcSize) sizeU32 = MEM_readLE32((BYTE const*)src + ZSTD_FRAMEIDSIZE); RETURN_ERROR_IF((U32)(sizeU32 + ZSTD_SKIPPABLEHEADERSIZE) < sizeU32, frameParameter_unsupported); - - return skippableHeaderSize + sizeU32; + { + size_t const skippableSize = skippableHeaderSize + sizeU32; + RETURN_ERROR_IF(skippableSize > srcSize, srcSize_wrong); + return skippableSize; + } } /** ZSTD_findDecompressedSize() : @@ -378,11 +381,10 @@ unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize) if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { size_t const skippableSize = readSkippableFrameSize(src, srcSize); - if (ZSTD_isError(skippableSize)) - return skippableSize; - if (srcSize < skippableSize) { + if (ZSTD_isError(skippableSize)) { return ZSTD_CONTENTSIZE_ERROR; } + assert(skippableSize <= srcSize); src = (const BYTE *)src + skippableSize; srcSize -= skippableSize; @@ -467,6 +469,8 @@ static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize if ((srcSize >= ZSTD_SKIPPABLEHEADERSIZE) && (MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { frameSizeInfo.compressedSize = readSkippableFrameSize(src, srcSize); + assert(ZSTD_isError(frameSizeInfo.compressedSize) || + frameSizeInfo.compressedSize <= srcSize); return frameSizeInfo; } else { const BYTE* ip = (const BYTE*)src; @@ -529,7 +533,6 @@ size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize) return frameSizeInfo.compressedSize; } - /** ZSTD_decompressBound() : * compatible with legacy mode * `src` must point to the start of a ZSTD frame or a skippeable frame @@ -546,6 +549,7 @@ unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize) unsigned long long const decompressedBound = frameSizeInfo.decompressedBound; if (ZSTD_isError(compressedSize) || decompressedBound == ZSTD_CONTENTSIZE_ERROR) return ZSTD_CONTENTSIZE_ERROR; + assert(srcSize >= compressedSize); src = (const BYTE*)src + compressedSize; srcSize -= compressedSize; bound += decompressedBound; @@ -738,9 +742,8 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx, (unsigned)magicNumber, ZSTD_MAGICNUMBER); if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { size_t const skippableSize = readSkippableFrameSize(src, srcSize); - if (ZSTD_isError(skippableSize)) - return skippableSize; - RETURN_ERROR_IF(srcSize < skippableSize, srcSize_wrong); + FORWARD_IF_ERROR(skippableSize); + assert(skippableSize <= srcSize); src = (const BYTE *)src + skippableSize; srcSize -= skippableSize; diff --git a/thirdparty/zstd/decompress/zstd_decompress_block.c b/thirdparty/zstd/decompress/zstd_decompress_block.c index a2a7eedcf2..24f4859c56 100644 --- a/thirdparty/zstd/decompress/zstd_decompress_block.c +++ b/thirdparty/zstd/decompress/zstd_decompress_block.c @@ -505,7 +505,7 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, *nbSeqPtr = nbSeq; /* FSE table descriptors */ - RETURN_ERROR_IF(ip+4 > iend, srcSize_wrong); /* minimum possible size */ + RETURN_ERROR_IF(ip+1 > iend, srcSize_wrong); /* minimum possible size: 1 byte for symbol encoding types */ { symbolEncodingType_e const LLtype = (symbolEncodingType_e)(*ip >> 6); symbolEncodingType_e const OFtype = (symbolEncodingType_e)((*ip >> 4) & 3); symbolEncodingType_e const MLtype = (symbolEncodingType_e)((*ip >> 2) & 3); @@ -637,9 +637,10 @@ size_t ZSTD_execSequence(BYTE* op, if (oLitEnd>oend_w) return ZSTD_execSequenceLast7(op, oend, sequence, litPtr, litLimit, prefixStart, virtualStart, dictEnd); /* copy Literals */ - ZSTD_copy8(op, *litPtr); if (sequence.litLength > 8) - ZSTD_wildcopy(op+8, (*litPtr)+8, sequence.litLength - 8); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */ + ZSTD_wildcopy_16min(op, (*litPtr), sequence.litLength, ZSTD_no_overlap); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */ + else + ZSTD_copy8(op, *litPtr); op = oLitEnd; *litPtr = iLitEnd; /* update for next sequence */ @@ -686,13 +687,13 @@ size_t ZSTD_execSequence(BYTE* op, if (oMatchEnd > oend-(16-MINMATCH)) { if (op < oend_w) { - ZSTD_wildcopy(op, match, oend_w - op); + ZSTD_wildcopy(op, match, oend_w - op, ZSTD_overlap_src_before_dst); match += oend_w - op; op = oend_w; } while (op < oMatchEnd) *op++ = *match++; } else { - ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */ + ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8, ZSTD_overlap_src_before_dst); /* works even if matchLength < 8 */ } return sequenceLength; } @@ -717,9 +718,11 @@ size_t ZSTD_execSequenceLong(BYTE* op, if (oLitEnd > oend_w) return ZSTD_execSequenceLast7(op, oend, sequence, litPtr, litLimit, prefixStart, dictStart, dictEnd); /* copy Literals */ - ZSTD_copy8(op, *litPtr); /* note : op <= oLitEnd <= oend_w == oend - 8 */ if (sequence.litLength > 8) - ZSTD_wildcopy(op+8, (*litPtr)+8, sequence.litLength - 8); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */ + ZSTD_wildcopy_16min(op, *litPtr, sequence.litLength, ZSTD_no_overlap); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */ + else + ZSTD_copy8(op, *litPtr); /* note : op <= oLitEnd <= oend_w == oend - 8 */ + op = oLitEnd; *litPtr = iLitEnd; /* update for next sequence */ @@ -766,13 +769,13 @@ size_t ZSTD_execSequenceLong(BYTE* op, if (oMatchEnd > oend-(16-MINMATCH)) { if (op < oend_w) { - ZSTD_wildcopy(op, match, oend_w - op); + ZSTD_wildcopy(op, match, oend_w - op, ZSTD_overlap_src_before_dst); match += oend_w - op; op = oend_w; } while (op < oMatchEnd) *op++ = *match++; } else { - ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */ + ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8, ZSTD_overlap_src_before_dst); /* works even if matchLength < 8 */ } return sequenceLength; } @@ -889,6 +892,7 @@ ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets) } FORCE_INLINE_TEMPLATE size_t +DONT_VECTORIZE ZSTD_decompressSequences_body( ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* seqStart, size_t seqSize, int nbSeq, @@ -918,6 +922,11 @@ ZSTD_decompressSequences_body( ZSTD_DCtx* dctx, ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr); ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); + ZSTD_STATIC_ASSERT( + BIT_DStream_unfinished < BIT_DStream_completed && + BIT_DStream_endOfBuffer < BIT_DStream_completed && + BIT_DStream_completed < BIT_DStream_overflow); + for ( ; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && nbSeq ; ) { nbSeq--; { seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset); @@ -930,6 +939,7 @@ ZSTD_decompressSequences_body( ZSTD_DCtx* dctx, /* check if reached exact end */ DEBUGLOG(5, "ZSTD_decompressSequences_body: after decode loop, remaining nbSeq : %i", nbSeq); RETURN_ERROR_IF(nbSeq, corruption_detected); + RETURN_ERROR_IF(BIT_reloadDStream(&seqState.DStream) < BIT_DStream_completed, corruption_detected); /* save reps for next block */ { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) dctx->entropy.rep[i] = (U32)(seqState.prevOffset[i]); } } @@ -1131,6 +1141,7 @@ ZSTD_decompressSequencesLong_default(ZSTD_DCtx* dctx, #ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG static TARGET_ATTRIBUTE("bmi2") size_t +DONT_VECTORIZE ZSTD_decompressSequences_bmi2(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* seqStart, size_t seqSize, int nbSeq, diff --git a/thirdparty/zstd/zstd.h b/thirdparty/zstd/zstd.h index 53470c18f3..a1910ee223 100644 --- a/thirdparty/zstd/zstd.h +++ b/thirdparty/zstd/zstd.h @@ -71,7 +71,7 @@ extern "C" { /*------ Version ------*/ #define ZSTD_VERSION_MAJOR 1 #define ZSTD_VERSION_MINOR 4 -#define ZSTD_VERSION_RELEASE 0 +#define ZSTD_VERSION_RELEASE 1 #define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE) ZSTDLIB_API unsigned ZSTD_versionNumber(void); /**< to check runtime library version */ @@ -82,16 +82,16 @@ ZSTDLIB_API unsigned ZSTD_versionNumber(void); /**< to check runtime library v #define ZSTD_VERSION_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_LIB_VERSION) ZSTDLIB_API const char* ZSTD_versionString(void); /* requires v1.3.0+ */ -/*************************************** -* Default constant -***************************************/ +/* ************************************* + * Default constant + ***************************************/ #ifndef ZSTD_CLEVEL_DEFAULT # define ZSTD_CLEVEL_DEFAULT 3 #endif -/*************************************** -* Constants -***************************************/ +/* ************************************* + * Constants + ***************************************/ /* All magic numbers are supposed read/written to/from files/memory using little-endian convention */ #define ZSTD_MAGICNUMBER 0xFD2FB528 /* valid since v0.8.0 */ @@ -183,9 +183,14 @@ ZSTDLIB_API int ZSTD_maxCLevel(void); /*!< maximum compres ***************************************/ /*= Compression context * When compressing many times, - * it is recommended to allocate a context just once, and re-use it for each successive compression operation. + * it is recommended to allocate a context just once, + * and re-use it for each successive compression operation. * This will make workload friendlier for system's memory. - * Use one context per thread for parallel execution in multi-threaded environments. */ + * Note : re-using context is just a speed / resource optimization. + * It doesn't change the compression ratio, which remains identical. + * Note 2 : In multi-threaded environments, + * use one different context per thread for parallel execution. + */ typedef struct ZSTD_CCtx_s ZSTD_CCtx; ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx(void); ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx); @@ -380,6 +385,7 @@ typedef enum { * ZSTD_c_forceMaxWindow * ZSTD_c_forceAttachDict * ZSTD_c_literalCompressionMode + * ZSTD_c_targetCBlockSize * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. * note : never ever use experimentalParam? names directly; * also, the enums values themselves are unstable and can still change. @@ -389,6 +395,7 @@ typedef enum { ZSTD_c_experimentalParam3=1000, ZSTD_c_experimentalParam4=1001, ZSTD_c_experimentalParam5=1002, + ZSTD_c_experimentalParam6=1003, } ZSTD_cParameter; typedef struct { @@ -657,17 +664,33 @@ ZSTDLIB_API size_t ZSTD_compressStream2( ZSTD_CCtx* cctx, ZSTD_inBuffer* input, ZSTD_EndDirective endOp); + +/* These buffer sizes are softly recommended. + * They are not required : ZSTD_compressStream*() happily accepts any buffer size, for both input and output. + * Respecting the recommended size just makes it a bit easier for ZSTD_compressStream*(), + * reducing the amount of memory shuffling and buffering, resulting in minor performance savings. + * + * However, note that these recommendations are from the perspective of a C caller program. + * If the streaming interface is invoked from some other language, + * especially managed ones such as Java or Go, through a foreign function interface such as jni or cgo, + * a major performance rule is to reduce crossing such interface to an absolute minimum. + * It's not rare that performance ends being spent more into the interface, rather than compression itself. + * In which cases, prefer using large buffers, as large as practical, + * for both input and output, to reduce the nb of roundtrips. + */ ZSTDLIB_API size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */ -ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block in all circumstances. */ +ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block. */ -/******************************************************************************* - * This is a legacy streaming API, and can be replaced by ZSTD_CCtx_reset() and - * ZSTD_compressStream2(). It is redundant, but is still fully supported. + +/* ***************************************************************************** + * This following is a legacy streaming API. + * It can be replaced by ZSTD_CCtx_reset() and ZSTD_compressStream2(). + * It is redundant, but remains fully supported. * Advanced parameters and dictionary compression can only be used through the * new API. ******************************************************************************/ -/** +/*! * Equivalent to: * * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); @@ -675,16 +698,16 @@ ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); */ ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel); -/** +/*! * Alternative for ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue). * NOTE: The return value is different. ZSTD_compressStream() returns a hint for * the next read size (if non-zero and not an error). ZSTD_compressStream2() - * returns the number of bytes left to flush (if non-zero and not an error). + * returns the minimum nb of bytes left to flush (if non-zero and not an error). */ ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input); -/** Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_flush). */ +/*! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_flush). */ ZSTDLIB_API size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); -/** Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_end). */ +/*! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_end). */ ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); @@ -969,7 +992,7 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); #endif /* ZSTD_H_235446 */ -/**************************************************************************************** +/* ************************************************************************************** * ADVANCED AND EXPERIMENTAL FUNCTIONS **************************************************************************************** * The definitions in the following section are considered experimental. @@ -1037,6 +1060,10 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); #define ZSTD_LDM_HASHRATELOG_MIN 0 #define ZSTD_LDM_HASHRATELOG_MAX (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN) +/* Advanced parameter bounds */ +#define ZSTD_TARGETCBLOCKSIZE_MIN 64 +#define ZSTD_TARGETCBLOCKSIZE_MAX ZSTD_BLOCKSIZE_MAX + /* internal */ #define ZSTD_HASHLOG3_MAX 17 @@ -1162,7 +1189,7 @@ typedef enum { * however it does mean that all frame data must be present and valid. */ ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize); -/** ZSTD_decompressBound() : +/*! ZSTD_decompressBound() : * `src` should point to the start of a series of ZSTD encoded and/or skippable frames * `srcSize` must be the _exact_ size of this series * (i.e. there should be a frame boundary at `src + srcSize`) @@ -1409,6 +1436,11 @@ ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* pre */ #define ZSTD_c_literalCompressionMode ZSTD_c_experimentalParam5 +/* Tries to fit compressed block size to be around targetCBlockSize. + * No target when targetCBlockSize == 0. + * There is no guarantee on compressed block size (default:0) */ +#define ZSTD_c_targetCBlockSize ZSTD_c_experimentalParam6 + /*! ZSTD_CCtx_getParameter() : * Get the requested compression parameter value, selected by enum ZSTD_cParameter, * and store it into int* value. @@ -1843,7 +1875,7 @@ typedef struct { unsigned checksumFlag; } ZSTD_frameHeader; -/** ZSTD_getFrameHeader() : +/*! ZSTD_getFrameHeader() : * decode Frame Header, or requires larger `srcSize`. * @return : 0, `zfhPtr` is correctly filled, * >0, `srcSize` is too small, value is wanted `srcSize` amount, |