Sort decals and lights based on camera origin

Also implement sort_offset for decals

Co-authored-by: Hugo Locurcio <hugo.locurcio@hugo.pro>

15 files changed, 56 insertions, 35 deletions

diff --git a/doc/classes/VisualInstance3D.xml b/doc/classes/VisualInstance3D.xml
index e069642e50..3781045c02 100644
--- a/doc/classes/VisualInstance3D.xml
+++ b/doc/classes/VisualInstance3D.xml
@@ -64,7 +64,7 @@
 		<member name="sorting_offset" type="float" setter="set_sorting_offset" getter="get_sorting_offset" default="0.0">
 			The sorting offset used by this [VisualInstance3D]. Adjusting it to a higher value will make the [VisualInstance3D] reliably draw on top of other [VisualInstance3D]s that are otherwise positioned at the same spot.
 		</member>
-		<member name="sorting_use_aabb_center" type="bool" setter="set_sorting_use_aabb_center" getter="is_sorting_use_aabb_center" default="true">
+		<member name="sorting_use_aabb_center" type="bool" setter="set_sorting_use_aabb_center" getter="is_sorting_use_aabb_center">
 			If [code]true[/code], the object is sorted based on the [AABB] center. The object will be sorted based on the global position otherwise.
 			The [AABB] center based sorting is generally more accurate for 3D models. The position based sorting instead allows to better control the drawing order when working with [GPUParticles3D] and [CPUParticles3D].
 		</member>
diff --git a/drivers/gles3/rasterizer_scene_gles3.cpp b/drivers/gles3/rasterizer_scene_gles3.cpp
index e4bcc424b3..8adca1db8e 100644
--- a/drivers/gles3/rasterizer_scene_gles3.cpp
+++ b/drivers/gles3/rasterizer_scene_gles3.cpp
@@ -1584,6 +1584,7 @@ void RasterizerSceneGLES3::_setup_lights(const RenderDataGLES3 *p_render_data, b
 		LightData &light_data = (i < r_omni_light_count) ? scene_state.omni_lights[index] : scene_state.spot_lights[index];
 		RS::LightType type = (i < r_omni_light_count) ? RS::LIGHT_OMNI : RS::LIGHT_SPOT;
 		GLES3::LightInstance *li = (i < r_omni_light_count) ? scene_state.omni_light_sort[index].instance : scene_state.spot_light_sort[index].instance;
+		real_t distance = (i < r_omni_light_count) ? scene_state.omni_light_sort[index].depth : scene_state.spot_light_sort[index].depth;
 		RID base = li->light;
 
 		Transform3D light_transform = li->transform;
@@ -1612,13 +1613,11 @@ void RasterizerSceneGLES3::_setup_lights(const RenderDataGLES3 *p_render_data, b
 		// Reuse fade begin, fade length and distance for shadow LOD determination later.
 		float fade_begin = 0.0;
 		float fade_length = 0.0;
-		real_t distance = 0.0;
 
 		float fade = 1.0;
 		if (light_storage->light_is_distance_fade_enabled(li->light)) {
 			fade_begin = light_storage->light_get_distance_fade_begin(li->light);
 			fade_length = light_storage->light_get_distance_fade_length(li->light);
-			distance = p_render_data->cam_transform.origin.distance_to(li->transform.origin);
 
 			if (distance > fade_begin) {
 				// Use `smoothstep()` to make opacity changes more gradual and less noticeable to the player.
diff --git a/drivers/gles3/storage/texture_storage.cpp b/drivers/gles3/storage/texture_storage.cpp
index 8818ab2118..12def52d50 100644
--- a/drivers/gles3/storage/texture_storage.cpp
+++ b/drivers/gles3/storage/texture_storage.cpp
@@ -1534,18 +1534,6 @@ AABB TextureStorage::decal_get_aabb(RID p_decal) const {
 	return AABB();
 }
 
-/* DECAL INSTANCE API */
-
-RID TextureStorage::decal_instance_create(RID p_decal) {
-	return RID();
-}
-
-void TextureStorage::decal_instance_free(RID p_decal_instance) {
-}
-
-void TextureStorage::decal_instance_set_transform(RID p_decal, const Transform3D &p_transform) {
-}
-
 /* RENDER TARGET API */
 
 GLuint TextureStorage::system_fbo = 0;
diff --git a/drivers/gles3/storage/texture_storage.h b/drivers/gles3/storage/texture_storage.h
index 9ffe16fd16..9d3f407226 100644
--- a/drivers/gles3/storage/texture_storage.h
+++ b/drivers/gles3/storage/texture_storage.h
@@ -561,9 +561,10 @@ public:
 
 	/* DECAL INSTANCE */
 
-	virtual RID decal_instance_create(RID p_decal) override;
-	virtual void decal_instance_free(RID p_decal_instance) override;
-	virtual void decal_instance_set_transform(RID p_decal, const Transform3D &p_transform) override;
+	virtual RID decal_instance_create(RID p_decal) override { return RID(); }
+	virtual void decal_instance_free(RID p_decal_instance) override {}
+	virtual void decal_instance_set_transform(RID p_decal, const Transform3D &p_transform) override {}
+	virtual void decal_instance_set_sorting_offset(RID p_decal_instance, float p_sorting_offset) override {}
 
 	/* RENDER TARGET API */
 
diff --git a/scene/3d/decal.cpp b/scene/3d/decal.cpp
index e9cc4e9479..fbcb1c8f2c 100644
--- a/scene/3d/decal.cpp
+++ b/scene/3d/decal.cpp
@@ -156,6 +156,10 @@ void Decal::_validate_property(PropertyInfo &p_property) const {
 	if (!distance_fade_enabled && (p_property.name == "distance_fade_begin" || p_property.name == "distance_fade_length")) {
 		p_property.usage = PROPERTY_USAGE_NO_EDITOR;
 	}
+
+	if (p_property.name == "sorting_offset") {
+		p_property.usage = PROPERTY_USAGE_DEFAULT;
+	}
 }
 
 PackedStringArray Decal::get_configuration_warnings() const {
diff --git a/scene/3d/visual_instance_3d.cpp b/scene/3d/visual_instance_3d.cpp
index 64fb0a7657..8026b12c2b 100644
--- a/scene/3d/visual_instance_3d.cpp
+++ b/scene/3d/visual_instance_3d.cpp
@@ -120,6 +120,12 @@ bool VisualInstance3D::is_sorting_use_aabb_center() const {
 	return sorting_use_aabb_center;
 }
 
+void VisualInstance3D::_validate_property(PropertyInfo &p_property) const {
+	if (p_property.name == "sorting_offset" || p_property.name == "sorting_use_aabb_center") {
+		p_property.usage = PROPERTY_USAGE_NONE;
+	}
+}
+
 void VisualInstance3D::_bind_methods() {
 	ClassDB::bind_method(D_METHOD("set_base", "base"), &VisualInstance3D::set_base);
 	ClassDB::bind_method(D_METHOD("get_base"), &VisualInstance3D::get_base);
@@ -437,6 +443,12 @@ PackedStringArray GeometryInstance3D::get_configuration_warnings() const {
 	return warnings;
 }
 
+void GeometryInstance3D::_validate_property(PropertyInfo &p_property) const {
+	if (p_property.name == "sorting_offset" || p_property.name == "sorting_use_aabb_center") {
+		p_property.usage = PROPERTY_USAGE_DEFAULT;
+	}
+}
+
 void GeometryInstance3D::_bind_methods() {
 	ClassDB::bind_method(D_METHOD("set_material_override", "material"), &GeometryInstance3D::set_material_override);
 	ClassDB::bind_method(D_METHOD("get_material_override"), &GeometryInstance3D::get_material_override);
diff --git a/scene/3d/visual_instance_3d.h b/scene/3d/visual_instance_3d.h
index 190ed17753..ef0f7966e2 100644
--- a/scene/3d/visual_instance_3d.h
+++ b/scene/3d/visual_instance_3d.h
@@ -47,6 +47,7 @@ protected:
 
 	void _notification(int p_what);
 	static void _bind_methods();
+	void _validate_property(PropertyInfo &p_property) const;
 
 	GDVIRTUAL0RC(AABB, _get_aabb)
 public:
@@ -140,6 +141,7 @@ protected:
 	bool _set(const StringName &p_name, const Variant &p_value);
 	bool _get(const StringName &p_name, Variant &r_ret) const;
 	void _get_property_list(List<PropertyInfo> *p_list) const;
+	void _validate_property(PropertyInfo &p_property) const;
 
 	static void _bind_methods();
 
diff --git a/servers/rendering/dummy/storage/texture_storage.h b/servers/rendering/dummy/storage/texture_storage.h
index 62e1eb326d..fd36e7ac10 100644
--- a/servers/rendering/dummy/storage/texture_storage.h
+++ b/servers/rendering/dummy/storage/texture_storage.h
@@ -154,6 +154,7 @@ public:
 	virtual RID decal_instance_create(RID p_decal) override { return RID(); }
 	virtual void decal_instance_free(RID p_decal_instance) override {}
 	virtual void decal_instance_set_transform(RID p_decal, const Transform3D &p_transform) override {}
+	virtual void decal_instance_set_sorting_offset(RID p_decal_instance, float p_sorting_offset) override {}
 
 	/* RENDER TARGET */
 
diff --git a/servers/rendering/renderer_rd/forward_clustered/render_forward_clustered.cpp b/servers/rendering/renderer_rd/forward_clustered/render_forward_clustered.cpp
index 93c741fd34..3fd8d55a71 100644
--- a/servers/rendering/renderer_rd/forward_clustered/render_forward_clustered.cpp
+++ b/servers/rendering/renderer_rd/forward_clustered/render_forward_clustered.cpp
@@ -1480,7 +1480,7 @@ void RenderForwardClustered::_pre_opaque_render(RenderDataRD *p_render_data, boo
 	uint32_t directional_light_count = 0;
 	uint32_t positional_light_count = 0;
 	light_storage->update_light_buffers(p_render_data, *p_render_data->lights, p_render_data->scene_data->cam_transform, p_render_data->shadow_atlas, using_shadows, directional_light_count, positional_light_count, p_render_data->directional_light_soft_shadows);
-	texture_storage->update_decal_buffer(*p_render_data->decals, p_render_data->scene_data->cam_transform.affine_inverse());
+	texture_storage->update_decal_buffer(*p_render_data->decals, p_render_data->scene_data->cam_transform);
 
 	p_render_data->directional_light_count = directional_light_count;
 
diff --git a/servers/rendering/renderer_rd/forward_mobile/render_forward_mobile.cpp b/servers/rendering/renderer_rd/forward_mobile/render_forward_mobile.cpp
index 816248567b..5d4e3c1ac1 100644
--- a/servers/rendering/renderer_rd/forward_mobile/render_forward_mobile.cpp
+++ b/servers/rendering/renderer_rd/forward_mobile/render_forward_mobile.cpp
@@ -632,7 +632,7 @@ void RenderForwardMobile::_pre_opaque_render(RenderDataRD *p_render_data) {
 	uint32_t directional_light_count = 0;
 	uint32_t positional_light_count = 0;
 	light_storage->update_light_buffers(p_render_data, *p_render_data->lights, p_render_data->scene_data->cam_transform, p_render_data->shadow_atlas, using_shadows, directional_light_count, positional_light_count, p_render_data->directional_light_soft_shadows);
-	texture_storage->update_decal_buffer(*p_render_data->decals, p_render_data->scene_data->cam_transform.affine_inverse());
+	texture_storage->update_decal_buffer(*p_render_data->decals, p_render_data->scene_data->cam_transform);
 
 	p_render_data->directional_light_count = directional_light_count;
 }
diff --git a/servers/rendering/renderer_rd/storage_rd/light_storage.cpp b/servers/rendering/renderer_rd/storage_rd/light_storage.cpp
index 673fc25595..33e35a7a64 100644
--- a/servers/rendering/renderer_rd/storage_rd/light_storage.cpp
+++ b/servers/rendering/renderer_rd/storage_rd/light_storage.cpp
@@ -568,8 +568,6 @@ void LightStorage::update_light_buffers(RenderDataRD *p_render_data, const Paged
 	r_directional_light_count = 0;
 	r_positional_light_count = 0;
 
-	Plane camera_plane(-p_camera_transform.basis.get_column(Vector3::AXIS_Z).normalized(), p_camera_transform.origin);
-
 	omni_light_count = 0;
 	spot_light_count = 0;
 
@@ -720,7 +718,7 @@ void LightStorage::update_light_buffers(RenderDataRD *p_render_data, const Paged
 				}
 
 				Transform3D light_transform = light_instance->transform;
-				const real_t distance = camera_plane.distance_to(light_transform.origin);
+				const real_t distance = p_camera_transform.origin.distance_to(light_transform.origin);
 
 				if (light->distance_fade) {
 					const float fade_begin = light->distance_fade_begin;
@@ -745,7 +743,7 @@ void LightStorage::update_light_buffers(RenderDataRD *p_render_data, const Paged
 				}
 
 				Transform3D light_transform = light_instance->transform;
-				const real_t distance = camera_plane.distance_to(light_transform.origin);
+				const real_t distance = p_camera_transform.origin.distance_to(light_transform.origin);
 
 				if (light->distance_fade) {
 					const float fade_begin = light->distance_fade_begin;
@@ -787,6 +785,7 @@ void LightStorage::update_light_buffers(RenderDataRD *p_render_data, const Paged
 		RS::LightType type = (i < omni_light_count) ? RS::LIGHT_OMNI : RS::LIGHT_SPOT;
 		LightInstance *light_instance = (i < omni_light_count) ? omni_light_sort[index].light_instance : spot_light_sort[index].light_instance;
 		Light *light = (i < omni_light_count) ? omni_light_sort[index].light : spot_light_sort[index].light;
+		real_t distance = (i < omni_light_count) ? omni_light_sort[index].depth : spot_light_sort[index].depth;
 
 		if (using_forward_ids) {
 			forward_id_storage->map_forward_id(type == RS::LIGHT_OMNI ? RendererRD::FORWARD_ID_TYPE_OMNI_LIGHT : RendererRD::FORWARD_ID_TYPE_SPOT_LIGHT, light_instance->forward_id, index);
@@ -803,7 +802,6 @@ void LightStorage::update_light_buffers(RenderDataRD *p_render_data, const Paged
 		float fade_begin = 0.0;
 		float fade_shadow = 0.0;
 		float fade_length = 0.0;
-		real_t distance = 0.0;
 
 		float fade = 1.0;
 		float shadow_opacity_fade = 1.0;
@@ -811,7 +809,6 @@ void LightStorage::update_light_buffers(RenderDataRD *p_render_data, const Paged
 			fade_begin = light->distance_fade_begin;
 			fade_shadow = light->distance_fade_shadow;
 			fade_length = light->distance_fade_length;
-			distance = camera_plane.distance_to(light_transform.origin);
 
 			// Use `smoothstep()` to make opacity changes more gradual and less noticeable to the player.
 			if (distance > fade_begin) {
diff --git a/servers/rendering/renderer_rd/storage_rd/texture_storage.cpp b/servers/rendering/renderer_rd/storage_rd/texture_storage.cpp
index 06fda8fa9e..e8d9f486bb 100644
--- a/servers/rendering/renderer_rd/storage_rd/texture_storage.cpp
+++ b/servers/rendering/renderer_rd/storage_rd/texture_storage.cpp
@@ -2206,6 +2206,12 @@ void TextureStorage::decal_instance_set_transform(RID p_decal_instance, const Tr
 	di->transform = p_transform;
 }
 
+void TextureStorage::decal_instance_set_sorting_offset(RID p_decal_instance, float p_sorting_offset) {
+	DecalInstance *di = decal_instance_owner.get_or_null(p_decal_instance);
+	ERR_FAIL_COND(!di);
+	di->sorting_offset = p_sorting_offset;
+}
+
 /* DECAL DATA API */
 
 void TextureStorage::free_decal_data() {
@@ -2233,7 +2239,7 @@ void TextureStorage::set_max_decals(const uint32_t p_max_decals) {
 	decal_buffer = RD::get_singleton()->storage_buffer_create(decal_buffer_size);
 }
 
-void TextureStorage::update_decal_buffer(const PagedArray<RID> &p_decals, const Transform3D &p_camera_inverse_xform) {
+void TextureStorage::update_decal_buffer(const PagedArray<RID> &p_decals, const Transform3D &p_camera_xform) {
 	ForwardIDStorage *forward_id_storage = ForwardIDStorage::get_singleton();
 
 	Transform3D uv_xform;
@@ -2257,7 +2263,7 @@ void TextureStorage::update_decal_buffer(const PagedArray<RID> &p_decals, const
 
 		Transform3D xform = decal_instance->transform;
 
-		real_t distance = -p_camera_inverse_xform.xform(xform.origin).z;
+		real_t distance = p_camera_xform.origin.distance_to(xform.origin);
 
 		if (decal->distance_fade) {
 			float fade_begin = decal->distance_fade_begin;
@@ -2272,7 +2278,7 @@ void TextureStorage::update_decal_buffer(const PagedArray<RID> &p_decals, const
 
 		decal_sort[decal_count].decal_instance = decal_instance;
 		decal_sort[decal_count].decal = decal;
-		decal_sort[decal_count].depth = distance;
+		decal_sort[decal_count].depth = distance - decal_instance->sorting_offset;
 		decal_count++;
 	}
 
@@ -2292,11 +2298,10 @@ void TextureStorage::update_decal_buffer(const PagedArray<RID> &p_decals, const
 
 		decal_instance->cull_mask = decal->cull_mask;
 
-		Transform3D xform = decal_instance->transform;
 		float fade = 1.0;
 
 		if (decal->distance_fade) {
-			const real_t distance = -p_camera_inverse_xform.xform(xform.origin).z;
+			const real_t distance = decal_sort[i].depth + decal_instance->sorting_offset;
 			const float fade_begin = decal->distance_fade_begin;
 			const float fade_length = decal->distance_fade_length;
 
@@ -2312,11 +2317,16 @@ void TextureStorage::update_decal_buffer(const PagedArray<RID> &p_decals, const
 
 		Transform3D scale_xform;
 		scale_xform.basis.scale(decal_extents);
-		Transform3D to_decal_xform = (p_camera_inverse_xform * xform * scale_xform * uv_xform).affine_inverse();
+
+		Transform3D xform = decal_instance->transform;
+
+		Transform3D camera_inverse_xform = p_camera_xform.affine_inverse();
+
+		Transform3D to_decal_xform = (camera_inverse_xform * xform * scale_xform * uv_xform).affine_inverse();
 		MaterialStorage::store_transform(to_decal_xform, dd.xform);
 
 		Vector3 normal = xform.basis.get_column(Vector3::AXIS_Y).normalized();
-		normal = p_camera_inverse_xform.basis.xform(normal); //camera is normalized, so fine
+		normal = camera_inverse_xform.basis.xform(normal); //camera is normalized, so fine
 
 		dd.normal[0] = normal.x;
 		dd.normal[1] = normal.y;
@@ -2350,7 +2360,7 @@ void TextureStorage::update_decal_buffer(const PagedArray<RID> &p_decals, const
 			dd.normal_rect[2] = rect.size.x;
 			dd.normal_rect[3] = rect.size.y;
 
-			Basis normal_xform = p_camera_inverse_xform.basis * xform.basis.orthonormalized();
+			Basis normal_xform = camera_inverse_xform.basis * xform.basis.orthonormalized();
 			MaterialStorage::store_basis_3x4(normal_xform, dd.normal_xform);
 		} else {
 			dd.normal_rect[0] = 0;
diff --git a/servers/rendering/renderer_rd/storage_rd/texture_storage.h b/servers/rendering/renderer_rd/storage_rd/texture_storage.h
index 1558342c3b..ea0df0b459 100644
--- a/servers/rendering/renderer_rd/storage_rd/texture_storage.h
+++ b/servers/rendering/renderer_rd/storage_rd/texture_storage.h
@@ -258,6 +258,7 @@ private:
 	struct DecalInstance {
 		RID decal;
 		Transform3D transform;
+		float sorting_offset = 0.0;
 		uint32_t cull_mask = 0;
 		RendererRD::ForwardID forward_id = -1;
 	};
@@ -646,6 +647,7 @@ public:
 	virtual RID decal_instance_create(RID p_decal) override;
 	virtual void decal_instance_free(RID p_decal_instance) override;
 	virtual void decal_instance_set_transform(RID p_decal_instance, const Transform3D &p_transform) override;
+	virtual void decal_instance_set_sorting_offset(RID p_decal_instance, float p_sorting_offset) override;
 
 	_FORCE_INLINE_ RID decal_instance_get_base(RID p_decal_instance) const {
 		DecalInstance *di = decal_instance_owner.get_or_null(p_decal_instance);
@@ -677,7 +679,7 @@ public:
 	void free_decal_data();
 	void set_max_decals(const uint32_t p_max_decals);
 	RID get_decal_buffer() { return decal_buffer; }
-	void update_decal_buffer(const PagedArray<RID> &p_decals, const Transform3D &p_camera_inverse_xform);
+	void update_decal_buffer(const PagedArray<RID> &p_decals, const Transform3D &p_camera_xform);
 
 	/* RENDER TARGET API */
 
diff --git a/servers/rendering/renderer_scene_cull.cpp b/servers/rendering/renderer_scene_cull.cpp
index 218bb7b736..2a8d0b3cb4 100644
--- a/servers/rendering/renderer_scene_cull.cpp
+++ b/servers/rendering/renderer_scene_cull.cpp
@@ -694,6 +694,7 @@ void RendererSceneCull::instance_set_base(RID p_instance, RID p_base) {
 				instance->base_data = decal;
 
 				decal->instance = RSG::texture_storage->decal_instance_create(p_base);
+				RSG::texture_storage->decal_instance_set_sorting_offset(decal->instance, instance->sorting_offset);
 			} break;
 			case RS::INSTANCE_LIGHTMAP: {
 				InstanceLightmapData *lightmap_data = memnew(InstanceLightmapData);
@@ -871,6 +872,9 @@ void RendererSceneCull::instance_set_pivot_data(RID p_instance, float p_sorting_
 		InstanceGeometryData *geom = static_cast<InstanceGeometryData *>(instance->base_data);
 		ERR_FAIL_NULL(geom->geometry_instance);
 		geom->geometry_instance->set_pivot_data(p_sorting_offset, p_use_aabb_center);
+	} else if (instance->base_type == RS::INSTANCE_DECAL && instance->base_data) {
+		InstanceDecalData *decal = static_cast<InstanceDecalData *>(instance->base_data);
+		RSG::texture_storage->decal_instance_set_sorting_offset(decal->instance, instance->sorting_offset);
 	}
 }
 
diff --git a/servers/rendering/storage/texture_storage.h b/servers/rendering/storage/texture_storage.h
index 92149b0064..4c4a84d04e 100644
--- a/servers/rendering/storage/texture_storage.h
+++ b/servers/rendering/storage/texture_storage.h
@@ -127,6 +127,7 @@ public:
 	virtual RID decal_instance_create(RID p_decal) = 0;
 	virtual void decal_instance_free(RID p_decal_instance) = 0;
 	virtual void decal_instance_set_transform(RID p_decal_instance, const Transform3D &p_transform) = 0;
+	virtual void decal_instance_set_sorting_offset(RID p_decal_instance, float p_sorting_offset) = 0;
 
 	/* RENDER TARGET */