Sort decals and lights based on camera origin

Also implement sort_offset for decals

Co-authored-by: Hugo Locurcio <hugo.locurcio@hugo.pro>

8 files changed, 32 insertions, 17 deletions

diff --git a/servers/rendering/dummy/storage/texture_storage.h b/servers/rendering/dummy/storage/texture_storage.h
index 62e1eb326d..fd36e7ac10 100644
--- a/servers/rendering/dummy/storage/texture_storage.h
+++ b/servers/rendering/dummy/storage/texture_storage.h
@@ -154,6 +154,7 @@ public:
 	virtual RID decal_instance_create(RID p_decal) override { return RID(); }
 	virtual void decal_instance_free(RID p_decal_instance) override {}
 	virtual void decal_instance_set_transform(RID p_decal, const Transform3D &p_transform) override {}
+	virtual void decal_instance_set_sorting_offset(RID p_decal_instance, float p_sorting_offset) override {}
 
 	/* RENDER TARGET */
 
diff --git a/servers/rendering/renderer_rd/forward_clustered/render_forward_clustered.cpp b/servers/rendering/renderer_rd/forward_clustered/render_forward_clustered.cpp
index 93c741fd34..3fd8d55a71 100644
--- a/servers/rendering/renderer_rd/forward_clustered/render_forward_clustered.cpp
+++ b/servers/rendering/renderer_rd/forward_clustered/render_forward_clustered.cpp
@@ -1480,7 +1480,7 @@ void RenderForwardClustered::_pre_opaque_render(RenderDataRD *p_render_data, boo
 	uint32_t directional_light_count = 0;
 	uint32_t positional_light_count = 0;
 	light_storage->update_light_buffers(p_render_data, *p_render_data->lights, p_render_data->scene_data->cam_transform, p_render_data->shadow_atlas, using_shadows, directional_light_count, positional_light_count, p_render_data->directional_light_soft_shadows);
-	texture_storage->update_decal_buffer(*p_render_data->decals, p_render_data->scene_data->cam_transform.affine_inverse());
+	texture_storage->update_decal_buffer(*p_render_data->decals, p_render_data->scene_data->cam_transform);
 
 	p_render_data->directional_light_count = directional_light_count;
 
diff --git a/servers/rendering/renderer_rd/forward_mobile/render_forward_mobile.cpp b/servers/rendering/renderer_rd/forward_mobile/render_forward_mobile.cpp
index 816248567b..5d4e3c1ac1 100644
--- a/servers/rendering/renderer_rd/forward_mobile/render_forward_mobile.cpp
+++ b/servers/rendering/renderer_rd/forward_mobile/render_forward_mobile.cpp
@@ -632,7 +632,7 @@ void RenderForwardMobile::_pre_opaque_render(RenderDataRD *p_render_data) {
 	uint32_t directional_light_count = 0;
 	uint32_t positional_light_count = 0;
 	light_storage->update_light_buffers(p_render_data, *p_render_data->lights, p_render_data->scene_data->cam_transform, p_render_data->shadow_atlas, using_shadows, directional_light_count, positional_light_count, p_render_data->directional_light_soft_shadows);
-	texture_storage->update_decal_buffer(*p_render_data->decals, p_render_data->scene_data->cam_transform.affine_inverse());
+	texture_storage->update_decal_buffer(*p_render_data->decals, p_render_data->scene_data->cam_transform);
 
 	p_render_data->directional_light_count = directional_light_count;
 }
diff --git a/servers/rendering/renderer_rd/storage_rd/light_storage.cpp b/servers/rendering/renderer_rd/storage_rd/light_storage.cpp
index 673fc25595..33e35a7a64 100644
--- a/servers/rendering/renderer_rd/storage_rd/light_storage.cpp
+++ b/servers/rendering/renderer_rd/storage_rd/light_storage.cpp
@@ -568,8 +568,6 @@ void LightStorage::update_light_buffers(RenderDataRD *p_render_data, const Paged
 	r_directional_light_count = 0;
 	r_positional_light_count = 0;
 
-	Plane camera_plane(-p_camera_transform.basis.get_column(Vector3::AXIS_Z).normalized(), p_camera_transform.origin);
-
 	omni_light_count = 0;
 	spot_light_count = 0;
 
@@ -720,7 +718,7 @@ void LightStorage::update_light_buffers(RenderDataRD *p_render_data, const Paged
 				}
 
 				Transform3D light_transform = light_instance->transform;
-				const real_t distance = camera_plane.distance_to(light_transform.origin);
+				const real_t distance = p_camera_transform.origin.distance_to(light_transform.origin);
 
 				if (light->distance_fade) {
 					const float fade_begin = light->distance_fade_begin;
@@ -745,7 +743,7 @@ void LightStorage::update_light_buffers(RenderDataRD *p_render_data, const Paged
 				}
 
 				Transform3D light_transform = light_instance->transform;
-				const real_t distance = camera_plane.distance_to(light_transform.origin);
+				const real_t distance = p_camera_transform.origin.distance_to(light_transform.origin);
 
 				if (light->distance_fade) {
 					const float fade_begin = light->distance_fade_begin;
@@ -787,6 +785,7 @@ void LightStorage::update_light_buffers(RenderDataRD *p_render_data, const Paged
 		RS::LightType type = (i < omni_light_count) ? RS::LIGHT_OMNI : RS::LIGHT_SPOT;
 		LightInstance *light_instance = (i < omni_light_count) ? omni_light_sort[index].light_instance : spot_light_sort[index].light_instance;
 		Light *light = (i < omni_light_count) ? omni_light_sort[index].light : spot_light_sort[index].light;
+		real_t distance = (i < omni_light_count) ? omni_light_sort[index].depth : spot_light_sort[index].depth;
 
 		if (using_forward_ids) {
 			forward_id_storage->map_forward_id(type == RS::LIGHT_OMNI ? RendererRD::FORWARD_ID_TYPE_OMNI_LIGHT : RendererRD::FORWARD_ID_TYPE_SPOT_LIGHT, light_instance->forward_id, index);
@@ -803,7 +802,6 @@ void LightStorage::update_light_buffers(RenderDataRD *p_render_data, const Paged
 		float fade_begin = 0.0;
 		float fade_shadow = 0.0;
 		float fade_length = 0.0;
-		real_t distance = 0.0;
 
 		float fade = 1.0;
 		float shadow_opacity_fade = 1.0;
@@ -811,7 +809,6 @@ void LightStorage::update_light_buffers(RenderDataRD *p_render_data, const Paged
 			fade_begin = light->distance_fade_begin;
 			fade_shadow = light->distance_fade_shadow;
 			fade_length = light->distance_fade_length;
-			distance = camera_plane.distance_to(light_transform.origin);
 
 			// Use `smoothstep()` to make opacity changes more gradual and less noticeable to the player.
 			if (distance > fade_begin) {
diff --git a/servers/rendering/renderer_rd/storage_rd/texture_storage.cpp b/servers/rendering/renderer_rd/storage_rd/texture_storage.cpp
index 06fda8fa9e..e8d9f486bb 100644
--- a/servers/rendering/renderer_rd/storage_rd/texture_storage.cpp
+++ b/servers/rendering/renderer_rd/storage_rd/texture_storage.cpp
@@ -2206,6 +2206,12 @@ void TextureStorage::decal_instance_set_transform(RID p_decal_instance, const Tr
 	di->transform = p_transform;
 }
 
+void TextureStorage::decal_instance_set_sorting_offset(RID p_decal_instance, float p_sorting_offset) {
+	DecalInstance *di = decal_instance_owner.get_or_null(p_decal_instance);
+	ERR_FAIL_COND(!di);
+	di->sorting_offset = p_sorting_offset;
+}
+
 /* DECAL DATA API */
 
 void TextureStorage::free_decal_data() {
@@ -2233,7 +2239,7 @@ void TextureStorage::set_max_decals(const uint32_t p_max_decals) {
 	decal_buffer = RD::get_singleton()->storage_buffer_create(decal_buffer_size);
 }
 
-void TextureStorage::update_decal_buffer(const PagedArray<RID> &p_decals, const Transform3D &p_camera_inverse_xform) {
+void TextureStorage::update_decal_buffer(const PagedArray<RID> &p_decals, const Transform3D &p_camera_xform) {
 	ForwardIDStorage *forward_id_storage = ForwardIDStorage::get_singleton();
 
 	Transform3D uv_xform;
@@ -2257,7 +2263,7 @@ void TextureStorage::update_decal_buffer(const PagedArray<RID> &p_decals, const
 
 		Transform3D xform = decal_instance->transform;
 
-		real_t distance = -p_camera_inverse_xform.xform(xform.origin).z;
+		real_t distance = p_camera_xform.origin.distance_to(xform.origin);
 
 		if (decal->distance_fade) {
 			float fade_begin = decal->distance_fade_begin;
@@ -2272,7 +2278,7 @@ void TextureStorage::update_decal_buffer(const PagedArray<RID> &p_decals, const
 
 		decal_sort[decal_count].decal_instance = decal_instance;
 		decal_sort[decal_count].decal = decal;
-		decal_sort[decal_count].depth = distance;
+		decal_sort[decal_count].depth = distance - decal_instance->sorting_offset;
 		decal_count++;
 	}
 
@@ -2292,11 +2298,10 @@ void TextureStorage::update_decal_buffer(const PagedArray<RID> &p_decals, const
 
 		decal_instance->cull_mask = decal->cull_mask;
 
-		Transform3D xform = decal_instance->transform;
 		float fade = 1.0;
 
 		if (decal->distance_fade) {
-			const real_t distance = -p_camera_inverse_xform.xform(xform.origin).z;
+			const real_t distance = decal_sort[i].depth + decal_instance->sorting_offset;
 			const float fade_begin = decal->distance_fade_begin;
 			const float fade_length = decal->distance_fade_length;
 
@@ -2312,11 +2317,16 @@ void TextureStorage::update_decal_buffer(const PagedArray<RID> &p_decals, const
 
 		Transform3D scale_xform;
 		scale_xform.basis.scale(decal_extents);
-		Transform3D to_decal_xform = (p_camera_inverse_xform * xform * scale_xform * uv_xform).affine_inverse();
+
+		Transform3D xform = decal_instance->transform;
+
+		Transform3D camera_inverse_xform = p_camera_xform.affine_inverse();
+
+		Transform3D to_decal_xform = (camera_inverse_xform * xform * scale_xform * uv_xform).affine_inverse();
 		MaterialStorage::store_transform(to_decal_xform, dd.xform);
 
 		Vector3 normal = xform.basis.get_column(Vector3::AXIS_Y).normalized();
-		normal = p_camera_inverse_xform.basis.xform(normal); //camera is normalized, so fine
+		normal = camera_inverse_xform.basis.xform(normal); //camera is normalized, so fine
 
 		dd.normal[0] = normal.x;
 		dd.normal[1] = normal.y;
@@ -2350,7 +2360,7 @@ void TextureStorage::update_decal_buffer(const PagedArray<RID> &p_decals, const
 			dd.normal_rect[2] = rect.size.x;
 			dd.normal_rect[3] = rect.size.y;
 
-			Basis normal_xform = p_camera_inverse_xform.basis * xform.basis.orthonormalized();
+			Basis normal_xform = camera_inverse_xform.basis * xform.basis.orthonormalized();
 			MaterialStorage::store_basis_3x4(normal_xform, dd.normal_xform);
 		} else {
 			dd.normal_rect[0] = 0;
diff --git a/servers/rendering/renderer_rd/storage_rd/texture_storage.h b/servers/rendering/renderer_rd/storage_rd/texture_storage.h
index 1558342c3b..ea0df0b459 100644
--- a/servers/rendering/renderer_rd/storage_rd/texture_storage.h
+++ b/servers/rendering/renderer_rd/storage_rd/texture_storage.h
@@ -258,6 +258,7 @@ private:
 	struct DecalInstance {
 		RID decal;
 		Transform3D transform;
+		float sorting_offset = 0.0;
 		uint32_t cull_mask = 0;
 		RendererRD::ForwardID forward_id = -1;
 	};
@@ -646,6 +647,7 @@ public:
 	virtual RID decal_instance_create(RID p_decal) override;
 	virtual void decal_instance_free(RID p_decal_instance) override;
 	virtual void decal_instance_set_transform(RID p_decal_instance, const Transform3D &p_transform) override;
+	virtual void decal_instance_set_sorting_offset(RID p_decal_instance, float p_sorting_offset) override;
 
 	_FORCE_INLINE_ RID decal_instance_get_base(RID p_decal_instance) const {
 		DecalInstance *di = decal_instance_owner.get_or_null(p_decal_instance);
@@ -677,7 +679,7 @@ public:
 	void free_decal_data();
 	void set_max_decals(const uint32_t p_max_decals);
 	RID get_decal_buffer() { return decal_buffer; }
-	void update_decal_buffer(const PagedArray<RID> &p_decals, const Transform3D &p_camera_inverse_xform);
+	void update_decal_buffer(const PagedArray<RID> &p_decals, const Transform3D &p_camera_xform);
 
 	/* RENDER TARGET API */
 
diff --git a/servers/rendering/renderer_scene_cull.cpp b/servers/rendering/renderer_scene_cull.cpp
index 218bb7b736..2a8d0b3cb4 100644
--- a/servers/rendering/renderer_scene_cull.cpp
+++ b/servers/rendering/renderer_scene_cull.cpp
@@ -694,6 +694,7 @@ void RendererSceneCull::instance_set_base(RID p_instance, RID p_base) {
 				instance->base_data = decal;
 
 				decal->instance = RSG::texture_storage->decal_instance_create(p_base);
+				RSG::texture_storage->decal_instance_set_sorting_offset(decal->instance, instance->sorting_offset);
 			} break;
 			case RS::INSTANCE_LIGHTMAP: {
 				InstanceLightmapData *lightmap_data = memnew(InstanceLightmapData);
@@ -871,6 +872,9 @@ void RendererSceneCull::instance_set_pivot_data(RID p_instance, float p_sorting_
 		InstanceGeometryData *geom = static_cast<InstanceGeometryData *>(instance->base_data);
 		ERR_FAIL_NULL(geom->geometry_instance);
 		geom->geometry_instance->set_pivot_data(p_sorting_offset, p_use_aabb_center);
+	} else if (instance->base_type == RS::INSTANCE_DECAL && instance->base_data) {
+		InstanceDecalData *decal = static_cast<InstanceDecalData *>(instance->base_data);
+		RSG::texture_storage->decal_instance_set_sorting_offset(decal->instance, instance->sorting_offset);
 	}
 }
 
diff --git a/servers/rendering/storage/texture_storage.h b/servers/rendering/storage/texture_storage.h
index 92149b0064..4c4a84d04e 100644
--- a/servers/rendering/storage/texture_storage.h
+++ b/servers/rendering/storage/texture_storage.h
@@ -127,6 +127,7 @@ public:
 	virtual RID decal_instance_create(RID p_decal) = 0;
 	virtual void decal_instance_free(RID p_decal_instance) = 0;
 	virtual void decal_instance_set_transform(RID p_decal_instance, const Transform3D &p_transform) = 0;
+	virtual void decal_instance_set_sorting_offset(RID p_decal_instance, float p_sorting_offset) = 0;
 
 	/* RENDER TARGET */