add support for soft shadows to the lightmapper

4 files changed, 98 insertions, 45 deletions

diff --git a/modules/lightmapper_rd/lightmapper_rd.cpp b/modules/lightmapper_rd/lightmapper_rd.cpp
index 214c60091c..2ce3715bea 100644
--- a/modules/lightmapper_rd/lightmapper_rd.cpp
+++ b/modules/lightmapper_rd/lightmapper_rd.cpp
@@ -51,7 +51,7 @@ void LightmapperRD::add_mesh(const MeshData &p_mesh) {
 	mesh_instances.push_back(mi);
 }
 
-void LightmapperRD::add_directional_light(bool p_static, const Vector3 &p_direction, const Color &p_color, float p_energy, float p_angular_distance) {
+void LightmapperRD::add_directional_light(bool p_static, const Vector3 &p_direction, const Color &p_color, float p_energy, float p_angular_distance, float p_shadow_blur) {
 	Light l;
 	l.type = LIGHT_TYPE_DIRECTIONAL;
 	l.direction[0] = p_direction.x;
@@ -62,11 +62,12 @@ void LightmapperRD::add_directional_light(bool p_static, const Vector3 &p_direct
 	l.color[2] = p_color.b;
 	l.energy = p_energy;
 	l.static_bake = p_static;
-	l.size = p_angular_distance;
+	l.size = Math::tan(Math::deg2rad(p_angular_distance));
+	l.shadow_blur = p_shadow_blur;
 	lights.push_back(l);
 }
 
-void LightmapperRD::add_omni_light(bool p_static, const Vector3 &p_position, const Color &p_color, float p_energy, float p_range, float p_attenuation, float p_size) {
+void LightmapperRD::add_omni_light(bool p_static, const Vector3 &p_position, const Color &p_color, float p_energy, float p_range, float p_attenuation, float p_size, float p_shadow_blur) {
 	Light l;
 	l.type = LIGHT_TYPE_OMNI;
 	l.position[0] = p_position.x;
@@ -80,10 +81,11 @@ void LightmapperRD::add_omni_light(bool p_static, const Vector3 &p_position, con
 	l.energy = p_energy;
 	l.static_bake = p_static;
 	l.size = p_size;
+	l.shadow_blur = p_shadow_blur;
 	lights.push_back(l);
 }
 
-void LightmapperRD::add_spot_light(bool p_static, const Vector3 &p_position, const Vector3 p_direction, const Color &p_color, float p_energy, float p_range, float p_attenuation, float p_spot_angle, float p_spot_attenuation, float p_size) {
+void LightmapperRD::add_spot_light(bool p_static, const Vector3 &p_position, const Vector3 p_direction, const Color &p_color, float p_energy, float p_range, float p_attenuation, float p_spot_angle, float p_spot_attenuation, float p_size, float p_shadow_blur) {
 	Light l;
 	l.type = LIGHT_TYPE_SPOT;
 	l.position[0] = p_position.x;
@@ -102,6 +104,7 @@ void LightmapperRD::add_spot_light(bool p_static, const Vector3 &p_position, con
 	l.energy = p_energy;
 	l.static_bake = p_static;
 	l.size = p_size;
+	l.shadow_blur = p_shadow_blur;
 	lights.push_back(l);
 }
 
@@ -1140,6 +1143,23 @@ LightmapperRD::BakeError LightmapperRD::bake(BakeQuality p_quality, bool p_use_d
 
 		RID light_uniform_set = rd->uniform_set_create(uniforms, compute_shader_primary, 1);
 
+		switch (p_quality) {
+			case BAKE_QUALITY_LOW: {
+				push_constant.ray_count = GLOBAL_GET("rendering/lightmapping/bake_quality/low_quality_ray_count");
+			} break;
+			case BAKE_QUALITY_MEDIUM: {
+				push_constant.ray_count = GLOBAL_GET("rendering/lightmapping/bake_quality/medium_quality_ray_count");
+			} break;
+			case BAKE_QUALITY_HIGH: {
+				push_constant.ray_count = GLOBAL_GET("rendering/lightmapping/bake_quality/high_quality_ray_count");
+			} break;
+			case BAKE_QUALITY_ULTRA: {
+				push_constant.ray_count = GLOBAL_GET("rendering/lightmapping/bake_quality/ultra_quality_ray_count");
+			} break;
+		}
+
+		push_constant.ray_count = CLAMP(push_constant.ray_count, 16u, 8192u);
+
 		RD::ComputeListID compute_list = rd->compute_list_begin();
 		rd->compute_list_bind_compute_pipeline(compute_list, compute_shader_primary_pipeline);
 		rd->compute_list_bind_uniform_set(compute_list, compute_base_uniform_set, 0);
@@ -1230,23 +1250,6 @@ LightmapperRD::BakeError LightmapperRD::bake(BakeQuality p_quality, bool p_use_d
 		uniforms.write[1].set_id(0, light_dest_tex);
 		secondary_uniform_set[1] = rd->uniform_set_create(uniforms, compute_shader_secondary, 1);
 
-		switch (p_quality) {
-			case BAKE_QUALITY_LOW: {
-				push_constant.ray_count = GLOBAL_GET("rendering/lightmapping/bake_quality/low_quality_ray_count");
-			} break;
-			case BAKE_QUALITY_MEDIUM: {
-				push_constant.ray_count = GLOBAL_GET("rendering/lightmapping/bake_quality/medium_quality_ray_count");
-			} break;
-			case BAKE_QUALITY_HIGH: {
-				push_constant.ray_count = GLOBAL_GET("rendering/lightmapping/bake_quality/high_quality_ray_count");
-			} break;
-			case BAKE_QUALITY_ULTRA: {
-				push_constant.ray_count = GLOBAL_GET("rendering/lightmapping/bake_quality/ultra_quality_ray_count");
-			} break;
-		}
-
-		push_constant.ray_count = CLAMP(push_constant.ray_count, 16u, 8192u);
-
 		int max_region_size = nearest_power_of_2_templated(int(GLOBAL_GET("rendering/lightmapping/bake_performance/region_size")));
 		int max_rays = GLOBAL_GET("rendering/lightmapping/bake_performance/max_rays_per_pass");
 
diff --git a/modules/lightmapper_rd/lightmapper_rd.h b/modules/lightmapper_rd/lightmapper_rd.h
index bf9f9b5954..8cb4b58a18 100644
--- a/modules/lightmapper_rd/lightmapper_rd.h
+++ b/modules/lightmapper_rd/lightmapper_rd.h
@@ -57,8 +57,9 @@ class LightmapperRD : public Lightmapper {
 		float attenuation = 0.0;
 		float cos_spot_angle = 0.0;
 		float inv_spot_attenuation = 0.0;
+		float shadow_blur = 0.0;
 		uint32_t static_bake = 0;
-		uint32_t pad[3] = {};
+		uint32_t pad[2] = {};
 
 		bool operator<(const Light &p_light) const {
 			return type < p_light.type;
@@ -236,9 +237,9 @@ class LightmapperRD : public Lightmapper {
 
 public:
 	virtual void add_mesh(const MeshData &p_mesh) override;
-	virtual void add_directional_light(bool p_static, const Vector3 &p_direction, const Color &p_color, float p_energy, float p_angular_distance) override;
-	virtual void add_omni_light(bool p_static, const Vector3 &p_position, const Color &p_color, float p_energy, float p_range, float p_attenuation, float p_size) override;
-	virtual void add_spot_light(bool p_static, const Vector3 &p_position, const Vector3 p_direction, const Color &p_color, float p_energy, float p_range, float p_attenuation, float p_spot_angle, float p_spot_attenuation, float p_size) override;
+	virtual void add_directional_light(bool p_static, const Vector3 &p_direction, const Color &p_color, float p_energy, float p_angular_distance, float p_shadow_blur) override;
+	virtual void add_omni_light(bool p_static, const Vector3 &p_position, const Color &p_color, float p_energy, float p_range, float p_attenuation, float p_size, float p_shadow_blur) override;
+	virtual void add_spot_light(bool p_static, const Vector3 &p_position, const Vector3 p_direction, const Color &p_color, float p_energy, float p_range, float p_attenuation, float p_spot_angle, float p_spot_attenuation, float p_size, float p_shadow_blur) override;
 	virtual void add_probe(const Vector3 &p_position) override;
 	virtual BakeError bake(BakeQuality p_quality, bool p_use_denoiser, int p_bounces, float p_bias, int p_max_texture_size, bool p_bake_sh, GenerateProbes p_generate_probes, const Ref<Image> &p_environment_panorama, const Basis &p_environment_transform, BakeStepFunc p_step_function = nullptr, void *p_bake_userdata = nullptr) override;
 
diff --git a/modules/lightmapper_rd/lm_common_inc.glsl b/modules/lightmapper_rd/lm_common_inc.glsl
index 58523dc1f8..e0e8000f56 100644
--- a/modules/lightmapper_rd/lm_common_inc.glsl
+++ b/modules/lightmapper_rd/lm_common_inc.glsl
@@ -51,8 +51,9 @@ struct Light {
 	float cos_spot_angle;
 	float inv_spot_attenuation;
 
+	float shadow_blur;
 	bool static_bake;
-	uint pad[3];
+	uint pad[2];
 };
 
 layout(set = 0, binding = 4, std430) restrict readonly buffer Lights {
diff --git a/modules/lightmapper_rd/lm_compute.glsl b/modules/lightmapper_rd/lm_compute.glsl
index 0b6b72a310..efa6cd50b4 100644
--- a/modules/lightmapper_rd/lm_compute.glsl
+++ b/modules/lightmapper_rd/lm_compute.glsl
@@ -316,19 +316,24 @@ void main() {
 
 	for (uint i = 0; i < params.light_count; i++) {
 		vec3 light_pos;
+		float dist;
 		float attenuation;
+		float soft_shadowing_disk_size;
 		if (lights.data[i].type == LIGHT_TYPE_DIRECTIONAL) {
 			vec3 light_vec = lights.data[i].direction;
 			light_pos = position - light_vec * length(params.world_size);
+			dist = length(params.world_size);
 			attenuation = 1.0;
+			soft_shadowing_disk_size = lights.data[i].size;
 		} else {
 			light_pos = lights.data[i].position;
-			float d = distance(position, light_pos);
-			if (d > lights.data[i].range) {
+			dist = distance(position, light_pos);
+			if (dist > lights.data[i].range) {
 				continue;
 			}
+			soft_shadowing_disk_size = lights.data[i].size / dist;
 
-			attenuation = get_omni_attenuation(d, 1.0 / lights.data[i].range, lights.data[i].attenuation);
+			attenuation = get_omni_attenuation(dist, 1.0 / lights.data[i].range, lights.data[i].attenuation);
 
 			if (lights.data[i].type == LIGHT_TYPE_SPOT) {
 				vec3 rel = normalize(position - light_pos);
@@ -352,27 +357,70 @@ void main() {
 			continue; //no need to do anything
 		}
 
-		if (trace_ray(position + light_dir * params.bias, light_pos) == RAY_MISS) {
-			vec3 light = lights.data[i].color * lights.data[i].energy * attenuation;
-			if (lights.data[i].static_bake) {
-				static_light += light;
-#ifdef USE_SH_LIGHTMAPS
+		float penumbra = 0.0;
+		if (lights.data[i].size > 0.0) {
+			vec3 light_to_point = -light_dir;
+			vec3 aux = light_to_point.y < 0.777 ? vec3(0.0, 1.0, 0.0) : vec3(1.0, 0.0, 0.0);
+			vec3 light_to_point_tan = normalize(cross(light_to_point, aux));
+			vec3 light_to_point_bitan = normalize(cross(light_to_point, light_to_point_tan));
+
+			const uint shadowing_rays_check_penumbra_denom = 2;
+			uint shadowing_ray_count = params.ray_count;
+
+			uint hits = 0;
+			uint noise = random_seed(ivec3(atlas_pos, 43573547 /* some prime */));
+			vec3 light_disk_to_point = light_to_point;
+			for (uint j = 0; j < shadowing_ray_count; j++) {
+				// Optimization:
+				// Once already traced an important proportion of rays, if all are hits or misses,
+				// assume we're not in the penumbra so we can infer the rest would have the same result
+				if (j == shadowing_ray_count / shadowing_rays_check_penumbra_denom) {
+					if (hits == j) {
+						// Assume totally lit
+						hits = shadowing_ray_count;
+						break;
+					} else if (hits == 0) {
+						// Assume totally dark
+						hits = 0;
+						break;
+					}
+				}
 
-				float c[4] = float[](
-						0.282095, //l0
-						0.488603 * light_dir.y, //l1n1
-						0.488603 * light_dir.z, //l1n0
-						0.488603 * light_dir.x //l1p1
-				);
+				float r = randomize(noise);
+				float a = randomize(noise) * 2.0 * PI;
+				vec2 disk_sample = (r * vec2(cos(a), sin(a))) * soft_shadowing_disk_size * lights.data[i].shadow_blur;
+				light_disk_to_point = normalize(light_to_point + disk_sample.x * light_to_point_tan + disk_sample.y * light_to_point_bitan);
 
-				for (uint j = 0; j < 4; j++) {
-					sh_accum[j].rgb += light * c[j] * (1.0 / 3.0);
+				if (trace_ray(position - light_disk_to_point * params.bias, position - light_disk_to_point * dist) == RAY_MISS) {
+					hits++;
 				}
-#endif
+			}
+			penumbra = float(hits) / float(shadowing_ray_count);
+		} else {
+			if (trace_ray(position + light_dir * params.bias, light_pos) == RAY_MISS) {
+				penumbra = 1.0;
+			}
+		}
 
-			} else {
-				dynamic_light += light;
+		vec3 light = lights.data[i].color * lights.data[i].energy * attenuation * penumbra;
+		if (lights.data[i].static_bake) {
+			static_light += light;
+#ifdef USE_SH_LIGHTMAPS
+
+			float c[4] = float[](
+					0.282095, //l0
+					0.488603 * light_dir.y, //l1n1
+					0.488603 * light_dir.z, //l1n0
+					0.488603 * light_dir.x //l1p1
+			);
+
+			for (uint j = 0; j < 4; j++) {
+				sh_accum[j].rgb += light * c[j] * (1.0 / 3.0);
 			}
+#endif
+
+		} else {
+			dynamic_light += light;
 		}
 	}