Merge pull request #49050 from reduz/implement-spirv-cache

Implement shader caching

8 files changed, 440 insertions, 40 deletions

diff --git a/servers/rendering/renderer_rd/renderer_compositor_rd.cpp b/servers/rendering/renderer_rd/renderer_compositor_rd.cpp
index 0012ba9c27..1337d36762 100644
--- a/servers/rendering/renderer_rd/renderer_compositor_rd.cpp
+++ b/servers/rendering/renderer_rd/renderer_compositor_rd.cpp
@@ -31,6 +31,7 @@
 #include "renderer_compositor_rd.h"
 
 #include "core/config/project_settings.h"
+#include "core/os/dir_access.h"
 
 void RendererCompositorRD::prepare_for_blitting_render_targets() {
 	RD::get_singleton()->prepare_screen_for_drawing();
@@ -155,6 +156,43 @@ void RendererCompositorRD::finalize() {
 RendererCompositorRD *RendererCompositorRD::singleton = nullptr;
 
 RendererCompositorRD::RendererCompositorRD() {
+	{
+		String shader_cache_dir = Engine::get_singleton()->get_shader_cache_path();
+		if (shader_cache_dir == String()) {
+			shader_cache_dir = "user://";
+		}
+		DirAccessRef da = DirAccess::open(shader_cache_dir);
+		if (!da) {
+			ERR_PRINT("Can't create shader cache folder, no shader caching will happen: " + shader_cache_dir);
+		} else {
+			Error err = da->change_dir("shader_cache");
+			if (err != OK) {
+				err = da->make_dir("shader_cache");
+			}
+			if (err != OK) {
+				ERR_PRINT("Can't create shader cache folder, no shader caching will happen: " + shader_cache_dir);
+			} else {
+				shader_cache_dir = shader_cache_dir.plus_file("shader_cache");
+
+				bool shader_cache_enabled = GLOBAL_GET("rendering/shader_compiler/shader_cache/enabled");
+				if (!Engine::get_singleton()->is_editor_hint() && !shader_cache_enabled) {
+					shader_cache_dir = String(); //disable only if not editor
+				}
+
+				if (shader_cache_dir != String()) {
+					bool compress = GLOBAL_GET("rendering/shader_compiler/shader_cache/compress");
+					bool use_zstd = GLOBAL_GET("rendering/shader_compiler/shader_cache/use_zstd_compression");
+					bool strip_debug = GLOBAL_GET("rendering/shader_compiler/shader_cache/strip_debug");
+
+					ShaderRD::set_shader_cache_dir(shader_cache_dir);
+					ShaderRD::set_shader_cache_save_compressed(compress);
+					ShaderRD::set_shader_cache_save_compressed_zstd(use_zstd);
+					ShaderRD::set_shader_cache_save_debug(!strip_debug);
+				}
+			}
+		}
+	}
+
 	singleton = this;
 	time = 0;
 
@@ -171,3 +209,7 @@ RendererCompositorRD::RendererCompositorRD() {
 		scene = memnew(RendererSceneRenderImplementation::RenderForwardClustered(storage));
 	}
 }
+
+RendererCompositorRD::~RendererCompositorRD() {
+	ShaderRD::set_shader_cache_dir(String());
+}
diff --git a/servers/rendering/renderer_rd/renderer_compositor_rd.h b/servers/rendering/renderer_rd/renderer_compositor_rd.h
index 52552f7ee3..7a78322051 100644
--- a/servers/rendering/renderer_rd/renderer_compositor_rd.h
+++ b/servers/rendering/renderer_rd/renderer_compositor_rd.h
@@ -118,6 +118,6 @@ public:
 
 	static RendererCompositorRD *singleton;
 	RendererCompositorRD();
-	~RendererCompositorRD() {}
+	~RendererCompositorRD();
 };
 #endif // RASTERIZER_RD_H
diff --git a/servers/rendering/renderer_rd/shader_compiler_rd.cpp b/servers/rendering/renderer_rd/shader_compiler_rd.cpp
index 056c8113a7..7deedb80c3 100644
--- a/servers/rendering/renderer_rd/shader_compiler_rd.cpp
+++ b/servers/rendering/renderer_rd/shader_compiler_rd.cpp
@@ -369,17 +369,24 @@ void ShaderCompilerRD::_dump_function_deps(const SL::ShaderNode *p_node, const S
 
 	ERR_FAIL_COND(fidx == -1);
 
+	Vector<StringName> uses_functions;
+
 	for (Set<StringName>::Element *E = p_node->functions[fidx].uses_function.front(); E; E = E->next()) {
-		if (added.has(E->get())) {
+		uses_functions.push_back(E->get());
+	}
+	uses_functions.sort_custom<StringName::AlphCompare>(); //ensure order is deterministic so the same shader is always produced
+
+	for (int k = 0; k < uses_functions.size(); k++) {
+		if (added.has(uses_functions[k])) {
 			continue; //was added already
 		}
 
-		_dump_function_deps(p_node, E->get(), p_func_code, r_to_add, added);
+		_dump_function_deps(p_node, uses_functions[k], p_func_code, r_to_add, added);
 
 		SL::FunctionNode *fnode = nullptr;
 
 		for (int i = 0; i < p_node->functions.size(); i++) {
-			if (p_node->functions[i].name == E->get()) {
+			if (p_node->functions[i].name == uses_functions[k]) {
 				fnode = p_node->functions[i].function;
 				break;
 			}
@@ -427,9 +434,9 @@ void ShaderCompilerRD::_dump_function_deps(const SL::ShaderNode *p_node, const S
 
 		header += ")\n";
 		r_to_add += header;
-		r_to_add += p_func_code[E->get()];
+		r_to_add += p_func_code[uses_functions[k]];
 
-		added.insert(E->get());
+		added.insert(uses_functions[k]);
 	}
 }
 
@@ -581,63 +588,74 @@ String ShaderCompilerRD::_dump_node_code(const SL::Node *p_node, int p_level, Ge
 			uniform_defines.resize(max_uniforms);
 			bool uses_uniforms = false;
 
+			Vector<StringName> uniform_names;
+
 			for (Map<StringName, SL::ShaderNode::Uniform>::Element *E = pnode->uniforms.front(); E; E = E->next()) {
+				uniform_names.push_back(E->key());
+			}
+
+			uniform_names.sort_custom<StringName::AlphCompare>(); //ensure order is deterministic so the same shader is always produced
+
+			for (int k = 0; k < uniform_names.size(); k++) {
+				StringName uniform_name = uniform_names[k];
+				const SL::ShaderNode::Uniform &uniform = pnode->uniforms[uniform_name];
+
 				String ucode;
 
-				if (E->get().scope == SL::ShaderNode::Uniform::SCOPE_INSTANCE) {
+				if (uniform.scope == SL::ShaderNode::Uniform::SCOPE_INSTANCE) {
 					//insert, but don't generate any code.
-					p_actions.uniforms->insert(E->key(), E->get());
+					p_actions.uniforms->insert(uniform_name, uniform);
 					continue; //instances are indexed directly, dont need index uniforms
 				}
-				if (SL::is_sampler_type(E->get().type)) {
-					ucode = "layout(set = " + itos(actions.texture_layout_set) + ", binding = " + itos(actions.base_texture_binding_index + E->get().texture_order) + ") uniform ";
+				if (SL::is_sampler_type(uniform.type)) {
+					ucode = "layout(set = " + itos(actions.texture_layout_set) + ", binding = " + itos(actions.base_texture_binding_index + uniform.texture_order) + ") uniform ";
 				}
 
-				bool is_buffer_global = !SL::is_sampler_type(E->get().type) && E->get().scope == SL::ShaderNode::Uniform::SCOPE_GLOBAL;
+				bool is_buffer_global = !SL::is_sampler_type(uniform.type) && uniform.scope == SL::ShaderNode::Uniform::SCOPE_GLOBAL;
 
 				if (is_buffer_global) {
 					//this is an integer to index the global table
 					ucode += _typestr(ShaderLanguage::TYPE_UINT);
 				} else {
-					ucode += _prestr(E->get().precision);
-					ucode += _typestr(E->get().type);
+					ucode += _prestr(uniform.precision);
+					ucode += _typestr(uniform.type);
 				}
 
-				ucode += " " + _mkid(E->key());
+				ucode += " " + _mkid(uniform_name);
 				ucode += ";\n";
-				if (SL::is_sampler_type(E->get().type)) {
+				if (SL::is_sampler_type(uniform.type)) {
 					for (int j = 0; j < STAGE_MAX; j++) {
 						r_gen_code.stage_globals[j] += ucode;
 					}
 
 					GeneratedCode::Texture texture;
-					texture.name = E->key();
-					texture.hint = E->get().hint;
-					texture.type = E->get().type;
-					texture.filter = E->get().filter;
-					texture.repeat = E->get().repeat;
-					texture.global = E->get().scope == ShaderLanguage::ShaderNode::Uniform::SCOPE_GLOBAL;
+					texture.name = uniform_name;
+					texture.hint = uniform.hint;
+					texture.type = uniform.type;
+					texture.filter = uniform.filter;
+					texture.repeat = uniform.repeat;
+					texture.global = uniform.scope == ShaderLanguage::ShaderNode::Uniform::SCOPE_GLOBAL;
 					if (texture.global) {
 						r_gen_code.uses_global_textures = true;
 					}
 
-					r_gen_code.texture_uniforms.write[E->get().texture_order] = texture;
+					r_gen_code.texture_uniforms.write[uniform.texture_order] = texture;
 				} else {
 					if (!uses_uniforms) {
 						uses_uniforms = true;
 					}
-					uniform_defines.write[E->get().order] = ucode;
+					uniform_defines.write[uniform.order] = ucode;
 					if (is_buffer_global) {
 						//globals are indices into the global table
-						uniform_sizes.write[E->get().order] = _get_datatype_size(ShaderLanguage::TYPE_UINT);
-						uniform_alignments.write[E->get().order] = _get_datatype_alignment(ShaderLanguage::TYPE_UINT);
+						uniform_sizes.write[uniform.order] = _get_datatype_size(ShaderLanguage::TYPE_UINT);
+						uniform_alignments.write[uniform.order] = _get_datatype_alignment(ShaderLanguage::TYPE_UINT);
 					} else {
-						uniform_sizes.write[E->get().order] = _get_datatype_size(E->get().type);
-						uniform_alignments.write[E->get().order] = _get_datatype_alignment(E->get().type);
+						uniform_sizes.write[uniform.order] = _get_datatype_size(uniform.type);
+						uniform_alignments.write[uniform.order] = _get_datatype_alignment(uniform.type);
 					}
 				}
 
-				p_actions.uniforms->insert(E->key(), E->get());
+				p_actions.uniforms->insert(uniform_name, uniform);
 			}
 
 			for (int i = 0; i < max_uniforms; i++) {
@@ -704,21 +722,32 @@ String ShaderCompilerRD::_dump_node_code(const SL::Node *p_node, int p_level, Ge
 
 			List<Pair<StringName, SL::ShaderNode::Varying>> var_frag_to_light;
 
+			Vector<StringName> varying_names;
+
 			for (Map<StringName, SL::ShaderNode::Varying>::Element *E = pnode->varyings.front(); E; E = E->next()) {
-				if (E->get().stage == SL::ShaderNode::Varying::STAGE_FRAGMENT_TO_LIGHT || E->get().stage == SL::ShaderNode::Varying::STAGE_FRAGMENT) {
-					var_frag_to_light.push_back(Pair<StringName, SL::ShaderNode::Varying>(E->key(), E->get()));
-					fragment_varyings.insert(E->key());
+				varying_names.push_back(E->key());
+			}
+
+			varying_names.sort_custom<StringName::AlphCompare>(); //ensure order is deterministic so the same shader is always produced
+
+			for (int k = 0; k < varying_names.size(); k++) {
+				StringName varying_name = varying_names[k];
+				const SL::ShaderNode::Varying &varying = pnode->varyings[varying_name];
+
+				if (varying.stage == SL::ShaderNode::Varying::STAGE_FRAGMENT_TO_LIGHT || varying.stage == SL::ShaderNode::Varying::STAGE_FRAGMENT) {
+					var_frag_to_light.push_back(Pair<StringName, SL::ShaderNode::Varying>(varying_name, varying));
+					fragment_varyings.insert(varying_name);
 					continue;
 				}
 
 				String vcode;
-				String interp_mode = _interpstr(E->get().interpolation);
-				vcode += _prestr(E->get().precision);
-				vcode += _typestr(E->get().type);
-				vcode += " " + _mkid(E->key());
-				if (E->get().array_size > 0) {
+				String interp_mode = _interpstr(varying.interpolation);
+				vcode += _prestr(varying.precision);
+				vcode += _typestr(varying.type);
+				vcode += " " + _mkid(varying_name);
+				if (varying.array_size > 0) {
 					vcode += "[";
-					vcode += itos(E->get().array_size);
+					vcode += itos(varying.array_size);
 					vcode += "]";
 				}
 				vcode += ";\n";
diff --git a/servers/rendering/renderer_rd/shader_rd.cpp b/servers/rendering/renderer_rd/shader_rd.cpp
index f7242a2b17..6f29ff42bc 100644
--- a/servers/rendering/renderer_rd/shader_rd.cpp
+++ b/servers/rendering/renderer_rd/shader_rd.cpp
@@ -30,8 +30,12 @@
 
 #include "shader_rd.h"
 
+#include "core/io/compression.h"
+#include "core/os/dir_access.h"
+#include "core/os/file_access.h"
 #include "renderer_compositor_rd.h"
 #include "servers/rendering/rendering_device.h"
+#include "thirdparty/misc/smolv.h"
 
 void ShaderRD::_add_stage(const char *p_code, StageType p_stage_type) {
 	Vector<String> lines = String(p_code).split("\n");
@@ -97,6 +101,7 @@ void ShaderRD::_add_stage(const char *p_code, StageType p_stage_type) {
 
 void ShaderRD::setup(const char *p_vertex_code, const char *p_fragment_code, const char *p_compute_code, const char *p_name) {
 	name = p_name;
+
 	if (p_compute_code) {
 		_add_stage(p_compute_code, STAGE_TYPE_COMPUTE);
 		is_compute = true;
@@ -109,6 +114,18 @@ void ShaderRD::setup(const char *p_vertex_code, const char *p_fragment_code, con
 			_add_stage(p_fragment_code, STAGE_TYPE_FRAGMENT);
 		}
 	}
+
+	StringBuilder tohash;
+	tohash.append("[VersionKey]");
+	tohash.append(RenderingDevice::get_singleton()->shader_get_cache_key());
+	tohash.append("[Vertex]");
+	tohash.append(p_vertex_code ? p_vertex_code : "");
+	tohash.append("[Fragment]");
+	tohash.append(p_fragment_code ? p_fragment_code : "");
+	tohash.append("[Compute]");
+	tohash.append(p_compute_code ? p_compute_code : "");
+
+	base_sha256 = tohash.as_string().sha256_text();
 }
 
 RID ShaderRD::version_create() {
@@ -131,6 +148,9 @@ void ShaderRD::_clear_version(Version *p_version) {
 		}
 
 		memdelete_arr(p_version->variants);
+		if (p_version->variant_stages) {
+			memdelete_arr(p_version->variant_stages);
+		}
 		p_version->variants = nullptr;
 	}
 }
@@ -183,7 +203,7 @@ void ShaderRD::_compile_variant(uint32_t p_variant, Version *p_version) {
 		return; //variant is disabled, return
 	}
 
-	Vector<RD::ShaderStageData> stages;
+	Vector<RD::ShaderStageData> &stages = p_version->variant_stages[p_variant];
 
 	String error;
 	String current_source;
@@ -313,6 +333,197 @@ RS::ShaderNativeSourceCode ShaderRD::version_get_native_source_code(RID p_versio
 	return source_code;
 }
 
+String ShaderRD::_version_get_sha1(Version *p_version) const {
+	StringBuilder hash_build;
+
+	hash_build.append("[uniforms]");
+	hash_build.append(p_version->uniforms.get_data());
+	hash_build.append("[vertex_globals]");
+	hash_build.append(p_version->vertex_globals.get_data());
+	hash_build.append("[fragment_globals]");
+	hash_build.append(p_version->fragment_globals.get_data());
+	hash_build.append("[compute_globals]");
+	hash_build.append(p_version->compute_globals.get_data());
+
+	Vector<StringName> code_sections;
+	for (Map<StringName, CharString>::Element *E = p_version->code_sections.front(); E; E = E->next()) {
+		code_sections.push_back(E->key());
+	}
+	code_sections.sort_custom<StringName::AlphCompare>();
+
+	for (int i = 0; i < code_sections.size(); i++) {
+		hash_build.append(String("[code:") + String(code_sections[i]) + "]");
+		hash_build.append(p_version->code_sections[code_sections[i]].get_data());
+	}
+	for (int i = 0; i < p_version->custom_defines.size(); i++) {
+		hash_build.append("[custom_defines:" + itos(i) + "]");
+		hash_build.append(p_version->custom_defines[i].get_data());
+	}
+
+	return hash_build.as_string().sha1_text();
+}
+
+static const char *shader_file_header = "GDSC";
+static const uint32_t cache_file_version = 1;
+
+bool ShaderRD::_load_from_cache(Version *p_version) {
+	String sha1 = _version_get_sha1(p_version);
+	String path = shader_cache_dir.plus_file(name).plus_file(base_sha256).plus_file(sha1) + ".cache";
+
+	uint64_t time_from = OS::get_singleton()->get_ticks_usec();
+
+	FileAccessRef f = FileAccess::open(path, FileAccess::READ);
+	if (!f) {
+		return false;
+	}
+
+	char header[5] = { 0, 0, 0, 0, 0 };
+	f->get_buffer((uint8_t *)header, 4);
+	ERR_FAIL_COND_V(header != String(shader_file_header), false);
+
+	uint32_t file_version = f->get_32();
+	if (file_version != cache_file_version) {
+		return false; // wrong version
+	}
+
+	uint32_t variant_count = f->get_32();
+
+	ERR_FAIL_COND_V(variant_count != (uint32_t)variant_defines.size(), false); //should not happen but check
+
+	bool success = true;
+	for (uint32_t i = 0; i < variant_count; i++) {
+		uint32_t stage_count = f->get_32();
+		p_version->variant_stages[i].resize(stage_count);
+		for (uint32_t j = 0; j < stage_count; j++) {
+			p_version->variant_stages[i].write[j].shader_stage = RD::ShaderStage(f->get_32());
+
+			int compression = f->get_32();
+			uint32_t length = f->get_32();
+
+			if (compression == 0) {
+				Vector<uint8_t> data;
+				data.resize(length);
+
+				f->get_buffer(data.ptrw(), length);
+
+				p_version->variant_stages[i].write[j].spir_v = data;
+			} else {
+				Vector<uint8_t> data;
+
+				if (compression == 2) {
+					//zstd
+					int smol_length = f->get_32();
+					Vector<uint8_t> zstd_data;
+
+					zstd_data.resize(smol_length);
+					f->get_buffer(zstd_data.ptrw(), smol_length);
+
+					data.resize(length);
+					Compression::decompress(data.ptrw(), data.size(), zstd_data.ptr(), zstd_data.size(), Compression::MODE_ZSTD);
+
+				} else {
+					data.resize(length);
+					f->get_buffer(data.ptrw(), length);
+				}
+
+				Vector<uint8_t> spirv;
+				uint32_t spirv_size = smolv::GetDecodedBufferSize(data.ptr(), data.size());
+				spirv.resize(spirv_size);
+				if (!smolv::Decode(data.ptr(), data.size(), spirv.ptrw(), spirv_size)) {
+					ERR_PRINT("Malformed smolv input uncompressing shader " + name + ", variant #" + itos(i) + " stage :" + itos(j));
+					success = false;
+					break;
+				}
+				p_version->variant_stages[i].write[j].spir_v = spirv;
+			}
+		}
+	}
+
+	if (!success) {
+		for (uint32_t i = 0; i < variant_count; i++) {
+			p_version->variant_stages[i].resize(0);
+		}
+		return false;
+	}
+
+	float time_ms = double(OS::get_singleton()->get_ticks_usec() - time_from) / 1000.0;
+
+	print_verbose("Shader cache load success '" + path + "' " + rtos(time_ms) + "ms.");
+
+	for (uint32_t i = 0; i < variant_count; i++) {
+		RID shader = RD::get_singleton()->shader_create(p_version->variant_stages[i]);
+		{
+			MutexLock lock(variant_set_mutex);
+			p_version->variants[i] = shader;
+		}
+	}
+
+	memdelete_arr(p_version->variant_stages); //clear stages
+	p_version->variant_stages = nullptr;
+	p_version->valid = true;
+	return true;
+}
+
+void ShaderRD::_save_to_cache(Version *p_version) {
+	String sha1 = _version_get_sha1(p_version);
+	String path = shader_cache_dir.plus_file(name).plus_file(base_sha256).plus_file(sha1) + ".cache";
+
+	FileAccessRef f = FileAccess::open(path, FileAccess::WRITE);
+	ERR_FAIL_COND(!f);
+	f->store_buffer((const uint8_t *)shader_file_header, 4);
+	f->store_32(cache_file_version); //file version
+	uint32_t variant_count = variant_defines.size();
+	f->store_32(variant_count); //variant count
+
+	for (uint32_t i = 0; i < variant_count; i++) {
+		f->store_32(p_version->variant_stages[i].size()); //stage count
+		for (int j = 0; j < p_version->variant_stages[i].size(); j++) {
+			f->store_32(p_version->variant_stages[i][j].shader_stage); //stage count
+			Vector<uint8_t> spirv = p_version->variant_stages[i][j].spir_v;
+
+			bool save_uncompressed = true;
+			if (shader_cache_save_compressed) {
+				smolv::ByteArray smolv;
+				bool strip_debug = !shader_cache_save_debug;
+				if (!smolv::Encode(spirv.ptr(), spirv.size(), smolv, strip_debug ? smolv::kEncodeFlagStripDebugInfo : 0)) {
+					ERR_PRINT("Error compressing shader " + name + ", variant #" + itos(i) + " stage :" + itos(i));
+				} else {
+					bool compress_zstd = shader_cache_save_compressed_zstd;
+
+					if (compress_zstd) {
+						Vector<uint8_t> zstd;
+						zstd.resize(Compression::get_max_compressed_buffer_size(smolv.size(), Compression::MODE_ZSTD));
+						int dst_size = Compression::compress(zstd.ptrw(), &smolv[0], smolv.size(), Compression::MODE_ZSTD);
+						if (dst_size >= 0 && (uint32_t)dst_size < smolv.size()) {
+							f->store_32(2); //compressed zstd
+							f->store_32(smolv.size()); //size of smolv buffer
+							f->store_32(dst_size); //size of smolv buffer
+							f->store_buffer(zstd.ptr(), dst_size); //smolv buffer
+						} else {
+							compress_zstd = false;
+						}
+					}
+
+					if (!compress_zstd) {
+						f->store_32(1); //compressed
+						f->store_32(smolv.size()); //size of smolv buffer
+						f->store_buffer(&smolv[0], smolv.size()); //smolv buffer
+					}
+					save_uncompressed = false;
+				}
+			}
+
+			if (save_uncompressed) {
+				f->store_32(0); //uncompressed
+				f->store_32(spirv.size()); //stage count
+				f->store_buffer(spirv.ptr(), spirv.size()); //stage count
+			}
+		}
+	}
+
+	f->close();
+}
+
 void ShaderRD::_compile_version(Version *p_version) {
 	_clear_version(p_version);
 
@@ -320,6 +531,15 @@ void ShaderRD::_compile_version(Version *p_version) {
 	p_version->dirty = false;
 
 	p_version->variants = memnew_arr(RID, variant_defines.size());
+	typedef Vector<RD::ShaderStageData> ShaderStageArray;
+	p_version->variant_stages = memnew_arr(ShaderStageArray, variant_defines.size());
+
+	if (shader_cache_dir_valid) {
+		if (_load_from_cache(p_version)) {
+			return;
+		}
+	}
+
 #if 1
 
 	RendererThreadPool::singleton->thread_work_pool.do_work(variant_defines.size(), this, &ShaderRD::_compile_variant, p_version);
@@ -351,10 +571,20 @@ void ShaderRD::_compile_version(Version *p_version) {
 			}
 		}
 		memdelete_arr(p_version->variants);
+		if (p_version->variant_stages) {
+			memdelete_arr(p_version->variant_stages);
+		}
 		p_version->variants = nullptr;
+		p_version->variant_stages = nullptr;
 		return;
+	} else if (shader_cache_dir_valid) {
+		//save shader cache
+		_save_to_cache(p_version);
 	}
 
+	memdelete_arr(p_version->variant_stages); //clear stages
+	p_version->variant_stages = nullptr;
+
 	p_version->valid = true;
 }
 
@@ -443,6 +673,8 @@ bool ShaderRD::is_variant_enabled(int p_variant) const {
 	return variants_enabled[p_variant];
 }
 
+bool ShaderRD::shader_cache_cleanup_on_start = false;
+
 ShaderRD::ShaderRD() {
 	// Do not feel forced to use this, in most cases it makes little to no difference.
 	bool use_32_threads = false;
@@ -469,8 +701,64 @@ void ShaderRD::initialize(const Vector<String> &p_variant_defines, const String
 		variant_defines.push_back(p_variant_defines[i].utf8());
 		variants_enabled.push_back(true);
 	}
+
+	if (shader_cache_dir != String()) {
+		StringBuilder hash_build;
+
+		hash_build.append("[base_hash]");
+		hash_build.append(base_sha256);
+		hash_build.append("[general_defines]");
+		hash_build.append(general_defines.get_data());
+		for (int i = 0; i < variant_defines.size(); i++) {
+			hash_build.append("[variant_defines:" + itos(i) + "]");
+			hash_build.append(variant_defines[i].get_data());
+		}
+
+		base_sha256 = hash_build.as_string().sha256_text();
+
+		DirAccessRef d = DirAccess::open(shader_cache_dir);
+		ERR_FAIL_COND(!d);
+		if (d->change_dir(name) != OK) {
+			Error err = d->make_dir(name);
+			ERR_FAIL_COND(err != OK);
+			d->change_dir(name);
+		}
+
+		//erase other versions?
+		if (shader_cache_cleanup_on_start) {
+		}
+		//
+		if (d->change_dir(base_sha256) != OK) {
+			Error err = d->make_dir(base_sha256);
+			ERR_FAIL_COND(err != OK);
+		}
+		shader_cache_dir_valid = true;
+
+		print_verbose("Shader '" + name + "' SHA256: " + base_sha256);
+	}
+}
+
+void ShaderRD::set_shader_cache_dir(const String &p_dir) {
+	shader_cache_dir = p_dir;
+}
+
+void ShaderRD::set_shader_cache_save_compressed(bool p_enable) {
+	shader_cache_save_compressed = p_enable;
 }
 
+void ShaderRD::set_shader_cache_save_compressed_zstd(bool p_enable) {
+	shader_cache_save_compressed_zstd = p_enable;
+}
+
+void ShaderRD::set_shader_cache_save_debug(bool p_enable) {
+	shader_cache_save_debug = p_enable;
+}
+
+String ShaderRD::shader_cache_dir;
+bool ShaderRD::shader_cache_save_compressed = true;
+bool ShaderRD::shader_cache_save_compressed_zstd = true;
+bool ShaderRD::shader_cache_save_debug = true;
+
 ShaderRD::~ShaderRD() {
 	List<RID> remaining;
 	version_owner.get_owned_list(&remaining);
diff --git a/servers/rendering/renderer_rd/shader_rd.h b/servers/rendering/renderer_rd/shader_rd.h
index f20d539621..9a68e02007 100644
--- a/servers/rendering/renderer_rd/shader_rd.h
+++ b/servers/rendering/renderer_rd/shader_rd.h
@@ -59,7 +59,8 @@ class ShaderRD {
 		Map<StringName, CharString> code_sections;
 		Vector<CharString> custom_defines;
 
-		RID *variants; //same size as version defines
+		Vector<RD::ShaderStageData> *variant_stages = nullptr;
+		RID *variants = nullptr; //same size as version defines
 
 		bool valid;
 		bool dirty;
@@ -96,10 +97,19 @@ class ShaderRD {
 
 	bool is_compute = false;
 
-	const char *name;
+	String name;
 
 	CharString base_compute_defines;
 
+	String base_sha256;
+
+	static String shader_cache_dir;
+	static bool shader_cache_cleanup_on_start;
+	static bool shader_cache_save_compressed;
+	static bool shader_cache_save_compressed_zstd;
+	static bool shader_cache_save_debug;
+	bool shader_cache_dir_valid = false;
+
 	enum StageType {
 		STAGE_TYPE_VERTEX,
 		STAGE_TYPE_FRAGMENT,
@@ -113,6 +123,10 @@ class ShaderRD {
 
 	void _add_stage(const char *p_code, StageType p_stage_type);
 
+	String _version_get_sha1(Version *p_version) const;
+	bool _load_from_cache(Version *p_version);
+	void _save_to_cache(Version *p_version);
+
 protected:
 	ShaderRD();
 	void setup(const char *p_vertex_code, const char *p_fragment_code, const char *p_compute_code, const char *p_name);
@@ -148,6 +162,11 @@ public:
 	void set_variant_enabled(int p_variant, bool p_enabled);
 	bool is_variant_enabled(int p_variant) const;
 
+	static void set_shader_cache_dir(const String &p_dir);
+	static void set_shader_cache_save_compressed(bool p_enable);
+	static void set_shader_cache_save_compressed_zstd(bool p_enable);
+	static void set_shader_cache_save_debug(bool p_enable);
+
 	RS::ShaderNativeSourceCode version_get_native_source_code(RID p_version);
 
 	void initialize(const Vector<String> &p_variant_defines, const String &p_general_defines = "");
diff --git a/servers/rendering/rendering_device.cpp b/servers/rendering/rendering_device.cpp
index e6ad001807..056cec4c1f 100644
--- a/servers/rendering/rendering_device.cpp
+++ b/servers/rendering/rendering_device.cpp
@@ -40,6 +40,7 @@ RenderingDevice *RenderingDevice::get_singleton() {
 
 RenderingDevice::ShaderCompileFunction RenderingDevice::compile_function = nullptr;
 RenderingDevice::ShaderCacheFunction RenderingDevice::cache_function = nullptr;
+RenderingDevice::ShaderGetCacheKeyFunction RenderingDevice::get_cache_key_function = nullptr;
 
 void RenderingDevice::shader_set_compile_function(ShaderCompileFunction p_function) {
 	compile_function = p_function;
@@ -49,6 +50,10 @@ void RenderingDevice::shader_set_cache_function(ShaderCacheFunction p_function)
 	cache_function = p_function;
 }
 
+void RenderingDevice::shader_set_get_cache_key_function(ShaderGetCacheKeyFunction p_function) {
+	get_cache_key_function = p_function;
+}
+
 Vector<uint8_t> RenderingDevice::shader_compile_from_source(ShaderStage p_stage, const String &p_source_code, ShaderLanguage p_language, String *r_error, bool p_allow_cache) {
 	if (p_allow_cache && cache_function) {
 		Vector<uint8_t> cache = cache_function(p_stage, p_source_code, p_language);
@@ -62,6 +67,13 @@ Vector<uint8_t> RenderingDevice::shader_compile_from_source(ShaderStage p_stage,
 	return compile_function(p_stage, p_source_code, p_language, r_error, &device_capabilities);
 }
 
+String RenderingDevice::shader_get_cache_key() const {
+	if (get_cache_key_function) {
+		return get_cache_key_function(&device_capabilities);
+	}
+	return String();
+}
+
 RID RenderingDevice::_texture_create(const Ref<RDTextureFormat> &p_format, const Ref<RDTextureView> &p_view, const TypedArray<PackedByteArray> &p_data) {
 	ERR_FAIL_COND_V(p_format.is_null(), RID());
 	ERR_FAIL_COND_V(p_view.is_null(), RID());
diff --git a/servers/rendering/rendering_device.h b/servers/rendering/rendering_device.h
index 27bded9810..4dcb9b963e 100644
--- a/servers/rendering/rendering_device.h
+++ b/servers/rendering/rendering_device.h
@@ -103,12 +103,14 @@ public:
 		bool supports_multiview = false; // If true this device supports multiview options
 	};
 
+	typedef String (*ShaderGetCacheKeyFunction)(const Capabilities *p_capabilities);
 	typedef Vector<uint8_t> (*ShaderCompileFunction)(ShaderStage p_stage, const String &p_source_code, ShaderLanguage p_language, String *r_error, const Capabilities *p_capabilities);
 	typedef Vector<uint8_t> (*ShaderCacheFunction)(ShaderStage p_stage, const String &p_source_code, ShaderLanguage p_language);
 
 private:
 	static ShaderCompileFunction compile_function;
 	static ShaderCacheFunction cache_function;
+	static ShaderGetCacheKeyFunction get_cache_key_function;
 
 	static RenderingDevice *singleton;
 
@@ -635,9 +637,11 @@ public:
 	const Capabilities *get_device_capabilities() const { return &device_capabilities; };
 
 	virtual Vector<uint8_t> shader_compile_from_source(ShaderStage p_stage, const String &p_source_code, ShaderLanguage p_language = SHADER_LANGUAGE_GLSL, String *r_error = nullptr, bool p_allow_cache = true);
+	virtual String shader_get_cache_key() const;
 
 	static void shader_set_compile_function(ShaderCompileFunction p_function);
 	static void shader_set_cache_function(ShaderCacheFunction p_function);
+	static void shader_set_get_cache_key_function(ShaderGetCacheKeyFunction p_function);
 
 	struct ShaderStageData {
 		ShaderStage shader_stage;
diff --git a/servers/rendering_server.cpp b/servers/rendering_server.cpp
index a9601fd661..4741e90a81 100644
--- a/servers/rendering_server.cpp
+++ b/servers/rendering_server.cpp
@@ -2306,6 +2306,12 @@ RenderingServer::RenderingServer() {
 					"rendering/vulkan/rendering/back_end",
 					PROPERTY_HINT_ENUM, "ForwardClustered,ForwardMobile"));
 
+	GLOBAL_DEF("rendering/shader_compiler/shader_cache/enabled", true);
+	GLOBAL_DEF("rendering/shader_compiler/shader_cache/compress", true);
+	GLOBAL_DEF("rendering/shader_compiler/shader_cache/use_zstd_compression", true);
+	GLOBAL_DEF("rendering/shader_compiler/shader_cache/strip_debug", false);
+	GLOBAL_DEF("rendering/shader_compiler/shader_cache/strip_debug.release", true);
+
 	GLOBAL_DEF("rendering/reflections/sky_reflections/roughness_layers", 8);
 	GLOBAL_DEF("rendering/reflections/sky_reflections/texture_array_reflections", true);
 	GLOBAL_DEF("rendering/reflections/sky_reflections/texture_array_reflections.mobile", false);