Vulkan: Make validation layers optional

They're now disabled by default, and can be enabled with the command line
argument `--vk-layers`.

When enabled, the errors about them being missing are now warnings, as
users were confused and thought this meant Vulkan is broken for them.

Fix crash in `~VulkanContext` when validation layers are disabled (exposed by
this PR since before they could not be disabled without source modification).

Also moved VulkanContext member initializations to header.

Fixes #37102.

6 files changed, 53 insertions, 47 deletions

diff --git a/core/engine.cpp b/core/engine.cpp
index d08cf92ecb..b0037ffb37 100644
--- a/core/engine.cpp
+++ b/core/engine.cpp
@@ -181,6 +181,14 @@ String Engine::get_license_text() const {
 	return String(GODOT_LICENSE_TEXT);
 }
 
+bool Engine::is_abort_on_gpu_errors_enabled() const {
+	return abort_on_gpu_errors;
+}
+
+bool Engine::is_validation_layers_enabled() const {
+	return use_validation_layers;
+}
+
 void Engine::add_singleton(const Singleton &p_singleton) {
 	singletons.push_back(p_singleton);
 	singleton_ptrs[p_singleton.name] = p_singleton.ptr;
@@ -208,10 +216,6 @@ Engine *Engine::get_singleton() {
 	return singleton;
 }
 
-bool Engine::is_abort_on_gpu_errors_enabled() const {
-	return abort_on_gpu_errors;
-}
-
 Engine::Engine() {
 	singleton = this;
 }
diff --git a/core/engine.h b/core/engine.h
index fef330c0c1..799773c687 100644
--- a/core/engine.h
+++ b/core/engine.h
@@ -64,6 +64,7 @@ private:
 	uint64_t _physics_frames = 0;
 	float _physics_interpolation_fraction = 0.0f;
 	bool abort_on_gpu_errors = false;
+	bool use_validation_layers = false;
 
 	uint64_t _idle_frames = 0;
 	bool _in_physics = false;
@@ -127,6 +128,7 @@ public:
 	String get_license_text() const;
 
 	bool is_abort_on_gpu_errors_enabled() const;
+	bool is_validation_layers_enabled() const;
 
 	Engine();
 	virtual ~Engine() {}
diff --git a/drivers/vulkan/vulkan_context.cpp b/drivers/vulkan/vulkan_context.cpp
index 997ed3935f..8840391966 100644
--- a/drivers/vulkan/vulkan_context.cpp
+++ b/drivers/vulkan/vulkan_context.cpp
@@ -154,7 +154,7 @@ VkBool32 VulkanContext::_check_layers(uint32_t check_count, const char **check_n
 			}
 		}
 		if (!found) {
-			ERR_PRINT("Can't find layer: " + String(check_names[i]));
+			WARN_PRINT("Can't find layer: " + String(check_names[i]));
 			return 0;
 		}
 	}
@@ -1479,23 +1479,6 @@ VkPhysicalDeviceLimits VulkanContext::get_device_limits() const {
 	return gpu_props.limits;
 }
 
-VulkanContext::VulkanContext() {
-	queue_props = nullptr;
-	command_buffer_count = 0;
-	instance_validation_layers = nullptr;
-	use_validation_layers = true;
-	VK_KHR_incremental_present_enabled = true;
-	VK_GOOGLE_display_timing_enabled = true;
-
-	command_buffer_queue.resize(1); //first one is the setup command always
-	command_buffer_queue.write[0] = nullptr;
-	command_buffer_count = 1;
-	queues_initialized = false;
-
-	buffers_prepared = false;
-	swapchainImageCount = 0;
-}
-
 RID VulkanContext::local_device_create() {
 	LocalDevice ld;
 
@@ -1583,6 +1566,13 @@ void VulkanContext::local_device_free(RID p_local_device) {
 	local_device_owner.free(p_local_device);
 }
 
+VulkanContext::VulkanContext() {
+	use_validation_layers = Engine::get_singleton()->is_validation_layers_enabled();
+
+	command_buffer_queue.resize(1); // First one is always the setup command.
+	command_buffer_queue.write[0] = nullptr;
+}
+
 VulkanContext::~VulkanContext() {
 	if (queue_props) {
 		free(queue_props);
@@ -1596,7 +1586,7 @@ VulkanContext::~VulkanContext() {
 				vkDestroySemaphore(device, image_ownership_semaphores[i], nullptr);
 			}
 		}
-		if (inst_initialized) {
+		if (inst_initialized && use_validation_layers) {
 			DestroyDebugUtilsMessengerEXT(inst, dbg_messenger, nullptr);
 		}
 		vkDestroyDevice(device, nullptr);
diff --git a/drivers/vulkan/vulkan_context.h b/drivers/vulkan/vulkan_context.h
index 9ebea42ecb..59e404512a 100644
--- a/drivers/vulkan/vulkan_context.h
+++ b/drivers/vulkan/vulkan_context.h
@@ -37,6 +37,7 @@
 #include "core/rid_owner.h"
 #include "core/ustring.h"
 #include "servers/display_server.h"
+
 #include <vulkan/vulkan.h>
 
 class VulkanContext {
@@ -51,13 +52,15 @@ class VulkanContext {
 	VkPhysicalDevice gpu;
 	VkPhysicalDeviceProperties gpu_props;
 	uint32_t queue_family_count;
-	VkQueueFamilyProperties *queue_props;
+	VkQueueFamilyProperties *queue_props = nullptr;
 	VkDevice device;
 	bool device_initialized = false;
 	bool inst_initialized = false;
 
-	//present
-	bool queues_initialized;
+	bool buffers_prepared = false;
+
+	// Present queue.
+	bool queues_initialized = false;
 	uint32_t graphics_queue_family_index;
 	uint32_t present_queue_family_index;
 	bool separate_present_queue;
@@ -78,7 +81,6 @@ class VulkanContext {
 		VkCommandBuffer graphics_to_present_cmd;
 		VkImageView view;
 		VkFramebuffer framebuffer;
-
 	} SwapchainImageResources;
 
 	struct Window {
@@ -89,7 +91,7 @@ class VulkanContext {
 		uint32_t current_buffer = 0;
 		int width = 0;
 		int height = 0;
-		VkCommandPool present_cmd_pool; //for separate present queue
+		VkCommandPool present_cmd_pool; // For separate present queue.
 		VkRenderPass render_pass = VK_NULL_HANDLE;
 	};
 
@@ -102,19 +104,24 @@ class VulkanContext {
 	RID_Owner<LocalDevice, true> local_device_owner;
 
 	Map<DisplayServer::WindowID, Window> windows;
-	uint32_t swapchainImageCount;
+	uint32_t swapchainImageCount = 0;
 
-	//commands
+	// Commands.
 
 	bool prepared;
 
-	//extensions
-	bool VK_KHR_incremental_present_enabled;
-	bool VK_GOOGLE_display_timing_enabled;
-	const char **instance_validation_layers;
-	uint32_t enabled_extension_count;
-	uint32_t enabled_layer_count;
+	Vector<VkCommandBuffer> command_buffer_queue;
+	int command_buffer_count = 1;
+
+	// Extensions.
+
+	bool VK_KHR_incremental_present_enabled = true;
+	bool VK_GOOGLE_display_timing_enabled = true;
+	uint32_t enabled_extension_count = 0;
 	const char *extension_names[MAX_EXTENSIONS];
+
+	const char **instance_validation_layers = nullptr;
+	uint32_t enabled_layer_count = 0;
 	const char *enabled_layers[MAX_LAYERS];
 
 	PFN_vkCreateDebugUtilsMessengerEXT CreateDebugUtilsMessengerEXT;
@@ -142,7 +149,8 @@ class VulkanContext {
 	Error _initialize_extensions();
 
 	VkBool32 _check_layers(uint32_t check_count, const char **check_names, uint32_t layer_count, VkLayerProperties *layers);
-	static VKAPI_ATTR VkBool32 VKAPI_CALL _debug_messenger_callback(VkDebugUtilsMessageSeverityFlagBitsEXT messageSeverity,
+	static VKAPI_ATTR VkBool32 VKAPI_CALL _debug_messenger_callback(
+			VkDebugUtilsMessageSeverityFlagBitsEXT messageSeverity,
 			VkDebugUtilsMessageTypeFlagsEXT messageType,
 			const VkDebugUtilsMessengerCallbackDataEXT *pCallbackData,
 			void *pUserData);
@@ -160,12 +168,11 @@ class VulkanContext {
 	Error _create_swap_chain();
 	Error _create_semaphores();
 
-	Vector<VkCommandBuffer> command_buffer_queue;
-	int command_buffer_count;
-
 protected:
 	virtual const char *_get_platform_surface_extension() const = 0;
-	//	virtual VkResult _create_surface(VkSurfaceKHR *surface, VkInstance p_instance) = 0;
+
+	// Enabled via command line argument.
+	bool use_validation_layers = false;
 
 	virtual Error _window_create(DisplayServer::WindowID p_window_id, VkSurfaceKHR p_surface, int p_width, int p_height);
 
@@ -173,10 +180,6 @@ protected:
 		return inst;
 	}
 
-	bool buffers_prepared;
-
-	bool use_validation_layers;
-
 public:
 	VkDevice get_device();
 	VkPhysicalDevice get_physical_device();
diff --git a/main/main.cpp b/main/main.cpp
index 5ebd0138d3..a0a993ec64 100644
--- a/main/main.cpp
+++ b/main/main.cpp
@@ -334,7 +334,10 @@ void Main::print_help(const char *p_binary) {
 	OS::get_singleton()->print("  -d, --debug                      Debug (local stdout debugger).\n");
 	OS::get_singleton()->print("  -b, --breakpoints                Breakpoint list as source::line comma-separated pairs, no spaces (use %%20 instead).\n");
 	OS::get_singleton()->print("  --profiling                      Enable profiling in the script debugger.\n");
+#if DEBUG_ENABLED
+	OS::get_singleton()->print("  --vk-layers                      Enable Vulkan Validation layers for debugging.\n");
 	OS::get_singleton()->print("  --gpu-abort                      Abort on GPU errors (usually validation layer errors), may help see the problem if your system freezes.\n");
+#endif
 	OS::get_singleton()->print("  --remote-debug <uri>             Remote debug (<protocol>://<host/IP>[:<port>], e.g. tcp://127.0.0.1:6007).\n");
 #if defined(DEBUG_ENABLED) && !defined(SERVER_ENABLED)
 	OS::get_singleton()->print("  --debug-collisions               Show collision shapes when running the scene.\n");
@@ -695,9 +698,12 @@ Error Main::setup(const char *execpath, int argc, char *argv[], bool p_second_ph
 		} else if (I->get() == "-w" || I->get() == "--windowed") { // force windowed window
 
 			init_windowed = true;
-		} else if (I->get() == "--gpu-abort") { // force windowed window
-
+#ifdef DEBUG_ENABLED
+		} else if (I->get() == "--vk-layers") {
+			Engine::singleton->use_validation_layers = true;
+		} else if (I->get() == "--gpu-abort") {
 			Engine::singleton->abort_on_gpu_errors = true;
+#endif
 		} else if (I->get() == "--tablet-driver") {
 			if (I->next()) {
 				tablet_driver = I->next()->get();
diff --git a/platform/android/vulkan/vulkan_context_android.cpp b/platform/android/vulkan/vulkan_context_android.cpp
index 5fb7a83da4..56ef99dfc7 100644
--- a/platform/android/vulkan/vulkan_context_android.cpp
+++ b/platform/android/vulkan/vulkan_context_android.cpp
@@ -29,6 +29,7 @@
 /*************************************************************************/
 
 #include "vulkan_context_android.h"
+
 #include <vulkan/vulkan_android.h>
 
 const char *VulkanContextAndroid::_get_platform_surface_extension() const {