Re-enable per-pixel transparency support on Linux, macOS, and Windows (for Vulkan and OpenGL rendering drivers).

10 files changed, 88 insertions, 57 deletions

diff --git a/doc/classes/ProjectSettings.xml b/doc/classes/ProjectSettings.xml
index 47837cd9e1..1145798240 100644
--- a/doc/classes/ProjectSettings.xml
+++ b/doc/classes/ProjectSettings.xml
@@ -557,6 +557,9 @@
 		<member name="display/window/ios/hide_home_indicator" type="bool" setter="" getter="" default="true">
 			If [code]true[/code], the home indicator is hidden automatically. This only affects iOS devices without a physical home button.
 		</member>
+		<member name="display/window/per_pixel_transparency/allowed" type="bool" setter="" getter="" default="false">
+			If [code]true[/code], allows per-pixel transparency for the window background. This affects performance, so leave it on [code]false[/code] unless you need it.
+		</member>
 		<member name="display/window/size/always_on_top" type="bool" setter="" getter="" default="false">
 			Forces the main window to be always on top.
 			[b]Note:[/b] This setting is ignored on iOS, Android, and Web.
diff --git a/drivers/gles3/rasterizer_gles3.cpp b/drivers/gles3/rasterizer_gles3.cpp
index 7537636356..cc96294ca5 100644
--- a/drivers/gles3/rasterizer_gles3.cpp
+++ b/drivers/gles3/rasterizer_gles3.cpp
@@ -108,19 +108,6 @@ void RasterizerGLES3::begin_frame(double frame_step) {
 }
 
 void RasterizerGLES3::end_frame(bool p_swap_buffers) {
-	//	if (OS::get_singleton()->is_layered_allowed()) {
-	//		if (!OS::get_singleton()->get_window_per_pixel_transparency_enabled()) {
-	//clear alpha
-	//			glColorMask(false, false, false, true);
-	//			glClearColor(0.5, 0, 0, 1);
-	//			glClear(GL_COLOR_BUFFER_BIT);
-	//			glColorMask(true, true, true, true);
-	//		}
-	//	}
-
-	//	glClearColor(1, 0, 0, 1);
-	//	glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT | GL_ACCUM_BUFFER_BIT | GL_STENCIL_BUFFER_BIT);
-
 	if (p_swap_buffers) {
 		DisplayServer::get_singleton()->swap_buffers();
 	} else {
diff --git a/drivers/vulkan/vulkan_context.cpp b/drivers/vulkan/vulkan_context.cpp
index b52179b4f3..99ef57abae 100644
--- a/drivers/vulkan/vulkan_context.cpp
+++ b/drivers/vulkan/vulkan_context.cpp
@@ -1706,10 +1706,10 @@ Error VulkanContext::_update_swap_chain(Window *window) {
 	// Find a supported composite alpha mode - one of these is guaranteed to be set.
 	VkCompositeAlphaFlagBitsKHR compositeAlpha = VK_COMPOSITE_ALPHA_OPAQUE_BIT_KHR;
 	VkCompositeAlphaFlagBitsKHR compositeAlphaFlags[4] = {
-		VK_COMPOSITE_ALPHA_OPAQUE_BIT_KHR,
 		VK_COMPOSITE_ALPHA_PRE_MULTIPLIED_BIT_KHR,
 		VK_COMPOSITE_ALPHA_POST_MULTIPLIED_BIT_KHR,
 		VK_COMPOSITE_ALPHA_INHERIT_BIT_KHR,
+		VK_COMPOSITE_ALPHA_OPAQUE_BIT_KHR,
 	};
 	for (uint32_t i = 0; i < ARRAY_SIZE(compositeAlphaFlags); i++) {
 		if (surfCapabilities.supportedCompositeAlpha & compositeAlphaFlags[i]) {
diff --git a/main/main.cpp b/main/main.cpp
index a0d2f594ac..650d1159e0 100644
--- a/main/main.cpp
+++ b/main/main.cpp
@@ -1555,17 +1555,11 @@ Error Main::setup(const char *execpath, int argc, char *argv[], bool p_second_ph
 	GLOBAL_DEF("internationalization/locale/include_text_server_data", false);
 
 	OS::get_singleton()->_allow_hidpi = GLOBAL_DEF("display/window/dpi/allow_hidpi", true);
-
-	// FIXME: Restore support.
-#if 0
-	//OS::get_singleton()->_allow_layered = GLOBAL_DEF("display/window/per_pixel_transparency/allowed", false);
-	video_mode.layered = GLOBAL_DEF("display/window/per_pixel_transparency/enabled", false);
-#endif
+	OS::get_singleton()->_allow_layered = GLOBAL_DEF("display/window/per_pixel_transparency/allowed", false);
 
 	if (editor || project_manager) {
 		// The editor and project manager always detect and use hiDPI if needed
 		OS::get_singleton()->_allow_hidpi = true;
-		OS::get_singleton()->_allow_layered = false;
 	}
 
 	if (rtm == -1) {
diff --git a/platform/linuxbsd/display_server_x11.cpp b/platform/linuxbsd/display_server_x11.cpp
index a329c7ee8c..a0039fbbf1 100644
--- a/platform/linuxbsd/display_server_x11.cpp
+++ b/platform/linuxbsd/display_server_x11.cpp
@@ -2221,7 +2221,7 @@ void DisplayServerX11::window_set_flag(WindowFlags p_flag, bool p_enabled, Windo
 
 		} break;
 		case WINDOW_FLAG_TRANSPARENT: {
-			//todo reimplement
+			wd.layered_window = p_enabled;
 		} break;
 		case WINDOW_FLAG_NO_FOCUS: {
 			wd.no_focus = p_enabled;
@@ -2274,7 +2274,7 @@ bool DisplayServerX11::window_get_flag(WindowFlags p_flag, WindowID p_window) co
 			return wd.on_top;
 		} break;
 		case WINDOW_FLAG_TRANSPARENT: {
-			//todo reimplement
+			return wd.layered_window;
 		} break;
 		case WINDOW_FLAG_NO_FOCUS: {
 			return wd.no_focus;
@@ -4426,13 +4426,41 @@ DisplayServer *DisplayServerX11::create_func(const String &p_rendering_driver, W
 DisplayServerX11::WindowID DisplayServerX11::_create_window(WindowMode p_mode, VSyncMode p_vsync_mode, uint32_t p_flags, const Rect2i &p_rect) {
 	//Create window
 
-	long visualMask = VisualScreenMask;
-	int numberOfVisuals;
-	XVisualInfo vInfoTemplate = {};
-	vInfoTemplate.screen = DefaultScreen(x11_display);
-	XVisualInfo *visualInfo = XGetVisualInfo(x11_display, visualMask, &vInfoTemplate, &numberOfVisuals);
+	XVisualInfo visualInfo;
+	bool vi_selected = false;
 
-	Colormap colormap = XCreateColormap(x11_display, RootWindow(x11_display, vInfoTemplate.screen), visualInfo->visual, AllocNone);
+#ifdef GLES3_ENABLED
+	if (gl_manager) {
+		visualInfo = gl_manager->get_vi(x11_display);
+		vi_selected = true;
+	}
+#endif
+
+	if (!vi_selected) {
+		long visualMask = VisualScreenMask;
+		int numberOfVisuals;
+		XVisualInfo vInfoTemplate = {};
+		vInfoTemplate.screen = DefaultScreen(x11_display);
+		XVisualInfo *vi_list = XGetVisualInfo(x11_display, visualMask, &vInfoTemplate, &numberOfVisuals);
+		ERR_FAIL_COND_V(!vi_list, INVALID_WINDOW_ID);
+
+		visualInfo = vi_list[0];
+		if (OS::get_singleton()->is_layered_allowed()) {
+			for (int i = 0; i < numberOfVisuals; i++) {
+				XRenderPictFormat *pict_format = XRenderFindVisualFormat(x11_display, vi_list[i].visual);
+				if (!pict_format) {
+					continue;
+				}
+				visualInfo = vi_list[i];
+				if (pict_format->direct.alphaMask > 0) {
+					break;
+				}
+			}
+		}
+		XFree(vi_list);
+	}
+
+	Colormap colormap = XCreateColormap(x11_display, RootWindow(x11_display, visualInfo.screen), visualInfo.visual, AllocNone);
 
 	XSetWindowAttributes windowAttributes = {};
 	windowAttributes.colormap = colormap;
@@ -4442,6 +4470,13 @@ DisplayServerX11::WindowID DisplayServerX11::_create_window(WindowMode p_mode, V
 
 	unsigned long valuemask = CWBorderPixel | CWColormap | CWEventMask;
 
+	if (OS::get_singleton()->is_layered_allowed()) {
+		windowAttributes.background_pixmap = None;
+		windowAttributes.background_pixel = 0;
+		windowAttributes.border_pixmap = None;
+		valuemask |= CWBackPixel;
+	}
+
 	WindowID id = window_id_counter++;
 	WindowData &wd = windows[id];
 
@@ -4465,7 +4500,7 @@ DisplayServerX11::WindowID DisplayServerX11::_create_window(WindowMode p_mode, V
 	}
 
 	{
-		wd.x11_window = XCreateWindow(x11_display, RootWindow(x11_display, visualInfo->screen), p_rect.position.x, p_rect.position.y, p_rect.size.width > 0 ? p_rect.size.width : 1, p_rect.size.height > 0 ? p_rect.size.height : 1, 0, visualInfo->depth, InputOutput, visualInfo->visual, valuemask, &windowAttributes);
+		wd.x11_window = XCreateWindow(x11_display, RootWindow(x11_display, visualInfo.screen), p_rect.position.x, p_rect.position.y, p_rect.size.width > 0 ? p_rect.size.width : 1, p_rect.size.height > 0 ? p_rect.size.height : 1, 0, visualInfo.depth, InputOutput, visualInfo.visual, valuemask, &windowAttributes);
 
 		// Enable receiving notification when the window is initialized (MapNotify)
 		// so the focus can be set at the right time.
@@ -4602,8 +4637,6 @@ DisplayServerX11::WindowID DisplayServerX11::_create_window(WindowMode p_mode, V
 
 		XSync(x11_display, False);
 		//XSetErrorHandler(oldHandler);
-
-		XFree(visualInfo);
 	}
 
 	window_set_mode(p_mode, id);
diff --git a/platform/linuxbsd/display_server_x11.h b/platform/linuxbsd/display_server_x11.h
index dd9ac088d9..ea03b2328c 100644
--- a/platform/linuxbsd/display_server_x11.h
+++ b/platform/linuxbsd/display_server_x11.h
@@ -159,6 +159,7 @@ class DisplayServerX11 : public DisplayServer {
 		bool minimized = false;
 		bool maximized = false;
 		bool is_popup = false;
+		bool layered_window = false;
 
 		Rect2i parent_safe_rect;
 
@@ -251,8 +252,6 @@ class DisplayServerX11 : public DisplayServer {
 	CursorShape current_cursor = CURSOR_ARROW;
 	HashMap<CursorShape, Vector<Variant>> cursors_cache;
 
-	bool layered_window = false;
-
 	String rendering_driver;
 	void set_wm_fullscreen(bool p_enabled);
 	void set_wm_above(bool p_enabled);
diff --git a/platform/linuxbsd/gl_manager_x11.cpp b/platform/linuxbsd/gl_manager_x11.cpp
index d3fb1d6705..04c1df71fb 100644
--- a/platform/linuxbsd/gl_manager_x11.cpp
+++ b/platform/linuxbsd/gl_manager_x11.cpp
@@ -127,10 +127,6 @@ Error GLManager_X11::_create_context(GLDisplay &gl_display) {
 	GLXFBConfig fbconfig = nullptr;
 	XVisualInfo *vi = nullptr;
 
-	gl_display.x_swa.event_mask = StructureNotifyMask;
-	gl_display.x_swa.border_pixel = 0;
-	gl_display.x_valuemask = CWBorderPixel | CWColormap | CWEventMask;
-
 	if (OS::get_singleton()->is_layered_allowed()) {
 		GLXFBConfig *fbc = glXChooseFBConfig(x11_display, DefaultScreen(x11_display), visual_attribs_layered, &fbcount);
 		ERR_FAIL_COND_V(!fbc, ERR_UNCONFIGURED);
@@ -156,12 +152,6 @@ Error GLManager_X11::_create_context(GLDisplay &gl_display) {
 		XFree(fbc);
 
 		ERR_FAIL_COND_V(!fbconfig, ERR_UNCONFIGURED);
-
-		gl_display.x_swa.background_pixmap = None;
-		gl_display.x_swa.background_pixel = 0;
-		gl_display.x_swa.border_pixmap = None;
-		gl_display.x_valuemask |= CWBackPixel;
-
 	} else {
 		GLXFBConfig *fbc = glXChooseFBConfig(x11_display, DefaultScreen(x11_display), visual_attribs, &fbcount);
 		ERR_FAIL_COND_V(!fbc, ERR_UNCONFIGURED);
@@ -189,8 +179,6 @@ Error GLManager_X11::_create_context(GLDisplay &gl_display) {
 		} break;
 	}
 
-	gl_display.x_swa.colormap = XCreateColormap(x11_display, RootWindow(x11_display, vi->screen), vi->visual, AllocNone);
-
 	XSync(x11_display, False);
 	XSetErrorHandler(oldHandler);
 
@@ -205,6 +193,10 @@ Error GLManager_X11::_create_context(GLDisplay &gl_display) {
 	return OK;
 }
 
+XVisualInfo GLManager_X11::get_vi(Display *p_display) {
+	return _displays[_find_or_create_display(p_display)].x_vi;
+}
+
 Error GLManager_X11::window_create(DisplayServer::WindowID p_window_id, ::Window p_window, Display *p_display, int p_width, int p_height) {
 	// make sure vector is big enough...
 	// we can mirror the external vector, it is simpler
@@ -223,8 +215,6 @@ Error GLManager_X11::window_create(DisplayServer::WindowID p_window_id, ::Window
 
 	// the display could be invalid .. check NYI
 	GLDisplay &gl_display = _displays[win.gldisplay_id];
-	//const XVisualInfo &vi = gl_display.x_vi;
-	//XSetWindowAttributes &swa = gl_display.x_swa;
 	::Display *x11_display = gl_display.x11_display;
 	::Window &x11_window = win.x11_window;
 
@@ -315,6 +305,16 @@ void GLManager_X11::swap_buffers() {
 		return;
 	}
 
+	// On X11, when enabled, transparancy is always active, so clear alpha manually.
+	if (OS::get_singleton()->is_layered_allowed()) {
+		if (!DisplayServer::get_singleton()->window_get_flag(DisplayServer::WINDOW_FLAG_TRANSPARENT, _current_window->window_id)) {
+			glColorMask(false, false, false, true);
+			glClearColor(0, 0, 0, 1);
+			glClear(GL_COLOR_BUFFER_BIT);
+			glColorMask(true, true, true, true);
+		}
+	}
+
 	//	print_line("\tswap_buffers");
 
 	// only for debugging without drawing anything
diff --git a/platform/linuxbsd/gl_manager_x11.h b/platform/linuxbsd/gl_manager_x11.h
index fb2c74a2b6..4f78c45c88 100644
--- a/platform/linuxbsd/gl_manager_x11.h
+++ b/platform/linuxbsd/gl_manager_x11.h
@@ -68,8 +68,6 @@ private:
 		GLManager_X11_Private *context = nullptr;
 		::Display *x11_display;
 		XVisualInfo x_vi;
-		XSetWindowAttributes x_swa;
-		unsigned long x_valuemask;
 	};
 
 	// just for convenience, window and display struct
@@ -102,6 +100,7 @@ private:
 	Error _create_context(GLDisplay &gl_display);
 
 public:
+	XVisualInfo get_vi(Display *p_display);
 	Error window_create(DisplayServer::WindowID p_window_id, ::Window p_window, Display *p_display, int p_width, int p_height);
 	void window_destroy(DisplayServer::WindowID p_window_id);
 	void window_resize(DisplayServer::WindowID p_window_id, int p_width, int p_height);
diff --git a/platform/windows/display_server_windows.cpp b/platform/windows/display_server_windows.cpp
index 6530846c15..b4949de3f7 100644
--- a/platform/windows/display_server_windows.cpp
+++ b/platform/windows/display_server_windows.cpp
@@ -1312,7 +1312,28 @@ void DisplayServerWindows::window_set_flag(WindowFlags p_flag, bool p_enabled, W
 			_update_window_style(p_window);
 		} break;
 		case WINDOW_FLAG_TRANSPARENT: {
-			// FIXME: Implement.
+			if (p_enabled) {
+				//enable per-pixel alpha
+
+				DWM_BLURBEHIND bb = { 0 };
+				HRGN hRgn = CreateRectRgn(0, 0, -1, -1);
+				bb.dwFlags = DWM_BB_ENABLE | DWM_BB_BLURREGION;
+				bb.hRgnBlur = hRgn;
+				bb.fEnable = TRUE;
+				DwmEnableBlurBehindWindow(wd.hWnd, &bb);
+
+				wd.layered_window = true;
+			} else {
+				//disable per-pixel alpha
+				wd.layered_window = false;
+
+				DWM_BLURBEHIND bb = { 0 };
+				HRGN hRgn = CreateRectRgn(0, 0, -1, -1);
+				bb.dwFlags = DWM_BB_ENABLE | DWM_BB_BLURREGION;
+				bb.hRgnBlur = hRgn;
+				bb.fEnable = FALSE;
+				DwmEnableBlurBehindWindow(wd.hWnd, &bb);
+			}
 		} break;
 		case WINDOW_FLAG_NO_FOCUS: {
 			wd.no_focus = p_enabled;
@@ -1344,7 +1365,7 @@ bool DisplayServerWindows::window_get_flag(WindowFlags p_flag, WindowID p_window
 			return wd.always_on_top;
 		} break;
 		case WINDOW_FLAG_TRANSPARENT: {
-			// FIXME: Implement.
+			return wd.layered_window;
 		} break;
 		case WINDOW_FLAG_NO_FOCUS: {
 			return wd.no_focus;
diff --git a/platform/windows/display_server_windows.h b/platform/windows/display_server_windows.h
index afdf78b380..dbc9821970 100644
--- a/platform/windows/display_server_windows.h
+++ b/platform/windows/display_server_windows.h
@@ -352,7 +352,6 @@ class DisplayServerWindows : public DisplayServer {
 
 	struct WindowData {
 		HWND hWnd;
-		//layered window
 
 		Vector<Vector2> mpath;
 
@@ -392,10 +391,6 @@ class DisplayServerWindows : public DisplayServer {
 		Vector2 last_tilt;
 		bool last_pen_inverted = false;
 
-		HBITMAP hBitmap; //DIB section for layered window
-		uint8_t *dib_data = nullptr;
-		Size2 dib_size;
-		HDC hDC_dib;
 		Size2 min_size;
 		Size2 max_size;
 		int width = 0, height = 0;