Merge pull request #14622 from bruvzg/non-rectangular-windows

Experimental support for windows with per-pixel transparency.

5 files changed, 107 insertions, 20 deletions

diff --git a/platform/x11/context_gl_x11.cpp b/platform/x11/context_gl_x11.cpp
index 1a7cbc0d6d..cd76667c64 100644
--- a/platform/x11/context_gl_x11.cpp
+++ b/platform/x11/context_gl_x11.cpp
@@ -116,32 +116,76 @@ Error ContextGL_X11::initialize() {
 		None
 	};
 
+	static int visual_attribs_layered[] = {
+		GLX_RENDER_TYPE, GLX_RGBA_BIT,
+		GLX_DRAWABLE_TYPE, GLX_WINDOW_BIT,
+		GLX_DOUBLEBUFFER, true,
+		GLX_RED_SIZE, 8,
+		GLX_GREEN_SIZE, 8,
+		GLX_BLUE_SIZE, 8,
+		GLX_ALPHA_SIZE, 8,
+		GLX_DEPTH_SIZE, 24,
+		None
+	};
+
 	int fbcount;
-	GLXFBConfig *fbc = glXChooseFBConfig(x11_display, DefaultScreen(x11_display), visual_attribs, &fbcount);
-	ERR_FAIL_COND_V(!fbc, ERR_UNCONFIGURED);
+	GLXFBConfig fbconfig;
+	XVisualInfo *vi = NULL;
+
+	if (OS::get_singleton()->is_layered_allowed()) {
+		GLXFBConfig *fbc = glXChooseFBConfig(x11_display, DefaultScreen(x11_display), visual_attribs_layered, &fbcount);
+		ERR_FAIL_COND_V(!fbc, ERR_UNCONFIGURED);
+
+		for (int i = 0; i < fbcount; i++) {
+			vi = (XVisualInfo *)glXGetVisualFromFBConfig(x11_display, fbc[i]);
+			if (!vi)
+				continue;
+
+			XRenderPictFormat *pict_format = XRenderFindVisualFormat(x11_display, vi->visual);
+			if (!pict_format) {
+				XFree(vi);
+				vi = NULL;
+				continue;
+			}
+
+			fbconfig = fbc[i];
+			if (pict_format->direct.alphaMask > 0) {
+				break;
+			}
+		}
+		ERR_FAIL_COND_V(!fbconfig, ERR_UNCONFIGURED);
+
+		XSetWindowAttributes swa;
+
+		swa.colormap = XCreateColormap(x11_display, RootWindow(x11_display, vi->screen), vi->visual, AllocNone);
+		swa.border_pixel = 0;
+		swa.background_pixmap = None;
+		swa.background_pixel = 0;
+		swa.border_pixmap = None;
+		swa.event_mask = StructureNotifyMask;
+
+		x11_window = XCreateWindow(x11_display, RootWindow(x11_display, vi->screen), 0, 0, OS::get_singleton()->get_video_mode().width, OS::get_singleton()->get_video_mode().height, 0, vi->depth, InputOutput, vi->visual, CWBorderPixel | CWColormap | CWEventMask | CWBackPixel, &swa);
 
-	XVisualInfo *vi = glXGetVisualFromFBConfig(x11_display, fbc[0]);
+	} else {
+		GLXFBConfig *fbc = glXChooseFBConfig(x11_display, DefaultScreen(x11_display), visual_attribs, &fbcount);
+		ERR_FAIL_COND_V(!fbc, ERR_UNCONFIGURED);
 
-	XSetWindowAttributes swa;
+		vi = glXGetVisualFromFBConfig(x11_display, fbc[0]);
 
-	swa.colormap = XCreateColormap(x11_display, RootWindow(x11_display, vi->screen), vi->visual, AllocNone);
-	swa.border_pixel = 0;
-	swa.event_mask = StructureNotifyMask;
+		fbconfig = fbc[0];
 
-	/*
-	char* windowid = getenv("GODOT_WINDOWID");
-	if (windowid) {
+		XSetWindowAttributes swa;
+
+		swa.colormap = XCreateColormap(x11_display, RootWindow(x11_display, vi->screen), vi->visual, AllocNone);
+		swa.border_pixel = 0;
+		swa.event_mask = StructureNotifyMask;
+
+		x11_window = XCreateWindow(x11_display, RootWindow(x11_display, vi->screen), 0, 0, OS::get_singleton()->get_video_mode().width, OS::get_singleton()->get_video_mode().height, 0, vi->depth, InputOutput, vi->visual, CWBorderPixel | CWColormap | CWEventMask, &swa);
+	}
 
-		//freopen("/home/punto/stdout", "w", stdout);
-		//reopen("/home/punto/stderr", "w", stderr);
-		x11_window = atol(windowid);
-	} else {
-	*/
-	x11_window = XCreateWindow(x11_display, RootWindow(x11_display, vi->screen), 0, 0, OS::get_singleton()->get_video_mode().width, OS::get_singleton()->get_video_mode().height, 0, vi->depth, InputOutput, vi->visual, CWBorderPixel | CWColormap | CWEventMask, &swa);
 	ERR_FAIL_COND_V(!x11_window, ERR_UNCONFIGURED);
 	set_class_hint(x11_display, x11_window);
 	XMapWindow(x11_display, x11_window);
-	//};
 
 	int (*oldHandler)(Display *, XErrorEvent *) =
 			XSetErrorHandler(&ctxErrorHandler);
@@ -160,7 +204,7 @@ Error ContextGL_X11::initialize() {
 				None
 			};
 
-			p->glx_context = glXCreateContextAttribsARB(x11_display, fbc[0], NULL, true, context_attribs);
+			p->glx_context = glXCreateContextAttribsARB(x11_display, fbconfig, NULL, true, context_attribs);
 			ERR_EXPLAIN("Could not obtain an OpenGL 3.0 context!");
 			ERR_FAIL_COND_V(!p->glx_context, ERR_UNCONFIGURED);
 		} break;
@@ -175,7 +219,7 @@ Error ContextGL_X11::initialize() {
 				None
 			};
 
-			p->glx_context = glXCreateContextAttribsARB(x11_display, fbc[0], NULL, true, context_attribs);
+			p->glx_context = glXCreateContextAttribsARB(x11_display, fbconfig, NULL, true, context_attribs);
 			ERR_EXPLAIN("Could not obtain an OpenGL 3.3 context!");
 			ERR_FAIL_COND_V(ctxErrorOccurred || !p->glx_context, ERR_UNCONFIGURED);
 		} break;
@@ -195,7 +239,6 @@ Error ContextGL_X11::initialize() {
 	//glXMakeCurrent(x11_display, None, NULL);
 
 	XFree(vi);
-	XFree(fbc);
 
 	return OK;
 }
diff --git a/platform/x11/context_gl_x11.h b/platform/x11/context_gl_x11.h
index b54cc84fac..b8f3eb95d4 100644
--- a/platform/x11/context_gl_x11.h
+++ b/platform/x11/context_gl_x11.h
@@ -41,6 +41,7 @@
 #include "drivers/gl_context/context_gl.h"
 #include "os/os.h"
 #include <X11/Xlib.h>
+#include <X11/extensions/Xrender.h>
 
 struct ContextGL_X11_Private;
 
diff --git a/platform/x11/detect.py b/platform/x11/detect.py
index 5820a926e9..ad2620c9f5 100644
--- a/platform/x11/detect.py
+++ b/platform/x11/detect.py
@@ -42,6 +42,11 @@ def can_build():
         print("xrandr not found.. x11 disabled.")
         return False
 
+    x11_error = os.system("pkg-config xrender --modversion > /dev/null ")
+    if (x11_error):
+        print("xrender not found.. x11 disabled.")
+        return False
+
     return True
 
 def get_opts():
@@ -141,6 +146,7 @@ def configure(env):
     env.ParseConfig('pkg-config xcursor --cflags --libs')
     env.ParseConfig('pkg-config xinerama --cflags --libs')
     env.ParseConfig('pkg-config xrandr --cflags --libs')
+    env.ParseConfig('pkg-config xrender --cflags --libs')
 
     if (env['touch']):
         x11_error = os.system("pkg-config xi --modversion > /dev/null ")
diff --git a/platform/x11/os_x11.cpp b/platform/x11/os_x11.cpp
index 336068cb1e..d1b87dac6f 100644
--- a/platform/x11/os_x11.cpp
+++ b/platform/x11/os_x11.cpp
@@ -517,6 +517,10 @@ Error OS_X11::initialize(const VideoMode &p_desired, int p_video_driver, int p_a
 
 	power_manager = memnew(PowerX11);
 
+	if (p_desired.layered_splash) {
+		set_window_per_pixel_transparency_enabled(true);
+	}
+
 	XEvent xevent;
 	while (XPending(x11_display) > 0) {
 		XNextEvent(x11_display, &xevent);
@@ -707,6 +711,25 @@ Point2 OS_X11::get_mouse_position() const {
 	return last_mouse_pos;
 }
 
+bool OS_X11::get_window_per_pixel_transparency_enabled() const {
+
+	if (!is_layered_allowed()) return false;
+	return layered_window;
+}
+
+void OS_X11::set_window_per_pixel_transparency_enabled(bool p_enabled) {
+
+	if (!is_layered_allowed()) return;
+	if (layered_window != p_enabled) {
+		if (p_enabled) {
+			set_borderless_window(true);
+			layered_window = true;
+		} else {
+			layered_window = false;
+		}
+	}
+}
+
 void OS_X11::set_window_title(const String &p_title) {
 	XStoreName(x11_display, x11_window, p_title.utf8().get_data());
 
@@ -1006,9 +1029,13 @@ void OS_X11::set_window_size(const Size2 p_size) {
 }
 
 void OS_X11::set_window_fullscreen(bool p_enabled) {
+
 	if (current_videomode.fullscreen == p_enabled)
 		return;
 
+	if (layered_window)
+		set_window_per_pixel_transparency_enabled(false);
+
 	if (p_enabled && current_videomode.always_on_top) {
 		// Fullscreen + Always-on-top requires a maximized window on some window managers (Metacity)
 		set_window_maximized(true);
@@ -1254,6 +1281,9 @@ void OS_X11::set_borderless_window(bool p_borderless) {
 	if (current_videomode.borderless_window == p_borderless)
 		return;
 
+	if (!p_borderless && layered_window)
+		set_window_per_pixel_transparency_enabled(false);
+
 	current_videomode.borderless_window = p_borderless;
 
 	Hints hints;
@@ -2715,6 +2745,7 @@ OS_X11::OS_X11() {
 	AudioDriverManager::add_driver(&driver_alsa);
 #endif
 
+	layered_window = false;
 	minimized = false;
 	xim_style = 0L;
 	mouse_mode = MOUSE_MODE_VISIBLE;
diff --git a/platform/x11/os_x11.h b/platform/x11/os_x11.h
index 0a39da77de..09ed9588c4 100644
--- a/platform/x11/os_x11.h
+++ b/platform/x11/os_x11.h
@@ -172,6 +172,8 @@ class OS_X11 : public OS_Unix {
 
 	PowerX11 *power_manager;
 
+	bool layered_window;
+
 	CrashHandler crash_handler;
 
 	int audio_driver_index;
@@ -263,6 +265,10 @@ public:
 
 	virtual void set_borderless_window(bool p_borderless);
 	virtual bool get_borderless_window();
+
+	virtual bool get_window_per_pixel_transparency_enabled() const;
+	virtual void set_window_per_pixel_transparency_enabled(bool p_enabled);
+
 	virtual void set_ime_position(const Point2 &p_pos);
 
 	virtual String get_unique_id() const;