Merge pull request #41332 from bruvzg/win_subsys_option

Revert #41164, add subsystem build option.

3 files changed, 19 insertions, 26 deletions

diff --git a/platform/windows/detect.py b/platform/windows/detect.py
index a9f25fa078..4e1da22bb0 100644
--- a/platform/windows/detect.py
+++ b/platform/windows/detect.py
@@ -65,6 +65,7 @@ def get_opts():
         # Vista support dropped after EOL due to GH-10243
         ("target_win_version", "Targeted Windows version, >= 0x0601 (Windows 7)", "0x0601"),
         EnumVariable("debug_symbols", "Add debugging symbols to release builds", "yes", ("yes", "no", "full")),
+        EnumVariable("windows_subsystem", "Windows subsystem", "gui", ("console", "gui")),
         BoolVariable("separate_debug_symbols", "Create a separate file containing debugging symbols", False),
         ("msvc_version", "MSVC version to use. Ignored if VCINSTALLDIR is set in shell env.", None),
         BoolVariable("use_mingw", "Use the Mingw compiler, even if MSVC is installed. Only used on Windows.", False),
@@ -177,6 +178,8 @@ def configure_msvc(env, manual_msvc_config):
     """Configure env to work with MSVC"""
 
     # Build type
+    if env["tests"]:
+        env["windows_subsystem"] = "console"
 
     if env["target"] == "release":
         if env["optimize"] == "speed":  # optimize for speed (default)
@@ -199,12 +202,15 @@ def configure_msvc(env, manual_msvc_config):
         env.AppendUnique(CPPDEFINES=["DEBUG_ENABLED"])
         env.Append(LINKFLAGS=["/DEBUG"])
 
-    env.Append(LINKFLAGS=["/SUBSYSTEM:WINDOWS"])
-
     if env["debug_symbols"] == "full" or env["debug_symbols"] == "yes":
         env.AppendUnique(CCFLAGS=["/Z7"])
         env.AppendUnique(LINKFLAGS=["/DEBUG"])
 
+    if env["windows_subsystem"] == "gui":
+        env.Append(LINKFLAGS=["/SUBSYSTEM:WINDOWS"])
+    else:
+        env.Append(LINKFLAGS=["/SUBSYSTEM:CONSOLE"])
+
     ## Compile/link flags
 
     env.AppendUnique(CCFLAGS=["/MT", "/Gd", "/GR", "/nologo"])
@@ -302,6 +308,9 @@ def configure_mingw(env):
 
     ## Build type
 
+    if env["tests"]:
+        env["windows_subsystem"] = "console"
+
     if env["target"] == "release":
         env.Append(CCFLAGS=["-msse2"])
 
@@ -334,7 +343,10 @@ def configure_mingw(env):
         env.Append(CCFLAGS=["-g3"])
         env.Append(CPPDEFINES=["DEBUG_ENABLED"])
 
-    env.Append(LINKFLAGS=["-Wl,--subsystem,windows"])
+    if env["windows_subsystem"] == "gui":
+        env.Append(LINKFLAGS=["-Wl,--subsystem,windows"])
+    else:
+        env.Append(LINKFLAGS=["-Wl,--subsystem,console"])
 
     ## Compiler configuration
 
diff --git a/platform/windows/display_server_windows.cpp b/platform/windows/display_server_windows.cpp
index cd7f28833b..11f58d1be0 100644
--- a/platform/windows/display_server_windows.cpp
+++ b/platform/windows/display_server_windows.cpp
@@ -1137,17 +1137,10 @@ void DisplayServerWindows::window_set_ime_position(const Point2i &p_pos, WindowI
 void DisplayServerWindows::console_set_visible(bool p_enabled) {
 	_THREAD_SAFE_METHOD_
 
-	if (console_visible == p_enabled) {
+	if (console_visible == p_enabled)
 		return;
-	}
-	if (p_enabled && GetConsoleWindow() == nullptr) { // Open new console if not attached.
-		own_console = true;
-		AllocConsole();
-	}
-	if (own_console) { // Note: Do not hide parent console.
-		ShowWindow(GetConsoleWindow(), p_enabled ? SW_SHOW : SW_HIDE);
-		console_visible = p_enabled;
-	}
+	ShowWindow(GetConsoleWindow(), p_enabled ? SW_SHOW : SW_HIDE);
+	console_visible = p_enabled;
 }
 
 bool DisplayServerWindows::is_console_visible() const {
@@ -3028,18 +3021,7 @@ DisplayServerWindows::DisplayServerWindows(const String &p_rendering_driver, Win
 	shift_mem = false;
 	control_mem = false;
 	meta_mem = false;
-
-	if (AttachConsole(ATTACH_PARENT_PROCESS)) {
-		FILE *_file = nullptr;
-		freopen_s(&_file, "CONOUT$", "w", stdout);
-		freopen_s(&_file, "CONOUT$", "w", stderr);
-		freopen_s(&_file, "CONIN$", "r", stdin);
-
-		printf("\n");
-		console_visible = true;
-	} else {
-		console_visible = false;
-	}
+	console_visible = IsWindowVisible(GetConsoleWindow());
 	hInstance = ((OS_Windows *)OS::get_singleton())->get_hinstance();
 
 	pressrc = 0;
diff --git a/platform/windows/display_server_windows.h b/platform/windows/display_server_windows.h
index 52f5b0f4a9..7bd93a7086 100644
--- a/platform/windows/display_server_windows.h
+++ b/platform/windows/display_server_windows.h
@@ -405,7 +405,6 @@ private:
 	bool drop_events = false;
 	bool in_dispatch_input_event = false;
 	bool console_visible = false;
-	bool own_console = false;
 
 	WNDCLASSEXW wc;