[Windows] Improve console handling and execute/create_process.

Always build with the GUI subsystem.
Redirect stdout and stderr output to the parent process console.
Use CreateProcessW for blocking `execute` calls with piped stdout and stderr (prevent console windows for popping up when used with the GUI subsystem build, and have more consistent behavior with `create_process`).
Add `open_console` argument to the `execute` and `create_process` to open a new console window.
Remove `interface/editor/hide_console_window` editor setting.
Remove `Toggle System Console` menu option.
Remove `set_console_visible` and `is_console_visible` functions.

5 files changed, 75 insertions, 158 deletions

diff --git a/platform/windows/detect.py b/platform/windows/detect.py
index e9ecc99ef5..249a0d2e79 100644
--- a/platform/windows/detect.py
+++ b/platform/windows/detect.py
@@ -65,7 +65,7 @@ def get_opts():
         # Vista support dropped after EOL due to GH-10243
         ("target_win_version", "Targeted Windows version, >= 0x0601 (Windows 7)", "0x0601"),
         BoolVariable("debug_symbols", "Add debugging symbols to release/release_debug builds", True),
-        EnumVariable("windows_subsystem", "Windows subsystem", "default", ("default", "console", "gui")),
+        EnumVariable("windows_subsystem", "Windows subsystem", "gui", ("gui", "console")),
         BoolVariable("separate_debug_symbols", "Create a separate file containing debugging symbols", False),
         ("msvc_version", "MSVC version to use. Ignored if VCINSTALLDIR is set in shell env.", None),
         BoolVariable("use_mingw", "Use the Mingw compiler, even if MSVC is installed.", False),
@@ -178,15 +178,6 @@ def configure_msvc(env, manual_msvc_config):
 
     # Build type
 
-    if env["tests"]:
-        env["windows_subsystem"] = "console"
-    elif env["windows_subsystem"] == "default":
-        # Default means we use console for debug, gui for release.
-        if "debug" in env["target"]:
-            env["windows_subsystem"] = "console"
-        else:
-            env["windows_subsystem"] = "gui"
-
     if env["target"] == "release":
         if env["optimize"] == "speed":  # optimize for speed (default)
             env.Append(CCFLAGS=["/O2"])
@@ -326,15 +317,6 @@ def configure_mingw(env):
 
     ## Build type
 
-    if env["tests"]:
-        env["windows_subsystem"] = "console"
-    elif env["windows_subsystem"] == "default":
-        # Default means we use console for debug, gui for release.
-        if "debug" in env["target"]:
-            env["windows_subsystem"] = "console"
-        else:
-            env["windows_subsystem"] = "gui"
-
     if env["target"] == "release":
         env.Append(CCFLAGS=["-msse2"])
 
diff --git a/platform/windows/display_server_windows.cpp b/platform/windows/display_server_windows.cpp
index f83029e662..ac07ff9529 100644
--- a/platform/windows/display_server_windows.cpp
+++ b/platform/windows/display_server_windows.cpp
@@ -68,7 +68,6 @@ bool DisplayServerWindows::has_feature(Feature p_feature) const {
 		case FEATURE_CLIPBOARD:
 		case FEATURE_CURSOR_SHAPE:
 		case FEATURE_CUSTOM_CURSOR_SHAPE:
-		case FEATURE_CONSOLE_WINDOW:
 		case FEATURE_IME:
 		case FEATURE_WINDOW_TRANSPARENCY:
 		case FEATURE_HIDPI:
@@ -1198,23 +1197,6 @@ void DisplayServerWindows::window_set_ime_position(const Point2i &p_pos, WindowI
 	ImmReleaseContext(wd.hWnd, himc);
 }
 
-void DisplayServerWindows::console_set_visible(bool p_enabled) {
-	_THREAD_SAFE_METHOD_
-
-	if (console_visible == p_enabled) {
-		return;
-	}
-	if (!((OS_Windows *)OS::get_singleton())->_is_win11_terminal()) {
-		// GetConsoleWindow is not supported by the Windows Terminal.
-		ShowWindow(GetConsoleWindow(), p_enabled ? SW_SHOW : SW_HIDE);
-		console_visible = p_enabled;
-	}
-}
-
-bool DisplayServerWindows::is_console_visible() const {
-	return console_visible;
-}
-
 void DisplayServerWindows::cursor_set_shape(CursorShape p_shape) {
 	_THREAD_SAFE_METHOD_
 
@@ -3246,7 +3228,6 @@ DisplayServerWindows::DisplayServerWindows(const String &p_rendering_driver, Win
 	shift_mem = false;
 	control_mem = false;
 	meta_mem = false;
-	console_visible = IsWindowVisible(GetConsoleWindow());
 	hInstance = ((OS_Windows *)OS::get_singleton())->get_hinstance();
 
 	pressrc = 0;
diff --git a/platform/windows/display_server_windows.h b/platform/windows/display_server_windows.h
index a59f8ebb44..eebe3c8715 100644
--- a/platform/windows/display_server_windows.h
+++ b/platform/windows/display_server_windows.h
@@ -414,7 +414,6 @@ class DisplayServerWindows : public DisplayServer {
 	bool use_raw_input = false;
 	bool drop_events = false;
 	bool in_dispatch_input_event = false;
-	bool console_visible = false;
 
 	WNDCLASSEXW wc;
 
@@ -477,7 +476,7 @@ public:
 
 	virtual void window_attach_instance_id(ObjectID p_instance, WindowID p_window = MAIN_WINDOW_ID) override;
 	virtual ObjectID window_get_attached_instance_id(WindowID p_window = MAIN_WINDOW_ID) const override;
-	virtual void gl_window_make_current(DisplayServer::WindowID p_window_id);
+	virtual void gl_window_make_current(DisplayServer::WindowID p_window_id) override;
 
 	virtual void window_set_rect_changed_callback(const Callable &p_callable, WindowID p_window = MAIN_WINDOW_ID) override;
 
@@ -529,9 +528,6 @@ public:
 	virtual void window_set_vsync_mode(DisplayServer::VSyncMode p_vsync_mode, WindowID p_window = MAIN_WINDOW_ID) override;
 	virtual DisplayServer::VSyncMode window_get_vsync_mode(WindowID p_vsync_mode) const override;
 
-	virtual void console_set_visible(bool p_enabled) override;
-	virtual bool is_console_visible() const override;
-
 	virtual void cursor_set_shape(CursorShape p_shape) override;
 	virtual CursorShape cursor_get_shape() const override;
 	virtual void cursor_set_custom_image(const RES &p_cursor, CursorShape p_shape = CURSOR_ARROW, const Vector2 &p_hotspot = Vector2()) override;
diff --git a/platform/windows/os_windows.cpp b/platform/windows/os_windows.cpp
index bb6a077a5d..f714ede62c 100644
--- a/platform/windows/os_windows.cpp
+++ b/platform/windows/os_windows.cpp
@@ -52,8 +52,6 @@
 #include <regstr.h>
 #include <shlobj.h>
 
-static const WORD MAX_CONSOLE_LINES = 1500;
-
 extern "C" {
 __declspec(dllexport) DWORD NvOptimusEnablement = 1;
 __declspec(dllexport) int AmdPowerXpressRequestHighPerformance = 1;
@@ -86,65 +84,17 @@ static String format_error_message(DWORD id) {
 }
 
 void RedirectIOToConsole() {
-	int hConHandle;
-
-	intptr_t lStdHandle;
-
-	CONSOLE_SCREEN_BUFFER_INFO coninfo;
-
-	FILE *fp;
-
-	// allocate a console for this app
-
-	AllocConsole();
-
-	// set the screen buffer to be big enough to let us scroll text
-
-	GetConsoleScreenBufferInfo(GetStdHandle(STD_OUTPUT_HANDLE), &coninfo);
-
-	coninfo.dwSize.Y = MAX_CONSOLE_LINES;
-
-	SetConsoleScreenBufferSize(GetStdHandle(STD_OUTPUT_HANDLE), coninfo.dwSize);
-
-	// redirect unbuffered STDOUT to the console
-
-	lStdHandle = (intptr_t)GetStdHandle(STD_OUTPUT_HANDLE);
-
-	hConHandle = _open_osfhandle(lStdHandle, _O_TEXT);
-
-	fp = _fdopen(hConHandle, "w");
-
-	*stdout = *fp;
-
-	setvbuf(stdout, nullptr, _IONBF, 0);
-
-	// redirect unbuffered STDIN to the console
-
-	lStdHandle = (intptr_t)GetStdHandle(STD_INPUT_HANDLE);
+	if (AttachConsole(ATTACH_PARENT_PROCESS)) {
+		FILE *fpstdin = stdin;
+		FILE *fpstdout = stdout;
+		FILE *fpstderr = stderr;
 
-	hConHandle = _open_osfhandle(lStdHandle, _O_TEXT);
+		freopen_s(&fpstdin, "CONIN$", "r", stdin);
+		freopen_s(&fpstdout, "CONOUT$", "w", stdout);
+		freopen_s(&fpstderr, "CONOUT$", "w", stderr);
 
-	fp = _fdopen(hConHandle, "r");
-
-	*stdin = *fp;
-
-	setvbuf(stdin, nullptr, _IONBF, 0);
-
-	// redirect unbuffered STDERR to the console
-
-	lStdHandle = (intptr_t)GetStdHandle(STD_ERROR_HANDLE);
-
-	hConHandle = _open_osfhandle(lStdHandle, _O_TEXT);
-
-	fp = _fdopen(hConHandle, "w");
-
-	*stderr = *fp;
-
-	setvbuf(stderr, nullptr, _IONBF, 0);
-
-	// make cout, wcout, cin, wcin, wcerr, cerr, wclog and clog
-
-	// point to console as well
+		printf("\n"); // Make sure our output is starting from the new line.
+	}
 }
 
 BOOL WINAPI HandlerRoutine(_In_ DWORD dwCtrlType) {
@@ -172,7 +122,9 @@ void OS_Windows::initialize_debugging() {
 void OS_Windows::initialize() {
 	crash_handler.initialize();
 
-	//RedirectIOToConsole();
+#ifndef WINDOWS_SUBSYSTEM_CONSOLE
+	RedirectIOToConsole();
+#endif
 
 	FileAccess::make_default<FileAccessWindows>(FileAccess::ACCESS_RESOURCES);
 	FileAccess::make_default<FileAccessWindows>(FileAccess::ACCESS_USERDATA);
@@ -406,78 +358,87 @@ String OS_Windows::_quote_command_line_argument(const String &p_text) const {
 	return p_text;
 }
 
-Error OS_Windows::execute(const String &p_path, const List<String> &p_arguments, String *r_pipe, int *r_exitcode, bool read_stderr, Mutex *p_pipe_mutex) {
+Error OS_Windows::execute(const String &p_path, const List<String> &p_arguments, String *r_pipe, int *r_exitcode, bool read_stderr, Mutex *p_pipe_mutex, bool p_open_console) {
 	String path = p_path.replace("/", "\\");
 	String command = _quote_command_line_argument(path);
 	for (const String &E : p_arguments) {
 		command += " " + _quote_command_line_argument(E);
 	}
 
+	ProcessInfo pi;
+	ZeroMemory(&pi.si, sizeof(pi.si));
+	pi.si.cb = sizeof(pi.si);
+	ZeroMemory(&pi.pi, sizeof(pi.pi));
+	LPSTARTUPINFOW si_w = (LPSTARTUPINFOW)&pi.si;
+
+	bool inherit_handles = false;
+	HANDLE pipe[2] = { nullptr, nullptr };
 	if (r_pipe) {
+		// Create pipe for StdOut and StdErr.
+		SECURITY_ATTRIBUTES sa;
+		sa.nLength = sizeof(SECURITY_ATTRIBUTES);
+		sa.bInheritHandle = true;
+		sa.lpSecurityDescriptor = nullptr;
+
+		ERR_FAIL_COND_V(!CreatePipe(&pipe[0], &pipe[1], &sa, 0), ERR_CANT_FORK);
+		ERR_FAIL_COND_V(!SetHandleInformation(pipe[0], HANDLE_FLAG_INHERIT, 0), ERR_CANT_FORK); // Read handle is for host process only and should not be inherited.
+
+		pi.si.dwFlags |= STARTF_USESTDHANDLES;
+		pi.si.hStdOutput = pipe[1];
 		if (read_stderr) {
-			command += " 2>&1"; // Include stderr
+			pi.si.hStdError = pipe[1];
 		}
-		// Add extra quotes around the full command, to prevent it from stripping quotes in the command,
-		// because _wpopen calls command as "cmd.exe /c command", instead of executing it directly
-		command = _quote_command_line_argument(command);
-
-		FILE *f = _wpopen((LPCWSTR)(command.utf16().get_data()), L"r");
-		ERR_FAIL_COND_V_MSG(!f, ERR_CANT_OPEN, "Cannot create pipe from command: " + command);
-		char buf[65535];
-		while (fgets(buf, 65535, f)) {
+		inherit_handles = true;
+	}
+	DWORD creaton_flags = NORMAL_PRIORITY_CLASS;
+	if (p_open_console) {
+		creaton_flags |= CREATE_NEW_CONSOLE;
+	} else {
+		creaton_flags |= CREATE_NO_WINDOW;
+	}
+
+	int ret = CreateProcessW(nullptr, (LPWSTR)(command.utf16().ptrw()), nullptr, nullptr, inherit_handles, creaton_flags, nullptr, nullptr, si_w, &pi.pi);
+	if (!ret && r_pipe) {
+		CloseHandle(pipe[0]); // Cleanup pipe handles.
+		CloseHandle(pipe[1]);
+	}
+	ERR_FAIL_COND_V_MSG(ret == 0, ERR_CANT_FORK, "Could not create child process: " + command);
+
+	if (r_pipe) {
+		CloseHandle(pipe[1]); // Close pipe write handle (only child process is writing).
+		char buf[4096];
+		DWORD read = 0;
+		for (;;) { // Read StdOut and StdErr from pipe.
+			bool success = ReadFile(pipe[0], buf, 4096, &read, NULL);
+			if (!success || read == 0) {
+				break;
+			}
 			if (p_pipe_mutex) {
 				p_pipe_mutex->lock();
 			}
-			(*r_pipe) += String::utf8(buf);
+			(*r_pipe) += String::utf8(buf, read);
 			if (p_pipe_mutex) {
 				p_pipe_mutex->unlock();
 			}
-		}
-		int rv = _pclose(f);
-
-		if (r_exitcode) {
-			*r_exitcode = rv;
-		}
-		return OK;
+		};
+		CloseHandle(pipe[0]); // Close pipe read handle.
+	} else {
+		WaitForSingleObject(pi.pi.hProcess, INFINITE);
 	}
 
-	ProcessInfo pi;
-	ZeroMemory(&pi.si, sizeof(pi.si));
-	pi.si.cb = sizeof(pi.si);
-	ZeroMemory(&pi.pi, sizeof(pi.pi));
-	LPSTARTUPINFOW si_w = (LPSTARTUPINFOW)&pi.si;
-
-	DWORD dwCreationFlags = NORMAL_PRIORITY_CLASS;
-#ifndef DEBUG_ENABLED
-	dwCreationFlags |= CREATE_NO_WINDOW;
-#endif
-
-	int ret = CreateProcessW(nullptr, (LPWSTR)(command.utf16().ptrw()), nullptr, nullptr, false, dwCreationFlags, nullptr, nullptr, si_w, &pi.pi);
-	ERR_FAIL_COND_V_MSG(ret == 0, ERR_CANT_FORK, "Could not create child process: " + command);
-
-	WaitForSingleObject(pi.pi.hProcess, INFINITE);
 	if (r_exitcode) {
 		DWORD ret2;
 		GetExitCodeProcess(pi.pi.hProcess, &ret2);
 		*r_exitcode = ret2;
 	}
+
 	CloseHandle(pi.pi.hProcess);
 	CloseHandle(pi.pi.hThread);
 
 	return OK;
 };
 
-bool OS_Windows::_is_win11_terminal() const {
-	HANDLE hStdOut = GetStdHandle(STD_OUTPUT_HANDLE);
-	DWORD dwMode = 0;
-	if (GetConsoleMode(hStdOut, &dwMode)) {
-		return ((dwMode & ENABLE_VIRTUAL_TERMINAL_PROCESSING) == ENABLE_VIRTUAL_TERMINAL_PROCESSING);
-	} else {
-		return false;
-	}
-}
-
-Error OS_Windows::create_process(const String &p_path, const List<String> &p_arguments, ProcessID *r_child_id) {
+Error OS_Windows::create_process(const String &p_path, const List<String> &p_arguments, ProcessID *r_child_id, bool p_open_console) {
 	String path = p_path.replace("/", "\\");
 	String command = _quote_command_line_argument(path);
 	for (const String &E : p_arguments) {
@@ -490,16 +451,14 @@ Error OS_Windows::create_process(const String &p_path, const List<String> &p_arg
 	ZeroMemory(&pi.pi, sizeof(pi.pi));
 	LPSTARTUPINFOW si_w = (LPSTARTUPINFOW)&pi.si;
 
-	DWORD dwCreationFlags = NORMAL_PRIORITY_CLASS;
-#ifndef DEBUG_ENABLED
-	dwCreationFlags |= CREATE_NO_WINDOW;
-#endif
-	if (p_path == get_executable_path() && GetConsoleWindow() != nullptr && _is_win11_terminal()) {
-		// Open a new terminal as a workaround for Windows Terminal bug.
-		dwCreationFlags |= CREATE_NEW_CONSOLE;
+	DWORD creaton_flags = NORMAL_PRIORITY_CLASS;
+	if (p_open_console) {
+		creaton_flags |= CREATE_NEW_CONSOLE;
+	} else {
+		creaton_flags |= CREATE_NO_WINDOW;
 	}
 
-	int ret = CreateProcessW(nullptr, (LPWSTR)(command.utf16().ptrw()), nullptr, nullptr, false, dwCreationFlags, nullptr, nullptr, si_w, &pi.pi);
+	int ret = CreateProcessW(nullptr, (LPWSTR)(command.utf16().ptrw()), nullptr, nullptr, false, creaton_flags, nullptr, nullptr, si_w, &pi.pi);
 	ERR_FAIL_COND_V_MSG(ret == 0, ERR_CANT_FORK, "Could not create child process: " + command);
 
 	ProcessID pid = pi.pi.dwProcessId;
diff --git a/platform/windows/os_windows.h b/platform/windows/os_windows.h
index 086f63eb3e..d9504fcd0c 100644
--- a/platform/windows/os_windows.h
+++ b/platform/windows/os_windows.h
@@ -128,8 +128,8 @@ public:
 	virtual void delay_usec(uint32_t p_usec) const override;
 	virtual uint64_t get_ticks_usec() const override;
 
-	virtual Error execute(const String &p_path, const List<String> &p_arguments, String *r_pipe = nullptr, int *r_exitcode = nullptr, bool read_stderr = false, Mutex *p_pipe_mutex = nullptr) override;
-	virtual Error create_process(const String &p_path, const List<String> &p_arguments, ProcessID *r_child_id = nullptr) override;
+	virtual Error execute(const String &p_path, const List<String> &p_arguments, String *r_pipe = nullptr, int *r_exitcode = nullptr, bool read_stderr = false, Mutex *p_pipe_mutex = nullptr, bool p_open_console = false) override;
+	virtual Error create_process(const String &p_path, const List<String> &p_arguments, ProcessID *r_child_id = nullptr, bool p_open_console = false) override;
 	virtual Error kill(const ProcessID &p_pid) override;
 	virtual int get_process_id() const override;
 
@@ -157,7 +157,6 @@ public:
 
 	void run();
 
-	bool _is_win11_terminal() const;
 	virtual bool _check_internal_feature_support(const String &p_feature) override;
 
 	virtual void disable_crash_handler() override;