C#: Replace libnethost dependency to find hostfxr

We want to replace libnethost as it gives us issues with some compilers.
Our implementation tries to mimic libnethost's hostfxr_resolver search
logic. We try to use the same function names for easier comparing in
case we need to update this in the future.

1 files changed, 30 insertions, 71 deletions

diff --git a/modules/mono/mono_gd/gd_mono.cpp b/modules/mono/mono_gd/gd_mono.cpp
index e698e92d7a..3b87d9248a 100644
--- a/modules/mono/mono_gd/gd_mono.cpp
+++ b/modules/mono/mono_gd/gd_mono.cpp
@@ -43,12 +43,13 @@
 #include "../utils/path_utils.h"
 #include "gd_mono_cache.h"
 
+#include "../thirdparty/coreclr_delegates.h"
+#include "../thirdparty/hostfxr.h"
+
 #ifdef TOOLS_ENABLED
-#include <nethost.h>
+#include "../editor/hostfxr_resolver.h"
 #endif
 
-#include <coreclr_delegates.h>
-#include <hostfxr.h>
 #ifdef UNIX_ENABLED
 #include <dlfcn.h>
 #endif
@@ -88,85 +89,43 @@ HostFxrCharString str_to_hostfxr(const String &p_str) {
 #endif
 }
 
-#ifdef TOOLS_ENABLED
-String str_from_hostfxr(const char_t *p_buffer) {
-#ifdef _WIN32
-	return String::utf16((const char16_t *)p_buffer);
-#else
-	return String::utf8((const char *)p_buffer);
-#endif
-}
-#endif
-
 const char_t *get_data(const HostFxrCharString &p_char_str) {
 	return (const char_t *)p_char_str.get_data();
 }
 
-#ifdef TOOLS_ENABLED
-String find_hostfxr(size_t p_known_buffer_size, get_hostfxr_parameters *p_get_hostfxr_params) {
-	// Pre-allocate a large buffer for the path to hostfxr
-	Vector<char_t> buffer;
-	buffer.resize(p_known_buffer_size);
-
-	int rc = get_hostfxr_path(buffer.ptrw(), &p_known_buffer_size, p_get_hostfxr_params);
-
-	ERR_FAIL_COND_V_MSG(rc != 0, String(), "get_hostfxr_path failed with code: " + itos(rc));
-
-	return str_from_hostfxr(buffer.ptr());
-}
-#endif
-
 String find_hostfxr() {
 #ifdef TOOLS_ENABLED
-	const int CoreHostLibMissingFailure = 0x80008083;
-	const int HostApiBufferTooSmall = 0x80008098;
-
-	size_t buffer_size = 0;
-	int rc = get_hostfxr_path(nullptr, &buffer_size, nullptr);
-
-	if (rc == HostApiBufferTooSmall) {
-		return find_hostfxr(buffer_size, nullptr);
+	String dotnet_root;
+	String fxr_path;
+	if (godotsharp::hostfxr_resolver::try_get_path(dotnet_root, fxr_path)) {
+		return fxr_path;
 	}
 
-	if (rc == CoreHostLibMissingFailure) {
-		// Apparently `get_hostfxr_path` doesn't look for dotnet in `PATH`? (I suppose it needs the
-		// `DOTNET_ROOT` environment variable). If it fails, we try to find the dotnet executable
-		// in `PATH` ourselves and pass its location as `dotnet_root` to `get_hostfxr_path`.
-		String dotnet_exe = path::find_executable("dotnet");
-
-		if (!dotnet_exe.is_empty()) {
-			// The file found in PATH may be a symlink
-			dotnet_exe = path::abspath(path::realpath(dotnet_exe));
-
-			// TODO:
-			// Sometimes, the symlink may not point to the dotnet executable in the dotnet root.
-			// That's the case with snaps. The snap install should have been found with the
-			// previous `get_hostfxr_path`, but it would still be better to do this properly
-			// and use something like `dotnet --list-sdks/runtimes` to find the actual location.
-			// This way we could also check if the proper sdk or runtime is installed. This would
-			// allow us to fail gracefully and show some helpful information in the editor.
-
-			HostFxrCharString dotnet_root = str_to_hostfxr(dotnet_exe.get_base_dir());
-
-			get_hostfxr_parameters get_hostfxr_parameters = {
-				sizeof(get_hostfxr_parameters),
-				nullptr,
-				get_data(dotnet_root)
-			};
-
-			buffer_size = 0;
-			rc = get_hostfxr_path(nullptr, &buffer_size, &get_hostfxr_parameters);
-			if (rc == HostApiBufferTooSmall) {
-				return find_hostfxr(buffer_size, &get_hostfxr_parameters);
-			}
+	// hostfxr_resolver doesn't look for dotnet in `PATH`. If it fails, we try to find the dotnet
+	// executable in `PATH` here and pass its location as `dotnet_root` to `get_hostfxr_path`.
+	String dotnet_exe = path::find_executable("dotnet");
+
+	if (!dotnet_exe.is_empty()) {
+		// The file found in PATH may be a symlink
+		dotnet_exe = path::abspath(path::realpath(dotnet_exe));
+
+		// TODO:
+		// Sometimes, the symlink may not point to the dotnet executable in the dotnet root.
+		// That's the case with snaps. The snap install should have been found with the
+		// previous `get_hostfxr_path`, but it would still be better to do this properly
+		// and use something like `dotnet --list-sdks/runtimes` to find the actual location.
+		// This way we could also check if the proper sdk or runtime is installed. This would
+		// allow us to fail gracefully and show some helpful information in the editor.
+
+		dotnet_root = dotnet_exe.get_base_dir();
+		if (godotsharp::hostfxr_resolver::try_get_path_from_dotnet_root(dotnet_root, fxr_path)) {
+			return fxr_path;
 		}
 	}
 
-	if (rc == CoreHostLibMissingFailure) {
-		ERR_PRINT(String() + ".NET: One of the dependent libraries is missing. " +
-				"Typically when the `hostfxr`, `hostpolicy` or `coreclr` dynamic " +
-				"libraries are not present in the expected locations.");
-	}
+	ERR_PRINT(String() + ".NET: One of the dependent libraries is missing. " +
+			"Typically when the `hostfxr`, `hostpolicy` or `coreclr` dynamic " +
+			"libraries are not present in the expected locations.");
 
 	return String();
 #else