From 64b5ee701070e26a90702d345a1ee3188024d7ac Mon Sep 17 00:00:00 2001 From: Ignacio Etcheverry Date: Fri, 23 Oct 2020 09:25:16 +0200 Subject: C#: Make editor create NuGet fallback folder for Godot packages Main benefits: - Projects can be built offline. Previously you needed internet access the first time building to download the packages. - Changes to packages like Godot.NET.Sdk can be easily tested before publishing. This was already possible but required too many manual steps. - First time builds are a bit faster, as the Sdk package doesn't need to be downloaded. In practice, the package is very small so it makes little difference. Bumped Godot.NET.Sdk to 4.0.0-dev3 in order to enable the recent changes regarding '.mono/' -> '.godot/mono/'. --- .../Godot.NET.Sdk/Godot.NET.Sdk.csproj | 19 +- .../Godot.NET.Sdk/Godot.NET.Sdk_PackageVersion.txt | 1 + .../GodotTools.ProjectEditor.csproj | 1 + .../GodotTools.ProjectEditor/ProjectGenerator.cs | 5 +- .../GenerateGodotNupkgsVersions.targets | 36 +++ .../GodotTools.Shared/GodotTools.Shared.csproj | 6 + modules/mono/editor/GodotTools/GodotTools.sln | 6 + .../GodotTools/GodotTools/Build/BuildManager.cs | 11 +- .../GodotTools/GodotTools/Build/MSBuildPanel.cs | 20 ++ .../GodotTools/GodotTools/Build/NuGetUtils.cs | 296 +++++++++++++++++++++ .../GodotTools/GodotTools/GodotSharpEditor.cs | 27 ++ 11 files changed, 422 insertions(+), 6 deletions(-) create mode 100644 modules/mono/editor/Godot.NET.Sdk/Godot.NET.Sdk/Godot.NET.Sdk_PackageVersion.txt create mode 100644 modules/mono/editor/GodotTools/GodotTools.Shared/GenerateGodotNupkgsVersions.targets create mode 100644 modules/mono/editor/GodotTools/GodotTools.Shared/GodotTools.Shared.csproj create mode 100644 modules/mono/editor/GodotTools/GodotTools/Build/NuGetUtils.cs (limited to 'modules/mono/editor') diff --git a/modules/mono/editor/Godot.NET.Sdk/Godot.NET.Sdk/Godot.NET.Sdk.csproj b/modules/mono/editor/Godot.NET.Sdk/Godot.NET.Sdk/Godot.NET.Sdk.csproj index 86a0a4393e..8304d9e321 100644 --- a/modules/mono/editor/Godot.NET.Sdk/Godot.NET.Sdk/Godot.NET.Sdk.csproj +++ b/modules/mono/editor/Godot.NET.Sdk/Godot.NET.Sdk/Godot.NET.Sdk.csproj @@ -1,13 +1,13 @@ netstandard2.0 + false MSBuild .NET Sdk for Godot projects. Godot Engine contributors Godot.NET.Sdk 4.0.0 - 4.0.0-dev2 https://github.com/godotengine/godot/tree/master/modules/mono/editor/Godot.NET.Sdk MSBuildSdk MSBuildSdk @@ -19,7 +19,13 @@ $(GenerateNuspecDependsOn);SetNuSpecProperties - + + + $([System.IO.File]::ReadAllText('$(ProjectDir)Godot.NET.Sdk_PackageVersion.txt').Trim()) + + + + id=$(PackageId); @@ -32,4 +38,13 @@ + + + + $(SolutionDir)\..\..\..\..\ + $(GodotSourceRootPath)\bin\GodotSharp\ + + + diff --git a/modules/mono/editor/Godot.NET.Sdk/Godot.NET.Sdk/Godot.NET.Sdk_PackageVersion.txt b/modules/mono/editor/Godot.NET.Sdk/Godot.NET.Sdk/Godot.NET.Sdk_PackageVersion.txt new file mode 100644 index 0000000000..34749489b9 --- /dev/null +++ b/modules/mono/editor/Godot.NET.Sdk/Godot.NET.Sdk/Godot.NET.Sdk_PackageVersion.txt @@ -0,0 +1 @@ +4.0.0-dev3 diff --git a/modules/mono/editor/GodotTools/GodotTools.ProjectEditor/GodotTools.ProjectEditor.csproj b/modules/mono/editor/GodotTools/GodotTools.ProjectEditor/GodotTools.ProjectEditor.csproj index e4d6b2e010..37123ba2b2 100644 --- a/modules/mono/editor/GodotTools/GodotTools.ProjectEditor/GodotTools.ProjectEditor.csproj +++ b/modules/mono/editor/GodotTools/GodotTools.ProjectEditor/GodotTools.ProjectEditor.csproj @@ -10,6 +10,7 @@ + + + + + $(SolutionDir)..\Godot.NET.Sdk\Godot.NET.Sdk\Godot.NET.Sdk_PackageVersion.txt + $(IntermediateOutputPath)GodotNupkgsVersions.g.cs + + + + + + + + + + + + + + + + diff --git a/modules/mono/editor/GodotTools/GodotTools.Shared/GodotTools.Shared.csproj b/modules/mono/editor/GodotTools/GodotTools.Shared/GodotTools.Shared.csproj new file mode 100644 index 0000000000..3bc1698c15 --- /dev/null +++ b/modules/mono/editor/GodotTools/GodotTools.Shared/GodotTools.Shared.csproj @@ -0,0 +1,6 @@ + + + netstandard2.0 + + + diff --git a/modules/mono/editor/GodotTools/GodotTools.sln b/modules/mono/editor/GodotTools/GodotTools.sln index ba5379e562..d3107a69db 100644 --- a/modules/mono/editor/GodotTools/GodotTools.sln +++ b/modules/mono/editor/GodotTools/GodotTools.sln @@ -13,6 +13,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GodotTools.IdeMessaging", " EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GodotTools.OpenVisualStudio", "GodotTools.OpenVisualStudio\GodotTools.OpenVisualStudio.csproj", "{EAFFF236-FA96-4A4D-BD23-0E51EF988277}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GodotTools.Shared", "GodotTools.Shared\GodotTools.Shared.csproj", "{2758FFAF-8237-4CF2-B569-66BF8B3587BB}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -43,5 +45,9 @@ Global {EAFFF236-FA96-4A4D-BD23-0E51EF988277}.Debug|Any CPU.Build.0 = Debug|Any CPU {EAFFF236-FA96-4A4D-BD23-0E51EF988277}.Release|Any CPU.ActiveCfg = Release|Any CPU {EAFFF236-FA96-4A4D-BD23-0E51EF988277}.Release|Any CPU.Build.0 = Release|Any CPU + {2758FFAF-8237-4CF2-B569-66BF8B3587BB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {2758FFAF-8237-4CF2-B569-66BF8B3587BB}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2758FFAF-8237-4CF2-B569-66BF8B3587BB}.Release|Any CPU.ActiveCfg = Release|Any CPU + {2758FFAF-8237-4CF2-B569-66BF8B3587BB}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection EndGlobal diff --git a/modules/mono/editor/GodotTools/GodotTools/Build/BuildManager.cs b/modules/mono/editor/GodotTools/GodotTools/Build/BuildManager.cs index 9b512d90f5..b96b0c8175 100644 --- a/modules/mono/editor/GodotTools/GodotTools/Build/BuildManager.cs +++ b/modules/mono/editor/GodotTools/GodotTools/Build/BuildManager.cs @@ -7,7 +7,6 @@ using JetBrains.Annotations; using static GodotTools.Internals.Globals; using File = GodotTools.Utils.File; using OS = GodotTools.Utils.OS; -using Path = System.IO.Path; namespace GodotTools.Build { @@ -209,6 +208,16 @@ namespace GodotTools.Build if (!File.Exists(GodotSharpDirs.ProjectSlnPath)) return true; // No solution to build + try + { + // Make sure our packages are added to the fallback folder + NuGetUtils.AddBundledPackagesToFallbackFolder(NuGetUtils.GodotFallbackFolderPath); + } + catch (Exception e) + { + Godot.GD.PushError("Failed to setup Godot NuGet Offline Packages: " + e.Message); + } + GenerateEditorScriptMetadata(); if (GodotSharpEditor.Instance.SkipBuildBeforePlaying) diff --git a/modules/mono/editor/GodotTools/GodotTools/Build/MSBuildPanel.cs b/modules/mono/editor/GodotTools/GodotTools/Build/MSBuildPanel.cs index c22dce005c..708ec73454 100644 --- a/modules/mono/editor/GodotTools/GodotTools/Build/MSBuildPanel.cs +++ b/modules/mono/editor/GodotTools/GodotTools/Build/MSBuildPanel.cs @@ -33,6 +33,16 @@ namespace GodotTools.Build if (!File.Exists(GodotSharpDirs.ProjectSlnPath)) return; // No solution to build + try + { + // Make sure our packages are added to the fallback folder + NuGetUtils.AddBundledPackagesToFallbackFolder(NuGetUtils.GodotFallbackFolderPath); + } + catch (Exception e) + { + GD.PushError("Failed to setup Godot NuGet Offline Packages: " + e.Message); + } + BuildManager.GenerateEditorScriptMetadata(); if (!BuildManager.BuildProjectBlocking("Debug")) @@ -54,6 +64,16 @@ namespace GodotTools.Build if (!File.Exists(GodotSharpDirs.ProjectSlnPath)) return; // No solution to build + try + { + // Make sure our packages are added to the fallback folder + NuGetUtils.AddBundledPackagesToFallbackFolder(NuGetUtils.GodotFallbackFolderPath); + } + catch (Exception e) + { + GD.PushError("Failed to setup Godot NuGet Offline Packages: " + e.Message); + } + BuildManager.GenerateEditorScriptMetadata(); if (!BuildManager.BuildProjectBlocking("Debug", targets: new[] {"Rebuild"})) diff --git a/modules/mono/editor/GodotTools/GodotTools/Build/NuGetUtils.cs b/modules/mono/editor/GodotTools/GodotTools/Build/NuGetUtils.cs new file mode 100644 index 0000000000..793ef7fd71 --- /dev/null +++ b/modules/mono/editor/GodotTools/GodotTools/Build/NuGetUtils.cs @@ -0,0 +1,296 @@ +using System; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Security.Cryptography; +using System.Text; +using System.Xml; +using Godot; +using GodotTools.Internals; +using GodotTools.Shared; +using Directory = GodotTools.Utils.Directory; +using Environment = System.Environment; +using File = GodotTools.Utils.File; + +namespace GodotTools.Build +{ + public static class NuGetUtils + { + public const string GodotFallbackFolderName = "Godot Offline Packages"; + + public static string GodotFallbackFolderPath + => Path.Combine(GodotSharpDirs.MonoUserDir, "GodotNuGetFallbackFolder"); + + private static void AddFallbackFolderToNuGetConfig(string nuGetConfigPath, string name, string path) + { + var xmlDoc = new XmlDocument(); + xmlDoc.Load(nuGetConfigPath); + + const string nuGetConfigRootName = "configuration"; + + var rootNode = xmlDoc.DocumentElement; + + if (rootNode == null) + { + // No root node, create it + rootNode = xmlDoc.CreateElement(nuGetConfigRootName); + xmlDoc.AppendChild(rootNode); + + // Since this can be considered pretty much a new NuGet.Config, add the default nuget.org source as well + XmlElement nugetOrgSourceEntry = xmlDoc.CreateElement("add"); + nugetOrgSourceEntry.Attributes.Append(xmlDoc.CreateAttribute("key")).Value = "nuget.org"; + nugetOrgSourceEntry.Attributes.Append(xmlDoc.CreateAttribute("value")).Value = "https://api.nuget.org/v3/index.json"; + nugetOrgSourceEntry.Attributes.Append(xmlDoc.CreateAttribute("protocolVersion")).Value = "3"; + rootNode.AppendChild(xmlDoc.CreateElement("packageSources")).AppendChild(nugetOrgSourceEntry); + } + else + { + // Check that the root node is the expected one + if (rootNode.Name != nuGetConfigRootName) + throw new Exception("Invalid root Xml node for NuGet.Config. " + + $"Expected '{nuGetConfigRootName}' got '{rootNode.Name}'."); + } + + var fallbackFoldersNode = rootNode["fallbackPackageFolders"] ?? + rootNode.AppendChild(xmlDoc.CreateElement("fallbackPackageFolders")); + + // Check if it already has our fallback package folder + for (var xmlNode = fallbackFoldersNode.FirstChild; xmlNode != null; xmlNode = xmlNode.NextSibling) + { + if (xmlNode.NodeType != XmlNodeType.Element) + continue; + + var xmlElement = (XmlElement)xmlNode; + if (xmlElement.Name == "add" && + xmlElement.Attributes["key"]?.Value == name && + xmlElement.Attributes["value"]?.Value == path) + { + return; + } + } + + XmlElement newEntry = xmlDoc.CreateElement("add"); + newEntry.Attributes.Append(xmlDoc.CreateAttribute("key")).Value = name; + newEntry.Attributes.Append(xmlDoc.CreateAttribute("value")).Value = path; + + fallbackFoldersNode.AppendChild(newEntry); + + xmlDoc.Save(nuGetConfigPath); + } + + /// + /// Returns all the paths where the user NuGet.Config files can be found. + /// Does not determine whether the returned files exist or not. + /// + private static string[] GetAllUserNuGetConfigFilePaths() + { + // Where to find 'NuGet/NuGet.Config': + // + // - Mono/.NETFramework (standalone NuGet): + // Uses Environment.SpecialFolder.ApplicationData + // - Windows: '%APPDATA%' + // - Linux/macOS: '$HOME/.config' + // - CoreCLR (dotnet CLI NuGet): + // - Windows: '%APPDATA%' + // - Linux/macOS: '$DOTNET_CLI_HOME/.nuget' otherwise '$HOME/.nuget' + + string applicationData = Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData); + + if (Utils.OS.IsWindows) + { + // %APPDATA% for both + return new[] {Path.Combine(applicationData, "NuGet", "NuGet.Config")}; + } + + var paths = new string[2]; + + // CoreCLR (dotnet CLI NuGet) + + string dotnetCliHome = Environment.GetEnvironmentVariable("DOTNET_CLI_HOME"); + if (!string.IsNullOrEmpty(dotnetCliHome)) + { + paths[0] = Path.Combine(dotnetCliHome, ".nuget", "NuGet", "NuGet.Config"); + } + else + { + string home = Environment.GetEnvironmentVariable("HOME"); + if (string.IsNullOrEmpty(home)) + throw new InvalidOperationException("Required environment variable 'HOME' is not set."); + paths[0] = Path.Combine(home, ".nuget", "NuGet", "NuGet.Config"); + } + + // Mono/.NETFramework (standalone NuGet) + + // ApplicationData is $HOME/.config on Linux/macOS + paths[1] = Path.Combine(applicationData, "NuGet", "NuGet.Config"); + + return paths; + } + + // nupkg extraction + // + // Exclude: (NuGet.Client -> NuGet.Packaging.PackageHelper.ExcludePaths) + // package/ + // _rels/ + // [Content_Types].xml + // + // Don't ignore files that begin with a dot (.) + // + // The nuspec is not lower case inside the nupkg but must be made lower case when extracted. + + /// + /// Adds the specified fallback folder to the user NuGet.Config files, + /// for both standalone NuGet (Mono/.NETFramework) and dotnet CLI NuGet. + /// + public static void AddFallbackFolderToUserNuGetConfigs(string name, string path) + { + foreach (string nuGetConfigPath in GetAllUserNuGetConfigFilePaths()) + { + if (!System.IO.File.Exists(nuGetConfigPath)) + { + // It doesn't exist, so we create a default one + const string defaultConfig = @" + + + + + +"; + System.IO.File.WriteAllText(nuGetConfigPath, defaultConfig, Encoding.UTF8); // UTF-8 with BOM + } + + AddFallbackFolderToNuGetConfig(nuGetConfigPath, name, path); + } + } + + private static void AddPackageToFallbackFolder(string fallbackFolder, + string nupkgPath, string packageId, string packageVersion) + { + // dotnet CLI provides no command for this, but we can do it manually. + // + // - The expected structure is as follows: + // fallback_folder/ + // // + // ..nupkg + // ..nupkg.sha512 + // .nuspec + // ... extracted nupkg files (check code for excluded files) ... + // + // - and must be in lower case. + // - The sha512 of the nupkg is base64 encoded. + // - We can get the nuspec from the nupkg which is a Zip file. + + string packageIdLower = packageId.ToLower(); + string packageVersionLower = packageVersion.ToLower(); + + string destDir = Path.Combine(fallbackFolder, packageIdLower, packageVersionLower); + string nupkgDestPath = Path.Combine(destDir, $"{packageIdLower}.{packageVersionLower}.nupkg"); + string nupkgSha512DestPath = Path.Combine(destDir, $"{packageIdLower}.{packageVersionLower}.nupkg.sha512"); + + if (File.Exists(nupkgDestPath) && File.Exists(nupkgSha512DestPath)) + return; // Already added (for speed we don't check if every file is properly extracted) + + Directory.CreateDirectory(destDir); + + // Generate .nupkg.sha512 file + + using (var alg = SHA512.Create()) + { + alg.ComputeHash(File.ReadAllBytes(nupkgPath)); + string base64Hash = Convert.ToBase64String(alg.Hash); + File.WriteAllText(nupkgSha512DestPath, base64Hash); + } + + // Extract nupkg + ExtractNupkg(destDir, nupkgPath, packageId, packageVersion); + + // Copy .nupkg + File.Copy(nupkgPath, nupkgDestPath); + } + + private static readonly string[] NupkgExcludePaths = + { + "_rels/", + "package/", + "[Content_Types].xml" + }; + + private static void ExtractNupkg(string destDir, string nupkgPath, string packageId, string packageVersion) + { + // NOTE: Must use SimplifyGodotPath to make sure we don't extract files outside the destination directory. + + using (var archive = ZipFile.OpenRead(nupkgPath)) + { + // Extract .nuspec manually as it needs to be in lower case + + var nuspecEntry = archive.GetEntry(packageId + ".nuspec"); + + if (nuspecEntry == null) + throw new InvalidOperationException($"Failed to extract package {packageId}.{packageVersion}. Could not find the nuspec file."); + + nuspecEntry.ExtractToFile(Path.Combine(destDir, nuspecEntry.Name.ToLower().SimplifyGodotPath())); + + // Extract the other package files + + foreach (var entry in archive.Entries) + { + // NOTE: SimplifyGodotPath() removes trailing slash and backslash, + // so we can't use the result to check if the entry is a directory. + + string entryFullName = entry.FullName.Replace('\\', '/'); + + // Check if the file must be ignored + if ( // Excluded files. + NupkgExcludePaths.Any(e => entryFullName.StartsWith(e, StringComparison.OrdinalIgnoreCase)) || + // Nupkg hash files and nupkg metadata files on all directory. + entryFullName.EndsWith(".nupkg.sha512", StringComparison.OrdinalIgnoreCase) || + entryFullName.EndsWith(".nupkg.metadata", StringComparison.OrdinalIgnoreCase) || + // Nuspec at root level. We already extracted it previously but in lower case. + entryFullName.IndexOf('/') == -1 && entryFullName.EndsWith(".nuspec")) + { + continue; + } + + string entryFullNameSimplified = entryFullName.SimplifyGodotPath(); + string destFilePath = Path.Combine(destDir, entryFullNameSimplified); + bool isDir = entryFullName.EndsWith("/"); + + if (isDir) + { + Directory.CreateDirectory(destFilePath); + } + else + { + Directory.CreateDirectory(Path.GetDirectoryName(destFilePath)); + entry.ExtractToFile(destFilePath, overwrite: true); + } + } + } + } + + /// + /// Copies and extracts all the Godot bundled packages to the Godot NuGet fallback folder. + /// Does nothing if the packages were already copied. + /// + public static void AddBundledPackagesToFallbackFolder(string fallbackFolder) + { + GD.Print("Copying Godot Offline Packages..."); + + string nupkgsLocation = Path.Combine(GodotSharpDirs.DataEditorToolsDir, "nupkgs"); + + void AddPackage(string packageId, string packageVersion) + { + string nupkgPath = Path.Combine(nupkgsLocation, $"{packageId}.{packageVersion}.nupkg"); + AddPackageToFallbackFolder(fallbackFolder, nupkgPath, packageId, packageVersion); + } + + foreach (var (packageId, packageVersion) in PackagesToAdd) + AddPackage(packageId, packageVersion); + } + + private static readonly (string packageId, string packageVersion)[] PackagesToAdd = + { + ("Godot.NET.Sdk", GeneratedGodotNupkgsVersions.GodotNETSdk) + }; + } +} diff --git a/modules/mono/editor/GodotTools/GodotTools/GodotSharpEditor.cs b/modules/mono/editor/GodotTools/GodotTools/GodotSharpEditor.cs index f5c99e87ef..58561c5097 100644 --- a/modules/mono/editor/GodotTools/GodotTools/GodotSharpEditor.cs +++ b/modules/mono/editor/GodotTools/GodotTools/GodotSharpEditor.cs @@ -147,6 +147,21 @@ namespace GodotTools case MenuOptions.AboutCSharp: _ShowAboutDialog(); break; + case MenuOptions.SetupGodotNugetFallbackFolder: + { + try + { + string fallbackFolder = NuGetUtils.GodotFallbackFolderPath; + NuGetUtils.AddFallbackFolderToUserNuGetConfigs(NuGetUtils.GodotFallbackFolderName, fallbackFolder); + NuGetUtils.AddBundledPackagesToFallbackFolder(fallbackFolder); + } + catch (Exception e) + { + ShowErrorDialog("Failed to setup Godot NuGet Offline Packages: " + e.Message); + } + + break; + } default: throw new ArgumentOutOfRangeException(nameof(id), id, "Invalid menu option"); } @@ -183,6 +198,7 @@ namespace GodotTools { CreateSln, AboutCSharp, + SetupGodotNugetFallbackFolder, } public void ShowErrorDialog(string message, string title = "Error") @@ -426,6 +442,7 @@ namespace GodotTools // TODO: Remove or edit this info dialog once Mono support is no longer in alpha { menuPopup.AddItem("About C# support".TTR(), (int)MenuOptions.AboutCSharp); + menuPopup.AddItem("Setup Godot NuGet Offline Packages".TTR(), (int)MenuOptions.SetupGodotNugetFallbackFolder); aboutDialog = new AcceptDialog(); editorBaseControl.AddChild(aboutDialog); aboutDialog.Title = "Important: C# support is not feature-complete"; @@ -535,6 +552,16 @@ namespace GodotTools exportPlugin.RegisterExportSettings(); exportPluginWeak = WeakRef(exportPlugin); + try + { + // At startup we make sure NuGet.Config files have our Godot NuGet fallback folder included + NuGetUtils.AddFallbackFolderToUserNuGetConfigs(NuGetUtils.GodotFallbackFolderName, NuGetUtils.GodotFallbackFolderPath); + } + catch (Exception e) + { + GD.PushError("Failed to add Godot NuGet Offline Packages to NuGet.Config: " + e.Message); + } + BuildManager.Initialize(); RiderPathManager.Initialize(); -- cgit v1.2.3