From c0cc41d6c1bb7b9fb4897debb15f1f083d3c1d02 Mon Sep 17 00:00:00 2001 From: bruvzg <7645683+bruvzg@users.noreply.github.com> Date: Mon, 20 Dec 2021 11:28:54 +0200 Subject: Improve embedded PCK loading and exporting. Windows export process: Limit size of executable with embedded PCK to 4 GB. Use "rcedit" before embedding PCK. Capture and process "rcedit" errors. Windows, Linux: Add support for PCK loading from executable "pck" section. --- core/io/file_access_pack.cpp | 68 +++++++++++++++++------ core/os/os.cpp | 5 ++ core/os/os.h | 2 + editor/editor_export.cpp | 89 ++++++++++++++++++------------- editor/editor_export.h | 4 ++ platform/linuxbsd/os_linuxbsd.cpp | 85 +++++++++++++++++++++++++++++ platform/linuxbsd/os_linuxbsd.h | 2 + platform/windows/export/export_plugin.cpp | 56 +++++++++++++------ platform/windows/export/export_plugin.h | 2 +- platform/windows/godot_windows.cpp | 18 ++++--- platform/windows/os_windows.cpp | 52 ++++++++++++++++++ platform/windows/os_windows.h | 2 + 12 files changed, 310 insertions(+), 75 deletions(-) diff --git a/core/io/file_access_pack.cpp b/core/io/file_access_pack.cpp index c6e14ffee7..ba120de68b 100644 --- a/core/io/file_access_pack.cpp +++ b/core/io/file_access_pack.cpp @@ -32,6 +32,7 @@ #include "core/io/file_access_encrypted.h" #include "core/object/script_language.h" +#include "core/os/os.h" #include "core/version.h" #include @@ -131,32 +132,69 @@ bool PackedSourcePCK::try_open_pack(const String &p_path, bool p_replace_files, return false; } - f->seek(p_offset); + bool pck_header_found = false; + // Search for the header at the start offset - standalone PCK file. + f->seek(p_offset); uint32_t magic = f->get_32(); + if (magic == PACK_HEADER_MAGIC) { + pck_header_found = true; + } - if (magic != PACK_HEADER_MAGIC) { - // loading with offset feature not supported for self contained exe files - ERR_FAIL_COND_V_MSG(p_offset != 0, false, "Loading self-contained executable with offset not supported."); + // Search for the header in the executable "pck" section - self contained executable. + if (!pck_header_found) { + // Loading with offset feature not supported for self contained exe files. + if (p_offset != 0) { + ERR_FAIL_V_MSG(false, "Loading self-contained executable with offset not supported."); + } - //maybe at the end.... self contained exe - f->seek_end(); - f->seek(f->get_position() - 4); - magic = f->get_32(); - if (magic != PACK_HEADER_MAGIC) { - return false; + int64_t pck_off = OS::get_singleton()->get_embedded_pck_offset(); + if (pck_off != 0) { + // Search for the header, in case PCK start and section have different alignment. + for (int i = 0; i < 8; i++) { + f->seek(pck_off); + magic = f->get_32(); + if (magic == PACK_HEADER_MAGIC) { +#ifdef DEBUG_ENABLED + print_verbose("PCK header found in executable pck section, loading from offset 0x" + String::num_int64(pck_off - 4, 16)); +#endif + pck_header_found = true; + break; + } + pck_off++; + } } - f->seek(f->get_position() - 12); + } - uint64_t ds = f->get_64(); - f->seek(f->get_position() - ds - 8); + // Search for the header at the end of file - self contained executable. + if (!pck_header_found) { + // Loading with offset feature not supported for self contained exe files. + if (p_offset != 0) { + ERR_FAIL_V_MSG(false, "Loading self-contained executable with offset not supported."); + } + f->seek_end(); + f->seek(f->get_position() - 4); magic = f->get_32(); - if (magic != PACK_HEADER_MAGIC) { - return false; + + if (magic == PACK_HEADER_MAGIC) { + f->seek(f->get_position() - 12); + uint64_t ds = f->get_64(); + f->seek(f->get_position() - ds - 8); + magic = f->get_32(); + if (magic == PACK_HEADER_MAGIC) { +#ifdef DEBUG_ENABLED + print_verbose("PCK header found at the end of executable, loading from offset 0x" + String::num_int64(f->get_position() - 4, 16)); +#endif + pck_header_found = true; + } } } + if (!pck_header_found) { + return false; + } + uint32_t version = f->get_32(); uint32_t ver_major = f->get_32(); uint32_t ver_minor = f->get_32(); diff --git a/core/os/os.cpp b/core/os/os.cpp index 846aeb16c5..4f7095b0fc 100644 --- a/core/os/os.cpp +++ b/core/os/os.cpp @@ -237,6 +237,11 @@ String OS::get_locale_language() const { return get_locale().left(3).replace("_", ""); } +// Embedded PCK offset. +uint64_t OS::get_embedded_pck_offset() const { + return 0; +} + // Helper function to ensure that a dir name/path will be valid on the OS String OS::get_safe_dir_name(const String &p_dir_name, bool p_allow_dir_separator) const { Vector invalid_chars = String(": * ? \" < > |").split(" "); diff --git a/core/os/os.h b/core/os/os.h index b3c66ff18c..9ec0dd7728 100644 --- a/core/os/os.h +++ b/core/os/os.h @@ -264,6 +264,8 @@ public: virtual String get_locale() const; String get_locale_language() const; + virtual uint64_t get_embedded_pck_offset() const; + String get_safe_dir_name(const String &p_dir_name, bool p_allow_dir_separator = false) const; virtual String get_godot_dir_name() const; diff --git a/editor/editor_export.cpp b/editor/editor_export.cpp index 58a9175df1..74993f567a 100644 --- a/editor/editor_export.cpp +++ b/editor/editor_export.cpp @@ -1820,6 +1820,18 @@ bool EditorExportPlatformPC::can_export(const Ref &p_preset, Error EditorExportPlatformPC::export_project(const Ref &p_preset, bool p_debug, const String &p_path, int p_flags) { ExportNotifier notifier(*this, p_preset, p_debug, p_path, p_flags); + Error err = prepare_template(p_preset, p_debug, p_path, p_flags); + if (err == OK) { + err = export_project_data(p_preset, p_debug, p_path, p_flags); + } + + return err; +} + +Error EditorExportPlatformPC::prepare_template(const Ref &p_preset, bool p_debug, const String &p_path, int p_flags) { + if (!DirAccess::exists(p_path.get_base_dir())) { + return ERR_FILE_BAD_PATH; + } String custom_debug = p_preset->get("custom_template/debug"); String custom_release = p_preset->get("custom_template/release"); @@ -1841,49 +1853,52 @@ Error EditorExportPlatformPC::export_project(const Ref &p_pr da->make_dir_recursive(p_path.get_base_dir()); Error err = da->copy(template_path, p_path, get_chmod_flags()); - if (err == OK) { - String pck_path; - if (p_preset->get("binary_format/embed_pck")) { - pck_path = p_path; - } else { - pck_path = p_path.get_basename() + ".pck"; - } + return err; +} - Vector so_files; +Error EditorExportPlatformPC::export_project_data(const Ref &p_preset, bool p_debug, const String &p_path, int p_flags) { + String pck_path; + if (p_preset->get("binary_format/embed_pck")) { + pck_path = p_path; + } else { + pck_path = p_path.get_basename() + ".pck"; + } - int64_t embedded_pos; - int64_t embedded_size; - err = save_pack(p_preset, p_debug, pck_path, &so_files, p_preset->get("binary_format/embed_pck"), &embedded_pos, &embedded_size); - if (err == OK && p_preset->get("binary_format/embed_pck")) { - if (embedded_size >= 0x100000000 && !p_preset->get("binary_format/64_bits")) { - EditorNode::get_singleton()->show_warning(TTR("On 32-bit exports the embedded PCK cannot be bigger than 4 GiB.")); - return ERR_INVALID_PARAMETER; - } + Vector so_files; - err = fixup_embedded_pck(p_path, embedded_pos, embedded_size); + int64_t embedded_pos; + int64_t embedded_size; + Error err = save_pack(p_preset, p_debug, pck_path, &so_files, p_preset->get("binary_format/embed_pck"), &embedded_pos, &embedded_size); + if (err == OK && p_preset->get("binary_format/embed_pck")) { + if (embedded_size >= 0x100000000 && !p_preset->get("binary_format/64_bits")) { + EditorNode::get_singleton()->show_warning(TTR("On 32-bit exports the embedded PCK cannot be bigger than 4 GiB.")); + return ERR_INVALID_PARAMETER; } - if (err == OK && !so_files.is_empty()) { - // If shared object files, copy them. - for (int i = 0; i < so_files.size() && err == OK; i++) { - String src_path = ProjectSettings::get_singleton()->globalize_path(so_files[i].path); - String target_path; - if (so_files[i].target.is_empty()) { - target_path = p_path.get_base_dir().plus_file(src_path.get_file()); - } else { - target_path = p_path.get_base_dir().plus_file(so_files[i].target).plus_file(src_path.get_file()); - } + err = fixup_embedded_pck(p_path, embedded_pos, embedded_size); + } - if (da->dir_exists(src_path)) { - err = da->make_dir_recursive(target_path); - if (err == OK) { - err = da->copy_dir(src_path, target_path, -1, true); - } - } else { - err = da->copy(src_path, target_path); - if (err == OK) { - err = sign_shared_object(p_preset, p_debug, target_path); - } + if (err == OK && !so_files.is_empty()) { + // If shared object files, copy them. + Ref da = DirAccess::create(DirAccess::ACCESS_FILESYSTEM); + for (int i = 0; i < so_files.size() && err == OK; i++) { + String src_path = ProjectSettings::get_singleton()->globalize_path(so_files[i].path); + String target_path; + if (so_files[i].target.is_empty()) { + target_path = p_path.get_base_dir().plus_file(src_path.get_file()); + } else { + target_path = p_path.get_base_dir().plus_file(so_files[i].target).plus_file(src_path.get_file()); + } + + if (da->dir_exists(src_path)) { + err = da->make_dir_recursive(target_path); + if (err == OK) { + err = da->copy_dir(src_path, target_path, -1, true); + } + } else { + err = da->copy(src_path, target_path); + if (err == OK) { + err = sign_shared_object(p_preset, p_debug, target_path); } } } diff --git a/editor/editor_export.h b/editor/editor_export.h index 236f4d129c..ab09c98941 100644 --- a/editor/editor_export.h +++ b/editor/editor_export.h @@ -442,6 +442,10 @@ public: virtual Error sign_shared_object(const Ref &p_preset, bool p_debug, const String &p_path); virtual String get_template_file_name(const String &p_target, const String &p_arch) const = 0; + Error prepare_template(const Ref &p_preset, bool p_debug, const String &p_path, int p_flags); + Error export_project_data(const Ref &p_preset, bool p_debug, const String &p_path, int p_flags); + + void set_extension(const String &p_extension, const String &p_feature_key = "default"); void set_name(const String &p_name); void set_os_name(const String &p_name); diff --git a/platform/linuxbsd/os_linuxbsd.cpp b/platform/linuxbsd/os_linuxbsd.cpp index bf142f8804..b73d4dc626 100644 --- a/platform/linuxbsd/os_linuxbsd.cpp +++ b/platform/linuxbsd/os_linuxbsd.cpp @@ -242,6 +242,91 @@ bool OS_LinuxBSD::_check_internal_feature_support(const String &p_feature) { return p_feature == "pc"; } +uint64_t OS_LinuxBSD::get_embedded_pck_offset() const { + Ref f = FileAccess::open(get_executable_path(), FileAccess::READ); + if (f.is_null()) { + return 0; + } + + // Read and check ELF magic number. + { + uint32_t magic = f->get_32(); + if (magic != 0x464c457f) { // 0x7F + "ELF" + return 0; + } + } + + // Read program architecture bits from class field. + int bits = f->get_8() * 32; + + // Get info about the section header table. + int64_t section_table_pos; + int64_t section_header_size; + if (bits == 32) { + section_header_size = 40; + f->seek(0x20); + section_table_pos = f->get_32(); + f->seek(0x30); + } else { // 64 + section_header_size = 64; + f->seek(0x28); + section_table_pos = f->get_64(); + f->seek(0x3c); + } + int num_sections = f->get_16(); + int string_section_idx = f->get_16(); + + // Load the strings table. + uint8_t *strings; + { + // Jump to the strings section header. + f->seek(section_table_pos + string_section_idx * section_header_size); + + // Read strings data size and offset. + int64_t string_data_pos; + int64_t string_data_size; + if (bits == 32) { + f->seek(f->get_position() + 0x10); + string_data_pos = f->get_32(); + string_data_size = f->get_32(); + } else { // 64 + f->seek(f->get_position() + 0x18); + string_data_pos = f->get_64(); + string_data_size = f->get_64(); + } + + // Read strings data. + f->seek(string_data_pos); + strings = (uint8_t *)memalloc(string_data_size); + if (!strings) { + return 0; + } + f->get_buffer(strings, string_data_size); + } + + // Search for the "pck" section. + int64_t off = 0; + for (int i = 0; i < num_sections; ++i) { + int64_t section_header_pos = section_table_pos + i * section_header_size; + f->seek(section_header_pos); + + uint32_t name_offset = f->get_32(); + if (strcmp((char *)strings + name_offset, "pck") == 0) { + if (bits == 32) { + f->seek(section_header_pos + 0x10); + off = f->get_32(); + } else { // 64 + f->seek(section_header_pos + 0x18); + off = f->get_64(); + } + break; + } + } + memfree(strings); + + return off; +} + String OS_LinuxBSD::get_config_path() const { if (has_environment("XDG_CONFIG_HOME")) { if (get_environment("XDG_CONFIG_HOME").is_absolute_path()) { diff --git a/platform/linuxbsd/os_linuxbsd.h b/platform/linuxbsd/os_linuxbsd.h index 7b912ddee3..3f97b86eae 100644 --- a/platform/linuxbsd/os_linuxbsd.h +++ b/platform/linuxbsd/os_linuxbsd.h @@ -78,6 +78,8 @@ public: virtual MainLoop *get_main_loop() const override; + virtual uint64_t get_embedded_pck_offset() const override; + virtual String get_config_path() const override; virtual String get_data_path() const override; virtual String get_cache_path() const override; diff --git a/platform/windows/export/export_plugin.cpp b/platform/windows/export/export_plugin.cpp index 917a0af90b..4c91163d2b 100644 --- a/platform/windows/export/export_plugin.cpp +++ b/platform/windows/export/export_plugin.cpp @@ -54,16 +54,23 @@ Error EditorExportPlatformWindows::_export_debug_script(const Ref &p_preset, bool p_debug, const String &p_path, int p_flags) { - Error err = EditorExportPlatformPC::export_project(p_preset, p_debug, p_path, p_flags); - - if (err != OK) { - return err; + String pck_path = p_path; + if (p_preset->get("binary_format/embed_pck")) { + pck_path = p_path.get_basename() + ".tmp"; + } + Error err = EditorExportPlatformPC::prepare_template(p_preset, p_debug, pck_path, p_flags); + if (p_preset->get("application/modify_resources") && err == OK) { + err = _rcedit_add_data(p_preset, pck_path); + } + if (err == OK) { + err = EditorExportPlatformPC::export_project_data(p_preset, p_debug, pck_path, p_flags); } - - _rcedit_add_data(p_preset, p_path); - if (p_preset->get("codesign/enable") && err == OK) { - err = _code_sign(p_preset, p_path); + err = _code_sign(p_preset, pck_path); + } + if (p_preset->get("binary_format/embed_pck") && err == OK) { + Ref tmp_dir = DirAccess::create_for_path(p_path.get_base_dir()); + err = tmp_dir->rename(pck_path, p_path); } String app_name; @@ -117,6 +124,7 @@ void EditorExportPlatformWindows::get_export_options(List *r_optio r_options->push_back(ExportOption(PropertyInfo(Variant::STRING, "codesign/description"), "")); r_options->push_back(ExportOption(PropertyInfo(Variant::PACKED_STRING_ARRAY, "codesign/custom_options"), PackedStringArray())); + r_options->push_back(ExportOption(PropertyInfo(Variant::BOOL, "application/modify_resources"), true)); r_options->push_back(ExportOption(PropertyInfo(Variant::STRING, "application/icon", PROPERTY_HINT_FILE, "*.ico"), "")); r_options->push_back(ExportOption(PropertyInfo(Variant::STRING, "application/file_version", PROPERTY_HINT_PLACEHOLDER_TEXT, "1.0.0.0"), "")); r_options->push_back(ExportOption(PropertyInfo(Variant::STRING, "application/product_version", PROPERTY_HINT_PLACEHOLDER_TEXT, "1.0.0.0"), "")); @@ -127,17 +135,20 @@ void EditorExportPlatformWindows::get_export_options(List *r_optio r_options->push_back(ExportOption(PropertyInfo(Variant::STRING, "application/trademarks"), "")); } -void EditorExportPlatformWindows::_rcedit_add_data(const Ref &p_preset, const String &p_path) { +Error EditorExportPlatformWindows::_rcedit_add_data(const Ref &p_preset, const String &p_path) { String rcedit_path = EditorSettings::get_singleton()->get("export/windows/rcedit"); if (rcedit_path.is_empty()) { WARN_PRINT("The rcedit tool is not configured in the Editor Settings (Export > Windows > Rcedit). No custom icon or app information data will be embedded in the exported executable."); - return; + return ERR_FILE_NOT_FOUND; } if (!FileAccess::exists(rcedit_path)) { ERR_PRINT("Could not find rcedit executable at " + rcedit_path + ", no icon or app information data will be included."); - return; + return ERR_FILE_NOT_FOUND; + } + if (rcedit_path.is_empty()) { + rcedit_path = "rcedit"; // try to run signtool from PATH } #ifndef WINDOWS_ENABLED @@ -146,7 +157,7 @@ void EditorExportPlatformWindows::_rcedit_add_data(const Ref if (!wine_path.is_empty() && !FileAccess::exists(wine_path)) { ERR_PRINT("Could not find wine executable at " + wine_path + ", no icon or app information data will be included."); - return; + return ERR_FILE_NOT_FOUND; } if (wine_path.is_empty()) { @@ -204,13 +215,22 @@ void EditorExportPlatformWindows::_rcedit_add_data(const Ref args.push_back(trademarks); } -#ifdef WINDOWS_ENABLED - OS::get_singleton()->execute(rcedit_path, args); -#else +#ifndef WINDOWS_ENABLED // On non-Windows we need WINE to run rcedit args.push_front(rcedit_path); - OS::get_singleton()->execute(wine_path, args); + rcedit_path = wine_path; #endif + + String str; + Error err = OS::get_singleton()->execute(rcedit_path, args, &str, nullptr, true); + ERR_FAIL_COND_V(err != OK, err); + print_line("rcedit (" + p_path + "): " + str); + + if (str.find("Fatal error") != -1) { + return FAILED; + } + + return OK; } Error EditorExportPlatformWindows::_code_sign(const Ref &p_preset, const String &p_path) { @@ -417,6 +437,10 @@ bool EditorExportPlatformWindows::can_export(const Ref &p_pr Error EditorExportPlatformWindows::fixup_embedded_pck(const String &p_path, int64_t p_embedded_start, int64_t p_embedded_size) const { // Patch the header of the "pck" section in the PE file so that it corresponds to the embedded data + if (p_embedded_size + p_embedded_start >= 0x100000000) { // Check for total executable size + ERR_FAIL_V_MSG(ERR_INVALID_DATA, "Windows executables cannot be >= 4 GiB."); + } + Ref f = FileAccess::open(p_path, FileAccess::READ_WRITE); if (f.is_null()) { return ERR_CANT_OPEN; diff --git a/platform/windows/export/export_plugin.h b/platform/windows/export/export_plugin.h index 39d1cf4c77..c33c7f1f63 100644 --- a/platform/windows/export/export_plugin.h +++ b/platform/windows/export/export_plugin.h @@ -38,7 +38,7 @@ #include "platform/windows/logo.gen.h" class EditorExportPlatformWindows : public EditorExportPlatformPC { - void _rcedit_add_data(const Ref &p_preset, const String &p_path); + Error _rcedit_add_data(const Ref &p_preset, const String &p_path); Error _code_sign(const Ref &p_preset, const String &p_path); Error _export_debug_script(const Ref &p_preset, const String &p_app_name, const String &p_pkg_name, const String &p_path); diff --git a/platform/windows/godot_windows.cpp b/platform/windows/godot_windows.cpp index ad4e3ae77c..8de3ef294a 100644 --- a/platform/windows/godot_windows.cpp +++ b/platform/windows/godot_windows.cpp @@ -39,7 +39,18 @@ #ifndef TOOLS_ENABLED #if defined _MSC_VER #pragma section("pck", read) -__declspec(allocate("pck")) static const char dummy[8] = { 0 }; +__declspec(allocate("pck")) static char dummy[8] = { 0 }; + +// Dummy function to prevent LTO from discarding "pck" section. +extern "C" char *__cdecl pck_section_dummy_call() { + return &dummy[0]; +}; +#if defined _AMD64_ +#pragma comment(linker, "/include:pck_section_dummy_call") +#elif defined _X86_ +#pragma comment(linker, "/include:_pck_section_dummy_call") +#endif + #elif defined __GNUC__ static const char dummy[8] __attribute__((section("pck"), used)) = { 0 }; #endif @@ -140,11 +151,6 @@ int widechar_main(int argc, wchar_t **argv) { setlocale(LC_CTYPE, ""); -#ifndef TOOLS_ENABLED - // Workaround to prevent LTCG (MSVC LTO) from removing "pck" section - const char *dummy_guard = dummy; -#endif - char **argv_utf8 = new char *[argc]; for (int i = 0; i < argc; ++i) { diff --git a/platform/windows/os_windows.cpp b/platform/windows/os_windows.cpp index b4669e452a..8755bc65dc 100644 --- a/platform/windows/os_windows.cpp +++ b/platform/windows/os_windows.cpp @@ -686,6 +686,58 @@ MainLoop *OS_Windows::get_main_loop() const { return main_loop; } +uint64_t OS_Windows::get_embedded_pck_offset() const { + Ref f = FileAccess::open(get_executable_path(), FileAccess::READ); + if (f.is_null()) { + return 0; + } + + // Process header. + { + f->seek(0x3c); + uint32_t pe_pos = f->get_32(); + + f->seek(pe_pos); + uint32_t magic = f->get_32(); + if (magic != 0x00004550) { + return 0; + } + } + + int num_sections; + { + int64_t header_pos = f->get_position(); + + f->seek(header_pos + 2); + num_sections = f->get_16(); + f->seek(header_pos + 16); + uint16_t opt_header_size = f->get_16(); + + // Skip rest of header + optional header to go to the section headers. + f->seek(f->get_position() + 2 + opt_header_size); + } + int64_t section_table_pos = f->get_position(); + + // Search for the "pck" section. + int64_t off = 0; + for (int i = 0; i < num_sections; ++i) { + int64_t section_header_pos = section_table_pos + i * 40; + f->seek(section_header_pos); + + uint8_t section_name[9]; + f->get_buffer(section_name, 8); + section_name[8] = '\0'; + + if (strcmp((char *)section_name, "pck") == 0) { + f->seek(section_header_pos + 20); + off = f->get_32(); + break; + } + } + + return off; +} + String OS_Windows::get_config_path() const { // The XDG Base Directory specification technically only applies on Linux/*BSD, but it doesn't hurt to support it on Windows as well. if (has_environment("XDG_CONFIG_HOME")) { diff --git a/platform/windows/os_windows.h b/platform/windows/os_windows.h index adeecf37c5..370cb77fde 100644 --- a/platform/windows/os_windows.h +++ b/platform/windows/os_windows.h @@ -144,6 +144,8 @@ public: virtual int get_processor_count() const override; virtual String get_processor_name() const override; + virtual uint64_t get_embedded_pck_offset() const override; + virtual String get_config_path() const override; virtual String get_data_path() const override; virtual String get_cache_path() const override; -- cgit v1.2.3