diff options
Diffstat (limited to 'editor/editor_file_system.cpp')
-rw-r--r-- | editor/editor_file_system.cpp | 647 |
1 files changed, 477 insertions, 170 deletions
diff --git a/editor/editor_file_system.cpp b/editor/editor_file_system.cpp index a5edcf5c22..8523833d52 100644 --- a/editor/editor_file_system.cpp +++ b/editor/editor_file_system.cpp @@ -5,8 +5,8 @@ /* GODOT ENGINE */ /* https://godotengine.org */ /*************************************************************************/ -/* Copyright (c) 2007-2020 Juan Linietsky, Ariel Manzur. */ -/* Copyright (c) 2014-2020 Godot Engine contributors (cf. AUTHORS.md). */ +/* Copyright (c) 2007-2021 Juan Linietsky, Ariel Manzur. */ +/* Copyright (c) 2014-2021 Godot Engine contributors (cf. AUTHORS.md). */ /* */ /* Permission is hereby granted, free of charge, to any person obtaining */ /* a copy of this software and associated documentation files (the */ @@ -30,20 +30,21 @@ #include "editor_file_system.h" +#include "core/config/project_settings.h" +#include "core/extension/native_extension_manager.h" +#include "core/io/file_access.h" #include "core/io/resource_importer.h" #include "core/io/resource_loader.h" #include "core/io/resource_saver.h" -#include "core/os/file_access.h" #include "core/os/os.h" -#include "core/project_settings.h" -#include "core/variant_parser.h" +#include "core/variant/variant_parser.h" #include "editor_node.h" #include "editor_resource_preview.h" #include "editor_settings.h" EditorFileSystem *EditorFileSystem::singleton = nullptr; //the name is the version, to keep compatibility with different versions of Godot -#define CACHE_FILE_NAME "filesystem_cache6" +#define CACHE_FILE_NAME "filesystem_cache7" void EditorFileSystemDirectory::sort_files() { files.sort_custom<FileInfoSort>(); @@ -68,6 +69,11 @@ int EditorFileSystemDirectory::find_dir_index(const String &p_dir) const { return -1; } +void EditorFileSystemDirectory::force_update() { + // We set modified_time to 0 to force `EditorFileSystem::_scan_fs_changes` to search changes in the directory + modified_time = 0; +} + int EditorFileSystemDirectory::get_subdir_count() const { return subdirs.size(); } @@ -111,7 +117,26 @@ String EditorFileSystemDirectory::get_file_path(int p_idx) const { Vector<String> EditorFileSystemDirectory::get_file_deps(int p_idx) const { ERR_FAIL_INDEX_V(p_idx, files.size(), Vector<String>()); - return files[p_idx]->deps; + Vector<String> deps; + + for (int i = 0; i < files[p_idx]->deps.size(); i++) { + String dep = files[p_idx]->deps[i]; + int sep_idx = dep.find("::"); //may contain type information, unwanted + if (sep_idx != -1) { + dep = dep.substr(0, sep_idx); + } + ResourceUID::ID uid = ResourceUID::get_singleton()->text_to_id(dep); + if (uid != ResourceUID::INVALID_ID) { + //return proper dependency resource from uid + if (ResourceUID::get_singleton()->has_id(uid)) { + dep = ResourceUID::get_singleton()->get_id_path(uid); + } else { + continue; + } + } + deps.push_back(dep); + } + return deps; } bool EditorFileSystemDirectory::get_file_import_is_valid(int p_idx) const { @@ -119,6 +144,11 @@ bool EditorFileSystemDirectory::get_file_import_is_valid(int p_idx) const { return files[p_idx]->import_valid; } +uint64_t EditorFileSystemDirectory::get_file_modified_time(int p_idx) const { + ERR_FAIL_INDEX_V(p_idx, files.size(), 0); + return files[p_idx]->modified_time; +} + String EditorFileSystemDirectory::get_file_script_class_name(int p_idx) const { return files[p_idx]->script_class_name; } @@ -224,7 +254,7 @@ void EditorFileSystem::_scan_filesystem() { } else { Vector<String> split = l.split("::"); - ERR_CONTINUE(split.size() != 8); + ERR_CONTINUE(split.size() != 9); String name = split[0]; String file; @@ -233,15 +263,16 @@ void EditorFileSystem::_scan_filesystem() { FileCache fc; fc.type = split[1]; - fc.modification_time = split[2].to_int(); - fc.import_modification_time = split[3].to_int(); - fc.import_valid = split[4].to_int() != 0; - fc.import_group_file = split[5].strip_edges(); - fc.script_class_name = split[6].get_slice("<>", 0); - fc.script_class_extends = split[6].get_slice("<>", 1); - fc.script_class_icon_path = split[6].get_slice("<>", 2); - - String deps = split[7].strip_edges(); + fc.uid = split[2].to_int(); + fc.modification_time = split[3].to_int(); + fc.import_modification_time = split[4].to_int(); + fc.import_valid = split[5].to_int() != 0; + fc.import_group_file = split[6].strip_edges(); + fc.script_class_name = split[7].get_slice("<>", 0); + fc.script_class_extends = split[7].get_slice("<>", 1); + fc.script_class_icon_path = split[7].get_slice("<>", 2); + + String deps = split[8].strip_edges(); if (deps.length()) { Vector<String> dp = deps.split("<>"); for (int i = 0; i < dp.size(); i++) { @@ -352,10 +383,13 @@ bool EditorFileSystem::_test_for_reimport(const String &p_path, bool p_only_impo List<String> to_check; + String importer_name; String source_file = ""; String source_md5 = ""; Vector<String> dest_files; String dest_md5 = ""; + int version = 0; + bool found_uid = false; while (true) { assign = Variant(); @@ -379,6 +413,12 @@ bool EditorFileSystem::_test_for_reimport(const String &p_path, bool p_only_impo for (int i = 0; i < fa.size(); i++) { to_check.push_back(fa[i]); } + } else if (assign == "importer_version") { + version = value; + } else if (assign == "importer") { + importer_name = value; + } else if (assign == "uid") { + found_uid = true; } else if (!p_only_imported_files) { if (assign == "source_file") { source_file = value; @@ -394,6 +434,24 @@ bool EditorFileSystem::_test_for_reimport(const String &p_path, bool p_only_impo memdelete(f); + if (importer_name == "keep") { + return false; //keep mode, do not reimport + } + + if (!found_uid) { + return true; //UUID not found, old format, reimport. + } + + Ref<ResourceImporter> importer = ResourceFormatImporter::get_singleton()->get_importer_by_name(importer_name); + + if (importer.is_null()) { + return true; // the importer has possibly changed, try to reimport. + } + + if (importer->get_format_version() > version) { + return true; // version changed, reimport + } + // Read the md5's from a separate file (so the import parameters aren't dependent on the file version String base_path = ResourceFormatImporter::get_singleton()->get_import_base_path(p_path); FileAccess *md5s = FileAccess::open(base_path + ".md5", FileAccess::READ, &err); @@ -431,8 +489,8 @@ bool EditorFileSystem::_test_for_reimport(const String &p_path, bool p_only_impo memdelete(md5s); //imported files are gone, reimport - for (List<String>::Element *E = to_check.front(); E; E = E->next()) { - if (!FileAccess::exists(E->get())) { + for (const String &E : to_check) { + if (!FileAccess::exists(E)) { return true; } } @@ -471,16 +529,14 @@ bool EditorFileSystem::_update_scan_actions() { Vector<String> reimports; Vector<String> reloads; - for (List<ItemAction>::Element *E = scan_actions.front(); E; E = E->next()) { - ItemAction &ia = E->get(); - + for (const ItemAction &ia : scan_actions) { switch (ia.action) { case ItemAction::ACTION_NONE: { } break; case ItemAction::ACTION_DIR_ADD: { int idx = 0; for (int i = 0; i < ia.dir->subdirs.size(); i++) { - if (ia.new_dir->name < ia.dir->subdirs[i]->name) { + if (ia.new_dir->name.naturalnocasecmp_to(ia.dir->subdirs[i]->name) < 0) { break; } idx++; @@ -502,7 +558,7 @@ bool EditorFileSystem::_update_scan_actions() { case ItemAction::ACTION_FILE_ADD: { int idx = 0; for (int i = 0; i < ia.dir->files.size(); i++) { - if (ia.new_file->file < ia.dir->files[i]->file) { + if (ia.new_file->file.naturalnocasecmp_to(ia.dir->files[i]->file) < 0) { break; } idx++; @@ -554,8 +610,23 @@ bool EditorFileSystem::_update_scan_actions() { } } + if (_scan_extensions()) { + //needs editor restart + //extensions also may provide filetypes to be imported, so they must run before importing + if (EditorNode::immediate_confirmation_dialog(TTR("Some extensions need the editor to restart to take effect."), first_scan ? TTR("Restart") : TTR("Save&Restart"), TTR("Continue"))) { + if (!first_scan) { + EditorNode::get_singleton()->save_all_scenes(); + } + EditorNode::get_singleton()->restart_editor(); + //do not import + return true; + } + } if (reimports.size()) { reimport_files(reimports); + } else { + //reimport files will update the uid cache file so if nothing was reimported, update it manually + ResourceUID::get_singleton()->update_cache(); } if (first_scan) { @@ -566,7 +637,7 @@ bool EditorFileSystem::_update_scan_actions() { } if (reloads.size()) { - emit_signal("resources_reload", reloads); + emit_signal(SNAME("resources_reload"), reloads); } scan_actions.clear(); @@ -578,7 +649,7 @@ void EditorFileSystem::scan() { return; } - if (scanning || scanning_changes || thread) { + if (scanning || scanning_changes || thread.is_started()) { return; } @@ -597,18 +668,18 @@ void EditorFileSystem::scan() { new_filesystem = nullptr; _update_scan_actions(); scanning = false; - emit_signal("filesystem_changed"); - emit_signal("sources_changed", sources_changed.size() > 0); + emit_signal(SNAME("filesystem_changed")); + emit_signal(SNAME("sources_changed"), sources_changed.size() > 0); _queue_update_script_classes(); first_scan = false; } else { - ERR_FAIL_COND(thread); + ERR_FAIL_COND(thread.is_started()); set_process(true); Thread::Settings s; scanning = true; scan_total = 0; s.priority = Thread::PRIORITY_LOW; - thread = Thread::create(_thread_func, this, s); + thread.start(_thread_func, this, s); //tree->hide(); //progress->show(); } @@ -652,10 +723,7 @@ void EditorFileSystem::_scan_new_dir(EditorFileSystemDirectory *p_dir, DirAccess continue; } - if (FileAccess::exists(cd.plus_file(f).plus_file("project.godot"))) { // skip if another project inside this - continue; - } - if (FileAccess::exists(cd.plus_file(f).plus_file(".gdignore"))) { // skip if another project inside this + if (_should_skip_directory(cd.plus_file(f))) { continue; } @@ -690,7 +758,7 @@ void EditorFileSystem::_scan_new_dir(EditorFileSystemDirectory *p_dir, DirAccess int idx2 = 0; for (int i = 0; i < p_dir->subdirs.size(); i++) { - if (efd->name < p_dir->subdirs[i]->name) { + if (efd->name.naturalnocasecmp_to(p_dir->subdirs[i]->name) < 0) { break; } idx2++; @@ -733,6 +801,7 @@ void EditorFileSystem::_scan_new_dir(EditorFileSystemDirectory *p_dir, DirAccess if (fc && fc->modification_time == mt && fc->import_modification_time == import_mt && !_test_for_reimport(path, true)) { fi->type = fc->type; + fi->uid = fc->uid; fi->deps = fc->deps; fi->modified_time = fc->modification_time; fi->import_modified_time = fc->import_modification_time; @@ -758,8 +827,14 @@ void EditorFileSystem::_scan_new_dir(EditorFileSystemDirectory *p_dir, DirAccess //note: I think this should not happen any longer.. } + if (fc->uid == ResourceUID::INVALID_ID) { + // imported files should always have a UUID, so attempt to fetch it. + fi->uid = ResourceLoader::get_resource_uid(path); + } + } else { fi->type = ResourceFormatImporter::get_singleton()->get_resource_type(path); + fi->uid = ResourceFormatImporter::get_singleton()->get_resource_uid(path); fi->import_group_file = ResourceFormatImporter::get_singleton()->get_import_group_file(path); fi->script_class_name = _get_global_script_class(fi->type, path, &fi->script_class_extends, &fi->script_class_icon_path); fi->modified_time = 0; @@ -776,6 +851,7 @@ void EditorFileSystem::_scan_new_dir(EditorFileSystemDirectory *p_dir, DirAccess if (fc && fc->modification_time == mt) { //not imported, so just update type if changed fi->type = fc->type; + fi->uid = fc->uid; fi->modified_time = fc->modification_time; fi->deps = fc->deps; fi->import_modified_time = 0; @@ -786,6 +862,7 @@ void EditorFileSystem::_scan_new_dir(EditorFileSystemDirectory *p_dir, DirAccess } else { //new or modified time fi->type = ResourceLoader::get_resource_type(path); + fi->uid = ResourceLoader::get_resource_uid(path); fi->script_class_name = _get_global_script_class(fi->type, path, &fi->script_class_extends, &fi->script_class_icon_path); fi->deps = _get_dependencies(path); fi->modified_time = mt; @@ -794,6 +871,28 @@ void EditorFileSystem::_scan_new_dir(EditorFileSystemDirectory *p_dir, DirAccess } } + if (fi->uid != ResourceUID::INVALID_ID) { + if (ResourceUID::get_singleton()->has_id(fi->uid)) { + ResourceUID::get_singleton()->set_id(fi->uid, path); + } else { + ResourceUID::get_singleton()->add_id(fi->uid, path); + } + } + + for (int i = 0; i < ScriptServer::get_language_count(); i++) { + ScriptLanguage *lang = ScriptServer::get_language(i); + if (lang->supports_documentation() && fi->type == lang->get_type()) { + Ref<Script> script = ResourceLoader::load(path); + if (script == nullptr) { + continue; + } + const Vector<DocData::ClassDoc> &docs = script->get_documentation(); + for (int j = 0; j < docs.size(); j++) { + EditorHelp::get_doc_data()->add_doc(docs[j]); + } + } + } + p_dir->files.push_back(fi); p_progress.update(idx, total); } @@ -822,9 +921,11 @@ void EditorFileSystem::_scan_fs_changes(EditorFileSystemDirectory *p_dir, const //then scan files and directories and check what's different - DirAccess *da = DirAccess::create(DirAccess::ACCESS_RESOURCES); + DirAccessRef da = DirAccess::create(DirAccess::ACCESS_RESOURCES); + + Error ret = da->change_dir(cd); + ERR_FAIL_COND_MSG(ret != OK, "Cannot change to '" + cd + "' folder."); - da->change_dir(cd); da->list_dir_begin(); while (true) { String f = da->get_next(); @@ -843,10 +944,7 @@ void EditorFileSystem::_scan_fs_changes(EditorFileSystemDirectory *p_dir, const int idx = p_dir->find_dir_index(f); if (idx == -1) { - if (FileAccess::exists(cd.plus_file(f).plus_file("project.godot"))) { // skip if another project inside this - continue; - } - if (FileAccess::exists(cd.plus_file(f).plus_file(".gdignore"))) { // skip if another project inside this + if (_should_skip_directory(cd.plus_file(f))) { continue; } @@ -915,7 +1013,6 @@ void EditorFileSystem::_scan_fs_changes(EditorFileSystemDirectory *p_dir, const } da->list_dir_end(); - memdelete(da); } for (int i = 0; i < p_dir->files.size(); i++) { @@ -992,8 +1089,8 @@ void EditorFileSystem::_delete_internal_files(String p_file) { List<String> paths; ResourceFormatImporter::get_singleton()->get_internal_resource_path_list(p_file, &paths); DirAccess *da = DirAccess::create(DirAccess::ACCESS_RESOURCES); - for (List<String>::Element *E = paths.front(); E; E = E->next()) { - da->remove(E->get()); + for (const String &E : paths) { + da->remove(E); } da->remove(p_file + ".import"); memdelete(da); @@ -1013,13 +1110,9 @@ void EditorFileSystem::_thread_func_sources(void *_userdata) { efs->scanning_changes_done = true; } -void EditorFileSystem::get_changed_sources(List<String> *r_changed) { - *r_changed = sources_changed; -} - void EditorFileSystem::scan_changes() { if (first_scan || // Prevent a premature changes scan from inhibiting the first full scan - scanning || scanning_changes || thread) { + scanning || scanning_changes || thread.is_started()) { scan_changes_pending = true; set_process(true); return; @@ -1042,40 +1135,37 @@ void EditorFileSystem::scan_changes() { scan_total = 0; _scan_fs_changes(filesystem, sp); if (_update_scan_actions()) { - emit_signal("filesystem_changed"); + emit_signal(SNAME("filesystem_changed")); } } scanning_changes = false; scanning_changes_done = true; - emit_signal("sources_changed", sources_changed.size() > 0); + emit_signal(SNAME("sources_changed"), sources_changed.size() > 0); } else { - ERR_FAIL_COND(thread_sources); + ERR_FAIL_COND(thread_sources.is_started()); set_process(true); scan_total = 0; Thread::Settings s; s.priority = Thread::PRIORITY_LOW; - thread_sources = Thread::create(_thread_func_sources, this, s); + thread_sources.start(_thread_func_sources, this, s); } } void EditorFileSystem::_notification(int p_what) { switch (p_what) { case NOTIFICATION_ENTER_TREE: { - call_deferred("scan"); //this should happen after every editor node entered the tree + call_deferred(SNAME("scan")); //this should happen after every editor node entered the tree } break; case NOTIFICATION_EXIT_TREE: { - Thread *active_thread = thread ? thread : thread_sources; - if (use_threads && active_thread) { + Thread &active_thread = thread.is_started() ? thread : thread_sources; + if (use_threads && active_thread.is_started()) { //abort thread if in progress abort_scan = true; while (scanning) { OS::get_singleton()->delay_usec(1000); } - Thread::wait_to_finish(active_thread); - memdelete(active_thread); - thread = nullptr; - thread_sources = nullptr; + active_thread.wait_to_finish(); WARN_PRINT("Scan thread aborted..."); set_process(false); } @@ -1098,17 +1188,15 @@ void EditorFileSystem::_notification(int p_what) { set_process(false); - Thread::wait_to_finish(thread_sources); - memdelete(thread_sources); - thread_sources = nullptr; + thread_sources.wait_to_finish(); if (_update_scan_actions()) { - emit_signal("filesystem_changed"); + emit_signal(SNAME("filesystem_changed")); } - emit_signal("sources_changed", sources_changed.size() > 0); + emit_signal(SNAME("sources_changed"), sources_changed.size() > 0); _queue_update_script_classes(); first_scan = false; } - } else if (!scanning && thread) { + } else if (!scanning && thread.is_started()) { set_process(false); if (filesystem) { @@ -1116,12 +1204,10 @@ void EditorFileSystem::_notification(int p_what) { } filesystem = new_filesystem; new_filesystem = nullptr; - Thread::wait_to_finish(thread); - memdelete(thread); - thread = nullptr; + thread.wait_to_finish(); _update_scan_actions(); - emit_signal("filesystem_changed"); - emit_signal("sources_changed", sources_changed.size() > 0); + emit_signal(SNAME("filesystem_changed")); + emit_signal(SNAME("sources_changed"), sources_changed.size() > 0); _queue_update_script_classes(); first_scan = false; } @@ -1157,7 +1243,7 @@ void EditorFileSystem::_save_filesystem_cache(EditorFileSystemDirectory *p_dir, if (p_dir->files[i]->import_group_file != String()) { group_file_cache.insert(p_dir->files[i]->import_group_file); } - String s = p_dir->files[i]->file + "::" + p_dir->files[i]->type + "::" + itos(p_dir->files[i]->modified_time) + "::" + itos(p_dir->files[i]->import_modified_time) + "::" + itos(p_dir->files[i]->import_valid) + "::" + p_dir->files[i]->import_group_file + "::" + p_dir->files[i]->script_class_name + "<>" + p_dir->files[i]->script_class_extends + "<>" + p_dir->files[i]->script_class_icon_path; + String s = p_dir->files[i]->file + "::" + p_dir->files[i]->type + "::" + itos(p_dir->files[i]->uid) + "::" + itos(p_dir->files[i]->modified_time) + "::" + itos(p_dir->files[i]->import_modified_time) + "::" + itos(p_dir->files[i]->import_valid) + "::" + p_dir->files[i]->import_group_file + "::" + p_dir->files[i]->script_class_name + "<>" + p_dir->files[i]->script_class_extends + "<>" + p_dir->files[i]->script_class_icon_path; s += "::"; for (int j = 0; j < p_dir->files[i]->deps.size(); j++) { if (j > 0) { @@ -1221,7 +1307,7 @@ bool EditorFileSystem::_find_file(const String &p_file, EditorFileSystemDirector int idx2 = 0; for (int j = 0; j < fs->get_subdir_count(); j++) { - if (efsd->name < fs->get_subdir(j)->get_name()) { + if (efsd->name.naturalnocasecmp_to(fs->get_subdir(j)->get_name()) < 0) { break; } idx2++; @@ -1344,8 +1430,8 @@ Vector<String> EditorFileSystem::_get_dependencies(const String &p_path) { ResourceLoader::get_dependencies(p_path, &deps); Vector<String> ret; - for (List<String>::Element *E = deps.front(); E; E = E->next()) { - ret.push_back(E->get()); + for (const String &E : deps) { + ret.push_back(E); } return ret; @@ -1393,11 +1479,11 @@ void EditorFileSystem::_scan_script_classes(EditorFileSystemDirectory *p_dir) { } void EditorFileSystem::update_script_classes() { - if (!update_script_classes_queued) { + if (!update_script_classes_queued.is_set()) { return; } - update_script_classes_queued = false; + update_script_classes_queued.clear(); ScriptServer::global_classes_clear(); if (get_filesystem()) { _scan_script_classes(get_filesystem()); @@ -1416,12 +1502,12 @@ void EditorFileSystem::update_script_classes() { } void EditorFileSystem::_queue_update_script_classes() { - if (update_script_classes_queued) { + if (update_script_classes_queued.is_set()) { return; } - update_script_classes_queued = true; - call_deferred("update_script_classes"); + update_script_classes_queued.set(); + call_deferred(SNAME("update_script_classes")); } void EditorFileSystem::update_file(const String &p_file) { @@ -1438,32 +1524,37 @@ void EditorFileSystem::update_file(const String &p_file) { //was removed _delete_internal_files(p_file); if (cpos != -1) { // Might've never been part of the editor file system (*.* files deleted in Open dialog). + if (fs->files[cpos]->uid != ResourceUID::INVALID_ID) { + if (ResourceUID::get_singleton()->has_id(fs->files[cpos]->uid)) { + ResourceUID::get_singleton()->remove_id(fs->files[cpos]->uid); + } + } memdelete(fs->files[cpos]); fs->files.remove(cpos); } - call_deferred("emit_signal", "filesystem_changed"); //update later + call_deferred(SNAME("emit_signal"), "filesystem_changed"); //update later _queue_update_script_classes(); return; } String type = ResourceLoader::get_resource_type(p_file); + ResourceUID::ID uid = ResourceLoader::get_resource_uid(p_file); if (cpos == -1) { - //the file did not exist, it was added - - late_added_files.insert(p_file); //remember that it was added. This mean it will be scanned and imported on editor restart + // The file did not exist, it was added. int idx = 0; + String file_name = p_file.get_file(); for (int i = 0; i < fs->files.size(); i++) { - if (p_file < fs->files[i]->file) { + if (p_file.naturalnocasecmp_to(fs->files[i]->file) < 0) { break; } idx++; } EditorFileSystemDirectory::FileInfo *fi = memnew(EditorFileSystemDirectory::FileInfo); - fi->file = p_file.get_file(); + fi->file = file_name; fi->import_modified_time = 0; fi->import_valid = ResourceLoader::is_import_valid(p_file); @@ -1481,16 +1572,26 @@ void EditorFileSystem::update_file(const String &p_file) { } fs->files[cpos]->type = type; + fs->files[cpos]->uid = uid; fs->files[cpos]->script_class_name = _get_global_script_class(type, p_file, &fs->files[cpos]->script_class_extends, &fs->files[cpos]->script_class_icon_path); fs->files[cpos]->import_group_file = ResourceLoader::get_import_group_file(p_file); fs->files[cpos]->modified_time = FileAccess::get_modified_time(p_file); fs->files[cpos]->deps = _get_dependencies(p_file); fs->files[cpos]->import_valid = ResourceLoader::is_import_valid(p_file); + if (uid != ResourceUID::INVALID_ID) { + if (ResourceUID::get_singleton()->has_id(uid)) { + ResourceUID::get_singleton()->set_id(uid, p_file); + } else { + ResourceUID::get_singleton()->add_id(uid, p_file); + } + + ResourceUID::get_singleton()->update_cache(); + } // Update preview EditorResourcePreview::get_singleton()->check_for_invalidation(p_file); - call_deferred("emit_signal", "filesystem_changed"); //update later + call_deferred(SNAME("emit_signal"), "filesystem_changed"); //update later _queue_update_script_classes(); } @@ -1501,7 +1602,7 @@ Error EditorFileSystem::_reimport_group(const String &p_group_file, const Vector Map<String, String> base_paths; for (int i = 0; i < p_files.size(); i++) { Ref<ConfigFile> config; - config.instance(); + config.instantiate(); Error err = config->load(p_files[i] + ".import"); ERR_CONTINUE(err != OK); ERR_CONTINUE(!config->has_section_key("remap", "importer")); @@ -1509,7 +1610,6 @@ Error EditorFileSystem::_reimport_group(const String &p_group_file, const Vector ERR_CONTINUE(file_importer_name == String()); if (importer_name != String() && importer_name != file_importer_name) { - print_line("one importer '" + importer_name + "' the other '" + file_importer_name + "'."); EditorNode::get_singleton()->show_warning(vformat(TTR("There are multiple importers for different types pointing to file %s, import aborted"), p_group_file)); ERR_FAIL_V(ERR_FILE_CORRUPT); } @@ -1517,20 +1617,23 @@ Error EditorFileSystem::_reimport_group(const String &p_group_file, const Vector source_file_options[p_files[i]] = Map<StringName, Variant>(); importer_name = file_importer_name; + if (importer_name == "keep") { + continue; //do nothing + } + Ref<ResourceImporter> importer = ResourceFormatImporter::get_singleton()->get_importer_by_name(importer_name); ERR_FAIL_COND_V(!importer.is_valid(), ERR_FILE_CORRUPT); List<ResourceImporter::ImportOption> options; importer->get_import_options(&options); //set default values - for (List<ResourceImporter::ImportOption>::Element *E = options.front(); E; E = E->next()) { - source_file_options[p_files[i]][E->get().option.name] = E->get().default_value; + for (const ResourceImporter::ImportOption &E : options) { + source_file_options[p_files[i]][E.option.name] = E.default_value; } if (config->has_section("params")) { List<String> sk; config->get_section_keys("params", &sk); - for (List<String>::Element *E = sk.front(); E; E = E->next()) { - String param = E->get(); + for (const String ¶m : sk) { Variant value = config->get_value("params", param); //override with whathever is in file source_file_options[p_files[i]][param] = value; @@ -1540,6 +1643,10 @@ Error EditorFileSystem::_reimport_group(const String &p_group_file, const Vector base_paths[p_files[i]] = ResourceFormatImporter::get_singleton()->get_import_base_path(p_files[i]); } + if (importer_name == "keep") { + return OK; // (do nothing) + } + ERR_FAIL_COND_V(importer_name == String(), ERR_UNCONFIGURED); Ref<ResourceImporter> importer = ResourceFormatImporter::get_singleton()->get_importer_by_name(importer_name); @@ -1557,6 +1664,10 @@ Error EditorFileSystem::_reimport_group(const String &p_group_file, const Vector f->store_line("[remap]"); f->store_line(""); f->store_line("importer=\"" + importer->get_importer_name() + "\""); + int version = importer->get_format_version(); + if (version > 0) { + f->store_line("importer_version=" + itos(version)); + } if (importer->get_resource_type() != "") { f->store_line("type=\"" + importer->get_resource_type() + "\""); } @@ -1596,9 +1707,9 @@ Error EditorFileSystem::_reimport_group(const String &p_group_file, const Vector List<ResourceImporter::ImportOption> options; importer->get_import_options(&options); //set default values - for (List<ResourceImporter::ImportOption>::Element *F = options.front(); F; F = F->next()) { - String base = F->get().option.name; - Variant v = F->get().default_value; + for (const ResourceImporter::ImportOption &F : options) { + String base = F.option.name; + Variant v = F.default_value; if (source_file_options[file].has(base)) { v = source_file_options[file][base]; } @@ -1649,7 +1760,7 @@ Error EditorFileSystem::_reimport_group(const String &p_group_file, const Vector return err; } -void EditorFileSystem::_reimport_file(const String &p_file) { +void EditorFileSystem::_reimport_file(const String &p_file, const Map<StringName, Variant> *p_custom_options, const String &p_custom_importer) { EditorFileSystemDirectory *fs = nullptr; int cpos = -1; bool found = _find_file(p_file, &fs, cpos); @@ -1658,30 +1769,56 @@ void EditorFileSystem::_reimport_file(const String &p_file) { //try to obtain existing params Map<StringName, Variant> params; - String importer_name; + String importer_name; //empty by default though + + if (p_custom_importer != String()) { + importer_name = p_custom_importer; + } + if (p_custom_options != nullptr) { + params = *p_custom_options; + } + + ResourceUID::ID uid = ResourceUID::INVALID_ID; if (FileAccess::exists(p_file + ".import")) { //use existing - Ref<ConfigFile> cf; - cf.instance(); - Error err = cf->load(p_file + ".import"); - if (err == OK) { - if (cf->has_section("params")) { - List<String> sk; - cf->get_section_keys("params", &sk); - for (List<String>::Element *E = sk.front(); E; E = E->next()) { - params[E->get()] = cf->get_value("params", E->get()); + if (p_custom_options == nullptr) { + Ref<ConfigFile> cf; + cf.instantiate(); + Error err = cf->load(p_file + ".import"); + if (err == OK) { + if (cf->has_section("params")) { + List<String> sk; + cf->get_section_keys("params", &sk); + for (const String &E : sk) { + params[E] = cf->get_value("params", E); + } + } + + if (cf->has_section("remap")) { + if (p_custom_importer == String()) { + importer_name = cf->get_value("remap", "importer"); + } + + if (cf->has_section_key("remap", "uid")) { + String uidt = cf->get_value("remap", "uid"); + uid = ResourceUID::get_singleton()->text_to_id(uidt); + } } - } - if (cf->has_section("remap")) { - importer_name = cf->get_value("remap", "importer"); } } - - } else { - late_added_files.insert(p_file); //imported files do not call update_file(), but just in case.. } + if (importer_name == "keep") { + //keep files, do nothing. + fs->files[cpos]->modified_time = FileAccess::get_modified_time(p_file); + fs->files[cpos]->import_modified_time = FileAccess::get_modified_time(p_file + ".import"); + fs->files[cpos]->deps.clear(); + fs->files[cpos]->type = ""; + fs->files[cpos]->import_valid = false; + EditorResourcePreview::get_singleton()->check_for_invalidation(p_file); + return; + } Ref<ResourceImporter> importer; bool load_default = false; //find the importer @@ -1694,7 +1831,7 @@ void EditorFileSystem::_reimport_file(const String &p_file) { importer = ResourceFormatImporter::get_singleton()->get_importer_by_extension(p_file.get_extension()); load_default = true; if (importer.is_null()) { - ERR_PRINT("BUG: File queued for import, but can't be imported!"); + ERR_PRINT("BUG: File queued for import, but can't be imported, importer for type '" + importer_name + "' not found."); ERR_FAIL(); } } @@ -1703,9 +1840,9 @@ void EditorFileSystem::_reimport_file(const String &p_file) { List<ResourceImporter::ImportOption> opts; importer->get_import_options(&opts); - for (List<ResourceImporter::ImportOption>::Element *E = opts.front(); E; E = E->next()) { - if (!params.has(E->get().option.name)) { //this one is not present - params[E->get().option.name] = E->get().default_value; + for (const ResourceImporter::ImportOption &E : opts) { + if (!params.has(E.option.name)) { //this one is not present + params[E.option.name] = E.default_value; } } @@ -1715,8 +1852,8 @@ void EditorFileSystem::_reimport_file(const String &p_file) { List<Variant> v; d.get_key_list(&v); - for (List<Variant>::Element *E = v.front(); E; E = E->next()) { - params[E->get()] = d[E->get()]; + for (const Variant &E : v) { + params[E] = d[E]; } } @@ -1741,10 +1878,20 @@ void EditorFileSystem::_reimport_file(const String &p_file) { f->store_line("[remap]"); f->store_line(""); f->store_line("importer=\"" + importer->get_importer_name() + "\""); + int version = importer->get_format_version(); + if (version > 0) { + f->store_line("importer_version=" + itos(version)); + } if (importer->get_resource_type() != "") { f->store_line("type=\"" + importer->get_resource_type() + "\""); } + if (uid == ResourceUID::INVALID_ID) { + uid = ResourceUID::get_singleton()->create_id(); + } + + f->store_line("uid=\"" + ResourceUID::get_singleton()->id_to_text(uid) + "\""); //store in readable format + Vector<String> dest_paths; if (err == OK) { @@ -1752,10 +1899,10 @@ void EditorFileSystem::_reimport_file(const String &p_file) { //no path } else if (import_variants.size()) { //import with variants - for (List<String>::Element *E = import_variants.front(); E; E = E->next()) { - String path = base_path.c_escape() + "." + E->get() + "." + importer->get_save_extension(); + for (const String &E : import_variants) { + String path = base_path.c_escape() + "." + E + "." + importer->get_save_extension(); - f->store_line("path." + E->get() + "=\"" + path + "\""); + f->store_line("path." + E + "=\"" + path + "\""); dest_paths.push_back(path); } } else { @@ -1778,9 +1925,9 @@ void EditorFileSystem::_reimport_file(const String &p_file) { if (gen_files.size()) { Array genf; - for (List<String>::Element *E = gen_files.front(); E; E = E->next()) { - genf.push_back(E->get()); - dest_paths.push_back(E->get()); + for (const String &E : gen_files) { + genf.push_back(E); + dest_paths.push_back(E); } String value; @@ -1804,8 +1951,8 @@ void EditorFileSystem::_reimport_file(const String &p_file) { //store options in provided order, to avoid file changing. Order is also important because first match is accepted first. - for (List<ResourceImporter::ImportOption>::Element *E = opts.front(); E; E = E->next()) { - String base = E->get().option.name; + for (const ResourceImporter::ImportOption &E : opts) { + String base = E.option.name; String value; VariantWriter::write_to_string(params[base], value); f->store_line(base + "=" + value); @@ -1830,8 +1977,15 @@ void EditorFileSystem::_reimport_file(const String &p_file) { fs->files[cpos]->import_modified_time = FileAccess::get_modified_time(p_file + ".import"); fs->files[cpos]->deps = _get_dependencies(p_file); fs->files[cpos]->type = importer->get_resource_type(); + fs->files[cpos]->uid = uid; fs->files[cpos]->import_valid = ResourceLoader::is_import_valid(p_file); + if (ResourceUID::get_singleton()->has_id(uid)) { + ResourceUID::get_singleton()->set_id(uid, p_file); + } else { + ResourceUID::get_singleton()->add_id(uid, p_file); + } + //if file is currently up, maybe the source it was loaded from changed, so import math must be updated for it //to reload properly if (ResourceCache::has(p_file)) { @@ -1864,31 +2018,36 @@ void EditorFileSystem::_find_group_files(EditorFileSystemDirectory *efd, Map<Str } } -void EditorFileSystem::reimport_files(const Vector<String> &p_files) { - { //check that .import folder exists - DirAccess *da = DirAccess::open("res://"); - if (da->change_dir(".import") != OK) { - Error err = da->make_dir(".import"); - if (err) { - memdelete(da); - ERR_FAIL_MSG("Failed to create 'res://.import' folder."); - } - } - memdelete(da); - } +void EditorFileSystem::reimport_file_with_custom_parameters(const String &p_file, const String &p_importer, const Map<StringName, Variant> &p_custom_params) { + _reimport_file(p_file, &p_custom_params, p_importer); +} +void EditorFileSystem::_reimport_thread(uint32_t p_index, ImportThreadData *p_import_data) { + p_import_data->max_index = MAX(p_import_data->reimport_from + int(p_index), p_import_data->max_index); + _reimport_file(p_import_data->reimport_files[p_import_data->reimport_from + p_index].path); +} + +void EditorFileSystem::reimport_files(const Vector<String> &p_files) { importing = true; EditorProgress pr("reimport", TTR("(Re)Importing Assets"), p_files.size()); - Vector<ImportFile> files; + Vector<ImportFile> reimport_files; + Set<String> groups_to_reimport; for (int i = 0; i < p_files.size(); i++) { - String group_file = ResourceFormatImporter::get_singleton()->get_import_group_file(p_files[i]); + String file = p_files[i]; + + ResourceUID::ID uid = ResourceUID::get_singleton()->text_to_id(file); + if (uid != ResourceUID::INVALID_ID && ResourceUID::get_singleton()->has_id(uid)) { + file = ResourceUID::get_singleton()->get_id_path(uid); + } - if (group_file_cache.has(p_files[i])) { + String group_file = ResourceFormatImporter::get_singleton()->get_import_group_file(file); + + if (group_file_cache.has(file)) { //maybe the file itself is a group! - groups_to_reimport.insert(p_files[i]); + groups_to_reimport.insert(file); //groups do not belong to grups group_file = String(); } else if (group_file != String()) { @@ -1897,24 +2056,64 @@ void EditorFileSystem::reimport_files(const Vector<String> &p_files) { } else { //it's a regular file ImportFile ifile; - ifile.path = p_files[i]; - ifile.order = ResourceFormatImporter::get_singleton()->get_import_order(p_files[i]); - files.push_back(ifile); + ifile.path = file; + ResourceFormatImporter::get_singleton()->get_import_order_threads_and_importer(file, ifile.order, ifile.threaded, ifile.importer); + reimport_files.push_back(ifile); } //group may have changed, so also update group reference EditorFileSystemDirectory *fs = nullptr; int cpos = -1; - if (_find_file(p_files[i], &fs, cpos)) { + if (_find_file(file, &fs, cpos)) { fs->files.write[cpos]->import_group_file = group_file; } } - files.sort(); + reimport_files.sort(); - for (int i = 0; i < files.size(); i++) { - pr.step(files[i].path.get_file(), i); - _reimport_file(files[i].path); + bool use_threads = GLOBAL_GET("editor/import/use_multiple_threads"); + + int from = 0; + for (int i = 0; i < reimport_files.size(); i++) { + if (use_threads && reimport_files[i].threaded) { + if (i + 1 == reimport_files.size() || reimport_files[i + 1].importer != reimport_files[from].importer) { + if (from - i == 0) { + //single file, do not use threads + pr.step(reimport_files[i].path.get_file(), i); + _reimport_file(reimport_files[i].path); + } else { + Ref<ResourceImporter> importer = ResourceFormatImporter::get_singleton()->get_importer_by_name(reimport_files[from].importer); + ERR_CONTINUE(!importer.is_valid()); + + importer->import_threaded_begin(); + + ImportThreadData data; + data.max_index = from; + data.reimport_from = from; + data.reimport_files = reimport_files.ptr(); + + import_threads.begin_work(i - from + 1, this, &EditorFileSystem::_reimport_thread, &data); + int current_index = from - 1; + do { + if (current_index < data.max_index) { + current_index = data.max_index; + pr.step(reimport_files[current_index].path.get_file(), current_index); + } + OS::get_singleton()->delay_usec(1); + } while (!import_threads.is_done_dispatching()); + + import_threads.end_work(); + + importer->import_threaded_end(); + } + + from = i + 1; + } + + } else { + pr.step(reimport_files[i].path.get_file(), i); + _reimport_file(reimport_files[i].path); + } } //reimport groups @@ -1930,13 +2129,15 @@ void EditorFileSystem::reimport_files(const Vector<String> &p_files) { } } + ResourceUID::get_singleton()->update_cache(); //after reimporting, update the cache + _save_filesystem_cache(); importing = false; if (!is_scanning()) { - emit_signal("filesystem_changed"); + emit_signal(SNAME("filesystem_changed")); } - emit_signal("resources_reimported", p_files); + emit_signal(SNAME("resources_reimported"), p_files); } Error EditorFileSystem::_resource_import(const String &p_path) { @@ -1949,6 +2150,20 @@ Error EditorFileSystem::_resource_import(const String &p_path) { return OK; } +bool EditorFileSystem::_should_skip_directory(const String &p_path) { + if (FileAccess::exists(p_path.plus_file("project.godot"))) { + // skip if another project inside this + return true; + } + + if (FileAccess::exists(p_path.plus_file(".gdignore"))) { + // skip if a `.gdignore` file is inside this + return true; + } + + return false; +} + bool EditorFileSystem::is_group_file(const String &p_path) const { return group_file_cache.has(p_path); } @@ -1961,7 +2176,7 @@ void EditorFileSystem::_move_group_files(EditorFileSystemDirectory *efd, const S files[i]->import_group_file = p_new_location; Ref<ConfigFile> config; - config.instance(); + config.instantiate(); String path = efd->get_file_path(i) + ".import"; Error err = config->load(path); if (err != OK) { @@ -1973,9 +2188,8 @@ void EditorFileSystem::_move_group_files(EditorFileSystemDirectory *efd, const S List<String> sk; config->get_section_keys("params", &sk); - for (List<String>::Element *E = sk.front(); E; E = E->next()) { + for (const String ¶m : sk) { //not very clean, but should work - String param = E->get(); String value = config->get_value("params", param); if (value == p_group_file) { config->set_value("params", param, p_new_location); @@ -2001,6 +2215,100 @@ void EditorFileSystem::move_group_file(const String &p_path, const String &p_new } } +ResourceUID::ID EditorFileSystem::_resource_saver_get_resource_id_for_path(const String &p_path, bool p_generate) { + if (!p_path.is_resource_file() || p_path.begins_with("res://.godot")) { + //saved externally (configuration file) or internal file, do not assign an ID. + return ResourceUID::INVALID_ID; + } + + EditorFileSystemDirectory *fs = nullptr; + int cpos = -1; + + if (!singleton->_find_file(p_path, &fs, cpos)) { + if (p_generate) { + return ResourceUID::get_singleton()->create_id(); //just create a new one, we will be notified of save anyway and fetch the right UUID at that time, to keep things simple. + } else { + return ResourceUID::INVALID_ID; + } + } else if (fs->files[cpos]->uid != ResourceUID::INVALID_ID) { + return fs->files[cpos]->uid; + } else if (p_generate) { + return ResourceUID::get_singleton()->create_id(); //just create a new one, we will be notified of save anyway and fetch the right UUID at that time, to keep things simple. + } else { + return ResourceUID::INVALID_ID; + } +} + +static void _scan_extensions_dir(EditorFileSystemDirectory *d, Set<String> &extensions) { + int fc = d->get_file_count(); + for (int i = 0; i < fc; i++) { + if (d->get_file_type(i) == SNAME("NativeExtension")) { + extensions.insert(d->get_file_path(i)); + } + } + int dc = d->get_subdir_count(); + for (int i = 0; i < dc; i++) { + _scan_extensions_dir(d->get_subdir(i), extensions); + } +} +bool EditorFileSystem::_scan_extensions() { + EditorFileSystemDirectory *d = get_filesystem(); + Set<String> extensions; + _scan_extensions_dir(d, extensions); + + //verify against loaded extensions + + Vector<String> extensions_added; + Vector<String> extensions_removed; + + for (const String &E : extensions) { + if (!NativeExtensionManager::get_singleton()->is_extension_loaded(E)) { + extensions_added.push_back(E); + } + } + + Vector<String> loaded_extensions = NativeExtensionManager::get_singleton()->get_loaded_extensions(); + for (int i = 0; i < loaded_extensions.size(); i++) { + if (!extensions.has(loaded_extensions[i])) { + extensions_removed.push_back(loaded_extensions[i]); + } + } + + if (extensions.size()) { + if (extensions_added.size() || extensions_removed.size()) { //extensions were added or removed + FileAccessRef f = FileAccess::open(NativeExtension::EXTENSION_LIST_CONFIG_FILE, FileAccess::WRITE); + for (const String &E : extensions) { + f->store_line(E); + } + } + } else { + if (loaded_extensions.size() || FileAccess::exists(NativeExtension::EXTENSION_LIST_CONFIG_FILE)) { //extensions were removed + DirAccessRef da = DirAccess::create(DirAccess::ACCESS_RESOURCES); + da->remove(NativeExtension::EXTENSION_LIST_CONFIG_FILE); + } + } + + bool needs_restart = false; + for (int i = 0; i < extensions_added.size(); i++) { + NativeExtensionManager::LoadStatus st = NativeExtensionManager::get_singleton()->load_extension(extensions_added[i]); + if (st == NativeExtensionManager::LOAD_STATUS_FAILED) { + EditorNode::get_singleton()->add_io_error("Error loading extension: " + extensions_added[i]); + } else if (st == NativeExtensionManager::LOAD_STATUS_NEEDS_RESTART) { + needs_restart = true; + } + } + for (int i = 0; i < extensions_removed.size(); i++) { + NativeExtensionManager::LoadStatus st = NativeExtensionManager::get_singleton()->unload_extension(extensions_removed[i]); + if (st == NativeExtensionManager::LOAD_STATUS_FAILED) { + EditorNode::get_singleton()->add_io_error("Error removing extension: " + extensions_added[i]); + } else if (st == NativeExtensionManager::LOAD_STATUS_NEEDS_RESTART) { + needs_restart = true; + } + } + + return needs_restart; +} + void EditorFileSystem::_bind_methods() { ClassDB::bind_method(D_METHOD("get_filesystem"), &EditorFileSystem::get_filesystem); ClassDB::bind_method(D_METHOD("is_scanning"), &EditorFileSystem::is_scanning); @@ -2024,50 +2332,49 @@ void EditorFileSystem::_update_extensions() { List<String> extensionsl; ResourceLoader::get_recognized_extensions_for_type("", &extensionsl); - for (List<String>::Element *E = extensionsl.front(); E; E = E->next()) { - valid_extensions.insert(E->get()); + for (const String &E : extensionsl) { + valid_extensions.insert(E); } extensionsl.clear(); ResourceFormatImporter::get_singleton()->get_recognized_extensions(&extensionsl); - for (List<String>::Element *E = extensionsl.front(); E; E = E->next()) { - import_extensions.insert(E->get()); + for (const String &E : extensionsl) { + import_extensions.insert(E); } } EditorFileSystem::EditorFileSystem() { ResourceLoader::import = _resource_import; - reimport_on_missing_imported_files = GLOBAL_DEF("editor/reimport_missing_imported_files", true); - + reimport_on_missing_imported_files = GLOBAL_DEF("editor/import/reimport_missing_imported_files", true); + GLOBAL_DEF("editor/import/use_multiple_threads", true); singleton = this; filesystem = memnew(EditorFileSystemDirectory); //like, empty filesystem->parent = nullptr; - thread = nullptr; scanning = false; importing = false; use_threads = true; - thread_sources = nullptr; new_filesystem = nullptr; abort_scan = false; scanning_changes = false; scanning_changes_done = false; - DirAccess *da = DirAccess::create(DirAccess::ACCESS_RESOURCES); - if (da->change_dir("res://.import") != OK) { - da->make_dir("res://.import"); - } // This should probably also work on Unix and use the string it returns for FAT32 or exFAT + DirAccessRef da = DirAccess::create(DirAccess::ACCESS_RESOURCES); using_fat32_or_exfat = (da->get_filesystem_type() == "FAT32" || da->get_filesystem_type() == "exFAT"); - memdelete(da); scan_total = 0; - update_script_classes_queued = false; + update_script_classes_queued.clear(); first_scan = true; scan_changes_pending = false; revalidate_import_files = false; + import_threads.init(); + ResourceUID::get_singleton()->clear(); //will be updated on scan + ResourceSaver::set_get_resource_id_for_path(_resource_saver_get_resource_id_for_path); } EditorFileSystem::~EditorFileSystem() { + import_threads.finish(); + ResourceSaver::set_get_resource_id_for_path(nullptr); } |