diff options
| author | Rémi Verschelde <rverschelde@gmail.com> | 2020-02-28 17:21:16 +0100 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2020-02-28 17:21:16 +0100 |
| commit | 620030b600b375545ecab5190615e08919e56da4 (patch) | |
| tree | 0ca3e64b0da0edf3d4b1fd14ed558311dd9041e4 | |
| parent | 324e5a60dd068cbbff9c433802f8ecb78cff3364 (diff) | |
| parent | 475e4ea67b7e650fd1df01eab6ea659afbc16ec5 (diff) | |
Merge pull request #36640 from reduz/resource-loader-refactor
Removed ResourceInteractiveLoader, add built-in threaded loading.
57 files changed, 856 insertions, 576 deletions
diff --git a/core/bind/core_bind.cpp b/core/bind/core_bind.cpp index d9cc77feb3..e99e69578c 100644 --- a/core/bind/core_bind.cpp +++ b/core/bind/core_bind.cpp @@ -64,8 +64,21 @@ static const unsigned int MONTH_DAYS_TABLE[2][12] = { _ResourceLoader *_ResourceLoader::singleton = NULL; -Ref<ResourceInteractiveLoader> _ResourceLoader::load_interactive(const String &p_path, const String &p_type_hint) { - return ResourceLoader::load_interactive(p_path, p_type_hint); +Error _ResourceLoader::load_threaded_request(const String &p_path, const String &p_type_hint, bool p_use_sub_threads) { + + return ResourceLoader::load_threaded_request(p_path, p_type_hint, p_use_sub_threads); +} +_ResourceLoader::ThreadLoadStatus _ResourceLoader::load_threaded_get_status(const String &p_path, Array r_progress) { + float progress = 0; + ResourceLoader::ThreadLoadStatus tls = ResourceLoader::load_threaded_get_status(p_path, &progress); + r_progress.resize(1); + r_progress[0] = progress; + return (ThreadLoadStatus)tls; +} +RES _ResourceLoader::load_threaded_get(const String &p_path) { + Error error; + RES res = ResourceLoader::load_threaded_get(p_path, &error); + return res; } RES _ResourceLoader::load(const String &p_path, const String &p_type_hint, bool p_no_cache) { @@ -120,13 +133,21 @@ bool _ResourceLoader::exists(const String &p_path, const String &p_type_hint) { void _ResourceLoader::_bind_methods() { - ClassDB::bind_method(D_METHOD("load_interactive", "path", "type_hint"), &_ResourceLoader::load_interactive, DEFVAL("")); + ClassDB::bind_method(D_METHOD("load_threaded_request", "path", "type_hint", "use_sub_threads"), &_ResourceLoader::load_threaded_request, DEFVAL(""), DEFVAL(false)); + ClassDB::bind_method(D_METHOD("load_threaded_get_status", "path", "r_progress"), &_ResourceLoader::load_threaded_get_status, DEFVAL(Array())); + ClassDB::bind_method(D_METHOD("load_threaded_get", "path"), &_ResourceLoader::load_threaded_get); + ClassDB::bind_method(D_METHOD("load", "path", "type_hint", "no_cache"), &_ResourceLoader::load, DEFVAL(""), DEFVAL(false)); ClassDB::bind_method(D_METHOD("get_recognized_extensions_for_type", "type"), &_ResourceLoader::get_recognized_extensions_for_type); ClassDB::bind_method(D_METHOD("set_abort_on_missing_resources", "abort"), &_ResourceLoader::set_abort_on_missing_resources); ClassDB::bind_method(D_METHOD("get_dependencies", "path"), &_ResourceLoader::get_dependencies); ClassDB::bind_method(D_METHOD("has_cached", "path"), &_ResourceLoader::has_cached); ClassDB::bind_method(D_METHOD("exists", "path", "type_hint"), &_ResourceLoader::exists, DEFVAL("")); + + BIND_ENUM_CONSTANT(THREAD_LOAD_INVALID_RESOURCE); + BIND_ENUM_CONSTANT(THREAD_LOAD_IN_PROGRESS); + BIND_ENUM_CONSTANT(THREAD_LOAD_FAILED); + BIND_ENUM_CONSTANT(THREAD_LOAD_LOADED); } _ResourceLoader::_ResourceLoader() { diff --git a/core/bind/core_bind.h b/core/bind/core_bind.h index e68c40c9e4..d4a7c00629 100644 --- a/core/bind/core_bind.h +++ b/core/bind/core_bind.h @@ -49,8 +49,19 @@ protected: static _ResourceLoader *singleton; public: + enum ThreadLoadStatus { + THREAD_LOAD_INVALID_RESOURCE, + THREAD_LOAD_IN_PROGRESS, + THREAD_LOAD_FAILED, + THREAD_LOAD_LOADED + }; + static _ResourceLoader *get_singleton() { return singleton; } - Ref<ResourceInteractiveLoader> load_interactive(const String &p_path, const String &p_type_hint = ""); + + Error load_threaded_request(const String &p_path, const String &p_type_hint = "", bool p_use_sub_threads = false); + ThreadLoadStatus load_threaded_get_status(const String &p_path, Array r_progress = Array()); + RES load_threaded_get(const String &p_path); + RES load(const String &p_path, const String &p_type_hint = "", bool p_no_cache = false); Vector<String> get_recognized_extensions_for_type(const String &p_type); void set_abort_on_missing_resources(bool p_abort); @@ -61,6 +72,8 @@ public: _ResourceLoader(); }; +VARIANT_ENUM_CAST(_ResourceLoader::ThreadLoadStatus); + class _ResourceSaver : public Object { GDCLASS(_ResourceSaver, Object); diff --git a/core/crypto/crypto.cpp b/core/crypto/crypto.cpp index 3711b26e47..793bf719b7 100644 --- a/core/crypto/crypto.cpp +++ b/core/crypto/crypto.cpp @@ -99,7 +99,7 @@ Crypto::Crypto() { /// Resource loader/saver -RES ResourceFormatLoaderCrypto::load(const String &p_path, const String &p_original_path, Error *r_error) { +RES ResourceFormatLoaderCrypto::load(const String &p_path, const String &p_original_path, Error *r_error, bool p_use_sub_threads, float *r_progress) { String el = p_path.get_extension().to_lower(); if (el == "crt") { diff --git a/core/crypto/crypto.h b/core/crypto/crypto.h index babd0bc4eb..3279c0620f 100644 --- a/core/crypto/crypto.h +++ b/core/crypto/crypto.h @@ -87,7 +87,7 @@ class ResourceFormatLoaderCrypto : public ResourceFormatLoader { GDCLASS(ResourceFormatLoaderCrypto, ResourceFormatLoader); public: - virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL); + virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL, bool p_use_sub_threads = false, float *r_progress = nullptr); virtual void get_recognized_extensions(List<String> *p_extensions) const; virtual bool handles_type(const String &p_type) const; virtual String get_resource_type(const String &p_path) const; diff --git a/core/io/image_loader.cpp b/core/io/image_loader.cpp index 720f25f91b..99ac5bcdd9 100644 --- a/core/io/image_loader.cpp +++ b/core/io/image_loader.cpp @@ -129,7 +129,7 @@ void ImageLoader::cleanup() { ///////////////// -RES ResourceFormatLoaderImage::load(const String &p_path, const String &p_original_path, Error *r_error) { +RES ResourceFormatLoaderImage::load(const String &p_path, const String &p_original_path, Error *r_error, bool p_use_sub_threads, float *r_progress) { FileAccess *f = FileAccess::open(p_path, FileAccess::READ); if (!f) { diff --git a/core/io/image_loader.h b/core/io/image_loader.h index d6dfd261ca..3ba028b99c 100644 --- a/core/io/image_loader.h +++ b/core/io/image_loader.h @@ -73,7 +73,7 @@ public: class ResourceFormatLoaderImage : public ResourceFormatLoader { public: - virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL); + virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL, bool p_use_sub_threads = false, float *r_progress = nullptr); virtual void get_recognized_extensions(List<String> *p_extensions) const; virtual bool handles_type(const String &p_type) const; virtual String get_resource_type(const String &p_path) const; diff --git a/core/io/resource_format_binary.cpp b/core/io/resource_format_binary.cpp index 8c343a0f43..54b75cc29d 100644 --- a/core/io/resource_format_binary.cpp +++ b/core/io/resource_format_binary.cpp @@ -93,7 +93,7 @@ enum { }; -void ResourceInteractiveLoaderBinary::_advance_padding(uint32_t p_len) { +void ResourceLoaderBinary::_advance_padding(uint32_t p_len) { uint32_t extra = 4 - (p_len % 4); if (extra < 4) { @@ -102,7 +102,7 @@ void ResourceInteractiveLoaderBinary::_advance_padding(uint32_t p_len) { } } -StringName ResourceInteractiveLoaderBinary::_get_string() { +StringName ResourceLoaderBinary::_get_string() { uint32_t id = f->get_32(); if (id & 0x80000000) { @@ -121,7 +121,7 @@ StringName ResourceInteractiveLoaderBinary::_get_string() { return string_map[id]; } -Error ResourceInteractiveLoaderBinary::parse_variant(Variant &r_v) { +Error ResourceLoaderBinary::parse_variant(Variant &r_v) { uint32_t type = f->get_32(); print_bl("find property of type: " + itos(type)); @@ -377,20 +377,26 @@ Error ResourceInteractiveLoaderBinary::parse_variant(Variant &r_v) { r_v = Variant(); } else { - String exttype = external_resources[erindex].type; - String path = external_resources[erindex].path; + if (external_resources[erindex].cache.is_null()) { + //cache not here yet, wait for it? + if (use_sub_threads) { + Error err; + external_resources.write[erindex].cache = ResourceLoader::load_threaded_get(external_resources[erindex].path, &err); - if (path.find("://") == -1 && path.is_rel_path()) { - // path is relative to file being loaded, so convert to a resource path - path = ProjectSettings::get_singleton()->localize_path(res_path.get_base_dir().plus_file(path)); - } + if (err != OK || external_resources[erindex].cache.is_null()) { + if (!ResourceLoader::get_abort_on_missing_resources()) { - RES res = ResourceLoader::load(path, exttype); + ResourceLoader::notify_dependency_error(local_path, external_resources[erindex].path, external_resources[erindex].type); + } else { - if (res.is_null()) { - WARN_PRINT(String("Couldn't load resource: " + path).utf8().get_data()); + error = ERR_FILE_MISSING_DEPENDENCIES; + ERR_FAIL_V_MSG(error, "Can't load dependency: " + external_resources[erindex].path + "."); + } + } + } } - r_v = res; + + r_v = external_resources[erindex].cache; } } break; @@ -638,160 +644,168 @@ Error ResourceInteractiveLoaderBinary::parse_variant(Variant &r_v) { return OK; //never reach anyway } -void ResourceInteractiveLoaderBinary::set_local_path(const String &p_local_path) { +void ResourceLoaderBinary::set_local_path(const String &p_local_path) { res_path = p_local_path; } -Ref<Resource> ResourceInteractiveLoaderBinary::get_resource() { +Ref<Resource> ResourceLoaderBinary::get_resource() { return resource; } -Error ResourceInteractiveLoaderBinary::poll() { +Error ResourceLoaderBinary::load() { if (error != OK) return error; - int s = stage; + int stage = 0; - if (s < external_resources.size()) { + for (int i = 0; i < external_resources.size(); i++) { - String path = external_resources[s].path; + String path = external_resources[i].path; if (remaps.has(path)) { path = remaps[path]; } - RES res = ResourceLoader::load(path, external_resources[s].type); - if (res.is_null()) { - if (!ResourceLoader::get_abort_on_missing_resources()) { + if (path.find("://") == -1 && path.is_rel_path()) { + // path is relative to file being loaded, so convert to a resource path + path = ProjectSettings::get_singleton()->localize_path(path.get_base_dir().plus_file(external_resources[i].path)); + } + + external_resources.write[i].path = path; //remap happens here, not on load because on load it can actually be used for filesystem dock resource remap - ResourceLoader::notify_dependency_error(local_path, path, external_resources[s].type); - } else { + if (!use_sub_threads) { + external_resources.write[i].cache = ResourceLoader::load(path, external_resources[i].type); - error = ERR_FILE_MISSING_DEPENDENCIES; - ERR_FAIL_V_MSG(error, "Can't load dependency: " + path + "."); + if (external_resources[i].cache.is_null()) { + if (!ResourceLoader::get_abort_on_missing_resources()) { + + ResourceLoader::notify_dependency_error(local_path, path, external_resources[i].type); + } else { + + error = ERR_FILE_MISSING_DEPENDENCIES; + ERR_FAIL_V_MSG(error, "Can't load dependency: " + path + "."); + } } } else { - resource_cache.push_back(res); + Error err = ResourceLoader::load_threaded_request(path, external_resources[i].type, use_sub_threads, local_path); + if (err != OK) { + if (!ResourceLoader::get_abort_on_missing_resources()) { + + ResourceLoader::notify_dependency_error(local_path, path, external_resources[i].type); + } else { + + error = ERR_FILE_MISSING_DEPENDENCIES; + ERR_FAIL_V_MSG(error, "Can't load dependency: " + path + "."); + } + } } stage++; - return error; } - s -= external_resources.size(); + for (int i = 0; i < internal_resources.size(); i++) { - if (s >= internal_resources.size()) { + bool main = i == (internal_resources.size() - 1); - error = ERR_BUG; - ERR_FAIL_COND_V(s >= internal_resources.size(), error); - } + //maybe it is loaded already + String path; + int subindex = 0; - bool main = s == (internal_resources.size() - 1); + if (!main) { - //maybe it is loaded already - String path; - int subindex = 0; + path = internal_resources[i].path; + if (path.begins_with("local://")) { + path = path.replace_first("local://", ""); + subindex = path.to_int(); + path = res_path + "::" + path; + } - if (!main) { + if (ResourceCache::has(path)) { + //already loaded, don't do anything + stage++; + error = OK; + continue; + } + } else { - path = internal_resources[s].path; - if (path.begins_with("local://")) { - path = path.replace_first("local://", ""); - subindex = path.to_int(); - path = res_path + "::" + path; + if (!ResourceCache::has(res_path)) + path = res_path; } - if (ResourceCache::has(path)) { - //already loaded, don't do anything - stage++; - error = OK; - return error; - } - } else { + uint64_t offset = internal_resources[i].offset; - if (!ResourceCache::has(res_path)) - path = res_path; - } - - uint64_t offset = internal_resources[s].offset; + f->seek(offset); - f->seek(offset); + String t = get_unicode_string(); - String t = get_unicode_string(); - - Object *obj = ClassDB::instance(t); - if (!obj) { - error = ERR_FILE_CORRUPT; - ERR_FAIL_V_MSG(ERR_FILE_CORRUPT, local_path + ":Resource of unrecognized type in file: " + t + "."); - } + Object *obj = ClassDB::instance(t); + if (!obj) { + error = ERR_FILE_CORRUPT; + ERR_FAIL_V_MSG(ERR_FILE_CORRUPT, local_path + ":Resource of unrecognized type in file: " + t + "."); + } - Resource *r = Object::cast_to<Resource>(obj); - if (!r) { - String obj_class = obj->get_class(); - error = ERR_FILE_CORRUPT; - memdelete(obj); //bye - ERR_FAIL_V_MSG(ERR_FILE_CORRUPT, local_path + ":Resource type in resource field not a resource, type is: " + obj_class + "."); - } + Resource *r = Object::cast_to<Resource>(obj); + if (!r) { + String obj_class = obj->get_class(); + error = ERR_FILE_CORRUPT; + memdelete(obj); //bye + ERR_FAIL_V_MSG(ERR_FILE_CORRUPT, local_path + ":Resource type in resource field not a resource, type is: " + obj_class + "."); + } - RES res = RES(r); + RES res = RES(r); - r->set_path(path); - r->set_subindex(subindex); + r->set_path(path); + r->set_subindex(subindex); - int pc = f->get_32(); + int pc = f->get_32(); - //set properties + //set properties - for (int i = 0; i < pc; i++) { + for (int j = 0; j < pc; j++) { - StringName name = _get_string(); + StringName name = _get_string(); - if (name == StringName()) { - error = ERR_FILE_CORRUPT; - ERR_FAIL_V(ERR_FILE_CORRUPT); - } + if (name == StringName()) { + error = ERR_FILE_CORRUPT; + ERR_FAIL_V(ERR_FILE_CORRUPT); + } - Variant value; + Variant value; - error = parse_variant(value); - if (error) - return error; + error = parse_variant(value); + if (error) + return error; - res->set(name, value); - } + res->set(name, value); + } #ifdef TOOLS_ENABLED - res->set_edited(false); + res->set_edited(false); #endif - stage++; + stage++; - resource_cache.push_back(res); + if (progress) { + *progress = (i + 1) / float(internal_resources.size()); + } - if (main) { + resource_cache.push_back(res); - f->close(); - resource = res; - resource->set_as_translation_remapped(translation_remapped); - error = ERR_FILE_EOF; + if (main) { - } else { - error = OK; + f->close(); + resource = res; + resource->set_as_translation_remapped(translation_remapped); + error = OK; + return OK; + } } - return OK; -} -int ResourceInteractiveLoaderBinary::get_stage() const { - - return stage; -} -int ResourceInteractiveLoaderBinary::get_stage_count() const { - - return external_resources.size() + internal_resources.size(); + return ERR_FILE_EOF; } -void ResourceInteractiveLoaderBinary::set_translation_remapped(bool p_remapped) { +void ResourceLoaderBinary::set_translation_remapped(bool p_remapped) { translation_remapped = p_remapped; } @@ -814,7 +828,7 @@ static String get_ustring(FileAccess *f) { return s; } -String ResourceInteractiveLoaderBinary::get_unicode_string() { +String ResourceLoaderBinary::get_unicode_string() { int len = f->get_32(); if (len > str_buf.size()) { @@ -828,7 +842,7 @@ String ResourceInteractiveLoaderBinary::get_unicode_string() { return s; } -void ResourceInteractiveLoaderBinary::get_dependencies(FileAccess *p_f, List<String> *p_dependencies, bool p_add_types) { +void ResourceLoaderBinary::get_dependencies(FileAccess *p_f, List<String> *p_dependencies, bool p_add_types) { open(p_f); if (error) @@ -846,7 +860,7 @@ void ResourceInteractiveLoaderBinary::get_dependencies(FileAccess *p_f, List<Str } } -void ResourceInteractiveLoaderBinary::open(FileAccess *p_f) { +void ResourceLoaderBinary::open(FileAccess *p_f) { error = OK; @@ -947,7 +961,7 @@ void ResourceInteractiveLoaderBinary::open(FileAccess *p_f) { } } -String ResourceInteractiveLoaderBinary::recognize(FileAccess *p_f) { +String ResourceLoaderBinary::recognize(FileAccess *p_f) { error = OK; @@ -992,20 +1006,22 @@ String ResourceInteractiveLoaderBinary::recognize(FileAccess *p_f) { return type; } -ResourceInteractiveLoaderBinary::ResourceInteractiveLoaderBinary() : +ResourceLoaderBinary::ResourceLoaderBinary() : translation_remapped(false), f(NULL), - error(OK), - stage(0) { + error(OK) { + + progress = nullptr; + use_sub_threads = false; } -ResourceInteractiveLoaderBinary::~ResourceInteractiveLoaderBinary() { +ResourceLoaderBinary::~ResourceLoaderBinary() { if (f) memdelete(f); } -Ref<ResourceInteractiveLoader> ResourceFormatLoaderBinary::load_interactive(const String &p_path, const String &p_original_path, Error *r_error) { +RES ResourceFormatLoaderBinary::load(const String &p_path, const String &p_original_path, Error *r_error, bool p_use_sub_threads, float *r_progress) { if (r_error) *r_error = ERR_FILE_CANT_OPEN; @@ -1013,16 +1029,27 @@ Ref<ResourceInteractiveLoader> ResourceFormatLoaderBinary::load_interactive(cons Error err; FileAccess *f = FileAccess::open(p_path, FileAccess::READ, &err); - ERR_FAIL_COND_V_MSG(err != OK, Ref<ResourceInteractiveLoader>(), "Cannot open file '" + p_path + "'."); + ERR_FAIL_COND_V_MSG(err != OK, RES(), "Cannot open file '" + p_path + "'."); - Ref<ResourceInteractiveLoaderBinary> ria = memnew(ResourceInteractiveLoaderBinary); + ResourceLoaderBinary loader; + loader.use_sub_threads = p_use_sub_threads; + loader.progress = r_progress; String path = p_original_path != "" ? p_original_path : p_path; - ria->local_path = ProjectSettings::get_singleton()->localize_path(path); - ria->res_path = ria->local_path; - //ria->set_local_path( Globals::get_singleton()->localize_path(p_path) ); - ria->open(f); + loader.local_path = ProjectSettings::get_singleton()->localize_path(path); + loader.res_path = loader.local_path; + //loader.set_local_path( Globals::get_singleton()->localize_path(p_path) ); + loader.open(f); + + err = loader.load(); + + if (r_error) { + *r_error = err; + } - return ria; + if (err) { + return RES(); + } + return loader.resource; } void ResourceFormatLoaderBinary::get_recognized_extensions_for_type(const String &p_type, List<String> *p_extensions) const { @@ -1064,11 +1091,11 @@ void ResourceFormatLoaderBinary::get_dependencies(const String &p_path, List<Str FileAccess *f = FileAccess::open(p_path, FileAccess::READ); ERR_FAIL_COND_MSG(!f, "Cannot open file '" + p_path + "'."); - Ref<ResourceInteractiveLoaderBinary> ria = memnew(ResourceInteractiveLoaderBinary); - ria->local_path = ProjectSettings::get_singleton()->localize_path(p_path); - ria->res_path = ria->local_path; - //ria->set_local_path( Globals::get_singleton()->localize_path(p_path) ); - ria->get_dependencies(f, p_dependencies, p_add_types); + ResourceLoaderBinary loader; + loader.local_path = ProjectSettings::get_singleton()->localize_path(p_path); + loader.res_path = loader.local_path; + //loader.set_local_path( Globals::get_singleton()->localize_path(p_path) ); + loader.get_dependencies(f, p_dependencies, p_add_types); } Error ResourceFormatLoaderBinary::rename_dependencies(const String &p_path, const Map<String, String> &p_map) { @@ -1153,21 +1180,17 @@ Error ResourceFormatLoaderBinary::rename_dependencies(const String &p_path, cons ERR_FAIL_COND_V_MSG(err != OK, ERR_FILE_CANT_OPEN, "Cannot open file '" + p_path + "'."); - Ref<ResourceInteractiveLoaderBinary> ria = memnew(ResourceInteractiveLoaderBinary); - ria->local_path = ProjectSettings::get_singleton()->localize_path(p_path); - ria->res_path = ria->local_path; - ria->remaps = p_map; - //ria->set_local_path( Globals::get_singleton()->localize_path(p_path) ); - ria->open(f); - - err = ria->poll(); + ResourceLoaderBinary loader; + loader.local_path = ProjectSettings::get_singleton()->localize_path(p_path); + loader.res_path = loader.local_path; + loader.remaps = p_map; + //loader.set_local_path( Globals::get_singleton()->localize_path(p_path) ); + loader.open(f); - while (err == OK) { - err = ria->poll(); - } + err = loader.load(); ERR_FAIL_COND_V(err != ERR_FILE_EOF, ERR_FILE_CORRUPT); - RES res = ria->get_resource(); + RES res = loader.get_resource(); ERR_FAIL_COND_V(!res.is_valid(), ERR_FILE_CORRUPT); return ResourceFormatSaverBinary::singleton->save(p_path, res); @@ -1283,11 +1306,11 @@ String ResourceFormatLoaderBinary::get_resource_type(const String &p_path) const return ""; //could not rwead } - Ref<ResourceInteractiveLoaderBinary> ria = memnew(ResourceInteractiveLoaderBinary); - ria->local_path = ProjectSettings::get_singleton()->localize_path(p_path); - ria->res_path = ria->local_path; - //ria->set_local_path( Globals::get_singleton()->localize_path(p_path) ); - String r = ria->recognize(f); + ResourceLoaderBinary loader; + loader.local_path = ProjectSettings::get_singleton()->localize_path(p_path); + loader.res_path = loader.local_path; + //loader.set_local_path( Globals::get_singleton()->localize_path(p_path) ); + String r = loader.recognize(f); return ClassDB::get_compatibility_remapped_class(r); } diff --git a/core/io/resource_format_binary.h b/core/io/resource_format_binary.h index f02dbaa0c2..0ffa2c3626 100644 --- a/core/io/resource_format_binary.h +++ b/core/io/resource_format_binary.h @@ -35,7 +35,7 @@ #include "core/io/resource_saver.h" #include "core/os/file_access.h" -class ResourceInteractiveLoaderBinary : public ResourceInteractiveLoader { +class ResourceLoaderBinary { bool translation_remapped; String local_path; @@ -58,8 +58,11 @@ class ResourceInteractiveLoaderBinary : public ResourceInteractiveLoader { struct ExtResource { String path; String type; + RES cache; }; + bool use_sub_threads; + float *progress; Vector<ExtResource> external_resources; struct IntResource { @@ -75,32 +78,30 @@ class ResourceInteractiveLoaderBinary : public ResourceInteractiveLoader { Map<String, String> remaps; Error error; - int stage; - friend class ResourceFormatLoaderBinary; Error parse_variant(Variant &r_v); + Map<String, RES> dependency_cache; + public: - virtual void set_local_path(const String &p_local_path); - virtual Ref<Resource> get_resource(); - virtual Error poll(); - virtual int get_stage() const; - virtual int get_stage_count() const; - virtual void set_translation_remapped(bool p_remapped); + void set_local_path(const String &p_local_path); + Ref<Resource> get_resource(); + Error load(); + void set_translation_remapped(bool p_remapped); void set_remaps(const Map<String, String> &p_remaps) { remaps = p_remaps; } void open(FileAccess *p_f); String recognize(FileAccess *p_f); void get_dependencies(FileAccess *p_f, List<String> *p_dependencies, bool p_add_types); - ResourceInteractiveLoaderBinary(); - ~ResourceInteractiveLoaderBinary(); + ResourceLoaderBinary(); + ~ResourceLoaderBinary(); }; class ResourceFormatLoaderBinary : public ResourceFormatLoader { public: - virtual Ref<ResourceInteractiveLoader> load_interactive(const String &p_path, const String &p_original_path = "", Error *r_error = NULL); + virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL, bool p_use_sub_threads = false, float *r_progress = nullptr); virtual void get_recognized_extensions_for_type(const String &p_type, List<String> *p_extensions) const; virtual void get_recognized_extensions(List<String> *p_extensions) const; virtual bool handles_type(const String &p_type) const; diff --git a/core/io/resource_importer.cpp b/core/io/resource_importer.cpp index f147170ff7..efaf958949 100644 --- a/core/io/resource_importer.cpp +++ b/core/io/resource_importer.cpp @@ -117,7 +117,7 @@ Error ResourceFormatImporter::_get_path_and_type(const String &p_path, PathAndTy return OK; } -RES ResourceFormatImporter::load(const String &p_path, const String &p_original_path, Error *r_error) { +RES ResourceFormatImporter::load(const String &p_path, const String &p_original_path, Error *r_error, bool p_use_sub_threads, float *r_progress) { PathAndType pat; Error err = _get_path_and_type(p_path, pat); @@ -130,7 +130,7 @@ RES ResourceFormatImporter::load(const String &p_path, const String &p_original_ return RES(); } - RES res = ResourceLoader::_load(pat.path, p_path, pat.type, false, r_error); + RES res = ResourceLoader::_load(pat.path, p_path, pat.type, false, r_error, p_use_sub_threads, r_progress); #ifdef TOOLS_ENABLED if (res.is_valid()) { diff --git a/core/io/resource_importer.h b/core/io/resource_importer.h index 4eb04586e6..65c148f2ac 100644 --- a/core/io/resource_importer.h +++ b/core/io/resource_importer.h @@ -58,7 +58,7 @@ class ResourceFormatImporter : public ResourceFormatLoader { public: static ResourceFormatImporter *get_singleton() { return singleton; } - virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL); + virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL, bool p_use_sub_threads = false, float *r_progress = nullptr); virtual void get_recognized_extensions(List<String> *p_extensions) const; virtual void get_recognized_extensions_for_type(const String &p_type, List<String> *p_extensions) const; virtual bool recognize_path(const String &p_path, const String &p_for_type = String()) const; diff --git a/core/io/resource_loader.cpp b/core/io/resource_loader.cpp index b43cd7f1a2..504dbe2d63 100644 --- a/core/io/resource_loader.cpp +++ b/core/io/resource_loader.cpp @@ -39,26 +39,16 @@ #include "core/translation.h" #include "core/variant_parser.h" +#ifdef DEBUG_LOAD_THREADED +#define print_lt(m_text) print_line(m_text) +#else +#define print_lt(m_text) +#endif + Ref<ResourceFormatLoader> ResourceLoader::loader[ResourceLoader::MAX_LOADERS]; int ResourceLoader::loader_count = 0; -Error ResourceInteractiveLoader::wait() { - - Error err = poll(); - while (err == OK) { - err = poll(); - } - - return err; -} - -ResourceInteractiveLoader::~ResourceInteractiveLoader() { - if (path_loading != String()) { - ResourceLoader::_remove_from_loading_map_and_thread(path_loading, path_loading_thread); - } -} - bool ResourceFormatLoader::recognize_path(const String &p_path, const String &p_for_type) const { String extension = p_path.get_extension(); @@ -111,45 +101,6 @@ void ResourceLoader::get_recognized_extensions_for_type(const String &p_type, Li } } -void ResourceInteractiveLoader::_bind_methods() { - - ClassDB::bind_method(D_METHOD("get_resource"), &ResourceInteractiveLoader::get_resource); - ClassDB::bind_method(D_METHOD("poll"), &ResourceInteractiveLoader::poll); - ClassDB::bind_method(D_METHOD("wait"), &ResourceInteractiveLoader::wait); - ClassDB::bind_method(D_METHOD("get_stage"), &ResourceInteractiveLoader::get_stage); - ClassDB::bind_method(D_METHOD("get_stage_count"), &ResourceInteractiveLoader::get_stage_count); -} - -class ResourceInteractiveLoaderDefault : public ResourceInteractiveLoader { - - GDCLASS(ResourceInteractiveLoaderDefault, ResourceInteractiveLoader); - -public: - Ref<Resource> resource; - - virtual void set_local_path(const String &p_local_path) { /*scene->set_filename(p_local_path);*/ - } - virtual Ref<Resource> get_resource() { return resource; } - virtual Error poll() { return ERR_FILE_EOF; } - virtual int get_stage() const { return 1; } - virtual int get_stage_count() const { return 1; } - virtual void set_translation_remapped(bool p_remapped) { resource->set_as_translation_remapped(p_remapped); } - - ResourceInteractiveLoaderDefault() {} -}; - -Ref<ResourceInteractiveLoader> ResourceFormatLoader::load_interactive(const String &p_path, const String &p_original_path, Error *r_error) { - - //either this - Ref<Resource> res = load(p_path, p_original_path, r_error); - if (res.is_null()) - return Ref<ResourceInteractiveLoader>(); - - Ref<ResourceInteractiveLoaderDefault> ril = Ref<ResourceInteractiveLoaderDefault>(memnew(ResourceInteractiveLoaderDefault)); - ril->resource = res; - return ril; -} - bool ResourceFormatLoader::exists(const String &p_path) const { return FileAccess::exists(p_path); //by default just check file } @@ -168,10 +119,10 @@ void ResourceFormatLoader::get_recognized_extensions(List<String> *p_extensions) } } -RES ResourceFormatLoader::load(const String &p_path, const String &p_original_path, Error *r_error) { +RES ResourceFormatLoader::load(const String &p_path, const String &p_original_path, Error *r_error, bool p_use_sub_threads, float *r_progress) { if (get_script_instance() && get_script_instance()->has_method("load")) { - Variant res = get_script_instance()->call("load", p_path, p_original_path); + Variant res = get_script_instance()->call("load", p_path, p_original_path, p_use_sub_threads); if (res.get_type() == Variant::INT) { @@ -184,29 +135,11 @@ RES ResourceFormatLoader::load(const String &p_path, const String &p_original_pa *r_error = OK; return res; } - } - - //or this must be implemented - Ref<ResourceInteractiveLoader> ril = load_interactive(p_path, p_original_path, r_error); - if (!ril.is_valid()) - return RES(); - ril->set_local_path(p_original_path); - - while (true) { - - Error err = ril->poll(); - - if (err == ERR_FILE_EOF) { - if (r_error) - *r_error = OK; - return ril->get_resource(); - } - if (r_error) - *r_error = err; - - ERR_FAIL_COND_V_MSG(err != OK, RES(), "Failed to load resource '" + p_path + "'."); + return res; } + + ERR_FAIL_V_MSG(RES(), "Failed to load resource '" + p_path + "', ResourceFormatLoader::load was not implemented for this resource type."); } void ResourceFormatLoader::get_dependencies(const String &p_path, List<String> *p_dependencies, bool p_add_types) { @@ -256,7 +189,7 @@ void ResourceFormatLoader::_bind_methods() { /////////////////////////////////// -RES ResourceLoader::_load(const String &p_path, const String &p_original_path, const String &p_type_hint, bool p_no_cache, Error *r_error) { +RES ResourceLoader::_load(const String &p_path, const String &p_original_path, const String &p_type_hint, bool p_no_cache, Error *r_error, bool p_use_sub_threads, float *r_progress) { bool found = false; @@ -267,7 +200,7 @@ RES ResourceLoader::_load(const String &p_path, const String &p_original_path, c continue; } found = true; - RES res = loader[i]->load(p_path, p_original_path != String() ? p_original_path : p_path, r_error); + RES res = loader[i]->load(p_path, p_original_path != String() ? p_original_path : p_path, r_error, p_use_sub_threads, r_progress); if (res.is_null()) { continue; } @@ -285,49 +218,68 @@ RES ResourceLoader::_load(const String &p_path, const String &p_original_path, c ERR_FAIL_V_MSG(RES(), "No loader found for resource: " + p_path + "."); } -bool ResourceLoader::_add_to_loading_map(const String &p_path) { +void ResourceLoader::_thread_load_function(void *p_userdata) { - bool success; - MutexLock lock(loading_map_mutex); + ThreadLoadTask &load_task = *(ThreadLoadTask *)p_userdata; + load_task.loader_id = Thread::get_caller_id(); - LoadingMapKey key; - key.path = p_path; - key.thread = Thread::get_caller_id(); + if (load_task.semaphore) { + //this is an actual thread, so wait for Ok fom semaphore + thread_load_semaphore->wait(); //wait until its ok to start loading + } + load_task.resource = _load(load_task.remapped_path, load_task.remapped_path != load_task.local_path ? load_task.local_path : String(), load_task.type_hint, false, &load_task.error, load_task.use_sub_threads, &load_task.progress); + + load_task.progress = 1.0; //it was fully loaded at this point, so force progress to 1.0 - if (loading_map.has(key)) { - success = false; + thread_load_mutex->lock(); + if (load_task.error != OK) { + load_task.status = THREAD_LOAD_FAILED; } else { - loading_map[key] = true; - success = true; + load_task.status = THREAD_LOAD_LOADED; } + if (load_task.semaphore) { - return success; -} + if (load_task.start_next && thread_waiting_count > 0) { + thread_waiting_count--; + //thread loading count remains constant, this ends but another one begins + thread_load_semaphore->post(); + } else { + thread_loading_count--; //no threads waiting, just reduce loading count + } -void ResourceLoader::_remove_from_loading_map(const String &p_path) { - MutexLock lock(loading_map_mutex); + print_lt("END: load count: " + itos(thread_loading_count) + " / wait count: " + itos(thread_waiting_count) + " / suspended count: " + itos(thread_suspended_count) + " / active: " + itos(thread_loading_count - thread_suspended_count)); - LoadingMapKey key; - key.path = p_path; - key.thread = Thread::get_caller_id(); + for (int i = 0; i < load_task.poll_requests; i++) { + load_task.semaphore->post(); + } + memdelete(load_task.semaphore); + load_task.semaphore = nullptr; + } - loading_map.erase(key); -} + if (load_task.resource.is_valid()) { + load_task.resource->set_path(load_task.local_path); -void ResourceLoader::_remove_from_loading_map_and_thread(const String &p_path, Thread::ID p_thread) { - MutexLock lock(loading_map_mutex); + if (load_task.xl_remapped) + load_task.resource->set_as_translation_remapped(true); - LoadingMapKey key; - key.path = p_path; - key.thread = p_thread; +#ifdef TOOLS_ENABLED - loading_map.erase(key); -} + load_task.resource->set_edited(false); + if (timestamp_on_load) { + uint64_t mt = FileAccess::get_modified_time(load_task.remapped_path); + //printf("mt %s: %lli\n",remapped_path.utf8().get_data(),mt); + load_task.resource->set_last_modified_time(mt); + } +#endif -RES ResourceLoader::load(const String &p_path, const String &p_type_hint, bool p_no_cache, Error *r_error) { + if (_loaded_callback) { + _loaded_callback(load_task.resource, load_task.local_path); + } + } - if (r_error) - *r_error = ERR_CANT_OPEN; + thread_load_mutex->unlock(); +} +Error ResourceLoader::load_threaded_request(const String &p_path, const String &p_type_hint, bool p_use_sub_threads, const String &p_source_resource) { String local_path; if (p_path.is_rel_path()) @@ -335,88 +287,154 @@ RES ResourceLoader::load(const String &p_path, const String &p_type_hint, bool p else local_path = ProjectSettings::get_singleton()->localize_path(p_path); - if (!p_no_cache) { + thread_load_mutex->lock(); - { - bool success = _add_to_loading_map(local_path); - ERR_FAIL_COND_V_MSG(!success, RES(), "Resource: '" + local_path + "' is already being loaded. Cyclic reference?"); + if (p_source_resource != String()) { + //must be loading from this resource + if (!thread_load_tasks.has(p_source_resource)) { + thread_load_mutex->unlock(); + ERR_FAIL_V_MSG(ERR_INVALID_PARAMETER, "There is no thread loading source resource '" + p_source_resource + "'."); + } + //must be loading from this thread + if (thread_load_tasks[p_source_resource].loader_id != Thread::get_caller_id()) { + thread_load_mutex->unlock(); + ERR_FAIL_V_MSG(ERR_INVALID_PARAMETER, "Threading loading resource'" + local_path + " failed: Source specified: '" + p_source_resource + "' but was not called by it."); } - //lock first if possible - if (ResourceCache::lock) { - ResourceCache::lock->read_lock(); + //must not be already added as s sub tasks + if (thread_load_tasks[p_source_resource].sub_tasks.has(local_path)) { + thread_load_mutex->unlock(); + ERR_FAIL_V_MSG(ERR_INVALID_PARAMETER, "Thread loading source resource '" + p_source_resource + "' already is loading '" + local_path + "'."); } + } - //get ptr - Resource **rptr = ResourceCache::resources.getptr(local_path); + if (thread_load_tasks.has(local_path)) { + thread_load_tasks[local_path].requests++; + if (p_source_resource != String()) { + thread_load_tasks[p_source_resource].sub_tasks.insert(local_path); + } + thread_load_mutex->unlock(); + return OK; + } - if (rptr) { - RES res(*rptr); - //it is possible this resource was just freed in a thread. If so, this referencing will not work and resource is considered not cached - if (res.is_valid()) { - //referencing is fine - if (r_error) - *r_error = OK; - if (ResourceCache::lock) { - ResourceCache::lock->read_unlock(); + { + //create load task + + ThreadLoadTask load_task; + + load_task.requests = 1; + load_task.remapped_path = _path_remap(local_path, &load_task.xl_remapped); + load_task.local_path = local_path; + load_task.type_hint = p_type_hint; + load_task.use_sub_threads = p_use_sub_threads; + + { //must check if resource is already loaded before attempting to load it in a thread + + if (load_task.loader_id == Thread::get_caller_id()) { + thread_load_mutex->unlock(); + ERR_FAIL_V_MSG(ERR_INVALID_PARAMETER, "Attempted to load a resource already being loaded from this thread, cyclic reference?"); + } + //lock first if possible + if (ResourceCache::lock) { + ResourceCache::lock->read_lock(); + } + + //get ptr + Resource **rptr = ResourceCache::resources.getptr(local_path); + + if (rptr) { + RES res(*rptr); + //it is possible this resource was just freed in a thread. If so, this referencing will not work and resource is considered not cached + if (res.is_valid()) { + //referencing is fine + load_task.resource = res; + load_task.status = THREAD_LOAD_LOADED; + load_task.progress = 1.0; } - _remove_from_loading_map(local_path); - return res; + } + if (ResourceCache::lock) { + ResourceCache::lock->read_unlock(); } } - if (ResourceCache::lock) { - ResourceCache::lock->read_unlock(); + + if (p_source_resource != String()) { + thread_load_tasks[p_source_resource].sub_tasks.insert(local_path); } + + thread_load_tasks[local_path] = load_task; } - bool xl_remapped = false; - String path = _path_remap(local_path, &xl_remapped); + ThreadLoadTask &load_task = thread_load_tasks[local_path]; - if (path == "") { - if (!p_no_cache) { - _remove_from_loading_map(local_path); + if (load_task.resource.is_null()) { //needs to be loaded in thread + + load_task.semaphore = memnew(Semaphore); + if (thread_loading_count < thread_load_max) { + thread_loading_count++; + thread_load_semaphore->post(); //we have free threads, so allow one + } else { + thread_waiting_count++; } - ERR_FAIL_V_MSG(RES(), "Remapping '" + local_path + "' failed."); + + print_lt("REQUEST: load count: " + itos(thread_loading_count) + " / wait count: " + itos(thread_waiting_count) + " / suspended count: " + itos(thread_suspended_count) + " / active: " + itos(thread_loading_count - thread_suspended_count)); + + load_task.thread = Thread::create(_thread_load_function, &thread_load_tasks[local_path]); + load_task.loader_id = load_task.thread->get_id(); } - print_verbose("Loading resource: " + path); - RES res = _load(path, local_path, p_type_hint, p_no_cache, r_error); + thread_load_mutex->unlock(); + + return OK; +} + +float ResourceLoader::_dependency_get_progress(const String &p_path) { - if (res.is_null()) { - if (!p_no_cache) { - _remove_from_loading_map(local_path); + if (thread_load_tasks.has(p_path)) { + ThreadLoadTask &load_task = thread_load_tasks[p_path]; + int dep_count = load_task.sub_tasks.size(); + if (dep_count > 0) { + float dep_progress = 0; + for (Set<String>::Element *E = load_task.sub_tasks.front(); E; E = E->next()) { + dep_progress += _dependency_get_progress(E->get()); + } + dep_progress /= float(dep_count); + dep_progress *= 0.5; + dep_progress += load_task.progress * 0.5; + return dep_progress; + } else { + return load_task.progress; } - print_verbose("Failed loading resource: " + path); - return RES(); + + } else { + return 1.0; //assume finished loading it so it no longer exists } - if (!p_no_cache) - res->set_path(local_path); +} - if (xl_remapped) - res->set_as_translation_remapped(true); +ResourceLoader::ThreadLoadStatus ResourceLoader::load_threaded_get_status(const String &p_path, float *r_progress) { -#ifdef TOOLS_ENABLED + String local_path; + if (p_path.is_rel_path()) + local_path = "res://" + p_path; + else + local_path = ProjectSettings::get_singleton()->localize_path(p_path); - res->set_edited(false); - if (timestamp_on_load) { - uint64_t mt = FileAccess::get_modified_time(path); - //printf("mt %s: %lli\n",remapped_path.utf8().get_data(),mt); - res->set_last_modified_time(mt); + thread_load_mutex->lock(); + if (!thread_load_tasks.has(local_path)) { + thread_load_mutex->unlock(); + return THREAD_LOAD_INVALID_RESOURCE; } -#endif - - if (!p_no_cache) { - _remove_from_loading_map(local_path); + ThreadLoadTask &load_task = thread_load_tasks[local_path]; + ThreadLoadStatus status; + status = load_task.status; + if (r_progress) { + *r_progress = _dependency_get_progress(local_path); } - if (_loaded_callback) { - _loaded_callback(res, p_path); - } + thread_load_mutex->unlock(); - return res; + return status; } - -bool ResourceLoader::exists(const String &p_path, const String &p_type_hint) { +RES ResourceLoader::load_threaded_get(const String &p_path, Error *r_error) { String local_path; if (p_path.is_rel_path()) @@ -424,29 +442,81 @@ bool ResourceLoader::exists(const String &p_path, const String &p_type_hint) { else local_path = ProjectSettings::get_singleton()->localize_path(p_path); - if (ResourceCache::has(local_path)) { - - return true; // If cached, it probably exists + thread_load_mutex->lock(); + if (!thread_load_tasks.has(local_path)) { + thread_load_mutex->unlock(); + if (r_error) { + *r_error = ERR_INVALID_PARAMETER; + } + return RES(); } - bool xl_remapped = false; - String path = _path_remap(local_path, &xl_remapped); + ThreadLoadTask &load_task = thread_load_tasks[local_path]; - // Try all loaders and pick the first match for the type hint - for (int i = 0; i < loader_count; i++) { + //semaphore still exists, meaning its still loading, request poll + Semaphore *semaphore = load_task.semaphore; + if (semaphore) { + load_task.poll_requests++; - if (!loader[i]->recognize_path(path, p_type_hint)) { - continue; + { + // As we got a semaphore, this means we are going to have to wait + // until the sub-resource is done loading + // + // As this thread will become 'blocked' we should "echange" its + // active status with a waiting one, to ensure load continues. + // + // This ensures loading is never blocked and that is also within + // the maximum number of active threads. + + if (thread_waiting_count > 0) { + thread_waiting_count--; + thread_loading_count++; + thread_load_semaphore->post(); + + load_task.start_next = false; //do not start next since we are doing it here + } + + thread_suspended_count++; + + print_lt("GET: load count: " + itos(thread_loading_count) + " / wait count: " + itos(thread_waiting_count) + " / suspended count: " + itos(thread_suspended_count) + " / active: " + itos(thread_loading_count - thread_suspended_count)); } - if (loader[i]->exists(path)) - return true; + thread_load_mutex->unlock(); + semaphore->wait(); + thread_load_mutex->lock(); + + thread_suspended_count--; + + if (!thread_load_tasks.has(local_path)) { //may have been erased during unlock and this was always an invalid call + thread_load_mutex->unlock(); + if (r_error) { + *r_error = ERR_INVALID_PARAMETER; + } + return RES(); + } } - return false; + RES resource = load_task.resource; + if (r_error) { + *r_error = load_task.error; + } + + load_task.requests--; + + if (load_task.requests == 0) { + if (load_task.thread) { //thread may not have been used + Thread::wait_to_finish(load_task.thread); + memdelete(load_task.thread); + } + thread_load_tasks.erase(local_path); + } + + thread_load_mutex->unlock(); + + return resource; } -Ref<ResourceInteractiveLoader> ResourceLoader::load_interactive(const String &p_path, const String &p_type_hint, bool p_no_cache, Error *r_error) { +RES ResourceLoader::load(const String &p_path, const String &p_type_hint, bool p_no_cache, Error *r_error) { if (r_error) *r_error = ERR_CANT_OPEN; @@ -459,61 +529,131 @@ Ref<ResourceInteractiveLoader> ResourceLoader::load_interactive(const String &p_ if (!p_no_cache) { - bool success = _add_to_loading_map(local_path); - ERR_FAIL_COND_V_MSG(!success, RES(), "Resource: '" + local_path + "' is already being loaded. Cyclic reference?"); + thread_load_mutex->lock(); - if (ResourceCache::has(local_path)) { + //Is it already being loaded? poll until done + if (thread_load_tasks.has(local_path)) { + Error err = load_threaded_request(p_path, p_type_hint); + if (err != OK) { + if (r_error) { + *r_error = err; + } + return RES(); + } + thread_load_mutex->unlock(); - print_verbose("Loading resource: " + local_path + " (cached)"); - Ref<Resource> res_cached = ResourceCache::get(local_path); - Ref<ResourceInteractiveLoaderDefault> ril = Ref<ResourceInteractiveLoaderDefault>(memnew(ResourceInteractiveLoaderDefault)); + return load_threaded_get(p_path, r_error); + } - ril->resource = res_cached; - ril->path_loading = local_path; - ril->path_loading_thread = Thread::get_caller_id(); - return ril; + //Is it cached? + if (ResourceCache::lock) { + ResourceCache::lock->read_lock(); } - } - bool xl_remapped = false; - String path = _path_remap(local_path, &xl_remapped); - if (path == "") { - if (!p_no_cache) { - _remove_from_loading_map(local_path); + Resource **rptr = ResourceCache::resources.getptr(local_path); + + if (rptr) { + RES res(*rptr); + + //it is possible this resource was just freed in a thread. If so, this referencing will not work and resource is considered not cached + if (res.is_valid()) { + if (ResourceCache::lock) { + ResourceCache::lock->read_unlock(); + } + thread_load_mutex->unlock(); + + if (r_error) { + *r_error = OK; + } + + return res; //use cached + } } - ERR_FAIL_V_MSG(RES(), "Remapping '" + local_path + "' failed."); - } - print_verbose("Loading resource: " + path); + if (ResourceCache::lock) { + ResourceCache::lock->read_unlock(); + } - bool found = false; - for (int i = 0; i < loader_count; i++) { + //load using task (but this thread) + ThreadLoadTask load_task; - if (!loader[i]->recognize_path(path, p_type_hint)) - continue; - found = true; - Ref<ResourceInteractiveLoader> ril = loader[i]->load_interactive(path, local_path, r_error); - if (ril.is_null()) - continue; - if (!p_no_cache) { - ril->set_local_path(local_path); - ril->path_loading = local_path; - ril->path_loading_thread = Thread::get_caller_id(); + load_task.requests = 1; + load_task.local_path = local_path; + load_task.remapped_path = _path_remap(local_path, &load_task.xl_remapped); + load_task.type_hint = p_type_hint; + load_task.loader_id = Thread::get_caller_id(); + + thread_load_tasks[local_path] = load_task; + + thread_load_mutex->unlock(); + + _thread_load_function(&thread_load_tasks[local_path]); + + return load_threaded_get(p_path, r_error); + + } else { + + bool xl_remapped = false; + String path = _path_remap(local_path, &xl_remapped); + + if (path == "") { + ERR_FAIL_V_MSG(RES(), "Remapping '" + local_path + "' failed."); + } + + print_verbose("Loading resource: " + path); + float p; + RES res = _load(path, local_path, p_type_hint, p_no_cache, r_error, false, &p); + + if (res.is_null()) { + print_verbose("Failed loading resource: " + path); + return RES(); } if (xl_remapped) - ril->set_translation_remapped(true); + res->set_as_translation_remapped(true); + +#ifdef TOOLS_ENABLED + + res->set_edited(false); + if (timestamp_on_load) { + uint64_t mt = FileAccess::get_modified_time(path); + //printf("mt %s: %lli\n",remapped_path.utf8().get_data(),mt); + res->set_last_modified_time(mt); + } +#endif - return ril; + return res; } +} - if (!p_no_cache) { - _remove_from_loading_map(local_path); +bool ResourceLoader::exists(const String &p_path, const String &p_type_hint) { + + String local_path; + if (p_path.is_rel_path()) + local_path = "res://" + p_path; + else + local_path = ProjectSettings::get_singleton()->localize_path(p_path); + + if (ResourceCache::has(local_path)) { + + return true; // If cached, it probably exists } - ERR_FAIL_COND_V_MSG(found, Ref<ResourceInteractiveLoader>(), "Failed loading resource: " + path + "."); + bool xl_remapped = false; + String path = _path_remap(local_path, &xl_remapped); - ERR_FAIL_V_MSG(Ref<ResourceInteractiveLoader>(), "No loader found for resource: " + path + "."); + // Try all loaders and pick the first match for the type hint + for (int i = 0; i < loader_count; i++) { + + if (!loader[i]->recognize_path(path, p_type_hint)) { + continue; + } + + if (loader[i]->exists(path)) + return true; + } + + return false; } void ResourceLoader::add_resource_format_loader(Ref<ResourceFormatLoader> p_format_loader, bool p_at_front) { @@ -984,20 +1124,19 @@ void ResourceLoader::remove_custom_loaders() { } } -Mutex ResourceLoader::loading_map_mutex; -HashMap<ResourceLoader::LoadingMapKey, int, ResourceLoader::LoadingMapKeyHasher> ResourceLoader::loading_map; - void ResourceLoader::initialize() { + thread_load_mutex = memnew(Mutex); + thread_load_max = OS::get_singleton()->get_processor_count(); + thread_loading_count = 0; + thread_waiting_count = 0; + thread_suspended_count = 0; + thread_load_semaphore = memnew(Semaphore); } void ResourceLoader::finalize() { -#ifndef NO_THREADS - const LoadingMapKey *K = NULL; - while ((K = loading_map.next(K))) { - ERR_PRINT("Exited while resource is being loaded: " + K->path); - } - loading_map.clear(); -#endif + + memdelete(thread_load_mutex); + memdelete(thread_load_semaphore); } ResourceLoadErrorNotify ResourceLoader::err_notify = NULL; @@ -1009,6 +1148,15 @@ void *ResourceLoader::dep_err_notify_ud = NULL; bool ResourceLoader::abort_on_missing_resource = true; bool ResourceLoader::timestamp_on_load = false; +Mutex *ResourceLoader::thread_load_mutex = nullptr; +HashMap<String, ResourceLoader::ThreadLoadTask> ResourceLoader::thread_load_tasks; +Semaphore *ResourceLoader::thread_load_semaphore = nullptr; + +int ResourceLoader::thread_loading_count = 0; +int ResourceLoader::thread_waiting_count = 0; +int ResourceLoader::thread_suspended_count = 0; +int ResourceLoader::thread_load_max = 0; + SelfList<Resource>::List ResourceLoader::remapped_list; HashMap<String, Vector<String> > ResourceLoader::translation_remaps; HashMap<String, String> ResourceLoader::path_remaps; diff --git a/core/io/resource_loader.h b/core/io/resource_loader.h index 172f8e979b..3b7a27f551 100644 --- a/core/io/resource_loader.h +++ b/core/io/resource_loader.h @@ -31,32 +31,10 @@ #ifndef RESOURCE_LOADER_H #define RESOURCE_LOADER_H +#include "core/os/semaphore.h" #include "core/os/thread.h" #include "core/resource.h" -class ResourceInteractiveLoader : public Reference { - - GDCLASS(ResourceInteractiveLoader, Reference); - friend class ResourceLoader; - String path_loading; - Thread::ID path_loading_thread; - -protected: - static void _bind_methods(); - -public: - virtual void set_local_path(const String &p_local_path) = 0; - virtual Ref<Resource> get_resource() = 0; - virtual Error poll() = 0; - virtual int get_stage() const = 0; - virtual int get_stage_count() const = 0; - virtual void set_translation_remapped(bool p_remapped) = 0; - virtual Error wait(); - - ResourceInteractiveLoader() {} - ~ResourceInteractiveLoader(); -}; - class ResourceFormatLoader : public Reference { GDCLASS(ResourceFormatLoader, Reference); @@ -65,8 +43,7 @@ protected: static void _bind_methods(); public: - virtual Ref<ResourceInteractiveLoader> load_interactive(const String &p_path, const String &p_original_path = "", Error *r_error = NULL); - virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL); + virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL, bool p_use_sub_threads = false, float *r_progress = nullptr); virtual bool exists(const String &p_path) const; virtual void get_recognized_extensions(List<String> *p_extensions) const; virtual void get_recognized_extensions_for_type(const String &p_type, List<String> *p_extensions) const; @@ -95,6 +72,15 @@ class ResourceLoader { MAX_LOADERS = 64 }; +public: + enum ThreadLoadStatus { + THREAD_LOAD_INVALID_RESOURCE, + THREAD_LOAD_IN_PROGRESS, + THREAD_LOAD_FAILED, + THREAD_LOAD_LOADED + }; + +private: static Ref<ResourceFormatLoader> loader[MAX_LOADERS]; static int loader_count; static bool timestamp_on_load; @@ -115,34 +101,47 @@ class ResourceLoader { friend class ResourceFormatImporter; friend class ResourceInteractiveLoader; //internal load function - static RES _load(const String &p_path, const String &p_original_path, const String &p_type_hint, bool p_no_cache, Error *r_error); + static RES _load(const String &p_path, const String &p_original_path, const String &p_type_hint, bool p_no_cache, Error *r_error, bool p_use_sub_threads, float *r_progress); static ResourceLoadedCallback _loaded_callback; static Ref<ResourceFormatLoader> _find_custom_resource_format_loader(String path); - static Mutex loading_map_mutex; - - //used to track paths being loaded in a thread, avoids cyclic recursion - struct LoadingMapKey { - String path; - Thread::ID thread; - bool operator==(const LoadingMapKey &p_key) const { - return (thread == p_key.thread && path == p_key.path); - } - }; - struct LoadingMapKeyHasher { - static _FORCE_INLINE_ uint32_t hash(const LoadingMapKey &p_key) { return p_key.path.hash() + HashMapHasherDefault::hash(p_key.thread); } + struct ThreadLoadTask { + Thread *thread = nullptr; + Thread::ID loader_id = 0; + Semaphore *semaphore = nullptr; + String local_path; + String remapped_path; + String type_hint; + float progress = 0.0; + ThreadLoadStatus status = THREAD_LOAD_IN_PROGRESS; + Error error; + RES resource; + bool xl_remapped = false; + bool use_sub_threads = false; + bool start_next = true; + int requests = 0; + int poll_requests = 0; + Set<String> sub_tasks; }; - static HashMap<LoadingMapKey, int, LoadingMapKeyHasher> loading_map; + static void _thread_load_function(void *p_userdata); + static Mutex *thread_load_mutex; + static HashMap<String, ThreadLoadTask> thread_load_tasks; + static Semaphore *thread_load_semaphore; + static int thread_waiting_count; + static int thread_loading_count; + static int thread_suspended_count; + static int thread_load_max; - static bool _add_to_loading_map(const String &p_path); - static void _remove_from_loading_map(const String &p_path); - static void _remove_from_loading_map_and_thread(const String &p_path, Thread::ID p_thread); + static float _dependency_get_progress(const String &p_path); public: - static Ref<ResourceInteractiveLoader> load_interactive(const String &p_path, const String &p_type_hint = "", bool p_no_cache = false, Error *r_error = NULL); + static Error load_threaded_request(const String &p_path, const String &p_type_hint = "", bool p_use_sub_threads = false, const String &p_source_resource = String()); + static ThreadLoadStatus load_threaded_get_status(const String &p_path, float *r_progress = nullptr); + static RES load_threaded_get(const String &p_path, Error *r_error = NULL); + static RES load(const String &p_path, const String &p_type_hint = "", bool p_no_cache = false, Error *r_error = NULL); static bool exists(const String &p_path, const String &p_type_hint = ""); diff --git a/core/io/translation_loader_po.cpp b/core/io/translation_loader_po.cpp index 4f7eeddc43..4051bf2947 100644 --- a/core/io/translation_loader_po.cpp +++ b/core/io/translation_loader_po.cpp @@ -176,7 +176,7 @@ RES TranslationLoaderPO::load_translation(FileAccess *f, Error *r_error, const S return translation; } -RES TranslationLoaderPO::load(const String &p_path, const String &p_original_path, Error *r_error) { +RES TranslationLoaderPO::load(const String &p_path, const String &p_original_path, Error *r_error, bool p_use_sub_threads, float *r_progress) { if (r_error) *r_error = ERR_CANT_OPEN; diff --git a/core/io/translation_loader_po.h b/core/io/translation_loader_po.h index 47e64276ca..fe3a75e5eb 100644 --- a/core/io/translation_loader_po.h +++ b/core/io/translation_loader_po.h @@ -38,7 +38,7 @@ class TranslationLoaderPO : public ResourceFormatLoader { public: static RES load_translation(FileAccess *f, Error *r_error, const String &p_path = String()); - virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL); + virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL, bool p_use_sub_threads = false, float *r_progress = nullptr); virtual void get_recognized_extensions(List<String> *p_extensions) const; virtual bool handles_type(const String &p_type) const; virtual String get_resource_type(const String &p_path) const; diff --git a/core/object.h b/core/object.h index dc7d49f534..59d3f06cfe 100644 --- a/core/object.h +++ b/core/object.h @@ -126,6 +126,7 @@ enum PropertyUsageFlags { PROPERTY_USAGE_NODE_PATH_FROM_SCENE_ROOT = 1 << 23, PROPERTY_USAGE_RESOURCE_NOT_PERSISTENT = 1 << 24, PROPERTY_USAGE_KEYING_INCREMENTS = 1 << 25, // Used in inspector to increment property when keyed in animation player + PROPERTY_USAGE_DEFERRED_SET_RESOURCE = 1 << 26, // when loading, the resource for this property can be set at the end of loading PROPERTY_USAGE_DEFAULT = PROPERTY_USAGE_STORAGE | PROPERTY_USAGE_EDITOR | PROPERTY_USAGE_NETWORK, PROPERTY_USAGE_DEFAULT_INTL = PROPERTY_USAGE_STORAGE | PROPERTY_USAGE_EDITOR | PROPERTY_USAGE_NETWORK | PROPERTY_USAGE_INTERNATIONALIZED, diff --git a/core/register_core_types.cpp b/core/register_core_types.cpp index ac60061c67..07a252ad31 100644 --- a/core/register_core_types.cpp +++ b/core/register_core_types.cpp @@ -186,8 +186,6 @@ void register_core_types() { ClassDB::register_class<HTTPClient>(); ClassDB::register_class<TriangleMesh>(); - ClassDB::register_virtual_class<ResourceInteractiveLoader>(); - ClassDB::register_class<ResourceFormatLoader>(); ClassDB::register_class<ResourceFormatSaver>(); diff --git a/drivers/dummy/texture_loader_dummy.cpp b/drivers/dummy/texture_loader_dummy.cpp index aff7bbd1bc..95876f5c7d 100644 --- a/drivers/dummy/texture_loader_dummy.cpp +++ b/drivers/dummy/texture_loader_dummy.cpp @@ -35,7 +35,7 @@ #include <string.h> -RES ResourceFormatDummyTexture::load(const String &p_path, const String &p_original_path, Error *r_error) { +RES ResourceFormatDummyTexture::load(const String &p_path, const String &p_original_path, Error *r_error, bool p_use_sub_threads, float *r_progress) { unsigned int width = 8; unsigned int height = 8; diff --git a/drivers/dummy/texture_loader_dummy.h b/drivers/dummy/texture_loader_dummy.h index 86c9a375a3..e5ae945706 100644 --- a/drivers/dummy/texture_loader_dummy.h +++ b/drivers/dummy/texture_loader_dummy.h @@ -36,7 +36,7 @@ class ResourceFormatDummyTexture : public ResourceFormatLoader { public: - virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL); + virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL, bool p_use_sub_threads = false, float *r_progress = nullptr); virtual void get_recognized_extensions(List<String> *p_extensions) const; virtual bool handles_type(const String &p_type) const; virtual String get_resource_type(const String &p_path) const; diff --git a/modules/dds/texture_loader_dds.cpp b/modules/dds/texture_loader_dds.cpp index a04989449c..ae21156b8b 100644 --- a/modules/dds/texture_loader_dds.cpp +++ b/modules/dds/texture_loader_dds.cpp @@ -94,7 +94,7 @@ static const DDSFormatInfo dds_format_info[DDS_MAX] = { { "GRAYSCALE_ALPHA", false, false, 1, 2, Image::FORMAT_LA8 } }; -RES ResourceFormatDDS::load(const String &p_path, const String &p_original_path, Error *r_error) { +RES ResourceFormatDDS::load(const String &p_path, const String &p_original_path, Error *r_error, bool p_use_sub_threads, float *r_progress) { if (r_error) *r_error = ERR_CANT_OPEN; diff --git a/modules/dds/texture_loader_dds.h b/modules/dds/texture_loader_dds.h index 34f4e9b548..de8088af90 100644 --- a/modules/dds/texture_loader_dds.h +++ b/modules/dds/texture_loader_dds.h @@ -36,7 +36,7 @@ class ResourceFormatDDS : public ResourceFormatLoader { public: - virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL); + virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL, bool p_use_sub_threads = false, float *r_progress = nullptr); virtual void get_recognized_extensions(List<String> *p_extensions) const; virtual bool handles_type(const String &p_type) const; virtual String get_resource_type(const String &p_path) const; diff --git a/modules/etc/texture_loader_pkm.cpp b/modules/etc/texture_loader_pkm.cpp index e460c70cec..ad0cc91c96 100644 --- a/modules/etc/texture_loader_pkm.cpp +++ b/modules/etc/texture_loader_pkm.cpp @@ -42,7 +42,7 @@ struct ETC1Header { uint16_t origHeight; }; -RES ResourceFormatPKM::load(const String &p_path, const String &p_original_path, Error *r_error) { +RES ResourceFormatPKM::load(const String &p_path, const String &p_original_path, Error *r_error, bool p_use_sub_threads, float *r_progress) { if (r_error) *r_error = ERR_CANT_OPEN; diff --git a/modules/etc/texture_loader_pkm.h b/modules/etc/texture_loader_pkm.h index 3d52a9ea95..d6011993e3 100644 --- a/modules/etc/texture_loader_pkm.h +++ b/modules/etc/texture_loader_pkm.h @@ -36,7 +36,7 @@ class ResourceFormatPKM : public ResourceFormatLoader { public: - virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL); + virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL, bool p_use_sub_threads = false, float *r_progress = nullptr); virtual void get_recognized_extensions(List<String> *p_extensions) const; virtual bool handles_type(const String &p_type) const; virtual String get_resource_type(const String &p_path) const; diff --git a/modules/gdnative/gdnative.cpp b/modules/gdnative/gdnative.cpp index 1571b821a5..33b734f672 100644 --- a/modules/gdnative/gdnative.cpp +++ b/modules/gdnative/gdnative.cpp @@ -493,7 +493,7 @@ Error GDNative::get_symbol(StringName p_procedure_name, void *&r_handle, bool p_ return result; } -RES GDNativeLibraryResourceLoader::load(const String &p_path, const String &p_original_path, Error *r_error) { +RES GDNativeLibraryResourceLoader::load(const String &p_path, const String &p_original_path, Error *r_error, bool p_use_sub_threads, float *r_progress) { Ref<GDNativeLibrary> lib; lib.instance(); diff --git a/modules/gdnative/gdnative.h b/modules/gdnative/gdnative.h index c0c2cdf24c..b4c5ec9d00 100644 --- a/modules/gdnative/gdnative.h +++ b/modules/gdnative/gdnative.h @@ -166,7 +166,7 @@ public: class GDNativeLibraryResourceLoader : public ResourceFormatLoader { public: - virtual RES load(const String &p_path, const String &p_original_path, Error *r_error); + virtual RES load(const String &p_path, const String &p_original_path, Error *r_error, bool p_use_sub_threads = false, float *r_progress = nullptr); virtual void get_recognized_extensions(List<String> *p_extensions) const; virtual bool handles_type(const String &p_type) const; virtual String get_resource_type(const String &p_path) const; diff --git a/modules/gdnative/nativescript/nativescript.cpp b/modules/gdnative/nativescript/nativescript.cpp index 0ec6ad71d9..d0e196b3e6 100644 --- a/modules/gdnative/nativescript/nativescript.cpp +++ b/modules/gdnative/nativescript/nativescript.cpp @@ -1931,7 +1931,7 @@ void NativeReloadNode::_notification(int p_what) { #endif } -RES ResourceFormatLoaderNativeScript::load(const String &p_path, const String &p_original_path, Error *r_error) { +RES ResourceFormatLoaderNativeScript::load(const String &p_path, const String &p_original_path, Error *r_error, bool p_use_sub_threads, float *r_progress) { return ResourceFormatLoaderText::singleton->load(p_path, p_original_path, r_error); } diff --git a/modules/gdnative/nativescript/nativescript.h b/modules/gdnative/nativescript/nativescript.h index 04670b952b..90542c96b7 100644 --- a/modules/gdnative/nativescript/nativescript.h +++ b/modules/gdnative/nativescript/nativescript.h @@ -406,7 +406,7 @@ public: class ResourceFormatLoaderNativeScript : public ResourceFormatLoader { public: - virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL); + virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL, bool p_use_sub_threads = false, float *r_progress = nullptr); virtual void get_recognized_extensions(List<String> *p_extensions) const; virtual bool handles_type(const String &p_type) const; virtual String get_resource_type(const String &p_path) const; diff --git a/modules/gdnative/pluginscript/pluginscript_loader.cpp b/modules/gdnative/pluginscript/pluginscript_loader.cpp index fe1cc83c69..46db20b6c2 100644 --- a/modules/gdnative/pluginscript/pluginscript_loader.cpp +++ b/modules/gdnative/pluginscript/pluginscript_loader.cpp @@ -39,7 +39,7 @@ ResourceFormatLoaderPluginScript::ResourceFormatLoaderPluginScript(PluginScriptL _language = language; } -RES ResourceFormatLoaderPluginScript::load(const String &p_path, const String &p_original_path, Error *r_error) { +RES ResourceFormatLoaderPluginScript::load(const String &p_path, const String &p_original_path, Error *r_error, bool p_use_sub_threads, float *r_progress) { if (r_error) *r_error = ERR_FILE_CANT_OPEN; diff --git a/modules/gdnative/pluginscript/pluginscript_loader.h b/modules/gdnative/pluginscript/pluginscript_loader.h index ede31c027e..a039072fdc 100644 --- a/modules/gdnative/pluginscript/pluginscript_loader.h +++ b/modules/gdnative/pluginscript/pluginscript_loader.h @@ -44,7 +44,7 @@ class ResourceFormatLoaderPluginScript : public ResourceFormatLoader { public: ResourceFormatLoaderPluginScript(PluginScriptLanguage *language); - virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL); + virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL, bool p_use_sub_threads = false, float *r_progress = nullptr); virtual void get_recognized_extensions(List<String> *p_extensions) const; virtual bool handles_type(const String &p_type) const; virtual String get_resource_type(const String &p_path) const; diff --git a/modules/gdnative/videodecoder/video_stream_gdnative.cpp b/modules/gdnative/videodecoder/video_stream_gdnative.cpp index bd16563ff0..f6e2bad739 100644 --- a/modules/gdnative/videodecoder/video_stream_gdnative.cpp +++ b/modules/gdnative/videodecoder/video_stream_gdnative.cpp @@ -373,7 +373,7 @@ void VideoStreamGDNative::set_audio_track(int p_track) { /* --- NOTE ResourceFormatLoaderVideoStreamGDNative starts here. ----- */ -RES ResourceFormatLoaderVideoStreamGDNative::load(const String &p_path, const String &p_original_path, Error *r_error) { +RES ResourceFormatLoaderVideoStreamGDNative::load(const String &p_path, const String &p_original_path, Error *r_error, bool p_use_sub_threads, float *r_progress) { FileAccess *f = FileAccess::open(p_path, FileAccess::READ); if (!f) { if (r_error) { diff --git a/modules/gdnative/videodecoder/video_stream_gdnative.h b/modules/gdnative/videodecoder/video_stream_gdnative.h index 024cdec196..21b5245a16 100644 --- a/modules/gdnative/videodecoder/video_stream_gdnative.h +++ b/modules/gdnative/videodecoder/video_stream_gdnative.h @@ -199,7 +199,7 @@ public: class ResourceFormatLoaderVideoStreamGDNative : public ResourceFormatLoader { public: - virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL); + virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL, bool p_use_sub_threads = false, float *r_progress = nullptr); virtual void get_recognized_extensions(List<String> *p_extensions) const; virtual bool handles_type(const String &p_type) const; virtual String get_resource_type(const String &p_path) const; diff --git a/modules/gdscript/gdscript.cpp b/modules/gdscript/gdscript.cpp index 6a4bf801f3..c641ce37c5 100644 --- a/modules/gdscript/gdscript.cpp +++ b/modules/gdscript/gdscript.cpp @@ -2251,7 +2251,7 @@ Ref<GDScript> GDScriptLanguage::get_orphan_subclass(const String &p_qualified_na /*************** RESOURCE ***************/ -RES ResourceFormatLoaderGDScript::load(const String &p_path, const String &p_original_path, Error *r_error) { +RES ResourceFormatLoaderGDScript::load(const String &p_path, const String &p_original_path, Error *r_error, bool p_use_sub_threads, float *r_progress) { if (r_error) *r_error = ERR_FILE_CANT_OPEN; diff --git a/modules/gdscript/gdscript.h b/modules/gdscript/gdscript.h index 6b53acd064..3a90f0fc20 100644 --- a/modules/gdscript/gdscript.h +++ b/modules/gdscript/gdscript.h @@ -542,7 +542,7 @@ public: class ResourceFormatLoaderGDScript : public ResourceFormatLoader { public: - virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL); + virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL, bool p_use_sub_threads = false, float *r_progress = nullptr); virtual void get_recognized_extensions(List<String> *p_extensions) const; virtual bool handles_type(const String &p_type) const; virtual String get_resource_type(const String &p_path) const; diff --git a/modules/mono/csharp_script.cpp b/modules/mono/csharp_script.cpp index 404d2eb022..6809cbdff9 100644 --- a/modules/mono/csharp_script.cpp +++ b/modules/mono/csharp_script.cpp @@ -3451,7 +3451,7 @@ CSharpScript::~CSharpScript() { /*************** RESOURCE ***************/ -RES ResourceFormatLoaderCSharpScript::load(const String &p_path, const String &p_original_path, Error *r_error) { +RES ResourceFormatLoaderCSharpScript::load(const String &p_path, const String &p_original_path, Error *r_error, bool p_use_sub_threads, float *r_progress) { if (r_error) *r_error = ERR_FILE_CANT_OPEN; diff --git a/modules/mono/csharp_script.h b/modules/mono/csharp_script.h index f44c4aebc4..18c53aab52 100644 --- a/modules/mono/csharp_script.h +++ b/modules/mono/csharp_script.h @@ -497,7 +497,7 @@ public: class ResourceFormatLoaderCSharpScript : public ResourceFormatLoader { public: - virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL); + virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL, bool p_use_sub_threads = false, float *r_progress = nullptr); virtual void get_recognized_extensions(List<String> *p_extensions) const; virtual bool handles_type(const String &p_type) const; virtual String get_resource_type(const String &p_path) const; diff --git a/modules/opus/audio_stream_opus.cpp b/modules/opus/audio_stream_opus.cpp index 67a07628d9..a983edee91 100644 --- a/modules/opus/audio_stream_opus.cpp +++ b/modules/opus/audio_stream_opus.cpp @@ -354,7 +354,7 @@ AudioStreamPlaybackOpus::~AudioStreamPlaybackOpus() { _clear_stream(); } -RES ResourceFormatLoaderAudioStreamOpus::load(const String &p_path, const String &p_original_path, Error *r_error) { +RES ResourceFormatLoaderAudioStreamOpus::load(const String &p_path, const String &p_original_path, Error *r_error, bool p_use_sub_threads, float *r_progress) { if (r_error) *r_error = OK; diff --git a/modules/opus/audio_stream_opus.h b/modules/opus/audio_stream_opus.h index 5c0a02a9d0..343cbc6b83 100644 --- a/modules/opus/audio_stream_opus.h +++ b/modules/opus/audio_stream_opus.h @@ -133,7 +133,7 @@ public: class ResourceFormatLoaderAudioStreamOpus : public ResourceFormatLoader { public: - virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL); + virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL, bool p_use_sub_threads = false, float *r_progress = nullptr); virtual void get_recognized_extensions(List<String> *p_extensions) const; virtual bool handles_type(const String &p_type) const; virtual String get_resource_type(const String &p_path) const; diff --git a/modules/pvr/texture_loader_pvr.cpp b/modules/pvr/texture_loader_pvr.cpp index dab4f64393..179c6f692b 100644 --- a/modules/pvr/texture_loader_pvr.cpp +++ b/modules/pvr/texture_loader_pvr.cpp @@ -51,7 +51,7 @@ enum PVRFLags { }; -RES ResourceFormatPVR::load(const String &p_path, const String &p_original_path, Error *r_error) { +RES ResourceFormatPVR::load(const String &p_path, const String &p_original_path, Error *r_error, bool p_use_sub_threads, float *r_progress) { if (r_error) *r_error = ERR_CANT_OPEN; diff --git a/modules/pvr/texture_loader_pvr.h b/modules/pvr/texture_loader_pvr.h index e384ed2b4c..c990c3612b 100644 --- a/modules/pvr/texture_loader_pvr.h +++ b/modules/pvr/texture_loader_pvr.h @@ -36,7 +36,7 @@ class ResourceFormatPVR : public ResourceFormatLoader { public: - virtual RES load(const String &p_path, const String &p_original_path, Error *r_error = NULL); + virtual RES load(const String &p_path, const String &p_original_path, Error *r_error = NULL, bool p_use_sub_threads = false, float *r_progress = nullptr); virtual void get_recognized_extensions(List<String> *p_extensions) const; virtual bool handles_type(const String &p_type) const; virtual String get_resource_type(const String &p_path) const; diff --git a/modules/theora/video_stream_theora.cpp b/modules/theora/video_stream_theora.cpp index a44ed0304d..ee44c6bf05 100644 --- a/modules/theora/video_stream_theora.cpp +++ b/modules/theora/video_stream_theora.cpp @@ -725,7 +725,7 @@ void VideoStreamTheora::_bind_methods() { //////////// -RES ResourceFormatLoaderTheora::load(const String &p_path, const String &p_original_path, Error *r_error) { +RES ResourceFormatLoaderTheora::load(const String &p_path, const String &p_original_path, Error *r_error, bool p_use_sub_threads, float *r_progress) { FileAccess *f = FileAccess::open(p_path, FileAccess::READ); if (!f) { diff --git a/modules/theora/video_stream_theora.h b/modules/theora/video_stream_theora.h index 258b3a1cce..da3558ce34 100644 --- a/modules/theora/video_stream_theora.h +++ b/modules/theora/video_stream_theora.h @@ -187,7 +187,7 @@ public: class ResourceFormatLoaderTheora : public ResourceFormatLoader { public: - virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL); + virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL, bool p_use_sub_threads = false, float *r_progress = nullptr); virtual void get_recognized_extensions(List<String> *p_extensions) const; virtual bool handles_type(const String &p_type) const; virtual String get_resource_type(const String &p_path) const; diff --git a/modules/vorbis/audio_stream_ogg_vorbis.cpp b/modules/vorbis/audio_stream_ogg_vorbis.cpp index 87067faf8e..1e8ee9b17e 100644 --- a/modules/vorbis/audio_stream_ogg_vorbis.cpp +++ b/modules/vorbis/audio_stream_ogg_vorbis.cpp @@ -381,7 +381,7 @@ AudioStreamPlaybackOGGVorbis::~AudioStreamPlaybackOGGVorbis() { _clear_stream(); } -RES ResourceFormatLoaderAudioStreamOGGVorbis::load(const String &p_path, const String &p_original_path, Error *r_error) { +RES ResourceFormatLoaderAudioStreamOGGVorbis::load(const String &p_path, const String &p_original_path, Error *r_error, bool p_use_sub_threads, float *r_progress) { if (r_error) *r_error = OK; diff --git a/modules/vorbis/audio_stream_ogg_vorbis.h b/modules/vorbis/audio_stream_ogg_vorbis.h index 739765a12f..db621f88d2 100644 --- a/modules/vorbis/audio_stream_ogg_vorbis.h +++ b/modules/vorbis/audio_stream_ogg_vorbis.h @@ -128,7 +128,7 @@ public: class ResourceFormatLoaderAudioStreamOGGVorbis : public ResourceFormatLoader { public: - virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL); + virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL, bool p_use_sub_threads = false, float *r_progress = nullptr); virtual void get_recognized_extensions(List<String> *p_extensions) const; virtual bool handles_type(const String &p_type) const; virtual String get_resource_type(const String &p_path) const; diff --git a/modules/webm/video_stream_webm.cpp b/modules/webm/video_stream_webm.cpp index 5768392fe5..265383831e 100644 --- a/modules/webm/video_stream_webm.cpp +++ b/modules/webm/video_stream_webm.cpp @@ -474,7 +474,7 @@ void VideoStreamWebm::set_audio_track(int p_track) { //////////// -RES ResourceFormatLoaderWebm::load(const String &p_path, const String &p_original_path, Error *r_error) { +RES ResourceFormatLoaderWebm::load(const String &p_path, const String &p_original_path, Error *r_error, bool p_use_sub_threads, float *r_progress) { FileAccess *f = FileAccess::open(p_path, FileAccess::READ); if (!f) { diff --git a/modules/webm/video_stream_webm.h b/modules/webm/video_stream_webm.h index a3d3c173f4..3feaa1278f 100644 --- a/modules/webm/video_stream_webm.h +++ b/modules/webm/video_stream_webm.h @@ -128,7 +128,7 @@ public: class ResourceFormatLoaderWebm : public ResourceFormatLoader { public: - virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL); + virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL, bool p_use_sub_threads = false, float *r_progress = nullptr); virtual void get_recognized_extensions(List<String> *p_extensions) const; virtual bool handles_type(const String &p_type) const; virtual String get_resource_type(const String &p_path) const; diff --git a/scene/3d/mesh_instance.cpp b/scene/3d/mesh_instance.cpp index a48d148d33..4ca139ebbc 100644 --- a/scene/3d/mesh_instance.cpp +++ b/scene/3d/mesh_instance.cpp @@ -100,7 +100,7 @@ void MeshInstance::_get_property_list(List<PropertyInfo> *p_list) const { if (mesh.is_valid()) { for (int i = 0; i < mesh->get_surface_count(); i++) { - p_list->push_back(PropertyInfo(Variant::OBJECT, "material/" + itos(i), PROPERTY_HINT_RESOURCE_TYPE, "ShaderMaterial,StandardMaterial3D")); + p_list->push_back(PropertyInfo(Variant::OBJECT, "material/" + itos(i), PROPERTY_HINT_RESOURCE_TYPE, "ShaderMaterial,StandardMaterial3D", PROPERTY_USAGE_DEFAULT | PROPERTY_USAGE_DEFERRED_SET_RESOURCE)); } } } diff --git a/scene/3d/visual_instance.cpp b/scene/3d/visual_instance.cpp index 73cffe9c89..105ce9ca74 100644 --- a/scene/3d/visual_instance.cpp +++ b/scene/3d/visual_instance.cpp @@ -294,7 +294,7 @@ void GeometryInstance::_bind_methods() { ClassDB::bind_method(D_METHOD("get_aabb"), &GeometryInstance::get_aabb); ADD_GROUP("Geometry", ""); - ADD_PROPERTY(PropertyInfo(Variant::OBJECT, "material_override", PROPERTY_HINT_RESOURCE_TYPE, "ShaderMaterial,StandardMaterial3D"), "set_material_override", "get_material_override"); + ADD_PROPERTY(PropertyInfo(Variant::OBJECT, "material_override", PROPERTY_HINT_RESOURCE_TYPE, "ShaderMaterial,StandardMaterial3D", PROPERTY_USAGE_DEFAULT | PROPERTY_USAGE_DEFERRED_SET_RESOURCE), "set_material_override", "get_material_override"); ADD_PROPERTY(PropertyInfo(Variant::INT, "cast_shadow", PROPERTY_HINT_ENUM, "Off,On,Double-Sided,Shadows Only"), "set_cast_shadows_setting", "get_cast_shadows_setting"); ADD_PROPERTY(PropertyInfo(Variant::FLOAT, "extra_cull_margin", PROPERTY_HINT_RANGE, "0,16384,0.01"), "set_extra_cull_margin", "get_extra_cull_margin"); ADD_PROPERTYI(PropertyInfo(Variant::BOOL, "use_in_baked_light"), "set_flag", "get_flag", FLAG_USE_BAKED_LIGHT); diff --git a/scene/resources/dynamic_font.cpp b/scene/resources/dynamic_font.cpp index fc53dfe376..ebd5b02dbc 100644 --- a/scene/resources/dynamic_font.cpp +++ b/scene/resources/dynamic_font.cpp @@ -1063,7 +1063,7 @@ void DynamicFont::update_oversampling() { ///////////////////////// -RES ResourceFormatLoaderDynamicFont::load(const String &p_path, const String &p_original_path, Error *r_error) { +RES ResourceFormatLoaderDynamicFont::load(const String &p_path, const String &p_original_path, Error *r_error, bool p_use_sub_threads, float *r_progress) { if (r_error) *r_error = ERR_FILE_CANT_OPEN; diff --git a/scene/resources/dynamic_font.h b/scene/resources/dynamic_font.h index 6a0b51fb93..c10f1e6681 100644 --- a/scene/resources/dynamic_font.h +++ b/scene/resources/dynamic_font.h @@ -302,7 +302,7 @@ VARIANT_ENUM_CAST(DynamicFont::SpacingType); class ResourceFormatLoaderDynamicFont : public ResourceFormatLoader { public: - virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL); + virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL, bool p_use_sub_threads = false, float *r_progress = nullptr); virtual void get_recognized_extensions(List<String> *p_extensions) const; virtual bool handles_type(const String &p_type) const; virtual String get_resource_type(const String &p_path) const; diff --git a/scene/resources/font.cpp b/scene/resources/font.cpp index 316d6f04d7..1f5e4b647a 100644 --- a/scene/resources/font.cpp +++ b/scene/resources/font.cpp @@ -646,7 +646,7 @@ BitmapFont::~BitmapFont() { //////////// -RES ResourceFormatLoaderBMFont::load(const String &p_path, const String &p_original_path, Error *r_error) { +RES ResourceFormatLoaderBMFont::load(const String &p_path, const String &p_original_path, Error *r_error, bool p_use_sub_threads, float *r_progress) { if (r_error) *r_error = ERR_FILE_CANT_OPEN; diff --git a/scene/resources/font.h b/scene/resources/font.h index 1ce8e79f09..85b295b5f8 100644 --- a/scene/resources/font.h +++ b/scene/resources/font.h @@ -200,7 +200,7 @@ public: class ResourceFormatLoaderBMFont : public ResourceFormatLoader { public: - virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL); + virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL, bool p_use_sub_threads = false, float *r_progress = nullptr); virtual void get_recognized_extensions(List<String> *p_extensions) const; virtual bool handles_type(const String &p_type) const; virtual String get_resource_type(const String &p_path) const; diff --git a/scene/resources/mesh.cpp b/scene/resources/mesh.cpp index 08c4169167..a063b7f74f 100644 --- a/scene/resources/mesh.cpp +++ b/scene/resources/mesh.cpp @@ -1004,7 +1004,6 @@ void ArrayMesh::_set_surfaces(const Array &p_surfaces) { } else { // if mesh does not exist (first time this is loaded, most likely), // we can create it with a single call, which is a lot more efficient and thread friendly - print_line("create mesh from surfaces: " + itos(surface_data.size())); mesh = VS::get_singleton()->mesh_create_from_surfaces(surface_data); VS::get_singleton()->mesh_set_blend_shape_mode(mesh, (VS::BlendShapeMode)blend_shape_mode); } diff --git a/scene/resources/resource_format_text.cpp b/scene/resources/resource_format_text.cpp index 956cc0bc4a..f07161d620 100644 --- a/scene/resources/resource_format_text.cpp +++ b/scene/resources/resource_format_text.cpp @@ -45,17 +45,17 @@ /// -void ResourceInteractiveLoaderText::set_local_path(const String &p_local_path) { +void ResourceLoaderText::set_local_path(const String &p_local_path) { res_path = p_local_path; } -Ref<Resource> ResourceInteractiveLoaderText::get_resource() { +Ref<Resource> ResourceLoaderText::get_resource() { return resource; } -Error ResourceInteractiveLoaderText::_parse_sub_resource_dummy(DummyReadData *p_data, VariantParser::Stream *p_stream, Ref<Resource> &r_res, int &line, String &r_err_str) { +Error ResourceLoaderText::_parse_sub_resource_dummy(DummyReadData *p_data, VariantParser::Stream *p_stream, Ref<Resource> &r_res, int &line, String &r_err_str) { VariantParser::Token token; VariantParser::get_token(p_stream, token, line, r_err_str); @@ -85,7 +85,7 @@ Error ResourceInteractiveLoaderText::_parse_sub_resource_dummy(DummyReadData *p_ return OK; } -Error ResourceInteractiveLoaderText::_parse_ext_resource_dummy(DummyReadData *p_data, VariantParser::Stream *p_stream, Ref<Resource> &r_res, int &line, String &r_err_str) { +Error ResourceLoaderText::_parse_ext_resource_dummy(DummyReadData *p_data, VariantParser::Stream *p_stream, Ref<Resource> &r_res, int &line, String &r_err_str) { VariantParser::Token token; VariantParser::get_token(p_stream, token, line, r_err_str); @@ -109,7 +109,7 @@ Error ResourceInteractiveLoaderText::_parse_ext_resource_dummy(DummyReadData *p_ return OK; } -Error ResourceInteractiveLoaderText::_parse_sub_resource(VariantParser::Stream *p_stream, Ref<Resource> &r_res, int &line, String &r_err_str) { +Error ResourceLoaderText::_parse_sub_resource(VariantParser::Stream *p_stream, Ref<Resource> &r_res, int &line, String &r_err_str) { VariantParser::Token token; VariantParser::get_token(p_stream, token, line, r_err_str); @@ -143,7 +143,7 @@ Error ResourceInteractiveLoaderText::_parse_sub_resource(VariantParser::Stream * return OK; } -Error ResourceInteractiveLoaderText::_parse_ext_resource(VariantParser::Stream *p_stream, Ref<Resource> &r_res, int &line, String &r_err_str) { +Error ResourceLoaderText::_parse_ext_resource(VariantParser::Stream *p_stream, Ref<Resource> &r_res, int &line, String &r_err_str) { VariantParser::Token token; VariantParser::get_token(p_stream, token, line, r_err_str); @@ -164,15 +164,30 @@ Error ResourceInteractiveLoaderText::_parse_ext_resource(VariantParser::Stream * String path = ext_resources[id].path; String type = ext_resources[id].type; - if (path.find("://") == -1 && path.is_rel_path()) { - // path is relative to file being loaded, so convert to a resource path - path = ProjectSettings::get_singleton()->localize_path(res_path.get_base_dir().plus_file(path)); - } + if (ext_resources[id].cache.is_valid()) { + r_res = ext_resources[id].cache; + } else if (use_sub_threads) { - r_res = ResourceLoader::load(path, type); + RES res = ResourceLoader::load_threaded_get(path); + if (res.is_null()) { - if (r_res.is_null()) { - WARN_PRINT(String("Couldn't load external resource: " + path).utf8().get_data()); + if (ResourceLoader::get_abort_on_missing_resources()) { + error = ERR_FILE_CORRUPT; + error_text = "[ext_resource] referenced nonexistent resource at: " + path; + _printerr(); + return error; + } else { + ResourceLoader::notify_dependency_error(local_path, path, type); + } + } else { + ext_resources[id].cache = res; + r_res = res; + } + } else { + error = ERR_FILE_CORRUPT; + error_text = "[ext_resource] referenced non-loaded resource at: " + path; + _printerr(); + return error; } } else { r_res = RES(); @@ -187,7 +202,7 @@ Error ResourceInteractiveLoaderText::_parse_ext_resource(VariantParser::Stream * return OK; } -Ref<PackedScene> ResourceInteractiveLoaderText::_parse_node_tag(VariantParser::ResourceParser &parser) { +Ref<PackedScene> ResourceLoaderText::_parse_node_tag(VariantParser::ResourceParser &parser) { Ref<PackedScene> packed_scene; packed_scene.instance(); @@ -321,7 +336,7 @@ Ref<PackedScene> ResourceInteractiveLoaderText::_parse_node_tag(VariantParser::R NodePath to = next_tag.fields["to"]; StringName method = next_tag.fields["method"]; StringName signal = next_tag.fields["signal"]; - int flags = CONNECT_PERSIST; + int flags = Object::CONNECT_PERSIST; Array binds; if (next_tag.fields.has("flags")) { @@ -389,12 +404,15 @@ Ref<PackedScene> ResourceInteractiveLoaderText::_parse_node_tag(VariantParser::R return packed_scene; } -Error ResourceInteractiveLoaderText::poll() { +Error ResourceLoaderText::load() { if (error != OK) return error; - if (next_tag.name == "ext_resource") { + while (true) { + if (next_tag.name != "ext_resource") { + break; + } if (!next_tag.fields.has("path")) { error = ERR_FILE_CORRUPT; @@ -430,30 +448,49 @@ Error ResourceInteractiveLoaderText::poll() { path = remaps[path]; } - RES res = ResourceLoader::load(path, type); + ExtResource er; + er.path = path; + er.type = type; + + if (use_sub_threads) { - if (res.is_null()) { + Error err = ResourceLoader::load_threaded_request(path, type, use_sub_threads, local_path); - if (ResourceLoader::get_abort_on_missing_resources()) { - error = ERR_FILE_CORRUPT; - error_text = "[ext_resource] referenced nonexistent resource at: " + path; - _printerr(); - return error; - } else { - ResourceLoader::notify_dependency_error(local_path, path, type); + if (err != OK) { + if (ResourceLoader::get_abort_on_missing_resources()) { + error = ERR_FILE_CORRUPT; + error_text = "[ext_resource] referenced broken resource at: " + path; + _printerr(); + return error; + } else { + ResourceLoader::notify_dependency_error(local_path, path, type); + } } + } else { + RES res = ResourceLoader::load(path, type); + + if (res.is_null()) { + + if (ResourceLoader::get_abort_on_missing_resources()) { + error = ERR_FILE_CORRUPT; + error_text = "[ext_resource] referenced nonexistent resource at: " + path; + _printerr(); + return error; + } else { + ResourceLoader::notify_dependency_error(local_path, path, type); + } + } else { - resource_cache.push_back(res); #ifdef TOOLS_ENABLED - //remember ID for saving - res->set_id_for_path(local_path, index); + //remember ID for saving + res->set_id_for_path(local_path, index); #endif + } + + er.cache = res; } - ExtResource er; - er.path = path; - er.type = type; ext_resources[index] = er; error = VariantParser::parse_tag(&stream, lines, error_text, next_tag, &rp); @@ -463,9 +500,16 @@ Error ResourceInteractiveLoaderText::poll() { } resource_current++; - return error; + } - } else if (next_tag.name == "sub_resource") { + //these are the ones that count + resources_total -= resource_current; + resource_current = 0; + + while (true) { + if (next_tag.name != "sub_resource") { + break; + } if (!next_tag.fields.has("type")) { error = ERR_FILE_CORRUPT; @@ -546,9 +590,15 @@ Error ResourceInteractiveLoaderText::poll() { } } - return OK; + if (progress) { + *progress = resource_current / float(resources_total); + } + } - } else if (next_tag.name == "resource") { + while (true) { + if (next_tag.name != "resource") { + break; + } if (is_scene) { @@ -609,14 +659,23 @@ Error ResourceInteractiveLoaderText::poll() { _printerr(); return error; } else { - error = ERR_FILE_EOF; + error = OK; + if (progress) { + *progress = resource_current / float(resources_total); + } + return error; } } - return OK; + if (progress) { + *progress = resource_current / float(resources_total); + } + } - } else if (next_tag.name == "node") { + //for scene files + + if (next_tag.name == "node") { if (!is_scene) { @@ -631,49 +690,54 @@ Error ResourceInteractiveLoaderText::poll() { if (!packed_scene.is_valid()) return error; - error = ERR_FILE_EOF; + error = OK; //get it here resource = packed_scene; if (!ResourceCache::has(res_path)) { packed_scene->set_path(res_path); } - return error; + resource_current++; + + if (progress) { + *progress = resource_current / float(resources_total); + } + return error; } else { error_text += "Unknown tag in file: " + next_tag.name; _printerr(); error = ERR_FILE_CORRUPT; return error; } - - return OK; } -int ResourceInteractiveLoaderText::get_stage() const { +int ResourceLoaderText::get_stage() const { return resource_current; } -int ResourceInteractiveLoaderText::get_stage_count() const { +int ResourceLoaderText::get_stage_count() const { return resources_total; //+ext_resources; } -void ResourceInteractiveLoaderText::set_translation_remapped(bool p_remapped) { +void ResourceLoaderText::set_translation_remapped(bool p_remapped) { translation_remapped = p_remapped; } -ResourceInteractiveLoaderText::ResourceInteractiveLoaderText() { +ResourceLoaderText::ResourceLoaderText() { + progress = nullptr; translation_remapped = false; + use_sub_threads = false; } -ResourceInteractiveLoaderText::~ResourceInteractiveLoaderText() { +ResourceLoaderText::~ResourceLoaderText() { memdelete(f); } -void ResourceInteractiveLoaderText::get_dependencies(FileAccess *p_f, List<String> *p_dependencies, bool p_add_types) { +void ResourceLoaderText::get_dependencies(FileAccess *p_f, List<String> *p_dependencies, bool p_add_types) { open(p_f); ignore_resource_parsing = true; @@ -720,7 +784,7 @@ void ResourceInteractiveLoaderText::get_dependencies(FileAccess *p_f, List<Strin } } -Error ResourceInteractiveLoaderText::rename_dependencies(FileAccess *p_f, const String &p_path, const Map<String, String> &p_map) { +Error ResourceLoaderText::rename_dependencies(FileAccess *p_f, const String &p_path, const Map<String, String> &p_map) { open(p_f, true); ERR_FAIL_COND_V(error != OK, error); @@ -822,7 +886,7 @@ Error ResourceInteractiveLoaderText::rename_dependencies(FileAccess *p_f, const return OK; } -void ResourceInteractiveLoaderText::open(FileAccess *p_f, bool p_skip_first_tag) { +void ResourceLoaderText::open(FileAccess *p_f, bool p_skip_first_tag) { error = OK; @@ -908,7 +972,7 @@ static void bs_save_unicode_string(FileAccess *f, const String &p_string, bool p f->store_buffer((const uint8_t *)utf8.get_data(), utf8.length() + 1); } -Error ResourceInteractiveLoaderText::save_as_binary(FileAccess *p_f, const String &p_path) { +Error ResourceLoaderText::save_as_binary(FileAccess *p_f, const String &p_path) { if (error) return error; @@ -1172,7 +1236,7 @@ Error ResourceInteractiveLoaderText::save_as_binary(FileAccess *p_f, const Strin return OK; } -String ResourceInteractiveLoaderText::recognize(FileAccess *p_f) { +String ResourceLoaderText::recognize(FileAccess *p_f) { error = OK; @@ -1217,24 +1281,34 @@ String ResourceInteractiveLoaderText::recognize(FileAccess *p_f) { ///////////////////// -Ref<ResourceInteractiveLoader> ResourceFormatLoaderText::load_interactive(const String &p_path, const String &p_original_path, Error *r_error) { +RES ResourceFormatLoaderText::load(const String &p_path, const String &p_original_path, Error *r_error, bool p_use_sub_threads, float *r_progress) { if (r_error) *r_error = ERR_CANT_OPEN; Error err; + FileAccess *f = FileAccess::open(p_path, FileAccess::READ, &err); - ERR_FAIL_COND_V_MSG(err != OK, Ref<ResourceInteractiveLoader>(), "Cannot open file '" + p_path + "'."); + ERR_FAIL_COND_V_MSG(err != OK, RES(), "Cannot open file '" + p_path + "'."); - Ref<ResourceInteractiveLoaderText> ria = memnew(ResourceInteractiveLoaderText); + ResourceLoaderText loader; String path = p_original_path != "" ? p_original_path : p_path; - ria->local_path = ProjectSettings::get_singleton()->localize_path(path); - ria->res_path = ria->local_path; - //ria->set_local_path( ProjectSettings::get_singleton()->localize_path(p_path) ); - ria->open(f); - - return ria; + loader.use_sub_threads = p_use_sub_threads; + loader.local_path = ProjectSettings::get_singleton()->localize_path(path); + loader.progress = r_progress; + loader.res_path = loader.local_path; + //loader.set_local_path( ProjectSettings::get_singleton()->localize_path(p_path) ); + loader.open(f); + err = loader.load(); + if (r_error) { + *r_error = err; + } + if (err == OK) { + return loader.get_resource(); + } else { + return RES(); + } } void ResourceFormatLoaderText::get_recognized_extensions_for_type(const String &p_type, List<String> *p_extensions) const { @@ -1276,11 +1350,11 @@ String ResourceFormatLoaderText::get_resource_type(const String &p_path) const { return ""; //could not rwead } - Ref<ResourceInteractiveLoaderText> ria = memnew(ResourceInteractiveLoaderText); - ria->local_path = ProjectSettings::get_singleton()->localize_path(p_path); - ria->res_path = ria->local_path; - //ria->set_local_path( ProjectSettings::get_singleton()->localize_path(p_path) ); - String r = ria->recognize(f); + ResourceLoaderText loader; + loader.local_path = ProjectSettings::get_singleton()->localize_path(p_path); + loader.res_path = loader.local_path; + //loader.set_local_path( ProjectSettings::get_singleton()->localize_path(p_path) ); + String r = loader.recognize(f); return ClassDB::get_compatibility_remapped_class(r); } @@ -1292,11 +1366,11 @@ void ResourceFormatLoaderText::get_dependencies(const String &p_path, List<Strin ERR_FAIL(); } - Ref<ResourceInteractiveLoaderText> ria = memnew(ResourceInteractiveLoaderText); - ria->local_path = ProjectSettings::get_singleton()->localize_path(p_path); - ria->res_path = ria->local_path; - //ria->set_local_path( ProjectSettings::get_singleton()->localize_path(p_path) ); - ria->get_dependencies(f, p_dependencies, p_add_types); + ResourceLoaderText loader; + loader.local_path = ProjectSettings::get_singleton()->localize_path(p_path); + loader.res_path = loader.local_path; + //loader.set_local_path( ProjectSettings::get_singleton()->localize_path(p_path) ); + loader.get_dependencies(f, p_dependencies, p_add_types); } Error ResourceFormatLoaderText::rename_dependencies(const String &p_path, const Map<String, String> &p_map) { @@ -1307,11 +1381,11 @@ Error ResourceFormatLoaderText::rename_dependencies(const String &p_path, const ERR_FAIL_V(ERR_CANT_OPEN); } - Ref<ResourceInteractiveLoaderText> ria = memnew(ResourceInteractiveLoaderText); - ria->local_path = ProjectSettings::get_singleton()->localize_path(p_path); - ria->res_path = ria->local_path; - //ria->set_local_path( ProjectSettings::get_singleton()->localize_path(p_path) ); - return ria->rename_dependencies(f, p_path, p_map); + ResourceLoaderText loader; + loader.local_path = ProjectSettings::get_singleton()->localize_path(p_path); + loader.res_path = loader.local_path; + //loader.set_local_path( ProjectSettings::get_singleton()->localize_path(p_path) ); + return loader.rename_dependencies(f, p_path, p_map); } ResourceFormatLoaderText *ResourceFormatLoaderText::singleton = NULL; @@ -1323,13 +1397,13 @@ Error ResourceFormatLoaderText::convert_file_to_binary(const String &p_src_path, ERR_FAIL_COND_V_MSG(err != OK, ERR_CANT_OPEN, "Cannot open file '" + p_src_path + "'."); - Ref<ResourceInteractiveLoaderText> ria = memnew(ResourceInteractiveLoaderText); + ResourceLoaderText loader; const String &path = p_src_path; - ria->local_path = ProjectSettings::get_singleton()->localize_path(path); - ria->res_path = ria->local_path; - //ria->set_local_path( ProjectSettings::get_singleton()->localize_path(p_path) ); - ria->open(f); - return ria->save_as_binary(f, p_dst_path); + loader.local_path = ProjectSettings::get_singleton()->localize_path(path); + loader.res_path = loader.local_path; + //loader.set_local_path( ProjectSettings::get_singleton()->localize_path(p_path) ); + loader.open(f); + return loader.save_as_binary(f, p_dst_path); } /*****************************************************************************************************/ diff --git a/scene/resources/resource_format_text.h b/scene/resources/resource_format_text.h index 66c69725e8..2425ac7f6c 100644 --- a/scene/resources/resource_format_text.h +++ b/scene/resources/resource_format_text.h @@ -37,7 +37,7 @@ #include "core/variant_parser.h" #include "scene/resources/packed_scene.h" -class ResourceInteractiveLoaderText : public ResourceInteractiveLoader { +class ResourceLoaderText { bool translation_remapped; String local_path; @@ -49,6 +49,7 @@ class ResourceInteractiveLoaderText : public ResourceInteractiveLoader { VariantParser::StreamFile stream; struct ExtResource { + RES cache; String path; String type; }; @@ -68,13 +69,16 @@ class ResourceInteractiveLoaderText : public ResourceInteractiveLoader { VariantParser::Tag next_tag; + bool use_sub_threads; + float *progress; + mutable int lines; Map<String, String> remaps; //void _printerr(); - static Error _parse_sub_resources(void *p_self, VariantParser::Stream *p_stream, Ref<Resource> &r_res, int &line, String &r_err_str) { return reinterpret_cast<ResourceInteractiveLoaderText *>(p_self)->_parse_sub_resource(p_stream, r_res, line, r_err_str); } - static Error _parse_ext_resources(void *p_self, VariantParser::Stream *p_stream, Ref<Resource> &r_res, int &line, String &r_err_str) { return reinterpret_cast<ResourceInteractiveLoaderText *>(p_self)->_parse_ext_resource(p_stream, r_res, line, r_err_str); } + static Error _parse_sub_resources(void *p_self, VariantParser::Stream *p_stream, Ref<Resource> &r_res, int &line, String &r_err_str) { return reinterpret_cast<ResourceLoaderText *>(p_self)->_parse_sub_resource(p_stream, r_res, line, r_err_str); } + static Error _parse_ext_resources(void *p_self, VariantParser::Stream *p_stream, Ref<Resource> &r_res, int &line, String &r_err_str) { return reinterpret_cast<ResourceLoaderText *>(p_self)->_parse_ext_resource(p_stream, r_res, line, r_err_str); } Error _parse_sub_resource(VariantParser::Stream *p_stream, Ref<Resource> &r_res, int &line, String &r_err_str); Error _parse_ext_resource(VariantParser::Stream *p_stream, Ref<Resource> &r_res, int &line, String &r_err_str); @@ -110,12 +114,12 @@ class ResourceInteractiveLoaderText : public ResourceInteractiveLoader { Ref<PackedScene> _parse_node_tag(VariantParser::ResourceParser &parser); public: - virtual void set_local_path(const String &p_local_path); - virtual Ref<Resource> get_resource(); - virtual Error poll(); - virtual int get_stage() const; - virtual int get_stage_count() const; - virtual void set_translation_remapped(bool p_remapped); + void set_local_path(const String &p_local_path); + Ref<Resource> get_resource(); + Error load(); + int get_stage() const; + int get_stage_count() const; + void set_translation_remapped(bool p_remapped); void open(FileAccess *p_f, bool p_skip_first_tag = false); String recognize(FileAccess *p_f); @@ -123,14 +127,14 @@ public: Error rename_dependencies(FileAccess *p_f, const String &p_path, const Map<String, String> &p_map); Error save_as_binary(FileAccess *p_f, const String &p_path); - ResourceInteractiveLoaderText(); - ~ResourceInteractiveLoaderText(); + ResourceLoaderText(); + ~ResourceLoaderText(); }; class ResourceFormatLoaderText : public ResourceFormatLoader { public: static ResourceFormatLoaderText *singleton; - virtual Ref<ResourceInteractiveLoader> load_interactive(const String &p_path, const String &p_original_path = "", Error *r_error = NULL); + virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL, bool p_use_sub_threads = false, float *r_progress = nullptr); virtual void get_recognized_extensions_for_type(const String &p_type, List<String> *p_extensions) const; virtual void get_recognized_extensions(List<String> *p_extensions) const; virtual bool handles_type(const String &p_type) const; diff --git a/scene/resources/shader.cpp b/scene/resources/shader.cpp index 79cb9754df..8f76c0165f 100644 --- a/scene/resources/shader.cpp +++ b/scene/resources/shader.cpp @@ -173,7 +173,7 @@ Shader::~Shader() { } //////////// -RES ResourceFormatLoaderShader::load(const String &p_path, const String &p_original_path, Error *r_error) { +RES ResourceFormatLoaderShader::load(const String &p_path, const String &p_original_path, Error *r_error, bool p_use_sub_threads, float *r_progress) { if (r_error) *r_error = ERR_FILE_CANT_OPEN; diff --git a/scene/resources/shader.h b/scene/resources/shader.h index 702e58aedc..5050632dd5 100644 --- a/scene/resources/shader.h +++ b/scene/resources/shader.h @@ -101,7 +101,7 @@ VARIANT_ENUM_CAST(Shader::Mode); class ResourceFormatLoaderShader : public ResourceFormatLoader { public: - virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL); + virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL, bool p_use_sub_threads = false, float *r_progress = nullptr); virtual void get_recognized_extensions(List<String> *p_extensions) const; virtual bool handles_type(const String &p_type) const; virtual String get_resource_type(const String &p_path) const; diff --git a/scene/resources/texture.cpp b/scene/resources/texture.cpp index ca5b6fa9d0..cb827c4b0b 100644 --- a/scene/resources/texture.cpp +++ b/scene/resources/texture.cpp @@ -363,7 +363,6 @@ Ref<Image> StreamTexture::load_image_from_file(FileAccess *f, int p_size_limit) uint32_t mipmaps = f->get_32(); Image::Format format = Image::Format(f->get_32()); - print_line("format: " + itos(data_format) + " size " + Size2i(w, h) + " mipmaps: " + itos(mipmaps)); if (data_format == DATA_FORMAT_LOSSLESS || data_format == DATA_FORMAT_LOSSY || data_format == DATA_FORMAT_BASIS_UNIVERSAL) { //look for a PNG or WEBP file inside @@ -797,7 +796,7 @@ StreamTexture::~StreamTexture() { } } -RES ResourceFormatLoaderStreamTexture::load(const String &p_path, const String &p_original_path, Error *r_error) { +RES ResourceFormatLoaderStreamTexture::load(const String &p_path, const String &p_original_path, Error *r_error, bool p_use_sub_threads, float *r_progress) { Ref<StreamTexture> st; st.instance(); @@ -2025,7 +2024,7 @@ TextureLayered::~TextureLayered() { } } -RES ResourceFormatLoaderTextureLayered::load(const String &p_path, const String &p_original_path, Error *r_error) { +RES ResourceFormatLoaderTextureLayered::load(const String &p_path, const String &p_original_path, Error *r_error, bool p_use_sub_threads, float *r_progress) { if (r_error) { *r_error = ERR_CANT_OPEN; diff --git a/scene/resources/texture.h b/scene/resources/texture.h index cd8576539b..237c02a8cd 100644 --- a/scene/resources/texture.h +++ b/scene/resources/texture.h @@ -213,7 +213,7 @@ public: class ResourceFormatLoaderStreamTexture : public ResourceFormatLoader { public: - virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL); + virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL, bool p_use_sub_threads = false, float *r_progress = nullptr); virtual void get_recognized_extensions(List<String> *p_extensions) const; virtual bool handles_type(const String &p_type) const; virtual String get_resource_type(const String &p_path) const; @@ -421,7 +421,7 @@ public: COMPRESSION_UNCOMPRESSED }; - virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL); + virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL, bool p_use_sub_threads = false, float *r_progress = nullptr); virtual void get_recognized_extensions(List<String> *p_extensions) const; virtual bool handles_type(const String &p_type) const; virtual String get_resource_type(const String &p_path) const; |