diff options
author | Juan Linietsky <juan@godotengine.org> | 2020-02-28 08:27:04 -0300 |
---|---|---|
committer | Juan Linietsky <juan@godotengine.org> | 2020-02-28 11:20:45 -0300 |
commit | 475e4ea67b7e650fd1df01eab6ea659afbc16ec5 (patch) | |
tree | 9e9603a3bd51a5d31a767334fa413b030a3ed402 /core | |
parent | 5beccaf86fc89166d0b7d28cfcf719cad8fb30ee (diff) |
Removed interactive loader, added proper thread loading.
Diffstat (limited to 'core')
-rw-r--r-- | core/bind/core_bind.cpp | 27 | ||||
-rw-r--r-- | core/bind/core_bind.h | 15 | ||||
-rw-r--r-- | core/crypto/crypto.cpp | 2 | ||||
-rw-r--r-- | core/crypto/crypto.h | 2 | ||||
-rw-r--r-- | core/io/image_loader.cpp | 2 | ||||
-rw-r--r-- | core/io/image_loader.h | 2 | ||||
-rw-r--r-- | core/io/resource_format_binary.cpp | 313 | ||||
-rw-r--r-- | core/io/resource_format_binary.h | 25 | ||||
-rw-r--r-- | core/io/resource_importer.cpp | 4 | ||||
-rw-r--r-- | core/io/resource_importer.h | 2 | ||||
-rw-r--r-- | core/io/resource_loader.cpp | 596 | ||||
-rw-r--r-- | core/io/resource_loader.h | 85 | ||||
-rw-r--r-- | core/io/translation_loader_po.cpp | 2 | ||||
-rw-r--r-- | core/io/translation_loader_po.h | 2 | ||||
-rw-r--r-- | core/object.h | 1 | ||||
-rw-r--r-- | core/register_core_types.cpp | 2 |
16 files changed, 643 insertions, 439 deletions
diff --git a/core/bind/core_bind.cpp b/core/bind/core_bind.cpp index d9cc77feb3..e99e69578c 100644 --- a/core/bind/core_bind.cpp +++ b/core/bind/core_bind.cpp @@ -64,8 +64,21 @@ static const unsigned int MONTH_DAYS_TABLE[2][12] = { _ResourceLoader *_ResourceLoader::singleton = NULL; -Ref<ResourceInteractiveLoader> _ResourceLoader::load_interactive(const String &p_path, const String &p_type_hint) { - return ResourceLoader::load_interactive(p_path, p_type_hint); +Error _ResourceLoader::load_threaded_request(const String &p_path, const String &p_type_hint, bool p_use_sub_threads) { + + return ResourceLoader::load_threaded_request(p_path, p_type_hint, p_use_sub_threads); +} +_ResourceLoader::ThreadLoadStatus _ResourceLoader::load_threaded_get_status(const String &p_path, Array r_progress) { + float progress = 0; + ResourceLoader::ThreadLoadStatus tls = ResourceLoader::load_threaded_get_status(p_path, &progress); + r_progress.resize(1); + r_progress[0] = progress; + return (ThreadLoadStatus)tls; +} +RES _ResourceLoader::load_threaded_get(const String &p_path) { + Error error; + RES res = ResourceLoader::load_threaded_get(p_path, &error); + return res; } RES _ResourceLoader::load(const String &p_path, const String &p_type_hint, bool p_no_cache) { @@ -120,13 +133,21 @@ bool _ResourceLoader::exists(const String &p_path, const String &p_type_hint) { void _ResourceLoader::_bind_methods() { - ClassDB::bind_method(D_METHOD("load_interactive", "path", "type_hint"), &_ResourceLoader::load_interactive, DEFVAL("")); + ClassDB::bind_method(D_METHOD("load_threaded_request", "path", "type_hint", "use_sub_threads"), &_ResourceLoader::load_threaded_request, DEFVAL(""), DEFVAL(false)); + ClassDB::bind_method(D_METHOD("load_threaded_get_status", "path", "r_progress"), &_ResourceLoader::load_threaded_get_status, DEFVAL(Array())); + ClassDB::bind_method(D_METHOD("load_threaded_get", "path"), &_ResourceLoader::load_threaded_get); + ClassDB::bind_method(D_METHOD("load", "path", "type_hint", "no_cache"), &_ResourceLoader::load, DEFVAL(""), DEFVAL(false)); ClassDB::bind_method(D_METHOD("get_recognized_extensions_for_type", "type"), &_ResourceLoader::get_recognized_extensions_for_type); ClassDB::bind_method(D_METHOD("set_abort_on_missing_resources", "abort"), &_ResourceLoader::set_abort_on_missing_resources); ClassDB::bind_method(D_METHOD("get_dependencies", "path"), &_ResourceLoader::get_dependencies); ClassDB::bind_method(D_METHOD("has_cached", "path"), &_ResourceLoader::has_cached); ClassDB::bind_method(D_METHOD("exists", "path", "type_hint"), &_ResourceLoader::exists, DEFVAL("")); + + BIND_ENUM_CONSTANT(THREAD_LOAD_INVALID_RESOURCE); + BIND_ENUM_CONSTANT(THREAD_LOAD_IN_PROGRESS); + BIND_ENUM_CONSTANT(THREAD_LOAD_FAILED); + BIND_ENUM_CONSTANT(THREAD_LOAD_LOADED); } _ResourceLoader::_ResourceLoader() { diff --git a/core/bind/core_bind.h b/core/bind/core_bind.h index e68c40c9e4..d4a7c00629 100644 --- a/core/bind/core_bind.h +++ b/core/bind/core_bind.h @@ -49,8 +49,19 @@ protected: static _ResourceLoader *singleton; public: + enum ThreadLoadStatus { + THREAD_LOAD_INVALID_RESOURCE, + THREAD_LOAD_IN_PROGRESS, + THREAD_LOAD_FAILED, + THREAD_LOAD_LOADED + }; + static _ResourceLoader *get_singleton() { return singleton; } - Ref<ResourceInteractiveLoader> load_interactive(const String &p_path, const String &p_type_hint = ""); + + Error load_threaded_request(const String &p_path, const String &p_type_hint = "", bool p_use_sub_threads = false); + ThreadLoadStatus load_threaded_get_status(const String &p_path, Array r_progress = Array()); + RES load_threaded_get(const String &p_path); + RES load(const String &p_path, const String &p_type_hint = "", bool p_no_cache = false); Vector<String> get_recognized_extensions_for_type(const String &p_type); void set_abort_on_missing_resources(bool p_abort); @@ -61,6 +72,8 @@ public: _ResourceLoader(); }; +VARIANT_ENUM_CAST(_ResourceLoader::ThreadLoadStatus); + class _ResourceSaver : public Object { GDCLASS(_ResourceSaver, Object); diff --git a/core/crypto/crypto.cpp b/core/crypto/crypto.cpp index 3711b26e47..793bf719b7 100644 --- a/core/crypto/crypto.cpp +++ b/core/crypto/crypto.cpp @@ -99,7 +99,7 @@ Crypto::Crypto() { /// Resource loader/saver -RES ResourceFormatLoaderCrypto::load(const String &p_path, const String &p_original_path, Error *r_error) { +RES ResourceFormatLoaderCrypto::load(const String &p_path, const String &p_original_path, Error *r_error, bool p_use_sub_threads, float *r_progress) { String el = p_path.get_extension().to_lower(); if (el == "crt") { diff --git a/core/crypto/crypto.h b/core/crypto/crypto.h index babd0bc4eb..3279c0620f 100644 --- a/core/crypto/crypto.h +++ b/core/crypto/crypto.h @@ -87,7 +87,7 @@ class ResourceFormatLoaderCrypto : public ResourceFormatLoader { GDCLASS(ResourceFormatLoaderCrypto, ResourceFormatLoader); public: - virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL); + virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL, bool p_use_sub_threads = false, float *r_progress = nullptr); virtual void get_recognized_extensions(List<String> *p_extensions) const; virtual bool handles_type(const String &p_type) const; virtual String get_resource_type(const String &p_path) const; diff --git a/core/io/image_loader.cpp b/core/io/image_loader.cpp index 720f25f91b..99ac5bcdd9 100644 --- a/core/io/image_loader.cpp +++ b/core/io/image_loader.cpp @@ -129,7 +129,7 @@ void ImageLoader::cleanup() { ///////////////// -RES ResourceFormatLoaderImage::load(const String &p_path, const String &p_original_path, Error *r_error) { +RES ResourceFormatLoaderImage::load(const String &p_path, const String &p_original_path, Error *r_error, bool p_use_sub_threads, float *r_progress) { FileAccess *f = FileAccess::open(p_path, FileAccess::READ); if (!f) { diff --git a/core/io/image_loader.h b/core/io/image_loader.h index d6dfd261ca..3ba028b99c 100644 --- a/core/io/image_loader.h +++ b/core/io/image_loader.h @@ -73,7 +73,7 @@ public: class ResourceFormatLoaderImage : public ResourceFormatLoader { public: - virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL); + virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL, bool p_use_sub_threads = false, float *r_progress = nullptr); virtual void get_recognized_extensions(List<String> *p_extensions) const; virtual bool handles_type(const String &p_type) const; virtual String get_resource_type(const String &p_path) const; diff --git a/core/io/resource_format_binary.cpp b/core/io/resource_format_binary.cpp index 8c343a0f43..54b75cc29d 100644 --- a/core/io/resource_format_binary.cpp +++ b/core/io/resource_format_binary.cpp @@ -93,7 +93,7 @@ enum { }; -void ResourceInteractiveLoaderBinary::_advance_padding(uint32_t p_len) { +void ResourceLoaderBinary::_advance_padding(uint32_t p_len) { uint32_t extra = 4 - (p_len % 4); if (extra < 4) { @@ -102,7 +102,7 @@ void ResourceInteractiveLoaderBinary::_advance_padding(uint32_t p_len) { } } -StringName ResourceInteractiveLoaderBinary::_get_string() { +StringName ResourceLoaderBinary::_get_string() { uint32_t id = f->get_32(); if (id & 0x80000000) { @@ -121,7 +121,7 @@ StringName ResourceInteractiveLoaderBinary::_get_string() { return string_map[id]; } -Error ResourceInteractiveLoaderBinary::parse_variant(Variant &r_v) { +Error ResourceLoaderBinary::parse_variant(Variant &r_v) { uint32_t type = f->get_32(); print_bl("find property of type: " + itos(type)); @@ -377,20 +377,26 @@ Error ResourceInteractiveLoaderBinary::parse_variant(Variant &r_v) { r_v = Variant(); } else { - String exttype = external_resources[erindex].type; - String path = external_resources[erindex].path; + if (external_resources[erindex].cache.is_null()) { + //cache not here yet, wait for it? + if (use_sub_threads) { + Error err; + external_resources.write[erindex].cache = ResourceLoader::load_threaded_get(external_resources[erindex].path, &err); - if (path.find("://") == -1 && path.is_rel_path()) { - // path is relative to file being loaded, so convert to a resource path - path = ProjectSettings::get_singleton()->localize_path(res_path.get_base_dir().plus_file(path)); - } + if (err != OK || external_resources[erindex].cache.is_null()) { + if (!ResourceLoader::get_abort_on_missing_resources()) { - RES res = ResourceLoader::load(path, exttype); + ResourceLoader::notify_dependency_error(local_path, external_resources[erindex].path, external_resources[erindex].type); + } else { - if (res.is_null()) { - WARN_PRINT(String("Couldn't load resource: " + path).utf8().get_data()); + error = ERR_FILE_MISSING_DEPENDENCIES; + ERR_FAIL_V_MSG(error, "Can't load dependency: " + external_resources[erindex].path + "."); + } + } + } } - r_v = res; + + r_v = external_resources[erindex].cache; } } break; @@ -638,160 +644,168 @@ Error ResourceInteractiveLoaderBinary::parse_variant(Variant &r_v) { return OK; //never reach anyway } -void ResourceInteractiveLoaderBinary::set_local_path(const String &p_local_path) { +void ResourceLoaderBinary::set_local_path(const String &p_local_path) { res_path = p_local_path; } -Ref<Resource> ResourceInteractiveLoaderBinary::get_resource() { +Ref<Resource> ResourceLoaderBinary::get_resource() { return resource; } -Error ResourceInteractiveLoaderBinary::poll() { +Error ResourceLoaderBinary::load() { if (error != OK) return error; - int s = stage; + int stage = 0; - if (s < external_resources.size()) { + for (int i = 0; i < external_resources.size(); i++) { - String path = external_resources[s].path; + String path = external_resources[i].path; if (remaps.has(path)) { path = remaps[path]; } - RES res = ResourceLoader::load(path, external_resources[s].type); - if (res.is_null()) { - if (!ResourceLoader::get_abort_on_missing_resources()) { + if (path.find("://") == -1 && path.is_rel_path()) { + // path is relative to file being loaded, so convert to a resource path + path = ProjectSettings::get_singleton()->localize_path(path.get_base_dir().plus_file(external_resources[i].path)); + } + + external_resources.write[i].path = path; //remap happens here, not on load because on load it can actually be used for filesystem dock resource remap - ResourceLoader::notify_dependency_error(local_path, path, external_resources[s].type); - } else { + if (!use_sub_threads) { + external_resources.write[i].cache = ResourceLoader::load(path, external_resources[i].type); - error = ERR_FILE_MISSING_DEPENDENCIES; - ERR_FAIL_V_MSG(error, "Can't load dependency: " + path + "."); + if (external_resources[i].cache.is_null()) { + if (!ResourceLoader::get_abort_on_missing_resources()) { + + ResourceLoader::notify_dependency_error(local_path, path, external_resources[i].type); + } else { + + error = ERR_FILE_MISSING_DEPENDENCIES; + ERR_FAIL_V_MSG(error, "Can't load dependency: " + path + "."); + } } } else { - resource_cache.push_back(res); + Error err = ResourceLoader::load_threaded_request(path, external_resources[i].type, use_sub_threads, local_path); + if (err != OK) { + if (!ResourceLoader::get_abort_on_missing_resources()) { + + ResourceLoader::notify_dependency_error(local_path, path, external_resources[i].type); + } else { + + error = ERR_FILE_MISSING_DEPENDENCIES; + ERR_FAIL_V_MSG(error, "Can't load dependency: " + path + "."); + } + } } stage++; - return error; } - s -= external_resources.size(); + for (int i = 0; i < internal_resources.size(); i++) { - if (s >= internal_resources.size()) { + bool main = i == (internal_resources.size() - 1); - error = ERR_BUG; - ERR_FAIL_COND_V(s >= internal_resources.size(), error); - } + //maybe it is loaded already + String path; + int subindex = 0; - bool main = s == (internal_resources.size() - 1); + if (!main) { - //maybe it is loaded already - String path; - int subindex = 0; + path = internal_resources[i].path; + if (path.begins_with("local://")) { + path = path.replace_first("local://", ""); + subindex = path.to_int(); + path = res_path + "::" + path; + } - if (!main) { + if (ResourceCache::has(path)) { + //already loaded, don't do anything + stage++; + error = OK; + continue; + } + } else { - path = internal_resources[s].path; - if (path.begins_with("local://")) { - path = path.replace_first("local://", ""); - subindex = path.to_int(); - path = res_path + "::" + path; + if (!ResourceCache::has(res_path)) + path = res_path; } - if (ResourceCache::has(path)) { - //already loaded, don't do anything - stage++; - error = OK; - return error; - } - } else { + uint64_t offset = internal_resources[i].offset; - if (!ResourceCache::has(res_path)) - path = res_path; - } - - uint64_t offset = internal_resources[s].offset; + f->seek(offset); - f->seek(offset); + String t = get_unicode_string(); - String t = get_unicode_string(); - - Object *obj = ClassDB::instance(t); - if (!obj) { - error = ERR_FILE_CORRUPT; - ERR_FAIL_V_MSG(ERR_FILE_CORRUPT, local_path + ":Resource of unrecognized type in file: " + t + "."); - } + Object *obj = ClassDB::instance(t); + if (!obj) { + error = ERR_FILE_CORRUPT; + ERR_FAIL_V_MSG(ERR_FILE_CORRUPT, local_path + ":Resource of unrecognized type in file: " + t + "."); + } - Resource *r = Object::cast_to<Resource>(obj); - if (!r) { - String obj_class = obj->get_class(); - error = ERR_FILE_CORRUPT; - memdelete(obj); //bye - ERR_FAIL_V_MSG(ERR_FILE_CORRUPT, local_path + ":Resource type in resource field not a resource, type is: " + obj_class + "."); - } + Resource *r = Object::cast_to<Resource>(obj); + if (!r) { + String obj_class = obj->get_class(); + error = ERR_FILE_CORRUPT; + memdelete(obj); //bye + ERR_FAIL_V_MSG(ERR_FILE_CORRUPT, local_path + ":Resource type in resource field not a resource, type is: " + obj_class + "."); + } - RES res = RES(r); + RES res = RES(r); - r->set_path(path); - r->set_subindex(subindex); + r->set_path(path); + r->set_subindex(subindex); - int pc = f->get_32(); + int pc = f->get_32(); - //set properties + //set properties - for (int i = 0; i < pc; i++) { + for (int j = 0; j < pc; j++) { - StringName name = _get_string(); + StringName name = _get_string(); - if (name == StringName()) { - error = ERR_FILE_CORRUPT; - ERR_FAIL_V(ERR_FILE_CORRUPT); - } + if (name == StringName()) { + error = ERR_FILE_CORRUPT; + ERR_FAIL_V(ERR_FILE_CORRUPT); + } - Variant value; + Variant value; - error = parse_variant(value); - if (error) - return error; + error = parse_variant(value); + if (error) + return error; - res->set(name, value); - } + res->set(name, value); + } #ifdef TOOLS_ENABLED - res->set_edited(false); + res->set_edited(false); #endif - stage++; + stage++; - resource_cache.push_back(res); + if (progress) { + *progress = (i + 1) / float(internal_resources.size()); + } - if (main) { + resource_cache.push_back(res); - f->close(); - resource = res; - resource->set_as_translation_remapped(translation_remapped); - error = ERR_FILE_EOF; + if (main) { - } else { - error = OK; + f->close(); + resource = res; + resource->set_as_translation_remapped(translation_remapped); + error = OK; + return OK; + } } - return OK; -} -int ResourceInteractiveLoaderBinary::get_stage() const { - - return stage; -} -int ResourceInteractiveLoaderBinary::get_stage_count() const { - - return external_resources.size() + internal_resources.size(); + return ERR_FILE_EOF; } -void ResourceInteractiveLoaderBinary::set_translation_remapped(bool p_remapped) { +void ResourceLoaderBinary::set_translation_remapped(bool p_remapped) { translation_remapped = p_remapped; } @@ -814,7 +828,7 @@ static String get_ustring(FileAccess *f) { return s; } -String ResourceInteractiveLoaderBinary::get_unicode_string() { +String ResourceLoaderBinary::get_unicode_string() { int len = f->get_32(); if (len > str_buf.size()) { @@ -828,7 +842,7 @@ String ResourceInteractiveLoaderBinary::get_unicode_string() { return s; } -void ResourceInteractiveLoaderBinary::get_dependencies(FileAccess *p_f, List<String> *p_dependencies, bool p_add_types) { +void ResourceLoaderBinary::get_dependencies(FileAccess *p_f, List<String> *p_dependencies, bool p_add_types) { open(p_f); if (error) @@ -846,7 +860,7 @@ void ResourceInteractiveLoaderBinary::get_dependencies(FileAccess *p_f, List<Str } } -void ResourceInteractiveLoaderBinary::open(FileAccess *p_f) { +void ResourceLoaderBinary::open(FileAccess *p_f) { error = OK; @@ -947,7 +961,7 @@ void ResourceInteractiveLoaderBinary::open(FileAccess *p_f) { } } -String ResourceInteractiveLoaderBinary::recognize(FileAccess *p_f) { +String ResourceLoaderBinary::recognize(FileAccess *p_f) { error = OK; @@ -992,20 +1006,22 @@ String ResourceInteractiveLoaderBinary::recognize(FileAccess *p_f) { return type; } -ResourceInteractiveLoaderBinary::ResourceInteractiveLoaderBinary() : +ResourceLoaderBinary::ResourceLoaderBinary() : translation_remapped(false), f(NULL), - error(OK), - stage(0) { + error(OK) { + + progress = nullptr; + use_sub_threads = false; } -ResourceInteractiveLoaderBinary::~ResourceInteractiveLoaderBinary() { +ResourceLoaderBinary::~ResourceLoaderBinary() { if (f) memdelete(f); } -Ref<ResourceInteractiveLoader> ResourceFormatLoaderBinary::load_interactive(const String &p_path, const String &p_original_path, Error *r_error) { +RES ResourceFormatLoaderBinary::load(const String &p_path, const String &p_original_path, Error *r_error, bool p_use_sub_threads, float *r_progress) { if (r_error) *r_error = ERR_FILE_CANT_OPEN; @@ -1013,16 +1029,27 @@ Ref<ResourceInteractiveLoader> ResourceFormatLoaderBinary::load_interactive(cons Error err; FileAccess *f = FileAccess::open(p_path, FileAccess::READ, &err); - ERR_FAIL_COND_V_MSG(err != OK, Ref<ResourceInteractiveLoader>(), "Cannot open file '" + p_path + "'."); + ERR_FAIL_COND_V_MSG(err != OK, RES(), "Cannot open file '" + p_path + "'."); - Ref<ResourceInteractiveLoaderBinary> ria = memnew(ResourceInteractiveLoaderBinary); + ResourceLoaderBinary loader; + loader.use_sub_threads = p_use_sub_threads; + loader.progress = r_progress; String path = p_original_path != "" ? p_original_path : p_path; - ria->local_path = ProjectSettings::get_singleton()->localize_path(path); - ria->res_path = ria->local_path; - //ria->set_local_path( Globals::get_singleton()->localize_path(p_path) ); - ria->open(f); + loader.local_path = ProjectSettings::get_singleton()->localize_path(path); + loader.res_path = loader.local_path; + //loader.set_local_path( Globals::get_singleton()->localize_path(p_path) ); + loader.open(f); + + err = loader.load(); + + if (r_error) { + *r_error = err; + } - return ria; + if (err) { + return RES(); + } + return loader.resource; } void ResourceFormatLoaderBinary::get_recognized_extensions_for_type(const String &p_type, List<String> *p_extensions) const { @@ -1064,11 +1091,11 @@ void ResourceFormatLoaderBinary::get_dependencies(const String &p_path, List<Str FileAccess *f = FileAccess::open(p_path, FileAccess::READ); ERR_FAIL_COND_MSG(!f, "Cannot open file '" + p_path + "'."); - Ref<ResourceInteractiveLoaderBinary> ria = memnew(ResourceInteractiveLoaderBinary); - ria->local_path = ProjectSettings::get_singleton()->localize_path(p_path); - ria->res_path = ria->local_path; - //ria->set_local_path( Globals::get_singleton()->localize_path(p_path) ); - ria->get_dependencies(f, p_dependencies, p_add_types); + ResourceLoaderBinary loader; + loader.local_path = ProjectSettings::get_singleton()->localize_path(p_path); + loader.res_path = loader.local_path; + //loader.set_local_path( Globals::get_singleton()->localize_path(p_path) ); + loader.get_dependencies(f, p_dependencies, p_add_types); } Error ResourceFormatLoaderBinary::rename_dependencies(const String &p_path, const Map<String, String> &p_map) { @@ -1153,21 +1180,17 @@ Error ResourceFormatLoaderBinary::rename_dependencies(const String &p_path, cons ERR_FAIL_COND_V_MSG(err != OK, ERR_FILE_CANT_OPEN, "Cannot open file '" + p_path + "'."); - Ref<ResourceInteractiveLoaderBinary> ria = memnew(ResourceInteractiveLoaderBinary); - ria->local_path = ProjectSettings::get_singleton()->localize_path(p_path); - ria->res_path = ria->local_path; - ria->remaps = p_map; - //ria->set_local_path( Globals::get_singleton()->localize_path(p_path) ); - ria->open(f); - - err = ria->poll(); + ResourceLoaderBinary loader; + loader.local_path = ProjectSettings::get_singleton()->localize_path(p_path); + loader.res_path = loader.local_path; + loader.remaps = p_map; + //loader.set_local_path( Globals::get_singleton()->localize_path(p_path) ); + loader.open(f); - while (err == OK) { - err = ria->poll(); - } + err = loader.load(); ERR_FAIL_COND_V(err != ERR_FILE_EOF, ERR_FILE_CORRUPT); - RES res = ria->get_resource(); + RES res = loader.get_resource(); ERR_FAIL_COND_V(!res.is_valid(), ERR_FILE_CORRUPT); return ResourceFormatSaverBinary::singleton->save(p_path, res); @@ -1283,11 +1306,11 @@ String ResourceFormatLoaderBinary::get_resource_type(const String &p_path) const return ""; //could not rwead } - Ref<ResourceInteractiveLoaderBinary> ria = memnew(ResourceInteractiveLoaderBinary); - ria->local_path = ProjectSettings::get_singleton()->localize_path(p_path); - ria->res_path = ria->local_path; - //ria->set_local_path( Globals::get_singleton()->localize_path(p_path) ); - String r = ria->recognize(f); + ResourceLoaderBinary loader; + loader.local_path = ProjectSettings::get_singleton()->localize_path(p_path); + loader.res_path = loader.local_path; + //loader.set_local_path( Globals::get_singleton()->localize_path(p_path) ); + String r = loader.recognize(f); return ClassDB::get_compatibility_remapped_class(r); } diff --git a/core/io/resource_format_binary.h b/core/io/resource_format_binary.h index f02dbaa0c2..0ffa2c3626 100644 --- a/core/io/resource_format_binary.h +++ b/core/io/resource_format_binary.h @@ -35,7 +35,7 @@ #include "core/io/resource_saver.h" #include "core/os/file_access.h" -class ResourceInteractiveLoaderBinary : public ResourceInteractiveLoader { +class ResourceLoaderBinary { bool translation_remapped; String local_path; @@ -58,8 +58,11 @@ class ResourceInteractiveLoaderBinary : public ResourceInteractiveLoader { struct ExtResource { String path; String type; + RES cache; }; + bool use_sub_threads; + float *progress; Vector<ExtResource> external_resources; struct IntResource { @@ -75,32 +78,30 @@ class ResourceInteractiveLoaderBinary : public ResourceInteractiveLoader { Map<String, String> remaps; Error error; - int stage; - friend class ResourceFormatLoaderBinary; Error parse_variant(Variant &r_v); + Map<String, RES> dependency_cache; + public: - virtual void set_local_path(const String &p_local_path); - virtual Ref<Resource> get_resource(); - virtual Error poll(); - virtual int get_stage() const; - virtual int get_stage_count() const; - virtual void set_translation_remapped(bool p_remapped); + void set_local_path(const String &p_local_path); + Ref<Resource> get_resource(); + Error load(); + void set_translation_remapped(bool p_remapped); void set_remaps(const Map<String, String> &p_remaps) { remaps = p_remaps; } void open(FileAccess *p_f); String recognize(FileAccess *p_f); void get_dependencies(FileAccess *p_f, List<String> *p_dependencies, bool p_add_types); - ResourceInteractiveLoaderBinary(); - ~ResourceInteractiveLoaderBinary(); + ResourceLoaderBinary(); + ~ResourceLoaderBinary(); }; class ResourceFormatLoaderBinary : public ResourceFormatLoader { public: - virtual Ref<ResourceInteractiveLoader> load_interactive(const String &p_path, const String &p_original_path = "", Error *r_error = NULL); + virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL, bool p_use_sub_threads = false, float *r_progress = nullptr); virtual void get_recognized_extensions_for_type(const String &p_type, List<String> *p_extensions) const; virtual void get_recognized_extensions(List<String> *p_extensions) const; virtual bool handles_type(const String &p_type) const; diff --git a/core/io/resource_importer.cpp b/core/io/resource_importer.cpp index f147170ff7..efaf958949 100644 --- a/core/io/resource_importer.cpp +++ b/core/io/resource_importer.cpp @@ -117,7 +117,7 @@ Error ResourceFormatImporter::_get_path_and_type(const String &p_path, PathAndTy return OK; } -RES ResourceFormatImporter::load(const String &p_path, const String &p_original_path, Error *r_error) { +RES ResourceFormatImporter::load(const String &p_path, const String &p_original_path, Error *r_error, bool p_use_sub_threads, float *r_progress) { PathAndType pat; Error err = _get_path_and_type(p_path, pat); @@ -130,7 +130,7 @@ RES ResourceFormatImporter::load(const String &p_path, const String &p_original_ return RES(); } - RES res = ResourceLoader::_load(pat.path, p_path, pat.type, false, r_error); + RES res = ResourceLoader::_load(pat.path, p_path, pat.type, false, r_error, p_use_sub_threads, r_progress); #ifdef TOOLS_ENABLED if (res.is_valid()) { diff --git a/core/io/resource_importer.h b/core/io/resource_importer.h index 4eb04586e6..65c148f2ac 100644 --- a/core/io/resource_importer.h +++ b/core/io/resource_importer.h @@ -58,7 +58,7 @@ class ResourceFormatImporter : public ResourceFormatLoader { public: static ResourceFormatImporter *get_singleton() { return singleton; } - virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL); + virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL, bool p_use_sub_threads = false, float *r_progress = nullptr); virtual void get_recognized_extensions(List<String> *p_extensions) const; virtual void get_recognized_extensions_for_type(const String &p_type, List<String> *p_extensions) const; virtual bool recognize_path(const String &p_path, const String &p_for_type = String()) const; diff --git a/core/io/resource_loader.cpp b/core/io/resource_loader.cpp index b43cd7f1a2..504dbe2d63 100644 --- a/core/io/resource_loader.cpp +++ b/core/io/resource_loader.cpp @@ -39,26 +39,16 @@ #include "core/translation.h" #include "core/variant_parser.h" +#ifdef DEBUG_LOAD_THREADED +#define print_lt(m_text) print_line(m_text) +#else +#define print_lt(m_text) +#endif + Ref<ResourceFormatLoader> ResourceLoader::loader[ResourceLoader::MAX_LOADERS]; int ResourceLoader::loader_count = 0; -Error ResourceInteractiveLoader::wait() { - - Error err = poll(); - while (err == OK) { - err = poll(); - } - - return err; -} - -ResourceInteractiveLoader::~ResourceInteractiveLoader() { - if (path_loading != String()) { - ResourceLoader::_remove_from_loading_map_and_thread(path_loading, path_loading_thread); - } -} - bool ResourceFormatLoader::recognize_path(const String &p_path, const String &p_for_type) const { String extension = p_path.get_extension(); @@ -111,45 +101,6 @@ void ResourceLoader::get_recognized_extensions_for_type(const String &p_type, Li } } -void ResourceInteractiveLoader::_bind_methods() { - - ClassDB::bind_method(D_METHOD("get_resource"), &ResourceInteractiveLoader::get_resource); - ClassDB::bind_method(D_METHOD("poll"), &ResourceInteractiveLoader::poll); - ClassDB::bind_method(D_METHOD("wait"), &ResourceInteractiveLoader::wait); - ClassDB::bind_method(D_METHOD("get_stage"), &ResourceInteractiveLoader::get_stage); - ClassDB::bind_method(D_METHOD("get_stage_count"), &ResourceInteractiveLoader::get_stage_count); -} - -class ResourceInteractiveLoaderDefault : public ResourceInteractiveLoader { - - GDCLASS(ResourceInteractiveLoaderDefault, ResourceInteractiveLoader); - -public: - Ref<Resource> resource; - - virtual void set_local_path(const String &p_local_path) { /*scene->set_filename(p_local_path);*/ - } - virtual Ref<Resource> get_resource() { return resource; } - virtual Error poll() { return ERR_FILE_EOF; } - virtual int get_stage() const { return 1; } - virtual int get_stage_count() const { return 1; } - virtual void set_translation_remapped(bool p_remapped) { resource->set_as_translation_remapped(p_remapped); } - - ResourceInteractiveLoaderDefault() {} -}; - -Ref<ResourceInteractiveLoader> ResourceFormatLoader::load_interactive(const String &p_path, const String &p_original_path, Error *r_error) { - - //either this - Ref<Resource> res = load(p_path, p_original_path, r_error); - if (res.is_null()) - return Ref<ResourceInteractiveLoader>(); - - Ref<ResourceInteractiveLoaderDefault> ril = Ref<ResourceInteractiveLoaderDefault>(memnew(ResourceInteractiveLoaderDefault)); - ril->resource = res; - return ril; -} - bool ResourceFormatLoader::exists(const String &p_path) const { return FileAccess::exists(p_path); //by default just check file } @@ -168,10 +119,10 @@ void ResourceFormatLoader::get_recognized_extensions(List<String> *p_extensions) } } -RES ResourceFormatLoader::load(const String &p_path, const String &p_original_path, Error *r_error) { +RES ResourceFormatLoader::load(const String &p_path, const String &p_original_path, Error *r_error, bool p_use_sub_threads, float *r_progress) { if (get_script_instance() && get_script_instance()->has_method("load")) { - Variant res = get_script_instance()->call("load", p_path, p_original_path); + Variant res = get_script_instance()->call("load", p_path, p_original_path, p_use_sub_threads); if (res.get_type() == Variant::INT) { @@ -184,29 +135,11 @@ RES ResourceFormatLoader::load(const String &p_path, const String &p_original_pa *r_error = OK; return res; } - } - - //or this must be implemented - Ref<ResourceInteractiveLoader> ril = load_interactive(p_path, p_original_path, r_error); - if (!ril.is_valid()) - return RES(); - ril->set_local_path(p_original_path); - - while (true) { - - Error err = ril->poll(); - - if (err == ERR_FILE_EOF) { - if (r_error) - *r_error = OK; - return ril->get_resource(); - } - if (r_error) - *r_error = err; - - ERR_FAIL_COND_V_MSG(err != OK, RES(), "Failed to load resource '" + p_path + "'."); + return res; } + + ERR_FAIL_V_MSG(RES(), "Failed to load resource '" + p_path + "', ResourceFormatLoader::load was not implemented for this resource type."); } void ResourceFormatLoader::get_dependencies(const String &p_path, List<String> *p_dependencies, bool p_add_types) { @@ -256,7 +189,7 @@ void ResourceFormatLoader::_bind_methods() { /////////////////////////////////// -RES ResourceLoader::_load(const String &p_path, const String &p_original_path, const String &p_type_hint, bool p_no_cache, Error *r_error) { +RES ResourceLoader::_load(const String &p_path, const String &p_original_path, const String &p_type_hint, bool p_no_cache, Error *r_error, bool p_use_sub_threads, float *r_progress) { bool found = false; @@ -267,7 +200,7 @@ RES ResourceLoader::_load(const String &p_path, const String &p_original_path, c continue; } found = true; - RES res = loader[i]->load(p_path, p_original_path != String() ? p_original_path : p_path, r_error); + RES res = loader[i]->load(p_path, p_original_path != String() ? p_original_path : p_path, r_error, p_use_sub_threads, r_progress); if (res.is_null()) { continue; } @@ -285,49 +218,68 @@ RES ResourceLoader::_load(const String &p_path, const String &p_original_path, c ERR_FAIL_V_MSG(RES(), "No loader found for resource: " + p_path + "."); } -bool ResourceLoader::_add_to_loading_map(const String &p_path) { +void ResourceLoader::_thread_load_function(void *p_userdata) { - bool success; - MutexLock lock(loading_map_mutex); + ThreadLoadTask &load_task = *(ThreadLoadTask *)p_userdata; + load_task.loader_id = Thread::get_caller_id(); - LoadingMapKey key; - key.path = p_path; - key.thread = Thread::get_caller_id(); + if (load_task.semaphore) { + //this is an actual thread, so wait for Ok fom semaphore + thread_load_semaphore->wait(); //wait until its ok to start loading + } + load_task.resource = _load(load_task.remapped_path, load_task.remapped_path != load_task.local_path ? load_task.local_path : String(), load_task.type_hint, false, &load_task.error, load_task.use_sub_threads, &load_task.progress); + + load_task.progress = 1.0; //it was fully loaded at this point, so force progress to 1.0 - if (loading_map.has(key)) { - success = false; + thread_load_mutex->lock(); + if (load_task.error != OK) { + load_task.status = THREAD_LOAD_FAILED; } else { - loading_map[key] = true; - success = true; + load_task.status = THREAD_LOAD_LOADED; } + if (load_task.semaphore) { - return success; -} + if (load_task.start_next && thread_waiting_count > 0) { + thread_waiting_count--; + //thread loading count remains constant, this ends but another one begins + thread_load_semaphore->post(); + } else { + thread_loading_count--; //no threads waiting, just reduce loading count + } -void ResourceLoader::_remove_from_loading_map(const String &p_path) { - MutexLock lock(loading_map_mutex); + print_lt("END: load count: " + itos(thread_loading_count) + " / wait count: " + itos(thread_waiting_count) + " / suspended count: " + itos(thread_suspended_count) + " / active: " + itos(thread_loading_count - thread_suspended_count)); - LoadingMapKey key; - key.path = p_path; - key.thread = Thread::get_caller_id(); + for (int i = 0; i < load_task.poll_requests; i++) { + load_task.semaphore->post(); + } + memdelete(load_task.semaphore); + load_task.semaphore = nullptr; + } - loading_map.erase(key); -} + if (load_task.resource.is_valid()) { + load_task.resource->set_path(load_task.local_path); -void ResourceLoader::_remove_from_loading_map_and_thread(const String &p_path, Thread::ID p_thread) { - MutexLock lock(loading_map_mutex); + if (load_task.xl_remapped) + load_task.resource->set_as_translation_remapped(true); - LoadingMapKey key; - key.path = p_path; - key.thread = p_thread; +#ifdef TOOLS_ENABLED - loading_map.erase(key); -} + load_task.resource->set_edited(false); + if (timestamp_on_load) { + uint64_t mt = FileAccess::get_modified_time(load_task.remapped_path); + //printf("mt %s: %lli\n",remapped_path.utf8().get_data(),mt); + load_task.resource->set_last_modified_time(mt); + } +#endif -RES ResourceLoader::load(const String &p_path, const String &p_type_hint, bool p_no_cache, Error *r_error) { + if (_loaded_callback) { + _loaded_callback(load_task.resource, load_task.local_path); + } + } - if (r_error) - *r_error = ERR_CANT_OPEN; + thread_load_mutex->unlock(); +} +Error ResourceLoader::load_threaded_request(const String &p_path, const String &p_type_hint, bool p_use_sub_threads, const String &p_source_resource) { String local_path; if (p_path.is_rel_path()) @@ -335,88 +287,154 @@ RES ResourceLoader::load(const String &p_path, const String &p_type_hint, bool p else local_path = ProjectSettings::get_singleton()->localize_path(p_path); - if (!p_no_cache) { + thread_load_mutex->lock(); - { - bool success = _add_to_loading_map(local_path); - ERR_FAIL_COND_V_MSG(!success, RES(), "Resource: '" + local_path + "' is already being loaded. Cyclic reference?"); + if (p_source_resource != String()) { + //must be loading from this resource + if (!thread_load_tasks.has(p_source_resource)) { + thread_load_mutex->unlock(); + ERR_FAIL_V_MSG(ERR_INVALID_PARAMETER, "There is no thread loading source resource '" + p_source_resource + "'."); + } + //must be loading from this thread + if (thread_load_tasks[p_source_resource].loader_id != Thread::get_caller_id()) { + thread_load_mutex->unlock(); + ERR_FAIL_V_MSG(ERR_INVALID_PARAMETER, "Threading loading resource'" + local_path + " failed: Source specified: '" + p_source_resource + "' but was not called by it."); } - //lock first if possible - if (ResourceCache::lock) { - ResourceCache::lock->read_lock(); + //must not be already added as s sub tasks + if (thread_load_tasks[p_source_resource].sub_tasks.has(local_path)) { + thread_load_mutex->unlock(); + ERR_FAIL_V_MSG(ERR_INVALID_PARAMETER, "Thread loading source resource '" + p_source_resource + "' already is loading '" + local_path + "'."); } + } - //get ptr - Resource **rptr = ResourceCache::resources.getptr(local_path); + if (thread_load_tasks.has(local_path)) { + thread_load_tasks[local_path].requests++; + if (p_source_resource != String()) { + thread_load_tasks[p_source_resource].sub_tasks.insert(local_path); + } + thread_load_mutex->unlock(); + return OK; + } - if (rptr) { - RES res(*rptr); - //it is possible this resource was just freed in a thread. If so, this referencing will not work and resource is considered not cached - if (res.is_valid()) { - //referencing is fine - if (r_error) - *r_error = OK; - if (ResourceCache::lock) { - ResourceCache::lock->read_unlock(); + { + //create load task + + ThreadLoadTask load_task; + + load_task.requests = 1; + load_task.remapped_path = _path_remap(local_path, &load_task.xl_remapped); + load_task.local_path = local_path; + load_task.type_hint = p_type_hint; + load_task.use_sub_threads = p_use_sub_threads; + + { //must check if resource is already loaded before attempting to load it in a thread + + if (load_task.loader_id == Thread::get_caller_id()) { + thread_load_mutex->unlock(); + ERR_FAIL_V_MSG(ERR_INVALID_PARAMETER, "Attempted to load a resource already being loaded from this thread, cyclic reference?"); + } + //lock first if possible + if (ResourceCache::lock) { + ResourceCache::lock->read_lock(); + } + + //get ptr + Resource **rptr = ResourceCache::resources.getptr(local_path); + + if (rptr) { + RES res(*rptr); + //it is possible this resource was just freed in a thread. If so, this referencing will not work and resource is considered not cached + if (res.is_valid()) { + //referencing is fine + load_task.resource = res; + load_task.status = THREAD_LOAD_LOADED; + load_task.progress = 1.0; } - _remove_from_loading_map(local_path); - return res; + } + if (ResourceCache::lock) { + ResourceCache::lock->read_unlock(); } } - if (ResourceCache::lock) { - ResourceCache::lock->read_unlock(); + + if (p_source_resource != String()) { + thread_load_tasks[p_source_resource].sub_tasks.insert(local_path); } + + thread_load_tasks[local_path] = load_task; } - bool xl_remapped = false; - String path = _path_remap(local_path, &xl_remapped); + ThreadLoadTask &load_task = thread_load_tasks[local_path]; - if (path == "") { - if (!p_no_cache) { - _remove_from_loading_map(local_path); + if (load_task.resource.is_null()) { //needs to be loaded in thread + + load_task.semaphore = memnew(Semaphore); + if (thread_loading_count < thread_load_max) { + thread_loading_count++; + thread_load_semaphore->post(); //we have free threads, so allow one + } else { + thread_waiting_count++; } - ERR_FAIL_V_MSG(RES(), "Remapping '" + local_path + "' failed."); + + print_lt("REQUEST: load count: " + itos(thread_loading_count) + " / wait count: " + itos(thread_waiting_count) + " / suspended count: " + itos(thread_suspended_count) + " / active: " + itos(thread_loading_count - thread_suspended_count)); + + load_task.thread = Thread::create(_thread_load_function, &thread_load_tasks[local_path]); + load_task.loader_id = load_task.thread->get_id(); } - print_verbose("Loading resource: " + path); - RES res = _load(path, local_path, p_type_hint, p_no_cache, r_error); + thread_load_mutex->unlock(); + + return OK; +} + +float ResourceLoader::_dependency_get_progress(const String &p_path) { - if (res.is_null()) { - if (!p_no_cache) { - _remove_from_loading_map(local_path); + if (thread_load_tasks.has(p_path)) { + ThreadLoadTask &load_task = thread_load_tasks[p_path]; + int dep_count = load_task.sub_tasks.size(); + if (dep_count > 0) { + float dep_progress = 0; + for (Set<String>::Element *E = load_task.sub_tasks.front(); E; E = E->next()) { + dep_progress += _dependency_get_progress(E->get()); + } + dep_progress /= float(dep_count); + dep_progress *= 0.5; + dep_progress += load_task.progress * 0.5; + return dep_progress; + } else { + return load_task.progress; } - print_verbose("Failed loading resource: " + path); - return RES(); + + } else { + return 1.0; //assume finished loading it so it no longer exists } - if (!p_no_cache) - res->set_path(local_path); +} - if (xl_remapped) - res->set_as_translation_remapped(true); +ResourceLoader::ThreadLoadStatus ResourceLoader::load_threaded_get_status(const String &p_path, float *r_progress) { -#ifdef TOOLS_ENABLED + String local_path; + if (p_path.is_rel_path()) + local_path = "res://" + p_path; + else + local_path = ProjectSettings::get_singleton()->localize_path(p_path); - res->set_edited(false); - if (timestamp_on_load) { - uint64_t mt = FileAccess::get_modified_time(path); - //printf("mt %s: %lli\n",remapped_path.utf8().get_data(),mt); - res->set_last_modified_time(mt); + thread_load_mutex->lock(); + if (!thread_load_tasks.has(local_path)) { + thread_load_mutex->unlock(); + return THREAD_LOAD_INVALID_RESOURCE; } -#endif - - if (!p_no_cache) { - _remove_from_loading_map(local_path); + ThreadLoadTask &load_task = thread_load_tasks[local_path]; + ThreadLoadStatus status; + status = load_task.status; + if (r_progress) { + *r_progress = _dependency_get_progress(local_path); } - if (_loaded_callback) { - _loaded_callback(res, p_path); - } + thread_load_mutex->unlock(); - return res; + return status; } - -bool ResourceLoader::exists(const String &p_path, const String &p_type_hint) { +RES ResourceLoader::load_threaded_get(const String &p_path, Error *r_error) { String local_path; if (p_path.is_rel_path()) @@ -424,29 +442,81 @@ bool ResourceLoader::exists(const String &p_path, const String &p_type_hint) { else local_path = ProjectSettings::get_singleton()->localize_path(p_path); - if (ResourceCache::has(local_path)) { - - return true; // If cached, it probably exists + thread_load_mutex->lock(); + if (!thread_load_tasks.has(local_path)) { + thread_load_mutex->unlock(); + if (r_error) { + *r_error = ERR_INVALID_PARAMETER; + } + return RES(); } - bool xl_remapped = false; - String path = _path_remap(local_path, &xl_remapped); + ThreadLoadTask &load_task = thread_load_tasks[local_path]; - // Try all loaders and pick the first match for the type hint - for (int i = 0; i < loader_count; i++) { + //semaphore still exists, meaning its still loading, request poll + Semaphore *semaphore = load_task.semaphore; + if (semaphore) { + load_task.poll_requests++; - if (!loader[i]->recognize_path(path, p_type_hint)) { - continue; + { + // As we got a semaphore, this means we are going to have to wait + // until the sub-resource is done loading + // + // As this thread will become 'blocked' we should "echange" its + // active status with a waiting one, to ensure load continues. + // + // This ensures loading is never blocked and that is also within + // the maximum number of active threads. + + if (thread_waiting_count > 0) { + thread_waiting_count--; + thread_loading_count++; + thread_load_semaphore->post(); + + load_task.start_next = false; //do not start next since we are doing it here + } + + thread_suspended_count++; + + print_lt("GET: load count: " + itos(thread_loading_count) + " / wait count: " + itos(thread_waiting_count) + " / suspended count: " + itos(thread_suspended_count) + " / active: " + itos(thread_loading_count - thread_suspended_count)); } - if (loader[i]->exists(path)) - return true; + thread_load_mutex->unlock(); + semaphore->wait(); + thread_load_mutex->lock(); + + thread_suspended_count--; + + if (!thread_load_tasks.has(local_path)) { //may have been erased during unlock and this was always an invalid call + thread_load_mutex->unlock(); + if (r_error) { + *r_error = ERR_INVALID_PARAMETER; + } + return RES(); + } } - return false; + RES resource = load_task.resource; + if (r_error) { + *r_error = load_task.error; + } + + load_task.requests--; + + if (load_task.requests == 0) { + if (load_task.thread) { //thread may not have been used + Thread::wait_to_finish(load_task.thread); + memdelete(load_task.thread); + } + thread_load_tasks.erase(local_path); + } + + thread_load_mutex->unlock(); + + return resource; } -Ref<ResourceInteractiveLoader> ResourceLoader::load_interactive(const String &p_path, const String &p_type_hint, bool p_no_cache, Error *r_error) { +RES ResourceLoader::load(const String &p_path, const String &p_type_hint, bool p_no_cache, Error *r_error) { if (r_error) *r_error = ERR_CANT_OPEN; @@ -459,61 +529,131 @@ Ref<ResourceInteractiveLoader> ResourceLoader::load_interactive(const String &p_ if (!p_no_cache) { - bool success = _add_to_loading_map(local_path); - ERR_FAIL_COND_V_MSG(!success, RES(), "Resource: '" + local_path + "' is already being loaded. Cyclic reference?"); + thread_load_mutex->lock(); - if (ResourceCache::has(local_path)) { + //Is it already being loaded? poll until done + if (thread_load_tasks.has(local_path)) { + Error err = load_threaded_request(p_path, p_type_hint); + if (err != OK) { + if (r_error) { + *r_error = err; + } + return RES(); + } + thread_load_mutex->unlock(); - print_verbose("Loading resource: " + local_path + " (cached)"); - Ref<Resource> res_cached = ResourceCache::get(local_path); - Ref<ResourceInteractiveLoaderDefault> ril = Ref<ResourceInteractiveLoaderDefault>(memnew(ResourceInteractiveLoaderDefault)); + return load_threaded_get(p_path, r_error); + } - ril->resource = res_cached; - ril->path_loading = local_path; - ril->path_loading_thread = Thread::get_caller_id(); - return ril; + //Is it cached? + if (ResourceCache::lock) { + ResourceCache::lock->read_lock(); } - } - bool xl_remapped = false; - String path = _path_remap(local_path, &xl_remapped); - if (path == "") { - if (!p_no_cache) { - _remove_from_loading_map(local_path); + Resource **rptr = ResourceCache::resources.getptr(local_path); + + if (rptr) { + RES res(*rptr); + + //it is possible this resource was just freed in a thread. If so, this referencing will not work and resource is considered not cached + if (res.is_valid()) { + if (ResourceCache::lock) { + ResourceCache::lock->read_unlock(); + } + thread_load_mutex->unlock(); + + if (r_error) { + *r_error = OK; + } + + return res; //use cached + } } - ERR_FAIL_V_MSG(RES(), "Remapping '" + local_path + "' failed."); - } - print_verbose("Loading resource: " + path); + if (ResourceCache::lock) { + ResourceCache::lock->read_unlock(); + } - bool found = false; - for (int i = 0; i < loader_count; i++) { + //load using task (but this thread) + ThreadLoadTask load_task; - if (!loader[i]->recognize_path(path, p_type_hint)) - continue; - found = true; - Ref<ResourceInteractiveLoader> ril = loader[i]->load_interactive(path, local_path, r_error); - if (ril.is_null()) - continue; - if (!p_no_cache) { - ril->set_local_path(local_path); - ril->path_loading = local_path; - ril->path_loading_thread = Thread::get_caller_id(); + load_task.requests = 1; + load_task.local_path = local_path; + load_task.remapped_path = _path_remap(local_path, &load_task.xl_remapped); + load_task.type_hint = p_type_hint; + load_task.loader_id = Thread::get_caller_id(); + + thread_load_tasks[local_path] = load_task; + + thread_load_mutex->unlock(); + + _thread_load_function(&thread_load_tasks[local_path]); + + return load_threaded_get(p_path, r_error); + + } else { + + bool xl_remapped = false; + String path = _path_remap(local_path, &xl_remapped); + + if (path == "") { + ERR_FAIL_V_MSG(RES(), "Remapping '" + local_path + "' failed."); + } + + print_verbose("Loading resource: " + path); + float p; + RES res = _load(path, local_path, p_type_hint, p_no_cache, r_error, false, &p); + + if (res.is_null()) { + print_verbose("Failed loading resource: " + path); + return RES(); } if (xl_remapped) - ril->set_translation_remapped(true); + res->set_as_translation_remapped(true); + +#ifdef TOOLS_ENABLED + + res->set_edited(false); + if (timestamp_on_load) { + uint64_t mt = FileAccess::get_modified_time(path); + //printf("mt %s: %lli\n",remapped_path.utf8().get_data(),mt); + res->set_last_modified_time(mt); + } +#endif - return ril; + return res; } +} - if (!p_no_cache) { - _remove_from_loading_map(local_path); +bool ResourceLoader::exists(const String &p_path, const String &p_type_hint) { + + String local_path; + if (p_path.is_rel_path()) + local_path = "res://" + p_path; + else + local_path = ProjectSettings::get_singleton()->localize_path(p_path); + + if (ResourceCache::has(local_path)) { + + return true; // If cached, it probably exists } - ERR_FAIL_COND_V_MSG(found, Ref<ResourceInteractiveLoader>(), "Failed loading resource: " + path + "."); + bool xl_remapped = false; + String path = _path_remap(local_path, &xl_remapped); - ERR_FAIL_V_MSG(Ref<ResourceInteractiveLoader>(), "No loader found for resource: " + path + "."); + // Try all loaders and pick the first match for the type hint + for (int i = 0; i < loader_count; i++) { + + if (!loader[i]->recognize_path(path, p_type_hint)) { + continue; + } + + if (loader[i]->exists(path)) + return true; + } + + return false; } void ResourceLoader::add_resource_format_loader(Ref<ResourceFormatLoader> p_format_loader, bool p_at_front) { @@ -984,20 +1124,19 @@ void ResourceLoader::remove_custom_loaders() { } } -Mutex ResourceLoader::loading_map_mutex; -HashMap<ResourceLoader::LoadingMapKey, int, ResourceLoader::LoadingMapKeyHasher> ResourceLoader::loading_map; - void ResourceLoader::initialize() { + thread_load_mutex = memnew(Mutex); + thread_load_max = OS::get_singleton()->get_processor_count(); + thread_loading_count = 0; + thread_waiting_count = 0; + thread_suspended_count = 0; + thread_load_semaphore = memnew(Semaphore); } void ResourceLoader::finalize() { -#ifndef NO_THREADS - const LoadingMapKey *K = NULL; - while ((K = loading_map.next(K))) { - ERR_PRINT("Exited while resource is being loaded: " + K->path); - } - loading_map.clear(); -#endif + + memdelete(thread_load_mutex); + memdelete(thread_load_semaphore); } ResourceLoadErrorNotify ResourceLoader::err_notify = NULL; @@ -1009,6 +1148,15 @@ void *ResourceLoader::dep_err_notify_ud = NULL; bool ResourceLoader::abort_on_missing_resource = true; bool ResourceLoader::timestamp_on_load = false; +Mutex *ResourceLoader::thread_load_mutex = nullptr; +HashMap<String, ResourceLoader::ThreadLoadTask> ResourceLoader::thread_load_tasks; +Semaphore *ResourceLoader::thread_load_semaphore = nullptr; + +int ResourceLoader::thread_loading_count = 0; +int ResourceLoader::thread_waiting_count = 0; +int ResourceLoader::thread_suspended_count = 0; +int ResourceLoader::thread_load_max = 0; + SelfList<Resource>::List ResourceLoader::remapped_list; HashMap<String, Vector<String> > ResourceLoader::translation_remaps; HashMap<String, String> ResourceLoader::path_remaps; diff --git a/core/io/resource_loader.h b/core/io/resource_loader.h index 172f8e979b..3b7a27f551 100644 --- a/core/io/resource_loader.h +++ b/core/io/resource_loader.h @@ -31,32 +31,10 @@ #ifndef RESOURCE_LOADER_H #define RESOURCE_LOADER_H +#include "core/os/semaphore.h" #include "core/os/thread.h" #include "core/resource.h" -class ResourceInteractiveLoader : public Reference { - - GDCLASS(ResourceInteractiveLoader, Reference); - friend class ResourceLoader; - String path_loading; - Thread::ID path_loading_thread; - -protected: - static void _bind_methods(); - -public: - virtual void set_local_path(const String &p_local_path) = 0; - virtual Ref<Resource> get_resource() = 0; - virtual Error poll() = 0; - virtual int get_stage() const = 0; - virtual int get_stage_count() const = 0; - virtual void set_translation_remapped(bool p_remapped) = 0; - virtual Error wait(); - - ResourceInteractiveLoader() {} - ~ResourceInteractiveLoader(); -}; - class ResourceFormatLoader : public Reference { GDCLASS(ResourceFormatLoader, Reference); @@ -65,8 +43,7 @@ protected: static void _bind_methods(); public: - virtual Ref<ResourceInteractiveLoader> load_interactive(const String &p_path, const String &p_original_path = "", Error *r_error = NULL); - virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL); + virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL, bool p_use_sub_threads = false, float *r_progress = nullptr); virtual bool exists(const String &p_path) const; virtual void get_recognized_extensions(List<String> *p_extensions) const; virtual void get_recognized_extensions_for_type(const String &p_type, List<String> *p_extensions) const; @@ -95,6 +72,15 @@ class ResourceLoader { MAX_LOADERS = 64 }; +public: + enum ThreadLoadStatus { + THREAD_LOAD_INVALID_RESOURCE, + THREAD_LOAD_IN_PROGRESS, + THREAD_LOAD_FAILED, + THREAD_LOAD_LOADED + }; + +private: static Ref<ResourceFormatLoader> loader[MAX_LOADERS]; static int loader_count; static bool timestamp_on_load; @@ -115,34 +101,47 @@ class ResourceLoader { friend class ResourceFormatImporter; friend class ResourceInteractiveLoader; //internal load function - static RES _load(const String &p_path, const String &p_original_path, const String &p_type_hint, bool p_no_cache, Error *r_error); + static RES _load(const String &p_path, const String &p_original_path, const String &p_type_hint, bool p_no_cache, Error *r_error, bool p_use_sub_threads, float *r_progress); static ResourceLoadedCallback _loaded_callback; static Ref<ResourceFormatLoader> _find_custom_resource_format_loader(String path); - static Mutex loading_map_mutex; - - //used to track paths being loaded in a thread, avoids cyclic recursion - struct LoadingMapKey { - String path; - Thread::ID thread; - bool operator==(const LoadingMapKey &p_key) const { - return (thread == p_key.thread && path == p_key.path); - } - }; - struct LoadingMapKeyHasher { - static _FORCE_INLINE_ uint32_t hash(const LoadingMapKey &p_key) { return p_key.path.hash() + HashMapHasherDefault::hash(p_key.thread); } + struct ThreadLoadTask { + Thread *thread = nullptr; + Thread::ID loader_id = 0; + Semaphore *semaphore = nullptr; + String local_path; + String remapped_path; + String type_hint; + float progress = 0.0; + ThreadLoadStatus status = THREAD_LOAD_IN_PROGRESS; + Error error; + RES resource; + bool xl_remapped = false; + bool use_sub_threads = false; + bool start_next = true; + int requests = 0; + int poll_requests = 0; + Set<String> sub_tasks; }; - static HashMap<LoadingMapKey, int, LoadingMapKeyHasher> loading_map; + static void _thread_load_function(void *p_userdata); + static Mutex *thread_load_mutex; + static HashMap<String, ThreadLoadTask> thread_load_tasks; + static Semaphore *thread_load_semaphore; + static int thread_waiting_count; + static int thread_loading_count; + static int thread_suspended_count; + static int thread_load_max; - static bool _add_to_loading_map(const String &p_path); - static void _remove_from_loading_map(const String &p_path); - static void _remove_from_loading_map_and_thread(const String &p_path, Thread::ID p_thread); + static float _dependency_get_progress(const String &p_path); public: - static Ref<ResourceInteractiveLoader> load_interactive(const String &p_path, const String &p_type_hint = "", bool p_no_cache = false, Error *r_error = NULL); + static Error load_threaded_request(const String &p_path, const String &p_type_hint = "", bool p_use_sub_threads = false, const String &p_source_resource = String()); + static ThreadLoadStatus load_threaded_get_status(const String &p_path, float *r_progress = nullptr); + static RES load_threaded_get(const String &p_path, Error *r_error = NULL); + static RES load(const String &p_path, const String &p_type_hint = "", bool p_no_cache = false, Error *r_error = NULL); static bool exists(const String &p_path, const String &p_type_hint = ""); diff --git a/core/io/translation_loader_po.cpp b/core/io/translation_loader_po.cpp index 4f7eeddc43..4051bf2947 100644 --- a/core/io/translation_loader_po.cpp +++ b/core/io/translation_loader_po.cpp @@ -176,7 +176,7 @@ RES TranslationLoaderPO::load_translation(FileAccess *f, Error *r_error, const S return translation; } -RES TranslationLoaderPO::load(const String &p_path, const String &p_original_path, Error *r_error) { +RES TranslationLoaderPO::load(const String &p_path, const String &p_original_path, Error *r_error, bool p_use_sub_threads, float *r_progress) { if (r_error) *r_error = ERR_CANT_OPEN; diff --git a/core/io/translation_loader_po.h b/core/io/translation_loader_po.h index 47e64276ca..fe3a75e5eb 100644 --- a/core/io/translation_loader_po.h +++ b/core/io/translation_loader_po.h @@ -38,7 +38,7 @@ class TranslationLoaderPO : public ResourceFormatLoader { public: static RES load_translation(FileAccess *f, Error *r_error, const String &p_path = String()); - virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL); + virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL, bool p_use_sub_threads = false, float *r_progress = nullptr); virtual void get_recognized_extensions(List<String> *p_extensions) const; virtual bool handles_type(const String &p_type) const; virtual String get_resource_type(const String &p_path) const; diff --git a/core/object.h b/core/object.h index dc7d49f534..59d3f06cfe 100644 --- a/core/object.h +++ b/core/object.h @@ -126,6 +126,7 @@ enum PropertyUsageFlags { PROPERTY_USAGE_NODE_PATH_FROM_SCENE_ROOT = 1 << 23, PROPERTY_USAGE_RESOURCE_NOT_PERSISTENT = 1 << 24, PROPERTY_USAGE_KEYING_INCREMENTS = 1 << 25, // Used in inspector to increment property when keyed in animation player + PROPERTY_USAGE_DEFERRED_SET_RESOURCE = 1 << 26, // when loading, the resource for this property can be set at the end of loading PROPERTY_USAGE_DEFAULT = PROPERTY_USAGE_STORAGE | PROPERTY_USAGE_EDITOR | PROPERTY_USAGE_NETWORK, PROPERTY_USAGE_DEFAULT_INTL = PROPERTY_USAGE_STORAGE | PROPERTY_USAGE_EDITOR | PROPERTY_USAGE_NETWORK | PROPERTY_USAGE_INTERNATIONALIZED, diff --git a/core/register_core_types.cpp b/core/register_core_types.cpp index ac60061c67..07a252ad31 100644 --- a/core/register_core_types.cpp +++ b/core/register_core_types.cpp @@ -186,8 +186,6 @@ void register_core_types() { ClassDB::register_class<HTTPClient>(); ClassDB::register_class<TriangleMesh>(); - ClassDB::register_virtual_class<ResourceInteractiveLoader>(); - ClassDB::register_class<ResourceFormatLoader>(); ClassDB::register_class<ResourceFormatSaver>(); |