summaryrefslogtreecommitdiff
path: root/core
diff options
context:
space:
mode:
Diffstat (limited to 'core')
-rw-r--r--core/bind/core_bind.cpp27
-rw-r--r--core/bind/core_bind.h15
-rw-r--r--core/crypto/crypto.cpp2
-rw-r--r--core/crypto/crypto.h2
-rw-r--r--core/io/image_loader.cpp2
-rw-r--r--core/io/image_loader.h2
-rw-r--r--core/io/resource_format_binary.cpp313
-rw-r--r--core/io/resource_format_binary.h25
-rw-r--r--core/io/resource_importer.cpp4
-rw-r--r--core/io/resource_importer.h2
-rw-r--r--core/io/resource_loader.cpp596
-rw-r--r--core/io/resource_loader.h85
-rw-r--r--core/io/translation_loader_po.cpp2
-rw-r--r--core/io/translation_loader_po.h2
-rw-r--r--core/object.h1
-rw-r--r--core/register_core_types.cpp2
16 files changed, 643 insertions, 439 deletions
diff --git a/core/bind/core_bind.cpp b/core/bind/core_bind.cpp
index d9cc77feb3..e99e69578c 100644
--- a/core/bind/core_bind.cpp
+++ b/core/bind/core_bind.cpp
@@ -64,8 +64,21 @@ static const unsigned int MONTH_DAYS_TABLE[2][12] = {
_ResourceLoader *_ResourceLoader::singleton = NULL;
-Ref<ResourceInteractiveLoader> _ResourceLoader::load_interactive(const String &p_path, const String &p_type_hint) {
- return ResourceLoader::load_interactive(p_path, p_type_hint);
+Error _ResourceLoader::load_threaded_request(const String &p_path, const String &p_type_hint, bool p_use_sub_threads) {
+
+ return ResourceLoader::load_threaded_request(p_path, p_type_hint, p_use_sub_threads);
+}
+_ResourceLoader::ThreadLoadStatus _ResourceLoader::load_threaded_get_status(const String &p_path, Array r_progress) {
+ float progress = 0;
+ ResourceLoader::ThreadLoadStatus tls = ResourceLoader::load_threaded_get_status(p_path, &progress);
+ r_progress.resize(1);
+ r_progress[0] = progress;
+ return (ThreadLoadStatus)tls;
+}
+RES _ResourceLoader::load_threaded_get(const String &p_path) {
+ Error error;
+ RES res = ResourceLoader::load_threaded_get(p_path, &error);
+ return res;
}
RES _ResourceLoader::load(const String &p_path, const String &p_type_hint, bool p_no_cache) {
@@ -120,13 +133,21 @@ bool _ResourceLoader::exists(const String &p_path, const String &p_type_hint) {
void _ResourceLoader::_bind_methods() {
- ClassDB::bind_method(D_METHOD("load_interactive", "path", "type_hint"), &_ResourceLoader::load_interactive, DEFVAL(""));
+ ClassDB::bind_method(D_METHOD("load_threaded_request", "path", "type_hint", "use_sub_threads"), &_ResourceLoader::load_threaded_request, DEFVAL(""), DEFVAL(false));
+ ClassDB::bind_method(D_METHOD("load_threaded_get_status", "path", "r_progress"), &_ResourceLoader::load_threaded_get_status, DEFVAL(Array()));
+ ClassDB::bind_method(D_METHOD("load_threaded_get", "path"), &_ResourceLoader::load_threaded_get);
+
ClassDB::bind_method(D_METHOD("load", "path", "type_hint", "no_cache"), &_ResourceLoader::load, DEFVAL(""), DEFVAL(false));
ClassDB::bind_method(D_METHOD("get_recognized_extensions_for_type", "type"), &_ResourceLoader::get_recognized_extensions_for_type);
ClassDB::bind_method(D_METHOD("set_abort_on_missing_resources", "abort"), &_ResourceLoader::set_abort_on_missing_resources);
ClassDB::bind_method(D_METHOD("get_dependencies", "path"), &_ResourceLoader::get_dependencies);
ClassDB::bind_method(D_METHOD("has_cached", "path"), &_ResourceLoader::has_cached);
ClassDB::bind_method(D_METHOD("exists", "path", "type_hint"), &_ResourceLoader::exists, DEFVAL(""));
+
+ BIND_ENUM_CONSTANT(THREAD_LOAD_INVALID_RESOURCE);
+ BIND_ENUM_CONSTANT(THREAD_LOAD_IN_PROGRESS);
+ BIND_ENUM_CONSTANT(THREAD_LOAD_FAILED);
+ BIND_ENUM_CONSTANT(THREAD_LOAD_LOADED);
}
_ResourceLoader::_ResourceLoader() {
diff --git a/core/bind/core_bind.h b/core/bind/core_bind.h
index e68c40c9e4..d4a7c00629 100644
--- a/core/bind/core_bind.h
+++ b/core/bind/core_bind.h
@@ -49,8 +49,19 @@ protected:
static _ResourceLoader *singleton;
public:
+ enum ThreadLoadStatus {
+ THREAD_LOAD_INVALID_RESOURCE,
+ THREAD_LOAD_IN_PROGRESS,
+ THREAD_LOAD_FAILED,
+ THREAD_LOAD_LOADED
+ };
+
static _ResourceLoader *get_singleton() { return singleton; }
- Ref<ResourceInteractiveLoader> load_interactive(const String &p_path, const String &p_type_hint = "");
+
+ Error load_threaded_request(const String &p_path, const String &p_type_hint = "", bool p_use_sub_threads = false);
+ ThreadLoadStatus load_threaded_get_status(const String &p_path, Array r_progress = Array());
+ RES load_threaded_get(const String &p_path);
+
RES load(const String &p_path, const String &p_type_hint = "", bool p_no_cache = false);
Vector<String> get_recognized_extensions_for_type(const String &p_type);
void set_abort_on_missing_resources(bool p_abort);
@@ -61,6 +72,8 @@ public:
_ResourceLoader();
};
+VARIANT_ENUM_CAST(_ResourceLoader::ThreadLoadStatus);
+
class _ResourceSaver : public Object {
GDCLASS(_ResourceSaver, Object);
diff --git a/core/crypto/crypto.cpp b/core/crypto/crypto.cpp
index 3711b26e47..793bf719b7 100644
--- a/core/crypto/crypto.cpp
+++ b/core/crypto/crypto.cpp
@@ -99,7 +99,7 @@ Crypto::Crypto() {
/// Resource loader/saver
-RES ResourceFormatLoaderCrypto::load(const String &p_path, const String &p_original_path, Error *r_error) {
+RES ResourceFormatLoaderCrypto::load(const String &p_path, const String &p_original_path, Error *r_error, bool p_use_sub_threads, float *r_progress) {
String el = p_path.get_extension().to_lower();
if (el == "crt") {
diff --git a/core/crypto/crypto.h b/core/crypto/crypto.h
index babd0bc4eb..3279c0620f 100644
--- a/core/crypto/crypto.h
+++ b/core/crypto/crypto.h
@@ -87,7 +87,7 @@ class ResourceFormatLoaderCrypto : public ResourceFormatLoader {
GDCLASS(ResourceFormatLoaderCrypto, ResourceFormatLoader);
public:
- virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL);
+ virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL, bool p_use_sub_threads = false, float *r_progress = nullptr);
virtual void get_recognized_extensions(List<String> *p_extensions) const;
virtual bool handles_type(const String &p_type) const;
virtual String get_resource_type(const String &p_path) const;
diff --git a/core/io/image_loader.cpp b/core/io/image_loader.cpp
index 720f25f91b..99ac5bcdd9 100644
--- a/core/io/image_loader.cpp
+++ b/core/io/image_loader.cpp
@@ -129,7 +129,7 @@ void ImageLoader::cleanup() {
/////////////////
-RES ResourceFormatLoaderImage::load(const String &p_path, const String &p_original_path, Error *r_error) {
+RES ResourceFormatLoaderImage::load(const String &p_path, const String &p_original_path, Error *r_error, bool p_use_sub_threads, float *r_progress) {
FileAccess *f = FileAccess::open(p_path, FileAccess::READ);
if (!f) {
diff --git a/core/io/image_loader.h b/core/io/image_loader.h
index d6dfd261ca..3ba028b99c 100644
--- a/core/io/image_loader.h
+++ b/core/io/image_loader.h
@@ -73,7 +73,7 @@ public:
class ResourceFormatLoaderImage : public ResourceFormatLoader {
public:
- virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL);
+ virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL, bool p_use_sub_threads = false, float *r_progress = nullptr);
virtual void get_recognized_extensions(List<String> *p_extensions) const;
virtual bool handles_type(const String &p_type) const;
virtual String get_resource_type(const String &p_path) const;
diff --git a/core/io/resource_format_binary.cpp b/core/io/resource_format_binary.cpp
index 8c343a0f43..54b75cc29d 100644
--- a/core/io/resource_format_binary.cpp
+++ b/core/io/resource_format_binary.cpp
@@ -93,7 +93,7 @@ enum {
};
-void ResourceInteractiveLoaderBinary::_advance_padding(uint32_t p_len) {
+void ResourceLoaderBinary::_advance_padding(uint32_t p_len) {
uint32_t extra = 4 - (p_len % 4);
if (extra < 4) {
@@ -102,7 +102,7 @@ void ResourceInteractiveLoaderBinary::_advance_padding(uint32_t p_len) {
}
}
-StringName ResourceInteractiveLoaderBinary::_get_string() {
+StringName ResourceLoaderBinary::_get_string() {
uint32_t id = f->get_32();
if (id & 0x80000000) {
@@ -121,7 +121,7 @@ StringName ResourceInteractiveLoaderBinary::_get_string() {
return string_map[id];
}
-Error ResourceInteractiveLoaderBinary::parse_variant(Variant &r_v) {
+Error ResourceLoaderBinary::parse_variant(Variant &r_v) {
uint32_t type = f->get_32();
print_bl("find property of type: " + itos(type));
@@ -377,20 +377,26 @@ Error ResourceInteractiveLoaderBinary::parse_variant(Variant &r_v) {
r_v = Variant();
} else {
- String exttype = external_resources[erindex].type;
- String path = external_resources[erindex].path;
+ if (external_resources[erindex].cache.is_null()) {
+ //cache not here yet, wait for it?
+ if (use_sub_threads) {
+ Error err;
+ external_resources.write[erindex].cache = ResourceLoader::load_threaded_get(external_resources[erindex].path, &err);
- if (path.find("://") == -1 && path.is_rel_path()) {
- // path is relative to file being loaded, so convert to a resource path
- path = ProjectSettings::get_singleton()->localize_path(res_path.get_base_dir().plus_file(path));
- }
+ if (err != OK || external_resources[erindex].cache.is_null()) {
+ if (!ResourceLoader::get_abort_on_missing_resources()) {
- RES res = ResourceLoader::load(path, exttype);
+ ResourceLoader::notify_dependency_error(local_path, external_resources[erindex].path, external_resources[erindex].type);
+ } else {
- if (res.is_null()) {
- WARN_PRINT(String("Couldn't load resource: " + path).utf8().get_data());
+ error = ERR_FILE_MISSING_DEPENDENCIES;
+ ERR_FAIL_V_MSG(error, "Can't load dependency: " + external_resources[erindex].path + ".");
+ }
+ }
+ }
}
- r_v = res;
+
+ r_v = external_resources[erindex].cache;
}
} break;
@@ -638,160 +644,168 @@ Error ResourceInteractiveLoaderBinary::parse_variant(Variant &r_v) {
return OK; //never reach anyway
}
-void ResourceInteractiveLoaderBinary::set_local_path(const String &p_local_path) {
+void ResourceLoaderBinary::set_local_path(const String &p_local_path) {
res_path = p_local_path;
}
-Ref<Resource> ResourceInteractiveLoaderBinary::get_resource() {
+Ref<Resource> ResourceLoaderBinary::get_resource() {
return resource;
}
-Error ResourceInteractiveLoaderBinary::poll() {
+Error ResourceLoaderBinary::load() {
if (error != OK)
return error;
- int s = stage;
+ int stage = 0;
- if (s < external_resources.size()) {
+ for (int i = 0; i < external_resources.size(); i++) {
- String path = external_resources[s].path;
+ String path = external_resources[i].path;
if (remaps.has(path)) {
path = remaps[path];
}
- RES res = ResourceLoader::load(path, external_resources[s].type);
- if (res.is_null()) {
- if (!ResourceLoader::get_abort_on_missing_resources()) {
+ if (path.find("://") == -1 && path.is_rel_path()) {
+ // path is relative to file being loaded, so convert to a resource path
+ path = ProjectSettings::get_singleton()->localize_path(path.get_base_dir().plus_file(external_resources[i].path));
+ }
+
+ external_resources.write[i].path = path; //remap happens here, not on load because on load it can actually be used for filesystem dock resource remap
- ResourceLoader::notify_dependency_error(local_path, path, external_resources[s].type);
- } else {
+ if (!use_sub_threads) {
+ external_resources.write[i].cache = ResourceLoader::load(path, external_resources[i].type);
- error = ERR_FILE_MISSING_DEPENDENCIES;
- ERR_FAIL_V_MSG(error, "Can't load dependency: " + path + ".");
+ if (external_resources[i].cache.is_null()) {
+ if (!ResourceLoader::get_abort_on_missing_resources()) {
+
+ ResourceLoader::notify_dependency_error(local_path, path, external_resources[i].type);
+ } else {
+
+ error = ERR_FILE_MISSING_DEPENDENCIES;
+ ERR_FAIL_V_MSG(error, "Can't load dependency: " + path + ".");
+ }
}
} else {
- resource_cache.push_back(res);
+ Error err = ResourceLoader::load_threaded_request(path, external_resources[i].type, use_sub_threads, local_path);
+ if (err != OK) {
+ if (!ResourceLoader::get_abort_on_missing_resources()) {
+
+ ResourceLoader::notify_dependency_error(local_path, path, external_resources[i].type);
+ } else {
+
+ error = ERR_FILE_MISSING_DEPENDENCIES;
+ ERR_FAIL_V_MSG(error, "Can't load dependency: " + path + ".");
+ }
+ }
}
stage++;
- return error;
}
- s -= external_resources.size();
+ for (int i = 0; i < internal_resources.size(); i++) {
- if (s >= internal_resources.size()) {
+ bool main = i == (internal_resources.size() - 1);
- error = ERR_BUG;
- ERR_FAIL_COND_V(s >= internal_resources.size(), error);
- }
+ //maybe it is loaded already
+ String path;
+ int subindex = 0;
- bool main = s == (internal_resources.size() - 1);
+ if (!main) {
- //maybe it is loaded already
- String path;
- int subindex = 0;
+ path = internal_resources[i].path;
+ if (path.begins_with("local://")) {
+ path = path.replace_first("local://", "");
+ subindex = path.to_int();
+ path = res_path + "::" + path;
+ }
- if (!main) {
+ if (ResourceCache::has(path)) {
+ //already loaded, don't do anything
+ stage++;
+ error = OK;
+ continue;
+ }
+ } else {
- path = internal_resources[s].path;
- if (path.begins_with("local://")) {
- path = path.replace_first("local://", "");
- subindex = path.to_int();
- path = res_path + "::" + path;
+ if (!ResourceCache::has(res_path))
+ path = res_path;
}
- if (ResourceCache::has(path)) {
- //already loaded, don't do anything
- stage++;
- error = OK;
- return error;
- }
- } else {
+ uint64_t offset = internal_resources[i].offset;
- if (!ResourceCache::has(res_path))
- path = res_path;
- }
-
- uint64_t offset = internal_resources[s].offset;
+ f->seek(offset);
- f->seek(offset);
+ String t = get_unicode_string();
- String t = get_unicode_string();
-
- Object *obj = ClassDB::instance(t);
- if (!obj) {
- error = ERR_FILE_CORRUPT;
- ERR_FAIL_V_MSG(ERR_FILE_CORRUPT, local_path + ":Resource of unrecognized type in file: " + t + ".");
- }
+ Object *obj = ClassDB::instance(t);
+ if (!obj) {
+ error = ERR_FILE_CORRUPT;
+ ERR_FAIL_V_MSG(ERR_FILE_CORRUPT, local_path + ":Resource of unrecognized type in file: " + t + ".");
+ }
- Resource *r = Object::cast_to<Resource>(obj);
- if (!r) {
- String obj_class = obj->get_class();
- error = ERR_FILE_CORRUPT;
- memdelete(obj); //bye
- ERR_FAIL_V_MSG(ERR_FILE_CORRUPT, local_path + ":Resource type in resource field not a resource, type is: " + obj_class + ".");
- }
+ Resource *r = Object::cast_to<Resource>(obj);
+ if (!r) {
+ String obj_class = obj->get_class();
+ error = ERR_FILE_CORRUPT;
+ memdelete(obj); //bye
+ ERR_FAIL_V_MSG(ERR_FILE_CORRUPT, local_path + ":Resource type in resource field not a resource, type is: " + obj_class + ".");
+ }
- RES res = RES(r);
+ RES res = RES(r);
- r->set_path(path);
- r->set_subindex(subindex);
+ r->set_path(path);
+ r->set_subindex(subindex);
- int pc = f->get_32();
+ int pc = f->get_32();
- //set properties
+ //set properties
- for (int i = 0; i < pc; i++) {
+ for (int j = 0; j < pc; j++) {
- StringName name = _get_string();
+ StringName name = _get_string();
- if (name == StringName()) {
- error = ERR_FILE_CORRUPT;
- ERR_FAIL_V(ERR_FILE_CORRUPT);
- }
+ if (name == StringName()) {
+ error = ERR_FILE_CORRUPT;
+ ERR_FAIL_V(ERR_FILE_CORRUPT);
+ }
- Variant value;
+ Variant value;
- error = parse_variant(value);
- if (error)
- return error;
+ error = parse_variant(value);
+ if (error)
+ return error;
- res->set(name, value);
- }
+ res->set(name, value);
+ }
#ifdef TOOLS_ENABLED
- res->set_edited(false);
+ res->set_edited(false);
#endif
- stage++;
+ stage++;
- resource_cache.push_back(res);
+ if (progress) {
+ *progress = (i + 1) / float(internal_resources.size());
+ }
- if (main) {
+ resource_cache.push_back(res);
- f->close();
- resource = res;
- resource->set_as_translation_remapped(translation_remapped);
- error = ERR_FILE_EOF;
+ if (main) {
- } else {
- error = OK;
+ f->close();
+ resource = res;
+ resource->set_as_translation_remapped(translation_remapped);
+ error = OK;
+ return OK;
+ }
}
- return OK;
-}
-int ResourceInteractiveLoaderBinary::get_stage() const {
-
- return stage;
-}
-int ResourceInteractiveLoaderBinary::get_stage_count() const {
-
- return external_resources.size() + internal_resources.size();
+ return ERR_FILE_EOF;
}
-void ResourceInteractiveLoaderBinary::set_translation_remapped(bool p_remapped) {
+void ResourceLoaderBinary::set_translation_remapped(bool p_remapped) {
translation_remapped = p_remapped;
}
@@ -814,7 +828,7 @@ static String get_ustring(FileAccess *f) {
return s;
}
-String ResourceInteractiveLoaderBinary::get_unicode_string() {
+String ResourceLoaderBinary::get_unicode_string() {
int len = f->get_32();
if (len > str_buf.size()) {
@@ -828,7 +842,7 @@ String ResourceInteractiveLoaderBinary::get_unicode_string() {
return s;
}
-void ResourceInteractiveLoaderBinary::get_dependencies(FileAccess *p_f, List<String> *p_dependencies, bool p_add_types) {
+void ResourceLoaderBinary::get_dependencies(FileAccess *p_f, List<String> *p_dependencies, bool p_add_types) {
open(p_f);
if (error)
@@ -846,7 +860,7 @@ void ResourceInteractiveLoaderBinary::get_dependencies(FileAccess *p_f, List<Str
}
}
-void ResourceInteractiveLoaderBinary::open(FileAccess *p_f) {
+void ResourceLoaderBinary::open(FileAccess *p_f) {
error = OK;
@@ -947,7 +961,7 @@ void ResourceInteractiveLoaderBinary::open(FileAccess *p_f) {
}
}
-String ResourceInteractiveLoaderBinary::recognize(FileAccess *p_f) {
+String ResourceLoaderBinary::recognize(FileAccess *p_f) {
error = OK;
@@ -992,20 +1006,22 @@ String ResourceInteractiveLoaderBinary::recognize(FileAccess *p_f) {
return type;
}
-ResourceInteractiveLoaderBinary::ResourceInteractiveLoaderBinary() :
+ResourceLoaderBinary::ResourceLoaderBinary() :
translation_remapped(false),
f(NULL),
- error(OK),
- stage(0) {
+ error(OK) {
+
+ progress = nullptr;
+ use_sub_threads = false;
}
-ResourceInteractiveLoaderBinary::~ResourceInteractiveLoaderBinary() {
+ResourceLoaderBinary::~ResourceLoaderBinary() {
if (f)
memdelete(f);
}
-Ref<ResourceInteractiveLoader> ResourceFormatLoaderBinary::load_interactive(const String &p_path, const String &p_original_path, Error *r_error) {
+RES ResourceFormatLoaderBinary::load(const String &p_path, const String &p_original_path, Error *r_error, bool p_use_sub_threads, float *r_progress) {
if (r_error)
*r_error = ERR_FILE_CANT_OPEN;
@@ -1013,16 +1029,27 @@ Ref<ResourceInteractiveLoader> ResourceFormatLoaderBinary::load_interactive(cons
Error err;
FileAccess *f = FileAccess::open(p_path, FileAccess::READ, &err);
- ERR_FAIL_COND_V_MSG(err != OK, Ref<ResourceInteractiveLoader>(), "Cannot open file '" + p_path + "'.");
+ ERR_FAIL_COND_V_MSG(err != OK, RES(), "Cannot open file '" + p_path + "'.");
- Ref<ResourceInteractiveLoaderBinary> ria = memnew(ResourceInteractiveLoaderBinary);
+ ResourceLoaderBinary loader;
+ loader.use_sub_threads = p_use_sub_threads;
+ loader.progress = r_progress;
String path = p_original_path != "" ? p_original_path : p_path;
- ria->local_path = ProjectSettings::get_singleton()->localize_path(path);
- ria->res_path = ria->local_path;
- //ria->set_local_path( Globals::get_singleton()->localize_path(p_path) );
- ria->open(f);
+ loader.local_path = ProjectSettings::get_singleton()->localize_path(path);
+ loader.res_path = loader.local_path;
+ //loader.set_local_path( Globals::get_singleton()->localize_path(p_path) );
+ loader.open(f);
+
+ err = loader.load();
+
+ if (r_error) {
+ *r_error = err;
+ }
- return ria;
+ if (err) {
+ return RES();
+ }
+ return loader.resource;
}
void ResourceFormatLoaderBinary::get_recognized_extensions_for_type(const String &p_type, List<String> *p_extensions) const {
@@ -1064,11 +1091,11 @@ void ResourceFormatLoaderBinary::get_dependencies(const String &p_path, List<Str
FileAccess *f = FileAccess::open(p_path, FileAccess::READ);
ERR_FAIL_COND_MSG(!f, "Cannot open file '" + p_path + "'.");
- Ref<ResourceInteractiveLoaderBinary> ria = memnew(ResourceInteractiveLoaderBinary);
- ria->local_path = ProjectSettings::get_singleton()->localize_path(p_path);
- ria->res_path = ria->local_path;
- //ria->set_local_path( Globals::get_singleton()->localize_path(p_path) );
- ria->get_dependencies(f, p_dependencies, p_add_types);
+ ResourceLoaderBinary loader;
+ loader.local_path = ProjectSettings::get_singleton()->localize_path(p_path);
+ loader.res_path = loader.local_path;
+ //loader.set_local_path( Globals::get_singleton()->localize_path(p_path) );
+ loader.get_dependencies(f, p_dependencies, p_add_types);
}
Error ResourceFormatLoaderBinary::rename_dependencies(const String &p_path, const Map<String, String> &p_map) {
@@ -1153,21 +1180,17 @@ Error ResourceFormatLoaderBinary::rename_dependencies(const String &p_path, cons
ERR_FAIL_COND_V_MSG(err != OK, ERR_FILE_CANT_OPEN, "Cannot open file '" + p_path + "'.");
- Ref<ResourceInteractiveLoaderBinary> ria = memnew(ResourceInteractiveLoaderBinary);
- ria->local_path = ProjectSettings::get_singleton()->localize_path(p_path);
- ria->res_path = ria->local_path;
- ria->remaps = p_map;
- //ria->set_local_path( Globals::get_singleton()->localize_path(p_path) );
- ria->open(f);
-
- err = ria->poll();
+ ResourceLoaderBinary loader;
+ loader.local_path = ProjectSettings::get_singleton()->localize_path(p_path);
+ loader.res_path = loader.local_path;
+ loader.remaps = p_map;
+ //loader.set_local_path( Globals::get_singleton()->localize_path(p_path) );
+ loader.open(f);
- while (err == OK) {
- err = ria->poll();
- }
+ err = loader.load();
ERR_FAIL_COND_V(err != ERR_FILE_EOF, ERR_FILE_CORRUPT);
- RES res = ria->get_resource();
+ RES res = loader.get_resource();
ERR_FAIL_COND_V(!res.is_valid(), ERR_FILE_CORRUPT);
return ResourceFormatSaverBinary::singleton->save(p_path, res);
@@ -1283,11 +1306,11 @@ String ResourceFormatLoaderBinary::get_resource_type(const String &p_path) const
return ""; //could not rwead
}
- Ref<ResourceInteractiveLoaderBinary> ria = memnew(ResourceInteractiveLoaderBinary);
- ria->local_path = ProjectSettings::get_singleton()->localize_path(p_path);
- ria->res_path = ria->local_path;
- //ria->set_local_path( Globals::get_singleton()->localize_path(p_path) );
- String r = ria->recognize(f);
+ ResourceLoaderBinary loader;
+ loader.local_path = ProjectSettings::get_singleton()->localize_path(p_path);
+ loader.res_path = loader.local_path;
+ //loader.set_local_path( Globals::get_singleton()->localize_path(p_path) );
+ String r = loader.recognize(f);
return ClassDB::get_compatibility_remapped_class(r);
}
diff --git a/core/io/resource_format_binary.h b/core/io/resource_format_binary.h
index f02dbaa0c2..0ffa2c3626 100644
--- a/core/io/resource_format_binary.h
+++ b/core/io/resource_format_binary.h
@@ -35,7 +35,7 @@
#include "core/io/resource_saver.h"
#include "core/os/file_access.h"
-class ResourceInteractiveLoaderBinary : public ResourceInteractiveLoader {
+class ResourceLoaderBinary {
bool translation_remapped;
String local_path;
@@ -58,8 +58,11 @@ class ResourceInteractiveLoaderBinary : public ResourceInteractiveLoader {
struct ExtResource {
String path;
String type;
+ RES cache;
};
+ bool use_sub_threads;
+ float *progress;
Vector<ExtResource> external_resources;
struct IntResource {
@@ -75,32 +78,30 @@ class ResourceInteractiveLoaderBinary : public ResourceInteractiveLoader {
Map<String, String> remaps;
Error error;
- int stage;
-
friend class ResourceFormatLoaderBinary;
Error parse_variant(Variant &r_v);
+ Map<String, RES> dependency_cache;
+
public:
- virtual void set_local_path(const String &p_local_path);
- virtual Ref<Resource> get_resource();
- virtual Error poll();
- virtual int get_stage() const;
- virtual int get_stage_count() const;
- virtual void set_translation_remapped(bool p_remapped);
+ void set_local_path(const String &p_local_path);
+ Ref<Resource> get_resource();
+ Error load();
+ void set_translation_remapped(bool p_remapped);
void set_remaps(const Map<String, String> &p_remaps) { remaps = p_remaps; }
void open(FileAccess *p_f);
String recognize(FileAccess *p_f);
void get_dependencies(FileAccess *p_f, List<String> *p_dependencies, bool p_add_types);
- ResourceInteractiveLoaderBinary();
- ~ResourceInteractiveLoaderBinary();
+ ResourceLoaderBinary();
+ ~ResourceLoaderBinary();
};
class ResourceFormatLoaderBinary : public ResourceFormatLoader {
public:
- virtual Ref<ResourceInteractiveLoader> load_interactive(const String &p_path, const String &p_original_path = "", Error *r_error = NULL);
+ virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL, bool p_use_sub_threads = false, float *r_progress = nullptr);
virtual void get_recognized_extensions_for_type(const String &p_type, List<String> *p_extensions) const;
virtual void get_recognized_extensions(List<String> *p_extensions) const;
virtual bool handles_type(const String &p_type) const;
diff --git a/core/io/resource_importer.cpp b/core/io/resource_importer.cpp
index f147170ff7..efaf958949 100644
--- a/core/io/resource_importer.cpp
+++ b/core/io/resource_importer.cpp
@@ -117,7 +117,7 @@ Error ResourceFormatImporter::_get_path_and_type(const String &p_path, PathAndTy
return OK;
}
-RES ResourceFormatImporter::load(const String &p_path, const String &p_original_path, Error *r_error) {
+RES ResourceFormatImporter::load(const String &p_path, const String &p_original_path, Error *r_error, bool p_use_sub_threads, float *r_progress) {
PathAndType pat;
Error err = _get_path_and_type(p_path, pat);
@@ -130,7 +130,7 @@ RES ResourceFormatImporter::load(const String &p_path, const String &p_original_
return RES();
}
- RES res = ResourceLoader::_load(pat.path, p_path, pat.type, false, r_error);
+ RES res = ResourceLoader::_load(pat.path, p_path, pat.type, false, r_error, p_use_sub_threads, r_progress);
#ifdef TOOLS_ENABLED
if (res.is_valid()) {
diff --git a/core/io/resource_importer.h b/core/io/resource_importer.h
index 4eb04586e6..65c148f2ac 100644
--- a/core/io/resource_importer.h
+++ b/core/io/resource_importer.h
@@ -58,7 +58,7 @@ class ResourceFormatImporter : public ResourceFormatLoader {
public:
static ResourceFormatImporter *get_singleton() { return singleton; }
- virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL);
+ virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL, bool p_use_sub_threads = false, float *r_progress = nullptr);
virtual void get_recognized_extensions(List<String> *p_extensions) const;
virtual void get_recognized_extensions_for_type(const String &p_type, List<String> *p_extensions) const;
virtual bool recognize_path(const String &p_path, const String &p_for_type = String()) const;
diff --git a/core/io/resource_loader.cpp b/core/io/resource_loader.cpp
index b43cd7f1a2..504dbe2d63 100644
--- a/core/io/resource_loader.cpp
+++ b/core/io/resource_loader.cpp
@@ -39,26 +39,16 @@
#include "core/translation.h"
#include "core/variant_parser.h"
+#ifdef DEBUG_LOAD_THREADED
+#define print_lt(m_text) print_line(m_text)
+#else
+#define print_lt(m_text)
+#endif
+
Ref<ResourceFormatLoader> ResourceLoader::loader[ResourceLoader::MAX_LOADERS];
int ResourceLoader::loader_count = 0;
-Error ResourceInteractiveLoader::wait() {
-
- Error err = poll();
- while (err == OK) {
- err = poll();
- }
-
- return err;
-}
-
-ResourceInteractiveLoader::~ResourceInteractiveLoader() {
- if (path_loading != String()) {
- ResourceLoader::_remove_from_loading_map_and_thread(path_loading, path_loading_thread);
- }
-}
-
bool ResourceFormatLoader::recognize_path(const String &p_path, const String &p_for_type) const {
String extension = p_path.get_extension();
@@ -111,45 +101,6 @@ void ResourceLoader::get_recognized_extensions_for_type(const String &p_type, Li
}
}
-void ResourceInteractiveLoader::_bind_methods() {
-
- ClassDB::bind_method(D_METHOD("get_resource"), &ResourceInteractiveLoader::get_resource);
- ClassDB::bind_method(D_METHOD("poll"), &ResourceInteractiveLoader::poll);
- ClassDB::bind_method(D_METHOD("wait"), &ResourceInteractiveLoader::wait);
- ClassDB::bind_method(D_METHOD("get_stage"), &ResourceInteractiveLoader::get_stage);
- ClassDB::bind_method(D_METHOD("get_stage_count"), &ResourceInteractiveLoader::get_stage_count);
-}
-
-class ResourceInteractiveLoaderDefault : public ResourceInteractiveLoader {
-
- GDCLASS(ResourceInteractiveLoaderDefault, ResourceInteractiveLoader);
-
-public:
- Ref<Resource> resource;
-
- virtual void set_local_path(const String &p_local_path) { /*scene->set_filename(p_local_path);*/
- }
- virtual Ref<Resource> get_resource() { return resource; }
- virtual Error poll() { return ERR_FILE_EOF; }
- virtual int get_stage() const { return 1; }
- virtual int get_stage_count() const { return 1; }
- virtual void set_translation_remapped(bool p_remapped) { resource->set_as_translation_remapped(p_remapped); }
-
- ResourceInteractiveLoaderDefault() {}
-};
-
-Ref<ResourceInteractiveLoader> ResourceFormatLoader::load_interactive(const String &p_path, const String &p_original_path, Error *r_error) {
-
- //either this
- Ref<Resource> res = load(p_path, p_original_path, r_error);
- if (res.is_null())
- return Ref<ResourceInteractiveLoader>();
-
- Ref<ResourceInteractiveLoaderDefault> ril = Ref<ResourceInteractiveLoaderDefault>(memnew(ResourceInteractiveLoaderDefault));
- ril->resource = res;
- return ril;
-}
-
bool ResourceFormatLoader::exists(const String &p_path) const {
return FileAccess::exists(p_path); //by default just check file
}
@@ -168,10 +119,10 @@ void ResourceFormatLoader::get_recognized_extensions(List<String> *p_extensions)
}
}
-RES ResourceFormatLoader::load(const String &p_path, const String &p_original_path, Error *r_error) {
+RES ResourceFormatLoader::load(const String &p_path, const String &p_original_path, Error *r_error, bool p_use_sub_threads, float *r_progress) {
if (get_script_instance() && get_script_instance()->has_method("load")) {
- Variant res = get_script_instance()->call("load", p_path, p_original_path);
+ Variant res = get_script_instance()->call("load", p_path, p_original_path, p_use_sub_threads);
if (res.get_type() == Variant::INT) {
@@ -184,29 +135,11 @@ RES ResourceFormatLoader::load(const String &p_path, const String &p_original_pa
*r_error = OK;
return res;
}
- }
-
- //or this must be implemented
- Ref<ResourceInteractiveLoader> ril = load_interactive(p_path, p_original_path, r_error);
- if (!ril.is_valid())
- return RES();
- ril->set_local_path(p_original_path);
-
- while (true) {
-
- Error err = ril->poll();
-
- if (err == ERR_FILE_EOF) {
- if (r_error)
- *r_error = OK;
- return ril->get_resource();
- }
- if (r_error)
- *r_error = err;
-
- ERR_FAIL_COND_V_MSG(err != OK, RES(), "Failed to load resource '" + p_path + "'.");
+ return res;
}
+
+ ERR_FAIL_V_MSG(RES(), "Failed to load resource '" + p_path + "', ResourceFormatLoader::load was not implemented for this resource type.");
}
void ResourceFormatLoader::get_dependencies(const String &p_path, List<String> *p_dependencies, bool p_add_types) {
@@ -256,7 +189,7 @@ void ResourceFormatLoader::_bind_methods() {
///////////////////////////////////
-RES ResourceLoader::_load(const String &p_path, const String &p_original_path, const String &p_type_hint, bool p_no_cache, Error *r_error) {
+RES ResourceLoader::_load(const String &p_path, const String &p_original_path, const String &p_type_hint, bool p_no_cache, Error *r_error, bool p_use_sub_threads, float *r_progress) {
bool found = false;
@@ -267,7 +200,7 @@ RES ResourceLoader::_load(const String &p_path, const String &p_original_path, c
continue;
}
found = true;
- RES res = loader[i]->load(p_path, p_original_path != String() ? p_original_path : p_path, r_error);
+ RES res = loader[i]->load(p_path, p_original_path != String() ? p_original_path : p_path, r_error, p_use_sub_threads, r_progress);
if (res.is_null()) {
continue;
}
@@ -285,49 +218,68 @@ RES ResourceLoader::_load(const String &p_path, const String &p_original_path, c
ERR_FAIL_V_MSG(RES(), "No loader found for resource: " + p_path + ".");
}
-bool ResourceLoader::_add_to_loading_map(const String &p_path) {
+void ResourceLoader::_thread_load_function(void *p_userdata) {
- bool success;
- MutexLock lock(loading_map_mutex);
+ ThreadLoadTask &load_task = *(ThreadLoadTask *)p_userdata;
+ load_task.loader_id = Thread::get_caller_id();
- LoadingMapKey key;
- key.path = p_path;
- key.thread = Thread::get_caller_id();
+ if (load_task.semaphore) {
+ //this is an actual thread, so wait for Ok fom semaphore
+ thread_load_semaphore->wait(); //wait until its ok to start loading
+ }
+ load_task.resource = _load(load_task.remapped_path, load_task.remapped_path != load_task.local_path ? load_task.local_path : String(), load_task.type_hint, false, &load_task.error, load_task.use_sub_threads, &load_task.progress);
+
+ load_task.progress = 1.0; //it was fully loaded at this point, so force progress to 1.0
- if (loading_map.has(key)) {
- success = false;
+ thread_load_mutex->lock();
+ if (load_task.error != OK) {
+ load_task.status = THREAD_LOAD_FAILED;
} else {
- loading_map[key] = true;
- success = true;
+ load_task.status = THREAD_LOAD_LOADED;
}
+ if (load_task.semaphore) {
- return success;
-}
+ if (load_task.start_next && thread_waiting_count > 0) {
+ thread_waiting_count--;
+ //thread loading count remains constant, this ends but another one begins
+ thread_load_semaphore->post();
+ } else {
+ thread_loading_count--; //no threads waiting, just reduce loading count
+ }
-void ResourceLoader::_remove_from_loading_map(const String &p_path) {
- MutexLock lock(loading_map_mutex);
+ print_lt("END: load count: " + itos(thread_loading_count) + " / wait count: " + itos(thread_waiting_count) + " / suspended count: " + itos(thread_suspended_count) + " / active: " + itos(thread_loading_count - thread_suspended_count));
- LoadingMapKey key;
- key.path = p_path;
- key.thread = Thread::get_caller_id();
+ for (int i = 0; i < load_task.poll_requests; i++) {
+ load_task.semaphore->post();
+ }
+ memdelete(load_task.semaphore);
+ load_task.semaphore = nullptr;
+ }
- loading_map.erase(key);
-}
+ if (load_task.resource.is_valid()) {
+ load_task.resource->set_path(load_task.local_path);
-void ResourceLoader::_remove_from_loading_map_and_thread(const String &p_path, Thread::ID p_thread) {
- MutexLock lock(loading_map_mutex);
+ if (load_task.xl_remapped)
+ load_task.resource->set_as_translation_remapped(true);
- LoadingMapKey key;
- key.path = p_path;
- key.thread = p_thread;
+#ifdef TOOLS_ENABLED
- loading_map.erase(key);
-}
+ load_task.resource->set_edited(false);
+ if (timestamp_on_load) {
+ uint64_t mt = FileAccess::get_modified_time(load_task.remapped_path);
+ //printf("mt %s: %lli\n",remapped_path.utf8().get_data(),mt);
+ load_task.resource->set_last_modified_time(mt);
+ }
+#endif
-RES ResourceLoader::load(const String &p_path, const String &p_type_hint, bool p_no_cache, Error *r_error) {
+ if (_loaded_callback) {
+ _loaded_callback(load_task.resource, load_task.local_path);
+ }
+ }
- if (r_error)
- *r_error = ERR_CANT_OPEN;
+ thread_load_mutex->unlock();
+}
+Error ResourceLoader::load_threaded_request(const String &p_path, const String &p_type_hint, bool p_use_sub_threads, const String &p_source_resource) {
String local_path;
if (p_path.is_rel_path())
@@ -335,88 +287,154 @@ RES ResourceLoader::load(const String &p_path, const String &p_type_hint, bool p
else
local_path = ProjectSettings::get_singleton()->localize_path(p_path);
- if (!p_no_cache) {
+ thread_load_mutex->lock();
- {
- bool success = _add_to_loading_map(local_path);
- ERR_FAIL_COND_V_MSG(!success, RES(), "Resource: '" + local_path + "' is already being loaded. Cyclic reference?");
+ if (p_source_resource != String()) {
+ //must be loading from this resource
+ if (!thread_load_tasks.has(p_source_resource)) {
+ thread_load_mutex->unlock();
+ ERR_FAIL_V_MSG(ERR_INVALID_PARAMETER, "There is no thread loading source resource '" + p_source_resource + "'.");
+ }
+ //must be loading from this thread
+ if (thread_load_tasks[p_source_resource].loader_id != Thread::get_caller_id()) {
+ thread_load_mutex->unlock();
+ ERR_FAIL_V_MSG(ERR_INVALID_PARAMETER, "Threading loading resource'" + local_path + " failed: Source specified: '" + p_source_resource + "' but was not called by it.");
}
- //lock first if possible
- if (ResourceCache::lock) {
- ResourceCache::lock->read_lock();
+ //must not be already added as s sub tasks
+ if (thread_load_tasks[p_source_resource].sub_tasks.has(local_path)) {
+ thread_load_mutex->unlock();
+ ERR_FAIL_V_MSG(ERR_INVALID_PARAMETER, "Thread loading source resource '" + p_source_resource + "' already is loading '" + local_path + "'.");
}
+ }
- //get ptr
- Resource **rptr = ResourceCache::resources.getptr(local_path);
+ if (thread_load_tasks.has(local_path)) {
+ thread_load_tasks[local_path].requests++;
+ if (p_source_resource != String()) {
+ thread_load_tasks[p_source_resource].sub_tasks.insert(local_path);
+ }
+ thread_load_mutex->unlock();
+ return OK;
+ }
- if (rptr) {
- RES res(*rptr);
- //it is possible this resource was just freed in a thread. If so, this referencing will not work and resource is considered not cached
- if (res.is_valid()) {
- //referencing is fine
- if (r_error)
- *r_error = OK;
- if (ResourceCache::lock) {
- ResourceCache::lock->read_unlock();
+ {
+ //create load task
+
+ ThreadLoadTask load_task;
+
+ load_task.requests = 1;
+ load_task.remapped_path = _path_remap(local_path, &load_task.xl_remapped);
+ load_task.local_path = local_path;
+ load_task.type_hint = p_type_hint;
+ load_task.use_sub_threads = p_use_sub_threads;
+
+ { //must check if resource is already loaded before attempting to load it in a thread
+
+ if (load_task.loader_id == Thread::get_caller_id()) {
+ thread_load_mutex->unlock();
+ ERR_FAIL_V_MSG(ERR_INVALID_PARAMETER, "Attempted to load a resource already being loaded from this thread, cyclic reference?");
+ }
+ //lock first if possible
+ if (ResourceCache::lock) {
+ ResourceCache::lock->read_lock();
+ }
+
+ //get ptr
+ Resource **rptr = ResourceCache::resources.getptr(local_path);
+
+ if (rptr) {
+ RES res(*rptr);
+ //it is possible this resource was just freed in a thread. If so, this referencing will not work and resource is considered not cached
+ if (res.is_valid()) {
+ //referencing is fine
+ load_task.resource = res;
+ load_task.status = THREAD_LOAD_LOADED;
+ load_task.progress = 1.0;
}
- _remove_from_loading_map(local_path);
- return res;
+ }
+ if (ResourceCache::lock) {
+ ResourceCache::lock->read_unlock();
}
}
- if (ResourceCache::lock) {
- ResourceCache::lock->read_unlock();
+
+ if (p_source_resource != String()) {
+ thread_load_tasks[p_source_resource].sub_tasks.insert(local_path);
}
+
+ thread_load_tasks[local_path] = load_task;
}
- bool xl_remapped = false;
- String path = _path_remap(local_path, &xl_remapped);
+ ThreadLoadTask &load_task = thread_load_tasks[local_path];
- if (path == "") {
- if (!p_no_cache) {
- _remove_from_loading_map(local_path);
+ if (load_task.resource.is_null()) { //needs to be loaded in thread
+
+ load_task.semaphore = memnew(Semaphore);
+ if (thread_loading_count < thread_load_max) {
+ thread_loading_count++;
+ thread_load_semaphore->post(); //we have free threads, so allow one
+ } else {
+ thread_waiting_count++;
}
- ERR_FAIL_V_MSG(RES(), "Remapping '" + local_path + "' failed.");
+
+ print_lt("REQUEST: load count: " + itos(thread_loading_count) + " / wait count: " + itos(thread_waiting_count) + " / suspended count: " + itos(thread_suspended_count) + " / active: " + itos(thread_loading_count - thread_suspended_count));
+
+ load_task.thread = Thread::create(_thread_load_function, &thread_load_tasks[local_path]);
+ load_task.loader_id = load_task.thread->get_id();
}
- print_verbose("Loading resource: " + path);
- RES res = _load(path, local_path, p_type_hint, p_no_cache, r_error);
+ thread_load_mutex->unlock();
+
+ return OK;
+}
+
+float ResourceLoader::_dependency_get_progress(const String &p_path) {
- if (res.is_null()) {
- if (!p_no_cache) {
- _remove_from_loading_map(local_path);
+ if (thread_load_tasks.has(p_path)) {
+ ThreadLoadTask &load_task = thread_load_tasks[p_path];
+ int dep_count = load_task.sub_tasks.size();
+ if (dep_count > 0) {
+ float dep_progress = 0;
+ for (Set<String>::Element *E = load_task.sub_tasks.front(); E; E = E->next()) {
+ dep_progress += _dependency_get_progress(E->get());
+ }
+ dep_progress /= float(dep_count);
+ dep_progress *= 0.5;
+ dep_progress += load_task.progress * 0.5;
+ return dep_progress;
+ } else {
+ return load_task.progress;
}
- print_verbose("Failed loading resource: " + path);
- return RES();
+
+ } else {
+ return 1.0; //assume finished loading it so it no longer exists
}
- if (!p_no_cache)
- res->set_path(local_path);
+}
- if (xl_remapped)
- res->set_as_translation_remapped(true);
+ResourceLoader::ThreadLoadStatus ResourceLoader::load_threaded_get_status(const String &p_path, float *r_progress) {
-#ifdef TOOLS_ENABLED
+ String local_path;
+ if (p_path.is_rel_path())
+ local_path = "res://" + p_path;
+ else
+ local_path = ProjectSettings::get_singleton()->localize_path(p_path);
- res->set_edited(false);
- if (timestamp_on_load) {
- uint64_t mt = FileAccess::get_modified_time(path);
- //printf("mt %s: %lli\n",remapped_path.utf8().get_data(),mt);
- res->set_last_modified_time(mt);
+ thread_load_mutex->lock();
+ if (!thread_load_tasks.has(local_path)) {
+ thread_load_mutex->unlock();
+ return THREAD_LOAD_INVALID_RESOURCE;
}
-#endif
-
- if (!p_no_cache) {
- _remove_from_loading_map(local_path);
+ ThreadLoadTask &load_task = thread_load_tasks[local_path];
+ ThreadLoadStatus status;
+ status = load_task.status;
+ if (r_progress) {
+ *r_progress = _dependency_get_progress(local_path);
}
- if (_loaded_callback) {
- _loaded_callback(res, p_path);
- }
+ thread_load_mutex->unlock();
- return res;
+ return status;
}
-
-bool ResourceLoader::exists(const String &p_path, const String &p_type_hint) {
+RES ResourceLoader::load_threaded_get(const String &p_path, Error *r_error) {
String local_path;
if (p_path.is_rel_path())
@@ -424,29 +442,81 @@ bool ResourceLoader::exists(const String &p_path, const String &p_type_hint) {
else
local_path = ProjectSettings::get_singleton()->localize_path(p_path);
- if (ResourceCache::has(local_path)) {
-
- return true; // If cached, it probably exists
+ thread_load_mutex->lock();
+ if (!thread_load_tasks.has(local_path)) {
+ thread_load_mutex->unlock();
+ if (r_error) {
+ *r_error = ERR_INVALID_PARAMETER;
+ }
+ return RES();
}
- bool xl_remapped = false;
- String path = _path_remap(local_path, &xl_remapped);
+ ThreadLoadTask &load_task = thread_load_tasks[local_path];
- // Try all loaders and pick the first match for the type hint
- for (int i = 0; i < loader_count; i++) {
+ //semaphore still exists, meaning its still loading, request poll
+ Semaphore *semaphore = load_task.semaphore;
+ if (semaphore) {
+ load_task.poll_requests++;
- if (!loader[i]->recognize_path(path, p_type_hint)) {
- continue;
+ {
+ // As we got a semaphore, this means we are going to have to wait
+ // until the sub-resource is done loading
+ //
+ // As this thread will become 'blocked' we should "echange" its
+ // active status with a waiting one, to ensure load continues.
+ //
+ // This ensures loading is never blocked and that is also within
+ // the maximum number of active threads.
+
+ if (thread_waiting_count > 0) {
+ thread_waiting_count--;
+ thread_loading_count++;
+ thread_load_semaphore->post();
+
+ load_task.start_next = false; //do not start next since we are doing it here
+ }
+
+ thread_suspended_count++;
+
+ print_lt("GET: load count: " + itos(thread_loading_count) + " / wait count: " + itos(thread_waiting_count) + " / suspended count: " + itos(thread_suspended_count) + " / active: " + itos(thread_loading_count - thread_suspended_count));
}
- if (loader[i]->exists(path))
- return true;
+ thread_load_mutex->unlock();
+ semaphore->wait();
+ thread_load_mutex->lock();
+
+ thread_suspended_count--;
+
+ if (!thread_load_tasks.has(local_path)) { //may have been erased during unlock and this was always an invalid call
+ thread_load_mutex->unlock();
+ if (r_error) {
+ *r_error = ERR_INVALID_PARAMETER;
+ }
+ return RES();
+ }
}
- return false;
+ RES resource = load_task.resource;
+ if (r_error) {
+ *r_error = load_task.error;
+ }
+
+ load_task.requests--;
+
+ if (load_task.requests == 0) {
+ if (load_task.thread) { //thread may not have been used
+ Thread::wait_to_finish(load_task.thread);
+ memdelete(load_task.thread);
+ }
+ thread_load_tasks.erase(local_path);
+ }
+
+ thread_load_mutex->unlock();
+
+ return resource;
}
-Ref<ResourceInteractiveLoader> ResourceLoader::load_interactive(const String &p_path, const String &p_type_hint, bool p_no_cache, Error *r_error) {
+RES ResourceLoader::load(const String &p_path, const String &p_type_hint, bool p_no_cache, Error *r_error) {
if (r_error)
*r_error = ERR_CANT_OPEN;
@@ -459,61 +529,131 @@ Ref<ResourceInteractiveLoader> ResourceLoader::load_interactive(const String &p_
if (!p_no_cache) {
- bool success = _add_to_loading_map(local_path);
- ERR_FAIL_COND_V_MSG(!success, RES(), "Resource: '" + local_path + "' is already being loaded. Cyclic reference?");
+ thread_load_mutex->lock();
- if (ResourceCache::has(local_path)) {
+ //Is it already being loaded? poll until done
+ if (thread_load_tasks.has(local_path)) {
+ Error err = load_threaded_request(p_path, p_type_hint);
+ if (err != OK) {
+ if (r_error) {
+ *r_error = err;
+ }
+ return RES();
+ }
+ thread_load_mutex->unlock();
- print_verbose("Loading resource: " + local_path + " (cached)");
- Ref<Resource> res_cached = ResourceCache::get(local_path);
- Ref<ResourceInteractiveLoaderDefault> ril = Ref<ResourceInteractiveLoaderDefault>(memnew(ResourceInteractiveLoaderDefault));
+ return load_threaded_get(p_path, r_error);
+ }
- ril->resource = res_cached;
- ril->path_loading = local_path;
- ril->path_loading_thread = Thread::get_caller_id();
- return ril;
+ //Is it cached?
+ if (ResourceCache::lock) {
+ ResourceCache::lock->read_lock();
}
- }
- bool xl_remapped = false;
- String path = _path_remap(local_path, &xl_remapped);
- if (path == "") {
- if (!p_no_cache) {
- _remove_from_loading_map(local_path);
+ Resource **rptr = ResourceCache::resources.getptr(local_path);
+
+ if (rptr) {
+ RES res(*rptr);
+
+ //it is possible this resource was just freed in a thread. If so, this referencing will not work and resource is considered not cached
+ if (res.is_valid()) {
+ if (ResourceCache::lock) {
+ ResourceCache::lock->read_unlock();
+ }
+ thread_load_mutex->unlock();
+
+ if (r_error) {
+ *r_error = OK;
+ }
+
+ return res; //use cached
+ }
}
- ERR_FAIL_V_MSG(RES(), "Remapping '" + local_path + "' failed.");
- }
- print_verbose("Loading resource: " + path);
+ if (ResourceCache::lock) {
+ ResourceCache::lock->read_unlock();
+ }
- bool found = false;
- for (int i = 0; i < loader_count; i++) {
+ //load using task (but this thread)
+ ThreadLoadTask load_task;
- if (!loader[i]->recognize_path(path, p_type_hint))
- continue;
- found = true;
- Ref<ResourceInteractiveLoader> ril = loader[i]->load_interactive(path, local_path, r_error);
- if (ril.is_null())
- continue;
- if (!p_no_cache) {
- ril->set_local_path(local_path);
- ril->path_loading = local_path;
- ril->path_loading_thread = Thread::get_caller_id();
+ load_task.requests = 1;
+ load_task.local_path = local_path;
+ load_task.remapped_path = _path_remap(local_path, &load_task.xl_remapped);
+ load_task.type_hint = p_type_hint;
+ load_task.loader_id = Thread::get_caller_id();
+
+ thread_load_tasks[local_path] = load_task;
+
+ thread_load_mutex->unlock();
+
+ _thread_load_function(&thread_load_tasks[local_path]);
+
+ return load_threaded_get(p_path, r_error);
+
+ } else {
+
+ bool xl_remapped = false;
+ String path = _path_remap(local_path, &xl_remapped);
+
+ if (path == "") {
+ ERR_FAIL_V_MSG(RES(), "Remapping '" + local_path + "' failed.");
+ }
+
+ print_verbose("Loading resource: " + path);
+ float p;
+ RES res = _load(path, local_path, p_type_hint, p_no_cache, r_error, false, &p);
+
+ if (res.is_null()) {
+ print_verbose("Failed loading resource: " + path);
+ return RES();
}
if (xl_remapped)
- ril->set_translation_remapped(true);
+ res->set_as_translation_remapped(true);
+
+#ifdef TOOLS_ENABLED
+
+ res->set_edited(false);
+ if (timestamp_on_load) {
+ uint64_t mt = FileAccess::get_modified_time(path);
+ //printf("mt %s: %lli\n",remapped_path.utf8().get_data(),mt);
+ res->set_last_modified_time(mt);
+ }
+#endif
- return ril;
+ return res;
}
+}
- if (!p_no_cache) {
- _remove_from_loading_map(local_path);
+bool ResourceLoader::exists(const String &p_path, const String &p_type_hint) {
+
+ String local_path;
+ if (p_path.is_rel_path())
+ local_path = "res://" + p_path;
+ else
+ local_path = ProjectSettings::get_singleton()->localize_path(p_path);
+
+ if (ResourceCache::has(local_path)) {
+
+ return true; // If cached, it probably exists
}
- ERR_FAIL_COND_V_MSG(found, Ref<ResourceInteractiveLoader>(), "Failed loading resource: " + path + ".");
+ bool xl_remapped = false;
+ String path = _path_remap(local_path, &xl_remapped);
- ERR_FAIL_V_MSG(Ref<ResourceInteractiveLoader>(), "No loader found for resource: " + path + ".");
+ // Try all loaders and pick the first match for the type hint
+ for (int i = 0; i < loader_count; i++) {
+
+ if (!loader[i]->recognize_path(path, p_type_hint)) {
+ continue;
+ }
+
+ if (loader[i]->exists(path))
+ return true;
+ }
+
+ return false;
}
void ResourceLoader::add_resource_format_loader(Ref<ResourceFormatLoader> p_format_loader, bool p_at_front) {
@@ -984,20 +1124,19 @@ void ResourceLoader::remove_custom_loaders() {
}
}
-Mutex ResourceLoader::loading_map_mutex;
-HashMap<ResourceLoader::LoadingMapKey, int, ResourceLoader::LoadingMapKeyHasher> ResourceLoader::loading_map;
-
void ResourceLoader::initialize() {
+ thread_load_mutex = memnew(Mutex);
+ thread_load_max = OS::get_singleton()->get_processor_count();
+ thread_loading_count = 0;
+ thread_waiting_count = 0;
+ thread_suspended_count = 0;
+ thread_load_semaphore = memnew(Semaphore);
}
void ResourceLoader::finalize() {
-#ifndef NO_THREADS
- const LoadingMapKey *K = NULL;
- while ((K = loading_map.next(K))) {
- ERR_PRINT("Exited while resource is being loaded: " + K->path);
- }
- loading_map.clear();
-#endif
+
+ memdelete(thread_load_mutex);
+ memdelete(thread_load_semaphore);
}
ResourceLoadErrorNotify ResourceLoader::err_notify = NULL;
@@ -1009,6 +1148,15 @@ void *ResourceLoader::dep_err_notify_ud = NULL;
bool ResourceLoader::abort_on_missing_resource = true;
bool ResourceLoader::timestamp_on_load = false;
+Mutex *ResourceLoader::thread_load_mutex = nullptr;
+HashMap<String, ResourceLoader::ThreadLoadTask> ResourceLoader::thread_load_tasks;
+Semaphore *ResourceLoader::thread_load_semaphore = nullptr;
+
+int ResourceLoader::thread_loading_count = 0;
+int ResourceLoader::thread_waiting_count = 0;
+int ResourceLoader::thread_suspended_count = 0;
+int ResourceLoader::thread_load_max = 0;
+
SelfList<Resource>::List ResourceLoader::remapped_list;
HashMap<String, Vector<String> > ResourceLoader::translation_remaps;
HashMap<String, String> ResourceLoader::path_remaps;
diff --git a/core/io/resource_loader.h b/core/io/resource_loader.h
index 172f8e979b..3b7a27f551 100644
--- a/core/io/resource_loader.h
+++ b/core/io/resource_loader.h
@@ -31,32 +31,10 @@
#ifndef RESOURCE_LOADER_H
#define RESOURCE_LOADER_H
+#include "core/os/semaphore.h"
#include "core/os/thread.h"
#include "core/resource.h"
-class ResourceInteractiveLoader : public Reference {
-
- GDCLASS(ResourceInteractiveLoader, Reference);
- friend class ResourceLoader;
- String path_loading;
- Thread::ID path_loading_thread;
-
-protected:
- static void _bind_methods();
-
-public:
- virtual void set_local_path(const String &p_local_path) = 0;
- virtual Ref<Resource> get_resource() = 0;
- virtual Error poll() = 0;
- virtual int get_stage() const = 0;
- virtual int get_stage_count() const = 0;
- virtual void set_translation_remapped(bool p_remapped) = 0;
- virtual Error wait();
-
- ResourceInteractiveLoader() {}
- ~ResourceInteractiveLoader();
-};
-
class ResourceFormatLoader : public Reference {
GDCLASS(ResourceFormatLoader, Reference);
@@ -65,8 +43,7 @@ protected:
static void _bind_methods();
public:
- virtual Ref<ResourceInteractiveLoader> load_interactive(const String &p_path, const String &p_original_path = "", Error *r_error = NULL);
- virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL);
+ virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL, bool p_use_sub_threads = false, float *r_progress = nullptr);
virtual bool exists(const String &p_path) const;
virtual void get_recognized_extensions(List<String> *p_extensions) const;
virtual void get_recognized_extensions_for_type(const String &p_type, List<String> *p_extensions) const;
@@ -95,6 +72,15 @@ class ResourceLoader {
MAX_LOADERS = 64
};
+public:
+ enum ThreadLoadStatus {
+ THREAD_LOAD_INVALID_RESOURCE,
+ THREAD_LOAD_IN_PROGRESS,
+ THREAD_LOAD_FAILED,
+ THREAD_LOAD_LOADED
+ };
+
+private:
static Ref<ResourceFormatLoader> loader[MAX_LOADERS];
static int loader_count;
static bool timestamp_on_load;
@@ -115,34 +101,47 @@ class ResourceLoader {
friend class ResourceFormatImporter;
friend class ResourceInteractiveLoader;
//internal load function
- static RES _load(const String &p_path, const String &p_original_path, const String &p_type_hint, bool p_no_cache, Error *r_error);
+ static RES _load(const String &p_path, const String &p_original_path, const String &p_type_hint, bool p_no_cache, Error *r_error, bool p_use_sub_threads, float *r_progress);
static ResourceLoadedCallback _loaded_callback;
static Ref<ResourceFormatLoader> _find_custom_resource_format_loader(String path);
- static Mutex loading_map_mutex;
-
- //used to track paths being loaded in a thread, avoids cyclic recursion
- struct LoadingMapKey {
- String path;
- Thread::ID thread;
- bool operator==(const LoadingMapKey &p_key) const {
- return (thread == p_key.thread && path == p_key.path);
- }
- };
- struct LoadingMapKeyHasher {
- static _FORCE_INLINE_ uint32_t hash(const LoadingMapKey &p_key) { return p_key.path.hash() + HashMapHasherDefault::hash(p_key.thread); }
+ struct ThreadLoadTask {
+ Thread *thread = nullptr;
+ Thread::ID loader_id = 0;
+ Semaphore *semaphore = nullptr;
+ String local_path;
+ String remapped_path;
+ String type_hint;
+ float progress = 0.0;
+ ThreadLoadStatus status = THREAD_LOAD_IN_PROGRESS;
+ Error error;
+ RES resource;
+ bool xl_remapped = false;
+ bool use_sub_threads = false;
+ bool start_next = true;
+ int requests = 0;
+ int poll_requests = 0;
+ Set<String> sub_tasks;
};
- static HashMap<LoadingMapKey, int, LoadingMapKeyHasher> loading_map;
+ static void _thread_load_function(void *p_userdata);
+ static Mutex *thread_load_mutex;
+ static HashMap<String, ThreadLoadTask> thread_load_tasks;
+ static Semaphore *thread_load_semaphore;
+ static int thread_waiting_count;
+ static int thread_loading_count;
+ static int thread_suspended_count;
+ static int thread_load_max;
- static bool _add_to_loading_map(const String &p_path);
- static void _remove_from_loading_map(const String &p_path);
- static void _remove_from_loading_map_and_thread(const String &p_path, Thread::ID p_thread);
+ static float _dependency_get_progress(const String &p_path);
public:
- static Ref<ResourceInteractiveLoader> load_interactive(const String &p_path, const String &p_type_hint = "", bool p_no_cache = false, Error *r_error = NULL);
+ static Error load_threaded_request(const String &p_path, const String &p_type_hint = "", bool p_use_sub_threads = false, const String &p_source_resource = String());
+ static ThreadLoadStatus load_threaded_get_status(const String &p_path, float *r_progress = nullptr);
+ static RES load_threaded_get(const String &p_path, Error *r_error = NULL);
+
static RES load(const String &p_path, const String &p_type_hint = "", bool p_no_cache = false, Error *r_error = NULL);
static bool exists(const String &p_path, const String &p_type_hint = "");
diff --git a/core/io/translation_loader_po.cpp b/core/io/translation_loader_po.cpp
index 4f7eeddc43..4051bf2947 100644
--- a/core/io/translation_loader_po.cpp
+++ b/core/io/translation_loader_po.cpp
@@ -176,7 +176,7 @@ RES TranslationLoaderPO::load_translation(FileAccess *f, Error *r_error, const S
return translation;
}
-RES TranslationLoaderPO::load(const String &p_path, const String &p_original_path, Error *r_error) {
+RES TranslationLoaderPO::load(const String &p_path, const String &p_original_path, Error *r_error, bool p_use_sub_threads, float *r_progress) {
if (r_error)
*r_error = ERR_CANT_OPEN;
diff --git a/core/io/translation_loader_po.h b/core/io/translation_loader_po.h
index 47e64276ca..fe3a75e5eb 100644
--- a/core/io/translation_loader_po.h
+++ b/core/io/translation_loader_po.h
@@ -38,7 +38,7 @@
class TranslationLoaderPO : public ResourceFormatLoader {
public:
static RES load_translation(FileAccess *f, Error *r_error, const String &p_path = String());
- virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL);
+ virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL, bool p_use_sub_threads = false, float *r_progress = nullptr);
virtual void get_recognized_extensions(List<String> *p_extensions) const;
virtual bool handles_type(const String &p_type) const;
virtual String get_resource_type(const String &p_path) const;
diff --git a/core/object.h b/core/object.h
index dc7d49f534..59d3f06cfe 100644
--- a/core/object.h
+++ b/core/object.h
@@ -126,6 +126,7 @@ enum PropertyUsageFlags {
PROPERTY_USAGE_NODE_PATH_FROM_SCENE_ROOT = 1 << 23,
PROPERTY_USAGE_RESOURCE_NOT_PERSISTENT = 1 << 24,
PROPERTY_USAGE_KEYING_INCREMENTS = 1 << 25, // Used in inspector to increment property when keyed in animation player
+ PROPERTY_USAGE_DEFERRED_SET_RESOURCE = 1 << 26, // when loading, the resource for this property can be set at the end of loading
PROPERTY_USAGE_DEFAULT = PROPERTY_USAGE_STORAGE | PROPERTY_USAGE_EDITOR | PROPERTY_USAGE_NETWORK,
PROPERTY_USAGE_DEFAULT_INTL = PROPERTY_USAGE_STORAGE | PROPERTY_USAGE_EDITOR | PROPERTY_USAGE_NETWORK | PROPERTY_USAGE_INTERNATIONALIZED,
diff --git a/core/register_core_types.cpp b/core/register_core_types.cpp
index ac60061c67..07a252ad31 100644
--- a/core/register_core_types.cpp
+++ b/core/register_core_types.cpp
@@ -186,8 +186,6 @@ void register_core_types() {
ClassDB::register_class<HTTPClient>();
ClassDB::register_class<TriangleMesh>();
- ClassDB::register_virtual_class<ResourceInteractiveLoader>();
-
ClassDB::register_class<ResourceFormatLoader>();
ClassDB::register_class<ResourceFormatSaver>();
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-23 17:09:09 +0000