diff options
Diffstat (limited to 'core/io')
-rw-r--r-- | core/io/file_access_network.cpp | 61 | ||||
-rw-r--r-- | core/io/file_access_network.h | 6 | ||||
-rw-r--r-- | core/io/image_loader.cpp | 2 | ||||
-rw-r--r-- | core/io/image_loader.h | 2 | ||||
-rw-r--r-- | core/io/ip.cpp | 42 | ||||
-rw-r--r-- | core/io/json.cpp | 8 | ||||
-rw-r--r-- | core/io/marshalls.cpp | 127 | ||||
-rw-r--r-- | core/io/multiplayer_api.cpp | 136 | ||||
-rw-r--r-- | core/io/resource_format_binary.cpp | 424 | ||||
-rw-r--r-- | core/io/resource_format_binary.h | 25 | ||||
-rw-r--r-- | core/io/resource_importer.cpp | 4 | ||||
-rw-r--r-- | core/io/resource_importer.h | 2 | ||||
-rw-r--r-- | core/io/resource_loader.cpp | 607 | ||||
-rw-r--r-- | core/io/resource_loader.h | 85 | ||||
-rw-r--r-- | core/io/translation_loader_po.cpp | 2 | ||||
-rw-r--r-- | core/io/translation_loader_po.h | 2 |
16 files changed, 935 insertions, 600 deletions
diff --git a/core/io/file_access_network.cpp b/core/io/file_access_network.cpp index 202eb89dbd..7f1eb6fd90 100644 --- a/core/io/file_access_network.cpp +++ b/core/io/file_access_network.cpp @@ -42,14 +42,14 @@ void FileAccessNetworkClient::lock_mutex() { - mutex->lock(); + mutex.lock(); lockcount++; } void FileAccessNetworkClient::unlock_mutex() { lockcount--; - mutex->unlock(); + mutex.unlock(); } void FileAccessNetworkClient::put_32(int p_32) { @@ -97,15 +97,16 @@ void FileAccessNetworkClient::_thread_func() { lock_mutex(); DEBUG_PRINT("MUTEX PASS"); - blockrequest_mutex->lock(); - while (block_requests.size()) { - put_32(block_requests.front()->get().id); - put_32(FileAccessNetwork::COMMAND_READ_BLOCK); - put_64(block_requests.front()->get().offset); - put_32(block_requests.front()->get().size); - block_requests.pop_front(); + { + MutexLock lock(blockrequest_mutex); + while (block_requests.size()) { + put_32(block_requests.front()->get().id); + put_32(FileAccessNetwork::COMMAND_READ_BLOCK); + put_64(block_requests.front()->get().offset); + put_32(block_requests.front()->get().size); + block_requests.pop_front(); + } } - blockrequest_mutex->unlock(); DEBUG_PRINT("THREAD ITER"); @@ -225,8 +226,6 @@ FileAccessNetworkClient *FileAccessNetworkClient::singleton = NULL; FileAccessNetworkClient::FileAccessNetworkClient() { thread = NULL; - mutex = Mutex::create(); - blockrequest_mutex = Mutex::create(); quit = false; singleton = this; last_id = 0; @@ -244,8 +243,6 @@ FileAccessNetworkClient::~FileAccessNetworkClient() { memdelete(thread); } - memdelete(blockrequest_mutex); - memdelete(mutex); memdelete(sem); } @@ -259,10 +256,11 @@ void FileAccessNetwork::_set_block(int p_offset, const Vector<uint8_t> &p_block) ERR_FAIL_COND((p_block.size() != (int)(total_size % page_size))); } - buffer_mutex->lock(); - pages.write[page].buffer = p_block; - pages.write[page].queued = false; - buffer_mutex->unlock(); + { + MutexLock lock(buffer_mutex); + pages.write[page].buffer = p_block; + pages.write[page].queued = false; + } if (waiting_on_page == page) { waiting_on_page = -1; @@ -384,15 +382,16 @@ void FileAccessNetwork::_queue_page(int p_page) const { if (pages[p_page].buffer.empty() && !pages[p_page].queued) { FileAccessNetworkClient *nc = FileAccessNetworkClient::singleton; - - nc->blockrequest_mutex->lock(); - FileAccessNetworkClient::BlockRequest br; - br.id = id; - br.offset = size_t(p_page) * page_size; - br.size = page_size; - nc->block_requests.push_back(br); - pages.write[p_page].queued = true; - nc->blockrequest_mutex->unlock(); + { + MutexLock lock(nc->blockrequest_mutex); + + FileAccessNetworkClient::BlockRequest br; + br.id = id; + br.offset = size_t(p_page) * page_size; + br.size = page_size; + nc->block_requests.push_back(br); + pages.write[p_page].queued = true; + } DEBUG_PRINT("QUEUE PAGE POST"); nc->sem->post(); DEBUG_PRINT("queued " + itos(p_page)); @@ -418,14 +417,14 @@ int FileAccessNetwork::get_buffer(uint8_t *p_dst, int p_length) const { int page = pos / page_size; if (page != last_page) { - buffer_mutex->lock(); + buffer_mutex.lock(); if (pages[page].buffer.empty()) { waiting_on_page = page; for (int j = 0; j < read_ahead; j++) { _queue_page(page + j); } - buffer_mutex->unlock(); + buffer_mutex.unlock(); DEBUG_PRINT("wait"); page_sem->wait(); DEBUG_PRINT("done"); @@ -436,7 +435,7 @@ int FileAccessNetwork::get_buffer(uint8_t *p_dst, int p_length) const { _queue_page(page + j); } //queue pages - buffer_mutex->unlock(); + buffer_mutex.unlock(); } buff = pages.write[page].buffer.ptrw(); @@ -524,7 +523,6 @@ FileAccessNetwork::FileAccessNetwork() { pos = 0; sem = SemaphoreOld::create(); page_sem = SemaphoreOld::create(); - buffer_mutex = Mutex::create(); FileAccessNetworkClient *nc = FileAccessNetworkClient::singleton; nc->lock_mutex(); id = nc->last_id++; @@ -542,7 +540,6 @@ FileAccessNetwork::~FileAccessNetwork() { close(); memdelete(sem); memdelete(page_sem); - memdelete(buffer_mutex); FileAccessNetworkClient *nc = FileAccessNetworkClient::singleton; nc->lock_mutex(); diff --git a/core/io/file_access_network.h b/core/io/file_access_network.h index f329abf7c5..38d9b8e8a6 100644 --- a/core/io/file_access_network.h +++ b/core/io/file_access_network.h @@ -52,8 +52,8 @@ class FileAccessNetworkClient { SemaphoreOld *sem; Thread *thread; bool quit; - Mutex *mutex; - Mutex *blockrequest_mutex; + Mutex mutex; + Mutex blockrequest_mutex; Map<int, FileAccessNetwork *> accesses; Ref<StreamPeerTCP> client; int last_id; @@ -87,7 +87,7 @@ class FileAccessNetwork : public FileAccess { SemaphoreOld *sem; SemaphoreOld *page_sem; - Mutex *buffer_mutex; + Mutex buffer_mutex; bool opened; size_t total_size; mutable size_t pos; diff --git a/core/io/image_loader.cpp b/core/io/image_loader.cpp index 720f25f91b..99ac5bcdd9 100644 --- a/core/io/image_loader.cpp +++ b/core/io/image_loader.cpp @@ -129,7 +129,7 @@ void ImageLoader::cleanup() { ///////////////// -RES ResourceFormatLoaderImage::load(const String &p_path, const String &p_original_path, Error *r_error) { +RES ResourceFormatLoaderImage::load(const String &p_path, const String &p_original_path, Error *r_error, bool p_use_sub_threads, float *r_progress) { FileAccess *f = FileAccess::open(p_path, FileAccess::READ); if (!f) { diff --git a/core/io/image_loader.h b/core/io/image_loader.h index d6dfd261ca..3ba028b99c 100644 --- a/core/io/image_loader.h +++ b/core/io/image_loader.h @@ -73,7 +73,7 @@ public: class ResourceFormatLoaderImage : public ResourceFormatLoader { public: - virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL); + virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL, bool p_use_sub_threads = false, float *r_progress = nullptr); virtual void get_recognized_extensions(List<String> *p_extensions) const; virtual bool handles_type(const String &p_type) const; virtual String get_resource_type(const String &p_path) const; diff --git a/core/io/ip.cpp b/core/io/ip.cpp index 7d18117711..af534e4bb7 100644 --- a/core/io/ip.cpp +++ b/core/io/ip.cpp @@ -70,7 +70,7 @@ struct _IP_ResolverPrivate { return IP::RESOLVER_INVALID_ID; } - Mutex *mutex; + Mutex mutex; SemaphoreOld *sem; Thread *thread; @@ -100,9 +100,8 @@ struct _IP_ResolverPrivate { ipr->sem->wait(); - ipr->mutex->lock(); + MutexLock lock(ipr->mutex); ipr->resolve_queues(); - ipr->mutex->unlock(); } } @@ -115,30 +114,27 @@ struct _IP_ResolverPrivate { IP_Address IP::resolve_hostname(const String &p_hostname, IP::Type p_type) { - resolver->mutex->lock(); + MutexLock lock(resolver->mutex); String key = _IP_ResolverPrivate::get_cache_key(p_hostname, p_type); if (resolver->cache.has(key) && resolver->cache[key].is_valid()) { IP_Address res = resolver->cache[key]; - resolver->mutex->unlock(); return res; } IP_Address res = _resolve_hostname(p_hostname, p_type); resolver->cache[key] = res; - resolver->mutex->unlock(); return res; } IP::ResolverID IP::resolve_hostname_queue_item(const String &p_hostname, IP::Type p_type) { - resolver->mutex->lock(); + MutexLock lock(resolver->mutex); ResolverID id = resolver->find_empty_id(); if (id == RESOLVER_INVALID_ID) { WARN_PRINT("Out of resolver queries"); - resolver->mutex->unlock(); return id; } @@ -157,7 +153,6 @@ IP::ResolverID IP::resolve_hostname_queue_item(const String &p_hostname, IP::Typ resolver->resolve_queues(); } - resolver->mutex->unlock(); return id; } @@ -165,50 +160,43 @@ IP::ResolverStatus IP::get_resolve_item_status(ResolverID p_id) const { ERR_FAIL_INDEX_V(p_id, IP::RESOLVER_MAX_QUERIES, IP::RESOLVER_STATUS_NONE); - resolver->mutex->lock(); + MutexLock lock(resolver->mutex); + if (resolver->queue[p_id].status == IP::RESOLVER_STATUS_NONE) { ERR_PRINT("Condition status == IP::RESOLVER_STATUS_NONE"); - resolver->mutex->unlock(); + resolver->mutex.unlock(); return IP::RESOLVER_STATUS_NONE; } - IP::ResolverStatus res = resolver->queue[p_id].status; - - resolver->mutex->unlock(); - return res; + return resolver->queue[p_id].status; } IP_Address IP::get_resolve_item_address(ResolverID p_id) const { ERR_FAIL_INDEX_V(p_id, IP::RESOLVER_MAX_QUERIES, IP_Address()); - resolver->mutex->lock(); + MutexLock lock(resolver->mutex); if (resolver->queue[p_id].status != IP::RESOLVER_STATUS_DONE) { ERR_PRINT("Resolve of '" + resolver->queue[p_id].hostname + "'' didn't complete yet."); - resolver->mutex->unlock(); + resolver->mutex.unlock(); return IP_Address(); } - IP_Address res = resolver->queue[p_id].response; - - resolver->mutex->unlock(); - return res; + return resolver->queue[p_id].response; } void IP::erase_resolve_item(ResolverID p_id) { ERR_FAIL_INDEX(p_id, IP::RESOLVER_MAX_QUERIES); - resolver->mutex->lock(); + MutexLock lock(resolver->mutex); resolver->queue[p_id].status = IP::RESOLVER_STATUS_NONE; - - resolver->mutex->unlock(); } void IP::clear_cache(const String &p_hostname) { - resolver->mutex->lock(); + MutexLock lock(resolver->mutex); if (p_hostname.empty()) { resolver->cache.clear(); @@ -218,8 +206,6 @@ void IP::clear_cache(const String &p_hostname) { resolver->cache.erase(_IP_ResolverPrivate::get_cache_key(p_hostname, IP::TYPE_IPV6)); resolver->cache.erase(_IP_ResolverPrivate::get_cache_key(p_hostname, IP::TYPE_ANY)); } - - resolver->mutex->unlock(); } Array IP::_get_local_addresses() const { @@ -315,7 +301,6 @@ IP::IP() { singleton = this; resolver = memnew(_IP_ResolverPrivate); resolver->sem = NULL; - resolver->mutex = Mutex::create(); #ifndef NO_THREADS @@ -349,6 +334,5 @@ IP::~IP() { #endif - memdelete(resolver->mutex); memdelete(resolver); } diff --git a/core/io/json.cpp b/core/io/json.cpp index 08a147032f..3a0edceb81 100644 --- a/core/io/json.cpp +++ b/core/io/json.cpp @@ -70,9 +70,11 @@ String JSON::_print_var(const Variant &p_var, const String &p_indent, int p_cur_ case Variant::NIL: return "null"; case Variant::BOOL: return p_var.operator bool() ? "true" : "false"; case Variant::INT: return itos(p_var); - case Variant::REAL: return rtos(p_var); - case Variant::PACKED_INT_ARRAY: - case Variant::PACKED_REAL_ARRAY: + case Variant::FLOAT: return rtos(p_var); + case Variant::PACKED_INT32_ARRAY: + case Variant::PACKED_INT64_ARRAY: + case Variant::PACKED_FLOAT32_ARRAY: + case Variant::PACKED_FLOAT64_ARRAY: case Variant::PACKED_STRING_ARRAY: case Variant::ARRAY: { diff --git a/core/io/marshalls.cpp b/core/io/marshalls.cpp index e97c26e05d..fbcaa582b7 100644 --- a/core/io/marshalls.cpp +++ b/core/io/marshalls.cpp @@ -147,7 +147,7 @@ Error decode_variant(Variant &r_variant, const uint8_t *p_buffer, int p_len, int } } break; - case Variant::REAL: { + case Variant::FLOAT: { if (type & ENCODE_FLAG_64) { ERR_FAIL_COND_V(len < 8, ERR_INVALID_DATA); @@ -186,7 +186,7 @@ Error decode_variant(Variant &r_variant, const uint8_t *p_buffer, int p_len, int if (r_len) (*r_len) += 4 * 2; - } break; // 5 + } break; case Variant::VECTOR2I: { ERR_FAIL_COND_V(len < 4 * 2, ERR_INVALID_DATA); @@ -198,7 +198,7 @@ Error decode_variant(Variant &r_variant, const uint8_t *p_buffer, int p_len, int if (r_len) (*r_len) += 4 * 2; - } break; // 5 + } break; case Variant::RECT2: { ERR_FAIL_COND_V(len < 4 * 4, ERR_INVALID_DATA); @@ -621,7 +621,7 @@ Error decode_variant(Variant &r_variant, const uint8_t *p_buffer, int p_len, int } } break; - case Variant::PACKED_INT_ARRAY: { + case Variant::PACKED_INT32_ARRAY: { ERR_FAIL_COND_V(len < 4, ERR_INVALID_DATA); int32_t count = decode_uint32(buf); @@ -630,12 +630,12 @@ Error decode_variant(Variant &r_variant, const uint8_t *p_buffer, int p_len, int ERR_FAIL_MUL_OF(count, 4, ERR_INVALID_DATA); ERR_FAIL_COND_V(count < 0 || count * 4 > len, ERR_INVALID_DATA); - Vector<int> data; + Vector<int32_t> data; if (count) { //const int*rbuf=(const int*)buf; data.resize(count); - int *w = data.ptrw(); + int32_t *w = data.ptrw(); for (int32_t i = 0; i < count; i++) { w[i] = decode_uint32(&buf[i * 4]); @@ -643,11 +643,37 @@ Error decode_variant(Variant &r_variant, const uint8_t *p_buffer, int p_len, int } r_variant = Variant(data); if (r_len) { - (*r_len) += 4 + count * sizeof(int); + (*r_len) += 4 + count * sizeof(int32_t); } } break; - case Variant::PACKED_REAL_ARRAY: { + case Variant::PACKED_INT64_ARRAY: { + + ERR_FAIL_COND_V(len < 4, ERR_INVALID_DATA); + int64_t count = decode_uint64(buf); + buf += 4; + len -= 4; + ERR_FAIL_MUL_OF(count, 8, ERR_INVALID_DATA); + ERR_FAIL_COND_V(count < 0 || count * 8 > len, ERR_INVALID_DATA); + + Vector<int64_t> data; + + if (count) { + //const int*rbuf=(const int*)buf; + data.resize(count); + int64_t *w = data.ptrw(); + for (int64_t i = 0; i < count; i++) { + + w[i] = decode_uint64(&buf[i * 8]); + } + } + r_variant = Variant(data); + if (r_len) { + (*r_len) += 4 + count * sizeof(int64_t); + } + + } break; + case Variant::PACKED_FLOAT32_ARRAY: { ERR_FAIL_COND_V(len < 4, ERR_INVALID_DATA); int32_t count = decode_uint32(buf); @@ -674,6 +700,33 @@ Error decode_variant(Variant &r_variant, const uint8_t *p_buffer, int p_len, int } } break; + case Variant::PACKED_FLOAT64_ARRAY: { + + ERR_FAIL_COND_V(len < 4, ERR_INVALID_DATA); + int64_t count = decode_uint64(buf); + buf += 4; + len -= 4; + ERR_FAIL_MUL_OF(count, 8, ERR_INVALID_DATA); + ERR_FAIL_COND_V(count < 0 || count * 8 > len, ERR_INVALID_DATA); + + Vector<double> data; + + if (count) { + //const double*rbuf=(const double*)buf; + data.resize(count); + double *w = data.ptrw(); + for (int64_t i = 0; i < count; i++) { + + w[i] = decode_double(&buf[i * 8]); + } + } + r_variant = data; + + if (r_len) { + (*r_len) += 4 + count * sizeof(double); + } + + } break; case Variant::PACKED_STRING_ARRAY: { ERR_FAIL_COND_V(len < 4, ERR_INVALID_DATA); @@ -851,7 +904,7 @@ Error encode_variant(const Variant &p_variant, uint8_t *r_buffer, int &r_len, bo flags |= ENCODE_FLAG_64; } } break; - case Variant::REAL: { + case Variant::FLOAT: { double d = p_variant; float f = d; @@ -918,7 +971,7 @@ Error encode_variant(const Variant &p_variant, uint8_t *r_buffer, int &r_len, bo r_len += 4; } } break; - case Variant::REAL: { + case Variant::FLOAT: { if (flags & ENCODE_FLAG_64) { if (buf) { @@ -1005,7 +1058,7 @@ Error encode_variant(const Variant &p_variant, uint8_t *r_buffer, int &r_len, bo r_len += 2 * 4; - } break; // 5 + } break; case Variant::VECTOR2I: { if (buf) { @@ -1016,7 +1069,7 @@ Error encode_variant(const Variant &p_variant, uint8_t *r_buffer, int &r_len, bo r_len += 2 * 4; - } break; // 5 + } break; case Variant::RECT2: { if (buf) { @@ -1333,33 +1386,50 @@ Error encode_variant(const Variant &p_variant, uint8_t *r_buffer, int &r_len, bo } } break; - case Variant::PACKED_INT_ARRAY: { + case Variant::PACKED_INT32_ARRAY: { - Vector<int> data = p_variant; + Vector<int32_t> data = p_variant; int datalen = data.size(); int datasize = sizeof(int32_t); if (buf) { encode_uint32(datalen, buf); buf += 4; - const int *r = data.ptr(); - for (int i = 0; i < datalen; i++) + const int32_t *r = data.ptr(); + for (int32_t i = 0; i < datalen; i++) encode_uint32(r[i], &buf[i * datasize]); } r_len += 4 + datalen * datasize; } break; - case Variant::PACKED_REAL_ARRAY: { + case Variant::PACKED_INT64_ARRAY: { + + Vector<int64_t> data = p_variant; + int datalen = data.size(); + int datasize = sizeof(int64_t); + + if (buf) { + encode_uint64(datalen, buf); + buf += 4; + const int64_t *r = data.ptr(); + for (int64_t i = 0; i < datalen; i++) + encode_uint64(r[i], &buf[i * datasize]); + } + + r_len += 4 + datalen * datasize; + + } break; + case Variant::PACKED_FLOAT32_ARRAY: { - Vector<real_t> data = p_variant; + Vector<float> data = p_variant; int datalen = data.size(); - int datasize = sizeof(real_t); + int datasize = sizeof(float); if (buf) { encode_uint32(datalen, buf); buf += 4; - const real_t *r = data.ptr(); + const float *r = data.ptr(); for (int i = 0; i < datalen; i++) encode_float(r[i], &buf[i * datasize]); } @@ -1367,6 +1437,23 @@ Error encode_variant(const Variant &p_variant, uint8_t *r_buffer, int &r_len, bo r_len += 4 + datalen * datasize; } break; + case Variant::PACKED_FLOAT64_ARRAY: { + + Vector<double> data = p_variant; + int datalen = data.size(); + int datasize = sizeof(double); + + if (buf) { + encode_uint32(datalen, buf); + buf += 4; + const double *r = data.ptr(); + for (int i = 0; i < datalen; i++) + encode_double(r[i], &buf[i * datasize]); + } + + r_len += 4 + datalen * datasize; + + } break; case Variant::PACKED_STRING_ARRAY: { Vector<String> data = p_variant; diff --git a/core/io/multiplayer_api.cpp b/core/io/multiplayer_api.cpp index 6a0eeea513..d7c82fddd9 100644 --- a/core/io/multiplayer_api.cpp +++ b/core/io/multiplayer_api.cpp @@ -34,6 +34,10 @@ #include "scene/main/node.h" #include <stdint.h> +#define NODE_ID_COMPRESSION_SHIFT 3 +#define NAME_ID_COMPRESSION_SHIFT 5 +#define BYTE_ONLY_OR_NO_ARGS_SHIFT 6 + #ifdef DEBUG_ENABLED #include "core/os/os.h" #endif @@ -145,22 +149,22 @@ void MultiplayerAPI::set_network_peer(const Ref<NetworkedMultiplayerPeer> &p_pee "Supplied NetworkedMultiplayerPeer must be connecting or connected."); if (network_peer.is_valid()) { - network_peer->disconnect_compat("peer_connected", this, "_add_peer"); - network_peer->disconnect_compat("peer_disconnected", this, "_del_peer"); - network_peer->disconnect_compat("connection_succeeded", this, "_connected_to_server"); - network_peer->disconnect_compat("connection_failed", this, "_connection_failed"); - network_peer->disconnect_compat("server_disconnected", this, "_server_disconnected"); + network_peer->disconnect("peer_connected", callable_mp(this, &MultiplayerAPI::_add_peer)); + network_peer->disconnect("peer_disconnected", callable_mp(this, &MultiplayerAPI::_del_peer)); + network_peer->disconnect("connection_succeeded", callable_mp(this, &MultiplayerAPI::_connected_to_server)); + network_peer->disconnect("connection_failed", callable_mp(this, &MultiplayerAPI::_connection_failed)); + network_peer->disconnect("server_disconnected", callable_mp(this, &MultiplayerAPI::_server_disconnected)); clear(); } network_peer = p_peer; if (network_peer.is_valid()) { - network_peer->connect_compat("peer_connected", this, "_add_peer"); - network_peer->connect_compat("peer_disconnected", this, "_del_peer"); - network_peer->connect_compat("connection_succeeded", this, "_connected_to_server"); - network_peer->connect_compat("connection_failed", this, "_connection_failed"); - network_peer->connect_compat("server_disconnected", this, "_server_disconnected"); + network_peer->connect("peer_connected", callable_mp(this, &MultiplayerAPI::_add_peer)); + network_peer->connect("peer_disconnected", callable_mp(this, &MultiplayerAPI::_del_peer)); + network_peer->connect("connection_succeeded", callable_mp(this, &MultiplayerAPI::_connected_to_server)); + network_peer->connect("connection_failed", callable_mp(this, &MultiplayerAPI::_connection_failed)); + network_peer->connect("server_disconnected", callable_mp(this, &MultiplayerAPI::_server_disconnected)); } } @@ -168,6 +172,16 @@ Ref<NetworkedMultiplayerPeer> MultiplayerAPI::get_network_peer() const { return network_peer; } +// Returns the packet size stripping the node path added when the node is not yet cached. +int get_packet_len(uint32_t p_node_target, int p_packet_len) { + if (p_node_target & 0x80000000) { + int ofs = p_node_target & 0x7FFFFFFF; + return p_packet_len - (p_packet_len - ofs); + } else { + return p_packet_len; + } +} + void MultiplayerAPI::_process_packet(int p_from, const uint8_t *p_packet, int p_packet_len) { ERR_FAIL_COND_MSG(root_node == NULL, "Multiplayer root node was not initialized. If you are using custom multiplayer, remember to set the root node via MultiplayerAPI.set_root_node before using it."); @@ -204,8 +218,8 @@ void MultiplayerAPI::_process_packet(int p_from, const uint8_t *p_packet, int p_ int name_id_offset = 1; ERR_FAIL_COND_MSG(p_packet_len < packet_min_size, "Invalid packet received. Size too small."); // Compute the meta size, which depends on the compression level. - int node_id_compression = (p_packet[0] & 24) >> 3; - int name_id_compression = (p_packet[0] & 32) >> 5; + int node_id_compression = (p_packet[0] & 24) >> NODE_ID_COMPRESSION_SHIFT; + int name_id_compression = (p_packet[0] & 32) >> NAME_ID_COMPRESSION_SHIFT; switch (node_id_compression) { case NETWORK_NODE_ID_COMPRESSION_8: @@ -250,6 +264,7 @@ void MultiplayerAPI::_process_packet(int p_from, const uint8_t *p_packet, int p_ // Unreachable, checked before. CRASH_NOW(); } + Node *node = _process_get_node(p_from, p_packet, node_target, p_packet_len); ERR_FAIL_COND_MSG(node == NULL, "Invalid packet received. Requested node was not found."); @@ -266,13 +281,14 @@ void MultiplayerAPI::_process_packet(int p_from, const uint8_t *p_packet, int p_ CRASH_NOW(); } + const int packet_len = get_packet_len(node_target, p_packet_len); if (packet_type == NETWORK_COMMAND_REMOTE_CALL) { - _process_rpc(node, name_id, p_from, p_packet, p_packet_len, packet_min_size); + _process_rpc(node, name_id, p_from, p_packet, packet_len, packet_min_size); } else { - _process_rset(node, name_id, p_from, p_packet, p_packet_len, packet_min_size); + _process_rset(node, name_id, p_from, p_packet, packet_len, packet_min_size); } } break; @@ -326,7 +342,7 @@ Node *MultiplayerAPI::_process_get_node(int p_from, const uint8_t *p_packet, uin void MultiplayerAPI::_process_rpc(Node *p_node, const uint16_t p_rpc_method_id, int p_from, const uint8_t *p_packet, int p_packet_len, int p_offset) { - ERR_FAIL_COND_MSG(p_offset >= p_packet_len, "Invalid packet received. Size too small."); + ERR_FAIL_COND_MSG(p_offset > p_packet_len, "Invalid packet received. Size too small."); // Check that remote can call the RPC on this node. StringName name = p_node->get_node_rpc_method(p_rpc_method_id); @@ -340,14 +356,30 @@ void MultiplayerAPI::_process_rpc(Node *p_node, const uint16_t p_rpc_method_id, bool can_call = _can_call_mode(p_node, rpc_mode, p_from); ERR_FAIL_COND_MSG(!can_call, "RPC '" + String(name) + "' is not allowed on node " + p_node->get_path() + " from: " + itos(p_from) + ". Mode is " + itos((int)rpc_mode) + ", master is " + itos(p_node->get_network_master()) + "."); - int argc = p_packet[p_offset]; + int argc = 0; + bool byte_only = false; + + const bool byte_only_or_no_args = ((p_packet[0] & 64) >> BYTE_ONLY_OR_NO_ARGS_SHIFT) == 1; + if (byte_only_or_no_args) { + if (p_offset < p_packet_len) { + // This packet contains only bytes. + argc = 1; + byte_only = true; + } else { + // This rpc calls a method without parameters. + } + } else { + // Normal variant, takes the argument count from the packet. + ERR_FAIL_COND_MSG(p_offset >= p_packet_len, "Invalid packet received. Size too small."); + argc = p_packet[p_offset]; + p_offset += 1; + } + Vector<Variant> args; Vector<const Variant *> argp; args.resize(argc); argp.resize(argc); - p_offset++; - #ifdef DEBUG_ENABLED if (profiling) { ObjectID id = p_node->get_instance_id(); @@ -356,16 +388,26 @@ void MultiplayerAPI::_process_rpc(Node *p_node, const uint16_t p_rpc_method_id, } #endif - for (int i = 0; i < argc; i++) { + if (byte_only) { + Vector<uint8_t> pure_data; + const int len = p_packet_len - p_offset; + pure_data.resize(len); + memcpy(pure_data.ptrw(), &p_packet[p_offset], len); + args.write[0] = pure_data; + argp.write[0] = &args[0]; + p_offset += len; + } else { + for (int i = 0; i < argc; i++) { - ERR_FAIL_COND_MSG(p_offset >= p_packet_len, "Invalid packet received. Size too small."); + ERR_FAIL_COND_MSG(p_offset >= p_packet_len, "Invalid packet received. Size too small."); - int vlen; - Error err = _decode_and_decompress_variant(args.write[i], &p_packet[p_offset], p_packet_len - p_offset, &vlen); - ERR_FAIL_COND_MSG(err != OK, "Invalid packet received. Unable to decode RPC argument."); + int vlen; + Error err = _decode_and_decompress_variant(args.write[i], &p_packet[p_offset], p_packet_len - p_offset, &vlen); + ERR_FAIL_COND_MSG(err != OK, "Invalid packet received. Unable to decode RPC argument."); - argp.write[i] = &args[i]; - p_offset += vlen; + argp.write[i] = &args[i]; + p_offset += vlen; + } } Callable::CallError ce; @@ -742,10 +784,12 @@ void MultiplayerAPI::_send_rpc(Node *p_from, int p_to, bool p_unreliable, bool p // - `NetworkCommands` in the first three bits. // - `NetworkNodeIdCompression` in the next 2 bits. // - `NetworkNameIdCompression` in the next 1 bit. - // - So we still have the last two bits free! + // - `byte_only_or_no_args` in the next 1 bit. + // - So we still have the last bit free! uint8_t command_type = p_set ? NETWORK_COMMAND_REMOTE_SET : NETWORK_COMMAND_REMOTE_CALL; uint8_t node_id_compression = UINT8_MAX; uint8_t name_id_compression = UINT8_MAX; + bool byte_only_or_no_args = false; MAKE_ROOM(1); // The meta is composed along the way, so just set 0 for now. @@ -835,17 +879,28 @@ void MultiplayerAPI::_send_rpc(Node *p_from, int p_to, bool p_unreliable, bool p ofs += 2; } - // Call arguments. - MAKE_ROOM(ofs + 1); - packet_cache.write[ofs] = p_argcount; - ofs += 1; - for (int i = 0; i < p_argcount; i++) { - int len(0); - Error err = _encode_and_compress_variant(*p_arg[i], NULL, len); - ERR_FAIL_COND_MSG(err != OK, "Unable to encode RPC argument. THIS IS LIKELY A BUG IN THE ENGINE!"); - MAKE_ROOM(ofs + len); - _encode_and_compress_variant(*p_arg[i], &(packet_cache.write[ofs]), len); - ofs += len; + if (p_argcount == 0) { + byte_only_or_no_args = true; + } else if (p_argcount == 1 && p_arg[0]->get_type() == Variant::PACKED_BYTE_ARRAY) { + byte_only_or_no_args = true; + // Special optimization when only the byte vector is sent. + const Vector<uint8_t> data = *p_arg[0]; + MAKE_ROOM(ofs + data.size()); + copymem(&(packet_cache.write[ofs]), data.ptr(), sizeof(uint8_t) * data.size()); + ofs += data.size(); + } else { + // Arguments + MAKE_ROOM(ofs + 1); + packet_cache.write[ofs] = p_argcount; + ofs += 1; + for (int i = 0; i < p_argcount; i++) { + int len(0); + Error err = _encode_and_compress_variant(*p_arg[i], NULL, len); + ERR_FAIL_COND_MSG(err != OK, "Unable to encode RPC argument. THIS IS LIKELY A BUG IN THE ENGINE!"); + MAKE_ROOM(ofs + len); + _encode_and_compress_variant(*p_arg[i], &(packet_cache.write[ofs]), len); + ofs += len; + } } } @@ -854,7 +909,7 @@ void MultiplayerAPI::_send_rpc(Node *p_from, int p_to, bool p_unreliable, bool p ERR_FAIL_COND(name_id_compression > 1); // We can now set the meta - packet_cache.write[0] = command_type + (node_id_compression << 3) + (name_id_compression << 5); + packet_cache.write[0] = command_type + (node_id_compression << NODE_ID_COMPRESSION_SHIFT) + (name_id_compression << NAME_ID_COMPRESSION_SHIFT) + ((byte_only_or_no_args ? 1 : 0) << BYTE_ONLY_OR_NO_ARGS_SHIFT); #ifdef DEBUG_ENABLED if (profiling) { @@ -1262,15 +1317,10 @@ void MultiplayerAPI::_bind_methods() { ClassDB::bind_method(D_METHOD("get_network_unique_id"), &MultiplayerAPI::get_network_unique_id); ClassDB::bind_method(D_METHOD("is_network_server"), &MultiplayerAPI::is_network_server); ClassDB::bind_method(D_METHOD("get_rpc_sender_id"), &MultiplayerAPI::get_rpc_sender_id); - ClassDB::bind_method(D_METHOD("_add_peer", "id"), &MultiplayerAPI::_add_peer); - ClassDB::bind_method(D_METHOD("_del_peer", "id"), &MultiplayerAPI::_del_peer); ClassDB::bind_method(D_METHOD("set_network_peer", "peer"), &MultiplayerAPI::set_network_peer); ClassDB::bind_method(D_METHOD("poll"), &MultiplayerAPI::poll); ClassDB::bind_method(D_METHOD("clear"), &MultiplayerAPI::clear); - ClassDB::bind_method(D_METHOD("_connected_to_server"), &MultiplayerAPI::_connected_to_server); - ClassDB::bind_method(D_METHOD("_connection_failed"), &MultiplayerAPI::_connection_failed); - ClassDB::bind_method(D_METHOD("_server_disconnected"), &MultiplayerAPI::_server_disconnected); ClassDB::bind_method(D_METHOD("get_network_connected_peers"), &MultiplayerAPI::get_network_connected_peers); ClassDB::bind_method(D_METHOD("set_refuse_new_network_connections", "refuse"), &MultiplayerAPI::set_refuse_new_network_connections); ClassDB::bind_method(D_METHOD("is_refusing_new_network_connections"), &MultiplayerAPI::is_refusing_new_network_connections); diff --git a/core/io/resource_format_binary.cpp b/core/io/resource_format_binary.cpp index 518323c5c4..54b75cc29d 100644 --- a/core/io/resource_format_binary.cpp +++ b/core/io/resource_format_binary.cpp @@ -46,7 +46,7 @@ enum { VARIANT_NIL = 1, VARIANT_BOOL = 2, VARIANT_INT = 3, - VARIANT_REAL = 4, + VARIANT_FLOAT = 4, VARIANT_STRING = 5, VARIANT_VECTOR2 = 10, VARIANT_RECT2 = 11, @@ -65,8 +65,8 @@ enum { VARIANT_DICTIONARY = 26, VARIANT_ARRAY = 30, VARIANT_RAW_ARRAY = 31, - VARIANT_INT_ARRAY = 32, - VARIANT_REAL_ARRAY = 33, + VARIANT_INT32_ARRAY = 32, + VARIANT_FLOAT32_ARRAY = 33, VARIANT_STRING_ARRAY = 34, VARIANT_VECTOR3_ARRAY = 35, VARIANT_COLOR_ARRAY = 36, @@ -79,6 +79,8 @@ enum { VARIANT_VECTOR2I = 45, VARIANT_RECT2I = 46, VARIANT_VECTOR3I = 47, + VARIANT_INT64_ARRAY = 48, + VARIANT_FLOAT64_ARRAY = 49, OBJECT_EMPTY = 0, OBJECT_EXTERNAL_RESOURCE = 1, OBJECT_INTERNAL_RESOURCE = 2, @@ -91,7 +93,7 @@ enum { }; -void ResourceInteractiveLoaderBinary::_advance_padding(uint32_t p_len) { +void ResourceLoaderBinary::_advance_padding(uint32_t p_len) { uint32_t extra = 4 - (p_len % 4); if (extra < 4) { @@ -100,7 +102,7 @@ void ResourceInteractiveLoaderBinary::_advance_padding(uint32_t p_len) { } } -StringName ResourceInteractiveLoaderBinary::_get_string() { +StringName ResourceLoaderBinary::_get_string() { uint32_t id = f->get_32(); if (id & 0x80000000) { @@ -119,7 +121,7 @@ StringName ResourceInteractiveLoaderBinary::_get_string() { return string_map[id]; } -Error ResourceInteractiveLoaderBinary::parse_variant(Variant &r_v) { +Error ResourceLoaderBinary::parse_variant(Variant &r_v) { uint32_t type = f->get_32(); print_bl("find property of type: " + itos(type)); @@ -142,7 +144,7 @@ Error ResourceInteractiveLoaderBinary::parse_variant(Variant &r_v) { r_v = int64_t(f->get_64()); } break; - case VARIANT_REAL: { + case VARIANT_FLOAT: { r_v = f->get_real(); } break; @@ -375,20 +377,26 @@ Error ResourceInteractiveLoaderBinary::parse_variant(Variant &r_v) { r_v = Variant(); } else { - String exttype = external_resources[erindex].type; - String path = external_resources[erindex].path; + if (external_resources[erindex].cache.is_null()) { + //cache not here yet, wait for it? + if (use_sub_threads) { + Error err; + external_resources.write[erindex].cache = ResourceLoader::load_threaded_get(external_resources[erindex].path, &err); - if (path.find("://") == -1 && path.is_rel_path()) { - // path is relative to file being loaded, so convert to a resource path - path = ProjectSettings::get_singleton()->localize_path(res_path.get_base_dir().plus_file(path)); - } + if (err != OK || external_resources[erindex].cache.is_null()) { + if (!ResourceLoader::get_abort_on_missing_resources()) { - RES res = ResourceLoader::load(path, exttype); + ResourceLoader::notify_dependency_error(local_path, external_resources[erindex].path, external_resources[erindex].type); + } else { - if (res.is_null()) { - WARN_PRINT(String("Couldn't load resource: " + path).utf8().get_data()); + error = ERR_FILE_MISSING_DEPENDENCIES; + ERR_FAIL_V_MSG(error, "Can't load dependency: " + external_resources[erindex].path + "."); + } + } + } } - r_v = res; + + r_v = external_resources[erindex].cache; } } break; @@ -452,14 +460,14 @@ Error ResourceInteractiveLoaderBinary::parse_variant(Variant &r_v) { r_v = array; } break; - case VARIANT_INT_ARRAY: { + case VARIANT_INT32_ARRAY: { uint32_t len = f->get_32(); - Vector<int> array; + Vector<int32_t> array; array.resize(len); - int *w = array.ptrw(); - f->get_buffer((uint8_t *)w, len * 4); + int32_t *w = array.ptrw(); + f->get_buffer((uint8_t *)w, len * sizeof(int32_t)); #ifdef BIG_ENDIAN_ENABLED { uint32_t *ptr = (uint32_t *)w.ptr(); @@ -473,14 +481,35 @@ Error ResourceInteractiveLoaderBinary::parse_variant(Variant &r_v) { r_v = array; } break; - case VARIANT_REAL_ARRAY: { + case VARIANT_INT64_ARRAY: { + + uint32_t len = f->get_32(); + + Vector<int64_t> array; + array.resize(len); + int64_t *w = array.ptrw(); + f->get_buffer((uint8_t *)w, len * sizeof(int64_t)); +#ifdef BIG_ENDIAN_ENABLED + { + uint64_t *ptr = (uint64_t *)w.ptr(); + for (int i = 0; i < len; i++) { + + ptr[i] = BSWAP64(ptr[i]); + } + } + +#endif + + r_v = array; + } break; + case VARIANT_FLOAT32_ARRAY: { uint32_t len = f->get_32(); - Vector<real_t> array; + Vector<float> array; array.resize(len); - real_t *w = array.ptrw(); - f->get_buffer((uint8_t *)w, len * sizeof(real_t)); + float *w = array.ptrw(); + f->get_buffer((uint8_t *)w, len * sizeof(float)); #ifdef BIG_ENDIAN_ENABLED { uint32_t *ptr = (uint32_t *)w.ptr(); @@ -494,6 +523,27 @@ Error ResourceInteractiveLoaderBinary::parse_variant(Variant &r_v) { r_v = array; } break; + case VARIANT_FLOAT64_ARRAY: { + + uint32_t len = f->get_32(); + + Vector<double> array; + array.resize(len); + double *w = array.ptrw(); + f->get_buffer((uint8_t *)w, len * sizeof(double)); +#ifdef BIG_ENDIAN_ENABLED + { + uint64_t *ptr = (uint64_t *)w.ptr(); + for (int i = 0; i < len; i++) { + + ptr[i] = BSWAP64(ptr[i]); + } + } + +#endif + + r_v = array; + } break; case VARIANT_STRING_ARRAY: { uint32_t len = f->get_32(); @@ -594,160 +644,168 @@ Error ResourceInteractiveLoaderBinary::parse_variant(Variant &r_v) { return OK; //never reach anyway } -void ResourceInteractiveLoaderBinary::set_local_path(const String &p_local_path) { +void ResourceLoaderBinary::set_local_path(const String &p_local_path) { res_path = p_local_path; } -Ref<Resource> ResourceInteractiveLoaderBinary::get_resource() { +Ref<Resource> ResourceLoaderBinary::get_resource() { return resource; } -Error ResourceInteractiveLoaderBinary::poll() { +Error ResourceLoaderBinary::load() { if (error != OK) return error; - int s = stage; + int stage = 0; - if (s < external_resources.size()) { + for (int i = 0; i < external_resources.size(); i++) { - String path = external_resources[s].path; + String path = external_resources[i].path; if (remaps.has(path)) { path = remaps[path]; } - RES res = ResourceLoader::load(path, external_resources[s].type); - if (res.is_null()) { - if (!ResourceLoader::get_abort_on_missing_resources()) { + if (path.find("://") == -1 && path.is_rel_path()) { + // path is relative to file being loaded, so convert to a resource path + path = ProjectSettings::get_singleton()->localize_path(path.get_base_dir().plus_file(external_resources[i].path)); + } - ResourceLoader::notify_dependency_error(local_path, path, external_resources[s].type); - } else { + external_resources.write[i].path = path; //remap happens here, not on load because on load it can actually be used for filesystem dock resource remap + + if (!use_sub_threads) { + external_resources.write[i].cache = ResourceLoader::load(path, external_resources[i].type); + + if (external_resources[i].cache.is_null()) { + if (!ResourceLoader::get_abort_on_missing_resources()) { + + ResourceLoader::notify_dependency_error(local_path, path, external_resources[i].type); + } else { - error = ERR_FILE_MISSING_DEPENDENCIES; - ERR_FAIL_V_MSG(error, "Can't load dependency: " + path + "."); + error = ERR_FILE_MISSING_DEPENDENCIES; + ERR_FAIL_V_MSG(error, "Can't load dependency: " + path + "."); + } } } else { - resource_cache.push_back(res); + Error err = ResourceLoader::load_threaded_request(path, external_resources[i].type, use_sub_threads, local_path); + if (err != OK) { + if (!ResourceLoader::get_abort_on_missing_resources()) { + + ResourceLoader::notify_dependency_error(local_path, path, external_resources[i].type); + } else { + + error = ERR_FILE_MISSING_DEPENDENCIES; + ERR_FAIL_V_MSG(error, "Can't load dependency: " + path + "."); + } + } } stage++; - return error; } - s -= external_resources.size(); - - if (s >= internal_resources.size()) { + for (int i = 0; i < internal_resources.size(); i++) { - error = ERR_BUG; - ERR_FAIL_COND_V(s >= internal_resources.size(), error); - } + bool main = i == (internal_resources.size() - 1); - bool main = s == (internal_resources.size() - 1); + //maybe it is loaded already + String path; + int subindex = 0; - //maybe it is loaded already - String path; - int subindex = 0; + if (!main) { - if (!main) { + path = internal_resources[i].path; + if (path.begins_with("local://")) { + path = path.replace_first("local://", ""); + subindex = path.to_int(); + path = res_path + "::" + path; + } - path = internal_resources[s].path; - if (path.begins_with("local://")) { - path = path.replace_first("local://", ""); - subindex = path.to_int(); - path = res_path + "::" + path; - } + if (ResourceCache::has(path)) { + //already loaded, don't do anything + stage++; + error = OK; + continue; + } + } else { - if (ResourceCache::has(path)) { - //already loaded, don't do anything - stage++; - error = OK; - return error; + if (!ResourceCache::has(res_path)) + path = res_path; } - } else { - if (!ResourceCache::has(res_path)) - path = res_path; - } + uint64_t offset = internal_resources[i].offset; - uint64_t offset = internal_resources[s].offset; + f->seek(offset); - f->seek(offset); + String t = get_unicode_string(); - String t = get_unicode_string(); - - Object *obj = ClassDB::instance(t); - if (!obj) { - error = ERR_FILE_CORRUPT; - ERR_FAIL_V_MSG(ERR_FILE_CORRUPT, local_path + ":Resource of unrecognized type in file: " + t + "."); - } + Object *obj = ClassDB::instance(t); + if (!obj) { + error = ERR_FILE_CORRUPT; + ERR_FAIL_V_MSG(ERR_FILE_CORRUPT, local_path + ":Resource of unrecognized type in file: " + t + "."); + } - Resource *r = Object::cast_to<Resource>(obj); - if (!r) { - String obj_class = obj->get_class(); - error = ERR_FILE_CORRUPT; - memdelete(obj); //bye - ERR_FAIL_V_MSG(ERR_FILE_CORRUPT, local_path + ":Resource type in resource field not a resource, type is: " + obj_class + "."); - } + Resource *r = Object::cast_to<Resource>(obj); + if (!r) { + String obj_class = obj->get_class(); + error = ERR_FILE_CORRUPT; + memdelete(obj); //bye + ERR_FAIL_V_MSG(ERR_FILE_CORRUPT, local_path + ":Resource type in resource field not a resource, type is: " + obj_class + "."); + } - RES res = RES(r); + RES res = RES(r); - r->set_path(path); - r->set_subindex(subindex); + r->set_path(path); + r->set_subindex(subindex); - int pc = f->get_32(); + int pc = f->get_32(); - //set properties + //set properties - for (int i = 0; i < pc; i++) { + for (int j = 0; j < pc; j++) { - StringName name = _get_string(); + StringName name = _get_string(); - if (name == StringName()) { - error = ERR_FILE_CORRUPT; - ERR_FAIL_V(ERR_FILE_CORRUPT); - } + if (name == StringName()) { + error = ERR_FILE_CORRUPT; + ERR_FAIL_V(ERR_FILE_CORRUPT); + } - Variant value; + Variant value; - error = parse_variant(value); - if (error) - return error; + error = parse_variant(value); + if (error) + return error; - res->set(name, value); - } + res->set(name, value); + } #ifdef TOOLS_ENABLED - res->set_edited(false); + res->set_edited(false); #endif - stage++; + stage++; - resource_cache.push_back(res); + if (progress) { + *progress = (i + 1) / float(internal_resources.size()); + } - if (main) { + resource_cache.push_back(res); - f->close(); - resource = res; - resource->set_as_translation_remapped(translation_remapped); - error = ERR_FILE_EOF; + if (main) { - } else { - error = OK; + f->close(); + resource = res; + resource->set_as_translation_remapped(translation_remapped); + error = OK; + return OK; + } } - return OK; -} -int ResourceInteractiveLoaderBinary::get_stage() const { - - return stage; + return ERR_FILE_EOF; } -int ResourceInteractiveLoaderBinary::get_stage_count() const { - return external_resources.size() + internal_resources.size(); -} - -void ResourceInteractiveLoaderBinary::set_translation_remapped(bool p_remapped) { +void ResourceLoaderBinary::set_translation_remapped(bool p_remapped) { translation_remapped = p_remapped; } @@ -770,7 +828,7 @@ static String get_ustring(FileAccess *f) { return s; } -String ResourceInteractiveLoaderBinary::get_unicode_string() { +String ResourceLoaderBinary::get_unicode_string() { int len = f->get_32(); if (len > str_buf.size()) { @@ -784,7 +842,7 @@ String ResourceInteractiveLoaderBinary::get_unicode_string() { return s; } -void ResourceInteractiveLoaderBinary::get_dependencies(FileAccess *p_f, List<String> *p_dependencies, bool p_add_types) { +void ResourceLoaderBinary::get_dependencies(FileAccess *p_f, List<String> *p_dependencies, bool p_add_types) { open(p_f); if (error) @@ -802,7 +860,7 @@ void ResourceInteractiveLoaderBinary::get_dependencies(FileAccess *p_f, List<Str } } -void ResourceInteractiveLoaderBinary::open(FileAccess *p_f) { +void ResourceLoaderBinary::open(FileAccess *p_f) { error = OK; @@ -903,7 +961,7 @@ void ResourceInteractiveLoaderBinary::open(FileAccess *p_f) { } } -String ResourceInteractiveLoaderBinary::recognize(FileAccess *p_f) { +String ResourceLoaderBinary::recognize(FileAccess *p_f) { error = OK; @@ -948,20 +1006,22 @@ String ResourceInteractiveLoaderBinary::recognize(FileAccess *p_f) { return type; } -ResourceInteractiveLoaderBinary::ResourceInteractiveLoaderBinary() : +ResourceLoaderBinary::ResourceLoaderBinary() : translation_remapped(false), f(NULL), - error(OK), - stage(0) { + error(OK) { + + progress = nullptr; + use_sub_threads = false; } -ResourceInteractiveLoaderBinary::~ResourceInteractiveLoaderBinary() { +ResourceLoaderBinary::~ResourceLoaderBinary() { if (f) memdelete(f); } -Ref<ResourceInteractiveLoader> ResourceFormatLoaderBinary::load_interactive(const String &p_path, const String &p_original_path, Error *r_error) { +RES ResourceFormatLoaderBinary::load(const String &p_path, const String &p_original_path, Error *r_error, bool p_use_sub_threads, float *r_progress) { if (r_error) *r_error = ERR_FILE_CANT_OPEN; @@ -969,16 +1029,27 @@ Ref<ResourceInteractiveLoader> ResourceFormatLoaderBinary::load_interactive(cons Error err; FileAccess *f = FileAccess::open(p_path, FileAccess::READ, &err); - ERR_FAIL_COND_V_MSG(err != OK, Ref<ResourceInteractiveLoader>(), "Cannot open file '" + p_path + "'."); + ERR_FAIL_COND_V_MSG(err != OK, RES(), "Cannot open file '" + p_path + "'."); - Ref<ResourceInteractiveLoaderBinary> ria = memnew(ResourceInteractiveLoaderBinary); + ResourceLoaderBinary loader; + loader.use_sub_threads = p_use_sub_threads; + loader.progress = r_progress; String path = p_original_path != "" ? p_original_path : p_path; - ria->local_path = ProjectSettings::get_singleton()->localize_path(path); - ria->res_path = ria->local_path; - //ria->set_local_path( Globals::get_singleton()->localize_path(p_path) ); - ria->open(f); + loader.local_path = ProjectSettings::get_singleton()->localize_path(path); + loader.res_path = loader.local_path; + //loader.set_local_path( Globals::get_singleton()->localize_path(p_path) ); + loader.open(f); + + err = loader.load(); - return ria; + if (r_error) { + *r_error = err; + } + + if (err) { + return RES(); + } + return loader.resource; } void ResourceFormatLoaderBinary::get_recognized_extensions_for_type(const String &p_type, List<String> *p_extensions) const { @@ -1020,11 +1091,11 @@ void ResourceFormatLoaderBinary::get_dependencies(const String &p_path, List<Str FileAccess *f = FileAccess::open(p_path, FileAccess::READ); ERR_FAIL_COND_MSG(!f, "Cannot open file '" + p_path + "'."); - Ref<ResourceInteractiveLoaderBinary> ria = memnew(ResourceInteractiveLoaderBinary); - ria->local_path = ProjectSettings::get_singleton()->localize_path(p_path); - ria->res_path = ria->local_path; - //ria->set_local_path( Globals::get_singleton()->localize_path(p_path) ); - ria->get_dependencies(f, p_dependencies, p_add_types); + ResourceLoaderBinary loader; + loader.local_path = ProjectSettings::get_singleton()->localize_path(p_path); + loader.res_path = loader.local_path; + //loader.set_local_path( Globals::get_singleton()->localize_path(p_path) ); + loader.get_dependencies(f, p_dependencies, p_add_types); } Error ResourceFormatLoaderBinary::rename_dependencies(const String &p_path, const Map<String, String> &p_map) { @@ -1109,21 +1180,17 @@ Error ResourceFormatLoaderBinary::rename_dependencies(const String &p_path, cons ERR_FAIL_COND_V_MSG(err != OK, ERR_FILE_CANT_OPEN, "Cannot open file '" + p_path + "'."); - Ref<ResourceInteractiveLoaderBinary> ria = memnew(ResourceInteractiveLoaderBinary); - ria->local_path = ProjectSettings::get_singleton()->localize_path(p_path); - ria->res_path = ria->local_path; - ria->remaps = p_map; - //ria->set_local_path( Globals::get_singleton()->localize_path(p_path) ); - ria->open(f); + ResourceLoaderBinary loader; + loader.local_path = ProjectSettings::get_singleton()->localize_path(p_path); + loader.res_path = loader.local_path; + loader.remaps = p_map; + //loader.set_local_path( Globals::get_singleton()->localize_path(p_path) ); + loader.open(f); - err = ria->poll(); - - while (err == OK) { - err = ria->poll(); - } + err = loader.load(); ERR_FAIL_COND_V(err != ERR_FILE_EOF, ERR_FILE_CORRUPT); - RES res = ria->get_resource(); + RES res = loader.get_resource(); ERR_FAIL_COND_V(!res.is_valid(), ERR_FILE_CORRUPT); return ResourceFormatSaverBinary::singleton->save(p_path, res); @@ -1239,11 +1306,11 @@ String ResourceFormatLoaderBinary::get_resource_type(const String &p_path) const return ""; //could not rwead } - Ref<ResourceInteractiveLoaderBinary> ria = memnew(ResourceInteractiveLoaderBinary); - ria->local_path = ProjectSettings::get_singleton()->localize_path(p_path); - ria->res_path = ria->local_path; - //ria->set_local_path( Globals::get_singleton()->localize_path(p_path) ); - String r = ria->recognize(f); + ResourceLoaderBinary loader; + loader.local_path = ProjectSettings::get_singleton()->localize_path(p_path); + loader.res_path = loader.local_path; + //loader.set_local_path( Globals::get_singleton()->localize_path(p_path) ); + String r = loader.recognize(f); return ClassDB::get_compatibility_remapped_class(r); } @@ -1293,7 +1360,7 @@ void ResourceFormatSaverBinaryInstance::write_variant(FileAccess *f, const Varia } } break; - case Variant::REAL: { + case Variant::FLOAT: { double d = p_property; float fl = d; @@ -1302,7 +1369,7 @@ void ResourceFormatSaverBinaryInstance::write_variant(FileAccess *f, const Varia f->store_double(d); } else { - f->store_32(VARIANT_REAL); + f->store_32(VARIANT_FLOAT); f->store_real(fl); } @@ -1573,29 +1640,52 @@ void ResourceFormatSaverBinaryInstance::write_variant(FileAccess *f, const Varia _pad_buffer(f, len); } break; - case Variant::PACKED_INT_ARRAY: { + case Variant::PACKED_INT32_ARRAY: { - f->store_32(VARIANT_INT_ARRAY); - Vector<int> arr = p_property; + f->store_32(VARIANT_INT32_ARRAY); + Vector<int32_t> arr = p_property; int len = arr.size(); f->store_32(len); - const int *r = arr.ptr(); + const int32_t *r = arr.ptr(); for (int i = 0; i < len; i++) f->store_32(r[i]); } break; - case Variant::PACKED_REAL_ARRAY: { + case Variant::PACKED_INT64_ARRAY: { - f->store_32(VARIANT_REAL_ARRAY); - Vector<real_t> arr = p_property; + f->store_32(VARIANT_INT64_ARRAY); + Vector<int64_t> arr = p_property; int len = arr.size(); f->store_32(len); - const real_t *r = arr.ptr(); + const int64_t *r = arr.ptr(); + for (int i = 0; i < len; i++) + f->store_64(r[i]); + + } break; + case Variant::PACKED_FLOAT32_ARRAY: { + + f->store_32(VARIANT_FLOAT32_ARRAY); + Vector<float> arr = p_property; + int len = arr.size(); + f->store_32(len); + const float *r = arr.ptr(); for (int i = 0; i < len; i++) { f->store_real(r[i]); } } break; + case Variant::PACKED_FLOAT64_ARRAY: { + + f->store_32(VARIANT_FLOAT64_ARRAY); + Vector<double> arr = p_property; + int len = arr.size(); + f->store_32(len); + const double *r = arr.ptr(); + for (int i = 0; i < len; i++) { + f->store_double(r[i]); + } + + } break; case Variant::PACKED_STRING_ARRAY: { f->store_32(VARIANT_STRING_ARRAY); diff --git a/core/io/resource_format_binary.h b/core/io/resource_format_binary.h index f02dbaa0c2..0ffa2c3626 100644 --- a/core/io/resource_format_binary.h +++ b/core/io/resource_format_binary.h @@ -35,7 +35,7 @@ #include "core/io/resource_saver.h" #include "core/os/file_access.h" -class ResourceInteractiveLoaderBinary : public ResourceInteractiveLoader { +class ResourceLoaderBinary { bool translation_remapped; String local_path; @@ -58,8 +58,11 @@ class ResourceInteractiveLoaderBinary : public ResourceInteractiveLoader { struct ExtResource { String path; String type; + RES cache; }; + bool use_sub_threads; + float *progress; Vector<ExtResource> external_resources; struct IntResource { @@ -75,32 +78,30 @@ class ResourceInteractiveLoaderBinary : public ResourceInteractiveLoader { Map<String, String> remaps; Error error; - int stage; - friend class ResourceFormatLoaderBinary; Error parse_variant(Variant &r_v); + Map<String, RES> dependency_cache; + public: - virtual void set_local_path(const String &p_local_path); - virtual Ref<Resource> get_resource(); - virtual Error poll(); - virtual int get_stage() const; - virtual int get_stage_count() const; - virtual void set_translation_remapped(bool p_remapped); + void set_local_path(const String &p_local_path); + Ref<Resource> get_resource(); + Error load(); + void set_translation_remapped(bool p_remapped); void set_remaps(const Map<String, String> &p_remaps) { remaps = p_remaps; } void open(FileAccess *p_f); String recognize(FileAccess *p_f); void get_dependencies(FileAccess *p_f, List<String> *p_dependencies, bool p_add_types); - ResourceInteractiveLoaderBinary(); - ~ResourceInteractiveLoaderBinary(); + ResourceLoaderBinary(); + ~ResourceLoaderBinary(); }; class ResourceFormatLoaderBinary : public ResourceFormatLoader { public: - virtual Ref<ResourceInteractiveLoader> load_interactive(const String &p_path, const String &p_original_path = "", Error *r_error = NULL); + virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL, bool p_use_sub_threads = false, float *r_progress = nullptr); virtual void get_recognized_extensions_for_type(const String &p_type, List<String> *p_extensions) const; virtual void get_recognized_extensions(List<String> *p_extensions) const; virtual bool handles_type(const String &p_type) const; diff --git a/core/io/resource_importer.cpp b/core/io/resource_importer.cpp index f147170ff7..efaf958949 100644 --- a/core/io/resource_importer.cpp +++ b/core/io/resource_importer.cpp @@ -117,7 +117,7 @@ Error ResourceFormatImporter::_get_path_and_type(const String &p_path, PathAndTy return OK; } -RES ResourceFormatImporter::load(const String &p_path, const String &p_original_path, Error *r_error) { +RES ResourceFormatImporter::load(const String &p_path, const String &p_original_path, Error *r_error, bool p_use_sub_threads, float *r_progress) { PathAndType pat; Error err = _get_path_and_type(p_path, pat); @@ -130,7 +130,7 @@ RES ResourceFormatImporter::load(const String &p_path, const String &p_original_ return RES(); } - RES res = ResourceLoader::_load(pat.path, p_path, pat.type, false, r_error); + RES res = ResourceLoader::_load(pat.path, p_path, pat.type, false, r_error, p_use_sub_threads, r_progress); #ifdef TOOLS_ENABLED if (res.is_valid()) { diff --git a/core/io/resource_importer.h b/core/io/resource_importer.h index 4eb04586e6..65c148f2ac 100644 --- a/core/io/resource_importer.h +++ b/core/io/resource_importer.h @@ -58,7 +58,7 @@ class ResourceFormatImporter : public ResourceFormatLoader { public: static ResourceFormatImporter *get_singleton() { return singleton; } - virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL); + virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL, bool p_use_sub_threads = false, float *r_progress = nullptr); virtual void get_recognized_extensions(List<String> *p_extensions) const; virtual void get_recognized_extensions_for_type(const String &p_type, List<String> *p_extensions) const; virtual bool recognize_path(const String &p_path, const String &p_for_type = String()) const; diff --git a/core/io/resource_loader.cpp b/core/io/resource_loader.cpp index 39bbebefa6..504dbe2d63 100644 --- a/core/io/resource_loader.cpp +++ b/core/io/resource_loader.cpp @@ -39,26 +39,16 @@ #include "core/translation.h" #include "core/variant_parser.h" +#ifdef DEBUG_LOAD_THREADED +#define print_lt(m_text) print_line(m_text) +#else +#define print_lt(m_text) +#endif + Ref<ResourceFormatLoader> ResourceLoader::loader[ResourceLoader::MAX_LOADERS]; int ResourceLoader::loader_count = 0; -Error ResourceInteractiveLoader::wait() { - - Error err = poll(); - while (err == OK) { - err = poll(); - } - - return err; -} - -ResourceInteractiveLoader::~ResourceInteractiveLoader() { - if (path_loading != String()) { - ResourceLoader::_remove_from_loading_map_and_thread(path_loading, path_loading_thread); - } -} - bool ResourceFormatLoader::recognize_path(const String &p_path, const String &p_for_type) const { String extension = p_path.get_extension(); @@ -111,45 +101,6 @@ void ResourceLoader::get_recognized_extensions_for_type(const String &p_type, Li } } -void ResourceInteractiveLoader::_bind_methods() { - - ClassDB::bind_method(D_METHOD("get_resource"), &ResourceInteractiveLoader::get_resource); - ClassDB::bind_method(D_METHOD("poll"), &ResourceInteractiveLoader::poll); - ClassDB::bind_method(D_METHOD("wait"), &ResourceInteractiveLoader::wait); - ClassDB::bind_method(D_METHOD("get_stage"), &ResourceInteractiveLoader::get_stage); - ClassDB::bind_method(D_METHOD("get_stage_count"), &ResourceInteractiveLoader::get_stage_count); -} - -class ResourceInteractiveLoaderDefault : public ResourceInteractiveLoader { - - GDCLASS(ResourceInteractiveLoaderDefault, ResourceInteractiveLoader); - -public: - Ref<Resource> resource; - - virtual void set_local_path(const String &p_local_path) { /*scene->set_filename(p_local_path);*/ - } - virtual Ref<Resource> get_resource() { return resource; } - virtual Error poll() { return ERR_FILE_EOF; } - virtual int get_stage() const { return 1; } - virtual int get_stage_count() const { return 1; } - virtual void set_translation_remapped(bool p_remapped) { resource->set_as_translation_remapped(p_remapped); } - - ResourceInteractiveLoaderDefault() {} -}; - -Ref<ResourceInteractiveLoader> ResourceFormatLoader::load_interactive(const String &p_path, const String &p_original_path, Error *r_error) { - - //either this - Ref<Resource> res = load(p_path, p_original_path, r_error); - if (res.is_null()) - return Ref<ResourceInteractiveLoader>(); - - Ref<ResourceInteractiveLoaderDefault> ril = Ref<ResourceInteractiveLoaderDefault>(memnew(ResourceInteractiveLoaderDefault)); - ril->resource = res; - return ril; -} - bool ResourceFormatLoader::exists(const String &p_path) const { return FileAccess::exists(p_path); //by default just check file } @@ -168,10 +119,10 @@ void ResourceFormatLoader::get_recognized_extensions(List<String> *p_extensions) } } -RES ResourceFormatLoader::load(const String &p_path, const String &p_original_path, Error *r_error) { +RES ResourceFormatLoader::load(const String &p_path, const String &p_original_path, Error *r_error, bool p_use_sub_threads, float *r_progress) { if (get_script_instance() && get_script_instance()->has_method("load")) { - Variant res = get_script_instance()->call("load", p_path, p_original_path); + Variant res = get_script_instance()->call("load", p_path, p_original_path, p_use_sub_threads); if (res.get_type() == Variant::INT) { @@ -184,29 +135,11 @@ RES ResourceFormatLoader::load(const String &p_path, const String &p_original_pa *r_error = OK; return res; } - } - - //or this must be implemented - Ref<ResourceInteractiveLoader> ril = load_interactive(p_path, p_original_path, r_error); - if (!ril.is_valid()) - return RES(); - ril->set_local_path(p_original_path); - - while (true) { - - Error err = ril->poll(); - - if (err == ERR_FILE_EOF) { - if (r_error) - *r_error = OK; - return ril->get_resource(); - } - if (r_error) - *r_error = err; - - ERR_FAIL_COND_V_MSG(err != OK, RES(), "Failed to load resource '" + p_path + "'."); + return res; } + + ERR_FAIL_V_MSG(RES(), "Failed to load resource '" + p_path + "', ResourceFormatLoader::load was not implemented for this resource type."); } void ResourceFormatLoader::get_dependencies(const String &p_path, List<String> *p_dependencies, bool p_add_types) { @@ -256,7 +189,7 @@ void ResourceFormatLoader::_bind_methods() { /////////////////////////////////// -RES ResourceLoader::_load(const String &p_path, const String &p_original_path, const String &p_type_hint, bool p_no_cache, Error *r_error) { +RES ResourceLoader::_load(const String &p_path, const String &p_original_path, const String &p_type_hint, bool p_no_cache, Error *r_error, bool p_use_sub_threads, float *r_progress) { bool found = false; @@ -267,7 +200,7 @@ RES ResourceLoader::_load(const String &p_path, const String &p_original_path, c continue; } found = true; - RES res = loader[i]->load(p_path, p_original_path != String() ? p_original_path : p_path, r_error); + RES res = loader[i]->load(p_path, p_original_path != String() ? p_original_path : p_path, r_error, p_use_sub_threads, r_progress); if (res.is_null()) { continue; } @@ -285,61 +218,302 @@ RES ResourceLoader::_load(const String &p_path, const String &p_original_path, c ERR_FAIL_V_MSG(RES(), "No loader found for resource: " + p_path + "."); } -bool ResourceLoader::_add_to_loading_map(const String &p_path) { +void ResourceLoader::_thread_load_function(void *p_userdata) { - bool success; - if (loading_map_mutex) { - loading_map_mutex->lock(); + ThreadLoadTask &load_task = *(ThreadLoadTask *)p_userdata; + load_task.loader_id = Thread::get_caller_id(); + + if (load_task.semaphore) { + //this is an actual thread, so wait for Ok fom semaphore + thread_load_semaphore->wait(); //wait until its ok to start loading } + load_task.resource = _load(load_task.remapped_path, load_task.remapped_path != load_task.local_path ? load_task.local_path : String(), load_task.type_hint, false, &load_task.error, load_task.use_sub_threads, &load_task.progress); - LoadingMapKey key; - key.path = p_path; - key.thread = Thread::get_caller_id(); + load_task.progress = 1.0; //it was fully loaded at this point, so force progress to 1.0 - if (loading_map.has(key)) { - success = false; + thread_load_mutex->lock(); + if (load_task.error != OK) { + load_task.status = THREAD_LOAD_FAILED; } else { - loading_map[key] = true; - success = true; + load_task.status = THREAD_LOAD_LOADED; + } + if (load_task.semaphore) { + + if (load_task.start_next && thread_waiting_count > 0) { + thread_waiting_count--; + //thread loading count remains constant, this ends but another one begins + thread_load_semaphore->post(); + } else { + thread_loading_count--; //no threads waiting, just reduce loading count + } + + print_lt("END: load count: " + itos(thread_loading_count) + " / wait count: " + itos(thread_waiting_count) + " / suspended count: " + itos(thread_suspended_count) + " / active: " + itos(thread_loading_count - thread_suspended_count)); + + for (int i = 0; i < load_task.poll_requests; i++) { + load_task.semaphore->post(); + } + memdelete(load_task.semaphore); + load_task.semaphore = nullptr; } - if (loading_map_mutex) { - loading_map_mutex->unlock(); + if (load_task.resource.is_valid()) { + load_task.resource->set_path(load_task.local_path); + + if (load_task.xl_remapped) + load_task.resource->set_as_translation_remapped(true); + +#ifdef TOOLS_ENABLED + + load_task.resource->set_edited(false); + if (timestamp_on_load) { + uint64_t mt = FileAccess::get_modified_time(load_task.remapped_path); + //printf("mt %s: %lli\n",remapped_path.utf8().get_data(),mt); + load_task.resource->set_last_modified_time(mt); + } +#endif + + if (_loaded_callback) { + _loaded_callback(load_task.resource, load_task.local_path); + } } - return success; + thread_load_mutex->unlock(); } +Error ResourceLoader::load_threaded_request(const String &p_path, const String &p_type_hint, bool p_use_sub_threads, const String &p_source_resource) { -void ResourceLoader::_remove_from_loading_map(const String &p_path) { - if (loading_map_mutex) { - loading_map_mutex->lock(); + String local_path; + if (p_path.is_rel_path()) + local_path = "res://" + p_path; + else + local_path = ProjectSettings::get_singleton()->localize_path(p_path); + + thread_load_mutex->lock(); + + if (p_source_resource != String()) { + //must be loading from this resource + if (!thread_load_tasks.has(p_source_resource)) { + thread_load_mutex->unlock(); + ERR_FAIL_V_MSG(ERR_INVALID_PARAMETER, "There is no thread loading source resource '" + p_source_resource + "'."); + } + //must be loading from this thread + if (thread_load_tasks[p_source_resource].loader_id != Thread::get_caller_id()) { + thread_load_mutex->unlock(); + ERR_FAIL_V_MSG(ERR_INVALID_PARAMETER, "Threading loading resource'" + local_path + " failed: Source specified: '" + p_source_resource + "' but was not called by it."); + } + + //must not be already added as s sub tasks + if (thread_load_tasks[p_source_resource].sub_tasks.has(local_path)) { + thread_load_mutex->unlock(); + ERR_FAIL_V_MSG(ERR_INVALID_PARAMETER, "Thread loading source resource '" + p_source_resource + "' already is loading '" + local_path + "'."); + } + } + + if (thread_load_tasks.has(local_path)) { + thread_load_tasks[local_path].requests++; + if (p_source_resource != String()) { + thread_load_tasks[p_source_resource].sub_tasks.insert(local_path); + } + thread_load_mutex->unlock(); + return OK; } - LoadingMapKey key; - key.path = p_path; - key.thread = Thread::get_caller_id(); + { + //create load task + + ThreadLoadTask load_task; + + load_task.requests = 1; + load_task.remapped_path = _path_remap(local_path, &load_task.xl_remapped); + load_task.local_path = local_path; + load_task.type_hint = p_type_hint; + load_task.use_sub_threads = p_use_sub_threads; + + { //must check if resource is already loaded before attempting to load it in a thread + + if (load_task.loader_id == Thread::get_caller_id()) { + thread_load_mutex->unlock(); + ERR_FAIL_V_MSG(ERR_INVALID_PARAMETER, "Attempted to load a resource already being loaded from this thread, cyclic reference?"); + } + //lock first if possible + if (ResourceCache::lock) { + ResourceCache::lock->read_lock(); + } + + //get ptr + Resource **rptr = ResourceCache::resources.getptr(local_path); + + if (rptr) { + RES res(*rptr); + //it is possible this resource was just freed in a thread. If so, this referencing will not work and resource is considered not cached + if (res.is_valid()) { + //referencing is fine + load_task.resource = res; + load_task.status = THREAD_LOAD_LOADED; + load_task.progress = 1.0; + } + } + if (ResourceCache::lock) { + ResourceCache::lock->read_unlock(); + } + } - loading_map.erase(key); + if (p_source_resource != String()) { + thread_load_tasks[p_source_resource].sub_tasks.insert(local_path); + } - if (loading_map_mutex) { - loading_map_mutex->unlock(); + thread_load_tasks[local_path] = load_task; } + + ThreadLoadTask &load_task = thread_load_tasks[local_path]; + + if (load_task.resource.is_null()) { //needs to be loaded in thread + + load_task.semaphore = memnew(Semaphore); + if (thread_loading_count < thread_load_max) { + thread_loading_count++; + thread_load_semaphore->post(); //we have free threads, so allow one + } else { + thread_waiting_count++; + } + + print_lt("REQUEST: load count: " + itos(thread_loading_count) + " / wait count: " + itos(thread_waiting_count) + " / suspended count: " + itos(thread_suspended_count) + " / active: " + itos(thread_loading_count - thread_suspended_count)); + + load_task.thread = Thread::create(_thread_load_function, &thread_load_tasks[local_path]); + load_task.loader_id = load_task.thread->get_id(); + } + + thread_load_mutex->unlock(); + + return OK; } -void ResourceLoader::_remove_from_loading_map_and_thread(const String &p_path, Thread::ID p_thread) { - if (loading_map_mutex) { - loading_map_mutex->lock(); +float ResourceLoader::_dependency_get_progress(const String &p_path) { + + if (thread_load_tasks.has(p_path)) { + ThreadLoadTask &load_task = thread_load_tasks[p_path]; + int dep_count = load_task.sub_tasks.size(); + if (dep_count > 0) { + float dep_progress = 0; + for (Set<String>::Element *E = load_task.sub_tasks.front(); E; E = E->next()) { + dep_progress += _dependency_get_progress(E->get()); + } + dep_progress /= float(dep_count); + dep_progress *= 0.5; + dep_progress += load_task.progress * 0.5; + return dep_progress; + } else { + return load_task.progress; + } + + } else { + return 1.0; //assume finished loading it so it no longer exists } +} - LoadingMapKey key; - key.path = p_path; - key.thread = p_thread; +ResourceLoader::ThreadLoadStatus ResourceLoader::load_threaded_get_status(const String &p_path, float *r_progress) { - loading_map.erase(key); + String local_path; + if (p_path.is_rel_path()) + local_path = "res://" + p_path; + else + local_path = ProjectSettings::get_singleton()->localize_path(p_path); - if (loading_map_mutex) { - loading_map_mutex->unlock(); + thread_load_mutex->lock(); + if (!thread_load_tasks.has(local_path)) { + thread_load_mutex->unlock(); + return THREAD_LOAD_INVALID_RESOURCE; + } + ThreadLoadTask &load_task = thread_load_tasks[local_path]; + ThreadLoadStatus status; + status = load_task.status; + if (r_progress) { + *r_progress = _dependency_get_progress(local_path); } + + thread_load_mutex->unlock(); + + return status; +} +RES ResourceLoader::load_threaded_get(const String &p_path, Error *r_error) { + + String local_path; + if (p_path.is_rel_path()) + local_path = "res://" + p_path; + else + local_path = ProjectSettings::get_singleton()->localize_path(p_path); + + thread_load_mutex->lock(); + if (!thread_load_tasks.has(local_path)) { + thread_load_mutex->unlock(); + if (r_error) { + *r_error = ERR_INVALID_PARAMETER; + } + return RES(); + } + + ThreadLoadTask &load_task = thread_load_tasks[local_path]; + + //semaphore still exists, meaning its still loading, request poll + Semaphore *semaphore = load_task.semaphore; + if (semaphore) { + load_task.poll_requests++; + + { + // As we got a semaphore, this means we are going to have to wait + // until the sub-resource is done loading + // + // As this thread will become 'blocked' we should "echange" its + // active status with a waiting one, to ensure load continues. + // + // This ensures loading is never blocked and that is also within + // the maximum number of active threads. + + if (thread_waiting_count > 0) { + thread_waiting_count--; + thread_loading_count++; + thread_load_semaphore->post(); + + load_task.start_next = false; //do not start next since we are doing it here + } + + thread_suspended_count++; + + print_lt("GET: load count: " + itos(thread_loading_count) + " / wait count: " + itos(thread_waiting_count) + " / suspended count: " + itos(thread_suspended_count) + " / active: " + itos(thread_loading_count - thread_suspended_count)); + } + + thread_load_mutex->unlock(); + semaphore->wait(); + thread_load_mutex->lock(); + + thread_suspended_count--; + + if (!thread_load_tasks.has(local_path)) { //may have been erased during unlock and this was always an invalid call + thread_load_mutex->unlock(); + if (r_error) { + *r_error = ERR_INVALID_PARAMETER; + } + return RES(); + } + } + + RES resource = load_task.resource; + if (r_error) { + *r_error = load_task.error; + } + + load_task.requests--; + + if (load_task.requests == 0) { + if (load_task.thread) { //thread may not have been used + Thread::wait_to_finish(load_task.thread); + memdelete(load_task.thread); + } + thread_load_tasks.erase(local_path); + } + + thread_load_mutex->unlock(); + + return resource; } RES ResourceLoader::load(const String &p_path, const String &p_type_hint, bool p_no_cache, Error *r_error) { @@ -355,83 +529,101 @@ RES ResourceLoader::load(const String &p_path, const String &p_type_hint, bool p if (!p_no_cache) { - { - bool success = _add_to_loading_map(local_path); - ERR_FAIL_COND_V_MSG(!success, RES(), "Resource: '" + local_path + "' is already being loaded. Cyclic reference?"); + thread_load_mutex->lock(); + + //Is it already being loaded? poll until done + if (thread_load_tasks.has(local_path)) { + Error err = load_threaded_request(p_path, p_type_hint); + if (err != OK) { + if (r_error) { + *r_error = err; + } + return RES(); + } + thread_load_mutex->unlock(); + + return load_threaded_get(p_path, r_error); } - //lock first if possible + //Is it cached? if (ResourceCache::lock) { ResourceCache::lock->read_lock(); } - //get ptr Resource **rptr = ResourceCache::resources.getptr(local_path); if (rptr) { RES res(*rptr); + //it is possible this resource was just freed in a thread. If so, this referencing will not work and resource is considered not cached if (res.is_valid()) { - //referencing is fine - if (r_error) - *r_error = OK; if (ResourceCache::lock) { ResourceCache::lock->read_unlock(); } - _remove_from_loading_map(local_path); - return res; + thread_load_mutex->unlock(); + + if (r_error) { + *r_error = OK; + } + + return res; //use cached } } + if (ResourceCache::lock) { ResourceCache::lock->read_unlock(); } - } - bool xl_remapped = false; - String path = _path_remap(local_path, &xl_remapped); + //load using task (but this thread) + ThreadLoadTask load_task; + + load_task.requests = 1; + load_task.local_path = local_path; + load_task.remapped_path = _path_remap(local_path, &load_task.xl_remapped); + load_task.type_hint = p_type_hint; + load_task.loader_id = Thread::get_caller_id(); - if (path == "") { - if (!p_no_cache) { - _remove_from_loading_map(local_path); + thread_load_tasks[local_path] = load_task; + + thread_load_mutex->unlock(); + + _thread_load_function(&thread_load_tasks[local_path]); + + return load_threaded_get(p_path, r_error); + + } else { + + bool xl_remapped = false; + String path = _path_remap(local_path, &xl_remapped); + + if (path == "") { + ERR_FAIL_V_MSG(RES(), "Remapping '" + local_path + "' failed."); } - ERR_FAIL_V_MSG(RES(), "Remapping '" + local_path + "' failed."); - } - print_verbose("Loading resource: " + path); - RES res = _load(path, local_path, p_type_hint, p_no_cache, r_error); + print_verbose("Loading resource: " + path); + float p; + RES res = _load(path, local_path, p_type_hint, p_no_cache, r_error, false, &p); - if (res.is_null()) { - if (!p_no_cache) { - _remove_from_loading_map(local_path); + if (res.is_null()) { + print_verbose("Failed loading resource: " + path); + return RES(); } - print_verbose("Failed loading resource: " + path); - return RES(); - } - if (!p_no_cache) - res->set_path(local_path); - if (xl_remapped) - res->set_as_translation_remapped(true); + if (xl_remapped) + res->set_as_translation_remapped(true); #ifdef TOOLS_ENABLED - res->set_edited(false); - if (timestamp_on_load) { - uint64_t mt = FileAccess::get_modified_time(path); - //printf("mt %s: %lli\n",remapped_path.utf8().get_data(),mt); - res->set_last_modified_time(mt); - } + res->set_edited(false); + if (timestamp_on_load) { + uint64_t mt = FileAccess::get_modified_time(path); + //printf("mt %s: %lli\n",remapped_path.utf8().get_data(),mt); + res->set_last_modified_time(mt); + } #endif - if (!p_no_cache) { - _remove_from_loading_map(local_path); - } - - if (_loaded_callback) { - _loaded_callback(res, p_path); + return res; } - - return res; } bool ResourceLoader::exists(const String &p_path, const String &p_type_hint) { @@ -464,76 +656,6 @@ bool ResourceLoader::exists(const String &p_path, const String &p_type_hint) { return false; } -Ref<ResourceInteractiveLoader> ResourceLoader::load_interactive(const String &p_path, const String &p_type_hint, bool p_no_cache, Error *r_error) { - - if (r_error) - *r_error = ERR_CANT_OPEN; - - String local_path; - if (p_path.is_rel_path()) - local_path = "res://" + p_path; - else - local_path = ProjectSettings::get_singleton()->localize_path(p_path); - - if (!p_no_cache) { - - bool success = _add_to_loading_map(local_path); - ERR_FAIL_COND_V_MSG(!success, RES(), "Resource: '" + local_path + "' is already being loaded. Cyclic reference?"); - - if (ResourceCache::has(local_path)) { - - print_verbose("Loading resource: " + local_path + " (cached)"); - Ref<Resource> res_cached = ResourceCache::get(local_path); - Ref<ResourceInteractiveLoaderDefault> ril = Ref<ResourceInteractiveLoaderDefault>(memnew(ResourceInteractiveLoaderDefault)); - - ril->resource = res_cached; - ril->path_loading = local_path; - ril->path_loading_thread = Thread::get_caller_id(); - return ril; - } - } - - bool xl_remapped = false; - String path = _path_remap(local_path, &xl_remapped); - if (path == "") { - if (!p_no_cache) { - _remove_from_loading_map(local_path); - } - ERR_FAIL_V_MSG(RES(), "Remapping '" + local_path + "' failed."); - } - - print_verbose("Loading resource: " + path); - - bool found = false; - for (int i = 0; i < loader_count; i++) { - - if (!loader[i]->recognize_path(path, p_type_hint)) - continue; - found = true; - Ref<ResourceInteractiveLoader> ril = loader[i]->load_interactive(path, local_path, r_error); - if (ril.is_null()) - continue; - if (!p_no_cache) { - ril->set_local_path(local_path); - ril->path_loading = local_path; - ril->path_loading_thread = Thread::get_caller_id(); - } - - if (xl_remapped) - ril->set_translation_remapped(true); - - return ril; - } - - if (!p_no_cache) { - _remove_from_loading_map(local_path); - } - - ERR_FAIL_COND_V_MSG(found, Ref<ResourceInteractiveLoader>(), "Failed loading resource: " + path + "."); - - ERR_FAIL_V_MSG(Ref<ResourceInteractiveLoader>(), "No loader found for resource: " + path + "."); -} - void ResourceLoader::add_resource_format_loader(Ref<ResourceFormatLoader> p_format_loader, bool p_at_front) { ERR_FAIL_COND(p_format_loader.is_null()); @@ -1002,25 +1124,19 @@ void ResourceLoader::remove_custom_loaders() { } } -Mutex *ResourceLoader::loading_map_mutex = NULL; -HashMap<ResourceLoader::LoadingMapKey, int, ResourceLoader::LoadingMapKeyHasher> ResourceLoader::loading_map; - void ResourceLoader::initialize() { -#ifndef NO_THREADS - loading_map_mutex = Mutex::create(); -#endif + thread_load_mutex = memnew(Mutex); + thread_load_max = OS::get_singleton()->get_processor_count(); + thread_loading_count = 0; + thread_waiting_count = 0; + thread_suspended_count = 0; + thread_load_semaphore = memnew(Semaphore); } void ResourceLoader::finalize() { -#ifndef NO_THREADS - const LoadingMapKey *K = NULL; - while ((K = loading_map.next(K))) { - ERR_PRINT("Exited while resource is being loaded: " + K->path); - } - loading_map.clear(); - memdelete(loading_map_mutex); - loading_map_mutex = NULL; -#endif + + memdelete(thread_load_mutex); + memdelete(thread_load_semaphore); } ResourceLoadErrorNotify ResourceLoader::err_notify = NULL; @@ -1032,6 +1148,15 @@ void *ResourceLoader::dep_err_notify_ud = NULL; bool ResourceLoader::abort_on_missing_resource = true; bool ResourceLoader::timestamp_on_load = false; +Mutex *ResourceLoader::thread_load_mutex = nullptr; +HashMap<String, ResourceLoader::ThreadLoadTask> ResourceLoader::thread_load_tasks; +Semaphore *ResourceLoader::thread_load_semaphore = nullptr; + +int ResourceLoader::thread_loading_count = 0; +int ResourceLoader::thread_waiting_count = 0; +int ResourceLoader::thread_suspended_count = 0; +int ResourceLoader::thread_load_max = 0; + SelfList<Resource>::List ResourceLoader::remapped_list; HashMap<String, Vector<String> > ResourceLoader::translation_remaps; HashMap<String, String> ResourceLoader::path_remaps; diff --git a/core/io/resource_loader.h b/core/io/resource_loader.h index 4e83427fae..3b7a27f551 100644 --- a/core/io/resource_loader.h +++ b/core/io/resource_loader.h @@ -31,32 +31,10 @@ #ifndef RESOURCE_LOADER_H #define RESOURCE_LOADER_H +#include "core/os/semaphore.h" #include "core/os/thread.h" #include "core/resource.h" -class ResourceInteractiveLoader : public Reference { - - GDCLASS(ResourceInteractiveLoader, Reference); - friend class ResourceLoader; - String path_loading; - Thread::ID path_loading_thread; - -protected: - static void _bind_methods(); - -public: - virtual void set_local_path(const String &p_local_path) = 0; - virtual Ref<Resource> get_resource() = 0; - virtual Error poll() = 0; - virtual int get_stage() const = 0; - virtual int get_stage_count() const = 0; - virtual void set_translation_remapped(bool p_remapped) = 0; - virtual Error wait(); - - ResourceInteractiveLoader() {} - ~ResourceInteractiveLoader(); -}; - class ResourceFormatLoader : public Reference { GDCLASS(ResourceFormatLoader, Reference); @@ -65,8 +43,7 @@ protected: static void _bind_methods(); public: - virtual Ref<ResourceInteractiveLoader> load_interactive(const String &p_path, const String &p_original_path = "", Error *r_error = NULL); - virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL); + virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL, bool p_use_sub_threads = false, float *r_progress = nullptr); virtual bool exists(const String &p_path) const; virtual void get_recognized_extensions(List<String> *p_extensions) const; virtual void get_recognized_extensions_for_type(const String &p_type, List<String> *p_extensions) const; @@ -95,6 +72,15 @@ class ResourceLoader { MAX_LOADERS = 64 }; +public: + enum ThreadLoadStatus { + THREAD_LOAD_INVALID_RESOURCE, + THREAD_LOAD_IN_PROGRESS, + THREAD_LOAD_FAILED, + THREAD_LOAD_LOADED + }; + +private: static Ref<ResourceFormatLoader> loader[MAX_LOADERS]; static int loader_count; static bool timestamp_on_load; @@ -115,34 +101,47 @@ class ResourceLoader { friend class ResourceFormatImporter; friend class ResourceInteractiveLoader; //internal load function - static RES _load(const String &p_path, const String &p_original_path, const String &p_type_hint, bool p_no_cache, Error *r_error); + static RES _load(const String &p_path, const String &p_original_path, const String &p_type_hint, bool p_no_cache, Error *r_error, bool p_use_sub_threads, float *r_progress); static ResourceLoadedCallback _loaded_callback; static Ref<ResourceFormatLoader> _find_custom_resource_format_loader(String path); - static Mutex *loading_map_mutex; - - //used to track paths being loaded in a thread, avoids cyclic recursion - struct LoadingMapKey { - String path; - Thread::ID thread; - bool operator==(const LoadingMapKey &p_key) const { - return (thread == p_key.thread && path == p_key.path); - } - }; - struct LoadingMapKeyHasher { - static _FORCE_INLINE_ uint32_t hash(const LoadingMapKey &p_key) { return p_key.path.hash() + HashMapHasherDefault::hash(p_key.thread); } + struct ThreadLoadTask { + Thread *thread = nullptr; + Thread::ID loader_id = 0; + Semaphore *semaphore = nullptr; + String local_path; + String remapped_path; + String type_hint; + float progress = 0.0; + ThreadLoadStatus status = THREAD_LOAD_IN_PROGRESS; + Error error; + RES resource; + bool xl_remapped = false; + bool use_sub_threads = false; + bool start_next = true; + int requests = 0; + int poll_requests = 0; + Set<String> sub_tasks; }; - static HashMap<LoadingMapKey, int, LoadingMapKeyHasher> loading_map; + static void _thread_load_function(void *p_userdata); + static Mutex *thread_load_mutex; + static HashMap<String, ThreadLoadTask> thread_load_tasks; + static Semaphore *thread_load_semaphore; + static int thread_waiting_count; + static int thread_loading_count; + static int thread_suspended_count; + static int thread_load_max; - static bool _add_to_loading_map(const String &p_path); - static void _remove_from_loading_map(const String &p_path); - static void _remove_from_loading_map_and_thread(const String &p_path, Thread::ID p_thread); + static float _dependency_get_progress(const String &p_path); public: - static Ref<ResourceInteractiveLoader> load_interactive(const String &p_path, const String &p_type_hint = "", bool p_no_cache = false, Error *r_error = NULL); + static Error load_threaded_request(const String &p_path, const String &p_type_hint = "", bool p_use_sub_threads = false, const String &p_source_resource = String()); + static ThreadLoadStatus load_threaded_get_status(const String &p_path, float *r_progress = nullptr); + static RES load_threaded_get(const String &p_path, Error *r_error = NULL); + static RES load(const String &p_path, const String &p_type_hint = "", bool p_no_cache = false, Error *r_error = NULL); static bool exists(const String &p_path, const String &p_type_hint = ""); diff --git a/core/io/translation_loader_po.cpp b/core/io/translation_loader_po.cpp index 4f7eeddc43..4051bf2947 100644 --- a/core/io/translation_loader_po.cpp +++ b/core/io/translation_loader_po.cpp @@ -176,7 +176,7 @@ RES TranslationLoaderPO::load_translation(FileAccess *f, Error *r_error, const S return translation; } -RES TranslationLoaderPO::load(const String &p_path, const String &p_original_path, Error *r_error) { +RES TranslationLoaderPO::load(const String &p_path, const String &p_original_path, Error *r_error, bool p_use_sub_threads, float *r_progress) { if (r_error) *r_error = ERR_CANT_OPEN; diff --git a/core/io/translation_loader_po.h b/core/io/translation_loader_po.h index 47e64276ca..fe3a75e5eb 100644 --- a/core/io/translation_loader_po.h +++ b/core/io/translation_loader_po.h @@ -38,7 +38,7 @@ class TranslationLoaderPO : public ResourceFormatLoader { public: static RES load_translation(FileAccess *f, Error *r_error, const String &p_path = String()); - virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL); + virtual RES load(const String &p_path, const String &p_original_path = "", Error *r_error = NULL, bool p_use_sub_threads = false, float *r_progress = nullptr); virtual void get_recognized_extensions(List<String> *p_extensions) const; virtual bool handles_type(const String &p_type) const; virtual String get_resource_type(const String &p_path) const; |