diff options
Diffstat (limited to 'core/string')
| -rw-r--r-- | core/string/node_path.cpp | 21 | ||||
| -rw-r--r-- | core/string/node_path.h | 6 | ||||
| -rw-r--r-- | core/string/optimized_translation.cpp (renamed from core/string/compressed_translation.cpp) | 58 | ||||
| -rw-r--r-- | core/string/optimized_translation.h (renamed from core/string/compressed_translation.h) | 18 | ||||
| -rw-r--r-- | core/string/print_string.cpp | 10 | ||||
| -rw-r--r-- | core/string/print_string.h | 16 | ||||
| -rw-r--r-- | core/string/string_buffer.h | 10 | ||||
| -rw-r--r-- | core/string/string_builder.cpp | 4 | ||||
| -rw-r--r-- | core/string/string_builder.h | 4 | ||||
| -rw-r--r-- | core/string/string_name.cpp | 112 | ||||
| -rw-r--r-- | core/string/string_name.h | 42 | ||||
| -rw-r--r-- | core/string/translation.cpp | 346 | ||||
| -rw-r--r-- | core/string/translation.h | 36 | ||||
| -rw-r--r-- | core/string/translation_po.cpp | 47 | ||||
| -rw-r--r-- | core/string/translation_po.h | 4 | ||||
| -rw-r--r-- | core/string/ucaps.h | 4 | ||||
| -rw-r--r-- | core/string/ustring.cpp | 846 | ||||
| -rw-r--r-- | core/string/ustring.h | 56 |
18 files changed, 1057 insertions, 583 deletions
diff --git a/core/string/node_path.cpp b/core/string/node_path.cpp index a63dde5b41..5fae13779e 100644 --- a/core/string/node_path.cpp +++ b/core/string/node_path.cpp @@ -5,8 +5,8 @@ /* GODOT ENGINE */ /* https://godotengine.org */ /*************************************************************************/ -/* Copyright (c) 2007-2020 Juan Linietsky, Ariel Manzur. */ -/* Copyright (c) 2014-2020 Godot Engine contributors (cf. AUTHORS.md). */ +/* Copyright (c) 2007-2021 Juan Linietsky, Ariel Manzur. */ +/* Copyright (c) 2014-2021 Godot Engine contributors (cf. AUTHORS.md). */ /* */ /* Permission is hereby granted, free of charge, to any person obtaining */ /* a copy of this software and associated documentation files (the */ @@ -240,19 +240,26 @@ NodePath NodePath::rel_path_to(const NodePath &p_np) const { common_parent--; Vector<StringName> relpath; + relpath.resize(src_dirs.size() + dst_dirs.size() + 1); - for (int i = src_dirs.size() - 1; i > common_parent; i--) { - relpath.push_back(".."); + StringName *relpath_ptr = relpath.ptrw(); + + int path_size = 0; + StringName back_str(".."); + for (int i = common_parent + 1; i < src_dirs.size(); i++) { + relpath_ptr[path_size++] = back_str; } for (int i = common_parent + 1; i < dst_dirs.size(); i++) { - relpath.push_back(dst_dirs[i]); + relpath_ptr[path_size++] = dst_dirs[i]; } - if (relpath.size() == 0) { - relpath.push_back("."); + if (path_size == 0) { + relpath_ptr[path_size++] = "."; } + relpath.resize(path_size); + return NodePath(relpath, p_np.get_subnames(), false); } diff --git a/core/string/node_path.h b/core/string/node_path.h index b4513ddb3c..a277ab26fa 100644 --- a/core/string/node_path.h +++ b/core/string/node_path.h @@ -5,8 +5,8 @@ /* GODOT ENGINE */ /* https://godotengine.org */ /*************************************************************************/ -/* Copyright (c) 2007-2020 Juan Linietsky, Ariel Manzur. */ -/* Copyright (c) 2014-2020 Godot Engine contributors (cf. AUTHORS.md). */ +/* Copyright (c) 2007-2021 Juan Linietsky, Ariel Manzur. */ +/* Copyright (c) 2014-2021 Godot Engine contributors (cf. AUTHORS.md). */ /* */ /* Permission is hereby granted, free of charge, to any person obtaining */ /* a copy of this software and associated documentation files (the */ @@ -66,8 +66,6 @@ public: void prepend_period(); - NodePath get_parent() const; - _FORCE_INLINE_ uint32_t hash() const { if (!data) { return 0; diff --git a/core/string/compressed_translation.cpp b/core/string/optimized_translation.cpp index bdb296a79b..f8be564740 100644 --- a/core/string/compressed_translation.cpp +++ b/core/string/optimized_translation.cpp @@ -1,12 +1,12 @@ /*************************************************************************/ -/* compressed_translation.cpp */ +/* optimized_translation.cpp */ /*************************************************************************/ /* This file is part of: */ /* GODOT ENGINE */ /* https://godotengine.org */ /*************************************************************************/ -/* Copyright (c) 2007-2020 Juan Linietsky, Ariel Manzur. */ -/* Copyright (c) 2014-2020 Godot Engine contributors (cf. AUTHORS.md). */ +/* Copyright (c) 2007-2021 Juan Linietsky, Ariel Manzur. */ +/* Copyright (c) 2014-2021 Godot Engine contributors (cf. AUTHORS.md). */ /* */ /* Permission is hereby granted, free of charge, to any person obtaining */ /* a copy of this software and associated documentation files (the */ @@ -28,7 +28,7 @@ /* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ /*************************************************************************/ -#include "compressed_translation.h" +#include "optimized_translation.h" #include "core/templates/pair.h" @@ -36,16 +36,17 @@ extern "C" { #include "thirdparty/misc/smaz.h" } -struct _PHashTranslationCmp { +struct CompressedString { int orig_len; CharString compressed; int offset; }; -void PHashTranslation::generate(const Ref<Translation> &p_from) { +void OptimizedTranslation::generate(const Ref<Translation> &p_from) { // This method compresses a Translation instance. - // Right now it doesn't handle context or plurals, so Translation subclasses using plurals or context (i.e TranslationPO) shouldn't be compressed. + // Right now, it doesn't handle context or plurals, so Translation subclasses using plurals or context (i.e TranslationPO) shouldn't be compressed. #ifdef TOOLS_ENABLED + ERR_FAIL_COND(p_from.is_null()); List<StringName> keys; p_from->get_message_list(&keys); @@ -54,7 +55,7 @@ void PHashTranslation::generate(const Ref<Translation> &p_from) { Vector<Vector<Pair<int, CharString>>> buckets; Vector<Map<uint32_t, int>> table; Vector<uint32_t> hfunc_table; - Vector<_PHashTranslationCmp> compressed; + Vector<CompressedString> compressed; table.resize(size); hfunc_table.resize(size); @@ -63,11 +64,10 @@ void PHashTranslation::generate(const Ref<Translation> &p_from) { int idx = 0; int total_compression_size = 0; - int total_string_size = 0; - for (List<StringName>::Element *E = keys.front(); E; E = E->next()) { + for (const StringName &E : keys) { //hash string - CharString cs = E->get().operator String().utf8(); + CharString cs = E.operator String().utf8(); uint32_t h = hash(0, cs.get_data()); Pair<int, CharString> p; p.first = idx; @@ -75,8 +75,8 @@ void PHashTranslation::generate(const Ref<Translation> &p_from) { buckets.write[h % size].push_back(p); //compress string - CharString src_s = p_from->get_message(E->get()).operator String().utf8(); - _PHashTranslationCmp ps; + CharString src_s = p_from->get_message(E).operator String().utf8(); + CompressedString ps; ps.orig_len = src_s.size(); ps.offset = total_compression_size; @@ -101,7 +101,6 @@ void PHashTranslation::generate(const Ref<Translation> &p_from) { compressed.write[idx] = ps; total_compression_size += ps.compressed.size(); - total_string_size += src_s.size(); idx++; } @@ -146,26 +145,23 @@ void PHashTranslation::generate(const Ref<Translation> &p_from) { uint32_t *btw = (uint32_t *)&btwb[0]; int btindex = 0; - int collisions = 0; for (int i = 0; i < size; i++) { const Map<uint32_t, int> &t = table[i]; if (t.size() == 0) { htw[i] = 0xFFFFFFFF; //nothing continue; - } else if (t.size() > 1) { - collisions += t.size() - 1; } htw[i] = btindex; btw[btindex++] = t.size(); btw[btindex++] = hfunc_table[i]; - for (Map<uint32_t, int>::Element *E = t.front(); E; E = E->next()) { - btw[btindex++] = E->key(); - btw[btindex++] = compressed[E->get()].offset; - btw[btindex++] = compressed[E->get()].compressed.size(); - btw[btindex++] = compressed[E->get()].orig_len; + for (const KeyValue<uint32_t, int> &E : t) { + btw[btindex++] = E.key; + btw[btindex++] = compressed[E.value].offset; + btw[btindex++] = compressed[E.value].compressed.size(); + btw[btindex++] = compressed[E.value].orig_len; } } @@ -182,7 +178,7 @@ void PHashTranslation::generate(const Ref<Translation> &p_from) { #endif } -bool PHashTranslation::_set(const StringName &p_name, const Variant &p_value) { +bool OptimizedTranslation::_set(const StringName &p_name, const Variant &p_value) { String name = p_name.operator String(); if (name == "hash_table") { hash_table = p_value; @@ -199,7 +195,7 @@ bool PHashTranslation::_set(const StringName &p_name, const Variant &p_value) { return true; } -bool PHashTranslation::_get(const StringName &p_name, Variant &r_ret) const { +bool OptimizedTranslation::_get(const StringName &p_name, Variant &r_ret) const { String name = p_name.operator String(); if (name == "hash_table") { r_ret = hash_table; @@ -214,8 +210,8 @@ bool PHashTranslation::_get(const StringName &p_name, Variant &r_ret) const { return true; } -StringName PHashTranslation::get_message(const StringName &p_src_text, const StringName &p_context) const { - // p_context passed in is ignore. The use of context is not yet supported in PHashTranslation. +StringName OptimizedTranslation::get_message(const StringName &p_src_text, const StringName &p_context) const { + // p_context passed in is ignore. The use of context is not yet supported in OptimizedTranslation. int htsize = hash_table.size(); @@ -271,18 +267,18 @@ StringName PHashTranslation::get_message(const StringName &p_src_text, const Str } } -StringName PHashTranslation::get_plural_message(const StringName &p_src_text, const StringName &p_plural_text, int p_n, const StringName &p_context) const { - // The use of plurals translation is not yet supported in PHashTranslation. +StringName OptimizedTranslation::get_plural_message(const StringName &p_src_text, const StringName &p_plural_text, int p_n, const StringName &p_context) const { + // The use of plurals translation is not yet supported in OptimizedTranslation. return get_message(p_src_text, p_context); } -void PHashTranslation::_get_property_list(List<PropertyInfo> *p_list) const { +void OptimizedTranslation::_get_property_list(List<PropertyInfo> *p_list) const { p_list->push_back(PropertyInfo(Variant::PACKED_INT32_ARRAY, "hash_table")); p_list->push_back(PropertyInfo(Variant::PACKED_INT32_ARRAY, "bucket_table")); p_list->push_back(PropertyInfo(Variant::PACKED_BYTE_ARRAY, "strings")); p_list->push_back(PropertyInfo(Variant::OBJECT, "load_from", PROPERTY_HINT_RESOURCE_TYPE, "Translation", PROPERTY_USAGE_EDITOR)); } -void PHashTranslation::_bind_methods() { - ClassDB::bind_method(D_METHOD("generate", "from"), &PHashTranslation::generate); +void OptimizedTranslation::_bind_methods() { + ClassDB::bind_method(D_METHOD("generate", "from"), &OptimizedTranslation::generate); } diff --git a/core/string/compressed_translation.h b/core/string/optimized_translation.h index efb3535362..bccf932383 100644 --- a/core/string/compressed_translation.h +++ b/core/string/optimized_translation.h @@ -1,12 +1,12 @@ /*************************************************************************/ -/* compressed_translation.h */ +/* optimized_translation.h */ /*************************************************************************/ /* This file is part of: */ /* GODOT ENGINE */ /* https://godotengine.org */ /*************************************************************************/ -/* Copyright (c) 2007-2020 Juan Linietsky, Ariel Manzur. */ -/* Copyright (c) 2014-2020 Godot Engine contributors (cf. AUTHORS.md). */ +/* Copyright (c) 2007-2021 Juan Linietsky, Ariel Manzur. */ +/* Copyright (c) 2014-2021 Godot Engine contributors (cf. AUTHORS.md). */ /* */ /* Permission is hereby granted, free of charge, to any person obtaining */ /* a copy of this software and associated documentation files (the */ @@ -28,13 +28,13 @@ /* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ /*************************************************************************/ -#ifndef COMPRESSED_TRANSLATION_H -#define COMPRESSED_TRANSLATION_H +#ifndef OPTIMIZED_TRANSLATION_H +#define OPTIMIZED_TRANSLATION_H #include "core/string/translation.h" -class PHashTranslation : public Translation { - GDCLASS(PHashTranslation, Translation); +class OptimizedTranslation : public Translation { + GDCLASS(OptimizedTranslation, Translation); //this translation uses a sort of modified perfect hash algorithm //it requires hashing strings twice and then does a binary search, @@ -83,7 +83,7 @@ public: virtual StringName get_plural_message(const StringName &p_src_text, const StringName &p_plural_text, int p_n, const StringName &p_context = "") const override; void generate(const Ref<Translation> &p_from); - PHashTranslation() {} + OptimizedTranslation() {} }; -#endif // COMPRESSED_TRANSLATION_H +#endif // OPTIMIZED_TRANSLATION_H diff --git a/core/string/print_string.cpp b/core/string/print_string.cpp index 54de229471..adc218f597 100644 --- a/core/string/print_string.cpp +++ b/core/string/print_string.cpp @@ -5,8 +5,8 @@ /* GODOT ENGINE */ /* https://godotengine.org */ /*************************************************************************/ -/* Copyright (c) 2007-2020 Juan Linietsky, Ariel Manzur. */ -/* Copyright (c) 2014-2020 Godot Engine contributors (cf. AUTHORS.md). */ +/* Copyright (c) 2007-2021 Juan Linietsky, Ariel Manzur. */ +/* Copyright (c) 2014-2021 Godot Engine contributors (cf. AUTHORS.md). */ /* */ /* Permission is hereby granted, free of charge, to any person obtaining */ /* a copy of this software and associated documentation files (the */ @@ -69,7 +69,7 @@ void remove_print_handler(PrintHandlerList *p_handler) { ERR_FAIL_COND(l == nullptr); } -void print_line(String p_string) { +void __print_line(String p_string) { if (!_print_line_enabled) { return; } @@ -108,3 +108,7 @@ void print_verbose(String p_string) { print_line(p_string); } } + +String stringify_variants(Variant p_var) { + return p_var.operator String(); +} diff --git a/core/string/print_string.h b/core/string/print_string.h index 3e8f244cc5..3cd170b68e 100644 --- a/core/string/print_string.h +++ b/core/string/print_string.h @@ -5,8 +5,8 @@ /* GODOT ENGINE */ /* https://godotengine.org */ /*************************************************************************/ -/* Copyright (c) 2007-2020 Juan Linietsky, Ariel Manzur. */ -/* Copyright (c) 2014-2020 Godot Engine contributors (cf. AUTHORS.md). */ +/* Copyright (c) 2007-2021 Juan Linietsky, Ariel Manzur. */ +/* Copyright (c) 2014-2021 Godot Engine contributors (cf. AUTHORS.md). */ /* */ /* Permission is hereby granted, free of charge, to any person obtaining */ /* a copy of this software and associated documentation files (the */ @@ -31,7 +31,7 @@ #ifndef PRINT_STRING_H #define PRINT_STRING_H -#include "core/string/ustring.h" +#include "core/variant/variant.h" extern void (*_print_func)(String); @@ -46,13 +46,21 @@ struct PrintHandlerList { PrintHandlerList() {} }; +String stringify_variants(Variant p_var); + +template <typename... Args> +String stringify_variants(Variant p_var, Args... p_args) { + return p_var.operator String() + " " + stringify_variants(p_args...); +} + void add_print_handler(PrintHandlerList *p_handler); void remove_print_handler(PrintHandlerList *p_handler); extern bool _print_line_enabled; extern bool _print_error_enabled; -extern void print_line(String p_string); +extern void __print_line(String p_string); extern void print_error(String p_string); extern void print_verbose(String p_string); +#define print_line(...) __print_line(stringify_variants(__VA_ARGS__)) #endif // PRINT_STRING_H diff --git a/core/string/string_buffer.h b/core/string/string_buffer.h index 1317b538d4..33897c3674 100644 --- a/core/string/string_buffer.h +++ b/core/string/string_buffer.h @@ -5,8 +5,8 @@ /* GODOT ENGINE */ /* https://godotengine.org */ /*************************************************************************/ -/* Copyright (c) 2007-2020 Juan Linietsky, Ariel Manzur. */ -/* Copyright (c) 2014-2020 Godot Engine contributors (cf. AUTHORS.md). */ +/* Copyright (c) 2007-2021 Juan Linietsky, Ariel Manzur. */ +/* Copyright (c) 2014-2021 Godot Engine contributors (cf. AUTHORS.md). */ /* */ /* Permission is hereby granted, free of charge, to any person obtaining */ /* a copy of this software and associated documentation files (the */ @@ -40,7 +40,7 @@ class StringBuffer { int string_length = 0; _FORCE_INLINE_ char32_t *current_buffer_ptr() { - return static_cast<String &>(buffer).empty() ? short_buffer : buffer.ptrw(); + return static_cast<String &>(buffer).is_empty() ? short_buffer : buffer.ptrw(); } public: @@ -122,7 +122,7 @@ StringBuffer<SHORT_BUFFER_SIZE> &StringBuffer<SHORT_BUFFER_SIZE>::reserve(int p_ return *this; } - bool need_copy = string_length > 0 && buffer.empty(); + bool need_copy = string_length > 0 && buffer.is_empty(); buffer.resize(next_power_of_2(p_size)); if (need_copy) { memcpy(buffer.ptrw(), short_buffer, string_length * sizeof(char32_t)); @@ -139,7 +139,7 @@ int StringBuffer<SHORT_BUFFER_SIZE>::length() const { template <int SHORT_BUFFER_SIZE> String StringBuffer<SHORT_BUFFER_SIZE>::as_string() { current_buffer_ptr()[string_length] = '\0'; - if (buffer.empty()) { + if (buffer.is_empty()) { return String(short_buffer); } else { buffer.resize(string_length + 1); diff --git a/core/string/string_builder.cpp b/core/string/string_builder.cpp index dec299ffa3..834c87c845 100644 --- a/core/string/string_builder.cpp +++ b/core/string/string_builder.cpp @@ -5,8 +5,8 @@ /* GODOT ENGINE */ /* https://godotengine.org */ /*************************************************************************/ -/* Copyright (c) 2007-2020 Juan Linietsky, Ariel Manzur. */ -/* Copyright (c) 2014-2020 Godot Engine contributors (cf. AUTHORS.md). */ +/* Copyright (c) 2007-2021 Juan Linietsky, Ariel Manzur. */ +/* Copyright (c) 2014-2021 Godot Engine contributors (cf. AUTHORS.md). */ /* */ /* Permission is hereby granted, free of charge, to any person obtaining */ /* a copy of this software and associated documentation files (the */ diff --git a/core/string/string_builder.h b/core/string/string_builder.h index c732f1b9ea..30ce2a06f7 100644 --- a/core/string/string_builder.h +++ b/core/string/string_builder.h @@ -5,8 +5,8 @@ /* GODOT ENGINE */ /* https://godotengine.org */ /*************************************************************************/ -/* Copyright (c) 2007-2020 Juan Linietsky, Ariel Manzur. */ -/* Copyright (c) 2014-2020 Godot Engine contributors (cf. AUTHORS.md). */ +/* Copyright (c) 2007-2021 Juan Linietsky, Ariel Manzur. */ +/* Copyright (c) 2014-2021 Godot Engine contributors (cf. AUTHORS.md). */ /* */ /* Permission is hereby granted, free of charge, to any person obtaining */ /* a copy of this software and associated documentation files (the */ diff --git a/core/string/string_name.cpp b/core/string/string_name.cpp index 34afdaee38..9024f60dae 100644 --- a/core/string/string_name.cpp +++ b/core/string/string_name.cpp @@ -5,8 +5,8 @@ /* GODOT ENGINE */ /* https://godotengine.org */ /*************************************************************************/ -/* Copyright (c) 2007-2020 Juan Linietsky, Ariel Manzur. */ -/* Copyright (c) 2014-2020 Godot Engine contributors (cf. AUTHORS.md). */ +/* Copyright (c) 2007-2021 Juan Linietsky, Ariel Manzur. */ +/* Copyright (c) 2014-2021 Godot Engine contributors (cf. AUTHORS.md). */ /* */ /* Permission is hereby granted, free of charge, to any person obtaining */ /* a copy of this software and associated documentation files (the */ @@ -41,13 +41,17 @@ StaticCString StaticCString::create(const char *p_ptr) { StringName::_Data *StringName::_table[STRING_TABLE_LEN]; -StringName _scs_create(const char *p_chr) { - return (p_chr[0] ? StringName(StaticCString::create(p_chr)) : StringName()); +StringName _scs_create(const char *p_chr, bool p_static) { + return (p_chr[0] ? StringName(StaticCString::create(p_chr), p_static) : StringName()); } bool StringName::configured = false; Mutex StringName::mutex; +#ifdef DEBUG_ENABLED +bool StringName::debug_stringname = false; +#endif + void StringName::setup() { ERR_FAIL_COND(configured); for (int i = 0; i < STRING_TABLE_LEN; i++) { @@ -59,12 +63,29 @@ void StringName::setup() { void StringName::cleanup() { MutexLock lock(mutex); +#ifdef DEBUG_ENABLED + if (unlikely(debug_stringname)) { + Vector<_Data *> data; + for (int i = 0; i < STRING_TABLE_LEN; i++) { + _Data *d = _table[i]; + while (d) { + data.push_back(d); + d = d->next; + } + } + print_line("\nStringName Reference Ranking:\n"); + data.sort_custom<DebugSortReferences>(); + for (int i = 0; i < MIN(100, data.size()); i++) { + print_line(itos(i + 1) + ": " + data[i]->get_name() + " - " + itos(data[i]->debug_references)); + } + } +#endif int lost_strings = 0; for (int i = 0; i < STRING_TABLE_LEN; i++) { while (_table[i]) { _Data *d = _table[i]; lost_strings++; - if (OS::get_singleton()->is_stdout_verbose()) { + if (d->static_count.get() != d->refcount.get() && OS::get_singleton()->is_stdout_verbose()) { if (d->cname) { print_line("Orphan StringName: " + String(d->cname)); } else { @@ -79,6 +100,7 @@ void StringName::cleanup() { if (lost_strings) { print_verbose("StringName: " + itos(lost_strings) + " unclaimed string names at exit."); } + configured = false; } void StringName::unref() { @@ -87,6 +109,13 @@ void StringName::unref() { if (_data && _data->refcount.unref()) { MutexLock lock(mutex); + if (_data->static_count.get() > 0) { + if (_data->cname) { + ERR_PRINT("BUG: Unreferenced static string to 0: " + String(_data->cname)); + } else { + ERR_PRINT("BUG: Unreferenced static string to 0: " + String(_data->name)); + } + } if (_data->prev) { _data->prev->next = _data->next; } else { @@ -153,7 +182,7 @@ StringName::StringName(const StringName &p_name) { } } -StringName::StringName(const char *p_name) { +StringName::StringName(const char *p_name, bool p_static) { _data = nullptr; ERR_FAIL_COND(!configured); @@ -181,25 +210,42 @@ StringName::StringName(const char *p_name) { if (_data) { if (_data->refcount.ref()) { // exists - return; + if (p_static) { + _data->static_count.increment(); + } +#ifdef DEBUG_ENABLED + if (unlikely(debug_stringname)) { + _data->debug_references++; + } +#endif } + + return; } _data = memnew(_Data); _data->name = p_name; _data->refcount.init(); + _data->static_count.set(p_static ? 1 : 0); _data->hash = hash; _data->idx = idx; _data->cname = nullptr; _data->next = _table[idx]; _data->prev = nullptr; +#ifdef DEBUG_ENABLED + if (unlikely(debug_stringname)) { + // Keep in memory, force static. + _data->refcount.ref(); + _data->static_count.increment(); + } +#endif if (_table[idx]) { _table[idx]->prev = _data; } _table[idx] = _data; } -StringName::StringName(const StaticCString &p_static_string) { +StringName::StringName(const StaticCString &p_static_string, bool p_static) { _data = nullptr; ERR_FAIL_COND(!configured); @@ -225,6 +271,14 @@ StringName::StringName(const StaticCString &p_static_string) { if (_data) { if (_data->refcount.ref()) { // exists + if (p_static) { + _data->static_count.increment(); + } +#ifdef DEBUG_ENABLED + if (unlikely(debug_stringname)) { + _data->debug_references++; + } +#endif return; } } @@ -232,18 +286,26 @@ StringName::StringName(const StaticCString &p_static_string) { _data = memnew(_Data); _data->refcount.init(); + _data->static_count.set(p_static ? 1 : 0); _data->hash = hash; _data->idx = idx; _data->cname = p_static_string.ptr; _data->next = _table[idx]; _data->prev = nullptr; +#ifdef DEBUG_ENABLED + if (unlikely(debug_stringname)) { + // Keep in memory, force static. + _data->refcount.ref(); + _data->static_count.increment(); + } +#endif if (_table[idx]) { _table[idx]->prev = _data; } _table[idx] = _data; } -StringName::StringName(const String &p_name) { +StringName::StringName(const String &p_name, bool p_static) { _data = nullptr; ERR_FAIL_COND(!configured); @@ -269,6 +331,14 @@ StringName::StringName(const String &p_name) { if (_data) { if (_data->refcount.ref()) { // exists + if (p_static) { + _data->static_count.increment(); + } +#ifdef DEBUG_ENABLED + if (unlikely(debug_stringname)) { + _data->debug_references++; + } +#endif return; } } @@ -276,11 +346,20 @@ StringName::StringName(const String &p_name) { _data = memnew(_Data); _data->name = p_name; _data->refcount.init(); + _data->static_count.set(p_static ? 1 : 0); _data->hash = hash; _data->idx = idx; _data->cname = nullptr; _data->next = _table[idx]; _data->prev = nullptr; +#ifdef DEBUG_ENABLED + if (unlikely(debug_stringname)) { + // Keep in memory, force static. + _data->refcount.ref(); + _data->static_count.increment(); + } +#endif + if (_table[idx]) { _table[idx]->prev = _data; } @@ -311,6 +390,12 @@ StringName StringName::search(const char *p_name) { } if (_data && _data->refcount.ref()) { +#ifdef DEBUG_ENABLED + if (unlikely(debug_stringname)) { + _data->debug_references++; + } +#endif + return StringName(_data); } @@ -368,16 +453,17 @@ StringName StringName::search(const String &p_name) { } if (_data && _data->refcount.ref()) { +#ifdef DEBUG_ENABLED + if (unlikely(debug_stringname)) { + _data->debug_references++; + } +#endif return StringName(_data); } return StringName(); //does not exist } -StringName::~StringName() { - unref(); -} - bool operator==(const String &p_name, const StringName &p_string_name) { return p_name == p_string_name.operator String(); } diff --git a/core/string/string_name.h b/core/string/string_name.h index 320f63bf68..ce7988744b 100644 --- a/core/string/string_name.h +++ b/core/string/string_name.h @@ -5,8 +5,8 @@ /* GODOT ENGINE */ /* https://godotengine.org */ /*************************************************************************/ -/* Copyright (c) 2007-2020 Juan Linietsky, Ariel Manzur. */ -/* Copyright (c) 2014-2020 Godot Engine contributors (cf. AUTHORS.md). */ +/* Copyright (c) 2007-2021 Juan Linietsky, Ariel Manzur. */ +/* Copyright (c) 2014-2021 Godot Engine contributors (cf. AUTHORS.md). */ /* */ /* Permission is hereby granted, free of charge, to any person obtaining */ /* a copy of this software and associated documentation files (the */ @@ -44,16 +44,19 @@ struct StaticCString { class StringName { enum { - STRING_TABLE_BITS = 12, + STRING_TABLE_BITS = 16, STRING_TABLE_LEN = 1 << STRING_TABLE_BITS, STRING_TABLE_MASK = STRING_TABLE_LEN - 1 }; struct _Data { SafeRefCount refcount; + SafeNumeric<uint32_t> static_count; const char *cname = nullptr; String name; - +#ifdef DEBUG_ENABLED + uint32_t debug_references = 0; +#endif String get_name() const { return cname ? String(cname) : name; } int idx = 0; uint32_t hash = 0; @@ -79,11 +82,20 @@ class StringName { static void setup(); static void cleanup(); static bool configured; +#ifdef DEBUG_ENABLED + struct DebugSortReferences { + bool operator()(const _Data *p_left, const _Data *p_right) const { + return p_left->debug_references > p_right->debug_references; + } + }; + + static bool debug_stringname; +#endif StringName(_Data *p_data) { _data = p_data; } public: - operator const void *() const { return (_data && (_data->cname || !_data->name.empty())) ? (void *)1 : nullptr; } + operator const void *() const { return (_data && (_data->cname || !_data->name.is_empty())) ? (void *)1 : nullptr; } bool operator==(const String &p_name) const; bool operator==(const char *p_name) const; @@ -146,12 +158,20 @@ public: }; void operator=(const StringName &p_name); - StringName(const char *p_name); + StringName(const char *p_name, bool p_static = false); StringName(const StringName &p_name); - StringName(const String &p_name); - StringName(const StaticCString &p_static_string); + StringName(const String &p_name, bool p_static = false); + StringName(const StaticCString &p_static_string, bool p_static = false); StringName() {} - ~StringName(); + _FORCE_INLINE_ ~StringName() { + if (likely(configured) && _data) { //only free if configured + unref(); + } + } + +#ifdef DEBUG_ENABLED + static void set_debug_stringnames(bool p_enable) { debug_stringname = p_enable; } +#endif }; bool operator==(const String &p_name, const StringName &p_string_name); @@ -159,6 +179,8 @@ bool operator!=(const String &p_name, const StringName &p_string_name); bool operator==(const char *p_name, const StringName &p_string_name); bool operator!=(const char *p_name, const StringName &p_string_name); -StringName _scs_create(const char *p_chr); +StringName _scs_create(const char *p_chr, bool p_static = false); + +#define SNAME(m_arg) ([]() -> const StringName & { static StringName sname = _scs_create(m_arg, true); return sname; })() #endif // STRING_NAME_H diff --git a/core/string/translation.cpp b/core/string/translation.cpp index 7b8c28e2e2..b98aad9b58 100644 --- a/core/string/translation.cpp +++ b/core/string/translation.cpp @@ -5,8 +5,8 @@ /* GODOT ENGINE */ /* https://godotengine.org */ /*************************************************************************/ -/* Copyright (c) 2007-2020 Juan Linietsky, Ariel Manzur. */ -/* Copyright (c) 2014-2020 Godot Engine contributors (cf. AUTHORS.md). */ +/* Copyright (c) 2007-2021 Juan Linietsky, Ariel Manzur. */ +/* Copyright (c) 2014-2021 Godot Engine contributors (cf. AUTHORS.md). */ /* */ /* Permission is hereby granted, free of charge, to any person obtaining */ /* a copy of this software and associated documentation files (the */ @@ -35,17 +35,17 @@ #include "core/os/os.h" #ifdef TOOLS_ENABLED -#include "editor/editor_settings.h" #include "main/main.h" #endif -// ISO 639-1 language codes, with the addition of glibc locales with their -// regional identifiers. This list must match the language names (in English) -// of locale_names. +// ISO 639-1 language codes (and a couple of three-letter ISO 639-2 codes), +// with the addition of glibc locales with their regional identifiers. +// This list must match the language names (in English) of locale_names. // // References: // - https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes // - https://lh.2xlibre.net/locales/ +// - https://iso639-3.sil.org/ static const char *locale_list[] = { "aa", // Afar @@ -83,6 +83,7 @@ static const char *locale_list[] = { "ast_ES", // Asturian (Spain) "ayc_PE", // Southern Aymara (Peru) "ay_PE", // Aymara (Peru) + "az", // Azerbaijani "az_AZ", // Azerbaijani (Azerbaijan) "be", // Belarusian "be_BY", // Belarusian (Belarus) @@ -100,6 +101,7 @@ static const char *locale_list[] = { "bo", // Tibetan "bo_CN", // Tibetan (China) "bo_IN", // Tibetan (India) + "br", // Breton "br_FR", // Breton (France) "brx_IN", // Bodo (India) "bs_BA", // Bosnian (Bosnia and Herzegovina) @@ -201,6 +203,7 @@ static const char *locale_list[] = { "gd_GB", // Scottish Gaelic (United Kingdom) "gez_ER", // Geez (Eritrea) "gez_ET", // Geez (Ethiopia) + "gl", // Galician "gl_ES", // Galician (Spain) "gu_IN", // Gujarati (India) "gv_GB", // Manx (United Kingdom) @@ -237,6 +240,7 @@ static const char *locale_list[] = { "ka_GE", // Georgian (Georgia) "kk_KZ", // Kazakh (Kazakhstan) "kl_GL", // Kalaallisut (Greenland) + "km", // Central Khmer "km_KH", // Central Khmer (Cambodia) "kn_IN", // Kannada (India) "kok_IN", // Konkani (India) @@ -273,6 +277,7 @@ static const char *locale_list[] = { "ml_IN", // Malayalam (India) "mni_IN", // Manipuri (India) "mn_MN", // Mongolian (Mongolia) + "mr", // Marathi "mr_IN", // Marathi (India) "ms", // Malay "ms_MY", // Malay (Malaysia) @@ -302,6 +307,7 @@ static const char *locale_list[] = { "om", // Oromo "om_ET", // Oromo (Ethiopia) "om_KE", // Oromo (Kenya) + "or", // Oriya "or_IN", // Oriya (India) "os_RU", // Ossetian (Russia) "pa_IN", // Panjabi (India) @@ -385,7 +391,10 @@ static const char *locale_list[] = { "tr_CY", // Turkish (Cyprus) "tr_TR", // Turkish (Turkey) "ts_ZA", // Tsonga (South Africa) + "tt", // Tatar "tt_RU", // Tatar (Russia) + "tzm", // Central Atlas Tamazight + "tzm_MA", // Central Atlas Tamazight (Marrocos) "ug_CN", // Uighur (China) "uk", // Ukrainian "uk_UA", // Ukrainian (Ukraine) @@ -451,6 +460,7 @@ static const char *locale_names[] = { "Asturian (Spain)", "Southern Aymara (Peru)", "Aymara (Peru)", + "Azerbaijani", "Azerbaijani (Azerbaijan)", "Belarusian", "Belarusian (Belarus)", @@ -468,6 +478,7 @@ static const char *locale_names[] = { "Tibetan", "Tibetan (China)", "Tibetan (India)", + "Breton", "Breton (France)", "Bodo (India)", "Bosnian (Bosnia and Herzegovina)", @@ -569,6 +580,7 @@ static const char *locale_names[] = { "Scottish Gaelic (United Kingdom)", "Geez (Eritrea)", "Geez (Ethiopia)", + "Galician", "Galician (Spain)", "Gujarati (India)", "Manx (United Kingdom)", @@ -605,6 +617,7 @@ static const char *locale_names[] = { "Georgian (Georgia)", "Kazakh (Kazakhstan)", "Kalaallisut (Greenland)", + "Central Khmer", "Central Khmer (Cambodia)", "Kannada (India)", "Konkani (India)", @@ -641,6 +654,7 @@ static const char *locale_names[] = { "Malayalam (India)", "Manipuri (India)", "Mongolian (Mongolia)", + "Marathi", "Marathi (India)", "Malay", "Malay (Malaysia)", @@ -670,6 +684,7 @@ static const char *locale_names[] = { "Oromo", "Oromo (Ethiopia)", "Oromo (Kenya)", + "Oriya", "Oriya (India)", "Ossetian (Russia)", "Panjabi (India)", @@ -753,7 +768,10 @@ static const char *locale_names[] = { "Turkish (Cyprus)", "Turkish (Turkey)", "Tsonga (South Africa)", + "Tatar", "Tatar (Russia)", + "Central Atlas Tamazight", + "Central Atlas Tamazight (Marrocos)", "Uighur (China)", "Ukrainian", "Ukrainian (Ukraine)", @@ -791,9 +809,12 @@ static const char *locale_names[] = { // - https://msdn.microsoft.com/en-us/library/windows/desktop/ms693062(v=vs.85).aspx static const char *locale_renames[][2] = { - { "in", "id" }, // Indonesian - { "iw", "he" }, // Hebrew - { "no", "nb" }, // Norwegian Bokmål + { "in", "id" }, // Indonesian + { "iw", "he" }, // Hebrew + { "no", "nb" }, // Norwegian Bokmål + { "C", "en" }, // "C" is the simple/default/untranslated Computer locale. + // ASCII-only, English, no currency symbols. Godot treats this as "en". + // See https://unix.stackexchange.com/a/87763/164141 "The C locale is"... { nullptr, nullptr } }; @@ -801,8 +822,8 @@ static const char *locale_renames[][2] = { Dictionary Translation::_get_messages() const { Dictionary d; - for (const Map<StringName, StringName>::Element *E = translation_map.front(); E; E = E->next()) { - d[E->key()] = E->value(); + for (const KeyValue<StringName, StringName> &E : translation_map) { + d[E.key] = E.value; } return d; } @@ -811,8 +832,8 @@ Vector<String> Translation::_get_message_list() const { Vector<String> msgs; msgs.resize(translation_map.size()); int idx = 0; - for (const Map<StringName, StringName>::Element *E = translation_map.front(); E; E = E->next()) { - msgs.set(idx, E->key()); + for (const KeyValue<StringName, StringName> &E : translation_map) { + msgs.set(idx, E.key); idx += 1; } @@ -822,8 +843,8 @@ Vector<String> Translation::_get_message_list() const { void Translation::_set_messages(const Dictionary &p_messages) { List<Variant> keys; p_messages.get_key_list(&keys); - for (auto E = keys.front(); E; E = E->next()) { - translation_map[E->get()] = p_messages[E->get()]; + for (const Variant &E : keys) { + translation_map[E] = p_messages[E]; } } @@ -840,7 +861,7 @@ void Translation::set_locale(const String &p_locale) { locale = univ_locale; } - if (OS::get_singleton()->get_main_loop()) { + if (OS::get_singleton()->get_main_loop() && TranslationServer::get_singleton()->get_loaded_locales().has(this)) { OS::get_singleton()->get_main_loop()->notification(MainLoop::NOTIFICATION_TRANSLATION_CHANGED); } } @@ -851,11 +872,16 @@ void Translation::add_message(const StringName &p_src_text, const StringName &p_ void Translation::add_plural_message(const StringName &p_src_text, const Vector<String> &p_plural_xlated_texts, const StringName &p_context) { WARN_PRINT("Translation class doesn't handle plural messages. Calling add_plural_message() on a Translation instance is probably a mistake. \nUse a derived Translation class that handles plurals, such as TranslationPO class"); - ERR_FAIL_COND_MSG(p_plural_xlated_texts.empty(), "Parameter vector p_plural_xlated_texts passed in is empty."); + ERR_FAIL_COND_MSG(p_plural_xlated_texts.is_empty(), "Parameter vector p_plural_xlated_texts passed in is empty."); translation_map[p_src_text] = p_plural_xlated_texts[0]; } StringName Translation::get_message(const StringName &p_src_text, const StringName &p_context) const { + StringName ret; + if (GDVIRTUAL_CALL(_get_message, p_src_text, p_context, ret)) { + return ret; + } + if (p_context != StringName()) { WARN_PRINT("Translation class doesn't handle context. Using context in get_message() on a Translation instance is probably a mistake. \nUse a derived Translation class that handles context, such as TranslationPO class"); } @@ -869,6 +895,11 @@ StringName Translation::get_message(const StringName &p_src_text, const StringNa } StringName Translation::get_plural_message(const StringName &p_src_text, const StringName &p_plural_text, int p_n, const StringName &p_context) const { + StringName ret; + if (GDVIRTUAL_CALL(_get_plural_message, p_src_text, p_plural_text, p_n, p_context, ret)) { + return ret; + } + WARN_PRINT("Translation class doesn't handle plural messages. Calling get_plural_message() on a Translation instance is probably a mistake. \nUse a derived Translation class that handles plurals, such as TranslationPO class"); return get_message(p_src_text); } @@ -882,8 +913,8 @@ void Translation::erase_message(const StringName &p_src_text, const StringName & } void Translation::get_message_list(List<StringName> *r_messages) const { - for (const Map<StringName, StringName>::Element *E = translation_map.front(); E; E = E->next()) { - r_messages->push_back(E->key()); + for (const KeyValue<StringName, StringName> &E : translation_map) { + r_messages->push_back(E.key); } } @@ -904,12 +935,75 @@ void Translation::_bind_methods() { ClassDB::bind_method(D_METHOD("_set_messages"), &Translation::_set_messages); ClassDB::bind_method(D_METHOD("_get_messages"), &Translation::_get_messages); - ADD_PROPERTY(PropertyInfo(Variant::DICTIONARY, "messages", PROPERTY_HINT_NONE, "", PROPERTY_USAGE_NOEDITOR | PROPERTY_USAGE_INTERNAL), "_set_messages", "_get_messages"); + GDVIRTUAL_BIND(_get_plural_message, "src_message", "src_plural_message", "n", "context"); + GDVIRTUAL_BIND(_get_message, "src_message", "context"); + + ADD_PROPERTY(PropertyInfo(Variant::DICTIONARY, "messages", PROPERTY_HINT_NONE, "", PROPERTY_USAGE_NO_EDITOR | PROPERTY_USAGE_INTERNAL), "_set_messages", "_get_messages"); ADD_PROPERTY(PropertyInfo(Variant::STRING, "locale"), "set_locale", "get_locale"); } /////////////////////////////////////////////// +struct _character_accent_pair { + const char32_t character; + const char32_t *accented_character; +}; + +static _character_accent_pair _character_to_accented[] = { + { 'A', U"Å" }, + { 'B', U"ß" }, + { 'C', U"Ç" }, + { 'D', U"Ð" }, + { 'E', U"É" }, + { 'F', U"F́" }, + { 'G', U"Ĝ" }, + { 'H', U"Ĥ" }, + { 'I', U"Ĩ" }, + { 'J', U"Ĵ" }, + { 'K', U"ĸ" }, + { 'L', U"Ł" }, + { 'M', U"Ḿ" }, + { 'N', U"й" }, + { 'O', U"Ö" }, + { 'P', U"Ṕ" }, + { 'Q', U"Q́" }, + { 'R', U"Ř" }, + { 'S', U"Ŝ" }, + { 'T', U"Ŧ" }, + { 'U', U"Ũ" }, + { 'V', U"Ṽ" }, + { 'W', U"Ŵ" }, + { 'X', U"X́" }, + { 'Y', U"Ÿ" }, + { 'Z', U"Ž" }, + { 'a', U"á" }, + { 'b', U"ḅ" }, + { 'c', U"ć" }, + { 'd', U"d́" }, + { 'e', U"é" }, + { 'f', U"f́" }, + { 'g', U"ǵ" }, + { 'h', U"h̀" }, + { 'i', U"í" }, + { 'j', U"ǰ" }, + { 'k', U"ḱ" }, + { 'l', U"ł" }, + { 'm', U"m̀" }, + { 'n', U"ή" }, + { 'o', U"ô" }, + { 'p', U"ṕ" }, + { 'q', U"q́" }, + { 'r', U"ŕ" }, + { 's', U"š" }, + { 't', U"ŧ" }, + { 'u', U"ü" }, + { 'v', U"ṽ" }, + { 'w', U"ŵ" }, + { 'x', U"x́" }, + { 'y', U"ý" }, + { 'z', U"ź" }, +}; + bool TranslationServer::is_locale_valid(const String &p_locale) { const char **ptr = locale_list; @@ -1082,10 +1176,10 @@ StringName TranslationServer::translate(const StringName &p_message, const Strin } if (!res) { - return p_message; + return pseudolocalization_enabled ? pseudolocalize(p_message) : p_message; } - return res; + return pseudolocalization_enabled ? pseudolocalize(res) : res; } StringName TranslationServer::translate_plural(const StringName &p_message, const StringName &p_message_plural, int p_n, const StringName &p_context) const { @@ -1191,14 +1285,25 @@ bool TranslationServer::_load_translations(const String &p_from) { } void TranslationServer::setup() { - String test = GLOBAL_DEF("locale/test", ""); + String test = GLOBAL_DEF("internationalization/locale/test", ""); test = test.strip_edges(); if (test != "") { set_locale(test); } else { set_locale(OS::get_singleton()->get_locale()); } - fallback = GLOBAL_DEF("locale/fallback", "en"); + + fallback = GLOBAL_DEF("internationalization/locale/fallback", "en"); + pseudolocalization_enabled = GLOBAL_DEF("internationalization/pseudolocalization/use_pseudolocalization", false); + pseudolocalization_accents_enabled = GLOBAL_DEF("internationalization/pseudolocalization/replace_with_accents", true); + pseudolocalization_double_vowels_enabled = GLOBAL_DEF("internationalization/pseudolocalization/double_vowels", false); + pseudolocalization_fake_bidi_enabled = GLOBAL_DEF("internationalization/pseudolocalization/fake_bidi", false); + pseudolocalization_override_enabled = GLOBAL_DEF("internationalization/pseudolocalization/override", false); + expansion_ratio = GLOBAL_DEF("internationalization/pseudolocalization/expansion_ratio", 0.0); + pseudolocalization_prefix = GLOBAL_DEF("internationalization/pseudolocalization/prefix", "["); + pseudolocalization_suffix = GLOBAL_DEF("internationalization/pseudolocalization/suffix", "]"); + pseudolocalization_skip_placeholders_enabled = GLOBAL_DEF("internationalization/pseudolocalization/skip_placeholders", true); + #ifdef TOOLS_ENABLED { String options = ""; @@ -1210,7 +1315,7 @@ void TranslationServer::setup() { options += locale_list[idx]; idx++; } - ProjectSettings::get_singleton()->set_custom_property_info("locale/fallback", PropertyInfo(Variant::STRING, "locale/fallback", PROPERTY_HINT_ENUM, options)); + ProjectSettings::get_singleton()->set_custom_property_info("internationalization/locale/fallback", PropertyInfo(Variant::STRING, "internationalization/locale/fallback", PROPERTY_HINT_ENUM, options)); } #endif } @@ -1239,10 +1344,10 @@ StringName TranslationServer::tool_translate(const StringName &p_message, const if (tool_translation.is_valid()) { StringName r = tool_translation->get_message(p_message, p_context); if (r) { - return r; + return editor_pseudolocalization ? tool_pseudolocalize(r) : r; } } - return p_message; + return editor_pseudolocalization ? tool_pseudolocalize(p_message) : p_message; } StringName TranslationServer::tool_translate_plural(const StringName &p_message, const StringName &p_message_plural, int p_n, const StringName &p_context) const { @@ -1287,6 +1392,181 @@ StringName TranslationServer::doc_translate_plural(const StringName &p_message, return p_message_plural; } +bool TranslationServer::is_pseudolocalization_enabled() const { + return pseudolocalization_enabled; +} + +void TranslationServer::set_pseudolocalization_enabled(bool p_enabled) { + pseudolocalization_enabled = p_enabled; + + if (OS::get_singleton()->get_main_loop()) { + OS::get_singleton()->get_main_loop()->notification(MainLoop::NOTIFICATION_TRANSLATION_CHANGED); + } + ResourceLoader::reload_translation_remaps(); +} + +void TranslationServer::set_editor_pseudolocalization(bool p_enabled) { + editor_pseudolocalization = p_enabled; +} + +void TranslationServer::reload_pseudolocalization() { + pseudolocalization_accents_enabled = GLOBAL_GET("internationalization/pseudolocalization/replace_with_accents"); + pseudolocalization_double_vowels_enabled = GLOBAL_GET("internationalization/pseudolocalization/double_vowels"); + pseudolocalization_fake_bidi_enabled = GLOBAL_GET("internationalization/pseudolocalization/fake_bidi"); + pseudolocalization_override_enabled = GLOBAL_GET("internationalization/pseudolocalization/override"); + expansion_ratio = GLOBAL_GET("internationalization/pseudolocalization/expansion_ratio"); + pseudolocalization_prefix = GLOBAL_GET("internationalization/pseudolocalization/prefix"); + pseudolocalization_suffix = GLOBAL_GET("internationalization/pseudolocalization/suffix"); + pseudolocalization_skip_placeholders_enabled = GLOBAL_GET("internationalization/pseudolocalization/skip_placeholders"); + + if (OS::get_singleton()->get_main_loop()) { + OS::get_singleton()->get_main_loop()->notification(MainLoop::NOTIFICATION_TRANSLATION_CHANGED); + } + ResourceLoader::reload_translation_remaps(); +} + +StringName TranslationServer::pseudolocalize(const StringName &p_message) const { + String message = p_message; + int length = message.length(); + if (pseudolocalization_override_enabled) { + message = get_override_string(message); + } + + if (pseudolocalization_double_vowels_enabled) { + message = double_vowels(message); + } + + if (pseudolocalization_accents_enabled) { + message = replace_with_accented_string(message); + } + + if (pseudolocalization_fake_bidi_enabled) { + message = wrap_with_fakebidi_characters(message); + } + + StringName res = add_padding(message, length); + return res; +} + +StringName TranslationServer::tool_pseudolocalize(const StringName &p_message) const { + String message = p_message; + message = double_vowels(message); + message = replace_with_accented_string(message); + StringName res = "[!!! " + message + " !!!]"; + return res; +} + +String TranslationServer::get_override_string(String &p_message) const { + String res; + for (int i = 0; i < p_message.size(); i++) { + if (pseudolocalization_skip_placeholders_enabled && is_placeholder(p_message, i)) { + res += p_message[i]; + res += p_message[i + 1]; + i++; + continue; + } + res += '*'; + } + return res; +} + +String TranslationServer::double_vowels(String &p_message) const { + String res; + for (int i = 0; i < p_message.size(); i++) { + if (pseudolocalization_skip_placeholders_enabled && is_placeholder(p_message, i)) { + res += p_message[i]; + res += p_message[i + 1]; + i++; + continue; + } + res += p_message[i]; + if (p_message[i] == 'a' || p_message[i] == 'e' || p_message[i] == 'i' || p_message[i] == 'o' || p_message[i] == 'u' || + p_message[i] == 'A' || p_message[i] == 'E' || p_message[i] == 'I' || p_message[i] == 'O' || p_message[i] == 'U') { + res += p_message[i]; + } + } + return res; +}; + +String TranslationServer::replace_with_accented_string(String &p_message) const { + String res; + for (int i = 0; i < p_message.size(); i++) { + if (pseudolocalization_skip_placeholders_enabled && is_placeholder(p_message, i)) { + res += p_message[i]; + res += p_message[i + 1]; + i++; + continue; + } + const char32_t *accented = get_accented_version(p_message[i]); + if (accented) { + res += accented; + } else { + res += p_message[i]; + } + } + return res; +} + +String TranslationServer::wrap_with_fakebidi_characters(String &p_message) const { + String res; + char32_t fakebidiprefix = U'\u202e'; + char32_t fakebidisuffix = U'\u202c'; + res += fakebidiprefix; + // The fake bidi unicode gets popped at every newline so pushing it back at every newline. + for (int i = 0; i < p_message.size(); i++) { + if (p_message[i] == '\n') { + res += fakebidisuffix; + res += p_message[i]; + res += fakebidiprefix; + } else if (pseudolocalization_skip_placeholders_enabled && is_placeholder(p_message, i)) { + res += fakebidisuffix; + res += p_message[i]; + res += p_message[i + 1]; + res += fakebidiprefix; + i++; + } else { + res += p_message[i]; + } + } + res += fakebidisuffix; + return res; +} + +String TranslationServer::add_padding(String &p_message, int p_length) const { + String res; + String prefix = pseudolocalization_prefix; + String suffix; + for (int i = 0; i < p_length * expansion_ratio / 2; i++) { + prefix += "_"; + suffix += "_"; + } + suffix += pseudolocalization_suffix; + res += prefix; + res += p_message; + res += suffix; + return res; +} + +const char32_t *TranslationServer::get_accented_version(char32_t p_character) const { + if (!((p_character >= 'a' && p_character <= 'z') || (p_character >= 'A' && p_character <= 'Z'))) { + return nullptr; + } + + for (unsigned int i = 0; i < sizeof(_character_to_accented) / sizeof(_character_to_accented[0]); i++) { + if (_character_to_accented[i].character == p_character) { + return _character_to_accented[i].accented_character; + } + } + + return nullptr; +} + +bool TranslationServer::is_placeholder(String &p_message, int p_index) const { + return p_message[p_index] == '%' && p_index < p_message.size() - 1 && + (p_message[p_index + 1] == 's' || p_message[p_index + 1] == 'c' || p_message[p_index + 1] == 'd' || + p_message[p_index + 1] == 'o' || p_message[p_index + 1] == 'x' || p_message[p_index + 1] == 'X' || p_message[p_index + 1] == 'f'); +} + void TranslationServer::_bind_methods() { ClassDB::bind_method(D_METHOD("set_locale", "locale"), &TranslationServer::set_locale); ClassDB::bind_method(D_METHOD("get_locale"), &TranslationServer::get_locale); @@ -1303,15 +1583,21 @@ void TranslationServer::_bind_methods() { ClassDB::bind_method(D_METHOD("clear"), &TranslationServer::clear); ClassDB::bind_method(D_METHOD("get_loaded_locales"), &TranslationServer::get_loaded_locales); + + ClassDB::bind_method(D_METHOD("is_pseudolocalization_enabled"), &TranslationServer::is_pseudolocalization_enabled); + ClassDB::bind_method(D_METHOD("set_pseudolocalization_enabled", "enabled"), &TranslationServer::set_pseudolocalization_enabled); + ClassDB::bind_method(D_METHOD("reload_pseudolocalization"), &TranslationServer::reload_pseudolocalization); + ClassDB::bind_method(D_METHOD("pseudolocalize", "message"), &TranslationServer::pseudolocalize); + ADD_PROPERTY(PropertyInfo(Variant::Type::BOOL, "pseudolocalization_enabled"), "set_pseudolocalization_enabled", "is_pseudolocalization_enabled"); } void TranslationServer::load_translations() { String locale = get_locale(); - _load_translations("locale/translations"); //all - _load_translations("locale/translations_" + locale.substr(0, 2)); + _load_translations("internationalization/locale/translations"); //all + _load_translations("internationalization/locale/translations_" + locale.substr(0, 2)); if (locale.substr(0, 2) != locale) { - _load_translations("locale/translations_" + locale); + _load_translations("internationalization/locale/translations_" + locale); } } diff --git a/core/string/translation.h b/core/string/translation.h index c7ffe4d065..6aec0bb8ea 100644 --- a/core/string/translation.h +++ b/core/string/translation.h @@ -5,8 +5,8 @@ /* GODOT ENGINE */ /* https://godotengine.org */ /*************************************************************************/ -/* Copyright (c) 2007-2020 Juan Linietsky, Ariel Manzur. */ -/* Copyright (c) 2014-2020 Godot Engine contributors (cf. AUTHORS.md). */ +/* Copyright (c) 2007-2021 Juan Linietsky, Ariel Manzur. */ +/* Copyright (c) 2014-2021 Godot Engine contributors (cf. AUTHORS.md). */ /* */ /* Permission is hereby granted, free of charge, to any person obtaining */ /* a copy of this software and associated documentation files (the */ @@ -32,6 +32,8 @@ #define TRANSLATION_H #include "core/io/resource.h" +#include "core/object/gdvirtual.gen.inc" +#include "core/object/script_language.h" class Translation : public Resource { GDCLASS(Translation, Resource); @@ -48,6 +50,9 @@ class Translation : public Resource { protected: static void _bind_methods(); + GDVIRTUAL2RC(StringName, _get_message, StringName, StringName); + GDVIRTUAL4RC(StringName, _get_plural_message, StringName, StringName, int, StringName); + public: void set_locale(const String &p_locale); _FORCE_INLINE_ String get_locale() const { return locale; } @@ -77,6 +82,26 @@ class TranslationServer : public Object { bool enabled = true; + bool pseudolocalization_enabled = false; + bool pseudolocalization_accents_enabled = false; + bool pseudolocalization_double_vowels_enabled = false; + bool pseudolocalization_fake_bidi_enabled = false; + bool pseudolocalization_override_enabled = false; + bool pseudolocalization_skip_placeholders_enabled = false; + bool editor_pseudolocalization = false; + float expansion_ratio = 0.0; + String pseudolocalization_prefix; + String pseudolocalization_suffix; + + StringName tool_pseudolocalize(const StringName &p_message) const; + String get_override_string(String &p_message) const; + String double_vowels(String &p_message) const; + String replace_with_accented_string(String &p_message) const; + String wrap_with_fakebidi_characters(String &p_message) const; + String add_padding(String &p_message, int p_length) const; + const char32_t *get_accented_version(char32_t p_character) const; + bool is_placeholder(String &p_message, int p_index) const; + static TranslationServer *singleton; bool _load_translations(const String &p_from); @@ -104,6 +129,13 @@ public: StringName translate(const StringName &p_message, const StringName &p_context = "") const; StringName translate_plural(const StringName &p_message, const StringName &p_message_plural, int p_n, const StringName &p_context = "") const; + StringName pseudolocalize(const StringName &p_message) const; + + bool is_pseudolocalization_enabled() const; + void set_pseudolocalization_enabled(bool p_enabled); + void set_editor_pseudolocalization(bool p_enabled); + void reload_pseudolocalization(); + static Vector<String> get_all_locales(); static Vector<String> get_all_locale_names(); static bool is_locale_valid(const String &p_locale); diff --git a/core/string/translation_po.cpp b/core/string/translation_po.cpp index 203f29026b..1da00aa54b 100644 --- a/core/string/translation_po.cpp +++ b/core/string/translation_po.cpp @@ -5,8 +5,8 @@ /* GODOT ENGINE */ /* https://godotengine.org */ /*************************************************************************/ -/* Copyright (c) 2007-2020 Juan Linietsky, Ariel Manzur. */ -/* Copyright (c) 2014-2020 Godot Engine contributors (cf. AUTHORS.md). */ +/* Copyright (c) 2007-2021 Juan Linietsky, Ariel Manzur. */ +/* Copyright (c) 2014-2021 Godot Engine contributors (cf. AUTHORS.md). */ /* */ /* Permission is hereby granted, free of charge, to any person obtaining */ /* a copy of this software and associated documentation files (the */ @@ -30,7 +30,7 @@ #include "translation_po.h" -#include "core/os/file_access.h" +#include "core/io/file_access.h" #ifdef DEBUG_TRANSLATION_PO void TranslationPO::print_translation_map() { @@ -47,14 +47,13 @@ void TranslationPO::print_translation_map() { List<StringName> context_l; translation_map.get_key_list(&context_l); - for (auto E = context_l.front(); E; E = E->next()) { - StringName ctx = E->get(); + for (const StringName &ctx : context_l) { file->store_line(" ===== Context: " + String::utf8(String(ctx).utf8()) + " ===== "); const HashMap<StringName, Vector<StringName>> &inner_map = translation_map[ctx]; List<StringName> id_l; inner_map.get_key_list(&id_l); - for (auto E2 = id_l.front(); E2; E2 = E2->next()) { + for (List<StringName>::Element *E2 = id_l.front(); E2; E2 = E2->next()) { StringName id = E2->get(); file->store_line("msgid: " + String::utf8(String(id).utf8())); for (int i = 0; i < inner_map[id].size(); i++) { @@ -74,15 +73,14 @@ Dictionary TranslationPO::_get_messages() const { List<StringName> context_l; translation_map.get_key_list(&context_l); - for (auto E = context_l.front(); E; E = E->next()) { - StringName ctx = E->get(); + for (const StringName &ctx : context_l) { const HashMap<StringName, Vector<StringName>> &id_str_map = translation_map[ctx]; Dictionary d2; List<StringName> id_l; id_str_map.get_key_list(&id_l); // Save list of id and strs associated with a context in a temporary dictionary. - for (auto E2 = id_l.front(); E2; E2 = E2->next()) { + for (List<StringName>::Element *E2 = id_l.front(); E2; E2 = E2->next()) { StringName id = E2->get(); d2[id] = id_str_map[id]; } @@ -98,14 +96,13 @@ void TranslationPO::_set_messages(const Dictionary &p_messages) { List<Variant> context_l; p_messages.get_key_list(&context_l); - for (auto E = context_l.front(); E; E = E->next()) { - StringName ctx = E->get(); + for (const Variant &ctx : context_l) { const Dictionary &id_str_map = p_messages[ctx]; HashMap<StringName, Vector<StringName>> temp_map; List<Variant> id_l; id_str_map.get_key_list(&id_l); - for (auto E2 = id_l.front(); E2; E2 = E2->next()) { + for (List<Variant>::Element *E2 = id_l.front(); E2; E2 = E2->next()) { StringName id = E2->get(); temp_map[id] = id_str_map[id]; } @@ -121,8 +118,8 @@ Vector<String> TranslationPO::_get_message_list() const { get_message_list(&msgs); Vector<String> v; - for (auto E = msgs.front(); E; E = E->next()) { - v.push_back(E->get()); + for (const StringName &E : msgs) { + v.push_back(E); } return v; @@ -158,7 +155,7 @@ int TranslationPO::_get_plural_index(int p_n) const { void TranslationPO::_cache_plural_tests(const String &p_plural_rule) { // Some examples of p_plural_rule passed in can have the form: // "n==0 ? 0 : n==1 ? 1 : n==2 ? 2 : n%100>=3 && n%100<=10 ? 3 : n%100>=11 && n%100<=99 ? 4 : 5" (Arabic) - // "n >= 2" (French) // When evaluating the last, esp careful with this one. + // "n >= 2" (French) // When evaluating the last, especially careful with this one. // "n != 1" (English) int first_ques_mark = p_plural_rule.find("?"); if (first_ques_mark == -1) { @@ -188,7 +185,7 @@ void TranslationPO::set_plural_rule(const String &p_plural_rule) { plural_rule = plural_rule.replacen("(", ""); plural_rule = plural_rule.replacen(")", ""); _cache_plural_tests(plural_rule); - expr.instance(); + expr.instantiate(); input_name.push_back("n"); } @@ -230,7 +227,7 @@ StringName TranslationPO::get_message(const StringName &p_src_text, const String if (!translation_map.has(p_context) || !translation_map[p_context].has(p_src_text)) { return StringName(); } - ERR_FAIL_COND_V_MSG(translation_map[p_context][p_src_text].empty(), StringName(), "Source text \"" + String(p_src_text) + "\" is registered but doesn't have a translation. Please report this bug."); + ERR_FAIL_COND_V_MSG(translation_map[p_context][p_src_text].is_empty(), StringName(), "Source text \"" + String(p_src_text) + "\" is registered but doesn't have a translation. Please report this bug."); return translation_map[p_context][p_src_text][0]; } @@ -246,7 +243,7 @@ StringName TranslationPO::get_plural_message(const StringName &p_src_text, const if (!translation_map.has(p_context) || !translation_map[p_context].has(p_src_text)) { return StringName(); } - ERR_FAIL_COND_V_MSG(translation_map[p_context][p_src_text].empty(), StringName(), "Source text \"" + String(p_src_text) + "\" is registered but doesn't have a translation. Please report this bug."); + ERR_FAIL_COND_V_MSG(translation_map[p_context][p_src_text].is_empty(), StringName(), "Source text \"" + String(p_src_text) + "\" is registered but doesn't have a translation. Please report this bug."); if (translation_map[p_context][p_src_text].size() == 1) { WARN_PRINT("Source string \"" + String(p_src_text) + "\" doesn't have plural translations. Use singular translation API for such as tr(), TTR() to translate \"" + String(p_src_text) + "\""); @@ -275,21 +272,21 @@ void TranslationPO::erase_message(const StringName &p_src_text, const StringName } void TranslationPO::get_message_list(List<StringName> *r_messages) const { - // PHashTranslation uses this function to get the list of msgid. + // OptimizedTranslation uses this function to get the list of msgid. // Return all the keys of translation_map under "" context. List<StringName> context_l; translation_map.get_key_list(&context_l); - for (auto E = context_l.front(); E; E = E->next()) { - if (String(E->get()) != "") { + for (const StringName &E : context_l) { + if (String(E) != "") { continue; } List<StringName> msgid_l; - translation_map[E->get()].get_key_list(&msgid_l); + translation_map[E].get_key_list(&msgid_l); - for (auto E2 = msgid_l.front(); E2; E2 = E2->next()) { + for (List<StringName>::Element *E2 = msgid_l.front(); E2; E2 = E2->next()) { r_messages->push_back(E2->get()); } } @@ -300,8 +297,8 @@ int TranslationPO::get_message_count() const { translation_map.get_key_list(&context_l); int count = 0; - for (auto E = context_l.front(); E; E = E->next()) { - count += translation_map[E->get()].size(); + for (const StringName &E : context_l) { + count += translation_map[E].size(); } return count; } diff --git a/core/string/translation_po.h b/core/string/translation_po.h index c8a47bec5a..0e1d03d6ca 100644 --- a/core/string/translation_po.h +++ b/core/string/translation_po.h @@ -5,8 +5,8 @@ /* GODOT ENGINE */ /* https://godotengine.org */ /*************************************************************************/ -/* Copyright (c) 2007-2020 Juan Linietsky, Ariel Manzur. */ -/* Copyright (c) 2014-2020 Godot Engine contributors (cf. AUTHORS.md). */ +/* Copyright (c) 2007-2021 Juan Linietsky, Ariel Manzur. */ +/* Copyright (c) 2014-2021 Godot Engine contributors (cf. AUTHORS.md). */ /* */ /* Permission is hereby granted, free of charge, to any person obtaining */ /* a copy of this software and associated documentation files (the */ diff --git a/core/string/ucaps.h b/core/string/ucaps.h index 79b346acba..b785ac7879 100644 --- a/core/string/ucaps.h +++ b/core/string/ucaps.h @@ -5,8 +5,8 @@ /* GODOT ENGINE */ /* https://godotengine.org */ /*************************************************************************/ -/* Copyright (c) 2007-2020 Juan Linietsky, Ariel Manzur. */ -/* Copyright (c) 2014-2020 Godot Engine contributors (cf. AUTHORS.md). */ +/* Copyright (c) 2007-2021 Juan Linietsky, Ariel Manzur. */ +/* Copyright (c) 2014-2021 Godot Engine contributors (cf. AUTHORS.md). */ /* */ /* Permission is hereby granted, free of charge, to any person obtaining */ /* a copy of this software and associated documentation files (the */ diff --git a/core/string/ustring.cpp b/core/string/ustring.cpp index fee168993b..70236231a2 100644 --- a/core/string/ustring.cpp +++ b/core/string/ustring.cpp @@ -5,8 +5,8 @@ /* GODOT ENGINE */ /* https://godotengine.org */ /*************************************************************************/ -/* Copyright (c) 2007-2020 Juan Linietsky, Ariel Manzur. */ -/* Copyright (c) 2014-2020 Godot Engine contributors (cf. AUTHORS.md). */ +/* Copyright (c) 2007-2021 Juan Linietsky, Ariel Manzur. */ +/* Copyright (c) 2014-2021 Godot Engine contributors (cf. AUTHORS.md). */ /* */ /* Permission is hereby granted, free of charge, to any person obtaining */ /* a copy of this software and associated documentation files (the */ @@ -38,13 +38,11 @@ #include "core/string/translation.h" #include "core/string/ucaps.h" #include "core/variant/variant.h" +#include "core/version_generated.gen.h" -#include <cstdint> - -#ifndef NO_USE_STDLIB #include <stdio.h> #include <stdlib.h> -#endif +#include <cstdint> #ifdef _MSC_VER #define _CRT_SECURE_NO_WARNINGS // to disable build-time warning which suggested to use strcpy_s instead strcpy @@ -54,11 +52,27 @@ #define snprintf _snprintf_s #endif -#define MAX_DIGITS 6 -#define UPPERCASE(m_c) (((m_c) >= 'a' && (m_c) <= 'z') ? ((m_c) - ('a' - 'A')) : (m_c)) -#define LOWERCASE(m_c) (((m_c) >= 'A' && (m_c) <= 'Z') ? ((m_c) + ('a' - 'A')) : (m_c)) -#define IS_DIGIT(m_d) ((m_d) >= '0' && (m_d) <= '9') -#define IS_HEX_DIGIT(m_d) (((m_d) >= '0' && (m_d) <= '9') || ((m_d) >= 'a' && (m_d) <= 'f') || ((m_d) >= 'A' && (m_d) <= 'F')) +static const int MAX_DECIMALS = 32; + +static _FORCE_INLINE_ bool is_digit(char32_t c) { + return (c >= '0' && c <= '9'); +} + +static _FORCE_INLINE_ bool is_hex_digit(char32_t c) { + return (is_digit(c) || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F')); +} + +static _FORCE_INLINE_ bool is_upper_case(char32_t c) { + return (c >= 'A' && c <= 'Z'); +} + +static _FORCE_INLINE_ bool is_lower_case(char32_t c) { + return (c >= 'a' && c <= 'z'); +} + +static _FORCE_INLINE_ char32_t lower_case(char32_t c) { + return (is_upper_case(c) ? (c + ('a' - 'A')) : c); +} const char CharString::_null = 0; const char16_t Char16String::_null = 0; @@ -240,6 +254,71 @@ String String::word_wrap(int p_chars_per_line) const { return ret; } +Error String::parse_url(String &r_scheme, String &r_host, int &r_port, String &r_path) const { + // Splits the URL into scheme, host, port, path. Strip credentials when present. + String base = *this; + r_scheme = ""; + r_host = ""; + r_port = 0; + r_path = ""; + int pos = base.find("://"); + // Scheme + if (pos != -1) { + r_scheme = base.substr(0, pos + 3).to_lower(); + base = base.substr(pos + 3, base.length() - pos - 3); + } + pos = base.find("/"); + // Path + if (pos != -1) { + r_path = base.substr(pos, base.length() - pos); + base = base.substr(0, pos); + } + // Host + pos = base.find("@"); + if (pos != -1) { + // Strip credentials + base = base.substr(pos + 1, base.length() - pos - 1); + } + if (base.begins_with("[")) { + // Literal IPv6 + pos = base.rfind("]"); + if (pos == -1) { + return ERR_INVALID_PARAMETER; + } + r_host = base.substr(1, pos - 1); + base = base.substr(pos + 1, base.length() - pos - 1); + } else { + // Anything else + if (base.get_slice_count(":") > 2) { + return ERR_INVALID_PARAMETER; + } + pos = base.rfind(":"); + if (pos == -1) { + r_host = base; + base = ""; + } else { + r_host = base.substr(0, pos); + base = base.substr(pos, base.length() - pos); + } + } + if (r_host.is_empty()) { + return ERR_INVALID_PARAMETER; + } + r_host = r_host.to_lower(); + // Port + if (base.begins_with(":")) { + base = base.substr(1, base.length() - 1); + if (!base.is_valid_int()) { + return ERR_INVALID_PARAMETER; + } + r_port = base.to_int(); + if (r_port < 1 || r_port > 65535) { + return ERR_INVALID_PARAMETER; + } + } + return OK; +} + void String::copy_from(const char *p_cstr) { // copy Latin-1 encoded c-string directly if (!p_cstr) { @@ -427,12 +506,12 @@ String operator+(char32_t p_chr, const String &p_str) { } String &String::operator+=(const String &p_str) { - if (empty()) { + if (is_empty()) { *this = p_str; return *this; } - if (p_str.empty()) { + if (p_str.is_empty()) { return *this; } @@ -519,7 +598,7 @@ bool String::operator==(const char *p_str) const { if (length() != len) { return false; } - if (empty()) { + if (is_empty()) { return true; } @@ -558,7 +637,7 @@ bool String::operator==(const char32_t *p_str) const { if (length() != len) { return false; } - if (empty()) { + if (is_empty()) { return true; } @@ -580,7 +659,7 @@ bool String::operator==(const String &p_str) const { if (length() != p_str.length()) { return false; } - if (empty()) { + if (is_empty()) { return true; } @@ -605,7 +684,7 @@ bool String::operator==(const StrRange &p_str_range) const { if (length() != len) { return false; } - if (empty()) { + if (is_empty()) { return true; } @@ -673,25 +752,26 @@ bool String::operator<=(const String &p_str) const { bool String::operator>(const String &p_str) const { return p_str < *this; } + bool String::operator>=(const String &p_str) const { return !(*this < p_str); } bool String::operator<(const char *p_str) const { - if (empty() && p_str[0] == 0) { + if (is_empty() && p_str[0] == 0) { return false; } - if (empty()) { + if (is_empty()) { return true; } return is_str_less(get_data(), p_str); } bool String::operator<(const wchar_t *p_str) const { - if (empty() && p_str[0] == 0) { + if (is_empty() && p_str[0] == 0) { return false; } - if (empty()) { + if (is_empty()) { return true; } @@ -705,10 +785,10 @@ bool String::operator<(const wchar_t *p_str) const { } bool String::operator<(const char32_t *p_str) const { - if (empty() && p_str[0] == 0) { + if (is_empty() && p_str[0] == 0) { return false; } - if (empty()) { + if (is_empty()) { return true; } @@ -720,13 +800,13 @@ bool String::operator<(const String &p_str) const { } signed char String::nocasecmp_to(const String &p_str) const { - if (empty() && p_str.empty()) { + if (is_empty() && p_str.is_empty()) { return 0; } - if (empty()) { + if (is_empty()) { return -1; } - if (p_str.empty()) { + if (p_str.is_empty()) { return 1; } @@ -752,13 +832,13 @@ signed char String::nocasecmp_to(const String &p_str) const { } signed char String::casecmp_to(const String &p_str) const { - if (empty() && p_str.empty()) { + if (is_empty() && p_str.is_empty()) { return 0; } - if (empty()) { + if (is_empty()) { return -1; } - if (p_str.empty()) { + if (p_str.is_empty()) { return 1; } @@ -806,8 +886,8 @@ signed char String::naturalnocasecmp_to(const String &p_str) const { while (*this_str) { if (!*that_str) { return 1; - } else if (IS_DIGIT(*this_str)) { - if (!IS_DIGIT(*that_str)) { + } else if (is_digit(*this_str)) { + if (!is_digit(*that_str)) { return -1; } @@ -816,10 +896,10 @@ signed char String::naturalnocasecmp_to(const String &p_str) const { const char32_t *that_substr = that_str; // Compare lengths of both numerical sequences, ignoring leading zeros - while (IS_DIGIT(*this_str)) { + while (is_digit(*this_str)) { this_str++; } - while (IS_DIGIT(*that_str)) { + while (is_digit(*that_str)) { that_str++; } while (*this_substr == '0') { @@ -847,7 +927,7 @@ signed char String::naturalnocasecmp_to(const String &p_str) const { this_substr++; that_substr++; } - } else if (IS_DIGIT(*that_str)) { + } else if (is_digit(*that_str)) { return 1; } else { if (_find_upper(*this_str) < _find_upper(*that_str)) { //more than @@ -873,10 +953,6 @@ const char32_t *String::get_data() const { return size() ? &operator[](0) : &zero; } -void String::erase(int p_pos, int p_chars) { - *this = left(p_pos) + substr(p_pos + p_chars, length() - ((p_pos + p_chars))); -} - String String::capitalize() const { String aux = this->camelcase_to_underscore(true).replace("_", " ").strip_edges(); String cap; @@ -897,26 +973,25 @@ String String::capitalize() const { String String::camelcase_to_underscore(bool lowercase) const { const char32_t *cstr = get_data(); String new_string; - const char A = 'A', Z = 'Z'; - const char a = 'a', z = 'z'; int start_index = 0; for (int i = 1; i < this->size(); i++) { - bool is_upper = cstr[i] >= A && cstr[i] <= Z; - bool is_number = cstr[i] >= '0' && cstr[i] <= '9'; + bool is_upper = is_upper_case(cstr[i]); + bool is_number = is_digit(cstr[i]); + bool are_next_2_lower = false; bool is_next_lower = false; bool is_next_number = false; - bool was_precedent_upper = cstr[i - 1] >= A && cstr[i - 1] <= Z; - bool was_precedent_number = cstr[i - 1] >= '0' && cstr[i - 1] <= '9'; + bool was_precedent_upper = is_upper_case(cstr[i - 1]); + bool was_precedent_number = is_digit(cstr[i - 1]); if (i + 2 < this->size()) { - are_next_2_lower = cstr[i + 1] >= a && cstr[i + 1] <= z && cstr[i + 2] >= a && cstr[i + 2] <= z; + are_next_2_lower = is_lower_case(cstr[i + 1]) && is_lower_case(cstr[i + 2]); } if (i + 1 < this->size()) { - is_next_lower = cstr[i + 1] >= a && cstr[i + 1] <= z; - is_next_number = cstr[i + 1] >= '0' && cstr[i + 1] <= '9'; + is_next_lower = is_lower_case(cstr[i + 1]); + is_next_number = is_digit(cstr[i + 1]); } const bool cond_a = is_upper && !was_precedent_upper && !was_precedent_number; @@ -949,10 +1024,10 @@ String String::get_with_code_lines() const { } int String::get_slice_count(String p_splitter) const { - if (empty()) { + if (is_empty()) { return 0; } - if (p_splitter.empty()) { + if (p_splitter.is_empty()) { return 0; } @@ -968,7 +1043,7 @@ int String::get_slice_count(String p_splitter) const { } String String::get_slice(String p_splitter, int p_slice) const { - if (empty() || p_splitter.empty()) { + if (is_empty() || p_splitter.is_empty()) { return ""; } @@ -1008,7 +1083,7 @@ String String::get_slice(String p_splitter, int p_slice) const { } String String::get_slicec(char32_t p_splitter, int p_slice) const { - if (empty()) { + if (is_empty()) { return String(); } @@ -1138,7 +1213,7 @@ Vector<String> String::rsplit(const String &p_splitter, bool p_allow_empty, int remaining_len = left_edge; } - ret.invert(); + ret.reverse(); return ret; } @@ -1312,10 +1387,18 @@ String String::num(double p_num, int p_decimals) { return "inf"; } } -#ifndef NO_USE_STDLIB - if (p_decimals > 16) { - p_decimals = 16; + if (p_decimals < 0) { + p_decimals = 14; + const double abs_num = ABS(p_num); + if (abs_num > 10) { + // We want to align the digits to the above sane default, so we only + // need to subtract log10 for numbers with a positive power of ten. + p_decimals -= (int)floor(log10(abs_num)); + } + } + if (p_decimals > MAX_DECIMALS) { + p_decimals = MAX_DECIMALS; } char fmt[7]; @@ -1326,7 +1409,6 @@ String String::num(double p_num, int p_decimals) { fmt[1] = 'l'; fmt[2] = 'f'; fmt[3] = 0; - } else if (p_decimals < 10) { fmt[2] = '0' + p_decimals; fmt[3] = 'l'; @@ -1377,83 +1459,6 @@ String String::num(double p_num, int p_decimals) { } return buf; -#else - - String s; - String sd; - /* integer part */ - - bool neg = p_num < 0; - p_num = ABS(p_num); - int intn = (int)p_num; - - /* decimal part */ - - if (p_decimals > 0 || (p_decimals == -1 && (int)p_num != p_num)) { - double dec = p_num - (double)((int)p_num); - - int digit = 0; - if (p_decimals > MAX_DIGITS) - p_decimals = MAX_DIGITS; - - int dec_int = 0; - int dec_max = 0; - - while (true) { - dec *= 10.0; - dec_int = dec_int * 10 + (int)dec % 10; - dec_max = dec_max * 10 + 9; - digit++; - - if (p_decimals == -1) { - if (digit == MAX_DIGITS) //no point in going to infinite - break; - - if ((dec - (double)((int)dec)) < 1e-6) - break; - } - - if (digit == p_decimals) - break; - } - dec *= 10; - int last = (int)dec % 10; - - if (last > 5) { - if (dec_int == dec_max) { - dec_int = 0; - intn++; - } else { - dec_int++; - } - } - - String decimal; - for (int i = 0; i < digit; i++) { - char num[2] = { 0, 0 }; - num[0] = '0' + dec_int % 10; - decimal = num + decimal; - dec_int /= 10; - } - sd = '.' + decimal; - } - - if (intn == 0) - - s = "0"; - else { - while (intn) { - char32_t num = '0' + (intn % 10); - intn /= 10; - s = num + s; - } - } - - s = s + sd; - if (neg) - s = "-" + s; - return s; -#endif } String String::num_int64(int64_t p_num, int base, bool capitalize_hex) { @@ -1523,7 +1528,7 @@ String String::num_uint64(uint64_t p_num, int base, bool capitalize_hex) { return s; } -String String::num_real(double p_num) { +String String::num_real(double p_num, bool p_trailing) { if (Math::is_nan(p_num)) { return "nan"; } @@ -1538,30 +1543,52 @@ String String::num_real(double p_num) { String s; String sd; - /* integer part */ + + // Integer part. bool neg = p_num < 0; p_num = ABS(p_num); - int intn = (int)p_num; + int64_t intn = (int64_t)p_num; - /* decimal part */ + // Decimal part. - if ((int)p_num != p_num) { - double dec = p_num - (double)((int)p_num); + if (intn != p_num) { + double dec = p_num - (double)intn; int digit = 0; - int decimals = MAX_DIGITS; - int dec_int = 0; - int dec_max = 0; +#ifdef REAL_T_IS_DOUBLE + int decimals = 14; + double tolerance = 1e-14; +#else + int decimals = 6; + double tolerance = 1e-6; +#endif + // We want to align the digits to the above sane default, so we only + // need to subtract log10 for numbers with a positive power of ten. + if (p_num > 10) { + decimals -= (int)floor(log10(p_num)); + } + + if (decimals > MAX_DECIMALS) { + decimals = MAX_DECIMALS; + } + + // In case the value ends up ending in "99999", we want to add a + // tiny bit to the value we're checking when deciding when to stop, + // so we multiply by slightly above 1 (1 + 1e-7 or 1e-15). + double check_multiplier = 1 + tolerance / 10; + + int64_t dec_int = 0; + int64_t dec_max = 0; while (true) { dec *= 10.0; - dec_int = dec_int * 10 + (int)dec % 10; + dec_int = dec_int * 10 + (int64_t)dec % 10; dec_max = dec_max * 10 + 9; digit++; - if ((dec - (double)((int)dec)) < 1e-6) { + if ((dec - (double)(int64_t)(dec * check_multiplier)) < tolerance) { break; } @@ -1571,7 +1598,7 @@ String String::num_real(double p_num) { } dec *= 10; - int last = (int)dec % 10; + int last = (int64_t)dec % 10; if (last > 5) { if (dec_int == dec_max) { @@ -1590,8 +1617,10 @@ String String::num_real(double p_num) { dec_int /= 10; } sd = '.' + decimal; - } else { + } else if (p_trailing) { sd = ".0"; + } else { + sd = ""; } if (intn == 0) { @@ -1623,19 +1652,18 @@ String String::num_scientific(double p_num) { return "inf"; } } -#ifndef NO_USE_STDLIB char buf[256]; #if defined(__GNUC__) || defined(_MSC_VER) -#if (defined(__MINGW32__) || (defined(_MSC_VER) && _MSC_VER < 1900)) && defined(_TWO_DIGIT_EXPONENT) && !defined(_UCRT) - // MinGW and old MSC require _set_output_format() to conform to C99 output for printf +#if defined(__MINGW32__) && defined(_TWO_DIGIT_EXPONENT) && !defined(_UCRT) + // MinGW requires _set_output_format() to conform to C99 output for printf unsigned int old_exponent_format = _set_output_format(_TWO_DIGIT_EXPONENT); #endif snprintf(buf, 256, "%lg", p_num); -#if (defined(__MINGW32__) || (defined(_MSC_VER) && _MSC_VER < 1900)) && defined(_TWO_DIGIT_EXPONENT) && !defined(_UCRT) +#if defined(__MINGW32__) && defined(_TWO_DIGIT_EXPONENT) && !defined(_UCRT) _set_output_format(old_exponent_format); #endif @@ -1646,10 +1674,6 @@ String String::num_scientific(double p_num) { buf[255] = 0; return buf; -#else - - return String::num(p_num); -#endif } String String::md5(const uint8_t *p_md5) { @@ -1763,7 +1787,7 @@ bool String::parse_utf8(const char *p_utf8, int p_len) { if (skip) { _UNICERROR("no space left"); - return true; //not enough spac + return true; //not enough space } } @@ -2096,8 +2120,9 @@ String::String(const StrRange &p_range) { copy_from(p_range.c_str, p_range.len); } -int64_t String::hex_to_int(bool p_with_prefix) const { - if (p_with_prefix && length() < 3) { +int64_t String::hex_to_int() const { + int len = length(); + if (len == 0) { return 0; } @@ -2109,24 +2134,21 @@ int64_t String::hex_to_int(bool p_with_prefix) const { s++; } - if (p_with_prefix) { - if (s[0] != '0' || s[1] != 'x') { - return 0; - } + if (len > 2 && s[0] == '0' && lower_case(s[1]) == 'x') { s += 2; } int64_t hex = 0; while (*s) { - char32_t c = LOWERCASE(*s); + char32_t c = lower_case(*s); int64_t n; - if (c >= '0' && c <= '9') { + if (is_digit(c)) { n = c - '0'; } else if (c >= 'a' && c <= 'f') { n = (c - 'a') + 10; } else { - return 0; + ERR_FAIL_COND_V_MSG(true, 0, "Invalid hexadecimal notation character \"" + chr(*s) + "\" in string \"" + *this + "\"."); } // Check for overflow/underflow, with special case to ensure INT64_MIN does not result in error bool overflow = ((hex > INT64_MAX / 16) && (sign == 1 || (sign == -1 && hex != (INT64_MAX >> 4) + 1))) || (sign == -1 && hex == (INT64_MAX >> 4) + 1 && c > '0'); @@ -2139,8 +2161,9 @@ int64_t String::hex_to_int(bool p_with_prefix) const { return hex * sign; } -int64_t String::bin_to_int(bool p_with_prefix) const { - if (p_with_prefix && length() < 3) { +int64_t String::bin_to_int() const { + int len = length(); + if (len == 0) { return 0; } @@ -2152,17 +2175,14 @@ int64_t String::bin_to_int(bool p_with_prefix) const { s++; } - if (p_with_prefix) { - if (s[0] != '0' || s[1] != 'b') { - return 0; - } + if (len > 2 && s[0] == '0' && lower_case(s[1]) == 'b') { s += 2; } int64_t binary = 0; while (*s) { - char32_t c = LOWERCASE(*s); + char32_t c = lower_case(*s); int64_t n; if (c == '0' || c == '1') { n = c - '0'; @@ -2192,7 +2212,7 @@ int64_t String::to_int() const { for (int i = 0; i < to; i++) { char32_t c = operator[](i); - if (c >= '0' && c <= '9') { + if (is_digit(c)) { bool overflow = (integer > INT64_MAX / 10) || (integer == INT64_MAX / 10 && ((sign == 1 && c > '7') || (sign == -1 && c > '8'))); ERR_FAIL_COND_V_MSG(overflow, sign == 1 ? INT64_MAX : INT64_MIN, "Cannot represent " + *this + " as 64-bit integer, provided value is " + (sign == 1 ? "too big." : "too small.")); integer *= 10; @@ -2221,7 +2241,7 @@ int64_t String::to_int(const char *p_str, int p_len) { for (int i = 0; i < to; i++) { char c = p_str[i]; - if (c >= '0' && c <= '9') { + if (is_digit(c)) { bool overflow = (integer > INT64_MAX / 10) || (integer == INT64_MAX / 10 && ((sign == 1 && c > '7') || (sign == -1 && c > '8'))); ERR_FAIL_COND_V_MSG(overflow, sign == 1 ? INT64_MAX : INT64_MIN, "Cannot represent " + String(p_str).substr(0, to) + " as integer, provided value is " + (sign == 1 ? "too big." : "too small.")); integer *= 10; @@ -2252,7 +2272,7 @@ int64_t String::to_int(const wchar_t *p_str, int p_len) { for (int i = 0; i < to; i++) { wchar_t c = p_str[i]; - if (c >= '0' && c <= '9') { + if (is_digit(c)) { bool overflow = (integer > INT64_MAX / 10) || (integer == INT64_MAX / 10 && ((sign == 1 && c > '7') || (sign == -1 && c > '8'))); ERR_FAIL_COND_V_MSG(overflow, sign == 1 ? INT64_MAX : INT64_MIN, "Cannot represent " + String(p_str).substr(0, to) + " as integer, provided value is " + (sign == 1 ? "too big." : "too small.")); integer *= 10; @@ -2294,28 +2314,33 @@ bool String::is_numeric() const { } template <class C> -static double built_in_strtod(const C *string, /* A decimal ASCII floating-point number, - * optionally preceded by white space. Must - * have form "-I.FE-X", where I is the integer - * part of the mantissa, F is the fractional - * part of the mantissa, and X is the - * exponent. Either of the signs may be "+", - * "-", or omitted. Either I or F may be - * omitted, or both. The decimal point isn't - * necessary unless F is present. The "E" may - * actually be an "e". E and X may both be - * omitted (but not just one). */ - C **endPtr = nullptr) /* If non-nullptr, store terminating Cacter's - * address here. */ -{ - static const int maxExponent = 511; /* Largest possible base 10 exponent. Any - * exponent larger than this will already - * produce underflow or overflow, so there's - * no need to worry about additional digits. - */ - static const double powersOf10[] = { /* Table giving binary powers of 10. Entry */ - 10., /* is 10^2^i. Used to convert decimal */ - 100., /* exponents into floating-point numbers. */ +static double built_in_strtod( + /* A decimal ASCII floating-point number, + * optionally preceded by white space. Must + * have form "-I.FE-X", where I is the integer + * part of the mantissa, F is the fractional + * part of the mantissa, and X is the + * exponent. Either of the signs may be "+", + * "-", or omitted. Either I or F may be + * omitted, or both. The decimal point isn't + * necessary unless F is present. The "E" may + * actually be an "e". E and X may both be + * omitted (but not just one). */ + const C *string, + /* If non-nullptr, store terminating Cacter's + * address here. */ + C **endPtr = nullptr) { + /* Largest possible base 10 exponent. Any + * exponent larger than this will already + * produce underflow or overflow, so there's + * no need to worry about additional digits. */ + static const int maxExponent = 511; + /* Table giving binary powers of 10. Entry + * is 10^2^i. Used to convert decimal + * exponents into floating-point numbers. */ + static const double powersOf10[] = { + 10., + 100., 1.0e4, 1.0e8, 1.0e16, @@ -2330,25 +2355,28 @@ static double built_in_strtod(const C *string, /* A decimal ASCII floating-point const double *d; const C *p; int c; - int exp = 0; /* Exponent read from "EX" field. */ - int fracExp = 0; /* Exponent that derives from the fractional - * part. Under normal circumstances, it is - * the negative of the number of digits in F. - * However, if I is very long, the last digits - * of I get dropped (otherwise a long I with a - * large negative exponent could cause an - * unnecessary overflow on I alone). In this - * case, fracExp is incremented one for each - * dropped digit. */ - int mantSize; /* Number of digits in mantissa. */ - int decPt; /* Number of mantissa digits BEFORE decimal - * point. */ - const C *pExp; /* Temporarily holds location of exponent in - * string. */ + /* Exponent read from "EX" field. */ + int exp = 0; + /* Exponent that derives from the fractional + * part. Under normal circumstances, it is + * the negative of the number of digits in F. + * However, if I is very long, the last digits + * of I get dropped (otherwise a long I with a + * large negative exponent could cause an + * unnecessary overflow on I alone). In this + * case, fracExp is incremented one for each + * dropped digit. */ + int fracExp = 0; + /* Number of digits in mantissa. */ + int mantSize; + /* Number of mantissa digits BEFORE decimal point. */ + int decPt; + /* Temporarily holds location of exponent in string. */ + const C *pExp; /* - * Strip off leading blanks and check for a sign. - */ + * Strip off leading blanks and check for a sign. + */ p = string; while (*p == ' ' || *p == '\t' || *p == '\n') { @@ -2365,14 +2393,14 @@ static double built_in_strtod(const C *string, /* A decimal ASCII floating-point } /* - * Count the number of digits in the mantissa (including the decimal - * point), and also locate the decimal point. - */ + * Count the number of digits in the mantissa (including the decimal + * point), and also locate the decimal point. + */ decPt = -1; for (mantSize = 0;; mantSize += 1) { c = *p; - if (!IS_DIGIT(c)) { + if (!is_digit(c)) { if ((c != '.') || (decPt >= 0)) { break; } @@ -2382,11 +2410,11 @@ static double built_in_strtod(const C *string, /* A decimal ASCII floating-point } /* - * Now suck up the digits in the mantissa. Use two integers to collect 9 - * digits each (this is faster than using floating-point). If the mantissa - * has more than 18 digits, ignore the extras, since they can't affect the - * value anyway. - */ + * Now suck up the digits in the mantissa. Use two integers to collect 9 + * digits each (this is faster than using floating-point). If the mantissa + * has more than 18 digits, ignore the extras, since they can't affect the + * value anyway. + */ pExp = p; p -= mantSize; @@ -2432,8 +2460,8 @@ static double built_in_strtod(const C *string, /* A decimal ASCII floating-point } /* - * Skim off the exponent. - */ + * Skim off the exponent. + */ p = pExp; if ((*p == 'E') || (*p == 'e')) { @@ -2447,11 +2475,11 @@ static double built_in_strtod(const C *string, /* A decimal ASCII floating-point } expSign = false; } - if (!IS_DIGIT(char32_t(*p))) { + if (!is_digit(char32_t(*p))) { p = pExp; goto done; } - while (IS_DIGIT(char32_t(*p))) { + while (is_digit(char32_t(*p))) { exp = exp * 10 + (*p - '0'); p += 1; } @@ -2463,10 +2491,10 @@ static double built_in_strtod(const C *string, /* A decimal ASCII floating-point } /* - * Generate a floating-point number that represents the exponent. Do this - * by processing the exponent one bit at a time to combine many powers of - * 2 of 10. Then combine the exponent with the fraction. - */ + * Generate a floating-point number that represents the exponent. Do this + * by processing the exponent one bit at a time to combine many powers of + * 2 of 10. Then combine the exponent with the fraction. + */ if (exp < 0) { expSign = true; @@ -2537,7 +2565,7 @@ int64_t String::to_int(const char32_t *p_str, int p_len, bool p_clamp) { char32_t c = *(str++); switch (reading) { case READING_SIGN: { - if (c >= '0' && c <= '9') { + if (is_digit(c)) { reading = READING_INT; // let it fallthrough } else if (c == '-') { @@ -2554,7 +2582,7 @@ int64_t String::to_int(const char32_t *p_str, int p_len, bool p_clamp) { [[fallthrough]]; } case READING_INT: { - if (c >= '0' && c <= '9') { + if (is_digit(c)) { if (integer > INT64_MAX / 10) { String number(""); str = p_str; @@ -2585,7 +2613,7 @@ int64_t String::to_int(const char32_t *p_str, int p_len, bool p_clamp) { } double String::to_float() const { - if (empty()) { + if (is_empty()) { return 0; } return built_in_strtod<char32_t>(get_data()); @@ -2767,7 +2795,7 @@ String String::substr(int p_from, int p_chars) const { p_chars = length() - p_from; } - if (empty() || p_from < 0 || p_from >= length() || p_chars <= 0) { + if (is_empty() || p_from < 0 || p_from >= length() || p_chars <= 0) { return ""; } @@ -3074,35 +3102,47 @@ int String::rfindn(const String &p_str, int p_from) const { } bool String::ends_with(const String &p_string) const { - int pos = rfind(p_string); - if (pos == -1) { + int l = p_string.length(); + if (l > length()) { return false; } - return pos + p_string.length() == length(); + + if (l == 0) { + return true; + } + + const char32_t *p = &p_string[0]; + const char32_t *s = &operator[](length() - l); + + for (int i = 0; i < l; i++) { + if (p[i] != s[i]) { + return false; + } + } + + return true; } bool String::begins_with(const String &p_string) const { - if (p_string.length() > length()) { + int l = p_string.length(); + if (l > length()) { return false; } - int l = p_string.length(); if (l == 0) { return true; } - const char32_t *src = &p_string[0]; - const char32_t *str = &operator[](0); + const char32_t *p = &p_string[0]; + const char32_t *s = &operator[](0); - int i = 0; - for (; i < l; i++) { - if (src[i] != str[i]) { + for (int i = 0; i < l; i++) { + if (p[i] != s[i]) { return false; } } - // only if i == l the p_string matches the beginning - return i == l; + return true; } bool String::begins_with(const char *p_string) const { @@ -3142,7 +3182,7 @@ bool String::is_quoted() const { } int String::_count(const String &p_string, int p_from, int p_to, bool p_case_insensitive) const { - if (p_string.empty()) { + if (p_string.is_empty()) { return 0; } int len = length(); @@ -3250,8 +3290,8 @@ float String::similarity(const String &p_string) const { int src_size = src_bigrams.size(); int tgt_size = tgt_bigrams.size(); - double sum = src_size + tgt_size; - double inter = 0; + int sum = src_size + tgt_size; + int inter = 0; for (int i = 0; i < src_size; i++) { for (int j = 0; j < tgt_size; j++) { if (src_bigrams[i] == tgt_bigrams[j]) { @@ -3308,17 +3348,10 @@ String String::format(const Variant &values, String placeholder) const { if (value_arr.size() == 2) { Variant v_key = value_arr[0]; String key = v_key; - if (key.left(1) == "\"" && key.right(key.length() - 1) == "\"") { - key = key.substr(1, key.length() - 2); - } Variant v_val = value_arr[1]; String val = v_val; - if (val.left(1) == "\"" && val.right(val.length() - 1) == "\"") { - val = val.substr(1, val.length() - 2); - } - new_string = new_string.replace(placeholder.replace("_", key), val); } else { ERR_PRINT(String("STRING.format Inner Array size != 2 ").ascii().get_data()); @@ -3327,10 +3360,6 @@ String String::format(const Variant &values, String placeholder) const { Variant v_val = values_arr[i]; String val = v_val; - if (val.left(1) == "\"" && val.right(val.length() - 1) == "\"") { - val = val.substr(1, val.length() - 2); - } - if (placeholder.find("_") > -1) { new_string = new_string.replace(placeholder.replace("_", i_as_str), val); } else { @@ -3343,19 +3372,8 @@ String String::format(const Variant &values, String placeholder) const { List<Variant> keys; d.get_key_list(&keys); - for (List<Variant>::Element *E = keys.front(); E; E = E->next()) { - String key = E->get(); - String val = d[E->get()]; - - if (key.left(1) == "\"" && key.right(key.length() - 1) == "\"") { - key = key.substr(1, key.length() - 2); - } - - if (val.left(1) == "\"" && val.right(val.length() - 1) == "\"") { - val = val.substr(1, val.length() - 2); - } - - new_string = new_string.replace(placeholder.replace("_", key), val); + for (const Variant &key : keys) { + new_string = new_string.replace(placeholder.replace("_", key), d[key]); } } else { ERR_PRINT(String("Invalid type: use Array or Dictionary.").ascii().get_data()); @@ -3455,6 +3473,10 @@ String String::repeat(int p_count) const { } String String::left(int p_pos) const { + if (p_pos < 0) { + p_pos = length() + p_pos; + } + if (p_pos <= 0) { return ""; } @@ -3467,18 +3489,22 @@ String String::left(int p_pos) const { } String String::right(int p_pos) const { - if (p_pos >= length()) { - return ""; + if (p_pos < 0) { + p_pos = length() + p_pos; } if (p_pos <= 0) { + return ""; + } + + if (p_pos >= length()) { return *this; } - return substr(p_pos, (length() - p_pos)); + return substr(length() - p_pos); } -char32_t String::ord_at(int p_idx) const { +char32_t String::unicode_at(int p_idx) const { ERR_FAIL_INDEX_V(p_idx, length(), 0); return operator[](p_idx); } @@ -3541,7 +3567,7 @@ String String::strip_edges(bool left, bool right) const { } if (right) { - for (int i = (int)(len - 1); i >= 0; i--) { + for (int i = len - 1; i >= 0; i--) { if (operator[](i) <= 32) { end--; } else { @@ -3704,7 +3730,7 @@ String String::humanize_size(uint64_t p_size) { return String::num(p_size / divisor).pad_decimals(digits) + " " + prefixes[prefix_idx]; } -bool String::is_abs_path() const { +bool String::is_absolute_path() const { if (length() > 1) { return (operator[](0) == '/' || operator[](0) == '\\' || find(":/") != -1 || find(":\\") != -1); } else if ((length()) == 1) { @@ -3725,12 +3751,12 @@ bool String::is_valid_identifier() const { for (int i = 0; i < len; i++) { if (i == 0) { - if (str[0] >= '0' && str[0] <= '9') { + if (is_digit(str[0])) { return false; // no start with number plz } } - bool valid_char = (str[i] >= '0' && str[i] <= '9') || (str[i] >= 'a' && str[i] <= 'z') || (str[i] >= 'A' && str[i] <= 'Z') || str[i] == '_'; + bool valid_char = is_digit(str[i]) || is_lower_case(str[i]) || is_upper_case(str[i]) || str[i] == '_'; if (!valid_char) { return false; @@ -3750,22 +3776,19 @@ bool String::is_valid_string() const { return valid; } -String String::http_escape() const { +String String::uri_encode() const { const CharString temp = utf8(); String res; for (int i = 0; i < temp.length(); ++i) { char ord = temp[i]; - if (ord == '.' || ord == '-' || ord == '_' || ord == '~' || - (ord >= 'a' && ord <= 'z') || - (ord >= 'A' && ord <= 'Z') || - (ord >= '0' && ord <= '9')) { + if (ord == '.' || ord == '-' || ord == '_' || ord == '~' || is_lower_case(ord) || is_upper_case(ord) || is_digit(ord)) { res += ord; } else { char h_Val[3]; #if defined(__GNUC__) || defined(_MSC_VER) - snprintf(h_Val, 3, "%hhX", ord); + snprintf(h_Val, 3, "%02hhX", ord); #else - sprintf(h_Val, "%hhX", ord); + sprintf(h_Val, "%02hhX", ord); #endif res += "%"; res += h_Val; @@ -3774,26 +3797,29 @@ String String::http_escape() const { return res; } -String String::http_unescape() const { - String res; - for (int i = 0; i < length(); ++i) { - if (ord_at(i) == '%' && i + 2 < length()) { - char32_t ord1 = ord_at(i + 1); - if ((ord1 >= '0' && ord1 <= '9') || (ord1 >= 'A' && ord1 <= 'Z')) { - char32_t ord2 = ord_at(i + 2); - if ((ord2 >= '0' && ord2 <= '9') || (ord2 >= 'A' && ord2 <= 'Z')) { +String String::uri_decode() const { + CharString src = utf8(); + CharString res; + for (int i = 0; i < src.length(); ++i) { + if (src[i] == '%' && i + 2 < src.length()) { + char ord1 = src[i + 1]; + if (is_digit(ord1) || is_upper_case(ord1)) { + char ord2 = src[i + 2]; + if (is_digit(ord2) || is_upper_case(ord2)) { char bytes[3] = { (char)ord1, (char)ord2, 0 }; res += (char)strtol(bytes, nullptr, 16); i += 2; } } else { - res += ord_at(i); + res += src[i]; } + } else if (src[i] == '+') { + res += ' '; } else { - res += ord_at(i); + res += src[i]; } } - return String::utf8(res.ascii()); + return String::utf8(res); } String String::c_unescape() const { @@ -3877,25 +3903,55 @@ static _FORCE_INLINE_ int _xml_unescape(const char32_t *p_src, int p_src_len, ch if (p_src_len >= 4 && p_src[1] == '#') { char32_t c = 0; - - for (int i = 2; i < p_src_len; i++) { - eat = i + 1; - char32_t ct = p_src[i]; - if (ct == ';') { - break; - } else if (ct >= '0' && ct <= '9') { - ct = ct - '0'; - } else if (ct >= 'a' && ct <= 'f') { - ct = (ct - 'a') + 10; - } else if (ct >= 'A' && ct <= 'F') { - ct = (ct - 'A') + 10; - } else { - continue; + bool overflow = false; + if (p_src[2] == 'x') { + // Hex entity &#x<num>; + for (int i = 3; i < p_src_len; i++) { + eat = i + 1; + char32_t ct = p_src[i]; + if (ct == ';') { + break; + } else if (is_digit(ct)) { + ct = ct - '0'; + } else if (ct >= 'a' && ct <= 'f') { + ct = (ct - 'a') + 10; + } else if (ct >= 'A' && ct <= 'F') { + ct = (ct - 'A') + 10; + } else { + break; + } + if (c > (UINT32_MAX >> 4)) { + overflow = true; + break; + } + c <<= 4; + c |= ct; + } + } else { + // Decimal entity &#<num>; + for (int i = 2; i < p_src_len; i++) { + eat = i + 1; + char32_t ct = p_src[i]; + if (ct == ';' || ct < '0' || ct > '9') { + break; + } + } + if (p_src[eat - 1] == ';') { + int64_t val = String::to_int(p_src + 2, eat - 3); + if (val > 0 && val <= UINT32_MAX) { + c = (char32_t)val; + } else { + overflow = true; + } } - c <<= 4; - c |= ct; } + // Value must be non-zero, in the range of char32_t, + // actually end with ';'. If invalid, leave the entity as-is + if (c == '\0' || overflow || p_src[eat - 1] != ';') { + eat = 1; + c = *p_src; + } if (p_dst) { *p_dst = c; } @@ -4038,7 +4094,7 @@ String String::trim_suffix(const String &p_suffix) const { return s; } -bool String::is_valid_integer() const { +bool String::is_valid_int() const { int len = length(); if (len == 0) { @@ -4083,7 +4139,7 @@ bool String::is_valid_hex_number(bool p_with_prefix) const { for (int i = from; i < len; i++) { char32_t c = operator[](i); - if ((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F')) { + if (is_hex_digit(c)) { continue; } return false; @@ -4111,7 +4167,7 @@ bool String::is_valid_float() const { bool numbers_found = false; for (int i = from; i < len; i++) { - if (operator[](i) >= '0' && operator[](i) <= '9') { + if (is_digit(operator[](i))) { if (exponent_found) { exponent_values_found = true; } else { @@ -4241,11 +4297,11 @@ bool String::is_valid_ip_address() const { Vector<String> ip = split(":"); for (int i = 0; i < ip.size(); i++) { String n = ip[i]; - if (n.empty()) { + if (n.is_empty()) { continue; } if (n.is_valid_hex_number(false)) { - int64_t nint = n.hex_to_int(false); + int64_t nint = n.hex_to_int(); if (nint < 0 || nint > 0xffff) { return false; } @@ -4263,7 +4319,7 @@ bool String::is_valid_ip_address() const { } for (int i = 0; i < ip.size(); i++) { String n = ip[i]; - if (!n.is_valid_integer()) { + if (!n.is_valid_int()) { return false; } int val = n.to_int(); @@ -4280,28 +4336,44 @@ bool String::is_resource_file() const { return begins_with("res://") && find("::") == -1; } -bool String::is_rel_path() const { - return !is_abs_path(); +bool String::is_relative_path() const { + return !is_absolute_path(); } String String::get_base_dir() const { - int basepos = find(":/"); - if (basepos == -1) { - basepos = find(":\\"); + int end = 0; + + // url scheme style base + int basepos = find("://"); + if (basepos != -1) { + end = basepos + 3; } + + // windows top level directory base + if (end == 0) { + basepos = find(":/"); + if (basepos == -1) { + basepos = find(":\\"); + } + if (basepos != -1) { + end = basepos + 2; + } + } + + // unix root directory base + if (end == 0) { + if (begins_with("/")) { + end = 1; + } + } + String rs; String base; - if (basepos != -1) { - int end = basepos + 3; + if (end != 0) { rs = substr(end, length()); base = substr(0, end); } else { - if (begins_with("/")) { - rs = substr(1, length()); - base = "/"; - } else { - rs = *this; - } + rs = *this; } int sep = MAX(rs.rfind("/"), rs.rfind("\\")); @@ -4331,7 +4403,7 @@ String String::get_extension() const { } String String::plus_file(const String &p_file) const { - if (empty()) { + if (is_empty()) { return p_file; } if (operator[](length() - 1) == '/' || (p_file.size() > 0 && p_file.operator[](0) == '/')) { @@ -4340,69 +4412,12 @@ String String::plus_file(const String &p_file) const { return *this + "/" + p_file; } -String String::percent_encode() const { - CharString cs = utf8(); - String encoded; - for (int i = 0; i < cs.length(); i++) { - uint8_t c = cs[i]; - if ((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '-' || c == '_' || c == '~' || c == '.') { - char p[2] = { (char)c, 0 }; - encoded += p; - } else { - char p[4] = { '%', 0, 0, 0 }; - static const char hex[16] = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' }; - - p[1] = hex[c >> 4]; - p[2] = hex[c & 0xF]; - encoded += p; - } - } - - return encoded; -} - -String String::percent_decode() const { - CharString pe; - - CharString cs = utf8(); - for (int i = 0; i < cs.length(); i++) { - uint8_t c = cs[i]; - if (c == '%' && i < length() - 2) { - uint8_t a = LOWERCASE(cs[i + 1]); - uint8_t b = LOWERCASE(cs[i + 2]); - - if (a >= '0' && a <= '9') { - c = (a - '0') << 4; - } else if (a >= 'a' && a <= 'f') { - c = (a - 'a' + 10) << 4; - } else { - continue; - } - - uint8_t d = 0; - - if (b >= '0' && b <= '9') { - d = (b - '0'); - } else if (b >= 'a' && b <= 'f') { - d = (b - 'a' + 10); - } else { - continue; - } - c += d; - i += 2; - } - pe += c; - } - - return String::utf8(pe.ptr()); -} - String String::property_name_encode() const { // Escape and quote strings with extended ASCII or further Unicode characters // as well as '"', '=' or ' ' (32) const char32_t *cstr = get_data(); for (int i = 0; cstr[i]; i++) { - if (cstr[i] == '=' || cstr[i] == '"' || cstr[i] < 33 || cstr[i] > 126) { + if (cstr[i] == '=' || cstr[i] == '"' || cstr[i] == ';' || cstr[i] == '[' || cstr[i] == ']' || cstr[i] < 33 || cstr[i] > 126) { return "\"" + c_escape_multiline() + "\""; } } @@ -4410,6 +4425,18 @@ String String::property_name_encode() const { return *this; } +// Changes made to the set of invalid characters must also be reflected in the String documentation. +const String String::invalid_node_name_characters = ". : @ / \""; + +String String::validate_node_name() const { + Vector<String> chars = String::invalid_node_name_characters.split(" "); + String name = this->replace(chars[0], ""); + for (int i = 1; i < chars.size(); i++) { + name = name.replace(chars[i], ""); + } + return name; +} + String String::get_basename() const { int pos = rfind("."); if (pos < 0 || pos < MAX(rfind("/"), rfind("\\"))) { @@ -4484,7 +4511,7 @@ String String::sprintf(const Array &values, bool *error) const { for (; *self; self++) { const char32_t c = *self; - if (in_format) { // We have % - lets see what else we get. + if (in_format) { // We have % - let's see what else we get. switch (c) { case '%': { // Replace %% with % formatted += chr(c); @@ -4760,7 +4787,7 @@ String String::unquote() const { Vector<uint8_t> String::to_ascii_buffer() const { const String *s = this; - if (s->empty()) { + if (s->is_empty()) { return Vector<uint8_t>(); } CharString charstr = s->ascii(); @@ -4769,14 +4796,14 @@ Vector<uint8_t> String::to_ascii_buffer() const { size_t len = charstr.length(); retval.resize(len); uint8_t *w = retval.ptrw(); - copymem(w, charstr.ptr(), len); + memcpy(w, charstr.ptr(), len); return retval; } Vector<uint8_t> String::to_utf8_buffer() const { const String *s = this; - if (s->empty()) { + if (s->is_empty()) { return Vector<uint8_t>(); } CharString charstr = s->utf8(); @@ -4785,14 +4812,14 @@ Vector<uint8_t> String::to_utf8_buffer() const { size_t len = charstr.length(); retval.resize(len); uint8_t *w = retval.ptrw(); - copymem(w, charstr.ptr(), len); + memcpy(w, charstr.ptr(), len); return retval; } Vector<uint8_t> String::to_utf16_buffer() const { const String *s = this; - if (s->empty()) { + if (s->is_empty()) { return Vector<uint8_t>(); } Char16String charstr = s->utf16(); @@ -4801,14 +4828,14 @@ Vector<uint8_t> String::to_utf16_buffer() const { size_t len = charstr.length() * sizeof(char16_t); retval.resize(len); uint8_t *w = retval.ptrw(); - copymem(w, (const void *)charstr.ptr(), len); + memcpy(w, (const void *)charstr.ptr(), len); return retval; } Vector<uint8_t> String::to_utf32_buffer() const { const String *s = this; - if (s->empty()) { + if (s->is_empty()) { return Vector<uint8_t>(); } @@ -4816,7 +4843,7 @@ Vector<uint8_t> String::to_utf32_buffer() const { size_t len = s->length() * sizeof(char32_t); retval.resize(len); uint8_t *w = retval.ptrw(); - copymem(w, (const void *)s->ptr(), len); + memcpy(w, (const void *)s->ptr(), len); return retval; } @@ -4842,15 +4869,20 @@ String TTRN(const String &p_text, const String &p_text_plural, int p_n, const St return p_text_plural; } +/* DTR and DTRN are used for the documentation, handling descriptions extracted + * from the XML. + * They also replace `$DOCS_URL` with the actual URL to the documentation's branch, + * to allow dehardcoding it in the XML and doing proper substitutions everywhere. + */ String DTR(const String &p_text, const String &p_context) { // Comes straight from the XML, so remove indentation and any trailing whitespace. const String text = p_text.dedent().strip_edges(); if (TranslationServer::get_singleton()) { - return TranslationServer::get_singleton()->doc_translate(text, p_context); + return String(TranslationServer::get_singleton()->doc_translate(text, p_context)).replace("$DOCS_URL", VERSION_DOCS_URL); } - return text; + return text.replace("$DOCS_URL", VERSION_DOCS_URL); } String DTRN(const String &p_text, const String &p_text_plural, int p_n, const String &p_context) { @@ -4858,14 +4890,14 @@ String DTRN(const String &p_text, const String &p_text_plural, int p_n, const St const String text_plural = p_text_plural.dedent().strip_edges(); if (TranslationServer::get_singleton()) { - return TranslationServer::get_singleton()->doc_translate_plural(text, text_plural, p_n, p_context); + return String(TranslationServer::get_singleton()->doc_translate_plural(text, text_plural, p_n, p_context)).replace("$DOCS_URL", VERSION_DOCS_URL); } // Return message based on English plural rule if translation is not possible. if (p_n == 1) { - return text; + return text.replace("$DOCS_URL", VERSION_DOCS_URL); } - return text_plural; + return text_plural.replace("$DOCS_URL", VERSION_DOCS_URL); } #endif diff --git a/core/string/ustring.h b/core/string/ustring.h index 7ff78b2d86..1d80ccf58d 100644 --- a/core/string/ustring.h +++ b/core/string/ustring.h @@ -5,8 +5,8 @@ /* GODOT ENGINE */ /* https://godotengine.org */ /*************************************************************************/ -/* Copyright (c) 2007-2020 Juan Linietsky, Ariel Manzur. */ -/* Copyright (c) 2014-2020 Godot Engine contributors (cf. AUTHORS.md). */ +/* Copyright (c) 2007-2021 Juan Linietsky, Ariel Manzur. */ +/* Copyright (c) 2014-2021 Godot Engine contributors (cf. AUTHORS.md). */ /* */ /* Permission is hereby granted, free of charge, to any person obtaining */ /* a copy of this software and associated documentation files (the */ @@ -51,11 +51,15 @@ class CharProxy { CowData<T> &_cowdata; static const T _null = 0; - _FORCE_INLINE_ CharProxy(const int &p_index, CowData<T> &cowdata) : + _FORCE_INLINE_ CharProxy(const int &p_index, CowData<T> &p_cowdata) : _index(p_index), - _cowdata(cowdata) {} + _cowdata(p_cowdata) {} public: + _FORCE_INLINE_ CharProxy(const CharProxy<T> &p_other) : + _index(p_other._index), + _cowdata(p_other._cowdata) {} + _FORCE_INLINE_ operator T() const { if (unlikely(_index == _cowdata.size())) { return _null; @@ -68,12 +72,12 @@ public: return _cowdata.ptr() + _index; } - _FORCE_INLINE_ void operator=(const T &other) const { - _cowdata.set(_index, other); + _FORCE_INLINE_ void operator=(const T &p_other) const { + _cowdata.set(_index, p_other); } - _FORCE_INLINE_ void operator=(const CharProxy<T> &other) const { - _cowdata.set(_index, other.operator T()); + _FORCE_INLINE_ void operator=(const CharProxy<T> &p_other) const { + _cowdata.set(_index, p_other.operator T()); } }; @@ -309,7 +313,7 @@ public: String unquote() const; static String num(double p_num, int p_decimals = -1); static String num_scientific(double p_num); - static String num_real(double p_num); + static String num_real(double p_num, bool p_trailing = true); static String num_int64(int64_t p_num, int base = 10, bool capitalize_hex = false); static String num_uint64(uint64_t p_num, int base = 10, bool capitalize_hex = false); static String chr(char32_t p_char); @@ -318,8 +322,8 @@ public: bool is_numeric() const; double to_float() const; - int64_t hex_to_int(bool p_with_prefix = true) const; - int64_t bin_to_int(bool p_with_prefix = true) const; + int64_t hex_to_int() const; + int64_t bin_to_int() const; int64_t to_int() const; static int64_t to_int(const char *p_str, int p_len = -1); @@ -366,7 +370,7 @@ public: String get_extension() const; String get_basename() const; String plus_file(const String &p_file) const; - char32_t ord_at(int p_idx) const; + char32_t unicode_at(int p_idx) const; void erase(int p_pos, int p_chars); @@ -394,11 +398,11 @@ public: Vector<uint8_t> sha1_buffer() const; Vector<uint8_t> sha256_buffer() const; - _FORCE_INLINE_ bool empty() const { return length() == 0; } + _FORCE_INLINE_ bool is_empty() const { return length() == 0; } // path functions - bool is_abs_path() const; - bool is_rel_path() const; + bool is_absolute_path() const; + bool is_relative_path() const; bool is_resource_file() const; String path_to(const String &p_path) const; String path_to_file(const String &p_path) const; @@ -409,21 +413,23 @@ public: String xml_escape(bool p_escape_quotes = false) const; String xml_unescape() const; - String http_escape() const; - String http_unescape() const; + String uri_encode() const; + String uri_decode() const; String c_escape() const; String c_escape_multiline() const; String c_unescape() const; String json_escape() const; String word_wrap(int p_chars_per_line) const; - - String percent_encode() const; - String percent_decode() const; + Error parse_url(String &r_scheme, String &r_host, int &r_port, String &r_path) const; String property_name_encode() const; + // node functions + static const String invalid_node_name_characters; + String validate_node_name() const; + bool is_valid_identifier() const; - bool is_valid_integer() const; + bool is_valid_int() const; bool is_valid_float() const; bool is_valid_hex_number(bool p_with_prefix) const; bool is_valid_html_color() const; @@ -521,10 +527,10 @@ String DTRN(const String &p_text, const String &p_text_plural, int p_n, const St #define TTRGET(m_value) TTR(m_value) #else -#define TTR(m_value) (String()) -#define TTRN(m_value) (String()) -#define DTR(m_value) (String()) -#define DTRN(m_value) (String()) +#define TTR(m_value) String() +#define TTRN(m_value) String() +#define DTR(m_value) String() +#define DTRN(m_value) String() #define TTRC(m_value) (m_value) #define TTRGET(m_value) (m_value) #endif |