Merge pull request #34103 from akien-mga/resourceloader-remap-locale-near-matching

Translation: Use proper language code for locale near matching, including in ResourceLoader

3 files changed, 97 insertions, 48 deletions

diff --git a/core/io/resource_loader.cpp b/core/io/resource_loader.cpp
index f3eba44973..6f64543b3e 100644
--- a/core/io/resource_loader.cpp
+++ b/core/io/resource_loader.cpp
@@ -734,26 +734,49 @@ String ResourceLoader::_path_remap(const String &p_path, bool *r_translation_rem
 
 	String new_path = p_path;
 
-	if (translation_remaps.has(new_path)) {
+	if (translation_remaps.has(p_path)) {
+		// translation_remaps has the following format:
+		//   { "res://path.png": PoolStringArray( "res://path-ru.png:ru", "res://path-de.png:de" ) }
+
+		// To find the path of the remapped resource, we extract the locale name after
+		// the last ':' to match the project locale.
+		// We also fall back in case of regional locales as done in TranslationServer::translate
+		// (e.g. 'ru_RU' -> 'ru' if the former has no specific mapping).
 
-		Vector<String> &v = *translation_remaps.getptr(new_path);
 		String locale = TranslationServer::get_singleton()->get_locale();
-		if (r_translation_remapped) {
-			*r_translation_remapped = true;
-		}
-		for (int i = 0; i < v.size(); i++) {
+		ERR_FAIL_COND_V_MSG(locale.length() < 2, p_path, "Could not remap path '" + p_path + "' for translation as configured locale '" + locale + "' is invalid.");
+		String lang = TranslationServer::get_language_code(locale);
 
-			int split = v[i].find_last(":");
-			if (split == -1)
-				continue;
-			String l = v[i].right(split + 1).strip_edges();
-			if (l == String())
+		Vector<String> &res_remaps = *translation_remaps.getptr(new_path);
+		bool near_match = false;
+
+		for (int i = 0; i < res_remaps.size(); i++) {
+			int split = res_remaps[i].find_last(":");
+			if (split == -1) {
 				continue;
+			}
 
-			if (l.begins_with(locale)) {
-				new_path = v[i].left(split);
+			String l = res_remaps[i].right(split + 1).strip_edges();
+			if (l == locale) { // Exact match.
+				new_path = res_remaps[i].left(split);
 				break;
+			} else if (near_match) {
+				continue; // Already found near match, keep going for potential exact match.
 			}
+
+			// No exact match (e.g. locale 'ru_RU' but remap is 'ru'), let's look further
+			// for a near match (same language code, i.e. first 2 or 3 letters before
+			// regional code, if included).
+			if (TranslationServer::get_language_code(l) == lang) {
+				// Language code matches, that's a near match. Keep looking for exact match.
+				near_match = true;
+				new_path = res_remaps[i].left(split);
+				continue;
+			}
+		}
+
+		if (r_translation_remapped) {
+			*r_translation_remapped = true;
 		}
 	}
 
@@ -761,8 +784,8 @@ String ResourceLoader::_path_remap(const String &p_path, bool *r_translation_rem
 		new_path = path_remaps[new_path];
 	}
 
-	if (new_path == p_path) { //did not remap
-		//try file remap
+	if (new_path == p_path) { // Did not remap.
+		// Try file remap.
 		Error err;
 		FileAccess *f = FileAccess::open(p_path + ".remap", FileAccess::READ, &err);
 
diff --git a/core/translation.cpp b/core/translation.cpp
index 4a1ac26433..4a662b2590 100644
--- a/core/translation.cpp
+++ b/core/translation.cpp
@@ -792,11 +792,6 @@ static const char *locale_renames[][2] = {
 	{ NULL, NULL }
 };
 
-static String get_trimmed_locale(const String &p_locale) {
-
-	return p_locale.substr(0, 2);
-}
-
 ///////////////////////////////////////////////
 
 PoolVector<String> Translation::_get_messages() const {
@@ -846,7 +841,7 @@ void Translation::set_locale(const String &p_locale) {
 	String univ_locale = TranslationServer::standardize_locale(p_locale);
 
 	if (!TranslationServer::is_locale_valid(univ_locale)) {
-		String trimmed_locale = get_trimmed_locale(univ_locale);
+		String trimmed_locale = TranslationServer::get_language_code(univ_locale);
 
 		ERR_FAIL_COND_MSG(!TranslationServer::is_locale_valid(trimmed_locale), "Invalid locale: " + trimmed_locale + ".");
 
@@ -945,12 +940,29 @@ String TranslationServer::standardize_locale(const String &p_locale) {
 	return univ_locale;
 }
 
+String TranslationServer::get_language_code(const String &p_locale) {
+
+	ERR_FAIL_COND_V_MSG(p_locale.length() < 2, p_locale, "Invalid locale '" + p_locale + "'.");
+	// Most language codes are two letters, but some are three,
+	// so we have to look for a regional code separator ('_' or '-')
+	// to extract the left part.
+	// For example we get 'nah_MX' as input and should return 'nah'.
+	int split = p_locale.find("_");
+	if (split == -1) {
+		split = p_locale.find("-");
+	}
+	if (split == -1) { // No separator, so the locale is already only a language code.
+		return p_locale;
+	}
+	return p_locale.left(split);
+}
+
 void TranslationServer::set_locale(const String &p_locale) {
 
 	String univ_locale = standardize_locale(p_locale);
 
 	if (!is_locale_valid(univ_locale)) {
-		String trimmed_locale = get_trimmed_locale(univ_locale);
+		String trimmed_locale = get_language_code(univ_locale);
 		print_verbose(vformat("Unsupported locale '%s', falling back to '%s'.", p_locale, trimmed_locale));
 
 		if (!is_locale_valid(trimmed_locale)) {
@@ -1039,11 +1051,13 @@ void TranslationServer::clear() {
 
 StringName TranslationServer::translate(const StringName &p_message) const {
 
-	//translate using locale
+	// Match given message against the translation catalog for the project locale.
 
 	if (!enabled)
 		return p_message;
 
+	ERR_FAIL_COND_V_MSG(locale.length() < 2, p_message, "Could not translate message as configured locale '" + locale + "' is invalid.");
+
 	// Locale can be of the form 'll_CC', i.e. language code and regional code,
 	// e.g. 'en_US', 'en_GB', etc. It might also be simply 'll', e.g. 'en'.
 	// To find the relevant translation, we look for those with locale starting
@@ -1051,67 +1065,78 @@ StringName TranslationServer::translate(const StringName &p_message) const {
 	// form. If not found, we fall back to a near match (another locale with
 	// same language code).
 
+	// Note: ResourceLoader::_path_remap reproduces this locale near matching
+	// logic, so be sure to propagate changes there when changing things here.
+
 	StringName res;
+	String lang = get_language_code(locale);
 	bool near_match = false;
-	const CharType *lptr = &locale[0];
 
 	for (const Set<Ref<Translation> >::Element *E = translations.front(); E; E = E->next()) {
-
 		const Ref<Translation> &t = E->get();
-		ERR_FAIL_COND_V(t.is_null(), StringName(""));
+		ERR_FAIL_COND_V(t.is_null(), p_message);
 		String l = t->get_locale();
-		if (lptr[0] != l[0] || lptr[1] != l[1])
-			continue; // Language code does not match.
 
 		bool exact_match = (l == locale);
-		if (!exact_match && near_match)
-			continue; // Only near-match once, but keep looking for exact matches.
+		if (!exact_match) {
+			if (near_match) {
+				continue; // Only near-match once, but keep looking for exact matches.
+			}
+			if (get_language_code(l) != lang) {
+				continue; // Language code does not match.
+			}
+		}
 
 		StringName r = t->get_message(p_message);
-
-		if (!r)
+		if (!r) {
 			continue;
-
+		}
 		res = r;
 
-		if (exact_match)
+		if (exact_match) {
 			break;
-		else
+		} else {
 			near_match = true;
+		}
 	}
 
 	if (!res && fallback.length() >= 2) {
 		// Try again with the fallback locale.
-		const CharType *fptr = &fallback[0];
+		String fallback_lang = get_language_code(fallback);
 		near_match = false;
-		for (const Set<Ref<Translation> >::Element *E = translations.front(); E; E = E->next()) {
 
+		for (const Set<Ref<Translation> >::Element *E = translations.front(); E; E = E->next()) {
 			const Ref<Translation> &t = E->get();
-			ERR_FAIL_COND_V(t.is_null(), StringName(""));
+			ERR_FAIL_COND_V(t.is_null(), p_message);
 			String l = t->get_locale();
-			if (fptr[0] != l[0] || fptr[1] != l[1])
-				continue; // Language code does not match.
 
 			bool exact_match = (l == fallback);
-			if (!exact_match && near_match)
-				continue; // Only near-match once, but keep looking for exact matches.
+			if (!exact_match) {
+				if (near_match) {
+					continue; // Only near-match once, but keep looking for exact matches.
+				}
+				if (get_language_code(l) != fallback_lang) {
+					continue; // Language code does not match.
+				}
+			}
 
 			StringName r = t->get_message(p_message);
-
-			if (!r)
+			if (!r) {
 				continue;
-
+			}
 			res = r;
 
-			if (exact_match)
+			if (exact_match) {
 				break;
-			else
+			} else {
 				near_match = true;
+			}
 		}
 	}
 
-	if (!res)
+	if (!res) {
 		return p_message;
+	}
 
 	return res;
 }
diff --git a/core/translation.h b/core/translation.h
index d172b9ecf2..834374bda6 100644
--- a/core/translation.h
+++ b/core/translation.h
@@ -105,6 +105,7 @@ public:
 	static Vector<String> get_all_locale_names();
 	static bool is_locale_valid(const String &p_locale);
 	static String standardize_locale(const String &p_locale);
+	static String get_language_code(const String &p_locale);
 
 	void set_tool_translation(const Ref<Translation> &p_translation);
 	StringName tool_translate(const StringName &p_message) const;