Merge pull request #40443 from SkyLucilfer/PluralsSupport

Added plurals and context support to Translation

8 files changed, 243 insertions, 85 deletions

diff --git a/editor/editor_translation_parser.cpp b/editor/editor_translation_parser.cpp
index da191fbc92..7a90d20000 100644
--- a/editor/editor_translation_parser.cpp
+++ b/editor/editor_translation_parser.cpp
@@ -37,15 +37,30 @@
 
 EditorTranslationParser *EditorTranslationParser::singleton = nullptr;
 
-Error EditorTranslationParserPlugin::parse_file(const String &p_path, Vector<String> *r_extracted_strings) {
+Error EditorTranslationParserPlugin::parse_file(const String &p_path, Vector<String> *r_ids, Vector<Vector<String>> *r_ids_ctx_plural) {
 	if (!get_script_instance())
 		return ERR_UNAVAILABLE;
 
 	if (get_script_instance()->has_method("parse_file")) {
-		Array extracted_strings;
-		get_script_instance()->call("parse_file", p_path, extracted_strings);
-		for (int i = 0; i < extracted_strings.size(); i++) {
-			r_extracted_strings->append(extracted_strings[i]);
+		Array ids;
+		Array ids_ctx_plural;
+		get_script_instance()->call("parse_file", p_path, ids, ids_ctx_plural);
+
+		// Add user's extracted translatable messages.
+		for (int i = 0; i < ids.size(); i++) {
+			r_ids->append(ids[i]);
+		}
+
+		// Add user's collected translatable messages with context or plurals.
+		for (int i = 0; i < ids_ctx_plural.size(); i++) {
+			Array arr = ids_ctx_plural[i];
+			ERR_FAIL_COND_V_MSG(arr.size() != 3, ERR_INVALID_DATA, "Array entries written into `msgids_context_plural` in `parse_file()` method should have the form [\"message\", \"context\", \"plural message\"]");
+
+			Vector<String> id_ctx_plural;
+			id_ctx_plural.push_back(arr[0]);
+			id_ctx_plural.push_back(arr[1]);
+			id_ctx_plural.push_back(arr[2]);
+			r_ids_ctx_plural->append(id_ctx_plural);
 		}
 		return OK;
 	} else {
@@ -69,7 +84,7 @@ void EditorTranslationParserPlugin::get_recognized_extensions(List<String> *r_ex
 }
 
 void EditorTranslationParserPlugin::_bind_methods() {
-	ClassDB::add_virtual_method(get_class_static(), MethodInfo(Variant::NIL, "parse_file", PropertyInfo(Variant::STRING, "path"), PropertyInfo(Variant::ARRAY, "extracted_strings")));
+	ClassDB::add_virtual_method(get_class_static(), MethodInfo(Variant::NIL, "parse_file", PropertyInfo(Variant::STRING, "path"), PropertyInfo(Variant::ARRAY, "msgids"), PropertyInfo(Variant::ARRAY, "msgids_context_plural")));
 	ClassDB::add_virtual_method(get_class_static(), MethodInfo(Variant::ARRAY, "get_recognized_extensions"));
 }
 
diff --git a/editor/editor_translation_parser.h b/editor/editor_translation_parser.h
index fb8aa6ec9b..18f49b3803 100644
--- a/editor/editor_translation_parser.h
+++ b/editor/editor_translation_parser.h
@@ -41,7 +41,7 @@ protected:
 	static void _bind_methods();
 
 public:
-	virtual Error parse_file(const String &p_path, Vector<String> *r_extracted_strings);
+	virtual Error parse_file(const String &p_path, Vector<String> *r_ids, Vector<Vector<String>> *r_ids_ctx_plural);
 	virtual void get_recognized_extensions(List<String> *r_extensions) const;
 };
 
diff --git a/editor/plugins/packed_scene_translation_parser_plugin.cpp b/editor/plugins/packed_scene_translation_parser_plugin.cpp
index 52af0008b7..608b5c3104 100644
--- a/editor/plugins/packed_scene_translation_parser_plugin.cpp
+++ b/editor/plugins/packed_scene_translation_parser_plugin.cpp
@@ -37,7 +37,7 @@ void PackedSceneEditorTranslationParserPlugin::get_recognized_extensions(List<St
 	ResourceLoader::get_recognized_extensions_for_type("PackedScene", r_extensions);
 }
 
-Error PackedSceneEditorTranslationParserPlugin::parse_file(const String &p_path, Vector<String> *r_extracted_strings) {
+Error PackedSceneEditorTranslationParserPlugin::parse_file(const String &p_path, Vector<String> *r_ids, Vector<Vector<String>> *r_ids_ctx_plural) {
 	// Parse specific scene Node's properties (see in constructor) that are auto-translated by the engine when set. E.g Label's text property.
 	// These properties are translated with the tr() function in the C++ code when being set or updated.
 
@@ -71,8 +71,10 @@ Error PackedSceneEditorTranslationParserPlugin::parse_file(const String &p_path,
 				String extension = s->get_language()->get_extension();
 				if (EditorTranslationParser::get_singleton()->can_parse(extension)) {
 					Vector<String> temp;
-					EditorTranslationParser::get_singleton()->get_parser(extension)->parse_file(s->get_path(), &temp);
+					Vector<Vector<String>> ids_context_plural;
+					EditorTranslationParser::get_singleton()->get_parser(extension)->parse_file(s->get_path(), &temp, &ids_context_plural);
 					parsed_strings.append_array(temp);
+					r_ids_ctx_plural->append_array(ids_context_plural);
 				}
 			} else if (property_name == "filters") {
 				// Extract FileDialog's filters property with values in format "*.png ; PNG Images","*.gd ; GDScript Files".
@@ -93,7 +95,7 @@ Error PackedSceneEditorTranslationParserPlugin::parse_file(const String &p_path,
 		}
 	}
 
-	r_extracted_strings->append_array(parsed_strings);
+	r_ids->append_array(parsed_strings);
 
 	return OK;
 }
diff --git a/editor/plugins/packed_scene_translation_parser_plugin.h b/editor/plugins/packed_scene_translation_parser_plugin.h
index 2bd4dae995..a0ffdf692c 100644
--- a/editor/plugins/packed_scene_translation_parser_plugin.h
+++ b/editor/plugins/packed_scene_translation_parser_plugin.h
@@ -40,7 +40,7 @@ class PackedSceneEditorTranslationParserPlugin : public EditorTranslationParserP
 	Set<String> lookup_properties;
 
 public:
-	virtual Error parse_file(const String &p_path, Vector<String> *r_extracted_strings) override;
+	virtual Error parse_file(const String &p_path, Vector<String> *r_ids, Vector<Vector<String>> *r_ids_ctx_plural) override;
 	virtual void get_recognized_extensions(List<String> *r_extensions) const override;
 
 	PackedSceneEditorTranslationParserPlugin();
diff --git a/editor/pot_generator.cpp b/editor/pot_generator.cpp
index f9b8722aad..f09750efdc 100644
--- a/editor/pot_generator.cpp
+++ b/editor/pot_generator.cpp
@@ -31,23 +31,25 @@
 #include "pot_generator.h"
 
 #include "core/error_macros.h"
-#include "core/os/file_access.h"
 #include "core/project_settings.h"
 #include "editor_translation_parser.h"
 #include "plugins/packed_scene_translation_parser_plugin.h"
 
 POTGenerator *POTGenerator::singleton = nullptr;
 
-//#define DEBUG_POT
-
 #ifdef DEBUG_POT
-void _print_all_translation_strings(const OrderedHashMap<String, Set<String>> &p_all_translation_strings) {
-	for (auto E_pair = p_all_translation_strings.front(); E_pair; E_pair = E_pair.next()) {
-		String msg = static_cast<String>(E_pair.key()) + " : ";
-		for (Set<String>::Element *E = E_pair.value().front(); E; E = E->next()) {
-			msg += E->get() + " ";
+void POTGenerator::_print_all_translation_strings() {
+	for (auto E = all_translation_strings.front(); E; E = E.next()) {
+		Vector<MsgidData> v_md = all_translation_strings[E.key()];
+		for (int i = 0; i < v_md.size(); i++) {
+			print_line("++++++");
+			print_line("msgid: " + E.key());
+			print_line("context: " + v_md[i].ctx);
+			print_line("msgid_plural: " + v_md[i].plural);
+			for (Set<String>::Element *E = v_md[i].locations.front(); E; E = E->next()) {
+				print_line("location: " + E->get());
+			}
 		}
-		print_line(msg);
 	}
 }
 #endif
@@ -65,27 +67,27 @@ void POTGenerator::generate_pot(const String &p_file) {
 
 	// Collect all translatable strings according to files order in "POT Generation" setting.
 	for (int i = 0; i < files.size(); i++) {
-		Vector<String> translation_strings;
+		Vector<String> msgids;
+		Vector<Vector<String>> msgids_context_plural;
 		String file_path = files[i];
 		String file_extension = file_path.get_extension();
 
 		if (EditorTranslationParser::get_singleton()->can_parse(file_extension)) {
-			EditorTranslationParser::get_singleton()->get_parser(file_extension)->parse_file(file_path, &translation_strings);
+			EditorTranslationParser::get_singleton()->get_parser(file_extension)->parse_file(file_path, &msgids, &msgids_context_plural);
 		} else {
 			ERR_PRINT("Unrecognized file extension " + file_extension + " in generate_pot()");
 			return;
 		}
 
-		// Store translation strings parsed in this iteration along with their corresponding source file - to write into POT later on.
-		for (int j = 0; j < translation_strings.size(); j++) {
-			all_translation_strings[translation_strings[j]].insert(file_path);
+		for (int j = 0; j < msgids_context_plural.size(); j++) {
+			Vector<String> entry = msgids_context_plural[j];
+			_add_new_msgid(entry[0], entry[1], entry[2], file_path);
+		}
+		for (int j = 0; j < msgids.size(); j++) {
+			_add_new_msgid(msgids[j], "", "", file_path);
 		}
 	}
 
-#ifdef DEBUG_POT
-	_print_all_translation_strings(all_translation_strings);
-#endif
-
 	_write_to_pot(p_file);
 }
 
@@ -119,35 +121,86 @@ void POTGenerator::_write_to_pot(const String &p_file) {
 
 	file->store_string(header);
 
-	for (OrderedHashMap<String, Set<String>>::Element E_pair = all_translation_strings.front(); E_pair; E_pair = E_pair.next()) {
-		String msg = E_pair.key();
+	for (OrderedHashMap<String, Vector<MsgidData>>::Element E_pair = all_translation_strings.front(); E_pair; E_pair = E_pair.next()) {
+		String msgid = E_pair.key();
+		Vector<MsgidData> v_msgid_data = E_pair.value();
+		for (int i = 0; i < v_msgid_data.size(); i++) {
+			String context = v_msgid_data[i].ctx;
+			String plural = v_msgid_data[i].plural;
+			const Set<String> &locations = v_msgid_data[i].locations;
+
+			// Write file locations.
+			for (Set<String>::Element *E = locations.front(); E; E = E->next()) {
+				file->store_line("#: " + E->get().trim_prefix("res://"));
+			}
 
-		// Write file locations.
-		for (Set<String>::Element *E = E_pair.value().front(); E; E = E->next()) {
-			file->store_line("#: " + E->get().trim_prefix("res://"));
-		}
+			// Write context.
+			if (!context.empty()) {
+				file->store_line("msgctxt \"" + context + "\"");
+			}
 
-		// Split \\n and \n.
-		Vector<String> temp = msg.split("\\n");
-		Vector<String> msg_lines;
-		for (int i = 0; i < temp.size(); i++) {
-			msg_lines.append_array(temp[i].split("\n"));
-			if (i < temp.size() - 1) {
-				// Add \n.
-				msg_lines.set(msg_lines.size() - 1, msg_lines[msg_lines.size() - 1] + "\\n");
+			// Write msgid.
+			_write_msgid(file, msgid, false);
+
+			// Write msgid_plural
+			if (!plural.empty()) {
+				_write_msgid(file, plural, true);
+				file->store_line("msgstr[0] \"\"");
+				file->store_line("msgstr[1] \"\"\n");
+			} else {
+				file->store_line("msgstr \"\"\n");
 			}
 		}
+	}
 
-		// Write msgid.
-		file->store_string("msgid ");
-		for (int i = 0; i < msg_lines.size(); i++) {
-			file->store_line("\"" + msg_lines[i] + "\"");
+	file->close();
+}
+
+void POTGenerator::_write_msgid(FileAccess *r_file, const String &p_id, bool p_plural) {
+	// Split \\n and \n.
+	Vector<String> temp = p_id.split("\\n");
+	Vector<String> msg_lines;
+	for (int i = 0; i < temp.size(); i++) {
+		msg_lines.append_array(temp[i].split("\n"));
+		if (i < temp.size() - 1) {
+			// Add \n.
+			msg_lines.set(msg_lines.size() - 1, msg_lines[msg_lines.size() - 1] + "\\n");
 		}
+	}
 
-		file->store_line("msgstr \"\"\n");
+	if (p_plural) {
+		r_file->store_string("msgid_plural ");
+	} else {
+		r_file->store_string("msgid ");
 	}
 
-	file->close();
+	for (int i = 0; i < msg_lines.size(); i++) {
+		r_file->store_line("\"" + msg_lines[i] + "\"");
+	}
+}
+
+void POTGenerator::_add_new_msgid(const String &p_msgid, const String &p_context, const String &p_plural, const String &p_location) {
+	// Insert new location if msgid under same context exists already.
+	if (all_translation_strings.has(p_msgid)) {
+		Vector<MsgidData> &v_mdata = all_translation_strings[p_msgid];
+		for (int i = 0; i < v_mdata.size(); i++) {
+			if (v_mdata[i].ctx == p_context) {
+				if (!v_mdata[i].plural.empty() && !p_plural.empty() && v_mdata[i].plural != p_plural) {
+					WARN_PRINT("Redefinition of plural message (msgid_plural), under the same message (msgid) and context (msgctxt)");
+				}
+				v_mdata.write[i].locations.insert(p_location);
+				return;
+			}
+		}
+	}
+
+	// Add a new entry of msgid, context, plural and location - context and plural might be empty if the inserted msgid doesn't associated
+	// context or plurals.
+	MsgidData mdata;
+	mdata.ctx = p_context;
+	mdata.plural = p_plural;
+	mdata.locations.insert(p_location);
+	all_translation_strings[p_msgid].push_back(mdata);
 }
 
 POTGenerator *POTGenerator::get_singleton() {
diff --git a/editor/pot_generator.h b/editor/pot_generator.h
index abe1a21d41..8853b784ed 100644
--- a/editor/pot_generator.h
+++ b/editor/pot_generator.h
@@ -32,14 +32,29 @@
 #define POT_GENERATOR_H
 
 #include "core/ordered_hash_map.h"
+#include "core/os/file_access.h"
 #include "core/set.h"
 
+//#define DEBUG_POT
+
 class POTGenerator {
 	static POTGenerator *singleton;
-	// Stores all translatable strings and the source files containing them.
-	OrderedHashMap<String, Set<String>> all_translation_strings;
+
+	struct MsgidData {
+		String ctx;
+		String plural;
+		Set<String> locations;
+	};
+	// Store msgid as key and the additional data around the msgid - if it's under a context, has plurals and its file locations.
+	OrderedHashMap<String, Vector<MsgidData>> all_translation_strings;
 
 	void _write_to_pot(const String &p_file);
+	void _write_msgid(FileAccess *r_file, const String &p_id, bool p_plural);
+	void _add_new_msgid(const String &p_msgid, const String &p_context, const String &p_plural, const String &p_location);
+
+#ifdef DEBUG_POT
+	void _print_all_translation_strings();
+#endif
 
 public:
 	static POTGenerator *get_singleton();
diff --git a/editor/scene_tree_editor.cpp b/editor/scene_tree_editor.cpp
index 7404c9779b..a62448169d 100644
--- a/editor/scene_tree_editor.cpp
+++ b/editor/scene_tree_editor.cpp
@@ -258,27 +258,35 @@ bool SceneTreeEditor::_add_nodes(Node *p_node, TreeItem *p_parent) {
 		int num_connections = p_node->get_persistent_signal_connection_count();
 		int num_groups = p_node->get_persistent_group_count();
 
+		String msg_temp;
+		if (num_connections >= 1) {
+			Array arr;
+			arr.push_back(num_connections);
+			msg_temp += TTRN("Node has one connection.", "Node has {num} connections.", num_connections).format(arr, "{num}");
+			msg_temp += " ";
+		}
+		if (num_groups >= 1) {
+			Array arr;
+			arr.push_back(num_groups);
+			msg_temp += TTRN("Node is in one group.", "Node is in {num} groups.", num_groups).format(arr, "{num}");
+		}
+		if (num_connections >= 1 || num_groups >= 1) {
+			msg_temp += "\n" + TTR("Click to show signals dock.");
+		}
+
+		Ref<Texture2D> icon_temp;
+		auto signal_temp = BUTTON_SIGNALS;
 		if (num_connections >= 1 && num_groups >= 1) {
-			item->add_button(
-					0,
-					get_theme_icon("SignalsAndGroups", "EditorIcons"),
-					BUTTON_SIGNALS,
-					false,
-					vformat(TTR("Node has %s connection(s) and %s group(s).\nClick to show signals dock."), num_connections, num_groups));
+			icon_temp = get_theme_icon("SignalsAndGroups", "EditorIcons");
 		} else if (num_connections >= 1) {
-			item->add_button(
-					0,
-					get_theme_icon("Signals", "EditorIcons"),
-					BUTTON_SIGNALS,
-					false,
-					vformat(TTR("Node has %s connection(s).\nClick to show signals dock."), num_connections));
+			icon_temp = get_theme_icon("Signals", "EditorIcons");
 		} else if (num_groups >= 1) {
-			item->add_button(
-					0,
-					get_theme_icon("Groups", "EditorIcons"),
-					BUTTON_GROUPS,
-					false,
-					vformat(TTR("Node is in %s group(s).\nClick to show groups dock."), num_groups));
+			icon_temp = get_theme_icon("Groups", "EditorIcons");
+			signal_temp = BUTTON_GROUPS;
+		}
+
+		if (num_connections >= 1 || num_groups >= 1) {
+			item->add_button(0, icon_temp, signal_temp, false, msg_temp);
 		}
 	}
 
diff --git a/editor/translations/extract.py b/editor/translations/extract.py
index 749bad5fff..02ed65131f 100755
--- a/editor/translations/extract.py
+++ b/editor/translations/extract.py
@@ -33,6 +33,7 @@ matches.sort()
 
 unique_str = []
 unique_loc = {}
+ctx_group = {}  # Store msgctx, msg, and locations.
 main_po = """
 # LANGUAGE translation of the Godot Engine editor.
 # Copyright (c) 2007-2020 Juan Linietsky, Ariel Manzur.
@@ -52,6 +53,34 @@ msgstr ""
 """
 
 
+def _write_message(msgctx, msg, msg_plural, location):
+    global main_po
+    main_po += "\n#: " + location + "\n"
+    if msgctx != "":
+        main_po += 'msgctxt "' + msgctx + '"\n'
+    main_po += 'msgid "' + msg + '"\n'
+    if msg_plural != "":
+        main_po += 'msgid_plural "' + msg_plural + '"\n'
+        main_po += 'msgstr[0] ""\n'
+        main_po += 'msgstr[1] ""\n'
+    else:
+        main_po += 'msgstr ""\n'
+
+
+def _add_additional_location(msgctx, msg, location):
+    global main_po
+    # Add additional location to previous occurrence
+    msg_pos = -1
+    if msgctx != "":
+        msg_pos = main_po.find('\nmsgctxt "' + msgctx + '"\nmsgid "' + msg + '"')
+    else:
+        msg_pos = main_po.find('\nmsgid "' + msg + '"')
+
+    if msg_pos == -1:
+        print("Someone apparently thought writing Python was as easy as GDScript. Ping Akien.")
+    main_po = main_po[:msg_pos] + " " + location + main_po[msg_pos:]
+
+
 def process_file(f, fname):
 
     global main_po, unique_str, unique_loc
@@ -60,10 +89,11 @@ def process_file(f, fname):
     lc = 1
     while l:
 
-        patterns = ['RTR("', 'TTR("', 'TTRC("']
+        patterns = ['RTR("', 'TTR("', 'TTRC("', 'TTRN("', 'RTRN("']
         idx = 0
         pos = 0
         while pos >= 0:
+            # Loop until a pattern is found. If not, next line.
             pos = l.find(patterns[idx], pos)
             if pos == -1:
                 if idx < len(patterns) - 1:
@@ -72,29 +102,64 @@ def process_file(f, fname):
                 continue
             pos += len(patterns[idx])
 
+            # Read msg until "
             msg = ""
             while pos < len(l) and (l[pos] != '"' or l[pos - 1] == "\\"):
                 msg += l[pos]
                 pos += 1
 
+            # Read plural.
+            msg_plural = ""
+            if patterns[idx] in ['TTRN("', 'RTRN("']:
+                pos = l.find('"', pos + 1)
+                pos += 1
+                while pos < len(l) and (l[pos] != '"' or l[pos - 1] == "\\"):
+                    msg_plural += l[pos]
+                    pos += 1
+
+            # Read context.
+            msgctx = ""
+            pos += 1
+            read_ctx = False
+            while pos < len(l):
+                if l[pos] == ")":
+                    break
+                elif l[pos] == '"':
+                    read_ctx = True
+                    break
+                pos += 1
+
+            pos += 1
+            if read_ctx:
+                while pos < len(l) and (l[pos] != '"' or l[pos - 1] == "\\"):
+                    msgctx += l[pos]
+                    pos += 1
+
+            # File location.
             location = os.path.relpath(fname).replace("\\", "/")
             if line_nb:
                 location += ":" + str(lc)
 
-            if not msg in unique_str:
-                main_po += "\n#: " + location + "\n"
-                main_po += 'msgid "' + msg + '"\n'
-                main_po += 'msgstr ""\n'
-                unique_str.append(msg)
-                unique_loc[msg] = [location]
-            elif not location in unique_loc[msg]:
-                # Add additional location to previous occurrence too
-                msg_pos = main_po.find('\nmsgid "' + msg + '"')
-                if msg_pos == -1:
-                    print("Someone apparently thought writing Python was as easy as GDScript. Ping Akien.")
-                main_po = main_po[:msg_pos] + " " + location + main_po[msg_pos:]
-                unique_loc[msg].append(location)
-
+            if msgctx != "":
+                # If it's a new context or a new message within an existing context, then write new msgid.
+                # Else add location to existing msgid.
+                if not msgctx in ctx_group:
+                    _write_message(msgctx, msg, msg_plural, location)
+                    ctx_group[msgctx] = {msg: [location]}
+                elif not msg in ctx_group[msgctx]:
+                    _write_message(msgctx, msg, msg_plural, location)
+                    ctx_group[msgctx][msg] = [location]
+                elif not location in ctx_group[msgctx][msg]:
+                    _add_additional_location(msgctx, msg, location)
+                    ctx_group[msgctx][msg].append(location)
+            else:
+                if not msg in unique_str:
+                    _write_message(msgctx, msg, msg_plural, location)
+                    unique_str.append(msg)
+                    unique_loc[msg] = [location]
+                elif not location in unique_loc[msg]:
+                    _add_additional_location(msgctx, msg, location)
+                    unique_loc[msg].append(location)
         l = f.readline()
         lc += 1
 
@@ -102,7 +167,7 @@ def process_file(f, fname):
 print("Updating the editor.pot template...")
 
 for fname in matches:
-    with open(fname, "r") as f:
+    with open(fname, "r", encoding="utf8") as f:
         process_file(f, fname)
 
 with open("editor.pot", "w") as f: