diff options
| -rw-r--r-- | core/io/translation_loader_po.cpp | 439 |
1 files changed, 273 insertions, 166 deletions
diff --git a/core/io/translation_loader_po.cpp b/core/io/translation_loader_po.cpp index 8d3e58cad1..801bd8b0bf 100644 --- a/core/io/translation_loader_po.cpp +++ b/core/io/translation_loader_po.cpp @@ -35,98 +35,160 @@ #include "core/string/translation_po.h" RES TranslationLoaderPO::load_translation(FileAccess *f, Error *r_error) { - enum Status { - STATUS_NONE, - STATUS_READING_ID, - STATUS_READING_STRING, - STATUS_READING_CONTEXT, - STATUS_READING_PLURAL, - }; - - Status status = STATUS_NONE; - - String msg_id; - String msg_str; - String msg_context; - Vector<String> msgs_plural; - String config; - if (r_error) { *r_error = ERR_FILE_CORRUPT; } - Ref<TranslationPO> translation = Ref<TranslationPO>(memnew(TranslationPO)); - int line = 1; - int plural_forms = 0; - int plural_index = -1; - bool entered_context = false; - bool skip_this = false; - bool skip_next = false; - bool is_eof = false; const String path = f->get_path(); + Ref<TranslationPO> translation = Ref<TranslationPO>(memnew(TranslationPO)); + String config; - while (!is_eof) { - String l = f->get_line().strip_edges(); - is_eof = f->eof_reached(); + uint32_t magic = f->get_32(); + if (magic == 0x950412de) { + // Load binary MO file. - // If we reached last line and it's not a content line, break, otherwise let processing that last loop - if (is_eof && l.is_empty()) { - if (status == STATUS_READING_ID || status == STATUS_READING_CONTEXT || (status == STATUS_READING_PLURAL && plural_index != plural_forms - 1)) { - memdelete(f); - ERR_FAIL_V_MSG(RES(), "Unexpected EOF while reading PO file at: " + path + ":" + itos(line)); - } else { - break; - } + uint16_t version_maj = f->get_16(); + uint16_t version_min = f->get_16(); + if (version_maj > 1) { + ERR_FAIL_V_MSG(RES(), vformat("Unsupported MO file %s, version %d.%d.", path, version_maj, version_min)); } - if (l.begins_with("msgctxt")) { - if (status != STATUS_READING_STRING && status != STATUS_READING_PLURAL) { - memdelete(f); - ERR_FAIL_V_MSG(RES(), "Unexpected 'msgctxt', was expecting 'msgid_plural' or 'msgstr' before 'msgctxt' while parsing: " + path + ":" + itos(line)); + uint32_t num_strings = f->get_32(); + uint32_t id_table_offset = f->get_32(); + uint32_t trans_table_offset = f->get_32(); + + // Read string tables. + for (uint32_t i = 0; i < num_strings; i++) { + String msg_id; + String msg_id_plural; + String msg_context; + + // Read id strings and context. + { + Vector<uint8_t> data; + f->seek(id_table_offset + i * 8); + uint32_t str_start = 0; + uint32_t str_len = f->get_32(); + uint32_t str_offset = f->get_32(); + + data.resize(str_len + 1); + f->seek(str_offset); + f->get_buffer(data.ptrw(), str_len); + data.write[str_len] = 0; + + bool is_plural = false; + for (uint32_t j = 0; j < str_len + 1; j++) { + if (data[j] == 0x04) { + msg_context.parse_utf8((const char *)data.ptr(), j); + str_start = j + 1; + } + if (data[j] == 0x00) { + if (is_plural) { + msg_id_plural.parse_utf8((const char *)(data.ptr() + str_start), j - str_start); + } else { + msg_id.parse_utf8((const char *)(data.ptr() + str_start), j - str_start); + is_plural = true; + } + str_start = j + 1; + } + } } - // In PO file, "msgctxt" appears before "msgid". If we encounter a "msgctxt", we add what we have read - // and set "entered_context" to true to prevent adding twice. - if (!skip_this && !msg_id.is_empty()) { - if (status == STATUS_READING_STRING) { - translation->add_message(msg_id, msg_str, msg_context); - } else if (status == STATUS_READING_PLURAL) { - if (plural_index != plural_forms - 1) { - memdelete(f); - ERR_FAIL_V_MSG(RES(), "Number of 'msgstr[]' doesn't match with number of plural forms: " + path + ":" + itos(line)); + // Read translated strings. + { + Vector<uint8_t> data; + f->seek(trans_table_offset + i * 8); + uint32_t str_start = 0; + uint32_t str_len = f->get_32(); + uint32_t str_offset = f->get_32(); + + data.resize(str_len + 1); + f->seek(str_offset); + f->get_buffer(data.ptrw(), str_len); + data.write[str_len] = 0; + + if (msg_id.is_empty()) { + config = String::utf8((const char *)data.ptr(), str_len); + // Record plural rule. + int p_start = config.find("Plural-Forms"); + if (p_start != -1) { + int p_end = config.find("\n", p_start); + translation->set_plural_rule(config.substr(p_start, p_end - p_start)); + } + } else { + Vector<String> plural_msg; + for (uint32_t j = 0; j < str_len + 1; j++) { + if (data[j] == 0x00) { + if (msg_id_plural.is_empty()) { + translation->add_message(msg_id, String::utf8((const char *)(data.ptr() + str_start), j - str_start), msg_context); + } else { + plural_msg.push_back(String::utf8((const char *)(data.ptr() + str_start), j - str_start)); + } + str_start = j + 1; + } + } + if (!plural_msg.is_empty()) { + translation->add_plural_message(msg_id, plural_msg, msg_context); } - translation->add_plural_message(msg_id, msgs_plural, msg_context); } } - msg_context = ""; - l = l.substr(7, l.length()).strip_edges(); - status = STATUS_READING_CONTEXT; - entered_context = true; } - if (l.begins_with("msgid_plural")) { - if (plural_forms == 0) { - memdelete(f); - ERR_FAIL_V_MSG(RES(), "PO file uses 'msgid_plural' but 'Plural-Forms' is invalid or missing in header: " + path + ":" + itos(line)); - } else if (status != STATUS_READING_ID) { - memdelete(f); - ERR_FAIL_V_MSG(RES(), "Unexpected 'msgid_plural', was expecting 'msgid' before 'msgid_plural' while parsing: " + path + ":" + itos(line)); - } - // We don't record the message in "msgid_plural" itself as tr_n(), TTRN(), RTRN() interfaces provide the plural string already. - // We just have to reset variables related to plurals for "msgstr[]" later on. - l = l.substr(12, l.length()).strip_edges(); - plural_index = -1; - msgs_plural.clear(); - msgs_plural.resize(plural_forms); - status = STATUS_READING_PLURAL; - } else if (l.begins_with("msgid")) { - if (status == STATUS_READING_ID) { - memdelete(f); - ERR_FAIL_V_MSG(RES(), "Unexpected 'msgid', was expecting 'msgstr' while parsing: " + path + ":" + itos(line)); + memdelete(f); + } else { + // Try to load as text PO file. + f->seek(0); + + enum Status { + STATUS_NONE, + STATUS_READING_ID, + STATUS_READING_STRING, + STATUS_READING_CONTEXT, + STATUS_READING_PLURAL, + }; + + Status status = STATUS_NONE; + + String msg_id; + String msg_str; + String msg_context; + Vector<String> msgs_plural; + + if (r_error) { + *r_error = ERR_FILE_CORRUPT; + } + + int line = 1; + int plural_forms = 0; + int plural_index = -1; + bool entered_context = false; + bool skip_this = false; + bool skip_next = false; + bool is_eof = false; + + while (!is_eof) { + String l = f->get_line().strip_edges(); + is_eof = f->eof_reached(); + + // If we reached last line and it's not a content line, break, otherwise let processing that last loop + if (is_eof && l.is_empty()) { + if (status == STATUS_READING_ID || status == STATUS_READING_CONTEXT || (status == STATUS_READING_PLURAL && plural_index != plural_forms - 1)) { + memdelete(f); + ERR_FAIL_V_MSG(RES(), "Unexpected EOF while reading PO file at: " + path + ":" + itos(line)); + } else { + break; + } } - if (!msg_id.is_empty()) { - if (!skip_this && !entered_context) { + if (l.begins_with("msgctxt")) { + if (status != STATUS_READING_STRING && status != STATUS_READING_PLURAL) { + memdelete(f); + ERR_FAIL_V_MSG(RES(), "Unexpected 'msgctxt', was expecting 'msgid_plural' or 'msgstr' before 'msgctxt' while parsing: " + path + ":" + itos(line)); + } + + // In PO file, "msgctxt" appears before "msgid". If we encounter a "msgctxt", we add what we have read + // and set "entered_context" to true to prevent adding twice. + if (!skip_this && !msg_id.is_empty()) { if (status == STATUS_READING_STRING) { translation->add_message(msg_id, msg_str, msg_context); } else if (status == STATUS_READING_PLURAL) { @@ -137,119 +199,163 @@ RES TranslationLoaderPO::load_translation(FileAccess *f, Error *r_error) { translation->add_plural_message(msg_id, msgs_plural, msg_context); } } - } else if (config.is_empty()) { - config = msg_str; - // Record plural rule. - int p_start = config.find("Plural-Forms"); - if (p_start != -1) { - int p_end = config.find("\n", p_start); - translation->set_plural_rule(config.substr(p_start, p_end - p_start)); - plural_forms = translation->get_plural_forms(); + msg_context = ""; + l = l.substr(7, l.length()).strip_edges(); + status = STATUS_READING_CONTEXT; + entered_context = true; + } + + if (l.begins_with("msgid_plural")) { + if (plural_forms == 0) { + memdelete(f); + ERR_FAIL_V_MSG(RES(), "PO file uses 'msgid_plural' but 'Plural-Forms' is invalid or missing in header: " + path + ":" + itos(line)); + } else if (status != STATUS_READING_ID) { + memdelete(f); + ERR_FAIL_V_MSG(RES(), "Unexpected 'msgid_plural', was expecting 'msgid' before 'msgid_plural' while parsing: " + path + ":" + itos(line)); + } + // We don't record the message in "msgid_plural" itself as tr_n(), TTRN(), RTRN() interfaces provide the plural string already. + // We just have to reset variables related to plurals for "msgstr[]" later on. + l = l.substr(12, l.length()).strip_edges(); + plural_index = -1; + msgs_plural.clear(); + msgs_plural.resize(plural_forms); + status = STATUS_READING_PLURAL; + } else if (l.begins_with("msgid")) { + if (status == STATUS_READING_ID) { + memdelete(f); + ERR_FAIL_V_MSG(RES(), "Unexpected 'msgid', was expecting 'msgstr' while parsing: " + path + ":" + itos(line)); } + + if (!msg_id.is_empty()) { + if (!skip_this && !entered_context) { + if (status == STATUS_READING_STRING) { + translation->add_message(msg_id, msg_str, msg_context); + } else if (status == STATUS_READING_PLURAL) { + if (plural_index != plural_forms - 1) { + memdelete(f); + ERR_FAIL_V_MSG(RES(), "Number of 'msgstr[]' doesn't match with number of plural forms: " + path + ":" + itos(line)); + } + translation->add_plural_message(msg_id, msgs_plural, msg_context); + } + } + } else if (config.is_empty()) { + config = msg_str; + // Record plural rule. + int p_start = config.find("Plural-Forms"); + if (p_start != -1) { + int p_end = config.find("\n", p_start); + translation->set_plural_rule(config.substr(p_start, p_end - p_start)); + plural_forms = translation->get_plural_forms(); + } + } + + l = l.substr(5, l.length()).strip_edges(); + status = STATUS_READING_ID; + // If we did not encounter msgctxt, we reset context to empty to reset it. + if (!entered_context) { + msg_context = ""; + } + msg_id = ""; + msg_str = ""; + skip_this = skip_next; + skip_next = false; + entered_context = false; } - l = l.substr(5, l.length()).strip_edges(); - status = STATUS_READING_ID; - // If we did not encounter msgctxt, we reset context to empty to reset it. - if (!entered_context) { - msg_context = ""; + if (l.begins_with("msgstr[")) { + if (status != STATUS_READING_PLURAL) { + memdelete(f); + ERR_FAIL_V_MSG(RES(), "Unexpected 'msgstr[]', was expecting 'msgid_plural' before 'msgstr[]' while parsing: " + path + ":" + itos(line)); + } + plural_index++; // Increment to add to the next slot in vector msgs_plural. + l = l.substr(9, l.length()).strip_edges(); + } else if (l.begins_with("msgstr")) { + if (status != STATUS_READING_ID) { + memdelete(f); + ERR_FAIL_V_MSG(RES(), "Unexpected 'msgstr', was expecting 'msgid' before 'msgstr' while parsing: " + path + ":" + itos(line)); + } + + l = l.substr(6, l.length()).strip_edges(); + status = STATUS_READING_STRING; } - msg_id = ""; - msg_str = ""; - skip_this = skip_next; - skip_next = false; - entered_context = false; - } - if (l.begins_with("msgstr[")) { - if (status != STATUS_READING_PLURAL) { - memdelete(f); - ERR_FAIL_V_MSG(RES(), "Unexpected 'msgstr[]', was expecting 'msgid_plural' before 'msgstr[]' while parsing: " + path + ":" + itos(line)); + if (l.is_empty() || l.begins_with("#")) { + if (l.contains("fuzzy")) { + skip_next = true; + } + line++; + continue; // Nothing to read or comment. } - plural_index++; // Increment to add to the next slot in vector msgs_plural. - l = l.substr(9, l.length()).strip_edges(); - } else if (l.begins_with("msgstr")) { - if (status != STATUS_READING_ID) { + + if (!l.begins_with("\"") || status == STATUS_NONE) { memdelete(f); - ERR_FAIL_V_MSG(RES(), "Unexpected 'msgstr', was expecting 'msgid' before 'msgstr' while parsing: " + path + ":" + itos(line)); + ERR_FAIL_V_MSG(RES(), "Invalid line '" + l + "' while parsing: " + path + ":" + itos(line)); } - l = l.substr(6, l.length()).strip_edges(); - status = STATUS_READING_STRING; - } - - if (l.is_empty() || l.begins_with("#")) { - if (l.contains("fuzzy")) { - skip_next = true; - } - line++; - continue; // Nothing to read or comment. - } + l = l.substr(1, l.length()); + // Find final quote, ignoring escaped ones (\"). + // The escape_next logic is necessary to properly parse things like \\" + // where the backslash is the one being escaped, not the quote. + int end_pos = -1; + bool escape_next = false; + for (int i = 0; i < l.length(); i++) { + if (l[i] == '\\' && !escape_next) { + escape_next = true; + continue; + } - if (!l.begins_with("\"") || status == STATUS_NONE) { - memdelete(f); - ERR_FAIL_V_MSG(RES(), "Invalid line '" + l + "' while parsing: " + path + ":" + itos(line)); - } + if (l[i] == '"' && !escape_next) { + end_pos = i; + break; + } - l = l.substr(1, l.length()); - // Find final quote, ignoring escaped ones (\"). - // The escape_next logic is necessary to properly parse things like \\" - // where the backslash is the one being escaped, not the quote. - int end_pos = -1; - bool escape_next = false; - for (int i = 0; i < l.length(); i++) { - if (l[i] == '\\' && !escape_next) { - escape_next = true; - continue; + escape_next = false; } - if (l[i] == '"' && !escape_next) { - end_pos = i; - break; + if (end_pos == -1) { + memdelete(f); + ERR_FAIL_V_MSG(RES(), "Expected '\"' at end of message while parsing: " + path + ":" + itos(line)); } - escape_next = false; - } + l = l.substr(0, end_pos); + l = l.c_unescape(); - if (end_pos == -1) { - memdelete(f); - ERR_FAIL_V_MSG(RES(), "Expected '\"' at end of message while parsing: " + path + ":" + itos(line)); - } + if (status == STATUS_READING_ID) { + msg_id += l; + } else if (status == STATUS_READING_STRING) { + msg_str += l; + } else if (status == STATUS_READING_CONTEXT) { + msg_context += l; + } else if (status == STATUS_READING_PLURAL && plural_index >= 0) { + if (plural_index >= plural_forms) { + memdelete(f); + ERR_FAIL_V_MSG(RES(), "Unexpected plural form while parsing: " + path + ":" + itos(line)); + } + msgs_plural.write[plural_index] = msgs_plural[plural_index] + l; + } - l = l.substr(0, end_pos); - l = l.c_unescape(); - - if (status == STATUS_READING_ID) { - msg_id += l; - } else if (status == STATUS_READING_STRING) { - msg_str += l; - } else if (status == STATUS_READING_CONTEXT) { - msg_context += l; - } else if (status == STATUS_READING_PLURAL && plural_index >= 0) { - msgs_plural.write[plural_index] = msgs_plural[plural_index] + l; + line++; } - line++; - } - - memdelete(f); + memdelete(f); - // Add the last set of data from last iteration. - if (status == STATUS_READING_STRING) { - if (!msg_id.is_empty()) { - if (!skip_this) { - translation->add_message(msg_id, msg_str, msg_context); + // Add the last set of data from last iteration. + if (status == STATUS_READING_STRING) { + if (!msg_id.is_empty()) { + if (!skip_this) { + translation->add_message(msg_id, msg_str, msg_context); + } + } else if (config.is_empty()) { + config = msg_str; } - } else if (config.is_empty()) { - config = msg_str; - } - } else if (status == STATUS_READING_PLURAL) { - if (!skip_this && !msg_id.is_empty()) { - if (plural_index != plural_forms - 1) { - memdelete(f); - ERR_FAIL_V_MSG(RES(), "Number of 'msgstr[]' doesn't match with number of plural forms: " + path + ":" + itos(line)); + } else if (status == STATUS_READING_PLURAL) { + if (!skip_this && !msg_id.is_empty()) { + if (plural_index != plural_forms - 1) { + memdelete(f); + ERR_FAIL_V_MSG(RES(), "Number of 'msgstr[]' doesn't match with number of plural forms: " + path + ":" + itos(line)); + } + translation->add_plural_message(msg_id, msgs_plural, msg_context); } - translation->add_plural_message(msg_id, msgs_plural, msg_context); } } @@ -290,6 +396,7 @@ RES TranslationLoaderPO::load(const String &p_path, const String &p_original_pat void TranslationLoaderPO::get_recognized_extensions(List<String> *p_extensions) const { p_extensions->push_back("po"); + p_extensions->push_back("mo"); } bool TranslationLoaderPO::handles_type(const String &p_type) const { @@ -297,7 +404,7 @@ bool TranslationLoaderPO::handles_type(const String &p_type) const { } String TranslationLoaderPO::get_resource_type(const String &p_path) const { - if (p_path.get_extension().to_lower() == "po") { + if (p_path.get_extension().to_lower() == "po" || p_path.get_extension().to_lower() == "mo") { return "Translation"; } return ""; |