From ebb96e2303c95e63458ad68fa350954fb2cf80c7 Mon Sep 17 00:00:00 2001 From: Haoyu Qiu Date: Mon, 21 Jul 2025 17:14:37 +0800 Subject: [PATCH 1/3] Move plural rules logic into a separate class - Extracts plural rules logic in `TranslationPO` into a new `PluralRules` class. - Changes caching the last used plural index in `TranslationPO` into an LRU cache in `PluralRules`. - Adds tests for `PluralRules`. --- core/string/plural_rules.cpp | 167 +++++++++++++++++++++++++++ core/string/plural_rules.h | 72 ++++++++++++ core/string/translation_po.cpp | 140 +++------------------- core/string/translation_po.h | 33 +----- tests/core/string/test_translation.h | 49 ++++++++ 5 files changed, 311 insertions(+), 150 deletions(-) create mode 100644 core/string/plural_rules.cpp create mode 100644 core/string/plural_rules.h diff --git a/core/string/plural_rules.cpp b/core/string/plural_rules.cpp new file mode 100644 index 00000000000..ccdb5cd1e61 --- /dev/null +++ b/core/string/plural_rules.cpp @@ -0,0 +1,167 @@ +/**************************************************************************/ +/* plural_rules.cpp */ +/**************************************************************************/ +/* This file is part of: */ +/* GODOT ENGINE */ +/* https://godotengine.org */ +/**************************************************************************/ +/* Copyright (c) 2014-present Godot Engine contributors (see AUTHORS.md). */ +/* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur. */ +/* */ +/* Permission is hereby granted, free of charge, to any person obtaining */ +/* a copy of this software and associated documentation files (the */ +/* "Software"), to deal in the Software without restriction, including */ +/* without limitation the rights to use, copy, modify, merge, publish, */ +/* distribute, sublicense, and/or sell copies of the Software, and to */ +/* permit persons to whom the Software is furnished to do so, subject to */ +/* the following conditions: */ +/* */ +/* The above copyright notice and this permission notice shall be */ +/* included in all copies or substantial portions of the Software. */ +/* */ +/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */ +/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */ +/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. */ +/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */ +/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */ +/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */ +/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +/**************************************************************************/ + +#include "plural_rules.h" + +#include "core/math/expression.h" + +int PluralRules::_eq_test(const Array &p_input_val, const Ref &p_node, const Variant &p_result) const { + if (p_node.is_null()) { + return p_result; + } + + static const Vector input_name = { "n" }; + + Error err = expr->parse(p_node->regex, input_name); + ERR_FAIL_COND_V_MSG(err != OK, 0, vformat("Cannot parse expression \"%s\". Error: %s", p_node->regex, expr->get_error_text())); + + Variant result = expr->execute(p_input_val); + ERR_FAIL_COND_V_MSG(expr->has_execute_failed(), 0, vformat("Cannot evaluate expression \"%s\".", p_node->regex)); + + if (bool(result)) { + return _eq_test(p_input_val, p_node->left, result); + } else { + return _eq_test(p_input_val, p_node->right, result); + } +} + +int PluralRules::_find_unquoted(const String &p_src, char32_t p_chr) const { + const int len = p_src.length(); + if (len == 0) { + return -1; + } + + const char32_t *src = p_src.get_data(); + bool in_quote = false; + for (int i = 0; i < len; i++) { + if (in_quote) { + if (src[i] == ')') { + in_quote = false; + } + } else { + if (src[i] == '(') { + in_quote = true; + } else if (src[i] == p_chr) { + return i; + } + } + } + + return -1; +} + +void PluralRules::_cache_plural_tests(const String &p_plural_rule, Ref &p_node) { + // Some examples of p_plural_rule passed in can have the form: + // "n==0 ? 0 : n==1 ? 1 : n==2 ? 2 : n%100>=3 && n%100<=10 ? 3 : n%100>=11 && n%100<=99 ? 4 : 5" (Arabic) + // "n >= 2" (French) // When evaluating the last, especially careful with this one. + // "n != 1" (English) + + String rule = p_plural_rule; + if (rule.begins_with("(") && rule.ends_with(")")) { + int bcount = 0; + for (int i = 1; i < rule.length() - 1 && bcount >= 0; i++) { + if (rule[i] == '(') { + bcount++; + } else if (rule[i] == ')') { + bcount--; + } + } + if (bcount == 0) { + rule = rule.substr(1, rule.length() - 2); + } + } + + int first_ques_mark = _find_unquoted(rule, '?'); + int first_colon = _find_unquoted(rule, ':'); + + if (first_ques_mark == -1) { + p_node->regex = rule.strip_edges(); + return; + } + + p_node->regex = rule.substr(0, first_ques_mark).strip_edges(); + + p_node->left.instantiate(); + _cache_plural_tests(rule.substr(first_ques_mark + 1, first_colon - first_ques_mark - 1).strip_edges(), p_node->left); + p_node->right.instantiate(); + _cache_plural_tests(rule.substr(first_colon + 1).strip_edges(), p_node->right); +} + +int PluralRules::evaluate(int p_n) const { + const int *cached = cache.getptr(p_n); + if (cached) { + return *cached; + } + + const Array &input_val = { p_n }; + int index = _eq_test(input_val, equi_tests, 0); + cache.insert(p_n, index); + return index; +} + +PluralRules::PluralRules(int p_nplurals, const String &p_plural) : + nplurals(p_nplurals), + plural(p_plural) { + equi_tests.instantiate(); + _cache_plural_tests(plural, equi_tests); + + expr.instantiate(); +} + +PluralRules *PluralRules::parse(const String &p_rules) { + // `p_rules` should be in the format "nplurals=; plural=;". + + const int nplurals_eq = p_rules.find_char('='); + ERR_FAIL_COND_V_MSG(nplurals_eq == -1, nullptr, "Invalid plural rules format. Missing equal sign for `nplurals`."); + + const int nplurals_semi_col = p_rules.find_char(';', nplurals_eq); + ERR_FAIL_COND_V_MSG(nplurals_semi_col == -1, nullptr, "Invalid plural rules format. Missing semicolon for `nplurals`."); + + const String nplurals_str = p_rules.substr(nplurals_eq + 1, nplurals_semi_col - (nplurals_eq + 1)).strip_edges(); + ERR_FAIL_COND_V_MSG(!nplurals_str.is_valid_int(), nullptr, "Invalid plural rules format. `nplurals` should be an integer."); + + const int nplurals = nplurals_str.to_int(); + ERR_FAIL_COND_V_MSG(nplurals < 1, nullptr, "Invalid plural rules format. `nplurals` should be at least 1."); + + const int expression_eq = p_rules.find_char('=', nplurals_semi_col + 1); + ERR_FAIL_COND_V_MSG(expression_eq == -1, nullptr, "Invalid plural rules format. Missing equal sign for `plural`."); + + int expression_end = p_rules.rfind_char(';'); + if (expression_end == -1) { + WARN_PRINT("Invalid plural rules format. Missing semicolon at the end of `plural` expression. Assuming ends at the end of the string."); + expression_end = p_rules.length(); + } + + const int expression_start = expression_eq + 1; + ERR_FAIL_COND_V_MSG(expression_end <= expression_start, nullptr, "Invalid plural rules format. `plural` expression is empty."); + + const String &plural = p_rules.substr(expression_start, expression_end - expression_start).strip_edges(); + return memnew(PluralRules(nplurals, plural)); +} diff --git a/core/string/plural_rules.h b/core/string/plural_rules.h new file mode 100644 index 00000000000..f549cc3c762 --- /dev/null +++ b/core/string/plural_rules.h @@ -0,0 +1,72 @@ +/**************************************************************************/ +/* plural_rules.h */ +/**************************************************************************/ +/* This file is part of: */ +/* GODOT ENGINE */ +/* https://godotengine.org */ +/**************************************************************************/ +/* Copyright (c) 2014-present Godot Engine contributors (see AUTHORS.md). */ +/* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur. */ +/* */ +/* Permission is hereby granted, free of charge, to any person obtaining */ +/* a copy of this software and associated documentation files (the */ +/* "Software"), to deal in the Software without restriction, including */ +/* without limitation the rights to use, copy, modify, merge, publish, */ +/* distribute, sublicense, and/or sell copies of the Software, and to */ +/* permit persons to whom the Software is furnished to do so, subject to */ +/* the following conditions: */ +/* */ +/* The above copyright notice and this permission notice shall be */ +/* included in all copies or substantial portions of the Software. */ +/* */ +/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */ +/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */ +/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. */ +/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */ +/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */ +/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */ +/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +/**************************************************************************/ + +#pragma once + +#include "core/object/ref_counted.h" +#include "core/templates/lru.h" + +class Expression; + +class PluralRules : public Object { + GDSOFTCLASS(PluralRules, Object); + + mutable LRUCache cache; + + // These two fields are initialized in the constructor. + const int nplurals; + const String plural; + + // Cache temporary variables related to `evaluate()` to make it faster. + class EQNode : public RefCounted { + GDSOFTCLASS(EQNode, RefCounted); + + public: + String regex; + Ref left; + Ref right; + }; + Ref equi_tests; + Ref expr; + + int _find_unquoted(const String &p_src, char32_t p_chr) const; + int _eq_test(const Array &p_input_val, const Ref &p_node, const Variant &p_result) const; + void _cache_plural_tests(const String &p_plural_rule, Ref &p_node); + + PluralRules(int p_nplurals, const String &p_plural); + +public: + int evaluate(int p_n) const; + + int get_nplurals() const { return nplurals; } + String get_plural() const { return plural; } + + static PluralRules *parse(const String &p_rules); +}; diff --git a/core/string/translation_po.cpp b/core/string/translation_po.cpp index 8ff9006a181..569efe594cc 100644 --- a/core/string/translation_po.cpp +++ b/core/string/translation_po.cpp @@ -30,6 +30,8 @@ #include "translation_po.h" +#include "core/string/plural_rules.h" + #ifdef DEBUG_TRANSLATION_PO #include "core/io/file_access.h" @@ -129,112 +131,11 @@ Vector TranslationPO::_get_message_list() const { return v; } -int TranslationPO::_get_plural_index(int p_n) const { - // Get a number between [0;number of plural forms). - - input_val.clear(); - input_val.push_back(p_n); - - return _eq_test(equi_tests, 0); -} - -int TranslationPO::_eq_test(const Ref &p_node, const Variant &p_result) const { - if (p_node.is_valid()) { - Error err = expr->parse(p_node->regex, input_name); - ERR_FAIL_COND_V_MSG(err != OK, 0, vformat("Cannot parse expression \"%s\". Error: %s", p_node->regex, expr->get_error_text())); - - Variant result = expr->execute(input_val); - ERR_FAIL_COND_V_MSG(expr->has_execute_failed(), 0, vformat("Cannot evaluate expression \"%s\".", p_node->regex)); - - if (bool(result)) { - return _eq_test(p_node->left, result); - } else { - return _eq_test(p_node->right, result); - } - } else { - return p_result; - } -} - -int TranslationPO::_find_unquoted(const String &p_src, char32_t p_chr) const { - const int len = p_src.length(); - if (len == 0) { - return -1; - } - - const char32_t *src = p_src.get_data(); - bool in_quote = false; - for (int i = 0; i < len; i++) { - if (in_quote) { - if (src[i] == ')') { - in_quote = false; - } - } else { - if (src[i] == '(') { - in_quote = true; - } else if (src[i] == p_chr) { - return i; - } - } - } - - return -1; -} - -void TranslationPO::_cache_plural_tests(const String &p_plural_rule, Ref &p_node) { - // Some examples of p_plural_rule passed in can have the form: - // "n==0 ? 0 : n==1 ? 1 : n==2 ? 2 : n%100>=3 && n%100<=10 ? 3 : n%100>=11 && n%100<=99 ? 4 : 5" (Arabic) - // "n >= 2" (French) // When evaluating the last, especially careful with this one. - // "n != 1" (English) - - String rule = p_plural_rule; - if (rule.begins_with("(") && rule.ends_with(")")) { - int bcount = 0; - for (int i = 1; i < rule.length() - 1 && bcount >= 0; i++) { - if (rule[i] == '(') { - bcount++; - } else if (rule[i] == ')') { - bcount--; - } - } - if (bcount == 0) { - rule = rule.substr(1, rule.length() - 2); - } - } - - int first_ques_mark = _find_unquoted(rule, '?'); - int first_colon = _find_unquoted(rule, ':'); - - if (first_ques_mark == -1) { - p_node->regex = rule.strip_edges(); - return; - } - - p_node->regex = rule.substr(0, first_ques_mark).strip_edges(); - - p_node->left.instantiate(); - _cache_plural_tests(rule.substr(first_ques_mark + 1, first_colon - first_ques_mark - 1).strip_edges(), p_node->left); - p_node->right.instantiate(); - _cache_plural_tests(rule.substr(first_colon + 1).strip_edges(), p_node->right); -} - void TranslationPO::set_plural_rule(const String &p_plural_rule) { - // Set plural_forms and plural_rule. - // p_plural_rule passed in has the form "Plural-Forms: nplurals=2; plural=(n >= 2);". - - int first_semi_col = p_plural_rule.find_char(';'); - plural_forms = p_plural_rule.substr(p_plural_rule.find_char('=') + 1, first_semi_col - (p_plural_rule.find_char('=') + 1)).to_int(); - - int expression_start = p_plural_rule.find_char('=', first_semi_col) + 1; - int second_semi_col = p_plural_rule.rfind_char(';'); - plural_rule = p_plural_rule.substr(expression_start, second_semi_col - expression_start).strip_edges(); - - // Setup the cache to make evaluating plural rule faster later on. - equi_tests.instantiate(); - _cache_plural_tests(plural_rule, equi_tests); - - expr.instantiate(); - input_name.push_back("n"); + if (plural_rules) { + memdelete(plural_rules); + } + plural_rules = PluralRules::parse(p_plural_rule); } void TranslationPO::add_message(const StringName &p_src_text, const StringName &p_xlated_text, const StringName &p_context) { @@ -249,7 +150,8 @@ void TranslationPO::add_message(const StringName &p_src_text, const StringName & } void TranslationPO::add_plural_message(const StringName &p_src_text, const Vector &p_plural_xlated_texts, const StringName &p_context) { - ERR_FAIL_COND_MSG(p_plural_xlated_texts.size() != plural_forms, vformat("Trying to add plural texts that don't match the required number of plural forms for locale \"%s\".", get_locale())); + ERR_FAIL_NULL_MSG(plural_rules, "Plural rules are not set. Please call set_plural_rule() before calling add_plural_message()."); + ERR_FAIL_COND_MSG(p_plural_xlated_texts.size() != plural_rules->get_nplurals(), vformat("Trying to add plural texts that don't match the required number of plural forms for locale \"%s\".", get_locale())); HashMap> &map_id_str = translation_map[p_context]; @@ -264,11 +166,11 @@ void TranslationPO::add_plural_message(const StringName &p_src_text, const Vecto } int TranslationPO::get_plural_forms() const { - return plural_forms; + return plural_rules ? plural_rules->get_nplurals() : 0; } String TranslationPO::get_plural_rule() const { - return plural_rule; + return plural_rules ? plural_rules->get_plural() : String(); } StringName TranslationPO::get_message(const StringName &p_src_text, const StringName &p_context) const { @@ -282,27 +184,16 @@ StringName TranslationPO::get_message(const StringName &p_src_text, const String StringName TranslationPO::get_plural_message(const StringName &p_src_text, const StringName &p_plural_text, int p_n, const StringName &p_context) const { ERR_FAIL_COND_V_MSG(p_n < 0, StringName(), "N passed into translation to get a plural message should not be negative. For negative numbers, use singular translation please. Search \"gettext PO Plural Forms\" online for the documentation on translating negative numbers."); - - // If the query is the same as last time, return the cached result. - if (p_n == last_plural_n && p_context == last_plural_context && p_src_text == last_plural_key) { - return translation_map[p_context][p_src_text][last_plural_mapped_index]; - } + ERR_FAIL_NULL_V_MSG(plural_rules, StringName(), "Plural rules are not set. Please call set_plural_rule() before calling get_plural_message()."); if (!translation_map.has(p_context) || !translation_map[p_context].has(p_src_text)) { return StringName(); } ERR_FAIL_COND_V_MSG(translation_map[p_context][p_src_text].is_empty(), StringName(), vformat("Source text \"%s\" is registered but doesn't have a translation. Please report this bug.", String(p_src_text))); - int plural_index = _get_plural_index(p_n); + int plural_index = plural_rules->evaluate(p_n); ERR_FAIL_COND_V_MSG(plural_index < 0 || translation_map[p_context][p_src_text].size() < plural_index + 1, StringName(), "Plural index returned or number of plural translations is not valid. Please report this bug."); - // Cache result so that if the next entry is the same, we can return directly. - // _get_plural_index(p_n) can get very costly, especially when evaluating long plural-rule (Arabic) - last_plural_key = p_src_text; - last_plural_context = p_context; - last_plural_n = p_n; - last_plural_mapped_index = plural_index; - return translation_map[p_context][p_src_text][plural_index]; } @@ -343,3 +234,10 @@ void TranslationPO::_bind_methods() { ClassDB::bind_method(D_METHOD("get_plural_forms"), &TranslationPO::get_plural_forms); ClassDB::bind_method(D_METHOD("get_plural_rule"), &TranslationPO::get_plural_rule); } + +TranslationPO::~TranslationPO() { + if (plural_rules) { + memdelete(plural_rules); + plural_rules = nullptr; + } +} diff --git a/core/string/translation_po.h b/core/string/translation_po.h index 3f92264a007..ab7d0e7b8f1 100644 --- a/core/string/translation_po.h +++ b/core/string/translation_po.h @@ -32,9 +32,10 @@ //#define DEBUG_TRANSLATION_PO -#include "core/math/expression.h" #include "core/string/translation.h" +class PluralRules; + class TranslationPO : public Translation { GDCLASS(TranslationPO, Translation); @@ -45,33 +46,7 @@ class TranslationPO : public Translation { // Strings without context have "" as first key. HashMap>> translation_map; - int plural_forms = 0; // 0 means no "Plural-Forms" is given in the PO header file. The min for all languages is 1. - String plural_rule; - - // Cache temporary variables related to _get_plural_index() to make it faster - class EQNode : public RefCounted { - GDSOFTCLASS(EQNode, RefCounted); - - public: - String regex; - Ref left; - Ref right; - }; - Ref equi_tests; - - int _find_unquoted(const String &p_src, char32_t p_chr) const; - int _eq_test(const Ref &p_node, const Variant &p_result) const; - - Vector input_name; - mutable Ref expr; - mutable Array input_val; - mutable StringName last_plural_key; - mutable StringName last_plural_context; - mutable int last_plural_n = -1; // Set it to an impossible value at the beginning. - mutable int last_plural_mapped_index = 0; - - void _cache_plural_tests(const String &p_plural_rule, Ref &p_node); - int _get_plural_index(int p_n) const; + PluralRules *plural_rules = nullptr; Vector _get_message_list() const override; Dictionary _get_messages() const override; @@ -98,5 +73,5 @@ public: void print_translation_map(); #endif - TranslationPO() {} + ~TranslationPO(); }; diff --git a/tests/core/string/test_translation.h b/tests/core/string/test_translation.h index d01137122d7..cf0a496634b 100644 --- a/tests/core/string/test_translation.h +++ b/tests/core/string/test_translation.h @@ -31,6 +31,7 @@ #pragma once #include "core/string/optimized_translation.h" +#include "core/string/plural_rules.h" #include "core/string/translation.h" #include "core/string/translation_po.h" #include "core/string/translation_server.h" @@ -129,6 +130,54 @@ TEST_CASE("[TranslationPO] Plural messages") { CHECK(vformat(translation->get_plural_message("There are %d apples", "", 2), 2) == "Il y a 2 pommes"); } +TEST_CASE("[TranslationPO] Plural rules parsing") { + ERR_PRINT_OFF; + { + CHECK(PluralRules::parse("") == nullptr); + + CHECK(PluralRules::parse("plurals=(n != 1);") == nullptr); + CHECK(PluralRules::parse("nplurals; plurals=(n != 1);") == nullptr); + CHECK(PluralRules::parse("nplurals=; plurals=(n != 1);") == nullptr); + CHECK(PluralRules::parse("nplurals=0; plurals=(n != 1);") == nullptr); + CHECK(PluralRules::parse("nplurals=-1; plurals=(n != 1);") == nullptr); + + CHECK(PluralRules::parse("nplurals=2;") == nullptr); + CHECK(PluralRules::parse("nplurals=2; plurals;") == nullptr); + CHECK(PluralRules::parse("nplurals=2; plurals=;") == nullptr); + } + ERR_PRINT_ON; + + { + PluralRules *pr = PluralRules::parse("nplurals=3; plural=(n==0 ? 0 : n==1 ? 1 : 2);"); + REQUIRE(pr != nullptr); + + CHECK(pr->get_nplurals() == 3); + CHECK(pr->get_plural() == "(n==0 ? 0 : n==1 ? 1 : 2)"); + + CHECK(pr->evaluate(0) == 0); + CHECK(pr->evaluate(1) == 1); + CHECK(pr->evaluate(2) == 2); + CHECK(pr->evaluate(3) == 2); + + memdelete(pr); + } + + { + PluralRules *pr = PluralRules::parse("nplurals=1; plural=0;"); + REQUIRE(pr != nullptr); + + CHECK(pr->get_nplurals() == 1); + CHECK(pr->get_plural() == "0"); + + CHECK(pr->evaluate(0) == 0); + CHECK(pr->evaluate(1) == 0); + CHECK(pr->evaluate(2) == 0); + CHECK(pr->evaluate(3) == 0); + + memdelete(pr); + } +} + #ifdef TOOLS_ENABLED TEST_CASE("[OptimizedTranslation] Generate from Translation and read messages") { Ref translation = memnew(Translation); From e882e42e1be615b0a4aefeaeb0bb52d7a6aa3fa0 Mon Sep 17 00:00:00 2001 From: Haoyu Qiu Date: Mon, 21 Jul 2025 17:59:14 +0800 Subject: [PATCH 2/3] Add default plural rules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This makes the PO loader correctly handle the situation where the optional `Plural-Forms` header field does not exist. The `Translation` class and its subclasses always have access to valid plural rules via `_get_plural_rules()`. Plural rules are prioritized: 1. `Translation.plural_rules_override` 2. `TranslationServer.get_plural_rules(locale)` 3. The English plural rules: `nplurals=2; plurals=(n != 1)` Co-Authored-By: Pāvels Nadtočajevs <7645683+bruvzg@users.noreply.github.com> --- core/io/translation_loader_po.cpp | 4 +-- core/string/locales.h | 47 ++++++++++++++++++++++++ core/string/translation.cpp | 54 ++++++++++++++++++++++++++++ core/string/translation.h | 12 ++++++- core/string/translation_po.cpp | 24 +++---------- core/string/translation_po.h | 7 ---- core/string/translation_server.cpp | 42 +++++++++++++++++----- core/string/translation_server.h | 16 +++++---- doc/classes/Translation.xml | 4 +++ doc/classes/TranslationServer.xml | 7 ++++ tests/core/string/test_translation.h | 46 ++++++++++++++++-------- 11 files changed, 202 insertions(+), 61 deletions(-) diff --git a/core/io/translation_loader_po.cpp b/core/io/translation_loader_po.cpp index 35773109a97..fcab40e653f 100644 --- a/core/io/translation_loader_po.cpp +++ b/core/io/translation_loader_po.cpp @@ -112,7 +112,7 @@ Ref TranslationLoaderPO::load_translation(Ref f, Error *r_ int p_start = config.find("Plural-Forms"); if (p_start != -1) { int p_end = config.find_char('\n', p_start); - translation->set_plural_rule(config.substr(p_start, p_end - p_start)); + translation->set_plural_rules_override(config.substr(p_start, p_end - p_start)); } } else { uint32_t str_start = 0; @@ -228,7 +228,7 @@ Ref TranslationLoaderPO::load_translation(Ref f, Error *r_ int p_start = config.find("Plural-Forms"); if (p_start != -1) { int p_end = config.find_char('\n', p_start); - translation->set_plural_rule(config.substr(p_start, p_end - p_start)); + translation->set_plural_rules_override(config.substr(p_start, p_end - p_start)); plural_forms = translation->get_plural_forms(); } } diff --git a/core/string/locales.h b/core/string/locales.h index f80e07789b7..6f1f5819094 100644 --- a/core/string/locales.h +++ b/core/string/locales.h @@ -1194,3 +1194,50 @@ static const char *script_list[][2] = { { "Zanabazar Square", "Zanb" }, { nullptr, nullptr } }; + +// Plural rules. +// Reference: +// - https://github.com/unicode-org/cldr/blob/main/common/supplemental/plurals.xml +static const char *plural_rules[][2] = { + { "bm bo dz hnj id ig ii in ja jbo jv jw kde kea km ko lkt lo ms my nqo osa root sah ses sg su th to tpi vi wo yo yue zh", "nplurals=1; plural=0;" }, + { "am as bn doi fa gu hi kn pcm zu", "nplurals=2; plural=(n==0 || n==1);" }, + { "ff hy kab", "nplurals=2; plural=(n > 1);" }, + { "ast de en et fi fy gl ia io ji lij nl sc sv sw ur yi", "nplurals=2; plural=(n != 1);" }, + { "si", "nplurals=2; plural=(n > 1);" }, + { "ak bho csw guw ln mg nso pa ti wa", "nplurals=2; plural=(n > 1);" }, + { "tzm", "nplurals=2; plural=(n<=1 || (n>=11 && n<=99));" }, + { "af an asa az bal bem bez bg brx ce cgg chr ckb dv ee el eo eu fo fur gsw ha haw hu jgo jmc ka kaj kcg kk kkj kl ks ksb ku ky lb lg mas mgo ml mn mr nah nb nd ne nn nnh no nr ny nyn om or os pap ps rm rof rwk saq sd sdh seh sn so sq ss ssy st syr ta te teo tig tk tn tr ts ug uz ve vo vun wae xh xog", "nplurals=2; plural=(n != 1);" }, + { "da", "nplurals=2; plural=(n != 1);" }, + { "is", "nplurals=2; plural=(n%10==1 && n%100!=11);" }, + { "mk", "nplurals=2; plural=(n%10==1 && n%100!=11);" }, + { "ceb fil tl", "nplurals=2; plural=(n==1 || n==2 || n==3 || (n%10!=4 && n%10!=6 && n%10!=9));" }, + { "lv prg", "nplurals=3; plural=(n%10==0 || (n%100>=11 && n%100<=19) ? 0 : n%10==1 && n%100!=11 ? 1 : 2);" }, + { "lag", "nplurals=3; plural=(n==0 ? 0 : (n==0 || n==1) && n!=0 ? 1 : 2);" }, + { "blo", "nplurals=3; plural=(n==0 ? 0 : n==1 ? 1 : 2);" }, + { "ksh", "nplurals=3; plural=(n==0 ? 0 : n==1 ? 1 : 2);" }, + { "he iw", "nplurals=3; plural=(n==1 ? 0 : n==2 ? 1 : 2);" }, + { "iu naq sat se sma smi smj smn sms", "nplurals=3; plural=(n==1 ? 0 : n==2 ? 1 : 2);" }, + { "shi", "nplurals=3; plural=(n==0 || n==1 ? 0 : n>=2 && n<=10 ? 1 : 2);" }, + { "mo ro", "nplurals=3; plural=(n==1 ? 0 : n==0 || (n!=1 && n%100>=1 && n%100<=19) ? 1 : 2);" }, + { "bs hr sh sr", "nplurals=3; plural=(n%10==1 && n%100!=11 ? 0 : n%10>=2 && n%10<=4 && (n%100<12 || n%100>14) ? 1 : 2);" }, + { "fr", "nplurals=3; plural=(n == 0 || n == 1) ? 0 : n != 0 && n % 1000000 == 0 ? 1 : 2);" }, + { "pt", "nplurals=3; plural=((n == 0 || n == 1) ? 0 : n != 0 && n % 1000000 == 0 ? 1 : 2);" }, + { "ca it lld pt_PT scn vec", "nplurals=3; plural=(n == 1 ? 0 : n != 0 && n % 1000000 == 0 ? 1 : 2);" }, + { "es", "nplurals=3; plural=(n == 1 ? 0 : n != 0 && n % 1000000 == 0 ? 1 : 2);" }, + { "gd", "nplurals=4; plural=(n==1 || n==11 ? 0 : n==2 || n==12 ? 1 : (n>=3 && n<=10) || (n>=13 && n<=19) ? 2 : 3);" }, + { "sl", "nplurals=4; plural=(n%100==1 ? 0 : n%100==2 ? 1 : n%100>=3 && n%100<=4 ? 2 : 3);" }, + { "dsb hsb", "nplurals=4; plural=(n%100==1 ? 0 : n%100==2 ? 1 : n%100>=3 && n%100<=4 ? 2 : 3);" }, + { "cs sk", "nplurals=3; plural=(n==1 ? 0 : n>=2 && n<=4 ? 1 : 2);" }, + { "pl", "nplurals=3; plural=(n==1 ? 0 : n%10>=2 && n%10<=4 && (n%100<12 || n%100>14) ? 1 : 2);" }, + { "be", "nplurals=3; plural=(n%10==1 && n%100!=11 ? 0 : n%10>=2 && n%10<=4 && (n%100<12 || n%100>14) ? 1 : 2);" }, + { "lt", "nplurals=3; plural=(n%10==1 && (n%100<11 || n%100>19) ? 0 : n%10>=2 && n%10<=9 && (n%100<11 || n%100>19) ? 1 : 2);" }, + { "ru uk", "nplurals=3; plural=(n%10==1 && n%100!=11 ? 0 : n%10>=2 && n%10<=4 && (n%100<12 || n%100>14) ? 1 : 2);" }, + { "br", "nplurals=5; plural=(n%10==1 && n%100!=11 && n%100!=71 && n%100!=91 ? 0 : n%10==2 && n%100!=12 && n%100!=72 && n%100!=92 ? 1 : ((n%10>=3 && n%10<=4) || n%10==9) && (n%100<10 || n%100>19) && (n%100<70 || n%100>79) && (n%100<90 || n%100>99) ? 2 : n!=0 && n%1000000==0 ? 3 : 4);" }, + { "mt", "nplurals=5; plural=(n==1 ? 0 : n==2 ? 1 : n==0 || (n%100>=3 && n%100<=10) ? 2 : n%100>=11 && n%100<=19 ? 3 : 4);" }, + { "ga", "nplurals=5; plural=(n==1 ? 0 : n==2 ? 1 : n>=3 && n<=6 ? 2 : n>=7 && n<=10 ? 3 : 4);" }, + { "gv", "nplurals=4; plural=(n%10==1 ? 0 : n%10==2 ? 1 : n%100==0 || n%100==20 || n%100==40 || n%100==60 || n%100==80 ? 2 : 3);" }, + { "kw", "nplurals=6; plural=(n==0 ? 0 : n==1 ? 1 : n%100==2 || n%100==22 || n%100==42 || n%100==62 || n%100==82 || (n%1000==0 && ((n%100000>=1000 && n%100000<=20000) || n%100000==40000 || n%100000==60000 || n%100000==80000)) || (n!=0 && n%1000000==100000) ? 2 : n%100==3 || n%100==23 || n%100==43 || n%100==63 || n%100==83 ? 3 : n!=1 && (n%100==1 || n%100==21 || n%100==41 || n%100==61 || n%100==81) ? 4 : 5);" }, + { "ar ars", "nplurals=6; plural=(n==0 ? 0 : n==1 ? 1 : n==2 ? 2 : n%100>=3 && n%100<=10 ? 3 : n%100>=11 && n%100<=99 ? 4 : 5);" }, + { "cy", "nplurals=6; plural=(n==0 ? 0 : n==1 ? 1 : n==2 ? 2 : n==3 ? 3 : n==6 ? 4 : 5);" }, + { nullptr, nullptr }, +}; diff --git a/core/string/translation.cpp b/core/string/translation.cpp index d7b4d86c28f..7fcfc13229b 100644 --- a/core/string/translation.cpp +++ b/core/string/translation.cpp @@ -31,6 +31,7 @@ #include "translation.h" #include "core/os/thread.h" +#include "core/string/plural_rules.h" #include "core/string/translation_server.h" Dictionary Translation::_get_messages() const { @@ -73,6 +74,11 @@ void Translation::_set_messages(const Dictionary &p_messages) { void Translation::set_locale(const String &p_locale) { locale = TranslationServer::get_singleton()->standardize_locale(p_locale); + + if (plural_rules_cache && plural_rules_override.is_empty()) { + memdelete(plural_rules_cache); + plural_rules_cache = nullptr; + } } void Translation::add_message(const StringName &p_src_text, const StringName &p_xlated_text, const StringName &p_context) { @@ -131,6 +137,44 @@ int Translation::get_message_count() const { return translation_map.size(); } +PluralRules *Translation::_get_plural_rules() const { + if (plural_rules_cache) { + return plural_rules_cache; + } + + if (!plural_rules_override.is_empty()) { + plural_rules_cache = PluralRules::parse(plural_rules_override); + } + + if (!plural_rules_cache) { + // Locale's default plural rules. + const String &default_rule = TranslationServer::get_singleton()->get_plural_rules(locale); + if (!default_rule.is_empty()) { + plural_rules_cache = PluralRules::parse(default_rule); + } + + // Use English plural rules as a fallback. + if (!plural_rules_cache) { + plural_rules_cache = PluralRules::parse("nplurals=2; plural=(n != 1);"); + } + } + + DEV_ASSERT(plural_rules_cache != nullptr); + return plural_rules_cache; +} + +void Translation::set_plural_rules_override(const String &p_rules) { + plural_rules_override = p_rules; + if (plural_rules_cache) { + memdelete(plural_rules_cache); + plural_rules_cache = nullptr; + } +} + +String Translation::get_plural_rules_override() const { + return plural_rules_override; +} + void Translation::_bind_methods() { ClassDB::bind_method(D_METHOD("set_locale", "locale"), &Translation::set_locale); ClassDB::bind_method(D_METHOD("get_locale"), &Translation::get_locale); @@ -144,10 +188,20 @@ void Translation::_bind_methods() { ClassDB::bind_method(D_METHOD("get_message_count"), &Translation::get_message_count); ClassDB::bind_method(D_METHOD("_set_messages", "messages"), &Translation::_set_messages); ClassDB::bind_method(D_METHOD("_get_messages"), &Translation::_get_messages); + ClassDB::bind_method(D_METHOD("set_plural_rules_override", "rules"), &Translation::set_plural_rules_override); + ClassDB::bind_method(D_METHOD("get_plural_rules_override"), &Translation::get_plural_rules_override); GDVIRTUAL_BIND(_get_plural_message, "src_message", "src_plural_message", "n", "context"); GDVIRTUAL_BIND(_get_message, "src_message", "context"); ADD_PROPERTY(PropertyInfo(Variant::DICTIONARY, "messages", PROPERTY_HINT_NONE, "", PROPERTY_USAGE_NO_EDITOR | PROPERTY_USAGE_INTERNAL), "_set_messages", "_get_messages"); ADD_PROPERTY(PropertyInfo(Variant::STRING, "locale"), "set_locale", "get_locale"); + ADD_PROPERTY(PropertyInfo(Variant::STRING, "plural_rules_override"), "set_plural_rules_override", "get_plural_rules_override"); +} + +Translation::~Translation() { + if (plural_rules_cache) { + memdelete(plural_rules_cache); + plural_rules_cache = nullptr; + } } diff --git a/core/string/translation.h b/core/string/translation.h index 3e445bedcb0..3136d8420f1 100644 --- a/core/string/translation.h +++ b/core/string/translation.h @@ -33,6 +33,8 @@ #include "core/io/resource.h" #include "core/object/gdvirtual.gen.inc" +class PluralRules; + class Translation : public Resource { GDCLASS(Translation, Resource); OBJ_SAVE_TYPE(Translation); @@ -41,6 +43,9 @@ class Translation : public Resource { String locale = "en"; HashMap translation_map; + mutable PluralRules *plural_rules_cache = nullptr; + String plural_rules_override; + virtual Vector _get_message_list() const; virtual Dictionary _get_messages() const; virtual void _set_messages(const Dictionary &p_messages); @@ -48,6 +53,8 @@ class Translation : public Resource { protected: static void _bind_methods(); + PluralRules *_get_plural_rules() const; + GDVIRTUAL2RC(StringName, _get_message, StringName, StringName); GDVIRTUAL4RC(StringName, _get_plural_message, StringName, StringName, int, StringName); @@ -64,5 +71,8 @@ public: virtual int get_message_count() const; virtual Vector get_translated_message_list() const; - Translation() {} + void set_plural_rules_override(const String &p_rules); + String get_plural_rules_override() const; + + ~Translation(); }; diff --git a/core/string/translation_po.cpp b/core/string/translation_po.cpp index 569efe594cc..3eaecbf5344 100644 --- a/core/string/translation_po.cpp +++ b/core/string/translation_po.cpp @@ -131,13 +131,6 @@ Vector TranslationPO::_get_message_list() const { return v; } -void TranslationPO::set_plural_rule(const String &p_plural_rule) { - if (plural_rules) { - memdelete(plural_rules); - } - plural_rules = PluralRules::parse(p_plural_rule); -} - void TranslationPO::add_message(const StringName &p_src_text, const StringName &p_xlated_text, const StringName &p_context) { HashMap> &map_id_str = translation_map[p_context]; @@ -150,8 +143,7 @@ void TranslationPO::add_message(const StringName &p_src_text, const StringName & } void TranslationPO::add_plural_message(const StringName &p_src_text, const Vector &p_plural_xlated_texts, const StringName &p_context) { - ERR_FAIL_NULL_MSG(plural_rules, "Plural rules are not set. Please call set_plural_rule() before calling add_plural_message()."); - ERR_FAIL_COND_MSG(p_plural_xlated_texts.size() != plural_rules->get_nplurals(), vformat("Trying to add plural texts that don't match the required number of plural forms for locale \"%s\".", get_locale())); + ERR_FAIL_COND_MSG(p_plural_xlated_texts.size() != _get_plural_rules()->get_nplurals(), vformat("Trying to add plural texts that don't match the required number of plural forms for locale \"%s\".", get_locale())); HashMap> &map_id_str = translation_map[p_context]; @@ -166,11 +158,11 @@ void TranslationPO::add_plural_message(const StringName &p_src_text, const Vecto } int TranslationPO::get_plural_forms() const { - return plural_rules ? plural_rules->get_nplurals() : 0; + return _get_plural_rules()->get_nplurals(); } String TranslationPO::get_plural_rule() const { - return plural_rules ? plural_rules->get_plural() : String(); + return _get_plural_rules()->get_plural(); } StringName TranslationPO::get_message(const StringName &p_src_text, const StringName &p_context) const { @@ -184,14 +176,13 @@ StringName TranslationPO::get_message(const StringName &p_src_text, const String StringName TranslationPO::get_plural_message(const StringName &p_src_text, const StringName &p_plural_text, int p_n, const StringName &p_context) const { ERR_FAIL_COND_V_MSG(p_n < 0, StringName(), "N passed into translation to get a plural message should not be negative. For negative numbers, use singular translation please. Search \"gettext PO Plural Forms\" online for the documentation on translating negative numbers."); - ERR_FAIL_NULL_V_MSG(plural_rules, StringName(), "Plural rules are not set. Please call set_plural_rule() before calling get_plural_message()."); if (!translation_map.has(p_context) || !translation_map[p_context].has(p_src_text)) { return StringName(); } ERR_FAIL_COND_V_MSG(translation_map[p_context][p_src_text].is_empty(), StringName(), vformat("Source text \"%s\" is registered but doesn't have a translation. Please report this bug.", String(p_src_text))); - int plural_index = plural_rules->evaluate(p_n); + int plural_index = _get_plural_rules()->evaluate(p_n); ERR_FAIL_COND_V_MSG(plural_index < 0 || translation_map[p_context][p_src_text].size() < plural_index + 1, StringName(), "Plural index returned or number of plural translations is not valid. Please report this bug."); return translation_map[p_context][p_src_text][plural_index]; @@ -234,10 +225,3 @@ void TranslationPO::_bind_methods() { ClassDB::bind_method(D_METHOD("get_plural_forms"), &TranslationPO::get_plural_forms); ClassDB::bind_method(D_METHOD("get_plural_rule"), &TranslationPO::get_plural_rule); } - -TranslationPO::~TranslationPO() { - if (plural_rules) { - memdelete(plural_rules); - plural_rules = nullptr; - } -} diff --git a/core/string/translation_po.h b/core/string/translation_po.h index ab7d0e7b8f1..dd0bc3c602b 100644 --- a/core/string/translation_po.h +++ b/core/string/translation_po.h @@ -34,8 +34,6 @@ #include "core/string/translation.h" -class PluralRules; - class TranslationPO : public Translation { GDCLASS(TranslationPO, Translation); @@ -46,8 +44,6 @@ class TranslationPO : public Translation { // Strings without context have "" as first key. HashMap>> translation_map; - PluralRules *plural_rules = nullptr; - Vector _get_message_list() const override; Dictionary _get_messages() const override; void _set_messages(const Dictionary &p_messages) override; @@ -65,13 +61,10 @@ public: StringName get_plural_message(const StringName &p_src_text, const StringName &p_plural_text, int p_n, const StringName &p_context = "") const override; void erase_message(const StringName &p_src_text, const StringName &p_context = "") override; - void set_plural_rule(const String &p_plural_rule); int get_plural_forms() const; String get_plural_rule() const; #ifdef DEBUG_TRANSLATION_PO void print_translation_map(); #endif - - ~TranslationPO(); }; diff --git a/core/string/translation_server.cpp b/core/string/translation_server.cpp index ea9c103e430..d9e72ea808b 100644 --- a/core/string/translation_server.cpp +++ b/core/string/translation_server.cpp @@ -36,15 +36,6 @@ #include "core/os/os.h" #include "core/string/locales.h" -Vector TranslationServer::locale_script_info; - -HashMap TranslationServer::language_map; -HashMap TranslationServer::script_map; -HashMap TranslationServer::locale_rename_map; -HashMap TranslationServer::country_name_map; -HashMap TranslationServer::variant_map; -HashMap TranslationServer::country_rename_map; - void TranslationServer::init_locale_info() { // Init locale info. language_map.clear(); @@ -113,6 +104,18 @@ void TranslationServer::init_locale_info() { } idx++; } + + // Init plural rules. + plural_rules_map.clear(); + idx = 0; + while (plural_rules[idx][0] != nullptr) { + const Vector rule_locs = String(plural_rules[idx][0]).split(" "); + const String rule = String(plural_rules[idx][1]); + for (const String &l : rule_locs) { + plural_rules_map[l] = rule; + } + idx++; + } } TranslationServer::Locale::operator String() const { @@ -305,6 +308,26 @@ String TranslationServer::get_locale_name(const String &p_locale) const { return name; } +String TranslationServer::get_plural_rules(const String &p_locale) const { + const String *rule = plural_rules_map.getptr(p_locale); + if (rule) { + return *rule; + } + + Locale l = Locale(*this, p_locale, false); + if (!l.country.is_empty()) { + rule = plural_rules_map.getptr(l.language + "_" + l.country); + if (rule) { + return *rule; + } + } + rule = plural_rules_map.getptr(l.language); + if (rule) { + return *rule; + } + return String(); +} + Vector TranslationServer::get_all_languages() const { Vector languages; @@ -584,6 +607,7 @@ void TranslationServer::_bind_methods() { ClassDB::bind_method(D_METHOD("get_country_name", "country"), &TranslationServer::get_country_name); ClassDB::bind_method(D_METHOD("get_locale_name", "locale"), &TranslationServer::get_locale_name); + ClassDB::bind_method(D_METHOD("get_plural_rules", "locale"), &TranslationServer::get_plural_rules); ClassDB::bind_method(D_METHOD("translate", "message", "context"), &TranslationServer::translate, DEFVAL(StringName())); ClassDB::bind_method(D_METHOD("translate_plural", "message", "plural_message", "n", "context"), &TranslationServer::translate_plural, DEFVAL(StringName())); diff --git a/core/string/translation_server.h b/core/string/translation_server.h index 48632953ce5..f904f074b89 100644 --- a/core/string/translation_server.h +++ b/core/string/translation_server.h @@ -62,7 +62,7 @@ class TranslationServer : public Object { String default_country; HashSet supported_countries; }; - static Vector locale_script_info; + static inline Vector locale_script_info; struct Locale { String language; @@ -82,12 +82,13 @@ class TranslationServer : public Object { Locale(const TranslationServer &p_server, const String &p_locale, bool p_add_defaults); }; - static HashMap language_map; - static HashMap script_map; - static HashMap locale_rename_map; - static HashMap country_name_map; - static HashMap country_rename_map; - static HashMap variant_map; + static inline HashMap language_map; + static inline HashMap script_map; + static inline HashMap locale_rename_map; + static inline HashMap country_name_map; + static inline HashMap country_rename_map; + static inline HashMap variant_map; + static inline HashMap plural_rules_map; void init_locale_info(); @@ -113,6 +114,7 @@ public: String get_country_name(const String &p_country) const; String get_locale_name(const String &p_locale) const; + String get_plural_rules(const String &p_locale) const; PackedStringArray get_loaded_locales() const; diff --git a/doc/classes/Translation.xml b/doc/classes/Translation.xml index 2d365b32a87..c57c29e68bb 100644 --- a/doc/classes/Translation.xml +++ b/doc/classes/Translation.xml @@ -102,5 +102,9 @@ The locale of the translation. + + The plural rules string to enforce. See [url=https://www.gnu.org/software/gettext/manual/html_node/Plural-forms.html]GNU gettext[/url] for examples and more info. + If empty or invalid, default plural rules from [method TranslationServer.get_plural_rules] are used. The English plural rules are used as a fallback. + diff --git a/doc/classes/TranslationServer.xml b/doc/classes/TranslationServer.xml index f30a1da014c..ee05b8091b5 100644 --- a/doc/classes/TranslationServer.xml +++ b/doc/classes/TranslationServer.xml @@ -92,6 +92,13 @@ Returns the translation domain with the specified name. An empty translation domain will be created and added if it does not exist. + + + + + Returns the default plural rules for the [param locale]. + + diff --git a/tests/core/string/test_translation.h b/tests/core/string/test_translation.h index cf0a496634b..35740e547c3 100644 --- a/tests/core/string/test_translation.h +++ b/tests/core/string/test_translation.h @@ -112,22 +112,38 @@ TEST_CASE("[TranslationPO] Messages with context") { } TEST_CASE("[TranslationPO] Plural messages") { - Ref translation = memnew(TranslationPO); - translation->set_locale("fr"); - translation->set_plural_rule("Plural-Forms: nplurals=2; plural=(n >= 2);"); - CHECK(translation->get_plural_forms() == 2); + { + Ref translation = memnew(TranslationPO); + translation->set_locale("fr"); + CHECK(translation->get_plural_forms() == 3); + CHECK(translation->get_plural_rule() == "(n == 0 || n == 1) ? 0 : n != 0 && n % 1000000 == 0 ? 1 : 2)"); + } - PackedStringArray plurals; - plurals.push_back("Il y a %d pomme"); - plurals.push_back("Il y a %d pommes"); - translation->add_plural_message("There are %d apples", plurals); - ERR_PRINT_OFF; - // This is invalid, as the number passed to `get_plural_message()` may not be negative. - CHECK(vformat(translation->get_plural_message("There are %d apples", "", -1), -1) == ""); - ERR_PRINT_ON; - CHECK(vformat(translation->get_plural_message("There are %d apples", "", 0), 0) == "Il y a 0 pomme"); - CHECK(vformat(translation->get_plural_message("There are %d apples", "", 1), 1) == "Il y a 1 pomme"); - CHECK(vformat(translation->get_plural_message("There are %d apples", "", 2), 2) == "Il y a 2 pommes"); + { + Ref translation = memnew(TranslationPO); + translation->set_locale("invalid"); + CHECK(translation->get_plural_forms() == 2); + CHECK(translation->get_plural_rule() == "(n != 1)"); + } + + { + Ref translation = memnew(TranslationPO); + translation->set_plural_rules_override("Plural-Forms: nplurals=2; plural=(n >= 2);"); + CHECK(translation->get_plural_forms() == 2); + CHECK(translation->get_plural_rule() == "(n >= 2)"); + + PackedStringArray plurals; + plurals.push_back("Il y a %d pomme"); + plurals.push_back("Il y a %d pommes"); + translation->add_plural_message("There are %d apples", plurals); + ERR_PRINT_OFF; + // This is invalid, as the number passed to `get_plural_message()` may not be negative. + CHECK(vformat(translation->get_plural_message("There are %d apples", "", -1), -1) == ""); + ERR_PRINT_ON; + CHECK(vformat(translation->get_plural_message("There are %d apples", "", 0), 0) == "Il y a 0 pomme"); + CHECK(vformat(translation->get_plural_message("There are %d apples", "", 1), 1) == "Il y a 1 pomme"); + CHECK(vformat(translation->get_plural_message("There are %d apples", "", 2), 2) == "Il y a 2 pommes"); + } } TEST_CASE("[TranslationPO] Plural rules parsing") { From 4e80190a46b06f5d5fc6d0d11abf573cdad2e5f2 Mon Sep 17 00:00:00 2001 From: Haoyu Qiu Date: Tue, 22 Jul 2025 15:40:46 +0800 Subject: [PATCH 3/3] Move context and plural support to Translation - `TranslationPO` is now an empty class. It exists for compatibility. - `OptimizedTranslation` stays the same, no context or plural support. --- core/io/translation_loader_po.cpp | 7 +- core/io/translation_loader_po.h | 1 - core/string/optimized_translation.cpp | 20 ++- core/string/translation.cpp | 168 ++++++++++++++++------ core/string/translation.h | 22 ++- core/string/translation_po.cpp | 197 +------------------------- core/string/translation_po.h | 33 ----- doc/classes/Translation.xml | 20 ++- tests/core/string/test_translation.h | 42 +++--- 9 files changed, 204 insertions(+), 306 deletions(-) diff --git a/core/io/translation_loader_po.cpp b/core/io/translation_loader_po.cpp index fcab40e653f..0f3cf852a49 100644 --- a/core/io/translation_loader_po.cpp +++ b/core/io/translation_loader_po.cpp @@ -31,7 +31,7 @@ #include "translation_loader_po.h" #include "core/io/file_access.h" -#include "core/string/translation_po.h" +#include "core/string/translation.h" Ref TranslationLoaderPO::load_translation(Ref f, Error *r_error) { if (r_error) { @@ -39,7 +39,8 @@ Ref TranslationLoaderPO::load_translation(Ref f, Error *r_ } const String path = f->get_path(); - Ref translation = Ref(memnew(TranslationPO)); + Ref translation; + translation.instantiate(); String config; uint32_t magic = f->get_32(); @@ -229,7 +230,7 @@ Ref TranslationLoaderPO::load_translation(Ref f, Error *r_ if (p_start != -1) { int p_end = config.find_char('\n', p_start); translation->set_plural_rules_override(config.substr(p_start, p_end - p_start)); - plural_forms = translation->get_plural_forms(); + plural_forms = translation->get_nplurals(); } } diff --git a/core/io/translation_loader_po.h b/core/io/translation_loader_po.h index 99eafdf28fc..baba9ace7ec 100644 --- a/core/io/translation_loader_po.h +++ b/core/io/translation_loader_po.h @@ -32,7 +32,6 @@ #include "core/io/file_access.h" #include "core/io/resource_loader.h" -#include "core/string/translation.h" class TranslationLoaderPO : public ResourceFormatLoader { GDSOFTCLASS(TranslationLoaderPO, ResourceFormatLoader); diff --git a/core/string/optimized_translation.cpp b/core/string/optimized_translation.cpp index d35b91436f8..0067997c393 100644 --- a/core/string/optimized_translation.cpp +++ b/core/string/optimized_translation.cpp @@ -44,11 +44,27 @@ struct CompressedString { void OptimizedTranslation::generate(const Ref &p_from) { // This method compresses a Translation instance. - // Right now, it doesn't handle context or plurals, so Translation subclasses using plurals or context (i.e TranslationPO) shouldn't be compressed. + // Right now, it doesn't handle context or plurals. #ifdef TOOLS_ENABLED ERR_FAIL_COND(p_from.is_null()); + List keys; - p_from->get_message_list(&keys); + { + List raw_keys; + p_from->get_message_list(&raw_keys); + + for (const StringName &key : raw_keys) { + const String key_str = key.operator String(); + int p = key_str.find_char(0x04); + if (p == -1) { + keys.push_back(key); + } else { + const String &msgctxt = key_str.substr(0, p); + const String &msgid = key_str.substr(p + 1); + WARN_PRINT(vformat("OptimizedTranslation does not support context, ignoring message '%s' with context '%s'.", msgid, msgctxt)); + } + } + } int size = Math::larger_prime(keys.size()); diff --git a/core/string/translation.cpp b/core/string/translation.cpp index 7fcfc13229b..d898e8b75d1 100644 --- a/core/string/translation.cpp +++ b/core/string/translation.cpp @@ -34,42 +34,102 @@ #include "core/string/plural_rules.h" #include "core/string/translation_server.h" +void _check_for_incompatibility(const String &p_msgctxt, const String &p_msgid) { + // Gettext PO and MO files use an empty untranslated string without context + // to store metadata. + if (p_msgctxt.is_empty() && p_msgid.is_empty()) { + WARN_PRINT("Both context and the untranslated string are empty. This may cause issues with the translation system and external tools."); + } + + // The EOT character (0x04) is used as a separator between context and + // untranslated string in the MO file format. This convention is also used + // by `get_message_list()`. + // + // It's unusual to have this character in the context or untranslated + // string. But it doesn't do any harm as long as you are aware of this when + // using the relevant APIs and tools. + if (p_msgctxt.contains_char(0x04)) { + WARN_PRINT(vformat("Found EOT character (0x04) within context '%s'. This may cause issues with the translation system and external tools.", p_msgctxt)); + } + if (p_msgid.contains_char(0x04)) { + WARN_PRINT(vformat("Found EOT character (0x04) within untranslated string '%s'. This may cause issues with the translation system and external tools.", p_msgid)); + } +} + Dictionary Translation::_get_messages() const { Dictionary d; - for (const KeyValue &E : translation_map) { - d[E.key] = E.value; + for (const KeyValue> &E : translation_map) { + const Array &storage_key = { E.key.msgctxt, E.key.msgid }; + + Array storage_value; + storage_value.resize(E.value.size()); + for (int i = 0; i < E.value.size(); i++) { + storage_value[i] = E.value[i]; + } + d[storage_key] = storage_value; } return d; } -Vector Translation::_get_message_list() const { - Vector msgs; - msgs.resize(translation_map.size()); - int idx = 0; - for (const KeyValue &E : translation_map) { - msgs.set(idx, E.key); - idx += 1; - } +void Translation::_set_messages(const Dictionary &p_messages) { + translation_map.clear(); - return msgs; + for (const KeyValue &kv : p_messages) { + switch (kv.key.get_type()) { + // Old version, no context or plural support. + case Variant::STRING_NAME: { + const MessageKey msg_key = { StringName(), kv.key }; + _check_for_incompatibility(msg_key.msgctxt, msg_key.msgid); + translation_map[msg_key] = { kv.value }; + } break; + + // Current version. + case Variant::ARRAY: { + const Array &storage_key = kv.key; + const MessageKey msg_key = { storage_key[0], storage_key[1] }; + + const Array &storage_value = kv.value; + ERR_CONTINUE_MSG(storage_value.is_empty(), vformat("No translated strings for untranslated string '%s' with context '%s'.", msg_key.msgid, msg_key.msgctxt)); + + Vector msgstrs; + msgstrs.resize(storage_value.size()); + for (int i = 0; i < storage_value.size(); i++) { + msgstrs.write[i] = storage_value[i]; + } + + _check_for_incompatibility(msg_key.msgctxt, msg_key.msgid); + translation_map[msg_key] = msgstrs; + } break; + + default: { + WARN_PRINT(vformat("Invalid key type in messages dictionary: %s.", Variant::get_type_name(kv.key.get_type()))); + continue; + } + } + } +} + +Vector Translation::_get_message_list() const { + List msgstrs; + get_message_list(&msgstrs); + + Vector keys; + keys.resize(msgstrs.size()); + int idx = 0; + for (const StringName &msgstr : msgstrs) { + keys.write[idx++] = msgstr; + } + return keys; } Vector Translation::get_translated_message_list() const { - Vector msgs; - msgs.resize(translation_map.size()); - int idx = 0; - for (const KeyValue &E : translation_map) { - msgs.set(idx, E.value); - idx += 1; - } - - return msgs; -} - -void Translation::_set_messages(const Dictionary &p_messages) { - for (const KeyValue &kv : p_messages) { - translation_map[kv.key] = kv.value; + Vector msgstrs; + for (const KeyValue> &E : translation_map) { + for (const StringName &msgstr : E.value) { + msgstrs.push_back(msgstr); + } } + return msgstrs; } void Translation::set_locale(const String &p_locale) { @@ -82,13 +142,21 @@ void Translation::set_locale(const String &p_locale) { } void Translation::add_message(const StringName &p_src_text, const StringName &p_xlated_text, const StringName &p_context) { - translation_map[p_src_text] = p_xlated_text; + _check_for_incompatibility(p_context, p_src_text); + translation_map[{ p_context, p_src_text }] = { p_xlated_text }; } void Translation::add_plural_message(const StringName &p_src_text, const Vector &p_plural_xlated_texts, const StringName &p_context) { - WARN_PRINT("Translation class doesn't handle plural messages. Calling add_plural_message() on a Translation instance is probably a mistake. \nUse a derived Translation class that handles plurals, such as TranslationPO class"); ERR_FAIL_COND_MSG(p_plural_xlated_texts.is_empty(), "Parameter vector p_plural_xlated_texts passed in is empty."); - translation_map[p_src_text] = p_plural_xlated_texts[0]; + + Vector msgstrs; + msgstrs.resize(p_plural_xlated_texts.size()); + for (int i = 0; i < p_plural_xlated_texts.size(); i++) { + msgstrs.write[i] = p_plural_xlated_texts[i]; + } + + _check_for_incompatibility(p_context, p_src_text); + translation_map[{ p_context, p_src_text }] = msgstrs; } StringName Translation::get_message(const StringName &p_src_text, const StringName &p_context) const { @@ -97,16 +165,13 @@ StringName Translation::get_message(const StringName &p_src_text, const StringNa return ret; } - if (p_context != StringName()) { - WARN_PRINT("Translation class doesn't handle context. Using context in get_message() on a Translation instance is probably a mistake. \nUse a derived Translation class that handles context, such as TranslationPO class"); - } - - HashMap::ConstIterator E = translation_map.find(p_src_text); - if (!E) { + const Vector *msgstrs = translation_map.getptr({ p_context, p_src_text }); + if (msgstrs == nullptr) { return StringName(); } - return E->value; + DEV_ASSERT(!msgstrs->is_empty()); // Should be prevented when adding messages. + return msgstrs->get(0); } StringName Translation::get_plural_message(const StringName &p_src_text, const StringName &p_plural_text, int p_n, const StringName &p_context) const { @@ -115,21 +180,30 @@ StringName Translation::get_plural_message(const StringName &p_src_text, const S return ret; } - WARN_PRINT("Translation class doesn't handle plural messages. Calling get_plural_message() on a Translation instance is probably a mistake. \nUse a derived Translation class that handles plurals, such as TranslationPO class"); - return get_message(p_src_text); + ERR_FAIL_COND_V_MSG(p_n < 0, StringName(), "N passed into translation to get a plural message should not be negative. For negative numbers, use singular translation please. Search \"gettext PO Plural Forms\" online for details on translating negative numbers."); + + const Vector *msgstrs = translation_map.getptr({ p_context, p_src_text }); + if (msgstrs == nullptr) { + return StringName(); + } + + const int index = _get_plural_rules()->evaluate(p_n); + ERR_FAIL_INDEX_V_MSG(index, msgstrs->size(), StringName(), "Plural index returned or number of plural translations is not valid."); + return msgstrs->get(index); } void Translation::erase_message(const StringName &p_src_text, const StringName &p_context) { - if (p_context != StringName()) { - WARN_PRINT("Translation class doesn't handle context. Using context in erase_message() on a Translation instance is probably a mistake. \nUse a derived Translation class that handles context, such as TranslationPO class"); - } - - translation_map.erase(p_src_text); + translation_map.erase({ p_context, p_src_text }); } void Translation::get_message_list(List *r_messages) const { - for (const KeyValue &E : translation_map) { - r_messages->push_back(E.key); + for (const KeyValue> &E : translation_map) { + if (E.key.msgctxt.is_empty()) { + r_messages->push_back(E.key.msgid); + } else { + // Separated by the EOT character. Compatible with the MO file format. + r_messages->push_back(vformat("%s\x04%s", E.key.msgctxt, E.key.msgid)); + } } } @@ -175,6 +249,10 @@ String Translation::get_plural_rules_override() const { return plural_rules_override; } +int Translation::get_nplurals() const { + return _get_plural_rules()->get_nplurals(); +} + void Translation::_bind_methods() { ClassDB::bind_method(D_METHOD("set_locale", "locale"), &Translation::set_locale); ClassDB::bind_method(D_METHOD("get_locale"), &Translation::get_locale); @@ -195,7 +273,7 @@ void Translation::_bind_methods() { GDVIRTUAL_BIND(_get_message, "src_message", "context"); ADD_PROPERTY(PropertyInfo(Variant::DICTIONARY, "messages", PROPERTY_HINT_NONE, "", PROPERTY_USAGE_NO_EDITOR | PROPERTY_USAGE_INTERNAL), "_set_messages", "_get_messages"); - ADD_PROPERTY(PropertyInfo(Variant::STRING, "locale"), "set_locale", "get_locale"); + ADD_PROPERTY(PropertyInfo(Variant::STRING, "locale", PROPERTY_HINT_LOCALE_ID), "set_locale", "get_locale"); ADD_PROPERTY(PropertyInfo(Variant::STRING, "plural_rules_override"), "set_plural_rules_override", "get_plural_rules_override"); } diff --git a/core/string/translation.h b/core/string/translation.h index 3136d8420f1..befb538079d 100644 --- a/core/string/translation.h +++ b/core/string/translation.h @@ -41,12 +41,29 @@ class Translation : public Resource { RES_BASE_EXTENSION("translation"); String locale = "en"; - HashMap translation_map; + + struct MessageKey { + StringName msgctxt; + StringName msgid; + + // Required to use this struct as a key in HashMap. + static uint32_t hash(const MessageKey &p_key) { + uint32_t h = hash_murmur3_one_32(HashMapHasherDefault::hash(p_key.msgctxt)); + return hash_fmix32(hash_murmur3_one_32(HashMapHasherDefault::hash(p_key.msgid), h)); + } + bool operator==(const MessageKey &p_key) const { + return msgctxt == p_key.msgctxt && msgid == p_key.msgid; + } + }; + + HashMap, MessageKey> translation_map; mutable PluralRules *plural_rules_cache = nullptr; String plural_rules_override; virtual Vector _get_message_list() const; + + // For data storage. virtual Dictionary _get_messages() const; virtual void _set_messages(const Dictionary &p_messages); @@ -74,5 +91,8 @@ public: void set_plural_rules_override(const String &p_rules); String get_plural_rules_override() const; + // This method is not exposed to scripting intentionally. It is only used by TranslationLoaderPO and tests. + int get_nplurals() const; + ~Translation(); }; diff --git a/core/string/translation_po.cpp b/core/string/translation_po.cpp index 3eaecbf5344..6c4a84f4b4c 100644 --- a/core/string/translation_po.cpp +++ b/core/string/translation_po.cpp @@ -30,198 +30,5 @@ #include "translation_po.h" -#include "core/string/plural_rules.h" - -#ifdef DEBUG_TRANSLATION_PO -#include "core/io/file_access.h" - -void TranslationPO::print_translation_map() { - Error err; - Ref file = FileAccess::open("translation_map_print_test.txt", FileAccess::WRITE, &err); - if (err != OK) { - ERR_PRINT("Failed to open translation_map_print_test.txt"); - return; - } - - file->store_line("NPlural : " + String::num_int64(get_plural_forms())); - file->store_line("Plural rule : " + get_plural_rule()); - file->store_line(""); - - List context_l; - translation_map.get_key_list(&context_l); - for (const StringName &ctx : context_l) { - file->store_line(" ===== Context: " + String::utf8(String(ctx).utf8()) + " ===== "); - const HashMap> &inner_map = translation_map[ctx]; - - List id_l; - inner_map.get_key_list(&id_l); - for (const StringName &id : id_l) { - file->store_line("msgid: " + String::utf8(String(id).utf8())); - for (int i = 0; i < inner_map[id].size(); i++) { - file->store_line("msgstr[" + String::num_int64(i) + "]: " + String::utf8(String(inner_map[id][i]).utf8())); - } - file->store_line(""); - } - } -} -#endif - -Dictionary TranslationPO::_get_messages() const { - // Return translation_map as a Dictionary. - - Dictionary d; - - for (const KeyValue>> &E : translation_map) { - Dictionary d2; - - for (const KeyValue> &E2 : E.value) { - d2[E2.key] = E2.value; - } - - d[E.key] = d2; - } - - return d; -} - -void TranslationPO::_set_messages(const Dictionary &p_messages) { - // Construct translation_map from a Dictionary. - - for (const KeyValue &kv : p_messages) { - const Dictionary &id_str_map = kv.value; - - HashMap> temp_map; - for (const KeyValue &kv_id : id_str_map) { - StringName id = kv_id.key; - temp_map[id] = kv_id.value; - } - - translation_map[kv.key] = temp_map; - } -} - -Vector TranslationPO::get_translated_message_list() const { - Vector msgs; - for (const KeyValue>> &E : translation_map) { - if (E.key != StringName()) { - continue; - } - - for (const KeyValue> &E2 : E.value) { - for (const StringName &E3 : E2.value) { - msgs.push_back(E3); - } - } - } - - return msgs; -} - -Vector TranslationPO::_get_message_list() const { - // Return all keys in translation_map. - - List msgs; - get_message_list(&msgs); - - Vector v; - for (const StringName &E : msgs) { - v.push_back(E); - } - - return v; -} - -void TranslationPO::add_message(const StringName &p_src_text, const StringName &p_xlated_text, const StringName &p_context) { - HashMap> &map_id_str = translation_map[p_context]; - - if (map_id_str.has(p_src_text)) { - WARN_PRINT(vformat("Double translations for \"%s\" under the same context \"%s\" for locale \"%s\".\nThere should only be one unique translation for a given string under the same context.", String(p_src_text), String(p_context), get_locale())); - map_id_str[p_src_text].set(0, p_xlated_text); - } else { - map_id_str[p_src_text].push_back(p_xlated_text); - } -} - -void TranslationPO::add_plural_message(const StringName &p_src_text, const Vector &p_plural_xlated_texts, const StringName &p_context) { - ERR_FAIL_COND_MSG(p_plural_xlated_texts.size() != _get_plural_rules()->get_nplurals(), vformat("Trying to add plural texts that don't match the required number of plural forms for locale \"%s\".", get_locale())); - - HashMap> &map_id_str = translation_map[p_context]; - - if (map_id_str.has(p_src_text)) { - WARN_PRINT(vformat("Double translations for \"%s\" under the same context \"%s\" for locale %s.\nThere should only be one unique translation for a given string under the same context.", p_src_text, p_context, get_locale())); - map_id_str[p_src_text].clear(); - } - - for (int i = 0; i < p_plural_xlated_texts.size(); i++) { - map_id_str[p_src_text].push_back(p_plural_xlated_texts[i]); - } -} - -int TranslationPO::get_plural_forms() const { - return _get_plural_rules()->get_nplurals(); -} - -String TranslationPO::get_plural_rule() const { - return _get_plural_rules()->get_plural(); -} - -StringName TranslationPO::get_message(const StringName &p_src_text, const StringName &p_context) const { - if (!translation_map.has(p_context) || !translation_map[p_context].has(p_src_text)) { - return StringName(); - } - ERR_FAIL_COND_V_MSG(translation_map[p_context][p_src_text].is_empty(), StringName(), vformat("Source text \"%s\" is registered but doesn't have a translation. Please report this bug.", String(p_src_text))); - - return translation_map[p_context][p_src_text][0]; -} - -StringName TranslationPO::get_plural_message(const StringName &p_src_text, const StringName &p_plural_text, int p_n, const StringName &p_context) const { - ERR_FAIL_COND_V_MSG(p_n < 0, StringName(), "N passed into translation to get a plural message should not be negative. For negative numbers, use singular translation please. Search \"gettext PO Plural Forms\" online for the documentation on translating negative numbers."); - - if (!translation_map.has(p_context) || !translation_map[p_context].has(p_src_text)) { - return StringName(); - } - ERR_FAIL_COND_V_MSG(translation_map[p_context][p_src_text].is_empty(), StringName(), vformat("Source text \"%s\" is registered but doesn't have a translation. Please report this bug.", String(p_src_text))); - - int plural_index = _get_plural_rules()->evaluate(p_n); - ERR_FAIL_COND_V_MSG(plural_index < 0 || translation_map[p_context][p_src_text].size() < plural_index + 1, StringName(), "Plural index returned or number of plural translations is not valid. Please report this bug."); - - return translation_map[p_context][p_src_text][plural_index]; -} - -void TranslationPO::erase_message(const StringName &p_src_text, const StringName &p_context) { - if (!translation_map.has(p_context)) { - return; - } - - translation_map[p_context].erase(p_src_text); -} - -void TranslationPO::get_message_list(List *r_messages) const { - // OptimizedTranslation uses this function to get the list of msgid. - // Return all the keys of translation_map under "" context. - - for (const KeyValue>> &E : translation_map) { - if (E.key != StringName()) { - continue; - } - - for (const KeyValue> &E2 : E.value) { - r_messages->push_back(E2.key); - } - } -} - -int TranslationPO::get_message_count() const { - int count = 0; - - for (const KeyValue>> &E : translation_map) { - count += E.value.size(); - } - - return count; -} - -void TranslationPO::_bind_methods() { - ClassDB::bind_method(D_METHOD("get_plural_forms"), &TranslationPO::get_plural_forms); - ClassDB::bind_method(D_METHOD("get_plural_rule"), &TranslationPO::get_plural_rule); -} +// This file is intentionally left empty. +// It makes sure that `TranslationPO` exists, for compatibility. diff --git a/core/string/translation_po.h b/core/string/translation_po.h index dd0bc3c602b..7548e227314 100644 --- a/core/string/translation_po.h +++ b/core/string/translation_po.h @@ -30,41 +30,8 @@ #pragma once -//#define DEBUG_TRANSLATION_PO - #include "core/string/translation.h" class TranslationPO : public Translation { GDCLASS(TranslationPO, Translation); - - // TLDR: Maps context to a list of source strings and translated strings. In PO terms, maps msgctxt to a list of msgid and msgstr. - // The first key corresponds to context, and the second key (of the contained HashMap) corresponds to source string. - // The value Vector in the second map stores the translated strings. Index 0, 1, 2 matches msgstr[0], msgstr[1], msgstr[2]... in the case of plurals. - // Otherwise index 0 matches to msgstr in a singular translation. - // Strings without context have "" as first key. - HashMap>> translation_map; - - Vector _get_message_list() const override; - Dictionary _get_messages() const override; - void _set_messages(const Dictionary &p_messages) override; - -protected: - static void _bind_methods(); - -public: - Vector get_translated_message_list() const override; - void get_message_list(List *r_messages) const override; - int get_message_count() const override; - void add_message(const StringName &p_src_text, const StringName &p_xlated_text, const StringName &p_context = "") override; - void add_plural_message(const StringName &p_src_text, const Vector &p_plural_xlated_texts, const StringName &p_context = "") override; - StringName get_message(const StringName &p_src_text, const StringName &p_context = "") const override; - StringName get_plural_message(const StringName &p_src_text, const StringName &p_plural_text, int p_n, const StringName &p_context = "") const override; - void erase_message(const StringName &p_src_text, const StringName &p_context = "") override; - - int get_plural_forms() const; - String get_plural_rule() const; - -#ifdef DEBUG_TRANSLATION_PO - void print_translation_map(); -#endif }; diff --git a/doc/classes/Translation.xml b/doc/classes/Translation.xml index c57c29e68bb..af1839602cf 100644 --- a/doc/classes/Translation.xml +++ b/doc/classes/Translation.xml @@ -4,7 +4,8 @@ A language translation that maps a collection of strings to their individual translations. - [Translation]s are resources that can be loaded and unloaded on demand. They map a collection of strings to their individual translations, and they also provide convenience methods for pluralization. + [Translation] maps a collection of strings to their individual translations, and also provides convenience methods for pluralization. + A [Translation] consists of messages. A message is identified by its context and untranslated string. Unlike [url=https://www.gnu.org/software/gettext/]gettext[/url], using an empty context string in Godot means not using any context. $DOCS_URL/tutorials/i18n/internationalizing_games.html @@ -48,7 +49,6 @@ Adds a message involving plural translation if nonexistent, followed by its translation. An additional context could be used to specify the translation context or differentiate polysemic words. - [b]Note:[/b] Plurals are only supported in [url=$DOCS_URL/tutorials/i18n/localization_using_gettext.html]gettext-based translations (PO)[/url], not CSV. @@ -76,7 +76,19 @@ - Returns all the messages (keys). + Returns the keys of all messages, that is, the context and untranslated strings of each message. + [b]Note:[/b] If a message does not use a context, the corresponding element is the untranslated string. Otherwise, the corresponding element is the context and untranslated string separated by the EOT character ([code]U+0004[/code]). This is done for compatibility purposes. + [codeblock] + for key in translation.get_message_list(): + var p = key.find("\u0004") + if p == -1: + var untranslated = key + print("Message %s" % untranslated) + else: + var context = key.substr(0, p) + var untranslated = key.substr(p + 1) + print("Message %s with context %s" % [untranslated, context]) + [/codeblock] @@ -94,7 +106,7 @@ - Returns all the messages (translated text). + Returns all the translated strings. diff --git a/tests/core/string/test_translation.h b/tests/core/string/test_translation.h index 35740e547c3..cce25db5b0f 100644 --- a/tests/core/string/test_translation.h +++ b/tests/core/string/test_translation.h @@ -33,7 +33,6 @@ #include "core/string/optimized_translation.h" #include "core/string/plural_rules.h" #include "core/string/translation.h" -#include "core/string/translation_po.h" #include "core/string/translation_server.h" #ifdef TOOLS_ENABLED @@ -46,7 +45,8 @@ namespace TestTranslation { TEST_CASE("[Translation] Messages") { - Ref translation = memnew(Translation); + Ref translation; + translation.instantiate(); translation->set_locale("fr"); translation->add_message("Hello", "Bonjour"); CHECK(translation->get_message("Hello") == "Bonjour"); @@ -71,8 +71,9 @@ TEST_CASE("[Translation] Messages") { CHECK(messages.find("Hello3")); } -TEST_CASE("[TranslationPO] Messages with context") { - Ref translation = memnew(TranslationPO); +TEST_CASE("[Translation] Messages with context") { + Ref translation; + translation.instantiate(); translation->set_locale("fr"); translation->add_message("Hello", "Bonjour"); translation->add_message("Hello", "Salut", "friendly"); @@ -90,11 +91,8 @@ TEST_CASE("[TranslationPO] Messages with context") { List messages; translation->get_message_list(&messages); - // `get_message_count()` takes all contexts into account. CHECK(translation->get_message_count() == 1); - // Only the default context is taken into account. - // Since "Hello" is now only present in a non-default context, it is not counted in the list of messages. - CHECK(messages.size() == 0); + CHECK(messages.size() == 1); translation->add_message("Hello2", "Bonjour2"); translation->add_message("Hello2", "Salut2", "friendly"); @@ -102,35 +100,35 @@ TEST_CASE("[TranslationPO] Messages with context") { messages.clear(); translation->get_message_list(&messages); - // `get_message_count()` takes all contexts into account. CHECK(translation->get_message_count() == 4); - // Only the default context is taken into account. - CHECK(messages.size() == 2); + CHECK(messages.size() == 4); // Messages are stored in a Map, don't assume ordering. CHECK(messages.find("Hello2")); CHECK(messages.find("Hello3")); + // Context and untranslated string are separated by EOT. + CHECK(messages.find("friendly\x04Hello2")); } -TEST_CASE("[TranslationPO] Plural messages") { +TEST_CASE("[Translation] Plural messages") { { - Ref translation = memnew(TranslationPO); + Ref translation; + translation.instantiate(); translation->set_locale("fr"); - CHECK(translation->get_plural_forms() == 3); - CHECK(translation->get_plural_rule() == "(n == 0 || n == 1) ? 0 : n != 0 && n % 1000000 == 0 ? 1 : 2)"); + CHECK(translation->get_nplurals() == 3); } { - Ref translation = memnew(TranslationPO); + Ref translation; + translation.instantiate(); translation->set_locale("invalid"); - CHECK(translation->get_plural_forms() == 2); - CHECK(translation->get_plural_rule() == "(n != 1)"); + CHECK(translation->get_nplurals() == 2); } { - Ref translation = memnew(TranslationPO); + Ref translation; + translation.instantiate(); translation->set_plural_rules_override("Plural-Forms: nplurals=2; plural=(n >= 2);"); - CHECK(translation->get_plural_forms() == 2); - CHECK(translation->get_plural_rule() == "(n >= 2)"); + CHECK(translation->get_nplurals() == 2); PackedStringArray plurals; plurals.push_back("Il y a %d pomme"); @@ -146,7 +144,7 @@ TEST_CASE("[TranslationPO] Plural messages") { } } -TEST_CASE("[TranslationPO] Plural rules parsing") { +TEST_CASE("[Translation] Plural rules parsing") { ERR_PRINT_OFF; { CHECK(PluralRules::parse("") == nullptr);