diff --git a/Cargo.lock b/Cargo.lock index e0e313d9..1b9bf783 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -791,6 +791,7 @@ name = "craft-i18n" version = "0.1.0" dependencies = [ "intl_pluralrules", + "log", "sys-locale", "unic-langid", ] diff --git a/crates/i18n/Cargo.toml b/crates/i18n/Cargo.toml index 45e21775..9a9b97e9 100644 --- a/crates/i18n/Cargo.toml +++ b/crates/i18n/Cargo.toml @@ -10,6 +10,7 @@ repository.workspace = true [dependencies] intl_pluralrules = "=7.0.2" unic-langid = "0.9" +log = "0.4" sys-locale = "0.3" [target.'cfg(target_arch = "wasm32")'.dependencies] diff --git a/crates/i18n/README.md b/crates/i18n/README.md index df337971..c2ce066b 100644 --- a/crates/i18n/README.md +++ b/crates/i18n/README.md @@ -20,8 +20,8 @@ include!(concat!(env!("OUT_DIR"), "/craft_i18n.rs")); // pub static CRAFT_I18N: pub static I18N: LazyLock = LazyLock::new(|| { let offered: Vec<&str> = LANGUAGES.iter().map(|l| l.code).collect(); // the app's own list - let (i18n, problems) = craft_i18n::I18n::new(CRAFT_I18N, &offered); - problems.iter().for_each(|p| log::warn!("translation catalog: {p}; that string shows in English")); + let (i18n, problems) = craft_i18n::I18n::new(CRAFT_I18N, &offered); // parses nothing yet + problems.iter().for_each(|p| log::warn!("translation catalog: {p}")); i18n }); @@ -51,15 +51,17 @@ The app keeps its `tl!` macro, its language setting (`"auto"` → `I18N.system_l placeholders, filled in one pass. - **Bad rows** (malformed, an unknown `@` context, a wrong plural form count, a row defined twice, of which the first is kept) and bad locales (`en`, `pseudo`, a code embedded twice in any case) - are skipped and returned by `I18n::new` for the app to log; craft-i18n's CI rejects them before - they ship. + are skipped; craft-i18n's CI rejects them before they ship. `I18n::new` parses nothing: it + returns the bad locales, and each locale's rows are parsed (and its bad rows logged with + `log::warn!`) on the first lookup that reaches it, including as a fallback. `I18n::check_all()` + parses every locale and returns all problems (`/: line N: …`), for a test. - **Plurals:** forms are listed in CLDR order (zero, one, two, few, many, other) for the categories whole numbers 0 to 1000 reach, from `intl_pluralrules`; a count in any other category takes the last form. CLDR's `pt` is Brazilian, so `pt` uses `pt-PT`'s rules (0 is plural) and `pt-br` `pt`'s. `intl_pluralrules` is pinned to the version craft-i18n's checker uses, and `plural_form_counts` holds the same counts. -- **Names:** a language's name comes from its shared catalog's `@id language.name` row, else its - code; English is "English". +- **Names:** a language's name comes from the app catalog's `@id language.name` row, else the + shared catalog's, else its code; English is "English". - **Right to left:** `rtl` is true for `ar`, `ckb`, `dv`, `fa`, `he`, `ps`, `sd`, `ug`, `ur` and `yi`; putting text into display order is the app's job. - **System locale:** `system_lang` matches the system's list once and caches it: on desktop Unix diff --git a/crates/i18n/src/catalog.rs b/crates/i18n/src/catalog.rs index 50d731b2..0cb51e36 100644 --- a/crates/i18n/src/catalog.rs +++ b/crates/i18n/src/catalog.rs @@ -37,7 +37,7 @@ pub(crate) fn parse(text: &str, forms: usize) -> (Vec, Vec) { (rows, problems) } -fn parse_row(text: &str, forms: usize) -> Result { +pub(crate) fn parse_row(text: &str, forms: usize) -> Result { let columns: Vec<&str> = text.split('\t').collect(); let [context, source, translation] = columns.as_slice() else { return Err(format!("{} columns; expected 3", columns.len())); diff --git a/crates/i18n/src/lib.rs b/crates/i18n/src/lib.rs index 7fa7a4ef..c040d074 100644 --- a/crates/i18n/src/lib.rs +++ b/crates/i18n/src/lib.rs @@ -31,7 +31,14 @@ impl Lang { /// One locale's translations: context → source text → translation, app rows over shared ones. struct Locale { code: String, - rows: HashMap>, + /// The shared and the app's catalog text, parsed on first use. + shared: &'static str, + app: &'static str, + /// Plural forms `@plural` rows must list, for [`catalog::parse`]. + forms: usize, + parsed: OnceLock, + /// [`I18n::name`], read from the text on first use. + name: OnceLock>, /// Itself, then its parent without the region if that is embedded too (`pt-br`, `pt`; /// `zh-hant-hk`, `zh-hant`). Empty for English and the pseudo locale. chain: Vec, @@ -42,7 +49,49 @@ struct Locale { plural: Option<(PluralRules, Vec)>, } -/// The embedded catalogs, ready for lookups. Build it once, at start-up. +/// A locale's rows, app over shared, and what was wrong with the catalogs it came from. +struct Parsed { + rows: HashMap>, + problems: Vec, +} + +impl Locale { + /// The rows; the catalogs are parsed on the first call and any problems logged then. + fn parsed(&self) -> &Parsed { + self.parsed.get_or_init(|| { + let mut problems = Vec::new(); + let mut rows: HashMap> = HashMap::new(); + for (name, text) in [("common", self.shared), ("app", self.app)] { + let (entries, errors) = catalog::parse(text, self.forms); + problems.extend(errors.into_iter().map(|e| format!("{}/{name}: {e}", self.code))); + for e in entries { + rows.entry(e.context).or_default().insert(e.source, e.translation); + } + } + problems.iter().for_each(|p| log::warn!("translation catalog: {p}; row skipped")); + Parsed { rows, problems } + }) + } + + /// The `@id language.name` row of the app's catalog, else the shared one, found by scanning the + /// text so the locale needn't be parsed. + fn language_name(&self) -> Option<&str> { + self.name + .get_or_init(|| { + [self.app, self.shared].into_iter().find_map(|text| { + let text = text.strip_prefix('\u{feff}').unwrap_or(text); + text.split('\n') + .filter(|line| line.starts_with("@id\tlanguage.name\t")) + .find_map(|line| catalog::parse_row(line.strip_suffix('\r').unwrap_or(line), self.forms).ok()) + .map(|row| row.translation) + }) + }) + .as_deref() + } +} + +/// The embedded catalogs, ready for lookups. Build it once, at start-up. Each locale is parsed on +/// its first lookup, which is also when its bad rows are logged. pub struct I18n { locales: Vec, /// Marked strings for the pseudo locale by source text, leaked so lookups can hand out `&str`. @@ -55,10 +104,11 @@ pub struct I18n { } impl I18n { - /// Read `embedded`, offering the locales in `offered` (any case; the app's own language list), - /// so an app never offers a language its interface isn't ready for (right to left, say). Rows - /// and locales that can't be used are skipped and come back as problems - /// (`/: line N: …`) for the app to log. + /// Index `embedded` without parsing it, offering the locales in `offered` (any case; the app's + /// own language list), so an app never offers a language its interface isn't ready for (right + /// to left, say). Locales that can't be used are skipped and come back as problems for the app + /// to log. A locale's bad rows are logged (`log::warn!`) when it is first looked up in, and + /// [`I18n::check_all`] lists them all. pub fn new(embedded: &'static Embedded, offered: &[&str]) -> (I18n, Vec) { let mut problems = Vec::new(); let mut locales = vec![bare("en"), bare("pseudo")]; @@ -73,16 +123,8 @@ impl I18n { problems.push(format!("{code}: no plural rules; plurals show in English")); } let forms = plural.as_ref().map_or(1, |(_, c)| c.len()); - let mut rows: HashMap> = HashMap::new(); - for (name, text) in [("common", shared), ("app", app)] { - let (entries, errors) = catalog::parse(text, forms); - problems.extend(errors.into_iter().map(|e| format!("{code}/{name}: {e}"))); - for e in entries { - rows.entry(e.context).or_default().insert(e.source, e.translation); - } - } let offered = offered.iter().any(|o| o.eq_ignore_ascii_case(&code)); - locales.push(Locale { code, rows, chain: Vec::new(), plural, offered }); + locales.push(Locale { code, shared, app, forms, parsed: OnceLock::new(), name: OnceLock::new(), chain: Vec::new(), plural, offered }); } for i in 2..locales.len() { let mut chain = vec![i]; @@ -105,6 +147,12 @@ impl I18n { (I18n { locales, pseudo: Mutex::default(), system: OnceLock::new() }, problems) } + /// Parse every locale and list every problem, `/: line N: …`, for a test or + /// an audit; the same ones [`I18n::new`] leaves to lookups to log. + pub fn check_all(&self) -> Vec { + self.locales.iter().flat_map(|l| l.parsed().problems.iter().cloned()).collect() + } + /// The language with this code (any case): `en`, an offered locale, or in debug builds /// `pseudo`. pub fn from_code(&self, code: &str) -> Option { @@ -167,7 +215,7 @@ impl I18n { .iter() .find_map(|&i| { let locale = self.locales.get(i)?; - let forms = locale.rows.get("@plural")?.get(&key)?; + let forms = locale.parsed().rows.get("@plural")?.get(&key)?; let (rules, categories) = locale.plural.as_ref()?; let category = rules.select(n).ok()?; // A category whole numbers up to 1000 never reach (French `many`) takes the last form. @@ -190,13 +238,14 @@ impl I18n { std::iter::once(Lang::EN).chain(embedded.into_iter().map(Lang)).collect() } - /// The language's name in itself, for a language picker: its shared catalog's - /// `@id language.name` row, or the code when there is none. + /// The language's name in itself, for a language picker: its `@id language.name` row (the + /// app's catalog over the shared one), or the code when there is none. Doesn't parse the + /// locale. pub fn name(&self, lang: Lang) -> &str { if lang == Lang::EN { return "English"; } - self.locale(lang).and_then(|l| l.rows.get("@id")?.get("language.name")).map_or_else(|| self.code(lang), String::as_str) + self.locale(lang).and_then(Locale::language_name).unwrap_or_else(|| self.code(lang)) } /// Whether the language is written right to left. Putting text into display order is the @@ -255,7 +304,7 @@ impl I18n { fn find(&self, lang: Lang, keys: &[(&str, &str)]) -> Option<&str> { let chain = self.locale(lang).map(|l| l.chain.as_slice())?; chain.iter().find_map(|&i| { - let rows = &self.locales.get(i)?.rows; + let rows = &self.locales.get(i)?.parsed().rows; keys.iter().find_map(|(context, source)| rows.get(*context)?.get(*source).map(String::as_str)) }) } @@ -360,7 +409,17 @@ fn is_region(subtag: &str) -> bool { /// A locale with no catalogs (English, the pseudo locale). fn bare(code: &str) -> Locale { - Locale { code: code.to_owned(), rows: HashMap::new(), chain: Vec::new(), plural: None, offered: false } + Locale { + code: code.to_owned(), + shared: "", + app: "", + forms: 1, + parsed: OnceLock::new(), + name: OnceLock::new(), + chain: Vec::new(), + plural: None, + offered: false, + } } /// `code`'s CLDR cardinal rules and the categories whole numbers 0..=1000 reach, in CLDR order @@ -398,6 +457,15 @@ fn rank(c: &PluralCategory) -> u8 { mod tests { use super::*; + impl I18n { + /// The embedded locales parsed so far, by code. + fn parsed(&self) -> Vec<&str> { + let mut codes: Vec<&str> = self.locales.iter().skip(2).filter(|l| l.parsed.get().is_some()).map(|l| l.code.as_str()).collect(); + codes.sort_unstable(); + codes + } + } + static E: &Embedded = &[ ("fr", "\tSave\tEnregistrer\n\tOpen\tOuvrir\nmenu\tOpen\tOuvrir le menu\n@id\tfile.save\tEnregistrer le fichier\n", "\tSave\tSauvegarder\n"), ("pt", "\tSave\tGuardar\n\tUndo\tDesfazer\n", ""), @@ -461,11 +529,9 @@ mod tests { #[test] fn bad_rows_skipped_and_reported() { - let (i, problems) = all(E); - assert_eq!(problems.len(), 2, "{problems:?}"); - assert!(problems[0].starts_with("de/common: line 1:"), "{problems:?}"); - assert!(problems[1].starts_with("de/common: line 2:"), "{problems:?}"); + let i = i18n(); assert_eq!(i.t(lang(&i, "de"), "Undo"), "Rückgängig"); + assert_eq!(i.check_all().len(), 2); } #[test] @@ -703,8 +769,8 @@ mod tests { #[test] fn duplicate_row_keeps_the_first() { - let (i, problems) = all(&[("fr", "\tSave\tEnregistrer\n\tSave\tSauver\n", "")]); - assert_eq!(problems, ["fr/common: line 2: defined twice (same context and source text); the first is kept"]); + let i = all(&[("fr", "\tSave\tEnregistrer\n\tSave\tSauver\n", "")]).0; + assert_eq!(i.check_all(), ["fr/common: line 2: defined twice (same context and source text); the first is kept"]); assert_eq!(i.t(lang(&i, "fr"), "Save"), "Enregistrer"); } @@ -740,6 +806,67 @@ mod tests { assert!(i.languages().into_iter().skip(1).all(|l| i.rtl(l))); } + #[test] + fn check_all_reports_every_bad_row() { + let (i, problems) = all(E); + assert!(i.parsed().is_empty(), "new parses nothing"); + assert!(problems.iter().all(|p| !p.contains("line")), "{problems:?}"); + let found = i.check_all(); + assert_eq!(found.len(), 2, "{found:?}"); + assert!(found[0].starts_with("de/common: line 1:"), "{found:?}"); + assert!(found[1].starts_with("de/common: line 2:"), "{found:?}"); + assert_eq!(i.check_all(), found); + assert_eq!(i.parsed(), ["de", "fr", "pt", "pt-br"]); + } + + #[test] + fn lookup_parses_only_what_it_reaches() { + static LAZY: &Embedded = &[ + ("pt", "\tUndo\tDesfazer\n", ""), + ("pt-br", "\tSave\tSalvar\n", ""), + ("de", "\tUndo\tRückgängig\n", ""), + ("fr", "\tUndo\tAnnuler\n", ""), + ]; + let i = all(LAZY).0; + assert!(i.parsed().is_empty()); + assert_eq!(i.t(lang(&i, "pt-br"), "Save"), "Salvar"); + assert_eq!(i.parsed(), ["pt-br"]); + assert_eq!(i.t(lang(&i, "pt-br"), "Undo"), "Desfazer"); + assert_eq!(i.parsed(), ["pt", "pt-br"]); + assert_eq!(i.t(lang(&i, "pt-br"), "Undo"), "Desfazer"); + assert_eq!(i.parsed(), ["pt", "pt-br"]); + assert_eq!(i.t(Lang::EN, "Undo"), "Undo"); + assert_eq!(i.t(Lang::PSEUDO, "Undo"), "⟦Undo⟧"); + assert_eq!(i.parsed(), ["pt", "pt-br"]); + } + + #[test] + fn name_reads_escapes_without_parsing() { + let i = all(&[("fr", "@id\tlanguage.name\tFran\\\\ais\\tx\n", ""), ("de", "\tA\tB\n@id\tlanguage.name\tDeutsch\n", "")]).0; + assert_eq!(i.name(lang(&i, "fr")), "Fran\\ais\tx"); + assert_eq!(i.name(lang(&i, "de")), "Deutsch"); + assert!(i.parsed().is_empty()); + } + + #[test] + fn app_name_overrides_shared() { + let i = all(&[ + ("fr", "@id\tlanguage.name\tFrançais\n", "@id\tlanguage.name\tFrançais (app)\n"), + ("de", "@id\tlanguage.name\tDeutsch\n", "\tA\tB\n"), + ("es", "", ""), + ]) + .0; + assert_eq!(i.name(lang(&i, "fr")), "Français (app)"); + assert_eq!(i.name(lang(&i, "de")), "Deutsch"); + assert_eq!(i.name(lang(&i, "es")), "es"); + } + + #[test] + fn i18n_is_send_and_sync() { + fn check() {} + check::(); + } + /// Prints what this machine reports, so CI logs show `system_lang` on each platform. #[test] fn system_lang_reads_the_system() {