From 7b797335cb25ae8e7b384c30949bb16cd95ad9ce Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Sat, 17 Oct 2026 00:40:17 +0100 Subject: [PATCH 01/10] i18n: catalog parsing and lookup with fallback Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01F9fM3mv1PUyKpk5rdwUQrW --- Cargo.lock | 37 ++++++ Cargo.toml | 1 + crates/i18n/Cargo.toml | 15 +++ crates/i18n/src/catalog.rs | 114 ++++++++++++++++++ crates/i18n/src/lib.rs | 231 +++++++++++++++++++++++++++++++++++++ 5 files changed, 398 insertions(+) create mode 100644 crates/i18n/Cargo.toml create mode 100644 crates/i18n/src/catalog.rs create mode 100644 crates/i18n/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 676a5220..55a956a2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -609,6 +609,14 @@ dependencies = [ name = "craft-docpath" version = "0.1.0" +[[package]] +name = "craft-i18n" +version = "0.1.0" +dependencies = [ + "intl_pluralrules", + "unic-langid", +] + [[package]] name = "craft-pdf" version = "0.1.0" @@ -1934,6 +1942,15 @@ dependencies = [ "hashbrown 0.17.1", ] +[[package]] +name = "intl_pluralrules" +version = "7.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "078ea7b7c29a2b4df841a7f6ac8775ff6074020c6776d48491ce2268e068f972" +dependencies = [ + "unic-langid", +] + [[package]] name = "ipnet" version = "2.12.2" @@ -4346,6 +4363,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" dependencies = [ "displaydoc", + "serde_core", "zerovec", ] @@ -4573,6 +4591,24 @@ version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" +[[package]] +name = "unic-langid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28ba52c9b05311f4f6e62d5d9d46f094bd6e84cb8df7b3ef952748d752a7d05" +dependencies = [ + "unic-langid-impl", +] + +[[package]] +name = "unic-langid-impl" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce1bf08044d4b7a94028c93786f8566047edc11110595914de93362559bc658" +dependencies = [ + "tinystr", +] + [[package]] name = "unicode-bidi" version = "0.3.18" @@ -5826,6 +5862,7 @@ version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" dependencies = [ + "serde", "yoke", "zerofrom", "zerovec-derive", diff --git a/Cargo.toml b/Cargo.toml index 336fd37b..868e568d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,6 +23,7 @@ thiserror = "2" craft-atomic = { path = "crates/atomic" } craft-cms = { path = "crates/cms" } craft-docpath = { path = "crates/docpath" } +craft-i18n = { path = "crates/i18n" } craft-print = { path = "crates/print" } craft-quit = { path = "crates/quit" } craft-settings = { path = "crates/settings" } diff --git a/crates/i18n/Cargo.toml b/crates/i18n/Cargo.toml new file mode 100644 index 00000000..f1f08217 --- /dev/null +++ b/crates/i18n/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "craft-i18n" +description = "Translations for the Craft apps: the embedded craft-i18n catalogs, lookup with fallback, plurals, language names and the system locale." +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true + +[dependencies] +intl_pluralrules = "7" +unic-langid = "0.9" + +[lints] +workspace = true diff --git a/crates/i18n/src/catalog.rs b/crates/i18n/src/catalog.rs new file mode 100644 index 00000000..43d0d733 --- /dev/null +++ b/crates/i18n/src/catalog.rs @@ -0,0 +1,114 @@ +//! Reading one catalog. The rules are craft-i18n's `docs/catalog-format.md`; craft-i18n's CI +//! applies them strictly, and here a row that breaks one is skipped and reported, so an app built +//! against a newer catalog still runs, with that string in English. + +/// A row that passed, with its columns unescaped. +pub(crate) struct Entry { + pub context: String, + pub source: String, + pub translation: String, +} + +/// The contexts with a meaning here. Any other context starting with `@` is skipped. +const RESERVED: &[&str] = &["@plural", "@id", "@msg"]; + +/// The rows of a catalog whose locale has `forms` plural forms, and one problem (`line N: …`) per +/// row skipped. +pub(crate) fn parse(text: &str, forms: usize) -> (Vec, Vec) { + let text = text.strip_prefix('\u{feff}').unwrap_or(text); + let mut rows = Vec::new(); + let mut problems = Vec::new(); + for (i, line) in text.split('\n').enumerate() { + let line = line.strip_suffix('\r').unwrap_or(line); + if line.is_empty() || line.starts_with('#') { + continue; + } + match parse_row(line, forms) { + Ok(row) => rows.push(row), + Err(e) => problems.push(format!("line {}: {e}", i + 1)), + } + } + (rows, problems) +} + +fn parse_row(text: &str, forms: usize) -> Result { + let columns: Vec<&str> = text.split('\t').collect(); + let [context, source, translation] = columns.as_slice() else { + return Err(format!("{} columns; expected 3", columns.len())); + }; + if text.chars().any(|c| c != '\t' && (c.is_control() || c == '\u{feff}')) { + return Err("invisible character in the row".into()); + } + let row = Entry { context: unescape(context)?, source: unescape(source)?, translation: unescape(translation)? }; + if row.source.is_empty() || row.translation.is_empty() { + return Err("empty source text or translation".into()); + } + match row.context.as_str() { + "@id" => {} + "@plural" => check_plural(&row.source, &row.translation, forms)?, + c if c.starts_with('@') && !RESERVED.contains(&c) => { + return Err(format!("unknown reserved context {c:?}")); + } + _ => { + if placeholders(&row.source) != placeholders(&row.translation) { + return Err("placeholders differ from the source text".into()); + } + if row.source.ends_with('…') != row.translation.ends_with('…') { + return Err("trailing … differs from the source text".into()); + } + } + } + Ok(row) +} + +fn check_plural(source: &str, translation: &str, forms: usize) -> Result<(), String> { + let english: Vec<&str> = source.split('|').collect(); + let [one, other] = english.as_slice() else { + return Err("plural source text must be one|other".into()); + }; + if one.is_empty() || other.is_empty() || placeholders(one) != placeholders(other) { + return Err("plural source forms must be nonempty with the same placeholders".into()); + } + let translated: Vec<&str> = translation.split('|').collect(); + if translated.len() != forms || translated.iter().any(|f| f.is_empty()) { + return Err(format!("expected {forms} nonempty plural forms")); + } + if translated.iter().any(|f| placeholders(f) != placeholders(other)) { + return Err("plural form placeholders differ from the source text".into()); + } + Ok(()) +} + +fn unescape(text: &str) -> Result { + let mut out = String::with_capacity(text.len()); + let mut chars = text.chars(); + while let Some(c) = chars.next() { + if c != '\\' { + out.push(c); + continue; + } + match chars.next() { + Some('n') => out.push('\n'), + Some('t') => out.push('\t'), + Some('\\') => out.push('\\'), + Some(other) => return Err(format!("unknown escape \\{other}")), + None => return Err("trailing backslash".into()), + } + } + Ok(out) +} + +/// `{name}` placeholders, sorted: translations may reorder them. +fn placeholders(text: &str) -> Vec<&str> { + let mut names = Vec::new(); + let mut rest = text; + while let Some((_, after)) = rest.split_once('{') { + let Some((name, after)) = after.split_once('}') else { + break; + }; + names.push(name); + rest = after; + } + names.sort_unstable(); + names +} diff --git a/crates/i18n/src/lib.rs b/crates/i18n/src/lib.rs new file mode 100644 index 00000000..bb324595 --- /dev/null +++ b/crates/i18n/src/lib.rs @@ -0,0 +1,231 @@ +//! Translations for the Craft apps. +//! +//! An app embeds the [craft-i18n](https://github.com/benandez/craft-i18n) catalogs at build time +//! as an [`Embedded`] static, builds one [`I18n`] from it, and looks English source text up in a +//! [`Lang`]. A missing translation is the English itself; nothing here fails. +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] + +mod catalog; + +use std::collections::HashMap; + +use intl_pluralrules::{PluralCategory, PluralRuleType, PluralRules}; +use unic_langid::LanguageIdentifier; + +/// The catalogs an app embeds, one entry per locale: (locale, shared catalog, the app's catalog). +/// A missing catalog is `""`. +pub type Embedded = [(&'static str, &'static str, &'static str)]; + +/// A language the interface can be shown in: English, the pseudo locale, or an embedded locale. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct Lang(usize); + +impl Lang { + /// English: the source text itself. + pub const EN: Lang = Lang(0); + /// A test-only locale that marks every looked-up string. + pub const PSEUDO: Lang = Lang(1); +} + +/// One locale's translations: context → source text → translation, app rows over shared ones. +struct Locale { + code: String, + rows: HashMap>, + /// Itself, then its base language if that is embedded too (`pt-br`, `pt`). Empty for English + /// and the pseudo locale. + chain: Vec, + /// Plural rules and the categories `@plural` forms list, in order. + #[expect(dead_code, reason = "read by plural lookups")] + plural: Option<(PluralRules, Vec)>, +} + +/// The embedded catalogs, ready for lookups. Build it once, at start-up. +pub struct I18n { + locales: Vec, +} + +impl I18n { + /// Read `embedded`. Rows that can't be used are skipped and come back as problems + /// (`/: line N: …`) for the app to log. + pub fn new(embedded: &'static Embedded) -> (I18n, Vec) { + let mut problems = Vec::new(); + let mut locales = vec![bare("en"), bare("pseudo")]; + for (code, shared, app) in embedded { + let code = code.to_ascii_lowercase(); + let plural = plural_rules(&code); + if plural.is_none() { + problems.push(format!("{code}: no plural rules; plurals show in English")); + } + let forms = plural.as_ref().map_or(1, |(_, c)| c.len()); + let mut rows: HashMap> = HashMap::new(); + for (name, text) in [("common", shared), ("app", app)] { + let (entries, errors) = catalog::parse(text, forms); + problems.extend(errors.into_iter().map(|e| format!("{code}/{name}: {e}"))); + for e in entries { + rows.entry(e.context).or_default().insert(e.source, e.translation); + } + } + locales.push(Locale { code, rows, chain: Vec::new(), plural }); + } + for i in 2..locales.len() { + let mut chain = vec![i]; + let base = locales.get(i).and_then(|l| l.code.split('-').next()).unwrap_or_default(); + if let Some(b) = locales.iter().skip(2).position(|l| l.code == base) + && b + 2 != i + { + chain.push(b + 2); + } + if let Some(l) = locales.get_mut(i) { + l.chain = chain; + } + } + (I18n { locales }, problems) + } + + /// The language with this code (any case): `en`, `pseudo`, or an embedded locale. + pub fn from_code(&self, code: &str) -> Option { + self.locales.iter().position(|l| l.code.eq_ignore_ascii_case(code)).map(Lang) + } + + /// The language's code, lowercase BCP 47 (`pt-br`). + pub fn code(&self, lang: Lang) -> &str { + self.locale(lang).map_or("en", |l| l.code.as_str()) + } + + /// `source` in `lang`, or `source` itself. + pub fn t<'a>(&'a self, lang: Lang, source: &'a str) -> &'a str { + self.find(lang, "", source).unwrap_or(source) + } + + /// `source` with a disambiguating `context` (`menu`, `@msg`); falls back to [`I18n::t`]. + pub fn tc<'a>(&'a self, lang: Lang, context: &str, source: &'a str) -> &'a str { + self.find(lang, context, source).unwrap_or_else(|| self.t(lang, source)) + } + + /// A command's label by its `id` (`@id` rows); falls back to [`I18n::t`] of the English `label`. + pub fn tid<'a>(&'a self, lang: Lang, id: &str, label: &'a str) -> &'a str { + self.find(lang, "@id", id).unwrap_or_else(|| self.t(lang, label)) + } + + fn locale(&self, lang: Lang) -> Option<&Locale> { + self.locales.get(lang.0) + } + + /// The first translation of (`context`, `source`) along `lang`'s chain. + fn find(&self, lang: Lang, context: &str, source: &str) -> Option<&str> { + let chain = self.locale(lang).map(|l| l.chain.as_slice())?; + chain.iter().find_map(|&i| self.locales.get(i)?.rows.get(context)?.get(source).map(String::as_str)) + } +} + +/// A locale with no catalogs (English, the pseudo locale). +fn bare(code: &str) -> Locale { + Locale { code: code.to_owned(), rows: HashMap::new(), chain: Vec::new(), plural: None } +} + +/// `code`'s CLDR cardinal rules and the categories whole numbers 0..=1000 reach, in CLDR order +/// (zero, one, two, few, many, other): one `@plural` form each. Regional and scripted codes fall +/// back to the language's rules. +fn plural_rules(code: &str) -> Option<(PluralRules, Vec)> { + let id: LanguageIdentifier = code.parse().ok()?; + let language = LanguageIdentifier::from_parts(id.language, None, None, &[]); + let rules = PluralRules::create(id, PluralRuleType::CARDINAL).or_else(|_| PluralRules::create(language, PluralRuleType::CARDINAL)).ok()?; + let mut categories = Vec::new(); + for n in 0..=1000_u32 { + let c = rules.select(n).ok()?; + if !categories.contains(&c) { + categories.push(c); + } + } + categories.sort_by_key(rank); + Some((rules, categories)) +} + +fn rank(c: &PluralCategory) -> u8 { + match c { + PluralCategory::ZERO => 0, + PluralCategory::ONE => 1, + PluralCategory::TWO => 2, + PluralCategory::FEW => 3, + PluralCategory::MANY => 4, + PluralCategory::OTHER => 5, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + static E: &Embedded = &[ + ("fr", "\tSave\tEnregistrer\n\tOpen\tOuvrir\nmenu\tOpen\tOuvrir le menu\n@id\tfile.save\tEnregistrer le fichier\n", "\tSave\tSauvegarder\n"), + ("pt", "\tSave\tGuardar\n\tUndo\tDesfazer\n", ""), + ("pt-br", "\tSave\tSalvar\n", ""), + ("de", "\tBad\n@nope\tA\tB\n\tUndo\tRückgängig\n", ""), + ]; + + fn i18n() -> I18n { + I18n::new(E).0 + } + + fn lang(i: &I18n, code: &str) -> Lang { + i.from_code(code).unwrap() + } + + #[test] + fn app_row_overrides_shared() { + let i = i18n(); + assert_eq!(i.t(lang(&i, "fr"), "Save"), "Sauvegarder"); + } + + #[test] + fn shared_row_when_no_override() { + let i = i18n(); + assert_eq!(i.t(lang(&i, "fr"), "Open"), "Ouvrir"); + } + + #[test] + fn region_falls_back_to_base() { + let i = i18n(); + assert_eq!(i.t(lang(&i, "pt-br"), "Undo"), "Desfazer"); + assert_eq!(i.t(lang(&i, "pt-br"), "Save"), "Salvar"); + } + + #[test] + fn missing_is_english() { + let i = i18n(); + assert_eq!(i.t(lang(&i, "fr"), "Close"), "Close"); + assert_eq!(i.t(Lang::EN, "Save"), "Save"); + } + + #[test] + fn context_then_plain() { + let i = i18n(); + assert_eq!(i.tc(lang(&i, "fr"), "menu", "Open"), "Ouvrir le menu"); + assert_eq!(i.tc(lang(&i, "fr"), "menu", "Save"), "Sauvegarder"); + } + + #[test] + fn id_then_label() { + let i = i18n(); + assert_eq!(i.tid(lang(&i, "fr"), "file.save", "Save"), "Enregistrer le fichier"); + assert_eq!(i.tid(lang(&i, "fr"), "file.nope", "Open"), "Ouvrir"); + } + + #[test] + fn bad_rows_skipped_and_reported() { + let (i, problems) = I18n::new(E); + assert_eq!(problems.len(), 2, "{problems:?}"); + assert!(problems[0].starts_with("de/common: line 1:"), "{problems:?}"); + assert!(problems[1].starts_with("de/common: line 2:"), "{problems:?}"); + assert_eq!(i.t(lang(&i, "de"), "Undo"), "Rückgängig"); + } + + #[test] + fn from_code_is_case_insensitive() { + let i = i18n(); + assert_eq!(i.from_code("PT-BR"), i.from_code("pt-br")); + assert_eq!(i.from_code("xx"), None); + assert_eq!(i.from_code("en"), Some(Lang::EN)); + assert_eq!(i.code(Lang::EN), "en"); + } +} From c84b14d6c23a5523bf72c982e849fae619b419f9 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Sat, 17 Oct 2026 00:40:58 +0100 Subject: [PATCH 02/10] i18n: plurals and placeholder filling Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01F9fM3mv1PUyKpk5rdwUQrW --- crates/i18n/src/lib.rs | 96 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 95 insertions(+), 1 deletion(-) diff --git a/crates/i18n/src/lib.rs b/crates/i18n/src/lib.rs index bb324595..a5baef50 100644 --- a/crates/i18n/src/lib.rs +++ b/crates/i18n/src/lib.rs @@ -35,7 +35,6 @@ struct Locale { /// and the pseudo locale. chain: Vec, /// Plural rules and the categories `@plural` forms list, in order. - #[expect(dead_code, reason = "read by plural lookups")] plural: Option<(PluralRules, Vec)>, } @@ -107,6 +106,26 @@ impl I18n { self.find(lang, "@id", id).unwrap_or_else(|| self.t(lang, label)) } + /// A count's wording in `lang`, `{n}` filled in. `one` and `other` are the English forms; + /// they are also the key of the `@plural` row, and the fallback. + pub fn trn(&self, lang: Lang, n: u64, one: &str, other: &str) -> String { + let key = format!("{one}|{other}"); + let chain = self.locale(lang).map_or(&[][..], |l| l.chain.as_slice()); + let text = chain + .iter() + .find_map(|&i| { + let locale = self.locales.get(i)?; + let forms = locale.rows.get("@plural")?.get(&key)?; + let (rules, categories) = locale.plural.as_ref()?; + let category = rules.select(n).ok()?; + // A category whole numbers up to 1000 never reach (French `many`) takes the last form. + let index = categories.iter().position(|c| *c == category).unwrap_or(categories.len().saturating_sub(1)); + forms.split('|').nth(index) + }) + .unwrap_or(if n == 1 { one } else { other }); + fmt(text, &[("n", &n.to_string())]) + } + fn locale(&self, lang: Lang) -> Option<&Locale> { self.locales.get(lang.0) } @@ -118,6 +137,32 @@ impl I18n { } } +/// Fill `{name}` placeholders from `args` in one pass. Unknown placeholders stay as written, and +/// inserted values are never read as templates, so a file name containing `{n}` stays intact. +pub fn fmt(template: &str, args: &[(&str, &str)]) -> String { + let mut out = String::with_capacity(template.len()); + let mut rest = template; + while let Some((before, after_open)) = rest.split_once('{') { + out.push_str(before); + let Some((name, after_close)) = after_open.split_once('}') else { + out.push('{'); + rest = after_open; + break; + }; + match args.iter().find(|(k, _)| *k == name) { + Some((_, value)) => out.push_str(value), + None => { + out.push('{'); + out.push_str(name); + out.push('}'); + } + } + rest = after_close; + } + out.push_str(rest); + out +} + /// A locale with no catalogs (English, the pseudo locale). fn bare(code: &str) -> Locale { Locale { code: code.to_owned(), rows: HashMap::new(), chain: Vec::new(), plural: None } @@ -228,4 +273,53 @@ mod tests { assert_eq!(i.from_code("en"), Some(Lang::EN)); assert_eq!(i.code(Lang::EN), "en"); } + + static P: &Embedded = &[ + ("fr", "@plural\t{n} page|{n} pages\t{n} page|{n} pages\n", ""), + ("ru", "@plural\t{n} file|{n} files\t{n} файл|{n} файла|{n} файлов\n", ""), + ("ja", "@plural\t{n} file|{n} files\t{n} 個\n", ""), + ]; + + #[test] + fn fmt_fills_once() { + assert_eq!(fmt("{a} to {b}", &[("a", "x"), ("b", "{a}")]), "x to {a}"); + assert_eq!(fmt("{z} {", &[]), "{z} {"); + } + + #[test] + fn one_other() { + let (i, problems) = I18n::new(P); + assert!(problems.is_empty(), "{problems:?}"); + let fr = lang(&i, "fr"); + assert_eq!(i.trn(fr, 0, "{n} page", "{n} pages"), "0 page"); + assert_eq!(i.trn(fr, 2, "{n} page", "{n} pages"), "2 pages"); + } + + #[test] + fn three_forms_ru() { + let i = I18n::new(P).0; + let ru = lang(&i, "ru"); + for (n, want) in [(1, "1 файл"), (3, "3 файла"), (5, "5 файлов"), (11, "11 файлов"), (21, "21 файл")] { + assert_eq!(i.trn(ru, n, "{n} file", "{n} files"), want); + } + } + + #[test] + fn one_form_ja() { + let i = I18n::new(P).0; + assert_eq!(i.trn(lang(&i, "ja"), 7, "{n} file", "{n} files"), "7 個"); + } + + #[test] + fn english_fallback() { + let i = I18n::new(P).0; + assert_eq!(i.trn(Lang::EN, 1, "{n} file", "{n} files"), "1 file"); + assert_eq!(i.trn(lang(&i, "fr"), 2, "{n} x", "{n} xs"), "2 xs"); + } + + #[test] + fn beyond_sweep() { + let i = I18n::new(P).0; + assert_eq!(i.trn(lang(&i, "fr"), 1_000_000, "{n} page", "{n} pages"), "1000000 pages"); + } } From cfbc6a1adfb947c94f0e9739dea12a4d4bed2165 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Sat, 17 Oct 2026 00:42:44 +0100 Subject: [PATCH 03/10] i18n: languages, names and system locale matching Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01F9fM3mv1PUyKpk5rdwUQrW --- Cargo.lock | 13 +++++ crates/i18n/Cargo.toml | 4 ++ crates/i18n/src/lib.rs | 123 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 140 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 55a956a2..6471411f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -614,6 +614,7 @@ name = "craft-i18n" version = "0.1.0" dependencies = [ "intl_pluralrules", + "sys-locale", "unic-langid", ] @@ -4211,6 +4212,18 @@ dependencies = [ "syn 3.0.6", ] +[[package]] +name = "sys-locale" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eab9a99a024a169fe8a903cf9d4a3b3601109bcc13bd9e3c6fff259138626c4" +dependencies = [ + "js-sys", + "libc", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "system-configuration" version = "0.7.0" diff --git a/crates/i18n/Cargo.toml b/crates/i18n/Cargo.toml index f1f08217..94f0e295 100644 --- a/crates/i18n/Cargo.toml +++ b/crates/i18n/Cargo.toml @@ -10,6 +10,10 @@ repository.workspace = true [dependencies] intl_pluralrules = "7" unic-langid = "0.9" +sys-locale = "0.3" + +[target.'cfg(target_arch = "wasm32")'.dependencies] +sys-locale = { version = "0.3", features = ["js"] } [lints] workspace = true diff --git a/crates/i18n/src/lib.rs b/crates/i18n/src/lib.rs index a5baef50..55a00da5 100644 --- a/crates/i18n/src/lib.rs +++ b/crates/i18n/src/lib.rs @@ -126,6 +126,53 @@ impl I18n { fmt(text, &[("n", &n.to_string())]) } + /// The languages to offer: English first, then the embedded locales by code. Never the + /// pseudo locale. + pub fn languages(&self) -> Vec { + let mut embedded: Vec = (2..self.locales.len()).collect(); + embedded.sort_by_key(|&i| self.locales.get(i).map(|l| l.code.as_str())); + std::iter::once(Lang::EN).chain(embedded.into_iter().map(Lang)).collect() + } + + /// The language's name in itself, for a language picker: its shared catalog's + /// `@id language.name` row, or the code when there is none. + pub fn name(&self, lang: Lang) -> &str { + if lang == Lang::EN { + return "English"; + } + self.locale(lang).and_then(|l| l.rows.get("@id")?.get("language.name")).map_or_else(|| self.code(lang), String::as_str) + } + + /// Whether the language is written right to left. Putting text into display order is the + /// app's job. + pub fn rtl(&self, lang: Lang) -> bool { + let code = self.code(lang); + let language = code.split('-').next().unwrap_or(code); + matches!(language, "ar" | "fa" | "he" | "ur") + } + + /// The language for a system locale tag (`fr_CA.UTF-8`, `zh-Hant-MO`); `C` and `POSIX` are + /// English. `None` when no embedded language fits. + pub fn match_tag(&self, tag: &str) -> Option { + let candidates = candidates(tag); + if matches!(candidates.first().map(String::as_str), Some("c" | "posix")) { + return Some(Lang::EN); + } + candidates.iter().filter(|c| *c != "pseudo").find_map(|c| self.from_code(c)) + } + + /// The first of `tags` (most preferred first) that [`I18n::match_tag`] knows; English if none. + pub fn match_tags<'t>(&self, tags: impl IntoIterator) -> Lang { + tags.into_iter().find_map(|t| self.match_tag(t)).unwrap_or(Lang::EN) + } + + /// The language the system asks for (`LANG` and friends, the macOS and Windows preferred + /// languages, the browser's on the web); English if none fits. + pub fn system_lang(&self) -> Lang { + let tags: Vec = sys_locale::get_locales().collect(); + self.match_tags(tags.iter().map(String::as_str)) + } + fn locale(&self, lang: Lang) -> Option<&Locale> { self.locales.get(lang.0) } @@ -163,6 +210,25 @@ pub fn fmt(template: &str, args: &[(&str, &str)]) -> String { out } +/// Codes to try for a locale tag, most specific first: `zh_TW.UTF-8` → `zh-tw`, `zh-hant`, `zh`. +/// Chinese without a script picks one by region; Portuguese regions without a catalog use `pt-br`. +fn candidates(tag: &str) -> Vec { + let base = tag.trim().split(['.', '@']).next().unwrap_or("").replace('_', "-").to_ascii_lowercase(); + let parts: Vec<&str> = base.split('-').filter(|p| !p.is_empty()).collect(); + let Some(&primary) = parts.first() else { + return Vec::new(); + }; + let mut out: Vec = (1..=parts.len()).rev().map(|n| parts.get(..n).unwrap_or_default().join("-")).collect(); + if primary == "zh" && !parts.iter().any(|p| matches!(*p, "hans" | "hant")) { + let script = if parts.iter().any(|p| matches!(*p, "tw" | "hk" | "mo")) { "zh-hant" } else { "zh-hans" }; + out.insert(out.len().saturating_sub(1), script.to_owned()); + } + if primary == "pt" && !out.iter().any(|c| c == "pt-br") { + out.insert(out.len().saturating_sub(1), "pt-br".to_owned()); + } + out +} + /// A locale with no catalogs (English, the pseudo locale). fn bare(code: &str) -> Locale { Locale { code: code.to_owned(), rows: HashMap::new(), chain: Vec::new(), plural: None } @@ -322,4 +388,61 @@ mod tests { let i = I18n::new(P).0; assert_eq!(i.trn(lang(&i, "fr"), 1_000_000, "{n} page", "{n} pages"), "1000000 pages"); } + + static L: &Embedded = + &[("fr", "@id\tlanguage.name\tFrançais\n", ""), ("pt-br", "", ""), ("zh-hans", "", ""), ("zh-hant", "", ""), ("ar", "", "")]; + + #[test] + fn languages_order() { + let i = I18n::new(L).0; + let codes: Vec<&str> = i.languages().into_iter().map(|l| i.code(l)).collect(); + assert_eq!(codes, ["en", "ar", "fr", "pt-br", "zh-hans", "zh-hant"]); + } + + #[test] + fn names() { + let i = I18n::new(L).0; + assert_eq!(i.name(Lang::EN), "English"); + assert_eq!(i.name(lang(&i, "fr")), "Français"); + assert_eq!(i.name(lang(&i, "ar")), "ar"); + } + + #[test] + fn rtl() { + let i = I18n::new(L).0; + assert!(i.rtl(lang(&i, "ar"))); + assert!(!i.rtl(lang(&i, "fr"))); + assert!(!i.rtl(Lang::EN)); + } + + #[test] + fn tags() { + let i = I18n::new(L).0; + for (tag, want) in [ + ("fr_CA.UTF-8", Some("fr")), + ("pt-PT", Some("pt-br")), + ("pt_BR", Some("pt-br")), + ("zh_TW.UTF-8", Some("zh-hant")), + ("zh-HK", Some("zh-hant")), + ("zh", Some("zh-hans")), + ("zh-Hant-MO", Some("zh-hant")), + ("zh_CN", Some("zh-hans")), + ("C", Some("en")), + ("POSIX", Some("en")), + ("en_US", Some("en")), + ("ja_JP", None), + ("", None), + ("@@", None), + ] { + assert_eq!(i.match_tag(tag).map(|l| i.code(l)), want, "{tag}"); + } + } + + #[test] + fn match_tags_takes_first_match() { + let i = I18n::new(L).0; + assert_eq!(i.match_tags(["ja_JP", "fr_FR"]), lang(&i, "fr")); + assert_eq!(i.match_tags(["ja"]), Lang::EN); + assert_eq!(i.match_tags([]), Lang::EN); + } } From 6346621e911c73a47832ca69fc52417a2cab7bd6 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Sat, 17 Oct 2026 00:43:15 +0100 Subject: [PATCH 04/10] i18n: pseudo locale Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01F9fM3mv1PUyKpk5rdwUQrW --- crates/i18n/src/lib.rs | 43 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/crates/i18n/src/lib.rs b/crates/i18n/src/lib.rs index 55a00da5..b868d5bb 100644 --- a/crates/i18n/src/lib.rs +++ b/crates/i18n/src/lib.rs @@ -8,6 +8,7 @@ mod catalog; use std::collections::HashMap; +use std::sync::{Mutex, PoisonError}; use intl_pluralrules::{PluralCategory, PluralRuleType, PluralRules}; use unic_langid::LanguageIdentifier; @@ -41,6 +42,8 @@ struct Locale { /// The embedded catalogs, ready for lookups. Build it once, at start-up. pub struct I18n { locales: Vec, + /// Marked strings for the pseudo locale, leaked so lookups can hand out `&str`. Test-only. + pseudo: Mutex>, } impl I18n { @@ -78,7 +81,7 @@ impl I18n { l.chain = chain; } } - (I18n { locales }, problems) + (I18n { locales, pseudo: Mutex::default() }, problems) } /// The language with this code (any case): `en`, `pseudo`, or an embedded locale. @@ -93,16 +96,25 @@ impl I18n { /// `source` in `lang`, or `source` itself. pub fn t<'a>(&'a self, lang: Lang, source: &'a str) -> &'a str { + if lang == Lang::PSEUDO { + return self.mark(source); + } self.find(lang, "", source).unwrap_or(source) } /// `source` with a disambiguating `context` (`menu`, `@msg`); falls back to [`I18n::t`]. pub fn tc<'a>(&'a self, lang: Lang, context: &str, source: &'a str) -> &'a str { + if lang == Lang::PSEUDO { + return self.mark(source); + } self.find(lang, context, source).unwrap_or_else(|| self.t(lang, source)) } /// A command's label by its `id` (`@id` rows); falls back to [`I18n::t`] of the English `label`. pub fn tid<'a>(&'a self, lang: Lang, id: &str, label: &'a str) -> &'a str { + if lang == Lang::PSEUDO { + return self.mark(label); + } self.find(lang, "@id", id).unwrap_or_else(|| self.t(lang, label)) } @@ -123,7 +135,11 @@ impl I18n { forms.split('|').nth(index) }) .unwrap_or(if n == 1 { one } else { other }); - fmt(text, &[("n", &n.to_string())]) + let text = fmt(text, &[("n", &n.to_string())]); + if lang == Lang::PSEUDO { + return marked(&text); + } + text } /// The languages to offer: English first, then the embedded locales by code. Never the @@ -173,6 +189,12 @@ impl I18n { self.match_tags(tags.iter().map(String::as_str)) } + /// `source` marked for the pseudo locale, made once per string. + fn mark(&self, source: &str) -> &'static str { + let mut cache = self.pseudo.lock().unwrap_or_else(PoisonError::into_inner); + cache.entry(source.to_owned()).or_insert_with(|| Box::leak(marked(source).into_boxed_str())) + } + fn locale(&self, lang: Lang) -> Option<&Locale> { self.locales.get(lang.0) } @@ -229,6 +251,11 @@ fn candidates(tag: &str) -> Vec { out } +/// `⟦text⟧`, each line marked on its own, since multi-line labels are often drawn a line at a time. +fn marked(text: &str) -> String { + format!("⟦{}⟧", text.replace('\n', "⟧\n⟦")) +} + /// A locale with no catalogs (English, the pseudo locale). fn bare(code: &str) -> Locale { Locale { code: code.to_owned(), rows: HashMap::new(), chain: Vec::new(), plural: None } @@ -445,4 +472,16 @@ mod tests { assert_eq!(i.match_tags(["ja"]), Lang::EN); assert_eq!(i.match_tags([]), Lang::EN); } + + #[test] + fn pseudo_marks_every_lookup() { + let i = i18n(); + assert_eq!(i.t(Lang::PSEUDO, "Save"), "⟦Save⟧"); + assert_eq!(i.t(Lang::PSEUDO, "A\nB"), "⟦A⟧\n⟦B⟧"); + assert_eq!(i.tc(Lang::PSEUDO, "menu", "Open"), "⟦Open⟧"); + assert_eq!(i.tid(Lang::PSEUDO, "file.save", "Save"), "⟦Save⟧"); + assert_eq!(i.trn(Lang::PSEUDO, 2, "{n} x", "{n} xs"), "⟦2 xs⟧"); + assert!(!i.languages().contains(&Lang::PSEUDO)); + assert_eq!(i.from_code("pseudo"), Some(Lang::PSEUDO)); + } } From c9bcc60832f013a659a463820113eb42e7ac0172 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Sat, 17 Oct 2026 00:50:44 +0100 Subject: [PATCH 05/10] i18n: README and library listing Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01F9fM3mv1PUyKpk5rdwUQrW --- README.md | 4 ++++ crates/i18n/README.md | 55 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+) create mode 100644 crates/i18n/README.md diff --git a/README.md b/README.md index 4b428ee8..78b24340 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,10 @@ craft- = { git = "https://github.com/storytold/craft-libs", rev = " = LazyLock::new(|| { + let (i18n, problems) = craft_i18n::I18n::new(CRAFT_I18N); + problems.iter().for_each(|p| log::warn!("translation catalog: {p}; that string shows in English")); + i18n +}); + +thread_local! { static CURRENT: Cell = const { Cell::new(craft_i18n::Lang::EN) }; } + +macro_rules! tl { ($s:expr) => { $crate::I18N.t(CURRENT.get(), $s) }; } +``` + +The app keeps its `tl!` macro, its language setting (`"auto"` → `I18N.system_lang()`, otherwise +`I18N.from_code`) and its picker (`I18N.languages()`, `I18N.name(lang)`). + +## Behaviour + +- **Fallback:** `pt-br` app rows → `pt-br` shared rows → `pt` → the English source text. App rows + override shared ones in their own locale. A missing translation never errors. +- **Lookups:** `t` (plain), `tc` (with a context such as `menu` or `@msg`, falling back to plain), + `tid` (an `@id` command id, falling back to the English label), `trn` (a count, `@plural` rows, + `{n}` filled), and `fmt` for `{name}` placeholders, filled in one pass. +- **Bad rows** (malformed, an unknown `@` context, a wrong plural form count) are skipped and + returned by `I18n::new` for the app to log; craft-i18n's CI rejects them before they ship. +- **Plurals:** forms are listed in CLDR order (zero, one, two, few, many, other) for the categories + whole numbers 0 to 1000 reach, from `intl_pluralrules`; a count in any other category takes the + last form. Keep `intl_pluralrules` on the same version as craft-i18n's checker. +- **Names:** a language's name comes from its shared catalog's `@id language.name` row, else its + code; English is "English". +- **Right to left:** `rtl` is true for `ar`, `fa`, `he` and `ur`; putting text into display order + is the app's job. +- **System locale:** `system_lang` matches `sys-locale`'s list (`LANG`, macOS and Windows preferred + languages, the browser's on the web): `fr_CA.UTF-8` → `fr`, `zh-TW` → `zh-hant`, `pt-PT` → + `pt-br` when that is the only Portuguese, `C`/`POSIX` → English. +- **Pseudo locale** (`pseudo`, test-only, never in `languages()`): every lookup comes back as + `⟦…⟧`, line by line, so text drawn without the marks never went through translation. From 841ebed58c33f09142d885a48d79533274eec9c1 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Sat, 17 Oct 2026 00:55:59 +0100 Subject: [PATCH 06/10] i18n: prefer pt over pt-br, no fallback across scripts, lookup without fallback Final review fixes: a European Portuguese locale wins over pt-br when both are embedded; sr-latn no longer falls back to Cyrillic sr; get() looks up one context with no plain or English fallback (key-cap names). Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01F9fM3mv1PUyKpk5rdwUQrW --- crates/i18n/README.md | 6 +++-- crates/i18n/src/lib.rs | 58 +++++++++++++++++++++++++++++++++++++++--- 2 files changed, 59 insertions(+), 5 deletions(-) diff --git a/crates/i18n/README.md b/crates/i18n/README.md index ceb67969..0f8cfaf9 100644 --- a/crates/i18n/README.md +++ b/crates/i18n/README.md @@ -35,10 +35,12 @@ The app keeps its `tl!` macro, its language setting (`"auto"` → `I18N.system_l ## Behaviour - **Fallback:** `pt-br` app rows → `pt-br` shared rows → `pt` → the English source text. App rows - override shared ones in their own locale. A missing translation never errors. + override shared ones in their own locale. Only a region falls back (`sr-latn` never falls back to + Cyrillic `sr`). A missing translation never errors. - **Lookups:** `t` (plain), `tc` (with a context such as `menu` or `@msg`, falling back to plain), `tid` (an `@id` command id, falling back to the English label), `trn` (a count, `@plural` rows, - `{n}` filled), and `fmt` for `{name}` placeholders, filled in one pass. + `{n}` filled), `get` (one context, no fallback: a key cap's name under `key`, or "is there a + translation?"), and `fmt` for `{name}` placeholders, filled in one pass. - **Bad rows** (malformed, an unknown `@` context, a wrong plural form count) are skipped and returned by `I18n::new` for the app to log; craft-i18n's CI rejects them before they ship. - **Plurals:** forms are listed in CLDR order (zero, one, two, few, many, other) for the categories diff --git a/crates/i18n/src/lib.rs b/crates/i18n/src/lib.rs index b868d5bb..a4edf131 100644 --- a/crates/i18n/src/lib.rs +++ b/crates/i18n/src/lib.rs @@ -71,7 +71,13 @@ impl I18n { } for i in 2..locales.len() { let mut chain = vec![i]; - let base = locales.get(i).and_then(|l| l.code.split('-').next()).unwrap_or_default(); + // Only a region falls back to its language (`pt-br` → `pt`); a script doesn't, since + // `sr-latn` → `sr` would show Cyrillic. + let code = locales.get(i).map_or("", |l| l.code.as_str()); + let base = match code.split_once('-') { + Some((language, region)) if is_region(region) => language, + _ => "", + }; if let Some(b) = locales.iter().skip(2).position(|l| l.code == base) && b + 2 != i { @@ -110,6 +116,13 @@ impl I18n { self.find(lang, context, source).unwrap_or_else(|| self.t(lang, source)) } + /// The translation of (`context`, `source`) along `lang`'s chain, with no fallback to the plain + /// row or the English: for strings that must not borrow another meaning's wording (a key cap's + /// name under context `key`), or to ask whether a translation exists. + pub fn get(&self, lang: Lang, context: &str, source: &str) -> Option<&str> { + self.find(lang, context, source) + } + /// A command's label by its `id` (`@id` rows); falls back to [`I18n::t`] of the English `label`. pub fn tid<'a>(&'a self, lang: Lang, id: &str, label: &'a str) -> &'a str { if lang == Lang::PSEUDO { @@ -233,7 +246,8 @@ pub fn fmt(template: &str, args: &[(&str, &str)]) -> String { } /// Codes to try for a locale tag, most specific first: `zh_TW.UTF-8` → `zh-tw`, `zh-hant`, `zh`. -/// Chinese without a script picks one by region; Portuguese regions without a catalog use `pt-br`. +/// Chinese without a script picks one by region; Portuguese falls back to `pt-br` last, when +/// there is no `pt`. fn candidates(tag: &str) -> Vec { let base = tag.trim().split(['.', '@']).next().unwrap_or("").replace('_', "-").to_ascii_lowercase(); let parts: Vec<&str> = base.split('-').filter(|p| !p.is_empty()).collect(); @@ -246,7 +260,7 @@ fn candidates(tag: &str) -> Vec { out.insert(out.len().saturating_sub(1), script.to_owned()); } if primary == "pt" && !out.iter().any(|c| c == "pt-br") { - out.insert(out.len().saturating_sub(1), "pt-br".to_owned()); + out.push("pt-br".to_owned()); } out } @@ -256,6 +270,11 @@ fn marked(text: &str) -> String { format!("⟦{}⟧", text.replace('\n', "⟧\n⟦")) } +/// A BCP 47 region subtag: two letters or three digits. +fn is_region(subtag: &str) -> bool { + (subtag.len() == 2 && subtag.bytes().all(|b| b.is_ascii_alphabetic())) || (subtag.len() == 3 && subtag.bytes().all(|b| b.is_ascii_digit())) +} + /// A locale with no catalogs (English, the pseudo locale). fn bare(code: &str) -> Locale { Locale { code: code.to_owned(), rows: HashMap::new(), chain: Vec::new(), plural: None } @@ -484,4 +503,37 @@ mod tests { assert!(!i.languages().contains(&Lang::PSEUDO)); assert_eq!(i.from_code("pseudo"), Some(Lang::PSEUDO)); } + + static F: &Embedded = &[ + ("pt", "", ""), + ("pt-br", "", ""), + ("sr", "\tSave\tСачувај\n", ""), + ("sr-latn", "", ""), + ("fr", "\tDelete\tSupprimer\nkey\tEsc\tÉchap\n", ""), + ]; + + #[test] + fn european_portuguese_prefers_pt() { + let i = I18n::new(F).0; + for tag in ["pt", "pt-PT", "pt_PT.UTF-8"] { + assert_eq!(i.match_tag(tag).map(|l| i.code(l)), Some("pt"), "{tag}"); + } + assert_eq!(i.match_tag("pt_BR").map(|l| i.code(l)), Some("pt-br")); + } + + #[test] + fn script_locale_does_not_fall_back_across_scripts() { + let i = I18n::new(F).0; + assert_eq!(i.t(lang(&i, "sr-latn"), "Save"), "Save"); + } + + #[test] + fn get_has_no_plain_fallback() { + let i = I18n::new(F).0; + let fr = lang(&i, "fr"); + assert_eq!(i.get(fr, "key", "Esc"), Some("Échap")); + assert_eq!(i.get(fr, "key", "Delete"), None); + assert_eq!(i.get(fr, "", "Delete"), Some("Supprimer")); + assert_eq!(i.get(Lang::EN, "", "Delete"), None); + } } From 10592ba4fd55a3dfca89a0f6f095522f79fea3f9 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Sun, 18 Oct 2026 10:07:11 +0100 Subject: [PATCH 07/10] ci(i18n): run clippy and tests for craft-i18n on all three platforms Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/workflows/i18n.yml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 .github/workflows/i18n.yml diff --git a/.github/workflows/i18n.yml b/.github/workflows/i18n.yml new file mode 100644 index 00000000..d7e6f5b5 --- /dev/null +++ b/.github/workflows/i18n.yml @@ -0,0 +1,29 @@ +name: Translations + +on: + workflow_dispatch: + push: + branches: [main] + paths: ['crates/i18n/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/i18n.yml'] + pull_request: + paths: ['crates/i18n/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/i18n.yml'] + +permissions: + contents: read + +jobs: + i18n: + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + with: + components: rustfmt, clippy + - run: cargo fmt --check + - run: cargo clippy -p craft-i18n --all-targets --locked -- -D warnings + - run: cargo test -p craft-i18n --locked -- --nocapture From 886a0b367f3a29f4b3a34c13d2750edde0272512 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Sun, 18 Oct 2026 10:38:04 +0100 Subject: [PATCH 08/10] i18n: sr_RS@latin picks sr-latn; README macro and get wording Review fixes (CodeRabbit): a glibc @latin/@cyrillic modifier becomes the script subtag and only codes with that script are tried. sys-locale drops the modifier, so on desktop Unix system_lang reads LANGUAGE, LC_ALL, LC_MESSAGES and LANG raw, in its order. README: tl! qualifies CURRENT with $crate; get follows the locale chain but never the plain row or English. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/i18n/README.md | 8 +++---- crates/i18n/src/lib.rs | 53 +++++++++++++++++++++++++++++++++++++++--- 2 files changed, 54 insertions(+), 7 deletions(-) diff --git a/crates/i18n/README.md b/crates/i18n/README.md index 0f8cfaf9..d9a7ebb4 100644 --- a/crates/i18n/README.md +++ b/crates/i18n/README.md @@ -26,7 +26,7 @@ pub static I18N: LazyLock = LazyLock::new(|| { thread_local! { static CURRENT: Cell = const { Cell::new(craft_i18n::Lang::EN) }; } -macro_rules! tl { ($s:expr) => { $crate::I18N.t(CURRENT.get(), $s) }; } +macro_rules! tl { ($s:expr) => { $crate::I18N.t($crate::CURRENT.get(), $s) }; } ``` The app keeps its `tl!` macro, its language setting (`"auto"` → `I18N.system_lang()`, otherwise @@ -36,11 +36,11 @@ The app keeps its `tl!` macro, its language setting (`"auto"` → `I18N.system_l - **Fallback:** `pt-br` app rows → `pt-br` shared rows → `pt` → the English source text. App rows override shared ones in their own locale. Only a region falls back (`sr-latn` never falls back to - Cyrillic `sr`). A missing translation never errors. + Cyrillic `sr`, and a system locale of `sr_RS@latin` picks `sr-latn`, never `sr`). A missing translation never errors. - **Lookups:** `t` (plain), `tc` (with a context such as `menu` or `@msg`, falling back to plain), `tid` (an `@id` command id, falling back to the English label), `trn` (a count, `@plural` rows, - `{n}` filled), `get` (one context, no fallback: a key cap's name under `key`, or "is there a - translation?"), and `fmt` for `{name}` placeholders, filled in one pass. + `{n}` filled), `get` (one context along the locale chain, so `pt-br` can return a `pt` row, but never the + plain row or English: a key cap's name under `key`, or "is there a translation?"), and `fmt` for `{name}` placeholders, filled in one pass. - **Bad rows** (malformed, an unknown `@` context, a wrong plural form count) are skipped and returned by `I18n::new` for the app to log; craft-i18n's CI rejects them before they ship. - **Plurals:** forms are listed in CLDR order (zero, one, two, few, many, other) for the categories diff --git a/crates/i18n/src/lib.rs b/crates/i18n/src/lib.rs index a4edf131..4387aa4f 100644 --- a/crates/i18n/src/lib.rs +++ b/crates/i18n/src/lib.rs @@ -198,6 +198,10 @@ impl I18n { /// The language the system asks for (`LANG` and friends, the macOS and Windows preferred /// languages, the browser's on the web); English if none fits. pub fn system_lang(&self) -> Lang { + // sys-locale drops glibc's script modifier (`sr_RS@latin` → `sr-RS`), so read its variables raw. + #[cfg(all(unix, not(any(target_vendor = "apple", target_os = "android"))))] + let tags = posix_env_tags(|k| std::env::var(k).ok()); + #[cfg(not(all(unix, not(any(target_vendor = "apple", target_os = "android")))))] let tags: Vec = sys_locale::get_locales().collect(); self.match_tags(tags.iter().map(String::as_str)) } @@ -247,14 +251,27 @@ pub fn fmt(template: &str, args: &[(&str, &str)]) -> String { /// Codes to try for a locale tag, most specific first: `zh_TW.UTF-8` → `zh-tw`, `zh-hant`, `zh`. /// Chinese without a script picks one by region; Portuguese falls back to `pt-br` last, when -/// there is no `pt`. +/// there is no `pt`. A glibc script modifier (`sr_RS@latin`) becomes the script subtag, and only +/// codes with that script are tried, so Latin Serbian never picks Cyrillic `sr`. fn candidates(tag: &str) -> Vec { - let base = tag.trim().split(['.', '@']).next().unwrap_or("").replace('_', "-").to_ascii_lowercase(); - let parts: Vec<&str> = base.split('-').filter(|p| !p.is_empty()).collect(); + let tag = tag.trim().to_ascii_lowercase(); + let base = tag.split(['.', '@']).next().unwrap_or("").replace('_', "-"); + let mut parts: Vec<&str> = base.split('-').filter(|p| !p.is_empty()).collect(); let Some(&primary) = parts.first() else { return Vec::new(); }; + let script = match tag.split_once('@').map(|(_, m)| m) { + Some("latin") => Some("latn"), + Some("cyrillic") => Some("cyrl"), + _ => None, + }; + if let Some(s) = script { + parts.insert(1, s); + } let mut out: Vec = (1..=parts.len()).rev().map(|n| parts.get(..n).unwrap_or_default().join("-")).collect(); + if let Some(s) = script { + out.retain(|c| c.split('-').any(|p| p == s)); + } if primary == "zh" && !parts.iter().any(|p| matches!(*p, "hans" | "hant")) { let script = if parts.iter().any(|p| matches!(*p, "tw" | "hk" | "mo")) { "zh-hant" } else { "zh-hans" }; out.insert(out.len().saturating_sub(1), script.to_owned()); @@ -265,6 +282,16 @@ fn candidates(tag: &str) -> Vec { out } +/// The locale variables sys-locale reads on Unix, in its order (`LANGUAGE`'s `:` list, then +/// `LC_ALL`, `LC_MESSAGES`, `LANG`), but unconverted, so `@latin` survives. +#[cfg(all(unix, not(any(target_vendor = "apple", target_os = "android"))))] +fn posix_env_tags(var: impl Fn(&str) -> Option) -> Vec { + let mut tags: Vec = var("LANGUAGE").map(|v| v.split(':').map(str::to_owned).collect()).unwrap_or_default(); + tags.extend(["LC_ALL", "LC_MESSAGES", "LANG"].into_iter().filter_map(&var)); + tags.retain(|t| !t.is_empty()); + tags +} + /// `⟦text⟧`, each line marked on its own, since multi-line labels are often drawn a line at a time. fn marked(text: &str) -> String { format!("⟦{}⟧", text.replace('\n', "⟧\n⟦")) @@ -527,6 +554,26 @@ mod tests { assert_eq!(i.t(lang(&i, "sr-latn"), "Save"), "Save"); } + #[test] + fn glibc_script_modifier_picks_the_script() { + let i = I18n::new(F).0; + for (tag, want) in [("sr_RS.UTF-8@latin", Some("sr-latn")), ("sr_RS@latin", Some("sr-latn")), ("sr_RS.UTF-8", Some("sr"))] { + assert_eq!(i.match_tag(tag).map(|l| i.code(l)), want, "{tag}"); + } + let cyrillic_only = I18n::new(&[("sr", "", "")]).0; + assert_eq!(cyrillic_only.match_tag("sr_RS@latin"), None); + } + + #[cfg(all(unix, not(any(target_vendor = "apple", target_os = "android"))))] + #[test] + fn posix_env_keeps_the_modifier() { + let env = |vars: &'static [(&'static str, &'static str)]| move |k: &str| vars.iter().find(|(n, _)| *n == k).map(|(_, v)| (*v).to_owned()); + assert_eq!(posix_env_tags(env(&[("LANG", "sr_RS.UTF-8@latin")])), ["sr_RS.UTF-8@latin"]); + assert_eq!(posix_env_tags(env(&[("LANGUAGE", "fr:de"), ("LC_ALL", ""), ("LANG", "sr_RS@latin")])), ["fr", "de", "sr_RS@latin"]); + let i = I18n::new(F).0; + assert_eq!(i.code(i.match_tags(posix_env_tags(env(&[("LANG", "sr_RS.UTF-8@latin")])).iter().map(String::as_str))), "sr-latn"); + } + #[test] fn get_has_no_plain_fallback() { let i = I18n::new(F).0; From c36ea1bf64fe47a40b4e6b7876266ebf2b74b361 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Sun, 18 Oct 2026 19:12:01 +0100 Subject: [PATCH 09/10] i18n: review fixes (offered locales, locale-first fallback, pt plurals, scripts) Skylar's review of 886a0b3: 1. tc/tid go locale by locale: in each locale the context or @id row, then the plain row, so pt-br's plain row beats pt's context row. 2. pt (and pt-* other than pt-br) uses pt-PT plural rules: 0 is plural. 3. A script subtag (sr-Latn-RS, sr_RS@latin, be_BY@latin) never picks a code of another script; sr-Cyrl-RS still picks sr. 4. A row defined twice is reported and the first kept, as in the checker. 5. The pseudo locale is debug-only via from_code, get marks too, and looked_up() lists what was looked up for an app's audit. 6. intl_pluralrules pinned to =7.0.2; plural_form_counts copied from the checker, plus pt. 7. I18n::new takes the app's offered codes: languages(), from_code, match_tag and system_lang only return those; others stay fallbacks. Smaller: en/pseudo/repeated codes are skipped and reported, system_lang is cached, zh-hant-hk falls back to zh-hant, rtl adds ckb, dv, ps, sd, ug and yi. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/i18n/Cargo.toml | 2 +- crates/i18n/README.md | 53 ++++--- crates/i18n/src/catalog.rs | 8 +- crates/i18n/src/lib.rs | 280 +++++++++++++++++++++++++++++-------- 4 files changed, 261 insertions(+), 82 deletions(-) diff --git a/crates/i18n/Cargo.toml b/crates/i18n/Cargo.toml index 94f0e295..45e21775 100644 --- a/crates/i18n/Cargo.toml +++ b/crates/i18n/Cargo.toml @@ -8,7 +8,7 @@ rust-version.workspace = true repository.workspace = true [dependencies] -intl_pluralrules = "7" +intl_pluralrules = "=7.0.2" unic-langid = "0.9" sys-locale = "0.3" diff --git a/crates/i18n/README.md b/crates/i18n/README.md index d9a7ebb4..df337971 100644 --- a/crates/i18n/README.md +++ b/crates/i18n/README.md @@ -19,7 +19,8 @@ use std::sync::LazyLock; include!(concat!(env!("OUT_DIR"), "/craft_i18n.rs")); // pub static CRAFT_I18N: &craft_i18n::Embedded pub static I18N: LazyLock = LazyLock::new(|| { - let (i18n, problems) = craft_i18n::I18n::new(CRAFT_I18N); + let offered: Vec<&str> = LANGUAGES.iter().map(|l| l.code).collect(); // the app's own list + let (i18n, problems) = craft_i18n::I18n::new(CRAFT_I18N, &offered); problems.iter().for_each(|p| log::warn!("translation catalog: {p}; that string shows in English")); i18n }); @@ -34,24 +35,40 @@ The app keeps its `tl!` macro, its language setting (`"auto"` → `I18N.system_l ## Behaviour -- **Fallback:** `pt-br` app rows → `pt-br` shared rows → `pt` → the English source text. App rows - override shared ones in their own locale. Only a region falls back (`sr-latn` never falls back to - Cyrillic `sr`, and a system locale of `sr_RS@latin` picks `sr-latn`, never `sr`). A missing translation never errors. -- **Lookups:** `t` (plain), `tc` (with a context such as `menu` or `@msg`, falling back to plain), - `tid` (an `@id` command id, falling back to the English label), `trn` (a count, `@plural` rows, - `{n}` filled), `get` (one context along the locale chain, so `pt-br` can return a `pt` row, but never the - plain row or English: a key cap's name under `key`, or "is there a translation?"), and `fmt` for `{name}` placeholders, filled in one pass. -- **Bad rows** (malformed, an unknown `@` context, a wrong plural form count) are skipped and - returned by `I18n::new` for the app to log; craft-i18n's CI rejects them before they ship. +- **Offered locales:** `I18n::new` takes the codes the app offers (its own language list), so an + app never offers a language its interface isn't ready for (Arabic without right-to-left layout, + say). `languages()`, `from_code`, `match_tag` and `system_lang` only return offered locales; an + embedded locale that isn't offered still serves as a fallback (`pt` for an offered `pt-br`). +- **Fallback:** locale by locale along the chain, `pt-br` → `pt` → the English source text. In + each locale, app rows override shared ones, and `tc`/`tid` try their context or `@id` row before + the plain row, so `pt-br`'s plain row beats `pt`'s `menu` row. Only a region falls back (`pt-br` + → `pt`, `zh-hant-hk` → `zh-hant`); `sr-latn` never falls back to Cyrillic `sr`. A missing + translation never errors. +- **Lookups:** `t` (plain), `tc` (with a context such as `menu` or `@msg`, else plain), `tid` (an + `@id` command id, else the English label), `trn` (a count, `@plural` rows, `{n}` filled), `get` + (one context along the locale chain, so `pt-br` can return a `pt` row, but never the plain row + or English: a key cap's name under `key`, or "is there a translation?"), and `fmt` for `{name}` + placeholders, filled in one pass. +- **Bad rows** (malformed, an unknown `@` context, a wrong plural form count, a row defined twice, + of which the first is kept) and bad locales (`en`, `pseudo`, a code embedded twice in any case) + are skipped and returned by `I18n::new` for the app to log; craft-i18n's CI rejects them before + they ship. - **Plurals:** forms are listed in CLDR order (zero, one, two, few, many, other) for the categories whole numbers 0 to 1000 reach, from `intl_pluralrules`; a count in any other category takes the - last form. Keep `intl_pluralrules` on the same version as craft-i18n's checker. + last form. CLDR's `pt` is Brazilian, so `pt` uses `pt-PT`'s rules (0 is plural) and `pt-br` + `pt`'s. `intl_pluralrules` is pinned to the version craft-i18n's checker uses, and + `plural_form_counts` holds the same counts. - **Names:** a language's name comes from its shared catalog's `@id language.name` row, else its code; English is "English". -- **Right to left:** `rtl` is true for `ar`, `fa`, `he` and `ur`; putting text into display order - is the app's job. -- **System locale:** `system_lang` matches `sys-locale`'s list (`LANG`, macOS and Windows preferred - languages, the browser's on the web): `fr_CA.UTF-8` → `fr`, `zh-TW` → `zh-hant`, `pt-PT` → - `pt-br` when that is the only Portuguese, `C`/`POSIX` → English. -- **Pseudo locale** (`pseudo`, test-only, never in `languages()`): every lookup comes back as - `⟦…⟧`, line by line, so text drawn without the marks never went through translation. +- **Right to left:** `rtl` is true for `ar`, `ckb`, `dv`, `fa`, `he`, `ps`, `sd`, `ug`, `ur` and + `yi`; putting text into display order is the app's job. +- **System locale:** `system_lang` matches the system's list once and caches it: on desktop Unix + `LANGUAGE`, `LC_ALL`, `LC_MESSAGES` and `LANG`, read raw; elsewhere `sys-locale`'s (macOS and + Windows preferred languages, the browser's on the web). `fr_CA.UTF-8` → `fr`, `zh-TW` → + `zh-hant`, `pt-PT` → `pt-br` when that is the only Portuguese, `C`/`POSIX` → English. A script + is never crossed: `sr_RS@latin` and `sr-Latn-RS` pick `sr-latn`, never `sr`, while `sr-Cyrl-RS` + picks `sr`. +- **Pseudo locale** (`pseudo`, debug builds only, never in `languages()`): every lookup, `get` + included, comes back as `⟦…⟧`, line by line, so text drawn without the marks never went through + translation. `looked_up()` lists the source texts looked up so far, for an audit against a + complete catalog. diff --git a/crates/i18n/src/catalog.rs b/crates/i18n/src/catalog.rs index 43d0d733..50d731b2 100644 --- a/crates/i18n/src/catalog.rs +++ b/crates/i18n/src/catalog.rs @@ -2,6 +2,8 @@ //! applies them strictly, and here a row that breaks one is skipped and reported, so an app built //! against a newer catalog still runs, with that string in English. +use std::collections::HashSet; + /// A row that passed, with its columns unescaped. pub(crate) struct Entry { pub context: String, @@ -13,17 +15,21 @@ pub(crate) struct Entry { const RESERVED: &[&str] = &["@plural", "@id", "@msg"]; /// The rows of a catalog whose locale has `forms` plural forms, and one problem (`line N: …`) per -/// row skipped. +/// row skipped. Of a row defined twice, the first is kept, as in craft-i18n's checker. pub(crate) fn parse(text: &str, forms: usize) -> (Vec, Vec) { let text = text.strip_prefix('\u{feff}').unwrap_or(text); let mut rows = Vec::new(); let mut problems = Vec::new(); + let mut seen = HashSet::new(); for (i, line) in text.split('\n').enumerate() { let line = line.strip_suffix('\r').unwrap_or(line); if line.is_empty() || line.starts_with('#') { continue; } match parse_row(line, forms) { + Ok(row) if !seen.insert((row.context.clone(), row.source.clone())) => { + problems.push(format!("line {}: defined twice (same context and source text); the first is kept", i + 1)); + } Ok(row) => rows.push(row), Err(e) => problems.push(format!("line {}: {e}", i + 1)), } diff --git a/crates/i18n/src/lib.rs b/crates/i18n/src/lib.rs index 4387aa4f..f44bb5a2 100644 --- a/crates/i18n/src/lib.rs +++ b/crates/i18n/src/lib.rs @@ -7,8 +7,8 @@ mod catalog; -use std::collections::HashMap; -use std::sync::{Mutex, PoisonError}; +use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::sync::{Mutex, OnceLock, PoisonError}; use intl_pluralrules::{PluralCategory, PluralRuleType, PluralRules}; use unic_langid::LanguageIdentifier; @@ -32,9 +32,12 @@ impl Lang { struct Locale { code: String, rows: HashMap>, - /// Itself, then its base language if that is embedded too (`pt-br`, `pt`). Empty for English - /// and the pseudo locale. + /// Itself, then its parent without the region if that is embedded too (`pt-br`, `pt`; + /// `zh-hant-hk`, `zh-hant`). Empty for English and the pseudo locale. chain: Vec, + /// The app offers it: in [`I18n::languages`], found by code and system tag. An embedded locale + /// the app doesn't offer still serves as a fallback. + offered: bool, /// Plural rules and the categories `@plural` forms list, in order. plural: Option<(PluralRules, Vec)>, } @@ -42,18 +45,29 @@ struct Locale { /// The embedded catalogs, ready for lookups. Build it once, at start-up. pub struct I18n { locales: Vec, - /// Marked strings for the pseudo locale, leaked so lookups can hand out `&str`. Test-only. - pseudo: Mutex>, + /// Marked strings for the pseudo locale by source text, leaked so lookups can hand out `&str`. + /// Debug builds only; the keys are what [`I18n::looked_up`] reports. + // ponytail: grows by one leak per distinct source text, fine for a test run; an arena if the + // pseudo locale ever ships. + pseudo: Mutex>, + /// [`I18n::system_lang`], read once. + system: OnceLock, } impl I18n { - /// Read `embedded`. Rows that can't be used are skipped and come back as problems + /// Read `embedded`, offering the locales in `offered` (any case; the app's own language list), + /// so an app never offers a language its interface isn't ready for (right to left, say). Rows + /// and locales that can't be used are skipped and come back as problems /// (`/: line N: …`) for the app to log. - pub fn new(embedded: &'static Embedded) -> (I18n, Vec) { + pub fn new(embedded: &'static Embedded, offered: &[&str]) -> (I18n, Vec) { let mut problems = Vec::new(); let mut locales = vec![bare("en"), bare("pseudo")]; for (code, shared, app) in embedded { let code = code.to_ascii_lowercase(); + if locales.iter().any(|l| l.code == code) { + problems.push(format!("{code}: reserved or embedded twice; skipped")); + continue; + } let plural = plural_rules(&code); if plural.is_none() { problems.push(format!("{code}: no plural rules; plurals show in English")); @@ -67,15 +81,16 @@ impl I18n { rows.entry(e.context).or_default().insert(e.source, e.translation); } } - locales.push(Locale { code, rows, chain: Vec::new(), plural }); + let offered = offered.iter().any(|o| o.eq_ignore_ascii_case(&code)); + locales.push(Locale { code, rows, chain: Vec::new(), plural, offered }); } for i in 2..locales.len() { let mut chain = vec![i]; - // Only a region falls back to its language (`pt-br` → `pt`); a script doesn't, since - // `sr-latn` → `sr` would show Cyrillic. + // Only a region falls back (`pt-br` → `pt`, `zh-hant-hk` → `zh-hant`); a script + // doesn't, since `sr-latn` → `sr` would show Cyrillic. let code = locales.get(i).map_or("", |l| l.code.as_str()); - let base = match code.split_once('-') { - Some((language, region)) if is_region(region) => language, + let base = match code.rsplit_once('-') { + Some((parent, region)) if is_region(region) => parent, _ => "", }; if let Some(b) = locales.iter().skip(2).position(|l| l.code == base) @@ -87,12 +102,19 @@ impl I18n { l.chain = chain; } } - (I18n { locales, pseudo: Mutex::default() }, problems) + (I18n { locales, pseudo: Mutex::default(), system: OnceLock::new() }, problems) } - /// The language with this code (any case): `en`, `pseudo`, or an embedded locale. + /// The language with this code (any case): `en`, an offered locale, or in debug builds + /// `pseudo`. pub fn from_code(&self, code: &str) -> Option { - self.locales.iter().position(|l| l.code.eq_ignore_ascii_case(code)).map(Lang) + let i = self.locales.iter().position(|l| l.code.eq_ignore_ascii_case(code))?; + let usable = match Lang(i) { + Lang::EN => true, + Lang::PSEUDO => cfg!(debug_assertions), + _ => self.locales.get(i).is_some_and(|l| l.offered), + }; + usable.then_some(Lang(i)) } /// The language's code, lowercase BCP 47 (`pt-br`). @@ -105,30 +127,35 @@ impl I18n { if lang == Lang::PSEUDO { return self.mark(source); } - self.find(lang, "", source).unwrap_or(source) + self.find(lang, &[("", source)]).unwrap_or(source) } - /// `source` with a disambiguating `context` (`menu`, `@msg`); falls back to [`I18n::t`]. + /// `source` with a disambiguating `context` (`menu`, `@msg`); in each locale of the chain the + /// context row, then the plain one, so `pt-br`'s plain row beats `pt`'s context row. pub fn tc<'a>(&'a self, lang: Lang, context: &str, source: &'a str) -> &'a str { if lang == Lang::PSEUDO { return self.mark(source); } - self.find(lang, context, source).unwrap_or_else(|| self.t(lang, source)) + self.find(lang, &[(context, source), ("", source)]).unwrap_or(source) } /// The translation of (`context`, `source`) along `lang`'s chain, with no fallback to the plain /// row or the English: for strings that must not borrow another meaning's wording (a key cap's /// name under context `key`), or to ask whether a translation exists. - pub fn get(&self, lang: Lang, context: &str, source: &str) -> Option<&str> { - self.find(lang, context, source) + pub fn get<'a>(&'a self, lang: Lang, context: &str, source: &'a str) -> Option<&'a str> { + if lang == Lang::PSEUDO { + return Some(self.mark(source)); + } + self.find(lang, &[(context, source)]) } - /// A command's label by its `id` (`@id` rows); falls back to [`I18n::t`] of the English `label`. + /// A command's label by its `id` (`@id` rows); in each locale of the chain the `@id` row, then + /// the plain row of the English `label`. pub fn tid<'a>(&'a self, lang: Lang, id: &str, label: &'a str) -> &'a str { if lang == Lang::PSEUDO { return self.mark(label); } - self.find(lang, "@id", id).unwrap_or_else(|| self.t(lang, label)) + self.find(lang, &[("@id", id), ("", label)]).unwrap_or(label) } /// A count's wording in `lang`, `{n}` filled in. `one` and `other` are the English forms; @@ -155,10 +182,10 @@ impl I18n { text } - /// The languages to offer: English first, then the embedded locales by code. Never the + /// The languages to offer: English first, then the offered locales by code. Never the /// pseudo locale. pub fn languages(&self) -> Vec { - let mut embedded: Vec = (2..self.locales.len()).collect(); + let mut embedded: Vec = (2..self.locales.len()).filter(|&i| self.locales.get(i).is_some_and(|l| l.offered)).collect(); embedded.sort_by_key(|&i| self.locales.get(i).map(|l| l.code.as_str())); std::iter::once(Lang::EN).chain(embedded.into_iter().map(Lang)).collect() } @@ -177,7 +204,7 @@ impl I18n { pub fn rtl(&self, lang: Lang) -> bool { let code = self.code(lang); let language = code.split('-').next().unwrap_or(code); - matches!(language, "ar" | "fa" | "he" | "ur") + matches!(language, "ar" | "ckb" | "dv" | "fa" | "he" | "ps" | "sd" | "ug" | "ur" | "yi") } /// The language for a system locale tag (`fr_CA.UTF-8`, `zh-Hant-MO`); `C` and `POSIX` are @@ -196,14 +223,22 @@ impl I18n { } /// The language the system asks for (`LANG` and friends, the macOS and Windows preferred - /// languages, the browser's on the web); English if none fits. + /// languages, the browser's on the web); English if none fits. Read once, then cached. pub fn system_lang(&self) -> Lang { - // sys-locale drops glibc's script modifier (`sr_RS@latin` → `sr-RS`), so read its variables raw. - #[cfg(all(unix, not(any(target_vendor = "apple", target_os = "android"))))] - let tags = posix_env_tags(|k| std::env::var(k).ok()); - #[cfg(not(all(unix, not(any(target_vendor = "apple", target_os = "android")))))] - let tags: Vec = sys_locale::get_locales().collect(); - self.match_tags(tags.iter().map(String::as_str)) + *self.system.get_or_init(|| { + // sys-locale drops glibc's script modifier (`sr_RS@latin` → `sr-RS`), so read its variables raw. + #[cfg(all(unix, not(any(target_vendor = "apple", target_os = "android"))))] + let tags = posix_env_tags(|k| std::env::var(k).ok()); + #[cfg(not(all(unix, not(any(target_vendor = "apple", target_os = "android")))))] + let tags: Vec = sys_locale::get_locales().collect(); + self.match_tags(tags.iter().map(String::as_str)) + }) + } + + /// Every source text looked up in the pseudo locale so far, for an audit to check against a + /// complete catalog. Plurals aren't recorded. + pub fn looked_up(&self) -> BTreeSet { + self.pseudo.lock().unwrap_or_else(PoisonError::into_inner).keys().cloned().collect() } /// `source` marked for the pseudo locale, made once per string. @@ -216,10 +251,13 @@ impl I18n { self.locales.get(lang.0) } - /// The first translation of (`context`, `source`) along `lang`'s chain. - fn find(&self, lang: Lang, context: &str, source: &str) -> Option<&str> { + /// Along `lang`'s chain, locale by locale, the first of `keys` (context, source text) found. + fn find(&self, lang: Lang, keys: &[(&str, &str)]) -> Option<&str> { let chain = self.locale(lang).map(|l| l.chain.as_slice())?; - chain.iter().find_map(|&i| self.locales.get(i)?.rows.get(context)?.get(source).map(String::as_str)) + chain.iter().find_map(|&i| { + let rows = &self.locales.get(i)?.rows; + keys.iter().find_map(|(context, source)| rows.get(*context)?.get(*source).map(String::as_str)) + }) } } @@ -251,8 +289,9 @@ pub fn fmt(template: &str, args: &[(&str, &str)]) -> String { /// Codes to try for a locale tag, most specific first: `zh_TW.UTF-8` → `zh-tw`, `zh-hant`, `zh`. /// Chinese without a script picks one by region; Portuguese falls back to `pt-br` last, when -/// there is no `pt`. A glibc script modifier (`sr_RS@latin`) becomes the script subtag, and only -/// codes with that script are tried, so Latin Serbian never picks Cyrillic `sr`. +/// there is no `pt`. A glibc script modifier (`sr_RS@latin`) becomes the script subtag. With a +/// script (`sr-Latn-RS`), a code without one is tried only when that is its language's usual +/// script, so Latin Serbian never picks Cyrillic `sr`, while `sr-Cyrl-RS` does. fn candidates(tag: &str) -> Vec { let tag = tag.trim().to_ascii_lowercase(); let base = tag.split(['.', '@']).next().unwrap_or("").replace('_', "-"); @@ -260,16 +299,19 @@ fn candidates(tag: &str) -> Vec { let Some(&primary) = parts.first() else { return Vec::new(); }; - let script = match tag.split_once('@').map(|(_, m)| m) { + let modifier = match tag.split_once('@').map(|(_, m)| m) { Some("latin") => Some("latn"), Some("cyrillic") => Some("cyrl"), _ => None, }; - if let Some(s) = script { + if let Some(s) = modifier { parts.insert(1, s); } + let script = parts.get(1).copied().filter(|s| s.len() == 4 && s.bytes().all(|b| b.is_ascii_alphabetic())); let mut out: Vec = (1..=parts.len()).rev().map(|n| parts.get(..n).unwrap_or_default().join("-")).collect(); - if let Some(s) = script { + if let Some(s) = script + && usual_script(primary).is_some_and(|usual| usual != s) + { out.retain(|c| c.split('-').any(|p| p == s)); } if primary == "zh" && !parts.iter().any(|p| matches!(*p, "hans" | "hant")) { @@ -297,6 +339,20 @@ fn marked(text: &str) -> String { format!("⟦{}⟧", text.replace('\n', "⟧\n⟦")) } +/// The script a code without one means, for languages written in more than one (CLDR likely +/// subtags). Any other language is taken to have one script. +// ponytail: the multi-script languages a Craft app could plausibly ship; add a row with the locale. +fn usual_script(language: &str) -> Option<&'static str> { + Some(match language { + "be" | "mn" | "sr" => "cyrl", + "zh" => "hans", + "az" | "bs" | "ha" | "uz" => "latn", + "ks" | "sd" => "arab", + "pa" => "guru", + _ => return None, + }) +} + /// A BCP 47 region subtag: two letters or three digits. fn is_region(subtag: &str) -> bool { (subtag.len() == 2 && subtag.bytes().all(|b| b.is_ascii_alphabetic())) || (subtag.len() == 3 && subtag.bytes().all(|b| b.is_ascii_digit())) @@ -304,13 +360,15 @@ fn is_region(subtag: &str) -> bool { /// A locale with no catalogs (English, the pseudo locale). fn bare(code: &str) -> Locale { - Locale { code: code.to_owned(), rows: HashMap::new(), chain: Vec::new(), plural: None } + Locale { code: code.to_owned(), rows: HashMap::new(), chain: Vec::new(), plural: None, offered: false } } /// `code`'s CLDR cardinal rules and the categories whole numbers 0..=1000 reach, in CLDR order /// (zero, one, two, few, many, other): one `@plural` form each. Regional and scripted codes fall -/// back to the language's rules. +/// back to the language's rules. CLDR's `pt` is Brazilian (0 is singular), so Portuguese other +/// than `pt-br` takes `pt-PT`'s. fn plural_rules(code: &str) -> Option<(PluralRules, Vec)> { + let code = if code == "pt" || (code.starts_with("pt-") && code != "pt-br") { "pt-pt" } else { code }; let id: LanguageIdentifier = code.parse().ok()?; let language = LanguageIdentifier::from_parts(id.language, None, None, &[]); let rules = PluralRules::create(id, PluralRuleType::CARDINAL).or_else(|_| PluralRules::create(language, PluralRuleType::CARDINAL)).ok()?; @@ -347,8 +405,14 @@ mod tests { ("de", "\tBad\n@nope\tA\tB\n\tUndo\tRückgängig\n", ""), ]; + /// `embedded`, offering every locale in it. + fn all(embedded: &'static Embedded) -> (I18n, Vec) { + let codes: Vec<&str> = embedded.iter().map(|(c, _, _)| *c).collect(); + I18n::new(embedded, &codes) + } + fn i18n() -> I18n { - I18n::new(E).0 + all(E).0 } fn lang(i: &I18n, code: &str) -> Lang { @@ -397,7 +461,7 @@ mod tests { #[test] fn bad_rows_skipped_and_reported() { - let (i, problems) = I18n::new(E); + let (i, problems) = all(E); assert_eq!(problems.len(), 2, "{problems:?}"); assert!(problems[0].starts_with("de/common: line 1:"), "{problems:?}"); assert!(problems[1].starts_with("de/common: line 2:"), "{problems:?}"); @@ -427,7 +491,7 @@ mod tests { #[test] fn one_other() { - let (i, problems) = I18n::new(P); + let (i, problems) = all(P); assert!(problems.is_empty(), "{problems:?}"); let fr = lang(&i, "fr"); assert_eq!(i.trn(fr, 0, "{n} page", "{n} pages"), "0 page"); @@ -436,7 +500,7 @@ mod tests { #[test] fn three_forms_ru() { - let i = I18n::new(P).0; + let i = all(P).0; let ru = lang(&i, "ru"); for (n, want) in [(1, "1 файл"), (3, "3 файла"), (5, "5 файлов"), (11, "11 файлов"), (21, "21 файл")] { assert_eq!(i.trn(ru, n, "{n} file", "{n} files"), want); @@ -445,20 +509,20 @@ mod tests { #[test] fn one_form_ja() { - let i = I18n::new(P).0; + let i = all(P).0; assert_eq!(i.trn(lang(&i, "ja"), 7, "{n} file", "{n} files"), "7 個"); } #[test] fn english_fallback() { - let i = I18n::new(P).0; + let i = all(P).0; assert_eq!(i.trn(Lang::EN, 1, "{n} file", "{n} files"), "1 file"); assert_eq!(i.trn(lang(&i, "fr"), 2, "{n} x", "{n} xs"), "2 xs"); } #[test] fn beyond_sweep() { - let i = I18n::new(P).0; + let i = all(P).0; assert_eq!(i.trn(lang(&i, "fr"), 1_000_000, "{n} page", "{n} pages"), "1000000 pages"); } @@ -467,14 +531,14 @@ mod tests { #[test] fn languages_order() { - let i = I18n::new(L).0; + let i = all(L).0; let codes: Vec<&str> = i.languages().into_iter().map(|l| i.code(l)).collect(); assert_eq!(codes, ["en", "ar", "fr", "pt-br", "zh-hans", "zh-hant"]); } #[test] fn names() { - let i = I18n::new(L).0; + let i = all(L).0; assert_eq!(i.name(Lang::EN), "English"); assert_eq!(i.name(lang(&i, "fr")), "Français"); assert_eq!(i.name(lang(&i, "ar")), "ar"); @@ -482,7 +546,7 @@ mod tests { #[test] fn rtl() { - let i = I18n::new(L).0; + let i = all(L).0; assert!(i.rtl(lang(&i, "ar"))); assert!(!i.rtl(lang(&i, "fr"))); assert!(!i.rtl(Lang::EN)); @@ -490,7 +554,7 @@ mod tests { #[test] fn tags() { - let i = I18n::new(L).0; + let i = all(L).0; for (tag, want) in [ ("fr_CA.UTF-8", Some("fr")), ("pt-PT", Some("pt-br")), @@ -513,7 +577,7 @@ mod tests { #[test] fn match_tags_takes_first_match() { - let i = I18n::new(L).0; + let i = all(L).0; assert_eq!(i.match_tags(["ja_JP", "fr_FR"]), lang(&i, "fr")); assert_eq!(i.match_tags(["ja"]), Lang::EN); assert_eq!(i.match_tags([]), Lang::EN); @@ -528,7 +592,9 @@ mod tests { assert_eq!(i.tid(Lang::PSEUDO, "file.save", "Save"), "⟦Save⟧"); assert_eq!(i.trn(Lang::PSEUDO, 2, "{n} x", "{n} xs"), "⟦2 xs⟧"); assert!(!i.languages().contains(&Lang::PSEUDO)); - assert_eq!(i.from_code("pseudo"), Some(Lang::PSEUDO)); + assert_eq!(i.from_code("pseudo"), cfg!(debug_assertions).then_some(Lang::PSEUDO)); + assert_eq!(i.get(Lang::PSEUDO, "key", "Esc"), Some("⟦Esc⟧")); + assert_eq!(i.looked_up().into_iter().collect::>(), ["A\nB", "Esc", "Open", "Save"]); } static F: &Embedded = &[ @@ -541,7 +607,7 @@ mod tests { #[test] fn european_portuguese_prefers_pt() { - let i = I18n::new(F).0; + let i = all(F).0; for tag in ["pt", "pt-PT", "pt_PT.UTF-8"] { assert_eq!(i.match_tag(tag).map(|l| i.code(l)), Some("pt"), "{tag}"); } @@ -550,17 +616,17 @@ mod tests { #[test] fn script_locale_does_not_fall_back_across_scripts() { - let i = I18n::new(F).0; + let i = all(F).0; assert_eq!(i.t(lang(&i, "sr-latn"), "Save"), "Save"); } #[test] fn glibc_script_modifier_picks_the_script() { - let i = I18n::new(F).0; + let i = all(F).0; for (tag, want) in [("sr_RS.UTF-8@latin", Some("sr-latn")), ("sr_RS@latin", Some("sr-latn")), ("sr_RS.UTF-8", Some("sr"))] { assert_eq!(i.match_tag(tag).map(|l| i.code(l)), want, "{tag}"); } - let cyrillic_only = I18n::new(&[("sr", "", "")]).0; + let cyrillic_only = all(&[("sr", "", "")]).0; assert_eq!(cyrillic_only.match_tag("sr_RS@latin"), None); } @@ -570,17 +636,107 @@ mod tests { let env = |vars: &'static [(&'static str, &'static str)]| move |k: &str| vars.iter().find(|(n, _)| *n == k).map(|(_, v)| (*v).to_owned()); assert_eq!(posix_env_tags(env(&[("LANG", "sr_RS.UTF-8@latin")])), ["sr_RS.UTF-8@latin"]); assert_eq!(posix_env_tags(env(&[("LANGUAGE", "fr:de"), ("LC_ALL", ""), ("LANG", "sr_RS@latin")])), ["fr", "de", "sr_RS@latin"]); - let i = I18n::new(F).0; + let i = all(F).0; assert_eq!(i.code(i.match_tags(posix_env_tags(env(&[("LANG", "sr_RS.UTF-8@latin")])).iter().map(String::as_str))), "sr-latn"); } #[test] fn get_has_no_plain_fallback() { - let i = I18n::new(F).0; + let i = all(F).0; let fr = lang(&i, "fr"); assert_eq!(i.get(fr, "key", "Esc"), Some("Échap")); assert_eq!(i.get(fr, "key", "Delete"), None); assert_eq!(i.get(fr, "", "Delete"), Some("Supprimer")); assert_eq!(i.get(Lang::EN, "", "Delete"), None); } + + static C: &Embedded = &[("pt", "menu\tOpen\tAbrir menu\n@id\tfile.open\tAbrir ficheiro\n", ""), ("pt-br", "\tOpen\tAbrir\n", "")]; + + #[test] + fn locale_before_context() { + let i = all(C).0; + let br = lang(&i, "pt-br"); + assert_eq!(i.tc(br, "menu", "Open"), "Abrir"); + assert_eq!(i.tid(br, "file.open", "Open"), "Abrir"); + assert_eq!(i.tc(lang(&i, "pt"), "menu", "Open"), "Abrir menu"); + } + + static PT: &Embedded = &[ + ("pt", "@plural\t{n} file|{n} files\t{n} ficheiro|{n} ficheiros\n", ""), + ("pt-br", "@plural\t{n} file|{n} files\t{n} arquivo|{n} arquivos\n", ""), + ]; + + #[test] + fn european_portuguese_zero_is_plural() { + let i = all(PT).0; + assert_eq!(i.trn(lang(&i, "pt"), 0, "{n} file", "{n} files"), "0 ficheiros"); + assert_eq!(i.trn(lang(&i, "pt"), 1, "{n} file", "{n} files"), "1 ficheiro"); + assert_eq!(i.trn(lang(&i, "pt-br"), 0, "{n} file", "{n} files"), "0 arquivo"); + } + + /// The same counts as craft-i18n's checker (`plural_form_counts`), which rejects a catalog + /// whose `@plural` rows list any other. + #[test] + fn plural_form_counts() { + for (locale, n) in + [("en", 2), ("fr", 2), ("ja", 1), ("vi", 1), ("ru", 3), ("pl", 3), ("cs", 3), ("sl", 4), ("ar", 6), ("pt", 2), ("pt-br", 2), ("tr", 2)] + { + assert_eq!(plural_rules(locale).map(|(_, c)| c.len()), Some(n), "{locale}"); + } + } + + #[test] + fn script_subtag_never_picks_another_script() { + let i = all(&[("sr", "", ""), ("zh-hant", "", ""), ("fr", "", ""), ("be", "", "")]).0; + for (tag, want) in [ + ("sr-Latn-RS", None), + ("sr-Cyrl-RS", Some("sr")), + ("zh-Hant-TW", Some("zh-hant")), + ("zh-Hans", None), + ("fr-Latn-FR", Some("fr")), + ("be_BY@latin", None), + ("be_BY", Some("be")), + ] { + assert_eq!(i.match_tag(tag).map(|l| i.code(l)), want, "{tag}"); + } + } + + #[test] + fn duplicate_row_keeps_the_first() { + let (i, problems) = all(&[("fr", "\tSave\tEnregistrer\n\tSave\tSauver\n", "")]); + assert_eq!(problems, ["fr/common: line 2: defined twice (same context and source text); the first is kept"]); + assert_eq!(i.t(lang(&i, "fr"), "Save"), "Enregistrer"); + } + + #[test] + fn reserved_and_repeated_codes_are_skipped() { + let (i, problems) = all(&[("en", "\tSave\tX\n", ""), ("pseudo", "", ""), ("fr", "\tSave\tEnregistrer\n", ""), ("FR", "\tSave\tY\n", "")]); + assert_eq!(problems.len(), 3, "{problems:?}"); + assert_eq!(i.t(Lang::EN, "Save"), "Save"); + assert_eq!(i.t(lang(&i, "fr"), "Save"), "Enregistrer"); + assert_eq!(i.languages().len(), 2); + } + + #[test] + fn only_offered_locales_are_offered() { + let (i, _) = I18n::new(&[("ar", "\tSave\tحفظ\n", ""), ("pt", "\tUndo\tDesfazer\n", ""), ("pt-br", "", "")], &["PT-BR"]); + let codes: Vec<&str> = i.languages().into_iter().map(|l| i.code(l)).collect(); + assert_eq!(codes, ["en", "pt-br"]); + assert_eq!(i.from_code("ar"), None); + assert_eq!(i.match_tag("ar_EG"), None); + assert_eq!(i.match_tag("pt_PT"), i.from_code("pt-br")); + assert_eq!(i.t(lang(&i, "pt-br"), "Undo"), "Desfazer"); + } + + #[test] + fn region_after_script_falls_back() { + let i = all(&[("zh-hant", "\tSave\t儲存\n", ""), ("zh-hant-hk", "", "")]).0; + assert_eq!(i.t(lang(&i, "zh-hant-hk"), "Save"), "儲存"); + } + + #[test] + fn more_right_to_left() { + let i = all(&[("ps", "", ""), ("ckb", "", ""), ("yi", "", ""), ("ug", "", "")]).0; + assert!(i.languages().into_iter().skip(1).all(|l| i.rtl(l))); + } } From bc9d71b27a3788eca364f0005c83841422bc42f4 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Sun, 18 Oct 2026 19:12:44 +0100 Subject: [PATCH 10/10] i18n: test prints system_lang per platform for CI logs Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/i18n/src/lib.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/crates/i18n/src/lib.rs b/crates/i18n/src/lib.rs index f44bb5a2..7fa7a4ef 100644 --- a/crates/i18n/src/lib.rs +++ b/crates/i18n/src/lib.rs @@ -739,4 +739,13 @@ mod tests { let i = all(&[("ps", "", ""), ("ckb", "", ""), ("yi", "", ""), ("ug", "", "")]).0; assert!(i.languages().into_iter().skip(1).all(|l| i.rtl(l))); } + + /// Prints what this machine reports, so CI logs show `system_lang` on each platform. + #[test] + fn system_lang_reads_the_system() { + let i = all(&[("fr", "", ""), ("de", "", ""), ("pt-br", "", ""), ("sr-latn", "", "")]).0; + let first = i.system_lang(); + println!("sys-locale: {:?}; system_lang: {}", sys_locale::get_locales().collect::>(), i.code(first)); + assert_eq!(i.system_lang(), first); + } }