From 7b797335cb25ae8e7b384c30949bb16cd95ad9ce Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Sat, 17 Oct 2026 00:40:17 +0100 Subject: [PATCH] i18n: catalog parsing and lookup with fallback Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01F9fM3mv1PUyKpk5rdwUQrW --- Cargo.lock | 37 ++++++ Cargo.toml | 1 + crates/i18n/Cargo.toml | 15 +++ crates/i18n/src/catalog.rs | 114 ++++++++++++++++++ crates/i18n/src/lib.rs | 231 +++++++++++++++++++++++++++++++++++++ 5 files changed, 398 insertions(+) create mode 100644 crates/i18n/Cargo.toml create mode 100644 crates/i18n/src/catalog.rs create mode 100644 crates/i18n/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 676a5220..55a956a2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -609,6 +609,14 @@ dependencies = [ name = "craft-docpath" version = "0.1.0" +[[package]] +name = "craft-i18n" +version = "0.1.0" +dependencies = [ + "intl_pluralrules", + "unic-langid", +] + [[package]] name = "craft-pdf" version = "0.1.0" @@ -1934,6 +1942,15 @@ dependencies = [ "hashbrown 0.17.1", ] +[[package]] +name = "intl_pluralrules" +version = "7.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "078ea7b7c29a2b4df841a7f6ac8775ff6074020c6776d48491ce2268e068f972" +dependencies = [ + "unic-langid", +] + [[package]] name = "ipnet" version = "2.12.2" @@ -4346,6 +4363,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" dependencies = [ "displaydoc", + "serde_core", "zerovec", ] @@ -4573,6 +4591,24 @@ version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" +[[package]] +name = "unic-langid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28ba52c9b05311f4f6e62d5d9d46f094bd6e84cb8df7b3ef952748d752a7d05" +dependencies = [ + "unic-langid-impl", +] + +[[package]] +name = "unic-langid-impl" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce1bf08044d4b7a94028c93786f8566047edc11110595914de93362559bc658" +dependencies = [ + "tinystr", +] + [[package]] name = "unicode-bidi" version = "0.3.18" @@ -5826,6 +5862,7 @@ version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" dependencies = [ + "serde", "yoke", "zerofrom", "zerovec-derive", diff --git a/Cargo.toml b/Cargo.toml index 336fd37b..868e568d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,6 +23,7 @@ thiserror = "2" craft-atomic = { path = "crates/atomic" } craft-cms = { path = "crates/cms" } craft-docpath = { path = "crates/docpath" } +craft-i18n = { path = "crates/i18n" } craft-print = { path = "crates/print" } craft-quit = { path = "crates/quit" } craft-settings = { path = "crates/settings" } diff --git a/crates/i18n/Cargo.toml b/crates/i18n/Cargo.toml new file mode 100644 index 00000000..f1f08217 --- /dev/null +++ b/crates/i18n/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "craft-i18n" +description = "Translations for the Craft apps: the embedded craft-i18n catalogs, lookup with fallback, plurals, language names and the system locale." +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true + +[dependencies] +intl_pluralrules = "7" +unic-langid = "0.9" + +[lints] +workspace = true diff --git a/crates/i18n/src/catalog.rs b/crates/i18n/src/catalog.rs new file mode 100644 index 00000000..43d0d733 --- /dev/null +++ b/crates/i18n/src/catalog.rs @@ -0,0 +1,114 @@ +//! Reading one catalog. The rules are craft-i18n's `docs/catalog-format.md`; craft-i18n's CI +//! applies them strictly, and here a row that breaks one is skipped and reported, so an app built +//! against a newer catalog still runs, with that string in English. + +/// A row that passed, with its columns unescaped. +pub(crate) struct Entry { + pub context: String, + pub source: String, + pub translation: String, +} + +/// The contexts with a meaning here. Any other context starting with `@` is skipped. +const RESERVED: &[&str] = &["@plural", "@id", "@msg"]; + +/// The rows of a catalog whose locale has `forms` plural forms, and one problem (`line N: …`) per +/// row skipped. +pub(crate) fn parse(text: &str, forms: usize) -> (Vec, Vec) { + let text = text.strip_prefix('\u{feff}').unwrap_or(text); + let mut rows = Vec::new(); + let mut problems = Vec::new(); + for (i, line) in text.split('\n').enumerate() { + let line = line.strip_suffix('\r').unwrap_or(line); + if line.is_empty() || line.starts_with('#') { + continue; + } + match parse_row(line, forms) { + Ok(row) => rows.push(row), + Err(e) => problems.push(format!("line {}: {e}", i + 1)), + } + } + (rows, problems) +} + +fn parse_row(text: &str, forms: usize) -> Result { + let columns: Vec<&str> = text.split('\t').collect(); + let [context, source, translation] = columns.as_slice() else { + return Err(format!("{} columns; expected 3", columns.len())); + }; + if text.chars().any(|c| c != '\t' && (c.is_control() || c == '\u{feff}')) { + return Err("invisible character in the row".into()); + } + let row = Entry { context: unescape(context)?, source: unescape(source)?, translation: unescape(translation)? }; + if row.source.is_empty() || row.translation.is_empty() { + return Err("empty source text or translation".into()); + } + match row.context.as_str() { + "@id" => {} + "@plural" => check_plural(&row.source, &row.translation, forms)?, + c if c.starts_with('@') && !RESERVED.contains(&c) => { + return Err(format!("unknown reserved context {c:?}")); + } + _ => { + if placeholders(&row.source) != placeholders(&row.translation) { + return Err("placeholders differ from the source text".into()); + } + if row.source.ends_with('…') != row.translation.ends_with('…') { + return Err("trailing … differs from the source text".into()); + } + } + } + Ok(row) +} + +fn check_plural(source: &str, translation: &str, forms: usize) -> Result<(), String> { + let english: Vec<&str> = source.split('|').collect(); + let [one, other] = english.as_slice() else { + return Err("plural source text must be one|other".into()); + }; + if one.is_empty() || other.is_empty() || placeholders(one) != placeholders(other) { + return Err("plural source forms must be nonempty with the same placeholders".into()); + } + let translated: Vec<&str> = translation.split('|').collect(); + if translated.len() != forms || translated.iter().any(|f| f.is_empty()) { + return Err(format!("expected {forms} nonempty plural forms")); + } + if translated.iter().any(|f| placeholders(f) != placeholders(other)) { + return Err("plural form placeholders differ from the source text".into()); + } + Ok(()) +} + +fn unescape(text: &str) -> Result { + let mut out = String::with_capacity(text.len()); + let mut chars = text.chars(); + while let Some(c) = chars.next() { + if c != '\\' { + out.push(c); + continue; + } + match chars.next() { + Some('n') => out.push('\n'), + Some('t') => out.push('\t'), + Some('\\') => out.push('\\'), + Some(other) => return Err(format!("unknown escape \\{other}")), + None => return Err("trailing backslash".into()), + } + } + Ok(out) +} + +/// `{name}` placeholders, sorted: translations may reorder them. +fn placeholders(text: &str) -> Vec<&str> { + let mut names = Vec::new(); + let mut rest = text; + while let Some((_, after)) = rest.split_once('{') { + let Some((name, after)) = after.split_once('}') else { + break; + }; + names.push(name); + rest = after; + } + names.sort_unstable(); + names +} diff --git a/crates/i18n/src/lib.rs b/crates/i18n/src/lib.rs new file mode 100644 index 00000000..bb324595 --- /dev/null +++ b/crates/i18n/src/lib.rs @@ -0,0 +1,231 @@ +//! Translations for the Craft apps. +//! +//! An app embeds the [craft-i18n](https://github.com/benandez/craft-i18n) catalogs at build time +//! as an [`Embedded`] static, builds one [`I18n`] from it, and looks English source text up in a +//! [`Lang`]. A missing translation is the English itself; nothing here fails. +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] + +mod catalog; + +use std::collections::HashMap; + +use intl_pluralrules::{PluralCategory, PluralRuleType, PluralRules}; +use unic_langid::LanguageIdentifier; + +/// The catalogs an app embeds, one entry per locale: (locale, shared catalog, the app's catalog). +/// A missing catalog is `""`. +pub type Embedded = [(&'static str, &'static str, &'static str)]; + +/// A language the interface can be shown in: English, the pseudo locale, or an embedded locale. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct Lang(usize); + +impl Lang { + /// English: the source text itself. + pub const EN: Lang = Lang(0); + /// A test-only locale that marks every looked-up string. + pub const PSEUDO: Lang = Lang(1); +} + +/// One locale's translations: context → source text → translation, app rows over shared ones. +struct Locale { + code: String, + rows: HashMap>, + /// Itself, then its base language if that is embedded too (`pt-br`, `pt`). Empty for English + /// and the pseudo locale. + chain: Vec, + /// Plural rules and the categories `@plural` forms list, in order. + #[expect(dead_code, reason = "read by plural lookups")] + plural: Option<(PluralRules, Vec)>, +} + +/// The embedded catalogs, ready for lookups. Build it once, at start-up. +pub struct I18n { + locales: Vec, +} + +impl I18n { + /// Read `embedded`. Rows that can't be used are skipped and come back as problems + /// (`/: line N: …`) for the app to log. + pub fn new(embedded: &'static Embedded) -> (I18n, Vec) { + let mut problems = Vec::new(); + let mut locales = vec![bare("en"), bare("pseudo")]; + for (code, shared, app) in embedded { + let code = code.to_ascii_lowercase(); + let plural = plural_rules(&code); + if plural.is_none() { + problems.push(format!("{code}: no plural rules; plurals show in English")); + } + let forms = plural.as_ref().map_or(1, |(_, c)| c.len()); + let mut rows: HashMap> = HashMap::new(); + for (name, text) in [("common", shared), ("app", app)] { + let (entries, errors) = catalog::parse(text, forms); + problems.extend(errors.into_iter().map(|e| format!("{code}/{name}: {e}"))); + for e in entries { + rows.entry(e.context).or_default().insert(e.source, e.translation); + } + } + locales.push(Locale { code, rows, chain: Vec::new(), plural }); + } + for i in 2..locales.len() { + let mut chain = vec![i]; + let base = locales.get(i).and_then(|l| l.code.split('-').next()).unwrap_or_default(); + if let Some(b) = locales.iter().skip(2).position(|l| l.code == base) + && b + 2 != i + { + chain.push(b + 2); + } + if let Some(l) = locales.get_mut(i) { + l.chain = chain; + } + } + (I18n { locales }, problems) + } + + /// The language with this code (any case): `en`, `pseudo`, or an embedded locale. + pub fn from_code(&self, code: &str) -> Option { + self.locales.iter().position(|l| l.code.eq_ignore_ascii_case(code)).map(Lang) + } + + /// The language's code, lowercase BCP 47 (`pt-br`). + pub fn code(&self, lang: Lang) -> &str { + self.locale(lang).map_or("en", |l| l.code.as_str()) + } + + /// `source` in `lang`, or `source` itself. + pub fn t<'a>(&'a self, lang: Lang, source: &'a str) -> &'a str { + self.find(lang, "", source).unwrap_or(source) + } + + /// `source` with a disambiguating `context` (`menu`, `@msg`); falls back to [`I18n::t`]. + pub fn tc<'a>(&'a self, lang: Lang, context: &str, source: &'a str) -> &'a str { + self.find(lang, context, source).unwrap_or_else(|| self.t(lang, source)) + } + + /// A command's label by its `id` (`@id` rows); falls back to [`I18n::t`] of the English `label`. + pub fn tid<'a>(&'a self, lang: Lang, id: &str, label: &'a str) -> &'a str { + self.find(lang, "@id", id).unwrap_or_else(|| self.t(lang, label)) + } + + fn locale(&self, lang: Lang) -> Option<&Locale> { + self.locales.get(lang.0) + } + + /// The first translation of (`context`, `source`) along `lang`'s chain. + fn find(&self, lang: Lang, context: &str, source: &str) -> Option<&str> { + let chain = self.locale(lang).map(|l| l.chain.as_slice())?; + chain.iter().find_map(|&i| self.locales.get(i)?.rows.get(context)?.get(source).map(String::as_str)) + } +} + +/// A locale with no catalogs (English, the pseudo locale). +fn bare(code: &str) -> Locale { + Locale { code: code.to_owned(), rows: HashMap::new(), chain: Vec::new(), plural: None } +} + +/// `code`'s CLDR cardinal rules and the categories whole numbers 0..=1000 reach, in CLDR order +/// (zero, one, two, few, many, other): one `@plural` form each. Regional and scripted codes fall +/// back to the language's rules. +fn plural_rules(code: &str) -> Option<(PluralRules, Vec)> { + let id: LanguageIdentifier = code.parse().ok()?; + let language = LanguageIdentifier::from_parts(id.language, None, None, &[]); + let rules = PluralRules::create(id, PluralRuleType::CARDINAL).or_else(|_| PluralRules::create(language, PluralRuleType::CARDINAL)).ok()?; + let mut categories = Vec::new(); + for n in 0..=1000_u32 { + let c = rules.select(n).ok()?; + if !categories.contains(&c) { + categories.push(c); + } + } + categories.sort_by_key(rank); + Some((rules, categories)) +} + +fn rank(c: &PluralCategory) -> u8 { + match c { + PluralCategory::ZERO => 0, + PluralCategory::ONE => 1, + PluralCategory::TWO => 2, + PluralCategory::FEW => 3, + PluralCategory::MANY => 4, + PluralCategory::OTHER => 5, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + static E: &Embedded = &[ + ("fr", "\tSave\tEnregistrer\n\tOpen\tOuvrir\nmenu\tOpen\tOuvrir le menu\n@id\tfile.save\tEnregistrer le fichier\n", "\tSave\tSauvegarder\n"), + ("pt", "\tSave\tGuardar\n\tUndo\tDesfazer\n", ""), + ("pt-br", "\tSave\tSalvar\n", ""), + ("de", "\tBad\n@nope\tA\tB\n\tUndo\tRückgängig\n", ""), + ]; + + fn i18n() -> I18n { + I18n::new(E).0 + } + + fn lang(i: &I18n, code: &str) -> Lang { + i.from_code(code).unwrap() + } + + #[test] + fn app_row_overrides_shared() { + let i = i18n(); + assert_eq!(i.t(lang(&i, "fr"), "Save"), "Sauvegarder"); + } + + #[test] + fn shared_row_when_no_override() { + let i = i18n(); + assert_eq!(i.t(lang(&i, "fr"), "Open"), "Ouvrir"); + } + + #[test] + fn region_falls_back_to_base() { + let i = i18n(); + assert_eq!(i.t(lang(&i, "pt-br"), "Undo"), "Desfazer"); + assert_eq!(i.t(lang(&i, "pt-br"), "Save"), "Salvar"); + } + + #[test] + fn missing_is_english() { + let i = i18n(); + assert_eq!(i.t(lang(&i, "fr"), "Close"), "Close"); + assert_eq!(i.t(Lang::EN, "Save"), "Save"); + } + + #[test] + fn context_then_plain() { + let i = i18n(); + assert_eq!(i.tc(lang(&i, "fr"), "menu", "Open"), "Ouvrir le menu"); + assert_eq!(i.tc(lang(&i, "fr"), "menu", "Save"), "Sauvegarder"); + } + + #[test] + fn id_then_label() { + let i = i18n(); + assert_eq!(i.tid(lang(&i, "fr"), "file.save", "Save"), "Enregistrer le fichier"); + assert_eq!(i.tid(lang(&i, "fr"), "file.nope", "Open"), "Ouvrir"); + } + + #[test] + fn bad_rows_skipped_and_reported() { + let (i, problems) = I18n::new(E); + assert_eq!(problems.len(), 2, "{problems:?}"); + assert!(problems[0].starts_with("de/common: line 1:"), "{problems:?}"); + assert!(problems[1].starts_with("de/common: line 2:"), "{problems:?}"); + assert_eq!(i.t(lang(&i, "de"), "Undo"), "Rückgängig"); + } + + #[test] + fn from_code_is_case_insensitive() { + let i = i18n(); + assert_eq!(i.from_code("PT-BR"), i.from_code("pt-br")); + assert_eq!(i.from_code("xx"), None); + assert_eq!(i.from_code("en"), Some(Lang::EN)); + assert_eq!(i.code(Lang::EN), "en"); + } +}