i18n: catalog parsing and lookup with fallback

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01F9fM3mv1PUyKpk5rdwUQrW

Cargo.lock

@@ -609,6 +609,14 @@ dependencies = [
 name = "craft-docpath"
 version = "0.1.0"
 
+[[package]]
+name = "craft-i18n"
+version = "0.1.0"
+dependencies = [
+ "intl_pluralrules",
+ "unic-langid",
+]
+
 [[package]]
 name = "craft-pdf"
 version = "0.1.0"
@@ -1934,6 +1942,15 @@ dependencies = [
  "hashbrown 0.17.1",
 ]
 
+[[package]]
+name = "intl_pluralrules"
+version = "7.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "078ea7b7c29a2b4df841a7f6ac8775ff6074020c6776d48491ce2268e068f972"
+dependencies = [
+ "unic-langid",
+]
+
 [[package]]
 name = "ipnet"
 version = "2.12.2"
@@ -4346,6 +4363,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643"
 dependencies = [
  "displaydoc",
+ "serde_core",
  "zerovec",
 ]
 
@@ -4573,6 +4591,24 @@ version = "1.20.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20"
 
+[[package]]
+name = "unic-langid"
+version = "0.9.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a28ba52c9b05311f4f6e62d5d9d46f094bd6e84cb8df7b3ef952748d752a7d05"
+dependencies = [
+ "unic-langid-impl",
+]
+
+[[package]]
+name = "unic-langid-impl"
+version = "0.9.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dce1bf08044d4b7a94028c93786f8566047edc11110595914de93362559bc658"
+dependencies = [
+ "tinystr",
+]
+
 [[package]]
 name = "unicode-bidi"
 version = "0.3.18"
@@ -5826,6 +5862,7 @@ version = "0.11.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8"
 dependencies = [
+ "serde",
  "yoke",
  "zerofrom",
  "zerovec-derive",

Cargo.toml

@@ -23,6 +23,7 @@ thiserror = "2"
 craft-atomic = { path = "crates/atomic" }
 craft-cms = { path = "crates/cms" }
 craft-docpath = { path = "crates/docpath" }
+craft-i18n = { path = "crates/i18n" }
 craft-print = { path = "crates/print" }
 craft-quit = { path = "crates/quit" }
 craft-settings = { path = "crates/settings" }

crates/i18n/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "craft-i18n"
+description = "Translations for the Craft apps: the embedded craft-i18n catalogs, lookup with fallback, plurals, language names and the system locale."
+version.workspace = true
+edition.workspace = true
+license.workspace = true
+rust-version.workspace = true
+repository.workspace = true
+
+[dependencies]
+intl_pluralrules = "7"
+unic-langid = "0.9"
+
+[lints]
+workspace = true

crates/i18n/src/catalog.rs

@@ -0,0 +1,114 @@
+//! Reading one catalog. The rules are craft-i18n's `docs/catalog-format.md`; craft-i18n's CI
+//! applies them strictly, and here a row that breaks one is skipped and reported, so an app built
+//! against a newer catalog still runs, with that string in English.
+
+/// A row that passed, with its columns unescaped.
+pub(crate) struct Entry {
+    pub context: String,
+    pub source: String,
+    pub translation: String,
+}
+
+/// The contexts with a meaning here. Any other context starting with `@` is skipped.
+const RESERVED: &[&str] = &["@plural", "@id", "@msg"];
+
+/// The rows of a catalog whose locale has `forms` plural forms, and one problem (`line N: …`) per
+/// row skipped.
+pub(crate) fn parse(text: &str, forms: usize) -> (Vec<Entry>, Vec<String>) {
+    let text = text.strip_prefix('\u{feff}').unwrap_or(text);
+    let mut rows = Vec::new();
+    let mut problems = Vec::new();
+    for (i, line) in text.split('\n').enumerate() {
+        let line = line.strip_suffix('\r').unwrap_or(line);
+        if line.is_empty() || line.starts_with('#') {
+            continue;
+        }
+        match parse_row(line, forms) {
+            Ok(row) => rows.push(row),
+            Err(e) => problems.push(format!("line {}: {e}", i + 1)),
+        }
+    }
+    (rows, problems)
+}
+
+fn parse_row(text: &str, forms: usize) -> Result<Entry, String> {
+    let columns: Vec<&str> = text.split('\t').collect();
+    let [context, source, translation] = columns.as_slice() else {
+        return Err(format!("{} columns; expected 3", columns.len()));
+    };
+    if text.chars().any(|c| c != '\t' && (c.is_control() || c == '\u{feff}')) {
+        return Err("invisible character in the row".into());
+    }
+    let row = Entry { context: unescape(context)?, source: unescape(source)?, translation: unescape(translation)? };
+    if row.source.is_empty() || row.translation.is_empty() {
+        return Err("empty source text or translation".into());
+    }
+    match row.context.as_str() {
+        "@id" => {}
+        "@plural" => check_plural(&row.source, &row.translation, forms)?,
+        c if c.starts_with('@') && !RESERVED.contains(&c) => {
+            return Err(format!("unknown reserved context {c:?}"));
+        }
+        _ => {
+            if placeholders(&row.source) != placeholders(&row.translation) {
+                return Err("placeholders differ from the source text".into());
+            }
+            if row.source.ends_with('…') != row.translation.ends_with('…') {
+                return Err("trailing … differs from the source text".into());
+            }
+        }
+    }
+    Ok(row)
+}
+
+fn check_plural(source: &str, translation: &str, forms: usize) -> Result<(), String> {
+    let english: Vec<&str> = source.split('|').collect();
+    let [one, other] = english.as_slice() else {
+        return Err("plural source text must be one|other".into());
+    };
+    if one.is_empty() || other.is_empty() || placeholders(one) != placeholders(other) {
+        return Err("plural source forms must be nonempty with the same placeholders".into());
+    }
+    let translated: Vec<&str> = translation.split('|').collect();
+    if translated.len() != forms || translated.iter().any(|f| f.is_empty()) {
+        return Err(format!("expected {forms} nonempty plural forms"));
+    }
+    if translated.iter().any(|f| placeholders(f) != placeholders(other)) {
+        return Err("plural form placeholders differ from the source text".into());
+    }
+    Ok(())
+}
+
+fn unescape(text: &str) -> Result<String, String> {
+    let mut out = String::with_capacity(text.len());
+    let mut chars = text.chars();
+    while let Some(c) = chars.next() {
+        if c != '\\' {
+            out.push(c);
+            continue;
+        }
+        match chars.next() {
+            Some('n') => out.push('\n'),
+            Some('t') => out.push('\t'),
+            Some('\\') => out.push('\\'),
+            Some(other) => return Err(format!("unknown escape \\{other}")),
+            None => return Err("trailing backslash".into()),
+        }
+    }
+    Ok(out)
+}
+
+/// `{name}` placeholders, sorted: translations may reorder them.
+fn placeholders(text: &str) -> Vec<&str> {
+    let mut names = Vec::new();
+    let mut rest = text;
+    while let Some((_, after)) = rest.split_once('{') {
+        let Some((name, after)) = after.split_once('}') else {
+            break;
+        };
+        names.push(name);
+        rest = after;
+    }
+    names.sort_unstable();
+    names
+}

crates/i18n/src/lib.rs

@@ -0,0 +1,231 @@
+//! Translations for the Craft apps.
+//!
+//! An app embeds the [craft-i18n](https://github.com/benandez/craft-i18n) catalogs at build time
+//! as an [`Embedded`] static, builds one [`I18n`] from it, and looks English source text up in a
+//! [`Lang`]. A missing translation is the English itself; nothing here fails.
+#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
+
+mod catalog;
+
+use std::collections::HashMap;
+
+use intl_pluralrules::{PluralCategory, PluralRuleType, PluralRules};
+use unic_langid::LanguageIdentifier;
+
+/// The catalogs an app embeds, one entry per locale: (locale, shared catalog, the app's catalog).
+/// A missing catalog is `""`.
+pub type Embedded = [(&'static str, &'static str, &'static str)];
+
+/// A language the interface can be shown in: English, the pseudo locale, or an embedded locale.
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
+pub struct Lang(usize);
+
+impl Lang {
+    /// English: the source text itself.
+    pub const EN: Lang = Lang(0);
+    /// A test-only locale that marks every looked-up string.
+    pub const PSEUDO: Lang = Lang(1);
+}
+
+/// One locale's translations: context → source text → translation, app rows over shared ones.
+struct Locale {
+    code: String,
+    rows: HashMap<String, HashMap<String, String>>,
+    /// Itself, then its base language if that is embedded too (`pt-br`, `pt`). Empty for English
+    /// and the pseudo locale.
+    chain: Vec<usize>,
+    /// Plural rules and the categories `@plural` forms list, in order.
+    #[expect(dead_code, reason = "read by plural lookups")]
+    plural: Option<(PluralRules, Vec<PluralCategory>)>,
+}
+
+/// The embedded catalogs, ready for lookups. Build it once, at start-up.
+pub struct I18n {
+    locales: Vec<Locale>,
+}
+
+impl I18n {
+    /// Read `embedded`. Rows that can't be used are skipped and come back as problems
+    /// (`<locale>/<common|app>: line N: …`) for the app to log.
+    pub fn new(embedded: &'static Embedded) -> (I18n, Vec<String>) {
+        let mut problems = Vec::new();
+        let mut locales = vec![bare("en"), bare("pseudo")];
+        for (code, shared, app) in embedded {
+            let code = code.to_ascii_lowercase();
+            let plural = plural_rules(&code);
+            if plural.is_none() {
+                problems.push(format!("{code}: no plural rules; plurals show in English"));
+            }
+            let forms = plural.as_ref().map_or(1, |(_, c)| c.len());
+            let mut rows: HashMap<String, HashMap<String, String>> = HashMap::new();
+            for (name, text) in [("common", shared), ("app", app)] {
+                let (entries, errors) = catalog::parse(text, forms);
+                problems.extend(errors.into_iter().map(|e| format!("{code}/{name}: {e}")));
+                for e in entries {
+                    rows.entry(e.context).or_default().insert(e.source, e.translation);
+                }
+            }
+            locales.push(Locale { code, rows, chain: Vec::new(), plural });
+        }
+        for i in 2..locales.len() {
+            let mut chain = vec![i];
+            let base = locales.get(i).and_then(|l| l.code.split('-').next()).unwrap_or_default();
+            if let Some(b) = locales.iter().skip(2).position(|l| l.code == base)
+                && b + 2 != i
+            {
+                chain.push(b + 2);
+            }
+            if let Some(l) = locales.get_mut(i) {
+                l.chain = chain;
+            }
+        }
+        (I18n { locales }, problems)
+    }
+
+    /// The language with this code (any case): `en`, `pseudo`, or an embedded locale.
+    pub fn from_code(&self, code: &str) -> Option<Lang> {
+        self.locales.iter().position(|l| l.code.eq_ignore_ascii_case(code)).map(Lang)
+    }
+
+    /// The language's code, lowercase BCP 47 (`pt-br`).
+    pub fn code(&self, lang: Lang) -> &str {
+        self.locale(lang).map_or("en", |l| l.code.as_str())
+    }
+
+    /// `source` in `lang`, or `source` itself.
+    pub fn t<'a>(&'a self, lang: Lang, source: &'a str) -> &'a str {
+        self.find(lang, "", source).unwrap_or(source)
+    }
+
+    /// `source` with a disambiguating `context` (`menu`, `@msg`); falls back to [`I18n::t`].
+    pub fn tc<'a>(&'a self, lang: Lang, context: &str, source: &'a str) -> &'a str {
+        self.find(lang, context, source).unwrap_or_else(|| self.t(lang, source))
+    }
+
+    /// A command's label by its `id` (`@id` rows); falls back to [`I18n::t`] of the English `label`.
+    pub fn tid<'a>(&'a self, lang: Lang, id: &str, label: &'a str) -> &'a str {
+        self.find(lang, "@id", id).unwrap_or_else(|| self.t(lang, label))
+    }
+
+    fn locale(&self, lang: Lang) -> Option<&Locale> {
+        self.locales.get(lang.0)
+    }
+
+    /// The first translation of (`context`, `source`) along `lang`'s chain.
+    fn find(&self, lang: Lang, context: &str, source: &str) -> Option<&str> {
+        let chain = self.locale(lang).map(|l| l.chain.as_slice())?;
+        chain.iter().find_map(|&i| self.locales.get(i)?.rows.get(context)?.get(source).map(String::as_str))
+    }
+}
+
+/// A locale with no catalogs (English, the pseudo locale).
+fn bare(code: &str) -> Locale {
+    Locale { code: code.to_owned(), rows: HashMap::new(), chain: Vec::new(), plural: None }
+}
+
+/// `code`'s CLDR cardinal rules and the categories whole numbers 0..=1000 reach, in CLDR order
+/// (zero, one, two, few, many, other): one `@plural` form each. Regional and scripted codes fall
+/// back to the language's rules.
+fn plural_rules(code: &str) -> Option<(PluralRules, Vec<PluralCategory>)> {
+    let id: LanguageIdentifier = code.parse().ok()?;
+    let language = LanguageIdentifier::from_parts(id.language, None, None, &[]);
+    let rules = PluralRules::create(id, PluralRuleType::CARDINAL).or_else(|_| PluralRules::create(language, PluralRuleType::CARDINAL)).ok()?;
+    let mut categories = Vec::new();
+    for n in 0..=1000_u32 {
+        let c = rules.select(n).ok()?;
+        if !categories.contains(&c) {
+            categories.push(c);
+        }
+    }
+    categories.sort_by_key(rank);
+    Some((rules, categories))
+}
+
+fn rank(c: &PluralCategory) -> u8 {
+    match c {
+        PluralCategory::ZERO => 0,
+        PluralCategory::ONE => 1,
+        PluralCategory::TWO => 2,
+        PluralCategory::FEW => 3,
+        PluralCategory::MANY => 4,
+        PluralCategory::OTHER => 5,
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    static E: &Embedded = &[
+        ("fr", "\tSave\tEnregistrer\n\tOpen\tOuvrir\nmenu\tOpen\tOuvrir le menu\n@id\tfile.save\tEnregistrer le fichier\n", "\tSave\tSauvegarder\n"),
+        ("pt", "\tSave\tGuardar\n\tUndo\tDesfazer\n", ""),
+        ("pt-br", "\tSave\tSalvar\n", ""),
+        ("de", "\tBad\n@nope\tA\tB\n\tUndo\tRückgängig\n", ""),
+    ];
+
+    fn i18n() -> I18n {
+        I18n::new(E).0
+    }
+
+    fn lang(i: &I18n, code: &str) -> Lang {
+        i.from_code(code).unwrap()
+    }
+
+    #[test]
+    fn app_row_overrides_shared() {
+        let i = i18n();
+        assert_eq!(i.t(lang(&i, "fr"), "Save"), "Sauvegarder");
+    }
+
+    #[test]
+    fn shared_row_when_no_override() {
+        let i = i18n();
+        assert_eq!(i.t(lang(&i, "fr"), "Open"), "Ouvrir");
+    }
+
+    #[test]
+    fn region_falls_back_to_base() {
+        let i = i18n();
+        assert_eq!(i.t(lang(&i, "pt-br"), "Undo"), "Desfazer");
+        assert_eq!(i.t(lang(&i, "pt-br"), "Save"), "Salvar");
+    }
+
+    #[test]
+    fn missing_is_english() {
+        let i = i18n();
+        assert_eq!(i.t(lang(&i, "fr"), "Close"), "Close");
+        assert_eq!(i.t(Lang::EN, "Save"), "Save");
+    }
+
+    #[test]
+    fn context_then_plain() {
+        let i = i18n();
+        assert_eq!(i.tc(lang(&i, "fr"), "menu", "Open"), "Ouvrir le menu");
+        assert_eq!(i.tc(lang(&i, "fr"), "menu", "Save"), "Sauvegarder");
+    }
+
+    #[test]
+    fn id_then_label() {
+        let i = i18n();
+        assert_eq!(i.tid(lang(&i, "fr"), "file.save", "Save"), "Enregistrer le fichier");
+        assert_eq!(i.tid(lang(&i, "fr"), "file.nope", "Open"), "Ouvrir");
+    }
+
+    #[test]
+    fn bad_rows_skipped_and_reported() {
+        let (i, problems) = I18n::new(E);
+        assert_eq!(problems.len(), 2, "{problems:?}");
+        assert!(problems[0].starts_with("de/common: line 1:"), "{problems:?}");
+        assert!(problems[1].starts_with("de/common: line 2:"), "{problems:?}");
+        assert_eq!(i.t(lang(&i, "de"), "Undo"), "Rückgängig");
+    }
+
+    #[test]
+    fn from_code_is_case_insensitive() {
+        let i = i18n();
+        assert_eq!(i.from_code("PT-BR"), i.from_code("pt-br"));
+        assert_eq!(i.from_code("xx"), None);
+        assert_eq!(i.from_code("en"), Some(Lang::EN));
+        assert_eq!(i.code(Lang::EN), "en");
+    }
+}