diff --git a/.github/workflows/i18n.yml b/.github/workflows/i18n.yml new file mode 100644 index 00000000..d7e6f5b5 --- /dev/null +++ b/.github/workflows/i18n.yml @@ -0,0 +1,29 @@ +name: Translations + +on: + workflow_dispatch: + push: + branches: [main] + paths: ['crates/i18n/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/i18n.yml'] + pull_request: + paths: ['crates/i18n/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/i18n.yml'] + +permissions: + contents: read + +jobs: + i18n: + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + with: + components: rustfmt, clippy + - run: cargo fmt --check + - run: cargo clippy -p craft-i18n --all-targets --locked -- -D warnings + - run: cargo test -p craft-i18n --locked -- --nocapture diff --git a/Cargo.lock b/Cargo.lock index 676a5220..6471411f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -609,6 +609,15 @@ dependencies = [ name = "craft-docpath" version = "0.1.0" +[[package]] +name = "craft-i18n" +version = "0.1.0" +dependencies = [ + "intl_pluralrules", + "sys-locale", + "unic-langid", +] + [[package]] name = "craft-pdf" version = "0.1.0" @@ -1934,6 +1943,15 @@ dependencies = [ "hashbrown 0.17.1", ] +[[package]] +name = "intl_pluralrules" +version = "7.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "078ea7b7c29a2b4df841a7f6ac8775ff6074020c6776d48491ce2268e068f972" +dependencies = [ + "unic-langid", +] + [[package]] name = "ipnet" version = "2.12.2" @@ -4194,6 +4212,18 @@ dependencies = [ "syn 3.0.6", ] +[[package]] +name = "sys-locale" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eab9a99a024a169fe8a903cf9d4a3b3601109bcc13bd9e3c6fff259138626c4" +dependencies = [ + "js-sys", + "libc", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "system-configuration" version = "0.7.0" @@ -4346,6 +4376,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" dependencies = [ "displaydoc", + "serde_core", "zerovec", ] @@ -4573,6 +4604,24 @@ version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" +[[package]] +name = "unic-langid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28ba52c9b05311f4f6e62d5d9d46f094bd6e84cb8df7b3ef952748d752a7d05" +dependencies = [ + "unic-langid-impl", +] + +[[package]] +name = "unic-langid-impl" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce1bf08044d4b7a94028c93786f8566047edc11110595914de93362559bc658" +dependencies = [ + "tinystr", +] + [[package]] name = "unicode-bidi" version = "0.3.18" @@ -5826,6 +5875,7 @@ version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" dependencies = [ + "serde", "yoke", "zerofrom", "zerovec-derive", diff --git a/Cargo.toml b/Cargo.toml index 336fd37b..868e568d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,6 +23,7 @@ thiserror = "2" craft-atomic = { path = "crates/atomic" } craft-cms = { path = "crates/cms" } craft-docpath = { path = "crates/docpath" } +craft-i18n = { path = "crates/i18n" } craft-print = { path = "crates/print" } craft-quit = { path = "crates/quit" } craft-settings = { path = "crates/settings" } diff --git a/README.md b/README.md index 4b428ee8..78b24340 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,10 @@ craft- = { git = "https://github.com/storytold/craft-libs", rev = " = LazyLock::new(|| { + let offered: Vec<&str> = LANGUAGES.iter().map(|l| l.code).collect(); // the app's own list + let (i18n, problems) = craft_i18n::I18n::new(CRAFT_I18N, &offered); + problems.iter().for_each(|p| log::warn!("translation catalog: {p}; that string shows in English")); + i18n +}); + +thread_local! { static CURRENT: Cell = const { Cell::new(craft_i18n::Lang::EN) }; } + +macro_rules! tl { ($s:expr) => { $crate::I18N.t($crate::CURRENT.get(), $s) }; } +``` + +The app keeps its `tl!` macro, its language setting (`"auto"` → `I18N.system_lang()`, otherwise +`I18N.from_code`) and its picker (`I18N.languages()`, `I18N.name(lang)`). + +## Behaviour + +- **Offered locales:** `I18n::new` takes the codes the app offers (its own language list), so an + app never offers a language its interface isn't ready for (Arabic without right-to-left layout, + say). `languages()`, `from_code`, `match_tag` and `system_lang` only return offered locales; an + embedded locale that isn't offered still serves as a fallback (`pt` for an offered `pt-br`). +- **Fallback:** locale by locale along the chain, `pt-br` → `pt` → the English source text. In + each locale, app rows override shared ones, and `tc`/`tid` try their context or `@id` row before + the plain row, so `pt-br`'s plain row beats `pt`'s `menu` row. Only a region falls back (`pt-br` + → `pt`, `zh-hant-hk` → `zh-hant`); `sr-latn` never falls back to Cyrillic `sr`. A missing + translation never errors. +- **Lookups:** `t` (plain), `tc` (with a context such as `menu` or `@msg`, else plain), `tid` (an + `@id` command id, else the English label), `trn` (a count, `@plural` rows, `{n}` filled), `get` + (one context along the locale chain, so `pt-br` can return a `pt` row, but never the plain row + or English: a key cap's name under `key`, or "is there a translation?"), and `fmt` for `{name}` + placeholders, filled in one pass. +- **Bad rows** (malformed, an unknown `@` context, a wrong plural form count, a row defined twice, + of which the first is kept) and bad locales (`en`, `pseudo`, a code embedded twice in any case) + are skipped and returned by `I18n::new` for the app to log; craft-i18n's CI rejects them before + they ship. +- **Plurals:** forms are listed in CLDR order (zero, one, two, few, many, other) for the categories + whole numbers 0 to 1000 reach, from `intl_pluralrules`; a count in any other category takes the + last form. CLDR's `pt` is Brazilian, so `pt` uses `pt-PT`'s rules (0 is plural) and `pt-br` + `pt`'s. `intl_pluralrules` is pinned to the version craft-i18n's checker uses, and + `plural_form_counts` holds the same counts. +- **Names:** a language's name comes from its shared catalog's `@id language.name` row, else its + code; English is "English". +- **Right to left:** `rtl` is true for `ar`, `ckb`, `dv`, `fa`, `he`, `ps`, `sd`, `ug`, `ur` and + `yi`; putting text into display order is the app's job. +- **System locale:** `system_lang` matches the system's list once and caches it: on desktop Unix + `LANGUAGE`, `LC_ALL`, `LC_MESSAGES` and `LANG`, read raw; elsewhere `sys-locale`'s (macOS and + Windows preferred languages, the browser's on the web). `fr_CA.UTF-8` → `fr`, `zh-TW` → + `zh-hant`, `pt-PT` → `pt-br` when that is the only Portuguese, `C`/`POSIX` → English. A script + is never crossed: `sr_RS@latin` and `sr-Latn-RS` pick `sr-latn`, never `sr`, while `sr-Cyrl-RS` + picks `sr`. +- **Pseudo locale** (`pseudo`, debug builds only, never in `languages()`): every lookup, `get` + included, comes back as `⟦…⟧`, line by line, so text drawn without the marks never went through + translation. `looked_up()` lists the source texts looked up so far, for an audit against a + complete catalog. diff --git a/crates/i18n/src/catalog.rs b/crates/i18n/src/catalog.rs new file mode 100644 index 00000000..50d731b2 --- /dev/null +++ b/crates/i18n/src/catalog.rs @@ -0,0 +1,120 @@ +//! Reading one catalog. The rules are craft-i18n's `docs/catalog-format.md`; craft-i18n's CI +//! applies them strictly, and here a row that breaks one is skipped and reported, so an app built +//! against a newer catalog still runs, with that string in English. + +use std::collections::HashSet; + +/// A row that passed, with its columns unescaped. +pub(crate) struct Entry { + pub context: String, + pub source: String, + pub translation: String, +} + +/// The contexts with a meaning here. Any other context starting with `@` is skipped. +const RESERVED: &[&str] = &["@plural", "@id", "@msg"]; + +/// The rows of a catalog whose locale has `forms` plural forms, and one problem (`line N: …`) per +/// row skipped. Of a row defined twice, the first is kept, as in craft-i18n's checker. +pub(crate) fn parse(text: &str, forms: usize) -> (Vec, Vec) { + let text = text.strip_prefix('\u{feff}').unwrap_or(text); + let mut rows = Vec::new(); + let mut problems = Vec::new(); + let mut seen = HashSet::new(); + for (i, line) in text.split('\n').enumerate() { + let line = line.strip_suffix('\r').unwrap_or(line); + if line.is_empty() || line.starts_with('#') { + continue; + } + match parse_row(line, forms) { + Ok(row) if !seen.insert((row.context.clone(), row.source.clone())) => { + problems.push(format!("line {}: defined twice (same context and source text); the first is kept", i + 1)); + } + Ok(row) => rows.push(row), + Err(e) => problems.push(format!("line {}: {e}", i + 1)), + } + } + (rows, problems) +} + +fn parse_row(text: &str, forms: usize) -> Result { + let columns: Vec<&str> = text.split('\t').collect(); + let [context, source, translation] = columns.as_slice() else { + return Err(format!("{} columns; expected 3", columns.len())); + }; + if text.chars().any(|c| c != '\t' && (c.is_control() || c == '\u{feff}')) { + return Err("invisible character in the row".into()); + } + let row = Entry { context: unescape(context)?, source: unescape(source)?, translation: unescape(translation)? }; + if row.source.is_empty() || row.translation.is_empty() { + return Err("empty source text or translation".into()); + } + match row.context.as_str() { + "@id" => {} + "@plural" => check_plural(&row.source, &row.translation, forms)?, + c if c.starts_with('@') && !RESERVED.contains(&c) => { + return Err(format!("unknown reserved context {c:?}")); + } + _ => { + if placeholders(&row.source) != placeholders(&row.translation) { + return Err("placeholders differ from the source text".into()); + } + if row.source.ends_with('…') != row.translation.ends_with('…') { + return Err("trailing … differs from the source text".into()); + } + } + } + Ok(row) +} + +fn check_plural(source: &str, translation: &str, forms: usize) -> Result<(), String> { + let english: Vec<&str> = source.split('|').collect(); + let [one, other] = english.as_slice() else { + return Err("plural source text must be one|other".into()); + }; + if one.is_empty() || other.is_empty() || placeholders(one) != placeholders(other) { + return Err("plural source forms must be nonempty with the same placeholders".into()); + } + let translated: Vec<&str> = translation.split('|').collect(); + if translated.len() != forms || translated.iter().any(|f| f.is_empty()) { + return Err(format!("expected {forms} nonempty plural forms")); + } + if translated.iter().any(|f| placeholders(f) != placeholders(other)) { + return Err("plural form placeholders differ from the source text".into()); + } + Ok(()) +} + +fn unescape(text: &str) -> Result { + let mut out = String::with_capacity(text.len()); + let mut chars = text.chars(); + while let Some(c) = chars.next() { + if c != '\\' { + out.push(c); + continue; + } + match chars.next() { + Some('n') => out.push('\n'), + Some('t') => out.push('\t'), + Some('\\') => out.push('\\'), + Some(other) => return Err(format!("unknown escape \\{other}")), + None => return Err("trailing backslash".into()), + } + } + Ok(out) +} + +/// `{name}` placeholders, sorted: translations may reorder them. +fn placeholders(text: &str) -> Vec<&str> { + let mut names = Vec::new(); + let mut rest = text; + while let Some((_, after)) = rest.split_once('{') { + let Some((name, after)) = after.split_once('}') else { + break; + }; + names.push(name); + rest = after; + } + names.sort_unstable(); + names +} diff --git a/crates/i18n/src/lib.rs b/crates/i18n/src/lib.rs new file mode 100644 index 00000000..7fa7a4ef --- /dev/null +++ b/crates/i18n/src/lib.rs @@ -0,0 +1,751 @@ +//! Translations for the Craft apps. +//! +//! An app embeds the [craft-i18n](https://github.com/benandez/craft-i18n) catalogs at build time +//! as an [`Embedded`] static, builds one [`I18n`] from it, and looks English source text up in a +//! [`Lang`]. A missing translation is the English itself; nothing here fails. +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] + +mod catalog; + +use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::sync::{Mutex, OnceLock, PoisonError}; + +use intl_pluralrules::{PluralCategory, PluralRuleType, PluralRules}; +use unic_langid::LanguageIdentifier; + +/// The catalogs an app embeds, one entry per locale: (locale, shared catalog, the app's catalog). +/// A missing catalog is `""`. +pub type Embedded = [(&'static str, &'static str, &'static str)]; + +/// A language the interface can be shown in: English, the pseudo locale, or an embedded locale. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct Lang(usize); + +impl Lang { + /// English: the source text itself. + pub const EN: Lang = Lang(0); + /// A test-only locale that marks every looked-up string. + pub const PSEUDO: Lang = Lang(1); +} + +/// One locale's translations: context → source text → translation, app rows over shared ones. +struct Locale { + code: String, + rows: HashMap>, + /// Itself, then its parent without the region if that is embedded too (`pt-br`, `pt`; + /// `zh-hant-hk`, `zh-hant`). Empty for English and the pseudo locale. + chain: Vec, + /// The app offers it: in [`I18n::languages`], found by code and system tag. An embedded locale + /// the app doesn't offer still serves as a fallback. + offered: bool, + /// Plural rules and the categories `@plural` forms list, in order. + plural: Option<(PluralRules, Vec)>, +} + +/// The embedded catalogs, ready for lookups. Build it once, at start-up. +pub struct I18n { + locales: Vec, + /// Marked strings for the pseudo locale by source text, leaked so lookups can hand out `&str`. + /// Debug builds only; the keys are what [`I18n::looked_up`] reports. + // ponytail: grows by one leak per distinct source text, fine for a test run; an arena if the + // pseudo locale ever ships. + pseudo: Mutex>, + /// [`I18n::system_lang`], read once. + system: OnceLock, +} + +impl I18n { + /// Read `embedded`, offering the locales in `offered` (any case; the app's own language list), + /// so an app never offers a language its interface isn't ready for (right to left, say). Rows + /// and locales that can't be used are skipped and come back as problems + /// (`/: line N: …`) for the app to log. + pub fn new(embedded: &'static Embedded, offered: &[&str]) -> (I18n, Vec) { + let mut problems = Vec::new(); + let mut locales = vec![bare("en"), bare("pseudo")]; + for (code, shared, app) in embedded { + let code = code.to_ascii_lowercase(); + if locales.iter().any(|l| l.code == code) { + problems.push(format!("{code}: reserved or embedded twice; skipped")); + continue; + } + let plural = plural_rules(&code); + if plural.is_none() { + problems.push(format!("{code}: no plural rules; plurals show in English")); + } + let forms = plural.as_ref().map_or(1, |(_, c)| c.len()); + let mut rows: HashMap> = HashMap::new(); + for (name, text) in [("common", shared), ("app", app)] { + let (entries, errors) = catalog::parse(text, forms); + problems.extend(errors.into_iter().map(|e| format!("{code}/{name}: {e}"))); + for e in entries { + rows.entry(e.context).or_default().insert(e.source, e.translation); + } + } + let offered = offered.iter().any(|o| o.eq_ignore_ascii_case(&code)); + locales.push(Locale { code, rows, chain: Vec::new(), plural, offered }); + } + for i in 2..locales.len() { + let mut chain = vec![i]; + // Only a region falls back (`pt-br` → `pt`, `zh-hant-hk` → `zh-hant`); a script + // doesn't, since `sr-latn` → `sr` would show Cyrillic. + let code = locales.get(i).map_or("", |l| l.code.as_str()); + let base = match code.rsplit_once('-') { + Some((parent, region)) if is_region(region) => parent, + _ => "", + }; + if let Some(b) = locales.iter().skip(2).position(|l| l.code == base) + && b + 2 != i + { + chain.push(b + 2); + } + if let Some(l) = locales.get_mut(i) { + l.chain = chain; + } + } + (I18n { locales, pseudo: Mutex::default(), system: OnceLock::new() }, problems) + } + + /// The language with this code (any case): `en`, an offered locale, or in debug builds + /// `pseudo`. + pub fn from_code(&self, code: &str) -> Option { + let i = self.locales.iter().position(|l| l.code.eq_ignore_ascii_case(code))?; + let usable = match Lang(i) { + Lang::EN => true, + Lang::PSEUDO => cfg!(debug_assertions), + _ => self.locales.get(i).is_some_and(|l| l.offered), + }; + usable.then_some(Lang(i)) + } + + /// The language's code, lowercase BCP 47 (`pt-br`). + pub fn code(&self, lang: Lang) -> &str { + self.locale(lang).map_or("en", |l| l.code.as_str()) + } + + /// `source` in `lang`, or `source` itself. + pub fn t<'a>(&'a self, lang: Lang, source: &'a str) -> &'a str { + if lang == Lang::PSEUDO { + return self.mark(source); + } + self.find(lang, &[("", source)]).unwrap_or(source) + } + + /// `source` with a disambiguating `context` (`menu`, `@msg`); in each locale of the chain the + /// context row, then the plain one, so `pt-br`'s plain row beats `pt`'s context row. + pub fn tc<'a>(&'a self, lang: Lang, context: &str, source: &'a str) -> &'a str { + if lang == Lang::PSEUDO { + return self.mark(source); + } + self.find(lang, &[(context, source), ("", source)]).unwrap_or(source) + } + + /// The translation of (`context`, `source`) along `lang`'s chain, with no fallback to the plain + /// row or the English: for strings that must not borrow another meaning's wording (a key cap's + /// name under context `key`), or to ask whether a translation exists. + pub fn get<'a>(&'a self, lang: Lang, context: &str, source: &'a str) -> Option<&'a str> { + if lang == Lang::PSEUDO { + return Some(self.mark(source)); + } + self.find(lang, &[(context, source)]) + } + + /// A command's label by its `id` (`@id` rows); in each locale of the chain the `@id` row, then + /// the plain row of the English `label`. + pub fn tid<'a>(&'a self, lang: Lang, id: &str, label: &'a str) -> &'a str { + if lang == Lang::PSEUDO { + return self.mark(label); + } + self.find(lang, &[("@id", id), ("", label)]).unwrap_or(label) + } + + /// A count's wording in `lang`, `{n}` filled in. `one` and `other` are the English forms; + /// they are also the key of the `@plural` row, and the fallback. + pub fn trn(&self, lang: Lang, n: u64, one: &str, other: &str) -> String { + let key = format!("{one}|{other}"); + let chain = self.locale(lang).map_or(&[][..], |l| l.chain.as_slice()); + let text = chain + .iter() + .find_map(|&i| { + let locale = self.locales.get(i)?; + let forms = locale.rows.get("@plural")?.get(&key)?; + let (rules, categories) = locale.plural.as_ref()?; + let category = rules.select(n).ok()?; + // A category whole numbers up to 1000 never reach (French `many`) takes the last form. + let index = categories.iter().position(|c| *c == category).unwrap_or(categories.len().saturating_sub(1)); + forms.split('|').nth(index) + }) + .unwrap_or(if n == 1 { one } else { other }); + let text = fmt(text, &[("n", &n.to_string())]); + if lang == Lang::PSEUDO { + return marked(&text); + } + text + } + + /// The languages to offer: English first, then the offered locales by code. Never the + /// pseudo locale. + pub fn languages(&self) -> Vec { + let mut embedded: Vec = (2..self.locales.len()).filter(|&i| self.locales.get(i).is_some_and(|l| l.offered)).collect(); + embedded.sort_by_key(|&i| self.locales.get(i).map(|l| l.code.as_str())); + std::iter::once(Lang::EN).chain(embedded.into_iter().map(Lang)).collect() + } + + /// The language's name in itself, for a language picker: its shared catalog's + /// `@id language.name` row, or the code when there is none. + pub fn name(&self, lang: Lang) -> &str { + if lang == Lang::EN { + return "English"; + } + self.locale(lang).and_then(|l| l.rows.get("@id")?.get("language.name")).map_or_else(|| self.code(lang), String::as_str) + } + + /// Whether the language is written right to left. Putting text into display order is the + /// app's job. + pub fn rtl(&self, lang: Lang) -> bool { + let code = self.code(lang); + let language = code.split('-').next().unwrap_or(code); + matches!(language, "ar" | "ckb" | "dv" | "fa" | "he" | "ps" | "sd" | "ug" | "ur" | "yi") + } + + /// The language for a system locale tag (`fr_CA.UTF-8`, `zh-Hant-MO`); `C` and `POSIX` are + /// English. `None` when no embedded language fits. + pub fn match_tag(&self, tag: &str) -> Option { + let candidates = candidates(tag); + if matches!(candidates.first().map(String::as_str), Some("c" | "posix")) { + return Some(Lang::EN); + } + candidates.iter().filter(|c| *c != "pseudo").find_map(|c| self.from_code(c)) + } + + /// The first of `tags` (most preferred first) that [`I18n::match_tag`] knows; English if none. + pub fn match_tags<'t>(&self, tags: impl IntoIterator) -> Lang { + tags.into_iter().find_map(|t| self.match_tag(t)).unwrap_or(Lang::EN) + } + + /// The language the system asks for (`LANG` and friends, the macOS and Windows preferred + /// languages, the browser's on the web); English if none fits. Read once, then cached. + pub fn system_lang(&self) -> Lang { + *self.system.get_or_init(|| { + // sys-locale drops glibc's script modifier (`sr_RS@latin` → `sr-RS`), so read its variables raw. + #[cfg(all(unix, not(any(target_vendor = "apple", target_os = "android"))))] + let tags = posix_env_tags(|k| std::env::var(k).ok()); + #[cfg(not(all(unix, not(any(target_vendor = "apple", target_os = "android")))))] + let tags: Vec = sys_locale::get_locales().collect(); + self.match_tags(tags.iter().map(String::as_str)) + }) + } + + /// Every source text looked up in the pseudo locale so far, for an audit to check against a + /// complete catalog. Plurals aren't recorded. + pub fn looked_up(&self) -> BTreeSet { + self.pseudo.lock().unwrap_or_else(PoisonError::into_inner).keys().cloned().collect() + } + + /// `source` marked for the pseudo locale, made once per string. + fn mark(&self, source: &str) -> &'static str { + let mut cache = self.pseudo.lock().unwrap_or_else(PoisonError::into_inner); + cache.entry(source.to_owned()).or_insert_with(|| Box::leak(marked(source).into_boxed_str())) + } + + fn locale(&self, lang: Lang) -> Option<&Locale> { + self.locales.get(lang.0) + } + + /// Along `lang`'s chain, locale by locale, the first of `keys` (context, source text) found. + fn find(&self, lang: Lang, keys: &[(&str, &str)]) -> Option<&str> { + let chain = self.locale(lang).map(|l| l.chain.as_slice())?; + chain.iter().find_map(|&i| { + let rows = &self.locales.get(i)?.rows; + keys.iter().find_map(|(context, source)| rows.get(*context)?.get(*source).map(String::as_str)) + }) + } +} + +/// Fill `{name}` placeholders from `args` in one pass. Unknown placeholders stay as written, and +/// inserted values are never read as templates, so a file name containing `{n}` stays intact. +pub fn fmt(template: &str, args: &[(&str, &str)]) -> String { + let mut out = String::with_capacity(template.len()); + let mut rest = template; + while let Some((before, after_open)) = rest.split_once('{') { + out.push_str(before); + let Some((name, after_close)) = after_open.split_once('}') else { + out.push('{'); + rest = after_open; + break; + }; + match args.iter().find(|(k, _)| *k == name) { + Some((_, value)) => out.push_str(value), + None => { + out.push('{'); + out.push_str(name); + out.push('}'); + } + } + rest = after_close; + } + out.push_str(rest); + out +} + +/// Codes to try for a locale tag, most specific first: `zh_TW.UTF-8` → `zh-tw`, `zh-hant`, `zh`. +/// Chinese without a script picks one by region; Portuguese falls back to `pt-br` last, when +/// there is no `pt`. A glibc script modifier (`sr_RS@latin`) becomes the script subtag. With a +/// script (`sr-Latn-RS`), a code without one is tried only when that is its language's usual +/// script, so Latin Serbian never picks Cyrillic `sr`, while `sr-Cyrl-RS` does. +fn candidates(tag: &str) -> Vec { + let tag = tag.trim().to_ascii_lowercase(); + let base = tag.split(['.', '@']).next().unwrap_or("").replace('_', "-"); + let mut parts: Vec<&str> = base.split('-').filter(|p| !p.is_empty()).collect(); + let Some(&primary) = parts.first() else { + return Vec::new(); + }; + let modifier = match tag.split_once('@').map(|(_, m)| m) { + Some("latin") => Some("latn"), + Some("cyrillic") => Some("cyrl"), + _ => None, + }; + if let Some(s) = modifier { + parts.insert(1, s); + } + let script = parts.get(1).copied().filter(|s| s.len() == 4 && s.bytes().all(|b| b.is_ascii_alphabetic())); + let mut out: Vec = (1..=parts.len()).rev().map(|n| parts.get(..n).unwrap_or_default().join("-")).collect(); + if let Some(s) = script + && usual_script(primary).is_some_and(|usual| usual != s) + { + out.retain(|c| c.split('-').any(|p| p == s)); + } + if primary == "zh" && !parts.iter().any(|p| matches!(*p, "hans" | "hant")) { + let script = if parts.iter().any(|p| matches!(*p, "tw" | "hk" | "mo")) { "zh-hant" } else { "zh-hans" }; + out.insert(out.len().saturating_sub(1), script.to_owned()); + } + if primary == "pt" && !out.iter().any(|c| c == "pt-br") { + out.push("pt-br".to_owned()); + } + out +} + +/// The locale variables sys-locale reads on Unix, in its order (`LANGUAGE`'s `:` list, then +/// `LC_ALL`, `LC_MESSAGES`, `LANG`), but unconverted, so `@latin` survives. +#[cfg(all(unix, not(any(target_vendor = "apple", target_os = "android"))))] +fn posix_env_tags(var: impl Fn(&str) -> Option) -> Vec { + let mut tags: Vec = var("LANGUAGE").map(|v| v.split(':').map(str::to_owned).collect()).unwrap_or_default(); + tags.extend(["LC_ALL", "LC_MESSAGES", "LANG"].into_iter().filter_map(&var)); + tags.retain(|t| !t.is_empty()); + tags +} + +/// `⟦text⟧`, each line marked on its own, since multi-line labels are often drawn a line at a time. +fn marked(text: &str) -> String { + format!("⟦{}⟧", text.replace('\n', "⟧\n⟦")) +} + +/// The script a code without one means, for languages written in more than one (CLDR likely +/// subtags). Any other language is taken to have one script. +// ponytail: the multi-script languages a Craft app could plausibly ship; add a row with the locale. +fn usual_script(language: &str) -> Option<&'static str> { + Some(match language { + "be" | "mn" | "sr" => "cyrl", + "zh" => "hans", + "az" | "bs" | "ha" | "uz" => "latn", + "ks" | "sd" => "arab", + "pa" => "guru", + _ => return None, + }) +} + +/// A BCP 47 region subtag: two letters or three digits. +fn is_region(subtag: &str) -> bool { + (subtag.len() == 2 && subtag.bytes().all(|b| b.is_ascii_alphabetic())) || (subtag.len() == 3 && subtag.bytes().all(|b| b.is_ascii_digit())) +} + +/// A locale with no catalogs (English, the pseudo locale). +fn bare(code: &str) -> Locale { + Locale { code: code.to_owned(), rows: HashMap::new(), chain: Vec::new(), plural: None, offered: false } +} + +/// `code`'s CLDR cardinal rules and the categories whole numbers 0..=1000 reach, in CLDR order +/// (zero, one, two, few, many, other): one `@plural` form each. Regional and scripted codes fall +/// back to the language's rules. CLDR's `pt` is Brazilian (0 is singular), so Portuguese other +/// than `pt-br` takes `pt-PT`'s. +fn plural_rules(code: &str) -> Option<(PluralRules, Vec)> { + let code = if code == "pt" || (code.starts_with("pt-") && code != "pt-br") { "pt-pt" } else { code }; + let id: LanguageIdentifier = code.parse().ok()?; + let language = LanguageIdentifier::from_parts(id.language, None, None, &[]); + let rules = PluralRules::create(id, PluralRuleType::CARDINAL).or_else(|_| PluralRules::create(language, PluralRuleType::CARDINAL)).ok()?; + let mut categories = Vec::new(); + for n in 0..=1000_u32 { + let c = rules.select(n).ok()?; + if !categories.contains(&c) { + categories.push(c); + } + } + categories.sort_by_key(rank); + Some((rules, categories)) +} + +fn rank(c: &PluralCategory) -> u8 { + match c { + PluralCategory::ZERO => 0, + PluralCategory::ONE => 1, + PluralCategory::TWO => 2, + PluralCategory::FEW => 3, + PluralCategory::MANY => 4, + PluralCategory::OTHER => 5, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + static E: &Embedded = &[ + ("fr", "\tSave\tEnregistrer\n\tOpen\tOuvrir\nmenu\tOpen\tOuvrir le menu\n@id\tfile.save\tEnregistrer le fichier\n", "\tSave\tSauvegarder\n"), + ("pt", "\tSave\tGuardar\n\tUndo\tDesfazer\n", ""), + ("pt-br", "\tSave\tSalvar\n", ""), + ("de", "\tBad\n@nope\tA\tB\n\tUndo\tRückgängig\n", ""), + ]; + + /// `embedded`, offering every locale in it. + fn all(embedded: &'static Embedded) -> (I18n, Vec) { + let codes: Vec<&str> = embedded.iter().map(|(c, _, _)| *c).collect(); + I18n::new(embedded, &codes) + } + + fn i18n() -> I18n { + all(E).0 + } + + fn lang(i: &I18n, code: &str) -> Lang { + i.from_code(code).unwrap() + } + + #[test] + fn app_row_overrides_shared() { + let i = i18n(); + assert_eq!(i.t(lang(&i, "fr"), "Save"), "Sauvegarder"); + } + + #[test] + fn shared_row_when_no_override() { + let i = i18n(); + assert_eq!(i.t(lang(&i, "fr"), "Open"), "Ouvrir"); + } + + #[test] + fn region_falls_back_to_base() { + let i = i18n(); + assert_eq!(i.t(lang(&i, "pt-br"), "Undo"), "Desfazer"); + assert_eq!(i.t(lang(&i, "pt-br"), "Save"), "Salvar"); + } + + #[test] + fn missing_is_english() { + let i = i18n(); + assert_eq!(i.t(lang(&i, "fr"), "Close"), "Close"); + assert_eq!(i.t(Lang::EN, "Save"), "Save"); + } + + #[test] + fn context_then_plain() { + let i = i18n(); + assert_eq!(i.tc(lang(&i, "fr"), "menu", "Open"), "Ouvrir le menu"); + assert_eq!(i.tc(lang(&i, "fr"), "menu", "Save"), "Sauvegarder"); + } + + #[test] + fn id_then_label() { + let i = i18n(); + assert_eq!(i.tid(lang(&i, "fr"), "file.save", "Save"), "Enregistrer le fichier"); + assert_eq!(i.tid(lang(&i, "fr"), "file.nope", "Open"), "Ouvrir"); + } + + #[test] + fn bad_rows_skipped_and_reported() { + let (i, problems) = all(E); + assert_eq!(problems.len(), 2, "{problems:?}"); + assert!(problems[0].starts_with("de/common: line 1:"), "{problems:?}"); + assert!(problems[1].starts_with("de/common: line 2:"), "{problems:?}"); + assert_eq!(i.t(lang(&i, "de"), "Undo"), "Rückgängig"); + } + + #[test] + fn from_code_is_case_insensitive() { + let i = i18n(); + assert_eq!(i.from_code("PT-BR"), i.from_code("pt-br")); + assert_eq!(i.from_code("xx"), None); + assert_eq!(i.from_code("en"), Some(Lang::EN)); + assert_eq!(i.code(Lang::EN), "en"); + } + + static P: &Embedded = &[ + ("fr", "@plural\t{n} page|{n} pages\t{n} page|{n} pages\n", ""), + ("ru", "@plural\t{n} file|{n} files\t{n} файл|{n} файла|{n} файлов\n", ""), + ("ja", "@plural\t{n} file|{n} files\t{n} 個\n", ""), + ]; + + #[test] + fn fmt_fills_once() { + assert_eq!(fmt("{a} to {b}", &[("a", "x"), ("b", "{a}")]), "x to {a}"); + assert_eq!(fmt("{z} {", &[]), "{z} {"); + } + + #[test] + fn one_other() { + let (i, problems) = all(P); + assert!(problems.is_empty(), "{problems:?}"); + let fr = lang(&i, "fr"); + assert_eq!(i.trn(fr, 0, "{n} page", "{n} pages"), "0 page"); + assert_eq!(i.trn(fr, 2, "{n} page", "{n} pages"), "2 pages"); + } + + #[test] + fn three_forms_ru() { + let i = all(P).0; + let ru = lang(&i, "ru"); + for (n, want) in [(1, "1 файл"), (3, "3 файла"), (5, "5 файлов"), (11, "11 файлов"), (21, "21 файл")] { + assert_eq!(i.trn(ru, n, "{n} file", "{n} files"), want); + } + } + + #[test] + fn one_form_ja() { + let i = all(P).0; + assert_eq!(i.trn(lang(&i, "ja"), 7, "{n} file", "{n} files"), "7 個"); + } + + #[test] + fn english_fallback() { + let i = all(P).0; + assert_eq!(i.trn(Lang::EN, 1, "{n} file", "{n} files"), "1 file"); + assert_eq!(i.trn(lang(&i, "fr"), 2, "{n} x", "{n} xs"), "2 xs"); + } + + #[test] + fn beyond_sweep() { + let i = all(P).0; + assert_eq!(i.trn(lang(&i, "fr"), 1_000_000, "{n} page", "{n} pages"), "1000000 pages"); + } + + static L: &Embedded = + &[("fr", "@id\tlanguage.name\tFrançais\n", ""), ("pt-br", "", ""), ("zh-hans", "", ""), ("zh-hant", "", ""), ("ar", "", "")]; + + #[test] + fn languages_order() { + let i = all(L).0; + let codes: Vec<&str> = i.languages().into_iter().map(|l| i.code(l)).collect(); + assert_eq!(codes, ["en", "ar", "fr", "pt-br", "zh-hans", "zh-hant"]); + } + + #[test] + fn names() { + let i = all(L).0; + assert_eq!(i.name(Lang::EN), "English"); + assert_eq!(i.name(lang(&i, "fr")), "Français"); + assert_eq!(i.name(lang(&i, "ar")), "ar"); + } + + #[test] + fn rtl() { + let i = all(L).0; + assert!(i.rtl(lang(&i, "ar"))); + assert!(!i.rtl(lang(&i, "fr"))); + assert!(!i.rtl(Lang::EN)); + } + + #[test] + fn tags() { + let i = all(L).0; + for (tag, want) in [ + ("fr_CA.UTF-8", Some("fr")), + ("pt-PT", Some("pt-br")), + ("pt_BR", Some("pt-br")), + ("zh_TW.UTF-8", Some("zh-hant")), + ("zh-HK", Some("zh-hant")), + ("zh", Some("zh-hans")), + ("zh-Hant-MO", Some("zh-hant")), + ("zh_CN", Some("zh-hans")), + ("C", Some("en")), + ("POSIX", Some("en")), + ("en_US", Some("en")), + ("ja_JP", None), + ("", None), + ("@@", None), + ] { + assert_eq!(i.match_tag(tag).map(|l| i.code(l)), want, "{tag}"); + } + } + + #[test] + fn match_tags_takes_first_match() { + let i = all(L).0; + assert_eq!(i.match_tags(["ja_JP", "fr_FR"]), lang(&i, "fr")); + assert_eq!(i.match_tags(["ja"]), Lang::EN); + assert_eq!(i.match_tags([]), Lang::EN); + } + + #[test] + fn pseudo_marks_every_lookup() { + let i = i18n(); + assert_eq!(i.t(Lang::PSEUDO, "Save"), "⟦Save⟧"); + assert_eq!(i.t(Lang::PSEUDO, "A\nB"), "⟦A⟧\n⟦B⟧"); + assert_eq!(i.tc(Lang::PSEUDO, "menu", "Open"), "⟦Open⟧"); + assert_eq!(i.tid(Lang::PSEUDO, "file.save", "Save"), "⟦Save⟧"); + assert_eq!(i.trn(Lang::PSEUDO, 2, "{n} x", "{n} xs"), "⟦2 xs⟧"); + assert!(!i.languages().contains(&Lang::PSEUDO)); + assert_eq!(i.from_code("pseudo"), cfg!(debug_assertions).then_some(Lang::PSEUDO)); + assert_eq!(i.get(Lang::PSEUDO, "key", "Esc"), Some("⟦Esc⟧")); + assert_eq!(i.looked_up().into_iter().collect::>(), ["A\nB", "Esc", "Open", "Save"]); + } + + static F: &Embedded = &[ + ("pt", "", ""), + ("pt-br", "", ""), + ("sr", "\tSave\tСачувај\n", ""), + ("sr-latn", "", ""), + ("fr", "\tDelete\tSupprimer\nkey\tEsc\tÉchap\n", ""), + ]; + + #[test] + fn european_portuguese_prefers_pt() { + let i = all(F).0; + for tag in ["pt", "pt-PT", "pt_PT.UTF-8"] { + assert_eq!(i.match_tag(tag).map(|l| i.code(l)), Some("pt"), "{tag}"); + } + assert_eq!(i.match_tag("pt_BR").map(|l| i.code(l)), Some("pt-br")); + } + + #[test] + fn script_locale_does_not_fall_back_across_scripts() { + let i = all(F).0; + assert_eq!(i.t(lang(&i, "sr-latn"), "Save"), "Save"); + } + + #[test] + fn glibc_script_modifier_picks_the_script() { + let i = all(F).0; + for (tag, want) in [("sr_RS.UTF-8@latin", Some("sr-latn")), ("sr_RS@latin", Some("sr-latn")), ("sr_RS.UTF-8", Some("sr"))] { + assert_eq!(i.match_tag(tag).map(|l| i.code(l)), want, "{tag}"); + } + let cyrillic_only = all(&[("sr", "", "")]).0; + assert_eq!(cyrillic_only.match_tag("sr_RS@latin"), None); + } + + #[cfg(all(unix, not(any(target_vendor = "apple", target_os = "android"))))] + #[test] + fn posix_env_keeps_the_modifier() { + let env = |vars: &'static [(&'static str, &'static str)]| move |k: &str| vars.iter().find(|(n, _)| *n == k).map(|(_, v)| (*v).to_owned()); + assert_eq!(posix_env_tags(env(&[("LANG", "sr_RS.UTF-8@latin")])), ["sr_RS.UTF-8@latin"]); + assert_eq!(posix_env_tags(env(&[("LANGUAGE", "fr:de"), ("LC_ALL", ""), ("LANG", "sr_RS@latin")])), ["fr", "de", "sr_RS@latin"]); + let i = all(F).0; + assert_eq!(i.code(i.match_tags(posix_env_tags(env(&[("LANG", "sr_RS.UTF-8@latin")])).iter().map(String::as_str))), "sr-latn"); + } + + #[test] + fn get_has_no_plain_fallback() { + let i = all(F).0; + let fr = lang(&i, "fr"); + assert_eq!(i.get(fr, "key", "Esc"), Some("Échap")); + assert_eq!(i.get(fr, "key", "Delete"), None); + assert_eq!(i.get(fr, "", "Delete"), Some("Supprimer")); + assert_eq!(i.get(Lang::EN, "", "Delete"), None); + } + + static C: &Embedded = &[("pt", "menu\tOpen\tAbrir menu\n@id\tfile.open\tAbrir ficheiro\n", ""), ("pt-br", "\tOpen\tAbrir\n", "")]; + + #[test] + fn locale_before_context() { + let i = all(C).0; + let br = lang(&i, "pt-br"); + assert_eq!(i.tc(br, "menu", "Open"), "Abrir"); + assert_eq!(i.tid(br, "file.open", "Open"), "Abrir"); + assert_eq!(i.tc(lang(&i, "pt"), "menu", "Open"), "Abrir menu"); + } + + static PT: &Embedded = &[ + ("pt", "@plural\t{n} file|{n} files\t{n} ficheiro|{n} ficheiros\n", ""), + ("pt-br", "@plural\t{n} file|{n} files\t{n} arquivo|{n} arquivos\n", ""), + ]; + + #[test] + fn european_portuguese_zero_is_plural() { + let i = all(PT).0; + assert_eq!(i.trn(lang(&i, "pt"), 0, "{n} file", "{n} files"), "0 ficheiros"); + assert_eq!(i.trn(lang(&i, "pt"), 1, "{n} file", "{n} files"), "1 ficheiro"); + assert_eq!(i.trn(lang(&i, "pt-br"), 0, "{n} file", "{n} files"), "0 arquivo"); + } + + /// The same counts as craft-i18n's checker (`plural_form_counts`), which rejects a catalog + /// whose `@plural` rows list any other. + #[test] + fn plural_form_counts() { + for (locale, n) in + [("en", 2), ("fr", 2), ("ja", 1), ("vi", 1), ("ru", 3), ("pl", 3), ("cs", 3), ("sl", 4), ("ar", 6), ("pt", 2), ("pt-br", 2), ("tr", 2)] + { + assert_eq!(plural_rules(locale).map(|(_, c)| c.len()), Some(n), "{locale}"); + } + } + + #[test] + fn script_subtag_never_picks_another_script() { + let i = all(&[("sr", "", ""), ("zh-hant", "", ""), ("fr", "", ""), ("be", "", "")]).0; + for (tag, want) in [ + ("sr-Latn-RS", None), + ("sr-Cyrl-RS", Some("sr")), + ("zh-Hant-TW", Some("zh-hant")), + ("zh-Hans", None), + ("fr-Latn-FR", Some("fr")), + ("be_BY@latin", None), + ("be_BY", Some("be")), + ] { + assert_eq!(i.match_tag(tag).map(|l| i.code(l)), want, "{tag}"); + } + } + + #[test] + fn duplicate_row_keeps_the_first() { + let (i, problems) = all(&[("fr", "\tSave\tEnregistrer\n\tSave\tSauver\n", "")]); + assert_eq!(problems, ["fr/common: line 2: defined twice (same context and source text); the first is kept"]); + assert_eq!(i.t(lang(&i, "fr"), "Save"), "Enregistrer"); + } + + #[test] + fn reserved_and_repeated_codes_are_skipped() { + let (i, problems) = all(&[("en", "\tSave\tX\n", ""), ("pseudo", "", ""), ("fr", "\tSave\tEnregistrer\n", ""), ("FR", "\tSave\tY\n", "")]); + assert_eq!(problems.len(), 3, "{problems:?}"); + assert_eq!(i.t(Lang::EN, "Save"), "Save"); + assert_eq!(i.t(lang(&i, "fr"), "Save"), "Enregistrer"); + assert_eq!(i.languages().len(), 2); + } + + #[test] + fn only_offered_locales_are_offered() { + let (i, _) = I18n::new(&[("ar", "\tSave\tحفظ\n", ""), ("pt", "\tUndo\tDesfazer\n", ""), ("pt-br", "", "")], &["PT-BR"]); + let codes: Vec<&str> = i.languages().into_iter().map(|l| i.code(l)).collect(); + assert_eq!(codes, ["en", "pt-br"]); + assert_eq!(i.from_code("ar"), None); + assert_eq!(i.match_tag("ar_EG"), None); + assert_eq!(i.match_tag("pt_PT"), i.from_code("pt-br")); + assert_eq!(i.t(lang(&i, "pt-br"), "Undo"), "Desfazer"); + } + + #[test] + fn region_after_script_falls_back() { + let i = all(&[("zh-hant", "\tSave\t儲存\n", ""), ("zh-hant-hk", "", "")]).0; + assert_eq!(i.t(lang(&i, "zh-hant-hk"), "Save"), "儲存"); + } + + #[test] + fn more_right_to_left() { + let i = all(&[("ps", "", ""), ("ckb", "", ""), ("yi", "", ""), ("ug", "", "")]).0; + assert!(i.languages().into_iter().skip(1).all(|l| i.rtl(l))); + } + + /// Prints what this machine reports, so CI logs show `system_lang` on each platform. + #[test] + fn system_lang_reads_the_system() { + let i = all(&[("fr", "", ""), ("de", "", ""), ("pt-br", "", ""), ("sr-latn", "", "")]).0; + let first = i.system_lang(); + println!("sys-locale: {:?}; system_lang: {}", sys_locale::get_locales().collect::>(), i.code(first)); + assert_eq!(i.system_lang(), first); + } +}