diff --git a/crates/i18n/Cargo.toml b/crates/i18n/Cargo.toml index 94f0e295..45e21775 100644 --- a/crates/i18n/Cargo.toml +++ b/crates/i18n/Cargo.toml @@ -8,7 +8,7 @@ rust-version.workspace = true repository.workspace = true [dependencies] -intl_pluralrules = "7" +intl_pluralrules = "=7.0.2" unic-langid = "0.9" sys-locale = "0.3" diff --git a/crates/i18n/README.md b/crates/i18n/README.md index d9a7ebb4..df337971 100644 --- a/crates/i18n/README.md +++ b/crates/i18n/README.md @@ -19,7 +19,8 @@ use std::sync::LazyLock; include!(concat!(env!("OUT_DIR"), "/craft_i18n.rs")); // pub static CRAFT_I18N: &craft_i18n::Embedded pub static I18N: LazyLock = LazyLock::new(|| { - let (i18n, problems) = craft_i18n::I18n::new(CRAFT_I18N); + let offered: Vec<&str> = LANGUAGES.iter().map(|l| l.code).collect(); // the app's own list + let (i18n, problems) = craft_i18n::I18n::new(CRAFT_I18N, &offered); problems.iter().for_each(|p| log::warn!("translation catalog: {p}; that string shows in English")); i18n }); @@ -34,24 +35,40 @@ The app keeps its `tl!` macro, its language setting (`"auto"` → `I18N.system_l ## Behaviour -- **Fallback:** `pt-br` app rows → `pt-br` shared rows → `pt` → the English source text. App rows - override shared ones in their own locale. Only a region falls back (`sr-latn` never falls back to - Cyrillic `sr`, and a system locale of `sr_RS@latin` picks `sr-latn`, never `sr`). A missing translation never errors. -- **Lookups:** `t` (plain), `tc` (with a context such as `menu` or `@msg`, falling back to plain), - `tid` (an `@id` command id, falling back to the English label), `trn` (a count, `@plural` rows, - `{n}` filled), `get` (one context along the locale chain, so `pt-br` can return a `pt` row, but never the - plain row or English: a key cap's name under `key`, or "is there a translation?"), and `fmt` for `{name}` placeholders, filled in one pass. -- **Bad rows** (malformed, an unknown `@` context, a wrong plural form count) are skipped and - returned by `I18n::new` for the app to log; craft-i18n's CI rejects them before they ship. +- **Offered locales:** `I18n::new` takes the codes the app offers (its own language list), so an + app never offers a language its interface isn't ready for (Arabic without right-to-left layout, + say). `languages()`, `from_code`, `match_tag` and `system_lang` only return offered locales; an + embedded locale that isn't offered still serves as a fallback (`pt` for an offered `pt-br`). +- **Fallback:** locale by locale along the chain, `pt-br` → `pt` → the English source text. In + each locale, app rows override shared ones, and `tc`/`tid` try their context or `@id` row before + the plain row, so `pt-br`'s plain row beats `pt`'s `menu` row. Only a region falls back (`pt-br` + → `pt`, `zh-hant-hk` → `zh-hant`); `sr-latn` never falls back to Cyrillic `sr`. A missing + translation never errors. +- **Lookups:** `t` (plain), `tc` (with a context such as `menu` or `@msg`, else plain), `tid` (an + `@id` command id, else the English label), `trn` (a count, `@plural` rows, `{n}` filled), `get` + (one context along the locale chain, so `pt-br` can return a `pt` row, but never the plain row + or English: a key cap's name under `key`, or "is there a translation?"), and `fmt` for `{name}` + placeholders, filled in one pass. +- **Bad rows** (malformed, an unknown `@` context, a wrong plural form count, a row defined twice, + of which the first is kept) and bad locales (`en`, `pseudo`, a code embedded twice in any case) + are skipped and returned by `I18n::new` for the app to log; craft-i18n's CI rejects them before + they ship. - **Plurals:** forms are listed in CLDR order (zero, one, two, few, many, other) for the categories whole numbers 0 to 1000 reach, from `intl_pluralrules`; a count in any other category takes the - last form. Keep `intl_pluralrules` on the same version as craft-i18n's checker. + last form. CLDR's `pt` is Brazilian, so `pt` uses `pt-PT`'s rules (0 is plural) and `pt-br` + `pt`'s. `intl_pluralrules` is pinned to the version craft-i18n's checker uses, and + `plural_form_counts` holds the same counts. - **Names:** a language's name comes from its shared catalog's `@id language.name` row, else its code; English is "English". -- **Right to left:** `rtl` is true for `ar`, `fa`, `he` and `ur`; putting text into display order - is the app's job. -- **System locale:** `system_lang` matches `sys-locale`'s list (`LANG`, macOS and Windows preferred - languages, the browser's on the web): `fr_CA.UTF-8` → `fr`, `zh-TW` → `zh-hant`, `pt-PT` → - `pt-br` when that is the only Portuguese, `C`/`POSIX` → English. -- **Pseudo locale** (`pseudo`, test-only, never in `languages()`): every lookup comes back as - `⟦…⟧`, line by line, so text drawn without the marks never went through translation. +- **Right to left:** `rtl` is true for `ar`, `ckb`, `dv`, `fa`, `he`, `ps`, `sd`, `ug`, `ur` and + `yi`; putting text into display order is the app's job. +- **System locale:** `system_lang` matches the system's list once and caches it: on desktop Unix + `LANGUAGE`, `LC_ALL`, `LC_MESSAGES` and `LANG`, read raw; elsewhere `sys-locale`'s (macOS and + Windows preferred languages, the browser's on the web). `fr_CA.UTF-8` → `fr`, `zh-TW` → + `zh-hant`, `pt-PT` → `pt-br` when that is the only Portuguese, `C`/`POSIX` → English. A script + is never crossed: `sr_RS@latin` and `sr-Latn-RS` pick `sr-latn`, never `sr`, while `sr-Cyrl-RS` + picks `sr`. +- **Pseudo locale** (`pseudo`, debug builds only, never in `languages()`): every lookup, `get` + included, comes back as `⟦…⟧`, line by line, so text drawn without the marks never went through + translation. `looked_up()` lists the source texts looked up so far, for an audit against a + complete catalog. diff --git a/crates/i18n/src/catalog.rs b/crates/i18n/src/catalog.rs index 43d0d733..50d731b2 100644 --- a/crates/i18n/src/catalog.rs +++ b/crates/i18n/src/catalog.rs @@ -2,6 +2,8 @@ //! applies them strictly, and here a row that breaks one is skipped and reported, so an app built //! against a newer catalog still runs, with that string in English. +use std::collections::HashSet; + /// A row that passed, with its columns unescaped. pub(crate) struct Entry { pub context: String, @@ -13,17 +15,21 @@ pub(crate) struct Entry { const RESERVED: &[&str] = &["@plural", "@id", "@msg"]; /// The rows of a catalog whose locale has `forms` plural forms, and one problem (`line N: …`) per -/// row skipped. +/// row skipped. Of a row defined twice, the first is kept, as in craft-i18n's checker. pub(crate) fn parse(text: &str, forms: usize) -> (Vec, Vec) { let text = text.strip_prefix('\u{feff}').unwrap_or(text); let mut rows = Vec::new(); let mut problems = Vec::new(); + let mut seen = HashSet::new(); for (i, line) in text.split('\n').enumerate() { let line = line.strip_suffix('\r').unwrap_or(line); if line.is_empty() || line.starts_with('#') { continue; } match parse_row(line, forms) { + Ok(row) if !seen.insert((row.context.clone(), row.source.clone())) => { + problems.push(format!("line {}: defined twice (same context and source text); the first is kept", i + 1)); + } Ok(row) => rows.push(row), Err(e) => problems.push(format!("line {}: {e}", i + 1)), } diff --git a/crates/i18n/src/lib.rs b/crates/i18n/src/lib.rs index 4387aa4f..f44bb5a2 100644 --- a/crates/i18n/src/lib.rs +++ b/crates/i18n/src/lib.rs @@ -7,8 +7,8 @@ mod catalog; -use std::collections::HashMap; -use std::sync::{Mutex, PoisonError}; +use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::sync::{Mutex, OnceLock, PoisonError}; use intl_pluralrules::{PluralCategory, PluralRuleType, PluralRules}; use unic_langid::LanguageIdentifier; @@ -32,9 +32,12 @@ impl Lang { struct Locale { code: String, rows: HashMap>, - /// Itself, then its base language if that is embedded too (`pt-br`, `pt`). Empty for English - /// and the pseudo locale. + /// Itself, then its parent without the region if that is embedded too (`pt-br`, `pt`; + /// `zh-hant-hk`, `zh-hant`). Empty for English and the pseudo locale. chain: Vec, + /// The app offers it: in [`I18n::languages`], found by code and system tag. An embedded locale + /// the app doesn't offer still serves as a fallback. + offered: bool, /// Plural rules and the categories `@plural` forms list, in order. plural: Option<(PluralRules, Vec)>, } @@ -42,18 +45,29 @@ struct Locale { /// The embedded catalogs, ready for lookups. Build it once, at start-up. pub struct I18n { locales: Vec, - /// Marked strings for the pseudo locale, leaked so lookups can hand out `&str`. Test-only. - pseudo: Mutex>, + /// Marked strings for the pseudo locale by source text, leaked so lookups can hand out `&str`. + /// Debug builds only; the keys are what [`I18n::looked_up`] reports. + // ponytail: grows by one leak per distinct source text, fine for a test run; an arena if the + // pseudo locale ever ships. + pseudo: Mutex>, + /// [`I18n::system_lang`], read once. + system: OnceLock, } impl I18n { - /// Read `embedded`. Rows that can't be used are skipped and come back as problems + /// Read `embedded`, offering the locales in `offered` (any case; the app's own language list), + /// so an app never offers a language its interface isn't ready for (right to left, say). Rows + /// and locales that can't be used are skipped and come back as problems /// (`/: line N: …`) for the app to log. - pub fn new(embedded: &'static Embedded) -> (I18n, Vec) { + pub fn new(embedded: &'static Embedded, offered: &[&str]) -> (I18n, Vec) { let mut problems = Vec::new(); let mut locales = vec![bare("en"), bare("pseudo")]; for (code, shared, app) in embedded { let code = code.to_ascii_lowercase(); + if locales.iter().any(|l| l.code == code) { + problems.push(format!("{code}: reserved or embedded twice; skipped")); + continue; + } let plural = plural_rules(&code); if plural.is_none() { problems.push(format!("{code}: no plural rules; plurals show in English")); @@ -67,15 +81,16 @@ impl I18n { rows.entry(e.context).or_default().insert(e.source, e.translation); } } - locales.push(Locale { code, rows, chain: Vec::new(), plural }); + let offered = offered.iter().any(|o| o.eq_ignore_ascii_case(&code)); + locales.push(Locale { code, rows, chain: Vec::new(), plural, offered }); } for i in 2..locales.len() { let mut chain = vec![i]; - // Only a region falls back to its language (`pt-br` → `pt`); a script doesn't, since - // `sr-latn` → `sr` would show Cyrillic. + // Only a region falls back (`pt-br` → `pt`, `zh-hant-hk` → `zh-hant`); a script + // doesn't, since `sr-latn` → `sr` would show Cyrillic. let code = locales.get(i).map_or("", |l| l.code.as_str()); - let base = match code.split_once('-') { - Some((language, region)) if is_region(region) => language, + let base = match code.rsplit_once('-') { + Some((parent, region)) if is_region(region) => parent, _ => "", }; if let Some(b) = locales.iter().skip(2).position(|l| l.code == base) @@ -87,12 +102,19 @@ impl I18n { l.chain = chain; } } - (I18n { locales, pseudo: Mutex::default() }, problems) + (I18n { locales, pseudo: Mutex::default(), system: OnceLock::new() }, problems) } - /// The language with this code (any case): `en`, `pseudo`, or an embedded locale. + /// The language with this code (any case): `en`, an offered locale, or in debug builds + /// `pseudo`. pub fn from_code(&self, code: &str) -> Option { - self.locales.iter().position(|l| l.code.eq_ignore_ascii_case(code)).map(Lang) + let i = self.locales.iter().position(|l| l.code.eq_ignore_ascii_case(code))?; + let usable = match Lang(i) { + Lang::EN => true, + Lang::PSEUDO => cfg!(debug_assertions), + _ => self.locales.get(i).is_some_and(|l| l.offered), + }; + usable.then_some(Lang(i)) } /// The language's code, lowercase BCP 47 (`pt-br`). @@ -105,30 +127,35 @@ impl I18n { if lang == Lang::PSEUDO { return self.mark(source); } - self.find(lang, "", source).unwrap_or(source) + self.find(lang, &[("", source)]).unwrap_or(source) } - /// `source` with a disambiguating `context` (`menu`, `@msg`); falls back to [`I18n::t`]. + /// `source` with a disambiguating `context` (`menu`, `@msg`); in each locale of the chain the + /// context row, then the plain one, so `pt-br`'s plain row beats `pt`'s context row. pub fn tc<'a>(&'a self, lang: Lang, context: &str, source: &'a str) -> &'a str { if lang == Lang::PSEUDO { return self.mark(source); } - self.find(lang, context, source).unwrap_or_else(|| self.t(lang, source)) + self.find(lang, &[(context, source), ("", source)]).unwrap_or(source) } /// The translation of (`context`, `source`) along `lang`'s chain, with no fallback to the plain /// row or the English: for strings that must not borrow another meaning's wording (a key cap's /// name under context `key`), or to ask whether a translation exists. - pub fn get(&self, lang: Lang, context: &str, source: &str) -> Option<&str> { - self.find(lang, context, source) + pub fn get<'a>(&'a self, lang: Lang, context: &str, source: &'a str) -> Option<&'a str> { + if lang == Lang::PSEUDO { + return Some(self.mark(source)); + } + self.find(lang, &[(context, source)]) } - /// A command's label by its `id` (`@id` rows); falls back to [`I18n::t`] of the English `label`. + /// A command's label by its `id` (`@id` rows); in each locale of the chain the `@id` row, then + /// the plain row of the English `label`. pub fn tid<'a>(&'a self, lang: Lang, id: &str, label: &'a str) -> &'a str { if lang == Lang::PSEUDO { return self.mark(label); } - self.find(lang, "@id", id).unwrap_or_else(|| self.t(lang, label)) + self.find(lang, &[("@id", id), ("", label)]).unwrap_or(label) } /// A count's wording in `lang`, `{n}` filled in. `one` and `other` are the English forms; @@ -155,10 +182,10 @@ impl I18n { text } - /// The languages to offer: English first, then the embedded locales by code. Never the + /// The languages to offer: English first, then the offered locales by code. Never the /// pseudo locale. pub fn languages(&self) -> Vec { - let mut embedded: Vec = (2..self.locales.len()).collect(); + let mut embedded: Vec = (2..self.locales.len()).filter(|&i| self.locales.get(i).is_some_and(|l| l.offered)).collect(); embedded.sort_by_key(|&i| self.locales.get(i).map(|l| l.code.as_str())); std::iter::once(Lang::EN).chain(embedded.into_iter().map(Lang)).collect() } @@ -177,7 +204,7 @@ impl I18n { pub fn rtl(&self, lang: Lang) -> bool { let code = self.code(lang); let language = code.split('-').next().unwrap_or(code); - matches!(language, "ar" | "fa" | "he" | "ur") + matches!(language, "ar" | "ckb" | "dv" | "fa" | "he" | "ps" | "sd" | "ug" | "ur" | "yi") } /// The language for a system locale tag (`fr_CA.UTF-8`, `zh-Hant-MO`); `C` and `POSIX` are @@ -196,14 +223,22 @@ impl I18n { } /// The language the system asks for (`LANG` and friends, the macOS and Windows preferred - /// languages, the browser's on the web); English if none fits. + /// languages, the browser's on the web); English if none fits. Read once, then cached. pub fn system_lang(&self) -> Lang { - // sys-locale drops glibc's script modifier (`sr_RS@latin` → `sr-RS`), so read its variables raw. - #[cfg(all(unix, not(any(target_vendor = "apple", target_os = "android"))))] - let tags = posix_env_tags(|k| std::env::var(k).ok()); - #[cfg(not(all(unix, not(any(target_vendor = "apple", target_os = "android")))))] - let tags: Vec = sys_locale::get_locales().collect(); - self.match_tags(tags.iter().map(String::as_str)) + *self.system.get_or_init(|| { + // sys-locale drops glibc's script modifier (`sr_RS@latin` → `sr-RS`), so read its variables raw. + #[cfg(all(unix, not(any(target_vendor = "apple", target_os = "android"))))] + let tags = posix_env_tags(|k| std::env::var(k).ok()); + #[cfg(not(all(unix, not(any(target_vendor = "apple", target_os = "android")))))] + let tags: Vec = sys_locale::get_locales().collect(); + self.match_tags(tags.iter().map(String::as_str)) + }) + } + + /// Every source text looked up in the pseudo locale so far, for an audit to check against a + /// complete catalog. Plurals aren't recorded. + pub fn looked_up(&self) -> BTreeSet { + self.pseudo.lock().unwrap_or_else(PoisonError::into_inner).keys().cloned().collect() } /// `source` marked for the pseudo locale, made once per string. @@ -216,10 +251,13 @@ impl I18n { self.locales.get(lang.0) } - /// The first translation of (`context`, `source`) along `lang`'s chain. - fn find(&self, lang: Lang, context: &str, source: &str) -> Option<&str> { + /// Along `lang`'s chain, locale by locale, the first of `keys` (context, source text) found. + fn find(&self, lang: Lang, keys: &[(&str, &str)]) -> Option<&str> { let chain = self.locale(lang).map(|l| l.chain.as_slice())?; - chain.iter().find_map(|&i| self.locales.get(i)?.rows.get(context)?.get(source).map(String::as_str)) + chain.iter().find_map(|&i| { + let rows = &self.locales.get(i)?.rows; + keys.iter().find_map(|(context, source)| rows.get(*context)?.get(*source).map(String::as_str)) + }) } } @@ -251,8 +289,9 @@ pub fn fmt(template: &str, args: &[(&str, &str)]) -> String { /// Codes to try for a locale tag, most specific first: `zh_TW.UTF-8` → `zh-tw`, `zh-hant`, `zh`. /// Chinese without a script picks one by region; Portuguese falls back to `pt-br` last, when -/// there is no `pt`. A glibc script modifier (`sr_RS@latin`) becomes the script subtag, and only -/// codes with that script are tried, so Latin Serbian never picks Cyrillic `sr`. +/// there is no `pt`. A glibc script modifier (`sr_RS@latin`) becomes the script subtag. With a +/// script (`sr-Latn-RS`), a code without one is tried only when that is its language's usual +/// script, so Latin Serbian never picks Cyrillic `sr`, while `sr-Cyrl-RS` does. fn candidates(tag: &str) -> Vec { let tag = tag.trim().to_ascii_lowercase(); let base = tag.split(['.', '@']).next().unwrap_or("").replace('_', "-"); @@ -260,16 +299,19 @@ fn candidates(tag: &str) -> Vec { let Some(&primary) = parts.first() else { return Vec::new(); }; - let script = match tag.split_once('@').map(|(_, m)| m) { + let modifier = match tag.split_once('@').map(|(_, m)| m) { Some("latin") => Some("latn"), Some("cyrillic") => Some("cyrl"), _ => None, }; - if let Some(s) = script { + if let Some(s) = modifier { parts.insert(1, s); } + let script = parts.get(1).copied().filter(|s| s.len() == 4 && s.bytes().all(|b| b.is_ascii_alphabetic())); let mut out: Vec = (1..=parts.len()).rev().map(|n| parts.get(..n).unwrap_or_default().join("-")).collect(); - if let Some(s) = script { + if let Some(s) = script + && usual_script(primary).is_some_and(|usual| usual != s) + { out.retain(|c| c.split('-').any(|p| p == s)); } if primary == "zh" && !parts.iter().any(|p| matches!(*p, "hans" | "hant")) { @@ -297,6 +339,20 @@ fn marked(text: &str) -> String { format!("⟦{}⟧", text.replace('\n', "⟧\n⟦")) } +/// The script a code without one means, for languages written in more than one (CLDR likely +/// subtags). Any other language is taken to have one script. +// ponytail: the multi-script languages a Craft app could plausibly ship; add a row with the locale. +fn usual_script(language: &str) -> Option<&'static str> { + Some(match language { + "be" | "mn" | "sr" => "cyrl", + "zh" => "hans", + "az" | "bs" | "ha" | "uz" => "latn", + "ks" | "sd" => "arab", + "pa" => "guru", + _ => return None, + }) +} + /// A BCP 47 region subtag: two letters or three digits. fn is_region(subtag: &str) -> bool { (subtag.len() == 2 && subtag.bytes().all(|b| b.is_ascii_alphabetic())) || (subtag.len() == 3 && subtag.bytes().all(|b| b.is_ascii_digit())) @@ -304,13 +360,15 @@ fn is_region(subtag: &str) -> bool { /// A locale with no catalogs (English, the pseudo locale). fn bare(code: &str) -> Locale { - Locale { code: code.to_owned(), rows: HashMap::new(), chain: Vec::new(), plural: None } + Locale { code: code.to_owned(), rows: HashMap::new(), chain: Vec::new(), plural: None, offered: false } } /// `code`'s CLDR cardinal rules and the categories whole numbers 0..=1000 reach, in CLDR order /// (zero, one, two, few, many, other): one `@plural` form each. Regional and scripted codes fall -/// back to the language's rules. +/// back to the language's rules. CLDR's `pt` is Brazilian (0 is singular), so Portuguese other +/// than `pt-br` takes `pt-PT`'s. fn plural_rules(code: &str) -> Option<(PluralRules, Vec)> { + let code = if code == "pt" || (code.starts_with("pt-") && code != "pt-br") { "pt-pt" } else { code }; let id: LanguageIdentifier = code.parse().ok()?; let language = LanguageIdentifier::from_parts(id.language, None, None, &[]); let rules = PluralRules::create(id, PluralRuleType::CARDINAL).or_else(|_| PluralRules::create(language, PluralRuleType::CARDINAL)).ok()?; @@ -347,8 +405,14 @@ mod tests { ("de", "\tBad\n@nope\tA\tB\n\tUndo\tRückgängig\n", ""), ]; + /// `embedded`, offering every locale in it. + fn all(embedded: &'static Embedded) -> (I18n, Vec) { + let codes: Vec<&str> = embedded.iter().map(|(c, _, _)| *c).collect(); + I18n::new(embedded, &codes) + } + fn i18n() -> I18n { - I18n::new(E).0 + all(E).0 } fn lang(i: &I18n, code: &str) -> Lang { @@ -397,7 +461,7 @@ mod tests { #[test] fn bad_rows_skipped_and_reported() { - let (i, problems) = I18n::new(E); + let (i, problems) = all(E); assert_eq!(problems.len(), 2, "{problems:?}"); assert!(problems[0].starts_with("de/common: line 1:"), "{problems:?}"); assert!(problems[1].starts_with("de/common: line 2:"), "{problems:?}"); @@ -427,7 +491,7 @@ mod tests { #[test] fn one_other() { - let (i, problems) = I18n::new(P); + let (i, problems) = all(P); assert!(problems.is_empty(), "{problems:?}"); let fr = lang(&i, "fr"); assert_eq!(i.trn(fr, 0, "{n} page", "{n} pages"), "0 page"); @@ -436,7 +500,7 @@ mod tests { #[test] fn three_forms_ru() { - let i = I18n::new(P).0; + let i = all(P).0; let ru = lang(&i, "ru"); for (n, want) in [(1, "1 файл"), (3, "3 файла"), (5, "5 файлов"), (11, "11 файлов"), (21, "21 файл")] { assert_eq!(i.trn(ru, n, "{n} file", "{n} files"), want); @@ -445,20 +509,20 @@ mod tests { #[test] fn one_form_ja() { - let i = I18n::new(P).0; + let i = all(P).0; assert_eq!(i.trn(lang(&i, "ja"), 7, "{n} file", "{n} files"), "7 個"); } #[test] fn english_fallback() { - let i = I18n::new(P).0; + let i = all(P).0; assert_eq!(i.trn(Lang::EN, 1, "{n} file", "{n} files"), "1 file"); assert_eq!(i.trn(lang(&i, "fr"), 2, "{n} x", "{n} xs"), "2 xs"); } #[test] fn beyond_sweep() { - let i = I18n::new(P).0; + let i = all(P).0; assert_eq!(i.trn(lang(&i, "fr"), 1_000_000, "{n} page", "{n} pages"), "1000000 pages"); } @@ -467,14 +531,14 @@ mod tests { #[test] fn languages_order() { - let i = I18n::new(L).0; + let i = all(L).0; let codes: Vec<&str> = i.languages().into_iter().map(|l| i.code(l)).collect(); assert_eq!(codes, ["en", "ar", "fr", "pt-br", "zh-hans", "zh-hant"]); } #[test] fn names() { - let i = I18n::new(L).0; + let i = all(L).0; assert_eq!(i.name(Lang::EN), "English"); assert_eq!(i.name(lang(&i, "fr")), "Français"); assert_eq!(i.name(lang(&i, "ar")), "ar"); @@ -482,7 +546,7 @@ mod tests { #[test] fn rtl() { - let i = I18n::new(L).0; + let i = all(L).0; assert!(i.rtl(lang(&i, "ar"))); assert!(!i.rtl(lang(&i, "fr"))); assert!(!i.rtl(Lang::EN)); @@ -490,7 +554,7 @@ mod tests { #[test] fn tags() { - let i = I18n::new(L).0; + let i = all(L).0; for (tag, want) in [ ("fr_CA.UTF-8", Some("fr")), ("pt-PT", Some("pt-br")), @@ -513,7 +577,7 @@ mod tests { #[test] fn match_tags_takes_first_match() { - let i = I18n::new(L).0; + let i = all(L).0; assert_eq!(i.match_tags(["ja_JP", "fr_FR"]), lang(&i, "fr")); assert_eq!(i.match_tags(["ja"]), Lang::EN); assert_eq!(i.match_tags([]), Lang::EN); @@ -528,7 +592,9 @@ mod tests { assert_eq!(i.tid(Lang::PSEUDO, "file.save", "Save"), "⟦Save⟧"); assert_eq!(i.trn(Lang::PSEUDO, 2, "{n} x", "{n} xs"), "⟦2 xs⟧"); assert!(!i.languages().contains(&Lang::PSEUDO)); - assert_eq!(i.from_code("pseudo"), Some(Lang::PSEUDO)); + assert_eq!(i.from_code("pseudo"), cfg!(debug_assertions).then_some(Lang::PSEUDO)); + assert_eq!(i.get(Lang::PSEUDO, "key", "Esc"), Some("⟦Esc⟧")); + assert_eq!(i.looked_up().into_iter().collect::>(), ["A\nB", "Esc", "Open", "Save"]); } static F: &Embedded = &[ @@ -541,7 +607,7 @@ mod tests { #[test] fn european_portuguese_prefers_pt() { - let i = I18n::new(F).0; + let i = all(F).0; for tag in ["pt", "pt-PT", "pt_PT.UTF-8"] { assert_eq!(i.match_tag(tag).map(|l| i.code(l)), Some("pt"), "{tag}"); } @@ -550,17 +616,17 @@ mod tests { #[test] fn script_locale_does_not_fall_back_across_scripts() { - let i = I18n::new(F).0; + let i = all(F).0; assert_eq!(i.t(lang(&i, "sr-latn"), "Save"), "Save"); } #[test] fn glibc_script_modifier_picks_the_script() { - let i = I18n::new(F).0; + let i = all(F).0; for (tag, want) in [("sr_RS.UTF-8@latin", Some("sr-latn")), ("sr_RS@latin", Some("sr-latn")), ("sr_RS.UTF-8", Some("sr"))] { assert_eq!(i.match_tag(tag).map(|l| i.code(l)), want, "{tag}"); } - let cyrillic_only = I18n::new(&[("sr", "", "")]).0; + let cyrillic_only = all(&[("sr", "", "")]).0; assert_eq!(cyrillic_only.match_tag("sr_RS@latin"), None); } @@ -570,17 +636,107 @@ mod tests { let env = |vars: &'static [(&'static str, &'static str)]| move |k: &str| vars.iter().find(|(n, _)| *n == k).map(|(_, v)| (*v).to_owned()); assert_eq!(posix_env_tags(env(&[("LANG", "sr_RS.UTF-8@latin")])), ["sr_RS.UTF-8@latin"]); assert_eq!(posix_env_tags(env(&[("LANGUAGE", "fr:de"), ("LC_ALL", ""), ("LANG", "sr_RS@latin")])), ["fr", "de", "sr_RS@latin"]); - let i = I18n::new(F).0; + let i = all(F).0; assert_eq!(i.code(i.match_tags(posix_env_tags(env(&[("LANG", "sr_RS.UTF-8@latin")])).iter().map(String::as_str))), "sr-latn"); } #[test] fn get_has_no_plain_fallback() { - let i = I18n::new(F).0; + let i = all(F).0; let fr = lang(&i, "fr"); assert_eq!(i.get(fr, "key", "Esc"), Some("Échap")); assert_eq!(i.get(fr, "key", "Delete"), None); assert_eq!(i.get(fr, "", "Delete"), Some("Supprimer")); assert_eq!(i.get(Lang::EN, "", "Delete"), None); } + + static C: &Embedded = &[("pt", "menu\tOpen\tAbrir menu\n@id\tfile.open\tAbrir ficheiro\n", ""), ("pt-br", "\tOpen\tAbrir\n", "")]; + + #[test] + fn locale_before_context() { + let i = all(C).0; + let br = lang(&i, "pt-br"); + assert_eq!(i.tc(br, "menu", "Open"), "Abrir"); + assert_eq!(i.tid(br, "file.open", "Open"), "Abrir"); + assert_eq!(i.tc(lang(&i, "pt"), "menu", "Open"), "Abrir menu"); + } + + static PT: &Embedded = &[ + ("pt", "@plural\t{n} file|{n} files\t{n} ficheiro|{n} ficheiros\n", ""), + ("pt-br", "@plural\t{n} file|{n} files\t{n} arquivo|{n} arquivos\n", ""), + ]; + + #[test] + fn european_portuguese_zero_is_plural() { + let i = all(PT).0; + assert_eq!(i.trn(lang(&i, "pt"), 0, "{n} file", "{n} files"), "0 ficheiros"); + assert_eq!(i.trn(lang(&i, "pt"), 1, "{n} file", "{n} files"), "1 ficheiro"); + assert_eq!(i.trn(lang(&i, "pt-br"), 0, "{n} file", "{n} files"), "0 arquivo"); + } + + /// The same counts as craft-i18n's checker (`plural_form_counts`), which rejects a catalog + /// whose `@plural` rows list any other. + #[test] + fn plural_form_counts() { + for (locale, n) in + [("en", 2), ("fr", 2), ("ja", 1), ("vi", 1), ("ru", 3), ("pl", 3), ("cs", 3), ("sl", 4), ("ar", 6), ("pt", 2), ("pt-br", 2), ("tr", 2)] + { + assert_eq!(plural_rules(locale).map(|(_, c)| c.len()), Some(n), "{locale}"); + } + } + + #[test] + fn script_subtag_never_picks_another_script() { + let i = all(&[("sr", "", ""), ("zh-hant", "", ""), ("fr", "", ""), ("be", "", "")]).0; + for (tag, want) in [ + ("sr-Latn-RS", None), + ("sr-Cyrl-RS", Some("sr")), + ("zh-Hant-TW", Some("zh-hant")), + ("zh-Hans", None), + ("fr-Latn-FR", Some("fr")), + ("be_BY@latin", None), + ("be_BY", Some("be")), + ] { + assert_eq!(i.match_tag(tag).map(|l| i.code(l)), want, "{tag}"); + } + } + + #[test] + fn duplicate_row_keeps_the_first() { + let (i, problems) = all(&[("fr", "\tSave\tEnregistrer\n\tSave\tSauver\n", "")]); + assert_eq!(problems, ["fr/common: line 2: defined twice (same context and source text); the first is kept"]); + assert_eq!(i.t(lang(&i, "fr"), "Save"), "Enregistrer"); + } + + #[test] + fn reserved_and_repeated_codes_are_skipped() { + let (i, problems) = all(&[("en", "\tSave\tX\n", ""), ("pseudo", "", ""), ("fr", "\tSave\tEnregistrer\n", ""), ("FR", "\tSave\tY\n", "")]); + assert_eq!(problems.len(), 3, "{problems:?}"); + assert_eq!(i.t(Lang::EN, "Save"), "Save"); + assert_eq!(i.t(lang(&i, "fr"), "Save"), "Enregistrer"); + assert_eq!(i.languages().len(), 2); + } + + #[test] + fn only_offered_locales_are_offered() { + let (i, _) = I18n::new(&[("ar", "\tSave\tحفظ\n", ""), ("pt", "\tUndo\tDesfazer\n", ""), ("pt-br", "", "")], &["PT-BR"]); + let codes: Vec<&str> = i.languages().into_iter().map(|l| i.code(l)).collect(); + assert_eq!(codes, ["en", "pt-br"]); + assert_eq!(i.from_code("ar"), None); + assert_eq!(i.match_tag("ar_EG"), None); + assert_eq!(i.match_tag("pt_PT"), i.from_code("pt-br")); + assert_eq!(i.t(lang(&i, "pt-br"), "Undo"), "Desfazer"); + } + + #[test] + fn region_after_script_falls_back() { + let i = all(&[("zh-hant", "\tSave\t儲存\n", ""), ("zh-hant-hk", "", "")]).0; + assert_eq!(i.t(lang(&i, "zh-hant-hk"), "Save"), "儲存"); + } + + #[test] + fn more_right_to_left() { + let i = all(&[("ps", "", ""), ("ckb", "", ""), ("yi", "", ""), ("ug", "", "")]).0; + assert!(i.languages().into_iter().skip(1).all(|l| i.rtl(l))); + } }