diff --git a/crates/i18n/Cargo.toml b/crates/i18n/Cargo.toml index 3daaca10..88e52d02 100644 --- a/crates/i18n/Cargo.toml +++ b/crates/i18n/Cargo.toml @@ -8,15 +8,16 @@ rust-version.workspace = true repository.workspace = true [dependencies] -# Numbers and plural rules, on CLDR data for the shipped locales (data/numbers.postcard). +# Numbers and plural rules, on CLDR data (data/numbers.postcard) generated for one ICU4X minor +# version, hence `~2.3`. fixed_decimal = { version = "0.7.2", default-features = false } -icu_decimal = { version = "2.3", default-features = false, features = ["serde"] } -icu_locale_core = { version = "2.3", default-features = false } -icu_locale_fallback = { version = "2.3", default-features = false, features = ["compiled_data"] } -icu_plurals = { version = "2.3", default-features = false, features = ["serde"] } -icu_provider = { version = "2.3", default-features = false, features = ["sync"] } -icu_provider_adapters = { version = "2.3", default-features = false } -icu_provider_blob = { version = "2.3", default-features = false } +icu_decimal = { version = "~2.3", default-features = false, features = ["serde"] } +icu_locale_core = { version = "~2.3", default-features = false } +icu_locale_fallback = { version = "~2.3", default-features = false, features = ["compiled_data"] } +icu_plurals = { version = "~2.3", default-features = false, features = ["serde"] } +icu_provider = { version = "~2.3", default-features = false, features = ["sync"] } +icu_provider_adapters = { version = "~2.3", default-features = false } +icu_provider_blob = { version = "~2.3", default-features = false } writeable = "0.6" sys-locale = "0.3" diff --git a/crates/i18n/README.md b/crates/i18n/README.md index d9f97cb7..63eb7184 100644 --- a/crates/i18n/README.md +++ b/crates/i18n/README.md @@ -46,8 +46,7 @@ The app keeps its `tl!` macro, its language setting (`"auto"` → `I18N.system_l translation never errors. - **Lookups:** `t` (plain), `tc` (with a context such as `menu` or `@msg`, else plain), `tid` (an `@id` command id, else the English label), `trn` and `trn_decimal` (a count, `@plural` rows, - `{n}` filled), `get` - (one context along the locale chain, so `pt-br` can return a `pt` row, but never the plain row + `{n}` filled), `get` (one context along the locale chain, so `pt-br` can return a `pt` row, but never the plain row or English: a key cap's name under `key`, or "is there a translation?"), and `fmt` for `{name}` placeholders, filled in one pass. - **Bad rows** (malformed, an unknown `@` context, a wrong plural form count, a row defined twice, @@ -59,8 +58,9 @@ The app keeps its `tl!` macro, its language setting (`"auto"` → `I18N.system_l craft-i18n's checker expects. `trn_decimal` picks the form from the digits shown (ICU4X plural operands), so English `1.0` is plural and French `1,5` singular. A count in a category integers never reach takes the `few` form if there is one (Russian and Ukrainian decimals, `1,5 файла`), - else the last. CLDR's `pt` is Brazilian, so `pt` uses `pt-PT`'s rules (0 is plural) and `pt-br` - `pt`'s. A language outside the baked data (below) has no plural rules and shows English forms. + else the last. Rules are per language, so `sr-latn` and `zh-hant` take `sr`'s and `zh`'s. CLDR's + `pt` is Brazilian, so `pt` uses `pt-PT`'s rules (0 is plural) and `pt-br` `pt`'s. A language + CLDR has no plural rules for shows English forms. - **Names:** a language's name comes from its shared catalog's `@id language.name` row, else its code; English is "English". - **Right to left:** `rtl` is true for `ar`, `ckb`, `dv`, `fa`, `he`, `ps`, `sd`, `ug`, `ur` and @@ -83,24 +83,28 @@ The app keeps its `tl!` macro, its language setting (`"auto"` → `I18N.system_l (`1.234.567`, Indian `12,34,567`), `number(value, decimals)` and `percent(ratio, decimals)` (`50 %`, Turkish `%50`). It follows the region, not the interface language, with the locale's digits (`١٬٢٣٤٫٥` in Egypt, `۱٬۲۳۴` in Iran) or, with `Digits::Latin`, `0`–`9`. With Latin - digits, a decimal or grouping separator the region sets replaces the locale's (English in Poland: - `1.234,5`). `I18n::format` is the format `trn`, `file_size` and `measure` use: the system - region's until `I18n::set_format` (the user's region setting or digit choice). -- **CLDR data:** decimal symbols, digits and plural rules (`data/numbers.postcard`, about 6 KB) and - percent patterns (`data/locales.rs`) cover the languages the catalogs and apps ship, with their - regional variants; `data/generate.sh` lists them and rebuilds both from CLDR. Add a language - there when a catalog adds one. The ICU4X code and data add about 90 KB to a stripped release - build and 80 KB to wasm. + digits, a decimal or grouping separator the system sets (Windows' and macOS's own settings) + replaces the locale's. `I18n::format` is the format `trn`, `file_size` and `measure` use: the + system region's until `I18n::set_format` (the user's region setting or digit choice), and again + after `I18n::reset_format`. A count in `trn` is therefore grouped and in the region's digits + (`1.234 Dateien`, `١٬٢٣٤ ملفًا`); for names written into documents or files (`Copy 1234`), use + `trn_plain`, which writes `0`–`9` without grouping. +- **CLDR data:** decimal symbols, digits and plural rules (`data/numbers.postcard`, about 10 KB) and + percent patterns (`data/locales.rs`) cover every CLDR locale; `data/generate.sh` rebuilds both + from CLDR. The ICU4X crates are pinned to one minor version (`~2.3`), the one the data is + generated for. The ICU4X code and data add about 90 KB to a stripped release build and 80 KB to + wasm. - **Formatting region:** `Region` holds how numbers, times and measurements are written, apart from the interface language: the number and time locales (BCP 47), metric or US measurement, A4 or Letter, and the decimal and grouping separators (or no grouping) and clock when the system - sets them; otherwise the number locale's CLDR data decides. `Region::system()` reads it once: Windows' regional format with the user's changes - (`HKCU\Control Panel\International`), Apple's current locale (`en_PL`, `en_GB@rg=uszzzz`), - `LC_ALL` or else `LC_NUMERIC`, `LC_TIME`, `LC_MEASUREMENT` and `LC_PAPER` each on its own over - `LANG` on Unix (`C` there is `en-US`; `LANG=C` gives way to the preferred language), the preferred language on the web and Android. `Region::from_setting` takes - the app's saved choice (`auto` or a tag like `pl-PL`). Paper and measurement follow CLDR's - territory lists; tags may carry `-u-ms-`, `-u-hc-` and `-u-rg-`. Ported from WordCraft's - `Region`. + sets them; otherwise the number locale's CLDR data decides. `Region::system()` reads it once: + Windows' regional format with the user's changes (`HKCU\Control Panel\International`), Apple's + current locale (`en_PL`, `en_GB@rg=uszzzz`), `LC_ALL` or else `LC_NUMERIC`, `LC_TIME`, + `LC_MEASUREMENT` and `LC_PAPER` each on its own over `LANG` on Unix (`C` there is `en-US`; + `LANG=C` gives way to the preferred language), the preferred language on the web and Android. + `Region::from_setting` takes the app's saved choice (`auto` or a tag like `pl-PL`). Paper and + measurement follow CLDR's territory lists; tags may carry `-u-ms-`, `-u-hc-` and `-u-rg-`. + Ported from WordCraft's `Region`. - **Pseudo locale** (`pseudo`, debug builds only, never in `languages()`): every lookup, `get` included, comes back as `⟦…⟧`, line by line, so text drawn without the marks never went through translation. `looked_up()` lists the source texts looked up so far, for an audit against a diff --git a/crates/i18n/data/generate.sh b/crates/i18n/data/generate.sh index 695a698a..91b6ab94 100755 --- a/crates/i18n/data/generate.sh +++ b/crates/i18n/data/generate.sh @@ -1,23 +1,22 @@ #!/usr/bin/env bash -# Regenerates the number and plural data in this directory from Unicode CLDR, for the languages -# the craft-i18n catalogs and the apps ship (each with its regional variants). +# Regenerates the number and plural data in this directory from Unicode CLDR, for every CLDR +# locale: 10 KB, of which CLDR's modern-coverage locales alone would be 7 KB. Lookups fall back +# at run time, so a locale whose data matches its parent's is left out. # -# Needs `cargo install icu4x-datagen --version 2.3.0` (the ICU4X version in Cargo.toml) and -# python3. Downloads CLDR's JSON release once into $CACHE. +# Needs `cargo install icu4x-datagen --version 2.3.0` (the ICU4X minor version in Cargo.toml) +# and python3. Downloads CLDR's JSON release once into $CACHE. set -euo pipefail CLDR=48.2.1 -LANGUAGES=(en ar bg cs da de el es et fa fi fr hi hr hu id it ja ka kk ko lt nl pl pt ru sl sr sv te tr uk vi zh) HERE=$(cd "$(dirname "$0")" && pwd) CACHE=${CACHE:-${XDG_CACHE_HOME:-$HOME/.cache}/craft-i18n-data} mkdir -p "$CACHE" -# `sr-Latn` and `zh-Hant` are named as well: their CLDR parent is the root, not `sr` or `zh`. TMPDIR=$CACHE icu4x-datagen --format blob --overwrite \ --cldr-tag "$CLDR" \ --markers DecimalSymbolsV1 DecimalDigitsV1 PluralsCardinalV1 \ - --locales "${LANGUAGES[@]}" sr-Latn zh-Hant \ + --locales full --deduplication maximal \ --out "$HERE/numbers.postcard" ZIP=$CACHE/icu4x-source-cache/github.com/unicode-org/cldr-json/releases/download/$CLDR/cldr-$CLDR-json-full.zip -python3 -I "$HERE/percent.py" "$ZIP" "$CLDR" "${LANGUAGES[@]}" > "$HERE/locales.rs" +python3 -I "$HERE/locales.py" "$ZIP" "$CLDR" > "$HERE/locales.rs" diff --git a/crates/i18n/data/percent.py b/crates/i18n/data/locales.py similarity index 75% rename from crates/i18n/data/percent.py rename to crates/i18n/data/locales.py index 3c00bb44..337bcfb9 100644 --- a/crates/i18n/data/percent.py +++ b/crates/i18n/data/locales.py @@ -1,6 +1,7 @@ -"""Writes locales.rs: the covered languages and the percent affixes from CLDR's JSON release. +"""Writes locales.rs from CLDR's JSON release: the languages with plural rules and every locale's +percent affixes. -Usage: percent.py ... +Usage: locales.py """ import json @@ -21,12 +22,14 @@ def affixes(numbers, numsys): def main(): - path, version, languages = sys.argv[1], sys.argv[2], sys.argv[3:] + path, version = sys.argv[1], sys.argv[2] found = {} with zipfile.ZipFile(path) as z: + plurals = json.loads(z.read("cldr-core/supplemental/plurals.json"))["supplemental"]["plurals-type-cardinal"] + languages = sorted(l for l in plurals if l != "root" and "-" not in l) for name in z.namelist(): m = re.fullmatch(r"cldr-numbers-full/main/([^/]+)/numbers\.json", name) - if not m or m.group(1).split("-")[0] not in languages: + if not m or m.group(1) == "root": continue locale = m.group(1) numbers = json.loads(z.read(name))["main"][locale]["numbers"] @@ -49,8 +52,11 @@ def main(): print(f"// @generated by generate.sh from Unicode CLDR {version} (Unicode License).") print() - print("/// The languages the baked number and plural data cover, each with its regional variants.") - print("pub(crate) const LANGUAGES: &[&str] = &[" + ", ".join(rust(l) for l in languages) + "];") + print("/// The languages CLDR has plural rules for; the baked data has them all.") + print("pub(crate) const PLURAL_LANGUAGES: &[&str] = &[") + for i in range(0, len(languages), 16): + print(" " + ", ".join(rust(l) for l in languages[i : i + 16]) + ",") + print("];") print() print("/// Percent signs around the number (CLDR's standard percent pattern and sign): locale, Latin") print("/// digits or the locale's own, text before the number, text after it. A locale missing here") diff --git a/crates/i18n/data/locales.rs b/crates/i18n/data/locales.rs index a2ad6e72..205a9661 100644 --- a/crates/i18n/data/locales.rs +++ b/crates/i18n/data/locales.rs @@ -1,7 +1,22 @@ // @generated by generate.sh from Unicode CLDR 48.2.1 (Unicode License). -/// The languages the baked number and plural data cover, each with its regional variants. -pub(crate) const LANGUAGES: &[&str] = &["en", "ar", "bg", "cs", "da", "de", "el", "es", "et", "fa", "fi", "fr", "hi", "hr", "hu", "id", "it", "ja", "ka", "kk", "ko", "lt", "nl", "pl", "pt", "ru", "sl", "sr", "sv", "te", "tr", "uk", "vi", "zh"]; +/// The languages CLDR has plural rules for; the baked data has them all. +pub(crate) const PLURAL_LANGUAGES: &[&str] = &[ + "af", "ak", "am", "an", "ar", "ars", "as", "asa", "ast", "az", "bal", "be", "bem", "bez", "bg", "bho", + "blo", "bm", "bn", "bo", "br", "brx", "bs", "ca", "ce", "ceb", "cgg", "chr", "ckb", "cs", "csw", "cv", + "cy", "da", "de", "doi", "dsb", "dv", "dz", "ee", "el", "en", "eo", "es", "et", "eu", "fa", "ff", + "fi", "fil", "fo", "fr", "fur", "fy", "ga", "gd", "gl", "gsw", "gu", "guw", "gv", "ha", "haw", "he", + "hi", "hnj", "hr", "hsb", "hu", "hy", "ia", "id", "ie", "ig", "ii", "io", "is", "it", "iu", "ja", + "jbo", "jgo", "jmc", "jv", "jw", "ka", "kab", "kaj", "kcg", "kde", "kea", "kk", "kkj", "kl", "km", "kn", + "ko", "kok", "ks", "ksb", "ksh", "ku", "kw", "ky", "lag", "lb", "lg", "lij", "lkt", "lld", "ln", "lo", + "lt", "lv", "mas", "mg", "mgo", "mk", "ml", "mn", "mo", "mr", "ms", "mt", "my", "nah", "naq", "nb", + "nd", "ne", "nl", "nn", "nnh", "no", "nqo", "nr", "nso", "ny", "nyn", "om", "or", "os", "osa", "pa", + "pap", "pcm", "pl", "prg", "ps", "pt", "rm", "ro", "rof", "ru", "rwk", "sah", "saq", "sat", "sc", "scn", + "sd", "sdh", "se", "seh", "ses", "sg", "sgs", "sh", "shi", "si", "sk", "sl", "sma", "smi", "smj", "smn", + "sms", "sn", "so", "sq", "sr", "ss", "ssy", "st", "su", "sv", "sw", "syr", "ta", "te", "teo", "th", + "ti", "tig", "tk", "tl", "tn", "to", "tpi", "tr", "ts", "tzm", "ug", "uk", "und", "ur", "uz", "ve", + "vec", "vi", "vo", "vun", "wa", "wae", "wo", "xh", "xog", "yi", "yo", "yue", "zh", "zu", +]; /// Percent signs around the number (CLDR's standard percent pattern and sign): locale, Latin /// digits or the locale's own, text before the number, text after it. A locale missing here @@ -31,11 +46,27 @@ pub(crate) const PERCENT: &[(&str, bool, &str, &str)] = &[ ("ar-SY", false, "", "\u{66a}\u{61c}"), ("ar-TD", false, "", "\u{66a}\u{61c}"), ("ar-YE", false, "", "\u{66a}\u{61c}"), + ("az-Arab", false, "", "\u{66a}"), + ("bas", true, "", "\u{a0}%"), + ("be", true, "", "\u{a0}%"), + ("bgn", false, "", "\u{66a}"), + ("bgn", true, "", "\u{66a}"), + ("blo", true, "%\u{a0}", ""), + ("br", true, "", "\u{a0}%"), + ("bua", true, "", "\u{a0}%"), + ("ca", true, "", "\u{a0}%"), + ("ce", true, "", "\u{a0}%"), + ("ckb", false, "", "\u{a0}\u{66a}"), ("cs", true, "", "\u{a0}%"), + ("cv", true, "", "\u{a0}%"), ("da", true, "", "\u{a0}%"), ("de", true, "", "\u{a0}%"), ("de-CH", true, "", "%"), ("de-LI", true, "", "%"), + ("dsb", true, "", "\u{a0}%"), + ("dua", true, "", "\u{a0}%"), + ("dz", false, "", "\u{a0}%"), + ("dz", true, "", "\u{a0}%"), ("en-AT", true, "", "\u{a0}%"), ("en-DE", true, "", "\u{a0}%"), ("en-DK", true, "", "\u{a0}%"), @@ -66,14 +97,48 @@ pub(crate) const PERCENT: &[(&str, bool, &str, &str)] = &[ ("es-US", true, "", "%"), ("es-UY", true, "", "%"), ("es-VE", true, "", "%"), + ("eu", true, "%\u{a0}", ""), ("fa", false, "", "\u{66a}"), ("fi", true, "", "\u{a0}%"), + ("fo", true, "", "\u{a0}%"), ("fr", true, "", "\u{a0}%"), ("fr-CH", true, "", "%"), + ("gl", true, "", "\u{a0}%"), + ("gsw", true, "", "\u{a0}%"), ("hr", true, "", "\u{a0}%"), + ("hsb", true, "", "\u{a0}%"), + ("ht", true, "", "\u{a0}%"), + ("kl", true, "", "\u{a0}%"), + ("ks", false, "", "\u{66a}"), + ("ksh", true, "", "\u{a0}%"), + ("ku", true, "%", ""), + ("ku-Arab", true, "", "%"), + ("lb", true, "", "\u{a0}%"), + ("lrc", false, "", "\u{66a}"), ("lt", true, "", "\u{a0}%"), + ("mk", true, "", "\u{a0}%"), + ("mzn", false, "", "\u{66a}"), + ("nb", true, "", "\u{a0}%"), + ("nds", true, "", "\u{a0}%"), + ("nn", true, "", "\u{a0}%"), + ("no", true, "", "\u{a0}%"), + ("oc", true, "", "\u{202f}%"), + ("pa-Arab", false, "", "\u{66a}"), + ("ps", false, "", "\u{66a}"), + ("qu", true, "", "\u{a0}%"), + ("rn", true, "", "\u{a0}%"), + ("ro", true, "", "\u{a0}%"), ("ru", true, "", "\u{a0}%"), + ("sd", false, "", "\u{66a}\u{61c}"), + ("sdh", false, "", "\u{66a}\u{61c}"), + ("se", true, "", "\u{a0}%"), + ("sk", true, "", "\u{a0}%"), ("sl", true, "", "\u{a0}%"), + ("smn", true, "", "\u{a0}%"), ("sv", true, "", "\u{a0}%"), + ("tk", true, "", "\u{a0}%"), ("tr", true, "%", ""), + ("tt", true, "", "\u{a0}%"), + ("uz-Arab", false, "", "\u{66a}"), + ("zgh", true, "", "\u{a0}%"), ]; diff --git a/crates/i18n/data/numbers.postcard b/crates/i18n/data/numbers.postcard index 53764d4c..5ee84e03 100644 Binary files a/crates/i18n/data/numbers.postcard and b/crates/i18n/data/numbers.postcard differ diff --git a/crates/i18n/src/format.rs b/crates/i18n/src/format.rs index 07dfd70c..6def7d18 100644 --- a/crates/i18n/src/format.rs +++ b/crates/i18n/src/format.rs @@ -1,5 +1,5 @@ //! Numbers, percentages and counts written for a formatting [`Region`], on ICU4X's decimal -//! formatter with CLDR data for the shipped languages (`data/generate.sh`). +//! formatter with CLDR data for every CLDR locale (`data/generate.sh`). use std::fmt; @@ -29,7 +29,7 @@ pub enum Digits { pub struct Format { /// `None` only if the data doesn't load: numbers are then written as Rust writes them. decimal: Option, - /// The region's separators where the system set them apart from the locale's. + /// The system's separators where they differ from the locale's; an empty `group` is none. group: Option, point: Option, percent: (&'static str, &'static str), @@ -37,8 +37,7 @@ pub struct Format { impl Format { /// The format for `region`'s number locale. With Latin digits, a decimal or grouping separator - /// the region sets (Windows' and macOS's own settings, English in continental Europe) replaces - /// the locale's. + /// the region carries (Windows' and macOS's own settings) replaces the locale's. pub fn new(region: &Region, digits: Digits) -> Format { let mut locale = Locale::try_from_str(®ion.numbers).unwrap_or(Locale::UNKNOWN); if digits == Digits::Latin { @@ -50,15 +49,14 @@ impl Format { let probe = format.parts(&Decimal::try_from_str("-1234567.5").unwrap_or_default()); let latin = probe.integer.starts_with(|c: char| c.is_ascii_digit()); if latin { - let point = region.decimal.to_string(); - if probe.point != point { + if let Some(point) = region.decimal.map(String::from).filter(|p| *p != probe.point) { // `1,234.5` with a decimal comma becomes `1.234,5`, never `1,234,5`. if probe.group == point && region.grouping.is_none() { format.group = Some(probe.point.clone()); } format.point = Some(point); } - if let Some(g) = region.grouping.map(String::from).filter(|g| *g != probe.group) { + if let Some(g) = region.grouping.map(|g| g.map(String::from).unwrap_or_default()).filter(|g| *g != probe.group) { format.group = Some(g); } } @@ -76,12 +74,12 @@ impl Format { self.write(&Decimal::from(n.into())) } - /// `value` rounded to `decimals` digits after the decimal separator. - /// Infinities and NaN are written as Rust writes them. + /// `value` rounded to `decimals` digits after the decimal separator; infinities and NaN are + /// `∞`, `-∞` and `NaN`. pub fn number(&self, value: f64, decimals: usize) -> String { match to_decimal(value, decimals) { Some(d) => self.write(&d), - None => value.to_string(), + None => non_finite(value).to_owned(), } } @@ -89,7 +87,7 @@ impl Format { /// `50%`, `50 %` (French, German), `%50` (Turkish), `٥٠٪؜` (Egyptian Arabic). pub fn percent(&self, ratio: f64, decimals: usize) -> String { let Some(d) = to_decimal(ratio * 100.0, decimals) else { - return format!("{}%", ratio * 100.0); + return format!("{}{}{}", self.percent.0, non_finite(ratio), self.percent.1); }; let p = self.parts(&d); format!("{}{}{}{}", p.sign, self.percent.0, p.text, self.percent.1) @@ -200,6 +198,15 @@ pub(crate) fn to_decimal(value: f64, decimals: usize) -> Option { Some(d) } +/// CLDR's root symbols for a value that isn't finite. +fn non_finite(value: f64) -> &'static str { + match value { + f64::INFINITY => "∞", + f64::NEG_INFINITY => "-∞", + _ => "NaN", + } +} + /// The percent affixes for a number locale (`ar-EG`), found by dropping trailing subtags. fn percent_affixes(tag: &str, latin: bool) -> (&'static str, &'static str) { let mut tag = tag; @@ -248,17 +255,39 @@ mod tests { } #[test] - fn region_separators() { - // English in Poland writes a decimal comma; the grouping moves out of its way. - assert_eq!(Format::from_tag("en-PL").number(1234.5, 1), "1.234,5"); + fn cldr_separators_for_every_region() { + for (tag, want) in [ + ("en-PL", "1.234.567,5"), + ("en-DE", "1.234.567,5"), + ("ar-MA", "1.234.567,5"), + ("de-CH", "1'234'567.5"), + ("nb-NO", "1\u{a0}234\u{a0}567,5"), + ("sk-SK", "1\u{a0}234\u{a0}567,5"), + ("bn-BD", "১২,৩৪,৫৬৭.৫"), + ("he-IL", "1,234,567.5"), + ("km", "1,234,567.5"), + ("lo", "1.234.567,5"), + ] { + assert_eq!(Format::from_tag(tag).number(1_234_567.5, 1), want, "{tag}"); + } + } + + #[test] + fn system_separators() { + // A decimal comma the system sets moves the grouping out of its way. + let mut region = Region::from_tag("en-US"); + region.decimal = Some(','); + assert_eq!(Format::new(®ion, Digits::Locale).number(1234.5, 1), "1.234,5"); let mut region = Region::from_tag("de-DE"); - region.grouping = Some('\u{a0}'); + region.grouping = Some(Some('\u{a0}')); assert_eq!(Format::new(®ion, Digits::Locale).number(1234.5, 1), "1\u{a0}234,5"); - region.decimal = '.'; + region.decimal = Some('.'); assert_eq!(Format::new(®ion, Digits::Locale).number(1234.5, 1), "1\u{a0}234.5"); + region.grouping = Some(None); + assert_eq!(Format::new(®ion, Digits::Locale).integer(1_234_567), "1234567", "no grouping"); // A system's separators don't apply to Arabic-Indic digits. let mut region = Region::from_tag("ar-EG"); - region.grouping = Some(','); + region.grouping = Some(Some(',')); assert_eq!(Format::new(®ion, Digits::Locale).number(1234.5, 1), "١٬٢٣٤٫٥"); } @@ -280,7 +309,10 @@ mod tests { let en = Format::from_tag("en"); assert_eq!(en.number(-1234.5, 0), "-1,234"); assert_eq!(en.number(-0.04, 1), "0.0", "no sign on zero"); - assert_eq!(en.number(f64::INFINITY, 1), "inf"); + assert_eq!(en.number(f64::INFINITY, 1), "∞"); + assert_eq!(en.number(f64::NEG_INFINITY, 1), "-∞"); + assert_eq!(en.number(f64::NAN, 1), "NaN"); + assert_eq!(Format::from_tag("de").percent(f64::NAN, 0), "NaN\u{a0}%"); assert_eq!(en.integer(-5_i64), "-5"); } } diff --git a/crates/i18n/src/lib.rs b/crates/i18n/src/lib.rs index 910041ed..a3924fe9 100644 --- a/crates/i18n/src/lib.rs +++ b/crates/i18n/src/lib.rs @@ -3,6 +3,11 @@ //! An app embeds the [craft-i18n](https://github.com/benandez/craft-i18n) catalogs at build time //! as an [`Embedded`] static, builds one [`I18n`] from it, and looks English source text up in a //! [`Lang`]. A missing translation is the English itself; nothing here fails. +//! +//! Counts ([`I18n::trn`]), file sizes and measures write their numbers as the formatting +//! [`Region`] does ([`I18n::format`]): grouped (`1.234`, `12,34,567`) and in the region's digits +//! (`١٬٢٣٤`). Text that goes into documents or file names should use [`I18n::trn_plain`], which +//! writes `0`–`9` without grouping. #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] mod catalog; @@ -10,7 +15,7 @@ mod format; mod region; mod size; -/// The CLDR data for the shipped languages, from `data/generate.sh`. +/// The CLDR number and plural data for every CLDR locale, from `data/generate.sh`. mod data { use std::sync::OnceLock; @@ -189,25 +194,34 @@ impl I18n { self.find(lang, &[("@id", id), ("", label)]).unwrap_or(label) } - /// A count's wording in `lang`, `{n}` filled in as [`I18n::format`] writes numbers. `one` and - /// `other` are the English forms; they are also the key of the `@plural` row, and the fallback. + /// A count's wording in `lang`, `{n}` filled in as [`I18n::format`] writes numbers: grouped and + /// in the region's digits. `one` and `other` are the English forms; they are also the key of + /// the `@plural` row, and the fallback. pub fn trn(&self, lang: Lang, n: u64, one: &str, other: &str) -> String { let format = self.format(); - self.plural(lang, PluralOperands::from(n), n == 1, &format.integer(n), one, other) + self.plural(lang, Some(PluralOperands::from(n)), n == 1, &format.integer(n), one, other) + } + + /// [`I18n::trn`] with `{n}` in `0`–`9` without grouping (`Copy 1234`), for text that goes into + /// documents, file names or anything read back later. + pub fn trn_plain(&self, lang: Lang, n: u64, one: &str, other: &str) -> String { + self.plural(lang, Some(PluralOperands::from(n)), n == 1, &n.to_string(), one, other) } /// [`I18n::trn`] for a count with `decimals` digits after the decimal separator (`1.5 files`). - /// The plural form follows the digits shown, so `1.0` takes English `other`. + /// The plural form follows the digits shown, so `1.0` takes English `other`; infinities and NaN + /// (`∞`, `NaN`) take `other` too. pub fn trn_decimal(&self, lang: Lang, n: f64, decimals: usize, one: &str, other: &str) -> String { let format = self.format(); let Some(d) = format::to_decimal(n, decimals) else { - return fmt(other, &[("n", &n.to_string())]); + return self.plural(lang, None, false, &format.number(n, decimals), one, other); }; let english_one = d.to_string() == "1"; - self.plural(lang, PluralOperands::from(&d), english_one, &format.write(&d), one, other) + self.plural(lang, Some(PluralOperands::from(&d)), english_one, &format.write(&d), one, other) } - fn plural(&self, lang: Lang, operands: PluralOperands, english_one: bool, n: &str, one: &str, other: &str) -> String { + /// `operands` `None` is the `other` category. + fn plural(&self, lang: Lang, operands: Option, english_one: bool, n: &str, one: &str, other: &str) -> String { let key = format!("{one}|{other}"); let chain = self.locale(lang).map_or(&[][..], |l| l.chain.as_slice()); let text = chain @@ -216,7 +230,7 @@ impl I18n { let locale = self.locales.get(i)?; let forms = locale.rows.get("@plural")?.get(&key)?; let (rules, categories) = locale.plural.as_ref()?; - let category = rules.category_for(operands); + let category = operands.map_or(PluralCategory::Other, |o| rules.category_for(o)); // A category whole numbers up to 1000 never reach has no form of its own. Russian // and Ukrainian decimals (`other`) take the `few` form (`1,5 файла`); any other // (French `many` for a million) takes the last. @@ -236,7 +250,7 @@ impl I18n { } /// How counts, file sizes and measures write numbers: [`Region::system`]'s format with the - /// locale's digits until [`I18n::set_format`]. + /// locale's digits until [`I18n::set_format`], and again after [`I18n::reset_format`]. pub fn format(&self) -> Arc { if let Some(f) = self.format.read().unwrap_or_else(PoisonError::into_inner).as_ref() { return Arc::clone(f); @@ -250,6 +264,11 @@ impl I18n { *self.format.write().unwrap_or_else(PoisonError::into_inner) = Some(Arc::new(format)); } + /// Write numbers as the system region does again, undoing [`I18n::set_format`]. + pub fn reset_format(&self) { + *self.format.write().unwrap_or_else(PoisonError::into_inner) = None; + } + /// The languages to offer: English first, then the offered locales by code. Never the /// pseudo locale. pub fn languages(&self) -> Vec { @@ -432,15 +451,18 @@ fn bare(code: &str) -> Locale { } /// `code`'s CLDR cardinal rules and the categories whole numbers 0..=1000 reach, in CLDR order -/// (zero, one, two, few, many, other): one `@plural` form each. Regional and scripted codes fall -/// back to the language's rules. CLDR's `pt` is Brazilian (0 is singular), so Portuguese other -/// than `pt-br` takes `pt-PT`'s. `None` for a language the baked data doesn't cover. +/// (zero, one, two, few, many, other): one `@plural` form each. The rules are looked up by +/// language alone: CLDR's parent of `sr-Latn` and `zh-Hant` is the root, whose only category is +/// `other`. CLDR's `pt` is Brazilian (0 is singular), so Portuguese other than `pt-br` takes +/// `pt-PT`'s. `None` for a language CLDR has no plural rules for. fn plural_rules(code: &str) -> Option<(PluralRules, Vec)> { - let code = if code == "pt" || (code.starts_with("pt-") && code != "pt-br") { "pt-pt" } else { code }; - let locale = IcuLocale::try_from_str(code).ok()?; - if !data::LANGUAGES.contains(&locale.id.language.as_str()) { + let id = IcuLocale::try_from_str(code).ok()?.id; + let language = id.language.as_str(); + if !data::PLURAL_LANGUAGES.contains(&language) { return None; } + let brazil = id.region.is_some_and(|r| r.as_str() == "BR"); + let locale = IcuLocale::try_from_str(if language == "pt" && !brazil { "pt-PT" } else { language }).ok()?; let rules = PluralRules::try_new_cardinal_with_buffer_provider(data::provider()?, (&locale).into()).ok()?; let mut categories = Vec::new(); for n in 0..=1000_u32 { @@ -475,10 +497,12 @@ mod tests { ("de", "\tBad\n@nope\tA\tB\n\tUndo\tRückgängig\n", ""), ]; - /// `embedded`, offering every locale in it. + /// `embedded`, offering every locale in it, with `en-US` numbers rather than the machine's. fn all(embedded: &'static Embedded) -> (I18n, Vec) { let codes: Vec<&str> = embedded.iter().map(|(c, _, _)| *c).collect(); - I18n::new(embedded, &codes) + let (i, problems) = I18n::new(embedded, &codes); + i.set_format(Format::from_tag("en-US")); + (i, problems) } fn i18n() -> I18n { @@ -610,6 +634,20 @@ mod tests { assert_eq!(i.trn(Lang::EN, 12_345, "{n} file", "{n} files"), "12.345 files"); } + #[test] + fn plain_counts_and_the_system_format() { + let i = all(P).0; + i.set_format(Format::from_tag("ar-EG")); + assert_eq!(i.trn_plain(lang(&i, "ar"), 1234, "{n} file", "{n} files"), "1234 ملفًا"); + i.set_format(Format::from_tag("de-DE")); + assert_eq!(i.trn_plain(Lang::EN, 12_345, "copy {n}", "copy {n}"), "copy 12345"); + i.reset_format(); + assert_eq!( + i.trn(Lang::EN, 5, "{n} file", "{n} files"), + fmt("{n} files", &[("n", &Format::new(&Region::system(), Digits::Locale).integer(5))]) + ); + } + #[test] fn decimal_counts() { let i = all(P).0; @@ -627,6 +665,9 @@ mod tests { let ar = lang(&i, "ar"); assert_eq!(i.trn_decimal(ar, 2.5, 1, "{n} file", "{n} files"), "٢٫٥ ملف"); assert_eq!(i.trn_decimal(ar, 3.0, 0, "{n} file", "{n} files"), "٣ ملفات"); + // Not a number: the translated `other` form. + assert_eq!(i.trn_decimal(ru, f64::NAN, 1, "{n} file", "{n} files"), "NaN файла"); + assert_eq!(i.trn_decimal(Lang::EN, f64::INFINITY, 1, "{n} file", "{n} files"), "∞ files"); } static L: &Embedded = @@ -777,15 +818,31 @@ mod tests { assert_eq!(i.trn(lang(&i, "pt-br"), 0, "{n} file", "{n} files"), "0 arquivo"); } - /// The same counts as craft-i18n's checker (`plural_form_counts`), which rejects a catalog - /// whose `@plural` rows list any other. + /// The counts craft-i18n's checker expects, which rejects a catalog whose `@plural` rows list + /// any other: every directory in craft-i18n's `locales/`, then languages it may add. #[test] fn plural_form_counts() { - for (locale, n) in - [("en", 2), ("fr", 2), ("ja", 1), ("vi", 1), ("ru", 3), ("pl", 3), ("cs", 3), ("sl", 4), ("ar", 6), ("pt", 2), ("pt-br", 2), ("tr", 2)] - { + #[rustfmt::skip] + let catalogs = [ + ("ar", 6), ("bg", 2), ("cs", 3), ("da", 2), ("de", 2), ("el", 2), ("es", 2), ("et", 2), + ("fi", 2), ("fr", 2), ("hr", 3), ("hu", 2), ("id", 1), ("it", 2), ("ja", 1), ("ka", 2), + ("kk", 2), ("ko", 1), ("lt", 3), ("nl", 2), ("pl", 3), ("pt-br", 2), ("ru", 3), ("sl", 4), + ("sr", 3), ("sr-latn", 3), ("sv", 2), ("te", 2), ("tr", 2), ("uk", 3), ("vi", 1), + ("zh-hans", 1), ("zh-hant", 1), + ]; + let more = [("en", 2), ("pt", 2), ("he", 3), ("ro", 3), ("nb", 2), ("ca", 2), ("sk", 3), ("th", 1), ("ms", 1), ("fa", 2), ("hi", 2)]; + for (locale, n) in catalogs.into_iter().chain(more) { assert_eq!(plural_rules(locale).map(|(_, c)| c.len()), Some(n), "{locale}"); } + assert!(plural_rules("tlh").is_none(), "no CLDR rules"); + } + + #[test] + fn serbian_latin_plurals() { + let i = all(&[("sr-latn", "@plural\t{n} file|{n} files\t{n} fajl|{n} fajla|{n} fajlova\n", "")]).0; + let sr = lang(&i, "sr-latn"); + let files = |n| i.trn(sr, n, "{n} file", "{n} files"); + assert_eq!((files(1), files(3), files(5)), ("1 fajl".into(), "3 fajla".into(), "5 fajlova".into())); } #[test]