From ce70b2ede876a87f2abc8af95e39e6ce801fd224 Mon Sep 17 00:00:00 2001 From: aiongg Date: Wed, 21 Oct 2026 04:41:41 +0000 Subject: [PATCH 1/7] i18n: formatting region apart from the interface language Region holds the number and time locales, metric or US measurement, A4 or Letter, the decimal separator, and the grouping separator and clock when the system sets them. Region::system reads Windows' regional format with the user's changes, Apple's current locale, the POSIX LC_* variables each on their own, and the preferred language on the web and Android. Region::from_setting takes an app's saved choice ("auto" or a tag). Ported from WordCraft's Region (storytold/wordcraft#857, #1494). Co-authored-by: Dmitrii Khlapov Co-authored-by: Brandon Thomas --- Cargo.lock | 2 + crates/i18n/Cargo.toml | 8 + crates/i18n/README.md | 10 + crates/i18n/src/lib.rs | 3 + crates/i18n/src/region.rs | 464 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 487 insertions(+) create mode 100644 crates/i18n/src/region.rs diff --git a/Cargo.lock b/Cargo.lock index 4d9de428..11eebe64 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -657,8 +657,10 @@ name = "craft-i18n" version = "0.1.0" dependencies = [ "intl_pluralrules", + "objc2-foundation 0.3.2", "sys-locale", "unic-langid", + "windows-registry", ] [[package]] diff --git a/crates/i18n/Cargo.toml b/crates/i18n/Cargo.toml index 45e21775..bfff93ce 100644 --- a/crates/i18n/Cargo.toml +++ b/crates/i18n/Cargo.toml @@ -15,5 +15,13 @@ sys-locale = "0.3" [target.'cfg(target_arch = "wasm32")'.dependencies] sys-locale = { version = "0.3", features = ["js"] } +# The regional format (Region::system); safe registry reads, MIT OR Apache-2.0. +[target.'cfg(windows)'.dependencies] +windows-registry = "0.6" + +# The current locale (Region::system); already in the apps' Cargo.lock through eframe. +[target.'cfg(target_vendor = "apple")'.dependencies] +objc2-foundation = { version = "0.3.2", default-features = false, features = ["std", "NSDateFormatter", "NSFormatter", "NSLocale", "NSString"] } + [lints] workspace = true diff --git a/crates/i18n/README.md b/crates/i18n/README.md index df337971..d916bbbb 100644 --- a/crates/i18n/README.md +++ b/crates/i18n/README.md @@ -68,6 +68,16 @@ The app keeps its `tl!` macro, its language setting (`"auto"` → `I18N.system_l `zh-hant`, `pt-PT` → `pt-br` when that is the only Portuguese, `C`/`POSIX` → English. A script is never crossed: `sr_RS@latin` and `sr-Latn-RS` pick `sr-latn`, never `sr`, while `sr-Cyrl-RS` picks `sr`. +- **Formatting region:** `Region` holds how numbers, times and measurements are written, apart + from the interface language: the number and time locales (BCP 47), metric or US measurement, A4 + or Letter, the decimal separator, and the grouping separator and clock when the system sets + them. `Region::system()` reads it once: Windows' regional format with the user's changes + (`HKCU\Control Panel\International`), Apple's current locale (`en_PL`, `en_GB@rg=uszzzz`), + `LC_ALL` or else `LC_NUMERIC`, `LC_TIME`, `LC_MEASUREMENT` and `LC_PAPER` each on its own over + `LANG` on Unix, the preferred language on the web and Android. `Region::from_setting` takes + the app's saved choice (`auto` or a tag like `pl-PL`). Paper and measurement follow CLDR's + territory lists; tags may carry `-u-ms-`, `-u-hc-` and `-u-rg-`. Ported from WordCraft's + `Region`. - **Pseudo locale** (`pseudo`, debug builds only, never in `languages()`): every lookup, `get` included, comes back as `⟦…⟧`, line by line, so text drawn without the marks never went through translation. `looked_up()` lists the source texts looked up so far, for an audit against a diff --git a/crates/i18n/src/lib.rs b/crates/i18n/src/lib.rs index 7fa7a4ef..9e407419 100644 --- a/crates/i18n/src/lib.rs +++ b/crates/i18n/src/lib.rs @@ -6,6 +6,7 @@ #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] mod catalog; +mod region; use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::sync::{Mutex, OnceLock, PoisonError}; @@ -13,6 +14,8 @@ use std::sync::{Mutex, OnceLock, PoisonError}; use intl_pluralrules::{PluralCategory, PluralRuleType, PluralRules}; use unic_langid::LanguageIdentifier; +pub use region::{HourCycle, Measurement, Paper, Region}; + /// The catalogs an app embeds, one entry per locale: (locale, shared catalog, the app's catalog). /// A missing catalog is `""`. pub type Embedded = [(&'static str, &'static str, &'static str)]; diff --git a/crates/i18n/src/region.rs b/crates/i18n/src/region.rs new file mode 100644 index 00000000..f9bb4720 --- /dev/null +++ b/crates/i18n/src/region.rs @@ -0,0 +1,464 @@ +//! The formatting region: how numbers, times and measurements are written, apart from the +//! interface language. An English interface in Poland still writes `2,54 cm` on A4. +//! +//! The territory lists come from Unicode CLDR's supplemental `measurementData` (Unicode License): +//! `paperSize` US-Letter and `measurementSystem` US. The decimal separators follow CLDR's number +//! symbols for the common locales; the system's own separator wins where the platform has one. + +use std::sync::OnceLock; + +/// Territories whose default paper is US Letter (CLDR `paperSize type="US-Letter"`). +const LETTER: &[&str] = &["BZ", "CA", "CL", "CO", "CR", "GT", "MX", "NI", "PA", "PH", "PR", "SV", "US", "VE"]; + +/// Territories that measure in inches (CLDR `measurementSystem type="US"`). +const INCHES: &[&str] = &["LR", "MM", "US"]; + +/// Languages whose numbers use a decimal comma in most of their territories. +const COMMA_LANGUAGES: &[&str] = &[ + "af", "az", "be", "bg", "bs", "ca", "cs", "da", "de", "el", "es", "et", "eu", "fi", "fo", "fr", "gl", "hr", "hu", "hy", "id", "is", "it", "ka", + "kk", "ky", "lb", "lt", "lv", "mk", "nb", "nl", "nn", "no", "pl", "pt", "ro", "ru", "sk", "sl", "sq", "sr", "sv", "tr", "uk", "uz", "vi", +]; + +/// Territories where those languages use a decimal point after all. +const POINT_EXCEPTIONS: &[(&str, &[&str])] = + &[("de", &["CH", "LI"]), ("it", &["CH"]), ("es", &["CU", "DO", "GT", "HN", "MX", "NI", "PA", "PE", "PR", "SV", "US", "419"])]; + +/// Territories where English is written with a decimal comma: most of continental Europe, South +/// Africa and much of South America. +const COMMA_TERRITORIES: &[&str] = &[ + "AR", "AT", "BE", "BG", "BO", "BR", "CL", "CO", "CZ", "DE", "DK", "EC", "EE", "ES", "FI", "FR", "GR", "HR", "HU", "ID", "IS", "IT", "LT", "LU", + "LV", "NL", "NO", "PL", "PT", "PY", "RO", "RS", "RU", "SE", "SI", "SK", "TR", "UA", "UY", "VE", "VN", "ZA", +]; + +/// How lengths, distances and weights are measured. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum Measurement { + /// Centimetres, kilometres and kilograms (CLDR `metric` and `UK`). + Metric, + /// Inches, miles and pounds (CLDR `US`). + Us, +} + +/// The paper a new document starts on. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum Paper { + A4, + Letter, +} + +/// A clock the user chose, where the system says so apart from the locale. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum HourCycle { + /// 1 to 12 with AM and PM. + H12, + /// 0 to 23. + H23, +} + +/// The conventions numbers, times and measurements follow. Read it with [`Region::system`] or, +/// for the user's saved choice, [`Region::from_setting`]; the fields are public so an app can +/// keep a separate choice of, say, measurement. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Region { + /// The locale numbers are written in, BCP 47 (`pl-PL`, `sr-Latn-RS`). + pub numbers: String, + /// The locale dates and times are written in, BCP 47. + pub time: String, + pub measurement: Measurement, + pub paper: Paper, + /// The decimal separator (`,` in Poland). + pub decimal: char, + /// The grouping separator the system sets (Windows, macOS); `None` follows the locale. + pub grouping: Option, + /// The clock the system sets (Windows, macOS); `None` follows the locale. + pub hour_cycle: Option, +} + +impl Default for Region { + /// `en-US`: decimal point, Letter, inches. + fn default() -> Self { + Region::from_parts(&Tag::default()) + } +} + +impl Region { + /// The region a BCP 47 or POSIX locale tag names (`pl-PL`, `de_AT.UTF-8@euro`, `zh-Hans-CN`, + /// `en-US-u-ms-metric`, Apple's `en_US@rg=plzzzz`). The Unicode extensions `ms` (measurement), + /// `hc` (clock) and `rg` (region override) are read. An empty or unreadable tag, `C` and + /// `POSIX` are `en-US`; a bare `en` is the United States, any other bare language has no + /// territory. + pub fn from_tag(tag: &str) -> Region { + Region::from_parts(&Tag::parse(tag).unwrap_or_default()) + } + + /// The region for an app's saved region setting: `""` or `auto` (any case) is + /// [`Region::system`], anything else a locale tag for [`Region::from_tag`]. + pub fn from_setting(setting: &str) -> Region { + let setting = setting.trim(); + if setting.is_empty() || setting.eq_ignore_ascii_case("auto") { Region::system() } else { Region::from_tag(setting) } + } + + /// The system's region, read once, then cached: + /// + /// - Windows: the regional format and its customised separators, measurement system, paper + /// and short time format (Settings › Time & language › Language & region). + /// - macOS and iOS: the current locale (its identifier carries the Region setting), its + /// separators and metric setting, and its hour format. + /// - Linux and the BSDs: `LC_ALL`, else each of `LC_NUMERIC`, `LC_TIME`, `LC_MEASUREMENT` and + /// `LC_PAPER` on its own, else `LANG`. + /// - The web and Android: the preferred language, the only locale they expose. + pub fn system() -> Region { + static SYSTEM: OnceLock = OnceLock::new(); + SYSTEM.get_or_init(read_system).clone() + } + + fn from_parts(tag: &Tag) -> Region { + let territory = tag.territory.as_str(); + let language = tag.language.as_str(); + let measurement = tag.measurement.unwrap_or(if INCHES.contains(&territory) { Measurement::Us } else { Measurement::Metric }); + let decimal = if COMMA_LANGUAGES.contains(&language) { + if POINT_EXCEPTIONS.iter().any(|(l, ts)| *l == language && ts.contains(&territory)) { '.' } else { ',' } + } else if language == "en" && COMMA_TERRITORIES.contains(&territory) { + ',' + } else { + '.' + }; + Region { + numbers: tag.bcp47(), + time: tag.bcp47(), + measurement, + paper: if LETTER.contains(&territory) { Paper::Letter } else { Paper::A4 }, + decimal, + grouping: None, + hour_cycle: tag.hour_cycle, + } + } +} + +/// The parts of a locale tag the region uses. +#[derive(Clone, Debug, PartialEq, Eq)] +struct Tag { + /// Lower case (`pl`). + language: String, + /// Title case (`Latn`); empty when the tag has none. + script: String, + /// Upper case (`PL`, or a UN M.49 number like `419`); empty when the tag has none. + territory: String, + measurement: Option, + hour_cycle: Option, +} + +impl Default for Tag { + fn default() -> Self { + Tag { language: "en".into(), script: String::new(), territory: "US".into(), measurement: None, hour_cycle: None } + } +} + +impl Tag { + fn parse(tag: &str) -> Option { + // Locale tags are short; anything long is junk and only its start is read. + let tag: String = tag.chars().take(64).collect(); + let (tag, modifier) = tag.split_once('@').unwrap_or((&tag, "")); + let tag = tag.split('.').next().unwrap_or_default().trim(); + if tag.eq_ignore_ascii_case("c") || tag.eq_ignore_ascii_case("posix") { + return None; + } + let mut parts = tag.split(['-', '_']); + let language = parts.next().unwrap_or_default(); + if !(2..=3).contains(&language.len()) || !language.chars().all(|c| c.is_ascii_alphabetic()) { + return None; + } + let mut out = + Tag { language: language.to_ascii_lowercase(), script: String::new(), territory: String::new(), measurement: None, hour_cycle: None }; + let mut extension = None; + for p in parts.by_ref() { + // A singleton starts the extensions (`-u-…`, `-x-…`). + if p.len() == 1 { + extension = Some(p.to_ascii_lowercase()); + break; + } + if out.territory.is_empty() && out.script.is_empty() && p.len() == 4 && p.chars().all(|c| c.is_ascii_alphabetic()) { + out.script = title_case(p); + } else if out.territory.is_empty() && is_territory(p) { + out.territory = p.to_ascii_uppercase(); + } + } + let mut keywords: Vec<(String, String)> = Vec::new(); + if extension.as_deref() == Some("u") { + let mut key = String::new(); + for p in parts { + if p.len() == 1 { + break; + } + if p.len() == 2 { + key = p.to_ascii_lowercase(); + } else if !key.is_empty() { + keywords.push((std::mem::take(&mut key), p.to_ascii_lowercase())); + } + } + } + // Apple's and ICU's `@rg=plzzzz;ms=metric` keywords, and glibc's `@latin` script modifier. + for m in modifier.split(';') { + match m.split_once('=') { + Some((k, v)) => keywords.push((k.trim().to_ascii_lowercase(), v.trim().to_ascii_lowercase())), + None if m.eq_ignore_ascii_case("latin") => out.script = "Latn".into(), + None if m.eq_ignore_ascii_case("cyrillic") => out.script = "Cyrl".into(), + None => {} + } + } + for (key, value) in keywords { + match (key.as_str(), value.as_str()) { + ("ms", "metric" | "uksystem") => out.measurement = Some(Measurement::Metric), + ("ms", "ussystem") => out.measurement = Some(Measurement::Us), + ("hc", "h11" | "h12") => out.hour_cycle = Some(HourCycle::H12), + ("hc", "h23" | "h24") => out.hour_cycle = Some(HourCycle::H23), + ("rg", v) if v.len() == 6 && v.ends_with("zzzz") && v.get(..2).is_some_and(is_territory) => { + out.territory = v.get(..2).unwrap_or_default().to_ascii_uppercase(); + } + _ => {} + } + } + if out.territory.is_empty() && out.language == "en" { + out.territory = "US".into(); + } + Some(out) + } + + fn bcp47(&self) -> String { + [self.language.as_str(), &self.script, &self.territory].into_iter().filter(|p| !p.is_empty()).collect::>().join("-") + } +} + +fn is_territory(p: &str) -> bool { + (p.len() == 2 && p.chars().all(|c| c.is_ascii_alphabetic())) || (p.len() == 3 && p.chars().all(|c| c.is_ascii_digit())) +} + +fn title_case(p: &str) -> String { + let lower = p.to_ascii_lowercase(); + let mut chars = lower.chars(); + chars.next().map(|c| c.to_ascii_uppercase().to_string() + chars.as_str()).unwrap_or_default() +} + +/// A locale variable that names a locale (not empty, `C` or `POSIX`). +#[cfg(any(test, all(unix, not(any(target_vendor = "apple", target_os = "android")))))] +fn usable_locale(v: &str) -> bool { + let v = v.trim(); + !v.is_empty() && !v.eq_ignore_ascii_case("c") && !v.eq_ignore_ascii_case("posix") && !v.starts_with("C.") +} + +/// The region from the POSIX locale variables `var` reads and, when neither `LC_ALL` nor the +/// variable for a convention nor `LANG` is set, the platform's preferred locale. +#[cfg(any(test, all(unix, not(any(target_vendor = "apple", target_os = "android")))))] +fn from_posix(var: impl Fn(&str) -> Option, platform: impl FnOnce() -> Option) -> Region { + let var = |name: &str| var(name).filter(|v| usable_locale(v)); + if let Some(all) = var("LC_ALL") { + return Region::from_tag(&all); + } + let lang = var("LANG").or_else(platform).unwrap_or_default(); + let tag = |name: &str| var(name).unwrap_or_else(|| lang.clone()); + let time = Region::from_tag(&tag("LC_TIME")); + Region { + time: time.time, + hour_cycle: time.hour_cycle, + measurement: Region::from_tag(&tag("LC_MEASUREMENT")).measurement, + paper: Region::from_tag(&tag("LC_PAPER")).paper, + ..Region::from_tag(&tag("LC_NUMERIC")) + } +} + +/// The region from the values under `HKEY_CURRENT_USER\Control Panel\International` that `value` +/// reads, where Windows keeps the regional format and the user's changes to it. `LocaleName` is +/// the format (`pl-PL`), else the platform's preferred locale. +#[cfg(any(test, windows))] +fn from_windows(value: impl Fn(&str) -> Option, platform: impl FnOnce() -> Option) -> Region { + let mut region = Region::from_tag(&value("LocaleName").or_else(platform).unwrap_or_default()); + if let Some(c) = value("sDecimal").as_deref().and_then(single_char) { + region.decimal = c; + } + if let Some(s) = value("sThousand") { + region.grouping = single_char(&s); + } + match value("iMeasure").as_deref().map(str::trim) { + Some("0") => region.measurement = Measurement::Metric, + Some("1") => region.measurement = Measurement::Us, + _ => {} + } + // DMPAPER_LETTER and DMPAPER_A4; other sizes keep the territory's. + match value("iPaperSize").as_deref().map(str::trim) { + Some("1") => region.paper = Paper::Letter, + Some("9") => region.paper = Paper::A4, + _ => {} + } + region.hour_cycle = value("sShortTime").as_deref().and_then(hour_cycle_of).or(region.hour_cycle); + region +} + +/// The region from Apple's current locale: its identifier (`en_PL`, `en_GB@rg=uszzzz`), its +/// metric setting and separators, and the hour format its `j` skeleton gives (`HH`, `h a`). +#[cfg(any(test, target_vendor = "apple"))] +fn from_apple(identifier: &str, metric: bool, decimal: &str, grouping: &str, hours: Option<&str>) -> Region { + let mut region = Region::from_tag(identifier); + region.measurement = if metric { Measurement::Metric } else { Measurement::Us }; + if let Some(c) = single_char(decimal) { + region.decimal = c; + } + region.grouping = single_char(grouping); + region.hour_cycle = hours.and_then(hour_cycle_of).or(region.hour_cycle); + region +} + +/// A separator that is one character; Windows allows up to three, which the region can't carry. +#[cfg(any(test, windows, target_vendor = "apple"))] +fn single_char(s: &str) -> Option { + let mut chars = s.chars(); + let c = chars.next()?; + chars.next().is_none().then_some(c) +} + +/// The clock a time pattern (`HH:mm`, `h:mm tt`, `h 'Uhr'`) uses; quoted literals are skipped. +#[cfg(any(test, windows, target_vendor = "apple"))] +fn hour_cycle_of(pattern: &str) -> Option { + let mut quoted = false; + for c in pattern.chars() { + match c { + '\'' => quoted = !quoted, + 'H' | 'k' if !quoted => return Some(HourCycle::H23), + 'h' | 'K' if !quoted => return Some(HourCycle::H12), + _ => {} + } + } + None +} + +#[cfg(windows)] +fn read_system() -> Region { + let key = windows_registry::CURRENT_USER.open(r"Control Panel\International").ok(); + from_windows(|name| key.as_ref()?.get_string(name).ok(), sys_locale::get_locale) +} + +#[cfg(target_vendor = "apple")] +fn read_system() -> Region { + use objc2_foundation::{NSDateFormatter, NSLocale, NSString}; + let locale = NSLocale::currentLocale(); + let hours = NSDateFormatter::dateFormatFromTemplate_options_locale(&NSString::from_str("j"), 0, Some(&locale)).map(|f| f.to_string()); + from_apple( + &locale.localeIdentifier().to_string(), + locale.usesMetricSystem(), + &locale.decimalSeparator().to_string(), + &locale.groupingSeparator().to_string(), + hours.as_deref(), + ) +} + +#[cfg(all(unix, not(any(target_vendor = "apple", target_os = "android"))))] +fn read_system() -> Region { + from_posix(|name| std::env::var(name).ok(), sys_locale::get_locale) +} + +#[cfg(not(any(windows, unix)))] +fn read_system() -> Region { + Region::from_tag(&sys_locale::get_locale().unwrap_or_default()) +} + +#[cfg(target_os = "android")] +fn read_system() -> Region { + Region::from_tag(&sys_locale::get_locale().unwrap_or_default()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn conventions(r: &Region) -> (Paper, Measurement, char) { + (r.paper, r.measurement, r.decimal) + } + + #[test] + fn reads_locale_tags() { + assert_eq!(Region::from_tag("pl-PL").numbers, "pl-PL"); + assert_eq!(Region::from_tag("de_AT.UTF-8@euro").numbers, "de-AT"); + assert_eq!(Region::from_tag("zh-hans-cn").numbers, "zh-Hans-CN"); + assert_eq!(Region::from_tag("sr_RS@latin").numbers, "sr-Latn-RS"); + assert_eq!(Region::from_tag("es-419").numbers, "es-419"); + assert_eq!(Region::from_tag("fr").numbers, "fr"); + assert_eq!(Region::from_tag("en").numbers, "en-US", "bare English is the US"); + for junk in ["", "C", "POSIX", "C.UTF-8", "1234", "x", "--", "ünïcode-ÄÄ", &"a".repeat(10_000)] { + assert_eq!(Region::from_tag(junk), Region::default(), "{junk:?}"); + } + } + + #[test] + fn unicode_extensions_and_apple_keywords() { + let r = Region::from_tag("en-US-u-ms-metric-hc-h23"); + assert_eq!((r.numbers.as_str(), r.measurement, r.hour_cycle), ("en-US", Measurement::Metric, Some(HourCycle::H23))); + assert_eq!(Region::from_tag("en-GB-u-hc-h12").hour_cycle, Some(HourCycle::H12)); + assert_eq!(Region::from_tag("de-DE-x-ms-ussystem").measurement, Measurement::Metric, "private use isn't read"); + let r = Region::from_tag("en_GB@rg=uszzzz"); + assert_eq!((r.numbers.as_str(), r.paper, r.measurement), ("en-US", Paper::Letter, Measurement::Us)); + assert_eq!(Region::from_tag("en_US@rg=plzzzz;ms=metric").numbers, "en-PL"); + } + + /// Paper and measurement follow the territory, numbers the language and territory. + #[test] + fn conventions_follow_the_territory() { + assert_eq!(conventions(&Region::from_tag("pl-PL")), (Paper::A4, Measurement::Metric, ',')); + assert_eq!(conventions(&Region::from_tag("en-US")), (Paper::Letter, Measurement::Us, '.')); + assert_eq!(conventions(&Region::from_tag("fr-CA")), (Paper::Letter, Measurement::Metric, ',')); + assert_eq!(conventions(&Region::from_tag("en-DE")), (Paper::A4, Measurement::Metric, ',')); + assert_eq!(conventions(&Region::from_tag("en-GB")), (Paper::A4, Measurement::Metric, '.')); + assert_eq!(Region::from_tag("de-CH").decimal, '.'); + assert_eq!(Region::from_tag("es-MX").paper, Paper::Letter); + assert_eq!(Region::from_tag("ja-JP").paper, Paper::A4); + } + + #[test] + fn a_saved_setting_overrides_the_system() { + assert_eq!(Region::from_setting(" pl-PL "), Region::from_tag("pl-PL")); + assert_eq!(Region::from_setting("AUTO"), Region::system()); + assert_eq!(Region::from_setting(""), Region::system()); + } + + #[test] + fn posix_variables_set_each_convention() { + let env = |pairs: &'static [(&'static str, &'static str)]| move |n: &str| pairs.iter().find(|(k, _)| *k == n).map(|(_, v)| v.to_string()); + let none = || None; + let r = from_posix(env(&[("LANG", "en_US.UTF-8"), ("LC_NUMERIC", "pl_PL.UTF-8"), ("LC_TIME", "en_GB.UTF-8"), ("LC_PAPER", "de_DE")]), none); + assert_eq!((r.numbers.as_str(), r.time.as_str(), r.decimal), ("pl-PL", "en-GB", ',')); + assert_eq!((r.paper, r.measurement), (Paper::A4, Measurement::Us), "paper from LC_PAPER, measurement from LANG"); + assert_eq!(from_posix(env(&[("LANG", "it_IT.UTF-8"), ("LC_ALL", "en_US"), ("LC_PAPER", "de_DE")]), none).paper, Paper::Letter); + assert_eq!(from_posix(env(&[("LANG", "en_US"), ("LC_MEASUREMENT", "it_IT")]), none).measurement, Measurement::Metric); + assert_eq!(from_posix(env(&[("LANG", "C.UTF-8"), ("LC_ALL", "")]), || Some("it-IT".into())).numbers, "it-IT"); + assert_eq!(from_posix(env(&[]), none), Region::default()); + } + + #[test] + fn windows_regional_format_and_its_changes() { + let reg = |pairs: &'static [(&'static str, &'static str)]| move |n: &str| pairs.iter().find(|(k, _)| *k == n).map(|(_, v)| v.to_string()); + let r = from_windows( + reg(&[("LocaleName", "pl-PL"), ("sDecimal", ","), ("sThousand", "\u{a0}"), ("iMeasure", "0"), ("sShortTime", "HH:mm")]), + || None, + ); + assert_eq!((r.numbers.as_str(), r.decimal, r.grouping, r.hour_cycle), ("pl-PL", ',', Some('\u{a0}'), Some(HourCycle::H23))); + // An English format in the US with the user's metric and A4 choices and a decimal comma. + let r = from_windows( + reg(&[("LocaleName", "en-US"), ("sDecimal", ","), ("iMeasure", "0"), ("iPaperSize", "9"), ("sShortTime", "h:mm tt")]), + || None, + ); + assert_eq!((r.decimal, r.measurement, r.paper, r.hour_cycle), (',', Measurement::Metric, Paper::A4, Some(HourCycle::H12))); + let r = from_windows(reg(&[("sDecimal", "dec"), ("sThousand", "")]), || Some("de-DE".into())); + assert_eq!((r.numbers.as_str(), r.decimal, r.grouping), ("de-DE", ',', None), "a separator longer than one character keeps the locale's"); + } + + #[test] + fn apple_locale_values() { + let r = from_apple("en_PL", true, ",", "\u{a0}", Some("HH")); + assert_eq!( + (r.numbers.as_str(), r.paper, r.decimal, r.grouping, r.hour_cycle), + ("en-PL", Paper::A4, ',', Some('\u{a0}'), Some(HourCycle::H23)) + ); + let r = from_apple("en_US", false, ".", ",", Some("h a")); + assert_eq!((r.measurement, r.hour_cycle), (Measurement::Us, Some(HourCycle::H12))); + assert_eq!(from_apple("de_DE", true, ",", ".", Some("H 'Uhr'")).hour_cycle, Some(HourCycle::H23)); + assert_eq!(hour_cycle_of("'h' mm"), None, "quoted letters are text"); + } +} From 3fcaddbdaeaefffb89563228d70d9743ad89d0aa Mon Sep 17 00:00:00 2001 From: aiongg Date: Wed, 21 Oct 2026 04:44:53 +0000 Subject: [PATCH 2/7] i18n: file sizes and measures with translatable unit names I18n::file_size writes a byte count in steps of 1024 (KB or KiB) or 1000 (KB), with decimals per unit, no decimals from a given value up, a smallest and largest unit, and bytes as B or in words. I18n::measure writes any value with a unit template such as "{n} px". Unit names are catalog rows in context `unit`, keyed by the English template, so a translation can change the symbol and its spacing. --- crates/i18n/README.md | 8 ++ crates/i18n/src/lib.rs | 3 + crates/i18n/src/size.rs | 242 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 253 insertions(+) create mode 100644 crates/i18n/src/size.rs diff --git a/crates/i18n/README.md b/crates/i18n/README.md index df337971..f3daff2c 100644 --- a/crates/i18n/README.md +++ b/crates/i18n/README.md @@ -68,6 +68,14 @@ The app keeps its `tl!` macro, its language setting (`"auto"` → `I18N.system_l `zh-hant`, `pt-PT` → `pt-br` when that is the only Portuguese, `C`/`POSIX` → English. A script is never crossed: `sr_RS@latin` and `sr-Latn-RS` pick `sr-latn`, never `sr`, while `sr-Cyrl-RS` picks `sr`. +- **Sizes and units:** `file_size(lang, bytes, style)` writes `1.5 MB` or `512 bytes`; `FileSize` + picks steps of 1024 named KB (`BINARY`), 1024 named KiB (`IEC`) or 1000 named KB (`DECIMAL`), + the decimals per unit, no decimals from a given value up, the smallest and largest unit (below + the smallest, sizes round up), and `B` or words for bytes. `measure(lang, value, decimals, + "{n} px")` writes any other measure. Unit names are rows in context `unit`, keyed by the English + template (`{n} MB` → `{n} Mo`, `{n}%` → `{n} %`) and read with `get`, so a plain `B` row (bold) + is never used; bytes in words are the `@plural` row `{n} byte|{n} bytes`. Numbers are + ASCII digits with a decimal point. - **Pseudo locale** (`pseudo`, debug builds only, never in `languages()`): every lookup, `get` included, comes back as `⟦…⟧`, line by line, so text drawn without the marks never went through translation. `looked_up()` lists the source texts looked up so far, for an audit against a diff --git a/crates/i18n/src/lib.rs b/crates/i18n/src/lib.rs index 7fa7a4ef..30eb6972 100644 --- a/crates/i18n/src/lib.rs +++ b/crates/i18n/src/lib.rs @@ -6,6 +6,7 @@ #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] mod catalog; +mod size; use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::sync::{Mutex, OnceLock, PoisonError}; @@ -13,6 +14,8 @@ use std::sync::{Mutex, OnceLock, PoisonError}; use intl_pluralrules::{PluralCategory, PluralRuleType, PluralRules}; use unic_langid::LanguageIdentifier; +pub use size::{FileSize, SizeUnit}; + /// The catalogs an app embeds, one entry per locale: (locale, shared catalog, the app's catalog). /// A missing catalog is `""`. pub type Embedded = [(&'static str, &'static str, &'static str)]; diff --git a/crates/i18n/src/size.rs b/crates/i18n/src/size.rs new file mode 100644 index 00000000..1466672b --- /dev/null +++ b/crates/i18n/src/size.rs @@ -0,0 +1,242 @@ +//! File sizes and other measures with a unit, in the interface language. +//! +//! Unit names are catalog rows in context `unit`, keyed by the English template with the +//! number as `{n}` (`{n} MB`, `{n} px`, `{n}%`), so a translation can change the symbol and its +//! spacing (`{n} Mo`, `{n} %`). They are looked up with [`I18n::get`], never the plain row: a +//! plain `B` row means bold. A byte count in words is the `@plural` row `{n} byte|{n} bytes`. + +use crate::{I18n, Lang, fmt}; + +/// The context unit rows are in. +const CONTEXT: &str = "unit"; + +/// A file-size unit. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum SizeUnit { + Byte, + Kilo, + Mega, + Giga, + Tera, +} + +impl SizeUnit { + const ALL: [SizeUnit; 5] = [SizeUnit::Byte, SizeUnit::Kilo, SizeUnit::Mega, SizeUnit::Giga, SizeUnit::Tera]; +} + +/// How [`I18n::file_size`] writes a byte count: the step between units, their names, and the +/// digits after the decimal point. Start from [`FileSize::BINARY`], [`FileSize::IEC`] or +/// [`FileSize::DECIMAL`] and adjust: +/// +/// ``` +/// use craft_i18n::{FileSize, SizeUnit}; +/// // `12 MB`, `3.4 MB`, `640 KB`: decimal steps, whole kilobytes, no decimals from 10 MB. +/// let style = FileSize::DECIMAL.smallest(SizeUnit::Kilo).largest(SizeUnit::Mega).unit_decimals(SizeUnit::Kilo, 0).whole_from(10); +/// ``` +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct FileSize { + step: u64, + iec: bool, + /// Per unit, in [`SizeUnit::ALL`] order; bytes are always whole. + decimals: [u8; 5], + whole_from: u32, + smallest: SizeUnit, + largest: SizeUnit, + byte_symbol: bool, +} + +impl FileSize { + /// Steps of 1024 named KB, MB, GB and TB, as Windows Explorer shows sizes; one decimal, bytes + /// in words. + pub const BINARY: FileSize = FileSize { + step: 1024, + iec: false, + decimals: [0, 1, 1, 1, 1], + whole_from: 0, + smallest: SizeUnit::Byte, + largest: SizeUnit::Tera, + byte_symbol: false, + }; + /// Steps of 1024 named KiB, MiB, GiB and TiB. + pub const IEC: FileSize = FileSize { iec: true, ..FileSize::BINARY }; + /// Steps of 1000 named KB, MB, GB and TB, as macOS shows sizes and as download and disk sizes + /// are sold. + pub const DECIMAL: FileSize = FileSize { step: 1000, ..FileSize::BINARY }; + + /// `n` digits after the decimal point in every unit above bytes. + pub const fn decimals(mut self, n: u8) -> FileSize { + self.decimals = [0, n, n, n, n]; + self + } + + /// `n` digits after the decimal point in `unit` (`0` for whole kilobytes, `2` for gigabytes). + pub const fn unit_decimals(mut self, unit: SizeUnit, n: u8) -> FileSize { + if !matches!(unit, SizeUnit::Byte) { + self.decimals[unit as usize] = n; + } + self + } + + /// No decimals from `n` of a unit up: `whole_from(10)` writes `9.5 MB` and `12 MB`. `0`, the + /// default, keeps them. + pub const fn whole_from(mut self, n: u32) -> FileSize { + self.whole_from = n; + self + } + + /// The smallest unit shown. Sizes below it are written in it, rounded up, so a file that + /// isn't empty never reads `0 KB`. + pub const fn smallest(mut self, unit: SizeUnit) -> FileSize { + self.smallest = unit; + self + } + + /// The largest unit shown; larger sizes are counted in it (`2048 GB`). + pub const fn largest(mut self, unit: SizeUnit) -> FileSize { + self.largest = unit; + self + } + + /// Bytes as the symbol `B` (`512 B`) instead of words (`512 bytes`). + pub const fn byte_symbol(mut self) -> FileSize { + self.byte_symbol = true; + self + } + + /// The unit `bytes` is written in and the number in it, in English. + fn parts(&self, bytes: u64) -> (SizeUnit, String) { + let (smallest, largest) = (self.smallest.min(self.largest), self.largest); + let mut unit = smallest; + for &u in SizeUnit::ALL.iter().filter(|u| (smallest..=largest).contains(u)) { + if bytes as f64 >= self.scale(u) { + unit = u; + } + } + loop { + let value = bytes as f64 / self.scale(unit); + let mut decimals = self.decimals.get(unit as usize).copied().unwrap_or(0); + if unit == SizeUnit::Byte || (self.whole_from > 0 && value >= f64::from(self.whole_from)) { + decimals = 0; + } + let factor = 10f64.powi(i32::from(decimals)); + let rounded = if unit == smallest && bytes > 0 && value < 1.0 { (value * factor).ceil() } else { (value * factor).round() } / factor; + // 1023.96 KB rounds to 1024.0 KB: the next unit writes it as 1.0 MB. + if rounded >= self.step as f64 + && let Some(&next) = SizeUnit::ALL.get(unit as usize + 1) + && next <= largest + { + unit = next; + continue; + } + return (unit, number(rounded, usize::from(decimals))); + } + } + + fn scale(&self, unit: SizeUnit) -> f64 { + (self.step as f64).powi(unit as i32) + } + + /// The English template for `unit`, which is also its catalog key. + fn template(&self, unit: SizeUnit) -> &'static str { + match (unit, self.iec) { + (SizeUnit::Byte, _) => "{n} B", + (SizeUnit::Kilo, false) => "{n} KB", + (SizeUnit::Mega, false) => "{n} MB", + (SizeUnit::Giga, false) => "{n} GB", + (SizeUnit::Tera, false) => "{n} TB", + (SizeUnit::Kilo, true) => "{n} KiB", + (SizeUnit::Mega, true) => "{n} MiB", + (SizeUnit::Giga, true) => "{n} GiB", + (SizeUnit::Tera, true) => "{n} TiB", + } + } +} + +impl I18n { + /// `bytes` as a file size in `lang`, written the way `style` says (`1.5 MB`, `512 bytes`). + pub fn file_size(&self, lang: Lang, bytes: u64, style: FileSize) -> String { + let (unit, n) = style.parts(bytes); + if unit == SizeUnit::Byte && !style.byte_symbol { + return self.trn(lang, bytes, "{n} byte", "{n} bytes"); + } + self.unit_text(lang, style.template(unit), &n) + } + + /// `value` with `decimals` digits after the decimal point in the unit template `unit`, an + /// English catalog key with the number as `{n}` (`{n} px`, `{n}%`, `{n} dB`). + pub fn measure(&self, lang: Lang, value: f64, decimals: usize, unit: &str) -> String { + self.unit_text(lang, unit, &number(value, decimals)) + } + + fn unit_text(&self, lang: Lang, template: &str, n: &str) -> String { + fmt(self.get(lang, CONTEXT, template).unwrap_or(template), &[("n", n)]) + } +} + +/// `value` with `decimals` digits after the decimal point, in ASCII digits with a decimal point. +/// The one place sizes and measures write numbers, for craft-i18n's locale number formatting +/// (digits, separators) to take over. +fn number(value: f64, decimals: usize) -> String { + format!("{value:.decimals$}") +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::Embedded; + + static E: &Embedded = &[( + "fr", + "unit\t{n} KB\t{n} Ko\nunit\t{n} MB\t{n} Mo\nunit\t{n} B\t{n} o\nunit\t{n}%\t{n} %\n\tB\tG\n@plural\t{n} byte|{n} bytes\t{n} octet|{n} octets\n", + "", + )]; + + fn en(bytes: u64, style: FileSize) -> String { + let (i18n, _) = I18n::new(&[], &[]); + i18n.file_size(Lang::EN, bytes, style) + } + + #[test] + fn binary_decimal_and_iec_steps() { + assert_eq!(en(512, FileSize::BINARY), "512 bytes"); + assert_eq!(en(1, FileSize::BINARY), "1 byte"); + assert_eq!(en(1536, FileSize::BINARY), "1.5 KB"); + assert_eq!(en(1536, FileSize::IEC), "1.5 KiB"); + assert_eq!(en(1500, FileSize::DECIMAL), "1.5 KB"); + assert_eq!(en(3 << 30, FileSize::BINARY), "3.0 GB"); + assert_eq!(en(5 << 40, FileSize::IEC.decimals(0)), "5 TiB"); + assert_eq!(en(512, FileSize::BINARY.byte_symbol()), "512 B"); + } + + #[test] + fn precision_and_unit_range() { + // Rounding up to the next unit's step moves to that unit. + assert_eq!(en(1_048_575, FileSize::BINARY), "1.0 MB"); + assert_eq!(en(999_999, FileSize::DECIMAL.decimals(0)), "1 MB"); + assert_eq!(en(1_610_612_736, FileSize::BINARY.unit_decimals(SizeUnit::Giga, 2)), "1.50 GB"); + // Lightcraft's denoise sizes: whole kilobytes, at least 1, no decimals from 10 MB. + let style = FileSize::DECIMAL.smallest(SizeUnit::Kilo).largest(SizeUnit::Mega).unit_decimals(SizeUnit::Kilo, 0).whole_from(10); + assert_eq!(en(1, style), "1 KB"); + assert_eq!(en(0, style), "0 KB"); + assert_eq!(en(640_400, style), "640 KB"); + assert_eq!(en(3_400_000, style), "3.4 MB"); + assert_eq!(en(12_600_000, style), "13 MB"); + assert_eq!(en(5_000_000_000, style), "5000 MB"); + // Only sizes below the smallest unit round up. + assert_eq!(en(1025, FileSize::BINARY.smallest(SizeUnit::Kilo).unit_decimals(SizeUnit::Kilo, 0)), "1 KB"); + assert_eq!(en(100, FileSize::BINARY.smallest(SizeUnit::Kilo).unit_decimals(SizeUnit::Kilo, 0)), "1 KB"); + } + + #[test] + fn unit_names_are_catalog_rows() { + let (i18n, problems) = I18n::new(E, &["fr"]); + assert!(problems.is_empty(), "{problems:?}"); + let fr = i18n.from_code("fr").unwrap(); + assert_eq!(i18n.file_size(fr, 1536, FileSize::BINARY), "1.5 Ko"); + assert_eq!(i18n.file_size(fr, 512, FileSize::BINARY), "512 octets"); + assert_eq!(i18n.file_size(fr, 512, FileSize::BINARY.byte_symbol()), "512 o", "not the plain row B (bold)"); + assert_eq!(i18n.file_size(fr, 3 << 30, FileSize::BINARY), "3.0 GB", "a missing row is English"); + assert_eq!(i18n.measure(fr, 50.0, 0, "{n}%"), "50 %"); + assert_eq!(i18n.measure(Lang::EN, 12.34, 1, "{n} px"), "12.3 px"); + } +} From cf36f3cb215323845809c9688db681af2d835af9 Mon Sep 17 00:00:00 2001 From: aiongg Date: Wed, 21 Oct 2026 05:14:23 +0000 Subject: [PATCH 3/7] i18n: numbers for a region on ICU4X, locale digits in counts and sizes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Format writes integers, decimals and percentages for a Region on ICU4X's decimal formatter: the locale's digits (Arabic-Indic in Egypt, Persian in Iran) or Latin ones on request, its separators and grouping sizes (Indian 12,34,567), CLDR's percent patterns, and the region's own separators where the system sets them. I18n::format holds the one trn, file_size and measure use, the system region's until set_format. Plural rules move from intl_pluralrules to ICU4X, and trn_decimal picks the form from the digits shown (1.0 files, 1,5 файла). The CLDR data (decimal symbols, digits, plural rules, percent patterns) covers the shipped languages and their regional variants; data/generate.sh rebuilds it. --- Cargo.lock | 170 +++++++++++++++--- crates/i18n/Cargo.toml | 14 +- crates/i18n/README.md | 29 ++- crates/i18n/data/generate.sh | 23 +++ crates/i18n/data/locales.rs | 79 +++++++++ crates/i18n/data/numbers.postcard | Bin 0 -> 5664 bytes crates/i18n/data/percent.py | 64 +++++++ crates/i18n/src/format.rs | 286 ++++++++++++++++++++++++++++++ crates/i18n/src/lib.rs | 148 +++++++++++++--- crates/i18n/src/size.rs | 49 ++--- 10 files changed, 777 insertions(+), 85 deletions(-) create mode 100755 crates/i18n/data/generate.sh create mode 100644 crates/i18n/data/locales.rs create mode 100644 crates/i18n/data/numbers.postcard create mode 100644 crates/i18n/data/percent.py create mode 100644 crates/i18n/src/format.rs diff --git a/Cargo.lock b/Cargo.lock index 11eebe64..7fc992df 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -491,6 +491,15 @@ dependencies = [ "x509-cert", ] +[[package]] +name = "cobs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa961b519f0b462e3a3b4a34b64d119eeaca1d59af726fe450bbba07a9fc0a1" +dependencies = [ + "thiserror 2.0.21", +] + [[package]] name = "codespan-reporting" version = "0.13.1" @@ -656,11 +665,18 @@ version = "0.1.0" name = "craft-i18n" version = "0.1.0" dependencies = [ - "intl_pluralrules", + "fixed_decimal", + "icu_decimal", + "icu_locale_core", + "icu_locale_fallback", + "icu_plurals", + "icu_provider", + "icu_provider_adapters", + "icu_provider_blob", "objc2-foundation 0.3.2", "sys-locale", - "unic-langid", "windows-registry", + "writeable", ] [[package]] @@ -1232,6 +1248,17 @@ version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" +[[package]] +name = "fixed_decimal" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79c3c892f121fff406e5dd6b28c1b30096b95111c30701a899d4f2b18da6d1bd" +dependencies = [ + "displaydoc", + "smallvec", + "writeable", +] + [[package]] name = "flagset" version = "0.4.7" @@ -1861,6 +1888,23 @@ dependencies = [ "zerovec", ] +[[package]] +name = "icu_decimal" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9eb8f655bba2d0c0459e43a5b0e6cd3cd388109898fb3d85d048165e8b0abd08" +dependencies = [ + "displaydoc", + "fixed_decimal", + "icu_locale_core", + "icu_pattern", + "icu_plurals", + "icu_provider", + "serde", + "writeable", + "zerovec", +] + [[package]] name = "icu_locale_core" version = "2.3.0" @@ -1869,11 +1913,32 @@ checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" dependencies = [ "displaydoc", "litemap", + "serde", "tinystr", "writeable", "zerovec", ] +[[package]] +name = "icu_locale_fallback" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "251af8e57c9400e3eb58242fe5b8b1152b2a64fdf4cf632f923c38ccee6f2fa9" +dependencies = [ + "icu_locale_core", + "icu_locale_fallback_data", + "icu_provider", + "potential_utf", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locale_fallback_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "decf2a22ec8fa68f1a0c1129a3f8583f8f8bc24e8b9ccbe98ead99f62a4dc3a8" + [[package]] name = "icu_normalizer" version = "2.3.0" @@ -1894,6 +1959,32 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" +[[package]] +name = "icu_pattern" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b3db611446bd92c246b03e4aa0bafa71ddb194af431ddbb42b9d70e4bfce1a0" +dependencies = [ + "displaydoc", + "either", + "serde", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_plurals" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e475e6766ef87b1d3c1f97be6363995b0bfaeddbc789671615df598a97ff0593" +dependencies = [ + "displaydoc", + "fixed_decimal", + "icu_provider", + "serde", + "zerovec", +] + [[package]] name = "icu_properties" version = "2.3.0" @@ -1923,6 +2014,9 @@ checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" dependencies = [ "displaydoc", "icu_locale_core", + "postcard", + "serde", + "stable_deref_trait", "writeable", "yoke", "zerofrom", @@ -1930,6 +2024,30 @@ dependencies = [ "zerovec", ] +[[package]] +name = "icu_provider_adapters" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df887bc4f55dbc773c7c45a0c7dfa75be886f7b7812830a71ab7271cc1801820" +dependencies = [ + "icu_locale_fallback", + "icu_provider", +] + +[[package]] +name = "icu_provider_blob" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5916b44fac319d3f58e4d8d805a627df19e27937b582cd99fcde3bb6b7ce1b0e" +dependencies = [ + "icu_provider", + "postcard", + "serde", + "writeable", + "zerotrie", + "zerovec", +] + [[package]] name = "idna" version = "1.1.0" @@ -1997,15 +2115,6 @@ dependencies = [ "hashbrown 0.17.1", ] -[[package]] -name = "intl_pluralrules" -version = "7.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "078ea7b7c29a2b4df841a7f6ac8775ff6074020c6776d48491ce2268e068f972" -dependencies = [ - "unic-langid", -] - [[package]] name = "ipnet" version = "2.12.2" @@ -2250,6 +2359,9 @@ name = "litemap" version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" +dependencies = [ + "serde_core", +] [[package]] name = "litrs" @@ -3081,12 +3193,24 @@ dependencies = [ "portable-atomic", ] +[[package]] +name = "postcard" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6764c3b5dd454e283a30e6dfe78e9b31096d9e32036b5d1eaac7a6119ccb9a24" +dependencies = [ + "cobs", + "serde", +] + [[package]] name = "potential_utf" version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" dependencies = [ + "serde_core", + "writeable", "zerovec", ] @@ -4717,24 +4841,6 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" -[[package]] -name = "unic-langid" -version = "0.9.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a28ba52c9b05311f4f6e62d5d9d46f094bd6e84cb8df7b3ef952748d752a7d05" -dependencies = [ - "unic-langid-impl", -] - -[[package]] -name = "unic-langid-impl" -version = "0.9.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dce1bf08044d4b7a94028c93786f8566047edc11110595914de93362559bc658" -dependencies = [ - "tinystr", -] - [[package]] name = "unicode-bidi" version = "0.3.18" @@ -5794,6 +5900,9 @@ name = "writeable" version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" +dependencies = [ + "either", +] [[package]] name = "x11rb" @@ -5987,8 +6096,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" dependencies = [ "displaydoc", + "litemap", + "serde_core", "yoke", "zerofrom", + "zerovec", ] [[package]] diff --git a/crates/i18n/Cargo.toml b/crates/i18n/Cargo.toml index bfff93ce..3daaca10 100644 --- a/crates/i18n/Cargo.toml +++ b/crates/i18n/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "craft-i18n" -description = "Translations for the Craft apps: the embedded craft-i18n catalogs, lookup with fallback, plurals, language names and the system locale." +description = "Translations for the Craft apps: the embedded craft-i18n catalogs, lookup with fallback, plurals, numbers for a region, language names and the system locale." version.workspace = true edition.workspace = true license.workspace = true @@ -8,8 +8,16 @@ rust-version.workspace = true repository.workspace = true [dependencies] -intl_pluralrules = "=7.0.2" -unic-langid = "0.9" +# Numbers and plural rules, on CLDR data for the shipped locales (data/numbers.postcard). +fixed_decimal = { version = "0.7.2", default-features = false } +icu_decimal = { version = "2.3", default-features = false, features = ["serde"] } +icu_locale_core = { version = "2.3", default-features = false } +icu_locale_fallback = { version = "2.3", default-features = false, features = ["compiled_data"] } +icu_plurals = { version = "2.3", default-features = false, features = ["serde"] } +icu_provider = { version = "2.3", default-features = false, features = ["sync"] } +icu_provider_adapters = { version = "2.3", default-features = false } +icu_provider_blob = { version = "2.3", default-features = false } +writeable = "0.6" sys-locale = "0.3" [target.'cfg(target_arch = "wasm32")'.dependencies] diff --git a/crates/i18n/README.md b/crates/i18n/README.md index 5d37b72e..bb48bbad 100644 --- a/crates/i18n/README.md +++ b/crates/i18n/README.md @@ -45,7 +45,8 @@ The app keeps its `tl!` macro, its language setting (`"auto"` → `I18N.system_l → `pt`, `zh-hant-hk` → `zh-hant`); `sr-latn` never falls back to Cyrillic `sr`. A missing translation never errors. - **Lookups:** `t` (plain), `tc` (with a context such as `menu` or `@msg`, else plain), `tid` (an - `@id` command id, else the English label), `trn` (a count, `@plural` rows, `{n}` filled), `get` + `@id` command id, else the English label), `trn` and `trn_decimal` (a count, `@plural` rows, + `{n}` filled), `get` (one context along the locale chain, so `pt-br` can return a `pt` row, but never the plain row or English: a key cap's name under `key`, or "is there a translation?"), and `fmt` for `{name}` placeholders, filled in one pass. @@ -54,10 +55,12 @@ The app keeps its `tl!` macro, its language setting (`"auto"` → `I18N.system_l are skipped and returned by `I18n::new` for the app to log; craft-i18n's CI rejects them before they ship. - **Plurals:** forms are listed in CLDR order (zero, one, two, few, many, other) for the categories - whole numbers 0 to 1000 reach, from `intl_pluralrules`; a count in any other category takes the - last form. CLDR's `pt` is Brazilian, so `pt` uses `pt-PT`'s rules (0 is plural) and `pt-br` - `pt`'s. `intl_pluralrules` is pinned to the version craft-i18n's checker uses, and - `plural_form_counts` holds the same counts. + whole numbers 0 to 1000 reach, from ICU4X's CLDR rules; `plural_form_counts` holds the counts + craft-i18n's checker expects. `trn_decimal` picks the form from the digits shown (ICU4X plural + operands), so English `1.0` is plural and French `1,5` singular. A count in a category integers + never reach takes the `few` form if there is one (Russian and Ukrainian decimals, `1,5 файла`), + else the last. CLDR's `pt` is Brazilian, so `pt` uses `pt-PT`'s rules (0 is plural) and `pt-br` + `pt`'s. A language outside the baked data (below) has no plural rules and shows English forms. - **Names:** a language's name comes from its shared catalog's `@id language.name` row, else its code; English is "English". - **Right to left:** `rtl` is true for `ar`, `ckb`, `dv`, `fa`, `he`, `ps`, `sd`, `ug`, `ur` and @@ -74,8 +77,20 @@ The app keeps its `tl!` macro, its language setting (`"auto"` → `I18N.system_l the smallest, sizes round up), and `B` or words for bytes. `measure(lang, value, decimals, "{n} px")` writes any other measure. Unit names are rows in context `unit`, keyed by the English template (`{n} MB` → `{n} Mo`, `{n}%` → `{n} %`) and read with `get`, so a plain `B` row (bold) - is never used; bytes in words are the `@plural` row `{n} byte|{n} bytes`. Numbers are - ASCII digits with a decimal point. + is never used; bytes in words are the `@plural` row `{n} byte|{n} bytes`. Numbers are written + by `I18n::format`. +- **Numbers:** `Format` writes numbers for a `Region` on ICU4X's decimal formatter: `integer` + (`1.234.567`, Indian `12,34,567`), `number(value, decimals)` and `percent(ratio, decimals)` + (`50 %`, Turkish `%50`). It follows the region, not the interface language, with the locale's + digits (`١٬٢٣٤٫٥` in Egypt, `۱٬۲۳۴` in Iran) or, with `Digits::Latin`, `0`–`9`. With Latin + digits, a decimal or grouping separator the region sets replaces the locale's (English in Poland: + `1.234,5`). `I18n::format` is the format `trn`, `file_size` and `measure` use: the system + region's until `I18n::set_format` (the user's region setting or digit choice). +- **CLDR data:** decimal symbols, digits and plural rules (`data/numbers.postcard`, about 6 KB) and + percent patterns (`data/locales.rs`) cover the languages the catalogs and apps ship, with their + regional variants; `data/generate.sh` lists them and rebuilds both from CLDR. Add a language + there when a catalog adds one. The ICU4X code and data add about 90 KB to a stripped release + build and 80 KB to wasm. - **Formatting region:** `Region` holds how numbers, times and measurements are written, apart from the interface language: the number and time locales (BCP 47), metric or US measurement, A4 or Letter, the decimal separator, and the grouping separator and clock when the system sets diff --git a/crates/i18n/data/generate.sh b/crates/i18n/data/generate.sh new file mode 100755 index 00000000..695a698a --- /dev/null +++ b/crates/i18n/data/generate.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# Regenerates the number and plural data in this directory from Unicode CLDR, for the languages +# the craft-i18n catalogs and the apps ship (each with its regional variants). +# +# Needs `cargo install icu4x-datagen --version 2.3.0` (the ICU4X version in Cargo.toml) and +# python3. Downloads CLDR's JSON release once into $CACHE. +set -euo pipefail + +CLDR=48.2.1 +LANGUAGES=(en ar bg cs da de el es et fa fi fr hi hr hu id it ja ka kk ko lt nl pl pt ru sl sr sv te tr uk vi zh) +HERE=$(cd "$(dirname "$0")" && pwd) +CACHE=${CACHE:-${XDG_CACHE_HOME:-$HOME/.cache}/craft-i18n-data} +mkdir -p "$CACHE" + +# `sr-Latn` and `zh-Hant` are named as well: their CLDR parent is the root, not `sr` or `zh`. +TMPDIR=$CACHE icu4x-datagen --format blob --overwrite \ + --cldr-tag "$CLDR" \ + --markers DecimalSymbolsV1 DecimalDigitsV1 PluralsCardinalV1 \ + --locales "${LANGUAGES[@]}" sr-Latn zh-Hant \ + --out "$HERE/numbers.postcard" + +ZIP=$CACHE/icu4x-source-cache/github.com/unicode-org/cldr-json/releases/download/$CLDR/cldr-$CLDR-json-full.zip +python3 -I "$HERE/percent.py" "$ZIP" "$CLDR" "${LANGUAGES[@]}" > "$HERE/locales.rs" diff --git a/crates/i18n/data/locales.rs b/crates/i18n/data/locales.rs new file mode 100644 index 00000000..a2ad6e72 --- /dev/null +++ b/crates/i18n/data/locales.rs @@ -0,0 +1,79 @@ +// @generated by generate.sh from Unicode CLDR 48.2.1 (Unicode License). + +/// The languages the baked number and plural data cover, each with its regional variants. +pub(crate) const LANGUAGES: &[&str] = &["en", "ar", "bg", "cs", "da", "de", "el", "es", "et", "fa", "fi", "fr", "hi", "hr", "hu", "id", "it", "ja", "ka", "kk", "ko", "lt", "nl", "pl", "pt", "ru", "sl", "sr", "sv", "te", "tr", "uk", "vi", "zh"]; + +/// Percent signs around the number (CLDR's standard percent pattern and sign): locale, Latin +/// digits or the locale's own, text before the number, text after it. A locale missing here +/// takes its parent's by dropping trailing subtags, then `{n}%`. +pub(crate) const PERCENT: &[(&str, bool, &str, &str)] = &[ + ("ar", true, "", "\u{200e}%\u{200e}"), + ("ar-BH", false, "", "\u{66a}\u{61c}"), + ("ar-DJ", false, "", "\u{66a}\u{61c}"), + ("ar-EG", false, "", "\u{66a}\u{61c}"), + ("ar-ER", false, "", "\u{66a}\u{61c}"), + ("ar-IL", false, "", "\u{66a}\u{61c}"), + ("ar-IQ", false, "", "\u{66a}\u{61c}"), + ("ar-JO", false, "", "\u{66a}\u{61c}"), + ("ar-KM", false, "", "\u{66a}\u{61c}"), + ("ar-KW", false, "", "\u{66a}\u{61c}"), + ("ar-LB", false, "", "\u{66a}\u{61c}"), + ("ar-MR", false, "", "\u{66a}\u{61c}"), + ("ar-OM", false, "", "\u{66a}\u{61c}"), + ("ar-PS", false, "", "\u{66a}\u{61c}"), + ("ar-QA", false, "", "\u{66a}\u{61c}"), + ("ar-SA", false, "", "\u{66a}\u{61c}"), + ("ar-SA", true, "", "\u{66a}"), + ("ar-SD", false, "", "\u{66a}\u{61c}"), + ("ar-SO", false, "", "\u{66a}\u{61c}"), + ("ar-SO", true, "", "\u{66a}"), + ("ar-SS", false, "", "\u{66a}\u{61c}"), + ("ar-SY", false, "", "\u{66a}\u{61c}"), + ("ar-TD", false, "", "\u{66a}\u{61c}"), + ("ar-YE", false, "", "\u{66a}\u{61c}"), + ("cs", true, "", "\u{a0}%"), + ("da", true, "", "\u{a0}%"), + ("de", true, "", "\u{a0}%"), + ("de-CH", true, "", "%"), + ("de-LI", true, "", "%"), + ("en-AT", true, "", "\u{a0}%"), + ("en-DE", true, "", "\u{a0}%"), + ("en-DK", true, "", "\u{a0}%"), + ("en-FI", true, "", "\u{a0}%"), + ("en-NO", true, "", "\u{a0}%"), + ("en-SE", true, "", "\u{a0}%"), + ("es", true, "", "\u{a0}%"), + ("es-419", true, "", "%"), + ("es-AR", true, "", "%"), + ("es-BO", true, "", "%"), + ("es-BR", true, "", "%"), + ("es-BZ", true, "", "%"), + ("es-CL", true, "", "%"), + ("es-CO", true, "", "%"), + ("es-CR", true, "", "%"), + ("es-CU", true, "", "%"), + ("es-DO", true, "", "%"), + ("es-EC", true, "", "%"), + ("es-GT", true, "", "%"), + ("es-HN", true, "", "%"), + ("es-MX", true, "", "%"), + ("es-NI", true, "", "%"), + ("es-PA", true, "", "%"), + ("es-PE", true, "", "%"), + ("es-PR", true, "", "%"), + ("es-PY", true, "", "%"), + ("es-SV", true, "", "%"), + ("es-US", true, "", "%"), + ("es-UY", true, "", "%"), + ("es-VE", true, "", "%"), + ("fa", false, "", "\u{66a}"), + ("fi", true, "", "\u{a0}%"), + ("fr", true, "", "\u{a0}%"), + ("fr-CH", true, "", "%"), + ("hr", true, "", "\u{a0}%"), + ("lt", true, "", "\u{a0}%"), + ("ru", true, "", "\u{a0}%"), + ("sl", true, "", "\u{a0}%"), + ("sv", true, "", "\u{a0}%"), + ("tr", true, "%", ""), +]; diff --git a/crates/i18n/data/numbers.postcard b/crates/i18n/data/numbers.postcard new file mode 100644 index 0000000000000000000000000000000000000000..53764d4cc9c73c84627d11699de9e5f564923c92 GIT binary patch literal 5664 zcmcJTTTola8OP7rBgEC%*sdMNvSJg!iH!)()wy&WA%sMP5FoB@XHwZvqacJw2-j|> zjbwwtsgtblCP^FLu&gSSD1_J!w6zaVr&(ZF0zuoLdJZzYsb#kwjZ{;pX=~= zcl)~iJ$ru4aHVP0D;<|MGiU1x>At5;&+h1*nlIRDEcfw}`5k(Lpmq6NJ9zPYm$9te zT=|Tx%I>JCb=J2C@k>;$EZq|wN6!D@Z)vOVfAW+&+TA zetS)cGo{9uQZsEVcQ{+}@*gTsY5F!QUn{g>#ST~&vkNT1LJzGn{V}ayZr)V%dHl+%IwyQ@(wG=RKpzYO^=D#2a&! zdcE4p3#QB#n;jjMKIk`Tu5&UCFMy$&#-#0a6jk|5F^k^E?*>+5cQ&-dhRa>HT!~hc zh^dS%+mNJuUT~w_TxG9z)W_$h&TIqPbdzR#4aQRZDRX5tMO33qn~c^Uaekw{P_tUMlMi zS`WR<9xzt$Qhn@^Z{kIUkC7WnBj!jO)zq@GbSurH=cj!y4h~(4Oq(kzpQ%#zuZYKg zZWznyH-U~@Mph1{8mnJRcf2ea-KUKe)it$?`_=OzJ1=VBIvi%~Y^}0dM99Td}!bE!FCC123xJMF>CT_W663uUX4C(285z_PF@^Dq8mj zzR^%-y{O zuElC_Z@{Ai*}`MJ@?RA%?7_%WWj1>QMrc773)=A;Q_c&Fyl~Rv+rvuveF#5YW~#C` zOWAAnMy}iE^BC;dZ(1ncUuDe_Ds9RUK*={5WMhTY%r{l?O>&zrlW)-b+dMt|UdSfA z>EsSC25IH_Zmw&`0n!^X+g)CFYuiqK|D@IKtnHL-F1(KPiCVAR?Z3w$TkT71$<;Wp z3(jqU>@CTSvZG~Dz~nYd@`pd)>2CpzHs`Jln@S&i^1ItAZQFma%k6u4SFrmK-udE- z{ZsTZ7nv07-``KiUt|aSdnet#9z47h#NLa60at)FT4cdsz~ATT$EJxd95B?>5A+GJQ>WaRAYb?+mRm;$N&Za5wk~_!O8$ zK{yR2z)euY1mQe*RuY7d!C{Rc+yJfUcolpI{sA(zg0K-dz>mP|;0E{-tWFn%ZD1F8 z8C(W)AU#769so_?c@P30fq#R0G6lf`_JIia3-}U=6)GMey(Ea+; zWlIyE<-`*nu;|!@jA0-Qa6MiTAN5-vV0F>}%7`?oGUB&9ke(qg*O2E#1_Q15KnY$! zro4Leh!ff_UHXCKLZ?V}?Hv2Z49b>cau-VIO_6nQ5%H{%WF39Hgv$+Fou9waA zvAKh6uAj{fu(?Ch&7q4R1TKLv_!WqNH^4CXH5id@hTw(Zh2VwYh2VwYh2VwYh2VwY zh2e$ah2e$ah2e$ah2e$ah2e$aMc_r?Mc_r?Mc_r?Mc_r?Mc_r?4Z|CTHwt`fH$Hx^*QW-Fo*h?uVq$UwswN$EDBvhVl6T z?h)zM)n7jj5=Br=!1*iy8v7d?q zRP<8ON5w%Z`l%S8;t+}Hnh#bB-|GA3yZn~OKnw_QmR z7ivnru7)L?l9XO*O_EbgUoa)M5V@pSiJV%KG3?v1N*- zh4oorB(5!!B#kCb``N(J&Bf|k?oh|_)Etx)$G^H^*gU0ky3#o@@SQ5zN=ee<3;rjs z89sS!^Gsys4f-FXh_F(l$;ime`YiC00W3`6w)%K7Byp8io0XNF0~do*k3`E_YRSo6 MXi-POcP0$ ... +""" + +import json +import re +import sys +import zipfile + + +def rust(s): + return '"' + "".join(c if " " <= c <= "~" and c not in '"\\' else f"\\u{{{ord(c):x}}}" for c in s) + '"' + + +def affixes(numbers, numsys): + pattern = numbers[f"percentFormats-numberSystem-{numsys}"]["standard"].split(";")[0].replace("'", "") + sign = numbers[f"symbols-numberSystem-{numsys}"]["percentSign"] + number = re.search(r"[#0,.]+", pattern) + return pattern[: number.start()].replace("%", sign), pattern[number.end() :].replace("%", sign) + + +def main(): + path, version, languages = sys.argv[1], sys.argv[2], sys.argv[3:] + found = {} + with zipfile.ZipFile(path) as z: + for name in z.namelist(): + m = re.fullmatch(r"cldr-numbers-full/main/([^/]+)/numbers\.json", name) + if not m or m.group(1).split("-")[0] not in languages: + continue + locale = m.group(1) + numbers = json.loads(z.read(name))["main"][locale]["numbers"] + found[(locale, True)] = affixes(numbers, "latn") + default = numbers["defaultNumberingSystem"] + if default != "latn": + found[(locale, False)] = affixes(numbers, default) + + # Kept only where the lookup by dropping trailing subtags would find something else. + kept = {} + for (locale, latin), value in sorted(found.items(), key=lambda e: (e[0][0].count("-"), e[0])): + parts = locale.split("-") + inherited = ("", "%") + for n in range(len(parts) - 1, 0, -1): + if ("-".join(parts[:n]), latin) in kept: + inherited = kept[("-".join(parts[:n]), latin)] + break + if value != inherited: + kept[(locale, latin)] = value + + print(f"// @generated by generate.sh from Unicode CLDR {version} (Unicode License).") + print() + print("/// The languages the baked number and plural data cover, each with its regional variants.") + print("pub(crate) const LANGUAGES: &[&str] = &[" + ", ".join(rust(l) for l in languages) + "];") + print() + print("/// Percent signs around the number (CLDR's standard percent pattern and sign): locale, Latin") + print("/// digits or the locale's own, text before the number, text after it. A locale missing here") + print("/// takes its parent's by dropping trailing subtags, then `{n}%`.") + print("pub(crate) const PERCENT: &[(&str, bool, &str, &str)] = &[") + for (locale, latin), (before, after) in sorted(kept.items()): + print(f" ({rust(locale)}, {str(latin).lower()}, {rust(before)}, {rust(after)}),") + print("];") + + +main() diff --git a/crates/i18n/src/format.rs b/crates/i18n/src/format.rs new file mode 100644 index 00000000..07dfd70c --- /dev/null +++ b/crates/i18n/src/format.rs @@ -0,0 +1,286 @@ +//! Numbers, percentages and counts written for a formatting [`Region`], on ICU4X's decimal +//! formatter with CLDR data for the shipped languages (`data/generate.sh`). + +use std::fmt; + +use fixed_decimal::{Decimal, Sign}; +use icu_decimal::{DecimalFormatter, parts}; +use icu_locale_core::Locale; +use icu_locale_core::extensions::unicode::{key, value}; +use writeable::{Part, PartsWrite, Writeable}; + +use crate::Region; +use crate::data::{self, PERCENT}; + +/// Which digits numbers are written in. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +pub enum Digits { + /// The locale's own: Arabic-Indic in Egypt (`١٢٣`), Persian in Iran (`۱۲۳`), else `0`–`9`. + #[default] + Locale, + /// `0`–`9` everywhere, for users who read Arabic or Persian with Latin digits. + Latin, +} + +/// How numbers are written in a formatting region: digits, decimal and grouping separators, +/// grouping sizes (`1,234,567`, Indian `12,34,567`) and the percent sign. Build it once per region +/// and keep it; [`crate::I18n`] holds one for counts, sizes and measures. +#[derive(Debug)] +pub struct Format { + /// `None` only if the data doesn't load: numbers are then written as Rust writes them. + decimal: Option, + /// The region's separators where the system set them apart from the locale's. + group: Option, + point: Option, + percent: (&'static str, &'static str), +} + +impl Format { + /// The format for `region`'s number locale. With Latin digits, a decimal or grouping separator + /// the region sets (Windows' and macOS's own settings, English in continental Europe) replaces + /// the locale's. + pub fn new(region: &Region, digits: Digits) -> Format { + let mut locale = Locale::try_from_str(®ion.numbers).unwrap_or(Locale::UNKNOWN); + if digits == Digits::Latin { + locale.extensions.unicode.keywords.set(key!("nu"), value!("latn")); + } + let decimal = data::provider().and_then(|p| DecimalFormatter::try_new_with_buffer_provider(p, (&locale).into(), Default::default()).ok()); + let mut format = Format { decimal, group: None, point: None, percent: ("", "%") }; + // The locale's own separators and digits, read from a number that shows them all. + let probe = format.parts(&Decimal::try_from_str("-1234567.5").unwrap_or_default()); + let latin = probe.integer.starts_with(|c: char| c.is_ascii_digit()); + if latin { + let point = region.decimal.to_string(); + if probe.point != point { + // `1,234.5` with a decimal comma becomes `1.234,5`, never `1,234,5`. + if probe.group == point && region.grouping.is_none() { + format.group = Some(probe.point.clone()); + } + format.point = Some(point); + } + if let Some(g) = region.grouping.map(String::from).filter(|g| *g != probe.group) { + format.group = Some(g); + } + } + format.percent = percent_affixes(®ion.numbers, latin); + format + } + + /// The format for a locale tag (`de-CH`, `ar-EG`), with the locale's own digits. + pub fn from_tag(tag: &str) -> Format { + Format::new(&Region::from_tag(tag), Digits::Locale) + } + + /// A whole number: `1,234,567`, `1.234.567`, `١٬٢٣٤٬٥٦٧`. + pub fn integer(&self, n: impl Into) -> String { + self.write(&Decimal::from(n.into())) + } + + /// `value` rounded to `decimals` digits after the decimal separator. + /// Infinities and NaN are written as Rust writes them. + pub fn number(&self, value: f64, decimals: usize) -> String { + match to_decimal(value, decimals) { + Some(d) => self.write(&d), + None => value.to_string(), + } + } + + /// `ratio` as a percentage with `decimals` digits after the decimal separator: `0.5` is + /// `50%`, `50 %` (French, German), `%50` (Turkish), `٥٠٪؜` (Egyptian Arabic). + pub fn percent(&self, ratio: f64, decimals: usize) -> String { + let Some(d) = to_decimal(ratio * 100.0, decimals) else { + return format!("{}%", ratio * 100.0); + }; + let p = self.parts(&d); + format!("{}{}{}{}", p.sign, self.percent.0, p.text, self.percent.1) + } + + /// `d` as text, the sign in place. + pub(crate) fn write(&self, d: &Decimal) -> String { + let p = self.parts(d); + p.sign + &p.text + } + + fn parts(&self, d: &Decimal) -> Written { + let mut sink = Sink { group: self.group.as_deref(), point: self.point.as_deref(), into: Target::Text, w: Written::default() }; + let written = match &self.decimal { + Some(f) => f.format(d).write_to_parts(&mut sink), + None => fmt::Write::write_str(&mut sink, &d.to_string()), + }; + if written.is_err() { + return Written { text: d.to_string(), ..Written::default() }; + } + sink.w + } +} + +/// A formatted number taken apart: its sign, the rest, and the separators and integer digits as +/// the locale wrote them. +#[derive(Default)] +struct Written { + sign: String, + text: String, + integer: String, + group: String, + point: String, +} + +#[derive(Clone, Copy, PartialEq)] +enum Target { + Text, + Sign, + /// A separator the region replaces; the locale's goes to `group` or `point` only. + Replaced, +} + +/// Writes the formatter's parts into [`Written`], swapping in the region's separators. +struct Sink<'a> { + group: Option<&'a str>, + point: Option<&'a str>, + into: Target, + w: Written, +} + +impl fmt::Write for Sink<'_> { + fn write_str(&mut self, s: &str) -> fmt::Result { + match self.into { + Target::Text => self.w.text.push_str(s), + Target::Sign => self.w.sign.push_str(s), + Target::Replaced => {} + } + Ok(()) + } +} + +impl PartsWrite for Sink<'_> { + type SubPartsWrite = Self; + + fn with_part(&mut self, part: Part, mut f: impl FnMut(&mut Self) -> fmt::Result) -> fmt::Result { + let outer = self.into; + let start = self.w.text.len(); + let replacement = match part { + p if p == parts::GROUP => self.group, + p if p == parts::DECIMAL => self.point, + _ => None, + }; + if let Some(r) = replacement { + self.w.text.push_str(r); + self.into = Target::Replaced; + } else if part == parts::MINUS_SIGN || part == parts::PLUS_SIGN { + self.into = Target::Sign; + } + let result = f(self); + self.into = outer; + if replacement.is_none() { + let written = self.w.text.get(start..).unwrap_or_default(); + let slot = match part { + p if p == parts::GROUP => &mut self.w.group, + p if p == parts::DECIMAL => &mut self.w.point, + p if p == parts::INTEGER => &mut self.w.integer, + _ => return result, + }; + if slot.is_empty() { + *slot = written.to_owned(); + } + } + result + } +} + +/// `value` rounded to `decimals` digits after the point; `None` for infinities and NaN. A value +/// that rounds to zero has no sign. +pub(crate) fn to_decimal(value: f64, decimals: usize) -> Option { + if !value.is_finite() { + return None; + } + let mut d = Decimal::try_from_str(&format!("{value:.decimals$}")).ok()?; + if d.absolute.is_zero() { + d.sign = Sign::None; + } + Some(d) +} + +/// The percent affixes for a number locale (`ar-EG`), found by dropping trailing subtags. +fn percent_affixes(tag: &str, latin: bool) -> (&'static str, &'static str) { + let mut tag = tag; + loop { + if let Some((_, _, before, after)) = PERCENT.iter().find(|(l, lat, _, _)| *lat == latin && l.eq_ignore_ascii_case(tag)) { + return (before, after); + } + match tag.rsplit_once('-') { + Some((parent, _)) => tag = parent, + None => return ("", "%"), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn latin(tag: &str) -> Format { + Format::new(&Region::from_tag(tag), Digits::Latin) + } + + #[test] + fn locale_digits_and_separators() { + for (tag, integer, number) in [ + ("en", "1,234,567", "1,234.5"), + ("ja", "1,234,567", "1,234.5"), + ("de", "1.234.567", "1.234,5"), + ("fr", "1\u{202f}234\u{202f}567", "1\u{202f}234,5"), + ("hi", "12,34,567", "1,234.5"), + ("ar-EG", "١٬٢٣٤٬٥٦٧", "١٬٢٣٤٫٥"), + ("fa", "۱٬۲۳۴٬۵۶۷", "۱٬۲۳۴٫۵"), + ] { + let f = Format::from_tag(tag); + assert_eq!((f.integer(1_234_567), f.number(1234.5, 1)), (integer.into(), number.into()), "{tag}"); + } + // CLDR writes Arabic without a region in Latin digits. + assert_eq!(Format::from_tag("ar").number(1234.5, 1), "1,234.5"); + } + + #[test] + fn forced_latin_digits() { + assert_eq!(latin("ar-EG").number(1234.5, 1), "1,234.5"); + assert_eq!(latin("fa").integer(1_234_567), "1,234,567"); + assert_eq!(latin("de").number(1234.5, 1), "1.234,5", "a Latin locale is unchanged"); + } + + #[test] + fn region_separators() { + // English in Poland writes a decimal comma; the grouping moves out of its way. + assert_eq!(Format::from_tag("en-PL").number(1234.5, 1), "1.234,5"); + let mut region = Region::from_tag("de-DE"); + region.grouping = Some('\u{a0}'); + assert_eq!(Format::new(®ion, Digits::Locale).number(1234.5, 1), "1\u{a0}234,5"); + region.decimal = '.'; + assert_eq!(Format::new(®ion, Digits::Locale).number(1234.5, 1), "1\u{a0}234.5"); + // A system's separators don't apply to Arabic-Indic digits. + let mut region = Region::from_tag("ar-EG"); + region.grouping = Some(','); + assert_eq!(Format::new(®ion, Digits::Locale).number(1234.5, 1), "١٬٢٣٤٫٥"); + } + + #[test] + fn percentages() { + for (tag, want) in + [("en", "50%"), ("de", "50\u{a0}%"), ("fr", "12,5\u{a0}%"), ("tr", "%50"), ("ar-EG", "٥٠٪\u{61c}"), ("fa", "۵۰٪"), ("hi", "50%")] + { + let decimals = if tag == "fr" { 1 } else { 0 }; + let ratio = if tag == "fr" { 0.125 } else { 0.5 }; + assert_eq!(Format::from_tag(tag).percent(ratio, decimals), want, "{tag}"); + } + assert_eq!(Format::from_tag("tr").percent(-0.2, 0), "-%20", "the sign leads the pattern"); + assert_eq!(latin("fa").percent(0.5, 0), "50%"); + } + + #[test] + fn signs_and_odd_values() { + let en = Format::from_tag("en"); + assert_eq!(en.number(-1234.5, 0), "-1,234"); + assert_eq!(en.number(-0.04, 1), "0.0", "no sign on zero"); + assert_eq!(en.number(f64::INFINITY, 1), "inf"); + assert_eq!(en.integer(-5_i64), "-5"); + } +} diff --git a/crates/i18n/src/lib.rs b/crates/i18n/src/lib.rs index 73e794f9..910041ed 100644 --- a/crates/i18n/src/lib.rs +++ b/crates/i18n/src/lib.rs @@ -6,17 +6,41 @@ #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] mod catalog; -mod size; +mod format; mod region; +mod size; + +/// The CLDR data for the shipped languages, from `data/generate.sh`. +mod data { + use std::sync::OnceLock; + + use icu_locale_fallback::LocaleFallbacker; + use icu_provider_adapters::fallback::LocaleFallbackProvider; + use icu_provider_blob::BlobDataProvider; + + include!("../data/locales.rs"); + + /// The number and plural data, read once; `None` if the blob doesn't load. + pub(crate) fn provider() -> Option<&'static LocaleFallbackProvider> { + static PROVIDER: OnceLock>> = OnceLock::new(); + PROVIDER + .get_or_init(|| { + let blob = BlobDataProvider::try_new_from_static_blob(include_bytes!("../data/numbers.postcard")).ok()?; + Some(LocaleFallbackProvider::new(blob, LocaleFallbacker::new().static_to_owned())) + }) + .as_ref() + } +} use std::collections::{BTreeMap, BTreeSet, HashMap}; -use std::sync::{Mutex, OnceLock, PoisonError}; +use std::sync::{Arc, Mutex, OnceLock, PoisonError, RwLock}; -use intl_pluralrules::{PluralCategory, PluralRuleType, PluralRules}; -use unic_langid::LanguageIdentifier; +use icu_locale_core::Locale as IcuLocale; +use icu_plurals::{PluralCategory, PluralOperands, PluralRules}; -pub use size::{FileSize, SizeUnit}; +pub use format::{Digits, Format}; pub use region::{HourCycle, Measurement, Paper, Region}; +pub use size::{FileSize, SizeUnit}; /// The catalogs an app embeds, one entry per locale: (locale, shared catalog, the app's catalog). /// A missing catalog is `""`. @@ -57,6 +81,8 @@ pub struct I18n { pseudo: Mutex>, /// [`I18n::system_lang`], read once. system: OnceLock, + /// [`I18n::format`]; the system region's until [`I18n::set_format`]. + format: RwLock>>, } impl I18n { @@ -107,7 +133,7 @@ impl I18n { l.chain = chain; } } - (I18n { locales, pseudo: Mutex::default(), system: OnceLock::new() }, problems) + (I18n { locales, pseudo: Mutex::default(), system: OnceLock::new(), format: RwLock::default() }, problems) } /// The language with this code (any case): `en`, an offered locale, or in debug builds @@ -163,9 +189,25 @@ impl I18n { self.find(lang, &[("@id", id), ("", label)]).unwrap_or(label) } - /// A count's wording in `lang`, `{n}` filled in. `one` and `other` are the English forms; - /// they are also the key of the `@plural` row, and the fallback. + /// A count's wording in `lang`, `{n}` filled in as [`I18n::format`] writes numbers. `one` and + /// `other` are the English forms; they are also the key of the `@plural` row, and the fallback. pub fn trn(&self, lang: Lang, n: u64, one: &str, other: &str) -> String { + let format = self.format(); + self.plural(lang, PluralOperands::from(n), n == 1, &format.integer(n), one, other) + } + + /// [`I18n::trn`] for a count with `decimals` digits after the decimal separator (`1.5 files`). + /// The plural form follows the digits shown, so `1.0` takes English `other`. + pub fn trn_decimal(&self, lang: Lang, n: f64, decimals: usize, one: &str, other: &str) -> String { + let format = self.format(); + let Some(d) = format::to_decimal(n, decimals) else { + return fmt(other, &[("n", &n.to_string())]); + }; + let english_one = d.to_string() == "1"; + self.plural(lang, PluralOperands::from(&d), english_one, &format.write(&d), one, other) + } + + fn plural(&self, lang: Lang, operands: PluralOperands, english_one: bool, n: &str, one: &str, other: &str) -> String { let key = format!("{one}|{other}"); let chain = self.locale(lang).map_or(&[][..], |l| l.chain.as_slice()); let text = chain @@ -174,19 +216,40 @@ impl I18n { let locale = self.locales.get(i)?; let forms = locale.rows.get("@plural")?.get(&key)?; let (rules, categories) = locale.plural.as_ref()?; - let category = rules.select(n).ok()?; - // A category whole numbers up to 1000 never reach (French `many`) takes the last form. - let index = categories.iter().position(|c| *c == category).unwrap_or(categories.len().saturating_sub(1)); + let category = rules.category_for(operands); + // A category whole numbers up to 1000 never reach has no form of its own. Russian + // and Ukrainian decimals (`other`) take the `few` form (`1,5 файла`); any other + // (French `many` for a million) takes the last. + let index = categories + .iter() + .position(|c| *c == category) + .or_else(|| categories.iter().position(|c| *c == PluralCategory::Few)) + .unwrap_or(categories.len().saturating_sub(1)); forms.split('|').nth(index) }) - .unwrap_or(if n == 1 { one } else { other }); - let text = fmt(text, &[("n", &n.to_string())]); + .unwrap_or(if english_one { one } else { other }); + let text = fmt(text, &[("n", n)]); if lang == Lang::PSEUDO { return marked(&text); } text } + /// How counts, file sizes and measures write numbers: [`Region::system`]'s format with the + /// locale's digits until [`I18n::set_format`]. + pub fn format(&self) -> Arc { + if let Some(f) = self.format.read().unwrap_or_else(PoisonError::into_inner).as_ref() { + return Arc::clone(f); + } + let mut slot = self.format.write().unwrap_or_else(PoisonError::into_inner); + Arc::clone(slot.get_or_insert_with(|| Arc::new(Format::new(&Region::system(), Digits::Locale)))) + } + + /// Write numbers as `format` does from now on: the user's region setting, or Latin digits. + pub fn set_format(&self, format: Format) { + *self.format.write().unwrap_or_else(PoisonError::into_inner) = Some(Arc::new(format)); + } + /// The languages to offer: English first, then the offered locales by code. Never the /// pseudo locale. pub fn languages(&self) -> Vec { @@ -371,15 +434,17 @@ fn bare(code: &str) -> Locale { /// `code`'s CLDR cardinal rules and the categories whole numbers 0..=1000 reach, in CLDR order /// (zero, one, two, few, many, other): one `@plural` form each. Regional and scripted codes fall /// back to the language's rules. CLDR's `pt` is Brazilian (0 is singular), so Portuguese other -/// than `pt-br` takes `pt-PT`'s. +/// than `pt-br` takes `pt-PT`'s. `None` for a language the baked data doesn't cover. fn plural_rules(code: &str) -> Option<(PluralRules, Vec)> { let code = if code == "pt" || (code.starts_with("pt-") && code != "pt-br") { "pt-pt" } else { code }; - let id: LanguageIdentifier = code.parse().ok()?; - let language = LanguageIdentifier::from_parts(id.language, None, None, &[]); - let rules = PluralRules::create(id, PluralRuleType::CARDINAL).or_else(|_| PluralRules::create(language, PluralRuleType::CARDINAL)).ok()?; + let locale = IcuLocale::try_from_str(code).ok()?; + if !data::LANGUAGES.contains(&locale.id.language.as_str()) { + return None; + } + let rules = PluralRules::try_new_cardinal_with_buffer_provider(data::provider()?, (&locale).into()).ok()?; let mut categories = Vec::new(); for n in 0..=1000_u32 { - let c = rules.select(n).ok()?; + let c = rules.category_for(n); if !categories.contains(&c) { categories.push(c); } @@ -390,12 +455,12 @@ fn plural_rules(code: &str) -> Option<(PluralRules, Vec)> { fn rank(c: &PluralCategory) -> u8 { match c { - PluralCategory::ZERO => 0, - PluralCategory::ONE => 1, - PluralCategory::TWO => 2, - PluralCategory::FEW => 3, - PluralCategory::MANY => 4, - PluralCategory::OTHER => 5, + PluralCategory::Zero => 0, + PluralCategory::One => 1, + PluralCategory::Two => 2, + PluralCategory::Few => 3, + PluralCategory::Many => 4, + PluralCategory::Other => 5, } } @@ -486,6 +551,7 @@ mod tests { ("fr", "@plural\t{n} page|{n} pages\t{n} page|{n} pages\n", ""), ("ru", "@plural\t{n} file|{n} files\t{n} файл|{n} файла|{n} файлов\n", ""), ("ja", "@plural\t{n} file|{n} files\t{n} 個\n", ""), + ("ar", "@plural\t{n} file|{n} files\t{n} ملفات|{n} ملف واحد|{n} ملفان|{n} ملفات|{n} ملفًا|{n} ملف\n", ""), ]; #[test] @@ -528,7 +594,39 @@ mod tests { #[test] fn beyond_sweep() { let i = all(P).0; - assert_eq!(i.trn(lang(&i, "fr"), 1_000_000, "{n} page", "{n} pages"), "1000000 pages"); + i.set_format(Format::from_tag("fr")); + assert_eq!(i.trn(lang(&i, "fr"), 1_000_000, "{n} page", "{n} pages"), "1\u{202f}000\u{202f}000 pages"); + } + + #[test] + fn counts_follow_the_region() { + let i = all(P).0; + i.set_format(Format::from_tag("ar-EG")); + assert_eq!(i.trn(lang(&i, "ar"), 1234, "{n} file", "{n} files"), "١٬٢٣٤ ملفًا"); + i.set_format(Format::new(&Region::from_tag("ar-EG"), Digits::Latin)); + assert_eq!(i.trn(lang(&i, "ar"), 1234, "{n} file", "{n} files"), "1,234 ملفًا"); + // An English interface with a German region. + i.set_format(Format::from_tag("de-DE")); + assert_eq!(i.trn(Lang::EN, 12_345, "{n} file", "{n} files"), "12.345 files"); + } + + #[test] + fn decimal_counts() { + let i = all(P).0; + i.set_format(Format::from_tag("en")); + assert_eq!(i.trn_decimal(Lang::EN, 1.5, 1, "{n} file", "{n} files"), "1.5 files"); + assert_eq!(i.trn_decimal(Lang::EN, 1.0, 1, "{n} file", "{n} files"), "1.0 files", "visible decimals are plural"); + assert_eq!(i.trn_decimal(Lang::EN, 1.0, 0, "{n} file", "{n} files"), "1 file"); + // Russian decimals are `other`, which integers never reach: the `few` form. + i.set_format(Format::from_tag("ru")); + let ru = lang(&i, "ru"); + assert_eq!(i.trn_decimal(ru, 1.5, 1, "{n} file", "{n} files"), "1,5 файла"); + assert_eq!(i.trn_decimal(ru, 21.0, 0, "{n} file", "{n} files"), "21 файл"); + // Arabic decimals are `other`. + i.set_format(Format::from_tag("ar-EG")); + let ar = lang(&i, "ar"); + assert_eq!(i.trn_decimal(ar, 2.5, 1, "{n} file", "{n} files"), "٢٫٥ ملف"); + assert_eq!(i.trn_decimal(ar, 3.0, 0, "{n} file", "{n} files"), "٣ ملفات"); } static L: &Embedded = diff --git a/crates/i18n/src/size.rs b/crates/i18n/src/size.rs index 1466672b..4dad2b57 100644 --- a/crates/i18n/src/size.rs +++ b/crates/i18n/src/size.rs @@ -103,8 +103,8 @@ impl FileSize { self } - /// The unit `bytes` is written in and the number in it, in English. - fn parts(&self, bytes: u64) -> (SizeUnit, String) { + /// The unit `bytes` is written in, the number in it and its digits after the point. + fn parts(&self, bytes: u64) -> (SizeUnit, f64, usize) { let (smallest, largest) = (self.smallest.min(self.largest), self.largest); let mut unit = smallest; for &u in SizeUnit::ALL.iter().filter(|u| (smallest..=largest).contains(u)) { @@ -128,7 +128,7 @@ impl FileSize { unit = next; continue; } - return (unit, number(rounded, usize::from(decimals))); + return (unit, rounded, usize::from(decimals)); } } @@ -153,19 +153,21 @@ impl FileSize { } impl I18n { - /// `bytes` as a file size in `lang`, written the way `style` says (`1.5 MB`, `512 bytes`). + /// `bytes` as a file size in `lang`, written the way `style` says (`1.5 MB`, `512 bytes`), the + /// number as [`I18n::format`] writes it (`1,5 Mo`). pub fn file_size(&self, lang: Lang, bytes: u64, style: FileSize) -> String { - let (unit, n) = style.parts(bytes); + let (unit, n, decimals) = style.parts(bytes); if unit == SizeUnit::Byte && !style.byte_symbol { return self.trn(lang, bytes, "{n} byte", "{n} bytes"); } - self.unit_text(lang, style.template(unit), &n) + self.unit_text(lang, style.template(unit), &self.format().number(n, decimals)) } - /// `value` with `decimals` digits after the decimal point in the unit template `unit`, an - /// English catalog key with the number as `{n}` (`{n} px`, `{n}%`, `{n} dB`). + /// `value` with `decimals` digits after the decimal separator in the unit template `unit`, an + /// English catalog key with the number as `{n}` (`{n} px`, `{n} dB`), the number as + /// [`I18n::format`] writes it. Percentages are [`crate::Format::percent`]. pub fn measure(&self, lang: Lang, value: f64, decimals: usize, unit: &str) -> String { - self.unit_text(lang, unit, &number(value, decimals)) + self.unit_text(lang, unit, &self.format().number(value, decimals)) } fn unit_text(&self, lang: Lang, template: &str, n: &str) -> String { @@ -173,17 +175,10 @@ impl I18n { } } -/// `value` with `decimals` digits after the decimal point, in ASCII digits with a decimal point. -/// The one place sizes and measures write numbers, for craft-i18n's locale number formatting -/// (digits, separators) to take over. -fn number(value: f64, decimals: usize) -> String { - format!("{value:.decimals$}") -} - #[cfg(test)] mod tests { use super::*; - use crate::Embedded; + use crate::{Embedded, Format}; static E: &Embedded = &[( "fr", @@ -193,12 +188,14 @@ mod tests { fn en(bytes: u64, style: FileSize) -> String { let (i18n, _) = I18n::new(&[], &[]); + i18n.set_format(Format::from_tag("en-US")); i18n.file_size(Lang::EN, bytes, style) } #[test] fn binary_decimal_and_iec_steps() { assert_eq!(en(512, FileSize::BINARY), "512 bytes"); + assert_eq!(en(1023, FileSize::BINARY), "1,023 bytes"); assert_eq!(en(1, FileSize::BINARY), "1 byte"); assert_eq!(en(1536, FileSize::BINARY), "1.5 KB"); assert_eq!(en(1536, FileSize::IEC), "1.5 KiB"); @@ -221,7 +218,7 @@ mod tests { assert_eq!(en(640_400, style), "640 KB"); assert_eq!(en(3_400_000, style), "3.4 MB"); assert_eq!(en(12_600_000, style), "13 MB"); - assert_eq!(en(5_000_000_000, style), "5000 MB"); + assert_eq!(en(5_000_000_000, style), "5,000 MB"); // Only sizes below the smallest unit round up. assert_eq!(en(1025, FileSize::BINARY.smallest(SizeUnit::Kilo).unit_decimals(SizeUnit::Kilo, 0)), "1 KB"); assert_eq!(en(100, FileSize::BINARY.smallest(SizeUnit::Kilo).unit_decimals(SizeUnit::Kilo, 0)), "1 KB"); @@ -232,11 +229,21 @@ mod tests { let (i18n, problems) = I18n::new(E, &["fr"]); assert!(problems.is_empty(), "{problems:?}"); let fr = i18n.from_code("fr").unwrap(); - assert_eq!(i18n.file_size(fr, 1536, FileSize::BINARY), "1.5 Ko"); + i18n.set_format(Format::from_tag("fr-FR")); + assert_eq!(i18n.file_size(fr, 1536, FileSize::BINARY), "1,5 Ko"); assert_eq!(i18n.file_size(fr, 512, FileSize::BINARY), "512 octets"); assert_eq!(i18n.file_size(fr, 512, FileSize::BINARY.byte_symbol()), "512 o", "not the plain row B (bold)"); - assert_eq!(i18n.file_size(fr, 3 << 30, FileSize::BINARY), "3.0 GB", "a missing row is English"); + assert_eq!(i18n.file_size(fr, 3 << 30, FileSize::BINARY), "3,0 GB", "a missing row is English"); assert_eq!(i18n.measure(fr, 50.0, 0, "{n}%"), "50 %"); - assert_eq!(i18n.measure(Lang::EN, 12.34, 1, "{n} px"), "12.3 px"); + assert_eq!(i18n.measure(Lang::EN, 12.34, 1, "{n} px"), "12,3 px", "numbers follow the region, not the language"); + } + + #[test] + fn locale_digits() { + let (i18n, _) = I18n::new(&[], &[]); + i18n.set_format(Format::from_tag("ar-EG")); + assert_eq!(i18n.file_size(Lang::EN, 1536, FileSize::BINARY), "١٫٥ KB"); + i18n.set_format(Format::from_tag("hi")); + assert_eq!(i18n.file_size(Lang::EN, 1_234_567, FileSize::DECIMAL.largest(SizeUnit::Byte).byte_symbol()), "12,34,567 B"); } } From 3e4cca97375628e1c5b007240b5b6fc29e4f13ca Mon Sep 17 00:00:00 2001 From: aiongg Date: Wed, 21 Oct 2026 05:44:34 +0000 Subject: [PATCH 4/7] i18n: Region separators only where the system sets them; C in LC_* is en-US Region::decimal and grouping are None unless Windows' or macOS's own settings set them, so the number locale's CLDR data decides otherwise. A cleared Windows sThousand is no grouping. LC_ALL=C and LC_NUMERIC=C (and the other LC_* variables) mean the C locale; only LANG=C gives way to the preferred language. --- crates/i18n/README.md | 6 +- crates/i18n/src/region.rs | 117 +++++++++++++++++--------------------- 2 files changed, 56 insertions(+), 67 deletions(-) diff --git a/crates/i18n/README.md b/crates/i18n/README.md index d916bbbb..7520f4ae 100644 --- a/crates/i18n/README.md +++ b/crates/i18n/README.md @@ -70,11 +70,11 @@ The app keeps its `tl!` macro, its language setting (`"auto"` → `I18N.system_l picks `sr`. - **Formatting region:** `Region` holds how numbers, times and measurements are written, apart from the interface language: the number and time locales (BCP 47), metric or US measurement, A4 - or Letter, the decimal separator, and the grouping separator and clock when the system sets - them. `Region::system()` reads it once: Windows' regional format with the user's changes + or Letter, and the decimal and grouping separators (or no grouping) and clock when the system + sets them; otherwise the number locale's CLDR data decides. `Region::system()` reads it once: Windows' regional format with the user's changes (`HKCU\Control Panel\International`), Apple's current locale (`en_PL`, `en_GB@rg=uszzzz`), `LC_ALL` or else `LC_NUMERIC`, `LC_TIME`, `LC_MEASUREMENT` and `LC_PAPER` each on its own over - `LANG` on Unix, the preferred language on the web and Android. `Region::from_setting` takes + `LANG` on Unix (`C` there is `en-US`; `LANG=C` gives way to the preferred language), the preferred language on the web and Android. `Region::from_setting` takes the app's saved choice (`auto` or a tag like `pl-PL`). Paper and measurement follow CLDR's territory lists; tags may carry `-u-ms-`, `-u-hc-` and `-u-rg-`. Ported from WordCraft's `Region`. diff --git a/crates/i18n/src/region.rs b/crates/i18n/src/region.rs index f9bb4720..bbebe4c9 100644 --- a/crates/i18n/src/region.rs +++ b/crates/i18n/src/region.rs @@ -2,8 +2,9 @@ //! interface language. An English interface in Poland still writes `2,54 cm` on A4. //! //! The territory lists come from Unicode CLDR's supplemental `measurementData` (Unicode License): -//! `paperSize` US-Letter and `measurementSystem` US. The decimal separators follow CLDR's number -//! symbols for the common locales; the system's own separator wins where the platform has one. +//! `paperSize` US-Letter and `measurementSystem` US. The separators numbers are written with +//! follow the number locale's CLDR data ([`crate::Format`]); the region carries them only where +//! the system sets its own. use std::sync::OnceLock; @@ -13,23 +14,6 @@ const LETTER: &[&str] = &["BZ", "CA", "CL", "CO", "CR", "GT", "MX", "NI", "PA", /// Territories that measure in inches (CLDR `measurementSystem type="US"`). const INCHES: &[&str] = &["LR", "MM", "US"]; -/// Languages whose numbers use a decimal comma in most of their territories. -const COMMA_LANGUAGES: &[&str] = &[ - "af", "az", "be", "bg", "bs", "ca", "cs", "da", "de", "el", "es", "et", "eu", "fi", "fo", "fr", "gl", "hr", "hu", "hy", "id", "is", "it", "ka", - "kk", "ky", "lb", "lt", "lv", "mk", "nb", "nl", "nn", "no", "pl", "pt", "ro", "ru", "sk", "sl", "sq", "sr", "sv", "tr", "uk", "uz", "vi", -]; - -/// Territories where those languages use a decimal point after all. -const POINT_EXCEPTIONS: &[(&str, &[&str])] = - &[("de", &["CH", "LI"]), ("it", &["CH"]), ("es", &["CU", "DO", "GT", "HN", "MX", "NI", "PA", "PE", "PR", "SV", "US", "419"])]; - -/// Territories where English is written with a decimal comma: most of continental Europe, South -/// Africa and much of South America. -const COMMA_TERRITORIES: &[&str] = &[ - "AR", "AT", "BE", "BG", "BO", "BR", "CL", "CO", "CZ", "DE", "DK", "EC", "EE", "ES", "FI", "FR", "GR", "HR", "HU", "ID", "IS", "IT", "LT", "LU", - "LV", "NL", "NO", "PL", "PT", "PY", "RO", "RS", "RU", "SE", "SI", "SK", "TR", "UA", "UY", "VE", "VN", "ZA", -]; - /// How lengths, distances and weights are measured. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub enum Measurement { @@ -66,16 +50,17 @@ pub struct Region { pub time: String, pub measurement: Measurement, pub paper: Paper, - /// The decimal separator (`,` in Poland). - pub decimal: char, - /// The grouping separator the system sets (Windows, macOS); `None` follows the locale. - pub grouping: Option, + /// The decimal separator the system sets (Windows, macOS); `None` follows the locale. + pub decimal: Option, + /// The grouping separator the system sets (Windows, macOS): `Some(None)` for none at all + /// (`1234567`); `None` follows the locale. + pub grouping: Option>, /// The clock the system sets (Windows, macOS); `None` follows the locale. pub hour_cycle: Option, } impl Default for Region { - /// `en-US`: decimal point, Letter, inches. + /// `en-US`: Letter, inches. fn default() -> Self { Region::from_parts(&Tag::default()) } @@ -114,21 +99,13 @@ impl Region { fn from_parts(tag: &Tag) -> Region { let territory = tag.territory.as_str(); - let language = tag.language.as_str(); let measurement = tag.measurement.unwrap_or(if INCHES.contains(&territory) { Measurement::Us } else { Measurement::Metric }); - let decimal = if COMMA_LANGUAGES.contains(&language) { - if POINT_EXCEPTIONS.iter().any(|(l, ts)| *l == language && ts.contains(&territory)) { '.' } else { ',' } - } else if language == "en" && COMMA_TERRITORIES.contains(&territory) { - ',' - } else { - '.' - }; Region { numbers: tag.bcp47(), time: tag.bcp47(), measurement, paper: if LETTER.contains(&territory) { Paper::Letter } else { Paper::A4 }, - decimal, + decimal: None, grouping: None, hour_cycle: tag.hour_cycle, } @@ -239,22 +216,23 @@ fn title_case(p: &str) -> String { chars.next().map(|c| c.to_ascii_uppercase().to_string() + chars.as_str()).unwrap_or_default() } -/// A locale variable that names a locale (not empty, `C` or `POSIX`). +/// Whether a locale variable names the C locale (`C`, `C.UTF-8`, `POSIX`). #[cfg(any(test, all(unix, not(any(target_vendor = "apple", target_os = "android")))))] -fn usable_locale(v: &str) -> bool { +fn is_c_locale(v: &str) -> bool { let v = v.trim(); - !v.is_empty() && !v.eq_ignore_ascii_case("c") && !v.eq_ignore_ascii_case("posix") && !v.starts_with("C.") + v.eq_ignore_ascii_case("c") || v.eq_ignore_ascii_case("posix") || v.starts_with("C.") } -/// The region from the POSIX locale variables `var` reads and, when neither `LC_ALL` nor the -/// variable for a convention nor `LANG` is set, the platform's preferred locale. +/// The region from the POSIX locale variables `var` reads. An empty variable is unset; `C` in +/// `LC_ALL` or an `LC_*` variable is the C locale, written as `en-US`. `LANG` set to `C` or unset +/// gives way to the platform's preferred locale. #[cfg(any(test, all(unix, not(any(target_vendor = "apple", target_os = "android")))))] fn from_posix(var: impl Fn(&str) -> Option, platform: impl FnOnce() -> Option) -> Region { - let var = |name: &str| var(name).filter(|v| usable_locale(v)); + let var = |name: &str| var(name).filter(|v| !v.trim().is_empty()); if let Some(all) = var("LC_ALL") { return Region::from_tag(&all); } - let lang = var("LANG").or_else(platform).unwrap_or_default(); + let lang = var("LANG").filter(|v| !is_c_locale(v)).or_else(platform).unwrap_or_default(); let tag = |name: &str| var(name).unwrap_or_else(|| lang.clone()); let time = Region::from_tag(&tag("LC_TIME")); Region { @@ -272,12 +250,8 @@ fn from_posix(var: impl Fn(&str) -> Option, platform: impl FnOnce() -> O #[cfg(any(test, windows))] fn from_windows(value: impl Fn(&str) -> Option, platform: impl FnOnce() -> Option) -> Region { let mut region = Region::from_tag(&value("LocaleName").or_else(platform).unwrap_or_default()); - if let Some(c) = value("sDecimal").as_deref().and_then(single_char) { - region.decimal = c; - } - if let Some(s) = value("sThousand") { - region.grouping = single_char(&s); - } + region.decimal = value("sDecimal").as_deref().and_then(single_char); + region.grouping = value("sThousand").as_deref().and_then(grouping); match value("iMeasure").as_deref().map(str::trim) { Some("0") => region.measurement = Measurement::Metric, Some("1") => region.measurement = Measurement::Us, @@ -299,14 +273,19 @@ fn from_windows(value: impl Fn(&str) -> Option, platform: impl FnOnce() fn from_apple(identifier: &str, metric: bool, decimal: &str, grouping: &str, hours: Option<&str>) -> Region { let mut region = Region::from_tag(identifier); region.measurement = if metric { Measurement::Metric } else { Measurement::Us }; - if let Some(c) = single_char(decimal) { - region.decimal = c; - } - region.grouping = single_char(grouping); + region.decimal = single_char(decimal); + region.grouping = self::grouping(grouping); region.hour_cycle = hours.and_then(hour_cycle_of).or(region.hour_cycle); region } +/// A grouping separator the system sets: none at all when empty (Windows' `sThousand` the user +/// cleared), else one character. +#[cfg(any(test, windows, target_vendor = "apple"))] +fn grouping(s: &str) -> Option> { + if s.is_empty() { Some(None) } else { single_char(s).map(Some) } +} + /// A separator that is one character; Windows allows up to three, which the region can't carry. #[cfg(any(test, windows, target_vendor = "apple"))] fn single_char(s: &str) -> Option { @@ -369,8 +348,8 @@ fn read_system() -> Region { mod tests { use super::*; - fn conventions(r: &Region) -> (Paper, Measurement, char) { - (r.paper, r.measurement, r.decimal) + fn conventions(r: &Region) -> (Paper, Measurement) { + (r.paper, r.measurement) } #[test] @@ -398,15 +377,15 @@ mod tests { assert_eq!(Region::from_tag("en_US@rg=plzzzz;ms=metric").numbers, "en-PL"); } - /// Paper and measurement follow the territory, numbers the language and territory. + /// Paper and measurement follow the territory; separators are left to the number locale. #[test] fn conventions_follow_the_territory() { - assert_eq!(conventions(&Region::from_tag("pl-PL")), (Paper::A4, Measurement::Metric, ',')); - assert_eq!(conventions(&Region::from_tag("en-US")), (Paper::Letter, Measurement::Us, '.')); - assert_eq!(conventions(&Region::from_tag("fr-CA")), (Paper::Letter, Measurement::Metric, ',')); - assert_eq!(conventions(&Region::from_tag("en-DE")), (Paper::A4, Measurement::Metric, ',')); - assert_eq!(conventions(&Region::from_tag("en-GB")), (Paper::A4, Measurement::Metric, '.')); - assert_eq!(Region::from_tag("de-CH").decimal, '.'); + assert_eq!(conventions(&Region::from_tag("pl-PL")), (Paper::A4, Measurement::Metric)); + assert_eq!(conventions(&Region::from_tag("en-US")), (Paper::Letter, Measurement::Us)); + assert_eq!(conventions(&Region::from_tag("fr-CA")), (Paper::Letter, Measurement::Metric)); + assert_eq!(conventions(&Region::from_tag("en-GB")), (Paper::A4, Measurement::Metric)); + let r = Region::from_tag("de-CH"); + assert_eq!((r.decimal, r.grouping), (None, None)); assert_eq!(Region::from_tag("es-MX").paper, Paper::Letter); assert_eq!(Region::from_tag("ja-JP").paper, Paper::A4); } @@ -423,12 +402,17 @@ mod tests { let env = |pairs: &'static [(&'static str, &'static str)]| move |n: &str| pairs.iter().find(|(k, _)| *k == n).map(|(_, v)| v.to_string()); let none = || None; let r = from_posix(env(&[("LANG", "en_US.UTF-8"), ("LC_NUMERIC", "pl_PL.UTF-8"), ("LC_TIME", "en_GB.UTF-8"), ("LC_PAPER", "de_DE")]), none); - assert_eq!((r.numbers.as_str(), r.time.as_str(), r.decimal), ("pl-PL", "en-GB", ',')); + assert_eq!((r.numbers.as_str(), r.time.as_str(), r.decimal), ("pl-PL", "en-GB", None)); assert_eq!((r.paper, r.measurement), (Paper::A4, Measurement::Us), "paper from LC_PAPER, measurement from LANG"); assert_eq!(from_posix(env(&[("LANG", "it_IT.UTF-8"), ("LC_ALL", "en_US"), ("LC_PAPER", "de_DE")]), none).paper, Paper::Letter); assert_eq!(from_posix(env(&[("LANG", "en_US"), ("LC_MEASUREMENT", "it_IT")]), none).measurement, Measurement::Metric); assert_eq!(from_posix(env(&[("LANG", "C.UTF-8"), ("LC_ALL", "")]), || Some("it-IT".into())).numbers, "it-IT"); assert_eq!(from_posix(env(&[]), none), Region::default()); + // `C` in an LC_* variable is the C locale's conventions, not an unset variable. + let r = from_posix(env(&[("LANG", "de_DE.UTF-8"), ("LC_NUMERIC", "C"), ("LC_TIME", "C.UTF-8")]), none); + assert_eq!((r.numbers.as_str(), r.time.as_str(), r.paper), ("en-US", "en-US", Paper::A4)); + assert_eq!(from_posix(env(&[("LANG", "de_DE.UTF-8"), ("LC_ALL", "C")]), none), Region::default()); + assert_eq!(from_posix(env(&[("LANG", "C"), ("LC_NUMERIC", "")]), || Some("it-IT".into())).numbers, "it-IT"); } #[test] @@ -438,15 +422,20 @@ mod tests { reg(&[("LocaleName", "pl-PL"), ("sDecimal", ","), ("sThousand", "\u{a0}"), ("iMeasure", "0"), ("sShortTime", "HH:mm")]), || None, ); - assert_eq!((r.numbers.as_str(), r.decimal, r.grouping, r.hour_cycle), ("pl-PL", ',', Some('\u{a0}'), Some(HourCycle::H23))); + assert_eq!((r.numbers.as_str(), r.decimal, r.grouping, r.hour_cycle), ("pl-PL", Some(','), Some(Some('\u{a0}')), Some(HourCycle::H23))); // An English format in the US with the user's metric and A4 choices and a decimal comma. let r = from_windows( reg(&[("LocaleName", "en-US"), ("sDecimal", ","), ("iMeasure", "0"), ("iPaperSize", "9"), ("sShortTime", "h:mm tt")]), || None, ); - assert_eq!((r.decimal, r.measurement, r.paper, r.hour_cycle), (',', Measurement::Metric, Paper::A4, Some(HourCycle::H12))); + assert_eq!( + (r.decimal, r.grouping, r.measurement, r.paper, r.hour_cycle), + (Some(','), None, Measurement::Metric, Paper::A4, Some(HourCycle::H12)) + ); let r = from_windows(reg(&[("sDecimal", "dec"), ("sThousand", "")]), || Some("de-DE".into())); - assert_eq!((r.numbers.as_str(), r.decimal, r.grouping), ("de-DE", ',', None), "a separator longer than one character keeps the locale's"); + assert_eq!((r.numbers.as_str(), r.decimal), ("de-DE", None), "a separator longer than one character keeps the locale's"); + assert_eq!(r.grouping, Some(None), "an empty grouping separator is no grouping"); + assert_eq!(from_windows(reg(&[("sThousand", "abc")]), || None).grouping, None); } #[test] @@ -454,7 +443,7 @@ mod tests { let r = from_apple("en_PL", true, ",", "\u{a0}", Some("HH")); assert_eq!( (r.numbers.as_str(), r.paper, r.decimal, r.grouping, r.hour_cycle), - ("en-PL", Paper::A4, ',', Some('\u{a0}'), Some(HourCycle::H23)) + ("en-PL", Paper::A4, Some(','), Some(Some('\u{a0}')), Some(HourCycle::H23)) ); let r = from_apple("en_US", false, ".", ",", Some("h a")); assert_eq!((r.measurement, r.hour_cycle), (Measurement::Us, Some(HourCycle::H12))); From c1f64180c99b50e397f30fa38fba9b5e45bb9ff3 Mon Sep 17 00:00:00 2001 From: aiongg Date: Wed, 21 Oct 2026 05:44:52 +0000 Subject: [PATCH 5/7] i18n: check whole_from after rounding 9,960,000 bytes in DECIMAL with whole_from(10) rounds to 10.0 MB, which is written 10 MB. --- crates/i18n/src/size.rs | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/crates/i18n/src/size.rs b/crates/i18n/src/size.rs index 1466672b..be724bde 100644 --- a/crates/i18n/src/size.rs +++ b/crates/i18n/src/size.rs @@ -114,12 +114,17 @@ impl FileSize { } loop { let value = bytes as f64 / self.scale(unit); - let mut decimals = self.decimals.get(unit as usize).copied().unwrap_or(0); - if unit == SizeUnit::Byte || (self.whole_from > 0 && value >= f64::from(self.whole_from)) { + let round = |decimals: u8| { + let factor = 10f64.powi(i32::from(decimals)); + (if unit == smallest && bytes > 0 && value < 1.0 { (value * factor).ceil() } else { (value * factor).round() }) / factor + }; + let mut decimals = if unit == SizeUnit::Byte { 0 } else { self.decimals.get(unit as usize).copied().unwrap_or(0) }; + let mut rounded = round(decimals); + // After rounding, so 9.96 MB, which rounds to 10.0, is written 10 MB. + if self.whole_from > 0 && rounded >= f64::from(self.whole_from) { decimals = 0; + rounded = round(0); } - let factor = 10f64.powi(i32::from(decimals)); - let rounded = if unit == smallest && bytes > 0 && value < 1.0 { (value * factor).ceil() } else { (value * factor).round() } / factor; // 1023.96 KB rounds to 1024.0 KB: the next unit writes it as 1.0 MB. if rounded >= self.step as f64 && let Some(&next) = SizeUnit::ALL.get(unit as usize + 1) @@ -221,6 +226,8 @@ mod tests { assert_eq!(en(640_400, style), "640 KB"); assert_eq!(en(3_400_000, style), "3.4 MB"); assert_eq!(en(12_600_000, style), "13 MB"); + assert_eq!(en(9_960_000, style), "10 MB", "whole once rounded to 10"); + assert_eq!(en(9_940_000, style), "9.9 MB"); assert_eq!(en(5_000_000_000, style), "5000 MB"); // Only sizes below the smallest unit round up. assert_eq!(en(1025, FileSize::BINARY.smallest(SizeUnit::Kilo).unit_decimals(SizeUnit::Kilo, 0)), "1 KB"); From 233e0b7aa0f2d3b1718241df143246dab641e849 Mon Sep 17 00:00:00 2001 From: aiongg Date: Wed, 21 Oct 2026 05:49:38 +0000 Subject: [PATCH 6/7] i18n: plural rules by language, CLDR data for every locale, trn_plain Plural rules are looked up by language alone: CLDR's parent of sr-Latn and zh-Hant is the root, whose only category is other, so sr-latn rows were rejected. The baked data covers every CLDR locale (10 KB): plural rules for he, ro, nb, ca, sk, th, ms and the rest, and decimal symbols for nb-NO, sk-SK, bn-BD, ar-MA and English in Europe. Format applies a Region's separators only where the system set them, and an empty grouping separator writes none. trn_plain writes counts in 0-9 without grouping, for documents and file names. reset_format goes back to the system region. trn_decimal and Format write infinities and NaN as CLDR does and take the other form. Tests that don't set a format use en-US, not the machine's region. The ICU4X crates are pinned to ~2.3, the minor version the data is for. --- crates/i18n/Cargo.toml | 17 ++-- crates/i18n/README.md | 42 ++++---- crates/i18n/data/generate.sh | 15 ++- crates/i18n/data/{percent.py => locales.py} | 18 ++-- crates/i18n/data/locales.rs | 69 ++++++++++++- crates/i18n/data/numbers.postcard | Bin 5664 -> 9553 bytes crates/i18n/src/format.rs | 68 +++++++++---- crates/i18n/src/lib.rs | 103 +++++++++++++++----- 8 files changed, 248 insertions(+), 84 deletions(-) rename crates/i18n/data/{percent.py => locales.py} (75%) diff --git a/crates/i18n/Cargo.toml b/crates/i18n/Cargo.toml index 3daaca10..88e52d02 100644 --- a/crates/i18n/Cargo.toml +++ b/crates/i18n/Cargo.toml @@ -8,15 +8,16 @@ rust-version.workspace = true repository.workspace = true [dependencies] -# Numbers and plural rules, on CLDR data for the shipped locales (data/numbers.postcard). +# Numbers and plural rules, on CLDR data (data/numbers.postcard) generated for one ICU4X minor +# version, hence `~2.3`. fixed_decimal = { version = "0.7.2", default-features = false } -icu_decimal = { version = "2.3", default-features = false, features = ["serde"] } -icu_locale_core = { version = "2.3", default-features = false } -icu_locale_fallback = { version = "2.3", default-features = false, features = ["compiled_data"] } -icu_plurals = { version = "2.3", default-features = false, features = ["serde"] } -icu_provider = { version = "2.3", default-features = false, features = ["sync"] } -icu_provider_adapters = { version = "2.3", default-features = false } -icu_provider_blob = { version = "2.3", default-features = false } +icu_decimal = { version = "~2.3", default-features = false, features = ["serde"] } +icu_locale_core = { version = "~2.3", default-features = false } +icu_locale_fallback = { version = "~2.3", default-features = false, features = ["compiled_data"] } +icu_plurals = { version = "~2.3", default-features = false, features = ["serde"] } +icu_provider = { version = "~2.3", default-features = false, features = ["sync"] } +icu_provider_adapters = { version = "~2.3", default-features = false } +icu_provider_blob = { version = "~2.3", default-features = false } writeable = "0.6" sys-locale = "0.3" diff --git a/crates/i18n/README.md b/crates/i18n/README.md index d9f97cb7..63eb7184 100644 --- a/crates/i18n/README.md +++ b/crates/i18n/README.md @@ -46,8 +46,7 @@ The app keeps its `tl!` macro, its language setting (`"auto"` → `I18N.system_l translation never errors. - **Lookups:** `t` (plain), `tc` (with a context such as `menu` or `@msg`, else plain), `tid` (an `@id` command id, else the English label), `trn` and `trn_decimal` (a count, `@plural` rows, - `{n}` filled), `get` - (one context along the locale chain, so `pt-br` can return a `pt` row, but never the plain row + `{n}` filled), `get` (one context along the locale chain, so `pt-br` can return a `pt` row, but never the plain row or English: a key cap's name under `key`, or "is there a translation?"), and `fmt` for `{name}` placeholders, filled in one pass. - **Bad rows** (malformed, an unknown `@` context, a wrong plural form count, a row defined twice, @@ -59,8 +58,9 @@ The app keeps its `tl!` macro, its language setting (`"auto"` → `I18N.system_l craft-i18n's checker expects. `trn_decimal` picks the form from the digits shown (ICU4X plural operands), so English `1.0` is plural and French `1,5` singular. A count in a category integers never reach takes the `few` form if there is one (Russian and Ukrainian decimals, `1,5 файла`), - else the last. CLDR's `pt` is Brazilian, so `pt` uses `pt-PT`'s rules (0 is plural) and `pt-br` - `pt`'s. A language outside the baked data (below) has no plural rules and shows English forms. + else the last. Rules are per language, so `sr-latn` and `zh-hant` take `sr`'s and `zh`'s. CLDR's + `pt` is Brazilian, so `pt` uses `pt-PT`'s rules (0 is plural) and `pt-br` `pt`'s. A language + CLDR has no plural rules for shows English forms. - **Names:** a language's name comes from its shared catalog's `@id language.name` row, else its code; English is "English". - **Right to left:** `rtl` is true for `ar`, `ckb`, `dv`, `fa`, `he`, `ps`, `sd`, `ug`, `ur` and @@ -83,24 +83,28 @@ The app keeps its `tl!` macro, its language setting (`"auto"` → `I18N.system_l (`1.234.567`, Indian `12,34,567`), `number(value, decimals)` and `percent(ratio, decimals)` (`50 %`, Turkish `%50`). It follows the region, not the interface language, with the locale's digits (`١٬٢٣٤٫٥` in Egypt, `۱٬۲۳۴` in Iran) or, with `Digits::Latin`, `0`–`9`. With Latin - digits, a decimal or grouping separator the region sets replaces the locale's (English in Poland: - `1.234,5`). `I18n::format` is the format `trn`, `file_size` and `measure` use: the system - region's until `I18n::set_format` (the user's region setting or digit choice). -- **CLDR data:** decimal symbols, digits and plural rules (`data/numbers.postcard`, about 6 KB) and - percent patterns (`data/locales.rs`) cover the languages the catalogs and apps ship, with their - regional variants; `data/generate.sh` lists them and rebuilds both from CLDR. Add a language - there when a catalog adds one. The ICU4X code and data add about 90 KB to a stripped release - build and 80 KB to wasm. + digits, a decimal or grouping separator the system sets (Windows' and macOS's own settings) + replaces the locale's. `I18n::format` is the format `trn`, `file_size` and `measure` use: the + system region's until `I18n::set_format` (the user's region setting or digit choice), and again + after `I18n::reset_format`. A count in `trn` is therefore grouped and in the region's digits + (`1.234 Dateien`, `١٬٢٣٤ ملفًا`); for names written into documents or files (`Copy 1234`), use + `trn_plain`, which writes `0`–`9` without grouping. +- **CLDR data:** decimal symbols, digits and plural rules (`data/numbers.postcard`, about 10 KB) and + percent patterns (`data/locales.rs`) cover every CLDR locale; `data/generate.sh` rebuilds both + from CLDR. The ICU4X crates are pinned to one minor version (`~2.3`), the one the data is + generated for. The ICU4X code and data add about 90 KB to a stripped release build and 80 KB to + wasm. - **Formatting region:** `Region` holds how numbers, times and measurements are written, apart from the interface language: the number and time locales (BCP 47), metric or US measurement, A4 or Letter, and the decimal and grouping separators (or no grouping) and clock when the system - sets them; otherwise the number locale's CLDR data decides. `Region::system()` reads it once: Windows' regional format with the user's changes - (`HKCU\Control Panel\International`), Apple's current locale (`en_PL`, `en_GB@rg=uszzzz`), - `LC_ALL` or else `LC_NUMERIC`, `LC_TIME`, `LC_MEASUREMENT` and `LC_PAPER` each on its own over - `LANG` on Unix (`C` there is `en-US`; `LANG=C` gives way to the preferred language), the preferred language on the web and Android. `Region::from_setting` takes - the app's saved choice (`auto` or a tag like `pl-PL`). Paper and measurement follow CLDR's - territory lists; tags may carry `-u-ms-`, `-u-hc-` and `-u-rg-`. Ported from WordCraft's - `Region`. + sets them; otherwise the number locale's CLDR data decides. `Region::system()` reads it once: + Windows' regional format with the user's changes (`HKCU\Control Panel\International`), Apple's + current locale (`en_PL`, `en_GB@rg=uszzzz`), `LC_ALL` or else `LC_NUMERIC`, `LC_TIME`, + `LC_MEASUREMENT` and `LC_PAPER` each on its own over `LANG` on Unix (`C` there is `en-US`; + `LANG=C` gives way to the preferred language), the preferred language on the web and Android. + `Region::from_setting` takes the app's saved choice (`auto` or a tag like `pl-PL`). Paper and + measurement follow CLDR's territory lists; tags may carry `-u-ms-`, `-u-hc-` and `-u-rg-`. + Ported from WordCraft's `Region`. - **Pseudo locale** (`pseudo`, debug builds only, never in `languages()`): every lookup, `get` included, comes back as `⟦…⟧`, line by line, so text drawn without the marks never went through translation. `looked_up()` lists the source texts looked up so far, for an audit against a diff --git a/crates/i18n/data/generate.sh b/crates/i18n/data/generate.sh index 695a698a..91b6ab94 100755 --- a/crates/i18n/data/generate.sh +++ b/crates/i18n/data/generate.sh @@ -1,23 +1,22 @@ #!/usr/bin/env bash -# Regenerates the number and plural data in this directory from Unicode CLDR, for the languages -# the craft-i18n catalogs and the apps ship (each with its regional variants). +# Regenerates the number and plural data in this directory from Unicode CLDR, for every CLDR +# locale: 10 KB, of which CLDR's modern-coverage locales alone would be 7 KB. Lookups fall back +# at run time, so a locale whose data matches its parent's is left out. # -# Needs `cargo install icu4x-datagen --version 2.3.0` (the ICU4X version in Cargo.toml) and -# python3. Downloads CLDR's JSON release once into $CACHE. +# Needs `cargo install icu4x-datagen --version 2.3.0` (the ICU4X minor version in Cargo.toml) +# and python3. Downloads CLDR's JSON release once into $CACHE. set -euo pipefail CLDR=48.2.1 -LANGUAGES=(en ar bg cs da de el es et fa fi fr hi hr hu id it ja ka kk ko lt nl pl pt ru sl sr sv te tr uk vi zh) HERE=$(cd "$(dirname "$0")" && pwd) CACHE=${CACHE:-${XDG_CACHE_HOME:-$HOME/.cache}/craft-i18n-data} mkdir -p "$CACHE" -# `sr-Latn` and `zh-Hant` are named as well: their CLDR parent is the root, not `sr` or `zh`. TMPDIR=$CACHE icu4x-datagen --format blob --overwrite \ --cldr-tag "$CLDR" \ --markers DecimalSymbolsV1 DecimalDigitsV1 PluralsCardinalV1 \ - --locales "${LANGUAGES[@]}" sr-Latn zh-Hant \ + --locales full --deduplication maximal \ --out "$HERE/numbers.postcard" ZIP=$CACHE/icu4x-source-cache/github.com/unicode-org/cldr-json/releases/download/$CLDR/cldr-$CLDR-json-full.zip -python3 -I "$HERE/percent.py" "$ZIP" "$CLDR" "${LANGUAGES[@]}" > "$HERE/locales.rs" +python3 -I "$HERE/locales.py" "$ZIP" "$CLDR" > "$HERE/locales.rs" diff --git a/crates/i18n/data/percent.py b/crates/i18n/data/locales.py similarity index 75% rename from crates/i18n/data/percent.py rename to crates/i18n/data/locales.py index 3c00bb44..337bcfb9 100644 --- a/crates/i18n/data/percent.py +++ b/crates/i18n/data/locales.py @@ -1,6 +1,7 @@ -"""Writes locales.rs: the covered languages and the percent affixes from CLDR's JSON release. +"""Writes locales.rs from CLDR's JSON release: the languages with plural rules and every locale's +percent affixes. -Usage: percent.py ... +Usage: locales.py """ import json @@ -21,12 +22,14 @@ def affixes(numbers, numsys): def main(): - path, version, languages = sys.argv[1], sys.argv[2], sys.argv[3:] + path, version = sys.argv[1], sys.argv[2] found = {} with zipfile.ZipFile(path) as z: + plurals = json.loads(z.read("cldr-core/supplemental/plurals.json"))["supplemental"]["plurals-type-cardinal"] + languages = sorted(l for l in plurals if l != "root" and "-" not in l) for name in z.namelist(): m = re.fullmatch(r"cldr-numbers-full/main/([^/]+)/numbers\.json", name) - if not m or m.group(1).split("-")[0] not in languages: + if not m or m.group(1) == "root": continue locale = m.group(1) numbers = json.loads(z.read(name))["main"][locale]["numbers"] @@ -49,8 +52,11 @@ def main(): print(f"// @generated by generate.sh from Unicode CLDR {version} (Unicode License).") print() - print("/// The languages the baked number and plural data cover, each with its regional variants.") - print("pub(crate) const LANGUAGES: &[&str] = &[" + ", ".join(rust(l) for l in languages) + "];") + print("/// The languages CLDR has plural rules for; the baked data has them all.") + print("pub(crate) const PLURAL_LANGUAGES: &[&str] = &[") + for i in range(0, len(languages), 16): + print(" " + ", ".join(rust(l) for l in languages[i : i + 16]) + ",") + print("];") print() print("/// Percent signs around the number (CLDR's standard percent pattern and sign): locale, Latin") print("/// digits or the locale's own, text before the number, text after it. A locale missing here") diff --git a/crates/i18n/data/locales.rs b/crates/i18n/data/locales.rs index a2ad6e72..205a9661 100644 --- a/crates/i18n/data/locales.rs +++ b/crates/i18n/data/locales.rs @@ -1,7 +1,22 @@ // @generated by generate.sh from Unicode CLDR 48.2.1 (Unicode License). -/// The languages the baked number and plural data cover, each with its regional variants. -pub(crate) const LANGUAGES: &[&str] = &["en", "ar", "bg", "cs", "da", "de", "el", "es", "et", "fa", "fi", "fr", "hi", "hr", "hu", "id", "it", "ja", "ka", "kk", "ko", "lt", "nl", "pl", "pt", "ru", "sl", "sr", "sv", "te", "tr", "uk", "vi", "zh"]; +/// The languages CLDR has plural rules for; the baked data has them all. +pub(crate) const PLURAL_LANGUAGES: &[&str] = &[ + "af", "ak", "am", "an", "ar", "ars", "as", "asa", "ast", "az", "bal", "be", "bem", "bez", "bg", "bho", + "blo", "bm", "bn", "bo", "br", "brx", "bs", "ca", "ce", "ceb", "cgg", "chr", "ckb", "cs", "csw", "cv", + "cy", "da", "de", "doi", "dsb", "dv", "dz", "ee", "el", "en", "eo", "es", "et", "eu", "fa", "ff", + "fi", "fil", "fo", "fr", "fur", "fy", "ga", "gd", "gl", "gsw", "gu", "guw", "gv", "ha", "haw", "he", + "hi", "hnj", "hr", "hsb", "hu", "hy", "ia", "id", "ie", "ig", "ii", "io", "is", "it", "iu", "ja", + "jbo", "jgo", "jmc", "jv", "jw", "ka", "kab", "kaj", "kcg", "kde", "kea", "kk", "kkj", "kl", "km", "kn", + "ko", "kok", "ks", "ksb", "ksh", "ku", "kw", "ky", "lag", "lb", "lg", "lij", "lkt", "lld", "ln", "lo", + "lt", "lv", "mas", "mg", "mgo", "mk", "ml", "mn", "mo", "mr", "ms", "mt", "my", "nah", "naq", "nb", + "nd", "ne", "nl", "nn", "nnh", "no", "nqo", "nr", "nso", "ny", "nyn", "om", "or", "os", "osa", "pa", + "pap", "pcm", "pl", "prg", "ps", "pt", "rm", "ro", "rof", "ru", "rwk", "sah", "saq", "sat", "sc", "scn", + "sd", "sdh", "se", "seh", "ses", "sg", "sgs", "sh", "shi", "si", "sk", "sl", "sma", "smi", "smj", "smn", + "sms", "sn", "so", "sq", "sr", "ss", "ssy", "st", "su", "sv", "sw", "syr", "ta", "te", "teo", "th", + "ti", "tig", "tk", "tl", "tn", "to", "tpi", "tr", "ts", "tzm", "ug", "uk", "und", "ur", "uz", "ve", + "vec", "vi", "vo", "vun", "wa", "wae", "wo", "xh", "xog", "yi", "yo", "yue", "zh", "zu", +]; /// Percent signs around the number (CLDR's standard percent pattern and sign): locale, Latin /// digits or the locale's own, text before the number, text after it. A locale missing here @@ -31,11 +46,27 @@ pub(crate) const PERCENT: &[(&str, bool, &str, &str)] = &[ ("ar-SY", false, "", "\u{66a}\u{61c}"), ("ar-TD", false, "", "\u{66a}\u{61c}"), ("ar-YE", false, "", "\u{66a}\u{61c}"), + ("az-Arab", false, "", "\u{66a}"), + ("bas", true, "", "\u{a0}%"), + ("be", true, "", "\u{a0}%"), + ("bgn", false, "", "\u{66a}"), + ("bgn", true, "", "\u{66a}"), + ("blo", true, "%\u{a0}", ""), + ("br", true, "", "\u{a0}%"), + ("bua", true, "", "\u{a0}%"), + ("ca", true, "", "\u{a0}%"), + ("ce", true, "", "\u{a0}%"), + ("ckb", false, "", "\u{a0}\u{66a}"), ("cs", true, "", "\u{a0}%"), + ("cv", true, "", "\u{a0}%"), ("da", true, "", "\u{a0}%"), ("de", true, "", "\u{a0}%"), ("de-CH", true, "", "%"), ("de-LI", true, "", "%"), + ("dsb", true, "", "\u{a0}%"), + ("dua", true, "", "\u{a0}%"), + ("dz", false, "", "\u{a0}%"), + ("dz", true, "", "\u{a0}%"), ("en-AT", true, "", "\u{a0}%"), ("en-DE", true, "", "\u{a0}%"), ("en-DK", true, "", "\u{a0}%"), @@ -66,14 +97,48 @@ pub(crate) const PERCENT: &[(&str, bool, &str, &str)] = &[ ("es-US", true, "", "%"), ("es-UY", true, "", "%"), ("es-VE", true, "", "%"), + ("eu", true, "%\u{a0}", ""), ("fa", false, "", "\u{66a}"), ("fi", true, "", "\u{a0}%"), + ("fo", true, "", "\u{a0}%"), ("fr", true, "", "\u{a0}%"), ("fr-CH", true, "", "%"), + ("gl", true, "", "\u{a0}%"), + ("gsw", true, "", "\u{a0}%"), ("hr", true, "", "\u{a0}%"), + ("hsb", true, "", "\u{a0}%"), + ("ht", true, "", "\u{a0}%"), + ("kl", true, "", "\u{a0}%"), + ("ks", false, "", "\u{66a}"), + ("ksh", true, "", "\u{a0}%"), + ("ku", true, "%", ""), + ("ku-Arab", true, "", "%"), + ("lb", true, "", "\u{a0}%"), + ("lrc", false, "", "\u{66a}"), ("lt", true, "", "\u{a0}%"), + ("mk", true, "", "\u{a0}%"), + ("mzn", false, "", "\u{66a}"), + ("nb", true, "", "\u{a0}%"), + ("nds", true, "", "\u{a0}%"), + ("nn", true, "", "\u{a0}%"), + ("no", true, "", "\u{a0}%"), + ("oc", true, "", "\u{202f}%"), + ("pa-Arab", false, "", "\u{66a}"), + ("ps", false, "", "\u{66a}"), + ("qu", true, "", "\u{a0}%"), + ("rn", true, "", "\u{a0}%"), + ("ro", true, "", "\u{a0}%"), ("ru", true, "", "\u{a0}%"), + ("sd", false, "", "\u{66a}\u{61c}"), + ("sdh", false, "", "\u{66a}\u{61c}"), + ("se", true, "", "\u{a0}%"), + ("sk", true, "", "\u{a0}%"), ("sl", true, "", "\u{a0}%"), + ("smn", true, "", "\u{a0}%"), ("sv", true, "", "\u{a0}%"), + ("tk", true, "", "\u{a0}%"), ("tr", true, "%", ""), + ("tt", true, "", "\u{a0}%"), + ("uz-Arab", false, "", "\u{66a}"), + ("zgh", true, "", "\u{a0}%"), ]; diff --git a/crates/i18n/data/numbers.postcard b/crates/i18n/data/numbers.postcard index 53764d4cc9c73c84627d11699de9e5f564923c92..5ee84e037e3467ca23300e955e2ba8ea6846436b 100644 GIT binary patch literal 9553 zcmcIq3vd+ImF>@H^hHQ~1QDet+ z+tOw&DXnyd_k8Qb+1r0CV{VJj5e_hKZ2k6B{f2pBYd0s{>ZI~#Cy-X^{bC!pPVhWd zv`v?Kpt^9g#5}x@XCuGzZqcs_^{eZJC6!yYrB0dsurB&C7qOBG-K3X!N=N&wj9J2F zb;u>m#%$JDVQ~h9DX7Jvv^nHswx2qjE;mh0s3DS+Ja=JkeqmYF3v6oxVq49sW3_=) zwmRN{_k1C!i-OEv-#^x4j+KjmizLvX09mQZ_ak*@+V)in& zVjWxIWLrI&fz>t42GtfV72p_qHm>)U(-#OL@0ujj>xZrmUT16I%g+moic3n@tSwtt zUa@KO7W;PDoS9>J1OX^0+qC7Snkr^1Wwuz^c8RTE_E_l#iOk1Jw@A=zi><@V+S&F} zwjHhxmQ++pCOk)=V^!7k#2zcJq#ig|spm>&8!TP70W@@H_CobP+s^W>x0b<`(oObi zQ*v6)tT}U+u3)8`nTS8i7IqC+vWZBzflE4G8ZhymVo?C=Cbk>=D{vmab$V*zcr zeoRSWozKsn*Y3Z;oa{xm-8xuYzA+^|R#-;vJqafYnJolewn1M*L`q_|CiXHW%N6m4 zSOJ3X3D8Z?GhFowCr%3Q`D?~4$e54t&x>j2TLle zGG({+$e8=bxguETVyVD^t|JnDKtos!_uq?~yR9%eS!uVc0#ds6ko=RmeE zujLD(lU{`Xq^Y?DW67FTxc;RYUxX(pa@9@UVa)Gq_OrDyt0#S9FyONI z(XcZ142?(iy>w&P>0#@5Xjf?$7)W8!x|DRxgte%G+3+73=b)+6XJLK3Pe6C>R-Kau zv@0ZMWM)sxnb!ahxI$)gF*`Zs46GE?2whc8iuwh`oXjzz7ys>iI^Dn$yQ|}M%YGkVhE$(o9o=ae`d^B<;G|hU+vIU^l z6|=bGapmO^Qx^;oBBP*TGVzpu_&%W7g9w@li`N-bPayEgHLryBxe3A-I0wh3ld)H4(vTwr6{ z+Q_zIAzydZ2SP1r8Iu>PE6=$*%&_k3SSfZfi#3%>ohn;b!9(uFP{tH`A5~rZHLPfV z&g=(06sbms$HTVJkS*FJv)9rTG@*Vug0hp+JmwBaY-MrOyXI?1}* z5%ADh9c7L{gGIlKx!kfFb#DwKJj%G!JodFfe=899f!71|khEjo8t<#e-@SSFucEEG z;pEvL{nOITb-N<3ymoU(y&ns%1@983)NG}#Lt)bz*tCwABP=RRRBox;8yHe5qL?+! zryPp;TSP@sf^9)IrGusIR@h{fg$=1$GN+_vWKCg`Qp2)cN=T_unB3IHrm(5k9nK)O zER#8vr8CjfQNvP|oAHfBluSwH6om}#0eD>&rR0><^p1l}^s#K^^$?!laMYpNHll+$ zb?*TtHYpuYZ@!7iL83RyP?MgaG%0LS2b)YL4&ETU;RqGeG$yM5vY6B!bNQv9QmcH7 zos@Prl$_+$j7iwS=FV4QHD2+c;!+f^qIA@EsF~aWsT(nwqOj=*OO4Cp3xpeG6G~jh zBp1s8nTd5wYI}ptyyc)e77T@@BpN|RgVGjMSSALTr0n|%iaR}QXf`J!T9Y!fV(wj{ z(oVy7tASMl${>n`DB!vCuUGt+7h)K!NyF+0q z^~x``b?;HP1bSC&Y-%>AXJ${G`M?~DHE+q|Pdv3e|LNE+Ol~_91DGeb((YCozQrbc zn7}fa%#yD=Jl+6|%cgcZ-~*eiG?RBs3@Nm+sWrMqG{Jq09jTzSAy$|oMe%=ATXG^C zhDUX+n9f0GFk(tlYGEy)INF>_%pDXFT@Mo@9g#qtD3#m z0}ZeJeXK5I!e0CYs^lzMst-6(=^u2|``P1Er{+Xr-jEz(b0f?WbNG|B6_p#<$(GUv znTJJa!j`uX``ts1STKy;^N|3Xam^PDH=9y2u!%ejeT%SHECKB?dI_5y^My>c^=vV= zws~0fZclRhWUVUA#}2mu)#ht>H+z&yfc3rw|eJO&lI#m!_fPNq+O%Pmw zDP0ikz#oA03_*Aq=m$QTBnWAlf?xqY20jPOlLg^npd1JTr-5t0pMYsug76Gb2OI+4 z2Yw5fvIQX*r~+OA&H+CI{vDVzMG#7WM&Kgw8{kjCe*;-l1z`cO3|Ip^2Y7*3fH#0M zz+fE~bY;C0|M@B`oq@C)EG;6H$r>6il`7g!B!2Asfd z;G4jAfcJnKz;A$m1*9Cr5m*GQ2C4uL@G5W=xCs0h_%-lnAbEx$%m-Eib|4IBhd5aj zuIM*og_wSUK!?s86^a{@gh@!CM}=j=O^v0sZHYHm3yLsJ@gv4B=#1>p4$8yv26T(a zJb{2imvJw20VnAK%2DLuXcsbv#7mIeTZu)&$gU8X9|d)|BAui{H9`U;uYk$dBvvxZbU-6KPQ6~EZ zN@c^NWX@8)&88T&P;38#16fOqoLc+i@N}?l(j&w?<>|e^PmM)J8nAd_?p> zGkYJ7VWe9068dmeu|y+d^xlK>|B*%NI3~82;53DgGQ{9CjZ-m0ws?$$3MXOO08n}15Y&R#Q%XlN)jUs+Q z=l>-?M(f*zzR+N~?oq_nupOucv@CALm3~}Fe322vr{enx zJ?EiQUu1ZPtNbF7pQVy-N}&ubomuPyDZ3;r-pCL74zU@m1a9PgM7KSvHRLIy&mdy+>P`INH7A zM*F%*Yj=@g6RHHqm?}rw`HYc61WHOm5uhJ!WP=7mpm$gTPh0AxAMN*Jf~D`_yoWB} zj(jPS-iy1f3%LKffV-;;?rB+P7?e^(NvRrU`ssv+xc19=x<^bBtisd6OM*}9%eCX@ zAc3$bj#W4lHpfx$IhlfTj;OJ_dg zt%*XCKQ1m6pAerEmx)h_%SD@*FRmDUNl;5C?@_=fgGy!*NqKayB|Z_yyzi(wbn~1^ zRl=x4opVTuYA7URWz+R2 zr{^yKoxnw)3-}(;4SXNy0e%4V%Aa;B|r5 z1zs0;UEpd0#*8^S;cs=0t zfY$?F4|qM`^@7(6UN3mP;Pryn3tlgHz2NnN*9%@0yeN24@S@;F!Ha?y1uqI-6uc;S zec<(h*9Tr7czxjYf!7CKA9#J>^?}z9UO#yK;Pr#o4_-fb{owV3*AHGlc$XM(7&rnP z1&#s7fj5B@z)9eg{OJIA1KOeU0nP#E0r+zn z{#=GXm%HRUZLL5Xpa6S-c3>~C57-YJ01nD`zIz5Z3%m`y1DpfS0~df!;35EhXBco8 zI0766jseGkH-QttN#K-xCkkE^yeN24@S@;F!Ha?y1uqI-^pyO&*8TElAGOM#{qPXZ z2j$fMvX$v9%|aD*-OnnYW7oefSQA7&NFHbQ*(rxqtqOu<~TKP zQgeculhmBb#NP?vx4_oLxsTE}HCYx%C3*Dy(I}yBi$)236-J#+2WD} zPHqi;3oJ{c{^V(r1P}9aYw=5BBZt1jOdv7zqR+QR4t=LHO5Fay&ct!$<9eNM7Jb0qzs~MGqjEbG1!AsIO(F816mh_H!TE|3I-t9y2(2lZh zABsmHOSNc*#k;M&ahfj6yL+h(-;?~%W{@TFOz1n*t((Y_W&F8~`R;)ui*xfhZ@f%O z;PLzwv-x_;%Z-2RgW&{W{8J}lIe}1RCMB7Zt#@1BSj=-r%vNnDtN!SrnMf#2m1X>a zQ0nc|*4w8S=M8}G!mzzY2u;oQMt*Kl9HMRW2!-Yu+xlk-H{xUr)Au2?Et?6 zA=5SGn9XVUP6SQxlAPCdjeDYNn@xQ(GDrJxefR_fr{^XoXJ%$ii91I+0VCyq02-l_ A*Z=?k literal 5664 zcmcJTTTola8OP7rBgEC%*sdMNvSJg!iH!)()wy&WA%sMP5FoB@XHwZvqacJw2-j|> zjbwwtsgtblCP^FLu&gSSD1_J!w6zaVr&(ZF0zuoLdJZzYsb#kwjZ{;pX=~= zcl)~iJ$ru4aHVP0D;<|MGiU1x>At5;&+h1*nlIRDEcfw}`5k(Lpmq6NJ9zPYm$9te zT=|Tx%I>JCb=J2C@k>;$EZq|wN6!D@Z)vOVfAW+&+TA zetS)cGo{9uQZsEVcQ{+}@*gTsY5F!QUn{g>#ST~&vkNT1LJzGn{V}ayZr)V%dHl+%IwyQ@(wG=RKpzYO^=D#2a&! zdcE4p3#QB#n;jjMKIk`Tu5&UCFMy$&#-#0a6jk|5F^k^E?*>+5cQ&-dhRa>HT!~hc zh^dS%+mNJuUT~w_TxG9z)W_$h&TIqPbdzR#4aQRZDRX5tMO33qn~c^Uaekw{P_tUMlMi zS`WR<9xzt$Qhn@^Z{kIUkC7WnBj!jO)zq@GbSurH=cj!y4h~(4Oq(kzpQ%#zuZYKg zZWznyH-U~@Mph1{8mnJRcf2ea-KUKe)it$?`_=OzJ1=VBIvi%~Y^}0dM99Td}!bE!FCC123xJMF>CT_W663uUX4C(285z_PF@^Dq8mj zzR^%-y{O zuElC_Z@{Ai*}`MJ@?RA%?7_%WWj1>QMrc773)=A;Q_c&Fyl~Rv+rvuveF#5YW~#C` zOWAAnMy}iE^BC;dZ(1ncUuDe_Ds9RUK*={5WMhTY%r{l?O>&zrlW)-b+dMt|UdSfA z>EsSC25IH_Zmw&`0n!^X+g)CFYuiqK|D@IKtnHL-F1(KPiCVAR?Z3w$TkT71$<;Wp z3(jqU>@CTSvZG~Dz~nYd@`pd)>2CpzHs`Jln@S&i^1ItAZQFma%k6u4SFrmK-udE- z{ZsTZ7nv07-``KiUt|aSdnet#9z47h#NLa60at)FT4cdsz~ATT$EJxd95B?>5A+GJQ>WaRAYb?+mRm;$N&Za5wk~_!O8$ zK{yR2z)euY1mQe*RuY7d!C{Rc+yJfUcolpI{sA(zg0K-dz>mP|;0E{-tWFn%ZD1F8 z8C(W)AU#769so_?c@P30fq#R0G6lf`_JIia3-}U=6)GMey(Ea+; zWlIyE<-`*nu;|!@jA0-Qa6MiTAN5-vV0F>}%7`?oGUB&9ke(qg*O2E#1_Q15KnY$! zro4Leh!ff_UHXCKLZ?V}?Hv2Z49b>cau-VIO_6nQ5%H{%WF39Hgv$+Fou9waA zvAKh6uAj{fu(?Ch&7q4R1TKLv_!WqNH^4CXH5id@hTw(Zh2VwYh2VwYh2VwYh2VwY zh2e$ah2e$ah2e$ah2e$ah2e$aMc_r?Mc_r?Mc_r?Mc_r?Mc_r?4Z|CTHwt`fH$Hx^*QW-Fo*h?uVq$UwswN$EDBvhVl6T z?h)zM)n7jj5=Br=!1*iy8v7d?q zRP<8ON5w%Z`l%S8;t+}Hnh#bB-|GA3yZn~OKnw_QmR z7ivnru7)L?l9XO*O_EbgUoa)M5V@pSiJV%KG3?v1N*- zh4oorB(5!!B#kCb``N(J&Bf|k?oh|_)Etx)$G^H^*gU0ky3#o@@SQ5zN=ee<3;rjs z89sS!^Gsys4f-FXh_F(l$;ime`YiC00W3`6w)%K7Byp8io0XNF0~do*k3`E_YRSo6 MXi-POcP0$, - /// The region's separators where the system set them apart from the locale's. + /// The system's separators where they differ from the locale's; an empty `group` is none. group: Option, point: Option, percent: (&'static str, &'static str), @@ -37,8 +37,7 @@ pub struct Format { impl Format { /// The format for `region`'s number locale. With Latin digits, a decimal or grouping separator - /// the region sets (Windows' and macOS's own settings, English in continental Europe) replaces - /// the locale's. + /// the region carries (Windows' and macOS's own settings) replaces the locale's. pub fn new(region: &Region, digits: Digits) -> Format { let mut locale = Locale::try_from_str(®ion.numbers).unwrap_or(Locale::UNKNOWN); if digits == Digits::Latin { @@ -50,15 +49,14 @@ impl Format { let probe = format.parts(&Decimal::try_from_str("-1234567.5").unwrap_or_default()); let latin = probe.integer.starts_with(|c: char| c.is_ascii_digit()); if latin { - let point = region.decimal.to_string(); - if probe.point != point { + if let Some(point) = region.decimal.map(String::from).filter(|p| *p != probe.point) { // `1,234.5` with a decimal comma becomes `1.234,5`, never `1,234,5`. if probe.group == point && region.grouping.is_none() { format.group = Some(probe.point.clone()); } format.point = Some(point); } - if let Some(g) = region.grouping.map(String::from).filter(|g| *g != probe.group) { + if let Some(g) = region.grouping.map(|g| g.map(String::from).unwrap_or_default()).filter(|g| *g != probe.group) { format.group = Some(g); } } @@ -76,12 +74,12 @@ impl Format { self.write(&Decimal::from(n.into())) } - /// `value` rounded to `decimals` digits after the decimal separator. - /// Infinities and NaN are written as Rust writes them. + /// `value` rounded to `decimals` digits after the decimal separator; infinities and NaN are + /// `∞`, `-∞` and `NaN`. pub fn number(&self, value: f64, decimals: usize) -> String { match to_decimal(value, decimals) { Some(d) => self.write(&d), - None => value.to_string(), + None => non_finite(value).to_owned(), } } @@ -89,7 +87,7 @@ impl Format { /// `50%`, `50 %` (French, German), `%50` (Turkish), `٥٠٪؜` (Egyptian Arabic). pub fn percent(&self, ratio: f64, decimals: usize) -> String { let Some(d) = to_decimal(ratio * 100.0, decimals) else { - return format!("{}%", ratio * 100.0); + return format!("{}{}{}", self.percent.0, non_finite(ratio), self.percent.1); }; let p = self.parts(&d); format!("{}{}{}{}", p.sign, self.percent.0, p.text, self.percent.1) @@ -200,6 +198,15 @@ pub(crate) fn to_decimal(value: f64, decimals: usize) -> Option { Some(d) } +/// CLDR's root symbols for a value that isn't finite. +fn non_finite(value: f64) -> &'static str { + match value { + f64::INFINITY => "∞", + f64::NEG_INFINITY => "-∞", + _ => "NaN", + } +} + /// The percent affixes for a number locale (`ar-EG`), found by dropping trailing subtags. fn percent_affixes(tag: &str, latin: bool) -> (&'static str, &'static str) { let mut tag = tag; @@ -248,17 +255,39 @@ mod tests { } #[test] - fn region_separators() { - // English in Poland writes a decimal comma; the grouping moves out of its way. - assert_eq!(Format::from_tag("en-PL").number(1234.5, 1), "1.234,5"); + fn cldr_separators_for_every_region() { + for (tag, want) in [ + ("en-PL", "1.234.567,5"), + ("en-DE", "1.234.567,5"), + ("ar-MA", "1.234.567,5"), + ("de-CH", "1'234'567.5"), + ("nb-NO", "1\u{a0}234\u{a0}567,5"), + ("sk-SK", "1\u{a0}234\u{a0}567,5"), + ("bn-BD", "১২,৩৪,৫৬৭.৫"), + ("he-IL", "1,234,567.5"), + ("km", "1,234,567.5"), + ("lo", "1.234.567,5"), + ] { + assert_eq!(Format::from_tag(tag).number(1_234_567.5, 1), want, "{tag}"); + } + } + + #[test] + fn system_separators() { + // A decimal comma the system sets moves the grouping out of its way. + let mut region = Region::from_tag("en-US"); + region.decimal = Some(','); + assert_eq!(Format::new(®ion, Digits::Locale).number(1234.5, 1), "1.234,5"); let mut region = Region::from_tag("de-DE"); - region.grouping = Some('\u{a0}'); + region.grouping = Some(Some('\u{a0}')); assert_eq!(Format::new(®ion, Digits::Locale).number(1234.5, 1), "1\u{a0}234,5"); - region.decimal = '.'; + region.decimal = Some('.'); assert_eq!(Format::new(®ion, Digits::Locale).number(1234.5, 1), "1\u{a0}234.5"); + region.grouping = Some(None); + assert_eq!(Format::new(®ion, Digits::Locale).integer(1_234_567), "1234567", "no grouping"); // A system's separators don't apply to Arabic-Indic digits. let mut region = Region::from_tag("ar-EG"); - region.grouping = Some(','); + region.grouping = Some(Some(',')); assert_eq!(Format::new(®ion, Digits::Locale).number(1234.5, 1), "١٬٢٣٤٫٥"); } @@ -280,7 +309,10 @@ mod tests { let en = Format::from_tag("en"); assert_eq!(en.number(-1234.5, 0), "-1,234"); assert_eq!(en.number(-0.04, 1), "0.0", "no sign on zero"); - assert_eq!(en.number(f64::INFINITY, 1), "inf"); + assert_eq!(en.number(f64::INFINITY, 1), "∞"); + assert_eq!(en.number(f64::NEG_INFINITY, 1), "-∞"); + assert_eq!(en.number(f64::NAN, 1), "NaN"); + assert_eq!(Format::from_tag("de").percent(f64::NAN, 0), "NaN\u{a0}%"); assert_eq!(en.integer(-5_i64), "-5"); } } diff --git a/crates/i18n/src/lib.rs b/crates/i18n/src/lib.rs index 910041ed..a3924fe9 100644 --- a/crates/i18n/src/lib.rs +++ b/crates/i18n/src/lib.rs @@ -3,6 +3,11 @@ //! An app embeds the [craft-i18n](https://github.com/benandez/craft-i18n) catalogs at build time //! as an [`Embedded`] static, builds one [`I18n`] from it, and looks English source text up in a //! [`Lang`]. A missing translation is the English itself; nothing here fails. +//! +//! Counts ([`I18n::trn`]), file sizes and measures write their numbers as the formatting +//! [`Region`] does ([`I18n::format`]): grouped (`1.234`, `12,34,567`) and in the region's digits +//! (`١٬٢٣٤`). Text that goes into documents or file names should use [`I18n::trn_plain`], which +//! writes `0`–`9` without grouping. #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] mod catalog; @@ -10,7 +15,7 @@ mod format; mod region; mod size; -/// The CLDR data for the shipped languages, from `data/generate.sh`. +/// The CLDR number and plural data for every CLDR locale, from `data/generate.sh`. mod data { use std::sync::OnceLock; @@ -189,25 +194,34 @@ impl I18n { self.find(lang, &[("@id", id), ("", label)]).unwrap_or(label) } - /// A count's wording in `lang`, `{n}` filled in as [`I18n::format`] writes numbers. `one` and - /// `other` are the English forms; they are also the key of the `@plural` row, and the fallback. + /// A count's wording in `lang`, `{n}` filled in as [`I18n::format`] writes numbers: grouped and + /// in the region's digits. `one` and `other` are the English forms; they are also the key of + /// the `@plural` row, and the fallback. pub fn trn(&self, lang: Lang, n: u64, one: &str, other: &str) -> String { let format = self.format(); - self.plural(lang, PluralOperands::from(n), n == 1, &format.integer(n), one, other) + self.plural(lang, Some(PluralOperands::from(n)), n == 1, &format.integer(n), one, other) + } + + /// [`I18n::trn`] with `{n}` in `0`–`9` without grouping (`Copy 1234`), for text that goes into + /// documents, file names or anything read back later. + pub fn trn_plain(&self, lang: Lang, n: u64, one: &str, other: &str) -> String { + self.plural(lang, Some(PluralOperands::from(n)), n == 1, &n.to_string(), one, other) } /// [`I18n::trn`] for a count with `decimals` digits after the decimal separator (`1.5 files`). - /// The plural form follows the digits shown, so `1.0` takes English `other`. + /// The plural form follows the digits shown, so `1.0` takes English `other`; infinities and NaN + /// (`∞`, `NaN`) take `other` too. pub fn trn_decimal(&self, lang: Lang, n: f64, decimals: usize, one: &str, other: &str) -> String { let format = self.format(); let Some(d) = format::to_decimal(n, decimals) else { - return fmt(other, &[("n", &n.to_string())]); + return self.plural(lang, None, false, &format.number(n, decimals), one, other); }; let english_one = d.to_string() == "1"; - self.plural(lang, PluralOperands::from(&d), english_one, &format.write(&d), one, other) + self.plural(lang, Some(PluralOperands::from(&d)), english_one, &format.write(&d), one, other) } - fn plural(&self, lang: Lang, operands: PluralOperands, english_one: bool, n: &str, one: &str, other: &str) -> String { + /// `operands` `None` is the `other` category. + fn plural(&self, lang: Lang, operands: Option, english_one: bool, n: &str, one: &str, other: &str) -> String { let key = format!("{one}|{other}"); let chain = self.locale(lang).map_or(&[][..], |l| l.chain.as_slice()); let text = chain @@ -216,7 +230,7 @@ impl I18n { let locale = self.locales.get(i)?; let forms = locale.rows.get("@plural")?.get(&key)?; let (rules, categories) = locale.plural.as_ref()?; - let category = rules.category_for(operands); + let category = operands.map_or(PluralCategory::Other, |o| rules.category_for(o)); // A category whole numbers up to 1000 never reach has no form of its own. Russian // and Ukrainian decimals (`other`) take the `few` form (`1,5 файла`); any other // (French `many` for a million) takes the last. @@ -236,7 +250,7 @@ impl I18n { } /// How counts, file sizes and measures write numbers: [`Region::system`]'s format with the - /// locale's digits until [`I18n::set_format`]. + /// locale's digits until [`I18n::set_format`], and again after [`I18n::reset_format`]. pub fn format(&self) -> Arc { if let Some(f) = self.format.read().unwrap_or_else(PoisonError::into_inner).as_ref() { return Arc::clone(f); @@ -250,6 +264,11 @@ impl I18n { *self.format.write().unwrap_or_else(PoisonError::into_inner) = Some(Arc::new(format)); } + /// Write numbers as the system region does again, undoing [`I18n::set_format`]. + pub fn reset_format(&self) { + *self.format.write().unwrap_or_else(PoisonError::into_inner) = None; + } + /// The languages to offer: English first, then the offered locales by code. Never the /// pseudo locale. pub fn languages(&self) -> Vec { @@ -432,15 +451,18 @@ fn bare(code: &str) -> Locale { } /// `code`'s CLDR cardinal rules and the categories whole numbers 0..=1000 reach, in CLDR order -/// (zero, one, two, few, many, other): one `@plural` form each. Regional and scripted codes fall -/// back to the language's rules. CLDR's `pt` is Brazilian (0 is singular), so Portuguese other -/// than `pt-br` takes `pt-PT`'s. `None` for a language the baked data doesn't cover. +/// (zero, one, two, few, many, other): one `@plural` form each. The rules are looked up by +/// language alone: CLDR's parent of `sr-Latn` and `zh-Hant` is the root, whose only category is +/// `other`. CLDR's `pt` is Brazilian (0 is singular), so Portuguese other than `pt-br` takes +/// `pt-PT`'s. `None` for a language CLDR has no plural rules for. fn plural_rules(code: &str) -> Option<(PluralRules, Vec)> { - let code = if code == "pt" || (code.starts_with("pt-") && code != "pt-br") { "pt-pt" } else { code }; - let locale = IcuLocale::try_from_str(code).ok()?; - if !data::LANGUAGES.contains(&locale.id.language.as_str()) { + let id = IcuLocale::try_from_str(code).ok()?.id; + let language = id.language.as_str(); + if !data::PLURAL_LANGUAGES.contains(&language) { return None; } + let brazil = id.region.is_some_and(|r| r.as_str() == "BR"); + let locale = IcuLocale::try_from_str(if language == "pt" && !brazil { "pt-PT" } else { language }).ok()?; let rules = PluralRules::try_new_cardinal_with_buffer_provider(data::provider()?, (&locale).into()).ok()?; let mut categories = Vec::new(); for n in 0..=1000_u32 { @@ -475,10 +497,12 @@ mod tests { ("de", "\tBad\n@nope\tA\tB\n\tUndo\tRückgängig\n", ""), ]; - /// `embedded`, offering every locale in it. + /// `embedded`, offering every locale in it, with `en-US` numbers rather than the machine's. fn all(embedded: &'static Embedded) -> (I18n, Vec) { let codes: Vec<&str> = embedded.iter().map(|(c, _, _)| *c).collect(); - I18n::new(embedded, &codes) + let (i, problems) = I18n::new(embedded, &codes); + i.set_format(Format::from_tag("en-US")); + (i, problems) } fn i18n() -> I18n { @@ -610,6 +634,20 @@ mod tests { assert_eq!(i.trn(Lang::EN, 12_345, "{n} file", "{n} files"), "12.345 files"); } + #[test] + fn plain_counts_and_the_system_format() { + let i = all(P).0; + i.set_format(Format::from_tag("ar-EG")); + assert_eq!(i.trn_plain(lang(&i, "ar"), 1234, "{n} file", "{n} files"), "1234 ملفًا"); + i.set_format(Format::from_tag("de-DE")); + assert_eq!(i.trn_plain(Lang::EN, 12_345, "copy {n}", "copy {n}"), "copy 12345"); + i.reset_format(); + assert_eq!( + i.trn(Lang::EN, 5, "{n} file", "{n} files"), + fmt("{n} files", &[("n", &Format::new(&Region::system(), Digits::Locale).integer(5))]) + ); + } + #[test] fn decimal_counts() { let i = all(P).0; @@ -627,6 +665,9 @@ mod tests { let ar = lang(&i, "ar"); assert_eq!(i.trn_decimal(ar, 2.5, 1, "{n} file", "{n} files"), "٢٫٥ ملف"); assert_eq!(i.trn_decimal(ar, 3.0, 0, "{n} file", "{n} files"), "٣ ملفات"); + // Not a number: the translated `other` form. + assert_eq!(i.trn_decimal(ru, f64::NAN, 1, "{n} file", "{n} files"), "NaN файла"); + assert_eq!(i.trn_decimal(Lang::EN, f64::INFINITY, 1, "{n} file", "{n} files"), "∞ files"); } static L: &Embedded = @@ -777,15 +818,31 @@ mod tests { assert_eq!(i.trn(lang(&i, "pt-br"), 0, "{n} file", "{n} files"), "0 arquivo"); } - /// The same counts as craft-i18n's checker (`plural_form_counts`), which rejects a catalog - /// whose `@plural` rows list any other. + /// The counts craft-i18n's checker expects, which rejects a catalog whose `@plural` rows list + /// any other: every directory in craft-i18n's `locales/`, then languages it may add. #[test] fn plural_form_counts() { - for (locale, n) in - [("en", 2), ("fr", 2), ("ja", 1), ("vi", 1), ("ru", 3), ("pl", 3), ("cs", 3), ("sl", 4), ("ar", 6), ("pt", 2), ("pt-br", 2), ("tr", 2)] - { + #[rustfmt::skip] + let catalogs = [ + ("ar", 6), ("bg", 2), ("cs", 3), ("da", 2), ("de", 2), ("el", 2), ("es", 2), ("et", 2), + ("fi", 2), ("fr", 2), ("hr", 3), ("hu", 2), ("id", 1), ("it", 2), ("ja", 1), ("ka", 2), + ("kk", 2), ("ko", 1), ("lt", 3), ("nl", 2), ("pl", 3), ("pt-br", 2), ("ru", 3), ("sl", 4), + ("sr", 3), ("sr-latn", 3), ("sv", 2), ("te", 2), ("tr", 2), ("uk", 3), ("vi", 1), + ("zh-hans", 1), ("zh-hant", 1), + ]; + let more = [("en", 2), ("pt", 2), ("he", 3), ("ro", 3), ("nb", 2), ("ca", 2), ("sk", 3), ("th", 1), ("ms", 1), ("fa", 2), ("hi", 2)]; + for (locale, n) in catalogs.into_iter().chain(more) { assert_eq!(plural_rules(locale).map(|(_, c)| c.len()), Some(n), "{locale}"); } + assert!(plural_rules("tlh").is_none(), "no CLDR rules"); + } + + #[test] + fn serbian_latin_plurals() { + let i = all(&[("sr-latn", "@plural\t{n} file|{n} files\t{n} fajl|{n} fajla|{n} fajlova\n", "")]).0; + let sr = lang(&i, "sr-latn"); + let files = |n| i.trn(sr, n, "{n} file", "{n} files"); + assert_eq!((files(1), files(3), files(5)), ("1 fajl".into(), "3 fajla".into(), "5 fajlova".into())); } #[test] From f2ab6dc7f3df71ad523a465e0ab5b0483ee323d0 Mon Sep 17 00:00:00 2001 From: aiongg Date: Wed, 21 Oct 2026 05:49:46 +0000 Subject: [PATCH 7/7] i18n: rewrap the Region README paragraph --- crates/i18n/README.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/crates/i18n/README.md b/crates/i18n/README.md index 7520f4ae..85f75c8b 100644 --- a/crates/i18n/README.md +++ b/crates/i18n/README.md @@ -71,13 +71,14 @@ The app keeps its `tl!` macro, its language setting (`"auto"` → `I18N.system_l - **Formatting region:** `Region` holds how numbers, times and measurements are written, apart from the interface language: the number and time locales (BCP 47), metric or US measurement, A4 or Letter, and the decimal and grouping separators (or no grouping) and clock when the system - sets them; otherwise the number locale's CLDR data decides. `Region::system()` reads it once: Windows' regional format with the user's changes - (`HKCU\Control Panel\International`), Apple's current locale (`en_PL`, `en_GB@rg=uszzzz`), - `LC_ALL` or else `LC_NUMERIC`, `LC_TIME`, `LC_MEASUREMENT` and `LC_PAPER` each on its own over - `LANG` on Unix (`C` there is `en-US`; `LANG=C` gives way to the preferred language), the preferred language on the web and Android. `Region::from_setting` takes - the app's saved choice (`auto` or a tag like `pl-PL`). Paper and measurement follow CLDR's - territory lists; tags may carry `-u-ms-`, `-u-hc-` and `-u-rg-`. Ported from WordCraft's - `Region`. + sets them; otherwise the number locale's CLDR data decides. `Region::system()` reads it once: + Windows' regional format with the user's changes (`HKCU\Control Panel\International`), Apple's + current locale (`en_PL`, `en_GB@rg=uszzzz`), `LC_ALL` or else `LC_NUMERIC`, `LC_TIME`, + `LC_MEASUREMENT` and `LC_PAPER` each on its own over `LANG` on Unix (`C` there is `en-US`; + `LANG=C` gives way to the preferred language), the preferred language on the web and Android. + `Region::from_setting` takes the app's saved choice (`auto` or a tag like `pl-PL`). Paper and + measurement follow CLDR's territory lists; tags may carry `-u-ms-`, `-u-hc-` and `-u-rg-`. + Ported from WordCraft's `Region`. - **Pseudo locale** (`pseudo`, debug builds only, never in `languages()`): every lookup, `get` included, comes back as `⟦…⟧`, line by line, so text drawn without the marks never went through translation. `looked_up()` lists the source texts looked up so far, for an audit against a