From 903dbd5fa342c4df1cd9fe81982b0440ae007f4b Mon Sep 17 00:00:00 2001 From: aiongg Date: Wed, 21 Oct 2026 04:38:22 +0000 Subject: [PATCH 1/9] i18n: scripts, direction and regional Han per language tag scripts(tag) lists the ISO 15924 scripts a language's interface text is written in, direction(tag) follows the main script, and han(tag) gives the regional Han forms (SC, TC, HK, JP, KR) for picking a CJK face. I18n::rtl uses direction. --- crates/i18n/README.md | 8 +- crates/i18n/src/lib.rs | 11 +- crates/i18n/src/scripts.rs | 363 +++++++++++++++++++++++++++++++++++++ 3 files changed, 375 insertions(+), 7 deletions(-) create mode 100644 crates/i18n/src/scripts.rs diff --git a/crates/i18n/README.md b/crates/i18n/README.md index df337971..cde3621d 100644 --- a/crates/i18n/README.md +++ b/crates/i18n/README.md @@ -60,8 +60,12 @@ The app keeps its `tl!` macro, its language setting (`"auto"` → `I18N.system_l `plural_form_counts` holds the same counts. - **Names:** a language's name comes from its shared catalog's `@id language.name` row, else its code; English is "English". -- **Right to left:** `rtl` is true for `ar`, `ckb`, `dv`, `fa`, `he`, `ps`, `sd`, `ug`, `ur` and - `yi`; putting text into display order is the app's job. +- **Scripts and direction:** for any language tag, `scripts` lists the ISO 15924 scripts its + interface text is written in (`ja` → Hani, Hira, Kana; `sr-latn` → Latn), `direction` is right + to left when the main script is (Arabic, Hebrew, Syriac, Thaana), and `han` gives the regional + forms of its Han characters for picking a CJK face: `zh-CN`/`zh-SG`/`zh-Hans` → SC, `zh-TW`/ + `zh-Hant` → TC, `zh-HK`/`zh-MO` → HK, `ja` → JP, `ko` → KR. `I18n::rtl` is `direction` of the + language's code; putting text into display order is the app's job. - **System locale:** `system_lang` matches the system's list once and caches it: on desktop Unix `LANGUAGE`, `LC_ALL`, `LC_MESSAGES` and `LANG`, read raw; elsewhere `sys-locale`'s (macOS and Windows preferred languages, the browser's on the web). `fr_CA.UTF-8` → `fr`, `zh-TW` → diff --git a/crates/i18n/src/lib.rs b/crates/i18n/src/lib.rs index 7fa7a4ef..84295a16 100644 --- a/crates/i18n/src/lib.rs +++ b/crates/i18n/src/lib.rs @@ -6,6 +6,7 @@ #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] mod catalog; +mod scripts; use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::sync::{Mutex, OnceLock, PoisonError}; @@ -13,6 +14,8 @@ use std::sync::{Mutex, OnceLock, PoisonError}; use intl_pluralrules::{PluralCategory, PluralRuleType, PluralRules}; use unic_langid::LanguageIdentifier; +pub use scripts::{Direction, Han, Script, direction, han, scripts}; + /// The catalogs an app embeds, one entry per locale: (locale, shared catalog, the app's catalog). /// A missing catalog is `""`. pub type Embedded = [(&'static str, &'static str, &'static str)]; @@ -199,12 +202,10 @@ impl I18n { self.locale(lang).and_then(|l| l.rows.get("@id")?.get("language.name")).map_or_else(|| self.code(lang), String::as_str) } - /// Whether the language is written right to left. Putting text into display order is the - /// app's job. + /// Whether the language is written right to left ([`direction`]). Putting text into display + /// order is the app's job. pub fn rtl(&self, lang: Lang) -> bool { - let code = self.code(lang); - let language = code.split('-').next().unwrap_or(code); - matches!(language, "ar" | "ckb" | "dv" | "fa" | "he" | "ps" | "sd" | "ug" | "ur" | "yi") + direction(self.code(lang)) == Direction::Rtl } /// The language for a system locale tag (`fr_CA.UTF-8`, `zh-Hant-MO`); `C` and `POSIX` are diff --git a/crates/i18n/src/scripts.rs b/crates/i18n/src/scripts.rs new file mode 100644 index 00000000..ecd2f8dd --- /dev/null +++ b/crates/i18n/src/scripts.rs @@ -0,0 +1,363 @@ +//! The scripts a language is written in, its direction, and which regional forms its Han +//! characters take, for picking fonts and laying out the interface. + +/// A writing system, by its ISO 15924 code. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub enum Script { + Arab, + Armn, + Beng, + Cyrl, + Deva, + Ethi, + Geor, + Grek, + Gujr, + Guru, + Hang, + Hani, + Hebr, + Hira, + Kana, + Khmr, + Knda, + Laoo, + Latn, + Mlym, + Mong, + Mymr, + Orya, + Sinh, + Syrc, + Taml, + Telu, + Thaa, + Thai, + Tibt, +} + +impl Script { + /// The ISO 15924 code, `Latn`. + pub fn code(self) -> &'static str { + match self { + Script::Arab => "Arab", + Script::Armn => "Armn", + Script::Beng => "Beng", + Script::Cyrl => "Cyrl", + Script::Deva => "Deva", + Script::Ethi => "Ethi", + Script::Geor => "Geor", + Script::Grek => "Grek", + Script::Gujr => "Gujr", + Script::Guru => "Guru", + Script::Hang => "Hang", + Script::Hani => "Hani", + Script::Hebr => "Hebr", + Script::Hira => "Hira", + Script::Kana => "Kana", + Script::Khmr => "Khmr", + Script::Knda => "Knda", + Script::Laoo => "Laoo", + Script::Latn => "Latn", + Script::Mlym => "Mlym", + Script::Mong => "Mong", + Script::Mymr => "Mymr", + Script::Orya => "Orya", + Script::Sinh => "Sinh", + Script::Syrc => "Syrc", + Script::Taml => "Taml", + Script::Telu => "Telu", + Script::Thaa => "Thaa", + Script::Thai => "Thai", + Script::Tibt => "Tibt", + } + } + + /// Whether text in this script runs right to left. + pub fn rtl(self) -> bool { + matches!(self, Script::Arab | Script::Hebr | Script::Syrc | Script::Thaa) + } +} + +/// Which way a language's text, and an interface in it, runs. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum Direction { + Ltr, + Rtl, +} + +/// The regional glyph forms Han characters take, as in the Noto CJK face names (SC, TC, HK, JP, +/// KR). The same code point is drawn differently in each, so a font stack picks its Han face by +/// this, not by script. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum Han { + /// Simplified Chinese, mainland China and Singapore (SC). + Simplified, + /// Traditional Chinese, Taiwan (TC). + Traditional, + /// Traditional Chinese, Hong Kong and Macau (HK). + HongKong, + /// Japanese kanji (JP). + Japanese, + /// Korean hanja (KR). + Korean, +} + +/// The scripts a language tag's interface text is written in (`ja` → Hani, Hira, Kana; `sr` → +/// Cyrl; `sr-latn` → Latn), the main one first. Any case and glibc form (`sr_RS.UTF-8@latin`). A +/// script subtag wins over the language's usual script. Latin product names and units appear in +/// every language, so Latn is listed only where it is the language's own script. A language not +/// in the table is taken to be written in Latin. +// ponytail: the languages a Craft app ships or could plausibly ship; add a row with the locale. +pub fn scripts(tag: &str) -> &'static [Script] { + use Script::*; + let tag = Subtags::parse(tag); + if let Some(s) = tag.script.as_deref().and_then(subtag_scripts) { + return s; + } + match tag.language.as_str() { + "ja" => &[Hani, Hira, Kana], + "ko" => &[Hang, Hani], + "zh" | "yue" | "wuu" => &[Hani], + "be" | "bg" | "ba" | "ce" | "cv" | "kk" | "ky" | "mk" | "mn" | "ru" | "sah" | "sr" | "tg" | "tt" | "uk" => &[Cyrl], + "el" => &[Grek], + "ar" | "ckb" | "fa" | "ks" | "ps" | "sd" | "ug" | "ur" => &[Arab], + "he" | "yi" => &[Hebr], + "syr" => &[Syrc], + "dv" => &[Thaa], + "ka" => &[Geor], + "hy" => &[Armn], + "hi" | "mr" | "ne" | "sa" | "kok" | "mai" | "bho" => &[Deva], + "bn" | "as" | "mni" => &[Beng], + "pa" => &[Guru], + "gu" => &[Gujr], + "or" => &[Orya], + "ta" => &[Taml], + "te" => &[Telu], + "kn" => &[Knda], + "ml" => &[Mlym], + "si" => &[Sinh], + "th" => &[Thai], + "lo" => &[Laoo], + "km" => &[Khmr], + "my" => &[Mymr], + "am" | "ti" => &[Ethi], + "bo" | "dz" => &[Tibt], + _ => &[Latn], + } +} + +/// The direction of a language tag's text: right to left when its main script is. +pub fn direction(tag: &str) -> Direction { + match scripts(tag).first() { + Some(s) if s.rtl() => Direction::Rtl, + _ => Direction::Ltr, + } +} + +/// The forms a language tag's Han characters take: `zh-CN`, `zh-SG`, `zh-Hans` and plain `zh` → +/// Simplified; `zh-TW` and `zh-Hant` → Traditional; `zh-HK`, `zh-MO` and `zh-Hant-HK` → HongKong; +/// `ja` → Japanese; `ko` → Korean. `None` for a language not written with Han, which leaves the +/// choice to the font stack. +pub fn han(tag: &str) -> Option { + let tag = Subtags::parse(tag); + let hong_kong = matches!(tag.region.as_deref(), Some("hk" | "mo")); + match (tag.language.as_str(), tag.script.as_deref()) { + ("ja", _) => Some(Han::Japanese), + ("ko", _) => Some(Han::Korean), + ("zh" | "yue" | "wuu", Some("hans")) => Some(Han::Simplified), + ("zh" | "yue" | "wuu", Some("hant")) => Some(if hong_kong { Han::HongKong } else { Han::Traditional }), + // Cantonese is written in Traditional characters with Hong Kong forms unless it says otherwise. + ("yue", _) => Some(if tag.region.as_deref() == Some("cn") { Han::Simplified } else { Han::HongKong }), + ("zh" | "wuu", _) => Some(match tag.region.as_deref() { + Some("tw") => Han::Traditional, + _ if hong_kong => Han::HongKong, + _ => Han::Simplified, + }), + _ => None, + } +} + +/// The scripts a script subtag stands for, including the CLDR unions (`jpan`, `kore`). `None` for +/// one not in [`Script`], so the language's own scripts apply. +fn subtag_scripts(subtag: &str) -> Option<&'static [Script]> { + use Script::*; + Some(match subtag { + "hans" | "hant" | "hani" => &[Hani], + "jpan" => &[Hani, Hira, Kana], + "kore" => &[Hang, Hani], + "arab" => &[Arab], + "armn" => &[Armn], + "beng" => &[Beng], + "cyrl" => &[Cyrl], + "deva" => &[Deva], + "ethi" => &[Ethi], + "geor" => &[Geor], + "grek" => &[Grek], + "gujr" => &[Gujr], + "guru" => &[Guru], + "hang" => &[Hang], + "hebr" => &[Hebr], + "khmr" => &[Khmr], + "knda" => &[Knda], + "laoo" => &[Laoo], + "latn" => &[Latn], + "mlym" => &[Mlym], + "mong" => &[Mong], + "mymr" => &[Mymr], + "orya" => &[Orya], + "sinh" => &[Sinh], + "syrc" => &[Syrc], + "taml" => &[Taml], + "telu" => &[Telu], + "thaa" => &[Thaa], + "thai" => &[Thai], + "tibt" => &[Tibt], + _ => return None, + }) +} + +/// A language tag's language, script and region subtags, lowercase. +struct Subtags { + language: String, + script: Option, + region: Option, +} + +impl Subtags { + /// `zh-Hant-HK`, `zh_TW.UTF-8` or `sr_RS@latin` (glibc's script modifier). + fn parse(tag: &str) -> Subtags { + let tag = tag.trim().to_ascii_lowercase(); + let base = tag.split(['.', '@']).next().unwrap_or("").replace('_', "-"); + let mut parts = base.split('-').filter(|p| !p.is_empty()); + let language = parts.next().unwrap_or("").to_owned(); + let mut script = match tag.split_once('@').map(|(_, m)| m) { + Some("latin") => Some("latn".to_owned()), + Some("cyrillic") => Some("cyrl".to_owned()), + _ => None, + }; + let mut region = None; + for part in parts { + if part.len() == 4 && part.bytes().all(|b| b.is_ascii_alphabetic()) { + script.get_or_insert_with(|| part.to_owned()); + } else if super::is_region(part) { + region = Some(part.to_owned()); + break; + } + } + Subtags { language, script, region } + } +} + +#[cfg(test)] +mod tests { + use super::Script::*; + use super::*; + + /// Every locale the craft-i18n catalogs or an app ship, as the apps spell them. + #[test] + fn shipped_locales() { + for (tag, want, han_forms) in [ + ("en", &[Latn][..], None), + ("ar", &[Arab], None), + ("bg", &[Cyrl], None), + ("cs", &[Latn], None), + ("da", &[Latn], None), + ("de", &[Latn], None), + ("el", &[Grek], None), + ("es", &[Latn], None), + ("es-419", &[Latn], None), + ("et", &[Latn], None), + ("fi", &[Latn], None), + ("fr", &[Latn], None), + ("hr", &[Latn], None), + ("hu", &[Latn], None), + ("id", &[Latn], None), + ("it", &[Latn], None), + ("ja", &[Hani, Hira, Kana], Some(Han::Japanese)), + ("ka", &[Geor], None), + ("kk", &[Cyrl], None), + ("ko", &[Hang, Hani], Some(Han::Korean)), + ("lt", &[Latn], None), + ("nl", &[Latn], None), + ("pl", &[Latn], None), + ("pt", &[Latn], None), + ("pt-br", &[Latn], None), + ("pt_BR", &[Latn], None), + ("ru", &[Cyrl], None), + ("sl", &[Latn], None), + ("sr", &[Cyrl], None), + ("sr-latn", &[Latn], None), + ("sv", &[Latn], None), + ("te", &[Telu], None), + ("tr", &[Latn], None), + ("uk", &[Cyrl], None), + ("vi", &[Latn], None), + ("zh", &[Hani], Some(Han::Simplified)), + ("zh-cn", &[Hani], Some(Han::Simplified)), + ("zh-CN", &[Hani], Some(Han::Simplified)), + ("zh-hans", &[Hani], Some(Han::Simplified)), + ("zh-Hans", &[Hani], Some(Han::Simplified)), + ("zh-hant", &[Hani], Some(Han::Traditional)), + ] { + assert_eq!(scripts(tag), want, "{tag}"); + assert_eq!(han(tag), han_forms, "{tag}"); + let rtl = tag == "ar"; + assert_eq!(direction(tag), if rtl { Direction::Rtl } else { Direction::Ltr }, "{tag}"); + } + } + + #[test] + fn regional_han() { + for (tag, want) in [ + ("zh-TW", Han::Traditional), + ("zh_TW.UTF-8", Han::Traditional), + ("zh-Hant-TW", Han::Traditional), + ("zh-HK", Han::HongKong), + ("zh-MO", Han::HongKong), + ("zh-Hant-HK", Han::HongKong), + ("zh-hant-hk", Han::HongKong), + ("zh-Hans-HK", Han::Simplified), + ("zh-SG", Han::Simplified), + ("zh-Hans-SG", Han::Simplified), + ("yue", Han::HongKong), + ("yue-Hans", Han::Simplified), + ("ja-JP", Han::Japanese), + ("ko_KR.UTF-8", Han::Korean), + ] { + assert_eq!(han(tag), Some(want), "{tag}"); + } + assert_eq!(han("en"), None); + assert_eq!(han(""), None); + } + + #[test] + fn script_subtag_wins() { + for (tag, want) in [ + ("sr-Latn-RS", &[Latn][..]), + ("sr_RS@latin", &[Latn]), + ("sr-Cyrl", &[Cyrl]), + ("sr_RS.UTF-8", &[Cyrl]), + ("az-Cyrl", &[Cyrl]), + ("pa-Arab", &[Arab]), + ("pa", &[Guru]), + ("mn-Mong", &[Mong]), + ("sd-Deva", &[Deva]), + ("fr-Zyyy", &[Latn]), + ] { + assert_eq!(scripts(tag), want, "{tag}"); + } + assert_eq!(direction("pa-Arab"), Direction::Rtl); + assert_eq!(direction("sd-Deva"), Direction::Ltr); + assert_eq!(direction("he_IL.UTF-8"), Direction::Rtl); + } + + /// The usual scripts tag matching relies on agree with the table. + #[test] + fn agrees_with_usual_script() { + for language in ["be", "mn", "sr", "zh", "az", "bs", "ha", "uz", "ks", "sd", "pa"] { + let usual = crate::usual_script(language).and_then(subtag_scripts); + assert_eq!(usual, Some(scripts(language)), "{language}"); + } + } +} From e896187c6c3a88c759035edaf4e6900f358bdc20 Mon Sep 17 00:00:00 2001 From: aiongg Date: Wed, 21 Oct 2026 04:40:00 +0000 Subject: [PATCH 2/9] i18n: isolate inserted values in fmt and trn MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fmt wraps each value in FSI…PDI and trn wraps {n} in LRI…PDI, so a value's direction can't reorder the message around it. ltr() isolates numbers and paths left to right; a value that already is one isolate goes in unchanged, and a stray PDI can't close the isolate early. fmt_plain and strip_isolates are for text that leaves the interface. --- crates/i18n/README.md | 7 ++ crates/i18n/src/lib.rs | 142 +++++++++++++++++++++++++++++++++++------ 2 files changed, 131 insertions(+), 18 deletions(-) diff --git a/crates/i18n/README.md b/crates/i18n/README.md index df337971..a34ce183 100644 --- a/crates/i18n/README.md +++ b/crates/i18n/README.md @@ -49,6 +49,13 @@ The app keeps its `tl!` macro, its language setting (`"auto"` → `I18N.system_l (one context along the locale chain, so `pt-br` can return a `pt` row, but never the plain row or English: a key cap's name under `key`, or "is there a translation?"), and `fmt` for `{name}` placeholders, filled in one pass. +- **Isolation:** `fmt` wraps each inserted value in a first-strong isolate (FSI…PDI), and `trn` + wraps `{n}` in a left-to-right one (LRI…PDI), so a Latin file name in an Arabic sentence, or a + Hebrew name in an English one, can't reorder the words around it. Wrap numbers, paths and codes + in `ltr` before passing them; a value that already is one isolate goes in as it is, and a stray + PDI in a value can't close its isolate early. egui draws the marks as nothing. For text that + leaves the interface (file names, logs, terminals, text parsed later), use `fmt_plain` or + `strip_isolates`. - **Bad rows** (malformed, an unknown `@` context, a wrong plural form count, a row defined twice, of which the first is kept) and bad locales (`en`, `pseudo`, a code embedded twice in any case) are skipped and returned by `I18n::new` for the app to log; craft-i18n's CI rejects them before diff --git a/crates/i18n/src/lib.rs b/crates/i18n/src/lib.rs index 7fa7a4ef..a394aef5 100644 --- a/crates/i18n/src/lib.rs +++ b/crates/i18n/src/lib.rs @@ -158,8 +158,9 @@ impl I18n { self.find(lang, &[("@id", id), ("", label)]).unwrap_or(label) } - /// A count's wording in `lang`, `{n}` filled in. `one` and `other` are the English forms; - /// they are also the key of the `@plural` row, and the fallback. + /// A count's wording in `lang`, `{n}` filled in as a left-to-right isolate ([`ltr`]). `one` + /// and `other` are the English forms; they are also the key of the `@plural` row, and the + /// fallback. pub fn trn(&self, lang: Lang, n: u64, one: &str, other: &str) -> String { let key = format!("{one}|{other}"); let chain = self.locale(lang).map_or(&[][..], |l| l.chain.as_slice()); @@ -175,7 +176,7 @@ impl I18n { forms.split('|').nth(index) }) .unwrap_or(if n == 1 { one } else { other }); - let text = fmt(text, &[("n", &n.to_string())]); + let text = fmt(text, &[("n", <r(&n.to_string()))]); if lang == Lang::PSEUDO { return marked(&text); } @@ -261,9 +262,88 @@ impl I18n { } } -/// Fill `{name}` placeholders from `args` in one pass. Unknown placeholders stay as written, and -/// inserted values are never read as templates, so a file name containing `{n}` stays intact. +/// Fill `{name}` placeholders from `args` in one pass, each value wrapped in a first-strong +/// isolate (FSI…PDI), so its direction can't reorder the sentence around it: a Latin file name +/// in an Arabic message stays where the translator put it. A value that already is one isolate +/// goes in as it is; pass numbers, paths and codes through [`ltr`]. Unknown placeholders stay as +/// written, and inserted values are never read as templates, so a file name containing `{n}` +/// stays intact. pub fn fmt(template: &str, args: &[(&str, &str)]) -> String { + fill(template, args, true) +} + +/// [`fmt`] without isolates, for text that goes where the marks would get in the way: file +/// names, logs, terminals, text compared or parsed later. +pub fn fmt_plain(template: &str, args: &[(&str, &str)]) -> String { + fill(template, args, false) +} + +const LRI: char = '\u{2066}'; +const RLI: char = '\u{2067}'; +const FSI: char = '\u{2068}'; +const PDI: char = '\u{2069}'; + +/// `value` as a first-strong isolate (FSI…PDI): it takes the direction of its first strong +/// letter and can't reorder the text around it. What [`fmt`] does to each value. +pub fn isolate(value: &str) -> String { + wrap(FSI, value) +} + +/// `value` as a left-to-right isolate (LRI…PDI), for numbers, paths, file sizes, timecodes and +/// codes, which read left to right in every language. In an RTL first-strong isolate, the +/// segments of `/home/مستخدم/file.txt` would run right to left. +pub fn ltr(value: &str) -> String { + wrap(LRI, value) +} + +/// `text` without isolate marks (LRI, RLI, FSI, PDI): a [`fmt`] or [`I18n::trn`] result made fit +/// for a file name, a log or a comparison. +pub fn strip_isolates(text: &str) -> String { + text.chars().filter(|c| !matches!(*c, LRI | RLI | FSI | PDI)).collect() +} + +/// `value` between `initiator` and PDI. A stray PDI in the value would close the isolate early, so +/// it is dropped; isolates the value leaves open are closed. +fn wrap(initiator: char, value: &str) -> String { + let mut out = String::with_capacity(value.len() + 6); + out.push(initiator); + let mut open = 0_usize; + for c in value.chars() { + match c { + LRI | RLI | FSI => open += 1, + PDI if open == 0 => continue, + PDI => open -= 1, + _ => {} + } + out.push(c); + } + for _ in 0..=open { + out.push(PDI); + } + out +} + +/// Whether `value` is exactly one isolate: an initiator, then the PDI that closes it last. +fn is_one_isolate(value: &str) -> bool { + let mut depth = 0_usize; + let mut chars = value.chars().peekable(); + while let Some(c) = chars.next() { + match c { + LRI | RLI | FSI => depth += 1, + PDI if depth > 0 => { + depth -= 1; + if depth == 0 { + return chars.peek().is_none(); + } + } + _ if depth == 0 => return false, + _ => {} + } + } + false +} + +fn fill(template: &str, args: &[(&str, &str)], isolated: bool) -> String { let mut out = String::with_capacity(template.len()); let mut rest = template; while let Some((before, after_open)) = rest.split_once('{') { @@ -274,7 +354,8 @@ pub fn fmt(template: &str, args: &[(&str, &str)]) -> String { break; }; match args.iter().find(|(k, _)| *k == name) { - Some((_, value)) => out.push_str(value), + Some((_, value)) if !isolated || value.is_empty() || is_one_isolate(value) => out.push_str(value), + Some((_, value)) => out.push_str(&isolate(value)), None => { out.push('{'); out.push_str(name); @@ -485,17 +566,42 @@ mod tests { #[test] fn fmt_fills_once() { - assert_eq!(fmt("{a} to {b}", &[("a", "x"), ("b", "{a}")]), "x to {a}"); + assert_eq!(fmt_plain("{a} to {b}", &[("a", "x"), ("b", "{a}")]), "x to {a}"); assert_eq!(fmt("{z} {", &[]), "{z} {"); } + #[test] + fn inserted_values_are_isolated() { + let saved = fmt("تم حفظ {name} في {dir}.", &[("name", "Report 2.pdf"), ("dir", <r("/home/سارة/docs"))]); + assert_eq!(saved, "تم حفظ \u{2068}Report 2.pdf\u{2069} في \u{2066}/home/سارة/docs\u{2069}."); + assert_eq!(strip_isolates(&saved), "تم حفظ Report 2.pdf في /home/سارة/docs."); + assert_eq!(fmt_plain("{a}", &[("a", "x")]), "x"); + assert_eq!(fmt("[{a}]", &[("a", "")]), "[]"); + } + + #[test] + fn a_value_cannot_escape_its_isolate() { + assert_eq!(isolate("a\u{2069}b"), "\u{2068}ab\u{2069}"); + assert_eq!(isolate("a\u{2067}b"), "\u{2068}a\u{2067}b\u{2069}\u{2069}"); + let one = ltr("12"); + assert_eq!(fmt("{n}", &[("n", &one)]), one); + // Two isolates side by side are not one, so they get wrapped. + assert_eq!(fmt("{v}", &[("v", "\u{2066}a\u{2069}\u{2066}b\u{2069}")]), "\u{2068}\u{2066}a\u{2069}\u{2066}b\u{2069}\u{2069}"); + } + + #[test] + fn counts_are_left_to_right_isolates() { + let i = all(P).0; + assert_eq!(i.trn(lang(&i, "fr"), 2, "{n} page", "{n} pages"), "\u{2066}2\u{2069} pages"); + } + #[test] fn one_other() { let (i, problems) = all(P); assert!(problems.is_empty(), "{problems:?}"); let fr = lang(&i, "fr"); - assert_eq!(i.trn(fr, 0, "{n} page", "{n} pages"), "0 page"); - assert_eq!(i.trn(fr, 2, "{n} page", "{n} pages"), "2 pages"); + assert_eq!(strip_isolates(&i.trn(fr, 0, "{n} page", "{n} pages")), "0 page"); + assert_eq!(strip_isolates(&i.trn(fr, 2, "{n} page", "{n} pages")), "2 pages"); } #[test] @@ -503,27 +609,27 @@ mod tests { let i = all(P).0; let ru = lang(&i, "ru"); for (n, want) in [(1, "1 файл"), (3, "3 файла"), (5, "5 файлов"), (11, "11 файлов"), (21, "21 файл")] { - assert_eq!(i.trn(ru, n, "{n} file", "{n} files"), want); + assert_eq!(strip_isolates(&i.trn(ru, n, "{n} file", "{n} files")), want); } } #[test] fn one_form_ja() { let i = all(P).0; - assert_eq!(i.trn(lang(&i, "ja"), 7, "{n} file", "{n} files"), "7 個"); + assert_eq!(strip_isolates(&i.trn(lang(&i, "ja"), 7, "{n} file", "{n} files")), "7 個"); } #[test] fn english_fallback() { let i = all(P).0; - assert_eq!(i.trn(Lang::EN, 1, "{n} file", "{n} files"), "1 file"); - assert_eq!(i.trn(lang(&i, "fr"), 2, "{n} x", "{n} xs"), "2 xs"); + assert_eq!(strip_isolates(&i.trn(Lang::EN, 1, "{n} file", "{n} files")), "1 file"); + assert_eq!(strip_isolates(&i.trn(lang(&i, "fr"), 2, "{n} x", "{n} xs")), "2 xs"); } #[test] fn beyond_sweep() { let i = all(P).0; - assert_eq!(i.trn(lang(&i, "fr"), 1_000_000, "{n} page", "{n} pages"), "1000000 pages"); + assert_eq!(strip_isolates(&i.trn(lang(&i, "fr"), 1_000_000, "{n} page", "{n} pages")), "1000000 pages"); } static L: &Embedded = @@ -590,7 +696,7 @@ mod tests { assert_eq!(i.t(Lang::PSEUDO, "A\nB"), "⟦A⟧\n⟦B⟧"); assert_eq!(i.tc(Lang::PSEUDO, "menu", "Open"), "⟦Open⟧"); assert_eq!(i.tid(Lang::PSEUDO, "file.save", "Save"), "⟦Save⟧"); - assert_eq!(i.trn(Lang::PSEUDO, 2, "{n} x", "{n} xs"), "⟦2 xs⟧"); + assert_eq!(strip_isolates(&i.trn(Lang::PSEUDO, 2, "{n} x", "{n} xs")), "⟦2 xs⟧"); assert!(!i.languages().contains(&Lang::PSEUDO)); assert_eq!(i.from_code("pseudo"), cfg!(debug_assertions).then_some(Lang::PSEUDO)); assert_eq!(i.get(Lang::PSEUDO, "key", "Esc"), Some("⟦Esc⟧")); @@ -669,9 +775,9 @@ mod tests { #[test] fn european_portuguese_zero_is_plural() { let i = all(PT).0; - assert_eq!(i.trn(lang(&i, "pt"), 0, "{n} file", "{n} files"), "0 ficheiros"); - assert_eq!(i.trn(lang(&i, "pt"), 1, "{n} file", "{n} files"), "1 ficheiro"); - assert_eq!(i.trn(lang(&i, "pt-br"), 0, "{n} file", "{n} files"), "0 arquivo"); + assert_eq!(strip_isolates(&i.trn(lang(&i, "pt"), 0, "{n} file", "{n} files")), "0 ficheiros"); + assert_eq!(strip_isolates(&i.trn(lang(&i, "pt"), 1, "{n} file", "{n} files")), "1 ficheiro"); + assert_eq!(strip_isolates(&i.trn(lang(&i, "pt-br"), 0, "{n} file", "{n} files")), "0 arquivo"); } /// The same counts as craft-i18n's checker (`plural_form_counts`), which rejects a catalog From 75b97a9192da103a0e0c6f782de307b42b7b9ec1 Mon Sep 17 00:00:00 2001 From: aiongg Date: Wed, 21 Oct 2026 05:42:51 +0000 Subject: [PATCH 3/9] i18n: non_exhaustive Script and Han; regions that imply a script pa-PK, uz-AF and az-IR are written in Arabic and sd-IN in Devanagari, as CLDR's likely subtags give them. A script subtag still wins. --- crates/i18n/README.md | 10 +++---- crates/i18n/src/scripts.rs | 59 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 61 insertions(+), 8 deletions(-) diff --git a/crates/i18n/README.md b/crates/i18n/README.md index cde3621d..ac6146b5 100644 --- a/crates/i18n/README.md +++ b/crates/i18n/README.md @@ -61,11 +61,11 @@ The app keeps its `tl!` macro, its language setting (`"auto"` → `I18N.system_l - **Names:** a language's name comes from its shared catalog's `@id language.name` row, else its code; English is "English". - **Scripts and direction:** for any language tag, `scripts` lists the ISO 15924 scripts its - interface text is written in (`ja` → Hani, Hira, Kana; `sr-latn` → Latn), `direction` is right - to left when the main script is (Arabic, Hebrew, Syriac, Thaana), and `han` gives the regional - forms of its Han characters for picking a CJK face: `zh-CN`/`zh-SG`/`zh-Hans` → SC, `zh-TW`/ - `zh-Hant` → TC, `zh-HK`/`zh-MO` → HK, `ja` → JP, `ko` → KR. `I18n::rtl` is `direction` of the - language's code; putting text into display order is the app's job. + interface text is written in (`ja` → Hani, Hira, Kana; `sr-latn` → Latn; `pa-PK` → Arab), + `direction` is right to left when the main script is (Arabic, Hebrew, Syriac, Thaana), and `han` + gives the regional forms of its Han characters for picking a CJK face: `zh-CN`/`zh-SG`/`zh-Hans` → + SC, `zh-TW`/`zh-Hant` → TC, `zh-HK`/`zh-MO` → HK, `ja` → JP, `ko` → KR. `I18n::rtl` is + `direction` of the language's code; putting text into display order is the app's job. - **System locale:** `system_lang` matches the system's list once and caches it: on desktop Unix `LANGUAGE`, `LC_ALL`, `LC_MESSAGES` and `LANG`, read raw; elsewhere `sys-locale`'s (macOS and Windows preferred languages, the browser's on the web). `fr_CA.UTF-8` → `fr`, `zh-TW` → diff --git a/crates/i18n/src/scripts.rs b/crates/i18n/src/scripts.rs index ecd2f8dd..037b3d40 100644 --- a/crates/i18n/src/scripts.rs +++ b/crates/i18n/src/scripts.rs @@ -3,6 +3,7 @@ /// A writing system, by its ISO 15924 code. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +#[non_exhaustive] pub enum Script { Arab, Armn, @@ -90,6 +91,7 @@ pub enum Direction { /// KR). The same code point is drawn differently in each, so a font stack picks its Han face by /// this, not by script. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +#[non_exhaustive] pub enum Han { /// Simplified Chinese, mainland China and Singapore (SC). Simplified, @@ -105,9 +107,10 @@ pub enum Han { /// The scripts a language tag's interface text is written in (`ja` → Hani, Hira, Kana; `sr` → /// Cyrl; `sr-latn` → Latn), the main one first. Any case and glibc form (`sr_RS.UTF-8@latin`). A -/// script subtag wins over the language's usual script. Latin product names and units appear in -/// every language, so Latn is listed only where it is the language's own script. A language not -/// in the table is taken to be written in Latin. +/// script subtag wins, then a region that implies a script (`pa-PK` → Arab), then the language's +/// usual script. Latin product names and units appear in every language, so Latn is listed only +/// where it is the language's own script. A language not in the table is taken to be written in +/// Latin. // ponytail: the languages a Craft app ships or could plausibly ship; add a row with the locale. pub fn scripts(tag: &str) -> &'static [Script] { use Script::*; @@ -115,6 +118,9 @@ pub fn scripts(tag: &str) -> &'static [Script] { if let Some(s) = tag.script.as_deref().and_then(subtag_scripts) { return s; } + if let Some(s) = tag.region.as_deref().and_then(|region| region_scripts(&tag.language, region)) { + return s; + } match tag.language.as_str() { "ja" => &[Hani, Hira, Kana], "ko" => &[Hang, Hani], @@ -178,6 +184,21 @@ pub fn han(tag: &str) -> Option { } } +/// The scripts a language takes in a region where CLDR's likely subtags give it a script other +/// than its usual one (`pa-PK` → `pa-Arab-PK`). Han regions are left to [`han`]. +// ponytail: mn-CN → Mong is left out until vertical Mongolian layout is supported. +fn region_scripts(language: &str, region: &str) -> Option<&'static [Script]> { + use Script::*; + Some(match (language, region) { + ("az", "iq" | "ir") | ("ha", "cm" | "sd") | ("kk", "af" | "cn" | "ir" | "mn") | ("ku", "iq" | "ir" | "lb") => &[Arab], + ("ky", "cn") | ("ms", "cc") | ("pa", "pk") | ("tg", "pk") | ("uz", "af") => &[Arab], + ("az", "ru") | ("ku", "am" | "az" | "ge" | "tm") | ("ug", "kz" | "mn") | ("uz", "cn") => &[Cyrl], + ("ky", "tr") | ("sr", "me" | "ro" | "tr") => &[Latn], + ("sd", "in") => &[Deva], + _ => return None, + }) +} + /// The scripts a script subtag stands for, including the CLDR unions (`jpan`, `kore`). `None` for /// one not in [`Script`], so the language's own scripts apply. fn subtag_scripts(subtag: &str) -> Option<&'static [Script]> { @@ -352,6 +373,38 @@ mod tests { assert_eq!(direction("he_IL.UTF-8"), Direction::Rtl); } + /// CLDR likely subtags: a region can switch a language to another script. + #[test] + fn region_implies_script() { + for (tag, want) in [ + ("pa-PK", &[Arab][..]), + ("pa_PK.UTF-8", &[Arab]), + ("pa-IN", &[Guru]), + ("uz-AF", &[Arab]), + ("uz-UZ", &[Latn]), + ("uz-CN", &[Cyrl]), + ("az-IR", &[Arab]), + ("az-AZ", &[Latn]), + ("sd-IN", &[Deva]), + ("sd-PK", &[Arab]), + ("ks-IN", &[Arab]), + ("ms-CC", &[Arab]), + ("ms-BN", &[Latn]), + ("sr-ME", &[Latn]), + ("kk-CN", &[Arab]), + ("ug-KZ", &[Cyrl]), + ("pa-Guru-PK", &[Guru]), + ("sd-Arab-IN", &[Arab]), + ] { + assert_eq!(scripts(tag), want, "{tag}"); + } + assert_eq!(direction("pa-PK"), Direction::Rtl); + assert_eq!(direction("uz-AF"), Direction::Rtl); + assert_eq!(direction("az-IR"), Direction::Rtl); + assert_eq!(direction("sd-IN"), Direction::Ltr); + assert_eq!(direction("ug-KZ"), Direction::Ltr); + } + /// The usual scripts tag matching relies on agree with the table. #[test] fn agrees_with_usual_script() { From 705e91e183ab3b63bc5d3aacf25ae6ab7e9d0cc6 Mon Sep 17 00:00:00 2001 From: aiongg Date: Wed, 21 Oct 2026 05:44:27 +0000 Subject: [PATCH 4/9] i18n: isolate only text with a right-to-left letter fmt and trn add isolates only when the template or an inserted value has a strong RTL character (bidi class R or AL). Text with none comes out as before, and a value's own isolate from ltr or isolate is taken off. --- Cargo.lock | 1 + crates/i18n/Cargo.toml | 1 + crates/i18n/README.md | 8 ++- crates/i18n/src/lib.rs | 124 ++++++++++++++++++++++++++--------------- 4 files changed, 85 insertions(+), 49 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4d9de428..ebc4055f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -659,6 +659,7 @@ dependencies = [ "intl_pluralrules", "sys-locale", "unic-langid", + "unicode-bidi", ] [[package]] diff --git a/crates/i18n/Cargo.toml b/crates/i18n/Cargo.toml index 45e21775..8283943f 100644 --- a/crates/i18n/Cargo.toml +++ b/crates/i18n/Cargo.toml @@ -11,6 +11,7 @@ repository.workspace = true intl_pluralrules = "=7.0.2" unic-langid = "0.9" sys-locale = "0.3" +unicode-bidi = "0.3" [target.'cfg(target_arch = "wasm32")'.dependencies] sys-locale = { version = "0.3", features = ["js"] } diff --git a/crates/i18n/README.md b/crates/i18n/README.md index a34ce183..951a6782 100644 --- a/crates/i18n/README.md +++ b/crates/i18n/README.md @@ -49,12 +49,14 @@ The app keeps its `tl!` macro, its language setting (`"auto"` → `I18N.system_l (one context along the locale chain, so `pt-br` can return a `pt` row, but never the plain row or English: a key cap's name under `key`, or "is there a translation?"), and `fmt` for `{name}` placeholders, filled in one pass. -- **Isolation:** `fmt` wraps each inserted value in a first-strong isolate (FSI…PDI), and `trn` +- **Isolation:** when a message or a value inserted into it has a right-to-left letter (bidi + class R or AL), `fmt` wraps each inserted value in a first-strong isolate (FSI…PDI), and `trn` wraps `{n}` in a left-to-right one (LRI…PDI), so a Latin file name in an Arabic sentence, or a Hebrew name in an English one, can't reorder the words around it. Wrap numbers, paths and codes in `ltr` before passing them; a value that already is one isolate goes in as it is, and a stray - PDI in a value can't close its isolate early. egui draws the marks as nothing. For text that - leaves the interface (file names, logs, terminals, text parsed later), use `fmt_plain` or + PDI in a value can't close its isolate early. Text with no right-to-left letter comes out + without marks, as if `fmt` didn't isolate at all. egui draws the marks as nothing. For text + that leaves the interface (file names, logs, terminals, text parsed later), use `fmt_plain` or `strip_isolates`. - **Bad rows** (malformed, an unknown `@` context, a wrong plural form count, a row defined twice, of which the first is kept) and bad locales (`en`, `pseudo`, a code embedded twice in any case) diff --git a/crates/i18n/src/lib.rs b/crates/i18n/src/lib.rs index a394aef5..8fde98ad 100644 --- a/crates/i18n/src/lib.rs +++ b/crates/i18n/src/lib.rs @@ -12,6 +12,7 @@ use std::sync::{Mutex, OnceLock, PoisonError}; use intl_pluralrules::{PluralCategory, PluralRuleType, PluralRules}; use unic_langid::LanguageIdentifier; +use unicode_bidi::{BidiClass, bidi_class}; /// The catalogs an app embeds, one entry per locale: (locale, shared catalog, the app's catalog). /// A missing catalog is `""`. @@ -158,9 +159,9 @@ impl I18n { self.find(lang, &[("@id", id), ("", label)]).unwrap_or(label) } - /// A count's wording in `lang`, `{n}` filled in as a left-to-right isolate ([`ltr`]). `one` - /// and `other` are the English forms; they are also the key of the `@plural` row, and the - /// fallback. + /// A count's wording in `lang`, `{n}` filled in, as a left-to-right isolate ([`ltr`]) when the + /// wording has a right-to-left letter. `one` and `other` are the English forms; they are also + /// the key of the `@plural` row, and the fallback. pub fn trn(&self, lang: Lang, n: u64, one: &str, other: &str) -> String { let key = format!("{one}|{other}"); let chain = self.locale(lang).map_or(&[][..], |l| l.chain.as_slice()); @@ -262,18 +263,20 @@ impl I18n { } } -/// Fill `{name}` placeholders from `args` in one pass, each value wrapped in a first-strong -/// isolate (FSI…PDI), so its direction can't reorder the sentence around it: a Latin file name -/// in an Arabic message stays where the translator put it. A value that already is one isolate -/// goes in as it is; pass numbers, paths and codes through [`ltr`]. Unknown placeholders stay as -/// written, and inserted values are never read as templates, so a file name containing `{n}` -/// stays intact. +/// Fill `{name}` placeholders from `args` in one pass. When the template or an inserted value +/// has a right-to-left letter (Hebrew, Arabic and the like), each value goes in as a first-strong +/// isolate (FSI…PDI), so its direction can't reorder the sentence around it: a Latin file name in +/// an Arabic message stays where the translator put it. A value that already is one isolate goes +/// in as it is; pass numbers, paths and codes through [`ltr`]. Text with no right-to-left letter +/// gets no marks, and a value's own isolate is taken off. Unknown placeholders stay as written, +/// and inserted values are never read as templates, so a file name containing `{n}` stays intact. pub fn fmt(template: &str, args: &[(&str, &str)]) -> String { fill(template, args, true) } /// [`fmt`] without isolates, for text that goes where the marks would get in the way: file -/// names, logs, terminals, text compared or parsed later. +/// names, logs, terminals, text compared or parsed later. A value that is one isolate goes in +/// without it. pub fn fmt_plain(template: &str, args: &[(&str, &str)]) -> String { fill(template, args, false) } @@ -284,14 +287,16 @@ const FSI: char = '\u{2068}'; const PDI: char = '\u{2069}'; /// `value` as a first-strong isolate (FSI…PDI): it takes the direction of its first strong -/// letter and can't reorder the text around it. What [`fmt`] does to each value. +/// letter and can't reorder the text around it. What [`fmt`] does to each value in text with a +/// right-to-left letter. pub fn isolate(value: &str) -> String { wrap(FSI, value) } /// `value` as a left-to-right isolate (LRI…PDI), for numbers, paths, file sizes, timecodes and /// codes, which read left to right in every language. In an RTL first-strong isolate, the -/// segments of `/home/مستخدم/file.txt` would run right to left. +/// segments of `/home/مستخدم/file.txt` would run right to left. [`fmt`] takes the isolate off +/// again in text with no right-to-left letter. pub fn ltr(value: &str) -> String { wrap(LRI, value) } @@ -323,6 +328,22 @@ fn wrap(initiator: char, value: &str) -> String { out } +/// Whether `text` has a strong right-to-left character (bidi class R or AL): the text that needs +/// isolates. +fn has_rtl(text: &str) -> bool { + !text.is_ascii() && text.chars().any(|c| matches!(bidi_class(c), BidiClass::R | BidiClass::AL)) +} + +/// `value` without its isolate, if it is exactly one. +fn unwrap_isolate(value: &str) -> &str { + if is_one_isolate(value) { + // The initiator and PDI are three bytes each in UTF-8. + value.get(3..value.len() - 3).unwrap_or(value) + } else { + value + } +} + /// Whether `value` is exactly one isolate: an initiator, then the PDI that closes it last. fn is_one_isolate(value: &str) -> bool { let mut depth = 0_usize; @@ -344,27 +365,32 @@ fn is_one_isolate(value: &str) -> bool { } fn fill(template: &str, args: &[(&str, &str)], isolated: bool) -> String { - let mut out = String::with_capacity(template.len()); + let mut pieces: Vec<(&str, bool)> = Vec::new(); let mut rest = template; while let Some((before, after_open)) = rest.split_once('{') { - out.push_str(before); + pieces.push((before, false)); let Some((name, after_close)) = after_open.split_once('}') else { - out.push('{'); + pieces.push(("{", false)); rest = after_open; break; }; match args.iter().find(|(k, _)| *k == name) { - Some((_, value)) if !isolated || value.is_empty() || is_one_isolate(value) => out.push_str(value), - Some((_, value)) => out.push_str(&isolate(value)), - None => { - out.push('{'); - out.push_str(name); - out.push('}'); - } + Some((_, value)) => pieces.push((value, true)), + None => pieces.extend([("{", false), (name, false), ("}", false)]), } rest = after_close; } - out.push_str(rest); + pieces.push((rest, false)); + let isolated = isolated && pieces.iter().any(|(text, _)| has_rtl(text)); + let mut out = String::with_capacity(template.len()); + for (text, value) in pieces { + match (value, isolated) { + (false, _) => out.push_str(text), + (true, false) => out.push_str(unwrap_isolate(text)), + (true, true) if text.is_empty() || is_one_isolate(text) => out.push_str(text), + (true, true) => out.push_str(&isolate(text)), + } + } out } @@ -562,21 +588,33 @@ mod tests { ("fr", "@plural\t{n} page|{n} pages\t{n} page|{n} pages\n", ""), ("ru", "@plural\t{n} file|{n} files\t{n} файл|{n} файла|{n} файлов\n", ""), ("ja", "@plural\t{n} file|{n} files\t{n} 個\n", ""), + ("fa", "@plural\t{n} page|{n} pages\t{n} صفحه|{n} صفحه\n", ""), ]; #[test] fn fmt_fills_once() { - assert_eq!(fmt_plain("{a} to {b}", &[("a", "x"), ("b", "{a}")]), "x to {a}"); + assert_eq!(fmt("{a} to {b}", &[("a", "x"), ("b", "{a}")]), "x to {a}"); assert_eq!(fmt("{z} {", &[]), "{z} {"); } #[test] - fn inserted_values_are_isolated() { + fn left_to_right_text_gets_no_marks() { + assert_eq!(fmt("Saved {name} in {dir}.", &[("name", "Report 2.pdf"), ("dir", <r("/home/sara"))]), "Saved Report 2.pdf in /home/sara."); + assert_eq!(fmt("{a}", &[("a", "x"), ("b", "שלום")]), "x", "an unused value doesn't count"); + assert_eq!(fmt_plain("{a}", &[("a", &isolate("x"))]), "x"); + let i = all(P).0; + assert_eq!(i.trn(Lang::EN, 2, "{n} page", "{n} pages"), "2 pages"); + } + + #[test] + fn right_to_left_text_isolates_values() { let saved = fmt("تم حفظ {name} في {dir}.", &[("name", "Report 2.pdf"), ("dir", <r("/home/سارة/docs"))]); assert_eq!(saved, "تم حفظ \u{2068}Report 2.pdf\u{2069} في \u{2066}/home/سارة/docs\u{2069}."); assert_eq!(strip_isolates(&saved), "تم حفظ Report 2.pdf في /home/سارة/docs."); - assert_eq!(fmt_plain("{a}", &[("a", "x")]), "x"); - assert_eq!(fmt("[{a}]", &[("a", "")]), "[]"); + assert_eq!(fmt("Opened {name}.", &[("name", "שלום.txt")]), "Opened \u{2068}שלום.txt\u{2069}."); + assert_eq!(fmt("[{a}] {b}", &[("a", ""), ("b", "ש")]), "[] \u{2068}ש\u{2069}"); + let i = all(P).0; + assert_eq!(i.trn(lang(&i, "fa"), 3, "{n} page", "{n} pages"), "\u{2066}3\u{2069} صفحه"); } #[test] @@ -584,15 +622,9 @@ mod tests { assert_eq!(isolate("a\u{2069}b"), "\u{2068}ab\u{2069}"); assert_eq!(isolate("a\u{2067}b"), "\u{2068}a\u{2067}b\u{2069}\u{2069}"); let one = ltr("12"); - assert_eq!(fmt("{n}", &[("n", &one)]), one); + assert_eq!(fmt("ש {n}", &[("n", &one)]), format!("ש {one}")); // Two isolates side by side are not one, so they get wrapped. - assert_eq!(fmt("{v}", &[("v", "\u{2066}a\u{2069}\u{2066}b\u{2069}")]), "\u{2068}\u{2066}a\u{2069}\u{2066}b\u{2069}\u{2069}"); - } - - #[test] - fn counts_are_left_to_right_isolates() { - let i = all(P).0; - assert_eq!(i.trn(lang(&i, "fr"), 2, "{n} page", "{n} pages"), "\u{2066}2\u{2069} pages"); + assert_eq!(fmt("ש{v}", &[("v", "\u{2066}a\u{2069}\u{2066}b\u{2069}")]), "ש\u{2068}\u{2066}a\u{2069}\u{2066}b\u{2069}\u{2069}"); } #[test] @@ -600,8 +632,8 @@ mod tests { let (i, problems) = all(P); assert!(problems.is_empty(), "{problems:?}"); let fr = lang(&i, "fr"); - assert_eq!(strip_isolates(&i.trn(fr, 0, "{n} page", "{n} pages")), "0 page"); - assert_eq!(strip_isolates(&i.trn(fr, 2, "{n} page", "{n} pages")), "2 pages"); + assert_eq!(i.trn(fr, 0, "{n} page", "{n} pages"), "0 page"); + assert_eq!(i.trn(fr, 2, "{n} page", "{n} pages"), "2 pages"); } #[test] @@ -609,27 +641,27 @@ mod tests { let i = all(P).0; let ru = lang(&i, "ru"); for (n, want) in [(1, "1 файл"), (3, "3 файла"), (5, "5 файлов"), (11, "11 файлов"), (21, "21 файл")] { - assert_eq!(strip_isolates(&i.trn(ru, n, "{n} file", "{n} files")), want); + assert_eq!(i.trn(ru, n, "{n} file", "{n} files"), want); } } #[test] fn one_form_ja() { let i = all(P).0; - assert_eq!(strip_isolates(&i.trn(lang(&i, "ja"), 7, "{n} file", "{n} files")), "7 個"); + assert_eq!(i.trn(lang(&i, "ja"), 7, "{n} file", "{n} files"), "7 個"); } #[test] fn english_fallback() { let i = all(P).0; - assert_eq!(strip_isolates(&i.trn(Lang::EN, 1, "{n} file", "{n} files")), "1 file"); - assert_eq!(strip_isolates(&i.trn(lang(&i, "fr"), 2, "{n} x", "{n} xs")), "2 xs"); + assert_eq!(i.trn(Lang::EN, 1, "{n} file", "{n} files"), "1 file"); + assert_eq!(i.trn(lang(&i, "fr"), 2, "{n} x", "{n} xs"), "2 xs"); } #[test] fn beyond_sweep() { let i = all(P).0; - assert_eq!(strip_isolates(&i.trn(lang(&i, "fr"), 1_000_000, "{n} page", "{n} pages")), "1000000 pages"); + assert_eq!(i.trn(lang(&i, "fr"), 1_000_000, "{n} page", "{n} pages"), "1000000 pages"); } static L: &Embedded = @@ -696,7 +728,7 @@ mod tests { assert_eq!(i.t(Lang::PSEUDO, "A\nB"), "⟦A⟧\n⟦B⟧"); assert_eq!(i.tc(Lang::PSEUDO, "menu", "Open"), "⟦Open⟧"); assert_eq!(i.tid(Lang::PSEUDO, "file.save", "Save"), "⟦Save⟧"); - assert_eq!(strip_isolates(&i.trn(Lang::PSEUDO, 2, "{n} x", "{n} xs")), "⟦2 xs⟧"); + assert_eq!(i.trn(Lang::PSEUDO, 2, "{n} x", "{n} xs"), "⟦2 xs⟧"); assert!(!i.languages().contains(&Lang::PSEUDO)); assert_eq!(i.from_code("pseudo"), cfg!(debug_assertions).then_some(Lang::PSEUDO)); assert_eq!(i.get(Lang::PSEUDO, "key", "Esc"), Some("⟦Esc⟧")); @@ -775,9 +807,9 @@ mod tests { #[test] fn european_portuguese_zero_is_plural() { let i = all(PT).0; - assert_eq!(strip_isolates(&i.trn(lang(&i, "pt"), 0, "{n} file", "{n} files")), "0 ficheiros"); - assert_eq!(strip_isolates(&i.trn(lang(&i, "pt"), 1, "{n} file", "{n} files")), "1 ficheiro"); - assert_eq!(strip_isolates(&i.trn(lang(&i, "pt-br"), 0, "{n} file", "{n} files")), "0 arquivo"); + assert_eq!(i.trn(lang(&i, "pt"), 0, "{n} file", "{n} files"), "0 ficheiros"); + assert_eq!(i.trn(lang(&i, "pt"), 1, "{n} file", "{n} files"), "1 ficheiro"); + assert_eq!(i.trn(lang(&i, "pt-br"), 0, "{n} file", "{n} files"), "0 arquivo"); } /// The same counts as craft-i18n's checker (`plural_form_counts`), which rejects a catalog From 502f7997167d3602920b63ac73be9f89c409ab17 Mon Sep 17 00:00:00 2001 From: aiongg Date: Wed, 21 Oct 2026 05:54:12 +0000 Subject: [PATCH 5/9] i18n: han and catalog matching follow region-implied scripts han(zh-US), zh-AU and the other regions CLDR writes in Traditional characters give Traditional, and pa-PK picks a pa-arab catalog, never Gurmukhi pa, as pa-Arab-PK does. --- crates/i18n/README.md | 10 ++++----- crates/i18n/src/lib.rs | 30 +++++++++++++++++++++++-- crates/i18n/src/scripts.rs | 46 +++++++++++++++++++++----------------- 3 files changed, 59 insertions(+), 27 deletions(-) diff --git a/crates/i18n/README.md b/crates/i18n/README.md index ac6146b5..4bf29ace 100644 --- a/crates/i18n/README.md +++ b/crates/i18n/README.md @@ -64,14 +64,14 @@ The app keeps its `tl!` macro, its language setting (`"auto"` → `I18N.system_l interface text is written in (`ja` → Hani, Hira, Kana; `sr-latn` → Latn; `pa-PK` → Arab), `direction` is right to left when the main script is (Arabic, Hebrew, Syriac, Thaana), and `han` gives the regional forms of its Han characters for picking a CJK face: `zh-CN`/`zh-SG`/`zh-Hans` → - SC, `zh-TW`/`zh-Hant` → TC, `zh-HK`/`zh-MO` → HK, `ja` → JP, `ko` → KR. `I18n::rtl` is + SC, `zh-TW`/`zh-US`/`zh-Hant` → TC, `zh-HK`/`zh-MO` → HK, `ja` → JP, `ko` → KR. `I18n::rtl` is `direction` of the language's code; putting text into display order is the app's job. - **System locale:** `system_lang` matches the system's list once and caches it: on desktop Unix `LANGUAGE`, `LC_ALL`, `LC_MESSAGES` and `LANG`, read raw; elsewhere `sys-locale`'s (macOS and - Windows preferred languages, the browser's on the web). `fr_CA.UTF-8` → `fr`, `zh-TW` → - `zh-hant`, `pt-PT` → `pt-br` when that is the only Portuguese, `C`/`POSIX` → English. A script - is never crossed: `sr_RS@latin` and `sr-Latn-RS` pick `sr-latn`, never `sr`, while `sr-Cyrl-RS` - picks `sr`. + Windows preferred languages, the browser's on the web). `fr_CA.UTF-8` → `fr`, `zh-TW` → `zh-hant`, + `pt-PT` → `pt-br` when that is the only Portuguese, `C`/`POSIX` → English. A script is never + crossed: `sr_RS@latin` and `sr-Latn-RS` pick `sr-latn`, never `sr`, while `sr-Cyrl-RS` picks `sr`. + A region that implies a script counts as that script: `pa-PK` picks `pa-arab`, not Gurmukhi `pa`. - **Pseudo locale** (`pseudo`, debug builds only, never in `languages()`): every lookup, `get` included, comes back as `⟦…⟧`, line by line, so text drawn without the marks never went through translation. `looked_up()` lists the source texts looked up so far, for an audit against a diff --git a/crates/i18n/src/lib.rs b/crates/i18n/src/lib.rs index 84295a16..6c841d1e 100644 --- a/crates/i18n/src/lib.rs +++ b/crates/i18n/src/lib.rs @@ -292,7 +292,9 @@ pub fn fmt(template: &str, args: &[(&str, &str)]) -> String { /// Chinese without a script picks one by region; Portuguese falls back to `pt-br` last, when /// there is no `pt`. A glibc script modifier (`sr_RS@latin`) becomes the script subtag. With a /// script (`sr-Latn-RS`), a code without one is tried only when that is its language's usual -/// script, so Latin Serbian never picks Cyrillic `sr`, while `sr-Cyrl-RS` does. +/// script, so Latin Serbian never picks Cyrillic `sr`, while `sr-Cyrl-RS` does. A region that +/// implies a script (CLDR likely subtags) counts as that script after the tag as written: +/// `pa-PK` → `pa-pk`, `pa-arab-pk`, `pa-arab`. fn candidates(tag: &str) -> Vec { let tag = tag.trim().to_ascii_lowercase(); let base = tag.split(['.', '@']).next().unwrap_or("").replace('_', "-"); @@ -309,6 +311,16 @@ fn candidates(tag: &str) -> Vec { parts.insert(1, s); } let script = parts.get(1).copied().filter(|s| s.len() == 4 && s.bytes().all(|b| b.is_ascii_alphabetic())); + // Chinese picks its script by region below, keeping plain `zh` as the last resort. + if script.is_none() + && primary != "zh" + && let Some(implied) = parts.get(1).filter(|r| is_region(r)).and_then(|r| scripts::region_script(primary, r)) + { + let mut out = vec![parts.join("-")]; + parts.insert(1, implied); + out.extend(candidates(&parts.join("-"))); + return out; + } let mut out: Vec = (1..=parts.len()).rev().map(|n| parts.get(..n).unwrap_or_default().join("-")).collect(); if let Some(s) = script && usual_script(primary).is_some_and(|usual| usual != s) @@ -316,7 +328,8 @@ fn candidates(tag: &str) -> Vec { out.retain(|c| c.split('-').any(|p| p == s)); } if primary == "zh" && !parts.iter().any(|p| matches!(*p, "hans" | "hant")) { - let script = if parts.iter().any(|p| matches!(*p, "tw" | "hk" | "mo")) { "zh-hant" } else { "zh-hans" }; + let hant = parts.iter().skip(1).any(|p| scripts::region_script("zh", p) == Some("hant")); + let script = if hant { "zh-hant" } else { "zh-hans" }; out.insert(out.len().saturating_sub(1), script.to_owned()); } if primary == "pt" && !out.iter().any(|c| c == "pt-br") { @@ -565,6 +578,8 @@ mod tests { ("zh", Some("zh-hans")), ("zh-Hant-MO", Some("zh-hant")), ("zh_CN", Some("zh-hans")), + ("zh-US", Some("zh-hant")), + ("zh_SG", Some("zh-hans")), ("C", Some("en")), ("POSIX", Some("en")), ("en_US", Some("en")), @@ -576,6 +591,17 @@ mod tests { } } + #[test] + fn region_implies_script() { + assert_eq!(candidates("pa-PK"), ["pa-pk", "pa-arab-pk", "pa-arab"]); + assert_eq!(candidates("pa_PK.UTF-8"), ["pa-pk", "pa-arab-pk", "pa-arab"]); + assert_eq!(candidates("pa-IN"), ["pa-in", "pa"]); + assert_eq!(candidates("sd-IN"), ["sd-in", "sd-deva-in", "sd-deva"]); + assert_eq!(candidates("pa-Guru-PK"), ["pa-guru-pk", "pa-guru", "pa"]); + assert_eq!(candidates("zh-US"), ["zh-us", "zh-hant", "zh"]); + assert_eq!(candidates("zh-SG"), ["zh-sg", "zh-hans", "zh"]); + } + #[test] fn match_tags_takes_first_match() { let i = all(L).0; diff --git a/crates/i18n/src/scripts.rs b/crates/i18n/src/scripts.rs index 037b3d40..56a1ba1f 100644 --- a/crates/i18n/src/scripts.rs +++ b/crates/i18n/src/scripts.rs @@ -118,7 +118,7 @@ pub fn scripts(tag: &str) -> &'static [Script] { if let Some(s) = tag.script.as_deref().and_then(subtag_scripts) { return s; } - if let Some(s) = tag.region.as_deref().and_then(|region| region_scripts(&tag.language, region)) { + if let Some(s) = tag.region.as_deref().and_then(|region| region_script(&tag.language, region)).and_then(subtag_scripts) { return s; } match tag.language.as_str() { @@ -162,39 +162,38 @@ pub fn direction(tag: &str) -> Direction { } /// The forms a language tag's Han characters take: `zh-CN`, `zh-SG`, `zh-Hans` and plain `zh` → -/// Simplified; `zh-TW` and `zh-Hant` → Traditional; `zh-HK`, `zh-MO` and `zh-Hant-HK` → HongKong; -/// `ja` → Japanese; `ko` → Korean. `None` for a language not written with Han, which leaves the -/// choice to the font stack. +/// Simplified; `zh-TW`, `zh-Hant` and the other regions CLDR writes in Traditional characters +/// (`zh-US`, `zh-AU`) → Traditional; `zh-HK`, `zh-MO` and `zh-Hant-HK` → HongKong; `ja` → +/// Japanese; `ko` → Korean. `None` for a language not written with Han, which leaves the choice +/// to the font stack. pub fn han(tag: &str) -> Option { let tag = Subtags::parse(tag); let hong_kong = matches!(tag.region.as_deref(), Some("hk" | "mo")); - match (tag.language.as_str(), tag.script.as_deref()) { + let script = tag.script.as_deref().or_else(|| region_script(&tag.language, tag.region.as_deref()?)); + match (tag.language.as_str(), script) { ("ja", _) => Some(Han::Japanese), ("ko", _) => Some(Han::Korean), ("zh" | "yue" | "wuu", Some("hans")) => Some(Han::Simplified), ("zh" | "yue" | "wuu", Some("hant")) => Some(if hong_kong { Han::HongKong } else { Han::Traditional }), // Cantonese is written in Traditional characters with Hong Kong forms unless it says otherwise. - ("yue", _) => Some(if tag.region.as_deref() == Some("cn") { Han::Simplified } else { Han::HongKong }), - ("zh" | "wuu", _) => Some(match tag.region.as_deref() { - Some("tw") => Han::Traditional, - _ if hong_kong => Han::HongKong, - _ => Han::Simplified, - }), + ("yue", _) => Some(Han::HongKong), + ("zh" | "wuu", _) => Some(Han::Simplified), _ => None, } } -/// The scripts a language takes in a region where CLDR's likely subtags give it a script other -/// than its usual one (`pa-PK` → `pa-Arab-PK`). Han regions are left to [`han`]. +/// The script subtag of a language in a region where CLDR's likely subtags give it a script +/// other than its usual one: `pa-PK` → `arab` (`pa-Arab-PK`), `zh-TW` → `hant`. // ponytail: mn-CN → Mong is left out until vertical Mongolian layout is supported. -fn region_scripts(language: &str, region: &str) -> Option<&'static [Script]> { - use Script::*; +pub(crate) fn region_script(language: &str, region: &str) -> Option<&'static str> { Some(match (language, region) { - ("az", "iq" | "ir") | ("ha", "cm" | "sd") | ("kk", "af" | "cn" | "ir" | "mn") | ("ku", "iq" | "ir" | "lb") => &[Arab], - ("ky", "cn") | ("ms", "cc") | ("pa", "pk") | ("tg", "pk") | ("uz", "af") => &[Arab], - ("az", "ru") | ("ku", "am" | "az" | "ge" | "tm") | ("ug", "kz" | "mn") | ("uz", "cn") => &[Cyrl], - ("ky", "tr") | ("sr", "me" | "ro" | "tr") => &[Latn], - ("sd", "in") => &[Deva], + ("az", "iq" | "ir") | ("ha", "cm" | "sd") | ("kk", "af" | "cn" | "ir" | "mn") | ("ku", "iq" | "ir" | "lb") => "arab", + ("ky", "cn") | ("ms", "cc") | ("pa", "pk") | ("tg", "pk") | ("uz", "af") => "arab", + ("az", "ru") | ("ku", "am" | "az" | "ge" | "tm") | ("ug", "kz" | "mn") | ("uz", "cn") => "cyrl", + ("ky", "tr") | ("sr", "me" | "ro" | "tr") => "latn", + ("sd", "in") => "deva", + ("zh", "au" | "bn" | "gb" | "gf" | "hk" | "id" | "mo" | "pa" | "pf" | "ph" | "sr" | "th" | "tw" | "us" | "vn") => "hant", + ("yue", "cn") => "hans", _ => return None, }) } @@ -340,6 +339,13 @@ mod tests { ("zh-hant-hk", Han::HongKong), ("zh-Hans-HK", Han::Simplified), ("zh-SG", Han::Simplified), + ("zh-US", Han::Traditional), + ("zh_AU.UTF-8", Han::Traditional), + ("zh-GB", Han::Traditional), + ("zh-Hans-US", Han::Simplified), + ("zh-MY", Han::Simplified), + ("yue-CN", Han::Simplified), + ("yue-HK", Han::HongKong), ("zh-Hans-SG", Han::Simplified), ("yue", Han::HongKong), ("yue-Hans", Han::Simplified), From 56bbf840a7ff71acd7f75f2bc5318df5ed8b3129 Mon Sep 17 00:00:00 2001 From: aiongg Date: Wed, 21 Oct 2026 06:06:08 +0000 Subject: [PATCH 6/9] i18n: strip every isolate mark in fmt_plain --- crates/i18n/src/lib.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/crates/i18n/src/lib.rs b/crates/i18n/src/lib.rs index 8fde98ad..2935b88f 100644 --- a/crates/i18n/src/lib.rs +++ b/crates/i18n/src/lib.rs @@ -275,10 +275,10 @@ pub fn fmt(template: &str, args: &[(&str, &str)]) -> String { } /// [`fmt`] without isolates, for text that goes where the marks would get in the way: file -/// names, logs, terminals, text compared or parsed later. A value that is one isolate goes in -/// without it. +/// names, logs, terminals, text compared or parsed later. Every isolate mark is taken off, in +/// the template and in the values. pub fn fmt_plain(template: &str, args: &[(&str, &str)]) -> String { - fill(template, args, false) + strip_isolates(&fill(template, args, false)) } const LRI: char = '\u{2066}'; @@ -602,6 +602,7 @@ mod tests { assert_eq!(fmt("Saved {name} in {dir}.", &[("name", "Report 2.pdf"), ("dir", <r("/home/sara"))]), "Saved Report 2.pdf in /home/sara."); assert_eq!(fmt("{a}", &[("a", "x"), ("b", "שלום")]), "x", "an unused value doesn't count"); assert_eq!(fmt_plain("{a}", &[("a", &isolate("x"))]), "x"); + assert_eq!(fmt_plain("\u{2066}{a} {b}", &[("a", "x\u{2069}"), ("b", &isolate(<r("y")))]), "x y"); let i = all(P).0; assert_eq!(i.trn(Lang::EN, 2, "{n} page", "{n} pages"), "2 pages"); } From c9013e08c7752a12c14f228e2d45ebf452c96a5b Mon Sep 17 00:00:00 2001 From: aiongg Date: Wed, 21 Oct 2026 06:06:36 +0000 Subject: [PATCH 7/9] i18n: Hani and yue-Hant pick Han forms by region; usual scripts for kk, ky, tg, ku, ms, ug --- crates/i18n/src/lib.rs | 9 ++++++--- crates/i18n/src/scripts.rs | 15 +++++++++++---- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/crates/i18n/src/lib.rs b/crates/i18n/src/lib.rs index 6c841d1e..db13b6db 100644 --- a/crates/i18n/src/lib.rs +++ b/crates/i18n/src/lib.rs @@ -358,10 +358,10 @@ fn marked(text: &str) -> String { // ponytail: the multi-script languages a Craft app could plausibly ship; add a row with the locale. fn usual_script(language: &str) -> Option<&'static str> { Some(match language { - "be" | "mn" | "sr" => "cyrl", + "be" | "kk" | "ky" | "mn" | "sr" | "tg" => "cyrl", "zh" => "hans", - "az" | "bs" | "ha" | "uz" => "latn", - "ks" | "sd" => "arab", + "az" | "bs" | "ha" | "ku" | "ms" | "uz" => "latn", + "ks" | "sd" | "ug" => "arab", "pa" => "guru", _ => return None, }) @@ -600,6 +600,9 @@ mod tests { assert_eq!(candidates("pa-Guru-PK"), ["pa-guru-pk", "pa-guru", "pa"]); assert_eq!(candidates("zh-US"), ["zh-us", "zh-hant", "zh"]); assert_eq!(candidates("zh-SG"), ["zh-sg", "zh-hans", "zh"]); + assert_eq!(candidates("kk-CN"), ["kk-cn", "kk-arab-cn", "kk-arab"]); + assert_eq!(candidates("kk-KZ"), ["kk-kz", "kk"]); + assert_eq!(candidates("ug-KZ"), ["ug-kz", "ug-cyrl-kz", "ug-cyrl"]); } #[test] diff --git a/crates/i18n/src/scripts.rs b/crates/i18n/src/scripts.rs index 56a1ba1f..ebe324f5 100644 --- a/crates/i18n/src/scripts.rs +++ b/crates/i18n/src/scripts.rs @@ -163,18 +163,21 @@ pub fn direction(tag: &str) -> Direction { /// The forms a language tag's Han characters take: `zh-CN`, `zh-SG`, `zh-Hans` and plain `zh` → /// Simplified; `zh-TW`, `zh-Hant` and the other regions CLDR writes in Traditional characters -/// (`zh-US`, `zh-AU`) → Traditional; `zh-HK`, `zh-MO` and `zh-Hant-HK` → HongKong; `ja` → +/// (`zh-US`, `zh-AU`) → Traditional; `zh-HK`, `zh-MO`, `zh-Hant-HK` and `yue` → HongKong; `ja` → /// Japanese; `ko` → Korean. `None` for a language not written with Han, which leaves the choice /// to the font stack. pub fn han(tag: &str) -> Option { let tag = Subtags::parse(tag); let hong_kong = matches!(tag.region.as_deref(), Some("hk" | "mo")); - let script = tag.script.as_deref().or_else(|| region_script(&tag.language, tag.region.as_deref()?)); + // `Hani` names Han without saying which forms, so the region still decides. + let script = tag.script.as_deref().filter(|s| *s != "hani"); + let script = script.or_else(|| region_script(&tag.language, tag.region.as_deref()?)); match (tag.language.as_str(), script) { ("ja", _) => Some(Han::Japanese), ("ko", _) => Some(Han::Korean), ("zh" | "yue" | "wuu", Some("hans")) => Some(Han::Simplified), - ("zh" | "yue" | "wuu", Some("hant")) => Some(if hong_kong { Han::HongKong } else { Han::Traditional }), + ("yue", Some("hant")) => Some(if tag.region.as_deref() == Some("tw") { Han::Traditional } else { Han::HongKong }), + ("zh" | "wuu", Some("hant")) => Some(if hong_kong { Han::HongKong } else { Han::Traditional }), // Cantonese is written in Traditional characters with Hong Kong forms unless it says otherwise. ("yue", _) => Some(Han::HongKong), ("zh" | "wuu", _) => Some(Han::Simplified), @@ -348,6 +351,10 @@ mod tests { ("yue-HK", Han::HongKong), ("zh-Hans-SG", Han::Simplified), ("yue", Han::HongKong), + ("yue-Hant", Han::HongKong), + ("yue-Hant-TW", Han::Traditional), + ("zh-Hani-HK", Han::HongKong), + ("zh-Hani", Han::Simplified), ("yue-Hans", Han::Simplified), ("ja-JP", Han::Japanese), ("ko_KR.UTF-8", Han::Korean), @@ -414,7 +421,7 @@ mod tests { /// The usual scripts tag matching relies on agree with the table. #[test] fn agrees_with_usual_script() { - for language in ["be", "mn", "sr", "zh", "az", "bs", "ha", "uz", "ks", "sd", "pa"] { + for language in ["be", "kk", "ky", "mn", "sr", "tg", "zh", "az", "bs", "ha", "ku", "ms", "uz", "ks", "sd", "ug", "pa"] { let usual = crate::usual_script(language).and_then(subtag_scripts); assert_eq!(usual, Some(scripts(language)), "{language}"); } From 2ff3f1165410a75d9051e9679446b8711f8a097b Mon Sep 17 00:00:00 2001 From: aiongg Date: Wed, 21 Oct 2026 06:13:49 +0000 Subject: [PATCH 8/9] i18n: base_direction of a paragraph from its first strong character --- Cargo.lock | 1 + crates/i18n/Cargo.toml | 1 + crates/i18n/src/lib.rs | 2 +- crates/i18n/src/scripts.rs | 26 ++++++++++++++++++++++++++ 4 files changed, 29 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 4d9de428..ebc4055f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -659,6 +659,7 @@ dependencies = [ "intl_pluralrules", "sys-locale", "unic-langid", + "unicode-bidi", ] [[package]] diff --git a/crates/i18n/Cargo.toml b/crates/i18n/Cargo.toml index 45e21775..8283943f 100644 --- a/crates/i18n/Cargo.toml +++ b/crates/i18n/Cargo.toml @@ -11,6 +11,7 @@ repository.workspace = true intl_pluralrules = "=7.0.2" unic-langid = "0.9" sys-locale = "0.3" +unicode-bidi = "0.3" [target.'cfg(target_arch = "wasm32")'.dependencies] sys-locale = { version = "0.3", features = ["js"] } diff --git a/crates/i18n/src/lib.rs b/crates/i18n/src/lib.rs index db13b6db..ca955f85 100644 --- a/crates/i18n/src/lib.rs +++ b/crates/i18n/src/lib.rs @@ -14,7 +14,7 @@ use std::sync::{Mutex, OnceLock, PoisonError}; use intl_pluralrules::{PluralCategory, PluralRuleType, PluralRules}; use unic_langid::LanguageIdentifier; -pub use scripts::{Direction, Han, Script, direction, han, scripts}; +pub use scripts::{Direction, Han, Script, base_direction, direction, han, scripts}; /// The catalogs an app embeds, one entry per locale: (locale, shared catalog, the app's catalog). /// A missing catalog is `""`. diff --git a/crates/i18n/src/scripts.rs b/crates/i18n/src/scripts.rs index ebe324f5..1ae0f29b 100644 --- a/crates/i18n/src/scripts.rs +++ b/crates/i18n/src/scripts.rs @@ -161,6 +161,18 @@ pub fn direction(tag: &str) -> Direction { } } +/// The direction of a paragraph of text, from its first strong character (UAX #9 rules P2 and +/// P3): the contents of isolates (LRI, RLI or FSI up to the matching PDI) are skipped, and the +/// search stops at the first paragraph separator. `None` when the paragraph has no strong +/// character; P3 then reads it as left to right, and an app may use the interface direction. +pub fn base_direction(text: &str) -> Option { + match unicode_bidi::get_base_direction(text) { + unicode_bidi::Direction::Ltr => Some(Direction::Ltr), + unicode_bidi::Direction::Rtl => Some(Direction::Rtl), + unicode_bidi::Direction::Mixed => None, + } +} + /// The forms a language tag's Han characters take: `zh-CN`, `zh-SG`, `zh-Hans` and plain `zh` → /// Simplified; `zh-TW`, `zh-Hant` and the other regions CLDR writes in Traditional characters /// (`zh-US`, `zh-AU`) → Traditional; `zh-HK`, `zh-MO`, `zh-Hant-HK` and `yue` → HongKong; `ja` → @@ -277,6 +289,20 @@ mod tests { use super::Script::*; use super::*; + #[test] + fn base_direction_of_text() { + assert_eq!(base_direction("Hello"), Some(Direction::Ltr)); + assert_eq!(base_direction("123 שלום"), Some(Direction::Rtl)); + assert_eq!(base_direction("(مرحبا) hi"), Some(Direction::Rtl)); + // The isolate's Hebrew is skipped; the Latin after it decides. + assert_eq!(base_direction("\u{2067}שלום\u{2069} hello"), Some(Direction::Ltr)); + assert_eq!(base_direction("\u{2068}abc\u{2069} שלום"), Some(Direction::Rtl)); + // Only the first paragraph counts. + assert_eq!(base_direction("123\nשלום"), None); + assert_eq!(base_direction("1920 × 1080"), None); + assert_eq!(base_direction(""), None); + } + /// Every locale the craft-i18n catalogs or an app ship, as the apps spell them. #[test] fn shipped_locales() { From 7143c25157037f9daa8d03d9b5e7922d3a09876a Mon Sep 17 00:00:00 2001 From: aiongg Date: Wed, 21 Oct 2026 17:20:34 +0000 Subject: [PATCH 9/9] i18n: contain values that carry bidi controls A value with an embedding, override, isolate or mark (LRE RLE PDF LRO RLO LRI RLI FSI PDI LRM RLM ALM) makes fmt isolate, even in text with no right-to-left letter, so an ASCII file name with an RLO can't reorder the sentence. Embeddings, overrides and isolates a value opens are closed inside its isolate, and stray PDF and PDI are dropped. --- crates/i18n/README.md | 12 +++-- crates/i18n/src/lib.rs | 103 +++++++++++++++++++++++++++++++++++------ 2 files changed, 95 insertions(+), 20 deletions(-) diff --git a/crates/i18n/README.md b/crates/i18n/README.md index 951a6782..dc2e3c46 100644 --- a/crates/i18n/README.md +++ b/crates/i18n/README.md @@ -50,11 +50,13 @@ The app keeps its `tl!` macro, its language setting (`"auto"` → `I18N.system_l or English: a key cap's name under `key`, or "is there a translation?"), and `fmt` for `{name}` placeholders, filled in one pass. - **Isolation:** when a message or a value inserted into it has a right-to-left letter (bidi - class R or AL), `fmt` wraps each inserted value in a first-strong isolate (FSI…PDI), and `trn` - wraps `{n}` in a left-to-right one (LRI…PDI), so a Latin file name in an Arabic sentence, or a - Hebrew name in an English one, can't reorder the words around it. Wrap numbers, paths and codes - in `ltr` before passing them; a value that already is one isolate goes in as it is, and a stray - PDI in a value can't close its isolate early. Text with no right-to-left letter comes out + class R or AL), or a value has a bidi control (LRE, RLE, PDF, LRO, RLO, LRI, RLI, FSI, PDI, + LRM, RLM, ALM), `fmt` wraps each inserted value in a first-strong isolate (FSI…PDI), and `trn` + wraps `{n}` in a left-to-right one (LRI…PDI), so a Latin file name in an Arabic sentence, a + Hebrew name in an English one, or an ASCII file name with a right-to-left override can't + reorder the words around it. Embeddings, overrides and isolates a value opens are closed inside + its isolate, and a stray PDF or PDI in it is dropped. Wrap numbers, paths and codes in `ltr` + before passing them; a value that already is one isolate keeps it. Other text comes out without marks, as if `fmt` didn't isolate at all. egui draws the marks as nothing. For text that leaves the interface (file names, logs, terminals, text parsed later), use `fmt_plain` or `strip_isolates`. diff --git a/crates/i18n/src/lib.rs b/crates/i18n/src/lib.rs index 2935b88f..bbffc260 100644 --- a/crates/i18n/src/lib.rs +++ b/crates/i18n/src/lib.rs @@ -264,11 +264,13 @@ impl I18n { } /// Fill `{name}` placeholders from `args` in one pass. When the template or an inserted value -/// has a right-to-left letter (Hebrew, Arabic and the like), each value goes in as a first-strong -/// isolate (FSI…PDI), so its direction can't reorder the sentence around it: a Latin file name in -/// an Arabic message stays where the translator put it. A value that already is one isolate goes -/// in as it is; pass numbers, paths and codes through [`ltr`]. Text with no right-to-left letter -/// gets no marks, and a value's own isolate is taken off. Unknown placeholders stay as written, +/// has a right-to-left letter (Hebrew, Arabic and the like), or a value has a bidi control +/// (an embedding, override, isolate or mark), each value goes in as a first-strong isolate +/// (FSI…PDI), so its direction can't reorder the sentence around it: a Latin file name in an +/// Arabic message stays where the translator put it, and an override in a file name ends with +/// the name. Embeddings, overrides and isolates a value opens are closed inside its isolate. A +/// value that already is one isolate keeps it; pass numbers, paths and codes through [`ltr`]. +/// Other text gets no marks, and a value's own isolate is taken off. Unknown placeholders stay as written, /// and inserted values are never read as templates, so a file name containing `{n}` stays intact. pub fn fmt(template: &str, args: &[(&str, &str)]) -> String { fill(template, args, true) @@ -285,6 +287,14 @@ const LRI: char = '\u{2066}'; const RLI: char = '\u{2067}'; const FSI: char = '\u{2068}'; const PDI: char = '\u{2069}'; +const LRE: char = '\u{202A}'; +const RLE: char = '\u{202B}'; +const PDF: char = '\u{202C}'; +const LRO: char = '\u{202D}'; +const RLO: char = '\u{202E}'; +const LRM: char = '\u{200E}'; +const RLM: char = '\u{200F}'; +const ALM: char = '\u{061C}'; /// `value` as a first-strong isolate (FSI…PDI): it takes the direction of its first strong /// letter and can't reorder the text around it. What [`fmt`] does to each value in text with a @@ -307,27 +317,46 @@ pub fn strip_isolates(text: &str) -> String { text.chars().filter(|c| !matches!(*c, LRI | RLI | FSI | PDI)).collect() } -/// `value` between `initiator` and PDI. A stray PDI in the value would close the isolate early, so -/// it is dropped; isolates the value leaves open are closed. +/// `value` between `initiator` and PDI, with every embedding, override and isolate it opens +/// closed inside: a stray PDF or PDI is dropped, and open ones are closed in order. Under the +/// bidi algorithm the outer PDI alone would close them, but a renderer that handles only +/// embeddings would let them run on into the rest of the text. fn wrap(initiator: char, value: &str) -> String { let mut out = String::with_capacity(value.len() + 6); out.push(initiator); - let mut open = 0_usize; + // One entry per open initiator: true for an isolate, false for an embedding or override. + let mut open: Vec = Vec::new(); for c in value.chars() { match c { - LRI | RLI | FSI => open += 1, - PDI if open == 0 => continue, - PDI => open -= 1, + LRI | RLI | FSI => open.push(true), + LRE | RLE | LRO | RLO => open.push(false), + PDF if open.last() == Some(&false) => { + open.pop(); + } + PDF => continue, + PDI if open.contains(&true) => { + while open.pop() == Some(false) { + out.push(PDF); + } + } + PDI => continue, _ => {} } out.push(c); } - for _ in 0..=open { - out.push(PDI); + while let Some(isolate) = open.pop() { + out.push(if isolate { PDI } else { PDF }); } + out.push(PDI); out } +/// Whether `value`, without its own isolate, has a bidi control: an embedding, override, +/// isolate or mark that could reorder the text around it unless it is contained. +fn has_bidi_control(value: &str) -> bool { + !value.is_ascii() && unwrap_isolate(value).chars().any(|c| matches!(c, LRE | RLE | PDF | LRO | RLO | LRI | RLI | FSI | PDI | LRM | RLM | ALM)) +} + /// Whether `text` has a strong right-to-left character (bidi class R or AL): the text that needs /// isolates. fn has_rtl(text: &str) -> bool { @@ -381,13 +410,18 @@ fn fill(template: &str, args: &[(&str, &str)], isolated: bool) -> String { rest = after_close; } pieces.push((rest, false)); - let isolated = isolated && pieces.iter().any(|(text, _)| has_rtl(text)); + let isolated = isolated && pieces.iter().any(|(text, value)| has_rtl(text) || (*value && has_bidi_control(text))); let mut out = String::with_capacity(template.len()); for (text, value) in pieces { match (value, isolated) { (false, _) => out.push_str(text), (true, false) => out.push_str(unwrap_isolate(text)), - (true, true) if text.is_empty() || is_one_isolate(text) => out.push_str(text), + (true, true) if text.is_empty() => {} + // Rebuilt rather than copied, so what it opens inside is closed too. + (true, true) if is_one_isolate(text) => { + let initiator = text.chars().next().unwrap_or(FSI); + out.push_str(&wrap(initiator, unwrap_isolate(text))); + } (true, true) => out.push_str(&isolate(text)), } } @@ -628,6 +662,45 @@ mod tests { assert_eq!(fmt("ש{v}", &[("v", "\u{2066}a\u{2069}\u{2066}b\u{2069}")]), "ש\u{2068}\u{2066}a\u{2069}\u{2066}b\u{2069}\u{2069}"); } + /// `text` in display order, as one line in a left-to-right paragraph, without bidi controls. + fn visual(text: &str) -> String { + let info = unicode_bidi::BidiInfo::new(text, Some(unicode_bidi::Level::ltr())); + let para = &info.paragraphs[0]; + let shown = info.reorder_line(para, para.range.clone()); + shown + .chars() + .filter(|c| bidi_class(*c) != BidiClass::BN && !"\u{202A}\u{202B}\u{202C}\u{202D}\u{202E}\u{2066}\u{2067}\u{2068}\u{2069}".contains(*c)) + .collect() + } + + #[test] + fn a_value_with_bidi_controls_cannot_reorder_the_sentence() { + let file_names = [ + "\u{202E}fdp.exe", + "\u{202B}abc def", + "\u{202D}abc def", + "\u{2067}abc def", + "\u{2068}abc\u{2069}\u{202E}def", + "\u{202E}abc\u{2069}", + "x\u{200F}", + "x\u{061C}", + ]; + for name in file_names { + let text = fmt("Open {name} -> {name} now?", &[("name", name)]); + let shown = visual(&text); + assert!(shown.starts_with("Open ") && shown.contains(" -> ") && shown.ends_with(" now?"), "{name:?}: {shown:?}"); + let i = all(P).0; + let wording = i.trn(Lang::EN, 2, "{n} page in {name}, done", "{n} pages in {name}, done"); + let shown = visual(&fmt(&wording, &[("name", name)])); + assert!(shown.starts_with("2 pages in ") && shown.ends_with(", done"), "{name:?}: {shown:?}"); + } + // An override the value opens is closed inside its isolate, and a stray PDF dropped. + assert_eq!(fmt("Open {name}.", &[("name", "\u{202E}abc")]), "Open \u{2068}\u{202E}abc\u{202C}\u{2069}."); + assert_eq!(fmt("Open {name}.", &[("name", "a\u{202C}b")]), "Open \u{2068}ab\u{2069}."); + assert_eq!(fmt("Open {n}.", &[("n", <r("\u{202B}1"))]), "Open \u{2066}\u{202B}1\u{202C}\u{2069}."); + assert_eq!(isolate("\u{2067}a\u{202E}b\u{2069}c"), "\u{2068}\u{2067}a\u{202E}b\u{202C}\u{2069}c\u{2069}"); + } + #[test] fn one_other() { let (i, problems) = all(P);