From 903dbd5fa342c4df1cd9fe81982b0440ae007f4b Mon Sep 17 00:00:00 2001 From: aiongg Date: Wed, 21 Oct 2026 04:38:22 +0000 Subject: [PATCH] i18n: scripts, direction and regional Han per language tag scripts(tag) lists the ISO 15924 scripts a language's interface text is written in, direction(tag) follows the main script, and han(tag) gives the regional Han forms (SC, TC, HK, JP, KR) for picking a CJK face. I18n::rtl uses direction. --- crates/i18n/README.md | 8 +- crates/i18n/src/lib.rs | 11 +- crates/i18n/src/scripts.rs | 363 +++++++++++++++++++++++++++++++++++++ 3 files changed, 375 insertions(+), 7 deletions(-) create mode 100644 crates/i18n/src/scripts.rs diff --git a/crates/i18n/README.md b/crates/i18n/README.md index df337971..cde3621d 100644 --- a/crates/i18n/README.md +++ b/crates/i18n/README.md @@ -60,8 +60,12 @@ The app keeps its `tl!` macro, its language setting (`"auto"` → `I18N.system_l `plural_form_counts` holds the same counts. - **Names:** a language's name comes from its shared catalog's `@id language.name` row, else its code; English is "English". -- **Right to left:** `rtl` is true for `ar`, `ckb`, `dv`, `fa`, `he`, `ps`, `sd`, `ug`, `ur` and - `yi`; putting text into display order is the app's job. +- **Scripts and direction:** for any language tag, `scripts` lists the ISO 15924 scripts its + interface text is written in (`ja` → Hani, Hira, Kana; `sr-latn` → Latn), `direction` is right + to left when the main script is (Arabic, Hebrew, Syriac, Thaana), and `han` gives the regional + forms of its Han characters for picking a CJK face: `zh-CN`/`zh-SG`/`zh-Hans` → SC, `zh-TW`/ + `zh-Hant` → TC, `zh-HK`/`zh-MO` → HK, `ja` → JP, `ko` → KR. `I18n::rtl` is `direction` of the + language's code; putting text into display order is the app's job. - **System locale:** `system_lang` matches the system's list once and caches it: on desktop Unix `LANGUAGE`, `LC_ALL`, `LC_MESSAGES` and `LANG`, read raw; elsewhere `sys-locale`'s (macOS and Windows preferred languages, the browser's on the web). `fr_CA.UTF-8` → `fr`, `zh-TW` → diff --git a/crates/i18n/src/lib.rs b/crates/i18n/src/lib.rs index 7fa7a4ef..84295a16 100644 --- a/crates/i18n/src/lib.rs +++ b/crates/i18n/src/lib.rs @@ -6,6 +6,7 @@ #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] mod catalog; +mod scripts; use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::sync::{Mutex, OnceLock, PoisonError}; @@ -13,6 +14,8 @@ use std::sync::{Mutex, OnceLock, PoisonError}; use intl_pluralrules::{PluralCategory, PluralRuleType, PluralRules}; use unic_langid::LanguageIdentifier; +pub use scripts::{Direction, Han, Script, direction, han, scripts}; + /// The catalogs an app embeds, one entry per locale: (locale, shared catalog, the app's catalog). /// A missing catalog is `""`. pub type Embedded = [(&'static str, &'static str, &'static str)]; @@ -199,12 +202,10 @@ impl I18n { self.locale(lang).and_then(|l| l.rows.get("@id")?.get("language.name")).map_or_else(|| self.code(lang), String::as_str) } - /// Whether the language is written right to left. Putting text into display order is the - /// app's job. + /// Whether the language is written right to left ([`direction`]). Putting text into display + /// order is the app's job. pub fn rtl(&self, lang: Lang) -> bool { - let code = self.code(lang); - let language = code.split('-').next().unwrap_or(code); - matches!(language, "ar" | "ckb" | "dv" | "fa" | "he" | "ps" | "sd" | "ug" | "ur" | "yi") + direction(self.code(lang)) == Direction::Rtl } /// The language for a system locale tag (`fr_CA.UTF-8`, `zh-Hant-MO`); `C` and `POSIX` are diff --git a/crates/i18n/src/scripts.rs b/crates/i18n/src/scripts.rs new file mode 100644 index 00000000..ecd2f8dd --- /dev/null +++ b/crates/i18n/src/scripts.rs @@ -0,0 +1,363 @@ +//! The scripts a language is written in, its direction, and which regional forms its Han +//! characters take, for picking fonts and laying out the interface. + +/// A writing system, by its ISO 15924 code. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub enum Script { + Arab, + Armn, + Beng, + Cyrl, + Deva, + Ethi, + Geor, + Grek, + Gujr, + Guru, + Hang, + Hani, + Hebr, + Hira, + Kana, + Khmr, + Knda, + Laoo, + Latn, + Mlym, + Mong, + Mymr, + Orya, + Sinh, + Syrc, + Taml, + Telu, + Thaa, + Thai, + Tibt, +} + +impl Script { + /// The ISO 15924 code, `Latn`. + pub fn code(self) -> &'static str { + match self { + Script::Arab => "Arab", + Script::Armn => "Armn", + Script::Beng => "Beng", + Script::Cyrl => "Cyrl", + Script::Deva => "Deva", + Script::Ethi => "Ethi", + Script::Geor => "Geor", + Script::Grek => "Grek", + Script::Gujr => "Gujr", + Script::Guru => "Guru", + Script::Hang => "Hang", + Script::Hani => "Hani", + Script::Hebr => "Hebr", + Script::Hira => "Hira", + Script::Kana => "Kana", + Script::Khmr => "Khmr", + Script::Knda => "Knda", + Script::Laoo => "Laoo", + Script::Latn => "Latn", + Script::Mlym => "Mlym", + Script::Mong => "Mong", + Script::Mymr => "Mymr", + Script::Orya => "Orya", + Script::Sinh => "Sinh", + Script::Syrc => "Syrc", + Script::Taml => "Taml", + Script::Telu => "Telu", + Script::Thaa => "Thaa", + Script::Thai => "Thai", + Script::Tibt => "Tibt", + } + } + + /// Whether text in this script runs right to left. + pub fn rtl(self) -> bool { + matches!(self, Script::Arab | Script::Hebr | Script::Syrc | Script::Thaa) + } +} + +/// Which way a language's text, and an interface in it, runs. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum Direction { + Ltr, + Rtl, +} + +/// The regional glyph forms Han characters take, as in the Noto CJK face names (SC, TC, HK, JP, +/// KR). The same code point is drawn differently in each, so a font stack picks its Han face by +/// this, not by script. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum Han { + /// Simplified Chinese, mainland China and Singapore (SC). + Simplified, + /// Traditional Chinese, Taiwan (TC). + Traditional, + /// Traditional Chinese, Hong Kong and Macau (HK). + HongKong, + /// Japanese kanji (JP). + Japanese, + /// Korean hanja (KR). + Korean, +} + +/// The scripts a language tag's interface text is written in (`ja` → Hani, Hira, Kana; `sr` → +/// Cyrl; `sr-latn` → Latn), the main one first. Any case and glibc form (`sr_RS.UTF-8@latin`). A +/// script subtag wins over the language's usual script. Latin product names and units appear in +/// every language, so Latn is listed only where it is the language's own script. A language not +/// in the table is taken to be written in Latin. +// ponytail: the languages a Craft app ships or could plausibly ship; add a row with the locale. +pub fn scripts(tag: &str) -> &'static [Script] { + use Script::*; + let tag = Subtags::parse(tag); + if let Some(s) = tag.script.as_deref().and_then(subtag_scripts) { + return s; + } + match tag.language.as_str() { + "ja" => &[Hani, Hira, Kana], + "ko" => &[Hang, Hani], + "zh" | "yue" | "wuu" => &[Hani], + "be" | "bg" | "ba" | "ce" | "cv" | "kk" | "ky" | "mk" | "mn" | "ru" | "sah" | "sr" | "tg" | "tt" | "uk" => &[Cyrl], + "el" => &[Grek], + "ar" | "ckb" | "fa" | "ks" | "ps" | "sd" | "ug" | "ur" => &[Arab], + "he" | "yi" => &[Hebr], + "syr" => &[Syrc], + "dv" => &[Thaa], + "ka" => &[Geor], + "hy" => &[Armn], + "hi" | "mr" | "ne" | "sa" | "kok" | "mai" | "bho" => &[Deva], + "bn" | "as" | "mni" => &[Beng], + "pa" => &[Guru], + "gu" => &[Gujr], + "or" => &[Orya], + "ta" => &[Taml], + "te" => &[Telu], + "kn" => &[Knda], + "ml" => &[Mlym], + "si" => &[Sinh], + "th" => &[Thai], + "lo" => &[Laoo], + "km" => &[Khmr], + "my" => &[Mymr], + "am" | "ti" => &[Ethi], + "bo" | "dz" => &[Tibt], + _ => &[Latn], + } +} + +/// The direction of a language tag's text: right to left when its main script is. +pub fn direction(tag: &str) -> Direction { + match scripts(tag).first() { + Some(s) if s.rtl() => Direction::Rtl, + _ => Direction::Ltr, + } +} + +/// The forms a language tag's Han characters take: `zh-CN`, `zh-SG`, `zh-Hans` and plain `zh` → +/// Simplified; `zh-TW` and `zh-Hant` → Traditional; `zh-HK`, `zh-MO` and `zh-Hant-HK` → HongKong; +/// `ja` → Japanese; `ko` → Korean. `None` for a language not written with Han, which leaves the +/// choice to the font stack. +pub fn han(tag: &str) -> Option { + let tag = Subtags::parse(tag); + let hong_kong = matches!(tag.region.as_deref(), Some("hk" | "mo")); + match (tag.language.as_str(), tag.script.as_deref()) { + ("ja", _) => Some(Han::Japanese), + ("ko", _) => Some(Han::Korean), + ("zh" | "yue" | "wuu", Some("hans")) => Some(Han::Simplified), + ("zh" | "yue" | "wuu", Some("hant")) => Some(if hong_kong { Han::HongKong } else { Han::Traditional }), + // Cantonese is written in Traditional characters with Hong Kong forms unless it says otherwise. + ("yue", _) => Some(if tag.region.as_deref() == Some("cn") { Han::Simplified } else { Han::HongKong }), + ("zh" | "wuu", _) => Some(match tag.region.as_deref() { + Some("tw") => Han::Traditional, + _ if hong_kong => Han::HongKong, + _ => Han::Simplified, + }), + _ => None, + } +} + +/// The scripts a script subtag stands for, including the CLDR unions (`jpan`, `kore`). `None` for +/// one not in [`Script`], so the language's own scripts apply. +fn subtag_scripts(subtag: &str) -> Option<&'static [Script]> { + use Script::*; + Some(match subtag { + "hans" | "hant" | "hani" => &[Hani], + "jpan" => &[Hani, Hira, Kana], + "kore" => &[Hang, Hani], + "arab" => &[Arab], + "armn" => &[Armn], + "beng" => &[Beng], + "cyrl" => &[Cyrl], + "deva" => &[Deva], + "ethi" => &[Ethi], + "geor" => &[Geor], + "grek" => &[Grek], + "gujr" => &[Gujr], + "guru" => &[Guru], + "hang" => &[Hang], + "hebr" => &[Hebr], + "khmr" => &[Khmr], + "knda" => &[Knda], + "laoo" => &[Laoo], + "latn" => &[Latn], + "mlym" => &[Mlym], + "mong" => &[Mong], + "mymr" => &[Mymr], + "orya" => &[Orya], + "sinh" => &[Sinh], + "syrc" => &[Syrc], + "taml" => &[Taml], + "telu" => &[Telu], + "thaa" => &[Thaa], + "thai" => &[Thai], + "tibt" => &[Tibt], + _ => return None, + }) +} + +/// A language tag's language, script and region subtags, lowercase. +struct Subtags { + language: String, + script: Option, + region: Option, +} + +impl Subtags { + /// `zh-Hant-HK`, `zh_TW.UTF-8` or `sr_RS@latin` (glibc's script modifier). + fn parse(tag: &str) -> Subtags { + let tag = tag.trim().to_ascii_lowercase(); + let base = tag.split(['.', '@']).next().unwrap_or("").replace('_', "-"); + let mut parts = base.split('-').filter(|p| !p.is_empty()); + let language = parts.next().unwrap_or("").to_owned(); + let mut script = match tag.split_once('@').map(|(_, m)| m) { + Some("latin") => Some("latn".to_owned()), + Some("cyrillic") => Some("cyrl".to_owned()), + _ => None, + }; + let mut region = None; + for part in parts { + if part.len() == 4 && part.bytes().all(|b| b.is_ascii_alphabetic()) { + script.get_or_insert_with(|| part.to_owned()); + } else if super::is_region(part) { + region = Some(part.to_owned()); + break; + } + } + Subtags { language, script, region } + } +} + +#[cfg(test)] +mod tests { + use super::Script::*; + use super::*; + + /// Every locale the craft-i18n catalogs or an app ship, as the apps spell them. + #[test] + fn shipped_locales() { + for (tag, want, han_forms) in [ + ("en", &[Latn][..], None), + ("ar", &[Arab], None), + ("bg", &[Cyrl], None), + ("cs", &[Latn], None), + ("da", &[Latn], None), + ("de", &[Latn], None), + ("el", &[Grek], None), + ("es", &[Latn], None), + ("es-419", &[Latn], None), + ("et", &[Latn], None), + ("fi", &[Latn], None), + ("fr", &[Latn], None), + ("hr", &[Latn], None), + ("hu", &[Latn], None), + ("id", &[Latn], None), + ("it", &[Latn], None), + ("ja", &[Hani, Hira, Kana], Some(Han::Japanese)), + ("ka", &[Geor], None), + ("kk", &[Cyrl], None), + ("ko", &[Hang, Hani], Some(Han::Korean)), + ("lt", &[Latn], None), + ("nl", &[Latn], None), + ("pl", &[Latn], None), + ("pt", &[Latn], None), + ("pt-br", &[Latn], None), + ("pt_BR", &[Latn], None), + ("ru", &[Cyrl], None), + ("sl", &[Latn], None), + ("sr", &[Cyrl], None), + ("sr-latn", &[Latn], None), + ("sv", &[Latn], None), + ("te", &[Telu], None), + ("tr", &[Latn], None), + ("uk", &[Cyrl], None), + ("vi", &[Latn], None), + ("zh", &[Hani], Some(Han::Simplified)), + ("zh-cn", &[Hani], Some(Han::Simplified)), + ("zh-CN", &[Hani], Some(Han::Simplified)), + ("zh-hans", &[Hani], Some(Han::Simplified)), + ("zh-Hans", &[Hani], Some(Han::Simplified)), + ("zh-hant", &[Hani], Some(Han::Traditional)), + ] { + assert_eq!(scripts(tag), want, "{tag}"); + assert_eq!(han(tag), han_forms, "{tag}"); + let rtl = tag == "ar"; + assert_eq!(direction(tag), if rtl { Direction::Rtl } else { Direction::Ltr }, "{tag}"); + } + } + + #[test] + fn regional_han() { + for (tag, want) in [ + ("zh-TW", Han::Traditional), + ("zh_TW.UTF-8", Han::Traditional), + ("zh-Hant-TW", Han::Traditional), + ("zh-HK", Han::HongKong), + ("zh-MO", Han::HongKong), + ("zh-Hant-HK", Han::HongKong), + ("zh-hant-hk", Han::HongKong), + ("zh-Hans-HK", Han::Simplified), + ("zh-SG", Han::Simplified), + ("zh-Hans-SG", Han::Simplified), + ("yue", Han::HongKong), + ("yue-Hans", Han::Simplified), + ("ja-JP", Han::Japanese), + ("ko_KR.UTF-8", Han::Korean), + ] { + assert_eq!(han(tag), Some(want), "{tag}"); + } + assert_eq!(han("en"), None); + assert_eq!(han(""), None); + } + + #[test] + fn script_subtag_wins() { + for (tag, want) in [ + ("sr-Latn-RS", &[Latn][..]), + ("sr_RS@latin", &[Latn]), + ("sr-Cyrl", &[Cyrl]), + ("sr_RS.UTF-8", &[Cyrl]), + ("az-Cyrl", &[Cyrl]), + ("pa-Arab", &[Arab]), + ("pa", &[Guru]), + ("mn-Mong", &[Mong]), + ("sd-Deva", &[Deva]), + ("fr-Zyyy", &[Latn]), + ] { + assert_eq!(scripts(tag), want, "{tag}"); + } + assert_eq!(direction("pa-Arab"), Direction::Rtl); + assert_eq!(direction("sd-Deva"), Direction::Ltr); + assert_eq!(direction("he_IL.UTF-8"), Direction::Rtl); + } + + /// The usual scripts tag matching relies on agree with the table. + #[test] + fn agrees_with_usual_script() { + for language in ["be", "mn", "sr", "zh", "az", "bs", "ha", "uz", "ks", "sd", "pa"] { + let usual = crate::usual_script(language).and_then(subtag_scripts); + assert_eq!(usual, Some(scripts(language)), "{language}"); + } + } +}