From 5354479ce303071e3bf06adac808ae43f956a808 Mon Sep 17 00:00:00 2001 From: aiongg <67543387+aiongg@users.noreply.github.com> Date: Fri, 23 Oct 2026 21:07:32 -0400 Subject: [PATCH] i18n: scripts, direction and regional Han per language tag (#200) * i18n: scripts, direction and regional Han per language tag scripts(tag) lists the ISO 15924 scripts a language's interface text is written in, direction(tag) follows the main script, and han(tag) gives the regional Han forms (SC, TC, HK, JP, KR) for picking a CJK face. I18n::rtl uses direction. * i18n: non_exhaustive Script and Han; regions that imply a script pa-PK, uz-AF and az-IR are written in Arabic and sd-IN in Devanagari, as CLDR's likely subtags give them. A script subtag still wins. * i18n: han and catalog matching follow region-implied scripts han(zh-US), zh-AU and the other regions CLDR writes in Traditional characters give Traditional, and pa-PK picks a pa-arab catalog, never Gurmukhi pa, as pa-Arab-PK does. * i18n: Hani and yue-Hant pick Han forms by region; usual scripts for kk, ky, tg, ku, ms, ug * i18n: base_direction of a paragraph from its first strong character --- Cargo.lock | 1 + crates/i18n/Cargo.toml | 1 + crates/i18n/README.md | 16 +- crates/i18n/src/lib.rs | 50 +++- crates/i18n/src/scripts.rs | 455 +++++++++++++++++++++++++++++++++++++ 5 files changed, 507 insertions(+), 16 deletions(-) create mode 100644 crates/i18n/src/scripts.rs diff --git a/Cargo.lock b/Cargo.lock index 48f04b90a..2b2ebaeb9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -741,6 +741,7 @@ dependencies = [ "log", "sys-locale", "unic-langid", + "unicode-bidi", ] [[package]] diff --git a/crates/i18n/Cargo.toml b/crates/i18n/Cargo.toml index 9a9b97e95..99b64a7d3 100644 --- a/crates/i18n/Cargo.toml +++ b/crates/i18n/Cargo.toml @@ -12,6 +12,7 @@ intl_pluralrules = "=7.0.2" unic-langid = "0.9" log = "0.4" sys-locale = "0.3" +unicode-bidi = "0.3" [target.'cfg(target_arch = "wasm32")'.dependencies] sys-locale = { version = "0.3", features = ["js"] } diff --git a/crates/i18n/README.md b/crates/i18n/README.md index c2ce066be..22ea77cff 100644 --- a/crates/i18n/README.md +++ b/crates/i18n/README.md @@ -62,14 +62,18 @@ The app keeps its `tl!` macro, its language setting (`"auto"` → `I18N.system_l `plural_form_counts` holds the same counts. - **Names:** a language's name comes from the app catalog's `@id language.name` row, else the shared catalog's, else its code; English is "English". -- **Right to left:** `rtl` is true for `ar`, `ckb`, `dv`, `fa`, `he`, `ps`, `sd`, `ug`, `ur` and - `yi`; putting text into display order is the app's job. +- **Scripts and direction:** for any language tag, `scripts` lists the ISO 15924 scripts its + interface text is written in (`ja` → Hani, Hira, Kana; `sr-latn` → Latn; `pa-PK` → Arab), + `direction` is right to left when the main script is (Arabic, Hebrew, Syriac, Thaana), and `han` + gives the regional forms of its Han characters for picking a CJK face: `zh-CN`/`zh-SG`/`zh-Hans` → + SC, `zh-TW`/`zh-US`/`zh-Hant` → TC, `zh-HK`/`zh-MO` → HK, `ja` → JP, `ko` → KR. `I18n::rtl` is + `direction` of the language's code; putting text into display order is the app's job. - **System locale:** `system_lang` matches the system's list once and caches it: on desktop Unix `LANGUAGE`, `LC_ALL`, `LC_MESSAGES` and `LANG`, read raw; elsewhere `sys-locale`'s (macOS and - Windows preferred languages, the browser's on the web). `fr_CA.UTF-8` → `fr`, `zh-TW` → - `zh-hant`, `pt-PT` → `pt-br` when that is the only Portuguese, `C`/`POSIX` → English. A script - is never crossed: `sr_RS@latin` and `sr-Latn-RS` pick `sr-latn`, never `sr`, while `sr-Cyrl-RS` - picks `sr`. + Windows preferred languages, the browser's on the web). `fr_CA.UTF-8` → `fr`, `zh-TW` → `zh-hant`, + `pt-PT` → `pt-br` when that is the only Portuguese, `C`/`POSIX` → English. A script is never + crossed: `sr_RS@latin` and `sr-Latn-RS` pick `sr-latn`, never `sr`, while `sr-Cyrl-RS` picks `sr`. + A region that implies a script counts as that script: `pa-PK` picks `pa-arab`, not Gurmukhi `pa`. - **Pseudo locale** (`pseudo`, debug builds only, never in `languages()`): every lookup, `get` included, comes back as `⟦…⟧`, line by line, so text drawn without the marks never went through translation. `looked_up()` lists the source texts looked up so far, for an audit against a diff --git a/crates/i18n/src/lib.rs b/crates/i18n/src/lib.rs index c040d0749..2d0e05a0f 100644 --- a/crates/i18n/src/lib.rs +++ b/crates/i18n/src/lib.rs @@ -6,6 +6,7 @@ #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] mod catalog; +mod scripts; use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::sync::{Mutex, OnceLock, PoisonError}; @@ -13,6 +14,8 @@ use std::sync::{Mutex, OnceLock, PoisonError}; use intl_pluralrules::{PluralCategory, PluralRuleType, PluralRules}; use unic_langid::LanguageIdentifier; +pub use scripts::{Direction, Han, Script, base_direction, direction, han, scripts}; + /// The catalogs an app embeds, one entry per locale: (locale, shared catalog, the app's catalog). /// A missing catalog is `""`. pub type Embedded = [(&'static str, &'static str, &'static str)]; @@ -248,12 +251,10 @@ impl I18n { self.locale(lang).and_then(Locale::language_name).unwrap_or_else(|| self.code(lang)) } - /// Whether the language is written right to left. Putting text into display order is the - /// app's job. + /// Whether the language is written right to left ([`direction`]). Putting text into display + /// order is the app's job. pub fn rtl(&self, lang: Lang) -> bool { - let code = self.code(lang); - let language = code.split('-').next().unwrap_or(code); - matches!(language, "ar" | "ckb" | "dv" | "fa" | "he" | "ps" | "sd" | "ug" | "ur" | "yi") + direction(self.code(lang)) == Direction::Rtl } /// The language for a system locale tag (`fr_CA.UTF-8`, `zh-Hant-MO`); `C` and `POSIX` are @@ -340,7 +341,9 @@ pub fn fmt(template: &str, args: &[(&str, &str)]) -> String { /// Chinese without a script picks one by region; Portuguese falls back to `pt-br` last, when /// there is no `pt`. A glibc script modifier (`sr_RS@latin`) becomes the script subtag. With a /// script (`sr-Latn-RS`), a code without one is tried only when that is its language's usual -/// script, so Latin Serbian never picks Cyrillic `sr`, while `sr-Cyrl-RS` does. +/// script, so Latin Serbian never picks Cyrillic `sr`, while `sr-Cyrl-RS` does. A region that +/// implies a script (CLDR likely subtags) counts as that script after the tag as written: +/// `pa-PK` → `pa-pk`, `pa-arab-pk`, `pa-arab`. fn candidates(tag: &str) -> Vec { let tag = tag.trim().to_ascii_lowercase(); let base = tag.split(['.', '@']).next().unwrap_or("").replace('_', "-"); @@ -357,6 +360,16 @@ fn candidates(tag: &str) -> Vec { parts.insert(1, s); } let script = parts.get(1).copied().filter(|s| s.len() == 4 && s.bytes().all(|b| b.is_ascii_alphabetic())); + // Chinese picks its script by region below, keeping plain `zh` as the last resort. + if script.is_none() + && primary != "zh" + && let Some(implied) = parts.get(1).filter(|r| is_region(r)).and_then(|r| scripts::region_script(primary, r)) + { + let mut out = vec![parts.join("-")]; + parts.insert(1, implied); + out.extend(candidates(&parts.join("-"))); + return out; + } let mut out: Vec = (1..=parts.len()).rev().map(|n| parts.get(..n).unwrap_or_default().join("-")).collect(); if let Some(s) = script && usual_script(primary).is_some_and(|usual| usual != s) @@ -364,7 +377,8 @@ fn candidates(tag: &str) -> Vec { out.retain(|c| c.split('-').any(|p| p == s)); } if primary == "zh" && !parts.iter().any(|p| matches!(*p, "hans" | "hant")) { - let script = if parts.iter().any(|p| matches!(*p, "tw" | "hk" | "mo")) { "zh-hant" } else { "zh-hans" }; + let hant = parts.iter().skip(1).any(|p| scripts::region_script("zh", p) == Some("hant")); + let script = if hant { "zh-hant" } else { "zh-hans" }; out.insert(out.len().saturating_sub(1), script.to_owned()); } if primary == "pt" && !out.iter().any(|c| c == "pt-br") { @@ -393,10 +407,10 @@ fn marked(text: &str) -> String { // ponytail: the multi-script languages a Craft app could plausibly ship; add a row with the locale. fn usual_script(language: &str) -> Option<&'static str> { Some(match language { - "be" | "mn" | "sr" => "cyrl", + "be" | "kk" | "ky" | "mn" | "sr" | "tg" => "cyrl", "zh" => "hans", - "az" | "bs" | "ha" | "uz" => "latn", - "ks" | "sd" => "arab", + "az" | "bs" | "ha" | "ku" | "ms" | "uz" => "latn", + "ks" | "sd" | "ug" => "arab", "pa" => "guru", _ => return None, }) @@ -630,6 +644,8 @@ mod tests { ("zh", Some("zh-hans")), ("zh-Hant-MO", Some("zh-hant")), ("zh_CN", Some("zh-hans")), + ("zh-US", Some("zh-hant")), + ("zh_SG", Some("zh-hans")), ("C", Some("en")), ("POSIX", Some("en")), ("en_US", Some("en")), @@ -641,6 +657,20 @@ mod tests { } } + #[test] + fn region_implies_script() { + assert_eq!(candidates("pa-PK"), ["pa-pk", "pa-arab-pk", "pa-arab"]); + assert_eq!(candidates("pa_PK.UTF-8"), ["pa-pk", "pa-arab-pk", "pa-arab"]); + assert_eq!(candidates("pa-IN"), ["pa-in", "pa"]); + assert_eq!(candidates("sd-IN"), ["sd-in", "sd-deva-in", "sd-deva"]); + assert_eq!(candidates("pa-Guru-PK"), ["pa-guru-pk", "pa-guru", "pa"]); + assert_eq!(candidates("zh-US"), ["zh-us", "zh-hant", "zh"]); + assert_eq!(candidates("zh-SG"), ["zh-sg", "zh-hans", "zh"]); + assert_eq!(candidates("kk-CN"), ["kk-cn", "kk-arab-cn", "kk-arab"]); + assert_eq!(candidates("kk-KZ"), ["kk-kz", "kk"]); + assert_eq!(candidates("ug-KZ"), ["ug-kz", "ug-cyrl-kz", "ug-cyrl"]); + } + #[test] fn match_tags_takes_first_match() { let i = all(L).0; diff --git a/crates/i18n/src/scripts.rs b/crates/i18n/src/scripts.rs new file mode 100644 index 000000000..1ae0f29bc --- /dev/null +++ b/crates/i18n/src/scripts.rs @@ -0,0 +1,455 @@ +//! The scripts a language is written in, its direction, and which regional forms its Han +//! characters take, for picking fonts and laying out the interface. + +/// A writing system, by its ISO 15924 code. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +#[non_exhaustive] +pub enum Script { + Arab, + Armn, + Beng, + Cyrl, + Deva, + Ethi, + Geor, + Grek, + Gujr, + Guru, + Hang, + Hani, + Hebr, + Hira, + Kana, + Khmr, + Knda, + Laoo, + Latn, + Mlym, + Mong, + Mymr, + Orya, + Sinh, + Syrc, + Taml, + Telu, + Thaa, + Thai, + Tibt, +} + +impl Script { + /// The ISO 15924 code, `Latn`. + pub fn code(self) -> &'static str { + match self { + Script::Arab => "Arab", + Script::Armn => "Armn", + Script::Beng => "Beng", + Script::Cyrl => "Cyrl", + Script::Deva => "Deva", + Script::Ethi => "Ethi", + Script::Geor => "Geor", + Script::Grek => "Grek", + Script::Gujr => "Gujr", + Script::Guru => "Guru", + Script::Hang => "Hang", + Script::Hani => "Hani", + Script::Hebr => "Hebr", + Script::Hira => "Hira", + Script::Kana => "Kana", + Script::Khmr => "Khmr", + Script::Knda => "Knda", + Script::Laoo => "Laoo", + Script::Latn => "Latn", + Script::Mlym => "Mlym", + Script::Mong => "Mong", + Script::Mymr => "Mymr", + Script::Orya => "Orya", + Script::Sinh => "Sinh", + Script::Syrc => "Syrc", + Script::Taml => "Taml", + Script::Telu => "Telu", + Script::Thaa => "Thaa", + Script::Thai => "Thai", + Script::Tibt => "Tibt", + } + } + + /// Whether text in this script runs right to left. + pub fn rtl(self) -> bool { + matches!(self, Script::Arab | Script::Hebr | Script::Syrc | Script::Thaa) + } +} + +/// Which way a language's text, and an interface in it, runs. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum Direction { + Ltr, + Rtl, +} + +/// The regional glyph forms Han characters take, as in the Noto CJK face names (SC, TC, HK, JP, +/// KR). The same code point is drawn differently in each, so a font stack picks its Han face by +/// this, not by script. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum Han { + /// Simplified Chinese, mainland China and Singapore (SC). + Simplified, + /// Traditional Chinese, Taiwan (TC). + Traditional, + /// Traditional Chinese, Hong Kong and Macau (HK). + HongKong, + /// Japanese kanji (JP). + Japanese, + /// Korean hanja (KR). + Korean, +} + +/// The scripts a language tag's interface text is written in (`ja` → Hani, Hira, Kana; `sr` → +/// Cyrl; `sr-latn` → Latn), the main one first. Any case and glibc form (`sr_RS.UTF-8@latin`). A +/// script subtag wins, then a region that implies a script (`pa-PK` → Arab), then the language's +/// usual script. Latin product names and units appear in every language, so Latn is listed only +/// where it is the language's own script. A language not in the table is taken to be written in +/// Latin. +// ponytail: the languages a Craft app ships or could plausibly ship; add a row with the locale. +pub fn scripts(tag: &str) -> &'static [Script] { + use Script::*; + let tag = Subtags::parse(tag); + if let Some(s) = tag.script.as_deref().and_then(subtag_scripts) { + return s; + } + if let Some(s) = tag.region.as_deref().and_then(|region| region_script(&tag.language, region)).and_then(subtag_scripts) { + return s; + } + match tag.language.as_str() { + "ja" => &[Hani, Hira, Kana], + "ko" => &[Hang, Hani], + "zh" | "yue" | "wuu" => &[Hani], + "be" | "bg" | "ba" | "ce" | "cv" | "kk" | "ky" | "mk" | "mn" | "ru" | "sah" | "sr" | "tg" | "tt" | "uk" => &[Cyrl], + "el" => &[Grek], + "ar" | "ckb" | "fa" | "ks" | "ps" | "sd" | "ug" | "ur" => &[Arab], + "he" | "yi" => &[Hebr], + "syr" => &[Syrc], + "dv" => &[Thaa], + "ka" => &[Geor], + "hy" => &[Armn], + "hi" | "mr" | "ne" | "sa" | "kok" | "mai" | "bho" => &[Deva], + "bn" | "as" | "mni" => &[Beng], + "pa" => &[Guru], + "gu" => &[Gujr], + "or" => &[Orya], + "ta" => &[Taml], + "te" => &[Telu], + "kn" => &[Knda], + "ml" => &[Mlym], + "si" => &[Sinh], + "th" => &[Thai], + "lo" => &[Laoo], + "km" => &[Khmr], + "my" => &[Mymr], + "am" | "ti" => &[Ethi], + "bo" | "dz" => &[Tibt], + _ => &[Latn], + } +} + +/// The direction of a language tag's text: right to left when its main script is. +pub fn direction(tag: &str) -> Direction { + match scripts(tag).first() { + Some(s) if s.rtl() => Direction::Rtl, + _ => Direction::Ltr, + } +} + +/// The direction of a paragraph of text, from its first strong character (UAX #9 rules P2 and +/// P3): the contents of isolates (LRI, RLI or FSI up to the matching PDI) are skipped, and the +/// search stops at the first paragraph separator. `None` when the paragraph has no strong +/// character; P3 then reads it as left to right, and an app may use the interface direction. +pub fn base_direction(text: &str) -> Option { + match unicode_bidi::get_base_direction(text) { + unicode_bidi::Direction::Ltr => Some(Direction::Ltr), + unicode_bidi::Direction::Rtl => Some(Direction::Rtl), + unicode_bidi::Direction::Mixed => None, + } +} + +/// The forms a language tag's Han characters take: `zh-CN`, `zh-SG`, `zh-Hans` and plain `zh` → +/// Simplified; `zh-TW`, `zh-Hant` and the other regions CLDR writes in Traditional characters +/// (`zh-US`, `zh-AU`) → Traditional; `zh-HK`, `zh-MO`, `zh-Hant-HK` and `yue` → HongKong; `ja` → +/// Japanese; `ko` → Korean. `None` for a language not written with Han, which leaves the choice +/// to the font stack. +pub fn han(tag: &str) -> Option { + let tag = Subtags::parse(tag); + let hong_kong = matches!(tag.region.as_deref(), Some("hk" | "mo")); + // `Hani` names Han without saying which forms, so the region still decides. + let script = tag.script.as_deref().filter(|s| *s != "hani"); + let script = script.or_else(|| region_script(&tag.language, tag.region.as_deref()?)); + match (tag.language.as_str(), script) { + ("ja", _) => Some(Han::Japanese), + ("ko", _) => Some(Han::Korean), + ("zh" | "yue" | "wuu", Some("hans")) => Some(Han::Simplified), + ("yue", Some("hant")) => Some(if tag.region.as_deref() == Some("tw") { Han::Traditional } else { Han::HongKong }), + ("zh" | "wuu", Some("hant")) => Some(if hong_kong { Han::HongKong } else { Han::Traditional }), + // Cantonese is written in Traditional characters with Hong Kong forms unless it says otherwise. + ("yue", _) => Some(Han::HongKong), + ("zh" | "wuu", _) => Some(Han::Simplified), + _ => None, + } +} + +/// The script subtag of a language in a region where CLDR's likely subtags give it a script +/// other than its usual one: `pa-PK` → `arab` (`pa-Arab-PK`), `zh-TW` → `hant`. +// ponytail: mn-CN → Mong is left out until vertical Mongolian layout is supported. +pub(crate) fn region_script(language: &str, region: &str) -> Option<&'static str> { + Some(match (language, region) { + ("az", "iq" | "ir") | ("ha", "cm" | "sd") | ("kk", "af" | "cn" | "ir" | "mn") | ("ku", "iq" | "ir" | "lb") => "arab", + ("ky", "cn") | ("ms", "cc") | ("pa", "pk") | ("tg", "pk") | ("uz", "af") => "arab", + ("az", "ru") | ("ku", "am" | "az" | "ge" | "tm") | ("ug", "kz" | "mn") | ("uz", "cn") => "cyrl", + ("ky", "tr") | ("sr", "me" | "ro" | "tr") => "latn", + ("sd", "in") => "deva", + ("zh", "au" | "bn" | "gb" | "gf" | "hk" | "id" | "mo" | "pa" | "pf" | "ph" | "sr" | "th" | "tw" | "us" | "vn") => "hant", + ("yue", "cn") => "hans", + _ => return None, + }) +} + +/// The scripts a script subtag stands for, including the CLDR unions (`jpan`, `kore`). `None` for +/// one not in [`Script`], so the language's own scripts apply. +fn subtag_scripts(subtag: &str) -> Option<&'static [Script]> { + use Script::*; + Some(match subtag { + "hans" | "hant" | "hani" => &[Hani], + "jpan" => &[Hani, Hira, Kana], + "kore" => &[Hang, Hani], + "arab" => &[Arab], + "armn" => &[Armn], + "beng" => &[Beng], + "cyrl" => &[Cyrl], + "deva" => &[Deva], + "ethi" => &[Ethi], + "geor" => &[Geor], + "grek" => &[Grek], + "gujr" => &[Gujr], + "guru" => &[Guru], + "hang" => &[Hang], + "hebr" => &[Hebr], + "khmr" => &[Khmr], + "knda" => &[Knda], + "laoo" => &[Laoo], + "latn" => &[Latn], + "mlym" => &[Mlym], + "mong" => &[Mong], + "mymr" => &[Mymr], + "orya" => &[Orya], + "sinh" => &[Sinh], + "syrc" => &[Syrc], + "taml" => &[Taml], + "telu" => &[Telu], + "thaa" => &[Thaa], + "thai" => &[Thai], + "tibt" => &[Tibt], + _ => return None, + }) +} + +/// A language tag's language, script and region subtags, lowercase. +struct Subtags { + language: String, + script: Option, + region: Option, +} + +impl Subtags { + /// `zh-Hant-HK`, `zh_TW.UTF-8` or `sr_RS@latin` (glibc's script modifier). + fn parse(tag: &str) -> Subtags { + let tag = tag.trim().to_ascii_lowercase(); + let base = tag.split(['.', '@']).next().unwrap_or("").replace('_', "-"); + let mut parts = base.split('-').filter(|p| !p.is_empty()); + let language = parts.next().unwrap_or("").to_owned(); + let mut script = match tag.split_once('@').map(|(_, m)| m) { + Some("latin") => Some("latn".to_owned()), + Some("cyrillic") => Some("cyrl".to_owned()), + _ => None, + }; + let mut region = None; + for part in parts { + if part.len() == 4 && part.bytes().all(|b| b.is_ascii_alphabetic()) { + script.get_or_insert_with(|| part.to_owned()); + } else if super::is_region(part) { + region = Some(part.to_owned()); + break; + } + } + Subtags { language, script, region } + } +} + +#[cfg(test)] +mod tests { + use super::Script::*; + use super::*; + + #[test] + fn base_direction_of_text() { + assert_eq!(base_direction("Hello"), Some(Direction::Ltr)); + assert_eq!(base_direction("123 שלום"), Some(Direction::Rtl)); + assert_eq!(base_direction("(مرحبا) hi"), Some(Direction::Rtl)); + // The isolate's Hebrew is skipped; the Latin after it decides. + assert_eq!(base_direction("\u{2067}שלום\u{2069} hello"), Some(Direction::Ltr)); + assert_eq!(base_direction("\u{2068}abc\u{2069} שלום"), Some(Direction::Rtl)); + // Only the first paragraph counts. + assert_eq!(base_direction("123\nשלום"), None); + assert_eq!(base_direction("1920 × 1080"), None); + assert_eq!(base_direction(""), None); + } + + /// Every locale the craft-i18n catalogs or an app ship, as the apps spell them. + #[test] + fn shipped_locales() { + for (tag, want, han_forms) in [ + ("en", &[Latn][..], None), + ("ar", &[Arab], None), + ("bg", &[Cyrl], None), + ("cs", &[Latn], None), + ("da", &[Latn], None), + ("de", &[Latn], None), + ("el", &[Grek], None), + ("es", &[Latn], None), + ("es-419", &[Latn], None), + ("et", &[Latn], None), + ("fi", &[Latn], None), + ("fr", &[Latn], None), + ("hr", &[Latn], None), + ("hu", &[Latn], None), + ("id", &[Latn], None), + ("it", &[Latn], None), + ("ja", &[Hani, Hira, Kana], Some(Han::Japanese)), + ("ka", &[Geor], None), + ("kk", &[Cyrl], None), + ("ko", &[Hang, Hani], Some(Han::Korean)), + ("lt", &[Latn], None), + ("nl", &[Latn], None), + ("pl", &[Latn], None), + ("pt", &[Latn], None), + ("pt-br", &[Latn], None), + ("pt_BR", &[Latn], None), + ("ru", &[Cyrl], None), + ("sl", &[Latn], None), + ("sr", &[Cyrl], None), + ("sr-latn", &[Latn], None), + ("sv", &[Latn], None), + ("te", &[Telu], None), + ("tr", &[Latn], None), + ("uk", &[Cyrl], None), + ("vi", &[Latn], None), + ("zh", &[Hani], Some(Han::Simplified)), + ("zh-cn", &[Hani], Some(Han::Simplified)), + ("zh-CN", &[Hani], Some(Han::Simplified)), + ("zh-hans", &[Hani], Some(Han::Simplified)), + ("zh-Hans", &[Hani], Some(Han::Simplified)), + ("zh-hant", &[Hani], Some(Han::Traditional)), + ] { + assert_eq!(scripts(tag), want, "{tag}"); + assert_eq!(han(tag), han_forms, "{tag}"); + let rtl = tag == "ar"; + assert_eq!(direction(tag), if rtl { Direction::Rtl } else { Direction::Ltr }, "{tag}"); + } + } + + #[test] + fn regional_han() { + for (tag, want) in [ + ("zh-TW", Han::Traditional), + ("zh_TW.UTF-8", Han::Traditional), + ("zh-Hant-TW", Han::Traditional), + ("zh-HK", Han::HongKong), + ("zh-MO", Han::HongKong), + ("zh-Hant-HK", Han::HongKong), + ("zh-hant-hk", Han::HongKong), + ("zh-Hans-HK", Han::Simplified), + ("zh-SG", Han::Simplified), + ("zh-US", Han::Traditional), + ("zh_AU.UTF-8", Han::Traditional), + ("zh-GB", Han::Traditional), + ("zh-Hans-US", Han::Simplified), + ("zh-MY", Han::Simplified), + ("yue-CN", Han::Simplified), + ("yue-HK", Han::HongKong), + ("zh-Hans-SG", Han::Simplified), + ("yue", Han::HongKong), + ("yue-Hant", Han::HongKong), + ("yue-Hant-TW", Han::Traditional), + ("zh-Hani-HK", Han::HongKong), + ("zh-Hani", Han::Simplified), + ("yue-Hans", Han::Simplified), + ("ja-JP", Han::Japanese), + ("ko_KR.UTF-8", Han::Korean), + ] { + assert_eq!(han(tag), Some(want), "{tag}"); + } + assert_eq!(han("en"), None); + assert_eq!(han(""), None); + } + + #[test] + fn script_subtag_wins() { + for (tag, want) in [ + ("sr-Latn-RS", &[Latn][..]), + ("sr_RS@latin", &[Latn]), + ("sr-Cyrl", &[Cyrl]), + ("sr_RS.UTF-8", &[Cyrl]), + ("az-Cyrl", &[Cyrl]), + ("pa-Arab", &[Arab]), + ("pa", &[Guru]), + ("mn-Mong", &[Mong]), + ("sd-Deva", &[Deva]), + ("fr-Zyyy", &[Latn]), + ] { + assert_eq!(scripts(tag), want, "{tag}"); + } + assert_eq!(direction("pa-Arab"), Direction::Rtl); + assert_eq!(direction("sd-Deva"), Direction::Ltr); + assert_eq!(direction("he_IL.UTF-8"), Direction::Rtl); + } + + /// CLDR likely subtags: a region can switch a language to another script. + #[test] + fn region_implies_script() { + for (tag, want) in [ + ("pa-PK", &[Arab][..]), + ("pa_PK.UTF-8", &[Arab]), + ("pa-IN", &[Guru]), + ("uz-AF", &[Arab]), + ("uz-UZ", &[Latn]), + ("uz-CN", &[Cyrl]), + ("az-IR", &[Arab]), + ("az-AZ", &[Latn]), + ("sd-IN", &[Deva]), + ("sd-PK", &[Arab]), + ("ks-IN", &[Arab]), + ("ms-CC", &[Arab]), + ("ms-BN", &[Latn]), + ("sr-ME", &[Latn]), + ("kk-CN", &[Arab]), + ("ug-KZ", &[Cyrl]), + ("pa-Guru-PK", &[Guru]), + ("sd-Arab-IN", &[Arab]), + ] { + assert_eq!(scripts(tag), want, "{tag}"); + } + assert_eq!(direction("pa-PK"), Direction::Rtl); + assert_eq!(direction("uz-AF"), Direction::Rtl); + assert_eq!(direction("az-IR"), Direction::Rtl); + assert_eq!(direction("sd-IN"), Direction::Ltr); + assert_eq!(direction("ug-KZ"), Direction::Ltr); + } + + /// The usual scripts tag matching relies on agree with the table. + #[test] + fn agrees_with_usual_script() { + for language in ["be", "kk", "ky", "mn", "sr", "tg", "zh", "az", "bs", "ha", "ku", "ms", "uz", "ks", "sd", "ug", "pa"] { + let usual = crate::usual_script(language).and_then(subtag_scripts); + assert_eq!(usual, Some(scripts(language)), "{language}"); + } + } +}