mirror of
https://github.com/storytold/craft-libs.git
synced 2026-11-19 16:55:21 +00:00
i18n: scripts, direction and regional Han per language tag (#200)
* i18n: scripts, direction and regional Han per language tag scripts(tag) lists the ISO 15924 scripts a language's interface text is written in, direction(tag) follows the main script, and han(tag) gives the regional Han forms (SC, TC, HK, JP, KR) for picking a CJK face. I18n::rtl uses direction. * i18n: non_exhaustive Script and Han; regions that imply a script pa-PK, uz-AF and az-IR are written in Arabic and sd-IN in Devanagari, as CLDR's likely subtags give them. A script subtag still wins. * i18n: han and catalog matching follow region-implied scripts han(zh-US), zh-AU and the other regions CLDR writes in Traditional characters give Traditional, and pa-PK picks a pa-arab catalog, never Gurmukhi pa, as pa-Arab-PK does. * i18n: Hani and yue-Hant pick Han forms by region; usual scripts for kk, ky, tg, ku, ms, ug * i18n: base_direction of a paragraph from its first strong character
This commit is contained in:
1 parent
3832c47ef6
commit
5354479ce3
5 files changed
+507
-16
No files matched your search
Generated
+1
@@ -741,6 +741,7 @@ dependencies = [
|
||||
"log",
|
||||
"sys-locale",
|
||||
"unic-langid",
|
||||
"unicode-bidi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
@@ -12,6 +12,7 @@ intl_pluralrules = "=7.0.2"
|
||||
unic-langid = "0.9"
|
||||
log = "0.4"
|
||||
sys-locale = "0.3"
|
||||
unicode-bidi = "0.3"
|
||||
|
||||
[target.'cfg(target_arch = "wasm32")'.dependencies]
|
||||
sys-locale = { version = "0.3", features = ["js"] }
|
||||
|
||||
+10
-6
@@ -62,14 +62,18 @@ The app keeps its `tl!` macro, its language setting (`"auto"` → `I18N.system_l
|
||||
`plural_form_counts` holds the same counts.
|
||||
- **Names:** a language's name comes from the app catalog's `@id language.name` row, else the
|
||||
shared catalog's, else its code; English is "English".
|
||||
- **Right to left:** `rtl` is true for `ar`, `ckb`, `dv`, `fa`, `he`, `ps`, `sd`, `ug`, `ur` and
|
||||
`yi`; putting text into display order is the app's job.
|
||||
- **Scripts and direction:** for any language tag, `scripts` lists the ISO 15924 scripts its
|
||||
interface text is written in (`ja` → Hani, Hira, Kana; `sr-latn` → Latn; `pa-PK` → Arab),
|
||||
`direction` is right to left when the main script is (Arabic, Hebrew, Syriac, Thaana), and `han`
|
||||
gives the regional forms of its Han characters for picking a CJK face: `zh-CN`/`zh-SG`/`zh-Hans` →
|
||||
SC, `zh-TW`/`zh-US`/`zh-Hant` → TC, `zh-HK`/`zh-MO` → HK, `ja` → JP, `ko` → KR. `I18n::rtl` is
|
||||
`direction` of the language's code; putting text into display order is the app's job.
|
||||
- **System locale:** `system_lang` matches the system's list once and caches it: on desktop Unix
|
||||
`LANGUAGE`, `LC_ALL`, `LC_MESSAGES` and `LANG`, read raw; elsewhere `sys-locale`'s (macOS and
|
||||
Windows preferred languages, the browser's on the web). `fr_CA.UTF-8` → `fr`, `zh-TW` →
|
||||
`zh-hant`, `pt-PT` → `pt-br` when that is the only Portuguese, `C`/`POSIX` → English. A script
|
||||
is never crossed: `sr_RS@latin` and `sr-Latn-RS` pick `sr-latn`, never `sr`, while `sr-Cyrl-RS`
|
||||
picks `sr`.
|
||||
Windows preferred languages, the browser's on the web). `fr_CA.UTF-8` → `fr`, `zh-TW` → `zh-hant`,
|
||||
`pt-PT` → `pt-br` when that is the only Portuguese, `C`/`POSIX` → English. A script is never
|
||||
crossed: `sr_RS@latin` and `sr-Latn-RS` pick `sr-latn`, never `sr`, while `sr-Cyrl-RS` picks `sr`.
|
||||
A region that implies a script counts as that script: `pa-PK` picks `pa-arab`, not Gurmukhi `pa`.
|
||||
- **Pseudo locale** (`pseudo`, debug builds only, never in `languages()`): every lookup, `get`
|
||||
included, comes back as `⟦…⟧`, line by line, so text drawn without the marks never went through
|
||||
translation. `looked_up()` lists the source texts looked up so far, for an audit against a
|
||||
|
||||
+40
-10
@@ -6,6 +6,7 @@
|
||||
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
|
||||
|
||||
mod catalog;
|
||||
mod scripts;
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet, HashMap};
|
||||
use std::sync::{Mutex, OnceLock, PoisonError};
|
||||
@@ -13,6 +14,8 @@ use std::sync::{Mutex, OnceLock, PoisonError};
|
||||
use intl_pluralrules::{PluralCategory, PluralRuleType, PluralRules};
|
||||
use unic_langid::LanguageIdentifier;
|
||||
|
||||
pub use scripts::{Direction, Han, Script, base_direction, direction, han, scripts};
|
||||
|
||||
/// The catalogs an app embeds, one entry per locale: (locale, shared catalog, the app's catalog).
|
||||
/// A missing catalog is `""`.
|
||||
pub type Embedded = [(&'static str, &'static str, &'static str)];
|
||||
@@ -248,12 +251,10 @@ impl I18n {
|
||||
self.locale(lang).and_then(Locale::language_name).unwrap_or_else(|| self.code(lang))
|
||||
}
|
||||
|
||||
/// Whether the language is written right to left. Putting text into display order is the
|
||||
/// app's job.
|
||||
/// Whether the language is written right to left ([`direction`]). Putting text into display
|
||||
/// order is the app's job.
|
||||
pub fn rtl(&self, lang: Lang) -> bool {
|
||||
let code = self.code(lang);
|
||||
let language = code.split('-').next().unwrap_or(code);
|
||||
matches!(language, "ar" | "ckb" | "dv" | "fa" | "he" | "ps" | "sd" | "ug" | "ur" | "yi")
|
||||
direction(self.code(lang)) == Direction::Rtl
|
||||
}
|
||||
|
||||
/// The language for a system locale tag (`fr_CA.UTF-8`, `zh-Hant-MO`); `C` and `POSIX` are
|
||||
@@ -340,7 +341,9 @@ pub fn fmt(template: &str, args: &[(&str, &str)]) -> String {
|
||||
/// Chinese without a script picks one by region; Portuguese falls back to `pt-br` last, when
|
||||
/// there is no `pt`. A glibc script modifier (`sr_RS@latin`) becomes the script subtag. With a
|
||||
/// script (`sr-Latn-RS`), a code without one is tried only when that is its language's usual
|
||||
/// script, so Latin Serbian never picks Cyrillic `sr`, while `sr-Cyrl-RS` does.
|
||||
/// script, so Latin Serbian never picks Cyrillic `sr`, while `sr-Cyrl-RS` does. A region that
|
||||
/// implies a script (CLDR likely subtags) counts as that script after the tag as written:
|
||||
/// `pa-PK` → `pa-pk`, `pa-arab-pk`, `pa-arab`.
|
||||
fn candidates(tag: &str) -> Vec<String> {
|
||||
let tag = tag.trim().to_ascii_lowercase();
|
||||
let base = tag.split(['.', '@']).next().unwrap_or("").replace('_', "-");
|
||||
@@ -357,6 +360,16 @@ fn candidates(tag: &str) -> Vec<String> {
|
||||
parts.insert(1, s);
|
||||
}
|
||||
let script = parts.get(1).copied().filter(|s| s.len() == 4 && s.bytes().all(|b| b.is_ascii_alphabetic()));
|
||||
// Chinese picks its script by region below, keeping plain `zh` as the last resort.
|
||||
if script.is_none()
|
||||
&& primary != "zh"
|
||||
&& let Some(implied) = parts.get(1).filter(|r| is_region(r)).and_then(|r| scripts::region_script(primary, r))
|
||||
{
|
||||
let mut out = vec![parts.join("-")];
|
||||
parts.insert(1, implied);
|
||||
out.extend(candidates(&parts.join("-")));
|
||||
return out;
|
||||
}
|
||||
let mut out: Vec<String> = (1..=parts.len()).rev().map(|n| parts.get(..n).unwrap_or_default().join("-")).collect();
|
||||
if let Some(s) = script
|
||||
&& usual_script(primary).is_some_and(|usual| usual != s)
|
||||
@@ -364,7 +377,8 @@ fn candidates(tag: &str) -> Vec<String> {
|
||||
out.retain(|c| c.split('-').any(|p| p == s));
|
||||
}
|
||||
if primary == "zh" && !parts.iter().any(|p| matches!(*p, "hans" | "hant")) {
|
||||
let script = if parts.iter().any(|p| matches!(*p, "tw" | "hk" | "mo")) { "zh-hant" } else { "zh-hans" };
|
||||
let hant = parts.iter().skip(1).any(|p| scripts::region_script("zh", p) == Some("hant"));
|
||||
let script = if hant { "zh-hant" } else { "zh-hans" };
|
||||
out.insert(out.len().saturating_sub(1), script.to_owned());
|
||||
}
|
||||
if primary == "pt" && !out.iter().any(|c| c == "pt-br") {
|
||||
@@ -393,10 +407,10 @@ fn marked(text: &str) -> String {
|
||||
// ponytail: the multi-script languages a Craft app could plausibly ship; add a row with the locale.
|
||||
fn usual_script(language: &str) -> Option<&'static str> {
|
||||
Some(match language {
|
||||
"be" | "mn" | "sr" => "cyrl",
|
||||
"be" | "kk" | "ky" | "mn" | "sr" | "tg" => "cyrl",
|
||||
"zh" => "hans",
|
||||
"az" | "bs" | "ha" | "uz" => "latn",
|
||||
"ks" | "sd" => "arab",
|
||||
"az" | "bs" | "ha" | "ku" | "ms" | "uz" => "latn",
|
||||
"ks" | "sd" | "ug" => "arab",
|
||||
"pa" => "guru",
|
||||
_ => return None,
|
||||
})
|
||||
@@ -630,6 +644,8 @@ mod tests {
|
||||
("zh", Some("zh-hans")),
|
||||
("zh-Hant-MO", Some("zh-hant")),
|
||||
("zh_CN", Some("zh-hans")),
|
||||
("zh-US", Some("zh-hant")),
|
||||
("zh_SG", Some("zh-hans")),
|
||||
("C", Some("en")),
|
||||
("POSIX", Some("en")),
|
||||
("en_US", Some("en")),
|
||||
@@ -641,6 +657,20 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn region_implies_script() {
|
||||
assert_eq!(candidates("pa-PK"), ["pa-pk", "pa-arab-pk", "pa-arab"]);
|
||||
assert_eq!(candidates("pa_PK.UTF-8"), ["pa-pk", "pa-arab-pk", "pa-arab"]);
|
||||
assert_eq!(candidates("pa-IN"), ["pa-in", "pa"]);
|
||||
assert_eq!(candidates("sd-IN"), ["sd-in", "sd-deva-in", "sd-deva"]);
|
||||
assert_eq!(candidates("pa-Guru-PK"), ["pa-guru-pk", "pa-guru", "pa"]);
|
||||
assert_eq!(candidates("zh-US"), ["zh-us", "zh-hant", "zh"]);
|
||||
assert_eq!(candidates("zh-SG"), ["zh-sg", "zh-hans", "zh"]);
|
||||
assert_eq!(candidates("kk-CN"), ["kk-cn", "kk-arab-cn", "kk-arab"]);
|
||||
assert_eq!(candidates("kk-KZ"), ["kk-kz", "kk"]);
|
||||
assert_eq!(candidates("ug-KZ"), ["ug-kz", "ug-cyrl-kz", "ug-cyrl"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn match_tags_takes_first_match() {
|
||||
let i = all(L).0;
|
||||
|
||||
@@ -0,0 +1,455 @@
|
||||
//! The scripts a language is written in, its direction, and which regional forms its Han
|
||||
//! characters take, for picking fonts and laying out the interface.
|
||||
|
||||
/// A writing system, by its ISO 15924 code.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
|
||||
#[non_exhaustive]
|
||||
pub enum Script {
|
||||
Arab,
|
||||
Armn,
|
||||
Beng,
|
||||
Cyrl,
|
||||
Deva,
|
||||
Ethi,
|
||||
Geor,
|
||||
Grek,
|
||||
Gujr,
|
||||
Guru,
|
||||
Hang,
|
||||
Hani,
|
||||
Hebr,
|
||||
Hira,
|
||||
Kana,
|
||||
Khmr,
|
||||
Knda,
|
||||
Laoo,
|
||||
Latn,
|
||||
Mlym,
|
||||
Mong,
|
||||
Mymr,
|
||||
Orya,
|
||||
Sinh,
|
||||
Syrc,
|
||||
Taml,
|
||||
Telu,
|
||||
Thaa,
|
||||
Thai,
|
||||
Tibt,
|
||||
}
|
||||
|
||||
impl Script {
|
||||
/// The ISO 15924 code, `Latn`.
|
||||
pub fn code(self) -> &'static str {
|
||||
match self {
|
||||
Script::Arab => "Arab",
|
||||
Script::Armn => "Armn",
|
||||
Script::Beng => "Beng",
|
||||
Script::Cyrl => "Cyrl",
|
||||
Script::Deva => "Deva",
|
||||
Script::Ethi => "Ethi",
|
||||
Script::Geor => "Geor",
|
||||
Script::Grek => "Grek",
|
||||
Script::Gujr => "Gujr",
|
||||
Script::Guru => "Guru",
|
||||
Script::Hang => "Hang",
|
||||
Script::Hani => "Hani",
|
||||
Script::Hebr => "Hebr",
|
||||
Script::Hira => "Hira",
|
||||
Script::Kana => "Kana",
|
||||
Script::Khmr => "Khmr",
|
||||
Script::Knda => "Knda",
|
||||
Script::Laoo => "Laoo",
|
||||
Script::Latn => "Latn",
|
||||
Script::Mlym => "Mlym",
|
||||
Script::Mong => "Mong",
|
||||
Script::Mymr => "Mymr",
|
||||
Script::Orya => "Orya",
|
||||
Script::Sinh => "Sinh",
|
||||
Script::Syrc => "Syrc",
|
||||
Script::Taml => "Taml",
|
||||
Script::Telu => "Telu",
|
||||
Script::Thaa => "Thaa",
|
||||
Script::Thai => "Thai",
|
||||
Script::Tibt => "Tibt",
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether text in this script runs right to left.
|
||||
pub fn rtl(self) -> bool {
|
||||
matches!(self, Script::Arab | Script::Hebr | Script::Syrc | Script::Thaa)
|
||||
}
|
||||
}
|
||||
|
||||
/// Which way a language's text, and an interface in it, runs.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
|
||||
pub enum Direction {
|
||||
Ltr,
|
||||
Rtl,
|
||||
}
|
||||
|
||||
/// The regional glyph forms Han characters take, as in the Noto CJK face names (SC, TC, HK, JP,
|
||||
/// KR). The same code point is drawn differently in each, so a font stack picks its Han face by
|
||||
/// this, not by script.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
|
||||
#[non_exhaustive]
|
||||
pub enum Han {
|
||||
/// Simplified Chinese, mainland China and Singapore (SC).
|
||||
Simplified,
|
||||
/// Traditional Chinese, Taiwan (TC).
|
||||
Traditional,
|
||||
/// Traditional Chinese, Hong Kong and Macau (HK).
|
||||
HongKong,
|
||||
/// Japanese kanji (JP).
|
||||
Japanese,
|
||||
/// Korean hanja (KR).
|
||||
Korean,
|
||||
}
|
||||
|
||||
/// The scripts a language tag's interface text is written in (`ja` → Hani, Hira, Kana; `sr` →
|
||||
/// Cyrl; `sr-latn` → Latn), the main one first. Any case and glibc form (`sr_RS.UTF-8@latin`). A
|
||||
/// script subtag wins, then a region that implies a script (`pa-PK` → Arab), then the language's
|
||||
/// usual script. Latin product names and units appear in every language, so Latn is listed only
|
||||
/// where it is the language's own script. A language not in the table is taken to be written in
|
||||
/// Latin.
|
||||
// ponytail: the languages a Craft app ships or could plausibly ship; add a row with the locale.
|
||||
pub fn scripts(tag: &str) -> &'static [Script] {
|
||||
use Script::*;
|
||||
let tag = Subtags::parse(tag);
|
||||
if let Some(s) = tag.script.as_deref().and_then(subtag_scripts) {
|
||||
return s;
|
||||
}
|
||||
if let Some(s) = tag.region.as_deref().and_then(|region| region_script(&tag.language, region)).and_then(subtag_scripts) {
|
||||
return s;
|
||||
}
|
||||
match tag.language.as_str() {
|
||||
"ja" => &[Hani, Hira, Kana],
|
||||
"ko" => &[Hang, Hani],
|
||||
"zh" | "yue" | "wuu" => &[Hani],
|
||||
"be" | "bg" | "ba" | "ce" | "cv" | "kk" | "ky" | "mk" | "mn" | "ru" | "sah" | "sr" | "tg" | "tt" | "uk" => &[Cyrl],
|
||||
"el" => &[Grek],
|
||||
"ar" | "ckb" | "fa" | "ks" | "ps" | "sd" | "ug" | "ur" => &[Arab],
|
||||
"he" | "yi" => &[Hebr],
|
||||
"syr" => &[Syrc],
|
||||
"dv" => &[Thaa],
|
||||
"ka" => &[Geor],
|
||||
"hy" => &[Armn],
|
||||
"hi" | "mr" | "ne" | "sa" | "kok" | "mai" | "bho" => &[Deva],
|
||||
"bn" | "as" | "mni" => &[Beng],
|
||||
"pa" => &[Guru],
|
||||
"gu" => &[Gujr],
|
||||
"or" => &[Orya],
|
||||
"ta" => &[Taml],
|
||||
"te" => &[Telu],
|
||||
"kn" => &[Knda],
|
||||
"ml" => &[Mlym],
|
||||
"si" => &[Sinh],
|
||||
"th" => &[Thai],
|
||||
"lo" => &[Laoo],
|
||||
"km" => &[Khmr],
|
||||
"my" => &[Mymr],
|
||||
"am" | "ti" => &[Ethi],
|
||||
"bo" | "dz" => &[Tibt],
|
||||
_ => &[Latn],
|
||||
}
|
||||
}
|
||||
|
||||
/// The direction of a language tag's text: right to left when its main script is.
|
||||
pub fn direction(tag: &str) -> Direction {
|
||||
match scripts(tag).first() {
|
||||
Some(s) if s.rtl() => Direction::Rtl,
|
||||
_ => Direction::Ltr,
|
||||
}
|
||||
}
|
||||
|
||||
/// The direction of a paragraph of text, from its first strong character (UAX #9 rules P2 and
|
||||
/// P3): the contents of isolates (LRI, RLI or FSI up to the matching PDI) are skipped, and the
|
||||
/// search stops at the first paragraph separator. `None` when the paragraph has no strong
|
||||
/// character; P3 then reads it as left to right, and an app may use the interface direction.
|
||||
pub fn base_direction(text: &str) -> Option<Direction> {
|
||||
match unicode_bidi::get_base_direction(text) {
|
||||
unicode_bidi::Direction::Ltr => Some(Direction::Ltr),
|
||||
unicode_bidi::Direction::Rtl => Some(Direction::Rtl),
|
||||
unicode_bidi::Direction::Mixed => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// The forms a language tag's Han characters take: `zh-CN`, `zh-SG`, `zh-Hans` and plain `zh` →
|
||||
/// Simplified; `zh-TW`, `zh-Hant` and the other regions CLDR writes in Traditional characters
|
||||
/// (`zh-US`, `zh-AU`) → Traditional; `zh-HK`, `zh-MO`, `zh-Hant-HK` and `yue` → HongKong; `ja` →
|
||||
/// Japanese; `ko` → Korean. `None` for a language not written with Han, which leaves the choice
|
||||
/// to the font stack.
|
||||
pub fn han(tag: &str) -> Option<Han> {
|
||||
let tag = Subtags::parse(tag);
|
||||
let hong_kong = matches!(tag.region.as_deref(), Some("hk" | "mo"));
|
||||
// `Hani` names Han without saying which forms, so the region still decides.
|
||||
let script = tag.script.as_deref().filter(|s| *s != "hani");
|
||||
let script = script.or_else(|| region_script(&tag.language, tag.region.as_deref()?));
|
||||
match (tag.language.as_str(), script) {
|
||||
("ja", _) => Some(Han::Japanese),
|
||||
("ko", _) => Some(Han::Korean),
|
||||
("zh" | "yue" | "wuu", Some("hans")) => Some(Han::Simplified),
|
||||
("yue", Some("hant")) => Some(if tag.region.as_deref() == Some("tw") { Han::Traditional } else { Han::HongKong }),
|
||||
("zh" | "wuu", Some("hant")) => Some(if hong_kong { Han::HongKong } else { Han::Traditional }),
|
||||
// Cantonese is written in Traditional characters with Hong Kong forms unless it says otherwise.
|
||||
("yue", _) => Some(Han::HongKong),
|
||||
("zh" | "wuu", _) => Some(Han::Simplified),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// The script subtag of a language in a region where CLDR's likely subtags give it a script
|
||||
/// other than its usual one: `pa-PK` → `arab` (`pa-Arab-PK`), `zh-TW` → `hant`.
|
||||
// ponytail: mn-CN → Mong is left out until vertical Mongolian layout is supported.
|
||||
pub(crate) fn region_script(language: &str, region: &str) -> Option<&'static str> {
|
||||
Some(match (language, region) {
|
||||
("az", "iq" | "ir") | ("ha", "cm" | "sd") | ("kk", "af" | "cn" | "ir" | "mn") | ("ku", "iq" | "ir" | "lb") => "arab",
|
||||
("ky", "cn") | ("ms", "cc") | ("pa", "pk") | ("tg", "pk") | ("uz", "af") => "arab",
|
||||
("az", "ru") | ("ku", "am" | "az" | "ge" | "tm") | ("ug", "kz" | "mn") | ("uz", "cn") => "cyrl",
|
||||
("ky", "tr") | ("sr", "me" | "ro" | "tr") => "latn",
|
||||
("sd", "in") => "deva",
|
||||
("zh", "au" | "bn" | "gb" | "gf" | "hk" | "id" | "mo" | "pa" | "pf" | "ph" | "sr" | "th" | "tw" | "us" | "vn") => "hant",
|
||||
("yue", "cn") => "hans",
|
||||
_ => return None,
|
||||
})
|
||||
}
|
||||
|
||||
/// The scripts a script subtag stands for, including the CLDR unions (`jpan`, `kore`). `None` for
|
||||
/// one not in [`Script`], so the language's own scripts apply.
|
||||
fn subtag_scripts(subtag: &str) -> Option<&'static [Script]> {
|
||||
use Script::*;
|
||||
Some(match subtag {
|
||||
"hans" | "hant" | "hani" => &[Hani],
|
||||
"jpan" => &[Hani, Hira, Kana],
|
||||
"kore" => &[Hang, Hani],
|
||||
"arab" => &[Arab],
|
||||
"armn" => &[Armn],
|
||||
"beng" => &[Beng],
|
||||
"cyrl" => &[Cyrl],
|
||||
"deva" => &[Deva],
|
||||
"ethi" => &[Ethi],
|
||||
"geor" => &[Geor],
|
||||
"grek" => &[Grek],
|
||||
"gujr" => &[Gujr],
|
||||
"guru" => &[Guru],
|
||||
"hang" => &[Hang],
|
||||
"hebr" => &[Hebr],
|
||||
"khmr" => &[Khmr],
|
||||
"knda" => &[Knda],
|
||||
"laoo" => &[Laoo],
|
||||
"latn" => &[Latn],
|
||||
"mlym" => &[Mlym],
|
||||
"mong" => &[Mong],
|
||||
"mymr" => &[Mymr],
|
||||
"orya" => &[Orya],
|
||||
"sinh" => &[Sinh],
|
||||
"syrc" => &[Syrc],
|
||||
"taml" => &[Taml],
|
||||
"telu" => &[Telu],
|
||||
"thaa" => &[Thaa],
|
||||
"thai" => &[Thai],
|
||||
"tibt" => &[Tibt],
|
||||
_ => return None,
|
||||
})
|
||||
}
|
||||
|
||||
/// A language tag's language, script and region subtags, lowercase.
|
||||
struct Subtags {
|
||||
language: String,
|
||||
script: Option<String>,
|
||||
region: Option<String>,
|
||||
}
|
||||
|
||||
impl Subtags {
|
||||
/// `zh-Hant-HK`, `zh_TW.UTF-8` or `sr_RS@latin` (glibc's script modifier).
|
||||
fn parse(tag: &str) -> Subtags {
|
||||
let tag = tag.trim().to_ascii_lowercase();
|
||||
let base = tag.split(['.', '@']).next().unwrap_or("").replace('_', "-");
|
||||
let mut parts = base.split('-').filter(|p| !p.is_empty());
|
||||
let language = parts.next().unwrap_or("").to_owned();
|
||||
let mut script = match tag.split_once('@').map(|(_, m)| m) {
|
||||
Some("latin") => Some("latn".to_owned()),
|
||||
Some("cyrillic") => Some("cyrl".to_owned()),
|
||||
_ => None,
|
||||
};
|
||||
let mut region = None;
|
||||
for part in parts {
|
||||
if part.len() == 4 && part.bytes().all(|b| b.is_ascii_alphabetic()) {
|
||||
script.get_or_insert_with(|| part.to_owned());
|
||||
} else if super::is_region(part) {
|
||||
region = Some(part.to_owned());
|
||||
break;
|
||||
}
|
||||
}
|
||||
Subtags { language, script, region }
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::Script::*;
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn base_direction_of_text() {
|
||||
assert_eq!(base_direction("Hello"), Some(Direction::Ltr));
|
||||
assert_eq!(base_direction("123 שלום"), Some(Direction::Rtl));
|
||||
assert_eq!(base_direction("(مرحبا) hi"), Some(Direction::Rtl));
|
||||
// The isolate's Hebrew is skipped; the Latin after it decides.
|
||||
assert_eq!(base_direction("\u{2067}שלום\u{2069} hello"), Some(Direction::Ltr));
|
||||
assert_eq!(base_direction("\u{2068}abc\u{2069} שלום"), Some(Direction::Rtl));
|
||||
// Only the first paragraph counts.
|
||||
assert_eq!(base_direction("123\nשלום"), None);
|
||||
assert_eq!(base_direction("1920 × 1080"), None);
|
||||
assert_eq!(base_direction(""), None);
|
||||
}
|
||||
|
||||
/// Every locale the craft-i18n catalogs or an app ship, as the apps spell them.
|
||||
#[test]
|
||||
fn shipped_locales() {
|
||||
for (tag, want, han_forms) in [
|
||||
("en", &[Latn][..], None),
|
||||
("ar", &[Arab], None),
|
||||
("bg", &[Cyrl], None),
|
||||
("cs", &[Latn], None),
|
||||
("da", &[Latn], None),
|
||||
("de", &[Latn], None),
|
||||
("el", &[Grek], None),
|
||||
("es", &[Latn], None),
|
||||
("es-419", &[Latn], None),
|
||||
("et", &[Latn], None),
|
||||
("fi", &[Latn], None),
|
||||
("fr", &[Latn], None),
|
||||
("hr", &[Latn], None),
|
||||
("hu", &[Latn], None),
|
||||
("id", &[Latn], None),
|
||||
("it", &[Latn], None),
|
||||
("ja", &[Hani, Hira, Kana], Some(Han::Japanese)),
|
||||
("ka", &[Geor], None),
|
||||
("kk", &[Cyrl], None),
|
||||
("ko", &[Hang, Hani], Some(Han::Korean)),
|
||||
("lt", &[Latn], None),
|
||||
("nl", &[Latn], None),
|
||||
("pl", &[Latn], None),
|
||||
("pt", &[Latn], None),
|
||||
("pt-br", &[Latn], None),
|
||||
("pt_BR", &[Latn], None),
|
||||
("ru", &[Cyrl], None),
|
||||
("sl", &[Latn], None),
|
||||
("sr", &[Cyrl], None),
|
||||
("sr-latn", &[Latn], None),
|
||||
("sv", &[Latn], None),
|
||||
("te", &[Telu], None),
|
||||
("tr", &[Latn], None),
|
||||
("uk", &[Cyrl], None),
|
||||
("vi", &[Latn], None),
|
||||
("zh", &[Hani], Some(Han::Simplified)),
|
||||
("zh-cn", &[Hani], Some(Han::Simplified)),
|
||||
("zh-CN", &[Hani], Some(Han::Simplified)),
|
||||
("zh-hans", &[Hani], Some(Han::Simplified)),
|
||||
("zh-Hans", &[Hani], Some(Han::Simplified)),
|
||||
("zh-hant", &[Hani], Some(Han::Traditional)),
|
||||
] {
|
||||
assert_eq!(scripts(tag), want, "{tag}");
|
||||
assert_eq!(han(tag), han_forms, "{tag}");
|
||||
let rtl = tag == "ar";
|
||||
assert_eq!(direction(tag), if rtl { Direction::Rtl } else { Direction::Ltr }, "{tag}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn regional_han() {
|
||||
for (tag, want) in [
|
||||
("zh-TW", Han::Traditional),
|
||||
("zh_TW.UTF-8", Han::Traditional),
|
||||
("zh-Hant-TW", Han::Traditional),
|
||||
("zh-HK", Han::HongKong),
|
||||
("zh-MO", Han::HongKong),
|
||||
("zh-Hant-HK", Han::HongKong),
|
||||
("zh-hant-hk", Han::HongKong),
|
||||
("zh-Hans-HK", Han::Simplified),
|
||||
("zh-SG", Han::Simplified),
|
||||
("zh-US", Han::Traditional),
|
||||
("zh_AU.UTF-8", Han::Traditional),
|
||||
("zh-GB", Han::Traditional),
|
||||
("zh-Hans-US", Han::Simplified),
|
||||
("zh-MY", Han::Simplified),
|
||||
("yue-CN", Han::Simplified),
|
||||
("yue-HK", Han::HongKong),
|
||||
("zh-Hans-SG", Han::Simplified),
|
||||
("yue", Han::HongKong),
|
||||
("yue-Hant", Han::HongKong),
|
||||
("yue-Hant-TW", Han::Traditional),
|
||||
("zh-Hani-HK", Han::HongKong),
|
||||
("zh-Hani", Han::Simplified),
|
||||
("yue-Hans", Han::Simplified),
|
||||
("ja-JP", Han::Japanese),
|
||||
("ko_KR.UTF-8", Han::Korean),
|
||||
] {
|
||||
assert_eq!(han(tag), Some(want), "{tag}");
|
||||
}
|
||||
assert_eq!(han("en"), None);
|
||||
assert_eq!(han(""), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn script_subtag_wins() {
|
||||
for (tag, want) in [
|
||||
("sr-Latn-RS", &[Latn][..]),
|
||||
("sr_RS@latin", &[Latn]),
|
||||
("sr-Cyrl", &[Cyrl]),
|
||||
("sr_RS.UTF-8", &[Cyrl]),
|
||||
("az-Cyrl", &[Cyrl]),
|
||||
("pa-Arab", &[Arab]),
|
||||
("pa", &[Guru]),
|
||||
("mn-Mong", &[Mong]),
|
||||
("sd-Deva", &[Deva]),
|
||||
("fr-Zyyy", &[Latn]),
|
||||
] {
|
||||
assert_eq!(scripts(tag), want, "{tag}");
|
||||
}
|
||||
assert_eq!(direction("pa-Arab"), Direction::Rtl);
|
||||
assert_eq!(direction("sd-Deva"), Direction::Ltr);
|
||||
assert_eq!(direction("he_IL.UTF-8"), Direction::Rtl);
|
||||
}
|
||||
|
||||
/// CLDR likely subtags: a region can switch a language to another script.
|
||||
#[test]
|
||||
fn region_implies_script() {
|
||||
for (tag, want) in [
|
||||
("pa-PK", &[Arab][..]),
|
||||
("pa_PK.UTF-8", &[Arab]),
|
||||
("pa-IN", &[Guru]),
|
||||
("uz-AF", &[Arab]),
|
||||
("uz-UZ", &[Latn]),
|
||||
("uz-CN", &[Cyrl]),
|
||||
("az-IR", &[Arab]),
|
||||
("az-AZ", &[Latn]),
|
||||
("sd-IN", &[Deva]),
|
||||
("sd-PK", &[Arab]),
|
||||
("ks-IN", &[Arab]),
|
||||
("ms-CC", &[Arab]),
|
||||
("ms-BN", &[Latn]),
|
||||
("sr-ME", &[Latn]),
|
||||
("kk-CN", &[Arab]),
|
||||
("ug-KZ", &[Cyrl]),
|
||||
("pa-Guru-PK", &[Guru]),
|
||||
("sd-Arab-IN", &[Arab]),
|
||||
] {
|
||||
assert_eq!(scripts(tag), want, "{tag}");
|
||||
}
|
||||
assert_eq!(direction("pa-PK"), Direction::Rtl);
|
||||
assert_eq!(direction("uz-AF"), Direction::Rtl);
|
||||
assert_eq!(direction("az-IR"), Direction::Rtl);
|
||||
assert_eq!(direction("sd-IN"), Direction::Ltr);
|
||||
assert_eq!(direction("ug-KZ"), Direction::Ltr);
|
||||
}
|
||||
|
||||
/// The usual scripts tag matching relies on agree with the table.
|
||||
#[test]
|
||||
fn agrees_with_usual_script() {
|
||||
for language in ["be", "kk", "ky", "mn", "sr", "tg", "zh", "az", "bs", "ha", "ku", "ms", "uz", "ks", "sd", "ug", "pa"] {
|
||||
let usual = crate::usual_script(language).and_then(subtag_scripts);
|
||||
assert_eq!(usual, Some(scripts(language)), "{language}");
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user