diff --git a/crates/i18n/README.md b/crates/i18n/README.md index 77e3d996..996d9268 100644 --- a/crates/i18n/README.md +++ b/crates/i18n/README.md @@ -73,12 +73,14 @@ The app keeps its `tl!` macro, its language setting (`"auto"` → `I18N.system_l translation. `looked_up()` lists the source texts looked up so far, for an audit against a complete catalog. - **Search** (`Folder`): `Folder::new(locale).fold(text)` gives a key for case- and - accent-insensitive matching (`keep_accents()` makes accents count) and a map back to the - original text, so `Folded::find` returns the byte range to highlight. Compatibility forms fold - to their plain letters (`fi`, full-width `Ａ`), `ß` to `ss`, `ς` to `σ`; invisible format - characters (soft hyphens, joiners, bidi isolates) are dropped. Turkish and Azerbaijani fold `I` - to `ı` when accents count; ignoring accents, `I`, `i`, `İ` and `ı` all match. + accent-insensitive matching (`keep_accents()` makes accents count) and a map back to the original + text, so `Folded::find` returns the byte range to highlight. Compatibility forms fold to their + plain letters (`fi`, full-width `Ａ`), `ß` to `ss`, `ς` to `σ`, digits of every script to ASCII + (`Layer ١٢` finds `layer 12`); ignoring accents, Persian and Arabic letter forms match (`ک` and + `ك`, `ی` and `ي`, `ة` and `ه`, `أ` and `ا`); invisible format characters (soft hyphens, joiners, + bidi isolates) are dropped. Turkish and Azerbaijani fold `I` to `ı` when accents count; ignoring + accents, `I`, `i`, `İ` and `ı` all match. - **Sorting** (`Collator`, feature `collate`): ICU4X collation for the locales craft-i18n ships, - with digits compared by value (`Layer 2` before `Layer 10`) and `-u-co-pinyin` / - `-u-co-stroke` for Chinese. The data (750 KB) is regenerated by `data/generate-collation.sh`; + Persian and Hindi, with digits compared by value (`Layer 2` before `Layer 10`) and `-u-co-pinyin` + / `-u-co-stroke` for Chinese. The data (754 KB) is regenerated by `data/generate-collation.sh`; the feature adds about 1 MB to a release binary or wasm module (320 KB gzipped). diff --git a/crates/i18n/data/collation.postcard b/crates/i18n/data/collation.postcard index 0ea89d72..7eabc2f2 100644 Binary files a/crates/i18n/data/collation.postcard and b/crates/i18n/data/collation.postcard differ diff --git a/crates/i18n/data/generate-collation.sh b/crates/i18n/data/generate-collation.sh index cc7bbd3d..35f605eb 100755 --- a/crates/i18n/data/generate-collation.sh +++ b/crates/i18n/data/generate-collation.sh @@ -1,7 +1,7 @@ #!/bin/sh # Regenerates the collation data `collate` embeds, a postcard blob for the locales craft-i18n -# ships (its `locales/` directory) plus English: each locale's default order, and Chinese -# pinyin and stroke orders. +# ships (its `locales/` directory) plus English, Persian and Hindi: each locale's default order, +# and Chinese pinyin and stroke orders. # Needs icu4x-datagen at the version of the icu_collator dependency: # cargo install icu4x-datagen --version 2.3.0 --locked set -eu @@ -11,5 +11,5 @@ icu4x-datagen --format blob --overwrite --out collation.postcard --deduplication --attribute-filter 'collator=/^(stroke|pinyin)?$/' \ --markers CollationRootV1 CollationTailoringV1 CollationDiacriticsV1 CollationJamoV1 \ CollationMetadataV1 CollationReorderingV1 CollationSpecialPrimariesV1 \ - --locales en ar bg cs da de el es et fi fr hr hu id it ja ka kk ko lt nl pl pt-BR ru sl sr \ - sr-Latn sv te tr uk vi zh-Hans zh-Hant + --locales en ar bg cs da de el es et fa fi fr hi hr hu id it ja ka kk ko lt nl pl pt-BR ru sl \ + sr sr-Latn sv te tr uk vi zh-Hans zh-Hant diff --git a/crates/i18n/src/collate.rs b/crates/i18n/src/collate.rs index d73b1efc..408f40b1 100644 --- a/crates/i18n/src/collate.rs +++ b/crates/i18n/src/collate.rs @@ -28,10 +28,10 @@ pub struct Collator(Option); impl Collator { /// Sorting for `locale`, a language code such as `sv`, `pt-br` or `zh-hant` (any case, `-` - /// or `_`). A BCP 47 collation type picks another order where there is one: - /// `zh-hant-u-co-pinyin`, `zh-hans-u-co-stroke`. + /// or `_`, glibc forms such as `sv_SE.UTF-8` and `sr_RS@latin`). A BCP 47 collation type + /// picks another order where there is one: `zh-hant-u-co-pinyin`, `zh-hans-u-co-stroke`. pub fn new(locale: &str) -> Collator { - let mut prefs: CollatorPreferences = Locale::try_from_str(&locale.replace('_', "-")).unwrap_or(Locale::UNKNOWN).into(); + let mut prefs: CollatorPreferences = Locale::try_from_str(&bcp47(locale)).unwrap_or(Locale::UNKNOWN).into(); prefs.numeric_ordering = Some(CollationNumericOrdering::True); let options = CollatorOptions::default(); let collator = Data::new().ok().and_then(|data| { @@ -58,6 +58,25 @@ impl Collator { } } +/// `locale` as a BCP 47 tag: `_` → `-`, a glibc codeset dropped and its script modifier made a +/// subtag (`sr_RS.UTF-8@latin` → `sr-latn-RS`). +fn bcp47(locale: &str) -> String { + let locale = locale.trim(); + let mut parts: Vec<&str> = locale.split(['.', '@']).next().unwrap_or_default().split(['-', '_']).collect(); + let script = match locale.split_once('@').map(|(_, m)| m.to_ascii_lowercase()).as_deref() { + Some("latin") => Some("latn"), + Some("cyrillic") => Some("cyrl"), + _ => None, + }; + let has_script = parts.get(1).is_some_and(|p| p.len() == 4 && p.bytes().all(|b| b.is_ascii_alphabetic())); + if let Some(script) = script + && !has_script + { + parts.insert(1.min(parts.len()), script); + } + parts.join("-") +} + /// The embedded collation data, with locale fallback, and the decomposition tables the collator /// also needs taken from `icu_normalizer`'s compiled data rather than embedded a second time. struct Data(LocaleFallbackProvider); @@ -123,8 +142,10 @@ mod tests { "el", "es", "et", + "fa", "fi", "fr", + "hi", "hr", "hu", "id", @@ -177,6 +198,23 @@ mod tests { assert_eq!(sorted("ja", &["漢字", "カメラ", "あめ", "いぬ", "Zoom"]), ["Zoom", "あめ", "いぬ", "カメラ", "漢字"]); } + #[test] + fn glibc_locale_names() { + let words = ["öl", "zebra", "år", "äpple", "apa", "oxe"]; + assert_eq!(sorted("sv_SE.UTF-8", &words), sorted("sv", &words)); + assert_ne!(sorted("sv_SE.UTF-8", &words), sorted("en", &words)); + let cities = ["天津", "北京", "上海"]; + assert_eq!(sorted("zh_TW.UTF-8", &cities), sorted("zh-hant", &cities)); + assert_eq!(sorted("zh_CN.UTF-8", &cities), sorted("zh-hans", &cities)); + // Serbian Cyrillic sorts Cyrillic first; Latin Serbian sorts Latin first, `lj` after `lu`. + let mixed = ["ljubav", "lula", "абв"]; + assert_eq!(sorted("sr_RS.UTF-8", &mixed), ["абв", "ljubav", "lula"]); + assert_eq!(sorted("sr_RS.UTF-8@latin", &mixed), ["lula", "ljubav", "абв"]); + assert_eq!(bcp47("sr_RS.UTF-8@latin"), "sr-latn-RS"); + assert_eq!(bcp47("sr_Latn_RS@latin"), "sr-Latn-RS"); + assert_eq!(bcp47("de_DE@euro"), "de-DE"); + } + #[test] fn chinese_sorts_by_pinyin_or_stroke() { // 北 běi (5 strokes), 上 shàng (3), 天 tiān (4). diff --git a/crates/i18n/src/fold.rs b/crates/i18n/src/fold.rs index 19289fe3..3a3330b1 100644 --- a/crates/i18n/src/fold.rs +++ b/crates/i18n/src/fold.rs @@ -4,8 +4,10 @@ //! [`Folder::fold`] turns text into a search key: compatibility forms decomposed (`fi` → `fi`, //! full-width `Ａ` → `a`, `㎏` → `kg`), lower case, `ß` → `ss`, final `ς` → `σ`, and with accents //! ignored (the default) combining accents, Hebrew points and Arabic vowel marks dropped and the -//! few letters with built-in strokes mapped to their base (`ø`, `ł`, `đ`, `ı`, `æ` → `ae`). -//! Invisible format characters (soft hyphens, zero-width joiners, bidi isolates) are dropped too. +//! few letters with built-in strokes mapped to their base (`ø`, `ł`, `đ`, `ı`, `æ` → `ae`), and +//! Persian letter forms mapped to the Arabic ones (`ک` → `ك`, `ی` → `ي`, `ة` → `ه`). Decimal +//! digits of every script fold to ASCII (`١٢` → `12`). Invisible format characters (soft +//! hyphens, zero-width joiners, bidi isolates) are dropped too. //! //! Dotted and dotless i follow the locale when accents count. In Turkish and Azerbaijani, `I` //! folds to `ı` and `İ` to `i`; elsewhere both fold to `i` and `ı` stays a letter of its own. @@ -33,7 +35,7 @@ impl Folder { /// Case- and accent-insensitive folding for `locale`, a language code such as `tr`, `pt-br` /// or `zh-hant` (any case, `-` or `_`). pub fn new(locale: &str) -> Folder { - let language = locale.split(['-', '_']).next().unwrap_or_default(); + let language = locale.split(['-', '_', '.', '@']).next().unwrap_or_default(); Folder { turkic: language.eq_ignore_ascii_case("tr") || language.eq_ignore_ascii_case("az"), accents: false } } @@ -87,7 +89,13 @@ impl Folder { out.push('i'); } 'I' if self.turkic && self.accents => out.push('ı'), + // Persian heh with hamza above (ۀ) decomposes to ae (ە) and hamza. + '\u{6D5}' if !self.accents && chars.peek() == Some(&'\u{654}') => { + chars.next(); + out.push('ه'); + } 'ß' | 'ẞ' => out.push_str("ss"), + c if let Some(d) = ascii_digit(c) => out.push(d), 'Σ' | 'σ' | 'ς' => out.push('σ'), c if ignorable(c) || (!self.accents && accent(c)) => {} c => { @@ -103,9 +111,15 @@ impl Folder { } } -/// The base of a lower-case letter whose diacritic is built in rather than combining. +/// The base of a lower-case letter whose diacritic is built in rather than combining, and the +/// Arabic form of a letter that Persian and Urdu write differently: keheh (ک) is kaf (ك), Farsi +/// yeh (ی) and alef maksura (ى) are yeh (ي), teh marbuta (ة) is heh (ه), alef wasla (ٱ) is alef. fn base_letter(c: char) -> Option<&'static str> { Some(match c { + 'ک' => "ك", + 'ی' | 'ى' => "ي", + 'ة' => "ه", + 'ٱ' => "ا", 'ı' => "i", 'ø' => "o", 'ł' => "l", @@ -139,6 +153,27 @@ fn accent(c: char) -> bool { | '\u{6DF}'..='\u{6E4}' | '\u{6E7}' | '\u{6E8}' | '\u{6EA}'..='\u{6ED}') } +/// The first code point of each run of ten Unicode decimal digits (general category Nd), Unicode +/// 16.0. +const ZEROS: [u32; 76] = [ + 0x30, 0x660, 0x6F0, 0x7C0, 0x966, 0x9E6, 0xA66, 0xAE6, 0xB66, 0xBE6, 0xC66, 0xCE6, 0xD66, 0xDE6, 0xE50, 0xED0, 0xF20, 0x1040, 0x1090, 0x17E0, + 0x1810, 0x1946, 0x19D0, 0x1A80, 0x1A90, 0x1B50, 0x1BB0, 0x1C40, 0x1C50, 0xA620, 0xA8D0, 0xA900, 0xA9D0, 0xA9F0, 0xAA50, 0xABF0, 0xFF10, 0x104A0, + 0x10D30, 0x10D40, 0x11066, 0x110F0, 0x11136, 0x111D0, 0x112F0, 0x11450, 0x114D0, 0x11650, 0x116C0, 0x116D0, 0x116DA, 0x11730, 0x118E0, 0x11950, + 0x11BF0, 0x11C50, 0x11D50, 0x11DA0, 0x11F50, 0x16130, 0x16A60, 0x16AC0, 0x16B50, 0x16D70, 0x1CCF0, 0x1D7CE, 0x1D7D8, 0x1D7E2, 0x1D7EC, 0x1D7F6, + 0x1E140, 0x1E2F0, 0x1E4F0, 0x1E5F1, 0x1E950, 0x1FBF0, +]; + +/// A decimal digit of any script as its ASCII digit: `١٢` and `१२` search as `12`. +fn ascii_digit(c: char) -> Option { + let c = u32::from(c); + let zero = match ZEROS.binary_search(&c) { + Ok(i) => ZEROS.get(i), + Err(i) => ZEROS.get(i.checked_sub(1)?), + }?; + let value = c - zero; + if value < 10 { char::from_digit(value, 10) } else { None } +} + /// Folded text and where each part came from. #[derive(Clone, Debug, PartialEq, Eq)] pub struct Folded { @@ -245,6 +280,10 @@ mod tests { ("he", "שָׁלוֹם", "שלום".into(), nfd("שָׁלוֹם")), ("en", "soft\u{AD}ware \u{2068}name\u{2069}", "software name".into(), "software name".into()), ("ru", "Ёлка", "елка".into(), nfd("ёлка")), + ("ar", "Layer ١٢ ۳۴ १२", "layer 12 34 12".into(), "layer 12 34 12".into()), + ("fa", "کتاب یک", "كتاب يك".into(), "کتاب یک".into()), + ("fa", "خانۀ", "خانه".into(), nfd("خانۀ")), + ("ar", "مدرسة على إسلام آمن", "مدرسه علي اسلام امن".into(), nfd("مدرسة على إسلام آمن")), ]; for (locale, input, ignoring, keeping) in table { assert_eq!(key(locale, input), ignoring, "{locale} {input}"); @@ -287,6 +326,22 @@ mod tests { assert_eq!(all, [0..2, 3..5, 6..8]); } + #[test] + fn persian_and_arabic_letters_match() { + let f = Folder::new("fa"); + // Persian keheh and Farsi yeh against Arabic kaf and yeh, and hamza forms of alef. + assert!(f.contains("كتاب عربي", "کتاب عربی")); + assert!(f.contains("کتاب عربی", "كتاب عربي")); + assert!(f.contains("أحمد إبراهيم", "احمد ابراهیم")); + assert!(f.contains("مدرسة", "مدرسه")); + assert_eq!(f.fold("Layer ١٢").find(&f.fold("layer 12")), Some(0.."Layer ١٢".len())); + assert!(Folder::new("en").keep_accents().contains("Layer ١٢", "layer 12")); + let digits = ZEROS.iter().map(|&zero| char::from_u32(zero + 7).unwrap()); + assert!(digits.clone().all(|d| d.is_numeric() && ascii_digit(d) == Some('7'))); + assert_eq!(ascii_digit('\u{1D800}'), None); + assert_eq!(ascii_digit('/'), None); + } + #[test] fn accents_count_only_whole_letters() { let f = Folder::new("fr").keep_accents();