i18n: glibc names in Collator; fa and hi collation; Persian letters and digits in folding

Collator::new takes sv_SE.UTF-8 and sr_RS@latin. Folding maps every
script's decimal digits to ASCII and, ignoring accents, Persian keheh,
Farsi yeh, alef maksura, teh marbuta and heh with hamza to the Arabic
letters. The collation data grows 4.8 KB (465 bytes gzipped).

crates/i18n/README.md

@@ -73,12 +73,14 @@ The app keeps its `tl!` macro, its language setting (`"auto"` → `I18N.system_l
   translation. `looked_up()` lists the source texts looked up so far, for an audit against a
   complete catalog.
 - **Search** (`Folder`): `Folder::new(locale).fold(text)` gives a key for case- and
-  accent-insensitive matching (`keep_accents()` makes accents count) and a map back to the
-  original text, so `Folded::find` returns the byte range to highlight. Compatibility forms fold
-  to their plain letters (`fi`, full-width `Ａ`), `ß` to `ss`, `ς` to `σ`; invisible format
-  characters (soft hyphens, joiners, bidi isolates) are dropped. Turkish and Azerbaijani fold `I`
-  to `ı` when accents count; ignoring accents, `I`, `i`, `İ` and `ı` all match.
+  accent-insensitive matching (`keep_accents()` makes accents count) and a map back to the original
+  text, so `Folded::find` returns the byte range to highlight. Compatibility forms fold to their
+  plain letters (`fi`, full-width `Ａ`), `ß` to `ss`, `ς` to `σ`, digits of every script to ASCII
+  (`Layer ١٢` finds `layer 12`); ignoring accents, Persian and Arabic letter forms match (`ک` and
+  `ك`, `ی` and `ي`, `ة` and `ه`, `أ` and `ا`); invisible format characters (soft hyphens, joiners,
+  bidi isolates) are dropped. Turkish and Azerbaijani fold `I` to `ı` when accents count; ignoring
+  accents, `I`, `i`, `İ` and `ı` all match.
 - **Sorting** (`Collator`, feature `collate`): ICU4X collation for the locales craft-i18n ships,
-  with digits compared by value (`Layer 2` before `Layer 10`) and `-u-co-pinyin` /
-  `-u-co-stroke` for Chinese. The data (750 KB) is regenerated by `data/generate-collation.sh`;
+  Persian and Hindi, with digits compared by value (`Layer 2` before `Layer 10`) and `-u-co-pinyin`
+  / `-u-co-stroke` for Chinese. The data (754 KB) is regenerated by `data/generate-collation.sh`;
   the feature adds about 1 MB to a release binary or wasm module (320 KB gzipped).

crates/i18n/data/generate-collation.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 # Regenerates the collation data `collate` embeds, a postcard blob for the locales craft-i18n
-# ships (its `locales/` directory) plus English: each locale's default order, and Chinese
-# pinyin and stroke orders.
+# ships (its `locales/` directory) plus English, Persian and Hindi: each locale's default order,
+# and Chinese pinyin and stroke orders.
 # Needs icu4x-datagen at the version of the icu_collator dependency:
 #   cargo install icu4x-datagen --version 2.3.0 --locked
 set -eu
@@ -11,5 +11,5 @@ icu4x-datagen --format blob --overwrite --out collation.postcard --deduplication
   --attribute-filter 'collator=/^(stroke|pinyin)?$/' \
   --markers CollationRootV1 CollationTailoringV1 CollationDiacriticsV1 CollationJamoV1 \
     CollationMetadataV1 CollationReorderingV1 CollationSpecialPrimariesV1 \
-  --locales en ar bg cs da de el es et fi fr hr hu id it ja ka kk ko lt nl pl pt-BR ru sl sr \
-    sr-Latn sv te tr uk vi zh-Hans zh-Hant
+  --locales en ar bg cs da de el es et fa fi fr hi hr hu id it ja ka kk ko lt nl pl pt-BR ru sl \
+    sr sr-Latn sv te tr uk vi zh-Hans zh-Hant

crates/i18n/src/collate.rs

@@ -28,10 +28,10 @@ pub struct Collator(Option<icu_collator::Collator>);
 
 impl Collator {
     /// Sorting for `locale`, a language code such as `sv`, `pt-br` or `zh-hant` (any case, `-`
-    /// or `_`). A BCP 47 collation type picks another order where there is one:
-    /// `zh-hant-u-co-pinyin`, `zh-hans-u-co-stroke`.
+    /// or `_`, glibc forms such as `sv_SE.UTF-8` and `sr_RS@latin`). A BCP 47 collation type
+    /// picks another order where there is one: `zh-hant-u-co-pinyin`, `zh-hans-u-co-stroke`.
     pub fn new(locale: &str) -> Collator {
-        let mut prefs: CollatorPreferences = Locale::try_from_str(&locale.replace('_', "-")).unwrap_or(Locale::UNKNOWN).into();
+        let mut prefs: CollatorPreferences = Locale::try_from_str(&bcp47(locale)).unwrap_or(Locale::UNKNOWN).into();
         prefs.numeric_ordering = Some(CollationNumericOrdering::True);
         let options = CollatorOptions::default();
         let collator = Data::new().ok().and_then(|data| {
@@ -58,6 +58,25 @@ impl Collator {
     }
 }
 
+/// `locale` as a BCP 47 tag: `_` → `-`, a glibc codeset dropped and its script modifier made a
+/// subtag (`sr_RS.UTF-8@latin` → `sr-latn-RS`).
+fn bcp47(locale: &str) -> String {
+    let locale = locale.trim();
+    let mut parts: Vec<&str> = locale.split(['.', '@']).next().unwrap_or_default().split(['-', '_']).collect();
+    let script = match locale.split_once('@').map(|(_, m)| m.to_ascii_lowercase()).as_deref() {
+        Some("latin") => Some("latn"),
+        Some("cyrillic") => Some("cyrl"),
+        _ => None,
+    };
+    let has_script = parts.get(1).is_some_and(|p| p.len() == 4 && p.bytes().all(|b| b.is_ascii_alphabetic()));
+    if let Some(script) = script
+        && !has_script
+    {
+        parts.insert(1.min(parts.len()), script);
+    }
+    parts.join("-")
+}
+
 /// The embedded collation data, with locale fallback, and the decomposition tables the collator
 /// also needs taken from `icu_normalizer`'s compiled data rather than embedded a second time.
 struct Data(LocaleFallbackProvider<BlobDataProvider>);
@@ -123,8 +142,10 @@ mod tests {
             "el",
             "es",
             "et",
+            "fa",
             "fi",
             "fr",
+            "hi",
             "hr",
             "hu",
             "id",
@@ -177,6 +198,23 @@ mod tests {
         assert_eq!(sorted("ja", &["漢字", "カメラ", "あめ", "いぬ", "Zoom"]), ["Zoom", "あめ", "いぬ", "カメラ", "漢字"]);
     }
 
+    #[test]
+    fn glibc_locale_names() {
+        let words = ["öl", "zebra", "år", "äpple", "apa", "oxe"];
+        assert_eq!(sorted("sv_SE.UTF-8", &words), sorted("sv", &words));
+        assert_ne!(sorted("sv_SE.UTF-8", &words), sorted("en", &words));
+        let cities = ["天津", "北京", "上海"];
+        assert_eq!(sorted("zh_TW.UTF-8", &cities), sorted("zh-hant", &cities));
+        assert_eq!(sorted("zh_CN.UTF-8", &cities), sorted("zh-hans", &cities));
+        // Serbian Cyrillic sorts Cyrillic first; Latin Serbian sorts Latin first, `lj` after `lu`.
+        let mixed = ["ljubav", "lula", "абв"];
+        assert_eq!(sorted("sr_RS.UTF-8", &mixed), ["абв", "ljubav", "lula"]);
+        assert_eq!(sorted("sr_RS.UTF-8@latin", &mixed), ["lula", "ljubav", "абв"]);
+        assert_eq!(bcp47("sr_RS.UTF-8@latin"), "sr-latn-RS");
+        assert_eq!(bcp47("sr_Latn_RS@latin"), "sr-Latn-RS");
+        assert_eq!(bcp47("de_DE@euro"), "de-DE");
+    }
+
     #[test]
     fn chinese_sorts_by_pinyin_or_stroke() {
         // 北 běi (5 strokes), 上 shàng (3), 天 tiān (4).

crates/i18n/src/fold.rs

@@ -4,8 +4,10 @@
 //! [`Folder::fold`] turns text into a search key: compatibility forms decomposed (`fi` → `fi`,
 //! full-width `Ａ` → `a`, `㎏` → `kg`), lower case, `ß` → `ss`, final `ς` → `σ`, and with accents
 //! ignored (the default) combining accents, Hebrew points and Arabic vowel marks dropped and the
-//! few letters with built-in strokes mapped to their base (`ø`, `ł`, `đ`, `ı`, `æ` → `ae`).
-//! Invisible format characters (soft hyphens, zero-width joiners, bidi isolates) are dropped too.
+//! few letters with built-in strokes mapped to their base (`ø`, `ł`, `đ`, `ı`, `æ` → `ae`), and
+//! Persian letter forms mapped to the Arabic ones (`ک` → `ك`, `ی` → `ي`, `ة` → `ه`). Decimal
+//! digits of every script fold to ASCII (`١٢` → `12`). Invisible format characters (soft
+//! hyphens, zero-width joiners, bidi isolates) are dropped too.
 //!
 //! Dotted and dotless i follow the locale when accents count. In Turkish and Azerbaijani, `I`
 //! folds to `ı` and `İ` to `i`; elsewhere both fold to `i` and `ı` stays a letter of its own.
@@ -33,7 +35,7 @@ impl Folder {
     /// Case- and accent-insensitive folding for `locale`, a language code such as `tr`, `pt-br`
     /// or `zh-hant` (any case, `-` or `_`).
     pub fn new(locale: &str) -> Folder {
-        let language = locale.split(['-', '_']).next().unwrap_or_default();
+        let language = locale.split(['-', '_', '.', '@']).next().unwrap_or_default();
         Folder { turkic: language.eq_ignore_ascii_case("tr") || language.eq_ignore_ascii_case("az"), accents: false }
     }
 
@@ -87,7 +89,13 @@ impl Folder {
                     out.push('i');
                 }
                 'I' if self.turkic && self.accents => out.push('ı'),
+                // Persian heh with hamza above (ۀ) decomposes to ae (ە) and hamza.
+                '\u{6D5}' if !self.accents && chars.peek() == Some(&'\u{654}') => {
+                    chars.next();
+                    out.push('ه');
+                }
                 'ß' | 'ẞ' => out.push_str("ss"),
+                c if let Some(d) = ascii_digit(c) => out.push(d),
                 'Σ' | 'σ' | 'ς' => out.push('σ'),
                 c if ignorable(c) || (!self.accents && accent(c)) => {}
                 c => {
@@ -103,9 +111,15 @@ impl Folder {
     }
 }
 
-/// The base of a lower-case letter whose diacritic is built in rather than combining.
+/// The base of a lower-case letter whose diacritic is built in rather than combining, and the
+/// Arabic form of a letter that Persian and Urdu write differently: keheh (ک) is kaf (ك), Farsi
+/// yeh (ی) and alef maksura (ى) are yeh (ي), teh marbuta (ة) is heh (ه), alef wasla (ٱ) is alef.
 fn base_letter(c: char) -> Option<&'static str> {
     Some(match c {
+        'ک' => "ك",
+        'ی' | 'ى' => "ي",
+        'ة' => "ه",
+        'ٱ' => "ا",
         'ı' => "i",
         'ø' => "o",
         'ł' => "l",
@@ -139,6 +153,27 @@ fn accent(c: char) -> bool {
         | '\u{6DF}'..='\u{6E4}' | '\u{6E7}' | '\u{6E8}' | '\u{6EA}'..='\u{6ED}')
 }
 
+/// The first code point of each run of ten Unicode decimal digits (general category Nd), Unicode
+/// 16.0.
+const ZEROS: [u32; 76] = [
+    0x30, 0x660, 0x6F0, 0x7C0, 0x966, 0x9E6, 0xA66, 0xAE6, 0xB66, 0xBE6, 0xC66, 0xCE6, 0xD66, 0xDE6, 0xE50, 0xED0, 0xF20, 0x1040, 0x1090, 0x17E0,
+    0x1810, 0x1946, 0x19D0, 0x1A80, 0x1A90, 0x1B50, 0x1BB0, 0x1C40, 0x1C50, 0xA620, 0xA8D0, 0xA900, 0xA9D0, 0xA9F0, 0xAA50, 0xABF0, 0xFF10, 0x104A0,
+    0x10D30, 0x10D40, 0x11066, 0x110F0, 0x11136, 0x111D0, 0x112F0, 0x11450, 0x114D0, 0x11650, 0x116C0, 0x116D0, 0x116DA, 0x11730, 0x118E0, 0x11950,
+    0x11BF0, 0x11C50, 0x11D50, 0x11DA0, 0x11F50, 0x16130, 0x16A60, 0x16AC0, 0x16B50, 0x16D70, 0x1CCF0, 0x1D7CE, 0x1D7D8, 0x1D7E2, 0x1D7EC, 0x1D7F6,
+    0x1E140, 0x1E2F0, 0x1E4F0, 0x1E5F1, 0x1E950, 0x1FBF0,
+];
+
+/// A decimal digit of any script as its ASCII digit: `١٢` and `१२` search as `12`.
+fn ascii_digit(c: char) -> Option<char> {
+    let c = u32::from(c);
+    let zero = match ZEROS.binary_search(&c) {
+        Ok(i) => ZEROS.get(i),
+        Err(i) => ZEROS.get(i.checked_sub(1)?),
+    }?;
+    let value = c - zero;
+    if value < 10 { char::from_digit(value, 10) } else { None }
+}
+
 /// Folded text and where each part came from.
 #[derive(Clone, Debug, PartialEq, Eq)]
 pub struct Folded {
@@ -245,6 +280,10 @@ mod tests {
             ("he", "שָׁלוֹם", "שלום".into(), nfd("שָׁלוֹם")),
             ("en", "soft\u{AD}ware \u{2068}name\u{2069}", "software name".into(), "software name".into()),
             ("ru", "Ёлка", "елка".into(), nfd("ёлка")),
+            ("ar", "Layer ١٢ ۳۴ १२", "layer 12 34 12".into(), "layer 12 34 12".into()),
+            ("fa", "کتاب یک", "كتاب يك".into(), "کتاب یک".into()),
+            ("fa", "خانۀ", "خانه".into(), nfd("خانۀ")),
+            ("ar", "مدرسة على إسلام آمن", "مدرسه علي اسلام امن".into(), nfd("مدرسة على إسلام آمن")),
         ];
         for (locale, input, ignoring, keeping) in table {
             assert_eq!(key(locale, input), ignoring, "{locale} {input}");
@@ -287,6 +326,22 @@ mod tests {
         assert_eq!(all, [0..2, 3..5, 6..8]);
     }
 
+    #[test]
+    fn persian_and_arabic_letters_match() {
+        let f = Folder::new("fa");
+        // Persian keheh and Farsi yeh against Arabic kaf and yeh, and hamza forms of alef.
+        assert!(f.contains("كتاب عربي", "کتاب عربی"));
+        assert!(f.contains("کتاب عربی", "كتاب عربي"));
+        assert!(f.contains("أحمد إبراهيم", "احمد ابراهیم"));
+        assert!(f.contains("مدرسة", "مدرسه"));
+        assert_eq!(f.fold("Layer ١٢").find(&f.fold("layer 12")), Some(0.."Layer ١٢".len()));
+        assert!(Folder::new("en").keep_accents().contains("Layer ١٢", "layer 12"));
+        let digits = ZEROS.iter().map(|&zero| char::from_u32(zero + 7).unwrap());
+        assert!(digits.clone().all(|d| d.is_numeric() && ascii_digit(d) == Some('7')));
+        assert_eq!(ascii_digit('\u{1D800}'), None);
+        assert_eq!(ascii_digit('/'), None);
+    }
+
     #[test]
     fn accents_count_only_whole_letters() {
         let f = Folder::new("fr").keep_accents();