From 44dbe322265ec67f0834362a468a395ce0faebc4 Mon Sep 17 00:00:00 2001 From: aiongg Date: Wed, 21 Oct 2026 05:45:40 +0000 Subject: [PATCH] i18n: glibc names in Collator; fa and hi collation; Persian letters and digits in folding Collator::new takes sv_SE.UTF-8 and sr_RS@latin. Folding maps every script's decimal digits to ASCII and, ignoring accents, Persian keheh, Farsi yeh, alef maksura, teh marbuta and heh with hamza to the Arabic letters. The collation data grows 4.8 KB (465 bytes gzipped). --- crates/i18n/README.md | 16 ++++--- crates/i18n/data/collation.postcard | Bin 749171 -> 753949 bytes crates/i18n/data/generate-collation.sh | 8 ++-- crates/i18n/src/collate.rs | 44 +++++++++++++++-- crates/i18n/src/fold.rs | 63 +++++++++++++++++++++++-- 5 files changed, 113 insertions(+), 18 deletions(-) diff --git a/crates/i18n/README.md b/crates/i18n/README.md index 77e3d996..996d9268 100644 --- a/crates/i18n/README.md +++ b/crates/i18n/README.md @@ -73,12 +73,14 @@ The app keeps its `tl!` macro, its language setting (`"auto"` → `I18N.system_l translation. `looked_up()` lists the source texts looked up so far, for an audit against a complete catalog. - **Search** (`Folder`): `Folder::new(locale).fold(text)` gives a key for case- and - accent-insensitive matching (`keep_accents()` makes accents count) and a map back to the - original text, so `Folded::find` returns the byte range to highlight. Compatibility forms fold - to their plain letters (`fi`, full-width `Ａ`), `ß` to `ss`, `ς` to `σ`; invisible format - characters (soft hyphens, joiners, bidi isolates) are dropped. Turkish and Azerbaijani fold `I` - to `ı` when accents count; ignoring accents, `I`, `i`, `İ` and `ı` all match. + accent-insensitive matching (`keep_accents()` makes accents count) and a map back to the original + text, so `Folded::find` returns the byte range to highlight. Compatibility forms fold to their + plain letters (`fi`, full-width `Ａ`), `ß` to `ss`, `ς` to `σ`, digits of every script to ASCII + (`Layer ١٢` finds `layer 12`); ignoring accents, Persian and Arabic letter forms match (`ک` and + `ك`, `ی` and `ي`, `ة` and `ه`, `أ` and `ا`); invisible format characters (soft hyphens, joiners, + bidi isolates) are dropped. Turkish and Azerbaijani fold `I` to `ı` when accents count; ignoring + accents, `I`, `i`, `İ` and `ı` all match. - **Sorting** (`Collator`, feature `collate`): ICU4X collation for the locales craft-i18n ships, - with digits compared by value (`Layer 2` before `Layer 10`) and `-u-co-pinyin` / - `-u-co-stroke` for Chinese. The data (750 KB) is regenerated by `data/generate-collation.sh`; + Persian and Hindi, with digits compared by value (`Layer 2` before `Layer 10`) and `-u-co-pinyin` + / `-u-co-stroke` for Chinese. The data (754 KB) is regenerated by `data/generate-collation.sh`; the feature adds about 1 MB to a release binary or wasm module (320 KB gzipped). diff --git a/crates/i18n/data/collation.postcard b/crates/i18n/data/collation.postcard index 0ea89d72b5b70cff1b93486909e2f2515ae38509..7eabc2f2f834412c2870fe1438d9b0e83d076a7f 100644 GIT binary patch delta 1501 zcmah{ZD?C%6h7zPyt!$c*``a^G)cSGjZxcWbRTTcjn1xm^Hrs`orMafiEFR79qE>S z;16dT)zrh+Llg zoO9myp7-4Md4B)i^WKX)cwOi_QG(;}Jj9^^)0n|a;mF|7v%@2?vGEDfTw=9XZ`sze zJI3t_w;kmVgtZiNSRgxD2304@pqfxa`0T{&Tf)F&SfMT3S}h?s4a-oD+D*YH z%$Lrh*^1gR@vez6m!hHBt_^Io2pQC7eEs{FZ%$bR00Sw^TwA%ylcx zY6LK8(-EBt2seBV|E=OkB@kF>_|xPFLKmg}Ums zJAz|&s!I^883afPlA01FmHzb31`CZW162T~-L!vrHq`Z?Sn`PIhDF4?f;Pokaw^>b zXI`cA@J@(z)rVjdrr=G|?<1Ip%Wwm3!=F%wPHe(f3}CV%#Oj5|^>5m?wu7G@HTTvd z_dlN3;E(^%-o*SXiuUB@t)$?deY#kN`Z^oR>oR4B*#A4gNB8a}1e{7T{;ODZP;M?Z zWNbawn=`h*xR%d5iaC?}=;Hg76kWKVsw>58MUmLLLn)Hi9lEC6@6a_R?@+R26B04cWd9d2n)vb#~-kd1t2V z5HW3$m*kNl!NN;+$PN(*iCUJkQBV+j2tkmJ9=du91YbJ*PXEh)`BC9;U!&Z^Eja>J z;BXI;_!{5f5B-iYRkOJ5xNJ7Ti?CZkS=*DZt6)Jf^Uy&GZ`>Xl9ETX>s2moF60 zUl<;_dVQ8ZsWxm+r|j2hW;+;H8rqF(+DzRxXFbmgOpcy*ag=%j?xK&VE4jgZ@yyvv zC6`mX&#*gVQbK4Q0DobR`8E- zvix@pVDbgR;s(O2O@wz_2p>KoY=1)dwvF)XGs5l{gg-k72LZyrCW6tF;rs12NQR?O zs>ETb7KfL@vM_9|1=2(bd;M0T=KsDqu%_ISq?MI%Z1fxJ{RR%zo|aoENP;Ycgf<~8 PM1-giTQ9fTf4)Bka1)rK diff --git a/crates/i18n/data/generate-collation.sh b/crates/i18n/data/generate-collation.sh index cc7bbd3d..35f605eb 100755 --- a/crates/i18n/data/generate-collation.sh +++ b/crates/i18n/data/generate-collation.sh @@ -1,7 +1,7 @@ #!/bin/sh # Regenerates the collation data `collate` embeds, a postcard blob for the locales craft-i18n -# ships (its `locales/` directory) plus English: each locale's default order, and Chinese -# pinyin and stroke orders. +# ships (its `locales/` directory) plus English, Persian and Hindi: each locale's default order, +# and Chinese pinyin and stroke orders. # Needs icu4x-datagen at the version of the icu_collator dependency: # cargo install icu4x-datagen --version 2.3.0 --locked set -eu @@ -11,5 +11,5 @@ icu4x-datagen --format blob --overwrite --out collation.postcard --deduplication --attribute-filter 'collator=/^(stroke|pinyin)?$/' \ --markers CollationRootV1 CollationTailoringV1 CollationDiacriticsV1 CollationJamoV1 \ CollationMetadataV1 CollationReorderingV1 CollationSpecialPrimariesV1 \ - --locales en ar bg cs da de el es et fi fr hr hu id it ja ka kk ko lt nl pl pt-BR ru sl sr \ - sr-Latn sv te tr uk vi zh-Hans zh-Hant + --locales en ar bg cs da de el es et fa fi fr hi hr hu id it ja ka kk ko lt nl pl pt-BR ru sl \ + sr sr-Latn sv te tr uk vi zh-Hans zh-Hant diff --git a/crates/i18n/src/collate.rs b/crates/i18n/src/collate.rs index d73b1efc..408f40b1 100644 --- a/crates/i18n/src/collate.rs +++ b/crates/i18n/src/collate.rs @@ -28,10 +28,10 @@ pub struct Collator(Option); impl Collator { /// Sorting for `locale`, a language code such as `sv`, `pt-br` or `zh-hant` (any case, `-` - /// or `_`). A BCP 47 collation type picks another order where there is one: - /// `zh-hant-u-co-pinyin`, `zh-hans-u-co-stroke`. + /// or `_`, glibc forms such as `sv_SE.UTF-8` and `sr_RS@latin`). A BCP 47 collation type + /// picks another order where there is one: `zh-hant-u-co-pinyin`, `zh-hans-u-co-stroke`. pub fn new(locale: &str) -> Collator { - let mut prefs: CollatorPreferences = Locale::try_from_str(&locale.replace('_', "-")).unwrap_or(Locale::UNKNOWN).into(); + let mut prefs: CollatorPreferences = Locale::try_from_str(&bcp47(locale)).unwrap_or(Locale::UNKNOWN).into(); prefs.numeric_ordering = Some(CollationNumericOrdering::True); let options = CollatorOptions::default(); let collator = Data::new().ok().and_then(|data| { @@ -58,6 +58,25 @@ impl Collator { } } +/// `locale` as a BCP 47 tag: `_` → `-`, a glibc codeset dropped and its script modifier made a +/// subtag (`sr_RS.UTF-8@latin` → `sr-latn-RS`). +fn bcp47(locale: &str) -> String { + let locale = locale.trim(); + let mut parts: Vec<&str> = locale.split(['.', '@']).next().unwrap_or_default().split(['-', '_']).collect(); + let script = match locale.split_once('@').map(|(_, m)| m.to_ascii_lowercase()).as_deref() { + Some("latin") => Some("latn"), + Some("cyrillic") => Some("cyrl"), + _ => None, + }; + let has_script = parts.get(1).is_some_and(|p| p.len() == 4 && p.bytes().all(|b| b.is_ascii_alphabetic())); + if let Some(script) = script + && !has_script + { + parts.insert(1.min(parts.len()), script); + } + parts.join("-") +} + /// The embedded collation data, with locale fallback, and the decomposition tables the collator /// also needs taken from `icu_normalizer`'s compiled data rather than embedded a second time. struct Data(LocaleFallbackProvider); @@ -123,8 +142,10 @@ mod tests { "el", "es", "et", + "fa", "fi", "fr", + "hi", "hr", "hu", "id", @@ -177,6 +198,23 @@ mod tests { assert_eq!(sorted("ja", &["漢字", "カメラ", "あめ", "いぬ", "Zoom"]), ["Zoom", "あめ", "いぬ", "カメラ", "漢字"]); } + #[test] + fn glibc_locale_names() { + let words = ["öl", "zebra", "år", "äpple", "apa", "oxe"]; + assert_eq!(sorted("sv_SE.UTF-8", &words), sorted("sv", &words)); + assert_ne!(sorted("sv_SE.UTF-8", &words), sorted("en", &words)); + let cities = ["天津", "北京", "上海"]; + assert_eq!(sorted("zh_TW.UTF-8", &cities), sorted("zh-hant", &cities)); + assert_eq!(sorted("zh_CN.UTF-8", &cities), sorted("zh-hans", &cities)); + // Serbian Cyrillic sorts Cyrillic first; Latin Serbian sorts Latin first, `lj` after `lu`. + let mixed = ["ljubav", "lula", "абв"]; + assert_eq!(sorted("sr_RS.UTF-8", &mixed), ["абв", "ljubav", "lula"]); + assert_eq!(sorted("sr_RS.UTF-8@latin", &mixed), ["lula", "ljubav", "абв"]); + assert_eq!(bcp47("sr_RS.UTF-8@latin"), "sr-latn-RS"); + assert_eq!(bcp47("sr_Latn_RS@latin"), "sr-Latn-RS"); + assert_eq!(bcp47("de_DE@euro"), "de-DE"); + } + #[test] fn chinese_sorts_by_pinyin_or_stroke() { // 北 běi (5 strokes), 上 shàng (3), 天 tiān (4). diff --git a/crates/i18n/src/fold.rs b/crates/i18n/src/fold.rs index 19289fe3..3a3330b1 100644 --- a/crates/i18n/src/fold.rs +++ b/crates/i18n/src/fold.rs @@ -4,8 +4,10 @@ //! [`Folder::fold`] turns text into a search key: compatibility forms decomposed (`fi` → `fi`, //! full-width `Ａ` → `a`, `㎏` → `kg`), lower case, `ß` → `ss`, final `ς` → `σ`, and with accents //! ignored (the default) combining accents, Hebrew points and Arabic vowel marks dropped and the -//! few letters with built-in strokes mapped to their base (`ø`, `ł`, `đ`, `ı`, `æ` → `ae`). -//! Invisible format characters (soft hyphens, zero-width joiners, bidi isolates) are dropped too. +//! few letters with built-in strokes mapped to their base (`ø`, `ł`, `đ`, `ı`, `æ` → `ae`), and +//! Persian letter forms mapped to the Arabic ones (`ک` → `ك`, `ی` → `ي`, `ة` → `ه`). Decimal +//! digits of every script fold to ASCII (`١٢` → `12`). Invisible format characters (soft +//! hyphens, zero-width joiners, bidi isolates) are dropped too. //! //! Dotted and dotless i follow the locale when accents count. In Turkish and Azerbaijani, `I` //! folds to `ı` and `İ` to `i`; elsewhere both fold to `i` and `ı` stays a letter of its own. @@ -33,7 +35,7 @@ impl Folder { /// Case- and accent-insensitive folding for `locale`, a language code such as `tr`, `pt-br` /// or `zh-hant` (any case, `-` or `_`). pub fn new(locale: &str) -> Folder { - let language = locale.split(['-', '_']).next().unwrap_or_default(); + let language = locale.split(['-', '_', '.', '@']).next().unwrap_or_default(); Folder { turkic: language.eq_ignore_ascii_case("tr") || language.eq_ignore_ascii_case("az"), accents: false } } @@ -87,7 +89,13 @@ impl Folder { out.push('i'); } 'I' if self.turkic && self.accents => out.push('ı'), + // Persian heh with hamza above (ۀ) decomposes to ae (ە) and hamza. + '\u{6D5}' if !self.accents && chars.peek() == Some(&'\u{654}') => { + chars.next(); + out.push('ه'); + } 'ß' | 'ẞ' => out.push_str("ss"), + c if let Some(d) = ascii_digit(c) => out.push(d), 'Σ' | 'σ' | 'ς' => out.push('σ'), c if ignorable(c) || (!self.accents && accent(c)) => {} c => { @@ -103,9 +111,15 @@ impl Folder { } } -/// The base of a lower-case letter whose diacritic is built in rather than combining. +/// The base of a lower-case letter whose diacritic is built in rather than combining, and the +/// Arabic form of a letter that Persian and Urdu write differently: keheh (ک) is kaf (ك), Farsi +/// yeh (ی) and alef maksura (ى) are yeh (ي), teh marbuta (ة) is heh (ه), alef wasla (ٱ) is alef. fn base_letter(c: char) -> Option<&'static str> { Some(match c { + 'ک' => "ك", + 'ی' | 'ى' => "ي", + 'ة' => "ه", + 'ٱ' => "ا", 'ı' => "i", 'ø' => "o", 'ł' => "l", @@ -139,6 +153,27 @@ fn accent(c: char) -> bool { | '\u{6DF}'..='\u{6E4}' | '\u{6E7}' | '\u{6E8}' | '\u{6EA}'..='\u{6ED}') } +/// The first code point of each run of ten Unicode decimal digits (general category Nd), Unicode +/// 16.0. +const ZEROS: [u32; 76] = [ + 0x30, 0x660, 0x6F0, 0x7C0, 0x966, 0x9E6, 0xA66, 0xAE6, 0xB66, 0xBE6, 0xC66, 0xCE6, 0xD66, 0xDE6, 0xE50, 0xED0, 0xF20, 0x1040, 0x1090, 0x17E0, + 0x1810, 0x1946, 0x19D0, 0x1A80, 0x1A90, 0x1B50, 0x1BB0, 0x1C40, 0x1C50, 0xA620, 0xA8D0, 0xA900, 0xA9D0, 0xA9F0, 0xAA50, 0xABF0, 0xFF10, 0x104A0, + 0x10D30, 0x10D40, 0x11066, 0x110F0, 0x11136, 0x111D0, 0x112F0, 0x11450, 0x114D0, 0x11650, 0x116C0, 0x116D0, 0x116DA, 0x11730, 0x118E0, 0x11950, + 0x11BF0, 0x11C50, 0x11D50, 0x11DA0, 0x11F50, 0x16130, 0x16A60, 0x16AC0, 0x16B50, 0x16D70, 0x1CCF0, 0x1D7CE, 0x1D7D8, 0x1D7E2, 0x1D7EC, 0x1D7F6, + 0x1E140, 0x1E2F0, 0x1E4F0, 0x1E5F1, 0x1E950, 0x1FBF0, +]; + +/// A decimal digit of any script as its ASCII digit: `١٢` and `१२` search as `12`. +fn ascii_digit(c: char) -> Option { + let c = u32::from(c); + let zero = match ZEROS.binary_search(&c) { + Ok(i) => ZEROS.get(i), + Err(i) => ZEROS.get(i.checked_sub(1)?), + }?; + let value = c - zero; + if value < 10 { char::from_digit(value, 10) } else { None } +} + /// Folded text and where each part came from. #[derive(Clone, Debug, PartialEq, Eq)] pub struct Folded { @@ -245,6 +280,10 @@ mod tests { ("he", "שָׁלוֹם", "שלום".into(), nfd("שָׁלוֹם")), ("en", "soft\u{AD}ware \u{2068}name\u{2069}", "software name".into(), "software name".into()), ("ru", "Ёлка", "елка".into(), nfd("ёлка")), + ("ar", "Layer ١٢ ۳۴ १२", "layer 12 34 12".into(), "layer 12 34 12".into()), + ("fa", "کتاب یک", "كتاب يك".into(), "کتاب یک".into()), + ("fa", "خانۀ", "خانه".into(), nfd("خانۀ")), + ("ar", "مدرسة على إسلام آمن", "مدرسه علي اسلام امن".into(), nfd("مدرسة على إسلام آمن")), ]; for (locale, input, ignoring, keeping) in table { assert_eq!(key(locale, input), ignoring, "{locale} {input}"); @@ -287,6 +326,22 @@ mod tests { assert_eq!(all, [0..2, 3..5, 6..8]); } + #[test] + fn persian_and_arabic_letters_match() { + let f = Folder::new("fa"); + // Persian keheh and Farsi yeh against Arabic kaf and yeh, and hamza forms of alef. + assert!(f.contains("كتاب عربي", "کتاب عربی")); + assert!(f.contains("کتاب عربی", "كتاب عربي")); + assert!(f.contains("أحمد إبراهيم", "احمد ابراهیم")); + assert!(f.contains("مدرسة", "مدرسه")); + assert_eq!(f.fold("Layer ١٢").find(&f.fold("layer 12")), Some(0.."Layer ١٢".len())); + assert!(Folder::new("en").keep_accents().contains("Layer ١٢", "layer 12")); + let digits = ZEROS.iter().map(|&zero| char::from_u32(zero + 7).unwrap()); + assert!(digits.clone().all(|d| d.is_numeric() && ascii_digit(d) == Some('7'))); + assert_eq!(ascii_digit('\u{1D800}'), None); + assert_eq!(ascii_digit('/'), None); + } + #[test] fn accents_count_only_whole_letters() { let f = Folder::new("fr").keep_accents();