diff --git a/.github/workflows/i18n.yml b/.github/workflows/i18n.yml index a973e37f..70f0f71e 100644 --- a/.github/workflows/i18n.yml +++ b/.github/workflows/i18n.yml @@ -25,5 +25,5 @@ jobs: with: components: rustfmt, clippy - run: cargo fmt --check - - run: cargo clippy -p craft-i18n --all-targets --locked -- -D warnings - - run: cargo test -p craft-i18n --locked -- --nocapture + - run: cargo clippy -p craft-i18n --all-targets --features collate --locked -- -D warnings + - run: cargo test -p craft-i18n --features collate --locked -- --nocapture diff --git a/ATTRIBUTION.md b/ATTRIBUTION.md index bf1d6ff3..7a2a751f 100644 --- a/ATTRIBUTION.md +++ b/ATTRIBUTION.md @@ -9,6 +9,7 @@ author, source and license. Library code and original assets are under the repos | `crates/cms/profiles/photocraft-coated-cmyk.icc` | PhotoCraft contributors; generated by `craft_cms::synth::coated_cmyk` | [PhotoCraft `crates/cms`](https://github.com/storytold/photocraft/tree/b3222ee8e7e6bcb4799c37f904c595f20f9baf5e/crates/cms) | CC0-1.0 | | `crates/cms/profiles/craft-coated-fogra39.icc` | craft-libs contributors; generated by `crates/cms/tests/fogra39.rs` from the FOGRA39 data below | this repository | CC0-1.0 | | `crates/cms/tests/data/FOGRA39.txt` (unmodified) | Fogra | [Fogra characterisation data](https://fogra.org/en/downloads/work-tools/characterisation-data), `FOGRA39_to_FOGRA60_v2.zip` | Fogra terms: free use and redistribution, unmodified, with Fogra identified as the source; "FOGRA39" identifies the reference data only and implies no certification, approval or endorsement by Fogra (full text in [NOTICE](NOTICE)) | +| `crates/i18n/data/collation.postcard` | Unicode, Inc. (ICU 78.1 collation data, CLDR 48.2.1); generated by `crates/i18n/data/generate-collation.sh` | [ICU](https://github.com/unicode-org/icu/releases/tag/release-78.1rc) | Unicode-3.0 | Original assets are covered by the repository licence. diff --git a/Cargo.lock b/Cargo.lock index 10c51c11..e87022a8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -624,6 +624,15 @@ dependencies = [ "x509-cert", ] +[[package]] +name = "cobs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa961b519f0b462e3a3b4a34b64d119eeaca1d59af726fe450bbba07a9fc0a1" +dependencies = [ + "thiserror 2.0.21", +] + [[package]] name = "codespan-reporting" version = "0.13.1" @@ -790,6 +799,14 @@ version = "0.1.0" name = "craft-i18n" version = "0.1.0" dependencies = [ + "icu_collator", + "icu_locale_core", + "icu_locale_fallback", + "icu_normalizer", + "icu_properties", + "icu_provider", + "icu_provider_adapters", + "icu_provider_blob", "intl_pluralrules", "log", "objc2-foundation 0.3.2", @@ -1254,6 +1271,18 @@ dependencies = [ "bytemuck", ] +[[package]] +name = "embedded-io" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced" + +[[package]] +name = "embedded-io" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" + [[package]] name = "encoding_rs" version = "0.8.42" @@ -2066,6 +2095,24 @@ dependencies = [ "cc", ] +[[package]] +name = "icu_collator" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08984ed58ac439ebf3e13d2cf26b0c46a60afcd21721c1d14087c0b240344dda" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_normalizer", + "icu_properties", + "icu_provider", + "serde", + "smallvec", + "utf16_iter", + "utf8_iter", + "zerovec", +] + [[package]] name = "icu_collections" version = "2.3.0" @@ -2074,6 +2121,7 @@ checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" dependencies = [ "displaydoc", "potential_utf", + "serde", "utf8_iter", "yoke", "zerofrom", @@ -2088,11 +2136,32 @@ checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" dependencies = [ "displaydoc", "litemap", + "serde", "tinystr", "writeable", "zerovec", ] +[[package]] +name = "icu_locale_fallback" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "251af8e57c9400e3eb58242fe5b8b1152b2a64fdf4cf632f923c38ccee6f2fa9" +dependencies = [ + "icu_locale_core", + "icu_locale_fallback_data", + "icu_provider", + "potential_utf", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locale_fallback_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "decf2a22ec8fa68f1a0c1129a3f8583f8f8bc24e8b9ccbe98ead99f62a4dc3a8" + [[package]] name = "icu_normalizer" version = "2.3.0" @@ -2103,7 +2172,11 @@ dependencies = [ "icu_normalizer_data", "icu_properties", "icu_provider", + "serde", "smallvec", + "utf16_iter", + "utf8_iter", + "write16", "zerovec", ] @@ -2124,6 +2197,7 @@ dependencies = [ "icu_locale_core", "icu_properties_data", "icu_provider", + "serde", "zerotrie", "zerovec", ] @@ -2142,6 +2216,9 @@ checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" dependencies = [ "displaydoc", "icu_locale_core", + "postcard", + "serde", + "stable_deref_trait", "writeable", "yoke", "zerofrom", @@ -2149,6 +2226,30 @@ dependencies = [ "zerovec", ] +[[package]] +name = "icu_provider_adapters" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df887bc4f55dbc773c7c45a0c7dfa75be886f7b7812830a71ab7271cc1801820" +dependencies = [ + "icu_locale_fallback", + "icu_provider", +] + +[[package]] +name = "icu_provider_blob" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5916b44fac319d3f58e4d8d805a627df19e27937b582cd99fcde3bb6b7ce1b0e" +dependencies = [ + "icu_provider", + "postcard", + "serde", + "writeable", + "zerotrie", + "zerovec", +] + [[package]] name = "idna" version = "1.1.0" @@ -2469,6 +2570,9 @@ name = "litemap" version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" +dependencies = [ + "serde_core", +] [[package]] name = "litrs" @@ -3336,12 +3440,26 @@ dependencies = [ "portable-atomic", ] +[[package]] +name = "postcard" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6764c3b5dd454e283a30e6dfe78e9b31096d9e32036b5d1eaac7a6119ccb9a24" +dependencies = [ + "cobs", + "embedded-io 0.4.0", + "embedded-io 0.6.1", + "serde", +] + [[package]] name = "potential_utf" version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" dependencies = [ + "serde_core", + "writeable", "zerovec", ] @@ -5131,6 +5249,12 @@ dependencies = [ "xmlwriter", ] +[[package]] +name = "utf16_iter" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" + [[package]] name = "utf8_iter" version = "1.0.4" @@ -6081,6 +6205,12 @@ dependencies = [ "read-fonts 0.39.2", ] +[[package]] +name = "write16" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" + [[package]] name = "writeable" version = "0.6.4" @@ -6349,8 +6479,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" dependencies = [ "displaydoc", + "litemap", + "serde_core", "yoke", "zerofrom", + "zerovec", ] [[package]] diff --git a/crates/i18n/Cargo.toml b/crates/i18n/Cargo.toml index b799005d..4a0abe43 100644 --- a/crates/i18n/Cargo.toml +++ b/crates/i18n/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "craft-i18n" -description = "Translations for the Craft apps: the embedded craft-i18n catalogs, lookup with fallback, plurals, language names and the system locale." +description = "Translations for the Craft apps: the embedded craft-i18n catalogs, lookup with fallback, plurals, language names, the system locale, search folding and locale sorting." version.workspace = true edition.workspace = true license.workspace = true @@ -12,6 +12,15 @@ intl_pluralrules = "=7.0.2" unic-langid = "0.9" log = "0.4" sys-locale = "0.3" +icu_normalizer = "~2.3" +icu_properties = "~2.3" +# Generated data fits one ICU4X minor version; data/generate-collation.sh names its datagen. +icu_collator = { version = "~2.3.1", default-features = false, features = ["serde"], optional = true } +icu_provider = { version = "~2.3", features = ["serde"], optional = true } +icu_provider_blob = { version = "~2.3", features = ["alloc"], optional = true } +icu_provider_adapters = { version = "~2.3", optional = true } +icu_locale_core = { version = "2.3", optional = true } +icu_locale_fallback = { version = "~2.3", features = ["compiled_data"], optional = true } unicode-bidi = "0.3" [target.'cfg(target_arch = "wasm32")'.dependencies] @@ -25,5 +34,16 @@ windows-registry = "0.6" [target.'cfg(target_vendor = "apple")'.dependencies] objc2-foundation = { version = "0.3.2", default-features = false, features = ["std", "NSDateFormatter", "NSFormatter", "NSLocale", "NSString"] } +[features] +# Locale sorting (`collate`), with ICU4X collation data for the suite's locales embedded. +collate = [ + "dep:icu_collator", + "dep:icu_locale_core", + "dep:icu_locale_fallback", + "dep:icu_provider", + "dep:icu_provider_adapters", + "dep:icu_provider_blob", +] + [lints] workspace = true diff --git a/crates/i18n/README.md b/crates/i18n/README.md index e151cdc4..cab76e07 100644 --- a/crates/i18n/README.md +++ b/crates/i18n/README.md @@ -110,3 +110,15 @@ The app keeps its `tl!` macro, its language setting (`"auto"` → `I18N.system_l included, comes back as `⟦…⟧`, line by line, so text drawn without the marks never went through translation. `looked_up()` lists the source texts looked up so far, for an audit against a complete catalog. +- **Search** (`Folder`): `Folder::new(locale).fold(text)` gives a key for case- and + accent-insensitive matching (`keep_accents()` makes accents count) and a map back to the original + text, so `Folded::find` returns the byte range to highlight. Compatibility forms fold to their + plain letters (`fi`, full-width `Ａ`), `ß` to `ss`, `ς` to `σ`, digits of every script to ASCII + (`Layer ١٢` finds `layer 12`); ignoring accents, Persian and Arabic letter forms match (`ک` and + `ك`, `ی` and `ي`, `ة` and `ه`, `أ` and `ا`); invisible format characters (soft hyphens, joiners, + bidi isolates) are dropped. Turkish and Azerbaijani fold `I` to `ı` when accents count; ignoring + accents, `I`, `i`, `İ` and `ı` all match. +- **Sorting** (`Collator`, feature `collate`): ICU4X collation for the locales craft-i18n ships, + Persian and Hindi, with digits compared by value (`Layer 2` before `Layer 10`) and `-u-co-pinyin` + / `-u-co-stroke` for Chinese. The data (754 KB) is regenerated by `data/generate-collation.sh`; + the feature adds about 1 MB to a release binary or wasm module (320 KB gzipped). diff --git a/crates/i18n/data/collation.postcard b/crates/i18n/data/collation.postcard new file mode 100644 index 00000000..7eabc2f2 Binary files /dev/null and b/crates/i18n/data/collation.postcard differ diff --git a/crates/i18n/data/generate-collation.sh b/crates/i18n/data/generate-collation.sh new file mode 100755 index 00000000..35f605eb --- /dev/null +++ b/crates/i18n/data/generate-collation.sh @@ -0,0 +1,15 @@ +#!/bin/sh +# Regenerates the collation data `collate` embeds, a postcard blob for the locales craft-i18n +# ships (its `locales/` directory) plus English, Persian and Hindi: each locale's default order, +# and Chinese pinyin and stroke orders. +# Needs icu4x-datagen at the version of the icu_collator dependency: +# cargo install icu4x-datagen --version 2.3.0 --locked +set -eu +cd "$(dirname "$0")" +icu4x-datagen --format blob --overwrite --out collation.postcard --deduplication maximal \ + --cldr-tag 48.2.1 --icuexport-tag release-78.1rc \ + --attribute-filter 'collator=/^(stroke|pinyin)?$/' \ + --markers CollationRootV1 CollationTailoringV1 CollationDiacriticsV1 CollationJamoV1 \ + CollationMetadataV1 CollationReorderingV1 CollationSpecialPrimariesV1 \ + --locales en ar bg cs da de el es et fa fi fr hi hr hu id it ja ka kk ko lt nl pl pt-BR ru sl \ + sr sr-Latn sv te tr uk vi zh-Hans zh-Hant diff --git a/crates/i18n/src/collate.rs b/crates/i18n/src/collate.rs new file mode 100644 index 00000000..06d41789 --- /dev/null +++ b/crates/i18n/src/collate.rs @@ -0,0 +1,234 @@ +//! Locale sorting for lists people see: fonts, layers, presets, files. +//! +//! ICU4X collation with data embedded for the locales craft-i18n ships (regenerated by +//! `data/generate-collation.sh`). A locale without data of its own sorts in the Unicode root +//! order, which suits most languages that have no tailoring. + +use std::cmp::Ordering; + +use icu_collator::CollatorPreferences; +use icu_collator::options::CollatorOptions; +use icu_collator::preferences::CollationNumericOrdering; +use icu_collator::provider::{ + CollationDiacriticsV1, CollationJamoV1, CollationMetadataV1, CollationReorderingV1, CollationRootV1, CollationSpecialPrimariesV1, + CollationTailoringV1, +}; +use icu_locale_core::Locale; +use icu_locale_fallback::LocaleFallbacker; +use icu_normalizer::provider::{NormalizerNfdDataV1, NormalizerNfdTablesV1}; +use icu_provider::buf::AsDeserializingBufferProvider; +use icu_provider::prelude::*; +use icu_provider_adapters::fallback::LocaleFallbackProvider; +use icu_provider_blob::BlobDataProvider; + +/// Compares text in one locale's order, with runs of digits by their value: `Layer 2` before +/// `Layer 10`. +#[derive(Debug)] +pub struct Collator(Option); + +impl Collator { + /// Sorting for `locale`, a language code such as `sv`, `pt-br` or `zh-hant` (any case, `-` + /// or `_`, glibc forms such as `sv_SE.UTF-8` and `sr_RS@latin`). A BCP 47 collation type + /// picks another order where there is one: `zh-hant-u-co-pinyin`, `zh-hans-u-co-stroke`. + pub fn new(locale: &str) -> Collator { + let mut prefs: CollatorPreferences = Locale::try_from_str(&bcp47(locale)).unwrap_or(Locale::UNKNOWN).into(); + prefs.numeric_ordering = Some(CollationNumericOrdering::True); + let options = CollatorOptions::default(); + let collator = Data::new().ok().and_then(|data| { + icu_collator::Collator::try_new_unstable(&data, prefs, options) + .or_else(|_| icu_collator::Collator::try_new_unstable(&data, CollatorPreferences::default(), options)) + .ok() + }); + Collator(collator) + } + + /// `a` against `b`. Text that differs only in case or accents still has an order, and text + /// the collation counts as equal (`Layer 2` and `Layer 02`) is in code point order, so a sort + /// doesn't depend on the input order. Without collation data (never with the embedded data) + /// this is code point order. + pub fn compare(&self, a: &str, b: &str) -> Ordering { + match &self.0 { + Some(c) => c.as_borrowed().compare(a, b).then_with(|| a.cmp(b)), + None => a.cmp(b), + } + } + + /// Sort `items` by the text `key` gives each one. + pub fn sort_by_key(&self, items: &mut [T], key: impl Fn(&T) -> &str) { + items.sort_by(|a, b| self.compare(key(a), key(b))); + } +} + +/// `locale` as a BCP 47 tag: `_` → `-`, a glibc codeset dropped and its script modifier made a +/// subtag (`sr_RS.UTF-8@latin` → `sr-latn-RS`). +fn bcp47(locale: &str) -> String { + let locale = locale.trim(); + let mut parts: Vec<&str> = locale.split(['.', '@']).next().unwrap_or_default().split(['-', '_']).collect(); + let script = match locale.split_once('@').map(|(_, m)| m.to_ascii_lowercase()).as_deref() { + Some("latin") => Some("latn"), + Some("cyrillic") => Some("cyrl"), + _ => None, + }; + let has_script = parts.get(1).is_some_and(|p| p.len() == 4 && p.bytes().all(|b| b.is_ascii_alphabetic())); + if let Some(script) = script + && !has_script + { + parts.insert(1.min(parts.len()), script); + } + parts.join("-") +} + +/// The embedded collation data, with locale fallback, and the decomposition tables the collator +/// also needs taken from `icu_normalizer`'s compiled data rather than embedded a second time. +struct Data(LocaleFallbackProvider); + +impl Data { + fn new() -> Result { + let blob = BlobDataProvider::try_new_from_static_blob(include_bytes!("../data/collation.postcard"))?; + Ok(Data(LocaleFallbackProvider::new(blob, LocaleFallbacker::new().static_to_owned()))) + } +} + +macro_rules! from_blob { + ($($marker:ty),*) => {$( + impl DataProvider<$marker> for Data { + fn load(&self, req: DataRequest) -> Result, DataError> { + self.0.as_deserializing().load(req) + } + } + )*}; +} + +from_blob!( + CollationRootV1, + CollationTailoringV1, + CollationDiacriticsV1, + CollationJamoV1, + CollationMetadataV1, + CollationReorderingV1, + CollationSpecialPrimariesV1 +); + +impl DataProvider for Data { + fn load(&self, req: DataRequest) -> Result, DataError> { + icu_normalizer::provider::Baked.load(req) + } +} + +impl DataProvider for Data { + fn load(&self, req: DataRequest) -> Result, DataError> { + icu_normalizer::provider::Baked.load(req) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn sorted<'a>(locale: &str, items: &[&'a str]) -> Vec<&'a str> { + let mut v = items.to_vec(); + Collator::new(locale).sort_by_key(&mut v, |s| s); + v + } + + /// Each tailored locale puts a pair in another order than the root collation does, which a + /// locale whose data is missing (it falls back to root) wouldn't. + #[test] + fn every_shipped_locale_has_its_data() { + let root = Collator::new("und"); + for (locale, first, second) in [ + ("ar", "ب", "a"), + ("bg", "б", "a"), + ("cs", "hrad", "chata"), + ("da", "zebra", "år"), + ("el", "β", "a"), + ("es", "nube", "ñu"), + ("et", "wc", "õun"), + ("fa", "ب", "a"), + ("fi", "zeta", "äiti"), + ("hi", "क", "a"), + ("hr", "cvijet", "čaj"), + ("hu", "cukor", "csak"), + ("ja", "亜", "一"), + ("ka", "ა", "a"), + ("kk", "б", "a"), + ("ko", "가", "a"), + ("lt", "yra", "jau"), + ("pl", "az", "ąb"), + ("ru", "б", "a"), + ("sl", "cvet", "čaj"), + ("sr", "б", "a"), + ("sr-latn", "cvijet", "čaj"), + ("sv", "zebra", "år"), + ("te", "క", "a"), + ("tr", "cz", "ça"), + ("uk", "б", "a"), + ("vi", "az", "ăn"), + ("zh-hans", "阿", "中"), + ("zh-hant", "人", "丈"), + ] { + assert_eq!(Collator::new(locale).compare(first, second), Ordering::Less, "{locale}: {first} < {second}"); + assert_eq!(root.compare(first, second), Ordering::Greater, "root: {second} < {first}"); + } + // CLDR doesn't tailor these: they sort in the root order. + for locale in ["en", "de", "fr", "id", "it", "nl", "pt-br", "not a locale"] { + let collator = Collator::new(locale); + assert!(collator.0.is_some(), "{locale}"); + assert_eq!(collator.compare("čaj", "cvijet"), Ordering::Less, "{locale}"); + } + } + + #[test] + fn digits_sort_by_value() { + let layers = ["Layer 10", "Layer 2", "layer 1", "Layer 1", "Layer 2b", "Layer 02"]; + assert_eq!(sorted("en", &layers), ["layer 1", "Layer 1", "Layer 02", "Layer 2", "Layer 2b", "Layer 10"]); + let reversed: Vec<&str> = layers.iter().rev().copied().collect(); + assert_eq!(sorted("en", &reversed), sorted("en", &layers)); + assert_eq!(sorted("de", &["Ebene ٣", "Ebene 10", "Ebene ２"]), ["Ebene ２", "Ebene ٣", "Ebene 10"]); + } + + #[test] + fn locales_sort_their_own_letters() { + let words = ["öl", "zebra", "år", "äpple", "apa", "oxe"]; + assert_eq!(sorted("sv", &words), ["apa", "oxe", "zebra", "år", "äpple", "öl"]); + assert_eq!(sorted("de", &words), ["apa", "äpple", "år", "öl", "oxe", "zebra"]); + assert_eq!(sorted("de", &["Muller", "Müller", "Mueller", "Mahler"]), ["Mahler", "Mueller", "Muller", "Müller"]); + assert_eq!( + sorted("tr", &["şeker", "sabun", "ılık", "iğne", "çay", "cam", "uzun", "üzüm"]), + ["cam", "çay", "ılık", "iğne", "sabun", "şeker", "uzun", "üzüm"] + ); + assert_eq!(sorted("en", &["ılık", "iğne", "cam", "çay"]), ["cam", "çay", "iğne", "ılık"]); + assert_eq!(sorted("es", &["ñu", "nube", "oso"]), ["nube", "ñu", "oso"]); + // Japanese: Latin, then kana in gojūon order (hiragana and katakana together), then kanji. + assert_eq!(sorted("ja", &["漢字", "カメラ", "あめ", "いぬ", "Zoom"]), ["Zoom", "あめ", "いぬ", "カメラ", "漢字"]); + } + + #[test] + fn glibc_locale_names() { + let words = ["öl", "zebra", "år", "äpple", "apa", "oxe"]; + assert_eq!(sorted("sv_SE.UTF-8", &words), sorted("sv", &words)); + assert_ne!(sorted("sv_SE.UTF-8", &words), sorted("en", &words)); + let cities = ["天津", "北京", "上海"]; + assert_eq!(sorted("zh_TW.UTF-8", &cities), sorted("zh-hant", &cities)); + assert_eq!(sorted("zh_CN.UTF-8", &cities), sorted("zh-hans", &cities)); + // Serbian Cyrillic sorts Cyrillic first; Latin Serbian sorts Latin first, `lj` after `lu`. + let mixed = ["ljubav", "lula", "абв"]; + assert_eq!(sorted("sr_RS.UTF-8", &mixed), ["абв", "ljubav", "lula"]); + assert_eq!(sorted("sr_RS.UTF-8@latin", &mixed), ["lula", "ljubav", "абв"]); + assert_eq!(bcp47("sr_RS.UTF-8@latin"), "sr-latn-RS"); + assert_eq!(bcp47("sr_Latn_RS@latin"), "sr-Latn-RS"); + assert_eq!(bcp47("de_DE@euro"), "de-DE"); + } + + #[test] + fn chinese_sorts_by_pinyin_or_stroke() { + // 北 běi (5 strokes), 上 shàng (3), 天 tiān (4). + let cities = ["天津", "北京", "上海"]; + let pinyin = ["北京", "上海", "天津"]; + let stroke = ["上海", "天津", "北京"]; + assert_eq!(sorted("zh-hans", &cities), pinyin); + assert_eq!(sorted("zh-hant", &cities), stroke); + assert_eq!(sorted("zh-hans-u-co-stroke", &cities), stroke); + assert_eq!(sorted("zh-hant-u-co-pinyin", &cities), pinyin); + } +} diff --git a/crates/i18n/src/fold.rs b/crates/i18n/src/fold.rs new file mode 100644 index 00000000..8d2e256b --- /dev/null +++ b/crates/i18n/src/fold.rs @@ -0,0 +1,424 @@ +//! Case- and accent-insensitive matching for search fields, with a map back to the original text +//! for highlighting. +//! +//! [`Folder::fold`] turns text into a search key: compatibility forms decomposed (`fi` → `fi`, +//! full-width `Ａ` → `a`, `㎏` → `kg`), lower case, `ß` → `ss`, final `ς` → `σ`, and with accents +//! ignored (the default) combining accents, Hebrew points and Arabic vowel marks dropped and the +//! few letters with built-in strokes mapped to their base (`ø`, `ł`, `đ`, `ı`, `æ` → `ae`), and +//! Persian letter forms mapped to the Arabic ones (`ک` → `ك`, `ی` → `ي`, `ة` → `ه`). Decimal +//! digits of every script fold to ASCII (`١٢` → `12`). Invisible format characters (soft +//! hyphens, zero-width joiners, bidi isolates) are dropped too. +//! +//! Dotted and dotless i follow the locale when accents count. In Turkish and Azerbaijani, `I` +//! folds to `ı` and `İ` to `i`; elsewhere both fold to `i` and `ı` stays a letter of its own. +//! Ignoring accents, all four fold to `i`, so `istanbul` finds `İSTANBUL` and `irmak` finds +//! `IRMAK` and `ırmak`. Plain `str::to_lowercase` follows no locale and turns `İ` into `i` plus a +//! combining dot, which nothing typed would match. +//! +//! The Turkish i handling and sigma folding come from PdfCraft's `fold_case` (snipeTR, +//! nevicia2010-hub). + +use std::ops::Range; + +use icu_normalizer::DecomposingNormalizerBorrowed; +use icu_properties::props::{GraphemeExtend, HangulSyllableType}; +use icu_properties::{CodePointMapData, CodePointSetData}; + +/// Folds text for one locale. Cheap to build and copy. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Folder { + turkic: bool, + accents: bool, +} + +impl Folder { + /// Case- and accent-insensitive folding for `locale`, a language code such as `tr`, `pt-br` + /// or `zh-hant` (any case, `-` or `_`). + pub fn new(locale: &str) -> Folder { + let language = locale.split(['-', '_', '.', '@']).next().unwrap_or_default(); + Folder { turkic: language.eq_ignore_ascii_case("tr") || language.eq_ignore_ascii_case("az"), accents: false } + } + + /// Accents count: `é` no longer matches `e`, and a match can't end on a letter whose accent it + /// leaves out. + pub fn keep_accents(self) -> Folder { + Folder { accents: true, ..self } + } + + /// `s` folded, with the map back to `s`. + pub fn fold(&self, s: &str) -> Folded { + let nfkd = DecomposingNormalizerBorrowed::new_nfkd(); + let extend = CodePointSetData::new::(); + let hangul = CodePointMapData::::new(); + let mut text = String::with_capacity(s.len()); + let mut starts = Vec::new(); + // A cluster is a character and the combining marks after it, folded and mapped as one; + // conjoining jamo that spell one syllable are one cluster too. + let mut cluster_start = 0; + let mut chars = s.char_indices(); + let mut prev = chars.next().map(|(_, c)| hangul.get(c)); + for (i, c) in chars { + let kind = hangul.get(c); + if !extend.contains(c) && !prev.is_some_and(|p| continues_syllable(p, kind)) { + starts.push((text.len(), cluster_start)); + self.fold_cluster(&nfkd.normalize(&s[cluster_start..i]), &mut text); + cluster_start = i; + } + prev = Some(kind); + } + if !s.is_empty() { + starts.push((text.len(), cluster_start)); + self.fold_cluster(&nfkd.normalize(&s[cluster_start..]), &mut text); + } + starts.push((text.len(), s.len())); + Folded { text, starts } + } + + /// `s` folded, without the map: a key to compare or store. + pub fn key(&self, s: &str) -> String { + self.fold(s).text + } + + /// Whether `haystack` contains `needle`, both folded. + pub fn contains(&self, haystack: &str, needle: &str) -> bool { + self.fold(haystack).find(&self.fold(needle)).is_some() + } + + /// One cluster, already in NFKD. + fn fold_cluster(&self, cluster: &str, out: &mut String) { + let mut chars = cluster.chars().peekable(); + while let Some(c) = chars.next() { + match c { + // `İ` decomposes to `I` and a combining dot above: the dot of a lower-case `i`. + 'I' if chars.peek() == Some(&'\u{307}') => { + chars.next(); + out.push('i'); + } + 'I' if self.turkic && self.accents => out.push('ı'), + // Persian heh with hamza above (ۀ) decomposes to ae (ە) and hamza. + '\u{6D5}' if !self.accents && chars.peek() == Some(&'\u{654}') => { + chars.next(); + out.push('ه'); + } + 'ß' | 'ẞ' => out.push_str("ss"), + c if let Some(d) = ascii_digit(c) => out.push(d), + 'Σ' | 'σ' | 'ς' => out.push('σ'), + c if ignorable(c) || (!self.accents && accent(c)) => {} + c => { + for l in c.to_lowercase() { + match base_letter(l) { + Some(base) if !self.accents => out.push_str(base), + _ => out.push(l), + } + } + } + } + } + } +} + +/// The base of a lower-case letter whose diacritic is built in rather than combining, and the +/// Arabic form of a letter that Persian and Urdu write differently: keheh (ک) is kaf (ك), Farsi +/// yeh (ی) and alef maksura (ى) are yeh (ي), teh marbuta (ة) is heh (ه), alef wasla (ٱ) is alef. +fn base_letter(c: char) -> Option<&'static str> { + Some(match c { + 'ک' => "ك", + 'ی' | 'ى' => "ي", + 'ة' => "ه", + 'ٱ' => "ا", + 'ı' => "i", + 'ø' => "o", + 'ł' => "l", + 'đ' => "d", + 'ħ' => "h", + 'ŧ' => "t", + 'æ' => "ae", + 'œ' => "oe", + _ => return None, + }) +} + +/// Invisible characters that only steer layout: soft hyphen, Arabic letter mark, the kashida +/// that stretches Arabic words, Mongolian variation selectors, zero-width space and joiners, direction marks, embeddings and isolates, +/// word joiner, variation selectors, byte order mark and tags. +fn ignorable(c: char) -> bool { + matches!(c, + '\u{AD}' | '\u{61C}' | '\u{640}' | '\u{180B}'..='\u{180F}' | '\u{200B}'..='\u{200F}' | '\u{202A}'..='\u{202E}' + | '\u{2060}'..='\u{206F}' | '\u{FE00}'..='\u{FE0F}' | '\u{FEFF}' | '\u{E0000}'..='\u{E0FFF}') +} + +/// Marks that searching without accents ignores: Latin, Greek and Cyrillic combining diacritics, +/// Hebrew points and cantillation, Arabic vowel marks. +/// Kana voicing marks, Indic and Thai vowel signs are letters' parts, not accents, and stay. +fn accent(c: char) -> bool { + matches!(c, + '\u{300}'..='\u{36F}' | '\u{1AB0}'..='\u{1AFF}' | '\u{1DC0}'..='\u{1DFF}' | '\u{20D0}'..='\u{20FF}' + | '\u{FE20}'..='\u{FE2F}' + | '\u{591}'..='\u{5BD}' | '\u{5BF}' | '\u{5C1}' | '\u{5C2}' | '\u{5C4}' | '\u{5C5}' | '\u{5C7}' + | '\u{610}'..='\u{61A}' | '\u{64B}'..='\u{65F}' | '\u{670}' | '\u{6D6}'..='\u{6DC}' + | '\u{6DF}'..='\u{6E4}' | '\u{6E7}' | '\u{6E8}' | '\u{6EA}'..='\u{6ED}') +} + +/// The first code point of each run of ten Unicode decimal digits (general category Nd), Unicode +/// 16.0. +const ZEROS: [u32; 76] = [ + 0x30, 0x660, 0x6F0, 0x7C0, 0x966, 0x9E6, 0xA66, 0xAE6, 0xB66, 0xBE6, 0xC66, 0xCE6, 0xD66, 0xDE6, 0xE50, 0xED0, 0xF20, 0x1040, 0x1090, 0x17E0, + 0x1810, 0x1946, 0x19D0, 0x1A80, 0x1A90, 0x1B50, 0x1BB0, 0x1C40, 0x1C50, 0xA620, 0xA8D0, 0xA900, 0xA9D0, 0xA9F0, 0xAA50, 0xABF0, 0xFF10, 0x104A0, + 0x10D30, 0x10D40, 0x11066, 0x110F0, 0x11136, 0x111D0, 0x112F0, 0x11450, 0x114D0, 0x11650, 0x116C0, 0x116D0, 0x116DA, 0x11730, 0x118E0, 0x11950, + 0x11BF0, 0x11C50, 0x11D50, 0x11DA0, 0x11F50, 0x16130, 0x16A60, 0x16AC0, 0x16B50, 0x16D70, 0x1CCF0, 0x1D7CE, 0x1D7D8, 0x1D7E2, 0x1D7EC, 0x1D7F6, + 0x1E140, 0x1E2F0, 0x1E4F0, 0x1E5F1, 0x1E950, 0x1FBF0, +]; + +/// A decimal digit of any script as its ASCII digit: `١٢` and `१२` search as `12`. +fn ascii_digit(c: char) -> Option { + let c = u32::from(c); + let zero = match ZEROS.binary_search(&c) { + Ok(i) => ZEROS.get(i), + Err(i) => ZEROS.get(i.checked_sub(1)?), + }?; + let value = c - zero; + if value < 10 { char::from_digit(value, 10) } else { None } +} + +/// Whether a conjoining jamo of kind `next` continues the syllable a character of kind `prev` +/// ends: a vowel after a leading consonant, a final consonant after a vowel or an open syllable +/// (`가` and `ᆨ` spell `각`). +fn continues_syllable(prev: HangulSyllableType, next: HangulSyllableType) -> bool { + matches!( + (prev, next), + (HangulSyllableType::LeadingJamo, HangulSyllableType::VowelJamo) + | (HangulSyllableType::VowelJamo | HangulSyllableType::LVSyllable, HangulSyllableType::TrailingJamo) + ) +} + +/// Folded text and where each part came from. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Folded { + text: String, + /// One (folded offset, source offset) per source cluster, in order, then both lengths. A + /// cluster folds to the text up to the next entry's folded offset, possibly none. + starts: Vec<(usize, usize)>, +} + +impl Folded { + /// The folded text. + pub fn as_str(&self) -> &str { + &self.text + } + + /// The source byte range that folded bytes `range` came from, widened to whole source + /// characters with their accents: `ss` in the folding of `Straße` maps back to `ß`, `e` in + /// that of `é` to the whole `é`. An empty range maps to an empty one. + pub fn source_range(&self, range: Range) -> Range { + // The last cluster starting at or before `at` is the one holding it: clusters that fold to + // nothing share their offset with the next one. + let cluster = |at: usize| self.starts.partition_point(|&(f, _)| f <= at).saturating_sub(1); + if range.is_empty() { + let s = self.starts[cluster(range.start)].1; + return s..s; + } + let first = cluster(range.start); + let last = cluster(range.end - 1); + self.starts[first].1..self.starts[(last + 1).min(self.starts.len() - 1)].1 + } + + /// The first match of `needle` (folded the same way), as a byte range of the source text. + /// An empty needle matches at the start. + pub fn find(&self, needle: &Folded) -> Option> { + self.matches(needle).next() + } + + /// Every non-overlapping match of `needle` (folded the same way), as byte ranges of the + /// source text. A match may start or end inside an expansion (`ss` of `ß`, `fi` of `fi`) but + /// not leave out a mark that follows a letter, so `re` doesn't match `ré` when accents count. + pub fn matches<'a>(&'a self, needle: &'a Folded) -> impl Iterator> + 'a { + let extend = CodePointSetData::new::(); + let hangul = CodePointMapData::::new(); + // The vowel and final consonant of a decomposed Hangul syllable continue it, like a mark, + // but only inside one source cluster: standalone jamo (`ㅋㅠ`, `ㅠㅠ`) are letters. + let starts_mark = move |at: usize| { + let Some(c) = self.text[at..].chars().next() else { return false }; + if extend.contains(c) { + return true; + } + let inside = self.starts.binary_search_by_key(&at, |&(f, _)| f).is_err(); + inside && self.text[..at].chars().next_back().is_some_and(|p| continues_syllable(hangul.get(p), hangul.get(c))) + }; + let n = needle.text.as_str(); + let mut from = 0; + let mut done = false; + std::iter::from_fn(move || { + if done { + return None; + } + if n.is_empty() { + done = true; + return Some(0..0); + } + while let Some(i) = self.text.get(from..).and_then(|t| t.find(n)) { + let start = from + i; + let end = start + n.len(); + if !starts_mark(start) && !starts_mark(end) { + from = end; + return Some(self.source_range(start..end)); + } + from = start + self.text[start..].chars().next().map_or(1, char::len_utf8); + } + done = true; + None + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn key(locale: &str, s: &str) -> String { + Folder::new(locale).key(s) + } + + fn key_accents(locale: &str, s: &str) -> String { + Folder::new(locale).keep_accents().key(s) + } + + #[test] + fn folding_table() { + let nfd = |s: &str| DecomposingNormalizerBorrowed::new_nfd().normalize(s).into_owned(); + // (locale, input, ignoring accents, keeping accents) + let table = [ + ("en", "Hello WORLD", "hello world".to_owned(), "hello world".to_owned()), + ("en", "İSTANBUL", "istanbul".into(), "istanbul".into()), + ("tr", "İSTANBUL", "istanbul".into(), "istanbul".into()), + ("en", "IRMAK ırmak", "irmak irmak".into(), "irmak ırmak".into()), + ("tr", "IRMAK ırmak", "irmak irmak".into(), "ırmak ırmak".into()), + ("az", "İIıi", "iiii".into(), "iııi".into()), + ("tr-TR", "Şişli", "sisli".into(), nfd("şişli")), + ("de", "Straße STRAẞE", "strasse strasse".into(), "strasse strasse".into()), + ("el", "ΟΔΥΣΣΕΥΣ οδυσσευς", "οδυσσευσ οδυσσευσ".into(), "οδυσσευσ οδυσσευσ".into()), + ("el", "Ἀθῆναι", "αθηναι".into(), nfd("ἀθῆναι")), + ("fr", "Œuvre ÉTÉ", "oeuvre ete".into(), nfd("œuvre été")), + ("da", "Ærø", "aero".into(), nfd("ærø")), + ("pl", "Łódź", "lodz".into(), nfd("łódź")), + ("vi", "Đà Nẵng", "da nang".into(), nfd("đà nẵng")), + ("en", "final ff", "final ff".into(), "final ff".into()), + ("ja", "ＡＢＣ１２３　ｶﾞ", nfd("abc123 ガ"), nfd("abc123 ガ")), + ("ar", "مُحَمَّد أحمـــد", nfd("محمد احمد"), nfd("مُحَمَّد أحمد")), + ("he", "שָׁלוֹם", "שלום".into(), nfd("שָׁלוֹם")), + ("en", "soft\u{AD}ware \u{2068}name\u{2069}", "software name".into(), "software name".into()), + ("ru", "Ёлка", "елка".into(), nfd("ёлка")), + ("ar", "Layer ١٢ ۳۴ १२", "layer 12 34 12".into(), "layer 12 34 12".into()), + ("fa", "کتاب یک", "كتاب يك".into(), "کتاب یک".into()), + ("fa", "خانۀ", "خانه".into(), nfd("خانۀ")), + ("ar", "مدرسة على إسلام آمن", "مدرسه علي اسلام امن".into(), nfd("مدرسة على إسلام آمن")), + ]; + for (locale, input, ignoring, keeping) in table { + assert_eq!(key(locale, input), ignoring, "{locale} {input}"); + assert_eq!(key_accents(locale, input), keeping, "{locale} {input}, accents kept"); + } + } + + #[test] + fn turkish_i_depends_on_locale_when_accents_count() { + let tr = Folder::new("tr").keep_accents(); + let en = Folder::new("en").keep_accents(); + assert!(tr.contains("IRMAK", "ırmak") && !tr.contains("IRMAK", "irmak")); + assert!(en.contains("IRMAK", "irmak") && !en.contains("IRMAK", "ırmak")); + assert!(tr.contains("İSTANBUL", "istanbul") && en.contains("İSTANBUL", "istanbul")); + for locale in ["tr", "en"] { + let f = Folder::new(locale); + assert!(f.contains("IRMAK", "ırmak") && f.contains("ırmak", "IRMAK") && f.contains("İzmir", "izmir")); + } + } + + #[test] + fn matches_map_back_to_the_source() { + let f = Folder::new("en"); + fn find<'a>(f: Folder, haystack: &'a str, needle: &str) -> Option<&'a str> { + f.fold(haystack).find(&f.fold(needle)).map(|r| &haystack[r]) + } + let find = |haystack, needle| find(f, haystack, needle); + assert_eq!(find("Große Straße", "strasse"), Some("Straße")); + assert_eq!(find("Große Straße", "SS"), Some("ß")); + assert_eq!(find("Große Straße", "sse"), Some("ße")); + assert_eq!(find("Café crème", "creme"), Some("crème")); + assert_eq!(find("Cafe\u{301} noir", "cafe"), Some("Cafe\u{301}")); + assert_eq!(find("Ｆｉｌｅ ２", "file 2"), Some("Ｆｉｌｅ ２")); + assert_eq!(find("the final cut", "fin"), Some("fin")); + assert_eq!(find("Layer \u{2068}İSTANBUL\u{2069}", "istanbul"), Some("İSTANBUL")); + assert_eq!(find("abc", "x"), None); + assert_eq!(find("abc", ""), Some("")); + + let all: Vec<_> = f.fold("Ab ab AB").matches(&f.fold("ab")).collect(); + assert_eq!(all, [0..2, 3..5, 6..8]); + } + + #[test] + fn persian_and_arabic_letters_match() { + let f = Folder::new("fa"); + // Persian keheh and Farsi yeh against Arabic kaf and yeh, and hamza forms of alef. + assert!(f.contains("كتاب عربي", "کتاب عربی")); + assert!(f.contains("کتاب عربی", "كتاب عربي")); + assert!(f.contains("أحمد إبراهيم", "احمد ابراهیم")); + assert!(f.contains("مدرسة", "مدرسه")); + assert_eq!(f.fold("Layer ١٢").find(&f.fold("layer 12")), Some(0.."Layer ١٢".len())); + assert!(Folder::new("en").keep_accents().contains("Layer ١٢", "layer 12")); + let digits = ZEROS.iter().map(|&zero| char::from_u32(zero + 7).unwrap()); + assert!(digits.clone().all(|d| d.is_numeric() && ascii_digit(d) == Some('7'))); + assert_eq!(ascii_digit('\u{1D800}'), None); + assert_eq!(ascii_digit('/'), None); + } + + #[test] + fn accents_count_only_whole_letters() { + let f = Folder::new("fr").keep_accents(); + assert!(!f.contains("ré", "re")); + assert!(f.contains("ré", "RÉ") && f.contains("re\u{301}", "ré")); + assert!(f.contains("ré", "é")); + // Kana voicing marks stay even with accents ignored: か doesn't find が. + assert!(!Folder::new("ja").contains("が", "か")); + // Nor does a Hangul syllable without its final consonant: 하 doesn't find 한. + let ko = Folder::new("ko"); + assert!(!ko.contains("한국", "하") && !ko.keep_accents().contains("한국", "하")); + assert!(ko.contains("한국", "한") && ko.contains("하늘", "하")); + // Conjoining jamo spell the same syllable; compatibility jamo stand alone. + assert!(!ko.contains("\u{1112}\u{1161}\u{11AB}", "하") && ko.contains("\u{1112}\u{1161}\u{11AB}", "한")); + assert!(!ko.contains("하\u{11AB}", "하")); + for (haystack, needle) in [("ㅠㅠ", "ㅠ"), ("ㅠㅠ", "ㅠㅠ"), ("ㅋㅠ", "ㅋ"), ("ㅋㅠ", "ㅠ"), ("ㅋㅋㅋ", "ㅋㅋ"), ("ㄳ", "ㄳ"), ("값ㄳ", "ㄳ")] + { + assert!(ko.contains(haystack, needle), "{haystack} {needle}"); + } + assert_eq!(ko.fold("ㅋㅠㅠ").find(&ko.fold("ㅠ")).map(|r| r.start), Some("ㅋ".len())); + } + + #[test] + fn source_ranges_round_trip() { + let f = Folder::new("de"); + for s in ["Straße", "ffi Ａ", "e\u{301}\u{AD}x", "", "İı", "\u{301}a"] { + let folded = f.fold(s); + let text = folded.as_str(); + // A leading mark folds to nothing and has no part in any match. + let whole = if s.starts_with('\u{301}') { 2..s.len() } else { 0..s.len() }; + assert_eq!(folded.source_range(0..text.len()), whole, "{s:?}"); + // Every folded character maps to a non-empty source range that folds back to text + // containing it. + for (i, c) in text.char_indices() { + let r = folded.source_range(i..i + c.len_utf8()); + assert!(!r.is_empty() && f.key(&s[r.clone()]).contains(c), "{s:?} {i} {r:?}"); + } + } + } + + #[test] + fn folding_is_idempotent() { + // A folded query folds to itself, so a stored key can be searched with a fresh fold. + for locale in ["en", "tr"] { + for f in [Folder::new(locale), Folder::new(locale).keep_accents()] { + for c in ('\0'..='\u{FFFF}').chain('\u{1D400}'..='\u{1D7FF}') { + let once = f.key(c.encode_utf8(&mut [0; 4])); + assert_eq!(f.key(&once), once, "{locale} {:?} U+{:04X}", f, c as u32); + } + } + } + } +} diff --git a/crates/i18n/src/lib.rs b/crates/i18n/src/lib.rs index 5bc6464a..3318b890 100644 --- a/crates/i18n/src/lib.rs +++ b/crates/i18n/src/lib.rs @@ -6,6 +6,9 @@ #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] mod catalog; +#[cfg(feature = "collate")] +mod collate; +mod fold; mod region; mod scripts; mod size; @@ -17,6 +20,9 @@ use intl_pluralrules::{PluralCategory, PluralRuleType, PluralRules}; use unic_langid::LanguageIdentifier; use unicode_bidi::{BidiClass, bidi_class}; +#[cfg(feature = "collate")] +pub use collate::Collator; +pub use fold::{Folded, Folder}; pub use region::{HourCycle, Measurement, Paper, Region}; pub use scripts::{Direction, Han, Script, base_direction, direction, han, scripts}; pub use size::{FileSize, SizeUnit};