mirror of
https://github.com/storytold/craft-libs.git
synced 2026-11-03 17:26:55 +00:00
235 lines
10 KiB
Rust
235 lines
10 KiB
Rust
//! Locale sorting for lists people see: fonts, layers, presets, files.
|
||
//!
|
||
//! ICU4X collation with data embedded for the locales craft-i18n ships (regenerated by
|
||
//! `data/generate-collation.sh`). A locale without data of its own sorts in the Unicode root
|
||
//! order, which suits most languages that have no tailoring.
|
||
|
||
use std::cmp::Ordering;
|
||
|
||
use icu_collator::CollatorPreferences;
|
||
use icu_collator::options::CollatorOptions;
|
||
use icu_collator::preferences::CollationNumericOrdering;
|
||
use icu_collator::provider::{
|
||
CollationDiacriticsV1, CollationJamoV1, CollationMetadataV1, CollationReorderingV1, CollationRootV1, CollationSpecialPrimariesV1,
|
||
CollationTailoringV1,
|
||
};
|
||
use icu_locale_core::Locale;
|
||
use icu_locale_fallback::LocaleFallbacker;
|
||
use icu_normalizer::provider::{NormalizerNfdDataV1, NormalizerNfdTablesV1};
|
||
use icu_provider::buf::AsDeserializingBufferProvider;
|
||
use icu_provider::prelude::*;
|
||
use icu_provider_adapters::fallback::LocaleFallbackProvider;
|
||
use icu_provider_blob::BlobDataProvider;
|
||
|
||
/// Compares text in one locale's order, with runs of digits by their value: `Layer 2` before
|
||
/// `Layer 10`.
|
||
#[derive(Debug)]
|
||
pub struct Collator(Option<icu_collator::Collator>);
|
||
|
||
impl Collator {
|
||
/// Sorting for `locale`, a language code such as `sv`, `pt-br` or `zh-hant` (any case, `-`
|
||
/// or `_`, glibc forms such as `sv_SE.UTF-8` and `sr_RS@latin`). A BCP 47 collation type
|
||
/// picks another order where there is one: `zh-hant-u-co-pinyin`, `zh-hans-u-co-stroke`.
|
||
pub fn new(locale: &str) -> Collator {
|
||
let mut prefs: CollatorPreferences = Locale::try_from_str(&bcp47(locale)).unwrap_or(Locale::UNKNOWN).into();
|
||
prefs.numeric_ordering = Some(CollationNumericOrdering::True);
|
||
let options = CollatorOptions::default();
|
||
let collator = Data::new().ok().and_then(|data| {
|
||
icu_collator::Collator::try_new_unstable(&data, prefs, options)
|
||
.or_else(|_| icu_collator::Collator::try_new_unstable(&data, CollatorPreferences::default(), options))
|
||
.ok()
|
||
});
|
||
Collator(collator)
|
||
}
|
||
|
||
/// `a` against `b`. Text that differs only in case or accents still has an order, and text
|
||
/// the collation counts as equal (`Layer 2` and `Layer 02`) is in code point order, so a sort
|
||
/// doesn't depend on the input order. Without collation data (never with the embedded data)
|
||
/// this is code point order.
|
||
pub fn compare(&self, a: &str, b: &str) -> Ordering {
|
||
match &self.0 {
|
||
Some(c) => c.as_borrowed().compare(a, b).then_with(|| a.cmp(b)),
|
||
None => a.cmp(b),
|
||
}
|
||
}
|
||
|
||
/// Sort `items` by the text `key` gives each one.
|
||
pub fn sort_by_key<T>(&self, items: &mut [T], key: impl Fn(&T) -> &str) {
|
||
items.sort_by(|a, b| self.compare(key(a), key(b)));
|
||
}
|
||
}
|
||
|
||
/// `locale` as a BCP 47 tag: `_` → `-`, a glibc codeset dropped and its script modifier made a
|
||
/// subtag (`sr_RS.UTF-8@latin` → `sr-latn-RS`).
|
||
fn bcp47(locale: &str) -> String {
|
||
let locale = locale.trim();
|
||
let mut parts: Vec<&str> = locale.split(['.', '@']).next().unwrap_or_default().split(['-', '_']).collect();
|
||
let script = match locale.split_once('@').map(|(_, m)| m.to_ascii_lowercase()).as_deref() {
|
||
Some("latin") => Some("latn"),
|
||
Some("cyrillic") => Some("cyrl"),
|
||
_ => None,
|
||
};
|
||
let has_script = parts.get(1).is_some_and(|p| p.len() == 4 && p.bytes().all(|b| b.is_ascii_alphabetic()));
|
||
if let Some(script) = script
|
||
&& !has_script
|
||
{
|
||
parts.insert(1.min(parts.len()), script);
|
||
}
|
||
parts.join("-")
|
||
}
|
||
|
||
/// The embedded collation data, with locale fallback, and the decomposition tables the collator
|
||
/// also needs taken from `icu_normalizer`'s compiled data rather than embedded a second time.
|
||
struct Data(LocaleFallbackProvider<BlobDataProvider>);
|
||
|
||
impl Data {
|
||
fn new() -> Result<Data, DataError> {
|
||
let blob = BlobDataProvider::try_new_from_static_blob(include_bytes!("../data/collation.postcard"))?;
|
||
Ok(Data(LocaleFallbackProvider::new(blob, LocaleFallbacker::new().static_to_owned())))
|
||
}
|
||
}
|
||
|
||
macro_rules! from_blob {
|
||
($($marker:ty),*) => {$(
|
||
impl DataProvider<$marker> for Data {
|
||
fn load(&self, req: DataRequest) -> Result<DataResponse<$marker>, DataError> {
|
||
self.0.as_deserializing().load(req)
|
||
}
|
||
}
|
||
)*};
|
||
}
|
||
|
||
from_blob!(
|
||
CollationRootV1,
|
||
CollationTailoringV1,
|
||
CollationDiacriticsV1,
|
||
CollationJamoV1,
|
||
CollationMetadataV1,
|
||
CollationReorderingV1,
|
||
CollationSpecialPrimariesV1
|
||
);
|
||
|
||
impl DataProvider<NormalizerNfdDataV1> for Data {
|
||
fn load(&self, req: DataRequest) -> Result<DataResponse<NormalizerNfdDataV1>, DataError> {
|
||
icu_normalizer::provider::Baked.load(req)
|
||
}
|
||
}
|
||
|
||
impl DataProvider<NormalizerNfdTablesV1> for Data {
|
||
fn load(&self, req: DataRequest) -> Result<DataResponse<NormalizerNfdTablesV1>, DataError> {
|
||
icu_normalizer::provider::Baked.load(req)
|
||
}
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use super::*;
|
||
|
||
fn sorted<'a>(locale: &str, items: &[&'a str]) -> Vec<&'a str> {
|
||
let mut v = items.to_vec();
|
||
Collator::new(locale).sort_by_key(&mut v, |s| s);
|
||
v
|
||
}
|
||
|
||
/// Each tailored locale puts a pair in another order than the root collation does, which a
|
||
/// locale whose data is missing (it falls back to root) wouldn't.
|
||
#[test]
|
||
fn every_shipped_locale_has_its_data() {
|
||
let root = Collator::new("und");
|
||
for (locale, first, second) in [
|
||
("ar", "ب", "a"),
|
||
("bg", "б", "a"),
|
||
("cs", "hrad", "chata"),
|
||
("da", "zebra", "år"),
|
||
("el", "β", "a"),
|
||
("es", "nube", "ñu"),
|
||
("et", "wc", "õun"),
|
||
("fa", "ب", "a"),
|
||
("fi", "zeta", "äiti"),
|
||
("hi", "क", "a"),
|
||
("hr", "cvijet", "čaj"),
|
||
("hu", "cukor", "csak"),
|
||
("ja", "亜", "一"),
|
||
("ka", "ა", "a"),
|
||
("kk", "б", "a"),
|
||
("ko", "가", "a"),
|
||
("lt", "yra", "jau"),
|
||
("pl", "az", "ąb"),
|
||
("ru", "б", "a"),
|
||
("sl", "cvet", "čaj"),
|
||
("sr", "б", "a"),
|
||
("sr-latn", "cvijet", "čaj"),
|
||
("sv", "zebra", "år"),
|
||
("te", "క", "a"),
|
||
("tr", "cz", "ça"),
|
||
("uk", "б", "a"),
|
||
("vi", "az", "ăn"),
|
||
("zh-hans", "阿", "中"),
|
||
("zh-hant", "人", "丈"),
|
||
] {
|
||
assert_eq!(Collator::new(locale).compare(first, second), Ordering::Less, "{locale}: {first} < {second}");
|
||
assert_eq!(root.compare(first, second), Ordering::Greater, "root: {second} < {first}");
|
||
}
|
||
// CLDR doesn't tailor these: they sort in the root order.
|
||
for locale in ["en", "de", "fr", "id", "it", "nl", "pt-br", "not a locale"] {
|
||
let collator = Collator::new(locale);
|
||
assert!(collator.0.is_some(), "{locale}");
|
||
assert_eq!(collator.compare("čaj", "cvijet"), Ordering::Less, "{locale}");
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn digits_sort_by_value() {
|
||
let layers = ["Layer 10", "Layer 2", "layer 1", "Layer 1", "Layer 2b", "Layer 02"];
|
||
assert_eq!(sorted("en", &layers), ["layer 1", "Layer 1", "Layer 02", "Layer 2", "Layer 2b", "Layer 10"]);
|
||
let reversed: Vec<&str> = layers.iter().rev().copied().collect();
|
||
assert_eq!(sorted("en", &reversed), sorted("en", &layers));
|
||
assert_eq!(sorted("de", &["Ebene ٣", "Ebene 10", "Ebene ２"]), ["Ebene ２", "Ebene ٣", "Ebene 10"]);
|
||
}
|
||
|
||
#[test]
|
||
fn locales_sort_their_own_letters() {
|
||
let words = ["öl", "zebra", "år", "äpple", "apa", "oxe"];
|
||
assert_eq!(sorted("sv", &words), ["apa", "oxe", "zebra", "år", "äpple", "öl"]);
|
||
assert_eq!(sorted("de", &words), ["apa", "äpple", "år", "öl", "oxe", "zebra"]);
|
||
assert_eq!(sorted("de", &["Muller", "Müller", "Mueller", "Mahler"]), ["Mahler", "Mueller", "Muller", "Müller"]);
|
||
assert_eq!(
|
||
sorted("tr", &["şeker", "sabun", "ılık", "iğne", "çay", "cam", "uzun", "üzüm"]),
|
||
["cam", "çay", "ılık", "iğne", "sabun", "şeker", "uzun", "üzüm"]
|
||
);
|
||
assert_eq!(sorted("en", &["ılık", "iğne", "cam", "çay"]), ["cam", "çay", "iğne", "ılık"]);
|
||
assert_eq!(sorted("es", &["ñu", "nube", "oso"]), ["nube", "ñu", "oso"]);
|
||
// Japanese: Latin, then kana in gojūon order (hiragana and katakana together), then kanji.
|
||
assert_eq!(sorted("ja", &["漢字", "カメラ", "あめ", "いぬ", "Zoom"]), ["Zoom", "あめ", "いぬ", "カメラ", "漢字"]);
|
||
}
|
||
|
||
#[test]
|
||
fn glibc_locale_names() {
|
||
let words = ["öl", "zebra", "år", "äpple", "apa", "oxe"];
|
||
assert_eq!(sorted("sv_SE.UTF-8", &words), sorted("sv", &words));
|
||
assert_ne!(sorted("sv_SE.UTF-8", &words), sorted("en", &words));
|
||
let cities = ["天津", "北京", "上海"];
|
||
assert_eq!(sorted("zh_TW.UTF-8", &cities), sorted("zh-hant", &cities));
|
||
assert_eq!(sorted("zh_CN.UTF-8", &cities), sorted("zh-hans", &cities));
|
||
// Serbian Cyrillic sorts Cyrillic first; Latin Serbian sorts Latin first, `lj` after `lu`.
|
||
let mixed = ["ljubav", "lula", "абв"];
|
||
assert_eq!(sorted("sr_RS.UTF-8", &mixed), ["абв", "ljubav", "lula"]);
|
||
assert_eq!(sorted("sr_RS.UTF-8@latin", &mixed), ["lula", "ljubav", "абв"]);
|
||
assert_eq!(bcp47("sr_RS.UTF-8@latin"), "sr-latn-RS");
|
||
assert_eq!(bcp47("sr_Latn_RS@latin"), "sr-Latn-RS");
|
||
assert_eq!(bcp47("de_DE@euro"), "de-DE");
|
||
}
|
||
|
||
#[test]
|
||
fn chinese_sorts_by_pinyin_or_stroke() {
|
||
// 北 běi (5 strokes), 上 shàng (3), 天 tiān (4).
|
||
let cities = ["天津", "北京", "上海"];
|
||
let pinyin = ["北京", "上海", "天津"];
|
||
let stroke = ["上海", "天津", "北京"];
|
||
assert_eq!(sorted("zh-hans", &cities), pinyin);
|
||
assert_eq!(sorted("zh-hant", &cities), stroke);
|
||
assert_eq!(sorted("zh-hans-u-co-stroke", &cities), stroke);
|
||
assert_eq!(sorted("zh-hant-u-co-pinyin", &cities), pinyin);
|
||
}
|
||
}
|