mirror of
https://github.com/storytold/craft-libs.git
synced 2026-10-31 07:39:23 +00:00
Merge #206: craft-i18n: search folding and locale sorting
# Conflicts: # crates/i18n/Cargo.toml # crates/i18n/src/lib.rs
This commit is contained in:
aiongg committed
commit
fa78cb52a5
10 files changed
+848
-3
No files matched your search
@@ -25,5 +25,5 @@ jobs:
|
||||
with:
|
||||
components: rustfmt, clippy
|
||||
- run: cargo fmt --check
|
||||
- run: cargo clippy -p craft-i18n --all-targets --locked -- -D warnings
|
||||
- run: cargo test -p craft-i18n --locked -- --nocapture
|
||||
- run: cargo clippy -p craft-i18n --all-targets --features collate --locked -- -D warnings
|
||||
- run: cargo test -p craft-i18n --features collate --locked -- --nocapture
|
||||
@@ -9,6 +9,7 @@ author, source and license. Library code and original assets are under the repos
|
||||
| `crates/cms/profiles/photocraft-coated-cmyk.icc` | PhotoCraft contributors; generated by `craft_cms::synth::coated_cmyk` | [PhotoCraft `crates/cms`](https://github.com/storytold/photocraft/tree/b3222ee8e7e6bcb4799c37f904c595f20f9baf5e/crates/cms) | CC0-1.0 |
|
||||
| `crates/cms/profiles/craft-coated-fogra39.icc` | craft-libs contributors; generated by `crates/cms/tests/fogra39.rs` from the FOGRA39 data below | this repository | CC0-1.0 |
|
||||
| `crates/cms/tests/data/FOGRA39.txt` (unmodified) | Fogra | [Fogra characterisation data](https://fogra.org/en/downloads/work-tools/characterisation-data), `FOGRA39_to_FOGRA60_v2.zip` | Fogra terms: free use and redistribution, unmodified, with Fogra identified as the source; "FOGRA39" identifies the reference data only and implies no certification, approval or endorsement by Fogra (full text in [NOTICE](NOTICE)) |
|
||||
| `crates/i18n/data/collation.postcard` | Unicode, Inc. (ICU 78.1 collation data, CLDR 48.2.1); generated by `crates/i18n/data/generate-collation.sh` | [ICU](https://github.com/unicode-org/icu/releases/tag/release-78.1rc) | Unicode-3.0 |
|
||||
|
||||
Original assets are covered by the repository licence.
|
||||
|
||||
|
||||
Generated
+133
@@ -624,6 +624,15 @@ dependencies = [
|
||||
"x509-cert",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cobs"
|
||||
version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0fa961b519f0b462e3a3b4a34b64d119eeaca1d59af726fe450bbba07a9fc0a1"
|
||||
dependencies = [
|
||||
"thiserror 2.0.21",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "codespan-reporting"
|
||||
version = "0.13.1"
|
||||
@@ -790,6 +799,14 @@ version = "0.1.0"
|
||||
name = "craft-i18n"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"icu_collator",
|
||||
"icu_locale_core",
|
||||
"icu_locale_fallback",
|
||||
"icu_normalizer",
|
||||
"icu_properties",
|
||||
"icu_provider",
|
||||
"icu_provider_adapters",
|
||||
"icu_provider_blob",
|
||||
"intl_pluralrules",
|
||||
"log",
|
||||
"objc2-foundation 0.3.2",
|
||||
@@ -1254,6 +1271,18 @@ dependencies = [
|
||||
"bytemuck",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "embedded-io"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced"
|
||||
|
||||
[[package]]
|
||||
name = "embedded-io"
|
||||
version = "0.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d"
|
||||
|
||||
[[package]]
|
||||
name = "encoding_rs"
|
||||
version = "0.8.42"
|
||||
@@ -2066,6 +2095,24 @@ dependencies = [
|
||||
"cc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "icu_collator"
|
||||
version = "2.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "08984ed58ac439ebf3e13d2cf26b0c46a60afcd21721c1d14087c0b240344dda"
|
||||
dependencies = [
|
||||
"icu_collections",
|
||||
"icu_locale_core",
|
||||
"icu_normalizer",
|
||||
"icu_properties",
|
||||
"icu_provider",
|
||||
"serde",
|
||||
"smallvec",
|
||||
"utf16_iter",
|
||||
"utf8_iter",
|
||||
"zerovec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "icu_collections"
|
||||
version = "2.3.0"
|
||||
@@ -2074,6 +2121,7 @@ checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513"
|
||||
dependencies = [
|
||||
"displaydoc",
|
||||
"potential_utf",
|
||||
"serde",
|
||||
"utf8_iter",
|
||||
"yoke",
|
||||
"zerofrom",
|
||||
@@ -2088,11 +2136,32 @@ checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb"
|
||||
dependencies = [
|
||||
"displaydoc",
|
||||
"litemap",
|
||||
"serde",
|
||||
"tinystr",
|
||||
"writeable",
|
||||
"zerovec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "icu_locale_fallback"
|
||||
version = "2.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "251af8e57c9400e3eb58242fe5b8b1152b2a64fdf4cf632f923c38ccee6f2fa9"
|
||||
dependencies = [
|
||||
"icu_locale_core",
|
||||
"icu_locale_fallback_data",
|
||||
"icu_provider",
|
||||
"potential_utf",
|
||||
"tinystr",
|
||||
"zerovec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "icu_locale_fallback_data"
|
||||
version = "2.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "decf2a22ec8fa68f1a0c1129a3f8583f8f8bc24e8b9ccbe98ead99f62a4dc3a8"
|
||||
|
||||
[[package]]
|
||||
name = "icu_normalizer"
|
||||
version = "2.3.0"
|
||||
@@ -2103,7 +2172,11 @@ dependencies = [
|
||||
"icu_normalizer_data",
|
||||
"icu_properties",
|
||||
"icu_provider",
|
||||
"serde",
|
||||
"smallvec",
|
||||
"utf16_iter",
|
||||
"utf8_iter",
|
||||
"write16",
|
||||
"zerovec",
|
||||
]
|
||||
|
||||
@@ -2124,6 +2197,7 @@ dependencies = [
|
||||
"icu_locale_core",
|
||||
"icu_properties_data",
|
||||
"icu_provider",
|
||||
"serde",
|
||||
"zerotrie",
|
||||
"zerovec",
|
||||
]
|
||||
@@ -2142,6 +2216,9 @@ checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73"
|
||||
dependencies = [
|
||||
"displaydoc",
|
||||
"icu_locale_core",
|
||||
"postcard",
|
||||
"serde",
|
||||
"stable_deref_trait",
|
||||
"writeable",
|
||||
"yoke",
|
||||
"zerofrom",
|
||||
@@ -2149,6 +2226,30 @@ dependencies = [
|
||||
"zerovec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "icu_provider_adapters"
|
||||
version = "2.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "df887bc4f55dbc773c7c45a0c7dfa75be886f7b7812830a71ab7271cc1801820"
|
||||
dependencies = [
|
||||
"icu_locale_fallback",
|
||||
"icu_provider",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "icu_provider_blob"
|
||||
version = "2.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5916b44fac319d3f58e4d8d805a627df19e27937b582cd99fcde3bb6b7ce1b0e"
|
||||
dependencies = [
|
||||
"icu_provider",
|
||||
"postcard",
|
||||
"serde",
|
||||
"writeable",
|
||||
"zerotrie",
|
||||
"zerovec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "idna"
|
||||
version = "1.1.0"
|
||||
@@ -2469,6 +2570,9 @@ name = "litemap"
|
||||
version = "0.8.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae"
|
||||
dependencies = [
|
||||
"serde_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "litrs"
|
||||
@@ -3336,12 +3440,26 @@ dependencies = [
|
||||
"portable-atomic",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "postcard"
|
||||
version = "1.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6764c3b5dd454e283a30e6dfe78e9b31096d9e32036b5d1eaac7a6119ccb9a24"
|
||||
dependencies = [
|
||||
"cobs",
|
||||
"embedded-io 0.4.0",
|
||||
"embedded-io 0.6.1",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "potential_utf"
|
||||
version = "0.1.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661"
|
||||
dependencies = [
|
||||
"serde_core",
|
||||
"writeable",
|
||||
"zerovec",
|
||||
]
|
||||
|
||||
@@ -5131,6 +5249,12 @@ dependencies = [
|
||||
"xmlwriter",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "utf16_iter"
|
||||
version = "1.0.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246"
|
||||
|
||||
[[package]]
|
||||
name = "utf8_iter"
|
||||
version = "1.0.4"
|
||||
@@ -6081,6 +6205,12 @@ dependencies = [
|
||||
"read-fonts 0.39.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "write16"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936"
|
||||
|
||||
[[package]]
|
||||
name = "writeable"
|
||||
version = "0.6.4"
|
||||
@@ -6349,8 +6479,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f"
|
||||
dependencies = [
|
||||
"displaydoc",
|
||||
"litemap",
|
||||
"serde_core",
|
||||
"yoke",
|
||||
"zerofrom",
|
||||
"zerovec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
+21
-1
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "craft-i18n"
|
||||
description = "Translations for the Craft apps: the embedded craft-i18n catalogs, lookup with fallback, plurals, language names and the system locale."
|
||||
description = "Translations for the Craft apps: the embedded craft-i18n catalogs, lookup with fallback, plurals, language names, the system locale, search folding and locale sorting."
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
@@ -12,6 +12,15 @@ intl_pluralrules = "=7.0.2"
|
||||
unic-langid = "0.9"
|
||||
log = "0.4"
|
||||
sys-locale = "0.3"
|
||||
icu_normalizer = "~2.3"
|
||||
icu_properties = "~2.3"
|
||||
# Generated data fits one ICU4X minor version; data/generate-collation.sh names its datagen.
|
||||
icu_collator = { version = "~2.3.1", default-features = false, features = ["serde"], optional = true }
|
||||
icu_provider = { version = "~2.3", features = ["serde"], optional = true }
|
||||
icu_provider_blob = { version = "~2.3", features = ["alloc"], optional = true }
|
||||
icu_provider_adapters = { version = "~2.3", optional = true }
|
||||
icu_locale_core = { version = "2.3", optional = true }
|
||||
icu_locale_fallback = { version = "~2.3", features = ["compiled_data"], optional = true }
|
||||
unicode-bidi = "0.3"
|
||||
|
||||
[target.'cfg(target_arch = "wasm32")'.dependencies]
|
||||
@@ -25,5 +34,16 @@ windows-registry = "0.6"
|
||||
[target.'cfg(target_vendor = "apple")'.dependencies]
|
||||
objc2-foundation = { version = "0.3.2", default-features = false, features = ["std", "NSDateFormatter", "NSFormatter", "NSLocale", "NSString"] }
|
||||
|
||||
[features]
|
||||
# Locale sorting (`collate`), with ICU4X collation data for the suite's locales embedded.
|
||||
collate = [
|
||||
"dep:icu_collator",
|
||||
"dep:icu_locale_core",
|
||||
"dep:icu_locale_fallback",
|
||||
"dep:icu_provider",
|
||||
"dep:icu_provider_adapters",
|
||||
"dep:icu_provider_blob",
|
||||
]
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
@@ -110,3 +110,15 @@ The app keeps its `tl!` macro, its language setting (`"auto"` → `I18N.system_l
|
||||
included, comes back as `⟦…⟧`, line by line, so text drawn without the marks never went through
|
||||
translation. `looked_up()` lists the source texts looked up so far, for an audit against a
|
||||
complete catalog.
|
||||
- **Search** (`Folder`): `Folder::new(locale).fold(text)` gives a key for case- and
|
||||
accent-insensitive matching (`keep_accents()` makes accents count) and a map back to the original
|
||||
text, so `Folded::find` returns the byte range to highlight. Compatibility forms fold to their
|
||||
plain letters (`fi`, full-width `Ａ`), `ß` to `ss`, `ς` to `σ`, digits of every script to ASCII
|
||||
(`Layer ١٢` finds `layer 12`); ignoring accents, Persian and Arabic letter forms match (`ک` and
|
||||
`ك`, `ی` and `ي`, `ة` and `ه`, `أ` and `ا`); invisible format characters (soft hyphens, joiners,
|
||||
bidi isolates) are dropped. Turkish and Azerbaijani fold `I` to `ı` when accents count; ignoring
|
||||
accents, `I`, `i`, `İ` and `ı` all match.
|
||||
- **Sorting** (`Collator`, feature `collate`): ICU4X collation for the locales craft-i18n ships,
|
||||
Persian and Hindi, with digits compared by value (`Layer 2` before `Layer 10`) and `-u-co-pinyin`
|
||||
/ `-u-co-stroke` for Chinese. The data (754 KB) is regenerated by `data/generate-collation.sh`;
|
||||
the feature adds about 1 MB to a release binary or wasm module (320 KB gzipped).
|
||||
Binary file not shown.
Executable
+15
@@ -0,0 +1,15 @@
|
||||
#!/bin/sh
|
||||
# Regenerates the collation data `collate` embeds, a postcard blob for the locales craft-i18n
|
||||
# ships (its `locales/` directory) plus English, Persian and Hindi: each locale's default order,
|
||||
# and Chinese pinyin and stroke orders.
|
||||
# Needs icu4x-datagen at the version of the icu_collator dependency:
|
||||
# cargo install icu4x-datagen --version 2.3.0 --locked
|
||||
set -eu
|
||||
cd "$(dirname "$0")"
|
||||
icu4x-datagen --format blob --overwrite --out collation.postcard --deduplication maximal \
|
||||
--cldr-tag 48.2.1 --icuexport-tag release-78.1rc \
|
||||
--attribute-filter 'collator=/^(stroke|pinyin)?$/' \
|
||||
--markers CollationRootV1 CollationTailoringV1 CollationDiacriticsV1 CollationJamoV1 \
|
||||
CollationMetadataV1 CollationReorderingV1 CollationSpecialPrimariesV1 \
|
||||
--locales en ar bg cs da de el es et fa fi fr hi hr hu id it ja ka kk ko lt nl pl pt-BR ru sl \
|
||||
sr sr-Latn sv te tr uk vi zh-Hans zh-Hant
|
||||
@@ -0,0 +1,234 @@
|
||||
//! Locale sorting for lists people see: fonts, layers, presets, files.
|
||||
//!
|
||||
//! ICU4X collation with data embedded for the locales craft-i18n ships (regenerated by
|
||||
//! `data/generate-collation.sh`). A locale without data of its own sorts in the Unicode root
|
||||
//! order, which suits most languages that have no tailoring.
|
||||
|
||||
use std::cmp::Ordering;
|
||||
|
||||
use icu_collator::CollatorPreferences;
|
||||
use icu_collator::options::CollatorOptions;
|
||||
use icu_collator::preferences::CollationNumericOrdering;
|
||||
use icu_collator::provider::{
|
||||
CollationDiacriticsV1, CollationJamoV1, CollationMetadataV1, CollationReorderingV1, CollationRootV1, CollationSpecialPrimariesV1,
|
||||
CollationTailoringV1,
|
||||
};
|
||||
use icu_locale_core::Locale;
|
||||
use icu_locale_fallback::LocaleFallbacker;
|
||||
use icu_normalizer::provider::{NormalizerNfdDataV1, NormalizerNfdTablesV1};
|
||||
use icu_provider::buf::AsDeserializingBufferProvider;
|
||||
use icu_provider::prelude::*;
|
||||
use icu_provider_adapters::fallback::LocaleFallbackProvider;
|
||||
use icu_provider_blob::BlobDataProvider;
|
||||
|
||||
/// Compares text in one locale's order, with runs of digits by their value: `Layer 2` before
|
||||
/// `Layer 10`.
|
||||
#[derive(Debug)]
|
||||
pub struct Collator(Option<icu_collator::Collator>);
|
||||
|
||||
impl Collator {
|
||||
/// Sorting for `locale`, a language code such as `sv`, `pt-br` or `zh-hant` (any case, `-`
|
||||
/// or `_`, glibc forms such as `sv_SE.UTF-8` and `sr_RS@latin`). A BCP 47 collation type
|
||||
/// picks another order where there is one: `zh-hant-u-co-pinyin`, `zh-hans-u-co-stroke`.
|
||||
pub fn new(locale: &str) -> Collator {
|
||||
let mut prefs: CollatorPreferences = Locale::try_from_str(&bcp47(locale)).unwrap_or(Locale::UNKNOWN).into();
|
||||
prefs.numeric_ordering = Some(CollationNumericOrdering::True);
|
||||
let options = CollatorOptions::default();
|
||||
let collator = Data::new().ok().and_then(|data| {
|
||||
icu_collator::Collator::try_new_unstable(&data, prefs, options)
|
||||
.or_else(|_| icu_collator::Collator::try_new_unstable(&data, CollatorPreferences::default(), options))
|
||||
.ok()
|
||||
});
|
||||
Collator(collator)
|
||||
}
|
||||
|
||||
/// `a` against `b`. Text that differs only in case or accents still has an order, and text
|
||||
/// the collation counts as equal (`Layer 2` and `Layer 02`) is in code point order, so a sort
|
||||
/// doesn't depend on the input order. Without collation data (never with the embedded data)
|
||||
/// this is code point order.
|
||||
pub fn compare(&self, a: &str, b: &str) -> Ordering {
|
||||
match &self.0 {
|
||||
Some(c) => c.as_borrowed().compare(a, b).then_with(|| a.cmp(b)),
|
||||
None => a.cmp(b),
|
||||
}
|
||||
}
|
||||
|
||||
/// Sort `items` by the text `key` gives each one.
|
||||
pub fn sort_by_key<T>(&self, items: &mut [T], key: impl Fn(&T) -> &str) {
|
||||
items.sort_by(|a, b| self.compare(key(a), key(b)));
|
||||
}
|
||||
}
|
||||
|
||||
/// `locale` as a BCP 47 tag: `_` → `-`, a glibc codeset dropped and its script modifier made a
|
||||
/// subtag (`sr_RS.UTF-8@latin` → `sr-latn-RS`).
|
||||
fn bcp47(locale: &str) -> String {
|
||||
let locale = locale.trim();
|
||||
let mut parts: Vec<&str> = locale.split(['.', '@']).next().unwrap_or_default().split(['-', '_']).collect();
|
||||
let script = match locale.split_once('@').map(|(_, m)| m.to_ascii_lowercase()).as_deref() {
|
||||
Some("latin") => Some("latn"),
|
||||
Some("cyrillic") => Some("cyrl"),
|
||||
_ => None,
|
||||
};
|
||||
let has_script = parts.get(1).is_some_and(|p| p.len() == 4 && p.bytes().all(|b| b.is_ascii_alphabetic()));
|
||||
if let Some(script) = script
|
||||
&& !has_script
|
||||
{
|
||||
parts.insert(1.min(parts.len()), script);
|
||||
}
|
||||
parts.join("-")
|
||||
}
|
||||
|
||||
/// The embedded collation data, with locale fallback, and the decomposition tables the collator
|
||||
/// also needs taken from `icu_normalizer`'s compiled data rather than embedded a second time.
|
||||
struct Data(LocaleFallbackProvider<BlobDataProvider>);
|
||||
|
||||
impl Data {
|
||||
fn new() -> Result<Data, DataError> {
|
||||
let blob = BlobDataProvider::try_new_from_static_blob(include_bytes!("../data/collation.postcard"))?;
|
||||
Ok(Data(LocaleFallbackProvider::new(blob, LocaleFallbacker::new().static_to_owned())))
|
||||
}
|
||||
}
|
||||
|
||||
macro_rules! from_blob {
|
||||
($($marker:ty),*) => {$(
|
||||
impl DataProvider<$marker> for Data {
|
||||
fn load(&self, req: DataRequest) -> Result<DataResponse<$marker>, DataError> {
|
||||
self.0.as_deserializing().load(req)
|
||||
}
|
||||
}
|
||||
)*};
|
||||
}
|
||||
|
||||
from_blob!(
|
||||
CollationRootV1,
|
||||
CollationTailoringV1,
|
||||
CollationDiacriticsV1,
|
||||
CollationJamoV1,
|
||||
CollationMetadataV1,
|
||||
CollationReorderingV1,
|
||||
CollationSpecialPrimariesV1
|
||||
);
|
||||
|
||||
impl DataProvider<NormalizerNfdDataV1> for Data {
|
||||
fn load(&self, req: DataRequest) -> Result<DataResponse<NormalizerNfdDataV1>, DataError> {
|
||||
icu_normalizer::provider::Baked.load(req)
|
||||
}
|
||||
}
|
||||
|
||||
impl DataProvider<NormalizerNfdTablesV1> for Data {
|
||||
fn load(&self, req: DataRequest) -> Result<DataResponse<NormalizerNfdTablesV1>, DataError> {
|
||||
icu_normalizer::provider::Baked.load(req)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn sorted<'a>(locale: &str, items: &[&'a str]) -> Vec<&'a str> {
|
||||
let mut v = items.to_vec();
|
||||
Collator::new(locale).sort_by_key(&mut v, |s| s);
|
||||
v
|
||||
}
|
||||
|
||||
/// Each tailored locale puts a pair in another order than the root collation does, which a
|
||||
/// locale whose data is missing (it falls back to root) wouldn't.
|
||||
#[test]
|
||||
fn every_shipped_locale_has_its_data() {
|
||||
let root = Collator::new("und");
|
||||
for (locale, first, second) in [
|
||||
("ar", "ب", "a"),
|
||||
("bg", "б", "a"),
|
||||
("cs", "hrad", "chata"),
|
||||
("da", "zebra", "år"),
|
||||
("el", "β", "a"),
|
||||
("es", "nube", "ñu"),
|
||||
("et", "wc", "õun"),
|
||||
("fa", "ب", "a"),
|
||||
("fi", "zeta", "äiti"),
|
||||
("hi", "क", "a"),
|
||||
("hr", "cvijet", "čaj"),
|
||||
("hu", "cukor", "csak"),
|
||||
("ja", "亜", "一"),
|
||||
("ka", "ა", "a"),
|
||||
("kk", "б", "a"),
|
||||
("ko", "가", "a"),
|
||||
("lt", "yra", "jau"),
|
||||
("pl", "az", "ąb"),
|
||||
("ru", "б", "a"),
|
||||
("sl", "cvet", "čaj"),
|
||||
("sr", "б", "a"),
|
||||
("sr-latn", "cvijet", "čaj"),
|
||||
("sv", "zebra", "år"),
|
||||
("te", "క", "a"),
|
||||
("tr", "cz", "ça"),
|
||||
("uk", "б", "a"),
|
||||
("vi", "az", "ăn"),
|
||||
("zh-hans", "阿", "中"),
|
||||
("zh-hant", "人", "丈"),
|
||||
] {
|
||||
assert_eq!(Collator::new(locale).compare(first, second), Ordering::Less, "{locale}: {first} < {second}");
|
||||
assert_eq!(root.compare(first, second), Ordering::Greater, "root: {second} < {first}");
|
||||
}
|
||||
// CLDR doesn't tailor these: they sort in the root order.
|
||||
for locale in ["en", "de", "fr", "id", "it", "nl", "pt-br", "not a locale"] {
|
||||
let collator = Collator::new(locale);
|
||||
assert!(collator.0.is_some(), "{locale}");
|
||||
assert_eq!(collator.compare("čaj", "cvijet"), Ordering::Less, "{locale}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn digits_sort_by_value() {
|
||||
let layers = ["Layer 10", "Layer 2", "layer 1", "Layer 1", "Layer 2b", "Layer 02"];
|
||||
assert_eq!(sorted("en", &layers), ["layer 1", "Layer 1", "Layer 02", "Layer 2", "Layer 2b", "Layer 10"]);
|
||||
let reversed: Vec<&str> = layers.iter().rev().copied().collect();
|
||||
assert_eq!(sorted("en", &reversed), sorted("en", &layers));
|
||||
assert_eq!(sorted("de", &["Ebene ٣", "Ebene 10", "Ebene ２"]), ["Ebene ２", "Ebene ٣", "Ebene 10"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn locales_sort_their_own_letters() {
|
||||
let words = ["öl", "zebra", "år", "äpple", "apa", "oxe"];
|
||||
assert_eq!(sorted("sv", &words), ["apa", "oxe", "zebra", "år", "äpple", "öl"]);
|
||||
assert_eq!(sorted("de", &words), ["apa", "äpple", "år", "öl", "oxe", "zebra"]);
|
||||
assert_eq!(sorted("de", &["Muller", "Müller", "Mueller", "Mahler"]), ["Mahler", "Mueller", "Muller", "Müller"]);
|
||||
assert_eq!(
|
||||
sorted("tr", &["şeker", "sabun", "ılık", "iğne", "çay", "cam", "uzun", "üzüm"]),
|
||||
["cam", "çay", "ılık", "iğne", "sabun", "şeker", "uzun", "üzüm"]
|
||||
);
|
||||
assert_eq!(sorted("en", &["ılık", "iğne", "cam", "çay"]), ["cam", "çay", "iğne", "ılık"]);
|
||||
assert_eq!(sorted("es", &["ñu", "nube", "oso"]), ["nube", "ñu", "oso"]);
|
||||
// Japanese: Latin, then kana in gojūon order (hiragana and katakana together), then kanji.
|
||||
assert_eq!(sorted("ja", &["漢字", "カメラ", "あめ", "いぬ", "Zoom"]), ["Zoom", "あめ", "いぬ", "カメラ", "漢字"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn glibc_locale_names() {
|
||||
let words = ["öl", "zebra", "år", "äpple", "apa", "oxe"];
|
||||
assert_eq!(sorted("sv_SE.UTF-8", &words), sorted("sv", &words));
|
||||
assert_ne!(sorted("sv_SE.UTF-8", &words), sorted("en", &words));
|
||||
let cities = ["天津", "北京", "上海"];
|
||||
assert_eq!(sorted("zh_TW.UTF-8", &cities), sorted("zh-hant", &cities));
|
||||
assert_eq!(sorted("zh_CN.UTF-8", &cities), sorted("zh-hans", &cities));
|
||||
// Serbian Cyrillic sorts Cyrillic first; Latin Serbian sorts Latin first, `lj` after `lu`.
|
||||
let mixed = ["ljubav", "lula", "абв"];
|
||||
assert_eq!(sorted("sr_RS.UTF-8", &mixed), ["абв", "ljubav", "lula"]);
|
||||
assert_eq!(sorted("sr_RS.UTF-8@latin", &mixed), ["lula", "ljubav", "абв"]);
|
||||
assert_eq!(bcp47("sr_RS.UTF-8@latin"), "sr-latn-RS");
|
||||
assert_eq!(bcp47("sr_Latn_RS@latin"), "sr-Latn-RS");
|
||||
assert_eq!(bcp47("de_DE@euro"), "de-DE");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chinese_sorts_by_pinyin_or_stroke() {
|
||||
// 北 běi (5 strokes), 上 shàng (3), 天 tiān (4).
|
||||
let cities = ["天津", "北京", "上海"];
|
||||
let pinyin = ["北京", "上海", "天津"];
|
||||
let stroke = ["上海", "天津", "北京"];
|
||||
assert_eq!(sorted("zh-hans", &cities), pinyin);
|
||||
assert_eq!(sorted("zh-hant", &cities), stroke);
|
||||
assert_eq!(sorted("zh-hans-u-co-stroke", &cities), stroke);
|
||||
assert_eq!(sorted("zh-hant-u-co-pinyin", &cities), pinyin);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,424 @@
|
||||
//! Case- and accent-insensitive matching for search fields, with a map back to the original text
|
||||
//! for highlighting.
|
||||
//!
|
||||
//! [`Folder::fold`] turns text into a search key: compatibility forms decomposed (`fi` → `fi`,
|
||||
//! full-width `Ａ` → `a`, `㎏` → `kg`), lower case, `ß` → `ss`, final `ς` → `σ`, and with accents
|
||||
//! ignored (the default) combining accents, Hebrew points and Arabic vowel marks dropped and the
|
||||
//! few letters with built-in strokes mapped to their base (`ø`, `ł`, `đ`, `ı`, `æ` → `ae`), and
|
||||
//! Persian letter forms mapped to the Arabic ones (`ک` → `ك`, `ی` → `ي`, `ة` → `ه`). Decimal
|
||||
//! digits of every script fold to ASCII (`١٢` → `12`). Invisible format characters (soft
|
||||
//! hyphens, zero-width joiners, bidi isolates) are dropped too.
|
||||
//!
|
||||
//! Dotted and dotless i follow the locale when accents count. In Turkish and Azerbaijani, `I`
|
||||
//! folds to `ı` and `İ` to `i`; elsewhere both fold to `i` and `ı` stays a letter of its own.
|
||||
//! Ignoring accents, all four fold to `i`, so `istanbul` finds `İSTANBUL` and `irmak` finds
|
||||
//! `IRMAK` and `ırmak`. Plain `str::to_lowercase` follows no locale and turns `İ` into `i` plus a
|
||||
//! combining dot, which nothing typed would match.
|
||||
//!
|
||||
//! The Turkish i handling and sigma folding come from PdfCraft's `fold_case` (snipeTR,
|
||||
//! nevicia2010-hub).
|
||||
|
||||
use std::ops::Range;
|
||||
|
||||
use icu_normalizer::DecomposingNormalizerBorrowed;
|
||||
use icu_properties::props::{GraphemeExtend, HangulSyllableType};
|
||||
use icu_properties::{CodePointMapData, CodePointSetData};
|
||||
|
||||
/// Folds text for one locale. Cheap to build and copy.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct Folder {
|
||||
turkic: bool,
|
||||
accents: bool,
|
||||
}
|
||||
|
||||
impl Folder {
|
||||
/// Case- and accent-insensitive folding for `locale`, a language code such as `tr`, `pt-br`
|
||||
/// or `zh-hant` (any case, `-` or `_`).
|
||||
pub fn new(locale: &str) -> Folder {
|
||||
let language = locale.split(['-', '_', '.', '@']).next().unwrap_or_default();
|
||||
Folder { turkic: language.eq_ignore_ascii_case("tr") || language.eq_ignore_ascii_case("az"), accents: false }
|
||||
}
|
||||
|
||||
/// Accents count: `é` no longer matches `e`, and a match can't end on a letter whose accent it
|
||||
/// leaves out.
|
||||
pub fn keep_accents(self) -> Folder {
|
||||
Folder { accents: true, ..self }
|
||||
}
|
||||
|
||||
/// `s` folded, with the map back to `s`.
|
||||
pub fn fold(&self, s: &str) -> Folded {
|
||||
let nfkd = DecomposingNormalizerBorrowed::new_nfkd();
|
||||
let extend = CodePointSetData::new::<GraphemeExtend>();
|
||||
let hangul = CodePointMapData::<HangulSyllableType>::new();
|
||||
let mut text = String::with_capacity(s.len());
|
||||
let mut starts = Vec::new();
|
||||
// A cluster is a character and the combining marks after it, folded and mapped as one;
|
||||
// conjoining jamo that spell one syllable are one cluster too.
|
||||
let mut cluster_start = 0;
|
||||
let mut chars = s.char_indices();
|
||||
let mut prev = chars.next().map(|(_, c)| hangul.get(c));
|
||||
for (i, c) in chars {
|
||||
let kind = hangul.get(c);
|
||||
if !extend.contains(c) && !prev.is_some_and(|p| continues_syllable(p, kind)) {
|
||||
starts.push((text.len(), cluster_start));
|
||||
self.fold_cluster(&nfkd.normalize(&s[cluster_start..i]), &mut text);
|
||||
cluster_start = i;
|
||||
}
|
||||
prev = Some(kind);
|
||||
}
|
||||
if !s.is_empty() {
|
||||
starts.push((text.len(), cluster_start));
|
||||
self.fold_cluster(&nfkd.normalize(&s[cluster_start..]), &mut text);
|
||||
}
|
||||
starts.push((text.len(), s.len()));
|
||||
Folded { text, starts }
|
||||
}
|
||||
|
||||
/// `s` folded, without the map: a key to compare or store.
|
||||
pub fn key(&self, s: &str) -> String {
|
||||
self.fold(s).text
|
||||
}
|
||||
|
||||
/// Whether `haystack` contains `needle`, both folded.
|
||||
pub fn contains(&self, haystack: &str, needle: &str) -> bool {
|
||||
self.fold(haystack).find(&self.fold(needle)).is_some()
|
||||
}
|
||||
|
||||
/// One cluster, already in NFKD.
|
||||
fn fold_cluster(&self, cluster: &str, out: &mut String) {
|
||||
let mut chars = cluster.chars().peekable();
|
||||
while let Some(c) = chars.next() {
|
||||
match c {
|
||||
// `İ` decomposes to `I` and a combining dot above: the dot of a lower-case `i`.
|
||||
'I' if chars.peek() == Some(&'\u{307}') => {
|
||||
chars.next();
|
||||
out.push('i');
|
||||
}
|
||||
'I' if self.turkic && self.accents => out.push('ı'),
|
||||
// Persian heh with hamza above (ۀ) decomposes to ae (ە) and hamza.
|
||||
'\u{6D5}' if !self.accents && chars.peek() == Some(&'\u{654}') => {
|
||||
chars.next();
|
||||
out.push('ه');
|
||||
}
|
||||
'ß' | 'ẞ' => out.push_str("ss"),
|
||||
c if let Some(d) = ascii_digit(c) => out.push(d),
|
||||
'Σ' | 'σ' | 'ς' => out.push('σ'),
|
||||
c if ignorable(c) || (!self.accents && accent(c)) => {}
|
||||
c => {
|
||||
for l in c.to_lowercase() {
|
||||
match base_letter(l) {
|
||||
Some(base) if !self.accents => out.push_str(base),
|
||||
_ => out.push(l),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The base of a lower-case letter whose diacritic is built in rather than combining, and the
|
||||
/// Arabic form of a letter that Persian and Urdu write differently: keheh (ک) is kaf (ك), Farsi
|
||||
/// yeh (ی) and alef maksura (ى) are yeh (ي), teh marbuta (ة) is heh (ه), alef wasla (ٱ) is alef.
|
||||
fn base_letter(c: char) -> Option<&'static str> {
|
||||
Some(match c {
|
||||
'ک' => "ك",
|
||||
'ی' | 'ى' => "ي",
|
||||
'ة' => "ه",
|
||||
'ٱ' => "ا",
|
||||
'ı' => "i",
|
||||
'ø' => "o",
|
||||
'ł' => "l",
|
||||
'đ' => "d",
|
||||
'ħ' => "h",
|
||||
'ŧ' => "t",
|
||||
'æ' => "ae",
|
||||
'œ' => "oe",
|
||||
_ => return None,
|
||||
})
|
||||
}
|
||||
|
||||
/// Invisible characters that only steer layout: soft hyphen, Arabic letter mark, the kashida
|
||||
/// that stretches Arabic words, Mongolian variation selectors, zero-width space and joiners, direction marks, embeddings and isolates,
|
||||
/// word joiner, variation selectors, byte order mark and tags.
|
||||
fn ignorable(c: char) -> bool {
|
||||
matches!(c,
|
||||
'\u{AD}' | '\u{61C}' | '\u{640}' | '\u{180B}'..='\u{180F}' | '\u{200B}'..='\u{200F}' | '\u{202A}'..='\u{202E}'
|
||||
| '\u{2060}'..='\u{206F}' | '\u{FE00}'..='\u{FE0F}' | '\u{FEFF}' | '\u{E0000}'..='\u{E0FFF}')
|
||||
}
|
||||
|
||||
/// Marks that searching without accents ignores: Latin, Greek and Cyrillic combining diacritics,
|
||||
/// Hebrew points and cantillation, Arabic vowel marks.
|
||||
/// Kana voicing marks, Indic and Thai vowel signs are letters' parts, not accents, and stay.
|
||||
fn accent(c: char) -> bool {
|
||||
matches!(c,
|
||||
'\u{300}'..='\u{36F}' | '\u{1AB0}'..='\u{1AFF}' | '\u{1DC0}'..='\u{1DFF}' | '\u{20D0}'..='\u{20FF}'
|
||||
| '\u{FE20}'..='\u{FE2F}'
|
||||
| '\u{591}'..='\u{5BD}' | '\u{5BF}' | '\u{5C1}' | '\u{5C2}' | '\u{5C4}' | '\u{5C5}' | '\u{5C7}'
|
||||
| '\u{610}'..='\u{61A}' | '\u{64B}'..='\u{65F}' | '\u{670}' | '\u{6D6}'..='\u{6DC}'
|
||||
| '\u{6DF}'..='\u{6E4}' | '\u{6E7}' | '\u{6E8}' | '\u{6EA}'..='\u{6ED}')
|
||||
}
|
||||
|
||||
/// The first code point of each run of ten Unicode decimal digits (general category Nd), Unicode
|
||||
/// 16.0.
|
||||
const ZEROS: [u32; 76] = [
|
||||
0x30, 0x660, 0x6F0, 0x7C0, 0x966, 0x9E6, 0xA66, 0xAE6, 0xB66, 0xBE6, 0xC66, 0xCE6, 0xD66, 0xDE6, 0xE50, 0xED0, 0xF20, 0x1040, 0x1090, 0x17E0,
|
||||
0x1810, 0x1946, 0x19D0, 0x1A80, 0x1A90, 0x1B50, 0x1BB0, 0x1C40, 0x1C50, 0xA620, 0xA8D0, 0xA900, 0xA9D0, 0xA9F0, 0xAA50, 0xABF0, 0xFF10, 0x104A0,
|
||||
0x10D30, 0x10D40, 0x11066, 0x110F0, 0x11136, 0x111D0, 0x112F0, 0x11450, 0x114D0, 0x11650, 0x116C0, 0x116D0, 0x116DA, 0x11730, 0x118E0, 0x11950,
|
||||
0x11BF0, 0x11C50, 0x11D50, 0x11DA0, 0x11F50, 0x16130, 0x16A60, 0x16AC0, 0x16B50, 0x16D70, 0x1CCF0, 0x1D7CE, 0x1D7D8, 0x1D7E2, 0x1D7EC, 0x1D7F6,
|
||||
0x1E140, 0x1E2F0, 0x1E4F0, 0x1E5F1, 0x1E950, 0x1FBF0,
|
||||
];
|
||||
|
||||
/// A decimal digit of any script as its ASCII digit: `١٢` and `१२` search as `12`.
|
||||
fn ascii_digit(c: char) -> Option<char> {
|
||||
let c = u32::from(c);
|
||||
let zero = match ZEROS.binary_search(&c) {
|
||||
Ok(i) => ZEROS.get(i),
|
||||
Err(i) => ZEROS.get(i.checked_sub(1)?),
|
||||
}?;
|
||||
let value = c - zero;
|
||||
if value < 10 { char::from_digit(value, 10) } else { None }
|
||||
}
|
||||
|
||||
/// Whether a conjoining jamo of kind `next` continues the syllable a character of kind `prev`
|
||||
/// ends: a vowel after a leading consonant, a final consonant after a vowel or an open syllable
|
||||
/// (`가` and `ᆨ` spell `각`).
|
||||
fn continues_syllable(prev: HangulSyllableType, next: HangulSyllableType) -> bool {
|
||||
matches!(
|
||||
(prev, next),
|
||||
(HangulSyllableType::LeadingJamo, HangulSyllableType::VowelJamo)
|
||||
| (HangulSyllableType::VowelJamo | HangulSyllableType::LVSyllable, HangulSyllableType::TrailingJamo)
|
||||
)
|
||||
}
|
||||
|
||||
/// Folded text and where each part came from.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct Folded {
|
||||
text: String,
|
||||
/// One (folded offset, source offset) per source cluster, in order, then both lengths. A
|
||||
/// cluster folds to the text up to the next entry's folded offset, possibly none.
|
||||
starts: Vec<(usize, usize)>,
|
||||
}
|
||||
|
||||
impl Folded {
|
||||
/// The folded text.
|
||||
pub fn as_str(&self) -> &str {
|
||||
&self.text
|
||||
}
|
||||
|
||||
/// The source byte range that folded bytes `range` came from, widened to whole source
|
||||
/// characters with their accents: `ss` in the folding of `Straße` maps back to `ß`, `e` in
|
||||
/// that of `é` to the whole `é`. An empty range maps to an empty one.
|
||||
pub fn source_range(&self, range: Range<usize>) -> Range<usize> {
|
||||
// The last cluster starting at or before `at` is the one holding it: clusters that fold to
|
||||
// nothing share their offset with the next one.
|
||||
let cluster = |at: usize| self.starts.partition_point(|&(f, _)| f <= at).saturating_sub(1);
|
||||
if range.is_empty() {
|
||||
let s = self.starts[cluster(range.start)].1;
|
||||
return s..s;
|
||||
}
|
||||
let first = cluster(range.start);
|
||||
let last = cluster(range.end - 1);
|
||||
self.starts[first].1..self.starts[(last + 1).min(self.starts.len() - 1)].1
|
||||
}
|
||||
|
||||
/// The first match of `needle` (folded the same way), as a byte range of the source text.
|
||||
/// An empty needle matches at the start.
|
||||
pub fn find(&self, needle: &Folded) -> Option<Range<usize>> {
|
||||
self.matches(needle).next()
|
||||
}
|
||||
|
||||
/// Every non-overlapping match of `needle` (folded the same way), as byte ranges of the
|
||||
/// source text. A match may start or end inside an expansion (`ss` of `ß`, `fi` of `fi`) but
|
||||
/// not leave out a mark that follows a letter, so `re` doesn't match `ré` when accents count.
|
||||
pub fn matches<'a>(&'a self, needle: &'a Folded) -> impl Iterator<Item = Range<usize>> + 'a {
|
||||
let extend = CodePointSetData::new::<GraphemeExtend>();
|
||||
let hangul = CodePointMapData::<HangulSyllableType>::new();
|
||||
// The vowel and final consonant of a decomposed Hangul syllable continue it, like a mark,
|
||||
// but only inside one source cluster: standalone jamo (`ㅋㅠ`, `ㅠㅠ`) are letters.
|
||||
let starts_mark = move |at: usize| {
|
||||
let Some(c) = self.text[at..].chars().next() else { return false };
|
||||
if extend.contains(c) {
|
||||
return true;
|
||||
}
|
||||
let inside = self.starts.binary_search_by_key(&at, |&(f, _)| f).is_err();
|
||||
inside && self.text[..at].chars().next_back().is_some_and(|p| continues_syllable(hangul.get(p), hangul.get(c)))
|
||||
};
|
||||
let n = needle.text.as_str();
|
||||
let mut from = 0;
|
||||
let mut done = false;
|
||||
std::iter::from_fn(move || {
|
||||
if done {
|
||||
return None;
|
||||
}
|
||||
if n.is_empty() {
|
||||
done = true;
|
||||
return Some(0..0);
|
||||
}
|
||||
while let Some(i) = self.text.get(from..).and_then(|t| t.find(n)) {
|
||||
let start = from + i;
|
||||
let end = start + n.len();
|
||||
if !starts_mark(start) && !starts_mark(end) {
|
||||
from = end;
|
||||
return Some(self.source_range(start..end));
|
||||
}
|
||||
from = start + self.text[start..].chars().next().map_or(1, char::len_utf8);
|
||||
}
|
||||
done = true;
|
||||
None
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn key(locale: &str, s: &str) -> String {
|
||||
Folder::new(locale).key(s)
|
||||
}
|
||||
|
||||
fn key_accents(locale: &str, s: &str) -> String {
|
||||
Folder::new(locale).keep_accents().key(s)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn folding_table() {
|
||||
let nfd = |s: &str| DecomposingNormalizerBorrowed::new_nfd().normalize(s).into_owned();
|
||||
// (locale, input, ignoring accents, keeping accents)
|
||||
let table = [
|
||||
("en", "Hello WORLD", "hello world".to_owned(), "hello world".to_owned()),
|
||||
("en", "İSTANBUL", "istanbul".into(), "istanbul".into()),
|
||||
("tr", "İSTANBUL", "istanbul".into(), "istanbul".into()),
|
||||
("en", "IRMAK ırmak", "irmak irmak".into(), "irmak ırmak".into()),
|
||||
("tr", "IRMAK ırmak", "irmak irmak".into(), "ırmak ırmak".into()),
|
||||
("az", "İIıi", "iiii".into(), "iııi".into()),
|
||||
("tr-TR", "Şişli", "sisli".into(), nfd("şişli")),
|
||||
("de", "Straße STRAẞE", "strasse strasse".into(), "strasse strasse".into()),
|
||||
("el", "ΟΔΥΣΣΕΥΣ οδυσσευς", "οδυσσευσ οδυσσευσ".into(), "οδυσσευσ οδυσσευσ".into()),
|
||||
("el", "Ἀθῆναι", "αθηναι".into(), nfd("ἀθῆναι")),
|
||||
("fr", "Œuvre ÉTÉ", "oeuvre ete".into(), nfd("œuvre été")),
|
||||
("da", "Ærø", "aero".into(), nfd("ærø")),
|
||||
("pl", "Łódź", "lodz".into(), nfd("łódź")),
|
||||
("vi", "Đà Nẵng", "da nang".into(), nfd("đà nẵng")),
|
||||
("en", "final ff", "final ff".into(), "final ff".into()),
|
||||
("ja", "ＡＢＣ１２３ ｶﾞ", nfd("abc123 ガ"), nfd("abc123 ガ")),
|
||||
("ar", "مُحَمَّد أحمـــد", nfd("محمد احمد"), nfd("مُحَمَّد أحمد")),
|
||||
("he", "שָׁלוֹם", "שלום".into(), nfd("שָׁלוֹם")),
|
||||
("en", "soft\u{AD}ware \u{2068}name\u{2069}", "software name".into(), "software name".into()),
|
||||
("ru", "Ёлка", "елка".into(), nfd("ёлка")),
|
||||
("ar", "Layer ١٢ ۳۴ १२", "layer 12 34 12".into(), "layer 12 34 12".into()),
|
||||
("fa", "کتاب یک", "كتاب يك".into(), "کتاب یک".into()),
|
||||
("fa", "خانۀ", "خانه".into(), nfd("خانۀ")),
|
||||
("ar", "مدرسة على إسلام آمن", "مدرسه علي اسلام امن".into(), nfd("مدرسة على إسلام آمن")),
|
||||
];
|
||||
for (locale, input, ignoring, keeping) in table {
|
||||
assert_eq!(key(locale, input), ignoring, "{locale} {input}");
|
||||
assert_eq!(key_accents(locale, input), keeping, "{locale} {input}, accents kept");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn turkish_i_depends_on_locale_when_accents_count() {
|
||||
let tr = Folder::new("tr").keep_accents();
|
||||
let en = Folder::new("en").keep_accents();
|
||||
assert!(tr.contains("IRMAK", "ırmak") && !tr.contains("IRMAK", "irmak"));
|
||||
assert!(en.contains("IRMAK", "irmak") && !en.contains("IRMAK", "ırmak"));
|
||||
assert!(tr.contains("İSTANBUL", "istanbul") && en.contains("İSTANBUL", "istanbul"));
|
||||
for locale in ["tr", "en"] {
|
||||
let f = Folder::new(locale);
|
||||
assert!(f.contains("IRMAK", "ırmak") && f.contains("ırmak", "IRMAK") && f.contains("İzmir", "izmir"));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn matches_map_back_to_the_source() {
|
||||
let f = Folder::new("en");
|
||||
fn find<'a>(f: Folder, haystack: &'a str, needle: &str) -> Option<&'a str> {
|
||||
f.fold(haystack).find(&f.fold(needle)).map(|r| &haystack[r])
|
||||
}
|
||||
let find = |haystack, needle| find(f, haystack, needle);
|
||||
assert_eq!(find("Große Straße", "strasse"), Some("Straße"));
|
||||
assert_eq!(find("Große Straße", "SS"), Some("ß"));
|
||||
assert_eq!(find("Große Straße", "sse"), Some("ße"));
|
||||
assert_eq!(find("Café crème", "creme"), Some("crème"));
|
||||
assert_eq!(find("Cafe\u{301} noir", "cafe"), Some("Cafe\u{301}"));
|
||||
assert_eq!(find("Ｆｉｌｅ ２", "file 2"), Some("Ｆｉｌｅ ２"));
|
||||
assert_eq!(find("the final cut", "fin"), Some("fin"));
|
||||
assert_eq!(find("Layer \u{2068}İSTANBUL\u{2069}", "istanbul"), Some("İSTANBUL"));
|
||||
assert_eq!(find("abc", "x"), None);
|
||||
assert_eq!(find("abc", ""), Some(""));
|
||||
|
||||
let all: Vec<_> = f.fold("Ab ab AB").matches(&f.fold("ab")).collect();
|
||||
assert_eq!(all, [0..2, 3..5, 6..8]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn persian_and_arabic_letters_match() {
|
||||
let f = Folder::new("fa");
|
||||
// Persian keheh and Farsi yeh against Arabic kaf and yeh, and hamza forms of alef.
|
||||
assert!(f.contains("كتاب عربي", "کتاب عربی"));
|
||||
assert!(f.contains("کتاب عربی", "كتاب عربي"));
|
||||
assert!(f.contains("أحمد إبراهيم", "احمد ابراهیم"));
|
||||
assert!(f.contains("مدرسة", "مدرسه"));
|
||||
assert_eq!(f.fold("Layer ١٢").find(&f.fold("layer 12")), Some(0.."Layer ١٢".len()));
|
||||
assert!(Folder::new("en").keep_accents().contains("Layer ١٢", "layer 12"));
|
||||
let digits = ZEROS.iter().map(|&zero| char::from_u32(zero + 7).unwrap());
|
||||
assert!(digits.clone().all(|d| d.is_numeric() && ascii_digit(d) == Some('7')));
|
||||
assert_eq!(ascii_digit('\u{1D800}'), None);
|
||||
assert_eq!(ascii_digit('/'), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn accents_count_only_whole_letters() {
|
||||
let f = Folder::new("fr").keep_accents();
|
||||
assert!(!f.contains("ré", "re"));
|
||||
assert!(f.contains("ré", "RÉ") && f.contains("re\u{301}", "ré"));
|
||||
assert!(f.contains("ré", "é"));
|
||||
// Kana voicing marks stay even with accents ignored: か doesn't find が.
|
||||
assert!(!Folder::new("ja").contains("が", "か"));
|
||||
// Nor does a Hangul syllable without its final consonant: 하 doesn't find 한.
|
||||
let ko = Folder::new("ko");
|
||||
assert!(!ko.contains("한국", "하") && !ko.keep_accents().contains("한국", "하"));
|
||||
assert!(ko.contains("한국", "한") && ko.contains("하늘", "하"));
|
||||
// Conjoining jamo spell the same syllable; compatibility jamo stand alone.
|
||||
assert!(!ko.contains("\u{1112}\u{1161}\u{11AB}", "하") && ko.contains("\u{1112}\u{1161}\u{11AB}", "한"));
|
||||
assert!(!ko.contains("하\u{11AB}", "하"));
|
||||
for (haystack, needle) in [("ㅠㅠ", "ㅠ"), ("ㅠㅠ", "ㅠㅠ"), ("ㅋㅠ", "ㅋ"), ("ㅋㅠ", "ㅠ"), ("ㅋㅋㅋ", "ㅋㅋ"), ("ㄳ", "ㄳ"), ("값ㄳ", "ㄳ")]
|
||||
{
|
||||
assert!(ko.contains(haystack, needle), "{haystack} {needle}");
|
||||
}
|
||||
assert_eq!(ko.fold("ㅋㅠㅠ").find(&ko.fold("ㅠ")).map(|r| r.start), Some("ㅋ".len()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn source_ranges_round_trip() {
|
||||
let f = Folder::new("de");
|
||||
for s in ["Straße", "ffi Ａ", "e\u{301}\u{AD}x", "", "İı", "\u{301}a"] {
|
||||
let folded = f.fold(s);
|
||||
let text = folded.as_str();
|
||||
// A leading mark folds to nothing and has no part in any match.
|
||||
let whole = if s.starts_with('\u{301}') { 2..s.len() } else { 0..s.len() };
|
||||
assert_eq!(folded.source_range(0..text.len()), whole, "{s:?}");
|
||||
// Every folded character maps to a non-empty source range that folds back to text
|
||||
// containing it.
|
||||
for (i, c) in text.char_indices() {
|
||||
let r = folded.source_range(i..i + c.len_utf8());
|
||||
assert!(!r.is_empty() && f.key(&s[r.clone()]).contains(c), "{s:?} {i} {r:?}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn folding_is_idempotent() {
|
||||
// A folded query folds to itself, so a stored key can be searched with a fresh fold.
|
||||
for locale in ["en", "tr"] {
|
||||
for f in [Folder::new(locale), Folder::new(locale).keep_accents()] {
|
||||
for c in ('\0'..='\u{FFFF}').chain('\u{1D400}'..='\u{1D7FF}') {
|
||||
let once = f.key(c.encode_utf8(&mut [0; 4]));
|
||||
assert_eq!(f.key(&once), once, "{locale} {:?} U+{:04X}", f, c as u32);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -6,6 +6,9 @@
|
||||
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
|
||||
|
||||
mod catalog;
|
||||
#[cfg(feature = "collate")]
|
||||
mod collate;
|
||||
mod fold;
|
||||
mod region;
|
||||
mod scripts;
|
||||
mod size;
|
||||
@@ -17,6 +20,9 @@ use intl_pluralrules::{PluralCategory, PluralRuleType, PluralRules};
|
||||
use unic_langid::LanguageIdentifier;
|
||||
use unicode_bidi::{BidiClass, bidi_class};
|
||||
|
||||
#[cfg(feature = "collate")]
|
||||
pub use collate::Collator;
|
||||
pub use fold::{Folded, Folder};
|
||||
pub use region::{HourCycle, Measurement, Paper, Region};
|
||||
pub use scripts::{Direction, Han, Script, base_direction, direction, han, scripts};
|
||||
pub use size::{FileSize, SizeUnit};
|
||||
|
||||
Reference in new issue
Block a user