diff --git a/Cargo.lock b/Cargo.lock index 97fa16be..16d7ef4d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -144,7 +144,16 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2f926cc3060f09db9ebc5b52823d85268d24bb917e472c0c4bea35780a7d" dependencies = [ - "bit-vec", + "bit-vec 0.9.1", +] + +[[package]] +name = "bit-set" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56d87354e4229f54a44f7bf2435906a4656dba36026ab6eaca629a2c436a691c" +dependencies = [ + "bit-vec 0.10.1", ] [[package]] @@ -153,6 +162,16 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b71798fca2c1fe1086445a7258a4bc81e6e49dcd24c8d0dd9a1e57395b603f51" +[[package]] +name = "bit-vec" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5727b15fa97d4f4fee0a3b7c3d550ed0269f54329207b86388de918604e31269" +dependencies = [ + "borsh", + "serde", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -183,6 +202,30 @@ dependencies = [ "objc2 0.6.5", ] +[[package]] +name = "borsh" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "553c5d846a6ba5150c65e3b1b8ec073bcf1abc20f9b7220de384a4443ea4e20a" +dependencies = [ + "borsh-derive", + "bytes", + "cfg_aliases", +] + +[[package]] +name = "borsh-derive" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12cdfe656708a01f89b451a7d36466e6fe6c414de0aa18fc54f864f6f9ca9f56" +dependencies = [ + "once_cell", + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 3.0.6", +] + [[package]] name = "bumpalo" version = "3.20.3" @@ -283,6 +326,17 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" +[[package]] +name = "chacha20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" +dependencies = [ + "cfg-if", + "cpufeatures", + "rand_core", +] + [[package]] name = "clipboard-win" version = "5.4.1" @@ -371,6 +425,21 @@ dependencies = [ "libc", ] +[[package]] +name = "core_detect" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f8f80099a98041a3d1622845c271458a2d73e688351bf3cb999266764b81d48" + +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + [[package]] name = "craft-print" version = "0.1.0" @@ -379,6 +448,14 @@ dependencies = [ "tempfile", ] +[[package]] +name = "craft-proof" +version = "0.1.0" +dependencies = [ + "miniz_oxide 0.8.9", + "proptest", +] + [[package]] name = "craft-ui" version = "0.1.0" @@ -752,6 +829,12 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "98de4bbd547a563b716d8dfa9aad1cb19bfab00f4fa09a6a4ed21dbcf44ce9c4" +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "foldhash" version = "0.2.0" @@ -837,6 +920,7 @@ dependencies = [ "cfg-if", "libc", "r-efi", + "rand_core", ] [[package]] @@ -1259,7 +1343,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a616d2fb8c89516ac2723a581f69d6c18576046bed761bd6b305e5618e6ae130" dependencies = [ "arrayvec", - "bit-set", + "bit-set 0.10.0", "bitflags 2.13.2", "cfg-if", "cfg_aliases", @@ -1878,12 +1962,38 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d595e54a326bc53c1c197b32d295e14b169e3cfeaa8dc82b529f947fba6bcf5" +[[package]] +name = "proptest" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8530004ccb15eae51c7e40009fbe317f341f804db54dc033eec1c50be28cfa0" +dependencies = [ + "bit-set 0.11.1", + "bit-vec 0.10.1", + "bitflags 2.13.2", + "chacha20", + "core_detect", + "num-traits", + "rand", + "rand_xorshift", + "regex-syntax", + "rusty-fork", + "tempfile", + "unarray", +] + [[package]] name = "pxfm" version = "0.1.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d55d956fa96f5ec02be2e13af0e20391a5aa83d6a074e3ad368959d0fab299ea" +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + [[package]] name = "quick-error" version = "2.0.1" @@ -1914,6 +2024,31 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "rand" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c9fb96cbc91e3478eaae79a69fcd3f1ae4ad052e471fe6732fff548984b4af" +dependencies = [ + "getrandom", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rand_xorshift" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60aa6af80be32871323012e02e6e65f8a7cc7890931ae421d217ad8fe0df2ccf" +dependencies = [ + "rand_core", +] + [[package]] name = "range-alloc" version = "0.1.5" @@ -1976,6 +2111,12 @@ dependencies = [ "bitflags 2.13.2", ] +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + [[package]] name = "renderdoc-sys" version = "1.1.0" @@ -2064,6 +2205,18 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" +[[package]] +name = "rusty-fork" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc6bf79ff24e648f6da1f8d1f011e9cac26491b619e6b9280f2b47f1774e6ee2" +dependencies = [ + "fnv", + "quick-error 1.2.3", + "tempfile", + "wait-timeout", +] + [[package]] name = "same-file" version = "1.0.6" @@ -2397,7 +2550,7 @@ dependencies = [ "fax", "flate2", "half", - "quick-error", + "quick-error 2.0.1", "weezl", "zune-jpeg", ] @@ -2506,6 +2659,12 @@ dependencies = [ "rustc-hash 2.1.3", ] +[[package]] +name = "unarray" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" + [[package]] name = "unicode-general-category" version = "1.1.0" @@ -2591,6 +2750,15 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "wait-timeout" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11" +dependencies = [ + "libc", +] + [[package]] name = "walkdir" version = "2.5.0" @@ -2842,8 +3010,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14c018fce9b6270aa203c2fdd56f3cce996713534bd757e4ea58c8560b121f14" dependencies = [ "arrayvec", - "bit-set", - "bit-vec", + "bit-set 0.10.0", + "bit-vec 0.9.1", "bitflags 2.13.2", "bytemuck", "cfg_aliases", @@ -2915,7 +3083,7 @@ dependencies = [ "android_system_properties", "arrayvec", "ash", - "bit-set", + "bit-set 0.10.0", "bitflags 2.13.2", "block2 0.6.2", "bytemuck", diff --git a/Cargo.toml b/Cargo.toml index 6880b4f8..976840b4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,6 +12,7 @@ repository = "https://github.com/storytold/craft-libs" [workspace.dependencies] # internal: add each crate here as `craft- = { path = "crates/" }` craft-print = { path = "crates/print" } +craft-proof = { path = "crates/proof" } craft-ui = { path = "crates/ui" } egui = { version = "0.36", default-features = false } diff --git a/crates/proof/Cargo.toml b/crates/proof/Cargo.toml new file mode 100644 index 00000000..986f90d4 --- /dev/null +++ b/crates/proof/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "craft-proof" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +description = "Spelling for the Craft apps: Hunspell affix dictionaries, installed-dictionary discovery and language tags" + +[lints] +workspace = true + +[dependencies] +miniz_oxide = "0.8" + +[dev-dependencies] +proptest = "1" diff --git a/crates/proof/README.md b/crates/proof/README.md new file mode 100644 index 00000000..6577c511 --- /dev/null +++ b/crates/proof/README.md @@ -0,0 +1,31 @@ +# craft-proof + +Spelling mechanism shared by the Craft apps: read a Hunspell `.aff`/`.dic` pair installed on the +system and check words against it. Seeded from WordCraft's `wordcraft-proof`; GridCraft, +DeckCraft and PdfCraft are the next consumers (storytold/craft-libs#30). No dependency on any +app or UI framework. Requires Rust 1.95. + +```rust,no_run +let search = craft_proof::hunspell::Search::for_app("WordCraft"); +if let Some((aff, dic)) = search.find(&["pl_PL", "pl"]) { + let dict = craft_proof::hunspell::load(&aff, &dic)?; + assert!(dict.check("kotami")); +} +# Ok::<(), String>(()) +``` + +- **Dictionaries are not bundled.** Apps read what the OS, a package manager or LibreOffice + installed; without one a language simply has no spelling check. +- **Search order** (`Search`): the app's dictionaries folder setting, `_DICTIONARIES`, the + app's folder under the Windows application-data folders, then the usual Hunspell and + LibreOffice locations for the platform. Nothing on wasm. +- **Affix engine** (`AffixDict`): the Hunspell subset real dictionaries such as SJP.PL (Polish) + and VERO (Brazilian Portuguese) use: one-character or `FLAG UTF-8` flags, prefixes and + suffixes with conditions and cross products, continuation classes, `FORBIDDENWORD`, and the + `TRY`/`MAP`/`REP` suggestion tables. Compounding and long or numeric flags are rejected with + an error rather than misread. +- **Compact form**: `AffixDict::to_bytes`/`from_bytes` store a loaded dictionary deflated + (Polish is under 1 MB), so an app can cache it instead of parsing the `.aff`/`.dic` each start. + +What stays in each app: which languages it proofs, suggestion ranking and grammar rules per +language, the user dictionary, and how marks are shown. diff --git a/crates/proof/src/affix.rs b/crates/proof/src/affix.rs new file mode 100644 index 00000000..e1aa1831 --- /dev/null +++ b/crates/proof/src/affix.rs @@ -0,0 +1,1075 @@ +//! Hunspell-style affix dictionaries — the subset the SJP.PL Polish dictionary (`pl_PL`) uses. +//! +//! A word is correct when it is a stem, a stem with one suffix, a stem with one prefix, or a stem +//! with a prefix and a suffix that both allow cross products; each rule applies only to stems +//! carrying its flag and ending (suffixes) or starting (prefixes) as its condition says. Highly +//! inflected languages store millions of forms this way in a few hundred thousand stems: the +//! Polish dictionary's ~350k stems and ~7,500 rules accept ~4.5 million forms. +//! +//! The rules and stems come from a Hunspell `.aff`/`.dic` pair ([`AffixDict::from_hunspell`]), +//! read at runtime from the dictionaries installed on the system ([`crate::hunspell`]). They can +//! also be stored in a compact format ([`AffixDict::to_bytes`]): the rules as text +//! lines and the stems front-coded, raw-deflated together (the Polish dictionary is under 1 MB). +//! +//! Supported: one-character flags (at most 128 distinct, any Unicode character — `FLAG UTF-8`), +//! `PFX`/`SFX` with strip, append and condition (`.`, letters, `[abc]`, `[^abc]`), cross +//! products, continuation classes on suffixes (at most two suffixes in a row, as in Hunspell), +//! `FORBIDDENWORD`, and the suggestion tables `TRY`, `MAP` and `REP`. Rejected: compounding, +//! long or numeric flags, `NEEDAFFIX` and the other directives that change which words are correct. +//! +//! Lookups never panic: stems sit in one string, found through an open-addressing hash table, and +//! a probe allocates only small scratch strings for the candidate roots it tries. + +use std::collections::HashMap; +use std::hash::Hasher; + +use crate::fx::{FxHasher, FxMap}; + +const MAGIC: &str = "WCAFF1"; +/// Longest word (in bytes) looked up; longer input is never a dictionary word. +pub const MAX_WORD_BYTES: usize = 120; +/// Empty slot in the stem hash table. +const EMPTY: u32 = u32::MAX; + +/// Directives that change which words are correct in ways this engine doesn't implement. +const UNSUPPORTED: &[&str] = &[ + "AF", + "AM", + "CHECKSHARPS", + "CIRCUMFIX", + "COMPLEXPREFIXES", + "COMPOUNDBEGIN", + "COMPOUNDEND", + "COMPOUNDFLAG", + "COMPOUNDMIDDLE", + "COMPOUNDMIN", + "COMPOUNDPERMITFLAG", + "COMPOUNDRULE", + "FULLSTRIP", + "ICONV", + "IGNORE", + "KEEPCASE", + "LEMMA_PRESENT", + "NEEDAFFIX", + "OCONV", + "ONLYINCOMPOUND", + "PSEUDOROOT", + "SUBSTANDARD", +]; + +/// One character class of a rule's condition. +#[derive(Clone, Debug, PartialEq, Eq)] +enum Cond { + Any, + Char(char), + Set { negated: bool, chars: Box<[char]> }, +} + +impl Cond { + fn matches(&self, c: char) -> bool { + match self { + Cond::Any => true, + Cond::Char(x) => *x == c, + Cond::Set { negated, chars } => chars.contains(&c) != *negated, + } + } +} + +/// Parse a Hunspell condition (`.`, `[^i]e`, `[km]anie`…). +fn parse_cond(text: &str) -> Result, String> { + if text == "." { + return Ok(Box::new([])); + } + let mut out = Vec::new(); + let mut chars = text.chars(); + while let Some(c) = chars.next() { + match c { + '.' => out.push(Cond::Any), + '[' => { + let mut set = Vec::new(); + let mut negated = false; + let mut closed = false; + let mut first = true; + for d in chars.by_ref() { + if d == ']' { + closed = true; + break; + } + if first && d == '^' { + negated = true; + } else { + set.push(d); + } + first = false; + } + if !closed { + return Err(format!("unclosed `[` in condition `{text}`")); + } + out.push(Cond::Set { negated, chars: set.into_boxed_slice() }); + } + ']' => return Err(format!("stray `]` in condition `{text}`")), + c => out.push(Cond::Char(c)), + } + } + Ok(out.into_boxed_slice()) +} + +/// How many suffixes a word may carry in a row through continuation classes (`work` + +/// `able/B` + `s` → `workables` is two). Hunspell strips at most two suffixes, so a third +/// chained suffix is not a word; the bound also keeps hostile input from walking deep recursions. +const MAX_CHAIN: u8 = 2; +/// How many suffixes may sit under the outermost one: the depth `form_has_flag` walks. +const INNER_CHAIN: u8 = MAX_CHAIN - 1; + +/// One prefix or suffix rule. +#[derive(Clone, Debug)] +struct Rule { + /// Bit of the rule's flag in the stems' flag masks. + bit: u128, + flag: char, + cross: bool, + strip: Box, + add: Box, + /// Continuation classes (`SFX A 0 able/B`): flags the derived form carries, licensing + /// further affixes. Chained application is bounded by [`MAX_CHAIN`]. + cont: u128, + cond: Box<[Cond]>, + cond_text: Box, +} + +impl Rule { + /// Does `root` end (suffix rule) or start (prefix rule) as the condition says? + fn cond_matches(&self, root: &str, suffix: bool) -> bool { + if self.cond.is_empty() { + return true; + } + if suffix { + let mut it = root.chars().rev(); + self.cond.iter().rev().all(|c| it.next().is_some_and(|x| c.matches(x))) + } else { + let mut it = root.chars(); + self.cond.iter().all(|c| it.next().is_some_and(|x| c.matches(x))) + } + } +} + +/// Suffix rules sharing an appended string and a stripped one: one stem lookup serves them all. +#[derive(Clone, Debug, Default)] +struct SfxGroup { + strip: Box, + rules: Vec, +} + +/// A stem list with affix rules (see the module docs). +#[derive(Clone, Debug, Default)] +pub struct AffixDict { + /// All stems, concatenated, sorted by bytes. + words: String, + offs: Vec, + /// Per stem: index into `sets`. + flags: Vec, + /// Distinct flag masks. + sets: Vec, + /// The `FORBIDDENWORD` flag's bit; stems carrying it are not words (0: none declared). + forbidden: u128, + /// Open-addressing hash table of stem indices (`EMPTY` = free). + table: Vec, + /// Flag char of each bit. + flag_chars: Vec, + prefixes: Vec, + suffixes: Vec, + /// Appended string → groups of suffix rules by stripped string. + sfx_index: FxMap, Vec>, + /// Longest appended suffix, in chars. + max_sfx_chars: usize, + try_chars: String, + map: Vec>, + rep: Vec<(String, String)>, + /// Provenance and licence lines, kept through [`AffixDict::to_bytes`]. + comments: Vec, +} + +fn hash_str(s: &str) -> u64 { + let mut h = FxHasher::default(); + h.write(s.as_bytes()); + h.finish() +} + +/// Letter case of a word. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Casing { + /// No capitals (`kot`). + Lower, + /// First letter capital, no other (`Kot`). + Initial, + /// Every cased letter a capital, at least two (`KOT`). + Upper, + /// Anything else (`McDonald`, `kOT`). + Mixed, +} + +/// The casing of `w`. +pub fn casing(w: &str) -> Casing { + let (mut upper, mut lower, mut first_upper) = (0usize, 0usize, false); + for (i, c) in w.chars().enumerate() { + if c.is_uppercase() { + upper += 1; + if i == 0 { + first_upper = true; + } + } else if c.is_lowercase() { + lower += 1; + } + } + if upper == 0 { + Casing::Lower + } else if upper == 1 && first_upper { + Casing::Initial + } else if lower == 0 { + Casing::Upper + } else { + Casing::Mixed + } +} + +/// `w` with its first letter a capital. +pub fn capitalize(w: &str) -> String { + let mut c = w.chars(); + match c.next() { + Some(f) => f.to_uppercase().chain(c).collect(), + None => String::new(), + } +} + +/// Parsed rules and tables, shared by the Hunspell reader and the compact format. +#[derive(Default)] +struct Rules { + prefixes: Vec, + suffixes: Vec, + try_chars: String, + map: Vec>, + rep: Vec<(String, String)>, + /// The `FORBIDDENWORD` flag: stems carrying it are listed but never correct. + forbidden: Option, +} + +/// A rule as parsed from text, before flags become bits: `(flag, cross, strip, add, cont, cond)`. +type RawRule = (char, bool, String, String, String, String); + +/// `0` means empty in Hunspell's strip and append fields. +fn zero_empty(s: &str) -> String { + if s == "0" { String::new() } else { s.to_string() } +} + +/// Read Hunspell `.aff` text (already decoded to UTF-8). +fn parse_aff(aff: &str) -> Result { + let mut r = Rules::default(); + let mut cross: HashMap<(bool, char), bool> = HashMap::new(); + let mut map_header = false; + let mut rep_header = false; + for (n, line) in aff.lines().enumerate() { + let f: Vec<&str> = line.split_whitespace().collect(); + let Some(&dir) = f.first() else { continue }; + if dir.starts_with('#') { + continue; + } + let at = |k: usize| f.get(k).copied().unwrap_or(""); + match dir { + "SET" | "LANG" | "NAME" | "VERSION" | "HOME" | "WORDCHARS" | "KEY" | "MAXNGRAMSUGS" | "MAXDIFF" | "ONLYMAXDIFF" | "NOSPLITSUGS" + | "SUGSWITHDOTS" | "NOSUGGEST" | "WARN" | "FORBIDWARN" | "MAXCPDSUGS" => {} + // Per-character flag modes (`FLAG UTF-8`, as VERO's pt_BR dictionary uses) work with + // the one-character parsing here; only the multi-character encodings differ. + "FLAG" if !matches!(at(1), "UTF-8" | "DEFAULT" | "utf-8" | "default" | "UTF8" | "") => { + return Err(format!("line {}: FLAG {} is not supported (one-character flags only)", n + 1, at(1))); + } + "FLAG" => {} + "FORBIDDENWORD" => { + let mut it = at(1).chars(); + match (it.next(), it.next()) { + (Some(c), None) if !c.is_whitespace() => r.forbidden = Some(c), + _ => return Err(format!("line {}: FORBIDDENWORD needs one flag character", n + 1)), + } + } + "TRY" => r.try_chars = at(1).to_string(), + "MAP" => { + if !map_header && at(1).parse::().is_ok() { + map_header = true; + } else if !at(1).contains('(') { + r.map.push(at(1).chars().collect()); + } + } + "REP" => { + if !rep_header && f.len() == 2 && at(1).parse::().is_ok() { + rep_header = true; + } else if f.len() >= 3 { + r.rep.push((at(1).replace('_', " "), at(2).replace('_', " "))); + } + } + "PFX" | "SFX" => { + let suffix = dir == "SFX"; + let mut flag_chars = at(1).chars(); + let (Some(flag), None) = (flag_chars.next(), flag_chars.next()) else { + return Err(format!("line {}: flag `{}` is not one character", n + 1, at(1))); + }; + let is_header = f.len() == 4 && matches!(at(2), "Y" | "N") && at(3).parse::().is_ok() && !cross.contains_key(&(suffix, flag)); + if is_header { + cross.insert((suffix, flag), at(2) == "Y"); + continue; + } + if f.len() < 4 { + return Err(format!("line {}: short affix rule", n + 1)); + } + // `able/B`: the appended text and its continuation classes. + let (add, cont) = match at(3).split_once('/') { + Some((a, c)) => (a, c), + None => (at(3), ""), + }; + let cond = if f.len() >= 5 { at(4) } else { "." }; + parse_cond(cond).map_err(|e| format!("line {}: {e}", n + 1))?; + let c = cross.get(&(suffix, flag)).copied().unwrap_or(false); + let rule: RawRule = (flag, c, zero_empty(at(2)), zero_empty(add), cont.to_string(), cond.to_string()); + if suffix { + r.suffixes.push(rule); + } else { + r.prefixes.push(rule); + } + } + d if UNSUPPORTED.contains(&d) => return Err(format!("line {}: {d} is not supported", n + 1)), + _ => {} + } + } + Ok(r) +} + +impl AffixDict { + /// Build from Hunspell `.aff` and `.dic` text (decoded to UTF-8; the `.dic` count line and + /// morphological fields are ignored). `comments` (provenance, licence) are kept with the data. + pub fn from_hunspell(aff: &str, dic: &str, comments: &[&str]) -> Result { + // Some dictionaries open with a UTF-8 BOM; it must not glue itself to the first + // directive or turn the `.dic` count line into a stem. + let aff = aff.strip_prefix('\u{feff}').unwrap_or(aff); + let dic = dic.strip_prefix('\u{feff}').unwrap_or(dic); + let rules = parse_aff(aff)?; + let mut stems: Vec<(String, String)> = Vec::new(); + for (n, line) in dic.lines().enumerate() { + let Some(tok) = line.split_whitespace().next() else { continue }; + if n == 0 && tok.parse::().is_ok() { + continue; + } + // `\/` is a literal slash in a word; the first plain `/` starts the flags. + let mut word = String::new(); + let mut flags = String::new(); + let mut chars = tok.chars().peekable(); + let mut in_flags = false; + while let Some(c) = chars.next() { + if in_flags { + flags.push(c); + } else if c == '\\' && chars.peek() == Some(&'/') { + word.push('/'); + chars.next(); + } else if c == '/' { + in_flags = true; + } else { + word.push(c); + } + } + if word.is_empty() || word.len() > MAX_WORD_BYTES || word.contains(['\n', '/']) { + continue; + } + stems.push((word, flags)); + } + AffixDict::build(rules, stems, comments.iter().map(|s| s.to_string()).collect()) + } + + fn build(rules: Rules, mut stems: Vec<(String, String)>, comments: Vec) -> Result { + // Flags: every one a rule or a stem uses gets a bit. + let mut flag_chars: Vec = rules.prefixes.iter().chain(&rules.suffixes).map(|r| r.0).collect(); + flag_chars.extend(stems.iter().flat_map(|s| s.1.chars())); + flag_chars.sort_unstable(); + flag_chars.dedup(); + if flag_chars.len() > 128 { + return Err(format!("{} distinct flags; at most 128 are supported", flag_chars.len())); + } + let bit_of = |c: char| flag_chars.iter().position(|&x| x == c).map_or(0, |i| 1u128 << i); + let forbidden = rules.forbidden.map_or(0, bit_of); + // Stems sorted by bytes; homonyms (the same word twice) are merged. + stems.sort_by(|a, b| a.0.as_bytes().cmp(b.0.as_bytes())); + let mut d = AffixDict { + words: String::with_capacity(stems.iter().map(|s| s.0.len()).sum()), + offs: Vec::with_capacity(stems.len() + 1), + flag_chars: flag_chars.clone(), + forbidden, + try_chars: rules.try_chars, + map: rules.map, + rep: rules.rep, + comments, + ..AffixDict::default() + }; + d.offs.push(0); + let mut set_ids: HashMap = HashMap::new(); + let mut i = 0; + while i < stems.len() { + let Some((word, _)) = stems.get(i) else { break }; + let mut mask = 0u128; + let mut j = i; + while let Some((w, f)) = stems.get(j) + && w == word + { + mask |= f.chars().map(bit_of).fold(0, |a, b| a | b); + j += 1; + } + let next_id = u16::try_from(d.sets.len()).map_err(|_| "too many distinct flag sets".to_string())?; + let id = *set_ids.entry(mask).or_insert(next_id); + if usize::from(id) == d.sets.len() { + d.sets.push(mask); + } + d.words.push_str(word); + d.offs.push(u32::try_from(d.words.len()).map_err(|_| "stems too large".to_string())?); + d.flags.push(id); + i = j; + } + let mk = |(flag, cross, strip, add, cont, cond): RawRule| -> Result { + let cont = cont.chars().fold(0u128, |m, c| m | bit_of(c)); + Ok(Rule { bit: bit_of(flag), flag, cross, strip: strip.into(), add: add.into(), cont, cond: parse_cond(&cond)?, cond_text: cond.into() }) + }; + d.prefixes = rules.prefixes.into_iter().map(mk).collect::>()?; + d.suffixes = rules.suffixes.into_iter().map(mk).collect::>()?; + d.index(); + Ok(d) + } + + /// Build the suffix index and the stem hash table. + fn index(&mut self) { + let mut idx: FxMap, Vec> = FxMap::default(); + for (i, r) in self.suffixes.iter().enumerate() { + let groups = idx.entry(r.add.clone()).or_default(); + let Ok(i) = u32::try_from(i) else { break }; + match groups.iter_mut().find(|g| g.strip == r.strip) { + Some(g) => g.rules.push(i), + None => groups.push(SfxGroup { strip: r.strip.clone(), rules: vec![i] }), + } + } + self.max_sfx_chars = self.suffixes.iter().map(|r| r.add.chars().count()).max().unwrap_or(0); + self.sfx_index = idx; + let n = self.len(); + let size = (n.max(1) * 2).next_power_of_two(); + let mut table = vec![EMPTY; size]; + let mask = size - 1; + for i in 0..n { + let Some(w) = self.word(i) else { continue }; + let mut slot = hash_str(w) as usize & mask; + while table.get(slot).is_some_and(|&t| t != EMPTY) { + slot = (slot + 1) & mask; + } + if let (Some(t), Ok(i)) = (table.get_mut(slot), u32::try_from(i)) { + *t = i; + } + } + self.table = table; + } + + /// Number of stems. + pub fn len(&self) -> usize { + self.flags.len() + } + + pub fn is_empty(&self) -> bool { + self.flags.is_empty() + } + + /// Number of prefix and suffix rules. + pub fn rule_count(&self) -> usize { + self.prefixes.len() + self.suffixes.len() + } + + fn word(&self, i: usize) -> Option<&str> { + let a = *self.offs.get(i)? as usize; + let b = *self.offs.get(i + 1)? as usize; + self.words.get(a..b) + } + + /// The flag characters of `mask`, in flag order (how the compact format stores flags). + fn flag_chars_for(&self, mask: u128) -> String { + self.flag_chars.iter().enumerate().filter(|&(i, _)| mask & (1u128 << i) != 0).map(|(_, c)| c).collect() + } + + /// The character of `mask` when it holds exactly one flag. + fn flag_char(&self, mask: u128) -> Option { + let s = self.flag_chars_for(mask); + let mut it = s.chars(); + match (it.next(), it.next()) { + (Some(c), None) => Some(c), + _ => None, + } + } + + /// The stems, in byte order. + pub fn stems(&self) -> impl Iterator + '_ { + (0..self.len()).filter_map(|i| self.word(i)) + } + + /// The stems starting with `prefix`, in byte order (a binary search, then a scan). + pub fn stems_with_prefix<'a>(&'a self, prefix: &str) -> impl Iterator + use<'a> { + let prefix = prefix.to_string(); + let first = { + let (mut lo, mut hi) = (0usize, self.len()); + while lo < hi { + let mid = lo + (hi - lo) / 2; + if self.word(mid).is_some_and(|w| w.as_bytes() < prefix.as_bytes()) { + lo = mid + 1; + } else { + hi = mid; + } + } + lo + }; + (first..self.len()).map_while(move |i| self.word(i).filter(|w| w.starts_with(prefix.as_str()))) + } + + /// Flag mask of a stem, if `w` is one. + fn stem_flags(&self, w: &str) -> Option { + if self.table.is_empty() { + return None; + } + let mask = self.table.len() - 1; + let mut slot = hash_str(w) as usize & mask; + for _ in 0..self.table.len() { + let t = *self.table.get(slot)?; + if t == EMPTY { + return None; + } + if self.word(t as usize) == Some(w) { + let id = *self.flags.get(t as usize)?; + let mask = self.sets.get(usize::from(id)).copied(); + // A stem carrying FORBIDDENWORD is listed but is not a word: no direct hit, + // and (this is the only stem lookup) no affixed form is built on it either. + return mask.filter(|m| m & self.forbidden == 0); + } + slot = (slot + 1) & mask; + } + None + } + + /// Does `w` carry flag `bit` — as a stem, or as the form of a suffix rule whose + /// continuation classes include `bit` (chained suffixation, depth-bounded)? + fn form_has_flag(&self, w: &str, bit: u128, depth: u8) -> bool { + if self.stem_flags(w).is_some_and(|f| f & bit != 0) { + return true; + } + if depth == 0 { + return false; + } + let mut root = String::with_capacity(w.len() + 8); + let cuts = std::iter::once(w.len()).chain(w.char_indices().rev().map(|(i, _)| i).filter(|&i| i > 0)); + for b in cuts.take(self.max_sfx_chars + 1) { + let (Some(base), Some(add)) = (w.get(..b), w.get(b..)) else { continue }; + let Some(groups) = self.sfx_index.get(add) else { continue }; + for g in groups { + root.clear(); + root.push_str(base); + root.push_str(&g.strip); + for r in g.rules.iter().filter_map(|&i| self.suffixes.get(i as usize)) { + if r.cont & bit != 0 && r.cond_matches(&root, true) && self.form_has_flag(&root, r.bit, depth - 1) { + return true; + } + } + } + } + false + } + + /// Is `w` exactly (case and all) a stem or an affixed form of one? + pub fn contains(&self, w: &str) -> bool { + if w.is_empty() || w.len() > MAX_WORD_BYTES { + return false; + } + if self.stem_flags(w).is_some() || self.suffixed(w, 0) { + return true; + } + let mut root = String::with_capacity(w.len() + 8); + for r in &self.prefixes { + let Some(rest) = w.strip_prefix(&*r.add) else { continue }; + if rest.is_empty() { + continue; + } + root.clear(); + root.push_str(&r.strip); + root.push_str(rest); + if !r.cond_matches(&root, false) { + continue; + } + if self.stem_flags(&root).is_some_and(|f| f & r.bit != 0) { + return true; + } + if self.form_has_flag(&root, r.bit, INNER_CHAIN) { + return true; + } + if r.cross && self.suffixed(&root, r.bit) { + return true; + } + } + false + } + + /// Is `w` a stem with one suffix? With `need` set (a prefix was taken off), the stem must + /// carry those flags too and the suffix must allow cross products. + fn suffixed(&self, w: &str, need: u128) -> bool { + let mut root = String::with_capacity(w.len() + 8); + // Each split point from the end: the appended string is `w[b..]`, at most the longest + // suffix, and something must stay before it. + let cuts = std::iter::once(w.len()).chain(w.char_indices().rev().map(|(i, _)| i).filter(|&i| i > 0)); + for b in cuts.take(self.max_sfx_chars + 1) { + let (Some(base), Some(add)) = (w.get(..b), w.get(b..)) else { continue }; + let Some(groups) = self.sfx_index.get(add) else { continue }; + for g in groups { + root.clear(); + root.push_str(base); + root.push_str(&g.strip); + let rules = || g.rules.iter().filter_map(|&i| self.suffixes.get(i as usize)); + // The stem path: `root` is a stem carrying the rule's flag (and the prefix's, + // when a cross product is being checked). + if let Some(f) = self.stem_flags(&root) + && f & need == need + && rules().any(|r| f & r.bit != 0 && (need == 0 || r.cross) && r.cond_matches(&root, true)) + { + return true; + } + // Continuation classes: `root` need not be a stem carrying this rule's flag — + // it may be a form that gained the flag from another rule's continuation. + if need == 0 && rules().any(|r| r.cond_matches(&root, true) && self.form_has_flag(&root, r.bit, INNER_CHAIN)) { + return true; + } + } + } + false + } + + /// Is `w` correct, Hunspell style: as written, or — for a capitalized word or one in capitals + /// — in lower case (and capitalized), since a sentence may start with any word and headings + /// may be in capitals. Lower-case input never matches a capitalized stem (`warszawa`). + pub fn check(&self, w: &str) -> bool { + if self.contains(w) { + return true; + } + match casing(w) { + Casing::Lower | Casing::Mixed => false, + Casing::Initial => self.contains(&w.to_lowercase()), + Casing::Upper => { + let lower = w.to_lowercase(); + self.contains(&capitalize(&lower)) || self.contains(&lower) + } + } + } + + /// Every form of the stem `stem` (itself included), unsorted, possibly with repeats. + pub fn forms(&self, stem: &str) -> Vec { + let Some(f) = self.stem_flags(stem) else { return Vec::new() }; + let mut out = vec![stem.to_string()]; + let mut cross_sfx = Vec::new(); + // Suffix forms queued with the continuation classes they carry, for chained expansion. + let mut chain: Vec<(String, u128)> = Vec::new(); + for r in &self.suffixes { + if f & r.bit == 0 || !r.cond_matches(stem, true) { + continue; + } + let Some(base) = stem.strip_suffix(&*r.strip) else { continue }; + let form = format!("{base}{}", r.add); + if r.cross { + cross_sfx.push(form.clone()); + } + if r.cont != 0 { + chain.push((form.clone(), r.cont)); + } + out.push(form); + } + // Continuation classes chain suffixes (`workable/B` → `workables`), up to [`MAX_CHAIN`] + // suffixes in all — the same bound the checker walks. + for _ in 0..INNER_CHAIN { + let level = std::mem::take(&mut chain); + if level.is_empty() { + break; + } + for (form, flags) in level { + for r in &self.suffixes { + if flags & r.bit == 0 || !r.cond_matches(&form, true) { + continue; + } + let Some(base) = form.strip_suffix(&*r.strip) else { continue }; + let next = format!("{base}{}", r.add); + if r.cont != 0 { + chain.push((next.clone(), r.cont)); + } + out.push(next); + } + } + } + for r in &self.prefixes { + if f & r.bit == 0 || !r.cond_matches(stem, false) { + continue; + } + let Some(rest) = stem.strip_prefix(&*r.strip) else { continue }; + out.push(format!("{}{rest}", r.add)); + if r.cross { + for s in &cross_sfx { + if let Some(rest) = s.strip_prefix(&*r.strip) { + out.push(format!("{}{rest}", r.add)); + } + } + } + } + out + } + + /// The `TRY` letters, most frequent first (suggestions try these). + pub fn try_chars(&self) -> &str { + &self.try_chars + } + + /// The `MAP` groups: letters often confused with each other (`aą`, `zżź`, `oóu`…). + pub fn map(&self) -> &[Vec] { + &self.map + } + + /// The `REP` table: common misspellings, `(wrong, right)`. + pub fn rep(&self) -> &[(String, String)] { + &self.rep + } + + /// The provenance and licence lines. + pub fn comments(&self) -> &[String] { + &self.comments + } + + /// Serialize to the compact format: raw-deflate of `WCAFF1`, the comments, the + /// tables and rules as text lines, a `WORDS` line, then one line per stem, front-coded + /// against the previous one (a byte with the shared prefix length in bytes, the rest of the + /// word, `/` and its flags if any). + pub fn to_bytes(&self) -> Vec { + let mut out = String::new(); + out.push_str(MAGIC); + out.push('\n'); + for c in &self.comments { + out.push_str("# "); + out.push_str(c.trim()); + out.push('\n'); + } + if !self.try_chars.is_empty() { + out.push_str(&format!("TRY {}\n", self.try_chars)); + } + for m in &self.map { + out.push_str(&format!("MAP {}\n", m.iter().collect::())); + } + for (a, b) in &self.rep { + out.push_str(&format!("REP {} {}\n", a.replace(' ', "_"), b.replace(' ', "_"))); + } + if self.forbidden != 0 + && let Some(c) = self.flag_char(self.forbidden) + { + out.push_str(&format!("FORBIDDENWORD {c}\n")); + } + let z = |s: &str| if s.is_empty() { "0".to_string() } else { s.to_string() }; + for (kind, rules) in [("PFX", &self.prefixes), ("SFX", &self.suffixes)] { + for r in rules { + let y = if r.cross { "Y" } else { "N" }; + // Continuation classes ride on the appended field, as in the `.aff` source. + let mut add = z(&r.add); + let cont = self.flag_chars_for(r.cont); + if !cont.is_empty() { + add.push('/'); + add.push_str(&cont); + } + out.push_str(&format!("{kind} {} {y} {} {} {}\n", r.flag, z(&r.strip), add, r.cond_text)); + } + } + out.push_str("WORDS\n"); + let mut raw = out.into_bytes(); + let mut prev: &[u8] = &[]; + for i in 0..self.len() { + let Some(w) = self.word(i) else { continue }; + let b = w.as_bytes(); + let p = prev.iter().zip(b).take_while(|(x, y)| x == y).count().min(255); + raw.push(p as u8); + raw.extend_from_slice(b.get(p..).unwrap_or_default()); + let mask = self.flags.get(i).and_then(|&id| self.sets.get(usize::from(id))).copied().unwrap_or(0); + if mask != 0 { + raw.push(b'/'); + raw.extend_from_slice(self.flag_chars_for(mask).as_bytes()); + } + raw.push(b'\n'); + prev = b; + } + miniz_oxide::deflate::compress_to_vec(&raw, 10) + } + + /// Read data written by [`AffixDict::to_bytes`]. + pub fn from_bytes(data: &[u8]) -> Result { + let raw = + miniz_oxide::inflate::decompress_to_vec_with_limit(data, 64 << 20).map_err(|e| format!("affix dictionary: inflate failed: {e:?}"))?; + let words_at = raw.windows(7).position(|w| w == b"\nWORDS\n").ok_or("affix dictionary: no WORDS section")?; + let head = std::str::from_utf8(raw.get(..words_at).unwrap_or_default()).map_err(|_| "affix dictionary: bad utf-8 in the rules")?; + let mut lines = head.lines(); + if lines.next() != Some(MAGIC) { + return Err("affix dictionary: bad header".into()); + } + let mut rules = Rules::default(); + let mut comments = Vec::new(); + for line in lines { + if let Some(c) = line.strip_prefix("# ") { + comments.push(c.to_string()); + continue; + } + let f: Vec<&str> = line.split_whitespace().collect(); + let at = |k: usize| f.get(k).copied().unwrap_or(""); + match at(0) { + "TRY" => rules.try_chars = at(1).to_string(), + "MAP" => rules.map.push(at(1).chars().collect()), + "REP" => rules.rep.push((at(1).replace('_', " "), at(2).replace('_', " "))), + "FORBIDDENWORD" => { + let mut it = at(1).chars(); + match (it.next(), it.next()) { + (Some(c), None) if !c.is_whitespace() => rules.forbidden = Some(c), + _ => return Err(format!("affix dictionary: bad FORBIDDENWORD in `{line}`")), + } + } + kind @ ("PFX" | "SFX") => { + let mut fc = at(1).chars(); + let (Some(flag), None) = (fc.next(), fc.next()) else { return Err(format!("affix dictionary: bad flag in `{line}`")) }; + if f.len() != 6 { + return Err(format!("affix dictionary: bad rule `{line}`")); + } + // Continuation classes ride on the appended field, as in the `.aff` source. + let (add, cont) = match at(4).split_once('/') { + Some((a, c)) => (zero_empty(a), c.to_string()), + None => (zero_empty(at(4)), String::new()), + }; + let rule: RawRule = (flag, at(2) == "Y", zero_empty(at(3)), add, cont, at(5).to_string()); + if kind == "SFX" { + rules.suffixes.push(rule); + } else { + rules.prefixes.push(rule); + } + } + "" => {} + other => return Err(format!("affix dictionary: unknown line `{other}`")), + } + } + let body = raw.get(words_at + 7..).unwrap_or_default(); + let mut stems: Vec<(String, String)> = Vec::new(); + let mut cur: Vec = Vec::new(); + let mut i = 0; + while let Some(&p) = body.get(i) { + let rest = body.get(i + 1..).unwrap_or_default(); + let len = rest.iter().position(|&b| b == b'\n').ok_or("affix dictionary: truncated")?; + let line = rest.get(..len).unwrap_or_default(); + i += len + 2; + let p = p as usize; + if p > cur.len() { + return Err("affix dictionary: bad prefix".into()); + } + let (tail, flags) = match line.iter().position(|&b| b == b'/') { + Some(k) => (line.get(..k).unwrap_or_default(), line.get(k + 1..).unwrap_or_default()), + None => (line, &[][..]), + }; + cur.truncate(p); + cur.extend_from_slice(tail); + let word = std::str::from_utf8(&cur).map_err(|_| "affix dictionary: bad utf-8 in a stem")?; + let flags = std::str::from_utf8(flags).map_err(|_| "affix dictionary: bad utf-8 in flags")?; + if word.is_empty() || word.len() > MAX_WORD_BYTES { + return Err("affix dictionary: bad stem".into()); + } + stems.push((word.to_string(), flags.to_string())); + } + AffixDict::build(rules, stems, comments) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const AFF: &str = "SET UTF-8\nTRY aeoiułkr\n\nPFX b Y 1\nPFX b 0 nie .\n\nSFX a Y 3\nSFX a 0 y [^y]\nSFX a ek ka ek\nSFX a 0 em .\n\nSFX c N 1\nSFX c a ą a\n\nMAP 2\nMAP aą\nMAP oóu\nREP 2\nREP ż rz\nREP u ó\n"; + const DIC: &str = "4\nkot/ab\npiesek/a\nkrowa/c\nWarszawa/c\n"; + + fn dict() -> AffixDict { + AffixDict::from_hunspell(AFF, DIC, &["test data"]).unwrap() + } + + #[test] + fn stems_suffixes_prefixes_and_cross_products() { + let d = dict(); + assert_eq!(d.len(), 4); + for w in ["kot", "koty", "kotem", "piesek", "pieska", "piesekem", "krowa", "krową", "niekot", "niekoty", "niekotem", "Warszawa", "Warszawą"] + { + assert!(d.contains(w), "{w}"); + } + for w in ["kotą", "pieseka", "pieskem", "niepiesek", "niekrową", "krowy", "warszawa", "", "nie", "y", "ka"] { + assert!(!d.contains(w), "{w}"); + } + } + + #[test] + fn case_variants() { + let d = dict(); + assert!(d.check("Kot") && d.check("KOT") && d.check("Koty") && d.check("WARSZAWĄ") && d.check("Warszawą")); + assert!(!d.check("warszawa") && !d.check("kOT") && !d.check("KoT")); + assert_eq!(casing("Kot"), Casing::Initial); + assert_eq!(casing("KOT"), Casing::Upper); + assert_eq!(casing("K"), Casing::Initial); + assert_eq!(casing("kot"), Casing::Lower); + assert_eq!(casing("McDonald"), Casing::Mixed); + assert_eq!(capitalize("żółw"), "Żółw"); + } + + #[test] + fn forms_are_all_accepted() { + let d = dict(); + let mut forms = d.forms("kot"); + forms.sort(); + forms.dedup(); + assert_eq!(forms, ["kot", "kotem", "koty", "niekot", "niekotem", "niekoty"]); + for stem in d.stems().map(str::to_string).collect::>() { + for f in d.forms(&stem) { + assert!(d.contains(&f), "{stem} → {f}"); + } + } + } + + #[test] + fn compact_format_round_trips() { + let d = dict(); + let back = AffixDict::from_bytes(&d.to_bytes()).unwrap(); + assert_eq!(back.len(), d.len()); + assert_eq!(back.rule_count(), d.rule_count()); + assert_eq!(back.comments(), ["test data"]); + assert_eq!(back.rep(), d.rep()); + assert_eq!(back.map(), d.map()); + assert_eq!(back.try_chars(), "aeoiułkr"); + for w in ["kotem", "niekoty", "pieska", "krową", "Warszawą"] { + assert!(back.contains(w), "{w}"); + } + assert!(!back.contains("kotą")); + } + + #[test] + fn flag_utf8_and_default_modes_are_accepted() { + // `FLAG UTF-8` (VERO pt_BR) and `FLAG DEFAULT` are per-character flag modes: the same + // mode the reader already assumes. Only `long` and `num` stay unsupported. + for mode in ["UTF-8", "DEFAULT"] { + let aff = format!("SET UTF-8\nFLAG {mode}\nSFX a Y 1\nSFX a 0 y [^y]\n"); + let d = AffixDict::from_hunspell(&aff, "1\nkot/a\n", &[]).unwrap(); + assert!(d.contains("kot") && d.contains("koty"), "{mode}"); + } + assert!(AffixDict::from_hunspell("FLAG UTF-8\n", "0\n", &[]).is_ok(), "FLAG alone is harmless"); + } + + #[test] + fn more_than_64_distinct_flags_load() { + // VERO pt_BR uses 106 distinct flags (FLAG UTF-8). Build one with every basic Latin + // letter, the digits and eight accented letters: 70 flags, 64 < n <= 128. + let mut flags = String::new(); + let mut aff = String::from("SET UTF-8\nFLAG UTF-8\n"); + let chars: Vec = ('a'..='z').chain('A'..='Z').chain('0'..='9').chain(['á', 'é', 'í', 'ó', 'ú', 'ç', 'ã', 'õ']).collect(); + assert_eq!(chars.len(), 70); + for c in chars { + aff.push_str(&format!("SFX {c} N 1\nSFX {c} 0 {c} .\n")); + flags.push(c); + } + let d = AffixDict::from_hunspell(&aff, &format!("1\nverbo/{flags}\n"), &[]).unwrap(); + assert!(d.contains("verbo")); + assert!(d.contains("verboa") && d.contains("verboz") && d.contains("verbo9")); + assert!(d.contains("verboç") && d.contains("verboã"), "multi-byte UTF-8 flags"); + assert!(!d.contains("verboß"), "ß is not a flag and not a stem"); + } + + #[test] + fn forbidden_words_are_rejected() { + // VERO pt_BR marks spellings it must reject (flag ý under FLAG UTF-8) as stems carrying + // FORBIDDENWORD: listed, but never correct and never inflected. + let aff = "SET UTF-8\nFLAG UTF-8\nFORBIDDENWORD ý\nSFX a Y 1\nSFX a 0 y [^y]\n"; + let d = AffixDict::from_hunspell(aff, "2\nkot/a\nimundo/ý\n", &[]).unwrap(); + assert!(d.check("kot") && d.check("koty") && d.check("Kot") && d.check("KOT")); + assert!(!d.check("imundo") && !d.check("Imundo") && !d.check("IMUNDO")); + assert!(!d.contains("imundoy"), "a forbidden stem must not inflect"); + assert!(d.forms("imundo").is_empty()); + } + + #[test] + fn continuation_classes_chain_suffixes() { + // `SFX A 0 able/B`: the form derived by A carries flag B, so a suffix of class B may + // follow (workable → workables). The bare stem has A only, so `works` stays wrong. + let aff = "SET UTF-8\nFLAG UTF-8\nSFX A N 1\nSFX A 0 able/B .\nSFX B N 1\nSFX B 0 s .\n"; + let d = AffixDict::from_hunspell(aff, "1\nwork/A\n", &[]).unwrap(); + assert!(d.contains("work") && d.contains("workable") && d.contains("workables")); + assert!(!d.contains("works"), "the stem has A, not B"); + assert!(!d.contains("workabless"), "B declares no continuation"); + assert!(d.check("Workable") && d.check("WORKABLES")); + } + + #[test] + fn continuation_chains_stop_at_two_suffixes() { + // Hunspell strips at most two suffixes: work + able/B + s/C is a word, a third (C) is not. + let aff = "SET UTF-8\nFLAG UTF-8\nSFX A N 1\nSFX A 0 able/B .\nSFX B N 1\nSFX B 0 s/C .\nSFX C N 1\nSFX C 0 x .\n"; + let d = AffixDict::from_hunspell(aff, "1\nwork/A\n", &[]).unwrap(); + assert!(d.contains("workable") && d.contains("workables")); + assert!(!d.contains("workablesx"), "three chained suffixes exceed the limit"); + assert!(!d.forms("work").iter().any(|f| f == "workablesx")); + } + + #[test] + fn forms_include_continuation_chains() { + let aff = "SET UTF-8\nFLAG UTF-8\nSFX A N 1\nSFX A 0 able/B .\nSFX B N 1\nSFX B 0 s .\n"; + let d = AffixDict::from_hunspell(aff, "1\nwork/A\n", &[]).unwrap(); + let mut f = d.forms("work"); + f.sort(); + f.dedup(); + assert_eq!(f, ["work", "workable", "workables"]); + for form in d.forms("work") { + assert!(d.contains(&form), "every listed form is accepted: {form}"); + } + } + + #[test] + fn compact_format_round_trips_continuations_and_forbidden() { + let aff = "SET UTF-8\nFLAG UTF-8\nFORBIDDENWORD ý\nSFX A N 1\nSFX A 0 able/B .\nSFX B N 1\nSFX B 0 s .\n"; + let d = AffixDict::from_hunspell(aff, "2\nwork/A\nimundo/ý\n", &[]).unwrap(); + let back = AffixDict::from_bytes(&d.to_bytes()).unwrap(); + assert!(back.contains("work") && back.contains("workable") && back.contains("workables")); + assert!(!back.contains("works"), "the stem has A, not B, after a round trip"); + assert!(!back.check("imundo") && !back.contains("imundoy"), "forbidden survives a round trip"); + } + + #[test] + fn dic_bom_glued_to_the_count_line_is_skipped() { + // VERO's .dic opens with a UTF-8 BOM glued to its count line; it must not become a stem. + let aff = "SET UTF-8\nSFX a Y 1\nSFX a 0 y [^y]\n"; + let d = AffixDict::from_hunspell(aff, "\u{feff}1\nkot/a\n", &[]).unwrap(); + assert_eq!(d.len(), 1); + assert!(d.contains("kot") && d.contains("koty")); + } + + #[test] + fn unsupported_and_broken_input_is_an_error() { + assert!(AffixDict::from_hunspell("FLAG long\n", DIC, &[]).is_err()); + assert!(AffixDict::from_hunspell("FLAG num\n", DIC, &[]).is_err()); + assert!(AffixDict::from_hunspell("COMPOUNDFLAG X\n", DIC, &[]).is_err()); + assert!(AffixDict::from_hunspell("SFX a Y 1\nSFX a 0 y [ab\n", DIC, &[]).is_err()); + assert!(AffixDict::from_bytes(b"not deflate").is_err()); + let ok = dict().to_bytes(); + assert!(AffixDict::from_bytes(&ok[..ok.len() / 2]).is_err()); + let garbage = miniz_oxide::deflate::compress_to_vec(b"WCAFF1\nWORDS\n\x09abc\n", 6); + assert!(AffixDict::from_bytes(&garbage).is_err(), "prefix longer than the previous word"); + let empty = AffixDict::default(); + assert!(!empty.contains("kot") && !empty.check("Kot") && empty.forms("kot").is_empty()); + } + + proptest::proptest! { + #[test] + fn lookups_never_panic(s in "\\PC{0,40}", bytes in proptest::collection::vec(proptest::num::u8::ANY, 0..200)) { + let d = dict(); + let _ = d.check(&s); + let _ = d.forms(&s); + let _ = AffixDict::from_bytes(&bytes); + let _ = AffixDict::from_bytes(&miniz_oxide::deflate::compress_to_vec(&bytes, 1)); + } + } +} diff --git a/crates/proof/src/fx.rs b/crates/proof/src/fx.rs new file mode 100644 index 00000000..7ec12d0a --- /dev/null +++ b/crates/proof/src/fx.rs @@ -0,0 +1,40 @@ +//! A small, fast multiplicative hasher (FxHash-style) for the affix engine's stem table and rule +//! maps: keys are short strings and packed integers, and HashDoS resistance is not needed for +//! dictionary data the app chose to load. + +use std::collections::HashMap; +use std::hash::{BuildHasherDefault, Hasher}; + +/// Small, fast multiplicative hasher for packed integer keys and short strings (FxHash-style). +#[derive(Default, Clone, Copy)] +pub struct FxHasher(u64); + +impl Hasher for FxHasher { + fn finish(&self) -> u64 { + self.0 + } + fn write(&mut self, bytes: &[u8]) { + let (chunks, rest) = bytes.as_chunks::<8>(); + for c in chunks { + self.write_u64(u64::from_le_bytes(*c)); + } + for &b in rest { + self.write_u64(b as u64); + } + } + fn write_u8(&mut self, i: u8) { + self.write_u64(i as u64); + } + fn write_usize(&mut self, i: usize) { + self.write_u64(i as u64); + } + fn write_u64(&mut self, i: u64) { + self.0 = (self.0.rotate_left(5) ^ i).wrapping_mul(0x51_7c_c1_b7_27_22_0a_95); + } + fn write_u128(&mut self, i: u128) { + self.write_u64(i as u64); + self.write_u64((i >> 64) as u64); + } +} + +pub type FxMap = HashMap>; diff --git a/crates/proof/src/hunspell.rs b/crates/proof/src/hunspell.rs new file mode 100644 index 00000000..ab5ecc03 --- /dev/null +++ b/crates/proof/src/hunspell.rs @@ -0,0 +1,208 @@ +//! Hunspell dictionaries installed on the system, read at runtime. +//! +//! The Craft apps don't bundle large spelling dictionaries. For a language with an affix +//! dictionary, an app reads the standard Hunspell `.aff`/`.dic` pair installed by the operating +//! system, a package manager or LibreOffice, with [`load`]. When none is installed the language +//! has no spelling check (no marks, no errors), like Word without the language's proofing tools. +//! +//! Where to look is per app ([`Search`]): its own dictionaries folder (a setting), an environment +//! variable and a folder under the user's application data, then the usual system locations. On +//! the web there are none. + +use std::path::{Path, PathBuf}; + +use crate::affix::AffixDict; + +/// Largest `.aff` or `.dic` file read (real ones are a few MB). +const MAX_FILE: u64 = 64 << 20; + +/// Where one app looks for Hunspell dictionaries, most specific first. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct Search { + /// A folder searched before everything else, such as a "dictionaries folder" setting. + pub extra_dir: Option, + /// An environment variable naming a folder searched next (`WORDCRAFT_DICTIONARIES`). + pub env_var: Option, + /// The app's folder name under the Windows application-data folders (`WordCraft` searches + /// `%LOCALAPPDATA%\WordCraft\dictionaries`). + pub app_dir: Option, +} + +impl Search { + /// The search for an app called `app` (`WordCraft`): its environment variable is the + /// upper-cased name with `_DICTIONARIES` (`WORDCRAFT_DICTIONARIES`) and its Windows folder the + /// name itself. + pub fn for_app(app: &str) -> Search { + Search { extra_dir: None, env_var: Some(format!("{}_DICTIONARIES", app.to_ascii_uppercase())), app_dir: Some(app.to_string()) } + } + + /// Directories that may hold Hunspell dictionaries, most specific first. `lo_exts`, when a + /// language is being searched for, adds LibreOffice's extension folders for it (`dict-pt-BR`). + pub fn dirs(&self, lo_exts: Option<&[String]>) -> Vec { + let mut dirs: Vec = self.extra_dir.iter().cloned().collect(); + if cfg!(target_arch = "wasm32") { + return dirs; + } + if let Some(d) = self.env_var.as_deref().and_then(std::env::var_os).filter(|v| !v.is_empty()) { + dirs.push(PathBuf::from(d)); + } + // LibreOffice keeps each language under `…/share/extensions/`. + let lo_dirs = |base: &Path| -> Vec { lo_exts.map_or_else(Vec::new, |exts| exts.iter().map(|e| base.join(e)).collect()) }; + let home = std::env::var_os("HOME").filter(|v| !v.is_empty()).map(PathBuf::from); + if cfg!(windows) { + for var in ["LOCALAPPDATA", "APPDATA"] { + if let Some(base) = std::env::var_os(var).filter(|v| !v.is_empty()).map(PathBuf::from) { + if let Some(app) = &self.app_dir { + dirs.push(base.join(app).join("dictionaries")); + } + dirs.push(base.join("hunspell")); + } + } + for var in ["ProgramFiles", "ProgramFiles(x86)"] { + if let Some(base) = std::env::var_os(var).filter(|v| !v.is_empty()).map(PathBuf::from) { + dirs.extend(lo_dirs(&base.join("LibreOffice").join("share").join("extensions"))); + } + } + } else if cfg!(target_os = "macos") { + if let Some(h) = &home { + dirs.push(h.join("Library/Spelling")); + } + dirs.push(PathBuf::from("/Library/Spelling")); + dirs.push(PathBuf::from("/opt/homebrew/share/hunspell")); + dirs.push(PathBuf::from("/usr/local/share/hunspell")); + dirs.extend(lo_dirs(Path::new("/Applications/LibreOffice.app/Contents/Resources/extensions"))); + } else { + if let Some(h) = &home { + dirs.push(h.join(".local/share/hunspell")); + } + for d in + ["/usr/share/hunspell", "/usr/share/myspell", "/usr/share/myspell/dicts", "/usr/local/share/hunspell", "/usr/local/share/myspell"] + { + dirs.push(PathBuf::from(d)); + } + for d in ["/usr/lib/libreoffice/share/extensions", "/opt/libreoffice/share/extensions"] { + dirs.extend(lo_dirs(Path::new(d))); + } + } + dirs + } + + /// The first `.aff`/`.dic` pair found for any of `names` (`pl_PL`, `pl`), + /// searching [`Search::dirs`] (including LibreOffice's extension folders for the first name). + pub fn find(&self, names: &[&str]) -> Option<(PathBuf, PathBuf)> { + let lo = names.first().copied().map(lo_ext_dirs); + self.dirs(lo.as_deref()).into_iter().find_map(|dir| find_in(&dir, names)) + } +} + +/// The LibreOffice extension directories a Hunspell dictionary name may live in: LibreOffice's +/// IDs are per dictionary — Polish is `dict-pl`, Brazilian Portuguese `dict-pt-BR` — so both +/// the primary-language folder and the dashed full name are tried. +fn lo_ext_dirs(name: &str) -> Vec { + let primary = name.split(['-', '_']).next().unwrap_or(name); + let dashed = name.replace('_', "-"); + let first = format!("dict-{primary}"); + let full = format!("dict-{dashed}"); + if full == first { vec![first] } else { vec![first, full] } +} + +/// The first `.aff`/`.dic` pair for any of `names` in `dir`. +pub fn find_in(dir: &Path, names: &[&str]) -> Option<(PathBuf, PathBuf)> { + names.iter().find_map(|n| { + let aff = dir.join(format!("{n}.aff")); + let dic = dir.join(format!("{n}.dic")); + (aff.is_file() && dic.is_file()).then_some((aff, dic)) + }) +} + +/// Read a Hunspell pair from disk (size-capped), decoded per the `.aff`'s `SET` line. +pub fn load(aff: &Path, dic: &Path) -> Result { + let read = |p: &Path| -> Result, String> { + let len = std::fs::metadata(p).map_err(|e| format!("{}: {e}", p.display()))?.len(); + if len > MAX_FILE { + return Err(format!("{}: too large ({len} bytes)", p.display())); + } + std::fs::read(p).map_err(|e| format!("{}: {e}", p.display())) + }; + let (aff_text, dic_text) = decode(&read(aff)?, &read(dic)?)?; + let source = format!("Hunspell dictionary {}", dic.display()); + AffixDict::from_hunspell(&aff_text, &dic_text, &[source.as_str()]) +} + +/// ISO 8859-2 code points 0xA0..=0xFF (0x00..=0x9F map to themselves). +const ISO_8859_2_HIGH: &str = "\u{a0}Ą˘Ł¤ĽŚ§¨ŠŞŤŹ\u{ad}ŽŻ°ą˛ł´ľśˇ¸šşťź˝žżŔÁÂĂÄĹĆÇČÉĘËĚÍÎĎĐŃŇÓÔŐÖ×ŘŮÚŰÜÝŢßŕáâăäĺćçčéęëěíîďđńňóôőö÷řůúűüýţ˙"; + +fn decode_latin2(bytes: &[u8]) -> String { + let high: Vec = ISO_8859_2_HIGH.chars().collect(); + bytes.iter().map(|&b| if b < 0xa0 { char::from(b) } else { high.get(usize::from(b - 0xa0)).copied().unwrap_or('\u{fffd}') }).collect() +} + +/// Decode an `.aff`/`.dic` pair to UTF-8 from the encoding the `.aff`'s `SET` line names +/// (UTF-8 when there is none; ISO 8859-2 is the other one Polish dictionaries use). +pub fn decode(aff: &[u8], dic: &[u8]) -> Result<(String, String), String> { + let head = String::from_utf8_lossy(aff.get(..aff.len().min(8192)).unwrap_or_default()).to_string(); + let set = head.lines().find_map(|l| l.trim_start_matches('\u{feff}').strip_prefix("SET ")).map(str::trim).unwrap_or("UTF-8").to_ascii_uppercase(); + match set.as_str() { + "UTF-8" | "UTF8" => { + let text = |b: &[u8], what: &str| String::from_utf8(b.to_vec()).map_err(|e| format!("{what}: not UTF-8: {e}")); + Ok((text(aff, ".aff")?, text(dic, ".dic")?)) + } + "ISO8859-2" | "ISO-8859-2" => Ok((decode_latin2(aff), decode_latin2(dic))), + other => Err(format!("unsupported encoding {other}")), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn decodes_latin2_and_utf8() { + let aff = b"SET ISO8859-2\nTRY a\n"; + let (a, d) = decode(aff, b"2\n\xbf\xf3\xb3w\n").unwrap(); + assert!(a.contains("TRY a")); + assert_eq!(d, "2\nżółw\n"); + let (_, d) = decode("SET UTF-8\n".as_bytes(), "żółw".as_bytes()).unwrap(); + assert_eq!(d, "żółw"); + assert!(decode(b"SET KOI8-R\n", b"").is_err()); + assert!(decode(b"SET UTF-8\n", b"\xff\xfe").is_err()); + } + + #[test] + fn libreoffice_extension_dirs_follow_the_language() { + assert_eq!(lo_ext_dirs("pl_PL"), ["dict-pl", "dict-pl-PL"]); + assert_eq!(lo_ext_dirs("pt_BR"), ["dict-pt", "dict-pt-BR"]); + assert_eq!(lo_ext_dirs("de-DE"), ["dict-de", "dict-de-DE"]); + let search = Search::default(); + let dirs = search.dirs(Some(&lo_ext_dirs("pt_BR"))); + assert!(dirs.iter().any(|d| d.ends_with("dict-pt-BR")), "{dirs:?}"); + assert!(search.dirs(None).iter().all(|d| !d.to_string_lossy().contains("dict-")), "no language: no LibreOffice guess"); + } + + #[test] + fn each_app_names_its_own_folder_and_variable() { + let s = Search::for_app("GridCraft"); + assert_eq!(s.env_var.as_deref(), Some("GRIDCRAFT_DICTIONARIES")); + assert_eq!(s.app_dir.as_deref(), Some("GridCraft")); + let extra = std::env::temp_dir().join("craft-proof-extra"); + let s = Search { extra_dir: Some(extra.clone()), ..Search::for_app("PdfCraft") }; + assert_eq!(s.dirs(None).first(), Some(&extra), "the setting comes first"); + } + + #[test] + fn finds_and_loads_a_pair_in_a_directory() { + let dir = std::env::temp_dir().join(format!("craft-proof-hunspell-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + assert!(find_in(&dir, &["xx_XX"]).is_none(), "no files: nothing found"); + std::fs::write(dir.join("xx_XX.aff"), "SET UTF-8\nSFX A Y 1\nSFX A 0 y .\n").unwrap(); + assert!(find_in(&dir, &["xx_XX"]).is_none(), "an .aff alone isn't a dictionary"); + std::fs::write(dir.join("xx_XX.dic"), "1\nkot/A\n").unwrap(); + let (aff, dic) = find_in(&dir, &["yy", "xx_XX"]).unwrap(); + let d = load(&aff, &dic).unwrap(); + assert!(d.check("kot") && d.check("koty") && !d.check("kotx")); + assert!(load(&dir.join("missing.aff"), &dic).is_err()); + let found = Search { extra_dir: Some(dir.clone()), ..Search::default() }.find(&["xx_XX"]); + assert_eq!(found, Some((aff, dic)), "the extra folder is searched"); + let _ = std::fs::remove_dir_all(&dir); + } +} diff --git a/crates/proof/src/lib.rs b/crates/proof/src/lib.rs new file mode 100644 index 00000000..379bae2a --- /dev/null +++ b/crates/proof/src/lib.rs @@ -0,0 +1,23 @@ +//! Spelling mechanism shared by the Craft apps: Hunspell affix dictionaries and where to find +//! them, with no opinion on which languages an app proofs or how it shows the result. +//! +//! - [`affix`]: an affix dictionary ([`affix::AffixDict`]) built from a Hunspell `.aff`/`.dic` +//! pair, word checks, generated forms, the suggestion tables (`TRY`, `MAP`, `REP`) and a +//! compact serialized form ([`affix::AffixDict::to_bytes`]) for fast loading. +//! - [`hunspell`]: finding an installed pair per app ([`hunspell::Search`]), decoding it from +//! UTF-8 or ISO 8859-2 and loading it. +//! - [`tag`]: canonical BCP 47 tags and primary subtags, for mapping a text's language to a +//! dictionary name. +//! +//! Per-app policy stays in the apps: the list of proofed languages, suggestion ranking and +//! grammar rules per language, user dictionaries and how marks are drawn. The code comes from +//! WordCraft's `wordcraft-proof` (storytold/craft-libs#30). + +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] + +pub mod affix; +mod fx; +pub mod hunspell; +pub mod tag; + +pub use affix::AffixDict; diff --git a/crates/proof/src/tag.rs b/crates/proof/src/tag.rs new file mode 100644 index 00000000..2206aff7 --- /dev/null +++ b/crates/proof/src/tag.rs @@ -0,0 +1,80 @@ +//! Language tags as apps receive them (`w:lang`, `a:rPr/@lang`, `LANG`, `navigator.language`): +//! canonical BCP 47 form and the primary language subtag. Which languages an app proofs, and +//! with which rules, stays in the app. + +/// A BCP 47 tag in canonical case (`pl-PL`, `en-GB`, `zh-Hant-TW`, `x-none`), or `None` when +/// `tag` isn't shaped like one: 1–35 ASCII letters, digits and `-`/`_` separators, starting with +/// a 2–8 letter language (or `x`/`i` for private and grandfathered tags). `_` becomes `-`. +pub fn normalize_tag(tag: &str) -> Option { + let tag = tag.trim(); + if tag.is_empty() || tag.len() > 35 || !tag.chars().all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_') { + return None; + } + let parts: Vec<&str> = tag.split(['-', '_']).collect(); + if parts.iter().any(|p| p.is_empty() || p.len() > 8) { + return None; + } + let first = parts.first()?; + let language = (2..=8).contains(&first.len()) && first.chars().all(|c| c.is_ascii_alphabetic()); + if !language && !first.eq_ignore_ascii_case("x") && !first.eq_ignore_ascii_case("i") { + return None; + } + let letters = |p: &str| p.chars().all(|c| c.is_ascii_alphabetic()); + // After a singleton (`x-…` private use, `u-…` extensions) subtags are lower case. + let singleton = parts.iter().position(|p| p.len() == 1).unwrap_or(usize::MAX); + let out: Vec = parts + .iter() + .enumerate() + .map(|(i, p)| { + if i >= singleton { + p.to_ascii_lowercase() + } else if i > 0 && p.len() == 2 && letters(p) { + p.to_ascii_uppercase() // region + } else if i > 0 && p.len() == 4 && letters(p) { + let lower = p.to_ascii_lowercase(); + let mut c = lower.chars(); + c.next().map(|f| f.to_ascii_uppercase().to_string() + c.as_str()).unwrap_or_default() // script + } else { + p.to_ascii_lowercase() + } + }) + .collect(); + Some(out.join("-")) +} + +/// The primary language subtag of a tag or POSIX locale, lower case (`pl` for `pl-PL`, +/// `pl_PL.UTF-8` and `PL`); `None` when there is none. +pub fn primary_subtag(tag: &str) -> Option { + let primary = tag.trim().split(['-', '_', '.', '@']).next().unwrap_or(""); + (!primary.is_empty() && primary.chars().all(|c| c.is_ascii_alphabetic())).then(|| primary.to_ascii_lowercase()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn tags_normalize() { + assert_eq!(normalize_tag("PL-pl").as_deref(), Some("pl-PL")); + assert_eq!(normalize_tag(" pl_pl ").as_deref(), Some("pl-PL")); + assert_eq!(normalize_tag("pl").as_deref(), Some("pl")); + assert_eq!(normalize_tag("zh-hant-tw").as_deref(), Some("zh-Hant-TW")); + assert_eq!(normalize_tag("es-419").as_deref(), Some("es-419")); + assert_eq!(normalize_tag("x-none").as_deref(), Some("x-none")); + assert_eq!(normalize_tag("de-DE-u-co-PHONEBK").as_deref(), Some("de-DE-u-co-phonebk")); + for bad in ["", " ", "p", "1pl", "pl--PL", "pl PL", "pl-PL;", "polski język", "-pl", "pl-", "abcdefghi", &"a".repeat(40)] { + assert_eq!(normalize_tag(bad), None, "{bad:?}"); + } + } + + #[test] + fn primary_subtags() { + for t in ["pl", "pl-PL", "PL_pl", "pl_PL.UTF-8", " pl-PL ", "pl_PL@euro"] { + assert_eq!(primary_subtag(t).as_deref(), Some("pl"), "{t}"); + } + assert_eq!(primary_subtag("zh-Hant-TW").as_deref(), Some("zh")); + for t in ["", " ", "-pl", "1pl"] { + assert_eq!(primary_subtag(t), None, "{t:?}"); + } + } +}