diff --git a/crates/proof/README.md b/crates/proof/README.md index 49a16473..97067014 100644 --- a/crates/proof/README.md +++ b/crates/proof/README.md @@ -13,7 +13,7 @@ craft-proof = { git = "https://github.com/storytold/craft-libs", rev = " | Language | Spelling | Grammar | Hyphenation | |---|---|---|---| -| English (en-US, en-CA, en-GB and the British-spelling Englishes such as en-AU, en-NZ) | SCOWL 2026 Hunspell dictionary per English + `en-extra.txt`, dialect policy (`en-variants.txt`); a dictionaries-folder English first; without SCOWL the installed Hunspell English, else the built-in Moby list ([below](#english-is-always-checked)) | yes | Moby list, our patterns, heuristic | +| English (en-US, en-CA, en-AU, en-GB and the British-spelling Englishes such as en-NZ) | SCOWL 2026 Hunspell dictionary per English + `en-extra.txt`, dialect policy (`en-variants.txt`); a dictionaries-folder English first; without SCOWL the installed Hunspell English, else the built-in Moby list ([below](#english-is-always-checked)) | yes | Moby list, our patterns, heuristic | | Polish, Portuguese, Italian | the installed Hunspell dictionary | yes | TeX patterns (embedded, or a Language pack) | | Croatian, Turkish, Russian, Czech, Indonesian, Spanish (es, es-MX, es-419, es-UY) | the installed Hunspell dictionary (Russian also `ru-extra.txt`) | yes | Turkish and Russian rules; English fallback otherwise | | German | — | — | TeX patterns (embedded, or a Language pack) | @@ -79,7 +79,7 @@ pub fn packs_dir() -> Option; // ＋ pub fn installed() -> Vec; // InstalledPack { locale, name, licence, path }, by locale pub fn licence_texts() -> Vec<(String, String)>; // (title, text): embedded English notices, then each pack's LICENCES/* pub fn load_errors() -> Vec<(String, String)>; // ＋ (locale or folder, reason): packs skipped, pattern files that failed -pub fn embedded_locales() -> &'static [&'static str]; // ＋ what this build embeds ("en-us", "de", "pl", "pt", "it", "ru" by default; "en-us" with CRAFT_PROOF_EMBED=en) +pub fn embedded_locales() -> &'static [&'static str]; // ＋ locales with embedded upstream data, not our supplements ("en-us", "en-gb", "en-ca", "en-au", "pl", "pt", "it", "de" by default; the four English ones with CRAFT_PROOF_EMBED=en) // craft_proof (crate root) pub enum ProofLang { …, Pack(PackLocale), Other } // ＋ Pack: any language without tools of its own (nl, fr, sr-Latn…) @@ -151,14 +151,16 @@ English spelling uses the first of these that is there; `english_source(lang)` s or in WordCraft's own `dictionaries` folder. The user's choice comes first. 2. **Installed English pack:** reserved for Language packs (craft-libs#485, CP4.2). 3. **Embedded SCOWL:** the English's own SCOWL `rel-2026.02.25` Hunspell dictionary from - `CRAFT_I18N_DIR` (`language-data/en-us`, `en-gb` (British `-ise`) or `en-ca`, `spelling.aff` + - `spelling.dic`), read by the affix engine. en-AU, en-NZ, en-IE and the other British-spelling - Englishes read en-GB; Canadian English reads en-CA. A word is right when the English's own + `CRAFT_I18N_DIR` (`language-data/en-us`, `en-gb` (British `-ise`), `en-au` or `en-ca`, + `spelling.aff` + `spelling.dic`), read by the affix engine. Australian English reads en-AU + (Australian words such as `billabong`, with en-GB's spelling policy); en-NZ, en-IE and the + other British-spelling Englishes read en-GB; Canadian English reads en-CA. A word is right when the English's own dictionary knows it (or `en-extra.txt` does), never because another English's does - (`colourblind` is wrong in en-US, `colorblind` in en-GB); suggestions come from the same - dictionary. SCOWL's size-60 lists keep one spelling per dialect, and Word accepts more, so + (`colourblind` is wrong in en-US, `colorblind` in en-GB); the one exception is Australian + English, which also takes en-GB's words (`enquiry`: en-AU's list keeps `inquiry` only). + Suggestions come from the English's own dictionary (en-AU's for Australian). SCOWL's size-60 lists keep one spelling per dialect, and Word accepts more, so British English also takes the `-ize` spellings of its `-ise` words that American or Canadian - English spell so (`organize`, `realization`; never `wize` or `advize`), and `en-variants.txt` flags the other dialect's spellings and accepts this + English spell so (`organize`, `realization`; never `wize` or `advize`, and never `-yze`: `reanalyze` stays American), and `en-variants.txt` flags the other dialect's spellings and accepts this dialect's other spellings of known words (`judgement` in en-US). English with no dialect (`is_correct`) takes a word any of the three knows. Suggestions offer a missing space only between two lower-case words of three letters or more, after every one-word candidate, and no @@ -199,7 +201,7 @@ craft-i18n's copy. The SCOWL notice (SCOWL's licence, the Ispell BSD licence of the affix files and Benjamin Titze's notice) is embedded with the dictionaries: `craft_proof::scowl::notice()`, for an About window. -Embedding SCOWL adds 1,542,396 bytes of Language data (the en-US, en-GB and en-CA `.aff`/`.dic` +Embedding SCOWL adds 2,056,533 bytes of Language data (the en-US, en-GB, en-CA and en-AU `.aff`/`.dic` and the notice); the Moby list (432,905 bytes) stays for hyphenation. ## Check diff --git a/crates/proof/build.rs b/crates/proof/build.rs index 8ff77fc4..fd34c22d 100644 --- a/crates/proof/build.rs +++ b/crates/proof/build.rs @@ -16,9 +16,8 @@ //! `include_str!` takes a string, so a `CRAFT_I18N_DIR` that isn't valid UTF-8 can't be embedded: //! it is treated like a folder without the data. //! -//! English: SCOWL's en-US, en-GB (-ise) and en-CA dictionaries; en-AU is not embedded, as -//! Australian English is checked as British-spelling English (`ProofLang::EnGb`). The SCOWL -//! notice is embedded once (the four upstream READMEs differ only in their title and build line). +//! English: SCOWL's en-US, en-GB (-ise), en-CA and en-AU dictionaries. The SCOWL notice is +//! embedded once (the four upstream READMEs differ only in their title and build line). use std::fmt::Write as _; use std::path::{Path, PathBuf}; @@ -34,6 +33,8 @@ const FILES: &[(&str, &str, bool)] = &[ ("EN_GB_SPELLING", "en-gb/spelling.dic", false), ("EN_CA_AFF", "en-ca/spelling.aff", false), ("EN_CA_SPELLING", "en-ca/spelling.dic", false), + ("EN_AU_AFF", "en-au/spelling.aff", false), + ("EN_AU_SPELLING", "en-au/spelling.dic", false), ("SCOWL_NOTICE", "en-us/LICENCE-SCOWL.txt", false), ("EN_US_PAT", "en-us/hyphenation.pat", false), ("EXTRA_EN", "en-us/en-extra.txt", false), @@ -152,8 +153,18 @@ fn main() { files.sort(); for f in files { println!("cargo::rerun-if-changed={}", f.display()); + // `include_str!` fails the whole build on a file that isn't UTF-8 (a Latin-1 + // licence text) and can't name a path that isn't either: leave such a file out. + let Some(path) = f.to_str() else { + println!("cargo::{level}=CRAFT_I18N_DIR: the path {} is not UTF-8; its licence text is not embedded", f.display()); + continue; + }; + if let Err(e) = std::fs::read_to_string(&f) { + println!("cargo::{level}=CRAFT_I18N_DIR: {path} can't be read as text ({e}); its licence text is not embedded"); + continue; + } let title = format!("{locale}: {}", f.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default()); - let _ = write!(licences, "({title:?}, include_str!({:?})), ", f.display().to_string()); + let _ = write!(licences, "({title:?}, include_str!({path:?})), "); } } } diff --git a/crates/proof/examples/affix_check.rs b/crates/proof/examples/affix_check.rs new file mode 100644 index 00000000..4319d506 --- /dev/null +++ b/crates/proof/examples/affix_check.rs @@ -0,0 +1,182 @@ +//! Compare the affix engine with another spell checker, black-box: load a Hunspell `.aff`/`.dic` +//! pair and answer for words, or make candidate compounds to ask both about. +//! +//! ```sh +//! # One word per line in, `word1` (right) or `word0` out, and the time taken on stderr. +//! cargo run --release -p craft-proof --example affix_check -- check de_DE.aff de_DE.dic < words.txt +//! # 2000 candidates (forms of random stems glued together, some with a letter changed), seed 7. +//! cargo run --release -p craft-proof --example affix_check -- gen de_DE.aff de_DE.dic 2000 7 > words.txt +//! # Single words instead: forms of random stems, and stems with another stem's ending. +//! cargo run --release -p craft-proof --example affix_check -- single de_DE.aff de_DE.dic 2000 7 > words.txt +//! # Suggestions: `wordmicrosecondssuggestion,suggestion…` per line. +//! cargo run --release -p craft-proof --example affix_check -- suggest de_DE.aff de_DE.dic < typos.txt +//! ``` +//! +//! The answers are compared with `hunspell -d -l` on the same words +//! (the words it lists are the wrong ones). Nothing of the other checker is read but its answers. + +use std::io::{BufRead, Write}; + +use craft_proof::affix::AffixDict; + +fn load(aff: &str, dic: &str) -> Result { + let aff = std::fs::read(aff).map_err(|e| format!("{aff}: {e}"))?; + let dic = std::fs::read(dic).map_err(|e| format!("{dic}: {e}"))?; + let (aff, dic) = craft_proof::hunspell::decode(&aff, &dic)?; + AffixDict::from_hunspell(&aff, &dic, &[]) +} + +/// xorshift64*: a reproducible stream of numbers, enough to pick stems. +struct Rng(u64); + +impl Rng { + fn next(&mut self) -> u64 { + self.0 ^= self.0 >> 12; + self.0 ^= self.0 << 25; + self.0 ^= self.0 >> 27; + self.0.wrapping_mul(0x2545_f491_4f6c_dd1d) + } + + fn below(&mut self, n: usize) -> usize { + if n == 0 { 0 } else { (self.next() % n as u64) as usize } + } +} + +/// `w` with its first letter in lower case (a compound's later parts). +fn decapitalize(w: &str) -> String { + let mut c = w.chars(); + c.next().map(|f| f.to_lowercase().chain(c).collect()).unwrap_or_default() +} + +/// Attempts per requested word before a generator gives up (a dictionary that yields none). +const MAX_TRIES_PER_WORD: usize = 1000; + +fn generate(d: &AffixDict, n: usize, seed: u64) -> Vec { + let stems: Vec<&str> = d.stems().filter(|s| s.chars().all(char::is_alphabetic)).collect(); + let mut rng = Rng(seed.max(1)); + let form = |rng: &mut Rng| -> String { + let s = stems.get(rng.below(stems.len())).copied().unwrap_or_default(); + let forms = d.forms(s); + forms.get(rng.below(forms.len())).cloned().unwrap_or_else(|| s.to_string()) + }; + let mut out = Vec::with_capacity(n); + // A dictionary without alphabetic stems yields no word: give up instead of looping forever. + let mut tries = n.saturating_mul(MAX_TRIES_PER_WORD); + while out.len() < n && !stems.is_empty() && tries > 0 { + tries -= 1; + let parts = 2 + rng.below(2); + let mut w = form(&mut rng); + for _ in 1..parts { + if rng.below(4) == 0 { + w.push('s'); + } + w.push_str(&decapitalize(&form(&mut rng))); + } + if rng.below(3) == 0 { + // A letter dropped, doubled or changed: mostly wrong words. + let chars: Vec = w.chars().collect(); + let i = rng.below(chars.len()); + let mut m: String = chars.iter().take(i).collect(); + match rng.below(3) { + 0 => {} + 1 => m.extend(chars.get(i).into_iter().chain(chars.get(i))), + _ => m.push('e'), + } + m.extend(chars.iter().skip(i + 1)); + w = m; + } + // Letters only: the other checker's command line splits words at anything else. + if w.chars().count() >= 4 && w.len() <= 100 && w.chars().all(char::is_alphabetic) { + out.push(w); + } + } + out +} + +/// Single words (no compounds): forms of random stems, and stems with another stem's ending +/// (mostly wrong, some right: an ending the stem's own homonym allows or not). +fn single(d: &AffixDict, n: usize, seed: u64) -> Vec { + let stems: Vec<&str> = d.stems().filter(|s| s.chars().all(char::is_alphabetic)).collect(); + let mut rng = Rng(seed.max(1)); + let mut endings: Vec = Vec::new(); + for _ in 0..200 { + let s = stems.get(rng.below(stems.len())).copied().unwrap_or_default(); + endings.extend(d.forms(s).iter().filter_map(|f| f.strip_prefix(s)).filter(|e| !e.is_empty()).map(str::to_string)); + } + endings.sort(); + endings.dedup(); + let mut out = Vec::with_capacity(n); + let mut tries = n.saturating_mul(MAX_TRIES_PER_WORD); + while out.len() < n && !stems.is_empty() && tries > 0 { + tries -= 1; + let s = stems.get(rng.below(stems.len())).copied().unwrap_or_default(); + let forms = d.forms(s); + if let Some(f) = forms.get(rng.below(forms.len())) { + out.push(f.clone()); + } + if let Some(e) = endings.get(rng.below(endings.len())) { + out.push(format!("{s}{e}")); + } + out.retain(|w| w.chars().all(char::is_alphabetic)); + } + // Letters only: the other checker's command line splits words at anything else. + out.retain(|w| w.chars().all(char::is_alphabetic)); + out.truncate(n); + out +} + +fn main() -> Result<(), String> { + let args: Vec = std::env::args().skip(1).collect(); + match args.as_slice() { + [cmd, aff, dic] if cmd == "check" => { + let d = load(aff, dic)?; + let words: Vec = std::io::stdin().lock().lines().map_while(Result::ok).collect(); + let t = std::time::Instant::now(); + let mut slowest: Vec<(std::time::Duration, &str)> = Vec::new(); + let answers: Vec = words + .iter() + .map(|w| { + let t = std::time::Instant::now(); + let ok = d.check(w); + slowest.push((t.elapsed(), w)); + ok + }) + .collect(); + let took = t.elapsed(); + slowest.sort_by_key(|s| std::cmp::Reverse(s.0)); + for (t, w) in slowest.iter().take(3) { + eprintln!("slow: {w} {t:?}"); + } + let mut out = std::io::stdout().lock(); + for (w, ok) in words.iter().zip(answers) { + writeln!(out, "{w}\t{}", u8::from(ok)).map_err(|e| e.to_string())?; + } + eprintln!("{} words in {took:?}", words.len()); + Ok(()) + } + [cmd, aff, dic] if cmd == "suggest" => { + let d = load(aff, dic)?; + let mut out = std::io::stdout().lock(); + for w in std::io::stdin().lock().lines().map_while(Result::ok) { + let t = std::time::Instant::now(); + let s = d.suggest(&w, 8); + writeln!(out, "{w}\t{}\t{}", t.elapsed().as_micros(), s.join(",")).map_err(|e| e.to_string())?; + } + Ok(()) + } + [cmd, aff, dic, n, seed] if cmd == "gen" || cmd == "single" => { + let d = load(aff, dic)?; + let n: usize = n.parse().map_err(|_| "n is a number")?; + let seed: u64 = seed.parse().map_err(|_| "seed is a number")?; + let words = if cmd == "gen" { generate(&d, n, seed) } else { single(&d, n, seed) }; + if words.len() < n { + return Err(format!("only {} of {n} words could be generated from this dictionary", words.len())); + } + for w in words { + println!("{w}"); + } + Ok(()) + } + _ => Err("usage: affix_check check|suggest < words | affix_check gen|single ".into()), + } +} diff --git a/crates/proof/src/affix.rs b/crates/proof/src/affix.rs index 63d6f289..0df68665 100644 --- a/crates/proof/src/affix.rs +++ b/crates/proof/src/affix.rs @@ -23,10 +23,11 @@ //! Compounds by `COMPOUNDRULE` (patterns of flags with `*` and `?`, as SCOWL's English //! dictionaries use for ordinals: `21st`, `112th`; with long or numeric flags each flag is written //! in parentheses, `(aa)*(bb)`), with `ONLYINCOMPOUND` and `COMPOUNDMIN`; the parts of such a -//! compound are stems, without affixes, as in Hunspell. -//! Morphological data (`AM` aliases, the fields after a stem) is skipped. Rejected: flag-based -//! compounding (`COMPOUNDFLAG`, `COMPOUNDBEGIN`…) and the other directives that change which -//! words are correct. +//! compound are stems, without affixes, as in Hunspell. Flag-based compounds (`COMPOUNDFLAG`, +//! `COMPOUNDBEGIN`/`MIDDLE`/`END`, `COMPOUNDPERMITFLAG`, `COMPOUNDFORBIDFLAG`, `COMPOUNDWORDMAX`, +//! the `CHECKCOMPOUND…` checks, `FORCEUCASE`…): see [`compound`]. +//! Morphological data (`AM` aliases, the fields after a stem) is skipped. Rejected: the other +//! directives that change which words are correct (`COMPLEXPREFIXES`, `COMPOUNDMORESUFFIXES`…). //! //! Each distinct flag gets a dense `u16` id; a stem's flags and a rule's continuation classes are //! a sorted set of ids, interned once in a flat table, so membership is a binary search in a few @@ -40,7 +41,11 @@ use std::hash::Hasher; use crate::patterns::{FxHasher, FxMap}; -const MAGIC: &str = "WCAFF1"; +mod compound; + +pub(crate) use compound::{MAX_SUGGEST_COMPOUND_WORK, Pieces}; + +const MAGIC: &str = "WCAFF2"; /// Most bytes the compact format inflates to, written or read. const MAX_INFLATED: usize = 64 << 20; /// Longest word (in bytes) looked up; longer input is never a dictionary word. @@ -74,22 +79,20 @@ const MAX_ALIAS_BYTES: usize = 4 << 20; /// Most flags all distinct flag sets (stems' and continuation classes') may hold together: a lookup /// tests each distinct continuation set it meets at most once, so this bounds its cost. const MAX_SET_DATA: usize = 1 << 23; +/// Most flags all stems may carry together before their sets are merged and interned (the same +/// set on many stems counts each time; an `AF` alias counts once). Checked while reading, so a +/// huge `.dic` is refused before its flags fill memory, not after (#495: a 760 MB `.dic` once +/// held 3 GB before [`MAX_SET_DATA`] refused it). +const MAX_STEM_FLAGS: usize = 4 * MAX_SET_DATA; +/// Most bytes of flag text per flag of [`MAX_STEM_FLAGS`] the stem lines may hold in all while +/// they are read (`FLAG num`: `65535,` is six). +const FLAG_TEXT_PER_FLAG: usize = 6; /// Most distinct flags a dictionary may use: each gets a `u16` id. const MAX_DISTINCT_FLAGS: usize = 1 << 16; /// Directives that change which words are correct in ways this engine doesn't implement. -const UNSUPPORTED: &[&str] = &[ - "COMPLEXPREFIXES", - "COMPOUNDBEGIN", - "COMPOUNDEND", - "COMPOUNDFIRST", - "COMPOUNDFLAG", - "COMPOUNDLAST", - "COMPOUNDMIDDLE", - "COMPOUNDPERMITFLAG", - "PSEUDOROOT", -]; +const UNSUPPORTED: &[&str] = &["COMPLEXPREFIXES", "PSEUDOROOT"]; /// Most `COMPOUNDRULE` patterns kept, and most tokens in one: a hostile `.aff` can't make a /// compound check cost more than a few thousand stem lookups per word. @@ -113,7 +116,29 @@ enum Repeat { fn parse_compound_rule(pattern: &str, mode: FlagMode) -> Result, String> { let mut out: Vec<(FlagKey, Repeat)> = Vec::new(); let mut chars = pattern.chars().filter(|&c| c != EMOJI_SELECTOR); + // With long or numeric flags, what isn't a flag in parentheses (the Mongolian dictionary + // writes such rules) is read as the CLI answers (black-box): a bracketed list (`[a0,a1]`) is + // skipped with its `*` or `?`, and a punctuation mark (`%?`, `.`) makes a rule that matches + // nothing (a token no stem carries). + let mut skipped = false; while let Some(c) = chars.next() { + if mode != FlagMode::Char { + let skip = match c { + '[' => { + if !chars.by_ref().any(|c| c == ']') { + return Err(format!("COMPOUNDRULE `{pattern}`: `[` is not closed")); + } + true + } + '*' | '?' => skipped, + c if !c.is_alphanumeric() && !matches!(c, '(' | ')') => return Ok(vec![(FlagKey::MAX, Repeat::Once)]), + _ => false, + }; + skipped = skip; + if skip { + continue; + } + } match c { '*' | '?' => { let Some(last) = out.last_mut().filter(|t| t.1 == Repeat::Once) else { @@ -133,6 +158,9 @@ fn parse_compound_rule(pattern: &str, mode: FlagMode) -> Result out.push((FlagKey::from(c), Repeat::Once)), ')' => return Err(format!("COMPOUNDRULE `{pattern}`: `)` closes no group")), c if mode == FlagMode::Char => out.push((FlagKey::from(c), Repeat::Once)), _ => return Err(format!("COMPOUNDRULE `{pattern}`: long and numeric flags are written in ( )")), @@ -152,7 +180,11 @@ fn parse_compound_rule(pattern: &str, mode: FlagMode) -> Result }, + /// Sorted and deduplicated. + Set { + negated: bool, + chars: Box<[char]>, + }, } impl Cond { @@ -160,7 +192,7 @@ impl Cond { match self { Cond::Any => true, Cond::Char(x) => *x == c, - Cond::Set { negated, chars } => chars.contains(&c) != *negated, + Cond::Set { negated, chars } => chars.binary_search(&c).is_ok() != *negated, } } } @@ -195,6 +227,10 @@ fn parse_cond(text: &str) -> Result, String> { if !closed { return Err(format!("unclosed `[` in condition `{text}`")); } + // Sorted, so a character is looked up, not compared with each (a hostile + // condition can list thousands). + set.sort_unstable(); + set.dedup(); out.push(Cond::Set { negated, chars: set.into_boxed_slice() }); } ']' => return Err(format!("stray `]` in condition `{text}`")), @@ -204,6 +240,21 @@ fn parse_cond(text: &str) -> Result, String> { Ok(out.into_boxed_slice()) } +/// Steps one simple-word check takes at most, all prefixes and suffix chains together. A step is +/// about one rule tried: a rule, a character class of its condition, [`FLAGS_PER_STEP`] flags +/// of a set looked through, and [`GROUP_STEPS`] for each group of rules a split of the word leads to. +const MAX_SIMPLE_WORK: usize = 1 << 16; +/// Flags of a set looked through for one step. +const FLAGS_PER_STEP: usize = 16; +/// Steps a group of rules costs before its rules (the root it leaves is built and looked up). +const GROUP_STEPS: usize = 4; +/// Steps one simple-word check of a suggestion candidate takes at most. +const MAX_SUGGEST_SIMPLE_WORK: usize = 1 << 13; +/// Steps the simple-word checks of one suggestion call take in all (real packs' longest typos +/// need about 2^24.5; hostile rules stop here in under a second). +const MAX_SUGGEST_STEPS: usize = 1 << 25; +/// Most homonyms of one stem kept apart ([`AffixDict::homonyms`]); past it their flags count merged. +const MAX_HOMONYMS: usize = 16; /// How many suffixes a word may carry in a row through continuation classes (`work` + /// `able/B` + `s` → `workables` is two). Hunspell strips at most two suffixes, so a third /// chained suffix is not a word; the bound also keeps hostile input from walking deep recursions. @@ -257,6 +308,9 @@ struct SfxGroup { rules: Vec, /// The rules of `rules` with continuation classes: the only ones a chain can go through. cont_rules: Vec, + /// The rules of `rules` whose flag some continuation class holds: the only ones that can be + /// the outer suffix of two in a row. + outer_rules: Vec, } /// A stem list with affix rules (see the module docs). @@ -334,6 +388,25 @@ pub struct AffixDict { /// `COMPOUNDRULE`), so a word starting otherwise is no compound: SCOWL's ordinals start with /// a digit, and a misspelled word costs one lookup here instead of one per cut. compound_firsts: Vec, + /// Flag-based compounding (`COMPOUNDFLAG`…), if the dictionary compounds by flags. + cpd: Option>, + /// Stems with `ONLYINCOMPOUND` homonyms beside others: stem index → those homonyms' flags + /// (compound parts only), while `flags` holds the others'. + cpd_sets: FxMap, + /// Those stems' sets with all their homonyms' flags, for `COMPOUNDRULE` parts. + rule_sets: FxMap, + /// Stems listed more than once with different flags (not the `ONLYINCOMPOUND` homonyms + /// beside others): stem index → each homonym's flags. `flags` holds them merged, which + /// answers for one affix; a prefix with a suffix must come from one homonym (Dutch `stop/Pj…` + /// takes `af`, `stop/Ca…` takes `je`: `afstopje` is wrong in the CLI). + homonyms: FxMap>, + /// Appended string → the prefix rules adding it, in rule order. + pfx_index: FxMap, Vec>, + /// Stems in mixed case (`kB`, `MSc`) by their lower case: a word in capitals may be one + /// (`KB`, `KBS`), as in the CLI. + mixed: FxMap, Vec>, + /// Longest appended prefix, in chars. + max_pfx_chars: usize, } /// An `ICONV` or `OCONV` table: `(from, to)` pairs in source order, found by the first character @@ -463,6 +536,12 @@ pub fn capitalize(w: &str) -> String { } } +/// `w` with its first letter in lower case. +fn decapitalize(w: &str) -> String { + let mut c = w.chars(); + c.next().map(|f| f.to_lowercase().chain(c).collect()).unwrap_or_default() +} + /// Parsed rules and tables, shared by the Hunspell reader and the compact format. #[derive(Default)] struct Rules { @@ -492,6 +571,8 @@ struct Rules { only_in_compound: Option, /// `COMPOUNDMIN`: the shortest part of a compound, in characters (Hunspell's default is 3). compound_min: Option, + /// Flag-based compounding directives. + cpd: compound::Raw, /// `AF` flag aliases: alias `n` (1-based) stands for `aliases[n - 1]`. aliases: Vec, /// The `AF` table opened with its count line, so flag fields are alias numbers. A table @@ -748,13 +829,42 @@ impl FlagMode { /// Stem flags that rule a lookup out: `FORBIDDENWORD` always, and per lookup `NOSUGGEST` and /// `SUBSTANDARD` (suggestions), `KEEPCASE` (case variants of a word) or `NEEDAFFIX` (a stem on /// its own). +/// The second field: no flag-based compounds; the third: a suggestion candidate, whose compound +/// search gets a small budget ([`compound::MAX_SUGGEST_WORK`]); the fourth: `KEEPCASE`, kept +/// apart since it holds for a compound's first part only (and may share its id with another). #[derive(Clone, Copy, Debug, Default)] -struct Excluded([Option; 5]); +struct Excluded([Option; 5], bool, bool, Option); impl Excluded { /// Does the flag set `set` carry an excluded flag? fn hits(self, set: &[u16]) -> bool { - self.0.iter().flatten().any(|f| has(set, *f)) + self.0.iter().chain(std::iter::once(&self.3)).flatten().any(|f| has(set, *f)) + } + + /// These exclusions, and no flag-based compounds (`MAXCPDSUGS 0` for suggestions). + fn without_compounds(mut self) -> Excluded { + self.1 = true; + self + } + + fn no_compounds(self) -> bool { + self.1 + } + + fn suggesting(self) -> bool { + self.2 + } + + /// These exclusions and `KEEPCASE` (`flag`): a case variant of the word is being checked. + fn keeping(mut self, flag: Option) -> Excluded { + self.3 = flag; + self + } + + /// These exclusions without `KEEPCASE` (a compound part after the first). + fn without_keep(mut self) -> Excluded { + self.3 = None; + self } /// These exclusions and `flag` too. @@ -798,27 +908,132 @@ impl<'a> FlagProbe<'a> { } } +// ponytail: a thread-local rather than a parameter through every check path; the guard below +// restores it, so nested or concurrent calls on other threads don't share it. +thread_local! { + /// Steps one suggestion call may still take in its simple-word checks, while it runs + /// ([`SuggestSteps`]); `None` outside one. + static SUGGEST_STEPS: std::cell::Cell> = const { std::cell::Cell::new(None) }; +} + +/// While alive, the simple-word checks on this thread share [`MAX_SUGGEST_STEPS`] steps +/// (one suggestion call: thousands of candidates can't each spend a full check's budget). +pub(crate) struct SuggestSteps(Option); + +impl SuggestSteps { + pub(crate) fn start() -> SuggestSteps { + SuggestSteps(SUGGEST_STEPS.with(|s| s.replace(Some(MAX_SUGGEST_STEPS)))) + } +} + +// Dropping restores the outer budget as it was: the steps a nested call took are not charged to +// it. Harmless while suggestion calls don't nest; a nested call would have to charge the outer one. +impl Drop for SuggestSteps { + fn drop(&mut self) { + SUGGEST_STEPS.with(|s| s.set(self.0)); + } +} + +/// Steps a word's check may still take ([`MAX_SIMPLE_WORK`] at first). +struct Steps(std::cell::Cell); + +impl Steps { + /// Pay for looking through a flag set of `len` flags: about [`FLAGS_PER_STEP`] flags a step. + fn scan(&self, len: usize) -> bool { + self.spend(len / FLAGS_PER_STEP + 1) + } + + /// Pay for a rule's condition check (a step per character class). + fn cond(&self, r: &Rule) -> bool { + self.spend(r.cond.len()) + } + + /// Take `n` steps; false when they ran out. + fn spend(&self, n: usize) -> bool { + let left = self.0.get(); + self.0.set(left.saturating_sub(n.max(1))); + left > 0 + } +} + +/// A stem's flag sets as [`AffixDict::build`] works them out from its entries. +struct StemSets { + id: u32, + cpd: Option, + rule: Option, + homonyms: Option>, + dropped: usize, + compound: bool, +} + +/// Flags a stem's entries hold in all from which its sets are worked out once per distinct +/// entry list ([`StemSets`]); real stems have a few dozen. +const MEMO_STEM_FLAGS: usize = 256; + /// Does the sorted flag set `set` hold `flag`? fn has(set: &[u16], flag: u16) -> bool { set.binary_search(&flag).is_ok() } +/// Note a homonym's flags among its group's (`group`), unless an equal one is there; false past +/// [`MAX_HOMONYMS`] (then the merged flags answer: hostile input can't make it costly). +fn note_homonym(group: &mut Vec>, fresh: &[u16]) -> bool { + if group.iter().any(|g| g == fresh) { + return true; + } + if group.len() >= MAX_HOMONYMS { + return false; + } + group.push(fresh.to_vec()); + true +} + +/// Add a homonym's sorted flags `fresh` to the sorted set `set`; `false` (and nothing added) when +/// that would take it past [`MAX_FLAGS_PER_STEM`]. +fn merge_homonym(set: &mut Vec, fresh: &[u16]) -> bool { + let before = set.len(); + let add: Vec = fresh.iter().copied().filter(|&f| !has(set, f)).collect(); + if add.is_empty() { + // A repeated homonym adds nothing: no sort (200,000 duplicate lines of a stem with + // thousands of flags would each pay one). + return true; + } + if before.saturating_add(add.len()) > MAX_FLAGS_PER_STEM { + return false; + } + set.extend(add); + // A sorted run and a sorted tail: the stable sort merges them in linear time. + set.sort(); + true +} + +/// Put the exclusion flags `carried` (FORBIDDENWORD, NOSUGGEST, SUBSTANDARD of any homonym) back +/// into the sorted set `set`, making room by dropping a flag that excludes nothing (there are at +/// most three exclusion flags, so a full set always has one). +fn carry_exclusions(set: &mut Vec, carried: &[u16], exclusions: &[u16]) { + for &f in carried { + if has(set, f) { + continue; + } + if set.len() >= MAX_FLAGS_PER_STEM + && let Some(at) = set.iter().rposition(|g| !exclusions.contains(g)) + { + set.remove(at); + } + if let Err(at) = set.binary_search(&f) { + set.insert(at, f); + } + } +} + /// Do two sorted flag sets share a flag? (A binary search per flag of the smaller one.) fn intersects(a: &[u16], b: &[u16]) -> bool { let (small, large) = if a.len() <= b.len() { (a, b) } else { (b, a) }; small.iter().any(|f| has(large, *f)) } -/// Directives an English dictionary may carry that this engine doesn't implement but that don't -/// change which English words are right, ignored by [`AffixDict::from_hunspell_english`]: -/// flag-based compounding (English dictionaries compound only ordinals, by `COMPOUNDRULE`, which -/// is read). Output conversion (`OCONV`) is read, for English as for every language: suggestions -/// are written as the dictionary says (`don’t`). -const IGNORED_IN_ENGLISH: &[&str] = - &["COMPOUNDBEGIN", "COMPOUNDEND", "COMPOUNDFIRST", "COMPOUNDFLAG", "COMPOUNDLAST", "COMPOUNDMIDDLE", "COMPOUNDPERMITFLAG"]; - -/// Read Hunspell `.aff` text (already decoded to UTF-8). `english`: ignore the directives in -/// [`IGNORED_IN_ENGLISH`] instead of refusing them. +/// Read Hunspell `.aff` text (already decoded to UTF-8). `english`: ignore the +/// flag-based compounding directives (see [`compound::is_directive`]). fn parse_aff(aff: &str, english: bool) -> Result { let mut r = Rules::default(); let mut cross: HashMap<(bool, FlagKey), bool> = HashMap::new(); @@ -836,8 +1051,8 @@ fn parse_aff(aff: &str, english: bool) -> Result { } let at = |k: usize| f.get(k).copied().unwrap_or(""); match dir { - "SET" | "LANG" | "NAME" | "VERSION" | "HOME" | "WORDCHARS" | "MAXNGRAMSUGS" | "MAXDIFF" | "ONLYMAXDIFF" | "NOSPLITSUGS" - | "SUGSWITHDOTS" | "WARN" | "FORBIDWARN" | "MAXCPDSUGS" => {} + "SET" | "NAME" | "VERSION" | "HOME" | "WORDCHARS" | "MAXNGRAMSUGS" | "MAXDIFF" | "ONLYMAXDIFF" | "SUGSWITHDOTS" | "WARN" + | "FORBIDWARN" => {} // Morphological aliases, and a flag that only matters for morphological generation: // the words it marks are correct. "AM" | "LEMMA_PRESENT" => {} @@ -935,9 +1150,16 @@ fn parse_aff(aff: &str, english: bool) -> Result { r.prefixes.push(rule); } } - d if english && IGNORED_IN_ENGLISH.contains(&d) => {} + // English dictionaries compound only ordinals, by `COMPOUNDRULE` (read above): their + // flag-based compounding directives don't change which English words are right. + d if english && compound::is_directive(d) => {} d if UNSUPPORTED.contains(&d) => return Err(format!("line {}: {d} is not supported", n + 1)), - _ => {} + _ => { + let mut cpd = std::mem::take(&mut r.cpd); + let read = cpd.read(&f, &mut |x| r.one_flag(x)); + r.cpd = cpd; + read.map_err(|e| format!("line {}: {e}", n + 1))?; + } } } // Aliases may be declared after the rules that use them; resolve once all are read. @@ -956,6 +1178,9 @@ fn parse_aff(aff: &str, english: bool) -> Result { Ok(r) } +/// A dictionary's stems as `(word, start, end, is a word)` ranges of one buffer of flag keys. +type StemFlags = (Vec<(String, usize, usize, bool)>, Vec); + impl AffixDict { /// Build from Hunspell `.aff` and `.dic` text (decoded to UTF-8; the `.dic` count line and /// morphological fields are ignored). `comments` (provenance, licence) are kept with the data. @@ -972,12 +1197,19 @@ impl AffixDict { } fn from_hunspell_in(aff: &str, dic: &str, comments: &[&str], english: bool) -> Result { + AffixDict::from_hunspell_capped(aff, dic, comments, english, MAX_STEM_FLAGS) + } + + /// [`AffixDict::from_hunspell_in`] with at most `max_flags` stem flags in all + /// ([`MAX_STEM_FLAGS`]; tests lower it). + fn from_hunspell_capped(aff: &str, dic: &str, comments: &[&str], english: bool, max_flags: usize) -> Result { // Some dictionaries open with a UTF-8 BOM; it must not glue itself to the first // directive or turn the `.dic` count line into a stem. let aff = aff.strip_prefix('\u{feff}').unwrap_or(aff); let dic = dic.strip_prefix('\u{feff}').unwrap_or(dic); let rules = parse_aff(aff, english)?; let mut stems: Vec<(String, String)> = Vec::new(); + let mut flag_text = FlagText::new(max_flags); for (n, line) in dic.lines().enumerate() { // The entry may hold spaces (`ad hoc/B\tpo:adv`); it ends where Hunspell ends it. // Leading blanks belong to it, as in Hunspell (` lab` is no `lab`). @@ -1013,14 +1245,15 @@ impl AffixDict { } // With an `AF` table the field stays an alias number here: build() reads each alias // once, and every stem using it shares those flags. + flag_text.add(&flags)?; stems.push((word, flags)); } - AffixDict::build(rules, stems, comments.iter().map(ToString::to_string).collect(), MAX_FLAGS_PER_ENTRY) + AffixDict::build(rules, stems, comments.iter().map(ToString::to_string).collect(), MAX_FLAGS_PER_ENTRY, max_flags) } - /// Build from parsed rules and stems whose flag fields carry at most `entry_cap` flags each - /// ([`MAX_FLAGS_PER_ENTRY`] for `.dic` entries, [`MAX_FLAGS_PER_STEM`] for compact data). - fn build(rules: Rules, stems: Vec<(String, String)>, comments: Vec, entry_cap: usize) -> Result { + /// Read each stem's flag field ([`AffixDict::build`] explains the rules) into ranges of one key + /// buffer, within the `max_flags` budget: the stems as `(word, start, end, is a word)`, and the keys. + fn read_stem_flags(rules: &Rules, stems: Vec<(String, String)>, entry_cap: usize, max_flags: usize) -> Result { let mode = rules.mode; // A `.dic` entry's malformed flag items read as Hunspell reads them ([`FlagMode`]); an // entry with more than MAX_FLAGS_PER_STEM flags, or compact data whose flags don't read, @@ -1049,23 +1282,36 @@ impl AffixDict { } Some((start, stem_keys.len(), word)) }; - let mut stems: Vec<(String, usize, usize, bool)> = stems - .into_iter() - .filter_map(|(w, f)| { - let range = match rules.alias_index(&f) { - Some(i) => *alias_ranges.entry(i).or_insert_with(|| read(rules.aliases.get(i).map_or("", String::as_str), &mut stem_keys)), - None if rules.aliases_on => Some((0, 0, true)), - None => read(&f, &mut stem_keys), - }; - if range.is_none_or(|r| !r.2) { - skipped += 1; - } - range.map(|(a, b, word)| (w, a, b, word)) - }) - .collect(); + let mut read_stems: Vec<(String, usize, usize, bool)> = Vec::with_capacity(stems.len()); + for (w, f) in stems { + let range = match rules.alias_index(&f) { + Some(i) => *alias_ranges.entry(i).or_insert_with(|| read(rules.aliases.get(i).map_or("", String::as_str), &mut stem_keys)), + None if rules.aliases_on => Some((0, 0, true)), + None => read(&f, &mut stem_keys), + }; + if stem_keys.len() > max_flags { + return Err(format!("the stems carry more than {max_flags} flags in all")); + } + if range.is_none_or(|r| !r.2) { + skipped += 1; + } + if let Some((a, b, word)) = range { + read_stems.push((w, a, b, word)); + } + } if skipped > 0 { log::debug!("affix dictionary: {skipped} entries with unreadable or too many flags skipped"); } + Ok((read_stems, stem_keys)) + } + + /// Build from parsed rules and stems whose flag fields carry at most `entry_cap` flags each + /// ([`MAX_FLAGS_PER_ENTRY`] for `.dic` entries, [`MAX_FLAGS_PER_STEM`] for compact data). + /// The stems' flags may number `max_flags` in all ([`MAX_STEM_FLAGS`]), counted as they are read. + fn build(rules: Rules, stems: Vec<(String, String)>, comments: Vec, entry_cap: usize, max_flags: usize) -> Result { + let mode = rules.mode; + let exclusion_keys = [rules.forbidden, rules.no_suggest, rules.substandard]; + let (mut stems, stem_keys) = AffixDict::read_stem_flags(&rules, stems, entry_cap, max_flags)?; let conts = |rs: &[RawRule]| rs.iter().map(|r| mode.parse(&r.4)).collect::, _>>(); let (pfx_cont, sfx_cont) = (conts(&rules.prefixes)?, conts(&rules.suffixes)?); // Flags: every one a rule, a stem or a directive uses gets an id, in key order. @@ -1075,7 +1321,8 @@ impl AffixDict { // NEEDAFFIX and CIRCUMFIX flags may only ever appear in rules' continuation classes. let directives = [rules.forbidden, rules.no_suggest, rules.substandard, rules.keepcase, rules.only_in_compound, rules.need_affix, rules.circumfix]; - for k in all.chain(directives.into_iter().flatten()) { + let cpd_keys = rules.cpd.keys(mode)?; + for k in all.chain(directives.into_iter().flatten()).chain(cpd_keys) { distinct.entry(k).or_insert(0); if distinct.len() > MAX_DISTINCT_FLAGS { return Err(format!("more than {MAX_DISTINCT_FLAGS} distinct flags")); @@ -1122,6 +1369,7 @@ impl AffixDict { compound_text: rules.compound_rules, only_in_compound: rules.only_in_compound.and_then(id_of), compound_min: rules.compound_min.unwrap_or(3), + cpd: rules.cpd.build(mode, &id_of)?, ..AffixDict::default() }; d.offs.push(0); @@ -1132,16 +1380,48 @@ impl AffixDict { let mut i = 0; let mut dropped = 0usize; let mut merged: Vec = Vec::new(); + let mut solo: Vec = Vec::new(); + let mut only_set: Vec = Vec::new(); let mut fresh: Vec = Vec::new(); let mut carried: Vec = Vec::new(); + // Each homonym's flags (ONLYINCOMPOUND ones apart), and whether all fit. + let mut each: Vec> = Vec::new(); + let mut each_only: Vec> = Vec::new(); + // A stem's sets depend only on its entries' flag ranges: stems whose entries share big + // `AF` aliases are worked out once (16 homonyms of 8,000 flags on 17,576 stems loaded in + // 55 s when each stem merged them again). + let mut memo: FxMap, Option> = FxMap::default(); while i < stems.len() { let Some((word, ..)) = stems.get(i) else { break }; + let end = stems.get(i..).and_then(|rest| rest.iter().position(|s| s.0 != *word)).map_or(stems.len(), |k| i + k); + let entries: Vec<(usize, usize, bool)> = stems.get(i..end).unwrap_or_default().iter().map(|s| (s.1, s.2, s.3)).collect(); + let heavy = entries.iter().map(|e| e.1.saturating_sub(e.0)).sum::() >= MEMO_STEM_FLAGS; + let at = u32::try_from(d.flags.len()).ok(); + if heavy && let Some(known) = memo.get(&entries) { + if let Some(m) = known { + dropped += m.dropped; + d.push_known(word, m)?; + } + i = end; + continue; + } + let dropped_before = dropped; merged.clear(); + solo.clear(); + only_set.clear(); carried.clear(); + each.clear(); + each_only.clear(); + let mut fits = true; let mut is_word = false; // NEEDAFFIX belongs to its homonym: the word stands alone if any homonym does // (`foo/NA` and `foo/B`: `foo` is right). let mut all_need = true; + // So does ONLYINCOMPOUND, with its flags: beside homonyms without it, the homonyms + // with it are kept apart (`cpd_sets`), only for compound parts (German + // `schulen/DIXY` beside `schulen/Sozm`: `schulen` alone, but not `schulens`), as in + // the CLI. + let (mut solo_need, mut only_need, mut any_solo, mut any_only) = (true, true, false, false); let mut j = i; while let Some((w, a, b, entry_is_word)) = stems.get(j) && w == word @@ -1158,52 +1438,78 @@ impl AffixDict { continue; } is_word = true; - all_need &= d.need_affix.is_some_and(|n| fresh.contains(&n)); + let need = d.need_affix.is_some_and(|n| fresh.contains(&n)); + let only = d.only_in_compound.is_some_and(|o| fresh.contains(&o)); + all_need &= need; fresh.sort_unstable(); fresh.dedup(); - fresh.retain(|&f| !has(&merged, f)); + let group_fits = if only { + any_only = true; + only_need &= need; + merge_homonym(&mut only_set, &fresh) + } else { + any_solo = true; + solo_need &= need; + merge_homonym(&mut solo, &fresh) + }; // Homonyms can add up to more flags than one entry holds; past MAX_FLAGS_PER_STEM // an entry's other flags are dropped (its exclusions are carried below), so the // compact data never holds a set its reader refuses. - if fresh.is_empty() { - // A repeated homonym adds nothing: no sort (200,000 duplicate lines of a - // stem with thousands of flags would each pay one). - } else if merged.len().saturating_add(fresh.len()) <= MAX_FLAGS_PER_STEM { - merged.extend_from_slice(&fresh); - // A sorted run and a sorted tail: the stable sort merges them in linear time. - merged.sort(); - } else { + if !merge_homonym(&mut merged, &fresh) || !group_fits { dropped += 1; + fits = false; } + fits &= note_homonym(if only { &mut each_only } else { &mut each }, &fresh); } if !is_word { + if heavy { + memo.insert(entries, None); + } i = j; continue; } if !all_need && let Some(n) = d.need_affix { merged.retain(|&f| f != n); } - for &f in &carried { - if has(&merged, f) { - continue; + let mixed = any_solo && any_only; + // The homonyms whose flags `flags` holds merged, if they differ. + let apart = if mixed { std::mem::take(&mut each) } else { each.drain(..).chain(each_only.drain(..)).collect() }; + if fits { + d.keep_apart(&mut set_ids, apart, &carried, &exclusions)?; + } + let cpd_id = if mixed { + for (set, all_need) in [(&mut solo, solo_need), (&mut only_set, only_need)] { + if !all_need && let Some(n) = d.need_affix { + set.retain(|&f| f != n); + } + carry_exclusions(set, &carried, &exclusions); } - // Make room by dropping a flag that excludes nothing (there are at most three - // exclusion flags, so a full set always has one). - if merged.len() >= MAX_FLAGS_PER_STEM - && let Some(at) = merged.iter().rposition(|g| !exclusions.contains(g)) - { - merged.remove(at); - } - if let Err(at) = merged.binary_search(&f) { - merged.insert(at, f); + // `COMPOUNDRULE` parts read all the homonyms' flags. + if !d.compound_rules.is_empty() { + carry_exclusions(&mut merged, &carried, &exclusions); + let all = d.intern(&mut set_ids, &mut merged)?; + if let Ok(at) = u32::try_from(d.flags.len()) { + d.rule_sets.insert(at, all); + } } + std::mem::swap(&mut merged, &mut solo); + Some(d.intern(&mut set_ids, &mut only_set)?) + } else { + carry_exclusions(&mut merged, &carried, &exclusions); + None + }; + if let Some(c) = cpd_id + && let Ok(at) = u32::try_from(d.flags.len()) + { + d.cpd_sets.insert(at, c); } let id = d.intern(&mut set_ids, &mut merged)?; - if merged.iter().any(|f| compound_flags.contains(f)) - && let Some(c) = word.chars().next() - && !d.compound_firsts.contains(&c) - { - d.compound_firsts.push(c); + let compound = [Some(id), cpd_id].into_iter().flatten().any(|s| d.set(s).iter().any(|f| compound_flags.contains(f))); + d.note_first(word, compound); + if heavy && let Some(at) = at { + let homonyms = d.homonyms.get(&at).cloned(); + let rule = d.rule_sets.get(&at).copied(); + memo.insert(entries, Some(StemSets { id, cpd: cpd_id, rule, homonyms, dropped: dropped - dropped_before, compound })); } d.words.push_str(word); d.offs.push(u32::try_from(d.words.len()).map_err(|_| "stems too large".to_string())?); @@ -1230,6 +1536,36 @@ impl AffixDict { Ok(d) } + /// Note the first letter of `word` among the compound parts' when it is one (`compound`). + fn note_first(&mut self, word: &str, compound: bool) { + if compound + && let Some(c) = word.chars().next() + && !self.compound_firsts.contains(&c) + { + self.compound_firsts.push(c); + } + } + + /// Push the stem `word` with the sets `m` another stem with the same entries got. + fn push_known(&mut self, word: &str, m: &StemSets) -> Result<(), String> { + if let Ok(at) = u32::try_from(self.flags.len()) { + if let Some(h) = &m.homonyms { + self.homonyms.insert(at, h.clone()); + } + if let Some(c) = m.cpd { + self.cpd_sets.insert(at, c); + } + if let Some(r) = m.rule { + self.rule_sets.insert(at, r); + } + } + self.note_first(word, m.compound); + self.words.push_str(word); + self.offs.push(u32::try_from(self.words.len()).map_err(|_| "stems too large".to_string())?); + self.flags.push(m.id); + Ok(()) + } + /// The id of the flag set `flags` (sorted and deduplicated here), adding it if it is new. fn intern(&mut self, ids: &mut FxMap, u32>, flags: &mut Vec) -> Result { flags.sort_unstable(); @@ -1247,6 +1583,23 @@ impl AffixDict { Ok(id) } + /// Keep the homonyms `apart` of the stem about to be pushed, each with the exclusions + /// `carried` (see [`AffixDict::homonyms`]), when there are two or more. + fn keep_apart(&mut self, ids: &mut FxMap, u32>, mut apart: Vec>, carried: &[u16], exclusions: &[u16]) -> Result<(), String> { + if apart.len() < 2 { + return Ok(()); + } + let mut sets = Vec::with_capacity(apart.len()); + for set in &mut apart { + carry_exclusions(set, carried, exclusions); + sets.push(self.intern(ids, set)?); + } + if let Ok(at) = u32::try_from(self.flags.len()) { + self.homonyms.insert(at, sets.into()); + } + Ok(()) + } + /// The flag set `id`, sorted (empty when unknown). fn set(&self, id: u32) -> &[u16] { let i = id as usize; @@ -1285,6 +1638,23 @@ impl AffixDict { } } self.sfx_by_flag = by_flag; + let mut pfx_index: FxMap, Vec> = FxMap::default(); + for (i, r) in self.prefixes.iter().enumerate() { + if let Ok(i) = u32::try_from(i) { + pfx_index.entry(r.add.clone()).or_default().push(i); + } + } + self.pfx_index = pfx_index; + let mut mixed: FxMap, Vec> = FxMap::default(); + for i in 0..self.len() { + if let (Some(w), Ok(i)) = (self.word(i), u32::try_from(i)) + && casing(w) == Casing::Mixed + { + mixed.entry(w.to_lowercase().into()).or_default().push(i); + } + } + self.mixed = mixed; + self.max_pfx_chars = self.prefixes.iter().map(|r| r.add.chars().count()).max().unwrap_or(0); let mut cont_flag = vec![false; self.flag_keys.len()]; for r in self.prefixes.iter().chain(&self.suffixes) { for &f in self.set(r.cont) { @@ -1294,6 +1664,10 @@ impl AffixDict { } } self.has_cont = cont_flag.contains(&true); + for g in idx.values_mut().flatten() { + let outer = |i: &&u32| self.suffixes.get(**i as usize).is_some_and(|r| cont_flag.get(usize::from(r.flag)).copied().unwrap_or(false)); + g.outer_rules = g.rules.iter().filter(outer).copied().collect(); + } self.cont_flag = cont_flag; self.max_sfx_chars = self.suffixes.iter().map(|r| r.add.chars().count()).max().unwrap_or(0); self.sfx_index = idx; @@ -1366,7 +1740,13 @@ impl AffixDict { /// suggestions. fn excluded(&self, suggestible: bool) -> Excluded { let e = Excluded::default().and(self.forbidden); - if suggestible { e.and(self.no_suggest).and(self.substandard) } else { e } + if !suggestible { + return e; + } + let mut e = e.and(self.no_suggest).and(self.substandard); + e.2 = true; + // `MAXCPDSUGS 0`: compounds are right but never suggested. + if self.cpd.as_ref().is_some_and(|c| c.no_suggestions) { e.without_compounds() } else { e } } /// Flag set of a stem, if `w` is one. @@ -1382,8 +1762,32 @@ impl AffixDict { self.stem_set(w).filter(|set| !excluded.hits(set) && !self.only_in_compound.is_some_and(|f| has(set, f))) } + /// Does one homonym of the stem `w` (excluded ones aside) have flags meeting `pred`? + fn homonym_meets(&self, w: &str, excluded: Excluded, pred: impl Fn(&[u16]) -> bool) -> bool { + let Some((at, set)) = self.stem_lookup(w) else { return false }; + let usable = |set: &[u16]| !excluded.hits(set) && !self.only_in_compound.is_some_and(|f| has(set, f)); + if !usable(set) { + return false; + } + match self.homonyms.get(&at) { + Some(sets) => sets.iter().map(|&h| self.set(h)).any(|h| usable(h) && pred(h)), + None => pred(set), + } + } + + /// Is `w` listed with `FORBIDDENWORD`? Then it is no word, however else it could be built + /// (affixes, compounds), as in Hunspell. + fn is_forbidden(&self, w: &str) -> bool { + self.forbidden.is_some_and(|f| self.stem_set(w).is_some_and(|set| has(set, f))) + } + /// Flag set of the stem `w`, whatever flags it carries. fn stem_set(&self, w: &str) -> Option<&[u16]> { + self.stem_lookup(w).map(|(_, set)| set) + } + + /// Index and flag set of the stem `w`, whatever flags it carries. + fn stem_lookup(&self, w: &str) -> Option<(u32, &[u16])> { if self.table.is_empty() { return None; } @@ -1395,7 +1799,7 @@ impl AffixDict { return None; } if self.word(t as usize) == Some(w) { - return Some(self.set(*self.flags.get(t as usize)?)); + return Some((t, self.set(*self.flags.get(t as usize)?))); } slot = (slot + 1) & mask; } @@ -1418,8 +1822,13 @@ impl AffixDict { /// The rules of a group that could lead to `w` are merged into one set of their flags (sorted) /// and followed once, not once per rule: the answer is the same, but thousands of rules of one /// flag cost one walk, not one each (#3613). - fn form_has_flag(&self, w: &str, flags: &[u16], depth: u8, excluded: Excluded) -> bool { - if self.stem_flags_except(w, excluded).is_some_and(|f| intersects(f, flags)) { + /// + /// With `pfx` set (a prefix was taken off: the flags of the prefixes that could be it), every + /// rule on the way must allow cross products and one prefix's flag must come from the stem or + /// a continuation class on the way (`meg` + `épít` + `és` + `e`). + fn form_has_flag(&self, w: &str, flags: &[u16], depth: u8, pfx: Option<&FlagProbe>, excluded: Excluded, steps: &Steps) -> bool { + let fits = |f: &[u16]| steps.scan(f.len()) && intersects(f, flags) && pfx.is_none_or(|p| p.meets(f)); + if if pfx.is_some() { self.homonym_meets(w, excluded, fits) } else { self.stem_flags_except(w, excluded).is_some_and(fits) } { return true; } // Without continuation classes no rule leads from a form to another. @@ -1428,30 +1837,52 @@ impl AffixDict { } let mut root = String::with_capacity(w.len() + 8); let mut inner: Vec = Vec::new(); + // Rules whose continuation classes give the prefix its flag: under them it is met. + let mut met: Vec = Vec::new(); // Does a continuation set hold one of `flags`? Answered once per distinct set (rules // share interned sets) and, for many flags, through a bitmap: a lookup costs at most the // total size of the distinct sets it meets, not rules × set size. + if !steps.scan(flags.len()) { + return false; + } let probe = FlagProbe::new(flags); let mut seen: FxMap = FxMap::default(); let cuts = std::iter::once(w.len()).chain(w.char_indices().rev().map(|(i, _)| i).filter(|&i| i > 0 || self.full_strip)); for b in cuts.take(self.max_sfx_chars + 1) { let (Some(base), Some(add)) = (w.get(..b), w.get(b..)) else { continue }; + // Each split is a lookup of its appended text. + if !steps.scan(add.len()) { + return false; + } let Some(groups) = self.sfx_index.get(add) else { continue }; for g in groups.iter().filter(|g| !g.cont_rules.is_empty()) { + if !steps.spend(g.cont_rules.len() + GROUP_STEPS) { + return false; + } root.clear(); root.push_str(base); root.push_str(&g.strip); inner.clear(); + met.clear(); for r in g.cont_rules.iter().filter_map(|&i| self.suffixes.get(i as usize)) { - let meets = *seen.entry(r.cont).or_insert_with(|| probe.meets(self.set(r.cont))); - if meets && !self.cont_holds(r, self.circumfix) && r.cond_matches(&root, true) { - inner.push(r.flag); + let cont = self.set(r.cont); + let meets = *seen.entry(r.cont).or_insert_with(|| steps.scan(cont.len()) && probe.meets(cont)); + if meets + && (pfx.is_none() || r.cross) + && !self.cont_holds(r, self.circumfix) + && !self.cont_holds(r, self.only_in_compound) + && steps.cond(r) + && r.cond_matches(&root, true) + { + if pfx.is_some_and(|p| steps.scan(cont.len()) && p.meets(cont)) { met.push(r.flag) } else { inner.push(r.flag) } } } - inner.sort_unstable(); - inner.dedup(); - if !inner.is_empty() && self.form_has_flag(&root, &inner, depth - 1, excluded) { - return true; + for (list, p) in [(&mut inner, pfx), (&mut met, None)] { + list.sort_unstable(); + list.dedup(); + if !list.is_empty() && self.form_has_flag(&root, list, depth - 1, p, excluded, steps) { + return true; + } } } } @@ -1470,6 +1901,37 @@ impl AffixDict { self.contains_except(&self.normalize(w), self.excluded(true)) } + /// [`AffixDict::contains_suggestible`], without flag-based compounds when `compounds` is + /// false (suggestion candidates past the compound check budget). + pub(crate) fn contains_suggestible_with(&self, w: &str, compounds: bool) -> bool { + let e = self.excluded(true); + self.contains_except(&self.normalize(w), if compounds { e } else { e.without_compounds() }) + } + + /// Suggestions for `word`, best first, at most `max`, from the dictionary's own tables (as + /// for a Language pack). + pub fn suggest(&self, word: &str, max: usize) -> Vec { + crate::affix_suggest::suggest_in(self, word, max, &crate::affix_suggest::Tuning::DEFAULT) + } + + /// Is `w`, known to be no suggestible simple word or `COMPOUNDRULE` compound, a flag-based + /// compound? The search spends its steps from `budget` (at most + /// [`compound::MAX_SUGGEST_WORK`] for this word); the word isn't looked up again. + pub(crate) fn flag_compound_spending_suggestible(&self, w: &str, budget: &mut usize, pieces: &mut compound::Pieces) -> bool { + let w = self.normalize(w); + let e = self.excluded(true); + let allowed = (*budget).min(compound::MAX_SUGGEST_WORK); + let mut left = allowed; + let found = self.flag_compound_spending(&w, e, matches!(casing(&w), Casing::Initial | Casing::Upper), &mut left, Some(pieces)); + *budget = budget.saturating_sub(allowed - left); + found + } + + /// Whether the dictionary compounds by flags (suggestion candidates then cost more). + pub(crate) fn flag_compounds(&self) -> bool { + self.cpd.is_some() + } + /// `w` after the `ICONV` conversions (at each position the longest match wins) and without /// the `IGNORE` characters. Input longer than [`MAX_INPUT_BYTES`] is no word and comes back /// unchanged, so the length check after this rejects it without converting it. @@ -1489,42 +1951,117 @@ impl AffixDict { /// [`AffixDict::contains`] on normalized input, where stems carrying an `excluded` flag /// don't count (forbidden words; case variants of `KEEPCASE` stems). fn contains_except(&self, w: &str, excluded: Excluded) -> bool { - if w.is_empty() || w.len() > MAX_WORD_BYTES { - return false; + self.contains_in(w, excluded, matches!(casing(w), Casing::Initial | Casing::Upper)) + } + + /// [`AffixDict::contains_except`], where `capitalized` says whether the word as written + /// starts with a capital (`w` may be its lower-case form): a compound whose last part has + /// `FORCEUCASE` is right only then. + fn contains_in(&self, w: &str, excluded: Excluded, capitalized: bool) -> bool { + self.simple_word(w, excluded) || self.compound(w, excluded) || self.flag_compound(w, excluded, capitalized) + } + + /// Is `w` a stem or an affixed form of one (no compound)? Affixes that are only valid inside + /// a compound (`ONLYINCOMPOUND`) don't count. + fn simple_word(&self, w: &str, excluded: Excluded) -> bool { + self.simple_word_with(w, excluded, true) + } + + /// [`AffixDict::simple_word`], with prefixes only when `prefixes` is true. + fn simple_word_with(&self, w: &str, excluded: Excluded, prefixes: bool) -> bool { + self.simple_word_budgeted(w, excluded, prefixes).unwrap_or(false) + } + + /// [`AffixDict::simple_word_with`], or `None` when its steps ran out before an answer. + fn simple_word_budgeted(&self, w: &str, excluded: Excluded, prefixes: bool) -> Option { + if w.is_empty() || w.len() > MAX_WORD_BYTES || self.is_forbidden(w) { + return Some(false); } + // The steps taken for this word, all ways together: past MAX_SIMPLE_WORK it is + // taken for no word (hostile rules can't make one word cost seconds). Inside a + // suggestion call the word gets at most MAX_SUGGEST_SIMPLE_WORK of the call's budget. + let shared = SUGGEST_STEPS.with(std::cell::Cell::get); + let cap = shared.map_or(MAX_SIMPLE_WORK, |left| left.min(MAX_SUGGEST_SIMPLE_WORK)); + let steps = Steps(std::cell::Cell::new(cap)); + let found = self.simple_word_in(w, excluded, prefixes, &steps); + if let Some(left) = shared { + let spent = cap.saturating_sub(steps.0.get()).max(1); + SUGGEST_STEPS.with(|s| s.set(Some(left.saturating_sub(spent)))); + } + (found || steps.0.get() > 0).then_some(found) + } + + /// [`AffixDict::simple_word_with`] in the steps `steps` allows. + fn simple_word_in(&self, w: &str, excluded: Excluded, prefixes: bool, steps: &Steps) -> bool { // A stem that needs an affix is no word on its own. - if self.stem_flags_except(w, excluded.and(self.need_affix)).is_some() || self.suffixed(w, None, false, false, excluded) { + if self.stem_flags_except(w, excluded.and(self.need_affix)).is_some() || self.suffixed(w, None, false, false, excluded, steps) { return true; } - let mut root = String::with_capacity(w.len() + 8); - for r in &self.prefixes { + if !prefixes { + return false; + } + // The prefix rules whose appended text starts `w`, found by that text (one hash lookup per + // length) rather than by trying every rule. + let ends = std::iter::once(0).chain(w.char_indices().map(|(i, c)| i + c.len_utf8())); + let rules = ends + .take(self.max_pfx_chars + 1) + .filter_map(|b| self.pfx_index.get(w.get(..b)?)) + .flatten() + .filter_map(|&i| self.prefixes.get(i as usize)); + // What each rule leaves: (root, circumfix, needs another affix, cross, flag). The rules + // that leave the same root alike are answered together, in one walk each. + let mut taken: Vec<(String, bool, bool, bool, u16)> = Vec::new(); + for r in rules { + if !steps.spend(1 + r.cond.len()) { + return false; + } let Some(rest) = w.strip_prefix(&*r.add) else { continue }; - if rest.is_empty() && !self.full_strip { + if (rest.is_empty() && !self.full_strip) || self.cont_holds(r, self.only_in_compound) { continue; } - root.clear(); - root.push_str(&r.strip); - root.push_str(rest); - if !r.cond_matches(&root, false) { - continue; + let root = format!("{}{rest}", r.strip); + if r.cond_matches(&root, false) { + // A circumfix prefix works only with its suffix, and one that needs another + // affix only with a suffix too. + taken.push((root, self.cont_holds(r, self.circumfix), self.cont_holds(r, self.need_affix), r.cross, r.flag)); } - // A circumfix prefix works only with its suffix, and one that needs another affix - // only with a suffix too. - let circ = self.cont_holds(r, self.circumfix); - let needs = self.cont_holds(r, self.need_affix); - if !circ && !needs { - if self.stem_flags_except(&root, excluded).is_some_and(|f| has(f, r.flag)) { + } + taken.sort_unstable(); + let mut plain: Vec = Vec::new(); + let mut crossed: Vec = Vec::new(); + for group in taken.chunk_by(|a, b| (&a.0, a.1, a.2) == (&b.0, b.1, b.2)) { + if !steps.spend(group.len() + GROUP_STEPS) { + return false; + } + let Some((root, circ, needs, ..)) = group.first() else { continue }; + plain.clear(); + crossed.clear(); + for &(.., cross, flag) in group { + if !circ && !needs { + plain.push(flag); + } + if cross { + crossed.push(flag); + } + } + plain.sort_unstable(); + plain.dedup(); + crossed.sort_unstable(); + crossed.dedup(); + if !plain.is_empty() { + if self.stem_flags_except(root, excluded).is_some_and(|f| steps.scan(f.len()) && intersects(f, &plain)) { return true; } - if self.is_cont_flag(r.flag) && self.form_has_flag(&root, &[r.flag], INNER_CHAIN, excluded) { + plain.retain(|&f| self.is_cont_flag(f)); + if !plain.is_empty() && self.form_has_flag(root, &plain, INNER_CHAIN, None, excluded, steps) { return true; } } - if r.cross && self.suffixed(&root, Some(r.flag), circ, needs, excluded) { + if !crossed.is_empty() && self.suffixed(root, Some(&FlagProbe::new(&crossed)), *circ, *needs, excluded, steps) { return true; } } - self.compound(w, excluded) + false } /// Is `w` a compound of two or more stems (no affixes) whose flags spell out one of the @@ -1536,7 +2073,7 @@ impl AffixDict { if self.compound_rules.is_empty() || w.len() > MAX_COMPOUND_BYTES || !w.chars().next().is_some_and(|c| self.compound_firsts.contains(&c)) - || self.forbidden.is_some_and(|f| self.stem_set(w).is_some_and(|set| has(set, f))) + || self.is_forbidden(w) { return false; } @@ -1558,7 +2095,10 @@ impl AffixDict { let slot = parts.get_mut(a * n + b)?; *slot.get_or_insert_with(|| { let (Some(&x), Some(&y)) = (cuts.get(a), cuts.get(b)) else { return None }; - w.get(x..y).and_then(|p| self.stem_set(p)).filter(|set| !excluded.hits(set) && bare(set)) + w.get(x..y) + .and_then(|p| self.stem_lookup(p)) + .map(|(stem, set)| self.rule_sets.get(&stem).map_or(set, |&s| self.set(s))) + .filter(|set| !excluded.hits(set) && bare(set)) }) }; self.compound_rules.iter().any(|rule| compound_matches(rule, &mut part, n)) @@ -1570,50 +2110,88 @@ impl AffixDict { /// one: the suffix must be one too, and otherwise must not be. `pfx_needs`: the prefix needs /// another affix (NEEDAFFIX), so the suffix may not need one too (Hunspell rejects `reworks` /// from `re/N` and `s/N`). - fn suffixed(&self, w: &str, need: Option, circ: bool, pfx_needs: bool, excluded: Excluded) -> bool { + /// `need` holds the flags of all the prefixes that could have been taken off (one walk for + /// them all: a thousand prefix rules adding nothing cost one walk, not a thousand). + fn suffixed(&self, w: &str, need: Option<&FlagProbe>, circ: bool, pfx_needs: bool, excluded: Excluded, steps: &Steps) -> bool { // Fits the suffix rule `r`: circumfix alike, and in need of another affix only when a // prefix that doesn't is there. - let fits = |r: &Rule| self.cont_holds(r, self.circumfix) == circ && (!self.cont_holds(r, self.need_affix) || (need.is_some() && !pfx_needs)); + let fits = |r: &Rule| { + self.cont_holds(r, self.circumfix) == circ + && (!self.cont_holds(r, self.need_affix) || (need.is_some() && !pfx_needs)) + && !self.cont_holds(r, self.only_in_compound) + }; let mut root = String::with_capacity(w.len() + 8); let mut flags: Vec = Vec::new(); + let mut met: Vec = Vec::new(); // Each split point from the end: the appended string is `w[b..]`, at most the longest // suffix, and something must stay before it. let cuts = std::iter::once(w.len()).chain(w.char_indices().rev().map(|(i, _)| i).filter(|&i| i > 0 || self.full_strip)); for b in cuts.take(self.max_sfx_chars + 1) { let (Some(base), Some(add)) = (w.get(..b), w.get(b..)) else { continue }; + // Each split is a lookup of its appended text. + if !steps.scan(add.len()) { + return false; + } let Some(groups) = self.sfx_index.get(add) else { continue }; for g in groups { + if !steps.spend(g.rules.len() + GROUP_STEPS) { + return false; + } root.clear(); root.push_str(base); root.push_str(&g.strip); let rules = || g.rules.iter().filter_map(|&i| self.suffixes.get(i as usize)); // The stem path: `root` is a stem carrying the rule's flag (and the prefix's, // when a cross product is being checked). - if let Some(f) = self.stem_flags_except(&root, excluded) - && rules().any(|r| { - has(f, r.flag) - && need.is_none_or(|n| has(f, n) || self.cont_holds(r, Some(n))) - && (need.is_none() || r.cross) - && fits(r) - && r.cond_matches(&root, true) - }) - { + let takes = |f: &[u16]| { + steps.spend(g.rules.len()) + && rules().any(|r| { + has(f, r.flag) + && need.is_none_or(|n| { + let cont = self.set(r.cont); + steps.scan(f.len() + cont.len()) && (n.meets(f) || n.meets(cont)) + }) + && (need.is_none() || r.cross) + && fits(r) + && steps.cond(r) + && r.cond_matches(&root, true) + }) + }; + if if need.is_some() { + self.homonym_meets(&root, excluded, takes) + } else { + self.stem_flags_except(&root, excluded).is_some_and(takes) + } { return true; } // Continuation classes: `root` need not be a stem carrying this rule's flag — // it may be a form that gained the flag from another rule's continuation. The // matching rules' flags are looked for together, in one walk. Without continuation // classes that walk would only repeat the stem path above. - if need.is_none() && self.has_cont { + if self.has_cont && !circ { flags.clear(); + met.clear(); // Only a flag some continuation class holds can come from another rule. The // outer suffix of a chain has another affix under it, so its NEEDAFFIX is met, // and Hunspell ignores its CIRCUMFIX there (`workables` from `s/c`). - flags.extend(rules().filter(|r| self.is_cont_flag(r.flag) && r.cond_matches(&root, true)).map(|r| r.flag)); - flags.sort_unstable(); - flags.dedup(); - if !flags.is_empty() && self.form_has_flag(&root, &flags, INNER_CHAIN, excluded) { - return true; + // After a prefix every rule must allow cross products, and a rule whose + // continuation classes hold the prefix's flag meets it for the whole chain. + for r in rules().filter(|r| { + self.is_cont_flag(r.flag) + && (need.is_none() || r.cross) + && !self.cont_holds(r, self.only_in_compound) + && steps.cond(r) + && r.cond_matches(&root, true) + }) { + let cont = self.set(r.cont); + if need.is_some_and(|n| steps.scan(cont.len()) && n.meets(cont)) { met.push(r.flag) } else { flags.push(r.flag) } + } + for (list, p) in [(&mut flags, need), (&mut met, None)] { + list.sort_unstable(); + list.dedup(); + if !list.is_empty() && self.form_has_flag(&root, list, INNER_CHAIN, p, excluded, steps) { + return true; + } } } } @@ -1637,7 +2215,7 @@ impl AffixDict { /// Whether `w` is a word only through a `KEEPCASE` stem, so it is correct in this letter case /// alone (`km` from `km/K`, which `Km` and `KM` are not). pub(crate) fn case_locked(&self, w: &str) -> bool { - self.keepcase.is_some() && self.contains(w) && !self.contains_except(&self.normalize(w), self.excluded(false).and(self.keepcase)) + self.keepcase.is_some() && self.contains(w) && !self.contains_except(&self.normalize(w), self.excluded(false).keeping(self.keepcase)) } fn check_except(&self, w: &str, excluded: Excluded) -> bool { @@ -1649,24 +2227,42 @@ impl AffixDict { if w.len() > MAX_INPUT_BYTES { return false; } - let keep = excluded.and(self.keepcase); + let keep = excluded.keeping(self.keepcase); match casing(&w) { - Casing::Lower | Casing::Mixed => false, + Casing::Lower => false, + // A compound may join a capitalized stem to others, and the word start with a + // capital (`FooQux` from `foo` and `Qux`: right in the CLI). + Casing::Mixed => w.chars().next().is_some_and(char::is_uppercase) && self.flag_compound(&decapitalize(&w), keep, true), Casing::Initial => { let lower = w.to_lowercase(); // CHECKSHARPS: a KEEPCASE word with `ß` may be capitalized (`Gruß` for `Gruß/k`). let ex = if self.check_sharps && lower.contains('ß') { excluded } else { keep }; - self.contains_except(&lower, ex) + self.contains_in(&lower, ex, true) } Casing::Upper => { let lower = w.to_lowercase(); self.contains_except(&capitalize(&lower), keep) - || self.contains_except(&lower, keep) + || self.contains_in(&lower, keep, true) + || self.mixed_case(&lower, keep) || (self.check_sharps && self.sharps(&lower, excluded)) } } } + /// Is `lower` (a word in capitals, lowercased) a stem in mixed case, or one with an ending + /// (`kbs` from `kB/S`)? Not a compound of one (`JQUERYKABEL` is wrong in the CLI). + fn mixed_case(&self, lower: &str, excluded: Excluded) -> bool { + if self.mixed.is_empty() { + return false; + } + let ends = lower.char_indices().map(|(i, c)| i + c.len_utf8()); + // At most MAX_HOMONYMS stems in all are tried, whatever the word's length. + ends.filter_map(|b| Some((self.mixed.get(lower.get(..b)?)?, lower.get(b..)?))) + .flat_map(|(stems, rest)| stems.iter().filter_map(|&i| self.word(i as usize)).map(move |m| (m, rest))) + .take(MAX_HOMONYMS) + .any(|(m, rest)| self.simple_word(&format!("{m}{rest}"), excluded)) + } + /// CHECKSHARPS: is `lower` (a word in capitals, lowercased) a word with some of its `ss` read /// as `ß`, in lower case or capitalized? Such a form may come from a `KEEPCASE` stem (in /// capitals a word with `ß` is written with `SS`). The first [`MAX_SHARPS`] `ss` pairs are @@ -1697,7 +2293,22 @@ impl AffixDict { /// cross products, so a dictionary with thousands of self-continuing rules can't make one stem /// cost millions of strings (#3613). pub fn forms(&self, stem: &str) -> Vec { + let mut budget = MAX_FORMS_WORK; + self.forms_spending(stem, &mut budget) + } + + /// [`AffixDict::forms`], with the rules it tries taken off `budget` (several stems' forms + /// share one budget: the suggestions' scan of 48 stems in a dictionary of 12,000 rules). + pub(crate) fn forms_spending(&self, stem: &str, budget: &mut usize) -> Vec { let Some(f) = self.stem_flags(stem) else { return Vec::new() }; + let start = *budget; + let out = self.forms_within(stem, f, (*budget).min(MAX_FORMS_WORK)); + *budget = start.saturating_sub(out.1); + out.0 + } + + /// The forms of `stem` (flags `f`) in at most `limit` rules tried, and the rules tried. + fn forms_within(&self, stem: &str, f: &[u16], limit: usize) -> (Vec, usize) { let mut out = Vec::new(); if !self.need_affix.is_some_and(|n| has(f, n)) { out.push(stem.to_string()); @@ -1712,7 +2323,14 @@ impl AffixDict { let mut queued: std::collections::HashSet<(String, u32)> = std::collections::HashSet::new(); let mut work = 0usize; for r in &self.suffixes { - if !has(f, r.flag) || !r.cond_matches(stem, true) { + if !has(f, r.flag) { + continue; + } + work += 1; + if work > limit { + return (out, work); + } + if !r.cond_matches(stem, true) || self.cont_holds(r, self.only_in_compound) { continue; } let Some(base) = stem.strip_suffix(&*r.strip) else { continue }; @@ -1734,7 +2352,7 @@ impl AffixDict { out.extend(non_empty(form)); } if out.len() >= MAX_FORMS { - return out; + return (out, work); } } // Continuation classes chain suffixes (`workable/B` → `workables`), up to [`MAX_CHAIN`] @@ -1747,10 +2365,10 @@ impl AffixDict { for (form, flags) in level { for r in self.suffixes_with(flags).into_iter().filter_map(|i| self.suffixes.get(i as usize)) { work += 1; - if work > MAX_FORMS_WORK { + if work > limit { break 'chain; } - if !r.cond_matches(&form, true) { + if !r.cond_matches(&form, true) || self.cont_holds(r, self.only_in_compound) { continue; } let Some(base) = form.strip_suffix(&*r.strip) else { continue }; @@ -1765,12 +2383,12 @@ impl AffixDict { // and Hunspell ignores its CIRCUMFIX there (`workables` from `s/c`). out.extend(non_empty(next)); if out.len() >= MAX_FORMS { - return out; + return (out, work); } } } } - for r in &self.prefixes { + for r in self.prefixes.iter().filter(|r| !self.cont_holds(r, self.only_in_compound)) { let on_stem = has(f, r.flag); let circ = self.cont_holds(r, self.circumfix); let needs = self.cont_holds(r, self.need_affix); @@ -1781,8 +2399,8 @@ impl AffixDict { // Every suffix form looked at counts against the work budget, matching or not. for (s, s_circ, s_needs, s_cont) in &cross_sfx { work += 1; - if out.len() >= MAX_FORMS || work > MAX_FORMS_WORK { - return out; + if out.len() >= MAX_FORMS || work > limit { + return (out, work); } // The prefix's flag comes from the stem or the suffix's continuation classes; // circumfix alike; not both in need of another affix. @@ -1793,10 +2411,10 @@ impl AffixDict { } } if out.len() >= MAX_FORMS { - return out; + return (out, work); } } - out + (out, work) } /// `root` with the prefix rule `r` added, when the checker takes it back off: the rule's @@ -1858,7 +2476,7 @@ impl AffixDict { &self.comments } - /// Serialize to WordCraft's compact format: raw-deflate of `WCAFF1`, a `FLAG` line when flags + /// Serialize to WordCraft's compact format: raw-deflate of `WCAFF2`, a `FLAG` line when flags /// aren't one character each, the comments, the tables and rules as text lines, a `WORDS` /// line, then one line per stem, front-coded against the previous one (a byte with the shared /// prefix length in bytes, the rest of the word, `/` and its flags if any). Flags are written @@ -1890,7 +2508,7 @@ impl AffixDict { // In one-character mode a directive flag that no stem or rule uses isn't written, as // before flag ids existed, so those dictionaries' compact data stays byte for byte the same. let used = |flag: u16| { - self.flags.iter().any(|&id| self.set(id).contains(&flag)) + self.flags.iter().chain(self.cpd_sets.values()).chain(self.homonyms.values().flatten()).any(|&id| self.set(id).contains(&flag)) || self.prefixes.iter().chain(&self.suffixes).any(|r| r.flag == flag || self.set(r.cont).contains(&flag)) }; if self.full_strip { @@ -1933,7 +2551,7 @@ impl AffixDict { out.push('\n'); } }; - if self.compound_text.is_empty() { + if self.compound_text.is_empty() && self.cpd.is_none() { write_only_in_compound(&mut out); } else { out.push_str(&format!("COMPOUNDMIN {}\n", self.compound_min)); @@ -1942,6 +2560,9 @@ impl AffixDict { out.push_str(&format!("COMPOUNDRULE {p}\n")); } } + if let Some(c) = &self.cpd { + c.write(&mut out, &|k, out: &mut String| self.flag_mode.write(std::iter::once(k), out)); + } let z = |s: &str| if s.is_empty() { "0".to_string() } else { s.to_string() }; for (kind, rules) in [("PFX", &self.prefixes), ("SFX", &self.suffixes)] { for r in rules { @@ -1967,13 +2588,25 @@ impl AffixDict { let p = prev.iter().zip(b).take_while(|(x, y)| x == y).count().min(255); raw.push(p as u8); raw.extend_from_slice(b.get(p..).unwrap_or_default()); - let set = self.flags.get(i).map_or(&[][..], |&id| self.set(id)); - if !set.is_empty() { - let mut flags = String::from("/"); - self.write_flags(set, &mut flags); - raw.extend_from_slice(flags.as_bytes()); + // The homonyms kept apart each on a line, or their flags merged; then the + // ONLYINCOMPOUND homonyms' flags. The same stem again is merged back on reading. + let at = u32::try_from(i).ok(); + let main = self.flags.get(i).copied().unwrap_or_default(); + let apart = at.and_then(|i| self.homonyms.get(&i)).map_or(std::slice::from_ref(&main), |h| &h[..]); + let cpd = at.and_then(|i| self.cpd_sets.get(&i)); + for (k, &id) in apart.iter().chain(cpd).enumerate() { + if k > 0 { + raw.push(b.len().min(255) as u8); + raw.extend_from_slice(b.get(b.len().min(255)..).unwrap_or_default()); + } + let set = self.set(id); + if !set.is_empty() { + let mut flags = String::from("/"); + self.write_flags(set, &mut flags); + raw.extend_from_slice(flags.as_bytes()); + } + raw.push(b'\n'); } - raw.push(b'\n'); prev = b; } // What [`AffixDict::from_bytes`] would refuse to inflate is refused here. @@ -2034,6 +2667,12 @@ impl AffixDict { } rules.compound_rules.push(at(1).to_string()); } + d if compound::is_directive(d) => { + let mut cpd = std::mem::take(&mut rules.cpd); + let read = cpd.read(&f, &mut |x| rules.one_flag(x)); + rules.cpd = cpd; + read.map_err(|e| format!("affix dictionary: bad `{line}`: {e}"))?; + } "MAP" => rules.map.push(at(1).chars().collect()), "REP" => rules.rep.push((at(1).replace('_', " "), at(2).replace('_', " "))), kind @ ("PFX" | "SFX") => { @@ -2059,6 +2698,7 @@ impl AffixDict { } let body = raw.get(words_at + 7..).unwrap_or_default(); let mut stems: Vec<(String, String)> = Vec::new(); + let mut flag_text = FlagText::new(MAX_STEM_FLAGS); let mut cur: Vec = Vec::new(); let mut i = 0; while let Some(&p) = body.get(i) { @@ -2081,9 +2721,32 @@ impl AffixDict { if word.is_empty() || word.len() > MAX_WORD_BYTES { return Err("affix dictionary: bad stem".into()); } + flag_text.add(flags).map_err(|e| format!("affix dictionary: {e}"))?; stems.push((word.to_string(), flags.to_string())); } - AffixDict::build(rules, stems, comments, MAX_FLAGS_PER_STEM) + AffixDict::build(rules, stems, comments, MAX_FLAGS_PER_STEM, MAX_STEM_FLAGS) + } +} + +/// A running count of the stems' flag text while a dictionary is read: past +/// [`FLAG_TEXT_PER_FLAG`] bytes for each of `max_flags` flags, reading stops with an error, before +/// the rest of a huge `.dic` is held ([`MAX_STEM_FLAGS`]). +struct FlagText { + bytes: usize, + cap: usize, +} + +impl FlagText { + fn new(max_flags: usize) -> Self { + FlagText { bytes: 0, cap: max_flags.saturating_mul(FLAG_TEXT_PER_FLAG) } + } + + fn add(&mut self, flags: &str) -> Result<(), String> { + self.bytes = self.bytes.saturating_add(flags.len()); + if self.bytes > self.cap { + return Err(format!("the stems' flags take more than {} bytes", self.cap)); + } + Ok(()) } } @@ -2133,6 +2796,30 @@ pub(crate) mod tests { AffixDict::from_hunspell(AFF, DIC, &["test data"]).unwrap() } + /// #495 review: the stems' flags are counted as they are read, so a `.dic` whose flags pass + /// the budget is refused while reading (its flag text) or while its flags are read (the + /// keys), before they are all held; one within it loads. + #[test] + fn stem_flags_are_budgeted_while_reading() { + let aff = "SET UTF-8\nSFX A Y 1\nSFX A 0 s .\nSFX B Y 1\nSFX B 0 x .\n"; + let many: String = (0..100).map(|i| format!("w{i}/AB\n")).collect(); + let dic = format!("100\n{many}"); + // 200 flags: over a budget of 50 keys (whose flag text, 300 bytes, still fits). + let e = AffixDict::from_hunspell_capped(aff, &dic, &[], false, 50).unwrap_err(); + assert!(e.contains("more than 50 flags"), "{e}"); + let d = AffixDict::from_hunspell_capped(aff, &dic, &[], false, 200).unwrap(); + assert!(d.check("w7s") && d.check("w99x")); + // Flag text past the budget (entries over the per-entry cap hold no keys, but their text + // still counts): refused while reading the `.dic`. + let long = "A".repeat(MAX_FLAGS_PER_ENTRY + 1); + let dic = format!("3\na/{long}\nb/{long}\nc/{long}\n"); + let e = AffixDict::from_hunspell_capped(aff, &dic, &[], false, 2000).unwrap_err(); + assert!(e.contains("more than 12000 bytes"), "{e}"); + assert!(AffixDict::from_hunspell(aff, &dic, &[]).is_ok(), "within the real budget: skipped entries, no error"); + let mut text = FlagText::new(1); + assert!(text.add("ABCDEF").is_ok() && text.add("G").is_err()); + } + #[test] fn affix_headers_allow_comments_but_rule_conditions_stay_literal() { let aff = "SET UTF-8\nPFX X Y 1 # prefix comment\nPFX X 0 ne . # rule comment\nSFX y Y 1 # suffix comment\nSFX y 0 y . # rule comment\n"; @@ -2994,6 +3681,193 @@ pub(crate) mod tests { assert!(d.check("neworkables")); } + #[test] + fn a_prefix_with_a_suffix_chain() { + // The CLI's answers (`meg` + `épít` + `és` + `e` in Hungarian): every rule must allow + // cross products, and the prefix's flag comes from the stem or a continuation class. + let rules = |p: char, a: char, b: char, a_cont: &str, b_cont: &str| { + format!("SET UTF-8\nPFX P {p} 1\nPFX P 0 un .\nSFX A {a} 1\nSFX A 0 a/B{a_cont} .\nSFX B {b} 1\nSFX B 0 b{b_cont} .\n") + }; + let d = |aff: &str, dic: &str| AffixDict::from_hunspell(aff, &format!("1\n{dic}\n"), &[]).unwrap(); + let yes = d(&rules('Y', 'Y', 'Y', "", ""), "foo/PA"); + assert!(yes.check("unfooab") && yes.check("unfooa") && yes.check("fooab")); + for (p, a, b) in [('Y', 'Y', 'N'), ('Y', 'N', 'Y'), ('N', 'Y', 'Y')] { + assert!(!d(&rules(p, a, b, "", ""), "foo/PA").check("unfooab"), "{p}{a}{b}"); + } + assert!(d(&rules('Y', 'Y', 'Y', "P", ""), "foo/A").check("unfooab"), "flag from the inner suffix"); + let outer = d(&rules('Y', 'Y', 'Y', "", "/P"), "foo/A"); + assert!(outer.check("unfooab") && !outer.check("unfooa"), "flag from the outer suffix"); + assert!(!d(&rules('Y', 'Y', 'Y', "", ""), "foo/A").check("unfooab"), "no prefix flag"); + } + + #[test] + fn a_prefix_and_a_suffix_come_from_one_homonym() { + // `stop/PA` and `stop/S`: `unstop` and `stops` are right, `unstops` wrong (the CLI's + // answers), also after a round trip through the compact form. + let aff = "SET UTF-8\nPFX P Y 1\nPFX P 0 un .\nSFX A Y 1\nSFX A 0 a/B .\nSFX B Y 1\nSFX B 0 b .\nSFX S Y 1\nSFX S 0 s .\n"; + let d = AffixDict::from_hunspell(aff, "3\nstop/PA\nstop/S\ngo/PS\n", &[]).unwrap(); + let again = AffixDict::from_bytes(&d.to_bytes().unwrap()).unwrap(); + for d in [d, again] { + assert!(d.check("unstop") && d.check("stops") && d.check("unstopa") && d.check("unstopab") && d.check("ungos")); + assert!(!d.check("unstops")); + } + } + + #[test] + fn a_word_in_capitals_may_be_a_mixed_case_stem() { + // The CLI's answers: `KB` and `KBS` from `kB/SU`, not `KBT`, `UNKB` or `Kb`. + let aff = "SET UTF-8\nSFX S Y 1\nSFX S 0 s .\nPFX U Y 1\nPFX U 0 un .\nSFX T Y 1\nSFX T 0 t .\n"; + let d = AffixDict::from_hunspell(aff, "2\nkB/SU\naBcD\n", &[]).unwrap(); + assert!(d.check("KB") && d.check("KBS") && d.check("kB") && d.check("ABCD")); + assert!(!d.check("KBT") && !d.check("UNKB") && !d.check("Kb") && !d.check("kb") && !d.check("Abcd")); + } + + #[test] + fn a_word_in_capitals_is_no_compound_of_a_mixed_case_stem() { + // The CLI's answers: `MSCS` and `ABC` right, `MSCFOO` and `ABCFOO` wrong. + let d = AffixDict::from_hunspell("SET UTF-8\nCOMPOUNDFLAG X\nSFX S Y 1\nSFX S 0 s .\n", "3\nMSc/SX\naBc/X\nfoo/X\n", &[]).unwrap(); + assert!(d.check("MSCS") && d.check("MSC") && d.check("ABC") && d.check("MScfoo")); + assert!(!d.check("MSCFOO") && !d.check("ABCFOO") && !d.check("FOOABC")); + } + + #[test] + fn homonyms_kept_apart_survive_the_compact_form() { + // `foo/XN` beside `foo/S`: `foos` right, `foobar` wrong (the CLI's answers), also after + // a round trip (the NEEDAFFIX flag is written). + let aff = "SET UTF-8\nCOMPOUNDFLAG X\nNEEDAFFIX N\nCOMPOUNDMIN 1\nSFX S Y 1\nSFX S 0 s .\n"; + let d = AffixDict::from_hunspell(aff, "3\nfoo/XN\nfoo/S\nbar/X\n", &[]).unwrap(); + let again = AffixDict::from_bytes(&d.to_bytes().unwrap()).unwrap(); + for d in [d, again] { + assert!(d.check("foos") && d.check("foo")); + assert!(!d.check("foobar") && !d.check("foosbar") && !d.check("barfoo")); + } + } + + #[test] + fn many_prefixes_adding_nothing_stay_cheap() { + // A thousand prefix rules that add nothing, and 40 chained suffix lengths of 50 flags + // each: every prefix leaves the same root, so they share one walk. + let mut aff = String::from("SET UTF-8\nFLAG num\n"); + for f in 1..=1000 { + aff.push_str(&format!("PFX {f} Y 1\nPFX {f} 0 0 .\n")); + } + let conts: Vec = (0..50).map(|k| (5000 + k * 7).to_string()).collect(); + for len in 1..=40 { + for k in 0..50 { + let f = 5000 + (len - 1) * 50 + k; + aff.push_str(&format!("SFX {f} Y 1\nSFX {f} 0 {}/{} .\n", "a".repeat(len), conts.join(","))); + } + } + let d = AffixDict::from_hunspell(&aff, "2\nb/1,5000\nab/2\n", &[]).unwrap(); + let t = std::time::Instant::now(); + for n in [40, 119] { + let _ = d.check(&"a".repeat(n)); + } + let _ = d.suggest(&format!("{}x", "a".repeat(39)), 8); + assert!(t.elapsed().as_secs() < 5, "{:?}", t.elapsed()); + } + + #[test] + fn prefixes_with_and_without_cross_products_on_one_root() { + // The CLI's answers: `redo` and `rework` right, from `c` (no cross products) beside `a` + // and `b` (cross products) adding the same `re`. + let aff = "SET UTF-8\nPFX a Y 1\nPFX a 0 re .\nPFX b Y 1\nPFX b 0 re .\nPFX c N 1\nPFX c 0 re .\nSFX S Y 1\nSFX S 0 s .\n"; + let d = AffixDict::from_hunspell(aff, "2\ndo/c\nwork/cS\n", &[]).unwrap(); + assert!(d.check("redo") && d.check("rework") && d.check("works")); + assert!(!d.check("reworks")); + let aff = "SET UTF-8\nFLAG num\nPFX 1 Y 1\nPFX 1 0 re .\nPFX 2 Y 1\nPFX 2 0 re .\nPFX 3 N 1\nPFX 3 0 re .\nPFX 4 N 1\nPFX 4 0 re .\nPFX 5 N 1\nPFX 5 0 re .\n"; + let d = AffixDict::from_hunspell(aff, "5\nfoo/3\nbar/4\nbaz/5\nqux/1\nquu/2\n", &[]).unwrap(); + assert!(["refoo", "rebar", "rebaz", "requx", "requu"].iter().all(|w| d.check(w))); + } + + #[test] + fn many_prefixes_with_their_own_roots_stay_cheap() { + // A thousand prefix rules each stripping its own letter (a thousand roots), and 40 + // chained suffix lengths of 50 flags each: one word's rules share one budget. + let mut aff = String::from("SET UTF-8\nFLAG num\n"); + for f in 1..=1000u32 { + let c = char::from_u32(0x4e00 + f).unwrap(); + aff.push_str(&format!("PFX {f} Y 1\nPFX {f} {c} 0 .\n")); + } + let conts: Vec = (0..50).map(|k| (5000 + k * 7).to_string()).collect(); + for len in 1..=40 { + for k in 0..50 { + let f = 5000 + (len - 1) * 50 + k; + aff.push_str(&format!("SFX {f} Y 1\nSFX {f} 0 {}/{} .\n", "a".repeat(len), conts.join(","))); + } + } + let d = AffixDict::from_hunspell(&aff, "2\nb/1,5000\nab/2\n", &[]).unwrap(); + let t = std::time::Instant::now(); + for n in [40, 119] { + let _ = d.check(&"a".repeat(n)); + } + let _ = d.suggest(&format!("{}x", "a".repeat(39)), 8); + assert!(t.elapsed().as_secs() < 10, "{:?}", t.elapsed()); + } + + #[test] + fn suggesting_with_a_long_try_line_shares_one_budget() { + // craft-libs#497 review: a thousand one-letter prefixes, 40 chained suffix lengths and + // a 128-letter TRY line. Each TRY candidate is a simple-word check; one suggest call + // shares one budget between them (17 s in a release build before CP7.7). + let mut aff = String::from("SET UTF-8\nFLAG num\nTRY "); + aff.extend(('a'..='z').chain((0x100..0x166).filter_map(char::from_u32))); + aff.push('\n'); + for f in 1..=1000u32 { + let c = char::from_u32(0x4e00 + f).unwrap(); + aff.push_str(&format!("PFX {f} Y 1\nPFX {f} {c} 0 .\n")); + } + let conts: Vec = (0..286).map(|k| (5000 + k * 7).to_string()).collect(); + for len in 1..=40 { + for k in 0..50 { + let f = 5000 + (len - 1) * 50 + k; + aff.push_str(&format!("SFX {f} Y 1\nSFX {f} 0 {}/{} .\n", "a".repeat(len), conts.join(","))); + } + } + assert_eq!(aff.lines().nth(2).map(|l| l.chars().count()), Some(4 + 128)); + let d = AffixDict::from_hunspell(&aff, "2\nb/1,5000\nab/2\n", &[]).unwrap(); + let t = std::time::Instant::now(); + let _ = d.suggest(&format!("{}x{}", "a".repeat(20), "a".repeat(19)), 8); + assert!(t.elapsed().as_secs() < 10, "{:?}", t.elapsed()); + } + + #[test] + fn a_check_out_of_steps_says_so() { + // craft-libs#497 review: the near-word check of a compound treats a candidate whose + // steps ran out as a word (the compound is refused), not as no word. + let d = AffixDict::from_hunspell("SET UTF-8\nSFX A Y 1\nSFX A 0 s .\n", "1\nkot/A\n", &[]).unwrap(); + let e = d.excluded(true); + assert_eq!(d.simple_word_budgeted("kots", e, true), Some(true)); + let _steps = SuggestSteps::start(); + SUGGEST_STEPS.with(|s| s.set(Some(0))); + assert_eq!(d.simple_word_budgeted("kots", e, true), None); + assert_eq!(d.simple_word_budgeted("kot", e, true), Some(true)); + } + + #[test] + fn many_stems_sharing_big_homonym_aliases_load_fast() { + // craft-libs#497 review: 16 homonyms per stem, each an `AF` alias of thousands of flags, + // on thousands of stems. Each stem's sets are worked out once per distinct entry list. + let n = 4000; + let mut aff = String::from("SET UTF-8\nFLAG num\nAF 16\n"); + for k in 0..16 { + let flags: Vec = (1..=n).filter(|&f| f != n - k).map(|f| f.to_string()).collect(); + aff.push_str(&format!("AF {}\n", flags.join(","))); + } + aff.push_str("SFX 1 Y 1\nSFX 1 0 s .\n"); + let stems: Vec = (0..2000).map(|i| format!("w{i}")).collect(); + let mut dic = format!("{}\n", stems.len() * 16); + for s in &stems { + for k in 1..=16 { + dic.push_str(&format!("{s}/{k}\n")); + } + } + let t = std::time::Instant::now(); + let d = AffixDict::from_hunspell(&aff, &dic, &[]).unwrap(); + assert!(t.elapsed().as_secs() < 10, "{:?}", t.elapsed()); + assert!(d.check("w7") && d.check("w1999s") && !d.check("w2000")); + } + #[test] fn hostile_compound_rules_are_refused_or_cheap() { for bad in ["COMPOUNDRULE *a", "COMPOUNDRULE (ab)", "COMPOUNDRULE a**", "COMPOUNDMIN x", "ONLYINCOMPOUND ab"] { @@ -3029,7 +3903,7 @@ pub(crate) mod tests { /// craft-libs#482 review: an English dictionary with compounding and output conversion, as the /// usual en_US and en_GB ones have, loads for English: `COMPOUNDRULE` makes its ordinals, its /// `ONLYINCOMPOUND` stems are words only inside them, `COMPOUNDFLAG` is ignored (other - /// languages still refuse it) and `OCONV` is applied to suggestions. (A synthetic `.aff` + /// languages read it) and `OCONV` is applied to suggestions. (A synthetic `.aff` /// written for this test.) #[test] fn english_dictionaries_ignore_compounding_and_output_conversion() { @@ -3037,7 +3911,7 @@ pub(crate) mod tests { COMPOUNDMIN 1\nONLYINCOMPOUND c\nCOMPOUNDRULE 2\nCOMPOUNDRULE n*1t\nCOMPOUNDRULE n*mp\nCOMPOUNDFLAG Z\n\ WORDCHARS 0123456789\nSFX S Y 1\nSFX S 0 s [^sxzhy]\nSFX D Y 1\nSFX D 0 ed [^ey]\n"; let dic = "7\nhouse/S\nwalk/SD\n0th/pc\n1st/p\n1th/tc\n1/n1\nthe\n"; - assert!(AffixDict::from_hunspell(aff, dic, &[]).is_err(), "refused for other languages"); + assert!(AffixDict::from_hunspell(&format!("{aff}COMPOUNDMORESUFFIXES\n"), dic, &[]).is_err(), "refused with COMPOUNDFLAG"); let d = AffixDict::from_hunspell_english(aff, dic, &[]).unwrap(); for w in ["house", "houses", "walked", "walks", "the"] { assert!(d.check(w), "{w}"); @@ -3073,12 +3947,12 @@ pub(crate) mod tests { assert!(AffixDict::from_hunspell("FLAG long\n", DIC, &[]).is_ok()); assert!(AffixDict::from_hunspell("FLAG num\n", DIC, &[]).is_ok()); assert!(AffixDict::from_hunspell("FLAG bogus\n", DIC, &[]).is_err()); - assert!(AffixDict::from_hunspell("COMPOUNDFLAG X\n", DIC, &[]).is_err()); + assert!(AffixDict::from_hunspell("COMPOUNDFLAG X\nCHECKCOMPOUNDPATTERN 1\nCHECKCOMPOUNDPATTERN a b c\n", DIC, &[]).is_err()); assert!(AffixDict::from_hunspell("SFX a Y 1\nSFX a 0 y [ab\n", DIC, &[]).is_err()); assert!(AffixDict::from_bytes(b"not deflate").is_err()); let ok = dict().to_bytes().unwrap(); assert!(AffixDict::from_bytes(&ok[..ok.len() / 2]).is_err()); - let garbage = miniz_oxide::deflate::compress_to_vec(b"WCAFF1\nWORDS\n\x09abc\n", 6); + let garbage = miniz_oxide::deflate::compress_to_vec(b"WCAFF2\nWORDS\n\x09abc\n", 6); assert!(AffixDict::from_bytes(&garbage).is_err(), "prefix longer than the previous word"); let empty = AffixDict::default(); assert!(!empty.contains("kot") && !empty.check("Kot") && empty.forms("kot").is_empty()); @@ -3118,7 +3992,7 @@ pub(crate) mod tests { let d = AffixDict::from_hunspell(NUM_AFF, NUM_DIC, &["num test"]).unwrap(); let bytes = d.to_bytes().unwrap(); let raw = miniz_oxide::inflate::decompress_to_vec(&bytes).unwrap(); - assert!(raw.starts_with(b"WCAFF1\nFLAG num\n"), "the mode is written before any flag"); + assert!(raw.starts_with(b"WCAFF2\nFLAG num\n"), "the mode is written before any flag"); let text = String::from_utf8_lossy(&raw); assert!(text.contains("FORBIDDENWORD 900\n") && text.contains("SFX 12 Y 0 om/1503 .\n") && text.contains("grad/7,12\n")); let back = AffixDict::from_bytes(&bytes).unwrap(); @@ -3177,7 +4051,7 @@ pub(crate) mod tests { assert_eq!(d.len(), 5); assert_long_dict(&d); let bytes = d.to_bytes().unwrap(); - assert!(miniz_oxide::inflate::decompress_to_vec(&bytes).unwrap().starts_with(b"WCAFF1\nFLAG long\n")); + assert!(miniz_oxide::inflate::decompress_to_vec(&bytes).unwrap().starts_with(b"WCAFF2\nFLAG long\n")); let back = AffixDict::from_bytes(&bytes).unwrap(); assert_long_dict(&back); assert_eq!(back.to_bytes().unwrap(), bytes); @@ -3470,15 +4344,18 @@ pub(crate) mod tests { #[test] fn compact_data_without_a_flag_line_still_loads() { - // Data written before the FLAG line existed: one-character flags, no FLAG line. - let old = miniz_oxide::deflate::compress_to_vec(b"WCAFF1\n# old data\nFORBIDDENWORD F\nSFX a Y 0 y .\nWORDS\n\x00kot/a\n\x00zly/F\n", 6); + // Data from before homonyms were kept apart (WCAFF1) is refused, not read with old answers. + let v1 = miniz_oxide::deflate::compress_to_vec(b"WCAFF1\nWORDS\n\x00kot\n", 6); + assert!(AffixDict::from_bytes(&v1).is_err()); + // One-character flags need no FLAG line. + let old = miniz_oxide::deflate::compress_to_vec(b"WCAFF2\n# old data\nFORBIDDENWORD F\nSFX a Y 0 y .\nWORDS\n\x00kot/a\n\x00zly/F\n", 6); let d = AffixDict::from_bytes(&old).unwrap(); assert!(d.check("kot") && d.check("koty") && !d.check("zly")); // One-character dictionaries still write no FLAG line, so their data is unchanged. let raw = miniz_oxide::inflate::decompress_to_vec(&dict().to_bytes().unwrap()).unwrap(); assert!(!String::from_utf8_lossy(&raw).contains("\nFLAG")); // A FLAG line after a flag, or an unknown one, is bad data. - for head in ["WCAFF1\nSFX a Y 0 y .\nFLAG num\n", "WCAFF1\nFLAG bogus\n"] { + for head in ["WCAFF2\nSFX a Y 0 y .\nFLAG num\n", "WCAFF2\nFLAG bogus\n"] { let bytes = miniz_oxide::deflate::compress_to_vec(format!("{head}WORDS\n\x00kot\n").as_bytes(), 6); assert!(AffixDict::from_bytes(&bytes).is_err(), "{head:?}"); } diff --git a/crates/proof/src/affix/compound.rs b/crates/proof/src/affix/compound.rs new file mode 100644 index 00000000..3530c095 --- /dev/null +++ b/crates/proof/src/affix/compound.rs @@ -0,0 +1,1992 @@ +//! Flag-based compounding, as the hunspell(5) man page describes it and the `hunspell` 1.7.5 CLI +//! answers on synthetic dictionaries (black-box; every rule below has a test pinned to the CLI's +//! answer in the tests below). +//! +//! A compound is two or more parts, each at least `COMPOUNDMIN` characters, each a stem or an +//! affixed form of one whose flags (the stem's, and the continuation classes of its affixes) +//! allow its place: `COMPOUNDFLAG` anywhere, `COMPOUNDBEGIN` first, `COMPOUNDMIDDLE` inside, +//! `COMPOUNDEND` last. (`COMPOUNDFIRST` and `COMPOUNDLAST` are ignored, as the CLI does.) +//! +//! - A prefix may sit on any part but the last and a suffix only on the last, unless the affix +//! carries `COMPOUNDPERMITFLAG`. Two suffixes in a row (continuation classes) only on the last. +//! - An affix carrying `ONLYINCOMPOUND` is only valid inside a compound: a prefix anywhere, a +//! suffix anywhere but on the last part (the CLI's answers; the word alone is no word). +//! - `COMPOUNDFORBIDFLAG` on a stem or a suffix: that part may only be last; on a prefix: no part. +//! - A stem with `NEEDAFFIX` is a part only with an affix; `FORBIDDENWORD` stems are no part, and +//! a forbidden word is never rebuilt from parts. +//! - `COMPOUNDWORDMAX`: most parts (a `COMPOUNDROOT` stem counts as two). With `COMPOUNDSYLLABLE` +//! and `LANG hu`, more parts are allowed while the word has at most that many vowels; with +//! `COMPOUNDSYLLABLE` and another language the CLI applies no limit, and neither does this. +//! - `CHECKCOMPOUNDDUP`: the last two parts may not be the same stem (`foofoo`; `foofoobar` is +//! fine, as in the CLI). +//! - `CHECKCOMPOUNDCASE`: no capital on either side of a boundary, unless the other side is `-`. +//! - `CHECKCOMPOUNDTRIPLE`: no three equal letters across a boundary (`schiff|fahrt`). +//! `SIMPLIFIEDTRIPLE`: a part ending in a double letter may share its last letter with the next +//! part (`schiffahrt` = `schiff` + `fahrt`), but with `CHECKCOMPOUNDTRIPLE` not leave three +//! equal letters (`busssen`). +//! - `CHECKCOMPOUNDPATTERN end[/flag] begin[/flag]`: no boundary where the part before ends with +//! `end` and the part after begins with `begin` (and they carry the flags); `0` for `end` means +//! a part without a suffix. The optional replacement field is refused. +//! - `CHECKCOMPOUNDREP`: no compound that a `REP` replacement turns into a word (not a compound), +//! checked on every two neighbouring parts and on every run of parts that ends the word. +//! - A compound of three or more parts is rejected when one slip turns it into a word (not a +//! compound): see [`AffixDict::near_word`]. The CLI's answers, with or without +//! `CHECKCOMPOUNDREP`. +//! - `FORCEUCASE` on the last part's stem: a word with no capital is wrong. +//! - Case: `KEEPCASE` holds for the first part only; a capitalized stem may be a part, and a word +//! with capitals inside may start with a capital (`FooQux`). +//! - `ONLYINCOMPOUND` homonyms beside others keep their own flags (compound parts only). +//! - `MAXCPDSUGS 0`: compounds are never suggested. +//! +//! Not read: `SYLLABLENUM` (Hungarian), which the man page doesn't describe; it is ignored. +//! +//! The work per word is bounded: words up to [`MAX_WORD_BYTES`](super::MAX_WORD_BYTES), at most +//! [`MAX_PARTS`] parts, each part's analyses computed once, and at most [`MAX_WORK`] steps (rules +//! tried and splits followed) per word; past that the word is no compound. + +use super::{AffixDict, Excluded, FlagKey, FlagMode, Rule, has}; + +/// Most parts a compound may have, whatever `COMPOUNDWORDMAX` says. +pub(super) const MAX_PARTS: usize = 16; +/// Most steps one compound check may take (affix rules tried, parts looked up, splits followed). +pub(super) const MAX_WORK: usize = 1 << 16; +/// Most steps the compound search of one suggestion candidate may take. +pub(crate) const MAX_SUGGEST_WORK: usize = 1 << 13; +/// Most steps the compound searches of one suggestion call may take in all; the candidates +/// left are checked as simple words only, so a long garbled word's suggestions stay quick. +pub(crate) const MAX_SUGGEST_COMPOUND_WORK: usize = 1 << 17; +/// Most `CHECKCOMPOUNDPATTERN` lines (the Dutch dictionary has 52). +const MAX_PATTERNS: usize = 512; +/// Longest text on either side of a `CHECKCOMPOUNDPATTERN`, in bytes. +const MAX_PATTERN_BYTES: usize = 64; +/// Most vowels a `COMPOUNDSYLLABLE` line may list. +const MAX_VOWELS: usize = 64; +/// Most ways one piece of a word may be a part (distinct flags and places). +const MAX_INFOS: usize = 256; +/// Most `MAP` variants, `TRY` letters and `KEY` row letters the three-part check tries. +const MAX_MAP_VARIANTS: usize = 256; +const MAX_TRY: usize = 128; +const MAX_KEY_ROW: usize = 128; +/// The keyboard rows the CLI assumes when a dictionary has no `KEY`. +const DEFAULT_KEY: &str = "qwertyuiop|asdfghjkl|zxcvbnm"; + +/// The part may come first, in the middle, last. +const FIRST: u8 = 1; +const MIDDLE: u8 = 2; +const LAST: u8 = 4; + +/// The single-flag compounding directives, in the order [`Raw::flags`] keeps them. +const FLAG_DIRECTIVES: [&str; 8] = + ["COMPOUNDFLAG", "COMPOUNDBEGIN", "COMPOUNDMIDDLE", "COMPOUNDEND", "COMPOUNDPERMITFLAG", "COMPOUNDFORBIDFLAG", "COMPOUNDROOT", "FORCEUCASE"]; +const FLAG: usize = 0; +const BEGIN: usize = 1; +const MIDDLE_FLAG: usize = 2; +const END: usize = 3; +const PERMIT: usize = 4; +const FORBID: usize = 5; +const ROOT: usize = 6; +const FORCE_UCASE: usize = 7; + +/// The yes/no compounding directives, in the order [`Raw::switches`] keeps them. +const SWITCHES: [&str; 5] = ["CHECKCOMPOUNDDUP", "CHECKCOMPOUNDREP", "CHECKCOMPOUNDCASE", "CHECKCOMPOUNDTRIPLE", "SIMPLIFIEDTRIPLE"]; +const DUP: usize = 0; +const REP: usize = 1; +const CASE: usize = 2; +const TRIPLE: usize = 3; +const SIMPLIFIED: usize = 4; + +/// Compounding directives as read from the `.aff` or the compact format, before flags get ids. +#[derive(Clone, Debug, Default)] +pub(super) struct Raw { + flags: [Option; 8], + switches: [bool; 5], + word_max: Option, + /// `COMPOUNDSYLLABLE`: most vowels, and the vowels. + syllable: Option<(usize, String)>, + /// `LANG hu…`: the Hungarian syllable rule applies. + hungarian: bool, + /// `CHECKCOMPOUNDPATTERN` lines, as written (`end[/flag] begin[/flag]`). + patterns: Vec<(String, String)>, + pattern_header: bool, + /// `MAXCPDSUGS 0`. + no_suggestions: bool, + /// `NOSPLITSUGS`. + no_split_suggestions: bool, + /// `COMPOUNDMORESUFFIXES`: refused with flag-based compounding, which it would change; + /// ignored without (Korean has it beside `COMPOUNDRULE`s only). + more_suffixes: bool, +} + +impl Raw { + /// Read the compounding directive `dir` of the line `f` (its fields); `Ok(false)` when `dir` + /// is none. `one_flag` reads a flag in the dictionary's `FLAG` mode. + pub(super) fn read(&mut self, f: &[&str], one_flag: &mut impl FnMut(&str) -> Result) -> Result { + let Some(&dir) = f.first() else { return Ok(false) }; + let at = |k: usize| f.get(k).copied().unwrap_or(""); + if let Some(i) = FLAG_DIRECTIVES.iter().position(|&d| d == dir) { + let k = one_flag(at(1)).map_err(|e| format!("{dir} needs one flag: {e}"))?; + if let Some(slot) = self.flags.get_mut(i) { + *slot = Some(k); + } + return Ok(true); + } + if let Some(i) = SWITCHES.iter().position(|&d| d == dir) { + if let Some(s) = self.switches.get_mut(i) { + *s = true; + } + return Ok(true); + } + match dir { + // `COMPOUNDWORDMAX 2 y` (Hungarian): the number is the first field. + "COMPOUNDWORDMAX" => { + let n = at(1).parse::().map_err(|_| "COMPOUNDWORDMAX needs a number".to_string())?; + self.word_max = Some(n.clamp(1, MAX_PARTS)); + } + "COMPOUNDSYLLABLE" => { + let n = at(1).parse::().map_err(|_| "COMPOUNDSYLLABLE needs a number".to_string())?; + let vowels: String = at(2).chars().take(MAX_VOWELS).collect(); + self.syllable = Some((n.min(super::MAX_WORD_BYTES), vowels)); + } + "LANG" => self.hungarian = at(1) == "hu" || at(1).starts_with("hu_") || at(1).starts_with("hu-"), + "MAXCPDSUGS" => self.no_suggestions = at(1) == "0", + "NOSPLITSUGS" => self.no_split_suggestions = true, + "COMPOUNDMORESUFFIXES" => self.more_suffixes = true, + "CHECKCOMPOUNDPATTERN" => self.read_pattern(f)?, + _ => return Ok(false), + } + Ok(true) + } + + /// A `CHECKCOMPOUNDPATTERN` line: the count line, or `end[/flag] begin[/flag]` with an + /// optional comment (`# …`). A replacement field is refused: no Language pack uses one. + fn read_pattern(&mut self, f: &[&str]) -> Result<(), String> { + let fields: Vec<&str> = f.iter().skip(1).copied().take_while(|x| !x.starts_with('#')).collect(); + if !self.pattern_header && fields.len() == 1 && fields.first().is_some_and(|x| x.parse::().is_ok()) { + self.pattern_header = true; + return Ok(()); + } + let [end, begin] = fields.as_slice() else { + return Err(format!("CHECKCOMPOUNDPATTERN `{}`: only `end[/flag] begin[/flag]` is supported", fields.join(" "))); + }; + if end.len() > MAX_PATTERN_BYTES || begin.len() > MAX_PATTERN_BYTES || self.patterns.len() >= MAX_PATTERNS { + return Err(format!("CHECKCOMPOUNDPATTERN: over {MAX_PATTERNS} patterns or {MAX_PATTERN_BYTES} bytes")); + } + self.patterns.push(((*end).to_string(), (*begin).to_string())); + Ok(()) + } + + /// Whether any directive makes words compound by flags (the rest only refine that). + fn compounds(&self) -> bool { + [FLAG, BEGIN, MIDDLE_FLAG, END].iter().any(|&i| self.flags.get(i).is_some_and(Option::is_some)) + } + + /// Every flag the directives and patterns name, so that each gets an id. + pub(super) fn keys(&self, mode: FlagMode) -> Result, String> { + let mut keys: Vec = self.flags.iter().flatten().copied().collect(); + for (end, begin) in &self.patterns { + for side in [end, begin] { + if let Some((_, flag)) = side.split_once('/') { + keys.push(mode.parse_one(flag).map_err(|e| format!("CHECKCOMPOUNDPATTERN `{side}`: {e}"))?); + } + } + } + Ok(keys) + } + + /// The compounding of a dictionary whose flags got ids by `id_of`; `None` when it doesn't + /// compound by flags. + pub(super) fn build(&self, mode: FlagMode, id_of: &impl Fn(FlagKey) -> Option) -> Result>, String> { + if !self.compounds() { + return Ok(None); + } + if self.more_suffixes { + return Err("COMPOUNDMORESUFFIXES is not supported with flag-based compounding".into()); + } + let id = |i: usize| self.flags.get(i).copied().flatten().and_then(id_of); + let switch = |i: usize| self.switches.get(i).copied().unwrap_or(false); + let mut c = Compounding { + flag: id(FLAG), + begin: id(BEGIN), + middle: id(MIDDLE_FLAG), + end: id(END), + permit: id(PERMIT), + forbid: id(FORBID), + root: id(ROOT), + force_ucase: id(FORCE_UCASE), + dup: switch(DUP), + rep: switch(REP), + case: switch(CASE), + triple: switch(TRIPLE), + simplified: switch(SIMPLIFIED), + no_suggestions: self.no_suggestions, + two_words: !self.no_split_suggestions, + raw: self.clone(), + ..Compounding::default() + }; + // `COMPOUNDSYLLABLE` turns `COMPOUNDWORDMAX` into the Hungarian syllable rule, or (other + // languages) lifts it: the CLI's answers. + match (&self.syllable, self.hungarian) { + (None, _) => c.word_max = self.word_max, + (Some((n, vowels)), true) => { + c.word_max = self.word_max; + c.syllables = Some((*n, vowels.chars().collect())); + } + (Some(_), false) => {} + } + if let Some(f) = c.force_ucase { + c.marks.push(f); + } + for (end, begin) in &self.patterns { + let side = |text: &str, c: &mut Compounding| -> Result<(String, Need), String> { + let (chars, flag) = text.split_once('/').map_or((text, None), |(a, b)| (a, Some(b))); + let need = match flag { + None => Need::Any, + Some(flag) => match id_of(mode.parse_one(flag)?) { + Some(f) => Need::Mark(c.mark(f)?), + None => Need::Never, + }, + }; + Ok((chars.to_string(), need)) + }; + let (end_chars, end_need) = side(end, &mut c)?; + let (begin, begin_need) = side(begin, &mut c)?; + let unmodified = end_chars == "0"; + let end = if unmodified { String::new() } else { end_chars }; + c.patterns.push(Pattern { end: end.into(), end_need, begin: begin.into(), begin_need, unmodified }); + } + Ok(Some(Box::new(c))) + } +} + +/// What a side of a `CHECKCOMPOUNDPATTERN` asks of a part's flags. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Need { + Any, + /// The flag of [`Compounding::marks`] at this index. + Mark(u8), + /// A flag no stem or rule carries. + Never, +} + +impl Need { + fn met(self, marks: u64) -> bool { + match self { + Need::Any => true, + Need::Mark(i) => marks & (1 << i) != 0, + Need::Never => false, + } + } +} + +#[derive(Clone, Debug)] +struct Pattern { + end: Box, + end_need: Need, + begin: Box, + begin_need: Need, + /// `0` for `end`: the part before is a stem without affixes. + unmodified: bool, +} + +/// The flag-based compounding of a dictionary (see the module docs). +#[derive(Clone, Debug, Default)] +pub(super) struct Compounding { + flag: Option, + begin: Option, + middle: Option, + end: Option, + permit: Option, + forbid: Option, + root: Option, + force_ucase: Option, + word_max: Option, + /// The Hungarian rule: past `word_max` parts, at most this many vowels in the word. + syllables: Option<(usize, Box<[char]>)>, + dup: bool, + rep: bool, + case: bool, + triple: bool, + simplified: bool, + pub(super) no_suggestions: bool, + /// A compound of three or more parts that splits into two words is wrong (unless + /// `NOSPLITSUGS`: the CLI ties the two to each other). + two_words: bool, + patterns: Vec, + /// Flags tracked per part (`FORCEUCASE`, the patterns' flags): bit `i` of [`Info::marks`]. + marks: Vec, + /// The directives as read, written back by [`Compounding::write`]. + raw: Raw, +} + +impl Compounding { + /// The mark index of `flag`, adding it. + fn mark(&mut self, flag: u16) -> Result { + let i = match self.marks.iter().position(|&m| m == flag) { + Some(i) => i, + None => { + self.marks.push(flag); + self.marks.len() - 1 + } + }; + u8::try_from(i).ok().filter(|&i| i < 64).ok_or_else(|| "CHECKCOMPOUNDPATTERN: more than 63 distinct flags".to_string()) + } + + /// The directives as compact-format lines (flags written by `write_flag`). + pub(super) fn write(&self, out: &mut String, write_flag: &impl Fn(FlagKey, &mut String)) { + let raw = &self.raw; + if raw.hungarian { + out.push_str("LANG hu\n"); + } + for (name, key) in FLAG_DIRECTIVES.iter().zip(&raw.flags) { + if let Some(k) = key { + out.push_str(name); + out.push(' '); + write_flag(*k, out); + out.push('\n'); + } + } + for (name, on) in SWITCHES.iter().zip(&raw.switches) { + if *on { + out.push_str(name); + out.push('\n'); + } + } + if let Some(n) = raw.word_max { + out.push_str(&format!("COMPOUNDWORDMAX {n}\n")); + } + if let Some((n, vowels)) = &raw.syllable { + out.push_str(&format!("COMPOUNDSYLLABLE {n} {vowels}\n")); + } + if raw.no_suggestions { + out.push_str("MAXCPDSUGS 0\n"); + } + if raw.no_split_suggestions { + out.push_str("NOSPLITSUGS\n"); + } + for (end, begin) in &raw.patterns { + out.push_str(&format!("CHECKCOMPOUNDPATTERN {end} {begin}\n")); + } + } +} + +/// One way a piece of the word can be a compound part. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +struct Info { + /// Where it may sit: [`FIRST`], [`MIDDLE`], [`LAST`]. + pos: u8, + /// Which of [`Compounding::marks`] its flags hold. + marks: u64, + /// No suffix (`CHECKCOMPOUNDPATTERN`'s `0`). + unmodified: bool, + /// The stem carries `FORCEUCASE`. + force: bool, + /// The stem's index (`CHECKCOMPOUNDDUP` compares stems). + stem: u32, + /// How many words it counts as (`COMPOUNDROOT`: two). + words: u8, + /// Syllables of its suffix the Hungarian rule leaves out (as the last part): a suffix with no + /// continuation classes doesn't count. + tail: u8, +} + +/// A stem found for a part, and the affixes that make the part from it. +#[derive(Clone, Copy)] +struct Analysis<'a> { + stem: u32, + set: &'a [u16], + pfx: Option<&'a Rule>, + sfx: Option<&'a Rule>, + /// The inner suffix of two in a row (`sfx` is the outer one). + inner: Option<&'a Rule>, +} + +/// Most pieces a [`Pieces`] cache keeps. +const MAX_CACHED_PIECES: usize = 1 << 14; + +/// The infos of pieces, kept across the compound checks of one suggestion call: its candidates +/// differ from the word by a letter or two, so most of their pieces are the same. Indexed by +/// whether the piece starts the word (KEEPCASE holds there). +#[derive(Default)] +pub(crate) struct Pieces([crate::patterns::FxMap, Box<[Info]>>; 2], usize); + +/// One compound check: the word, its parts' analyses (each piece looked up once), the parts +/// chosen so far and the work done. +struct Search<'a> { + c: &'a Compounding, + w: &'a str, + /// Byte offset of each char, and `w.len()`. + cuts: Vec, + chars: Vec, + excluded: Excluded, + /// The first part's (KEEPCASE holds there). + first_excluded: Excluded, + capitalized: bool, + /// Per piece `(start, end)` in chars: its infos as a range of `infos`. + memo: Vec>, + infos: Vec, + /// The parts so far: start and end char, and the info. + path: Vec<(usize, usize, Info)>, + work: usize, + /// [`AffixDict::near_word`], once answered. + near: Option, + /// Pieces analysed by earlier checks with the same exclusions. + cache: Option<&'a mut Pieces>, +} + +impl AffixDict { + /// Is `w` a compound by the flag-based rules (see the module docs)? `capitalized`: the word + /// as the user wrote it starts with a capital (`FORCEUCASE`). + pub(super) fn flag_compound(&self, w: &str, excluded: Excluded, capitalized: bool) -> bool { + // A suggestion candidate's search gets MAX_SUGGEST_WORK steps. + let mut budget = if excluded.suggesting() { MAX_SUGGEST_WORK } else { MAX_WORK }; + self.flag_compound_spending(w, excluded, capitalized, &mut budget, None) + } + + /// [`AffixDict::flag_compound`] in at most `budget` steps (and [`MAX_WORK`]), taking the + /// steps it spends off `budget`. `cache`: pieces analysed before, with the same `excluded`. + pub(super) fn flag_compound_spending( + &self, + w: &str, + excluded: Excluded, + capitalized: bool, + budget: &mut usize, + cache: Option<&mut Pieces>, + ) -> bool { + let Some(c) = self.cpd.as_deref() else { return false }; + if excluded.no_compounds() || w.is_empty() || w.len() > super::MAX_WORD_BYTES || *budget == 0 { + return false; + } + if self.is_forbidden(w) { + return false; + } + // KEEPCASE holds for the first part only (from `baz/K`, `Foobaz` and `FOOBAZ` are right + // in the CLI, `Bazfoo` and `BAZFOO` wrong). + let first_excluded = excluded; + let excluded = excluded.without_keep(); + let cuts: Vec = w.char_indices().map(|(i, _)| i).chain(std::iter::once(w.len())).collect(); + let chars: Vec = w.chars().collect(); + let n = cuts.len(); + let mut s = Search { + c, + w, + cuts, + chars, + excluded, + first_excluded, + capitalized, + memo: vec![None; n * n], + infos: Vec::new(), + path: Vec::with_capacity(MAX_PARTS), + work: MAX_WORK - (*budget).min(MAX_WORK), + near: None, + cache, + }; + let start = s.work; + let found = self.parts_from(&mut s, 0); + *budget = budget.saturating_sub(s.work.saturating_sub(start).max(1)); + found + } + + /// Where a slip in `w` (no compound) likely is, in chars: from the furthest point its start + /// reaches as compound parts to the nearest its end does, if anything reaches. A candidate + /// changed there is checked as a compound first. `capitalized`: the same with a capital + /// first letter counts too. + pub(crate) fn compound_gap(&self, w: &str, capitalized: bool, budget: &mut usize, pieces: &mut Pieces) -> Option<(usize, usize)> { + let c = self.cpd.as_deref()?; + if w.is_empty() || w.len() > super::MAX_WORD_BYTES || *budget == 0 { + return None; + } + let excluded = self.excluded(true); + let n = w.chars().count(); + let min = self.compound_min.max(1); + let mut reach = vec![false; n + 1]; + let mut back = vec![false; n + 1]; + let cap = capitalized.then(|| super::capitalize(w)); + for word in std::iter::once(w).chain(cap.as_deref()) { + let mut s = Search { + c, + w: word, + cuts: word.char_indices().map(|(i, _)| i).chain(std::iter::once(word.len())).collect(), + chars: word.chars().collect(), + excluded: excluded.without_keep(), + first_excluded: excluded, + capitalized, + memo: vec![None; (n + 1) * (n + 1)], + infos: Vec::new(), + path: Vec::new(), + work: MAX_WORK - (*budget).min(MAX_WORK), + near: None, + cache: Some(&mut *pieces), + }; + let start = s.work; + let fits = |s: &mut Search, a: usize, b: usize, place: u8| { + let (x, y) = self.infos_of(s, a, b); + s.infos.get(x as usize..y as usize).is_some_and(|infos| infos.iter().any(|i| i.pos & place != 0)) + }; + for a in 0..n { + if a > 0 && !reach.get(a).copied().unwrap_or(false) { + continue; + } + for b in a.saturating_add(min)..n { + if s.work <= MAX_WORK + && fits(&mut s, a, b, if a == 0 { FIRST } else { MIDDLE }) + && let Some(r) = reach.get_mut(b) + { + *r = true; + } + } + } + for b in (1..=n).rev() { + if b < n && !back.get(b).copied().unwrap_or(false) { + continue; + } + for a in 1..=b.saturating_sub(min) { + if s.work <= MAX_WORK + && fits(&mut s, a, b, if b == n { LAST } else { MIDDLE }) + && let Some(r) = back.get_mut(a) + { + *r = true; + } + } + } + *budget = budget.saturating_sub(s.work.saturating_sub(start)); + } + let from = reach.iter().rposition(|&r| r).unwrap_or(0); + let to = back.iter().position(|&r| r).unwrap_or(n); + Some((from.min(to), from.max(to))) + } + + /// Can the rest of the word, from char `start`, be split into parts after `s.path`? + fn parts_from(&self, s: &mut Search, start: usize) -> bool { + let total = s.chars.len(); + let count = s.path.len(); + let min = self.compound_min.max(1); + for end in start.saturating_add(min)..=total { + s.work += 1; + if s.work > MAX_WORK { + return false; + } + let last = end == total; + if !last && count + 1 >= MAX_PARTS { + break; + } + let place = if last { + LAST + } else if count == 0 { + FIRST + } else { + MIDDLE + }; + let (a, b) = self.infos_of(s, start, end); + for k in a..b { + // Each way of the piece costs a step, and a step per pattern at the boundary. + s.work += 1 + s.c.patterns.len(); + if s.work > MAX_WORK { + return false; + } + let Some(info) = s.infos.get(k as usize).copied() else { continue }; + if info.pos & place == 0 || !self.boundary_ok(s, start, &info) { + continue; + } + s.path.push((start, end, info)); + let ok = if last { self.whole_ok(s) } else { self.words_ok(s, false) && self.next_parts(s, end) }; + s.path.pop(); + if ok { + return true; + } + } + } + false + } + + /// After a part ending at char `end`: the next part starts there, or (`SIMPLIFIEDTRIPLE`) + /// one char earlier when the part ends in a double letter. + fn next_parts(&self, s: &mut Search, end: usize) -> bool { + if self.parts_from(s, end) { + return true; + } + let double = end >= 2 && s.chars.get(end - 1).is_some_and(|c| s.chars.get(end - 2) == Some(c)); + s.c.simplified && double && self.parts_from(s, end - 1) + } + + /// The analyses of the piece from char `a` to char `b`, computed once: a range of `s.infos`. + fn infos_of(&self, s: &mut Search, a: usize, b: usize) -> (u32, u32) { + let n = s.cuts.len(); + let slot = a.saturating_mul(n).saturating_add(b); + if let Some(Some(r)) = s.memo.get(slot) { + return *r; + } + let from = u32::try_from(s.infos.len()).unwrap_or(u32::MAX); + let cached = s.cuts.get(a).zip(s.cuts.get(b)).and_then(|(&x, &y)| s.w.get(x..y)).and_then(|piece| { + let map = s.cache.as_deref()?.0.get(usize::from(a == 0))?; + map.get(piece).map(|v| v.to_vec()) + }); + if let Some(v) = cached { + s.work += 1; + s.infos.extend(v); + } else if let (Some(&x), Some(&y)) = (s.cuts.get(a), s.cuts.get(b)) + && let Some(piece) = s.w.get(x..y) + { + // Deduplicated by hashing, and at most MAX_INFOS per piece (each one is tried at + // every split, against every pattern). + let mut seen: std::collections::HashSet> = Default::default(); + let mut found: Vec = Vec::new(); + let mut work = s.work; + let excluded = if a == 0 { s.first_excluded } else { s.excluded }; + self.analyses(piece, excluded, &mut work, &mut |an: Analysis| { + if found.len() < MAX_INFOS + && let Some(info) = self.info(s.c, &an) + && seen.insert(info) + { + found.push(info); + } + }); + s.work = work; + // Only a piece analysed in full is kept (one cut short by the budget isn't). + if work <= MAX_WORK + && let Some(c) = s.cache.as_deref_mut() + && c.1 < MAX_CACHED_PIECES + && let Some(map) = c.0.get_mut(usize::from(a == 0)) + { + map.insert(piece.into(), found.clone().into()); + c.1 += 1; + } + s.infos.extend(found); + } + let to = u32::try_from(s.infos.len()).unwrap_or(u32::MAX); + if let Some(m) = s.memo.get_mut(slot) { + *m = Some((from, to)); + } + (from, to) + } + + /// Where the analysis `an` lets its piece sit in a compound, if anywhere. + fn info(&self, c: &Compounding, an: &Analysis) -> Option { + let sets = [Some(an.set), an.pfx.map(|r| self.set(r.cont)), an.sfx.map(|r| self.set(r.cont)), an.inner.map(|r| self.set(r.cont))]; + let carries = |f: Option| f.is_some_and(|f| sets.iter().flatten().any(|s| has(s, f))); + // COMPOUNDFORBIDFLAG on a prefix: no part anywhere (`bar|defoo` is wrong in the CLI). + if an.pfx.is_some_and(|r| self.cont_holds(r, c.forbid)) { + return None; + } + let mut pos = 0; + if carries(c.flag) { + pos = FIRST | MIDDLE | LAST; + } + for (flag, p) in [(c.begin, FIRST), (c.middle, MIDDLE), (c.end, LAST)] { + if carries(flag) { + pos |= p; + } + } + // A prefix without COMPOUNDPERMITFLAG: any part but the last (`foo|unfoo|bar` is right + // in the CLI, `foo|unfoo` wrong). + if an.pfx.is_some_and(|r| !self.cont_holds(r, c.permit)) { + pos &= !LAST; + } + if let Some(r) = an.sfx { + if !self.cont_holds(r, c.permit) { + pos &= LAST; + } + if self.cont_holds(r, self.only_in_compound) { + pos &= !LAST; + } + } + if an.inner.is_some() { + pos &= LAST; + } + if carries(c.forbid) { + pos &= LAST; + } + if pos == 0 { + return None; + } + let mut marks = 0u64; + for (i, &m) in c.marks.iter().enumerate().take(64) { + if carries(Some(m)) { + marks |= 1 << i; + } + } + // COMPOUNDROOT and FORCEUCASE count on the stem only (an affix's continuation class + // holding them is ignored); + // a pattern's `0` means no suffix (a prefix doesn't count): the CLI's answers. + let words = if c.root.is_some_and(|f| has(an.set, f)) { 2 } else { 1 }; + let force = c.force_ucase.is_some_and(|f| has(an.set, f)); + // The Hungarian syllable rule leaves out a suffix with no continuation classes (the + // CLI's answers: `bab|bab|baata` from `ata` is right, from `ata/Z` wrong). + let tail = match (&c.syllables, an.sfx) { + (Some((_, vowels)), Some(r)) if self.set(r.cont).is_empty() => r.add.chars().filter(|ch| vowels.contains(ch)).count(), + _ => 0, + }; + let tail = u8::try_from(tail).unwrap_or(u8::MAX); + Some(Info { pos, marks, unmodified: an.sfx.is_none(), force, stem: an.stem, words, tail }) + } + + /// Every way `p` is a stem, or a stem with a prefix, one or two suffixes, or a prefix and a + /// suffix, as a compound part may be (`ONLYINCOMPOUND` affixes and stems included). + fn analyses<'a>(&'a self, p: &str, excluded: Excluded, work: &mut usize, emit: &mut impl FnMut(Analysis<'a>)) { + for (stem, set) in self.part_stems(p, excluded) { + if !self.need_affix.is_some_and(|n| has(set, n)) { + emit(Analysis { stem, set, pfx: None, sfx: None, inner: None }); + } + } + self.suffix_analyses(p, None, excluded, work, emit); + let mut root = String::with_capacity(p.len() + 8); + let ends = std::iter::once(0).chain(p.char_indices().map(|(i, c)| i + c.len_utf8())); + for b in ends.take(self.max_pfx_chars + 1) { + let (Some(add), Some(rest)) = (p.get(..b), p.get(b..)) else { continue }; + if rest.is_empty() && !self.full_strip { + continue; + } + let Some(ids) = self.pfx_index.get(add) else { continue }; + for r in ids.iter().filter_map(|&i| self.prefixes.get(i as usize)) { + *work += 1; + if *work > MAX_WORK { + return; + } + root.clear(); + root.push_str(&r.strip); + root.push_str(rest); + if !r.cond_matches(&root, false) { + continue; + } + let (circ, needs) = (self.cont_holds(r, self.circumfix), self.cont_holds(r, self.need_affix)); + if !circ && !needs { + for (stem, set) in self.part_stems(&root, excluded) { + if has(set, r.flag) { + emit(Analysis { stem, set, pfx: Some(r), sfx: None, inner: None }); + } + } + } + if r.cross { + self.suffix_analyses(&root, Some(r), excluded, work, emit); + } + } + } + } + + /// The analyses of `p` that end in a suffix: on a stem (after the prefix `pfx`, if any), or + /// (without a prefix) on a form of another suffix. + fn suffix_analyses<'a>(&'a self, p: &str, pfx: Option<&'a Rule>, excluded: Excluded, work: &mut usize, emit: &mut impl FnMut(Analysis<'a>)) { + let mut root = String::with_capacity(p.len() + 8); + let cuts = std::iter::once(p.len()).chain(p.char_indices().rev().map(|(i, _)| i).filter(|&i| i > 0 || self.full_strip)); + for b in cuts.take(self.max_sfx_chars + 1) { + let (Some(base), Some(add)) = (p.get(..b), p.get(b..)) else { continue }; + let Some(groups) = self.sfx_index.get(add) else { continue }; + for g in groups { + root.clear(); + root.push_str(base); + root.push_str(&g.strip); + let rules = || g.rules.iter().filter_map(|&i| self.suffixes.get(i as usize)); + for (stem, set) in self.part_stems(&root, excluded) { + for r in rules() { + *work += 1; + if *work > MAX_WORK { + return; + } + if self.suffix_fits(r, set, pfx) && r.cond_matches(&root, true) { + emit(Analysis { stem, set, pfx, sfx: Some(r), inner: None }); + } + } + } + // Two suffixes in a row: the outer ones that another suffix's continuation + // classes can license, looked for together under one inner scan. + if pfx.is_none() && !g.outer_rules.is_empty() { + let outers: Vec<&Rule> = g + .outer_rules + .iter() + .filter_map(|&i| self.suffixes.get(i as usize)) + .filter(|r| !self.cont_holds(r, self.only_in_compound) && r.cond_matches(&root, true)) + .collect(); + *work += outers.len(); + if !outers.is_empty() { + self.inner_analyses(&root, &outers, excluded, work, emit); + } + } + } + } + } + + /// Does the suffix `r` apply to a stem with the flags `set`, after the prefix `pfx` (if any)? + /// Circumfix alike; a suffix that needs another affix only after a prefix that doesn't. + fn suffix_fits(&self, r: &Rule, set: &[u16], pfx: Option<&Rule>) -> bool { + if !has(set, r.flag) { + return false; + } + let circ = self.cont_holds(r, self.circumfix); + let needs = self.cont_holds(r, self.need_affix); + match pfx { + None => !circ && !needs, + Some(p) => { + r.cross + && (has(set, p.flag) || self.cont_holds(r, Some(p.flag))) + && circ == self.cont_holds(p, self.circumfix) + && !(needs && self.cont_holds(p, self.need_affix)) + } + } + } + + /// `form` (what an outer suffix of `outers` was taken off) as a stem with an inner suffix + /// whose continuation classes hold that outer suffix's flag. + fn inner_analyses<'a>(&'a self, form: &str, outers: &[&'a Rule], excluded: Excluded, work: &mut usize, emit: &mut impl FnMut(Analysis<'a>)) { + let mut root = String::with_capacity(form.len() + 8); + let cuts = std::iter::once(form.len()).chain(form.char_indices().rev().map(|(i, _)| i).filter(|&i| i > 0 || self.full_strip)); + for b in cuts.take(self.max_sfx_chars + 1) { + let (Some(base), Some(add)) = (form.get(..b), form.get(b..)) else { continue }; + let Some(groups) = self.sfx_index.get(add) else { continue }; + for g in groups.iter().filter(|g| !g.cont_rules.is_empty()) { + root.clear(); + root.push_str(base); + root.push_str(&g.strip); + for (stem, set) in self.part_stems(&root, excluded) { + for r in g.cont_rules.iter().filter_map(|&i| self.suffixes.get(i as usize)) { + *work += 1; + if *work > MAX_WORK { + return; + } + if !has(set, r.flag) + || self.cont_holds(r, self.circumfix) + || self.cont_holds(r, self.only_in_compound) + || !r.cond_matches(&root, true) + { + continue; + } + for &outer in outers.iter().filter(|o| self.cont_holds(r, Some(o.flag))) { + *work += 1; + emit(Analysis { stem, set, pfx: None, sfx: Some(outer), inner: Some(r) }); + } + } + } + } + } + } + + /// The stem `w` as a compound part: its index and flags, unless it is forbidden or excluded. + /// A stem with `ONLYINCOMPOUND` homonyms beside others is two: each group's flags apply on + /// their own (`foo/XOS` and `foo/T`: `foos` is a part, `foot` is not, as in the CLI). + /// + /// Homonyms kept apart ([`AffixDict::homonyms`]) are each a part of their own: a suffix and + /// the compound flag must come from one (Dutch `poeder` the noun compounds, `poederden` the + /// verb form doesn't: `schuurpoederden` is wrong in the CLI). + fn part_stems(&self, w: &str, excluded: Excluded) -> impl Iterator { + let found = self.stem_lookup(w); + let apart = found.and_then(|(stem, _)| self.homonyms.get(&stem).map(|h| (stem, h))); + let main = found.filter(|_| apart.is_none()); + let each = apart.into_iter().flat_map(move |(stem, h)| h.iter().map(move |&id| (stem, self.set(id)))); + let only = found.and_then(|(stem, _)| self.cpd_sets.get(&stem).map(|&c| (stem, self.set(c)))); + main.into_iter().chain(each).chain(only).filter(move |(_, set)| !excluded.hits(set) && !self.forbidden.is_some_and(|f| has(set, f))) + } + + /// May a part with `info` start at char `start`, after the last part of `s.path` + /// (`CHECKCOMPOUNDCASE`, `CHECKCOMPOUNDTRIPLE`, `CHECKCOMPOUNDPATTERN`)? + fn boundary_ok(&self, s: &Search, start: usize, info: &Info) -> bool { + let Some(&(ps, pe, prev)) = s.path.last() else { return true }; + let c = s.c; + let (Some(&a), Some(&b)) = (s.chars.get(pe.saturating_sub(1)), s.chars.get(start)) else { return true }; + if c.case && (a.is_uppercase() || b.is_uppercase()) && a != '-' && b != '-' { + return false; + } + if c.triple && start == pe && a == b { + let before = pe >= 2 && s.chars.get(pe - 2) == Some(&a); + let after = s.chars.get(start + 1) == Some(&b); + if before || after { + return false; + } + } + // A SIMPLIFIEDTRIPLE join (the parts share a letter) still may not leave three equal + // letters in the word (`buss|ssen` as `busssen`, `boo|oob` as `booob`). + if c.triple && start + 1 == pe && s.chars.get(pe) == Some(&a) { + return false; + } + if c.patterns.is_empty() { + return true; + } + let text = |x: usize, y: usize| s.cuts.get(x).zip(s.cuts.get(y)).and_then(|(&x, &y)| s.w.get(x..y)).unwrap_or_default(); + let (before, after) = (text(ps, pe), text(start, s.chars.len())); + !c.patterns.iter().any(|p| { + before.ends_with(&*p.end) + && after.starts_with(&*p.begin) + && (!p.unmodified || prev.unmodified) + && p.end_need.met(prev.marks) + && p.begin_need.met(info.marks) + }) + } + + /// Are the parts of `s.path` few enough (`COMPOUNDWORDMAX`, `COMPOUNDROOT`)? `whole`: the + /// word is complete (the Hungarian syllable rule counts its vowels). + fn words_ok(&self, s: &Search, whole: bool) -> bool { + let Some(max) = s.c.word_max else { return true }; + let words: usize = s.path.iter().map(|p| usize::from(p.2.words)).sum(); + if words <= max { + return true; + } + match &s.c.syllables { + None => false, + Some(_) if !whole => true, + Some((most, vowels)) => { + let tail = s.path.last().map_or(0, |p| usize::from(p.2.tail)); + s.chars.iter().filter(|c| vowels.contains(c)).count().saturating_sub(tail) <= *most + } + } + } + + /// The checks on a complete split: two parts at least, `COMPOUNDWORDMAX`, `CHECKCOMPOUNDDUP`, + /// `FORCEUCASE`, the three-part rule and `CHECKCOMPOUNDREP`. + fn whole_ok(&self, s: &mut Search) -> bool { + let n = s.path.len(); + if n < 2 || !self.words_ok(s, true) { + return false; + } + let (Some(&(_, _, last)), Some(&(_, _, prev))) = (s.path.last(), s.path.get(n - 2)) else { return false }; + if s.c.dup && last.stem == prev.stem { + return false; + } + // FORCEUCASE: a word with no capital at all is wrong (`fooQuxbar` is right in the CLI). + if last.force && !s.capitalized && !s.chars.iter().any(|c| c.is_uppercase()) { + return false; + } + let span = |s: &Search, i: usize, j: usize| -> String { + let (Some(&(a, ..)), Some(&(_, b, _))) = (s.path.get(i), s.path.get(j)) else { return String::new() }; + s.cuts.get(a).zip(s.cuts.get(b)).and_then(|(&x, &y)| s.w.get(x..y)).unwrap_or_default().to_string() + }; + if n >= 3 && self.near_word(s) { + return false; + } + if s.c.rep && !self.rep.is_empty() { + let pairs = (0..n - 1).map(|i| (i, i + 1)); + let tails = (0..n - 1).map(|i| (i, n - 1)); + let spans: Vec = pairs.chain(tails).map(|(i, j)| span(s, i, j)).collect(); + if spans.iter().any(|sp| self.rep_fault(sp, s)) { + return false; + } + } + true + } + + /// The check on a compound of three or more parts: is the word one slip away from a word + /// (not a compound)? Then it is taken for a misspelling of that word. The slips are the + /// CLI's (black-box, see the tests): a letter too many, a pair typed twice, two letters + /// swapped or one moved (up to 4 apart), a keyboard neighbour (`KEY`), a `TRY` letter, a + /// `REP` or `MAP` change, the word capitalized, and (without `NOSPLITSUGS`) two words. It + /// depends on the word only, so it is answered once per check. + fn near_word(&self, s: &mut Search) -> bool { + if let Some(near) = s.near { + return near; + } + let excluded = s.excluded; + let mut work = s.work; + // A word listed as a stem that can't stand alone (ONLYINCOMPOUND, NEEDAFFIX) counts here + // only with a suffix, not with a prefix (`ver` + `kehr` beside `verkehr/zo`: + // `Fahrradverkehr` is right in the CLI). + let mut is_word = |cand: &[char]| { + work += 1; + let cand: String = cand.iter().collect(); + // Past its budget a candidate counts as a word: the compound is then rejected, as + // it is past MAX_WORK, so a suggestion never offers one the check would refuse. + work > MAX_WORK || self.simple_word_budgeted(&cand, excluded, self.stem_set(&cand).is_none()).unwrap_or(true) + }; + let near = self.near_miss(&s.chars, s.c.two_words, &mut is_word); + s.work = work; + s.near = Some(near); + near + } + + fn near_miss(&self, w: &[char], two_words: bool, is_word: &mut impl FnMut(&[char]) -> bool) -> bool { + let n = w.len(); + let mut cand: Vec = Vec::with_capacity(n + 8); + let mut first = w.iter().take(1).flat_map(|c| c.to_uppercase()).collect::>(); + first.extend(w.iter().skip(1)); + if is_word(&first) || is_word(&w.iter().flat_map(|c| c.to_uppercase()).collect::>()) { + return true; + } + for i in 0..n { + // A letter too many, and a pair typed twice (`abab` for `ab`). + cand.clear(); + cand.extend(w.iter().take(i).chain(w.iter().skip(i + 1))); + if is_word(&cand) { + return true; + } + if i + 3 < n && w.get(i) == w.get(i + 2) && w.get(i + 1) == w.get(i + 3) { + cand.clear(); + cand.extend(w.iter().take(i).chain(w.iter().skip(i + 2))); + if is_word(&cand) { + return true; + } + } + for d in 1..=4 { + if i + d >= n { + break; + } + // Swapped with a letter up to 4 further on. + cand.clear(); + cand.extend_from_slice(w); + cand.swap(i, i + d); + if cand != w && is_word(&cand) { + return true; + } + // Moved 2 to 4 places on or back (one place is a swap). + if d >= 2 { + cand.clear(); + cand.extend_from_slice(w); + for turn in [1, 2] { + if let Some(slice) = cand.get_mut(i..=i + d) { + // Left by one: `w[i]` moves on; then right by two: `w[i + d]` back. + if turn == 1 { slice.rotate_left(1) } else { slice.rotate_right(2) } + } + if is_word(&cand) { + return true; + } + } + } + } + } + // One letter for another: a keyboard neighbour, a TRY letter. + let key = if self.key.is_empty() { DEFAULT_KEY } else { self.key.as_str() }; + let neighbours = |c: char| { + key.split('|').flat_map(move |row| { + let row: Vec = row.chars().take(MAX_KEY_ROW).collect(); + let at: Vec = row.iter().enumerate().filter(|(_, k)| **k == c).map(|(i, _)| i).collect(); + at.into_iter() + .flat_map(move |i| [i.checked_sub(1).and_then(|j| row.get(j)).copied(), row.get(i + 1).copied()]) + .flatten() + .collect::>() + }) + }; + let tries: Vec = self.try_chars.chars().take(MAX_TRY).collect(); + for i in 0..n { + let Some(&c) = w.get(i) else { continue }; + for r in neighbours(c).chain(tries.iter().copied()) { + if r == c { + continue; + } + cand.clear(); + cand.extend_from_slice(w); + if let Some(x) = cand.get_mut(i) { + *x = r; + } + if is_word(&cand) { + return true; + } + } + } + // `REP` (here `^` and `$` tie to the word's start and end) and `MAP` changes. + let text: String = w.iter().collect(); + for (from, to) in &self.rep { + let (start, from) = from.strip_prefix('^').map_or((false, from.as_str()), |f| (true, f)); + let (end, from) = from.strip_suffix('$').map_or((false, from), |f| (true, f)); + if from.is_empty() || to.contains(' ') { + continue; + } + for (i, _) in text.match_indices(from) { + if (start && i != 0) || (end && i + from.len() != text.len()) { + continue; + } + let changed: Vec = text + .get(..i) + .unwrap_or_default() + .chars() + .chain(to.chars()) + .chain(text.get(i + from.len()..).unwrap_or_default().chars()) + .collect(); + if is_word(&changed) { + return true; + } + } + } + let mut budget = MAX_MAP_VARIANTS; + if !self.map.is_empty() && self.map_variant(w, 0, &mut w.to_vec(), &mut budget, is_word) { + return true; + } + two_words + && (1..n).any(|i| { + let (head, tail) = w.split_at(i); + is_word(head) && is_word(tail) + }) + } + + /// Does changing letters of `cand` from `i` on to others of their `MAP` group make a word? + fn map_variant(&self, w: &[char], i: usize, cand: &mut Vec, budget: &mut usize, is_word: &mut impl FnMut(&[char]) -> bool) -> bool { + let Some(&c) = w.get(i) else { return cand.as_slice() != w && is_word(cand) }; + if self.map_variant(w, i + 1, cand, budget, is_word) { + return true; + } + for group in self.map.iter().filter(|g| g.contains(&c)) { + for &r in group.iter().filter(|&&r| r != c) { + if *budget == 0 { + return false; + } + *budget -= 1; + if let Some(x) = cand.get_mut(i) { + *x = r; + } + let hit = self.map_variant(w, i + 1, cand, budget, is_word); + if let Some(x) = cand.get_mut(i) { + *x = c; + } + if hit { + return true; + } + } + } + false + } + + /// Does a `REP` replacement turn `text` into a word (not a compound)? + fn rep_fault(&self, text: &str, s: &mut Search) -> bool { + let mut cand = String::with_capacity(text.len() + 8); + for (from, to) in &self.rep { + // Replacements tied to a word's start or end (`^`, `$`) are for suggestions: the CLI + // never applies them here. + if from.is_empty() || from.starts_with('^') || from.ends_with('$') { + continue; + } + for (i, _) in text.match_indices(from.as_str()) { + s.work += 1; + if s.work > MAX_WORK { + return true; + } + cand.clear(); + cand.push_str(text.get(..i).unwrap_or_default()); + cand.push_str(to); + cand.push_str(text.get(i + from.len()..).unwrap_or_default()); + if self.simple_word(&cand, s.excluded) { + return true; + } + } + } + false + } +} + +/// The compounding directives a compact-format line may hold, read like the `.aff`'s. +pub(super) fn is_directive(dir: &str) -> bool { + FLAG_DIRECTIVES.contains(&dir) + || SWITCHES.contains(&dir) + || matches!(dir, "COMPOUNDWORDMAX" | "COMPOUNDSYLLABLE" | "LANG" | "MAXCPDSUGS" | "NOSPLITSUGS" | "CHECKCOMPOUNDPATTERN") +} + +#[cfg(test)] +mod tests { + use std::collections::HashSet; + + use crate::affix::AffixDict; + + /// A synthetic dictionary and the `hunspell` 1.7.5 CLI's answers on it (black-box: `hunspell + /// -a`), for every word of one to three `pieces` and the `extra` words. `listed` are the + /// right words when `right`, else the wrong ones; every other word is the opposite. + struct Pinned { + name: &'static str, + aff: &'static str, + dic: &'static str, + pieces: &'static str, + extra: &'static str, + right: bool, + listed: &'static str, + } + + fn load(aff: &str, dic: &str) -> AffixDict { + let entries: Vec<&str> = dic.split_whitespace().collect(); + let dic = format!("{}\n{}\n", entries.len(), entries.join("\n")); + AffixDict::from_hunspell(&format!("SET UTF-8\n{aff}\n"), &dic, &[]).unwrap() + } + + fn candidates(p: &Pinned) -> Vec { + let pieces: Vec<&str> = p.pieces.split_whitespace().collect(); + let mut out: Vec = pieces.iter().map(|a| (*a).to_string()).collect(); + for a in &pieces { + for b in &pieces { + out.push(format!("{a}{b}")); + } + } + for a in &pieces { + for b in &pieces { + for c in &pieces { + out.push(format!("{a}{b}{c}")); + } + } + } + out.extend(p.extra.split_whitespace().map(str::to_string)); + out + } + + /// One case per directive (or pair that only means something together), each word's answer + /// pinned to the CLI's, before and after the compact format. + #[test] + fn flag_compounds_answer_as_the_cli_does() { + let mut failed = Vec::new(); + for p in PINNED { + let d = load(p.aff, p.dic); + let back = AffixDict::from_bytes(&d.to_bytes().unwrap()).unwrap(); + let listed: HashSet<&str> = p.listed.split_whitespace().collect(); + for w in candidates(p) { + let want = listed.contains(w.as_str()) == p.right; + if d.check(&w) != want || back.check(&w) != want { + failed.push(format!("{}: {w} should be {}", p.name, if want { "right" } else { "wrong" })); + } + } + } + assert!(failed.is_empty(), "{failed:#?}"); + } + + /// Real dictionaries (the Language packs' `.aff`/`.dic`): words the CLI answers right and + /// wrong on the same files (black-box; mostly random compounds of the dictionary's forms). + /// Run when a craft-i18n checkout is in `CRAFT_I18N_DIR` and its harness has fetched the pack + /// files (`CRAFT_I18N_CACHE`, default `$CRAFT_I18N_DIR/target/language-data-cache`, files named + /// by their sha256); each language whose files are missing is skipped. + #[test] + fn pack_dictionaries_compound_as_the_cli_does() { + let Some(dir) = std::env::var_os("CRAFT_I18N_DIR").map(std::path::PathBuf::from) else { return }; + let Ok(manifest) = std::fs::read_to_string(dir.join("language-data/manifest.toml")) else { return }; + let cache = std::env::var_os("CRAFT_I18N_CACHE").map_or_else(|| dir.join("target/language-data-cache"), std::path::PathBuf::from); + let manifest: toml::Table = manifest.parse().unwrap(); + let files = manifest.get("file").and_then(toml::Value::as_array).cloned().unwrap_or_default(); + let read = |locale: &str, kind: &str| { + let f = files.iter().find(|f| { + f.get("locale").and_then(toml::Value::as_str) == Some(locale) && f.get("kind").and_then(toml::Value::as_str) == Some(kind) + })?; + std::fs::read(cache.join(f.get("sha256")?.as_str()?)).ok() + }; + for (locale, right, wrong) in REAL { + let (Some(aff), Some(dic)) = (read(locale, "spelling-aff"), read(locale, "spelling-dic")) else { + eprintln!("{locale}: pack files not fetched, skipped"); + continue; + }; + let (aff, dic) = crate::hunspell::decode(&aff, &dic).unwrap(); + let d = AffixDict::from_hunspell(&aff, &dic, &[]).unwrap_or_else(|e| panic!("{locale}: {e}")); + for w in right.split_whitespace() { + assert!(d.check(w), "{locale}: {w} is right"); + } + for w in wrong.split_whitespace() { + assert!(!d.check(w), "{locale}: {w} is wrong"); + } + // A long garbled word (wrong words run together): its suggestions stay quick. + let garbled: String = wrong.chars().filter(|c| !c.is_whitespace()).take(38).collect(); + let t = std::time::Instant::now(); + let _ = crate::affix_suggest::suggest_in(&d, &garbled, 8, &crate::affix_suggest::Tuning::DEFAULT); + let took = t.elapsed(); + eprintln!("{locale}: suggestions for {garbled} in {took:?}"); + assert!(took.as_secs() < 5, "{locale}: {took:?}"); + // A one-letter slip of a long compound gets the compound back. + for (typo, meant) in TYPOS.iter().filter(|t| t.0 == *locale).flat_map(|t| t.1.iter()) { + let got = d.suggest(typo, 8); + assert!(got.iter().any(|g| g == meant), "{locale}: {typo} → {got:?}, not {meant}"); + } + } + } + + /// (locale, [(typo, the compound meant)]): each one's suggestions hold it. + const TYPOS: &[(&str, &[(&str, &str)])] = &[ + ( + "de", + &[ + ("Donaudampfschff", "Donaudampfschiff"), + ("Arbetszimmer", "Arbeitszimmer"), + ("Kinerspielplatz", "Kinderspielplatz"), + ("Bundesverfassugsgericht", "Bundesverfassungsgericht"), + ], + ), + ( + "nl", + &[ + ("ziekenhusopname", "ziekenhuisopname"), + ("voetbawledstrijd", "voetbalwedstrijd"), + ("arbeidsongeschiktheidsverzeekring", "arbeidsongeschiktheidsverzekering"), + ("tandartsafsparak", "tandartsafspraak"), + ("belastingaagifte", "belastingaangifte"), + ], + ), + ]; + + /// (locale, right words, wrong words), the CLI's answers. + const REAL: &[(&str, &str, &str)] = &[ + ( + "da", + "sugammadexgårdfacadens kraftudtagssmutterbuttetheden pebermakrellensudvalgsarbejde Helsinkifirbladen", + "arbejdslagernebegivende visdommensrustet farfarentraveport egyptologiskopladesvoldtoges", + ), + ( + "de", + "Südwestbeifallsturm Machtfeudalideologie Schleifhärtering Gesprächskontur schule Verfilmungen Aufwärmungen", + "chilisoßeswitttenaus Magmabewegungaminosäurebasis ausgezählterzehntagefieber heerfolgetemperatechnik haus buch verfilmungs verfilmungen aufwärmungen", + ), + ( + "de-at", + "Verträglichkeitsnackenleder Sichelmangrovenwäldern Wehrpflichtbeiruter Kühlerblockigelnothilfe", + "Jungenschulemakromere Dagösnestküken vogtlandbahnelektronenarmem qualitativmedwedew", + ), + ( + "de-ch", + "Königsstahlextensionsfähigkeit Barlachpregelinsel Euroasiendysprosiums Hexendiabasgesteine", + "eingeglittenswohlbeleibtesten plakatflächewürdigens durchzuwandernliebediener zurückerinnernskolonnadenform", + ), + ( + "eo", + "elipsoidolumŝirmila resonancosonarkivojn bazanumeralosukersciuraj vivspertoellistiĝkapablo", + "nekapneatsgvidrolosmitozon Alaholaŭpove kapbalanadajsaŭtomatiĝadajn filamentecukrematoriosilviajn", + ), + ( + "et", + "jahvakemgisesski talõššidekäunuvaikski kõhnunuimatessegikräunujaisse nekroosiupsusina", + "spinningtoismele kollektivismessüngimainagi elektrolüüsispilgutelebki fosforisesverääni", + ), + ("fa", "", "بنمایاندمنتراوی زرشکتبهترانملجاجتکارترینید پارامترمبرقصاندیم"), + ("hr", "truležne", "vaeašteka razuvjeriliuzdanajpsujućemu pilencadiparieovom časomicevoziti"), + ( + "hu", + "eurofelsálazásiabbjáéin", + "síposékéibahetvenegyszerezni BMsorbánocskáérakönigesedéknek antihősekorgalvántelepeitekig Grazcákompendiumnyijaimmal", + ), + ("mn", "Канейдианхалифат Балжойрд", "өехийлчхөгсудлагчдархуутайгүй зээнцрүүдийгээбачиг цуглавруудынхаатөвшинт"), + ("nb", "", "fellesomkostningasweeperrollen kremfløytenalkoholdebatter Elsepålestikka translatøreksamenerkeldgufsene"), + ( + "nl", + "pikjesboekbinderspersen samengingengeloofsverdeeldheid sleepliftenerfgoed stortputtenbehadeling", + "onbeleefdheidhortensiastraat verwindjuilee Lotharengersvolontair gechromeerdsrietpluimen", + ), + ("nn", "", "orstuvastavantgardistar påskemåndagarkjempekenguruar piggtrådmusikkarreknemaskinar stimreinselikniårekeiper"), + ( + "sv", + "målvärdenservername sunniglädjepirrets kedjelängdendöbattangerna basfärgmetropoliterna Norges Sveriges norge", + "ogenomskådligsbåtsmanstorps tännkarheiskanens stängningstaggensbehängas råseleedemonlägeouppvärmt norges sveriges afrikas", + ), + ( + "sv-fi", + "Ausåsciseleringar uppsatsämnesinsatsens pimpelfiskelagstiftningsarbete orderstocksglutenet", + "smitartäbyeors triialiseringarkronkoloni databasenslågstadiumcardiganen springformsknegarensronneby", + ), + ]; + + /// craft-libs#497 review: hundreds of prefixes and suffixes that each add a part to the same + /// one-letter stem, with distinct pattern flags, make tens of thousands of ways for one + /// piece. They are capped and counted, so a word still answers at once. + #[test] + fn many_ways_for_one_piece_stay_cheap() { + let marks: Vec = (0..62).filter_map(|i| char::from_u32(0x3041 + i)).collect(); + let mut aff = String::from("SET UTF-8\nFLAG UTF-8\nCOMPOUNDFLAG X\nCOMPOUNDMIN 1\nCOMPOUNDPERMITFLAG P\nCOMPOUNDWORDMAX 99\n"); + let mut pats: Vec = marks.iter().map(|m| format!("zz/{m} zz/{m}")).collect(); + pats.push("b a".into()); + aff.push_str(&format!("CHECKCOMPOUNDPATTERN {}\n", pats.len())); + for p in &pats { + aff.push_str(&format!("CHECKCOMPOUNDPATTERN {p}\n")); + } + let mut flags = String::new(); + for (kind, base, shift) in [("PFX", 0x4e00, 0), ("SFX", 0x5e00, 31)] { + for i in 0..250u32 { + let Some(f) = char::from_u32(base + i) else { continue }; + let m: String = (0..31).filter(|b| (i >> b) & 1 == 1).filter_map(|b| marks.get(shift + b)).collect(); + aff.push_str(&format!("{kind} {f} Y 1\n{kind} {f} 0 a/XP{m} .\n")); + flags.push(f); + } + } + let d = AffixDict::from_hunspell(&aff, &format!("2\na/X{flags}\nb/X\n"), &[]).unwrap(); + let t = std::time::Instant::now(); + for w in ["baaa", "bbbbbbbbbbbbbbbbbbbbaaa", "ab"] { + let _ = d.check(w); + } + assert!(t.elapsed().as_millis() < 2000, "{:?}", t.elapsed()); + } + + /// `COMPOUNDRULE` as the Swedish (`)k`) and Mongolian (`[a0,a1]`, `%?`, `.`) dictionaries + /// write it, read as the CLI answers on the same rules. + #[test] + fn compound_rules_with_odd_marks_answer_as_the_cli_does() { + let d = load("COMPOUNDMIN 1\nCOMPOUNDRULE 1\nCOMPOUNDRULE )k", "a/) b/k"); + let back = AffixDict::from_bytes(&d.to_bytes().unwrap()).unwrap(); + for d in [&d, &back] { + assert!(d.check("ab") && !d.check("abb") && !d.check("aab")); + } + let d = load("FLAG long\nCOMPOUNDMIN 1\nCOMPOUNDRULE 1\nCOMPOUNDRULE (aa)*[a0,a1](bb)", "x/aa y/bb"); + assert!(d.check("xy") && d.check("xxy") && !d.check("xyy") && !d.check("yx")); + for rule in ["(nn)*%?", "(nn)*.(nn)*%?", "(aa)*%?(bb)"] { + let d = load(&format!("FLAG long\nCOMPOUNDMIN 1\nCOMPOUNDRULE 1\nCOMPOUNDRULE {rule}"), "x/nn y/nn z/aa w/bb"); + let back = AffixDict::from_bytes(&d.to_bytes().unwrap()).unwrap(); + for d in [&d, &back] { + assert!(!d.check("xy") && !d.check("xxy") && !d.check("zw") && d.check("x"), "{rule}"); + } + } + } + + /// The worst dictionary for the search: every letter and pair of letters a part anywhere, + /// every check on, hundreds of `REP`s, a long `TRY`, `MAP` groups and chained suffixes that + /// permit compounding. Each word still answers within the work bound. + #[test] + fn hostile_compounding_stays_cheap() { + let letters: Vec = ('a'..='z').collect(); + let mut aff = String::from( + "COMPOUNDFLAG X\nCOMPOUNDMIN 1\nCOMPOUNDWORDMAX 99\nCOMPOUNDPERMITFLAG P\nONLYINCOMPOUND O\nCHECKCOMPOUNDDUP\nCHECKCOMPOUNDREP\n\ + CHECKCOMPOUNDCASE\nCHECKCOMPOUNDTRIPLE\nSIMPLIFIEDTRIPLE\nTRY abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ\n\ + MAP 3\nMAP aeiou\nMAP bcdfg\nMAP hjklm\nCHECKCOMPOUNDPATTERN 2\nCHECKCOMPOUNDPATTERN a/X b/X\nCHECKCOMPOUNDPATTERN 0/P c\n", + ); + aff.push_str(&format!("REP {}\n", letters.len() * letters.len())); + for a in &letters { + for b in &letters { + aff.push_str(&format!("REP {a} {b}{a}\n")); + } + } + for (i, a) in letters.iter().enumerate() { + aff.push_str(&format!("SFX {a} Y 1\nSFX {a} 0 {a}/XPO{} .\n", letters.get(i + 1).copied().unwrap_or('a'))); + } + let flags: String = letters.iter().collect(); + let mut dic: Vec = letters.iter().map(|a| format!("{a}/X{flags}")).collect(); + for a in &letters { + for b in &letters { + dic.push(format!("{a}{b}/X{flags}")); + } + } + let d = AffixDict::from_hunspell(&format!("SET UTF-8\n{aff}"), &format!("{}\n{}\n", dic.len(), dic.join("\n")), &[]).unwrap(); + let mut rng = 7u64; + let random: String = (0..120) + .map(|_| { + rng = rng.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407); + letters.get((rng >> 33) as usize % 26).copied().unwrap_or('a') + }) + .collect(); + let mut slowest = std::time::Duration::ZERO; + let mut words: Vec = vec!["a".repeat(120), "ab".repeat(60), random.clone()]; + for n in [8, 16, 24, 32, 40, 48, 64] { + words.extend([random.chars().take(n).collect(), "a".repeat(n), "ab".repeat(n / 2), "aab".repeat(n / 3)]); + } + for w in &words { + let t = std::time::Instant::now(); + let _ = d.check(w); + slowest = slowest.max(t.elapsed()); + } + let t = std::time::Instant::now(); + let _ = crate::affix_suggest::suggest_in(&d, &"qwertyuiopasdfghjklzxcvbnm".repeat(2)[..38], 8, &crate::affix_suggest::Tuning::DEFAULT); + let suggest = t.elapsed(); + eprintln!("hostile compounding: slowest word {slowest:?}, suggestions {suggest:?}"); + assert!(suggest.as_secs() < 10, "{suggest:?}"); + assert!(slowest.as_secs() < 5, "{slowest:?}"); + } + + const PINNED: &[Pinned] = &[ + Pinned { + name: "flag", + aff: "COMPOUNDFLAG X", + dic: "foo/X bar/X ba/X baz", + pieces: "foo bar ba baz", + extra: "", + right: true, + listed: "foo bar ba baz foofoo foobar barfoo barbar foofoofoo foofoobar foobarfoo foobarbar barfoofoo barfoobar \ + barbarfoo barbarbar", + }, + Pinned { name: "min", aff: "COMPOUNDFLAG X\nCOMPOUNDMIN 1", dic: "f/X bar/X", pieces: "f bar", extra: "", right: false, listed: "" }, + Pinned { + name: "affixes", + aff: "COMPOUNDFLAG X\nCOMPOUNDPERMITFLAG Z\nSFX S Y 1\nSFX S 0 s .\nPFX P Y 1\nPFX P 0 un .\nSFX T Y 1\nSFX T 0 t/Z .\nPFX Q Y 1\nPFX Q 0 re/Z .", + dic: "foo/XSPTQ bar/XSPTQ baz/S", + pieces: "foo foos unfoo foot refoo bar", + extra: "barunfoo unfoobar barfoot footbar refoobar barrefoo barfoos foosbar baz", + right: false, + listed: "foounfoo foosfoo foosfoos foosunfoo foosfoot foosrefoo foosbar unfoounfoo footunfoo refoounfoo barunfoo \ + foofoounfoo foofoosfoo foofoosfoos foofoosunfoo foofoosfoot foofoosrefoo foofoosbar foounfoounfoo foofootunfoo \ + foorefoounfoo foobarunfoo foosfoofoo foosfoofoos foosfoounfoo foosfoofoot foosfoorefoo foosfoobar foosfoosfoo \ + foosfoosfoos foosfoosunfoo foosfoosfoot foosfoosrefoo foosfoosbar foosunfoofoo foosunfoofoos foosunfoounfoo \ + foosunfoofoot foosunfoorefoo foosunfoobar foosfootfoo foosfootfoos foosfootunfoo foosfootfoot foosfootrefoo \ + foosfootbar foosrefoofoo foosrefoofoos foosrefoounfoo foosrefoofoot foosrefoorefoo foosrefoobar foosbarfoo \ + foosbarfoos foosbarunfoo foosbarfoot foosbarrefoo foosbarbar unfoofoounfoo unfoofoosfoo unfoofoosfoos \ + unfoofoosunfoo unfoofoosfoot unfoofoosrefoo unfoofoosbar unfoounfoounfoo unfoofootunfoo unfoorefoounfoo \ + unfoobarunfoo footfoounfoo footfoosfoo footfoosfoos footfoosunfoo footfoosfoot footfoosrefoo footfoosbar \ + footunfoounfoo footfootunfoo footrefoounfoo footbarunfoo refoofoounfoo refoofoosfoo refoofoosfoos \ + refoofoosunfoo refoofoosfoot refoofoosrefoo refoofoosbar refoounfoounfoo refoofootunfoo refoorefoounfoo \ + refoobarunfoo barfoounfoo barfoosfoo barfoosfoos barfoosunfoo barfoosfoot barfoosrefoo barfoosbar \ + barunfoounfoo barfootunfoo barrefoounfoo barbarunfoo", + }, + Pinned { + name: "affix_flags", + aff: "COMPOUNDFLAG X\nSFX A Y 1\nSFX A 0 s/X .\nPFX B Y 1\nPFX B 0 un/X .\nSFX C Y 1\nSFX C 0 en .", + dic: "arbeit/AC comp/XC haus/B", + pieces: "arbeit arbeits comp compen haus unhaus", + extra: "", + right: true, + listed: "arbeit arbeits comp compen haus unhaus comparbeits compcomp compcompen unhausarbeits unhauscomp unhauscompen \ + compcomparbeits compcompcomp compcompcompen compunhausarbeits compunhauscomp compunhauscompen \ + unhauscomparbeits unhauscompcomp unhauscompcompen unhausunhausarbeits unhausunhauscomp unhausunhauscompen", + }, + Pinned { + name: "positions", + aff: "COMPOUNDFLAG X\nCOMPOUNDBEGIN B\nCOMPOUNDMIDDLE M\nCOMPOUNDEND E", + dic: "beg/B mid/M end/E any/X all/BME", + pieces: "beg mid end any all", + extra: "", + right: true, + listed: "beg mid end any all begend begany begall anyend anyany anyall allend allany allall begmidend begmidany \ + begmidall beganyend beganyany beganyall begallend begallany begallall anymidend anymidany anymidall anyanyend \ + anyanyany anyanyall anyallend anyallany anyallall allmidend allmidany allmidall allanyend allanyany allanyall \ + allallend allallany allallall", + }, + Pinned { + name: "first_last", + aff: "COMPOUNDFIRST B\nCOMPOUNDLAST E", + dic: "beg/B end/E", + pieces: "beg end", + extra: "", + right: true, + listed: "beg end", + }, + Pinned { + name: "onlyincompound", + aff: "COMPOUNDFLAG X\nONLYINCOMPOUND O\nSFX A Y 1\nSFX A 0 er/B .\nSFX B Y 1\nSFX B 0 s .\nSFX F Y 1\nSFX F 0 s/O .", + dic: "foo/XAF bar/XAF zap/XO qux/A", + pieces: "foo foos fooers bar zap", + extra: "zap foos fooer fooers quxer quxers", + right: true, + listed: "foo fooers bar foofoo foofooers foobar foozap barfoo barfooers barbar barzap zapfoo zapfooers zapbar zapzap \ + foofoofoo foofoofooers foofoobar foofoozap foobarfoo foobarfooers foobarbar foobarzap foozapfoo foozapfooers \ + foozapbar foozapzap barfoofoo barfoofooers barfoobar barfoozap barbarfoo barbarfooers barbarbar barbarzap \ + barzapfoo barzapfooers barzapbar barzapzap zapfoofoo zapfoofooers zapfoobar zapfoozap zapbarfoo zapbarfooers \ + zapbarbar zapbarzap zapzapfoo zapzapfooers zapzapbar zapzapzap fooer quxer quxers", + }, + Pinned { + name: "onlyincompound_permit", + aff: "COMPOUNDFLAG X\nONLYINCOMPOUND O\nCOMPOUNDPERMITFLAG P\nSFX F Y 1\nSFX F 0 s/OP .\nSFX G Y 1\nSFX G 0 s/O .\nPFX H Y 1\nPFX H 0 un/O .\nPFX I Y 1\nPFX I 0 re/OP .", + dic: "foo/XFHI bar/XFHI baz/XG", + pieces: "foo foos unfoo refoo bazs", + extra: "foos unfoo refoo bazs baz", + right: true, + listed: "foo foofoo foorefoo foosfoo foosrefoo unfoofoo unfoorefoo refoofoo refoorefoo foofoofoo foofoorefoo foofoosfoo \ + foofoosrefoo foounfoofoo foounfoorefoo foorefoofoo foorefoorefoo foosfoofoo foosfoorefoo foosfoosfoo \ + foosfoosrefoo foosunfoofoo foosunfoorefoo foosrefoofoo foosrefoorefoo unfoofoofoo unfoofoorefoo unfoofoosfoo \ + unfoofoosrefoo unfoounfoofoo unfoounfoorefoo unfoorefoofoo unfoorefoorefoo refoofoofoo refoofoorefoo \ + refoofoosfoo refoofoosrefoo refoounfoofoo refoounfoorefoo refoorefoofoo refoorefoorefoo baz", + }, + Pinned { + name: "forbid_needaffix", + aff: "COMPOUNDFLAG X\nCOMPOUNDFORBIDFLAG Q\nCOMPOUNDPERMITFLAG P\nNEEDAFFIX N\nSFX S Y 1\nSFX S 0 s .\nSFX T Y 1\nSFX T 0 t/Q .\nSFX U Y 1\nSFX U 0 u/QP .\nSFX V Y 1\nSFX V 0 v/P .", + dic: "foo/XSTUV bar/XSTUV qoo/XQS nee/XNSV noo/NSV", + pieces: "foo foot foou qoo nee nees neev", + extra: "nee noo noos", + right: true, + listed: "foo foot foou qoo nees neev foofoo foofoot foofoou fooqoo foonees fooneev neevfoo neevfoot neevfoou neevqoo \ + neevnees neevneev foofoofoo foofoofoot foofoofoou foofooqoo foofoonees foofooneev fooneevfoo fooneevfoot \ + fooneevfoou fooneevqoo fooneevnees fooneevneev neevfoofoo neevfoofoot neevfoofoou neevfooqoo neevfoonees \ + neevfooneev neevneevfoo neevneevfoot neevneevfoou neevneevqoo neevneevnees neevneevneev noos", + }, + Pinned { + name: "dup", + aff: "COMPOUNDFLAG X\nCHECKCOMPOUNDDUP\nSFX S Y 1\nSFX S 0 s .", + dic: "foo/XS bar/XS foos/X", + pieces: "foo foos bar", + extra: "", + right: false, + listed: "foofoo barbar foofoofoo foobarbar foosfoofoo foosbarbar barfoofoo barbarbar", + }, + Pinned { + name: "case", + aff: "COMPOUNDFLAG X\nCHECKCOMPOUNDCASE", + dic: "foo/X Bar/X baz/X QUX/X", + pieces: "foo Bar baz QUX", + extra: "", + right: true, + listed: "foo Bar baz QUX foofoo foobaz Barfoo Barbaz bazfoo bazbaz foofoofoo foofoobaz foobazfoo foobazbaz Barfoofoo \ + Barfoobaz Barbazfoo Barbazbaz bazfoofoo bazfoobaz bazbazfoo bazbazbaz", + }, + Pinned { + name: "triple", + aff: "COMPOUNDFLAG X\nCHECKCOMPOUNDTRIPLE", + dic: "foo/X bar/X schiff/X fahrt/X ooze/X baa/X", + pieces: "foo schiff fahrt ooze baa", + extra: "", + right: false, + listed: "fooooze schifffoo schifffahrt foofooooze fooschifffoo fooschifffahrt foooozefoo foooozeschiff foooozefahrt \ + foooozeooze foooozebaa schifffoofoo schifffooschiff schifffoofahrt schifffooooze schifffoobaa schiffschifffoo \ + schiffschifffahrt schifffahrtfoo schifffahrtschiff schifffahrtfahrt schifffahrtooze schifffahrtbaa \ + fahrtfooooze fahrtschifffoo fahrtschifffahrt oozefooooze oozeschifffoo oozeschifffahrt baafooooze baaschifffoo \ + baaschifffahrt", + }, + Pinned { + name: "simplified_triple", + aff: "COMPOUNDFLAG X\nCHECKCOMPOUNDTRIPLE\nSIMPLIFIEDTRIPLE", + dic: "foo/X bar/X schiff/X fahrt/X ooze/X baa/X", + pieces: "foo schiff fahrt ooze baa", + extra: "schiffahrt", + right: false, + listed: "fooooze schifffoo schifffahrt foofooooze fooschifffoo fooschifffahrt foooozefoo foooozeschiff foooozefahrt \ + foooozeooze foooozebaa schifffoofoo schifffooschiff schifffoofahrt schifffooooze schifffoobaa schiffschifffoo \ + schiffschifffahrt schifffahrtfoo schifffahrtschiff schifffahrtfahrt schifffahrtooze schifffahrtbaa \ + fahrtfooooze fahrtschifffoo fahrtschifffahrt oozefooooze oozeschifffoo oozeschifffahrt baafooooze baaschifffoo \ + baaschifffahrt", + }, + Pinned { + name: "simplified_only", + aff: "COMPOUNDFLAG X\nSIMPLIFIEDTRIPLE", + dic: "foo/X schiff/X fahrt/X ooze/X", + pieces: "foo schiff fahrt ooze", + extra: "schiffahrt fooze", + right: false, + listed: "fooze", + }, + Pinned { + name: "rep", + aff: "COMPOUNDFLAG X\nCHECKCOMPOUNDREP\nREP 4\nREP oo o\nREP rb r\nREP ^ba x\nREP az$ x\nSFX S Y 1\nSFX S 0 s .", + dic: "foo/XS bar/XS baz/X fobar/S barb/X baraz/S fobaz bx", + pieces: "foo foos bar bars baz barb", + extra: "", + right: true, + listed: "foo foos bar bars baz barb foofoo foofoos foobarb barfoo barfoos barbar barbars barbarb bazfoo bazfoos bazbar \ + bazbars bazbaz bazbarb barbfoo barbfoos barbbar barbbars barbbaz barbbarb foofoofoo foofoofoos foofoobarb \ + foobarbfoo foobarbfoos foobarbbar foobarbbars foobarbbaz foobarbbarb barfoofoo barfoofoos barfoobarb barbarfoo \ + barbarfoos barbarbar barbarbars barbarbarb barbarbfoo barbarbfoos barbarbbar barbarbbars barbarbbaz \ + barbarbbarb bazfoofoo bazfoofoos bazfoobarb bazbarfoo bazbarfoos bazbarbar bazbarbars bazbarbarb bazbazfoo \ + bazbazfoos bazbazbar bazbazbars bazbazbaz bazbazbarb bazbarbfoo bazbarbfoos bazbarbbar bazbarbbars bazbarbbaz \ + bazbarbbarb barbfoofoo barbfoofoos barbfoobarb barbbarfoo barbbarfoos barbbarbar barbbarbars barbbarbarb \ + barbbazfoo barbbazfoos barbbazbar barbbazbars barbbazbaz barbbazbarb barbbarbfoo barbbarbfoos barbbarbbar \ + barbbarbbars barbbarbbaz barbbarbbarb", + }, + Pinned { + name: "rep_spans", + aff: "COMPOUNDFLAG X\nCHECKCOMPOUNDREP\nREP 1\nREP oo o", + dic: "foo/X bar/X baz/X qux/X fobarbaz quxfobar", + pieces: "foo bar baz qux", + extra: "", + right: false, + listed: "foobarbaz quxfoobar", + }, + Pinned { + name: "three_parts", + aff: "COMPOUNDFLAG X", + dic: "foo/X bar/X baz/X qux/X foobar fooba barbazqux", + pieces: "foo bar baz qux", + extra: "fooxbarbaz", + right: false, + listed: "foofoobar foobarfoo foobarbar foobarbaz foobarqux barfoobar bazfoobar bazbarqux quxfoobar fooxbarbaz", + }, + Pinned { + name: "three_parts_nosplitsugs", + aff: "COMPOUNDFLAG X\nNOSPLITSUGS", + dic: "foo/X bar/X baz/X qux/X foobar fooba", + pieces: "foo bar baz qux", + extra: "", + right: false, + listed: "", + }, + Pinned { + name: "pattern", + aff: "COMPOUNDFLAG X\nCOMPOUNDPERMITFLAG P\nCHECKCOMPOUNDPATTERN 3\nCHECKCOMPOUNDPATTERN oo ba # comment\nCHECKCOMPOUNDPATTERN /F /G\nCHECKCOMPOUNDPATTERN er /H\nSFX S Y 1\nSFX S 0 s/P .\nSFX T Y 1\nSFX T 0 t/PF .\nSFX E Y 1\nSFX E 0 er/P .", + dic: "foo/XST bar/XSG baz/XS qux/XF zed/XH hey/XSE heyer/XS", + pieces: "foo foot bar baz qux zed heyer hey", + extra: "", + right: false, + listed: "foobar foobaz footbar quxbar heyerzed foofoobar foofoobaz foofootbar foobarfoo foobarfoot foobarbar foobarbaz \ + foobarqux foobarzed foobarheyer foobarhey foobazfoo foobazfoot foobazbar foobazbaz foobazqux foobazzed \ + foobazheyer foobazhey fooquxbar fooheyerzed footfoobar footfoobaz footfootbar footbarfoo footbarfoot \ + footbarbar footbarbaz footbarqux footbarzed footbarheyer footbarhey footquxbar footheyerzed barfoobar \ + barfoobaz barfootbar barquxbar barheyerzed bazfoobar bazfoobaz bazfootbar bazquxbar bazheyerzed quxfoobar \ + quxfoobaz quxfootbar quxbarfoo quxbarfoot quxbarbar quxbarbaz quxbarqux quxbarzed quxbarheyer quxbarhey \ + quxquxbar quxheyerzed zedfoobar zedfoobaz zedfootbar zedquxbar zedheyerzed heyerfoobar heyerfoobaz \ + heyerfootbar heyerquxbar heyerzedfoo heyerzedfoot heyerzedbar heyerzedbaz heyerzedqux heyerzedzed \ + heyerzedheyer heyerzedhey heyerheyerzed heyfoobar heyfoobaz heyfootbar heyquxbar heyheyerzed", + }, + Pinned { + name: "pattern_unmodified_forceucase", + aff: "COMPOUNDFLAG X\nFORCEUCASE U\nCHECKCOMPOUNDPATTERN 1\nCHECKCOMPOUNDPATTERN 0/A b\nCOMPOUNDPERMITFLAG P\nSFX S Y 1\nSFX S 0 s .\nSFX R Y 1\nSFX R 0 r/P .", + dic: "foo/XSR straat/XUS bar/X zoo/XAR big/X", + pieces: "foo straat bar zoo zoor big", + extra: "Foostraat FOOSTRAAT Foostraats", + right: false, + listed: "foostraat straatstraat barstraat zoostraat zoobar zoobig zoorstraat bigstraat foofoostraat foostraatstraat \ + foobarstraat foozoostraat foozoobar foozoobig foozoorstraat foobigstraat straatfoostraat straatstraatstraat \ + straatbarstraat straatzoostraat straatzoobar straatzoobig straatzoorstraat straatbigstraat barfoostraat \ + barstraatstraat barbarstraat barzoostraat barzoobar barzoobig barzoorstraat barbigstraat zoofoostraat \ + zoostraatstraat zoobarfoo zoobarstraat zoobarbar zoobarzoo zoobarzoor zoobarbig zoozoostraat zoozoobar \ + zoozoobig zoozoorstraat zoobigfoo zoobigstraat zoobigbar zoobigzoo zoobigzoor zoobigbig zoorfoostraat \ + zoorstraatstraat zoorbarstraat zoorzoostraat zoorzoobar zoorzoobig zoorzoorstraat zoorbigstraat bigfoostraat \ + bigstraatstraat bigbarstraat bigzoostraat bigzoobar bigzoobig bigzoorstraat bigbigstraat", + }, + Pinned { + name: "wordmax_root", + aff: "COMPOUNDFLAG X\nCOMPOUNDWORDMAX 2\nCOMPOUNDROOT R", + dic: "foo/X bar/X foobaz/XR", + pieces: "foo bar foobaz", + extra: "", + right: true, + listed: "foo bar foobaz foofoo foobar barfoo barbar", + }, + Pinned { + name: "wordmax", + aff: "COMPOUNDFLAG X\nCOMPOUNDWORDMAX 3\nCOMPOUNDMIN 1", + dic: "fo/X ba/X", + pieces: "fo ba", + extra: "fobafoba fobafobafo", + right: false, + listed: "fobafoba fobafobafo", + }, + Pinned { + name: "syllable", + aff: "COMPOUNDFLAG X\nCOMPOUNDWORDMAX 2\nCOMPOUNDSYLLABLE 6 aeiou\nCOMPOUNDMIN 1", + dic: "ba/X be/X baaa/X", + pieces: "ba be baaa", + extra: "babababa", + right: false, + listed: "", + }, + Pinned { + name: "syllable_hu", + aff: "LANG hu_HU\nCOMPOUNDFLAG X\nCOMPOUNDWORDMAX 2 y\nCOMPOUNDSYLLABLE 6 aeiou\nCOMPOUNDMIN 1", + dic: "ba/X be/X baaa/X", + pieces: "ba be baaa", + extra: "babababa", + right: false, + listed: "babaaabaaa bebaaabaaa baaababaaa baaabebaaa baaabaaaba baaabaaabe baaabaaabaaa", + }, + Pinned { + name: "near_slips", + aff: "COMPOUNDFLAG X\nCOMPOUNDMIN 4\nTRY x\nREP 1\nREP ^ab yz\nMAP 1\nMAP eë", + dic: "abcd/X efgh/X ijkl/X mnop/X abcdefghikjl efghabcdjkli ijklabcdefgg xnopabcdefgh yzcdmnopijkl ëfghmnopabcd Ijklmnopabcd", + pieces: "abcd efgh ijkl mnop", + extra: "", + right: false, + listed: "abcdefghijkl abcdmnopijkl efghabcdijkl efghmnopabcd ijklabcdefgh ijklmnopabcd mnopabcdefgh", + }, + Pinned { + name: "onlyincompound_homonyms", + aff: "COMPOUNDFLAG X\nONLYINCOMPOUND O\nSFX S Y 1\nSFX S 0 s .\nSFX T Y 1\nSFX T 0 t .", + dic: "foo/XOS foo/T bar/X baz/XO baz/XT", + pieces: "foo foos foot bar baz bazt bazs", + extra: "", + right: true, + listed: "foo foot bar baz bazt foofoo foofoos foobar foobaz foobazt barfoo barfoos barbar barbaz barbazt bazfoo bazfoos \ + bazbar bazbaz bazbazt foofoofoo foofoofoos foofoobar foofoobaz foofoobazt foobarfoo foobarfoos foobarbar \ + foobarbaz foobarbazt foobazfoo foobazfoos foobazbar foobazbaz foobazbazt barfoofoo barfoofoos barfoobar \ + barfoobaz barfoobazt barbarfoo barbarfoos barbarbar barbarbaz barbarbazt barbazfoo barbazfoos barbazbar \ + barbazbaz barbazbazt bazfoofoo bazfoofoos bazfoobar bazfoobaz bazfoobazt bazbarfoo bazbarfoos bazbarbar \ + bazbarbaz bazbarbazt bazbazfoo bazbazfoos bazbazbar bazbazbaz bazbazbazt", + }, + Pinned { + name: "onlyincompound_homonyms_alone", + aff: "ONLYINCOMPOUND O\nNEEDAFFIX N\nSFX S Y 1\nSFX S 0 s .\nPFX U Y 1\nPFX U 0 un .", + dic: "bar/OS bar/U qux/O qux/S zed/OS", + pieces: "", + extra: "bar bars unbar qux quxs unqux zed zeds", + right: true, + listed: "bar unbar qux quxs", + }, + Pinned { + name: "simplified_triple_checked", + aff: "COMPOUNDFLAG X\nCHECKCOMPOUNDTRIPLE\nSIMPLIFIEDTRIPLE", + dic: "boo/X oob/X ob/X xaa/X aay/X ay/X schiff/X fahrt/X ffahrt/X buss/X ssen/X fjell/X llama/X", + pieces: "", + extra: "booob boob boooob boobob xaaay xaay xaaaay schiffahrt schifffahrt schiffffahrt booobboo boobboo busssen bussen fjelllama fjellama", + right: true, + listed: "schiffahrt", + }, + Pinned { + name: "simplified_triple_min", + aff: "COMPOUNDFLAG X\nCOMPOUNDMIN 3\nCHECKCOMPOUNDTRIPLE\nSIMPLIFIEDTRIPLE", + dic: "foo/X ooa/X oo/X bb/X bbb/X abb/X bba/X", + pieces: "foo ooa oo abb bba bbb", + extra: "fooa abba abbba abbbba", + right: true, + listed: "foo ooa oo abb bba bbb foofoo fooabb foobba foobbb ooafoo ooaooa ooaabb ooabba ooabbb abbfoo abbooa abbabb \ + bbafoo bbaooa bbaabb bbabba bbabbb bbbfoo bbbooa bbbabb foofoofoo foofooabb foofoobba foofoobbb fooabbfoo \ + fooabbooa fooabbabb foobbafoo foobbaooa foobbaabb foobbabba foobbabbb foobbbfoo foobbbooa foobbbabb ooafoofoo \ + ooafooabb ooafoobba ooafoobbb ooaooafoo ooaooaooa ooaooaabb ooaooabba ooaooabbb ooaabbfoo ooaabbooa ooaabbabb \ + ooabbafoo ooabbaooa ooabbaabb ooabbabba ooabbabbb ooabbbfoo ooabbbooa ooabbbabb abbfoofoo abbfooabb abbfoobba \ + abbfoobbb abbooafoo abbooaooa abbooaabb abbooabba abbooabbb abbabbfoo abbabbooa abbabbabb bbafoofoo bbafooabb \ + bbafoobba bbafoobbb bbaooafoo bbaooaooa bbaooaabb bbaooabba bbaooabbb bbaabbfoo bbaabbooa bbaabbabb bbabbafoo \ + bbabbaooa bbabbaabb bbabbabba bbabbabbb bbabbbfoo bbabbbooa bbabbbabb bbbfoofoo bbbfooabb bbbfoobba bbbfoobbb \ + bbbooafoo bbbooaooa bbbooaabb bbbooabba bbbooabbb bbbabbfoo bbbabbooa bbbabbabb", + }, + Pinned { + name: "keepcase_parts", + aff: "COMPOUNDFLAG X\nKEEPCASE K", + dic: "foo/X baz/XK bar/X", + pieces: "foo baz bar Foo Baz BAZ FOO", + extra: "", + right: true, + listed: "foo baz bar Foo FOO foofoo foobaz foobar bazfoo bazbaz bazbar barfoo barbaz barbar Foofoo Foobaz Foobar FOOBAZ \ + FOOFOO foofoofoo foofoobaz foofoobar foobazfoo foobazbaz foobazbar foobarfoo foobarbaz foobarbar bazfoofoo \ + bazfoobaz bazfoobar bazbazfoo bazbazbaz bazbazbar bazbarfoo bazbarbaz bazbarbar barfoofoo barfoobaz barfoobar \ + barbazfoo barbazbaz barbazbar barbarfoo barbarbaz barbarbar Foofoofoo Foofoobaz Foofoobar Foobazfoo Foobazbaz \ + Foobazbar Foobarfoo Foobarbaz Foobarbar FOOBAZBAZ FOOBAZFOO FOOFOOBAZ FOOFOOFOO", + }, + Pinned { + name: "capitalized_stem_parts", + aff: "COMPOUNDFLAG X", + dic: "foo/X Qux/X bar/X", + pieces: "foo Qux qux bar Foo QUX", + extra: "", + right: true, + listed: "foo Qux bar Foo QUX foofoo fooQux foobar Quxfoo QuxQux Quxbar barfoo barQux barbar Foofoo FooQux Foobar \ + foofoofoo foofooQux foofoobar fooQuxfoo fooQuxQux fooQuxbar foobarfoo foobarQux foobarbar Quxfoofoo QuxfooQux \ + Quxfoobar QuxQuxfoo QuxQuxQux QuxQuxbar Quxbarfoo QuxbarQux Quxbarbar barfoofoo barfooQux barfoobar barQuxfoo \ + barQuxQux barQuxbar barbarfoo barbarQux barbarbar Foofoofoo FoofooQux Foofoobar FooQuxfoo FooQuxQux FooQuxbar \ + Foobarfoo FoobarQux Foobarbar", + }, + Pinned { + name: "forceucase_mixed", + aff: "COMPOUNDFLAG X\nFORCEUCASE U\nKEEPCASE K", + dic: "foo/X bar/XU baz/XK Qux/X", + pieces: "foo bar baz Foo Bar Qux", + extra: "FOOBAR FooBar fooBar FOOBAZ Foobaz QuxBar Quxbar QUXBAR", + right: true, + listed: "foo bar baz Foo Bar Qux foofoo foobaz fooQux barfoo barbaz barQux bazfoo bazbaz bazQux Foofoo Foobar Foobaz \ + FooQux Barfoo Barbar Barbaz BarQux Quxfoo Quxbar Quxbaz QuxQux foofoofoo foofoobaz foofooQux foobarfoo \ + foobarbaz foobarQux foobazfoo foobazbaz foobazQux fooQuxfoo fooQuxbar fooQuxbaz fooQuxQux barfoofoo barfoobaz \ + barfooQux barbarfoo barbarbaz barbarQux barbazfoo barbazbaz barbazQux barQuxfoo barQuxbar barQuxbaz barQuxQux \ + bazfoofoo bazfoobaz bazfooQux bazbarfoo bazbarbaz bazbarQux bazbazfoo bazbazbaz bazbazQux bazQuxfoo bazQuxbar \ + bazQuxbaz bazQuxQux Foofoofoo Foofoobar Foofoobaz FoofooQux Foobarfoo Foobarbar Foobarbaz FoobarQux Foobazfoo \ + Foobazbar Foobazbaz FoobazQux FooQuxfoo FooQuxbar FooQuxbaz FooQuxQux Barfoofoo Barfoobar Barfoobaz BarfooQux \ + Barbarfoo Barbarbar Barbarbaz BarbarQux Barbazfoo Barbazbar Barbazbaz BarbazQux BarQuxfoo BarQuxbar BarQuxbaz \ + BarQuxQux Quxfoofoo Quxfoobar Quxfoobaz QuxfooQux Quxbarfoo Quxbarbar Quxbarbaz QuxbarQux Quxbazfoo Quxbazbar \ + Quxbazbaz QuxbazQux QuxQuxfoo QuxQuxbar QuxQuxbaz QuxQuxQux FOOBAR FOOBAZ QUXBAR", + }, + Pinned { + name: "forceucase_suffix", + aff: "COMPOUNDFLAG X\nFORCEUCASE U\nSFX S Y 1\nSFX S 0 s/U .", + dic: "foo/X baz/XS", + pieces: "foo baz bazs", + extra: "Foobazs", + right: false, + listed: "bazsfoo bazsbaz bazsbazs foobazsfoo foobazsbaz foobazsbazs bazbazsfoo bazbazsbaz bazbazsbazs bazsfoofoo \ + bazsfoobaz bazsfoobazs bazsbazfoo bazsbazbaz bazsbazbazs bazsbazsfoo bazsbazsbaz bazsbazsbazs", + }, + Pinned { + name: "pattern_unsuffixed", + aff: "COMPOUNDFLAG X\nCOMPOUNDPERMITFLAG P\nSFX S Y 1\nSFX S 0 s/P .\nPFX U Y 1\nPFX U 0 un/P .\nCHECKCOMPOUNDPATTERN 2\nCHECKCOMPOUNDPATTERN 0 b\nCHECKCOMPOUNDPATTERN 0 un", + dic: "foo/XSU bar/XSU oof/X", + pieces: "foo foos unfoo bar bars unbar oof", + extra: "", + right: true, + listed: "foo foos unfoo bar bars unbar oof foofoo foofoos foooof foosfoo foosfoos foosunfoo foosbar foosbars foosunbar \ + foosoof unfoofoo unfoofoos unfoooof barfoo barfoos baroof barsfoo barsfoos barsunfoo barsbar barsbars \ + barsunbar barsoof unbarfoo unbarfoos unbaroof ooffoo ooffoos oofoof foofoofoo foofoofoos foofoooof foofoosfoo \ + foofoosfoos foofoosunfoo foofoosbar foofoosbars foofoosunbar foofoosoof fooooffoo fooooffoos foooofoof \ + foosfoofoo foosfoofoos foosfoooof foosfoosfoo foosfoosfoos foosfoosunfoo foosfoosbar foosfoosbars \ + foosfoosunbar foosfoosoof foosunfoofoo foosunfoofoos foosunfoooof foosbarfoo foosbarfoos foosbaroof \ + foosbarsfoo foosbarsfoos foosbarsunfoo foosbarsbar foosbarsbars foosbarsunbar foosbarsoof foosunbarfoo \ + foosunbarfoos foosunbaroof foosooffoo foosooffoos foosoofoof unfoofoofoo unfoofoofoos unfoofoooof unfoofoosfoo \ + unfoofoosfoos unfoofoosunfoo unfoofoosbar unfoofoosbars unfoofoosunbar unfoofoosoof unfooooffoo unfooooffoos \ + unfoooofoof barfoofoo barfoofoos barfoooof barfoosfoo barfoosfoos barfoosunfoo barfoosbar barfoosbars \ + barfoosunbar barfoosoof barooffoo barooffoos baroofoof barsfoofoo barsfoofoos barsfoooof barsfoosfoo \ + barsfoosfoos barsfoosunfoo barsfoosbar barsfoosbars barsfoosunbar barsfoosoof barsunfoofoo barsunfoofoos \ + barsunfoooof barsbarfoo barsbarfoos barsbaroof barsbarsfoo barsbarsfoos barsbarsunfoo barsbarsbar barsbarsbars \ + barsbarsunbar barsbarsoof barsunbarfoo barsunbarfoos barsunbaroof barsooffoo barsooffoos barsoofoof \ + unbarfoofoo unbarfoofoos unbarfoooof unbarfoosfoo unbarfoosfoos unbarfoosunfoo unbarfoosbar unbarfoosbars \ + unbarfoosunbar unbarfoosoof unbarooffoo unbarooffoos unbaroofoof ooffoofoo ooffoofoos ooffoooof ooffoosfoo \ + ooffoosfoos ooffoosunfoo ooffoosbar ooffoosbars ooffoosunbar ooffoosoof oofooffoo oofooffoos oofoofoof", + }, + Pinned { + name: "forbidflag_prefix", + aff: "COMPOUNDFLAG X\nCOMPOUNDPERMITFLAG P\nCOMPOUNDFORBIDFLAG Q\nPFX C Y 1\nPFX C 0 de/PQ .", + dic: "foo/XC bar/X", + pieces: "foo defoo bar", + extra: "", + right: true, + listed: "foo defoo bar foofoo foobar barfoo barbar foofoofoo foofoobar foobarfoo foobarbar barfoofoo barfoobar \ + barbarfoo barbarbar", + }, + Pinned { + name: "forbidden", + aff: "COMPOUNDFLAG X\nFORBIDDENWORD F\nSFX S Y 1\nSFX S 0 en .", + dic: "foo/X brok/XS broken/F bar/XF foobar/F", + pieces: "foo brok broken bar", + extra: "foobar broken", + right: true, + listed: "foo brok foofoo foobrok foobroken brokfoo brokbrok brokbroken foofoofoo foofoobrok foofoobroken foobrokfoo \ + foobrokbrok foobrokbroken brokfoofoo brokfoobrok brokfoobroken brokbrokfoo brokbrokbrok brokbrokbroken", + }, + Pinned { + name: "mixed_keep", + aff: "COMPOUNDFLAG X\nKEEPCASE K", + dic: "foo/X baz/XK Qux/X bar/X", + pieces: "foo baz Qux Baz Foo BAZ QUX", + extra: "BazQux BazfooQux FooQux FooBaz fooBaz FOOQux BAZQux", + right: true, + listed: "foo baz Qux Foo QUX foofoo foobaz fooQux bazfoo bazbaz bazQux Quxfoo Quxbaz QuxQux Foofoo Foobaz FooQux QUXBAZ \ + foofoofoo foofoobaz foofooQux foobazfoo foobazbaz foobazQux fooQuxfoo fooQuxbaz fooQuxQux bazfoofoo bazfoobaz \ + bazfooQux bazbazfoo bazbazbaz bazbazQux bazQuxfoo bazQuxbaz bazQuxQux Quxfoofoo Quxfoobaz QuxfooQux Quxbazfoo \ + Quxbazbaz QuxbazQux QuxQuxfoo QuxQuxbaz QuxQuxQux Foofoofoo Foofoobaz FoofooQux Foobazfoo Foobazbaz FoobazQux \ + FooQuxfoo FooQuxbaz FooQuxQux QUXBAZBAZ", + }, + Pinned { + name: "mixed_case_chk", + aff: "COMPOUNDFLAG X\nCHECKCOMPOUNDCASE", + dic: "foo/X Qux/X bar/X", + pieces: "foo Qux Foo bar Bar", + extra: "", + right: true, + listed: "foo Qux Foo bar Bar foofoo foobar Quxfoo Quxbar Foofoo Foobar barfoo barbar Barfoo Barbar foofoofoo foofoobar \ + foobarfoo foobarbar Quxfoofoo Quxfoobar Quxbarfoo Quxbarbar Foofoofoo Foofoobar Foobarfoo Foobarbar barfoofoo \ + barfoobar barbarfoo barbarbar Barfoofoo Barfoobar Barbarfoo Barbarbar", + }, + Pinned { + name: "mixed_forbid", + aff: "COMPOUNDFLAG X\nFORBIDDENWORD F", + dic: "foo/X Qux/X fooQux/F bar/X", + pieces: "foo Qux Foo bar", + extra: "FooQux fooQux FOOQUX", + right: false, + listed: "fooQux fooFoo QuxFoo FooQux FooFoo barFoo foofooFoo fooQuxFoo fooFoofoo fooFooQux fooFooFoo fooFoobar \ + foobarFoo QuxfooFoo QuxQuxFoo QuxFoofoo QuxFooQux QuxFooFoo QuxFoobar QuxbarFoo FoofooFoo FooQuxFoo FooFoofoo \ + FooFooQux FooFooFoo FooFoobar FoobarFoo barfooFoo barQuxFoo barFoofoo barFooQux barFooFoo barFoobar barbarFoo \ + FOOQUX", + }, + Pinned { + name: "mixed_onlyc", + aff: "COMPOUNDFLAG X\nONLYINCOMPOUND O", + dic: "foo/XO Qux/X bar/X", + pieces: "foo Qux Foo bar Bar", + extra: "", + right: true, + listed: "Qux bar Bar foofoo fooQux foobar Quxfoo QuxQux Quxbar Foofoo FooQux Foobar barfoo barQux barbar Barfoo BarQux \ + Barbar foofoofoo foofooQux foofoobar fooQuxfoo fooQuxQux fooQuxbar foobarfoo foobarQux foobarbar Quxfoofoo \ + QuxfooQux Quxfoobar QuxQuxfoo QuxQuxQux QuxQuxbar Quxbarfoo QuxbarQux Quxbarbar Foofoofoo FoofooQux Foofoobar \ + FooQuxfoo FooQuxQux FooQuxbar Foobarfoo FoobarQux Foobarbar barfoofoo barfooQux barfoobar barQuxfoo barQuxQux \ + barQuxbar barbarfoo barbarQux barbarbar Barfoofoo BarfooQux Barfoobar BarQuxfoo BarQuxQux BarQuxbar Barbarfoo \ + BarbarQux Barbarbar", + }, + Pinned { + name: "mixed_force", + aff: "COMPOUNDFLAG X\nFORCEUCASE U", + dic: "foo/X Qux/XU bar/XU", + pieces: "foo Qux Foo bar Bar", + extra: "", + right: true, + listed: "foo Qux Foo bar Bar foofoo fooQux Quxfoo QuxQux Quxbar Foofoo FooQux Foobar barfoo barQux Barfoo BarQux Barbar \ + foofoofoo foofooQux fooQuxfoo fooQuxQux fooQuxbar foobarfoo foobarQux Quxfoofoo QuxfooQux Quxfoobar QuxQuxfoo \ + QuxQuxQux QuxQuxbar Quxbarfoo QuxbarQux Quxbarbar Foofoofoo FoofooQux Foofoobar FooQuxfoo FooQuxQux FooQuxbar \ + Foobarfoo FoobarQux Foobarbar barfoofoo barfooQux barQuxfoo barQuxQux barQuxbar barbarfoo barbarQux Barfoofoo \ + BarfooQux Barfoobar BarQuxfoo BarQuxQux BarQuxbar Barbarfoo BarbarQux Barbarbar", + }, + Pinned { + name: "keep_nosug", + aff: "COMPOUNDFLAG X\nKEEPCASE K\nNOSUGGEST K", + dic: "foo/X baz/XK", + pieces: "foo baz Foo Baz", + extra: "", + right: true, + listed: "foo baz Foo foofoo foobaz bazfoo bazbaz Foofoo Foobaz foofoofoo foofoobaz foobazfoo foobazbaz bazfoofoo \ + bazfoobaz bazbazfoo bazbazbaz Foofoofoo Foofoobaz Foobazfoo Foobazbaz", + }, + Pinned { + name: "root_affix", + aff: "COMPOUNDFLAG Y\nCOMPOUNDWORDMAX 2\nCOMPOUNDROOT R\nCOMPOUNDPERMITFLAG P\nSFX A Y 1\nSFX A 0 s/RP .\nPFX B Y 1\nPFX B 0 un/RP .", + dic: "foo/YAB bar/YAB baz/YR", + pieces: "foo foos unfoo bar bars baz", + extra: "", + right: true, + listed: "foo foos unfoo bar bars baz foofoo foofoos foounfoo foobar foobars foosfoo foosfoos foosunfoo foosbar foosbars \ + unfoofoo unfoofoos unfoounfoo unfoobar unfoobars barfoo barfoos barunfoo barbar barbars barsfoo barsfoos \ + barsunfoo barsbar barsbars", + }, + Pinned { + name: "syllable_hu_suffix", + aff: "LANG hu_HU\nCOMPOUNDMIN 1\nCOMPOUNDFLAG Y\nCOMPOUNDWORDMAX 2\nCOMPOUNDSYLLABLE 4 aeiou\nSFX C Y 1\nSFX C 0 ata .\nSFX D Y 1\nSFX D 0 ete/Z .\nSFX E Y 1\nSFX E 0 t/F .\nSFX F Y 1\nSFX F 0 ete .", + dic: "ba/Y bab/YCDE", + pieces: "ba bab babata babete babtete", + extra: "bababababata babababababete bababababtete", + right: true, + listed: "ba bab babata babete babtete baba babab bababata bababete bababtete babba babbab babbabata babbabete \ + babbabtete bababa bababab babababata babababtete bababba bababbab bababbabata bababbabtete babbaba babbabab \ + babbababata babbababtete babbabba babbabbab babbabbabata babbabbabtete bababababata bababababtete", + }, + Pinned { + name: "near_prefix_shadow", + aff: "COMPOUNDBEGIN x\nCOMPOUNDMIDDLE y\nCOMPOUNDEND z\nNEEDAFFIX h\nONLYINCOMPOUND o\nSFX S Y 1\nSFX S 0 ad .\nPFX P Y 1\nPFX P 0 fa .\nSFX T Y 1\nSFX T 0 hr .\nPFX V Y 1\nPFX V 0 ve .", + dic: "fahr/x rad/y verkehr/zo hrrad/P fahrrad/h rke/VT fahrr/S", + pieces: "fahr rad verkehr", + extra: "fahrradverkehr fahrradverkehrr fahrradverkehrad", + right: true, + listed: "fahr rad verkehr fahrrad fahrverkehr fahrradverkehr", + }, + ]; +} diff --git a/crates/proof/src/affix_suggest.rs b/crates/proof/src/affix_suggest.rs index 3c9b06bc..3b498542 100644 --- a/crates/proof/src/affix_suggest.rs +++ b/crates/proof/src/affix_suggest.rs @@ -25,8 +25,12 @@ use crate::affix::{AffixDict, Casing, capitalize, casing}; /// Longest word suggestions are looked for, in letters. pub(crate) const MAX_LETTERS: usize = 40; const MAX_MAP_VARIANTS: usize = 4096; +/// Most candidates kept waiting for the compound check in one call (the rest are dropped). +const MAX_DEFERRED: usize = 1 << 14; /// Stems the scan of round 6 expands. const SCAN_STEMS: usize = 48; +/// Most affix rules tried in all to make those stems' forms (Mongolian has 12,000 rules). +const SCAN_FORMS_WORK: usize = 1 << 16; const COST_CASE: u32 = 0; pub(crate) const COST_ADD_DIACRITIC: u32 = 4; @@ -125,6 +129,12 @@ struct Collector<'a> { names: bool, /// May half of a missing space be one ([`Tuning::strict_case`])? split_names: bool, + /// Candidates no simple word, left for the compound check: (cost, order, candidate). + deferred: Vec<(u32, usize, String)>, + /// Steps left for the compound searches of this call (`MAX_SUGGEST_COMPOUND_WORK` at first). + compound_work: usize, + /// The pieces those searches analysed, shared between them. + pieces: crate::affix::Pieces, } impl Collector<'_> { @@ -141,18 +151,68 @@ impl Collector<'_> { self.tested.insert(cand.to_string(), cost); } } + // A candidate is checked as simple words here; one of a single word that isn't one waits + // for the compound check, which the cheapest candidates get first ([`Self::compounds`]). + if !self.accept(cand, cost, false) && self.d.flag_compounds() && !cand.contains(' ') && self.deferred.len() < MAX_DEFERRED { + self.seq += 1; + self.deferred.push((cost, self.seq, cand.to_string())); + } + } + + /// Check the waiting candidates as compounds, cheapest (then earliest) first, in one budget + /// of steps per call: compound checks cost far more than lookups, and a one-letter slip of a + /// long compound is the likeliest intent. + /// + /// A candidate changed where the word stops being parts (`ziekenhusopname`: `zieken` and + /// `opname` are, `hus` isn't) goes first. + fn compounds(&mut self) { + let mut waiting = std::mem::take(&mut self.deferred); + if waiting.is_empty() { + return; + } + let gap = self.d.compound_gap(&self.input, self.names, &mut self.compound_work, &mut self.pieces); + let input: Vec = self.input.chars().collect(); + let outside = |cand: &str| { + let Some((from, to)) = gap else { return false }; + let at = cand.chars().zip(&input).position(|(a, b)| a != *b).unwrap_or(input.len().min(cand.chars().count())); + at + 1 < from || at > to + 1 + }; + let mut waiting: Vec<(bool, u32, usize, String)> = waiting.drain(..).map(|(cost, seq, cand)| (outside(&cand), cost, seq, cand)).collect(); + waiting.sort_unstable(); + for (_, cost, _, cand) in waiting { + if self.compound_work == 0 { + break; + } + // Offered again at a lower cost meanwhile: that offer is the one to check. + if self.tested.get(&cand).is_some_and(|&c| c < cost) || self.found.contains_key(&cand) { + continue; + } + self.accept(&cand, cost, true); + } + } + + /// Keep `cand` if each of its words is in the dictionary (`compounds`: flag-based compounds + /// count); whether it was. + fn accept(&mut self, cand: &str, cost: u32, compounds: bool) -> bool { let mut words = Vec::new(); let mut cost = cost; let names = if cand.contains(' ') { self.split_names } else { self.names }; + let d = self.d; + let mut work = self.compound_work; + let pieces = &mut self.pieces; + let mut is_word = + |w: &str| if compounds { d.flag_compound_spending_suggestible(w, &mut work, pieces) } else { d.contains_suggestible_with(w, false) }; for part in cand.split(' ') { - if self.d.contains_suggestible(part) { + if is_word(part) { words.push(part.to_string()); } else if !names { - return; + self.compound_work = work; + return false; } else { let c = capitalize(part); - if !self.d.contains_suggestible(&c) { - return; + if !is_word(&c) { + self.compound_work = work; + return false; } words.push(c); // A name ranks after a common word at the same distance (`moze` → `może`, then @@ -160,7 +220,9 @@ impl Collector<'_> { cost += 1; } } + self.compound_work = work; self.keep(words.join(" "), cost); + true } /// Keep a suggestion known to be correct, unless the dictionary says never to offer it. @@ -303,12 +365,25 @@ pub(crate) fn suggest_in(d: &AffixDict, word: &str, max: usize, t: &Tuning) -> V if n == 0 || n > MAX_LETTERS || max == 0 || d.is_empty() || word.contains(char::is_whitespace) { return Vec::new(); } + // The simple-word checks of this call share one budget of affix rules. + let _steps = crate::affix::SuggestSteps::start(); let case = casing(word); let input = word.to_lowercase(); let chars: Vec = input.chars().collect(); let names = !t.strict_case || case != Casing::Lower; let split_names = !t.strict_case; - let mut col = Collector { d, input: input.clone(), tested: HashMap::new(), found: HashMap::new(), seq: 0, names, split_names }; + let mut col = Collector { + d, + input: input.clone(), + tested: HashMap::new(), + found: HashMap::new(), + seq: 0, + names, + split_names, + deferred: Vec::new(), + compound_work: crate::affix::MAX_SUGGEST_COMPOUND_WORK, + pieces: Default::default(), + }; add_case(&mut col, word, &input); let map = groups(d, t); add_map_slips(&mut col, &chars, &map, t); @@ -316,9 +391,11 @@ pub(crate) fn suggest_in(d: &AffixDict, word: &str, max: usize, t: &Tuning) -> V add_splits(&mut col, &input, &t.split); add_key_neighbours(&mut col, &chars); add_typos(&mut col, &chars, t); + col.compounds(); // 6. Further off: scan the stems that start like the word. if col.count_within(COST_MISTYPE) < max { scan(&mut col, &chars, &map); + col.compounds(); } let mut found: Vec<(String, (u32, usize))> = col.found.into_iter().collect(); // Among equal costs, the change nearest the word's start is the likelier intent @@ -521,8 +598,9 @@ fn scan(col: &mut Collector, chars: &[char], map: &[Vec]) { best.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(b.1))); let related = |a: char, b: char| fold(a, map) == fold(b, map); let limit = 4; + let mut budget = SCAN_FORMS_WORK; for (_, stem) in best { - for form in d.forms(stem) { + for form in d.forms_spending(stem, &mut budget) { let lf: Vec = form.to_lowercase().chars().collect(); if lf.len().abs_diff(chars.len()) > 2 { continue; @@ -640,8 +718,18 @@ mod tests { #[test] fn a_cheaper_later_offer_wins() { let d = AffixDict::from_hunspell("", "1\ncafe\n", &[]).unwrap(); - let mut col = - Collector { d: &d, input: "cafè".into(), tested: HashMap::new(), found: HashMap::new(), seq: 0, names: true, split_names: true }; + let mut col = Collector { + d: &d, + input: "cafè".into(), + tested: HashMap::new(), + found: HashMap::new(), + seq: 0, + names: true, + split_names: true, + deferred: Vec::new(), + compound_work: crate::affix::MAX_SUGGEST_COMPOUND_WORK, + pieces: Default::default(), + }; col.offer("cafe", COST_DROP_DIACRITIC); col.offer("cafe", COST_REP); col.offer("cafe", COST_SCAN); diff --git a/crates/proof/src/cs/grammar.rs b/crates/proof/src/cs/grammar.rs index 8a2590e6..e2d29954 100644 --- a/crates/proof/src/cs/grammar.rs +++ b/crates/proof/src/cs/grammar.rs @@ -62,11 +62,8 @@ fn spaces(text: &str, v: &mut Vec) { v.push(issue(i, i + 1 + p.len_utf8(), "Mezera před interpunkcí".into(), vec![p.to_string()])); } } - for (i, _) in text.match_indices(" ") { - if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) { - continue; - } - v.push(issue(i, i + 2, "Dvojitá mezera".into(), vec![" ".into()])); + for run in crate::space_runs(text) { + v.push(issue(run.start, run.end, "Dvojitá mezera".into(), vec![" ".into()])); } } diff --git a/crates/proof/src/cs/mod.rs b/crates/proof/src/cs/mod.rs index 2be5e3f0..b1695140 100644 --- a/crates/proof/src/cs/mod.rs +++ b/crates/proof/src/cs/mod.rs @@ -63,7 +63,8 @@ fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool { return true; }; if w.contains('-') { - return w.split('-').all(|p| p.chars().count() <= 1 || known(d, p)); + // The whole word first, as Hunspell does: a hyphenated entry or user word. + return known(d, w) || w.split('-').all(|p| p.chars().count() <= 1 || known(d, p)); } known(d, w) } @@ -142,6 +143,14 @@ mod tests { } } + /// craft-libs#496 review: a hyphenated dictionary entry is a word even when its parts aren't. + #[test] + fn a_hyphenated_entry_is_looked_up_whole() { + let d = AffixDict::from_hunspell("SET UTF-8\n", "1\ntic-tac\n", &[]).unwrap(); + assert!(is_correct_with(&d, "tic-tac")); + assert!(!is_correct_with(&d, "tic-tacc")); + } + #[test] fn missing_dictionary_and_ignored_inputs_are_quiet() { let empty = AffixDict::default(); diff --git a/crates/proof/src/cs/suggest.rs b/crates/proof/src/cs/suggest.rs index 05eba74a..1e31496e 100644 --- a/crates/proof/src/cs/suggest.rs +++ b/crates/proof/src/cs/suggest.rs @@ -75,4 +75,13 @@ mod tests { assert!(!suggest("město", 4).contains(&"město".into())); assert!(!suggest("zqzakázane", 8).contains(&"zqzakázané".into())); } + + #[test] + fn a_letter_that_lowercases_to_two_chars_is_still_suggested_for() { + // wordcraft#4683: "İx" lowercases to three chars ("i", U+0307, "x") but has two typed + // letters; the one-letter edits stay in range and dropping the dot still finds "ix". + let d = AffixDict::from_hunspell("", "1\nix\n", &[]).unwrap(); + let suggestions = suggest_in(&d, "İx", 4); + assert!(suggestions.contains(&"Ix".into()), "{suggestions:?}"); + } } diff --git a/crates/proof/src/en_variants.rs b/crates/proof/src/en_variants.rs index 07c16bb1..c8a63fd8 100644 --- a/crates/proof/src/en_variants.rs +++ b/crates/proof/src/en_variants.rs @@ -1,5 +1,5 @@ //! American and British spellings of the same English word (#1281, #1045): which forms each -//! English ([`ProofLang::En`], [`ProofLang::EnCa`], [`ProofLang::EnGb`]) accepts, which it flags, +//! English ([`ProofLang::En`], [`ProofLang::EnCa`], [`ProofLang::EnGb`] and [`ProofLang::EnAu`]) accepts, which it flags, //! and what it suggests instead (`colour` → `color` in American English). //! //! The pairs come from craft-i18n's `language-data/en-us/en-variants.txt`, our own list of base forms with a @@ -77,7 +77,8 @@ fn column(lang: ProofLang) -> Option { match lang { ProofLang::En => Some(0), ProofLang::EnCa => Some(1), - ProofLang::EnGb => Some(2), + // Australian English follows the British policy (`colour`, `centre`, `organise`). + ProofLang::EnGb | ProofLang::EnAu => Some(2), _ => None, } } diff --git a/crates/proof/src/english.rs b/crates/proof/src/english.rs index 0c2c2e13..e821e801 100644 --- a/crates/proof/src/english.rs +++ b/crates/proof/src/english.rs @@ -4,10 +4,11 @@ //! user keeps for WordCraft ([`crate::hunspell::user_dirs`]: the folder set in the options, //! `WORDCRAFT_DICTIONARIES`, WordCraft's own `dictionaries` folder). //! 2. **Installed English pack**: reserved for Language packs (craft-libs#485, CP4.2). -//! 3. **Embedded SCOWL**: the run's English's SCOWL dictionary (en-US, en-GB or en-CA) embedded +//! 3. **Embedded SCOWL**: the run's English's SCOWL dictionary (en-US, en-GB, en-AU or en-CA) embedded //! from craft-i18n at build time ([`crate::scowl`]); a word it knows (British English: also //! the `-ize` spelling of one it knows, [`ise_spelling`]), or the supplement ([`EXTRA`]), is -//! right; never one only another English knows. When it is there, the system locations aren't +//! right; never one only another English knows, except that Australian English also takes +//! en-GB's words (its suggestions still come from en-AU). When it is there, the system locations aren't //! searched. //! 4. **System Hunspell**: a pair for the run's own English in the system locations (none on the //! web). `COMPOUNDRULE` (ordinals) and `OCONV` are read; flag-based compounding is ignored @@ -18,8 +19,9 @@ //! embedded from craft-i18n is the same list; it is used for hyphenation only.) //! //! Hunspell dictionaries are found by [`crate::hunspell`] by their own English's names only -//! (`en_US` for American, `en_GB` for British, `en_CA` for Canadian English), each name in every -//! folder before the next name. Another English's dictionary is never used: it would mark this +//! (`en_US` for American, `en_GB` for British, `en_CA` for Canadian English; `en_AU`, then +//! `en_GB`, for Australian), each name in every folder before the next name. Another English's +//! dictionary is never used: it would mark this //! English's spellings (`colour` in an `en_US` one). A file that can't be read is skipped for the //! next name or folder, and one file is read once for every English that uses it. `COMPOUNDRULE` //! (ordinals) and `OCONV` (suggestions are written as the dictionary says: `don’t`) are read; @@ -90,15 +92,47 @@ fn names(lang: ProofLang) -> &'static [&'static str] { match lang { ProofLang::EnGb => &["en_GB", "en-GB"], ProofLang::EnCa => &["en_CA", "en-CA"], + // Australian English's own, else a British one (a system en_GB on a build without SCOWL): + // [`exact_candidates`] reads every en_AU source before any en_GB one. + ProofLang::EnAu => &["en_AU", "en-AU", "en_GB", "en-GB"], _ => &["en_US", "en-US"], } } +/// The Hunspell pairs of `lang` without SCOWL, from `find` (a folder search for some names, +/// [`crate::hunspell::candidates_exact`]): its own names first, and only then Australian English's +/// British fallback, so every en_AU source (a dictionaries folder, a pack, the system) comes before +/// any en_GB one. +fn exact_candidates(lang: ProofLang, find: impl Fn(&[&str]) -> Vec<(PathBuf, PathBuf)>) -> Vec<(PathBuf, PathBuf)> { + let mut out = find(own_names(lang)); + if lang == ProofLang::EnAu { + for c in find(&names(lang)[own_names(lang).len()..]) { + if !out.iter().any(|(_, d)| *d == c.1) { + out.push(c); + } + } + } + out +} + +/// The names [`names`] gives that outrank the dialect's own embedded SCOWL dictionary: its own pair +/// only, not Australian English's British fallback (a user's `en_GB` is for British text). +fn own_names(lang: ProofLang) -> &'static [&'static str] { + match lang { + ProofLang::EnAu => &["en_AU", "en-AU"], + _ => names(lang), + } +} + +/// How many Englishes have a slot of their own ([`slot`]). +const ENGLISHES: usize = 4; + /// The slot of each English in [`State::slots`]. fn slot(lang: ProofLang) -> usize { match lang { ProofLang::EnGb => 1, ProofLang::EnCa => 2, + ProofLang::EnAu => 3, _ => 0, } } @@ -109,7 +143,7 @@ type Loaded = Option<(Arc, PathBuf)>; /// The English Hunspell dictionaries, under one lock. struct State { /// Per English (see [`slot`]): `None` until first use, then what was found. - slots: [Option; 3], + slots: [Option; ENGLISHES], /// Every `.dic` file read, so Englishes that use the same file share one copy. files: Vec<(PathBuf, Arc)>, /// Counts reloads, so a load that started before one is not kept. @@ -118,7 +152,7 @@ struct State { impl State { const fn new() -> Self { - State { slots: [None, None, None], files: Vec::new(), reloads: 0 } + State { slots: [None, None, None, None], files: Vec::new(), reloads: 0 } } } @@ -181,7 +215,7 @@ fn already_read(state: &RwLock, dic: &Path) -> Option> { /// Forget what `state` holds, so the next check looks again. fn forget(state: &RwLock) { let mut s = state.write().unwrap_or_else(PoisonError::into_inner); - s.slots = [None, None, None]; + s.slots = [None, None, None, None]; s.files.clear(); s.reloads = s.reloads.wrapping_add(1); } @@ -198,9 +232,9 @@ fn key(lang: ProofLang) -> &'static str { /// would rather use), for [`note_error`]. fn load(lang: ProofLang) -> (Loaded, Option<(PathBuf, String)>) { let candidates = if crate::scowl::dict(lang).is_some() { - crate::hunspell::user_candidates(names(lang)) + crate::hunspell::user_candidates(own_names(lang)) } else { - crate::hunspell::candidates_exact(names(lang)) + exact_candidates(lang, crate::hunspell::candidates_exact) }; if candidates.is_empty() { log::info!("English spelling: no Hunspell {} dictionary to read first; using SCOWL or the built-in word list", key(lang)); @@ -240,7 +274,7 @@ fn note_error(lang: ProofLang, error: Option<(PathBuf, String)>) { /// folder changed). pub(crate) fn reload() { forget(&HUNSPELL); - for lang in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa] { + for lang in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa, ProofLang::EnAu] { crate::hunspell::clear_load_error(key(lang)); } } @@ -283,10 +317,16 @@ fn pick(found: Loaded, lang: ProofLang) -> Speller { if let Some((d, dic)) = found { return Speller::Hunspell(d, dic); } - match crate::scowl::dict(lang) { - Some(d) => Speller::Scowl(vec![d], crate::scowl::is_british(lang)), - None => Speller::List, + let Some(d) = crate::scowl::dict(lang) else { return Speller::List }; + let mut ds = vec![d]; + // SCOWL's en-AU list keeps one spelling per word (`inquiry`; `enquiry` is only in the en-GB + // list), and Australian English takes both. Suggestions come from the first, en-AU. + if lang == ProofLang::EnAu + && let Some(gb) = crate::scowl::dict(ProofLang::EnGb) + { + ds.push(gb); } + Speller::Scowl(ds, crate::scowl::is_british(lang)) } /// [`speller`] for English with no dialect ([`crate::is_correct`]): a SCOWL word of any English @@ -348,11 +388,12 @@ pub(crate) fn is_correct(word: &str, lang: ProofLang, opts: &Options, policy: bo || (policy && !parts.is_empty() && parts.iter().all(|p| s.knows_lower(p) || crate::en_variants::known_variant(p, lang, |x| s.knows_lower(x)))) } -/// `w` with its `-ize`/`-yze` endings (and their forms: `-ization`, `-izing`…) spelled `-ise`/ -/// `-yse`, when it has any: British English takes both (Word accepts `organize` and `organise` in -/// en-GB), and SCOWL's en-GB dictionary has only `-ise`. A `z` between `i`/`y` and a vowel is -/// swapped; the `-ize` form counts only when American or Canadian English spells the word so -/// ([`ize_elsewhere`]), or `wize` would pass as `wise`. +/// `w` with its `-ize` endings (and their forms: `-ization`, `-izing`…) spelled `-ise`, when it +/// has any: British English takes both (Word accepts `organize` and `organise` in en-GB), and +/// SCOWL's en-GB dictionary has only `-ise`. A `z` between `i` and a vowel is swapped; the `-ize` +/// form counts only when American or Canadian English spells the word so ([`ize_elsewhere`]), or +/// `wize` would pass as `wise`. Never `-yze`: Oxford spelling has `analyse`, `paralyse`, so +/// `reanalyze` or `breathalyzer` is American in British English, as Hunspell's en_GB says. pub(crate) fn ise_spelling(w: &str) -> Option { let b = w.as_bytes(); let mut out = b.to_vec(); @@ -360,7 +401,7 @@ pub(crate) fn ise_spelling(w: &str) -> Option { for i in 1..b.len() { let after = b.get(i + 1).is_some_and(|c| b"aeiAEI".contains(c)); if after - && matches!(b.get(i - 1), Some(b'i' | b'y' | b'I' | b'Y')) + && matches!(b.get(i - 1), Some(b'i' | b'I')) && let Some(c) = out.get_mut(i) { match *c { @@ -512,6 +553,18 @@ fn list_suggest(word: &str, max: usize) -> Vec { mod tests { use super::*; + /// Review follow-up: only `-ize` is swapped for `-ise`, never `-yze` for `-yse` (Oxford + /// spelling has `analyse`), so `reanalyze`, `breathalyzer`, `paralyzingly` stay American. + #[test] + fn ise_spelling_swaps_ize_only() { + assert_eq!(ise_spelling("organize").as_deref(), Some("organise")); + assert_eq!(ise_spelling("Realizations").as_deref(), Some("Realisations")); + assert_eq!(ise_spelling("ORGANIZING").as_deref(), Some("ORGANISING")); + for w in ["reanalyze", "reanalyzed", "breathalyze", "breathalyzer", "paralyzingly", "catalyze", "Analyzer", "ANALYZE"] { + assert_eq!(ise_spelling(w), None, "{w}"); + } + } + #[test] fn built_in_list_is_the_whole_moby_list() { let d = words(); @@ -596,16 +649,16 @@ mod tests { fn lookup_order() { let tiny = Arc::new(AffixDict::from_hunspell("", "1\nzzyzx\n", &[]).unwrap()); let file = PathBuf::from("/folder/en_US.dic"); - for lang in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa] { + for lang in [ProofLang::En, ProofLang::EnGb, ProofLang::EnAu, ProofLang::EnCa] { assert!(matches!(pick(Some((tiny.clone(), file.clone())), lang), Speller::Hunspell(_, p) if p == file), "{lang:?}"); let fallback = pick(None, lang); if crate::scowl::dict(lang).is_some() { assert!( - matches!(&fallback, Speller::Scowl(ds, _) if ds.len() == 1 && std::ptr::eq(ds[0], crate::scowl::dict(lang).unwrap())), + matches!(&fallback, Speller::Scowl(ds, _) if ds.len() == if lang == ProofLang::EnAu { 2 } else { 1 } && std::ptr::eq(ds[0], crate::scowl::dict(lang).unwrap())), "{lang:?}" ); assert!(fallback.knows("21st") && !fallback.knows("becuase"), "{lang:?}"); - assert_eq!(fallback.knows("colour"), lang != ProofLang::En, "{lang:?}: its own dictionary only"); + assert_eq!(fallback.knows("colour"), lang != ProofLang::En, "{lang:?}: its own dictionary (en-AU: and en-GB's)"); } else { assert!(matches!(fallback, Speller::List), "{lang:?}"); } @@ -625,6 +678,44 @@ mod tests { assert_eq!(found(names(ProofLang::EnGb)), ["en_GB"], "a user's en_US doesn't stand in for SCOWL's en-GB"); assert_eq!(found(names(ProofLang::EnCa)), ["en_CA"]); assert_eq!(found(names(ProofLang::En)), ["en_US"]); + // Australian English: its own, else the British one it read before it had its own. + assert_eq!(found(names(ProofLang::EnAu)), ["en_GB"]); + std::fs::write(dir.join("en_AU.aff"), "SET UTF-8\n").unwrap(); + std::fs::write(dir.join("en_AU.dic"), "1\nzzyzx\n").unwrap(); + assert_eq!(found(names(ProofLang::EnAu)), ["en_AU", "en_GB"]); + // craft-libs#496 review: with SCOWL embedded, a user's en_GB must not outrank its en-AU. + assert_eq!(found(own_names(ProofLang::EnAu)), ["en_AU"]); + std::fs::remove_file(dir.join("en_AU.aff")).unwrap(); + std::fs::remove_file(dir.join("en_AU.dic")).unwrap(); + assert!(found(own_names(ProofLang::EnAu)).is_empty(), "an en_GB alone is not Australian English's own"); + for lang in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa] { + assert_eq!(own_names(lang), names(lang)); + } + let _ = std::fs::remove_dir_all(&dir); + } + + /// Without SCOWL, every en_AU source comes before any en_GB one (a dictionaries folder's, an + /// installed pack's, the system's): a user's en_GB doesn't outrank a system en_AU. + #[test] + fn australian_english_reads_every_en_au_before_any_en_gb() { + let dir = std::env::temp_dir().join(format!("wc-proof-en-au-order-{}", std::process::id())); + for sub in ["user", "system"] { + std::fs::create_dir_all(dir.join(sub)).unwrap(); + } + let write = |sub: &str, name: &str| { + std::fs::write(dir.join(sub).join(format!("{name}.aff")), "SET UTF-8\n").unwrap(); + std::fs::write(dir.join(sub).join(format!("{name}.dic")), "1\nzzyzx\n").unwrap(); + }; + write("user", "en_GB"); + write("system", "en_AU"); + let (user, system) = (vec![dir.join("user")], vec![dir.join("system")]); + let find = |n: &[&str]| crate::hunspell::ordered(n, &user, &[], &system, &[]); + let stems = |v: Vec<(PathBuf, PathBuf)>| -> Vec { + v.iter().filter_map(|(_, d)| d.file_stem().map(|s| s.to_string_lossy().into_owned())).collect() + }; + assert_eq!(stems(exact_candidates(ProofLang::EnAu, find)), ["en_AU", "en_GB"]); + assert_eq!(stems(exact_candidates(ProofLang::EnGb, find)), ["en_GB"]); + assert_eq!(stems(exact_candidates(ProofLang::En, find)), Vec::::new()); let _ = std::fs::remove_dir_all(&dir); } diff --git a/crates/proof/src/es/grammar.rs b/crates/proof/src/es/grammar.rs index 42eede50..00b20789 100644 --- a/crates/proof/src/es/grammar.rs +++ b/crates/proof/src/es/grammar.rs @@ -75,11 +75,8 @@ fn spaces(text: &str, lang: ProofLang, v: &mut Vec) { v.push(issue(i, i + 1 + p.len_utf8(), "Espacio antes de un signo de puntuación".into(), vec![p.to_string()], lang)); } } - for (i, _) in text.match_indices(" ") { - if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) { - continue; - } - v.push(issue(i, i + 2, "Espacio doble".into(), vec![" ".into()], lang)); + for run in crate::space_runs(text) { + v.push(issue(run.start, run.end, "Espacio doble".into(), vec![" ".into()], lang)); } } @@ -194,6 +191,16 @@ mod tests { .collect() } + /// craft-libs#496 review: a run of spaces is one issue over the whole run. + #[test] + fn a_run_of_spaces_is_reported_whole() { + for (text, run) in [("casa roja", " "), ("casa roja", " "), ("casa roja", " "), ("casa roja", " ")] { + let m: Vec<_> = messages(text).into_iter().filter(|(_, msg)| msg == "Espacio doble").collect(); + assert_eq!(m, [(run.to_string(), "Espacio doble".to_string())], "{text}"); + } + assert!(messages("casa roja").is_empty()); + } + #[test] fn repeated_words_and_spaces_have_spanish_messages() { let text = "Es es la casa , dijo."; diff --git a/crates/proof/src/hr/grammar.rs b/crates/proof/src/hr/grammar.rs index 2605ae88..f4997e9d 100644 --- a/crates/proof/src/hr/grammar.rs +++ b/crates/proof/src/hr/grammar.rs @@ -105,11 +105,8 @@ fn spaces(text: &str, v: &mut Vec) { v.push(issue(i, i + 1 + p.len_utf8(), "Razmak ispred interpunkcijskog znaka".into(), vec![p.to_string()])); } } - for (i, _) in text.match_indices(" ") { - if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) { - continue; - } - v.push(issue(i, i + 2, "Dvostruki razmak".into(), vec![" ".into()])); + for run in crate::space_runs(text) { + v.push(issue(run.start, run.end, "Dvostruki razmak".into(), vec![" ".into()])); } } diff --git a/crates/proof/src/hr/mod.rs b/crates/proof/src/hr/mod.rs index 53edf067..e0bda809 100644 --- a/crates/proof/src/hr/mod.rs +++ b/crates/proof/src/hr/mod.rs @@ -65,7 +65,8 @@ fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool { return true; }; if w.contains('-') { - return w.split('-').all(|p| p.chars().count() <= 1 || known(d, p)); + // The whole word first, as Hunspell does: a hyphenated entry or user word. + return known(d, w) || w.split('-').all(|p| p.chars().count() <= 1 || known(d, p)); } known(d, w) } @@ -105,6 +106,14 @@ mod tests { } } + /// craft-libs#496 review: a hyphenated dictionary entry is a word even when its parts aren't. + #[test] + fn a_hyphenated_entry_is_looked_up_whole() { + let d = AffixDict::from_hunspell("SET UTF-8\n", "1\ntic-tac\n", &[]).unwrap(); + assert!(is_correct_with(&d, "tic-tac")); + assert!(!is_correct_with(&d, "tic-tacc")); + } + #[test] fn missing_dictionary_is_silent() { assert!(is_correct_with(&AffixDict::default(), "nepostojećariječ")); diff --git a/crates/proof/src/hunspell.rs b/crates/proof/src/hunspell.rs index 274af9eb..966ded72 100644 --- a/crates/proof/src/hunspell.rs +++ b/crates/proof/src/hunspell.rs @@ -196,19 +196,22 @@ pub(crate) fn ordered( let first = names.first().map(|n| norm(n)); let (region, rest): (Vec<&str>, Vec<&str>) = names.iter().copied().partition(|n| Some(norm(n)) == first); let pack: &[&str] = &[crate::packs::SPELLING]; + // (folder, names, a Language pack's: never read through a symbolic link, as the pack scan + // checks; a system folder's dictionaries often are links, `en_GB.dic` → `en_GB-ise.dic`). let tiers = dirs .iter() - .map(|d| (d, names)) - .chain(region_packs.iter().map(|d| (d, pack))) - .chain(system.iter().map(|d| (d, ®ion[..]))) - .chain(other_packs.iter().map(|d| (d, pack))) - .chain(system.iter().map(|d| (d, &rest[..]))); + .map(|d| (d, names, false)) + .chain(region_packs.iter().map(|d| (d, pack, true))) + .chain(system.iter().map(|d| (d, ®ion[..], false))) + .chain(other_packs.iter().map(|d| (d, pack, true))) + .chain(system.iter().map(|d| (d, &rest[..], false))); + let file = |p: &Path, pack: bool| if pack { std::fs::symlink_metadata(p).is_ok_and(|m| m.is_file()) } else { p.is_file() }; let mut out: Vec<(PathBuf, PathBuf)> = Vec::new(); - for (dir, names) in tiers { + for (dir, names, in_pack) in tiers { for n in names { let aff = dir.join(format!("{n}.aff")); let dic = dir.join(format!("{n}.dic")); - if aff.is_file() && dic.is_file() && !out.iter().any(|(_, d)| *d == dic) { + if file(&aff, in_pack) && file(&dic, in_pack) && !out.iter().any(|(_, d)| *d == dic) { out.push((aff, dic)); } } @@ -696,6 +699,34 @@ mod tests { } } + /// Review follow-up (craft-libs#485): a pack's `spelling.aff`/`.dic` that is a symbolic link + /// (swapped in after the pack scan) is never a candidate; one in a system folder still is. + #[cfg(unix)] + #[test] + fn pack_files_are_never_read_through_a_link() { + let root = std::env::temp_dir().join(format!("wc-proof-pack-links-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&root); + let (pack, system, elsewhere) = (root.join("packs").join("nl"), root.join("system"), root.join("elsewhere")); + for d in [&pack, &system, &elsewhere] { + std::fs::create_dir_all(d).unwrap(); + } + std::fs::write(elsewhere.join("x.aff"), "SET UTF-8\n").unwrap(); + std::fs::write(elsewhere.join("x.dic"), "1\nx\n").unwrap(); + std::fs::write(pack.join("spelling.aff"), "SET UTF-8\n").unwrap(); + std::os::unix::fs::symlink(elsewhere.join("x.dic"), pack.join("spelling.dic")).unwrap(); + std::os::unix::fs::symlink(elsewhere.join("x.aff"), system.join("nl.aff")).unwrap(); + std::os::unix::fs::symlink(elsewhere.join("x.dic"), system.join("nl.dic")).unwrap(); + let got: Vec = + ordered(&["nl"], &[], std::slice::from_ref(&pack), std::slice::from_ref(&system), &[]).into_iter().map(|(_, d)| d).collect(); + assert_eq!(got, [system.join("nl.dic")], "the system's link is followed, the pack's never"); + std::fs::remove_file(pack.join("spelling.dic")).unwrap(); + std::fs::write(pack.join("spelling.dic"), "1\nx\n").unwrap(); + let got: Vec = + ordered(&["nl"], &[], std::slice::from_ref(&pack), std::slice::from_ref(&system), &[]).into_iter().map(|(_, d)| d).collect(); + assert_eq!(got, [pack.join("spelling.dic"), system.join("nl.dic")]); + let _ = std::fs::remove_dir_all(&root); + } + /// WordCraft's own dictionaries folder under the user's application data is searched on every /// desktop system, after the setting and before the shared system locations (#1063). #[test] diff --git a/crates/proof/src/hyphen.rs b/crates/proof/src/hyphen.rs index 6bc804b2..cc9d4fbd 100644 --- a/crates/proof/src/hyphen.rs +++ b/crates/proof/src/hyphen.rs @@ -217,18 +217,25 @@ fn composed_chars(core: &[char]) -> (Vec, Vec>) { } /// Does a pack's `pats` hyphenate `core` in `lang`? Russian, Bulgarian and Turkish: a word in -/// their script (the guard their rules use). Polish, Portuguese, Italian and German: every word, -/// as their embedded patterns do. Any other language: a word whose letters the patterns know. +/// their script (the guard their rules use) whose letters the patterns know, so a word with a +/// letter the pack never mentions (`ё`) goes on to the built-in rules. Polish, Portuguese, Italian +/// and German: every word, as their embedded patterns do. Any other language: a word whose letters +/// the patterns know. fn pack_patterns_fit(lang: ProofLang, pats: &Patterns, core: &[char]) -> bool { match lang { - ProofLang::Ru => core.iter().all(|&c| crate::ru::is_cyrillic(c)), - ProofLang::Bg => core.iter().all(|&c| crate::bg::is_letter(c)), - ProofLang::Tr => core.iter().all(|&c| crate::tr::hyphen::is_letter(c)), + ProofLang::Ru => core.iter().all(|&c| crate::ru::is_cyrillic(c)) && pack_knows(pats, core), + ProofLang::Bg => core.iter().all(|&c| crate::bg::is_letter(c)) && pack_knows(pats, core), + ProofLang::Tr => core.iter().all(|&c| crate::tr::hyphen::is_letter(c)) && pack_knows(pats, core), ProofLang::Pl | ProofLang::Pt | ProofLang::It | ProofLang::De => true, _ => pats.values(core).is_some(), } } +/// Do `pats` know `core`: an exception word, or a word whose letters the patterns mention? +fn pack_knows(pats: &Patterns, core: &[char]) -> bool { + pats.exception(&core.iter().collect::()).is_some() || pats.values(core).is_some() +} + /// Polish: exception words, then the patterns, then the Polish syllable heuristic. fn core_breaks_pl(core: &[char]) -> (Vec, Source) { core_breaks_pl_with(Patterns::pl(), core) @@ -534,6 +541,30 @@ fn heuristic_points_pl(c: &[char]) -> Vec { mod tests { use super::*; + /// craft-libs#497 review: a Russian, Bulgarian or Turkish pack's patterns only take a word + /// whose letters they know; any other goes to the built-in rules, not the English heuristic. + #[test] + fn pack_patterns_for_ru_bg_tr_need_every_letter_of_the_word() { + let chars = |w: &str| w.chars().collect::>(); + let cases = [(ProofLang::Ru, "1на 1ка", "нака", "ёка"), (ProofLang::Bg, "1на 1ка", "нака", "жка"), (ProofLang::Tr, "a1b", "abab", "abğ")]; + for (lang, pat, known, unknown) in cases { + let p = Patterns::parse(pat); + assert!(pack_patterns_fit(lang, &p, &chars(known)), "{lang:?} {known}"); + assert!(!pack_patterns_fit(lang, &p, &chars(unknown)), "{lang:?} {unknown}"); + } + } + + /// An exception word of a pack keeps its breaks even with a letter the patterns never mention. + #[test] + fn pack_exception_words_need_no_pattern_letters() { + let chars = |w: &str| w.chars().collect::>(); + let p = Patterns::parse("1на 1ка ё-ка"); + let w = chars("ёка"); + assert!(pack_patterns_fit(ProofLang::Ru, &p, &w)); + assert_eq!(core_breaks_patterns(&p, &w), (vec![1], Source::Dictionary)); + assert!(!pack_patterns_fit(ProofLang::Ru, &p, &chars("ёна"))); + } + /// CodeRabbit on craft-libs#484: a Turkish word with a suffix after an apostrophe is /// hyphenated by the Turkish rules, part by part, never next to the apostrophe. #[test] diff --git a/crates/proof/src/id/grammar.rs b/crates/proof/src/id/grammar.rs index 72ca10c9..6c13b7f1 100644 --- a/crates/proof/src/id/grammar.rs +++ b/crates/proof/src/id/grammar.rs @@ -56,11 +56,8 @@ fn spaces(text: &str, v: &mut Vec) { v.push(issue(i, i + 1 + p.len_utf8(), "Spasi sebelum tanda baca".into(), vec![p.to_string()])); } } - for (i, _) in text.match_indices(" ") { - if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) { - continue; - } - v.push(issue(i, i + 2, "Spasi ganda".into(), vec![" ".into()])); + for run in crate::space_runs(text) { + v.push(issue(run.start, run.end, "Spasi ganda".into(), vec![" ".into()])); } } diff --git a/crates/proof/src/id/mod.rs b/crates/proof/src/id/mod.rs index c78e25d4..85dc825a 100644 --- a/crates/proof/src/id/mod.rs +++ b/crates/proof/src/id/mod.rs @@ -76,7 +76,7 @@ fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool { || w.split('-').all(|p| { p.chars().count() <= 1 || CLITICS.iter().any(|c| p.eq_ignore_ascii_case(c)) - || (opts.ignore_uppercase && p.chars().all(|c| !c.is_lowercase()) && p.chars().count() <= 6) + || (opts.ignore_uppercase && crate::spelling::capitals_part(p, 6)) || known(d, p) }); } @@ -141,6 +141,19 @@ mod tests { } } + /// Review follow-up (craft-libs#485's cased-letter rule): a part in a caseless script is no + /// abbreviation, so `buku-日本` is checked part by part; a part in capitals or without letters + /// still is one (`KTP-nya`, `COVID-19` with numbers checked). + #[test] + fn caseless_parts_are_not_abbreviations() { + test_dict::install(); + let d = dictionary(); + let defaults = crate::Options::default(); + assert!(!check(&d, "buku-日本", &defaults) && !check(&d, "buku-ไทย", &defaults)); + let numbers = crate::Options { ignore_numbers: false, ..defaults }; + assert!(check(&d, "KTP-nya", &defaults) && check(&d, "buku-19", &numbers)); + } + #[test] fn forbidden_words_are_rejected_in_every_case() { test_dict::install(); diff --git a/crates/proof/src/it/grammar.rs b/crates/proof/src/it/grammar.rs index 57c62e28..9019218c 100644 --- a/crates/proof/src/it/grammar.rs +++ b/crates/proof/src/it/grammar.rs @@ -150,10 +150,12 @@ fn abbreviation_or_number(prev: &str) -> bool { || ABBREVIATIONS.contains(&prev.to_lowercase().as_str()) } +const CLOSING: [char; 8] = ['"', '”', '’', '\'', ')', '»', ']', '\u{1}']; + /// Does the gap between two words end a sentence: a `.`, `!` or `?` (not an ellipsis, not after /// an abbreviation), then space, with nothing but quotes and brackets around? fn ends_sentence(gap: &str, prev: &str) -> bool { - let t = gap.trim_start_matches(['"', '”', '’', '\'', ')', '»', ']', '\u{1}']); + let t = gap.trim_start_matches(CLOSING); let Some(end) = t.chars().next() else { return false }; if !matches!(end, '.' | '!' | '?') || t.starts_with("..") { return false; @@ -161,8 +163,16 @@ fn ends_sentence(gap: &str, prev: &str) -> bool { if end == '.' && abbreviation_or_number(prev) { return false; } - let rest = t.trim_start_matches(['.', '!', '?']).trim_start_matches(['"', '”', '’', '\'', ')', '»', ']']); - rest.starts_with(char::is_whitespace) && rest.trim_start().chars().all(|c| matches!(c, '"' | '“' | '«' | '(' | '[' | '\'' | '—')) + let after_marks = t.trim_start_matches(['.', '!', '?']); + let rest = after_marks.trim_start_matches(CLOSING); + // (Not the object placeholder `\u{1}`: `Chi era?\u{1} era Marco.` still starts a sentence.) + // `«Vai?» disse lui.` and `—Vai? —disse lui.`: the narration after a quoted question or + // exclamation (not after a period: `Finì. —domani` starts a sentence), or after a dialogue dash, goes on in lower case. + if end != '.' && (rest.contains('—') || after_marks.trim_start_matches('\u{1}').starts_with(CLOSING)) { + return false; + } + rest.starts_with(char::is_whitespace) + && rest.trim_start().chars().all(|c| c.is_whitespace() || matches!(c, '"' | '“' | '«' | '(' | '[' | '\'' | '—')) } fn repeated_words(text: &str, ws: &[(usize, usize)], v: &mut Vec) { @@ -203,11 +213,8 @@ fn spaces(text: &str, v: &mut Vec) { v.push(issue(i, i + 1 + p.len_utf8(), "Spazio prima della punteggiatura".into(), vec![p.to_string()])); } } - for (i, _) in text.match_indices(" ") { - if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) { - continue; - } - v.push(issue(i, i + 2, "Spazio doppio".into(), vec![" ".into()])); + for run in crate::space_runs(text) { + v.push(issue(run.start, run.end, "Spazio doppio".into(), vec![" ".into()])); } } @@ -403,6 +410,23 @@ mod tests { } } + /// craft-libs#496 review: narration after a quoted or dashed question or exclamation goes on in + /// lower case. + #[test] + fn narration_after_a_quoted_question_is_not_a_new_sentence() { + for t in ["«Vai?» disse lui.", "«Vieni!» gridò Anna.", "\"Vai?\" disse lui.", "—Vai? —disse lui.", "— Vai! — disse lui."] { + assert!(fixes(t).is_empty(), "{t}: {:?}", fixes(t)); + } + assert!(fixes("Vado a Roma? sì.").contains(&pair("sì", "Sì")), "a bare question still ends the sentence"); + // After a period the dash opens the next line of speech: a sentence starts. + assert!(fixes("Finì. —domani partiamo.").contains(&pair("domani", "Domani")), "{:?}", fixes("Finì. —domani partiamo.")); + assert!(fixes("Finì. — domani partiamo.").contains(&pair("domani", "Domani")), "{:?}", fixes("Finì. — domani partiamo.")); + // An inline object (`\u{1}`) before a real closing quote is skipped. + assert!(fixes("«Vai?\u{1}» disse lui.").is_empty(), "{:?}", fixes("«Vai?\u{1}» disse lui.")); + // An inline object (`\u{1}`) is not a closing quote. + assert!(fixes("Chi era?\u{1} era Marco.").contains(&pair("era", "Era")), "{:?}", fixes("Chi era?\u{1} era Marco.")); + } + #[test] fn po_takes_an_apostrophe() { assert_eq!(fixes("Dammi un pò di pane"), [pair("pò", "po'")]); diff --git a/crates/proof/src/it/mod.rs b/crates/proof/src/it/mod.rs index a742eaa2..44550501 100644 --- a/crates/proof/src/it/mod.rs +++ b/crates/proof/src/it/mod.rs @@ -97,8 +97,14 @@ fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool { } // The whole word first, as Hunspell does: a hyphenated entry or user word. if w.contains('-') { - return known(d, w) || w.split('-').all(|p| p.chars().count() <= 1 || known(d, p)); + return known(d, w) || w.split('-').all(|p| p.chars().count() <= 1 || known_elided(d, p)); } + known_elided(d, w) +} + +/// [`known`] for a word that may start with an elided proclitic (`l'amico`, `l'ex`): the word after +/// the apostrophe is what gets checked. +fn known_elided(d: &AffixDict, w: &str) -> bool { if let Some((head, tail)) = w.split_once(['\'', '’']) && PROCLITICS.contains(&head.to_lowercase().as_str()) && !tail.is_empty() @@ -148,6 +154,16 @@ mod tests { assert!(!is_correct_with(&d, "tic-tacc")); } + /// craft-libs#496 review: an elision inside a hyphenated word (`l'ex-amico`) is read per part. + #[test] + fn an_elision_in_a_hyphenated_part_is_read() { + let d = AffixDict::from_hunspell("SET UTF-8\n", "2\nex\namico\n", &[]).unwrap(); + for w in ["l'ex-amico", "dell’ex-amico", "ex-amico"] { + assert!(is_correct_with(&d, w), "{w}"); + } + assert!(!is_correct_with(&d, "l'exx-amico")); + } + #[test] fn without_a_dictionary_nothing_is_flagged() { let none = AffixDict::default(); diff --git a/crates/proof/src/lang.rs b/crates/proof/src/lang.rs index 993e4030..463406a9 100644 --- a/crates/proof/src/lang.rs +++ b/crates/proof/src/lang.rs @@ -5,7 +5,7 @@ /// A language WordCraft can proof, or [`ProofLang::Other`] for one it can't. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, PartialOrd, Ord)] pub enum ProofLang { - /// American English (`en`, `en-US`, and every region not listed for [`ProofLang::EnGb`] or + /// American English (`en`, `en-US`, and every region not listed for [`ProofLang::EnGb`], [`ProofLang::EnAu`] or /// [`ProofLang::EnCa`]): SCOWL's en-US dictionary ([`crate::scowl`]), the English grammar rules and the /// English hyphenation dictionary and patterns. British spellings (`colour`, `centre`, /// `organise`) are flagged, with the American form suggested ([`crate::en_variants`]). Also @@ -13,10 +13,14 @@ pub enum ProofLang { /// Word's). #[default] En, - /// English with British spelling (`en-GB`, `en-AU`, `en-NZ`, `en-IE`, `en-ZA`, `en-IN`…): + /// English with British spelling (`en-GB`, `en-NZ`, `en-IE`, `en-ZA`, `en-IN`…): /// [`ProofLang::En`]'s tools, but American spellings (`color`, `center`, `analyze`) are /// flagged and `-ize` and `-ise` are both right, as in Oxford spelling. EnGb, + /// Australian English (`en-AU`): [`ProofLang::EnGb`]'s spelling policy, against SCOWL's own + /// en-AU dictionary, which knows Australian words (`billabong`, `brumby`, `larrikin`) and + /// spells some words the Australian way (`program`, `inquiry`). + EnAu, /// Canadian English (`en-CA`): British `-our`, `-re` and `-ence` (`colour`, `centre`, /// `defence`) with American `-ize` (`organize`), `tire` and `program`. EnCa, @@ -86,9 +90,10 @@ pub enum ProofLang { impl ProofLang { /// Every language with a spelling dictionary, in display order. - pub const PROOFED: [ProofLang; 15] = [ + pub const PROOFED: [ProofLang; 16] = [ ProofLang::En, ProofLang::EnGb, + ProofLang::EnAu, ProofLang::EnCa, ProofLang::Pl, ProofLang::Pt, @@ -155,12 +160,13 @@ impl ProofLang { } } - /// The English of an English tag, by its [`region`]: Canada is [`ProofLang::EnCa`], regions - /// that spell the British way are [`ProofLang::EnGb`], and the United States, any other + /// The English of an English tag, by its [`region`]: Canada is [`ProofLang::EnCa`], Australia + /// [`ProofLang::EnAu`], other regions that spell the British way are [`ProofLang::EnGb`], and the United States, any other /// region and no region are [`ProofLang::En`]. fn english(tag: &str) -> ProofLang { match region(tag) { Some(r) if r.eq_ignore_ascii_case("CA") => ProofLang::EnCa, + Some(r) if r.eq_ignore_ascii_case("AU") => ProofLang::EnAu, Some(r) if BRITISH_SPELLING.iter().any(|b| b.eq_ignore_ascii_case(r)) => ProofLang::EnGb, _ => ProofLang::En, } @@ -189,9 +195,10 @@ impl ProofLang { } } - /// Is this one of the Englishes ([`ProofLang::En`], [`ProofLang::EnGb`], [`ProofLang::EnCa`])? + /// Is this one of the Englishes ([`ProofLang::En`], [`ProofLang::EnGb`], [`ProofLang::EnAu`], + /// [`ProofLang::EnCa`])? pub fn is_english(self) -> bool { - matches!(self, ProofLang::En | ProofLang::EnGb | ProofLang::EnCa) + matches!(self, ProofLang::En | ProofLang::EnGb | ProofLang::EnAu | ProofLang::EnCa) } /// Is this one of the Spanishes ([`ProofLang::Es`], [`ProofLang::EsMx`], [`ProofLang::Es419`], @@ -208,7 +215,7 @@ impl ProofLang { pub fn code(self) -> &'static str { match self { ProofLang::Pack(_) => "", - ProofLang::En | ProofLang::EnGb | ProofLang::EnCa => "en", + ProofLang::En | ProofLang::EnGb | ProofLang::EnAu | ProofLang::EnCa => "en", ProofLang::Pl => "pl", ProofLang::Bg => "bg", ProofLang::Pt => "pt", @@ -371,12 +378,12 @@ pub(crate) fn region(tag: &str) -> Option<&str> { const LATIN_AMERICA: [&str; 22] = ["419", "005", "013", "029", "AR", "BO", "CL", "CO", "CR", "CU", "DO", "EC", "GT", "HN", "NI", "PA", "PE", "PR", "PY", "SV", "US", "VE"]; -/// Regions whose English uses British spelling ([`ProofLang::EnGb`]): the United Kingdom (and -/// the non-standard `UK`), Ireland, Australia, New Zealand, South Africa, India and other +/// Regions whose English uses British spelling ([`ProofLang::EnGb`]; Australia has its own, +/// [`ProofLang::EnAu`]): the United Kingdom (and the non-standard `UK`), Ireland, New Zealand, South Africa, India and other /// Commonwealth countries, the Crown dependencies and Europe (`150`). -const BRITISH_SPELLING: [&str; 26] = [ - "GB", "UK", "IE", "AU", "NZ", "ZA", "IN", "SG", "HK", "MY", "MT", "CY", "GI", "IM", "JE", "GG", "JM", "TT", "BS", "BB", "BZ", "KE", "NG", "GH", - "PK", "150", +const BRITISH_SPELLING: [&str; 25] = [ + "GB", "UK", "IE", "NZ", "ZA", "IN", "SG", "HK", "MY", "MT", "CY", "GI", "IM", "JE", "GG", "JM", "TT", "BS", "BB", "BZ", "KE", "NG", "GH", "PK", + "150", ]; /// A BCP 47 tag in canonical case (`pl-PL`, `en-GB`, `zh-Hant-TW`, `x-none`), or `None` when @@ -469,26 +476,12 @@ mod tests { fn tags_resolve_english_regions() { assert_tags(&["en", "en-US", "EN_us", "en-Latn-US", "en-PH", "en-001", "eng", "en_US.UTF-8", "en-x-gb"], ProofLang::En); assert_tags( - &[ - "en-GB", - "EN_gb", - "en_GB.UTF-8", - "en-AU", - "en_AU", - "en-NZ", - "en-IE", - "en-ZA", - "en-IN", - "en-Latn-GB", - "en-UK", - "en-150", - "eng-GB", - "en_GB@euro", - ], + &["en-GB", "EN_gb", "en_GB.UTF-8", "en-NZ", "en-IE", "en-ZA", "en-IN", "en-Latn-GB", "en-UK", "en-150", "eng-GB", "en_GB@euro"], ProofLang::EnGb, ); assert_tags(&["en-CA", "en_CA", "EN-ca", "en_CA.UTF-8", "en-Latn-CA"], ProofLang::EnCa); - for language in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa] { + assert_tags(&["en-AU", "en_AU", "EN-au", "en_AU.UTF-8", "en-Latn-AU"], ProofLang::EnAu); + for language in [ProofLang::En, ProofLang::EnGb, ProofLang::EnAu, ProofLang::EnCa] { assert!(language.is_english() && language.is_proofed() && ProofLang::PROOFED.contains(&language), "{language:?}"); assert_eq!(language.code(), "en"); } diff --git a/crates/proof/src/lib.rs b/crates/proof/src/lib.rs index 5b58768c..3df6cd01 100644 --- a/crates/proof/src/lib.rs +++ b/crates/proof/src/lib.rs @@ -152,6 +152,28 @@ fn dictionary_key(w: &str) -> String { composed(&w.to_lowercase()).into_owned() } +/// Each run of two or more spaces in `text`, once, over the whole run (a fix replaces it with one +/// space). A run of three is one issue over three, not a pair and a leftover. +pub(crate) fn space_runs(text: &str) -> Vec> { + let bytes = text.as_bytes(); + let mut runs = Vec::new(); + let mut i = 0; + while i < bytes.len() { + if bytes.get(i) != Some(&b' ') { + i += 1; + continue; + } + let start = i; + while bytes.get(i) == Some(&b' ') { + i += 1; + } + if i - start >= 2 { + runs.push(start..i); + } + } + runs +} + /// Add a word to the user dictionary ("Add to Dictionary"). It applies to every language. pub fn add_word(w: &str) { if user_words().write().unwrap_or_else(std::sync::PoisonError::into_inner).insert(dictionary_key(w)) { @@ -340,7 +362,7 @@ pub fn installed_dictionaries() -> Vec<(ProofLang, Option)> /// checked yet reports `None` (#2830). For an English, its embedded SCOWL dictionary or the /// English Hunspell dictionary that failed; English then uses the next source ([`english_source`]). pub fn dictionary_load_error(lang: ProofLang) -> Option { - if matches!(lang, ProofLang::En | ProofLang::EnGb | ProofLang::EnCa) { + if lang.is_english() { return english::load_error(lang); } if let Some(l) = lang.spelling_locale() { @@ -669,7 +691,7 @@ pub(crate) fn accepted_by_options(word: &str, opts: &Options) -> bool { || !w.chars().any(char::is_alphabetic) || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) // Only a word with a cased letter is in capitals: a Hebrew or Thai word has none. - || (opts.ignore_uppercase && w.chars().any(char::is_uppercase) && w.chars().all(|c| !c.is_lowercase())) + || (opts.ignore_uppercase && spelling::in_capitals(w)) // The whole word: the languages split a hyphenated word into its parts, and an added // `Unigranrio-RJ` is none of them (#2082). || in_user_dictionary(&w.to_lowercase()) @@ -953,7 +975,7 @@ pub fn check_grammar_in(text: &str, lang: ProofLang) -> Vec { fn grammar_in(text: &str, lang: ProofLang, capital_first: bool) -> Vec { match lang { ProofLang::En => grammar_en(text, capital_first), - ProofLang::EnGb | ProofLang::EnCa => grammar_en(text, capital_first).into_iter().map(|i| Issue { lang, ..i }).collect(), + ProofLang::EnGb | ProofLang::EnAu | ProofLang::EnCa => grammar_en(text, capital_first).into_iter().map(|i| Issue { lang, ..i }).collect(), // `pt-pt` shares Portuguese's rules, its issues in its own language. _ => language(lang.built_in()) .map_or_else(Vec::new, |l| (l.grammar)(text, lang, capital_first).into_iter().map(|i| Issue { lang, ..i }).collect()), @@ -1098,13 +1120,10 @@ fn grammar_en(text: &str, capital_first: bool) -> Vec { lang: ProofLang::En, }); } - for (i, _) in text.match_indices(" ") { - if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) { - continue; - } + for run in space_runs(text) { v.push(Issue { - start: i, - end: i + 2, + start: run.start, + end: run.end, kind: IssueKind::Grammar, message: "Extra space".into(), suggestions: vec![" ".into()], @@ -1197,7 +1216,7 @@ mod tests { } // Every proofed language other than English has a Hunspell dictionary to report. for l in ProofLang::PROOFED { - let english = matches!(l, ProofLang::En | ProofLang::EnGb | ProofLang::EnCa); + let english = l.is_english(); assert_eq!(dictionary_needed(l, false).is_some(), !english, "{l:?}"); } for l in [ProofLang::En, ProofLang::Other] { @@ -1321,7 +1340,8 @@ mod tests { } // Text in a language without proofing is never checked with English. assert!(check_spelling_in("hous teh zxqv", ProofLang::Other).is_empty()); - assert!(check_text("hous teh", &[(0..8, ProofLang::De)]).is_empty()); + // `Other` has no dictionary on any machine (German may: a system `de_DE` Hunspell). + assert!(check_text("hous teh", &[(0..8, ProofLang::Other)]).is_empty()); assert_eq!(english_source(ProofLang::Pl), None); drop_english_files(&dir, english); assert_ne!(english_source(ProofLang::En), Some(EnglishSource::Hunspell(dir.join("en_US.dic")))); @@ -1709,6 +1729,31 @@ mod tests { let _ = std::fs::remove_dir_all(&root); } + /// Review follow-up (craft-libs#485): a word in another script than a pack language's + /// dictionary (a Russian name in a Dutch run, a Dutch word in a Greek one) isn't marked; a + /// misspelling in the dictionary's own script still is. + #[test] + fn pack_languages_leave_other_scripts_alone() { + crate::hold_dictionary_tests(); + set_dictionaries_folder(Some(shared_test_folder())); + let root = std::env::temp_dir().join(format!("wc-proof-other-script-packs-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&root); + write_pack(&root, "nl", &[("spelling.aff", b"SET UTF-8\n"), ("spelling.dic", "2\nhuis\nkat\n".as_bytes())]); + write_pack(&root, "el", &[("spelling.aff", b"SET UTF-8\n"), ("spelling.dic", "1\nσπίτι\n".as_bytes())]); + packs::set_packs_dir(Some(root.clone())); + let (nl, el) = (ProofLang::from_tag(Some("nl-NL")), ProofLang::from_tag(Some("el-GR"))); + assert!(is_correct_in("huis", nl) && !is_correct_in("huiss", nl)); + assert!(!is_correct_in("zzzz", nl), "a misspelling in the dictionary's script, whatever its letters"); + assert!(is_correct_in("Пушкин", nl) && is_correct_in("σπίτι", nl)); + assert!(is_correct_in("σπίτι", el) && !is_correct_in("σπίτιι", el) && is_correct_in("huis", el)); + let text = "Пушкин huiss"; + let issues = check_text(text, &[(0..text.len(), nl)]); + assert_eq!(issues.iter().map(|i| &text[i.start..i.end]).collect::>(), ["huiss"], "{issues:?}"); + packs::set_packs_dir(None); + set_dictionaries_folder(Some(shared_test_folder())); + let _ = std::fs::remove_dir_all(&root); + } + /// The `.dic` file `lang`'s spelling is read from with the installed packs and `system` as the /// system's only dictionary folder. fn chosen_dictionary(lang: ProofLang, system: &std::path::Path) -> Option { @@ -2465,6 +2510,49 @@ mod tests { assert_eq!(suggest_in("programmes", 5, us).first().map(String::as_str), Some("programs")); } + /// Review follow-up: British English takes `-ize` (Oxford spelling) but never `-yze`, also in + /// forms en-variants.txt doesn't list. Hunspell 1.7.5 with SCOWL's en_GB rejects these too. + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] + #[test] + fn british_english_rejects_unlisted_yze_forms() { + crate::hold_dictionary_tests(); + let gb = ProofLang::EnGb; + for w in ["reanalyze", "reanalyzed", "breathalyze", "breathalyzer", "paralyzingly", "catalyze", "analyzer"] { + assert!(!is_correct_in(w, gb), "en-GB: {w}"); + assert!(is_correct_in(w, ProofLang::En), "en-US: {w}"); + } + for w in ["reanalyse", "breathalyser", "paralysingly", "catalyse", "organize", "realization"] { + assert!(is_correct_in(w, gb), "en-GB: {w}"); + } + } + + /// Review follow-up: Australian English reads SCOWL's own en-AU dictionary (craft-i18n ships + /// it), not en-GB's: its Australian words are right and spelled as Hunspell's en_AU spells + /// them; British English doesn't take them. The British policy still applies: American + /// spellings are wrong, `-ise` and `-ize` both right. + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] + #[test] + fn australian_english_reads_its_own_scowl() { + crate::hold_dictionary_tests(); + let (au, gb) = (ProofLang::EnAu, ProofLang::EnGb); + assert!(scowl::is_british(au)); + assert_eq!(english_source(au), Some(EnglishSource::Embedded)); + for w in ["billabong", "brumby", "larrikinism", "bunyip", "corroboree", "Bundaberg", "Rockhampton"] { + assert!(is_correct_in(w, au), "en-AU: {w}"); + assert!(!is_correct_in(w, gb), "en-GB: {w}"); + assert!(is_correct_for(w, Some("en-AU")), "en-AU tag: {w}"); + } + for w in ["colour", "centre", "organise", "organize", "analyse", "travelled", "defence"] { + assert!(is_correct_in(w, au), "en-AU: {w}"); + } + for w in ["color", "center", "analyze", "reanalyze", "defense"] { + assert!(!is_correct_in(w, au), "en-AU: {w}"); + } + let text = "The larrikin saw a brumby by the billabong and analysed the colour of the sky."; + assert!(check_spelling_in(text, au).is_empty(), "{:?}", check_spelling_in(text, au)); + assert_eq!(check_spelling_in(text, gb).len(), 3, "larrikin, brumby, billabong: {:?}", check_spelling_in(text, gb)); + } + /// More British spellings that must stay right in en-GB and en-AU (#1281): -re, single l, ae, /// with their inflections. #[test] @@ -2472,7 +2560,7 @@ mod tests { // English results depend on the dictionaries other tests install. crate::hold_dictionary_tests(); let gb = ProofLang::EnGb; - assert_eq!(ProofLang::from_tag(Some("en-AU")), gb); + assert_eq!(ProofLang::from_tag(Some("en-AU")), ProofLang::EnAu); for w in [ "goitre", "goitres", @@ -2497,11 +2585,66 @@ mod tests { "encyclopaedias", ] { assert!(is_correct_in(w, gb), "en-GB: {w}"); + assert!(is_correct_in(w, ProofLang::EnAu), "en-AU: {w}"); } let text = "The wilful knight took the sceptre to fulfil his enrolment, paid the instalment and read the encyclopaedia."; assert!(check_spelling_in(text, gb).is_empty(), "{:?}", check_spelling_in(text, gb)); } + /// Review follow-up: SCOWL's en-AU list keeps one spelling per word, so Australian English also + /// takes a word the en-GB list has (`enquiry`, `guerrilla`, `liquorice`); American-only + /// spellings, wrong in both, stay wrong. Suggestions still come from the en-AU list. + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] + #[test] + fn australian_english_also_takes_british_words() { + crate::hold_dictionary_tests(); + let (au, gb) = (ProofLang::EnAu, ProofLang::EnGb); + for w in ["enquiry", "enquire", "guerrilla", "guerrillas", "euthanise", "liveable", "liquorice", "veranda", "burka", "Anthea"] { + assert!(is_correct_in(w, gb), "en-GB: {w}"); + assert!(is_correct_in(w, au), "en-AU: {w}"); + } + for w in ["analyze", "catalyze", "color", "center", "defense", "zzyzxq"] { + assert!(!is_correct_in(w, gb), "en-GB: {w}"); + assert!(!is_correct_in(w, au), "en-AU: {w}"); + } + assert!(is_correct_in("billabong", au) && !is_correct_in("billabong", gb), "en-AU still has its own"); + assert!(suggest_in("billabon", 5, au).iter().any(|s| s == "billabong"), "{:?}", suggest_in("billabon", 5, au)); + } + + /// Review follow-up: a run of spaces is one issue over the whole run, in every language with a + /// double-space rule (a run of three used to leave two after the fix, four three). + #[test] + fn a_run_of_spaces_is_one_issue_in_every_language() { + assert_eq!(space_runs("a b c d "), [3..5, 6..9, 10..14]); + assert!(space_runs("a b c").is_empty() && space_runs("").is_empty() && space_runs(" ").is_empty()); + let langs = [ + ProofLang::En, + ProofLang::EnGb, + ProofLang::Es, + ProofLang::It, + ProofLang::Pt, + ProofLang::Cs, + ProofLang::Hr, + ProofLang::Id, + ProofLang::Pl, + ProofLang::Ru, + ProofLang::Tr, + ProofLang::EnAu, + ProofLang::EnCa, + ] + .into_iter() + .chain(PackLocale::new("pt-PT").map(ProofLang::Pack)); + for lang in langs { + for n in 2..=5 { + let text = format!("Aaa{}bbb", " ".repeat(n)); + let hits: Vec<_> = + check_grammar_in(&text, lang).into_iter().filter(|i| i.start <= 3 && i.end > 3 && i.suggestions == [" "]).collect(); + assert_eq!(hits.len(), 1, "{lang:?} {n}: {hits:?}"); + assert_eq!((hits[0].start, hits[0].end), (3, 3 + n), "{lang:?} {n}"); + } + } + } + /// English spelling reads SCOWL 2026 through the affix engine: one dictionary per English, /// its notice embedded, and its ordinals (`COMPOUNDRULE`) understood. #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] @@ -2513,7 +2656,8 @@ mod tests { assert_eq!(english_source(l), Some(EnglishSource::Embedded)); assert_eq!(dictionary_load_error(l), None); } - assert!(std::ptr::eq(scowl::dict(ProofLang::EnGb).unwrap(), scowl::dict(ProofLang::from_tag(Some("en-AU"))).unwrap()), "en-AU reads en-GB"); + assert!(!std::ptr::eq(scowl::dict(ProofLang::EnGb).unwrap(), scowl::dict(ProofLang::EnAu).unwrap()), "en-AU reads its own"); + assert!(std::ptr::eq(scowl::dict(ProofLang::EnGb).unwrap(), scowl::dict(ProofLang::from_tag(Some("en-NZ"))).unwrap()), "en-NZ reads en-GB"); let notice = scowl::notice().unwrap(); for credit in ["Kevin Atkinson", "Geoff Kuenning", "clause 4 removed", "Benjamin Titze"] { assert!(notice.contains(credit), "{credit}"); diff --git a/crates/proof/src/no_data.rs b/crates/proof/src/no_data.rs index 2f429985..cdd9025d 100644 --- a/crates/proof/src/no_data.rs +++ b/crates/proof/src/no_data.rs @@ -7,9 +7,10 @@ use crate::hyphen::{self, Limits, Source}; use crate::{EnglishSource, Options, ProofLang, dict, en_variants, patterns}; -const LANGS: [ProofLang; 18] = [ +const LANGS: [ProofLang; 19] = [ ProofLang::En, ProofLang::EnGb, + ProofLang::EnAu, ProofLang::EnCa, ProofLang::Pl, ProofLang::Bg, @@ -45,9 +46,14 @@ const SAMPLES: [&str; 10] = [ #[test] fn nothing_is_embedded() { assert!(crate::data::EN_US_DIC.is_empty()); - assert!(crate::data::EN_US_SPELLING.is_empty() && crate::data::EN_GB_SPELLING.is_empty() && crate::data::EN_CA_SPELLING.is_empty()); + assert!( + crate::data::EN_US_SPELLING.is_empty() + && crate::data::EN_GB_SPELLING.is_empty() + && crate::data::EN_CA_SPELLING.is_empty() + && crate::data::EN_AU_SPELLING.is_empty() + ); assert!(crate::scowl::notice().is_none()); - for l in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa] { + for l in [ProofLang::En, ProofLang::EnGb, ProofLang::EnAu, ProofLang::EnCa] { assert!(crate::scowl::dict(l).is_none() && crate::scowl::load_error(l).is_none()); } assert!(dict::Dictionary::en_us().is_empty()); @@ -63,7 +69,7 @@ fn nothing_is_embedded() { #[test] fn english_spelling_still_marks_misspellings() { crate::hold_dictionary_tests(); - for l in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa] { + for l in [ProofLang::En, ProofLang::EnGb, ProofLang::EnAu, ProofLang::EnCa] { let source = crate::english_source(l); assert!(matches!(source, Some(EnglishSource::Hunspell(_) | EnglishSource::CoreList)), "{l:?}"); // Any English dictionary marks junk and knows `the`; the word-by-word results below are diff --git a/crates/proof/src/pack_lang.rs b/crates/proof/src/pack_lang.rs index 6c116653..7b4d1931 100644 --- a/crates/proof/src/pack_lang.rs +++ b/crates/proof/src/pack_lang.rs @@ -6,6 +6,10 @@ //! files); the one used least recently goes first, but never one used during the current //! [`in_one_check`] call, so a paragraph in many languages loads each dictionary once. Locales //! with no dictionary are remembered apart and take no room. +//! +//! A word none of whose letters is in a script the dictionary's words are written in (a Cyrillic +//! word in a Dutch run) isn't this language's: it is never marked, as Word leaves a word in another +//! script alone. A word in the dictionary's script is always checked, whatever its letters. use std::cell::Cell; use std::path::PathBuf; @@ -20,7 +24,7 @@ use crate::lang::PackLocale; pub(crate) const MAX_LOADED_BYTES: u64 = 256 << 20; /// Estimated memory per byte of `.aff` and `.dic` file. const BYTES_PER_FILE_BYTE: u64 = 6; -/// Most locales remembered as having no dictionary (the list is cleared when full). +/// Most locales remembered as having no dictionary (the oldest is forgotten first when full). const MAX_MISSING: usize = 256; /// The budget in force (tests lower it). @@ -36,6 +40,8 @@ thread_local! { struct Loaded { locale: PackLocale, dict: Arc, + /// The scripts its words are written in ([`scripts`]), sorted. + scripts: Arc<[u32]>, /// The `.dic` it came from. file: Option, /// Estimated memory. @@ -66,15 +72,88 @@ pub(crate) fn reload() { /// over the budget. pub(crate) fn in_one_check(f: impl FnOnce() -> R) -> R { let outer = CHECK_FROM.with(Cell::get); - if outer.is_none() { + let _end = outer.is_none().then(|| { let now = CACHE.read().unwrap_or_else(PoisonError::into_inner).clock; CHECK_FROM.with(|c| c.set(Some(now.wrapping_add(1)))); - } - let r = f(); - if outer.is_none() { + EndCheck + }); + f() +} + +/// Ends the outermost [`in_one_check`] when dropped: also when `f` panics (`Session::run` catches +/// it), or every dictionary would stay pinned on this thread and the budget never hold again. +struct EndCheck; + +impl Drop for EndCheck { + fn drop(&mut self) { CHECK_FROM.with(|c| c.set(None)); } - r +} + +/// Remember that `locale` has no dictionary; when [`MAX_MISSING`] are remembered, the oldest is +/// forgotten (clearing them all would look every other one up again). +fn remember_missing(missing: &mut Vec, locale: PackLocale) { + if missing.len() >= MAX_MISSING { + missing.remove(0); + } + missing.push(locale); +} + +/// The script of the letter `c`, as a number: the main alphabetic scripts by their Unicode blocks +/// (Latin with its extensions, Greek, Cyrillic, Armenian, Hebrew, Arabic, Georgian, Hangul, kana, +/// Han, Ethiopic); any other letter by its 128-character block. Only told apart, never named. +fn script_of(c: char) -> u32 { + const SCRIPTS: &[(u32, u32, u32)] = &[ + (0x0041, 0x024F, 1), + (0x0250, 0x02AF, 1), + (0x1E00, 0x1EFF, 1), + (0x2C60, 0x2C7F, 1), + (0xA720, 0xA7FF, 1), + (0xAB30, 0xAB6F, 1), + (0xFF21, 0xFF5A, 1), + (0x0370, 0x03FF, 2), + (0x1F00, 0x1FFF, 2), + (0x0400, 0x052F, 3), + (0x1C80, 0x1C8F, 3), + (0x2DE0, 0x2DFF, 3), + (0xA640, 0xA69F, 3), + (0x0530, 0x058F, 4), + (0xFB13, 0xFB17, 4), + (0x0590, 0x05FF, 5), + (0xFB1D, 0xFB4F, 5), + (0x0600, 0x06FF, 6), + (0x0750, 0x077F, 6), + (0x08A0, 0x08FF, 6), + (0xFB50, 0xFDFF, 6), + (0xFE70, 0xFEFF, 6), + (0x10A0, 0x10FF, 7), + (0x1C90, 0x1CBF, 7), + (0x2D00, 0x2D2F, 7), + (0x1100, 0x11FF, 8), + (0x3130, 0x318F, 8), + (0xAC00, 0xD7AF, 8), + (0x3040, 0x30FF, 9), + (0x3400, 0x4DBF, 10), + (0x4E00, 0x9FFF, 10), + (0x1200, 0x139F, 11), + ]; + let u = u32::from(c); + SCRIPTS.iter().find(|(lo, hi, _)| (*lo..=*hi).contains(&u)).map_or(0x100 + (u >> 7), |(_, _, s)| *s) +} + +/// The scripts the words of `d` are written in ([`script_of`] of their letters), sorted. +fn scripts(d: &AffixDict) -> Arc<[u32]> { + let mut seen = std::collections::BTreeSet::new(); + for c in d.stems().flat_map(str::chars).filter(|c| c.is_alphabetic()) { + seen.insert(script_of(c)); + } + seen.into_iter().collect() +} + +/// Is `word` in another script than the dictionary's (`scripts`): none of its letters is in one +/// of them? A word in the dictionary's script is never, whatever its letters (`zzzz` in Dutch). +fn other_script(word: &str, scripts: &[u32]) -> bool { + !scripts.is_empty() && !word.chars().filter(|c| c.is_alphabetic()).any(|c| scripts.binary_search(&script_of(c)).is_ok()) } #[cfg(test)] @@ -149,30 +228,34 @@ pub(crate) fn names(locale: PackLocale) -> Vec { /// `locale`'s dictionary, loaded on first use; an empty one when there is none. fn get(locale: PackLocale) -> (Arc, Option) { + let (d, file, _) = get_with_scripts(locale); + (d, file) +} + +/// [`get`], with the dictionary's [`scripts`]. +fn get_with_scripts(locale: PackLocale) -> (Arc, Option, Arc<[u32]>) { let reloads = { let mut c = CACHE.write().unwrap_or_else(PoisonError::into_inner); c.clock = c.clock.wrapping_add(1); let now = c.clock; if let Some(l) = c.loaded.iter_mut().find(|l| l.locale == locale) { l.used = now; - return (l.dict.clone(), l.file.clone()); + return (l.dict.clone(), l.file.clone(), l.scripts.clone()); } if c.missing.contains(&locale) { - return (Arc::default(), None); + return (Arc::default(), None, Arc::from([])); } c.reloads }; let (dict, file, bytes) = load(locale); + let scripts = scripts(&dict); let mut c = CACHE.write().unwrap_or_else(PoisonError::into_inner); if c.reloads != reloads || c.loaded.iter().any(|l| l.locale == locale) || c.missing.contains(&locale) { - return (dict, file); + return (dict, file, scripts); } if dict.is_empty() { - if c.missing.len() >= MAX_MISSING { - c.missing.clear(); - } - c.missing.push(locale); - return (dict, file); + remember_missing(&mut c.missing, locale); + return (dict, file, scripts); } LOADS.fetch_add(1, Ordering::Relaxed); let budget = BUDGET.load(Ordering::Relaxed); @@ -187,8 +270,8 @@ fn get(locale: PackLocale) -> (Arc, Option) { total = total.saturating_sub(gone.bytes); } let used = c.clock; - c.loaded.push(Loaded { locale, dict: dict.clone(), file: file.clone(), bytes, used }); - (dict, file) + c.loaded.push(Loaded { locale, dict: dict.clone(), scripts: scripts.clone(), file: file.clone(), bytes, used }); + (dict, file, scripts) } /// Load `locale`'s dictionary, with its estimated memory. @@ -229,11 +312,14 @@ pub(crate) fn load_error(locale: PackLocale) -> Option { /// Is `word` right in `locale`? Every word is when there is no dictionary. pub(crate) fn is_correct(word: &str, locale: PackLocale, opts: &crate::Options) -> bool { - let (d, _) = get(locale); + let (d, _, scripts) = get_with_scripts(locale); if d.is_empty() { return true; } let Some(w) = crate::spelling::word_to_check_in(&d, word, opts) else { return true }; + if other_script(w, &scripts) { + return true; + } let known = |w: &str| d.check(w) || crate::in_user_dictionary(&w.to_lowercase()); if w.contains('-') && !known(w) { return w.split('-').all(|p| p.chars().count() <= 1 || known(p)); @@ -264,6 +350,57 @@ mod tests { PackLocale::new(t).unwrap() } + /// Review follow-up (craft-libs#485): a full list of locales with no dictionary forgets its + /// oldest one, not all of them, so a document in more languages doesn't look every one up again. + #[test] + fn missing_locales_forget_the_oldest_first() { + let tag = |i: usize| l(&format!("q{}{}", (b'a' + (i / 26 % 26) as u8) as char, (b'a' + (i % 26) as u8) as char)); + let mut missing = Vec::new(); + for i in 0..MAX_MISSING + 10 { + remember_missing(&mut missing, tag(i)); + assert!(missing.len() <= MAX_MISSING); + } + assert_eq!(missing.len(), MAX_MISSING); + assert!(!missing.contains(&tag(0)) && !missing.contains(&tag(9)), "the oldest went first"); + assert!(missing.contains(&tag(10)) && missing.contains(&tag(MAX_MISSING + 9)), "the rest stay"); + } + + /// Review follow-up (craft-libs#485): a panic inside a check (caught by the app) doesn't leave + /// this thread's dictionaries pinned past the budget for good. + #[test] + fn a_panic_inside_a_check_ends_it() { + let caught = std::panic::catch_unwind(|| in_one_check(|| panic!("a bug inside a check"))); + assert!(caught.is_err()); + assert_eq!(CHECK_FROM.with(Cell::get), None); + in_one_check(|| { + assert!(CHECK_FROM.with(Cell::get).is_some()); + in_one_check(|| ()); + assert!(CHECK_FROM.with(Cell::get).is_some(), "an inner check doesn't end the outer one"); + }); + assert_eq!(CHECK_FROM.with(Cell::get), None); + } + + /// Review follow-up (craft-libs#485): a word with no letter in the dictionary's scripts is + /// in another script (a Cyrillic word in a Dutch run); a word in its script is always checked, + /// even with letters no dictionary word uses (CodeRabbit on #496: `zzzz`). + #[test] + fn words_in_another_script_are_not_the_dictionarys() { + let nl = AffixDict::from_hunspell("SET UTF-8\n", "3\nhuis\nkat\nÉén\n", &[]).unwrap(); + let latin = scripts(&nl); + assert_eq!(latin.len(), 1); + for w in ["привет", "Привет", "αβγ", "שלום", "日本"] { + assert!(other_script(w, &latin), "{w}"); + } + for w in ["huus", "kät", "huisп", "ÉÉN", "zzzz", "qxqw", "ŋʒ"] { + assert!(!other_script(w, &latin), "{w}"); + } + assert!(!other_script("привет", &[]), "no scripts: every word is checked"); + let ru = AffixDict::from_hunspell("SET UTF-8\n", "1\nдом\n", &[]).unwrap(); + assert!(!other_script("щщщ", &scripts(&ru)) && other_script("dom", &scripts(&ru))); + assert_ne!(script_of('ა'), script_of('a')); + assert_eq!(script_of('ă'), script_of('Ạ')); + } + #[test] fn locales_and_file_names() { assert_eq!(l("nl-NL").as_str(), "nl-nl"); diff --git a/crates/proof/src/packs.rs b/crates/proof/src/packs.rs index 2fb291e9..2e558393 100644 --- a/crates/proof/src/packs.rs +++ b/crates/proof/src/packs.rs @@ -34,6 +34,13 @@ pub(crate) const SPELLING: &str = "spelling"; const HYPHENATION: &str = "hyphenation.pat"; /// Most languages whose pack patterns are remembered (a document may tag thousands). const MAX_HYPHENATION_LOOKUPS: usize = 256; +/// The memory budget of the remembered pack patterns, in estimated bytes ([`pattern_bytes`]): +/// a pack's patterns may be as large as a dictionary (64 MB of file), and up to +/// [`MAX_HYPHENATION_LOOKUPS`] of them are kept, so they get a budget of their own, as the +/// dictionaries do (`pack_lang::MAX_LOADED_BYTES`). +const MAX_HYPHENATION_BYTES: u64 = 256 << 20; +/// Estimated memory per pattern (its packed key, its values and the map's own room). +const BYTES_PER_PATTERN: u64 = 64; /// A pack's description. const PACK_TOML: &str = "pack.toml"; /// A pack's licence texts. @@ -80,15 +87,18 @@ struct State { dir: Option, scan: Option>, /// Each language's pack patterns once looked up (`None`: no pack has them), at most - /// [`MAX_HYPHENATION_LOOKUPS`] languages (cleared when full). + /// [`MAX_HYPHENATION_LOOKUPS`] languages and [`MAX_HYPHENATION_BYTES`] (cleared when full). hyphenation: BTreeMap>>, + /// The estimated memory of the patterns in [`State::hyphenation`]. + hyphenation_bytes: u64, /// (locale, reason) of pack hyphenation files that failed to load. hyphenation_errors: Vec<(String, String)>, /// Counts changes, so a scan or load that started before one is not kept. changes: u64, } -static STATE: RwLock = RwLock::new(State { dir: None, scan: None, hyphenation: BTreeMap::new(), hyphenation_errors: Vec::new(), changes: 0 }); +static STATE: RwLock = + RwLock::new(State { dir: None, scan: None, hyphenation: BTreeMap::new(), hyphenation_bytes: 0, hyphenation_errors: Vec::new(), changes: 0 }); fn read() -> RwLockReadGuard<'static, State> { STATE.read().unwrap_or_else(PoisonError::into_inner) @@ -121,6 +131,7 @@ pub fn reload() { fn forget(s: &mut State) { s.scan = None; s.hyphenation.clear(); + s.hyphenation_bytes = 0; s.hyphenation_errors.clear(); s.changes = s.changes.wrapping_add(1); } @@ -190,8 +201,9 @@ const BUILT_IN_MOBY_NOTICE: &str = "The English word list built into craft-proof const EXTRA_EN_NOTICE: &str = "The English word supplement (en-extra.txt), common words the Moby list lacks, is written by \ WordCraft contributors and licensed MIT OR Apache-2.0."; -/// The locales whose Language data is embedded in this build: `en-us`, and by default `de`, -/// `pl`, `pt`, `it`, `ru` (until Language packs ship, CP4.4); a build with `CRAFT_PROOF_EMBED` +/// The locales whose upstream Language data is embedded in this build: `en-us`, and by default `de`, +/// `pl`, `pt`, `it` (until Language packs ship, CP4.4; our own `ru-extra.txt` is embedded but not +/// listed); a build with `CRAFT_PROOF_EMBED` /// embeds English and the ones it names (`en`: English only). pub fn embedded_locales() -> &'static [&'static str] { crate::data::EMBEDDED_LOCALES @@ -531,7 +543,7 @@ fn hyphenation_locales(lang: ProofLang) -> Vec { ProofLang::EsUy => &["es-uy", "es-419", "es"], ProofLang::De => &["de", "de-1996", "de-de"], ProofLang::Pack(l) => return crate::pack_lang::hyphenation_locales(l), - ProofLang::En | ProofLang::EnGb | ProofLang::EnCa | ProofLang::Other => &[], + ProofLang::En | ProofLang::EnGb | ProofLang::EnAu | ProofLang::EnCa | ProofLang::Other => &[], }; names.iter().map(|n| (*n).to_string()).collect() } @@ -566,10 +578,8 @@ pub(crate) fn hyphenation(lang: ProofLang) -> Option> { let mut s = write(); if s.changes == changes && !s.hyphenation.contains_key(&lang) { // ponytail: cleared when full, an LRU if many languages each need their patterns. - if s.hyphenation.len() >= MAX_HYPHENATION_LOOKUPS { - s.hyphenation.clear(); - } - s.hyphenation.insert(lang, found.clone()); + let bytes = found.as_deref().map_or(0, pattern_bytes); + remember_patterns(&mut s, lang, found.clone(), bytes, MAX_HYPHENATION_BYTES); for e in errors { if !s.hyphenation_errors.contains(&e) { s.hyphenation_errors.push(e); @@ -579,6 +589,23 @@ pub(crate) fn hyphenation(lang: ProofLang) -> Option> { found } +/// Estimated memory of `p`. +fn pattern_bytes(p: &Patterns) -> u64 { + (p.len() as u64).saturating_add(p.exception_count() as u64).saturating_mul(BYTES_PER_PATTERN) +} + +/// Remember `lang`'s pack patterns (`bytes` of them): over [`MAX_HYPHENATION_LOOKUPS`] languages +/// or `budget` bytes, the ones remembered so far are forgotten first. Patterns larger than the +/// whole budget are still kept alone, or they would be read from disk for every word. +fn remember_patterns(s: &mut State, lang: ProofLang, found: Option>, bytes: u64, budget: u64) { + if s.hyphenation.len() >= MAX_HYPHENATION_LOOKUPS || s.hyphenation_bytes.saturating_add(bytes) > budget { + s.hyphenation.clear(); + s.hyphenation_bytes = 0; + } + s.hyphenation.insert(lang, found); + s.hyphenation_bytes = s.hyphenation_bytes.saturating_add(bytes); +} + /// How many languages' pack patterns are remembered now. #[cfg(test)] pub(crate) fn hyphenation_lookups() -> usize { @@ -600,6 +627,29 @@ fn load_patterns(path: &Path) -> Result { mod tests { use super::*; + /// Review follow-up (craft-libs#485): the remembered pack patterns have a memory budget, not + /// only a count, so 256 languages of large patterns can't each stay in memory. + #[test] + fn remembered_patterns_keep_to_a_memory_budget() { + let mut s = State { dir: None, scan: None, hyphenation: BTreeMap::new(), hyphenation_bytes: 0, hyphenation_errors: Vec::new(), changes: 0 }; + let p = Arc::new(Patterns::parse("hy3ph he2n hena4 hen5at 1na n2at 1tio 2io o2n")); + let bytes = pattern_bytes(&p); + assert_eq!(bytes, 9 * BYTES_PER_PATTERN); + let lang = |t: &str| ProofLang::from_tag(Some(t)); + let budget = bytes * 2; + remember_patterns(&mut s, lang("nl"), Some(p.clone()), bytes, budget); + remember_patterns(&mut s, lang("fr"), Some(p.clone()), bytes, budget); + remember_patterns(&mut s, lang("sv"), None, 0, budget); + assert_eq!((s.hyphenation.len(), s.hyphenation_bytes), (3, budget), "a language with no patterns takes no room"); + remember_patterns(&mut s, lang("da"), Some(p.clone()), bytes, budget); + assert_eq!((s.hyphenation.len(), s.hyphenation_bytes), (1, bytes), "over the budget: the others are forgotten"); + assert!(s.hyphenation.contains_key(&lang("da"))); + remember_patterns(&mut s, lang("fi"), Some(p.clone()), budget * 4, budget); + assert_eq!((s.hyphenation.len(), s.hyphenation_bytes), (1, budget * 4), "patterns over the whole budget are kept alone"); + forget(&mut s); + assert_eq!((s.hyphenation.len(), s.hyphenation_bytes), (0, 0)); + } + /// Review of #485: a pack's name loses bidirectional and other format characters. #[test] fn pack_names_are_plain_text() { diff --git a/crates/proof/src/pl/grammar.rs b/crates/proof/src/pl/grammar.rs index d514429e..2fd93f86 100644 --- a/crates/proof/src/pl/grammar.rs +++ b/crates/proof/src/pl/grammar.rs @@ -304,11 +304,8 @@ fn spaces(text: &str, v: &mut Vec) { v.push(issue(i, i + 1 + p.len_utf8(), "Zbędna spacja przed znakiem interpunkcyjnym".into(), vec![p.to_string()])); } } - for (i, _) in text.match_indices(" ") { - if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) { - continue; - } - v.push(issue(i, i + 2, "Podwójna spacja".into(), vec![" ".into()])); + for run in crate::space_runs(text) { + v.push(issue(run.start, run.end, "Podwójna spacja".into(), vec![" ".into()])); } } diff --git a/crates/proof/src/pt/grammar.rs b/crates/proof/src/pt/grammar.rs index 303929de..eb0f89a6 100644 --- a/crates/proof/src/pt/grammar.rs +++ b/crates/proof/src/pt/grammar.rs @@ -62,7 +62,12 @@ fn ends_sentence(gap: &str, prev: &str) -> bool { if end == '.' && abbreviation_or_number(prev) { return false; } - let rest = t.trim_start_matches(['.', '!', '?']).trim_start_matches(CLOSING); + let after_marks = t.trim_start_matches(['.', '!', '?']); + let rest = after_marks.trim_start_matches(CLOSING); + // `"Vai?" disse ele.` (not the object placeholder `\u{1}`): narration after a quoted question or exclamation goes on in lower case. + if end != '.' && after_marks.trim_start_matches('\u{1}').starts_with(CLOSING) { + return false; + } // A dialogue dash after the mark brings back the narration (`—Vai? —disse ele.`), which goes // on in lower case. rest.starts_with(char::is_whitespace) && !rest.contains('—') && rest.trim_start().chars().all(|c| OPENING.contains(&c)) @@ -124,11 +129,8 @@ fn spaces(text: &str, v: &mut Vec) { v.push(issue(i, i + 1 + p.len_utf8(), "Espaço antes da pontuação".into(), vec![p.to_string()])); } } - for (i, _) in text.match_indices(" ") { - if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) { - continue; - } - v.push(issue(i, i + 2, "Espaço duplo".into(), vec![" ".into()])); + for run in crate::space_runs(text) { + v.push(issue(run.start, run.end, "Espaço duplo".into(), vec![" ".into()])); } } @@ -225,6 +227,14 @@ mod tests { // craft-libs#488 review: no space after the dash. clean("—Vai? —disse ele."); clean("— Vai? — disse ele."); + // craft-libs#496 review: a closing quote after the mark. + clean("\"Vai?\" disse ele."); + clean("«Vai!» gritou ela."); + assert!(!messages("Vai? \"sim\".").is_empty(), "a bare question still ends the sentence"); + // An inline object (`\u{1}`) before a real closing quote is skipped. + clean("\"Vai?\u{1}\" disse ele."); + // An inline object (`\u{1}`) is not a closing quote. + assert!(!messages("Quem era?\u{1} era Marco.").is_empty()); clean("Ele disse: \"Volto amanhã\". E saiu."); // A list item may start in lower case. clean("maçãs,"); diff --git a/crates/proof/src/ru/grammar.rs b/crates/proof/src/ru/grammar.rs index ff8711fb..4d1ce3c2 100644 --- a/crates/proof/src/ru/grammar.rs +++ b/crates/proof/src/ru/grammar.rs @@ -503,11 +503,8 @@ fn spaces(text: &str, v: &mut Vec) { v.push(issue(i, i + 1 + p.len_utf8(), "Лишний пробел перед знаком препинания".into(), vec![p.to_string()])); } } - for (i, _) in text.match_indices(" ") { - if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) { - continue; - } - v.push(issue(i, i + 2, "Двойной пробел".into(), vec![" ".into()])); + for run in crate::space_runs(text) { + v.push(issue(run.start, run.end, "Двойной пробел".into(), vec![" ".into()])); } } diff --git a/crates/proof/src/ru/mod.rs b/crates/proof/src/ru/mod.rs index b88978af..4a7cd3e0 100644 --- a/crates/proof/src/ru/mod.rs +++ b/crates/proof/src/ru/mod.rs @@ -105,7 +105,7 @@ fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool { if n <= 1 || (opts.ignore_numbers && w.chars().any(char::is_numeric)) || !w.chars().any(char::is_alphabetic) { return true; } - if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= ABBREVIATION_LETTERS { + if opts.ignore_uppercase && crate::spelling::in_capitals(w) && n <= ABBREVIATION_LETTERS { return true; } // Camel case (`ВКонтакте`, `МирКрасок`): a brand or an identifier, not a dictionary word. @@ -321,7 +321,7 @@ fn hyphenated(d: &AffixDict, w: &str, opts: &crate::Options) -> bool { } match lp.as_slice() { // `кое-кто`, `кой-какой`. - [prefix, rest] if PRONOUN_PREFIXES.contains(prefix) => return parts.get(1).is_some_and(|r| known(d, r)), + [prefix, _] if PRONOUN_PREFIXES.contains(prefix) => return parts.get(1).is_some_and(|r| known(d, r)), // `по-русски` (from `русский`), `по-новому`, `по-моему`. ["по", rest] => { let Some(r) = parts.get(1) else { return false }; @@ -333,7 +333,7 @@ fn hyphenated(d: &AffixDict, w: &str, opts: &crate::Options) -> bool { let last = parts.len() - 1; parts.iter().enumerate().all(|(i, p)| { // An abbreviation part (`ИИ-консультант`, `SMS-рассылка`) or a known word. - if known(d, p) || (opts.ignore_uppercase && p.chars().all(|c| !c.is_lowercase()) && p.chars().count() <= ABBREVIATION_LETTERS) { + if known(d, p) || (opts.ignore_uppercase && crate::spelling::capitals_part(p, ABBREVIATION_LETTERS)) { return true; } // A first part with a linking vowel: the stem (`северо` → `север`) or its adjective @@ -485,5 +485,7 @@ mod tests { assert!(!crate::is_correct_in_with("м²", lang, &check_digits), "a superscript is a number too"); assert!(crate::is_correct_in_with("2026", lang, &check_digits), "numbers alone stay correct"); assert!(is_correct("ПРЕВЕТ"), "direct checks keep the defaults"); + // The cased-letter rule (craft-libs#485): capitals and letterless parts are abbreviations. + assert!(crate::is_correct_in_with("ИИ-2", lang, &check_digits) && crate::is_correct_in_with("ТУ-154М", lang, &check_digits)); } } diff --git a/crates/proof/src/scowl.rs b/crates/proof/src/scowl.rs index 9a71188f..cb101cd8 100644 --- a/crates/proof/src/scowl.rs +++ b/crates/proof/src/scowl.rs @@ -2,10 +2,15 @@ //! dictionaries of craft-i18n's `language-data/en-us`, `en-gb` (British `-ise`) and `en-ca`, //! embedded unmodified at build time and read by the affix engine ([`crate::affix`]) on first use. //! -//! Each English has its own: American English reads en-US, the British-spelling Englishes -//! (en-GB, en-AU, en-NZ, en-IE…) read en-GB and Canadian English reads en-CA. Its suggestions +//! Each English has its own: American English reads en-US, Australian English en-AU (SCOWL's +//! Australian words: `billabong`, `brumby`; and `program`, `inquiry`), the other British-spelling +//! Englishes (en-GB, en-NZ, en-IE…) read en-GB and Canadian English reads en-CA. SCOWL has no +//! en-NZ: New Zealand English keeps `programme` and has few words of its own in SCOWL's en-AU +//! data, so it stays on en-GB. Its suggestions //! come from its own dictionary, and a word is right when its own dictionary knows it: the other -//! dialects' dictionaries are never asked (en-US would take `colourblind` from en-GB's). SCOWL +//! dialects' dictionaries are never asked (en-US would take `colourblind` from en-GB's), except +//! that en-AU's list keeps one spelling per word, so Australian English also takes en-GB's words +//! (`enquiry`; its suggestions still come from en-AU). SCOWL //! picks one spelling per dialect (en-GB `-ise` has no `organize`, en-US no `judgement`), and Word //! accepts more, so British English also takes the `-ize` spellings of its `-ise` words that //! en-US or en-CA spells so, and `en-variants.txt` widens each dialect by its pairs @@ -19,16 +24,17 @@ use std::sync::OnceLock; use crate::ProofLang; use crate::affix::AffixDict; -use crate::data::{EN_CA_AFF, EN_CA_SPELLING, EN_GB_AFF, EN_GB_SPELLING, EN_US_AFF, EN_US_SPELLING, SCOWL_NOTICE}; +use crate::data::{EN_AU_AFF, EN_AU_SPELLING, EN_CA_AFF, EN_CA_SPELLING, EN_GB_AFF, EN_GB_SPELLING, EN_US_AFF, EN_US_SPELLING, SCOWL_NOTICE}; -/// Per English (American, British, Canadian): the loaded dictionary, or why it isn't there. -static DICTS: [OnceLock>; 3] = [OnceLock::new(), OnceLock::new(), OnceLock::new()]; +/// Per English (American, British, Canadian, Australian): the loaded dictionary, or why it isn't there. +static DICTS: [OnceLock>; 4] = [OnceLock::new(), OnceLock::new(), OnceLock::new(), OnceLock::new()]; /// The slot, `.aff` and `.dic` text of `lang`'s dictionary. fn source(lang: ProofLang) -> (usize, &'static str, &'static str) { match lang { ProofLang::EnGb => (1, EN_GB_AFF, EN_GB_SPELLING), ProofLang::EnCa => (2, EN_CA_AFF, EN_CA_SPELLING), + ProofLang::EnAu => (3, EN_AU_AFF, EN_AU_SPELLING), _ => (0, EN_US_AFF, EN_US_SPELLING), } } @@ -50,15 +56,17 @@ pub(crate) fn dict(lang: ProofLang) -> Option<&'static AffixDict> { loaded.as_ref().ok() } -/// Does `lang` read the British `-ise` dictionary (en-GB, en-AU, en-NZ…)? +/// Does `lang` read a British `-ise` dictionary (en-GB, and en-NZ… through it; en-AU)? pub(crate) fn is_british(lang: ProofLang) -> bool { - source(lang).0 == 1 + matches!(source(lang).0, 1 | 3) } /// Why `lang`'s embedded SCOWL dictionary failed to load, once something tried to load it. pub(crate) fn load_error(lang: ProofLang) -> Option { let (i, ..) = source(lang); - DICTS.get(i)?.get()?.as_ref().err().map(|e| format!("embedded SCOWL dictionary: {e}")) + let own = DICTS.get(i).and_then(|d| d.get()).and_then(|r| r.as_ref().err()).map(|e| format!("embedded SCOWL dictionary: {e}")); + // Australian English also reads the en-GB list (`crate::english`): its failure is reported too. + own.or_else(|| if lang == ProofLang::EnAu { load_error(ProofLang::EnGb).map(|e| format!("{e} (en-GB, also read for en-AU)")) } else { None }) } /// The SCOWL copyright and licence notices (the upstream README, with the Ispell BSD licence of diff --git a/crates/proof/src/spelling.rs b/crates/proof/src/spelling.rs index c80f42fe..1b42f5ff 100644 --- a/crates/proof/src/spelling.rs +++ b/crates/proof/src/spelling.rs @@ -32,17 +32,43 @@ pub(crate) fn word_to_check_in<'a>(d: &AffixDict, word: &'a str, opts: &Options) if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) { return None; } - // Only a word with a cased letter can be in capitals: Hebrew, Arabic, Thai… words have none. - if opts.ignore_uppercase && w.chars().any(char::is_uppercase) && w.chars().all(|c| !c.is_lowercase()) && n <= 6 { + if opts.ignore_uppercase && in_capitals(w) && n <= 6 { return None; } Some(w) } +/// Is `w` in capitals: a cased letter and no lower-case one? Only a word with a cased letter can +/// be: Hebrew, Arabic, Thai, Chinese… words have none, so they are never taken for acronyms. +pub(crate) fn in_capitals(w: &str) -> bool { + w.chars().any(char::is_uppercase) && w.chars().all(|c| !c.is_lowercase()) +} + +/// Is `part` of a hyphenated word an abbreviation Ignore words in UPPERCASE skips: at most `max` +/// characters, [`in_capitals`] or without letters (`SMS` in `SMS-ка`, `19` in `COVID-19`)? A +/// part in a caseless script is no abbreviation. +pub(crate) fn capitals_part(part: &str, max: usize) -> bool { + part.chars().count() <= max && (in_capitals(part) || !part.chars().any(char::is_alphabetic)) +} + #[cfg(test)] mod tests { use super::*; + /// craft-libs#485's cased-letter rule, shared by every language: a word or part is in + /// capitals only with a cased letter; a part without letters counts as an abbreviation. + #[test] + fn capitals_need_a_cased_letter() { + for w in ["PKO", "SMS", "ÉTÉ", "ВКП", "B2B"] { + assert!(in_capitals(w) && capitals_part(w, 6), "{w}"); + } + for w in ["日本", "שלום", "ไทย", "Pko", "pko", "123", ""] { + assert!(!in_capitals(w), "{w}"); + } + assert!(capitals_part("19", 6) && capitals_part("", 6)); + assert!(!capitals_part("日本", 6) && !capitals_part("ไทย", 6) && !capitals_part("ABCDEFG", 6)); + } + #[test] fn words_needing_no_lookup() { let d = AffixDict::from_hunspell("", "1\nkot\n", &[]).unwrap(); diff --git a/crates/proof/src/tr/grammar.rs b/crates/proof/src/tr/grammar.rs index 81ba93c2..4dcdc84b 100644 --- a/crates/proof/src/tr/grammar.rs +++ b/crates/proof/src/tr/grammar.rs @@ -128,8 +128,10 @@ const ABBREVIATIONS: &[&str] = &[ /// Conjunctions that must not be preceded by a comma. const CONJUNCTIONS_NO_PRE_COMMA: &[&str] = &["ve", "veya", "yahut", "velev", "veyahut", "de", "da"]; -/// Conjunctions that must not be followed by a comma. -const CONJUNCTIONS_NO_POST_COMMA: &[&str] = &["ve", "veya", "yahut", "veyahut", "hem", "gerek", "ister"]; +/// Conjunctions that must not be followed by a comma. Not `ister` and `gerek`: they are +/// conjunctions only in `ister … ister` / `gerek … gerek`; otherwise `ister` is the verb +/// (`Kim ister, gelsin.`) and `gerek` a noun or adjective, and a comma after them is correct. +const CONJUNCTIONS_NO_POST_COMMA: &[&str] = &["ve", "veya", "yahut", "veyahut", "hem"]; /// Check grammar and punctuation in a Turkish text block. pub fn check(text: &str) -> Vec { @@ -209,23 +211,17 @@ pub fn check_with(text: &str, capital_first: bool) -> Vec { }); } } - // Double space - if c == ' ' && i + 1 < chars.len() && chars[i + 1].1 == ' ' && (i == 0 || chars[i - 1].1 != ' ') { - let mut end = idx + 1; - while end < text.len() && text.as_bytes().get(end) == Some(&b' ') { - end += 1; - } - v.push(Issue { - start: idx, - end, - kind: IssueKind::Grammar, - message: "Fazladan boşluk".into(), - suggestions: vec![" ".into()], - lang: ProofLang::Tr, - }); - } } - + for run in crate::space_runs(text) { + v.push(Issue { + start: run.start, + end: run.end, + kind: IssueKind::Grammar, + message: "Fazladan boşluk".into(), + suggestions: vec![" ".into()], + lang: ProofLang::Tr, + }); + } // 3. Comma before or after conjunctions (`ali, ve veli` or `ve, veli`) for (i, &(w_start, w_end)) in ws.iter().enumerate() { let word = match text.get(w_start..w_end) { @@ -411,7 +407,15 @@ mod tests { /// (craft-libs#488 review). #[test] fn a_comma_after_de_ne_or_ya_is_fine() { - for text in ["Ali de, Veli de geldi.", "Ne, sen mi geldin?", "Ya, bilmiyorum.", "Bu mesele de, haklı olan bizdik."] { + for text in [ + "Ali de, Veli de geldi.", + "Ne, sen mi geldin?", + "Ya, bilmiyorum.", + "Bu mesele de, haklı olan bizdik.", + "Kim ister, gelsin.", + "Ali ister, Veli istemez.", + "Gerek, yok.", + ] { assert!(check(text).is_empty(), "{text}: {:?}", check(text)); } } diff --git a/crates/proof/src/tr/suggest.rs b/crates/proof/src/tr/suggest.rs index 0bb705b8..9dbff6f1 100644 --- a/crates/proof/src/tr/suggest.rs +++ b/crates/proof/src/tr/suggest.rs @@ -97,8 +97,8 @@ pub fn suggest_with(d: &TrDict, word: &str, max: usize) -> Vec { } } - // 3. Detached suffix check: `-de`, `-da`, `-mi`, `-mı`, `-mu`, `-mü` - for suffix in ["de", "da", "te", "ta", "mi", "mı", "mu", "mü"] { + // 3. Detached particle check: `-de`, `-da`, `-mi`, `-mı`, `-mu`, `-mü` + for suffix in ["de", "da", "mi", "mı", "mu", "mü"] { if lower.ends_with(suffix) && lower.len() > suffix.len() + 1 { let stem = &lower[..lower.len() - suffix.len()]; if d.is_empty() || d.contains(stem) { @@ -333,6 +333,19 @@ mod tests { assert!(sugs.contains(&"geldin mi".to_string())); } + /// craft-libs#496 review: the particle is `de`/`da` whatever consonant comes before it. `te` + /// and `ta` are only the locative suffix after a voiceless consonant (`kitapta`), written + /// attached, so `kitapte` is never offered as `kitap te`. + #[test] + fn the_detached_particle_is_never_te_or_ta() { + let d = TrDict::from_hunspell("SET UTF-8\n", "2\nkitap\nsonra\n").unwrap(); + assert!(suggest_with(&d, "kitapde", 8).contains(&"kitap de".to_string())); + for w in ["kitapte", "kitapta"] { + let sugs = suggest_with(&d, w, 8); + assert!(!sugs.iter().any(|s| s == "kitap te" || s == "kitap ta"), "{w}: {sugs:?}"); + } + } + #[test] fn overlong_words_get_no_suggestions_promptly() { let d = TrDict::from_hunspell("SET UTF-8\nSFX 1 Y 1\nSFX 1 0 lar .\n", "2\nkitap/1\nsonra\n").unwrap();