diff --git a/crates/proof/examples/affix_check.rs b/crates/proof/examples/affix_check.rs index 329c4fcb..78c3f9bc 100644 --- a/crates/proof/examples/affix_check.rs +++ b/crates/proof/examples/affix_check.rs @@ -8,6 +8,8 @@ //! cargo run --release -p craft-proof --example affix_check -- gen de_DE.aff de_DE.dic 2000 7 > words.txt //! # Single words instead: forms of random stems, and stems with another stem's ending. //! cargo run --release -p craft-proof --example affix_check -- single de_DE.aff de_DE.dic 2000 7 > words.txt +//! # Suggestions: `wordmicrosecondssuggestion,suggestion…` per line. +//! cargo run --release -p craft-proof --example affix_check -- suggest de_DE.aff de_DE.dic < typos.txt //! ``` //! //! The answers are compared with `hunspell -d -l` on the same words @@ -144,6 +146,16 @@ fn main() -> Result<(), String> { eprintln!("{} words in {took:?}", words.len()); Ok(()) } + [cmd, aff, dic] if cmd == "suggest" => { + let d = load(aff, dic)?; + let mut out = std::io::stdout().lock(); + for w in std::io::stdin().lock().lines().map_while(Result::ok) { + let t = std::time::Instant::now(); + let s = d.suggest(&w, 8); + writeln!(out, "{w}\t{}\t{}", t.elapsed().as_micros(), s.join(",")).map_err(|e| e.to_string())?; + } + Ok(()) + } [cmd, aff, dic, n, seed] if cmd == "gen" || cmd == "single" => { let d = load(aff, dic)?; let n: usize = n.parse().map_err(|_| "n is a number")?; @@ -154,6 +166,6 @@ fn main() -> Result<(), String> { } Ok(()) } - _ => Err("usage: affix_check check < words | affix_check gen|single ".into()), + _ => Err("usage: affix_check check|suggest < words | affix_check gen|single ".into()), } } diff --git a/crates/proof/src/affix.rs b/crates/proof/src/affix.rs index 2eccbc29..cdc2ccc5 100644 --- a/crates/proof/src/affix.rs +++ b/crates/proof/src/affix.rs @@ -43,7 +43,7 @@ use crate::patterns::{FxHasher, FxMap}; mod compound; -pub(crate) use compound::MAX_SUGGEST_COMPOUNDS; +pub(crate) use compound::{MAX_SUGGEST_COMPOUND_WORK, Pieces}; const MAGIC: &str = "WCAFF1"; /// Most bytes the compact format inflates to, written or read. @@ -224,6 +224,8 @@ fn parse_cond(text: &str) -> Result, String> { Ok(out.into_boxed_slice()) } +/// Most homonyms of one stem kept apart ([`AffixDict::homonyms`]); past it their flags count merged. +const MAX_HOMONYMS: usize = 16; /// How many suffixes a word may carry in a row through continuation classes (`work` + /// `able/B` + `s` → `workables` is two). Hunspell strips at most two suffixes, so a third /// chained suffix is not a word; the bound also keeps hostile input from walking deep recursions. @@ -364,8 +366,16 @@ pub struct AffixDict { cpd_sets: FxMap, /// Those stems' sets with all their homonyms' flags, for `COMPOUNDRULE` parts. rule_sets: FxMap, + /// Stems listed more than once with different flags (not the `ONLYINCOMPOUND` homonyms + /// beside others): stem index → each homonym's flags. `flags` holds them merged, which + /// answers for one affix; a prefix with a suffix must come from one homonym (Dutch `stop/Pj…` + /// takes `af`, `stop/Ca…` takes `je`: `afstopje` is wrong in the CLI). + homonyms: FxMap>, /// Appended string → the prefix rules adding it, in rule order. pfx_index: FxMap, Vec>, + /// Stems in mixed case (`kB`, `MSc`) by their lower case: a word in capitals may be one + /// (`KB`, `KBS`), as in the CLI. + mixed: FxMap, Vec>, /// Longest appended prefix, in chars. max_pfx_chars: usize, } @@ -791,14 +801,15 @@ impl FlagMode { /// `SUBSTANDARD` (suggestions), `KEEPCASE` (case variants of a word) or `NEEDAFFIX` (a stem on /// its own). /// The second field: no flag-based compounds; the third: a suggestion candidate, whose compound -/// search gets a small budget ([`compound::MAX_SUGGEST_WORK`]). +/// search gets a small budget ([`compound::MAX_SUGGEST_WORK`]); the fourth: `KEEPCASE`, kept +/// apart since it holds for a compound's first part only (and may share its id with another). #[derive(Clone, Copy, Debug, Default)] -struct Excluded([Option; 5], bool, bool); +struct Excluded([Option; 5], bool, bool, Option); impl Excluded { /// Does the flag set `set` carry an excluded flag? fn hits(self, set: &[u16]) -> bool { - self.0.iter().flatten().any(|f| has(set, *f)) + self.0.iter().chain(std::iter::once(&self.3)).flatten().any(|f| has(set, *f)) } /// These exclusions, and no flag-based compounds (`MAXCPDSUGS 0` for suggestions). @@ -815,13 +826,15 @@ impl Excluded { self.2 } - /// These exclusions without `flag`. - fn without(mut self, flag: Option) -> Excluded { - for slot in &mut self.0 { - if *slot == flag { - *slot = None; - } - } + /// These exclusions and `KEEPCASE` (`flag`): a case variant of the word is being checked. + fn keeping(mut self, flag: Option) -> Excluded { + self.3 = flag; + self + } + + /// These exclusions without `KEEPCASE` (a compound part after the first). + fn without_keep(mut self) -> Excluded { + self.3 = None; self } @@ -871,6 +884,19 @@ fn has(set: &[u16], flag: u16) -> bool { set.binary_search(&flag).is_ok() } +/// Note a homonym's flags among its group's (`group`), unless an equal one is there; false past +/// [`MAX_HOMONYMS`] (then the merged flags answer: hostile input can't make it costly). +fn note_homonym(group: &mut Vec>, fresh: &[u16]) -> bool { + if group.iter().any(|g| g == fresh) { + return true; + } + if group.len() >= MAX_HOMONYMS { + return false; + } + group.push(fresh.to_vec()); + true +} + /// Add a homonym's sorted flags `fresh` to the sorted set `set`; `false` (and nothing added) when /// that would take it past [`MAX_FLAGS_PER_STEM`]. fn merge_homonym(set: &mut Vec, fresh: &[u16]) -> bool { @@ -1243,12 +1269,18 @@ impl AffixDict { let mut only_set: Vec = Vec::new(); let mut fresh: Vec = Vec::new(); let mut carried: Vec = Vec::new(); + // Each homonym's flags (ONLYINCOMPOUND ones apart), and whether all fit. + let mut each: Vec> = Vec::new(); + let mut each_only: Vec> = Vec::new(); while i < stems.len() { let Some((word, ..)) = stems.get(i) else { break }; merged.clear(); solo.clear(); only_set.clear(); carried.clear(); + each.clear(); + each_only.clear(); + let mut fits = true; let mut is_word = false; // NEEDAFFIX belongs to its homonym: the word stands alone if any homonym does // (`foo/NA` and `foo/B`: `foo` is right). @@ -1279,21 +1311,23 @@ impl AffixDict { all_need &= need; fresh.sort_unstable(); fresh.dedup(); - if only { + let group_fits = if only { any_only = true; only_need &= need; - merge_homonym(&mut only_set, &fresh); + merge_homonym(&mut only_set, &fresh) } else { any_solo = true; solo_need &= need; - merge_homonym(&mut solo, &fresh); - } + merge_homonym(&mut solo, &fresh) + }; // Homonyms can add up to more flags than one entry holds; past MAX_FLAGS_PER_STEM // an entry's other flags are dropped (its exclusions are carried below), so the // compact data never holds a set its reader refuses. - if !merge_homonym(&mut merged, &fresh) { + if !merge_homonym(&mut merged, &fresh) || !group_fits { dropped += 1; + fits = false; } + fits &= note_homonym(if only { &mut each_only } else { &mut each }, &fresh); } if !is_word { i = j; @@ -1303,6 +1337,11 @@ impl AffixDict { merged.retain(|&f| f != n); } let mixed = any_solo && any_only; + // The homonyms whose flags `flags` holds merged, if they differ. + let apart = if mixed { std::mem::take(&mut each) } else { each.drain(..).chain(each_only.drain(..)).collect() }; + if fits { + d.keep_apart(&mut set_ids, apart, &carried, &exclusions)?; + } let cpd_id = if mixed { for (set, all_need) in [(&mut solo, solo_need), (&mut only_set, only_need)] { if !all_need && let Some(n) = d.need_affix { @@ -1378,6 +1417,23 @@ impl AffixDict { Ok(id) } + /// Keep the homonyms `apart` of the stem about to be pushed, each with the exclusions + /// `carried` (see [`AffixDict::homonyms`]), when there are two or more. + fn keep_apart(&mut self, ids: &mut FxMap, u32>, mut apart: Vec>, carried: &[u16], exclusions: &[u16]) -> Result<(), String> { + if apart.len() < 2 { + return Ok(()); + } + let mut sets = Vec::with_capacity(apart.len()); + for set in &mut apart { + carry_exclusions(set, carried, exclusions); + sets.push(self.intern(ids, set)?); + } + if let Ok(at) = u32::try_from(self.flags.len()) { + self.homonyms.insert(at, sets.into()); + } + Ok(()) + } + /// The flag set `id`, sorted (empty when unknown). fn set(&self, id: u32) -> &[u16] { let i = id as usize; @@ -1423,6 +1479,15 @@ impl AffixDict { } } self.pfx_index = pfx_index; + let mut mixed: FxMap, Vec> = FxMap::default(); + for i in 0..self.len() { + if let (Some(w), Ok(i)) = (self.word(i), u32::try_from(i)) + && casing(w) == Casing::Mixed + { + mixed.entry(w.to_lowercase().into()).or_default().push(i); + } + } + self.mixed = mixed; self.max_pfx_chars = self.prefixes.iter().map(|r| r.add.chars().count()).max().unwrap_or(0); let mut cont_flag = vec![false; self.flag_keys.len()]; for r in self.prefixes.iter().chain(&self.suffixes) { @@ -1531,6 +1596,19 @@ impl AffixDict { self.stem_set(w).filter(|set| !excluded.hits(set) && !self.only_in_compound.is_some_and(|f| has(set, f))) } + /// Does one homonym of the stem `w` (excluded ones aside) have flags meeting `pred`? + fn homonym_meets(&self, w: &str, excluded: Excluded, pred: impl Fn(&[u16]) -> bool) -> bool { + let Some((at, set)) = self.stem_lookup(w) else { return false }; + let usable = |set: &[u16]| !excluded.hits(set) && !self.only_in_compound.is_some_and(|f| has(set, f)); + if !usable(set) { + return false; + } + match self.homonyms.get(&at) { + Some(sets) => sets.iter().map(|&h| self.set(h)).any(|h| usable(h) && pred(h)), + None => pred(set), + } + } + /// Is `w` listed with `FORBIDDENWORD`? Then it is no word, however else it could be built /// (affixes, compounds), as in Hunspell. fn is_forbidden(&self, w: &str) -> bool { @@ -1578,8 +1656,13 @@ impl AffixDict { /// The rules of a group that could lead to `w` are merged into one set of their flags (sorted) /// and followed once, not once per rule: the answer is the same, but thousands of rules of one /// flag cost one walk, not one each (#3613). - fn form_has_flag(&self, w: &str, flags: &[u16], depth: u8, excluded: Excluded) -> bool { - if self.stem_flags_except(w, excluded).is_some_and(|f| intersects(f, flags)) { + /// + /// With `pfx` set (a prefix was taken off), every rule on the way must allow cross products + /// and the prefix's flag must come from the stem or a continuation class on the way + /// (`meg` + `épít` + `és` + `e`). + fn form_has_flag(&self, w: &str, flags: &[u16], depth: u8, pfx: Option, excluded: Excluded) -> bool { + let fits = |f: &[u16]| intersects(f, flags) && pfx.is_none_or(|p| has(f, p)); + if if pfx.is_some() { self.homonym_meets(w, excluded, fits) } else { self.stem_flags_except(w, excluded).is_some_and(fits) } { return true; } // Without continuation classes no rule leads from a form to another. @@ -1588,6 +1671,8 @@ impl AffixDict { } let mut root = String::with_capacity(w.len() + 8); let mut inner: Vec = Vec::new(); + // Rules whose continuation classes give the prefix its flag: under them it is met. + let mut met: Vec = Vec::new(); // Does a continuation set hold one of `flags`? Answered once per distinct set (rules // share interned sets) and, for many flags, through a bitmap: a lookup costs at most the // total size of the distinct sets it meets, not rules × set size. @@ -1602,16 +1687,24 @@ impl AffixDict { root.push_str(base); root.push_str(&g.strip); inner.clear(); + met.clear(); for r in g.cont_rules.iter().filter_map(|&i| self.suffixes.get(i as usize)) { let meets = *seen.entry(r.cont).or_insert_with(|| probe.meets(self.set(r.cont))); - if meets && !self.cont_holds(r, self.circumfix) && !self.cont_holds(r, self.only_in_compound) && r.cond_matches(&root, true) { - inner.push(r.flag); + if meets + && (pfx.is_none() || r.cross) + && !self.cont_holds(r, self.circumfix) + && !self.cont_holds(r, self.only_in_compound) + && r.cond_matches(&root, true) + { + if pfx.is_some_and(|p| self.cont_holds(r, Some(p))) { met.push(r.flag) } else { inner.push(r.flag) } } } - inner.sort_unstable(); - inner.dedup(); - if !inner.is_empty() && self.form_has_flag(&root, &inner, depth - 1, excluded) { - return true; + for (list, p) in [(&mut inner, pfx), (&mut met, None)] { + list.sort_unstable(); + list.dedup(); + if !list.is_empty() && self.form_has_flag(&root, list, depth - 1, p, excluded) { + return true; + } } } } @@ -1631,12 +1724,33 @@ impl AffixDict { } /// [`AffixDict::contains_suggestible`], without flag-based compounds when `compounds` is - /// false (suggestion candidates past the first [`compound::MAX_SUGGEST_COMPOUNDS`]). + /// false (suggestion candidates past the compound check budget). pub(crate) fn contains_suggestible_with(&self, w: &str, compounds: bool) -> bool { let e = self.excluded(true); self.contains_except(&self.normalize(w), if compounds { e } else { e.without_compounds() }) } + /// Suggestions for `word`, best first, at most `max`, from the dictionary's own tables (as + /// for a Language pack). + pub fn suggest(&self, word: &str, max: usize) -> Vec { + crate::affix_suggest::suggest_in(self, word, max, &crate::affix_suggest::Tuning::DEFAULT) + } + + /// [`AffixDict::contains_suggestible`], where the flag-based compound search spends its steps + /// from `budget` (at most [`compound::MAX_SUGGEST_WORK`] for this word). + pub(crate) fn contains_suggestible_spending(&self, w: &str, budget: &mut usize, pieces: &mut compound::Pieces) -> bool { + let w = self.normalize(w); + let e = self.excluded(true); + if self.simple_word(&w, e) || self.compound(&w, e) { + return true; + } + let allowed = (*budget).min(compound::MAX_SUGGEST_WORK); + let mut left = allowed; + let found = self.flag_compound_spending(&w, e, matches!(casing(&w), Casing::Initial | Casing::Upper), &mut left, Some(pieces)); + *budget = budget.saturating_sub(allowed - left); + found + } + /// Whether the dictionary compounds by flags (suggestion candidates then cost more). pub(crate) fn flag_compounds(&self) -> bool { self.cpd.is_some() @@ -1674,6 +1788,11 @@ impl AffixDict { /// Is `w` a stem or an affixed form of one (no compound)? Affixes that are only valid inside /// a compound (`ONLYINCOMPOUND`) don't count. fn simple_word(&self, w: &str, excluded: Excluded) -> bool { + self.simple_word_with(w, excluded, true) + } + + /// [`AffixDict::simple_word`], with prefixes only when `prefixes` is true. + fn simple_word_with(&self, w: &str, excluded: Excluded, prefixes: bool) -> bool { if w.is_empty() || w.len() > MAX_WORD_BYTES || self.is_forbidden(w) { return false; } @@ -1681,6 +1800,9 @@ impl AffixDict { if self.stem_flags_except(w, excluded.and(self.need_affix)).is_some() || self.suffixed(w, None, false, false, excluded) { return true; } + if !prefixes { + return false; + } let mut root = String::with_capacity(w.len() + 8); // The prefix rules whose appended text starts `w`, found by that text (one hash lookup per // length) rather than by trying every rule. @@ -1709,7 +1831,7 @@ impl AffixDict { if self.stem_flags_except(&root, excluded).is_some_and(|f| has(f, r.flag)) { return true; } - if self.is_cont_flag(r.flag) && self.form_has_flag(&root, &[r.flag], INNER_CHAIN, excluded) { + if self.is_cont_flag(r.flag) && self.form_has_flag(&root, &[r.flag], INNER_CHAIN, None, excluded) { return true; } } @@ -1776,6 +1898,7 @@ impl AffixDict { }; let mut root = String::with_capacity(w.len() + 8); let mut flags: Vec = Vec::new(); + let mut met: Vec = Vec::new(); // Each split point from the end: the appended string is `w[b..]`, at most the longest // suffix, and something must stay before it. let cuts = std::iter::once(w.len()).chain(w.char_indices().rev().map(|(i, _)| i).filter(|&i| i > 0 || self.full_strip)); @@ -1789,35 +1912,48 @@ impl AffixDict { let rules = || g.rules.iter().filter_map(|&i| self.suffixes.get(i as usize)); // The stem path: `root` is a stem carrying the rule's flag (and the prefix's, // when a cross product is being checked). - if let Some(f) = self.stem_flags_except(&root, excluded) - && rules().any(|r| { + let takes = |f: &[u16]| { + rules().any(|r| { has(f, r.flag) && need.is_none_or(|n| has(f, n) || self.cont_holds(r, Some(n))) && (need.is_none() || r.cross) && fits(r) && r.cond_matches(&root, true) }) - { + }; + if if need.is_some() { + self.homonym_meets(&root, excluded, takes) + } else { + self.stem_flags_except(&root, excluded).is_some_and(takes) + } { return true; } // Continuation classes: `root` need not be a stem carrying this rule's flag — // it may be a form that gained the flag from another rule's continuation. The // matching rules' flags are looked for together, in one walk. Without continuation // classes that walk would only repeat the stem path above. - if need.is_none() && self.has_cont { + if self.has_cont && !circ { flags.clear(); + met.clear(); // Only a flag some continuation class holds can come from another rule. The // outer suffix of a chain has another affix under it, so its NEEDAFFIX is met, // and Hunspell ignores its CIRCUMFIX there (`workables` from `s/c`). - flags.extend( - rules() - .filter(|r| self.is_cont_flag(r.flag) && !self.cont_holds(r, self.only_in_compound) && r.cond_matches(&root, true)) - .map(|r| r.flag), - ); - flags.sort_unstable(); - flags.dedup(); - if !flags.is_empty() && self.form_has_flag(&root, &flags, INNER_CHAIN, excluded) { - return true; + // After a prefix every rule must allow cross products, and a rule whose + // continuation classes hold the prefix's flag meets it for the whole chain. + for r in rules().filter(|r| { + self.is_cont_flag(r.flag) + && (need.is_none() || r.cross) + && !self.cont_holds(r, self.only_in_compound) + && r.cond_matches(&root, true) + }) { + if need.is_some_and(|n| self.cont_holds(r, Some(n))) { met.push(r.flag) } else { flags.push(r.flag) } + } + for (list, p) in [(&mut flags, need), (&mut met, None)] { + list.sort_unstable(); + list.dedup(); + if !list.is_empty() && self.form_has_flag(&root, list, INNER_CHAIN, p, excluded) { + return true; + } } } } @@ -1841,7 +1977,7 @@ impl AffixDict { /// Whether `w` is a word only through a `KEEPCASE` stem, so it is correct in this letter case /// alone (`km` from `km/K`, which `Km` and `KM` are not). pub(crate) fn case_locked(&self, w: &str) -> bool { - self.keepcase.is_some() && self.contains(w) && !self.contains_except(&self.normalize(w), self.excluded(false).and(self.keepcase)) + self.keepcase.is_some() && self.contains(w) && !self.contains_except(&self.normalize(w), self.excluded(false).keeping(self.keepcase)) } fn check_except(&self, w: &str, excluded: Excluded) -> bool { @@ -1853,12 +1989,12 @@ impl AffixDict { if w.len() > MAX_INPUT_BYTES { return false; } - let keep = excluded.and(self.keepcase); + let keep = excluded.keeping(self.keepcase); match casing(&w) { Casing::Lower => false, // A compound may join a capitalized stem to others, and the word start with a // capital (`FooQux` from `foo` and `Qux`: right in the CLI). - Casing::Mixed => w.chars().next().is_some_and(char::is_uppercase) && self.flag_compound(&decapitalize(&w), excluded, true), + Casing::Mixed => w.chars().next().is_some_and(char::is_uppercase) && self.flag_compound(&decapitalize(&w), keep, true), Casing::Initial => { let lower = w.to_lowercase(); // CHECKSHARPS: a KEEPCASE word with `ß` may be capitalized (`Gruß` for `Gruß/k`). @@ -1869,11 +2005,24 @@ impl AffixDict { let lower = w.to_lowercase(); self.contains_except(&capitalize(&lower), keep) || self.contains_in(&lower, keep, true) + || self.mixed_case(&lower, keep) || (self.check_sharps && self.sharps(&lower, excluded)) } } } + /// Is `lower` (a word in capitals, lowercased) a stem in mixed case, or one with an ending + /// (`kbs` from `kB/S`)? + fn mixed_case(&self, lower: &str, excluded: Excluded) -> bool { + if self.mixed.is_empty() { + return false; + } + let ends = lower.char_indices().map(|(i, c)| i + c.len_utf8()); + ends.filter_map(|b| Some((self.mixed.get(lower.get(..b)?)?, lower.get(b..)?))).any(|(stems, rest)| { + stems.iter().take(MAX_HOMONYMS).filter_map(|&i| self.word(i as usize)).any(|m| self.contains_except(&format!("{m}{rest}"), excluded)) + }) + } + /// CHECKSHARPS: is `lower` (a word in capitals, lowercased) a word with some of its `ss` read /// as `ß`, in lower case or capitalized? Such a form may come from a `KEEPCASE` stem (in /// capitals a word with `ß` is written with `SS`). The first [`MAX_SHARPS`] `ss` pairs are @@ -1904,7 +2053,22 @@ impl AffixDict { /// cross products, so a dictionary with thousands of self-continuing rules can't make one stem /// cost millions of strings (#3613). pub fn forms(&self, stem: &str) -> Vec { + let mut budget = MAX_FORMS_WORK; + self.forms_spending(stem, &mut budget) + } + + /// [`AffixDict::forms`], with the rules it tries taken off `budget` (several stems' forms + /// share one budget: the suggestions' scan of 48 stems in a dictionary of 12,000 rules). + pub(crate) fn forms_spending(&self, stem: &str, budget: &mut usize) -> Vec { let Some(f) = self.stem_flags(stem) else { return Vec::new() }; + let start = *budget; + let out = self.forms_within(stem, f, (*budget).min(MAX_FORMS_WORK)); + *budget = start.saturating_sub(out.1); + out.0 + } + + /// The forms of `stem` (flags `f`) in at most `limit` rules tried, and the rules tried. + fn forms_within(&self, stem: &str, f: &[u16], limit: usize) -> (Vec, usize) { let mut out = Vec::new(); if !self.need_affix.is_some_and(|n| has(f, n)) { out.push(stem.to_string()); @@ -1919,7 +2083,14 @@ impl AffixDict { let mut queued: std::collections::HashSet<(String, u32)> = std::collections::HashSet::new(); let mut work = 0usize; for r in &self.suffixes { - if !has(f, r.flag) || !r.cond_matches(stem, true) || self.cont_holds(r, self.only_in_compound) { + if !has(f, r.flag) { + continue; + } + work += 1; + if work > limit { + return (out, work); + } + if !r.cond_matches(stem, true) || self.cont_holds(r, self.only_in_compound) { continue; } let Some(base) = stem.strip_suffix(&*r.strip) else { continue }; @@ -1941,7 +2112,7 @@ impl AffixDict { out.extend(non_empty(form)); } if out.len() >= MAX_FORMS { - return out; + return (out, work); } } // Continuation classes chain suffixes (`workable/B` → `workables`), up to [`MAX_CHAIN`] @@ -1954,7 +2125,7 @@ impl AffixDict { for (form, flags) in level { for r in self.suffixes_with(flags).into_iter().filter_map(|i| self.suffixes.get(i as usize)) { work += 1; - if work > MAX_FORMS_WORK { + if work > limit { break 'chain; } if !r.cond_matches(&form, true) || self.cont_holds(r, self.only_in_compound) { @@ -1972,7 +2143,7 @@ impl AffixDict { // and Hunspell ignores its CIRCUMFIX there (`workables` from `s/c`). out.extend(non_empty(next)); if out.len() >= MAX_FORMS { - return out; + return (out, work); } } } @@ -1988,8 +2159,8 @@ impl AffixDict { // Every suffix form looked at counts against the work budget, matching or not. for (s, s_circ, s_needs, s_cont) in &cross_sfx { work += 1; - if out.len() >= MAX_FORMS || work > MAX_FORMS_WORK { - return out; + if out.len() >= MAX_FORMS || work > limit { + return (out, work); } // The prefix's flag comes from the stem or the suffix's continuation classes; // circumfix alike; not both in need of another affix. @@ -2000,10 +2171,10 @@ impl AffixDict { } } if out.len() >= MAX_FORMS { - return out; + return (out, work); } } - out + (out, work) } /// `root` with the prefix rule `r` added, when the checker takes it back off: the rule's @@ -2177,20 +2348,23 @@ impl AffixDict { let p = prev.iter().zip(b).take_while(|(x, y)| x == y).count().min(255); raw.push(p as u8); raw.extend_from_slice(b.get(p..).unwrap_or_default()); - let set = self.flags.get(i).map_or(&[][..], |&id| self.set(id)); - if !set.is_empty() { - let mut flags = String::from("/"); - self.write_flags(set, &mut flags); - raw.extend_from_slice(flags.as_bytes()); - } - raw.push(b'\n'); - // The ONLYINCOMPOUND homonyms' flags: the same stem again, merged back on reading. - if let Some(&c) = u32::try_from(i).ok().and_then(|i| self.cpd_sets.get(&i)) { - raw.push(b.len().min(255) as u8); - raw.extend_from_slice(b.get(b.len().min(255)..).unwrap_or_default()); - let mut flags = String::from("/"); - self.write_flags(self.set(c), &mut flags); - raw.extend_from_slice(flags.as_bytes()); + // The homonyms kept apart each on a line, or their flags merged; then the + // ONLYINCOMPOUND homonyms' flags. The same stem again is merged back on reading. + let at = u32::try_from(i).ok(); + let main = self.flags.get(i).copied().unwrap_or_default(); + let apart = at.and_then(|i| self.homonyms.get(&i)).map_or(std::slice::from_ref(&main), |h| &h[..]); + let cpd = at.and_then(|i| self.cpd_sets.get(&i)); + for (k, &id) in apart.iter().chain(cpd).enumerate() { + if k > 0 { + raw.push(b.len().min(255) as u8); + raw.extend_from_slice(b.get(b.len().min(255)..).unwrap_or_default()); + } + let set = self.set(id); + if !set.is_empty() { + let mut flags = String::from("/"); + self.write_flags(set, &mut flags); + raw.extend_from_slice(flags.as_bytes()); + } raw.push(b'\n'); } prev = b; @@ -3219,6 +3393,47 @@ pub(crate) mod tests { assert!(d.check("neworkables")); } + #[test] + fn a_prefix_with_a_suffix_chain() { + // The CLI's answers (`meg` + `épít` + `és` + `e` in Hungarian): every rule must allow + // cross products, and the prefix's flag comes from the stem or a continuation class. + let rules = |p: char, a: char, b: char, a_cont: &str, b_cont: &str| { + format!("SET UTF-8\nPFX P {p} 1\nPFX P 0 un .\nSFX A {a} 1\nSFX A 0 a/B{a_cont} .\nSFX B {b} 1\nSFX B 0 b{b_cont} .\n") + }; + let d = |aff: &str, dic: &str| AffixDict::from_hunspell(aff, &format!("1\n{dic}\n"), &[]).unwrap(); + let yes = d(&rules('Y', 'Y', 'Y', "", ""), "foo/PA"); + assert!(yes.check("unfooab") && yes.check("unfooa") && yes.check("fooab")); + for (p, a, b) in [('Y', 'Y', 'N'), ('Y', 'N', 'Y'), ('N', 'Y', 'Y')] { + assert!(!d(&rules(p, a, b, "", ""), "foo/PA").check("unfooab"), "{p}{a}{b}"); + } + assert!(d(&rules('Y', 'Y', 'Y', "P", ""), "foo/A").check("unfooab"), "flag from the inner suffix"); + let outer = d(&rules('Y', 'Y', 'Y', "", "/P"), "foo/A"); + assert!(outer.check("unfooab") && !outer.check("unfooa"), "flag from the outer suffix"); + assert!(!d(&rules('Y', 'Y', 'Y', "", ""), "foo/A").check("unfooab"), "no prefix flag"); + } + + #[test] + fn a_prefix_and_a_suffix_come_from_one_homonym() { + // `stop/PA` and `stop/S`: `unstop` and `stops` are right, `unstops` wrong (the CLI's + // answers), also after a round trip through the compact form. + let aff = "SET UTF-8\nPFX P Y 1\nPFX P 0 un .\nSFX A Y 1\nSFX A 0 a/B .\nSFX B Y 1\nSFX B 0 b .\nSFX S Y 1\nSFX S 0 s .\n"; + let d = AffixDict::from_hunspell(aff, "3\nstop/PA\nstop/S\ngo/PS\n", &[]).unwrap(); + let again = AffixDict::from_bytes(&d.to_bytes().unwrap()).unwrap(); + for d in [d, again] { + assert!(d.check("unstop") && d.check("stops") && d.check("unstopa") && d.check("unstopab") && d.check("ungos")); + assert!(!d.check("unstops")); + } + } + + #[test] + fn a_word_in_capitals_may_be_a_mixed_case_stem() { + // The CLI's answers: `KB` and `KBS` from `kB/SU`, not `KBT`, `UNKB` or `Kb`. + let aff = "SET UTF-8\nSFX S Y 1\nSFX S 0 s .\nPFX U Y 1\nPFX U 0 un .\nSFX T Y 1\nSFX T 0 t .\n"; + let d = AffixDict::from_hunspell(aff, "2\nkB/SU\naBcD\n", &[]).unwrap(); + assert!(d.check("KB") && d.check("KBS") && d.check("kB") && d.check("ABCD")); + assert!(!d.check("KBT") && !d.check("UNKB") && !d.check("Kb") && !d.check("kb") && !d.check("Abcd")); + } + #[test] fn hostile_compound_rules_are_refused_or_cheap() { for bad in ["COMPOUNDRULE *a", "COMPOUNDRULE (ab)", "COMPOUNDRULE a**", "COMPOUNDMIN x", "ONLYINCOMPOUND ab"] { diff --git a/crates/proof/src/affix/compound.rs b/crates/proof/src/affix/compound.rs index bc5b513d..4edf3df8 100644 --- a/crates/proof/src/affix/compound.rs +++ b/crates/proof/src/affix/compound.rs @@ -51,10 +51,10 @@ pub(super) const MAX_PARTS: usize = 16; /// Most steps one compound check may take (affix rules tried, parts looked up, splits followed). pub(super) const MAX_WORK: usize = 1 << 16; /// Most steps the compound search of one suggestion candidate may take. -pub(crate) const MAX_SUGGEST_WORK: usize = 1 << 11; -/// Most candidates of one suggestion call whose compound search runs; the later ones are -/// checked as simple words only, so a long garbled word's suggestions stay quick. -pub(crate) const MAX_SUGGEST_COMPOUNDS: usize = 128; +pub(crate) const MAX_SUGGEST_WORK: usize = 1 << 13; +/// Most steps the compound searches of one suggestion call may take in all; the candidates +/// left are checked as simple words only, so a long garbled word's suggestions stay quick. +pub(crate) const MAX_SUGGEST_COMPOUND_WORK: usize = 1 << 17; /// Most `CHECKCOMPOUNDPATTERN` lines (the Dutch dictionary has 52). const MAX_PATTERNS: usize = 512; /// Longest text on either side of a `CHECKCOMPOUNDPATTERN`, in bytes. @@ -384,6 +384,9 @@ struct Info { stem: u32, /// How many words it counts as (`COMPOUNDROOT`: two). words: u8, + /// Syllables of its suffix the Hungarian rule leaves out (as the last part): a suffix with no + /// continuation classes doesn't count. + tail: u8, } /// A stem found for a part, and the affixes that make the part from it. @@ -397,6 +400,15 @@ struct Analysis<'a> { inner: Option<&'a Rule>, } +/// Most pieces a [`Pieces`] cache keeps. +const MAX_CACHED_PIECES: usize = 1 << 14; + +/// The infos of pieces, kept across the compound checks of one suggestion call: its candidates +/// differ from the word by a letter or two, so most of their pieces are the same. Indexed by +/// whether the piece starts the word (KEEPCASE holds there). +#[derive(Default)] +pub(crate) struct Pieces([crate::patterns::FxMap, Box<[Info]>>; 2], usize); + /// One compound check: the word, its parts' analyses (each piece looked up once), the parts /// chosen so far and the work done. struct Search<'a> { @@ -417,14 +429,31 @@ struct Search<'a> { work: usize, /// [`AffixDict::near_word`], once answered. near: Option, + /// Pieces analysed by earlier checks with the same exclusions. + cache: Option<&'a mut Pieces>, } impl AffixDict { /// Is `w` a compound by the flag-based rules (see the module docs)? `capitalized`: the word /// as the user wrote it starts with a capital (`FORCEUCASE`). pub(super) fn flag_compound(&self, w: &str, excluded: Excluded, capitalized: bool) -> bool { + // A suggestion candidate's search gets MAX_SUGGEST_WORK steps. + let mut budget = if excluded.suggesting() { MAX_SUGGEST_WORK } else { MAX_WORK }; + self.flag_compound_spending(w, excluded, capitalized, &mut budget, None) + } + + /// [`AffixDict::flag_compound`] in at most `budget` steps (and [`MAX_WORK`]), taking the + /// steps it spends off `budget`. `cache`: pieces analysed before, with the same `excluded`. + pub(super) fn flag_compound_spending( + &self, + w: &str, + excluded: Excluded, + capitalized: bool, + budget: &mut usize, + cache: Option<&mut Pieces>, + ) -> bool { let Some(c) = self.cpd.as_deref() else { return false }; - if excluded.no_compounds() || w.is_empty() || w.len() > super::MAX_WORD_BYTES { + if excluded.no_compounds() || w.is_empty() || w.len() > super::MAX_WORD_BYTES || *budget == 0 { return false; } if self.is_forbidden(w) { @@ -433,7 +462,7 @@ impl AffixDict { // KEEPCASE holds for the first part only (from `baz/K`, `Foobaz` and `FOOBAZ` are right // in the CLI, `Bazfoo` and `BAZFOO` wrong). let first_excluded = excluded; - let excluded = excluded.without(self.keepcase); + let excluded = excluded.without_keep(); let cuts: Vec = w.char_indices().map(|(i, _)| i).chain(std::iter::once(w.len())).collect(); let chars: Vec = w.chars().collect(); let n = cuts.len(); @@ -448,11 +477,83 @@ impl AffixDict { memo: vec![None; n * n], infos: Vec::new(), path: Vec::with_capacity(MAX_PARTS), - // A suggestion candidate's search gets MAX_SUGGEST_WORK steps of the budget. - work: if excluded.suggesting() { MAX_WORK - MAX_SUGGEST_WORK } else { 0 }, + work: MAX_WORK - (*budget).min(MAX_WORK), near: None, + cache, }; - self.parts_from(&mut s, 0) + let start = s.work; + let found = self.parts_from(&mut s, 0); + *budget = budget.saturating_sub(s.work.saturating_sub(start).max(1)); + found + } + + /// Where a slip in `w` (no compound) likely is, in chars: from the furthest point its start + /// reaches as compound parts to the nearest its end does, if anything reaches. A candidate + /// changed there is checked as a compound first. `capitalized`: the same with a capital + /// first letter counts too. + pub(crate) fn compound_gap(&self, w: &str, capitalized: bool, budget: &mut usize, pieces: &mut Pieces) -> Option<(usize, usize)> { + let c = self.cpd.as_deref()?; + if w.is_empty() || w.len() > super::MAX_WORD_BYTES || *budget == 0 { + return None; + } + let excluded = self.excluded(true); + let n = w.chars().count(); + let min = self.compound_min.max(1); + let mut reach = vec![false; n + 1]; + let mut back = vec![false; n + 1]; + let cap = capitalized.then(|| super::capitalize(w)); + for word in std::iter::once(w).chain(cap.as_deref()) { + let mut s = Search { + c, + w: word, + cuts: word.char_indices().map(|(i, _)| i).chain(std::iter::once(word.len())).collect(), + chars: word.chars().collect(), + excluded: excluded.without_keep(), + first_excluded: excluded, + capitalized, + memo: vec![None; (n + 1) * (n + 1)], + infos: Vec::new(), + path: Vec::new(), + work: MAX_WORK - (*budget).min(MAX_WORK), + near: None, + cache: Some(&mut *pieces), + }; + let start = s.work; + let fits = |s: &mut Search, a: usize, b: usize, place: u8| { + let (x, y) = self.infos_of(s, a, b); + s.infos.get(x as usize..y as usize).is_some_and(|infos| infos.iter().any(|i| i.pos & place != 0)) + }; + for a in 0..n { + if a > 0 && !reach.get(a).copied().unwrap_or(false) { + continue; + } + for b in a.saturating_add(min)..n { + if s.work <= MAX_WORK + && fits(&mut s, a, b, if a == 0 { FIRST } else { MIDDLE }) + && let Some(r) = reach.get_mut(b) + { + *r = true; + } + } + } + for b in (1..=n).rev() { + if b < n && !back.get(b).copied().unwrap_or(false) { + continue; + } + for a in 1..=b.saturating_sub(min) { + if s.work <= MAX_WORK + && fits(&mut s, a, b, if b == n { LAST } else { MIDDLE }) + && let Some(r) = back.get_mut(a) + { + *r = true; + } + } + } + *budget = budget.saturating_sub(s.work.saturating_sub(start)); + } + let from = reach.iter().rposition(|&r| r).unwrap_or(0); + let to = back.iter().position(|&r| r).unwrap_or(n); + Some((from.min(to), from.max(to))) } /// Can the rest of the word, from char `start`, be split into parts after `s.path`? @@ -516,7 +617,14 @@ impl AffixDict { return *r; } let from = u32::try_from(s.infos.len()).unwrap_or(u32::MAX); - if let (Some(&x), Some(&y)) = (s.cuts.get(a), s.cuts.get(b)) + let cached = s.cuts.get(a).zip(s.cuts.get(b)).and_then(|(&x, &y)| s.w.get(x..y)).and_then(|piece| { + let map = s.cache.as_deref()?.0.get(usize::from(a == 0))?; + map.get(piece).map(|v| v.to_vec()) + }); + if let Some(v) = cached { + s.work += 1; + s.infos.extend(v); + } else if let (Some(&x), Some(&y)) = (s.cuts.get(a), s.cuts.get(b)) && let Some(piece) = s.w.get(x..y) { // Deduplicated by hashing, and at most MAX_INFOS per piece (each one is tried at @@ -534,6 +642,15 @@ impl AffixDict { } }); s.work = work; + // Only a piece analysed in full is kept (one cut short by the budget isn't). + if work <= MAX_WORK + && let Some(c) = s.cache.as_deref_mut() + && c.1 < MAX_CACHED_PIECES + && let Some(map) = c.0.get_mut(usize::from(a == 0)) + { + map.insert(piece.into(), found.clone().into()); + c.1 += 1; + } s.infos.extend(found); } let to = u32::try_from(s.infos.len()).unwrap_or(u32::MAX); @@ -588,11 +705,19 @@ impl AffixDict { marks |= 1 << i; } } - let words = if carries(c.root) { 2 } else { 1 }; - // FORCEUCASE counts on the stem only (a suffix's continuation class holding it is ignored); + // COMPOUNDROOT and FORCEUCASE count on the stem only (an affix's continuation class + // holding them is ignored); // a pattern's `0` means no suffix (a prefix doesn't count): the CLI's answers. + let words = if c.root.is_some_and(|f| has(an.set, f)) { 2 } else { 1 }; let force = c.force_ucase.is_some_and(|f| has(an.set, f)); - Some(Info { pos, marks, unmodified: an.sfx.is_none(), force, stem: an.stem, words }) + // The Hungarian syllable rule leaves out a suffix with no continuation classes (the + // CLI's answers: `bab|bab|baata` from `ata` is right, from `ata/Z` wrong). + let tail = match (&c.syllables, an.sfx) { + (Some((_, vowels)), Some(r)) if self.set(r.cont).is_empty() => r.add.chars().filter(|ch| vowels.contains(ch)).count(), + _ => 0, + }; + let tail = u8::try_from(tail).unwrap_or(u8::MAX); + Some(Info { pos, marks, unmodified: an.sfx.is_none(), force, stem: an.stem, words, tail }) } /// Every way `p` is a stem, or a stem with a prefix, one or two suffixes, or a prefix and a @@ -737,10 +862,17 @@ impl AffixDict { /// The stem `w` as a compound part: its index and flags, unless it is forbidden or excluded. /// A stem with `ONLYINCOMPOUND` homonyms beside others is two: each group's flags apply on /// their own (`foo/XOS` and `foo/T`: `foos` is a part, `foot` is not, as in the CLI). + /// + /// Homonyms kept apart ([`AffixDict::homonyms`]) are each a part of their own: a suffix and + /// the compound flag must come from one (Dutch `poeder` the noun compounds, `poederden` the + /// verb form doesn't: `schuurpoederden` is wrong in the CLI). fn part_stems(&self, w: &str, excluded: Excluded) -> impl Iterator { let found = self.stem_lookup(w); + let apart = found.and_then(|(stem, _)| self.homonyms.get(&stem).map(|h| (stem, h))); + let main = found.filter(|_| apart.is_none()); + let each = apart.into_iter().flat_map(move |(stem, h)| h.iter().map(move |&id| (stem, self.set(id)))); let only = found.and_then(|(stem, _)| self.cpd_sets.get(&stem).map(|&c| (stem, self.set(c)))); - found.into_iter().chain(only).filter(move |(_, set)| !excluded.hits(set) && !self.forbidden.is_some_and(|f| has(set, f))) + main.into_iter().chain(each).chain(only).filter(move |(_, set)| !excluded.hits(set) && !self.forbidden.is_some_and(|f| has(set, f))) } /// May a part with `info` start at char `start`, after the last part of `s.path` @@ -789,7 +921,10 @@ impl AffixDict { match &s.c.syllables { None => false, Some(_) if !whole => true, - Some((most, vowels)) => s.chars.iter().filter(|c| vowels.contains(c)).count() <= *most, + Some((most, vowels)) => { + let tail = s.path.last().map_or(0, |p| usize::from(p.2.tail)); + s.chars.iter().filter(|c| vowels.contains(c)).count().saturating_sub(tail) <= *most + } } } @@ -838,9 +973,13 @@ impl AffixDict { } let excluded = s.excluded; let mut work = s.work; + // A word listed as a stem that can't stand alone (ONLYINCOMPOUND, NEEDAFFIX) counts here + // only with a suffix, not with a prefix (`ver` + `kehr` beside `verkehr/zo`: + // `Fahrradverkehr` is right in the CLI). let mut is_word = |cand: &[char]| { work += 1; - work > MAX_WORK || self.simple_word(&cand.iter().collect::(), excluded) + let cand: String = cand.iter().collect(); + work > MAX_WORK || self.simple_word_with(&cand, excluded, self.stem_set(&cand).is_none()) }; let near = self.near_miss(&s.chars, s.c.two_words, &mut is_word); s.work = work; @@ -1123,9 +1262,37 @@ mod tests { let took = t.elapsed(); eprintln!("{locale}: suggestions for {garbled} in {took:?}"); assert!(took.as_secs() < 5, "{locale}: {took:?}"); + // A one-letter slip of a long compound gets the compound back. + for (typo, meant) in TYPOS.iter().filter(|t| t.0 == *locale).flat_map(|t| t.1.iter()) { + let got = d.suggest(typo, 8); + assert!(got.iter().any(|g| g == meant), "{locale}: {typo} → {got:?}, not {meant}"); + } } } + /// (locale, [(typo, the compound meant)]): each one's suggestions hold it. + const TYPOS: &[(&str, &[(&str, &str)])] = &[ + ( + "de", + &[ + ("Donaudampfschff", "Donaudampfschiff"), + ("Arbetszimmer", "Arbeitszimmer"), + ("Kinerspielplatz", "Kinderspielplatz"), + ("Bundesverfassugsgericht", "Bundesverfassungsgericht"), + ], + ), + ( + "nl", + &[ + ("ziekenhusopname", "ziekenhuisopname"), + ("voetbawledstrijd", "voetbalwedstrijd"), + ("arbeidsongeschiktheidsverzeekring", "arbeidsongeschiktheidsverzekering"), + ("tandartsafsparak", "tandartsafspraak"), + ("belastingaagifte", "belastingaangifte"), + ], + ), + ]; + /// (locale, right words, wrong words), the CLI's answers. const REAL: &[(&str, &str, &str)] = &[ ( @@ -1715,5 +1882,109 @@ mod tests { listed: "foo brok foofoo foobrok foobroken brokfoo brokbrok brokbroken foofoofoo foofoobrok foofoobroken foobrokfoo \ foobrokbrok foobrokbroken brokfoofoo brokfoobrok brokfoobroken brokbrokfoo brokbrokbrok brokbrokbroken", }, + Pinned { + name: "mixed_keep", + aff: "COMPOUNDFLAG X\nKEEPCASE K", + dic: "foo/X baz/XK Qux/X bar/X", + pieces: "foo baz Qux Baz Foo BAZ QUX", + extra: "BazQux BazfooQux FooQux FooBaz fooBaz FOOQux BAZQux", + right: true, + listed: "foo baz Qux Foo QUX foofoo foobaz fooQux bazfoo bazbaz bazQux Quxfoo Quxbaz QuxQux Foofoo Foobaz FooQux QUXBAZ \ + foofoofoo foofoobaz foofooQux foobazfoo foobazbaz foobazQux fooQuxfoo fooQuxbaz fooQuxQux bazfoofoo bazfoobaz \ + bazfooQux bazbazfoo bazbazbaz bazbazQux bazQuxfoo bazQuxbaz bazQuxQux Quxfoofoo Quxfoobaz QuxfooQux Quxbazfoo \ + Quxbazbaz QuxbazQux QuxQuxfoo QuxQuxbaz QuxQuxQux Foofoofoo Foofoobaz FoofooQux Foobazfoo Foobazbaz FoobazQux \ + FooQuxfoo FooQuxbaz FooQuxQux QUXBAZBAZ", + }, + Pinned { + name: "mixed_case_chk", + aff: "COMPOUNDFLAG X\nCHECKCOMPOUNDCASE", + dic: "foo/X Qux/X bar/X", + pieces: "foo Qux Foo bar Bar", + extra: "", + right: true, + listed: "foo Qux Foo bar Bar foofoo foobar Quxfoo Quxbar Foofoo Foobar barfoo barbar Barfoo Barbar foofoofoo foofoobar \ + foobarfoo foobarbar Quxfoofoo Quxfoobar Quxbarfoo Quxbarbar Foofoofoo Foofoobar Foobarfoo Foobarbar barfoofoo \ + barfoobar barbarfoo barbarbar Barfoofoo Barfoobar Barbarfoo Barbarbar", + }, + Pinned { + name: "mixed_forbid", + aff: "COMPOUNDFLAG X\nFORBIDDENWORD F", + dic: "foo/X Qux/X fooQux/F bar/X", + pieces: "foo Qux Foo bar", + extra: "FooQux fooQux FOOQUX", + right: false, + listed: "fooQux fooFoo QuxFoo FooQux FooFoo barFoo foofooFoo fooQuxFoo fooFoofoo fooFooQux fooFooFoo fooFoobar \ + foobarFoo QuxfooFoo QuxQuxFoo QuxFoofoo QuxFooQux QuxFooFoo QuxFoobar QuxbarFoo FoofooFoo FooQuxFoo FooFoofoo \ + FooFooQux FooFooFoo FooFoobar FoobarFoo barfooFoo barQuxFoo barFoofoo barFooQux barFooFoo barFoobar barbarFoo \ + FOOQUX", + }, + Pinned { + name: "mixed_onlyc", + aff: "COMPOUNDFLAG X\nONLYINCOMPOUND O", + dic: "foo/XO Qux/X bar/X", + pieces: "foo Qux Foo bar Bar", + extra: "", + right: true, + listed: "Qux bar Bar foofoo fooQux foobar Quxfoo QuxQux Quxbar Foofoo FooQux Foobar barfoo barQux barbar Barfoo BarQux \ + Barbar foofoofoo foofooQux foofoobar fooQuxfoo fooQuxQux fooQuxbar foobarfoo foobarQux foobarbar Quxfoofoo \ + QuxfooQux Quxfoobar QuxQuxfoo QuxQuxQux QuxQuxbar Quxbarfoo QuxbarQux Quxbarbar Foofoofoo FoofooQux Foofoobar \ + FooQuxfoo FooQuxQux FooQuxbar Foobarfoo FoobarQux Foobarbar barfoofoo barfooQux barfoobar barQuxfoo barQuxQux \ + barQuxbar barbarfoo barbarQux barbarbar Barfoofoo BarfooQux Barfoobar BarQuxfoo BarQuxQux BarQuxbar Barbarfoo \ + BarbarQux Barbarbar", + }, + Pinned { + name: "mixed_force", + aff: "COMPOUNDFLAG X\nFORCEUCASE U", + dic: "foo/X Qux/XU bar/XU", + pieces: "foo Qux Foo bar Bar", + extra: "", + right: true, + listed: "foo Qux Foo bar Bar foofoo fooQux Quxfoo QuxQux Quxbar Foofoo FooQux Foobar barfoo barQux Barfoo BarQux Barbar \ + foofoofoo foofooQux fooQuxfoo fooQuxQux fooQuxbar foobarfoo foobarQux Quxfoofoo QuxfooQux Quxfoobar QuxQuxfoo \ + QuxQuxQux QuxQuxbar Quxbarfoo QuxbarQux Quxbarbar Foofoofoo FoofooQux Foofoobar FooQuxfoo FooQuxQux FooQuxbar \ + Foobarfoo FoobarQux Foobarbar barfoofoo barfooQux barQuxfoo barQuxQux barQuxbar barbarfoo barbarQux Barfoofoo \ + BarfooQux Barfoobar BarQuxfoo BarQuxQux BarQuxbar Barbarfoo BarbarQux Barbarbar", + }, + Pinned { + name: "keep_nosug", + aff: "COMPOUNDFLAG X\nKEEPCASE K\nNOSUGGEST K", + dic: "foo/X baz/XK", + pieces: "foo baz Foo Baz", + extra: "", + right: true, + listed: "foo baz Foo foofoo foobaz bazfoo bazbaz Foofoo Foobaz foofoofoo foofoobaz foobazfoo foobazbaz bazfoofoo \ + bazfoobaz bazbazfoo bazbazbaz Foofoofoo Foofoobaz Foobazfoo Foobazbaz", + }, + Pinned { + name: "root_affix", + aff: "COMPOUNDFLAG Y\nCOMPOUNDWORDMAX 2\nCOMPOUNDROOT R\nCOMPOUNDPERMITFLAG P\nSFX A Y 1\nSFX A 0 s/RP .\nPFX B Y 1\nPFX B 0 un/RP .", + dic: "foo/YAB bar/YAB baz/YR", + pieces: "foo foos unfoo bar bars baz", + extra: "", + right: true, + listed: "foo foos unfoo bar bars baz foofoo foofoos foounfoo foobar foobars foosfoo foosfoos foosunfoo foosbar foosbars \ + unfoofoo unfoofoos unfoounfoo unfoobar unfoobars barfoo barfoos barunfoo barbar barbars barsfoo barsfoos \ + barsunfoo barsbar barsbars", + }, + Pinned { + name: "syllable_hu_suffix", + aff: "LANG hu_HU\nCOMPOUNDMIN 1\nCOMPOUNDFLAG Y\nCOMPOUNDWORDMAX 2\nCOMPOUNDSYLLABLE 4 aeiou\nSFX C Y 1\nSFX C 0 ata .\nSFX D Y 1\nSFX D 0 ete/Z .\nSFX E Y 1\nSFX E 0 t/F .\nSFX F Y 1\nSFX F 0 ete .", + dic: "ba/Y bab/YCDE", + pieces: "ba bab babata babete babtete", + extra: "bababababata babababababete bababababtete", + right: true, + listed: "ba bab babata babete babtete baba babab bababata bababete bababtete babba babbab babbabata babbabete \ + babbabtete bababa bababab babababata babababtete bababba bababbab bababbabata bababbabtete babbaba babbabab \ + babbababata babbababtete babbabba babbabbab babbabbabata babbabbabtete bababababata bababababtete", + }, + Pinned { + name: "near_prefix_shadow", + aff: "COMPOUNDBEGIN x\nCOMPOUNDMIDDLE y\nCOMPOUNDEND z\nNEEDAFFIX h\nONLYINCOMPOUND o\nSFX S Y 1\nSFX S 0 ad .\nPFX P Y 1\nPFX P 0 fa .\nSFX T Y 1\nSFX T 0 hr .\nPFX V Y 1\nPFX V 0 ve .", + dic: "fahr/x rad/y verkehr/zo hrrad/P fahrrad/h rke/VT fahrr/S", + pieces: "fahr rad verkehr", + extra: "fahrradverkehr fahrradverkehrr fahrradverkehrad", + right: true, + listed: "fahr rad verkehr fahrrad fahrverkehr fahrradverkehr", + }, ]; } diff --git a/crates/proof/src/affix_suggest.rs b/crates/proof/src/affix_suggest.rs index 7d488209..5e49183e 100644 --- a/crates/proof/src/affix_suggest.rs +++ b/crates/proof/src/affix_suggest.rs @@ -25,8 +25,12 @@ use crate::affix::{AffixDict, Casing, capitalize, casing}; /// Longest word suggestions are looked for, in letters. pub(crate) const MAX_LETTERS: usize = 40; const MAX_MAP_VARIANTS: usize = 4096; +/// Most candidates kept waiting for the compound check in one call (the rest are dropped). +const MAX_DEFERRED: usize = 1 << 14; /// Stems the scan of round 6 expands. const SCAN_STEMS: usize = 48; +/// Most affix rules tried in all to make those stems' forms (Mongolian has 12,000 rules). +const SCAN_FORMS_WORK: usize = 1 << 16; const COST_CASE: u32 = 0; pub(crate) const COST_ADD_DIACRITIC: u32 = 4; @@ -125,8 +129,12 @@ struct Collector<'a> { names: bool, /// May half of a missing space be one ([`Tuning::strict_case`])? split_names: bool, - /// Candidates checked with compounds so far (at most `MAX_SUGGEST_COMPOUNDS`). - compound_checks: usize, + /// Candidates no simple word, left for the compound check: (cost, order, candidate). + deferred: Vec<(u32, usize, String)>, + /// Steps left for the compound searches of this call (`MAX_SUGGEST_COMPOUND_WORK` at first). + compound_work: usize, + /// The pieces those searches analysed, shared between them. + pieces: crate::affix::Pieces, } impl Collector<'_> { @@ -143,20 +151,68 @@ impl Collector<'_> { self.tested.insert(cand.to_string(), cost); } } + // A candidate is checked as simple words here; one of a single word that isn't one waits + // for the compound check, which the cheapest candidates get first ([`Self::compounds`]). + if !self.accept(cand, cost, false) && self.d.flag_compounds() && !cand.contains(' ') && self.deferred.len() < MAX_DEFERRED { + self.seq += 1; + self.deferred.push((cost, self.seq, cand.to_string())); + } + } + + /// Check the waiting candidates as compounds, cheapest (then earliest) first, in one budget + /// of steps per call: compound checks cost far more than lookups, and a one-letter slip of a + /// long compound is the likeliest intent. + /// + /// A candidate changed where the word stops being parts (`ziekenhusopname`: `zieken` and + /// `opname` are, `hus` isn't) goes first. + fn compounds(&mut self) { + let mut waiting = std::mem::take(&mut self.deferred); + if waiting.is_empty() { + return; + } + let gap = self.d.compound_gap(&self.input, self.names, &mut self.compound_work, &mut self.pieces); + let input: Vec = self.input.chars().collect(); + let outside = |cand: &str| { + let Some((from, to)) = gap else { return false }; + let at = cand.chars().zip(&input).position(|(a, b)| a != *b).unwrap_or(input.len().min(cand.chars().count())); + at + 1 < from || at > to + 1 + }; + let mut waiting: Vec<(bool, u32, usize, String)> = waiting.drain(..).map(|(cost, seq, cand)| (outside(&cand), cost, seq, cand)).collect(); + waiting.sort_unstable(); + for (_, cost, _, cand) in waiting { + if self.compound_work == 0 { + break; + } + // Offered again at a lower cost meanwhile: that offer is the one to check. + if self.tested.get(&cand).is_some_and(|&c| c < cost) || self.found.contains_key(&cand) { + continue; + } + self.accept(&cand, cost, true); + } + } + + /// Keep `cand` if each of its words is in the dictionary (`compounds`: flag-based compounds + /// count); whether it was. + fn accept(&mut self, cand: &str, cost: u32, compounds: bool) -> bool { let mut words = Vec::new(); let mut cost = cost; let names = if cand.contains(' ') { self.split_names } else { self.names }; - let compounds = self.d.flag_compounds() && self.compound_checks < crate::affix::MAX_SUGGEST_COMPOUNDS; - self.compound_checks += usize::from(compounds); + let d = self.d; + let mut work = self.compound_work; + let pieces = &mut self.pieces; + let mut is_word = + |w: &str| if compounds { d.contains_suggestible_spending(w, &mut work, pieces) } else { d.contains_suggestible_with(w, false) }; for part in cand.split(' ') { - if self.d.contains_suggestible_with(part, compounds) { + if is_word(part) { words.push(part.to_string()); } else if !names { - return; + self.compound_work = work; + return false; } else { let c = capitalize(part); - if !self.d.contains_suggestible_with(&c, compounds) { - return; + if !is_word(&c) { + self.compound_work = work; + return false; } words.push(c); // A name ranks after a common word at the same distance (`moze` → `może`, then @@ -164,7 +220,9 @@ impl Collector<'_> { cost += 1; } } + self.compound_work = work; self.keep(words.join(" "), cost); + true } /// Keep a suggestion known to be correct, unless the dictionary says never to offer it. @@ -312,8 +370,18 @@ pub(crate) fn suggest_in(d: &AffixDict, word: &str, max: usize, t: &Tuning) -> V let chars: Vec = input.chars().collect(); let names = !t.strict_case || case != Casing::Lower; let split_names = !t.strict_case; - let mut col = - Collector { d, input: input.clone(), tested: HashMap::new(), found: HashMap::new(), seq: 0, names, split_names, compound_checks: 0 }; + let mut col = Collector { + d, + input: input.clone(), + tested: HashMap::new(), + found: HashMap::new(), + seq: 0, + names, + split_names, + deferred: Vec::new(), + compound_work: crate::affix::MAX_SUGGEST_COMPOUND_WORK, + pieces: Default::default(), + }; add_case(&mut col, word, &input); let map = groups(d, t); add_map_slips(&mut col, &chars, &map, t); @@ -321,9 +389,11 @@ pub(crate) fn suggest_in(d: &AffixDict, word: &str, max: usize, t: &Tuning) -> V add_splits(&mut col, &input, &t.split); add_key_neighbours(&mut col, &chars); add_typos(&mut col, &chars, t); + col.compounds(); // 6. Further off: scan the stems that start like the word. if col.count_within(COST_MISTYPE) < max { scan(&mut col, &chars, &map); + col.compounds(); } let mut found: Vec<(String, (u32, usize))> = col.found.into_iter().collect(); // Among equal costs, the change nearest the word's start is the likelier intent @@ -526,8 +596,9 @@ fn scan(col: &mut Collector, chars: &[char], map: &[Vec]) { best.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(b.1))); let related = |a: char, b: char| fold(a, map) == fold(b, map); let limit = 4; + let mut budget = SCAN_FORMS_WORK; for (_, stem) in best { - for form in d.forms(stem) { + for form in d.forms_spending(stem, &mut budget) { let lf: Vec = form.to_lowercase().chars().collect(); if lf.len().abs_diff(chars.len()) > 2 { continue; @@ -653,7 +724,9 @@ mod tests { seq: 0, names: true, split_names: true, - compound_checks: 0, + deferred: Vec::new(), + compound_work: crate::affix::MAX_SUGGEST_COMPOUND_WORK, + pieces: Default::default(), }; col.offer("cafe", COST_DROP_DIACRITIC); col.offer("cafe", COST_REP);