mirror of
https://github.com/storytold/craft-libs.git
synced 2026-11-09 20:12:57 +00:00
CP7.4: craft-libs#497 re-review — hu prefixed suffix chains, homonyms kept apart, all-caps mixed-case stems; compound suggestions cheapest-first in one budget
- A prefix with two chained suffixes (hu `meg`+`épít`+`és`+`e`, eo `enjugo`): every rule cross, the prefix flag from the stem or a continuation class (CLI answers). - Homonyms with different flags are kept apart (and written apart in the compact form): a prefix and a suffix, or a compound flag and a suffix, must come from one homonym (nl `afstopje`, `schuurpoederden` wrong in the CLI). nl now 100% in both comparisons. - Hungarian syllable rule leaves out a last suffix with no continuation classes; COMPOUNDROOT counts on the stem only. - Near-word check: a word listed as an ONLYINCOMPOUND/NEEDAFFIX stem counts only with a suffix, not a prefix (de `Fahrradverkehr` right). - A word in capitals may be a mixed-case stem (`KB`/`KBS` from `kB/S`, hu `DNY`, nl `BSC`). - Mixed-case retry keeps KEEPCASE for the first part; KEEPCASE kept apart from the other exclusions (NOSUGGEST sharing its flag is no longer cleared); overflow of the homonym groups counts as dropped. - Suggestions: candidates are checked as simple words first; the rest get the compound check cheapest first, those changed where the word stops being parts before others, in one step budget per call, with piece analyses shared between candidates. The suggestions' stem scan makes forms in one shared budget (mn). - affix_check example: `suggest` command; AffixDict::suggest. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01FypgCX8LsYYE8fsj4byBm2
This commit is contained in:
Martin BallandClaude Opus 5.5 committed
1 parent
3eaa276786
commit
b8efc5add9
4 files changed
+664
-93
No files matched your search
@@ -8,6 +8,8 @@
|
||||
//! cargo run --release -p craft-proof --example affix_check -- gen de_DE.aff de_DE.dic 2000 7 > words.txt
|
||||
//! # Single words instead: forms of random stems, and stems with another stem's ending.
|
||||
//! cargo run --release -p craft-proof --example affix_check -- single de_DE.aff de_DE.dic 2000 7 > words.txt
|
||||
//! # Suggestions: `word<TAB>microseconds<TAB>suggestion,suggestion…` per line.
|
||||
//! cargo run --release -p craft-proof --example affix_check -- suggest de_DE.aff de_DE.dic < typos.txt
|
||||
//! ```
|
||||
//!
|
||||
//! The answers are compared with `hunspell -d <path without extension> -l` on the same words
|
||||
@@ -144,6 +146,16 @@ fn main() -> Result<(), String> {
|
||||
eprintln!("{} words in {took:?}", words.len());
|
||||
Ok(())
|
||||
}
|
||||
[cmd, aff, dic] if cmd == "suggest" => {
|
||||
let d = load(aff, dic)?;
|
||||
let mut out = std::io::stdout().lock();
|
||||
for w in std::io::stdin().lock().lines().map_while(Result::ok) {
|
||||
let t = std::time::Instant::now();
|
||||
let s = d.suggest(&w, 8);
|
||||
writeln!(out, "{w}\t{}\t{}", t.elapsed().as_micros(), s.join(",")).map_err(|e| e.to_string())?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
[cmd, aff, dic, n, seed] if cmd == "gen" || cmd == "single" => {
|
||||
let d = load(aff, dic)?;
|
||||
let n: usize = n.parse().map_err(|_| "n is a number")?;
|
||||
@@ -154,6 +166,6 @@ fn main() -> Result<(), String> {
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
_ => Err("usage: affix_check check <aff> <dic> < words | affix_check gen|single <aff> <dic> <n> <seed>".into()),
|
||||
_ => Err("usage: affix_check check|suggest <aff> <dic> < words | affix_check gen|single <aff> <dic> <n> <seed>".into()),
|
||||
}
|
||||
}
|
||||
+279
-64
@@ -43,7 +43,7 @@ use crate::patterns::{FxHasher, FxMap};
|
||||
|
||||
mod compound;
|
||||
|
||||
pub(crate) use compound::MAX_SUGGEST_COMPOUNDS;
|
||||
pub(crate) use compound::{MAX_SUGGEST_COMPOUND_WORK, Pieces};
|
||||
|
||||
const MAGIC: &str = "WCAFF1";
|
||||
/// Most bytes the compact format inflates to, written or read.
|
||||
@@ -224,6 +224,8 @@ fn parse_cond(text: &str) -> Result<Box<[Cond]>, String> {
|
||||
Ok(out.into_boxed_slice())
|
||||
}
|
||||
|
||||
/// Most homonyms of one stem kept apart ([`AffixDict::homonyms`]); past it their flags count merged.
|
||||
const MAX_HOMONYMS: usize = 16;
|
||||
/// How many suffixes a word may carry in a row through continuation classes (`work` +
|
||||
/// `able/B` + `s` → `workables` is two). Hunspell strips at most two suffixes, so a third
|
||||
/// chained suffix is not a word; the bound also keeps hostile input from walking deep recursions.
|
||||
@@ -364,8 +366,16 @@ pub struct AffixDict {
|
||||
cpd_sets: FxMap<u32, u32>,
|
||||
/// Those stems' sets with all their homonyms' flags, for `COMPOUNDRULE` parts.
|
||||
rule_sets: FxMap<u32, u32>,
|
||||
/// Stems listed more than once with different flags (not the `ONLYINCOMPOUND` homonyms
|
||||
/// beside others): stem index → each homonym's flags. `flags` holds them merged, which
|
||||
/// answers for one affix; a prefix with a suffix must come from one homonym (Dutch `stop/Pj…`
|
||||
/// takes `af`, `stop/Ca…` takes `je`: `afstopje` is wrong in the CLI).
|
||||
homonyms: FxMap<u32, Box<[u32]>>,
|
||||
/// Appended string → the prefix rules adding it, in rule order.
|
||||
pfx_index: FxMap<Box<str>, Vec<u32>>,
|
||||
/// Stems in mixed case (`kB`, `MSc`) by their lower case: a word in capitals may be one
|
||||
/// (`KB`, `KBS`), as in the CLI.
|
||||
mixed: FxMap<Box<str>, Vec<u32>>,
|
||||
/// Longest appended prefix, in chars.
|
||||
max_pfx_chars: usize,
|
||||
}
|
||||
@@ -791,14 +801,15 @@ impl FlagMode {
|
||||
/// `SUBSTANDARD` (suggestions), `KEEPCASE` (case variants of a word) or `NEEDAFFIX` (a stem on
|
||||
/// its own).
|
||||
/// The second field: no flag-based compounds; the third: a suggestion candidate, whose compound
|
||||
/// search gets a small budget ([`compound::MAX_SUGGEST_WORK`]).
|
||||
/// search gets a small budget ([`compound::MAX_SUGGEST_WORK`]); the fourth: `KEEPCASE`, kept
|
||||
/// apart since it holds for a compound's first part only (and may share its id with another).
|
||||
#[derive(Clone, Copy, Debug, Default)]
|
||||
struct Excluded([Option<u16>; 5], bool, bool);
|
||||
struct Excluded([Option<u16>; 5], bool, bool, Option<u16>);
|
||||
|
||||
impl Excluded {
|
||||
/// Does the flag set `set` carry an excluded flag?
|
||||
fn hits(self, set: &[u16]) -> bool {
|
||||
self.0.iter().flatten().any(|f| has(set, *f))
|
||||
self.0.iter().chain(std::iter::once(&self.3)).flatten().any(|f| has(set, *f))
|
||||
}
|
||||
|
||||
/// These exclusions, and no flag-based compounds (`MAXCPDSUGS 0` for suggestions).
|
||||
@@ -815,13 +826,15 @@ impl Excluded {
|
||||
self.2
|
||||
}
|
||||
|
||||
/// These exclusions without `flag`.
|
||||
fn without(mut self, flag: Option<u16>) -> Excluded {
|
||||
for slot in &mut self.0 {
|
||||
if *slot == flag {
|
||||
*slot = None;
|
||||
}
|
||||
}
|
||||
/// These exclusions and `KEEPCASE` (`flag`): a case variant of the word is being checked.
|
||||
fn keeping(mut self, flag: Option<u16>) -> Excluded {
|
||||
self.3 = flag;
|
||||
self
|
||||
}
|
||||
|
||||
/// These exclusions without `KEEPCASE` (a compound part after the first).
|
||||
fn without_keep(mut self) -> Excluded {
|
||||
self.3 = None;
|
||||
self
|
||||
}
|
||||
|
||||
@@ -871,6 +884,19 @@ fn has(set: &[u16], flag: u16) -> bool {
|
||||
set.binary_search(&flag).is_ok()
|
||||
}
|
||||
|
||||
/// Note a homonym's flags among its group's (`group`), unless an equal one is there; false past
|
||||
/// [`MAX_HOMONYMS`] (then the merged flags answer: hostile input can't make it costly).
|
||||
fn note_homonym(group: &mut Vec<Vec<u16>>, fresh: &[u16]) -> bool {
|
||||
if group.iter().any(|g| g == fresh) {
|
||||
return true;
|
||||
}
|
||||
if group.len() >= MAX_HOMONYMS {
|
||||
return false;
|
||||
}
|
||||
group.push(fresh.to_vec());
|
||||
true
|
||||
}
|
||||
|
||||
/// Add a homonym's sorted flags `fresh` to the sorted set `set`; `false` (and nothing added) when
|
||||
/// that would take it past [`MAX_FLAGS_PER_STEM`].
|
||||
fn merge_homonym(set: &mut Vec<u16>, fresh: &[u16]) -> bool {
|
||||
@@ -1243,12 +1269,18 @@ impl AffixDict {
|
||||
let mut only_set: Vec<u16> = Vec::new();
|
||||
let mut fresh: Vec<u16> = Vec::new();
|
||||
let mut carried: Vec<u16> = Vec::new();
|
||||
// Each homonym's flags (ONLYINCOMPOUND ones apart), and whether all fit.
|
||||
let mut each: Vec<Vec<u16>> = Vec::new();
|
||||
let mut each_only: Vec<Vec<u16>> = Vec::new();
|
||||
while i < stems.len() {
|
||||
let Some((word, ..)) = stems.get(i) else { break };
|
||||
merged.clear();
|
||||
solo.clear();
|
||||
only_set.clear();
|
||||
carried.clear();
|
||||
each.clear();
|
||||
each_only.clear();
|
||||
let mut fits = true;
|
||||
let mut is_word = false;
|
||||
// NEEDAFFIX belongs to its homonym: the word stands alone if any homonym does
|
||||
// (`foo/NA` and `foo/B`: `foo` is right).
|
||||
@@ -1279,21 +1311,23 @@ impl AffixDict {
|
||||
all_need &= need;
|
||||
fresh.sort_unstable();
|
||||
fresh.dedup();
|
||||
if only {
|
||||
let group_fits = if only {
|
||||
any_only = true;
|
||||
only_need &= need;
|
||||
merge_homonym(&mut only_set, &fresh);
|
||||
merge_homonym(&mut only_set, &fresh)
|
||||
} else {
|
||||
any_solo = true;
|
||||
solo_need &= need;
|
||||
merge_homonym(&mut solo, &fresh);
|
||||
}
|
||||
merge_homonym(&mut solo, &fresh)
|
||||
};
|
||||
// Homonyms can add up to more flags than one entry holds; past MAX_FLAGS_PER_STEM
|
||||
// an entry's other flags are dropped (its exclusions are carried below), so the
|
||||
// compact data never holds a set its reader refuses.
|
||||
if !merge_homonym(&mut merged, &fresh) {
|
||||
if !merge_homonym(&mut merged, &fresh) || !group_fits {
|
||||
dropped += 1;
|
||||
fits = false;
|
||||
}
|
||||
fits &= note_homonym(if only { &mut each_only } else { &mut each }, &fresh);
|
||||
}
|
||||
if !is_word {
|
||||
i = j;
|
||||
@@ -1303,6 +1337,11 @@ impl AffixDict {
|
||||
merged.retain(|&f| f != n);
|
||||
}
|
||||
let mixed = any_solo && any_only;
|
||||
// The homonyms whose flags `flags` holds merged, if they differ.
|
||||
let apart = if mixed { std::mem::take(&mut each) } else { each.drain(..).chain(each_only.drain(..)).collect() };
|
||||
if fits {
|
||||
d.keep_apart(&mut set_ids, apart, &carried, &exclusions)?;
|
||||
}
|
||||
let cpd_id = if mixed {
|
||||
for (set, all_need) in [(&mut solo, solo_need), (&mut only_set, only_need)] {
|
||||
if !all_need && let Some(n) = d.need_affix {
|
||||
@@ -1378,6 +1417,23 @@ impl AffixDict {
|
||||
Ok(id)
|
||||
}
|
||||
|
||||
/// Keep the homonyms `apart` of the stem about to be pushed, each with the exclusions
|
||||
/// `carried` (see [`AffixDict::homonyms`]), when there are two or more.
|
||||
fn keep_apart(&mut self, ids: &mut FxMap<Vec<u16>, u32>, mut apart: Vec<Vec<u16>>, carried: &[u16], exclusions: &[u16]) -> Result<(), String> {
|
||||
if apart.len() < 2 {
|
||||
return Ok(());
|
||||
}
|
||||
let mut sets = Vec::with_capacity(apart.len());
|
||||
for set in &mut apart {
|
||||
carry_exclusions(set, carried, exclusions);
|
||||
sets.push(self.intern(ids, set)?);
|
||||
}
|
||||
if let Ok(at) = u32::try_from(self.flags.len()) {
|
||||
self.homonyms.insert(at, sets.into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The flag set `id`, sorted (empty when unknown).
|
||||
fn set(&self, id: u32) -> &[u16] {
|
||||
let i = id as usize;
|
||||
@@ -1423,6 +1479,15 @@ impl AffixDict {
|
||||
}
|
||||
}
|
||||
self.pfx_index = pfx_index;
|
||||
let mut mixed: FxMap<Box<str>, Vec<u32>> = FxMap::default();
|
||||
for i in 0..self.len() {
|
||||
if let (Some(w), Ok(i)) = (self.word(i), u32::try_from(i))
|
||||
&& casing(w) == Casing::Mixed
|
||||
{
|
||||
mixed.entry(w.to_lowercase().into()).or_default().push(i);
|
||||
}
|
||||
}
|
||||
self.mixed = mixed;
|
||||
self.max_pfx_chars = self.prefixes.iter().map(|r| r.add.chars().count()).max().unwrap_or(0);
|
||||
let mut cont_flag = vec![false; self.flag_keys.len()];
|
||||
for r in self.prefixes.iter().chain(&self.suffixes) {
|
||||
@@ -1531,6 +1596,19 @@ impl AffixDict {
|
||||
self.stem_set(w).filter(|set| !excluded.hits(set) && !self.only_in_compound.is_some_and(|f| has(set, f)))
|
||||
}
|
||||
|
||||
/// Does one homonym of the stem `w` (excluded ones aside) have flags meeting `pred`?
|
||||
fn homonym_meets(&self, w: &str, excluded: Excluded, pred: impl Fn(&[u16]) -> bool) -> bool {
|
||||
let Some((at, set)) = self.stem_lookup(w) else { return false };
|
||||
let usable = |set: &[u16]| !excluded.hits(set) && !self.only_in_compound.is_some_and(|f| has(set, f));
|
||||
if !usable(set) {
|
||||
return false;
|
||||
}
|
||||
match self.homonyms.get(&at) {
|
||||
Some(sets) => sets.iter().map(|&h| self.set(h)).any(|h| usable(h) && pred(h)),
|
||||
None => pred(set),
|
||||
}
|
||||
}
|
||||
|
||||
/// Is `w` listed with `FORBIDDENWORD`? Then it is no word, however else it could be built
|
||||
/// (affixes, compounds), as in Hunspell.
|
||||
fn is_forbidden(&self, w: &str) -> bool {
|
||||
@@ -1578,8 +1656,13 @@ impl AffixDict {
|
||||
/// The rules of a group that could lead to `w` are merged into one set of their flags (sorted)
|
||||
/// and followed once, not once per rule: the answer is the same, but thousands of rules of one
|
||||
/// flag cost one walk, not one each (#3613).
|
||||
fn form_has_flag(&self, w: &str, flags: &[u16], depth: u8, excluded: Excluded) -> bool {
|
||||
if self.stem_flags_except(w, excluded).is_some_and(|f| intersects(f, flags)) {
|
||||
///
|
||||
/// With `pfx` set (a prefix was taken off), every rule on the way must allow cross products
|
||||
/// and the prefix's flag must come from the stem or a continuation class on the way
|
||||
/// (`meg` + `épít` + `és` + `e`).
|
||||
fn form_has_flag(&self, w: &str, flags: &[u16], depth: u8, pfx: Option<u16>, excluded: Excluded) -> bool {
|
||||
let fits = |f: &[u16]| intersects(f, flags) && pfx.is_none_or(|p| has(f, p));
|
||||
if if pfx.is_some() { self.homonym_meets(w, excluded, fits) } else { self.stem_flags_except(w, excluded).is_some_and(fits) } {
|
||||
return true;
|
||||
}
|
||||
// Without continuation classes no rule leads from a form to another.
|
||||
@@ -1588,6 +1671,8 @@ impl AffixDict {
|
||||
}
|
||||
let mut root = String::with_capacity(w.len() + 8);
|
||||
let mut inner: Vec<u16> = Vec::new();
|
||||
// Rules whose continuation classes give the prefix its flag: under them it is met.
|
||||
let mut met: Vec<u16> = Vec::new();
|
||||
// Does a continuation set hold one of `flags`? Answered once per distinct set (rules
|
||||
// share interned sets) and, for many flags, through a bitmap: a lookup costs at most the
|
||||
// total size of the distinct sets it meets, not rules × set size.
|
||||
@@ -1602,16 +1687,24 @@ impl AffixDict {
|
||||
root.push_str(base);
|
||||
root.push_str(&g.strip);
|
||||
inner.clear();
|
||||
met.clear();
|
||||
for r in g.cont_rules.iter().filter_map(|&i| self.suffixes.get(i as usize)) {
|
||||
let meets = *seen.entry(r.cont).or_insert_with(|| probe.meets(self.set(r.cont)));
|
||||
if meets && !self.cont_holds(r, self.circumfix) && !self.cont_holds(r, self.only_in_compound) && r.cond_matches(&root, true) {
|
||||
inner.push(r.flag);
|
||||
if meets
|
||||
&& (pfx.is_none() || r.cross)
|
||||
&& !self.cont_holds(r, self.circumfix)
|
||||
&& !self.cont_holds(r, self.only_in_compound)
|
||||
&& r.cond_matches(&root, true)
|
||||
{
|
||||
if pfx.is_some_and(|p| self.cont_holds(r, Some(p))) { met.push(r.flag) } else { inner.push(r.flag) }
|
||||
}
|
||||
}
|
||||
inner.sort_unstable();
|
||||
inner.dedup();
|
||||
if !inner.is_empty() && self.form_has_flag(&root, &inner, depth - 1, excluded) {
|
||||
return true;
|
||||
for (list, p) in [(&mut inner, pfx), (&mut met, None)] {
|
||||
list.sort_unstable();
|
||||
list.dedup();
|
||||
if !list.is_empty() && self.form_has_flag(&root, list, depth - 1, p, excluded) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1631,12 +1724,33 @@ impl AffixDict {
|
||||
}
|
||||
|
||||
/// [`AffixDict::contains_suggestible`], without flag-based compounds when `compounds` is
|
||||
/// false (suggestion candidates past the first [`compound::MAX_SUGGEST_COMPOUNDS`]).
|
||||
/// false (suggestion candidates past the compound check budget).
|
||||
pub(crate) fn contains_suggestible_with(&self, w: &str, compounds: bool) -> bool {
|
||||
let e = self.excluded(true);
|
||||
self.contains_except(&self.normalize(w), if compounds { e } else { e.without_compounds() })
|
||||
}
|
||||
|
||||
/// Suggestions for `word`, best first, at most `max`, from the dictionary's own tables (as
|
||||
/// for a Language pack).
|
||||
pub fn suggest(&self, word: &str, max: usize) -> Vec<String> {
|
||||
crate::affix_suggest::suggest_in(self, word, max, &crate::affix_suggest::Tuning::DEFAULT)
|
||||
}
|
||||
|
||||
/// [`AffixDict::contains_suggestible`], where the flag-based compound search spends its steps
|
||||
/// from `budget` (at most [`compound::MAX_SUGGEST_WORK`] for this word).
|
||||
pub(crate) fn contains_suggestible_spending(&self, w: &str, budget: &mut usize, pieces: &mut compound::Pieces) -> bool {
|
||||
let w = self.normalize(w);
|
||||
let e = self.excluded(true);
|
||||
if self.simple_word(&w, e) || self.compound(&w, e) {
|
||||
return true;
|
||||
}
|
||||
let allowed = (*budget).min(compound::MAX_SUGGEST_WORK);
|
||||
let mut left = allowed;
|
||||
let found = self.flag_compound_spending(&w, e, matches!(casing(&w), Casing::Initial | Casing::Upper), &mut left, Some(pieces));
|
||||
*budget = budget.saturating_sub(allowed - left);
|
||||
found
|
||||
}
|
||||
|
||||
/// Whether the dictionary compounds by flags (suggestion candidates then cost more).
|
||||
pub(crate) fn flag_compounds(&self) -> bool {
|
||||
self.cpd.is_some()
|
||||
@@ -1674,6 +1788,11 @@ impl AffixDict {
|
||||
/// Is `w` a stem or an affixed form of one (no compound)? Affixes that are only valid inside
|
||||
/// a compound (`ONLYINCOMPOUND`) don't count.
|
||||
fn simple_word(&self, w: &str, excluded: Excluded) -> bool {
|
||||
self.simple_word_with(w, excluded, true)
|
||||
}
|
||||
|
||||
/// [`AffixDict::simple_word`], with prefixes only when `prefixes` is true.
|
||||
fn simple_word_with(&self, w: &str, excluded: Excluded, prefixes: bool) -> bool {
|
||||
if w.is_empty() || w.len() > MAX_WORD_BYTES || self.is_forbidden(w) {
|
||||
return false;
|
||||
}
|
||||
@@ -1681,6 +1800,9 @@ impl AffixDict {
|
||||
if self.stem_flags_except(w, excluded.and(self.need_affix)).is_some() || self.suffixed(w, None, false, false, excluded) {
|
||||
return true;
|
||||
}
|
||||
if !prefixes {
|
||||
return false;
|
||||
}
|
||||
let mut root = String::with_capacity(w.len() + 8);
|
||||
// The prefix rules whose appended text starts `w`, found by that text (one hash lookup per
|
||||
// length) rather than by trying every rule.
|
||||
@@ -1709,7 +1831,7 @@ impl AffixDict {
|
||||
if self.stem_flags_except(&root, excluded).is_some_and(|f| has(f, r.flag)) {
|
||||
return true;
|
||||
}
|
||||
if self.is_cont_flag(r.flag) && self.form_has_flag(&root, &[r.flag], INNER_CHAIN, excluded) {
|
||||
if self.is_cont_flag(r.flag) && self.form_has_flag(&root, &[r.flag], INNER_CHAIN, None, excluded) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -1776,6 +1898,7 @@ impl AffixDict {
|
||||
};
|
||||
let mut root = String::with_capacity(w.len() + 8);
|
||||
let mut flags: Vec<u16> = Vec::new();
|
||||
let mut met: Vec<u16> = Vec::new();
|
||||
// Each split point from the end: the appended string is `w[b..]`, at most the longest
|
||||
// suffix, and something must stay before it.
|
||||
let cuts = std::iter::once(w.len()).chain(w.char_indices().rev().map(|(i, _)| i).filter(|&i| i > 0 || self.full_strip));
|
||||
@@ -1789,35 +1912,48 @@ impl AffixDict {
|
||||
let rules = || g.rules.iter().filter_map(|&i| self.suffixes.get(i as usize));
|
||||
// The stem path: `root` is a stem carrying the rule's flag (and the prefix's,
|
||||
// when a cross product is being checked).
|
||||
if let Some(f) = self.stem_flags_except(&root, excluded)
|
||||
&& rules().any(|r| {
|
||||
let takes = |f: &[u16]| {
|
||||
rules().any(|r| {
|
||||
has(f, r.flag)
|
||||
&& need.is_none_or(|n| has(f, n) || self.cont_holds(r, Some(n)))
|
||||
&& (need.is_none() || r.cross)
|
||||
&& fits(r)
|
||||
&& r.cond_matches(&root, true)
|
||||
})
|
||||
{
|
||||
};
|
||||
if if need.is_some() {
|
||||
self.homonym_meets(&root, excluded, takes)
|
||||
} else {
|
||||
self.stem_flags_except(&root, excluded).is_some_and(takes)
|
||||
} {
|
||||
return true;
|
||||
}
|
||||
// Continuation classes: `root` need not be a stem carrying this rule's flag —
|
||||
// it may be a form that gained the flag from another rule's continuation. The
|
||||
// matching rules' flags are looked for together, in one walk. Without continuation
|
||||
// classes that walk would only repeat the stem path above.
|
||||
if need.is_none() && self.has_cont {
|
||||
if self.has_cont && !circ {
|
||||
flags.clear();
|
||||
met.clear();
|
||||
// Only a flag some continuation class holds can come from another rule. The
|
||||
// outer suffix of a chain has another affix under it, so its NEEDAFFIX is met,
|
||||
// and Hunspell ignores its CIRCUMFIX there (`workables` from `s/c`).
|
||||
flags.extend(
|
||||
rules()
|
||||
.filter(|r| self.is_cont_flag(r.flag) && !self.cont_holds(r, self.only_in_compound) && r.cond_matches(&root, true))
|
||||
.map(|r| r.flag),
|
||||
);
|
||||
flags.sort_unstable();
|
||||
flags.dedup();
|
||||
if !flags.is_empty() && self.form_has_flag(&root, &flags, INNER_CHAIN, excluded) {
|
||||
return true;
|
||||
// After a prefix every rule must allow cross products, and a rule whose
|
||||
// continuation classes hold the prefix's flag meets it for the whole chain.
|
||||
for r in rules().filter(|r| {
|
||||
self.is_cont_flag(r.flag)
|
||||
&& (need.is_none() || r.cross)
|
||||
&& !self.cont_holds(r, self.only_in_compound)
|
||||
&& r.cond_matches(&root, true)
|
||||
}) {
|
||||
if need.is_some_and(|n| self.cont_holds(r, Some(n))) { met.push(r.flag) } else { flags.push(r.flag) }
|
||||
}
|
||||
for (list, p) in [(&mut flags, need), (&mut met, None)] {
|
||||
list.sort_unstable();
|
||||
list.dedup();
|
||||
if !list.is_empty() && self.form_has_flag(&root, list, INNER_CHAIN, p, excluded) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1841,7 +1977,7 @@ impl AffixDict {
|
||||
/// Whether `w` is a word only through a `KEEPCASE` stem, so it is correct in this letter case
|
||||
/// alone (`km` from `km/K`, which `Km` and `KM` are not).
|
||||
pub(crate) fn case_locked(&self, w: &str) -> bool {
|
||||
self.keepcase.is_some() && self.contains(w) && !self.contains_except(&self.normalize(w), self.excluded(false).and(self.keepcase))
|
||||
self.keepcase.is_some() && self.contains(w) && !self.contains_except(&self.normalize(w), self.excluded(false).keeping(self.keepcase))
|
||||
}
|
||||
|
||||
fn check_except(&self, w: &str, excluded: Excluded) -> bool {
|
||||
@@ -1853,12 +1989,12 @@ impl AffixDict {
|
||||
if w.len() > MAX_INPUT_BYTES {
|
||||
return false;
|
||||
}
|
||||
let keep = excluded.and(self.keepcase);
|
||||
let keep = excluded.keeping(self.keepcase);
|
||||
match casing(&w) {
|
||||
Casing::Lower => false,
|
||||
// A compound may join a capitalized stem to others, and the word start with a
|
||||
// capital (`FooQux` from `foo` and `Qux`: right in the CLI).
|
||||
Casing::Mixed => w.chars().next().is_some_and(char::is_uppercase) && self.flag_compound(&decapitalize(&w), excluded, true),
|
||||
Casing::Mixed => w.chars().next().is_some_and(char::is_uppercase) && self.flag_compound(&decapitalize(&w), keep, true),
|
||||
Casing::Initial => {
|
||||
let lower = w.to_lowercase();
|
||||
// CHECKSHARPS: a KEEPCASE word with `ß` may be capitalized (`Gruß` for `Gruß/k`).
|
||||
@@ -1869,11 +2005,24 @@ impl AffixDict {
|
||||
let lower = w.to_lowercase();
|
||||
self.contains_except(&capitalize(&lower), keep)
|
||||
|| self.contains_in(&lower, keep, true)
|
||||
|| self.mixed_case(&lower, keep)
|
||||
|| (self.check_sharps && self.sharps(&lower, excluded))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Is `lower` (a word in capitals, lowercased) a stem in mixed case, or one with an ending
|
||||
/// (`kbs` from `kB/S`)?
|
||||
fn mixed_case(&self, lower: &str, excluded: Excluded) -> bool {
|
||||
if self.mixed.is_empty() {
|
||||
return false;
|
||||
}
|
||||
let ends = lower.char_indices().map(|(i, c)| i + c.len_utf8());
|
||||
ends.filter_map(|b| Some((self.mixed.get(lower.get(..b)?)?, lower.get(b..)?))).any(|(stems, rest)| {
|
||||
stems.iter().take(MAX_HOMONYMS).filter_map(|&i| self.word(i as usize)).any(|m| self.contains_except(&format!("{m}{rest}"), excluded))
|
||||
})
|
||||
}
|
||||
|
||||
/// CHECKSHARPS: is `lower` (a word in capitals, lowercased) a word with some of its `ss` read
|
||||
/// as `ß`, in lower case or capitalized? Such a form may come from a `KEEPCASE` stem (in
|
||||
/// capitals a word with `ß` is written with `SS`). The first [`MAX_SHARPS`] `ss` pairs are
|
||||
@@ -1904,7 +2053,22 @@ impl AffixDict {
|
||||
/// cross products, so a dictionary with thousands of self-continuing rules can't make one stem
|
||||
/// cost millions of strings (#3613).
|
||||
pub fn forms(&self, stem: &str) -> Vec<String> {
|
||||
let mut budget = MAX_FORMS_WORK;
|
||||
self.forms_spending(stem, &mut budget)
|
||||
}
|
||||
|
||||
/// [`AffixDict::forms`], with the rules it tries taken off `budget` (several stems' forms
|
||||
/// share one budget: the suggestions' scan of 48 stems in a dictionary of 12,000 rules).
|
||||
pub(crate) fn forms_spending(&self, stem: &str, budget: &mut usize) -> Vec<String> {
|
||||
let Some(f) = self.stem_flags(stem) else { return Vec::new() };
|
||||
let start = *budget;
|
||||
let out = self.forms_within(stem, f, (*budget).min(MAX_FORMS_WORK));
|
||||
*budget = start.saturating_sub(out.1);
|
||||
out.0
|
||||
}
|
||||
|
||||
/// The forms of `stem` (flags `f`) in at most `limit` rules tried, and the rules tried.
|
||||
fn forms_within(&self, stem: &str, f: &[u16], limit: usize) -> (Vec<String>, usize) {
|
||||
let mut out = Vec::new();
|
||||
if !self.need_affix.is_some_and(|n| has(f, n)) {
|
||||
out.push(stem.to_string());
|
||||
@@ -1919,7 +2083,14 @@ impl AffixDict {
|
||||
let mut queued: std::collections::HashSet<(String, u32)> = std::collections::HashSet::new();
|
||||
let mut work = 0usize;
|
||||
for r in &self.suffixes {
|
||||
if !has(f, r.flag) || !r.cond_matches(stem, true) || self.cont_holds(r, self.only_in_compound) {
|
||||
if !has(f, r.flag) {
|
||||
continue;
|
||||
}
|
||||
work += 1;
|
||||
if work > limit {
|
||||
return (out, work);
|
||||
}
|
||||
if !r.cond_matches(stem, true) || self.cont_holds(r, self.only_in_compound) {
|
||||
continue;
|
||||
}
|
||||
let Some(base) = stem.strip_suffix(&*r.strip) else { continue };
|
||||
@@ -1941,7 +2112,7 @@ impl AffixDict {
|
||||
out.extend(non_empty(form));
|
||||
}
|
||||
if out.len() >= MAX_FORMS {
|
||||
return out;
|
||||
return (out, work);
|
||||
}
|
||||
}
|
||||
// Continuation classes chain suffixes (`workable/B` → `workables`), up to [`MAX_CHAIN`]
|
||||
@@ -1954,7 +2125,7 @@ impl AffixDict {
|
||||
for (form, flags) in level {
|
||||
for r in self.suffixes_with(flags).into_iter().filter_map(|i| self.suffixes.get(i as usize)) {
|
||||
work += 1;
|
||||
if work > MAX_FORMS_WORK {
|
||||
if work > limit {
|
||||
break 'chain;
|
||||
}
|
||||
if !r.cond_matches(&form, true) || self.cont_holds(r, self.only_in_compound) {
|
||||
@@ -1972,7 +2143,7 @@ impl AffixDict {
|
||||
// and Hunspell ignores its CIRCUMFIX there (`workables` from `s/c`).
|
||||
out.extend(non_empty(next));
|
||||
if out.len() >= MAX_FORMS {
|
||||
return out;
|
||||
return (out, work);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1988,8 +2159,8 @@ impl AffixDict {
|
||||
// Every suffix form looked at counts against the work budget, matching or not.
|
||||
for (s, s_circ, s_needs, s_cont) in &cross_sfx {
|
||||
work += 1;
|
||||
if out.len() >= MAX_FORMS || work > MAX_FORMS_WORK {
|
||||
return out;
|
||||
if out.len() >= MAX_FORMS || work > limit {
|
||||
return (out, work);
|
||||
}
|
||||
// The prefix's flag comes from the stem or the suffix's continuation classes;
|
||||
// circumfix alike; not both in need of another affix.
|
||||
@@ -2000,10 +2171,10 @@ impl AffixDict {
|
||||
}
|
||||
}
|
||||
if out.len() >= MAX_FORMS {
|
||||
return out;
|
||||
return (out, work);
|
||||
}
|
||||
}
|
||||
out
|
||||
(out, work)
|
||||
}
|
||||
|
||||
/// `root` with the prefix rule `r` added, when the checker takes it back off: the rule's
|
||||
@@ -2177,20 +2348,23 @@ impl AffixDict {
|
||||
let p = prev.iter().zip(b).take_while(|(x, y)| x == y).count().min(255);
|
||||
raw.push(p as u8);
|
||||
raw.extend_from_slice(b.get(p..).unwrap_or_default());
|
||||
let set = self.flags.get(i).map_or(&[][..], |&id| self.set(id));
|
||||
if !set.is_empty() {
|
||||
let mut flags = String::from("/");
|
||||
self.write_flags(set, &mut flags);
|
||||
raw.extend_from_slice(flags.as_bytes());
|
||||
}
|
||||
raw.push(b'\n');
|
||||
// The ONLYINCOMPOUND homonyms' flags: the same stem again, merged back on reading.
|
||||
if let Some(&c) = u32::try_from(i).ok().and_then(|i| self.cpd_sets.get(&i)) {
|
||||
raw.push(b.len().min(255) as u8);
|
||||
raw.extend_from_slice(b.get(b.len().min(255)..).unwrap_or_default());
|
||||
let mut flags = String::from("/");
|
||||
self.write_flags(self.set(c), &mut flags);
|
||||
raw.extend_from_slice(flags.as_bytes());
|
||||
// The homonyms kept apart each on a line, or their flags merged; then the
|
||||
// ONLYINCOMPOUND homonyms' flags. The same stem again is merged back on reading.
|
||||
let at = u32::try_from(i).ok();
|
||||
let main = self.flags.get(i).copied().unwrap_or_default();
|
||||
let apart = at.and_then(|i| self.homonyms.get(&i)).map_or(std::slice::from_ref(&main), |h| &h[..]);
|
||||
let cpd = at.and_then(|i| self.cpd_sets.get(&i));
|
||||
for (k, &id) in apart.iter().chain(cpd).enumerate() {
|
||||
if k > 0 {
|
||||
raw.push(b.len().min(255) as u8);
|
||||
raw.extend_from_slice(b.get(b.len().min(255)..).unwrap_or_default());
|
||||
}
|
||||
let set = self.set(id);
|
||||
if !set.is_empty() {
|
||||
let mut flags = String::from("/");
|
||||
self.write_flags(set, &mut flags);
|
||||
raw.extend_from_slice(flags.as_bytes());
|
||||
}
|
||||
raw.push(b'\n');
|
||||
}
|
||||
prev = b;
|
||||
@@ -3219,6 +3393,47 @@ pub(crate) mod tests {
|
||||
assert!(d.check("neworkables"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_prefix_with_a_suffix_chain() {
|
||||
// The CLI's answers (`meg` + `épít` + `és` + `e` in Hungarian): every rule must allow
|
||||
// cross products, and the prefix's flag comes from the stem or a continuation class.
|
||||
let rules = |p: char, a: char, b: char, a_cont: &str, b_cont: &str| {
|
||||
format!("SET UTF-8\nPFX P {p} 1\nPFX P 0 un .\nSFX A {a} 1\nSFX A 0 a/B{a_cont} .\nSFX B {b} 1\nSFX B 0 b{b_cont} .\n")
|
||||
};
|
||||
let d = |aff: &str, dic: &str| AffixDict::from_hunspell(aff, &format!("1\n{dic}\n"), &[]).unwrap();
|
||||
let yes = d(&rules('Y', 'Y', 'Y', "", ""), "foo/PA");
|
||||
assert!(yes.check("unfooab") && yes.check("unfooa") && yes.check("fooab"));
|
||||
for (p, a, b) in [('Y', 'Y', 'N'), ('Y', 'N', 'Y'), ('N', 'Y', 'Y')] {
|
||||
assert!(!d(&rules(p, a, b, "", ""), "foo/PA").check("unfooab"), "{p}{a}{b}");
|
||||
}
|
||||
assert!(d(&rules('Y', 'Y', 'Y', "P", ""), "foo/A").check("unfooab"), "flag from the inner suffix");
|
||||
let outer = d(&rules('Y', 'Y', 'Y', "", "/P"), "foo/A");
|
||||
assert!(outer.check("unfooab") && !outer.check("unfooa"), "flag from the outer suffix");
|
||||
assert!(!d(&rules('Y', 'Y', 'Y', "", ""), "foo/A").check("unfooab"), "no prefix flag");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_prefix_and_a_suffix_come_from_one_homonym() {
|
||||
// `stop/PA` and `stop/S`: `unstop` and `stops` are right, `unstops` wrong (the CLI's
|
||||
// answers), also after a round trip through the compact form.
|
||||
let aff = "SET UTF-8\nPFX P Y 1\nPFX P 0 un .\nSFX A Y 1\nSFX A 0 a/B .\nSFX B Y 1\nSFX B 0 b .\nSFX S Y 1\nSFX S 0 s .\n";
|
||||
let d = AffixDict::from_hunspell(aff, "3\nstop/PA\nstop/S\ngo/PS\n", &[]).unwrap();
|
||||
let again = AffixDict::from_bytes(&d.to_bytes().unwrap()).unwrap();
|
||||
for d in [d, again] {
|
||||
assert!(d.check("unstop") && d.check("stops") && d.check("unstopa") && d.check("unstopab") && d.check("ungos"));
|
||||
assert!(!d.check("unstops"));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_word_in_capitals_may_be_a_mixed_case_stem() {
|
||||
// The CLI's answers: `KB` and `KBS` from `kB/SU`, not `KBT`, `UNKB` or `Kb`.
|
||||
let aff = "SET UTF-8\nSFX S Y 1\nSFX S 0 s .\nPFX U Y 1\nPFX U 0 un .\nSFX T Y 1\nSFX T 0 t .\n";
|
||||
let d = AffixDict::from_hunspell(aff, "2\nkB/SU\naBcD\n", &[]).unwrap();
|
||||
assert!(d.check("KB") && d.check("KBS") && d.check("kB") && d.check("ABCD"));
|
||||
assert!(!d.check("KBT") && !d.check("UNKB") && !d.check("Kb") && !d.check("kb") && !d.check("Abcd"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hostile_compound_rules_are_refused_or_cheap() {
|
||||
for bad in ["COMPOUNDRULE *a", "COMPOUNDRULE (ab)", "COMPOUNDRULE a**", "COMPOUNDMIN x", "ONLYINCOMPOUND ab"] {
|
||||
|
||||
@@ -51,10 +51,10 @@ pub(super) const MAX_PARTS: usize = 16;
|
||||
/// Most steps one compound check may take (affix rules tried, parts looked up, splits followed).
|
||||
pub(super) const MAX_WORK: usize = 1 << 16;
|
||||
/// Most steps the compound search of one suggestion candidate may take.
|
||||
pub(crate) const MAX_SUGGEST_WORK: usize = 1 << 11;
|
||||
/// Most candidates of one suggestion call whose compound search runs; the later ones are
|
||||
/// checked as simple words only, so a long garbled word's suggestions stay quick.
|
||||
pub(crate) const MAX_SUGGEST_COMPOUNDS: usize = 128;
|
||||
pub(crate) const MAX_SUGGEST_WORK: usize = 1 << 13;
|
||||
/// Most steps the compound searches of one suggestion call may take in all; the candidates
|
||||
/// left are checked as simple words only, so a long garbled word's suggestions stay quick.
|
||||
pub(crate) const MAX_SUGGEST_COMPOUND_WORK: usize = 1 << 17;
|
||||
/// Most `CHECKCOMPOUNDPATTERN` lines (the Dutch dictionary has 52).
|
||||
const MAX_PATTERNS: usize = 512;
|
||||
/// Longest text on either side of a `CHECKCOMPOUNDPATTERN`, in bytes.
|
||||
@@ -384,6 +384,9 @@ struct Info {
|
||||
stem: u32,
|
||||
/// How many words it counts as (`COMPOUNDROOT`: two).
|
||||
words: u8,
|
||||
/// Syllables of its suffix the Hungarian rule leaves out (as the last part): a suffix with no
|
||||
/// continuation classes doesn't count.
|
||||
tail: u8,
|
||||
}
|
||||
|
||||
/// A stem found for a part, and the affixes that make the part from it.
|
||||
@@ -397,6 +400,15 @@ struct Analysis<'a> {
|
||||
inner: Option<&'a Rule>,
|
||||
}
|
||||
|
||||
/// Most pieces a [`Pieces`] cache keeps.
|
||||
const MAX_CACHED_PIECES: usize = 1 << 14;
|
||||
|
||||
/// The infos of pieces, kept across the compound checks of one suggestion call: its candidates
|
||||
/// differ from the word by a letter or two, so most of their pieces are the same. Indexed by
|
||||
/// whether the piece starts the word (KEEPCASE holds there).
|
||||
#[derive(Default)]
|
||||
pub(crate) struct Pieces([crate::patterns::FxMap<Box<str>, Box<[Info]>>; 2], usize);
|
||||
|
||||
/// One compound check: the word, its parts' analyses (each piece looked up once), the parts
|
||||
/// chosen so far and the work done.
|
||||
struct Search<'a> {
|
||||
@@ -417,14 +429,31 @@ struct Search<'a> {
|
||||
work: usize,
|
||||
/// [`AffixDict::near_word`], once answered.
|
||||
near: Option<bool>,
|
||||
/// Pieces analysed by earlier checks with the same exclusions.
|
||||
cache: Option<&'a mut Pieces>,
|
||||
}
|
||||
|
||||
impl AffixDict {
|
||||
/// Is `w` a compound by the flag-based rules (see the module docs)? `capitalized`: the word
|
||||
/// as the user wrote it starts with a capital (`FORCEUCASE`).
|
||||
pub(super) fn flag_compound(&self, w: &str, excluded: Excluded, capitalized: bool) -> bool {
|
||||
// A suggestion candidate's search gets MAX_SUGGEST_WORK steps.
|
||||
let mut budget = if excluded.suggesting() { MAX_SUGGEST_WORK } else { MAX_WORK };
|
||||
self.flag_compound_spending(w, excluded, capitalized, &mut budget, None)
|
||||
}
|
||||
|
||||
/// [`AffixDict::flag_compound`] in at most `budget` steps (and [`MAX_WORK`]), taking the
|
||||
/// steps it spends off `budget`. `cache`: pieces analysed before, with the same `excluded`.
|
||||
pub(super) fn flag_compound_spending(
|
||||
&self,
|
||||
w: &str,
|
||||
excluded: Excluded,
|
||||
capitalized: bool,
|
||||
budget: &mut usize,
|
||||
cache: Option<&mut Pieces>,
|
||||
) -> bool {
|
||||
let Some(c) = self.cpd.as_deref() else { return false };
|
||||
if excluded.no_compounds() || w.is_empty() || w.len() > super::MAX_WORD_BYTES {
|
||||
if excluded.no_compounds() || w.is_empty() || w.len() > super::MAX_WORD_BYTES || *budget == 0 {
|
||||
return false;
|
||||
}
|
||||
if self.is_forbidden(w) {
|
||||
@@ -433,7 +462,7 @@ impl AffixDict {
|
||||
// KEEPCASE holds for the first part only (from `baz/K`, `Foobaz` and `FOOBAZ` are right
|
||||
// in the CLI, `Bazfoo` and `BAZFOO` wrong).
|
||||
let first_excluded = excluded;
|
||||
let excluded = excluded.without(self.keepcase);
|
||||
let excluded = excluded.without_keep();
|
||||
let cuts: Vec<usize> = w.char_indices().map(|(i, _)| i).chain(std::iter::once(w.len())).collect();
|
||||
let chars: Vec<char> = w.chars().collect();
|
||||
let n = cuts.len();
|
||||
@@ -448,11 +477,83 @@ impl AffixDict {
|
||||
memo: vec![None; n * n],
|
||||
infos: Vec::new(),
|
||||
path: Vec::with_capacity(MAX_PARTS),
|
||||
// A suggestion candidate's search gets MAX_SUGGEST_WORK steps of the budget.
|
||||
work: if excluded.suggesting() { MAX_WORK - MAX_SUGGEST_WORK } else { 0 },
|
||||
work: MAX_WORK - (*budget).min(MAX_WORK),
|
||||
near: None,
|
||||
cache,
|
||||
};
|
||||
self.parts_from(&mut s, 0)
|
||||
let start = s.work;
|
||||
let found = self.parts_from(&mut s, 0);
|
||||
*budget = budget.saturating_sub(s.work.saturating_sub(start).max(1));
|
||||
found
|
||||
}
|
||||
|
||||
/// Where a slip in `w` (no compound) likely is, in chars: from the furthest point its start
|
||||
/// reaches as compound parts to the nearest its end does, if anything reaches. A candidate
|
||||
/// changed there is checked as a compound first. `capitalized`: the same with a capital
|
||||
/// first letter counts too.
|
||||
pub(crate) fn compound_gap(&self, w: &str, capitalized: bool, budget: &mut usize, pieces: &mut Pieces) -> Option<(usize, usize)> {
|
||||
let c = self.cpd.as_deref()?;
|
||||
if w.is_empty() || w.len() > super::MAX_WORD_BYTES || *budget == 0 {
|
||||
return None;
|
||||
}
|
||||
let excluded = self.excluded(true);
|
||||
let n = w.chars().count();
|
||||
let min = self.compound_min.max(1);
|
||||
let mut reach = vec![false; n + 1];
|
||||
let mut back = vec![false; n + 1];
|
||||
let cap = capitalized.then(|| super::capitalize(w));
|
||||
for word in std::iter::once(w).chain(cap.as_deref()) {
|
||||
let mut s = Search {
|
||||
c,
|
||||
w: word,
|
||||
cuts: word.char_indices().map(|(i, _)| i).chain(std::iter::once(word.len())).collect(),
|
||||
chars: word.chars().collect(),
|
||||
excluded: excluded.without_keep(),
|
||||
first_excluded: excluded,
|
||||
capitalized,
|
||||
memo: vec![None; (n + 1) * (n + 1)],
|
||||
infos: Vec::new(),
|
||||
path: Vec::new(),
|
||||
work: MAX_WORK - (*budget).min(MAX_WORK),
|
||||
near: None,
|
||||
cache: Some(&mut *pieces),
|
||||
};
|
||||
let start = s.work;
|
||||
let fits = |s: &mut Search, a: usize, b: usize, place: u8| {
|
||||
let (x, y) = self.infos_of(s, a, b);
|
||||
s.infos.get(x as usize..y as usize).is_some_and(|infos| infos.iter().any(|i| i.pos & place != 0))
|
||||
};
|
||||
for a in 0..n {
|
||||
if a > 0 && !reach.get(a).copied().unwrap_or(false) {
|
||||
continue;
|
||||
}
|
||||
for b in a.saturating_add(min)..n {
|
||||
if s.work <= MAX_WORK
|
||||
&& fits(&mut s, a, b, if a == 0 { FIRST } else { MIDDLE })
|
||||
&& let Some(r) = reach.get_mut(b)
|
||||
{
|
||||
*r = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
for b in (1..=n).rev() {
|
||||
if b < n && !back.get(b).copied().unwrap_or(false) {
|
||||
continue;
|
||||
}
|
||||
for a in 1..=b.saturating_sub(min) {
|
||||
if s.work <= MAX_WORK
|
||||
&& fits(&mut s, a, b, if b == n { LAST } else { MIDDLE })
|
||||
&& let Some(r) = back.get_mut(a)
|
||||
{
|
||||
*r = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
*budget = budget.saturating_sub(s.work.saturating_sub(start));
|
||||
}
|
||||
let from = reach.iter().rposition(|&r| r).unwrap_or(0);
|
||||
let to = back.iter().position(|&r| r).unwrap_or(n);
|
||||
Some((from.min(to), from.max(to)))
|
||||
}
|
||||
|
||||
/// Can the rest of the word, from char `start`, be split into parts after `s.path`?
|
||||
@@ -516,7 +617,14 @@ impl AffixDict {
|
||||
return *r;
|
||||
}
|
||||
let from = u32::try_from(s.infos.len()).unwrap_or(u32::MAX);
|
||||
if let (Some(&x), Some(&y)) = (s.cuts.get(a), s.cuts.get(b))
|
||||
let cached = s.cuts.get(a).zip(s.cuts.get(b)).and_then(|(&x, &y)| s.w.get(x..y)).and_then(|piece| {
|
||||
let map = s.cache.as_deref()?.0.get(usize::from(a == 0))?;
|
||||
map.get(piece).map(|v| v.to_vec())
|
||||
});
|
||||
if let Some(v) = cached {
|
||||
s.work += 1;
|
||||
s.infos.extend(v);
|
||||
} else if let (Some(&x), Some(&y)) = (s.cuts.get(a), s.cuts.get(b))
|
||||
&& let Some(piece) = s.w.get(x..y)
|
||||
{
|
||||
// Deduplicated by hashing, and at most MAX_INFOS per piece (each one is tried at
|
||||
@@ -534,6 +642,15 @@ impl AffixDict {
|
||||
}
|
||||
});
|
||||
s.work = work;
|
||||
// Only a piece analysed in full is kept (one cut short by the budget isn't).
|
||||
if work <= MAX_WORK
|
||||
&& let Some(c) = s.cache.as_deref_mut()
|
||||
&& c.1 < MAX_CACHED_PIECES
|
||||
&& let Some(map) = c.0.get_mut(usize::from(a == 0))
|
||||
{
|
||||
map.insert(piece.into(), found.clone().into());
|
||||
c.1 += 1;
|
||||
}
|
||||
s.infos.extend(found);
|
||||
}
|
||||
let to = u32::try_from(s.infos.len()).unwrap_or(u32::MAX);
|
||||
@@ -588,11 +705,19 @@ impl AffixDict {
|
||||
marks |= 1 << i;
|
||||
}
|
||||
}
|
||||
let words = if carries(c.root) { 2 } else { 1 };
|
||||
// FORCEUCASE counts on the stem only (a suffix's continuation class holding it is ignored);
|
||||
// COMPOUNDROOT and FORCEUCASE count on the stem only (an affix's continuation class
|
||||
// holding them is ignored);
|
||||
// a pattern's `0` means no suffix (a prefix doesn't count): the CLI's answers.
|
||||
let words = if c.root.is_some_and(|f| has(an.set, f)) { 2 } else { 1 };
|
||||
let force = c.force_ucase.is_some_and(|f| has(an.set, f));
|
||||
Some(Info { pos, marks, unmodified: an.sfx.is_none(), force, stem: an.stem, words })
|
||||
// The Hungarian syllable rule leaves out a suffix with no continuation classes (the
|
||||
// CLI's answers: `bab|bab|baata` from `ata` is right, from `ata/Z` wrong).
|
||||
let tail = match (&c.syllables, an.sfx) {
|
||||
(Some((_, vowels)), Some(r)) if self.set(r.cont).is_empty() => r.add.chars().filter(|ch| vowels.contains(ch)).count(),
|
||||
_ => 0,
|
||||
};
|
||||
let tail = u8::try_from(tail).unwrap_or(u8::MAX);
|
||||
Some(Info { pos, marks, unmodified: an.sfx.is_none(), force, stem: an.stem, words, tail })
|
||||
}
|
||||
|
||||
/// Every way `p` is a stem, or a stem with a prefix, one or two suffixes, or a prefix and a
|
||||
@@ -737,10 +862,17 @@ impl AffixDict {
|
||||
/// The stem `w` as a compound part: its index and flags, unless it is forbidden or excluded.
|
||||
/// A stem with `ONLYINCOMPOUND` homonyms beside others is two: each group's flags apply on
|
||||
/// their own (`foo/XOS` and `foo/T`: `foos` is a part, `foot` is not, as in the CLI).
|
||||
///
|
||||
/// Homonyms kept apart ([`AffixDict::homonyms`]) are each a part of their own: a suffix and
|
||||
/// the compound flag must come from one (Dutch `poeder` the noun compounds, `poederden` the
|
||||
/// verb form doesn't: `schuurpoederden` is wrong in the CLI).
|
||||
fn part_stems(&self, w: &str, excluded: Excluded) -> impl Iterator<Item = (u32, &[u16])> {
|
||||
let found = self.stem_lookup(w);
|
||||
let apart = found.and_then(|(stem, _)| self.homonyms.get(&stem).map(|h| (stem, h)));
|
||||
let main = found.filter(|_| apart.is_none());
|
||||
let each = apart.into_iter().flat_map(move |(stem, h)| h.iter().map(move |&id| (stem, self.set(id))));
|
||||
let only = found.and_then(|(stem, _)| self.cpd_sets.get(&stem).map(|&c| (stem, self.set(c))));
|
||||
found.into_iter().chain(only).filter(move |(_, set)| !excluded.hits(set) && !self.forbidden.is_some_and(|f| has(set, f)))
|
||||
main.into_iter().chain(each).chain(only).filter(move |(_, set)| !excluded.hits(set) && !self.forbidden.is_some_and(|f| has(set, f)))
|
||||
}
|
||||
|
||||
/// May a part with `info` start at char `start`, after the last part of `s.path`
|
||||
@@ -789,7 +921,10 @@ impl AffixDict {
|
||||
match &s.c.syllables {
|
||||
None => false,
|
||||
Some(_) if !whole => true,
|
||||
Some((most, vowels)) => s.chars.iter().filter(|c| vowels.contains(c)).count() <= *most,
|
||||
Some((most, vowels)) => {
|
||||
let tail = s.path.last().map_or(0, |p| usize::from(p.2.tail));
|
||||
s.chars.iter().filter(|c| vowels.contains(c)).count().saturating_sub(tail) <= *most
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -838,9 +973,13 @@ impl AffixDict {
|
||||
}
|
||||
let excluded = s.excluded;
|
||||
let mut work = s.work;
|
||||
// A word listed as a stem that can't stand alone (ONLYINCOMPOUND, NEEDAFFIX) counts here
|
||||
// only with a suffix, not with a prefix (`ver` + `kehr` beside `verkehr/zo`:
|
||||
// `Fahrradverkehr` is right in the CLI).
|
||||
let mut is_word = |cand: &[char]| {
|
||||
work += 1;
|
||||
work > MAX_WORK || self.simple_word(&cand.iter().collect::<String>(), excluded)
|
||||
let cand: String = cand.iter().collect();
|
||||
work > MAX_WORK || self.simple_word_with(&cand, excluded, self.stem_set(&cand).is_none())
|
||||
};
|
||||
let near = self.near_miss(&s.chars, s.c.two_words, &mut is_word);
|
||||
s.work = work;
|
||||
@@ -1123,9 +1262,37 @@ mod tests {
|
||||
let took = t.elapsed();
|
||||
eprintln!("{locale}: suggestions for {garbled} in {took:?}");
|
||||
assert!(took.as_secs() < 5, "{locale}: {took:?}");
|
||||
// A one-letter slip of a long compound gets the compound back.
|
||||
for (typo, meant) in TYPOS.iter().filter(|t| t.0 == *locale).flat_map(|t| t.1.iter()) {
|
||||
let got = d.suggest(typo, 8);
|
||||
assert!(got.iter().any(|g| g == meant), "{locale}: {typo} → {got:?}, not {meant}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// (locale, [(typo, the compound meant)]): each one's suggestions hold it.
|
||||
const TYPOS: &[(&str, &[(&str, &str)])] = &[
|
||||
(
|
||||
"de",
|
||||
&[
|
||||
("Donaudampfschff", "Donaudampfschiff"),
|
||||
("Arbetszimmer", "Arbeitszimmer"),
|
||||
("Kinerspielplatz", "Kinderspielplatz"),
|
||||
("Bundesverfassugsgericht", "Bundesverfassungsgericht"),
|
||||
],
|
||||
),
|
||||
(
|
||||
"nl",
|
||||
&[
|
||||
("ziekenhusopname", "ziekenhuisopname"),
|
||||
("voetbawledstrijd", "voetbalwedstrijd"),
|
||||
("arbeidsongeschiktheidsverzeekring", "arbeidsongeschiktheidsverzekering"),
|
||||
("tandartsafsparak", "tandartsafspraak"),
|
||||
("belastingaagifte", "belastingaangifte"),
|
||||
],
|
||||
),
|
||||
];
|
||||
|
||||
/// (locale, right words, wrong words), the CLI's answers.
|
||||
const REAL: &[(&str, &str, &str)] = &[
|
||||
(
|
||||
@@ -1715,5 +1882,109 @@ mod tests {
|
||||
listed: "foo brok foofoo foobrok foobroken brokfoo brokbrok brokbroken foofoofoo foofoobrok foofoobroken foobrokfoo \
|
||||
foobrokbrok foobrokbroken brokfoofoo brokfoobrok brokfoobroken brokbrokfoo brokbrokbrok brokbrokbroken",
|
||||
},
|
||||
Pinned {
|
||||
name: "mixed_keep",
|
||||
aff: "COMPOUNDFLAG X\nKEEPCASE K",
|
||||
dic: "foo/X baz/XK Qux/X bar/X",
|
||||
pieces: "foo baz Qux Baz Foo BAZ QUX",
|
||||
extra: "BazQux BazfooQux FooQux FooBaz fooBaz FOOQux BAZQux",
|
||||
right: true,
|
||||
listed: "foo baz Qux Foo QUX foofoo foobaz fooQux bazfoo bazbaz bazQux Quxfoo Quxbaz QuxQux Foofoo Foobaz FooQux QUXBAZ \
|
||||
foofoofoo foofoobaz foofooQux foobazfoo foobazbaz foobazQux fooQuxfoo fooQuxbaz fooQuxQux bazfoofoo bazfoobaz \
|
||||
bazfooQux bazbazfoo bazbazbaz bazbazQux bazQuxfoo bazQuxbaz bazQuxQux Quxfoofoo Quxfoobaz QuxfooQux Quxbazfoo \
|
||||
Quxbazbaz QuxbazQux QuxQuxfoo QuxQuxbaz QuxQuxQux Foofoofoo Foofoobaz FoofooQux Foobazfoo Foobazbaz FoobazQux \
|
||||
FooQuxfoo FooQuxbaz FooQuxQux QUXBAZBAZ",
|
||||
},
|
||||
Pinned {
|
||||
name: "mixed_case_chk",
|
||||
aff: "COMPOUNDFLAG X\nCHECKCOMPOUNDCASE",
|
||||
dic: "foo/X Qux/X bar/X",
|
||||
pieces: "foo Qux Foo bar Bar",
|
||||
extra: "",
|
||||
right: true,
|
||||
listed: "foo Qux Foo bar Bar foofoo foobar Quxfoo Quxbar Foofoo Foobar barfoo barbar Barfoo Barbar foofoofoo foofoobar \
|
||||
foobarfoo foobarbar Quxfoofoo Quxfoobar Quxbarfoo Quxbarbar Foofoofoo Foofoobar Foobarfoo Foobarbar barfoofoo \
|
||||
barfoobar barbarfoo barbarbar Barfoofoo Barfoobar Barbarfoo Barbarbar",
|
||||
},
|
||||
Pinned {
|
||||
name: "mixed_forbid",
|
||||
aff: "COMPOUNDFLAG X\nFORBIDDENWORD F",
|
||||
dic: "foo/X Qux/X fooQux/F bar/X",
|
||||
pieces: "foo Qux Foo bar",
|
||||
extra: "FooQux fooQux FOOQUX",
|
||||
right: false,
|
||||
listed: "fooQux fooFoo QuxFoo FooQux FooFoo barFoo foofooFoo fooQuxFoo fooFoofoo fooFooQux fooFooFoo fooFoobar \
|
||||
foobarFoo QuxfooFoo QuxQuxFoo QuxFoofoo QuxFooQux QuxFooFoo QuxFoobar QuxbarFoo FoofooFoo FooQuxFoo FooFoofoo \
|
||||
FooFooQux FooFooFoo FooFoobar FoobarFoo barfooFoo barQuxFoo barFoofoo barFooQux barFooFoo barFoobar barbarFoo \
|
||||
FOOQUX",
|
||||
},
|
||||
Pinned {
|
||||
name: "mixed_onlyc",
|
||||
aff: "COMPOUNDFLAG X\nONLYINCOMPOUND O",
|
||||
dic: "foo/XO Qux/X bar/X",
|
||||
pieces: "foo Qux Foo bar Bar",
|
||||
extra: "",
|
||||
right: true,
|
||||
listed: "Qux bar Bar foofoo fooQux foobar Quxfoo QuxQux Quxbar Foofoo FooQux Foobar barfoo barQux barbar Barfoo BarQux \
|
||||
Barbar foofoofoo foofooQux foofoobar fooQuxfoo fooQuxQux fooQuxbar foobarfoo foobarQux foobarbar Quxfoofoo \
|
||||
QuxfooQux Quxfoobar QuxQuxfoo QuxQuxQux QuxQuxbar Quxbarfoo QuxbarQux Quxbarbar Foofoofoo FoofooQux Foofoobar \
|
||||
FooQuxfoo FooQuxQux FooQuxbar Foobarfoo FoobarQux Foobarbar barfoofoo barfooQux barfoobar barQuxfoo barQuxQux \
|
||||
barQuxbar barbarfoo barbarQux barbarbar Barfoofoo BarfooQux Barfoobar BarQuxfoo BarQuxQux BarQuxbar Barbarfoo \
|
||||
BarbarQux Barbarbar",
|
||||
},
|
||||
Pinned {
|
||||
name: "mixed_force",
|
||||
aff: "COMPOUNDFLAG X\nFORCEUCASE U",
|
||||
dic: "foo/X Qux/XU bar/XU",
|
||||
pieces: "foo Qux Foo bar Bar",
|
||||
extra: "",
|
||||
right: true,
|
||||
listed: "foo Qux Foo bar Bar foofoo fooQux Quxfoo QuxQux Quxbar Foofoo FooQux Foobar barfoo barQux Barfoo BarQux Barbar \
|
||||
foofoofoo foofooQux fooQuxfoo fooQuxQux fooQuxbar foobarfoo foobarQux Quxfoofoo QuxfooQux Quxfoobar QuxQuxfoo \
|
||||
QuxQuxQux QuxQuxbar Quxbarfoo QuxbarQux Quxbarbar Foofoofoo FoofooQux Foofoobar FooQuxfoo FooQuxQux FooQuxbar \
|
||||
Foobarfoo FoobarQux Foobarbar barfoofoo barfooQux barQuxfoo barQuxQux barQuxbar barbarfoo barbarQux Barfoofoo \
|
||||
BarfooQux Barfoobar BarQuxfoo BarQuxQux BarQuxbar Barbarfoo BarbarQux Barbarbar",
|
||||
},
|
||||
Pinned {
|
||||
name: "keep_nosug",
|
||||
aff: "COMPOUNDFLAG X\nKEEPCASE K\nNOSUGGEST K",
|
||||
dic: "foo/X baz/XK",
|
||||
pieces: "foo baz Foo Baz",
|
||||
extra: "",
|
||||
right: true,
|
||||
listed: "foo baz Foo foofoo foobaz bazfoo bazbaz Foofoo Foobaz foofoofoo foofoobaz foobazfoo foobazbaz bazfoofoo \
|
||||
bazfoobaz bazbazfoo bazbazbaz Foofoofoo Foofoobaz Foobazfoo Foobazbaz",
|
||||
},
|
||||
Pinned {
|
||||
name: "root_affix",
|
||||
aff: "COMPOUNDFLAG Y\nCOMPOUNDWORDMAX 2\nCOMPOUNDROOT R\nCOMPOUNDPERMITFLAG P\nSFX A Y 1\nSFX A 0 s/RP .\nPFX B Y 1\nPFX B 0 un/RP .",
|
||||
dic: "foo/YAB bar/YAB baz/YR",
|
||||
pieces: "foo foos unfoo bar bars baz",
|
||||
extra: "",
|
||||
right: true,
|
||||
listed: "foo foos unfoo bar bars baz foofoo foofoos foounfoo foobar foobars foosfoo foosfoos foosunfoo foosbar foosbars \
|
||||
unfoofoo unfoofoos unfoounfoo unfoobar unfoobars barfoo barfoos barunfoo barbar barbars barsfoo barsfoos \
|
||||
barsunfoo barsbar barsbars",
|
||||
},
|
||||
Pinned {
|
||||
name: "syllable_hu_suffix",
|
||||
aff: "LANG hu_HU\nCOMPOUNDMIN 1\nCOMPOUNDFLAG Y\nCOMPOUNDWORDMAX 2\nCOMPOUNDSYLLABLE 4 aeiou\nSFX C Y 1\nSFX C 0 ata .\nSFX D Y 1\nSFX D 0 ete/Z .\nSFX E Y 1\nSFX E 0 t/F .\nSFX F Y 1\nSFX F 0 ete .",
|
||||
dic: "ba/Y bab/YCDE",
|
||||
pieces: "ba bab babata babete babtete",
|
||||
extra: "bababababata babababababete bababababtete",
|
||||
right: true,
|
||||
listed: "ba bab babata babete babtete baba babab bababata bababete bababtete babba babbab babbabata babbabete \
|
||||
babbabtete bababa bababab babababata babababtete bababba bababbab bababbabata bababbabtete babbaba babbabab \
|
||||
babbababata babbababtete babbabba babbabbab babbabbabata babbabbabtete bababababata bababababtete",
|
||||
},
|
||||
Pinned {
|
||||
name: "near_prefix_shadow",
|
||||
aff: "COMPOUNDBEGIN x\nCOMPOUNDMIDDLE y\nCOMPOUNDEND z\nNEEDAFFIX h\nONLYINCOMPOUND o\nSFX S Y 1\nSFX S 0 ad .\nPFX P Y 1\nPFX P 0 fa .\nSFX T Y 1\nSFX T 0 hr .\nPFX V Y 1\nPFX V 0 ve .",
|
||||
dic: "fahr/x rad/y verkehr/zo hrrad/P fahrrad/h rke/VT fahrr/S",
|
||||
pieces: "fahr rad verkehr",
|
||||
extra: "fahrradverkehr fahrradverkehrr fahrradverkehrad",
|
||||
right: true,
|
||||
listed: "fahr rad verkehr fahrrad fahrverkehr fahrradverkehr",
|
||||
},
|
||||
];
|
||||
}
|
||||
@@ -25,8 +25,12 @@ use crate::affix::{AffixDict, Casing, capitalize, casing};
|
||||
/// Longest word suggestions are looked for, in letters.
|
||||
pub(crate) const MAX_LETTERS: usize = 40;
|
||||
const MAX_MAP_VARIANTS: usize = 4096;
|
||||
/// Most candidates kept waiting for the compound check in one call (the rest are dropped).
|
||||
const MAX_DEFERRED: usize = 1 << 14;
|
||||
/// Stems the scan of round 6 expands.
|
||||
const SCAN_STEMS: usize = 48;
|
||||
/// Most affix rules tried in all to make those stems' forms (Mongolian has 12,000 rules).
|
||||
const SCAN_FORMS_WORK: usize = 1 << 16;
|
||||
|
||||
const COST_CASE: u32 = 0;
|
||||
pub(crate) const COST_ADD_DIACRITIC: u32 = 4;
|
||||
@@ -125,8 +129,12 @@ struct Collector<'a> {
|
||||
names: bool,
|
||||
/// May half of a missing space be one ([`Tuning::strict_case`])?
|
||||
split_names: bool,
|
||||
/// Candidates checked with compounds so far (at most `MAX_SUGGEST_COMPOUNDS`).
|
||||
compound_checks: usize,
|
||||
/// Candidates no simple word, left for the compound check: (cost, order, candidate).
|
||||
deferred: Vec<(u32, usize, String)>,
|
||||
/// Steps left for the compound searches of this call (`MAX_SUGGEST_COMPOUND_WORK` at first).
|
||||
compound_work: usize,
|
||||
/// The pieces those searches analysed, shared between them.
|
||||
pieces: crate::affix::Pieces,
|
||||
}
|
||||
|
||||
impl Collector<'_> {
|
||||
@@ -143,20 +151,68 @@ impl Collector<'_> {
|
||||
self.tested.insert(cand.to_string(), cost);
|
||||
}
|
||||
}
|
||||
// A candidate is checked as simple words here; one of a single word that isn't one waits
|
||||
// for the compound check, which the cheapest candidates get first ([`Self::compounds`]).
|
||||
if !self.accept(cand, cost, false) && self.d.flag_compounds() && !cand.contains(' ') && self.deferred.len() < MAX_DEFERRED {
|
||||
self.seq += 1;
|
||||
self.deferred.push((cost, self.seq, cand.to_string()));
|
||||
}
|
||||
}
|
||||
|
||||
/// Check the waiting candidates as compounds, cheapest (then earliest) first, in one budget
|
||||
/// of steps per call: compound checks cost far more than lookups, and a one-letter slip of a
|
||||
/// long compound is the likeliest intent.
|
||||
///
|
||||
/// A candidate changed where the word stops being parts (`ziekenhusopname`: `zieken` and
|
||||
/// `opname` are, `hus` isn't) goes first.
|
||||
fn compounds(&mut self) {
|
||||
let mut waiting = std::mem::take(&mut self.deferred);
|
||||
if waiting.is_empty() {
|
||||
return;
|
||||
}
|
||||
let gap = self.d.compound_gap(&self.input, self.names, &mut self.compound_work, &mut self.pieces);
|
||||
let input: Vec<char> = self.input.chars().collect();
|
||||
let outside = |cand: &str| {
|
||||
let Some((from, to)) = gap else { return false };
|
||||
let at = cand.chars().zip(&input).position(|(a, b)| a != *b).unwrap_or(input.len().min(cand.chars().count()));
|
||||
at + 1 < from || at > to + 1
|
||||
};
|
||||
let mut waiting: Vec<(bool, u32, usize, String)> = waiting.drain(..).map(|(cost, seq, cand)| (outside(&cand), cost, seq, cand)).collect();
|
||||
waiting.sort_unstable();
|
||||
for (_, cost, _, cand) in waiting {
|
||||
if self.compound_work == 0 {
|
||||
break;
|
||||
}
|
||||
// Offered again at a lower cost meanwhile: that offer is the one to check.
|
||||
if self.tested.get(&cand).is_some_and(|&c| c < cost) || self.found.contains_key(&cand) {
|
||||
continue;
|
||||
}
|
||||
self.accept(&cand, cost, true);
|
||||
}
|
||||
}
|
||||
|
||||
/// Keep `cand` if each of its words is in the dictionary (`compounds`: flag-based compounds
|
||||
/// count); whether it was.
|
||||
fn accept(&mut self, cand: &str, cost: u32, compounds: bool) -> bool {
|
||||
let mut words = Vec::new();
|
||||
let mut cost = cost;
|
||||
let names = if cand.contains(' ') { self.split_names } else { self.names };
|
||||
let compounds = self.d.flag_compounds() && self.compound_checks < crate::affix::MAX_SUGGEST_COMPOUNDS;
|
||||
self.compound_checks += usize::from(compounds);
|
||||
let d = self.d;
|
||||
let mut work = self.compound_work;
|
||||
let pieces = &mut self.pieces;
|
||||
let mut is_word =
|
||||
|w: &str| if compounds { d.contains_suggestible_spending(w, &mut work, pieces) } else { d.contains_suggestible_with(w, false) };
|
||||
for part in cand.split(' ') {
|
||||
if self.d.contains_suggestible_with(part, compounds) {
|
||||
if is_word(part) {
|
||||
words.push(part.to_string());
|
||||
} else if !names {
|
||||
return;
|
||||
self.compound_work = work;
|
||||
return false;
|
||||
} else {
|
||||
let c = capitalize(part);
|
||||
if !self.d.contains_suggestible_with(&c, compounds) {
|
||||
return;
|
||||
if !is_word(&c) {
|
||||
self.compound_work = work;
|
||||
return false;
|
||||
}
|
||||
words.push(c);
|
||||
// A name ranks after a common word at the same distance (`moze` → `może`, then
|
||||
@@ -164,7 +220,9 @@ impl Collector<'_> {
|
||||
cost += 1;
|
||||
}
|
||||
}
|
||||
self.compound_work = work;
|
||||
self.keep(words.join(" "), cost);
|
||||
true
|
||||
}
|
||||
|
||||
/// Keep a suggestion known to be correct, unless the dictionary says never to offer it.
|
||||
@@ -312,8 +370,18 @@ pub(crate) fn suggest_in(d: &AffixDict, word: &str, max: usize, t: &Tuning) -> V
|
||||
let chars: Vec<char> = input.chars().collect();
|
||||
let names = !t.strict_case || case != Casing::Lower;
|
||||
let split_names = !t.strict_case;
|
||||
let mut col =
|
||||
Collector { d, input: input.clone(), tested: HashMap::new(), found: HashMap::new(), seq: 0, names, split_names, compound_checks: 0 };
|
||||
let mut col = Collector {
|
||||
d,
|
||||
input: input.clone(),
|
||||
tested: HashMap::new(),
|
||||
found: HashMap::new(),
|
||||
seq: 0,
|
||||
names,
|
||||
split_names,
|
||||
deferred: Vec::new(),
|
||||
compound_work: crate::affix::MAX_SUGGEST_COMPOUND_WORK,
|
||||
pieces: Default::default(),
|
||||
};
|
||||
add_case(&mut col, word, &input);
|
||||
let map = groups(d, t);
|
||||
add_map_slips(&mut col, &chars, &map, t);
|
||||
@@ -321,9 +389,11 @@ pub(crate) fn suggest_in(d: &AffixDict, word: &str, max: usize, t: &Tuning) -> V
|
||||
add_splits(&mut col, &input, &t.split);
|
||||
add_key_neighbours(&mut col, &chars);
|
||||
add_typos(&mut col, &chars, t);
|
||||
col.compounds();
|
||||
// 6. Further off: scan the stems that start like the word.
|
||||
if col.count_within(COST_MISTYPE) < max {
|
||||
scan(&mut col, &chars, &map);
|
||||
col.compounds();
|
||||
}
|
||||
let mut found: Vec<(String, (u32, usize))> = col.found.into_iter().collect();
|
||||
// Among equal costs, the change nearest the word's start is the likelier intent
|
||||
@@ -526,8 +596,9 @@ fn scan(col: &mut Collector, chars: &[char], map: &[Vec<char>]) {
|
||||
best.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(b.1)));
|
||||
let related = |a: char, b: char| fold(a, map) == fold(b, map);
|
||||
let limit = 4;
|
||||
let mut budget = SCAN_FORMS_WORK;
|
||||
for (_, stem) in best {
|
||||
for form in d.forms(stem) {
|
||||
for form in d.forms_spending(stem, &mut budget) {
|
||||
let lf: Vec<char> = form.to_lowercase().chars().collect();
|
||||
if lf.len().abs_diff(chars.len()) > 2 {
|
||||
continue;
|
||||
@@ -653,7 +724,9 @@ mod tests {
|
||||
seq: 0,
|
||||
names: true,
|
||||
split_names: true,
|
||||
compound_checks: 0,
|
||||
deferred: Vec::new(),
|
||||
compound_work: crate::affix::MAX_SUGGEST_COMPOUND_WORK,
|
||||
pieces: Default::default(),
|
||||
};
|
||||
col.offer("cafe", COST_DROP_DIACRITIC);
|
||||
col.offer("cafe", COST_REP);
|
||||
|
||||
Reference in new issue
Block a user