From e36fb5d06951c5813beb3668d2e6c31db38c0aa5 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Sun, 1 Nov 2026 23:57:16 +0000 Subject: [PATCH 01/40] =?UTF-8?q?FU.1:=20craft-proof=20=E2=80=94=20British?= =?UTF-8?q?=20English=20never=20swaps=20-yze=20for=20-yse=20(reanalyze,=20?= =?UTF-8?q?breathalyzer,=20paralyzingly=20stay=20American,=20as=20Hunspell?= =?UTF-8?q?=20en=5FGB=20says)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FypgCX8LsYYE8fsj4byBm2 --- crates/proof/src/english.rs | 25 +++++++++++++++++++------ crates/proof/src/lib.rs | 16 ++++++++++++++++ 2 files changed, 35 insertions(+), 6 deletions(-) diff --git a/crates/proof/src/english.rs b/crates/proof/src/english.rs index 0c2c2e13..e818465f 100644 --- a/crates/proof/src/english.rs +++ b/crates/proof/src/english.rs @@ -348,11 +348,12 @@ pub(crate) fn is_correct(word: &str, lang: ProofLang, opts: &Options, policy: bo || (policy && !parts.is_empty() && parts.iter().all(|p| s.knows_lower(p) || crate::en_variants::known_variant(p, lang, |x| s.knows_lower(x)))) } -/// `w` with its `-ize`/`-yze` endings (and their forms: `-ization`, `-izing`…) spelled `-ise`/ -/// `-yse`, when it has any: British English takes both (Word accepts `organize` and `organise` in -/// en-GB), and SCOWL's en-GB dictionary has only `-ise`. A `z` between `i`/`y` and a vowel is -/// swapped; the `-ize` form counts only when American or Canadian English spells the word so -/// ([`ize_elsewhere`]), or `wize` would pass as `wise`. +/// `w` with its `-ize` endings (and their forms: `-ization`, `-izing`…) spelled `-ise`, when it +/// has any: British English takes both (Word accepts `organize` and `organise` in en-GB), and +/// SCOWL's en-GB dictionary has only `-ise`. A `z` between `i` and a vowel is swapped; the `-ize` +/// form counts only when American or Canadian English spells the word so ([`ize_elsewhere`]), or +/// `wize` would pass as `wise`. Never `-yze`: Oxford spelling has `analyse`, `paralyse`, so +/// `reanalyze` or `breathalyzer` is American in British English, as Hunspell's en_GB says. pub(crate) fn ise_spelling(w: &str) -> Option { let b = w.as_bytes(); let mut out = b.to_vec(); @@ -360,7 +361,7 @@ pub(crate) fn ise_spelling(w: &str) -> Option { for i in 1..b.len() { let after = b.get(i + 1).is_some_and(|c| b"aeiAEI".contains(c)); if after - && matches!(b.get(i - 1), Some(b'i' | b'y' | b'I' | b'Y')) + && matches!(b.get(i - 1), Some(b'i' | b'I')) && let Some(c) = out.get_mut(i) { match *c { @@ -512,6 +513,18 @@ fn list_suggest(word: &str, max: usize) -> Vec { mod tests { use super::*; + /// Review follow-up: only `-ize` is swapped for `-ise`, never `-yze` for `-yse` (Oxford + /// spelling has `analyse`), so `reanalyze`, `breathalyzer`, `paralyzingly` stay American. + #[test] + fn ise_spelling_swaps_ize_only() { + assert_eq!(ise_spelling("organize").as_deref(), Some("organise")); + assert_eq!(ise_spelling("Realizations").as_deref(), Some("Realisations")); + assert_eq!(ise_spelling("ORGANIZING").as_deref(), Some("ORGANISING")); + for w in ["reanalyze", "reanalyzed", "breathalyze", "breathalyzer", "paralyzingly", "catalyze", "Analyzer", "ANALYZE"] { + assert_eq!(ise_spelling(w), None, "{w}"); + } + } + #[test] fn built_in_list_is_the_whole_moby_list() { let d = words(); diff --git a/crates/proof/src/lib.rs b/crates/proof/src/lib.rs index 5b58768c..9c19999d 100644 --- a/crates/proof/src/lib.rs +++ b/crates/proof/src/lib.rs @@ -2465,6 +2465,22 @@ mod tests { assert_eq!(suggest_in("programmes", 5, us).first().map(String::as_str), Some("programs")); } + /// Review follow-up: British English takes `-ize` (Oxford spelling) but never `-yze`, also in + /// forms en-variants.txt doesn't list. Hunspell 1.7.5 with SCOWL's en_GB rejects these too. + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] + #[test] + fn british_english_rejects_unlisted_yze_forms() { + crate::hold_dictionary_tests(); + let gb = ProofLang::EnGb; + for w in ["reanalyze", "reanalyzed", "breathalyze", "breathalyzer", "paralyzingly", "catalyze", "analyzer"] { + assert!(!is_correct_in(w, gb), "en-GB: {w}"); + assert!(is_correct_in(w, ProofLang::En), "en-US: {w}"); + } + for w in ["reanalyse", "breathalyser", "paralysingly", "catalyse", "organize", "realization"] { + assert!(is_correct_in(w, gb), "en-GB: {w}"); + } + } + /// More British spellings that must stay right in en-GB and en-AU (#1281): -re, single l, ae, /// with their inflections. #[test] From 820695e95121f5b0959a0de6847b2f01812a1fb3 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Mon, 2 Nov 2026 00:00:22 +0000 Subject: [PATCH 02/40] =?UTF-8?q?FU.2:=20craft-proof=20=E2=80=94=20Austral?= =?UTF-8?q?ian=20English=20reads=20SCOWL's=20own=20en-AU=20dictionary=20(P?= =?UTF-8?q?roofLang::EnAu),=20with=20en-GB's=20spelling=20policy;=20en-NZ?= =?UTF-8?q?=20stays=20on=20en-GB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SCOWL en-AU has 238 words en-GB lacks (billabong, brumby, larrikinism, Australian place names) and spells program, inquiry, livable, licorice, verandah the Australian way (no programme, enquiry, liveable, dialog or veranda, as Hunspell en_AU says); -ise/-ize policy and affix file are the same as en-GB's. craft-i18n already ships language-data/en-au. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FypgCX8LsYYE8fsj4byBm2 --- crates/proof/README.md | 11 +++---- crates/proof/build.rs | 7 +++-- crates/proof/src/en_variants.rs | 5 ++-- crates/proof/src/english.rs | 29 +++++++++++++----- crates/proof/src/lang.rs | 53 ++++++++++++++------------------- crates/proof/src/lib.rs | 39 ++++++++++++++++++++---- crates/proof/src/no_data.rs | 14 ++++++--- crates/proof/src/packs.rs | 2 +- crates/proof/src/scowl.rs | 18 ++++++----- 9 files changed, 113 insertions(+), 65 deletions(-) diff --git a/crates/proof/README.md b/crates/proof/README.md index 49a16473..4aeb677b 100644 --- a/crates/proof/README.md +++ b/crates/proof/README.md @@ -13,7 +13,7 @@ craft-proof = { git = "https://github.com/storytold/craft-libs", rev = " | Language | Spelling | Grammar | Hyphenation | |---|---|---|---| -| English (en-US, en-CA, en-GB and the British-spelling Englishes such as en-AU, en-NZ) | SCOWL 2026 Hunspell dictionary per English + `en-extra.txt`, dialect policy (`en-variants.txt`); a dictionaries-folder English first; without SCOWL the installed Hunspell English, else the built-in Moby list ([below](#english-is-always-checked)) | yes | Moby list, our patterns, heuristic | +| English (en-US, en-CA, en-AU, en-GB and the British-spelling Englishes such as en-NZ) | SCOWL 2026 Hunspell dictionary per English + `en-extra.txt`, dialect policy (`en-variants.txt`); a dictionaries-folder English first; without SCOWL the installed Hunspell English, else the built-in Moby list ([below](#english-is-always-checked)) | yes | Moby list, our patterns, heuristic | | Polish, Portuguese, Italian | the installed Hunspell dictionary | yes | TeX patterns (embedded, or a Language pack) | | Croatian, Turkish, Russian, Czech, Indonesian, Spanish (es, es-MX, es-419, es-UY) | the installed Hunspell dictionary (Russian also `ru-extra.txt`) | yes | Turkish and Russian rules; English fallback otherwise | | German | — | — | TeX patterns (embedded, or a Language pack) | @@ -151,14 +151,15 @@ English spelling uses the first of these that is there; `english_source(lang)` s or in WordCraft's own `dictionaries` folder. The user's choice comes first. 2. **Installed English pack:** reserved for Language packs (craft-libs#485, CP4.2). 3. **Embedded SCOWL:** the English's own SCOWL `rel-2026.02.25` Hunspell dictionary from - `CRAFT_I18N_DIR` (`language-data/en-us`, `en-gb` (British `-ise`) or `en-ca`, `spelling.aff` + - `spelling.dic`), read by the affix engine. en-AU, en-NZ, en-IE and the other British-spelling - Englishes read en-GB; Canadian English reads en-CA. A word is right when the English's own + `CRAFT_I18N_DIR` (`language-data/en-us`, `en-gb` (British `-ise`), `en-au` or `en-ca`, + `spelling.aff` + `spelling.dic`), read by the affix engine. Australian English reads en-AU + (Australian words such as `billabong`, with en-GB's spelling policy); en-NZ, en-IE and the + other British-spelling Englishes read en-GB; Canadian English reads en-CA. A word is right when the English's own dictionary knows it (or `en-extra.txt` does), never because another English's does (`colourblind` is wrong in en-US, `colorblind` in en-GB); suggestions come from the same dictionary. SCOWL's size-60 lists keep one spelling per dialect, and Word accepts more, so British English also takes the `-ize` spellings of its `-ise` words that American or Canadian - English spell so (`organize`, `realization`; never `wize` or `advize`), and `en-variants.txt` flags the other dialect's spellings and accepts this + English spell so (`organize`, `realization`; never `wize` or `advize`, and never `-yze`: `reanalyze` stays American), and `en-variants.txt` flags the other dialect's spellings and accepts this dialect's other spellings of known words (`judgement` in en-US). English with no dialect (`is_correct`) takes a word any of the three knows. Suggestions offer a missing space only between two lower-case words of three letters or more, after every one-word candidate, and no diff --git a/crates/proof/build.rs b/crates/proof/build.rs index 8ff77fc4..82f5f87f 100644 --- a/crates/proof/build.rs +++ b/crates/proof/build.rs @@ -16,9 +16,8 @@ //! `include_str!` takes a string, so a `CRAFT_I18N_DIR` that isn't valid UTF-8 can't be embedded: //! it is treated like a folder without the data. //! -//! English: SCOWL's en-US, en-GB (-ise) and en-CA dictionaries; en-AU is not embedded, as -//! Australian English is checked as British-spelling English (`ProofLang::EnGb`). The SCOWL -//! notice is embedded once (the four upstream READMEs differ only in their title and build line). +//! English: SCOWL's en-US, en-GB (-ise), en-CA and en-AU dictionaries. The SCOWL notice is +//! embedded once (the four upstream READMEs differ only in their title and build line). use std::fmt::Write as _; use std::path::{Path, PathBuf}; @@ -34,6 +33,8 @@ const FILES: &[(&str, &str, bool)] = &[ ("EN_GB_SPELLING", "en-gb/spelling.dic", false), ("EN_CA_AFF", "en-ca/spelling.aff", false), ("EN_CA_SPELLING", "en-ca/spelling.dic", false), + ("EN_AU_AFF", "en-au/spelling.aff", false), + ("EN_AU_SPELLING", "en-au/spelling.dic", false), ("SCOWL_NOTICE", "en-us/LICENCE-SCOWL.txt", false), ("EN_US_PAT", "en-us/hyphenation.pat", false), ("EXTRA_EN", "en-us/en-extra.txt", false), diff --git a/crates/proof/src/en_variants.rs b/crates/proof/src/en_variants.rs index 07c16bb1..c8a63fd8 100644 --- a/crates/proof/src/en_variants.rs +++ b/crates/proof/src/en_variants.rs @@ -1,5 +1,5 @@ //! American and British spellings of the same English word (#1281, #1045): which forms each -//! English ([`ProofLang::En`], [`ProofLang::EnCa`], [`ProofLang::EnGb`]) accepts, which it flags, +//! English ([`ProofLang::En`], [`ProofLang::EnCa`], [`ProofLang::EnGb`] and [`ProofLang::EnAu`]) accepts, which it flags, //! and what it suggests instead (`colour` → `color` in American English). //! //! The pairs come from craft-i18n's `language-data/en-us/en-variants.txt`, our own list of base forms with a @@ -77,7 +77,8 @@ fn column(lang: ProofLang) -> Option { match lang { ProofLang::En => Some(0), ProofLang::EnCa => Some(1), - ProofLang::EnGb => Some(2), + // Australian English follows the British policy (`colour`, `centre`, `organise`). + ProofLang::EnGb | ProofLang::EnAu => Some(2), _ => None, } } diff --git a/crates/proof/src/english.rs b/crates/proof/src/english.rs index e818465f..6617b1fb 100644 --- a/crates/proof/src/english.rs +++ b/crates/proof/src/english.rs @@ -4,7 +4,7 @@ //! user keeps for WordCraft ([`crate::hunspell::user_dirs`]: the folder set in the options, //! `WORDCRAFT_DICTIONARIES`, WordCraft's own `dictionaries` folder). //! 2. **Installed English pack**: reserved for Language packs (craft-libs#485, CP4.2). -//! 3. **Embedded SCOWL**: the run's English's SCOWL dictionary (en-US, en-GB or en-CA) embedded +//! 3. **Embedded SCOWL**: the run's English's SCOWL dictionary (en-US, en-GB, en-AU or en-CA) embedded //! from craft-i18n at build time ([`crate::scowl`]); a word it knows (British English: also //! the `-ize` spelling of one it knows, [`ise_spelling`]), or the supplement ([`EXTRA`]), is //! right; never one only another English knows. When it is there, the system locations aren't @@ -18,8 +18,9 @@ //! embedded from craft-i18n is the same list; it is used for hyphenation only.) //! //! Hunspell dictionaries are found by [`crate::hunspell`] by their own English's names only -//! (`en_US` for American, `en_GB` for British, `en_CA` for Canadian English), each name in every -//! folder before the next name. Another English's dictionary is never used: it would mark this +//! (`en_US` for American, `en_GB` for British, `en_CA` for Canadian English; `en_AU`, then +//! `en_GB`, for Australian), each name in every folder before the next name. Another English's +//! dictionary is never used: it would mark this //! English's spellings (`colour` in an `en_US` one). A file that can't be read is skipped for the //! next name or folder, and one file is read once for every English that uses it. `COMPOUNDRULE` //! (ordinals) and `OCONV` (suggestions are written as the dictionary says: `don’t`) are read; @@ -90,15 +91,22 @@ fn names(lang: ProofLang) -> &'static [&'static str] { match lang { ProofLang::EnGb => &["en_GB", "en-GB"], ProofLang::EnCa => &["en_CA", "en-CA"], + // Australian English's own, else a British one: the British-spelling dictionary it read + // before it had its own (a system en_GB on a build without SCOWL). + ProofLang::EnAu => &["en_AU", "en-AU", "en_GB", "en-GB"], _ => &["en_US", "en-US"], } } +/// How many Englishes have a slot of their own ([`slot`]). +const ENGLISHES: usize = 4; + /// The slot of each English in [`State::slots`]. fn slot(lang: ProofLang) -> usize { match lang { ProofLang::EnGb => 1, ProofLang::EnCa => 2, + ProofLang::EnAu => 3, _ => 0, } } @@ -109,7 +117,7 @@ type Loaded = Option<(Arc, PathBuf)>; /// The English Hunspell dictionaries, under one lock. struct State { /// Per English (see [`slot`]): `None` until first use, then what was found. - slots: [Option; 3], + slots: [Option; ENGLISHES], /// Every `.dic` file read, so Englishes that use the same file share one copy. files: Vec<(PathBuf, Arc)>, /// Counts reloads, so a load that started before one is not kept. @@ -118,7 +126,7 @@ struct State { impl State { const fn new() -> Self { - State { slots: [None, None, None], files: Vec::new(), reloads: 0 } + State { slots: [None, None, None, None], files: Vec::new(), reloads: 0 } } } @@ -181,7 +189,7 @@ fn already_read(state: &RwLock, dic: &Path) -> Option> { /// Forget what `state` holds, so the next check looks again. fn forget(state: &RwLock) { let mut s = state.write().unwrap_or_else(PoisonError::into_inner); - s.slots = [None, None, None]; + s.slots = [None, None, None, None]; s.files.clear(); s.reloads = s.reloads.wrapping_add(1); } @@ -240,7 +248,7 @@ fn note_error(lang: ProofLang, error: Option<(PathBuf, String)>) { /// folder changed). pub(crate) fn reload() { forget(&HUNSPELL); - for lang in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa] { + for lang in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa, ProofLang::EnAu] { crate::hunspell::clear_load_error(key(lang)); } } @@ -609,7 +617,7 @@ mod tests { fn lookup_order() { let tiny = Arc::new(AffixDict::from_hunspell("", "1\nzzyzx\n", &[]).unwrap()); let file = PathBuf::from("/folder/en_US.dic"); - for lang in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa] { + for lang in [ProofLang::En, ProofLang::EnGb, ProofLang::EnAu, ProofLang::EnCa] { assert!(matches!(pick(Some((tiny.clone(), file.clone())), lang), Speller::Hunspell(_, p) if p == file), "{lang:?}"); let fallback = pick(None, lang); if crate::scowl::dict(lang).is_some() { @@ -638,6 +646,11 @@ mod tests { assert_eq!(found(names(ProofLang::EnGb)), ["en_GB"], "a user's en_US doesn't stand in for SCOWL's en-GB"); assert_eq!(found(names(ProofLang::EnCa)), ["en_CA"]); assert_eq!(found(names(ProofLang::En)), ["en_US"]); + // Australian English: its own, else the British one it read before it had its own. + assert_eq!(found(names(ProofLang::EnAu)), ["en_GB"]); + std::fs::write(dir.join("en_AU.aff"), "SET UTF-8\n").unwrap(); + std::fs::write(dir.join("en_AU.dic"), "1\nzzyzx\n").unwrap(); + assert_eq!(found(names(ProofLang::EnAu)), ["en_AU", "en_GB"]); let _ = std::fs::remove_dir_all(&dir); } diff --git a/crates/proof/src/lang.rs b/crates/proof/src/lang.rs index 993e4030..463406a9 100644 --- a/crates/proof/src/lang.rs +++ b/crates/proof/src/lang.rs @@ -5,7 +5,7 @@ /// A language WordCraft can proof, or [`ProofLang::Other`] for one it can't. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, PartialOrd, Ord)] pub enum ProofLang { - /// American English (`en`, `en-US`, and every region not listed for [`ProofLang::EnGb`] or + /// American English (`en`, `en-US`, and every region not listed for [`ProofLang::EnGb`], [`ProofLang::EnAu`] or /// [`ProofLang::EnCa`]): SCOWL's en-US dictionary ([`crate::scowl`]), the English grammar rules and the /// English hyphenation dictionary and patterns. British spellings (`colour`, `centre`, /// `organise`) are flagged, with the American form suggested ([`crate::en_variants`]). Also @@ -13,10 +13,14 @@ pub enum ProofLang { /// Word's). #[default] En, - /// English with British spelling (`en-GB`, `en-AU`, `en-NZ`, `en-IE`, `en-ZA`, `en-IN`…): + /// English with British spelling (`en-GB`, `en-NZ`, `en-IE`, `en-ZA`, `en-IN`…): /// [`ProofLang::En`]'s tools, but American spellings (`color`, `center`, `analyze`) are /// flagged and `-ize` and `-ise` are both right, as in Oxford spelling. EnGb, + /// Australian English (`en-AU`): [`ProofLang::EnGb`]'s spelling policy, against SCOWL's own + /// en-AU dictionary, which knows Australian words (`billabong`, `brumby`, `larrikin`) and + /// spells some words the Australian way (`program`, `inquiry`). + EnAu, /// Canadian English (`en-CA`): British `-our`, `-re` and `-ence` (`colour`, `centre`, /// `defence`) with American `-ize` (`organize`), `tire` and `program`. EnCa, @@ -86,9 +90,10 @@ pub enum ProofLang { impl ProofLang { /// Every language with a spelling dictionary, in display order. - pub const PROOFED: [ProofLang; 15] = [ + pub const PROOFED: [ProofLang; 16] = [ ProofLang::En, ProofLang::EnGb, + ProofLang::EnAu, ProofLang::EnCa, ProofLang::Pl, ProofLang::Pt, @@ -155,12 +160,13 @@ impl ProofLang { } } - /// The English of an English tag, by its [`region`]: Canada is [`ProofLang::EnCa`], regions - /// that spell the British way are [`ProofLang::EnGb`], and the United States, any other + /// The English of an English tag, by its [`region`]: Canada is [`ProofLang::EnCa`], Australia + /// [`ProofLang::EnAu`], other regions that spell the British way are [`ProofLang::EnGb`], and the United States, any other /// region and no region are [`ProofLang::En`]. fn english(tag: &str) -> ProofLang { match region(tag) { Some(r) if r.eq_ignore_ascii_case("CA") => ProofLang::EnCa, + Some(r) if r.eq_ignore_ascii_case("AU") => ProofLang::EnAu, Some(r) if BRITISH_SPELLING.iter().any(|b| b.eq_ignore_ascii_case(r)) => ProofLang::EnGb, _ => ProofLang::En, } @@ -189,9 +195,10 @@ impl ProofLang { } } - /// Is this one of the Englishes ([`ProofLang::En`], [`ProofLang::EnGb`], [`ProofLang::EnCa`])? + /// Is this one of the Englishes ([`ProofLang::En`], [`ProofLang::EnGb`], [`ProofLang::EnAu`], + /// [`ProofLang::EnCa`])? pub fn is_english(self) -> bool { - matches!(self, ProofLang::En | ProofLang::EnGb | ProofLang::EnCa) + matches!(self, ProofLang::En | ProofLang::EnGb | ProofLang::EnAu | ProofLang::EnCa) } /// Is this one of the Spanishes ([`ProofLang::Es`], [`ProofLang::EsMx`], [`ProofLang::Es419`], @@ -208,7 +215,7 @@ impl ProofLang { pub fn code(self) -> &'static str { match self { ProofLang::Pack(_) => "", - ProofLang::En | ProofLang::EnGb | ProofLang::EnCa => "en", + ProofLang::En | ProofLang::EnGb | ProofLang::EnAu | ProofLang::EnCa => "en", ProofLang::Pl => "pl", ProofLang::Bg => "bg", ProofLang::Pt => "pt", @@ -371,12 +378,12 @@ pub(crate) fn region(tag: &str) -> Option<&str> { const LATIN_AMERICA: [&str; 22] = ["419", "005", "013", "029", "AR", "BO", "CL", "CO", "CR", "CU", "DO", "EC", "GT", "HN", "NI", "PA", "PE", "PR", "PY", "SV", "US", "VE"]; -/// Regions whose English uses British spelling ([`ProofLang::EnGb`]): the United Kingdom (and -/// the non-standard `UK`), Ireland, Australia, New Zealand, South Africa, India and other +/// Regions whose English uses British spelling ([`ProofLang::EnGb`]; Australia has its own, +/// [`ProofLang::EnAu`]): the United Kingdom (and the non-standard `UK`), Ireland, New Zealand, South Africa, India and other /// Commonwealth countries, the Crown dependencies and Europe (`150`). -const BRITISH_SPELLING: [&str; 26] = [ - "GB", "UK", "IE", "AU", "NZ", "ZA", "IN", "SG", "HK", "MY", "MT", "CY", "GI", "IM", "JE", "GG", "JM", "TT", "BS", "BB", "BZ", "KE", "NG", "GH", - "PK", "150", +const BRITISH_SPELLING: [&str; 25] = [ + "GB", "UK", "IE", "NZ", "ZA", "IN", "SG", "HK", "MY", "MT", "CY", "GI", "IM", "JE", "GG", "JM", "TT", "BS", "BB", "BZ", "KE", "NG", "GH", "PK", + "150", ]; /// A BCP 47 tag in canonical case (`pl-PL`, `en-GB`, `zh-Hant-TW`, `x-none`), or `None` when @@ -469,26 +476,12 @@ mod tests { fn tags_resolve_english_regions() { assert_tags(&["en", "en-US", "EN_us", "en-Latn-US", "en-PH", "en-001", "eng", "en_US.UTF-8", "en-x-gb"], ProofLang::En); assert_tags( - &[ - "en-GB", - "EN_gb", - "en_GB.UTF-8", - "en-AU", - "en_AU", - "en-NZ", - "en-IE", - "en-ZA", - "en-IN", - "en-Latn-GB", - "en-UK", - "en-150", - "eng-GB", - "en_GB@euro", - ], + &["en-GB", "EN_gb", "en_GB.UTF-8", "en-NZ", "en-IE", "en-ZA", "en-IN", "en-Latn-GB", "en-UK", "en-150", "eng-GB", "en_GB@euro"], ProofLang::EnGb, ); assert_tags(&["en-CA", "en_CA", "EN-ca", "en_CA.UTF-8", "en-Latn-CA"], ProofLang::EnCa); - for language in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa] { + assert_tags(&["en-AU", "en_AU", "EN-au", "en_AU.UTF-8", "en-Latn-AU"], ProofLang::EnAu); + for language in [ProofLang::En, ProofLang::EnGb, ProofLang::EnAu, ProofLang::EnCa] { assert!(language.is_english() && language.is_proofed() && ProofLang::PROOFED.contains(&language), "{language:?}"); assert_eq!(language.code(), "en"); } diff --git a/crates/proof/src/lib.rs b/crates/proof/src/lib.rs index 9c19999d..d10708cf 100644 --- a/crates/proof/src/lib.rs +++ b/crates/proof/src/lib.rs @@ -340,7 +340,7 @@ pub fn installed_dictionaries() -> Vec<(ProofLang, Option)> /// checked yet reports `None` (#2830). For an English, its embedded SCOWL dictionary or the /// English Hunspell dictionary that failed; English then uses the next source ([`english_source`]). pub fn dictionary_load_error(lang: ProofLang) -> Option { - if matches!(lang, ProofLang::En | ProofLang::EnGb | ProofLang::EnCa) { + if lang.is_english() { return english::load_error(lang); } if let Some(l) = lang.spelling_locale() { @@ -953,7 +953,7 @@ pub fn check_grammar_in(text: &str, lang: ProofLang) -> Vec { fn grammar_in(text: &str, lang: ProofLang, capital_first: bool) -> Vec { match lang { ProofLang::En => grammar_en(text, capital_first), - ProofLang::EnGb | ProofLang::EnCa => grammar_en(text, capital_first).into_iter().map(|i| Issue { lang, ..i }).collect(), + ProofLang::EnGb | ProofLang::EnAu | ProofLang::EnCa => grammar_en(text, capital_first).into_iter().map(|i| Issue { lang, ..i }).collect(), // `pt-pt` shares Portuguese's rules, its issues in its own language. _ => language(lang.built_in()) .map_or_else(Vec::new, |l| (l.grammar)(text, lang, capital_first).into_iter().map(|i| Issue { lang, ..i }).collect()), @@ -1197,7 +1197,7 @@ mod tests { } // Every proofed language other than English has a Hunspell dictionary to report. for l in ProofLang::PROOFED { - let english = matches!(l, ProofLang::En | ProofLang::EnGb | ProofLang::EnCa); + let english = l.is_english(); assert_eq!(dictionary_needed(l, false).is_some(), !english, "{l:?}"); } for l in [ProofLang::En, ProofLang::Other] { @@ -2481,6 +2481,33 @@ mod tests { } } + /// Review follow-up: Australian English reads SCOWL's own en-AU dictionary (craft-i18n ships + /// it), not en-GB's: its Australian words are right and spelled as Hunspell's en_AU spells + /// them; British English doesn't take them. The British policy still applies: American + /// spellings are wrong, `-ise` and `-ize` both right. + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] + #[test] + fn australian_english_reads_its_own_scowl() { + crate::hold_dictionary_tests(); + let (au, gb) = (ProofLang::EnAu, ProofLang::EnGb); + assert!(scowl::is_british(au)); + assert_eq!(english_source(au), Some(EnglishSource::Embedded)); + for w in ["billabong", "brumby", "larrikinism", "bunyip", "corroboree", "Bundaberg", "Rockhampton"] { + assert!(is_correct_in(w, au), "en-AU: {w}"); + assert!(!is_correct_in(w, gb), "en-GB: {w}"); + assert!(is_correct_for(w, Some("en-AU")), "en-AU tag: {w}"); + } + for w in ["colour", "centre", "organise", "organize", "analyse", "travelled", "defence"] { + assert!(is_correct_in(w, au), "en-AU: {w}"); + } + for w in ["color", "center", "analyze", "reanalyze", "defense"] { + assert!(!is_correct_in(w, au), "en-AU: {w}"); + } + let text = "The larrikin saw a brumby by the billabong and analysed the colour of the sky."; + assert!(check_spelling_in(text, au).is_empty(), "{:?}", check_spelling_in(text, au)); + assert_eq!(check_spelling_in(text, gb).len(), 3, "larrikin, brumby, billabong: {:?}", check_spelling_in(text, gb)); + } + /// More British spellings that must stay right in en-GB and en-AU (#1281): -re, single l, ae, /// with their inflections. #[test] @@ -2488,7 +2515,7 @@ mod tests { // English results depend on the dictionaries other tests install. crate::hold_dictionary_tests(); let gb = ProofLang::EnGb; - assert_eq!(ProofLang::from_tag(Some("en-AU")), gb); + assert_eq!(ProofLang::from_tag(Some("en-AU")), ProofLang::EnAu); for w in [ "goitre", "goitres", @@ -2513,6 +2540,7 @@ mod tests { "encyclopaedias", ] { assert!(is_correct_in(w, gb), "en-GB: {w}"); + assert!(is_correct_in(w, ProofLang::EnAu), "en-AU: {w}"); } let text = "The wilful knight took the sceptre to fulfil his enrolment, paid the instalment and read the encyclopaedia."; assert!(check_spelling_in(text, gb).is_empty(), "{:?}", check_spelling_in(text, gb)); @@ -2529,7 +2557,8 @@ mod tests { assert_eq!(english_source(l), Some(EnglishSource::Embedded)); assert_eq!(dictionary_load_error(l), None); } - assert!(std::ptr::eq(scowl::dict(ProofLang::EnGb).unwrap(), scowl::dict(ProofLang::from_tag(Some("en-AU"))).unwrap()), "en-AU reads en-GB"); + assert!(!std::ptr::eq(scowl::dict(ProofLang::EnGb).unwrap(), scowl::dict(ProofLang::EnAu).unwrap()), "en-AU reads its own"); + assert!(std::ptr::eq(scowl::dict(ProofLang::EnGb).unwrap(), scowl::dict(ProofLang::from_tag(Some("en-NZ"))).unwrap()), "en-NZ reads en-GB"); let notice = scowl::notice().unwrap(); for credit in ["Kevin Atkinson", "Geoff Kuenning", "clause 4 removed", "Benjamin Titze"] { assert!(notice.contains(credit), "{credit}"); diff --git a/crates/proof/src/no_data.rs b/crates/proof/src/no_data.rs index 2f429985..cdd9025d 100644 --- a/crates/proof/src/no_data.rs +++ b/crates/proof/src/no_data.rs @@ -7,9 +7,10 @@ use crate::hyphen::{self, Limits, Source}; use crate::{EnglishSource, Options, ProofLang, dict, en_variants, patterns}; -const LANGS: [ProofLang; 18] = [ +const LANGS: [ProofLang; 19] = [ ProofLang::En, ProofLang::EnGb, + ProofLang::EnAu, ProofLang::EnCa, ProofLang::Pl, ProofLang::Bg, @@ -45,9 +46,14 @@ const SAMPLES: [&str; 10] = [ #[test] fn nothing_is_embedded() { assert!(crate::data::EN_US_DIC.is_empty()); - assert!(crate::data::EN_US_SPELLING.is_empty() && crate::data::EN_GB_SPELLING.is_empty() && crate::data::EN_CA_SPELLING.is_empty()); + assert!( + crate::data::EN_US_SPELLING.is_empty() + && crate::data::EN_GB_SPELLING.is_empty() + && crate::data::EN_CA_SPELLING.is_empty() + && crate::data::EN_AU_SPELLING.is_empty() + ); assert!(crate::scowl::notice().is_none()); - for l in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa] { + for l in [ProofLang::En, ProofLang::EnGb, ProofLang::EnAu, ProofLang::EnCa] { assert!(crate::scowl::dict(l).is_none() && crate::scowl::load_error(l).is_none()); } assert!(dict::Dictionary::en_us().is_empty()); @@ -63,7 +69,7 @@ fn nothing_is_embedded() { #[test] fn english_spelling_still_marks_misspellings() { crate::hold_dictionary_tests(); - for l in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa] { + for l in [ProofLang::En, ProofLang::EnGb, ProofLang::EnAu, ProofLang::EnCa] { let source = crate::english_source(l); assert!(matches!(source, Some(EnglishSource::Hunspell(_) | EnglishSource::CoreList)), "{l:?}"); // Any English dictionary marks junk and knows `the`; the word-by-word results below are diff --git a/crates/proof/src/packs.rs b/crates/proof/src/packs.rs index 2fb291e9..7424cd11 100644 --- a/crates/proof/src/packs.rs +++ b/crates/proof/src/packs.rs @@ -531,7 +531,7 @@ fn hyphenation_locales(lang: ProofLang) -> Vec { ProofLang::EsUy => &["es-uy", "es-419", "es"], ProofLang::De => &["de", "de-1996", "de-de"], ProofLang::Pack(l) => return crate::pack_lang::hyphenation_locales(l), - ProofLang::En | ProofLang::EnGb | ProofLang::EnCa | ProofLang::Other => &[], + ProofLang::En | ProofLang::EnGb | ProofLang::EnAu | ProofLang::EnCa | ProofLang::Other => &[], }; names.iter().map(|n| (*n).to_string()).collect() } diff --git a/crates/proof/src/scowl.rs b/crates/proof/src/scowl.rs index 9a71188f..e9f41b9e 100644 --- a/crates/proof/src/scowl.rs +++ b/crates/proof/src/scowl.rs @@ -2,8 +2,11 @@ //! dictionaries of craft-i18n's `language-data/en-us`, `en-gb` (British `-ise`) and `en-ca`, //! embedded unmodified at build time and read by the affix engine ([`crate::affix`]) on first use. //! -//! Each English has its own: American English reads en-US, the British-spelling Englishes -//! (en-GB, en-AU, en-NZ, en-IE…) read en-GB and Canadian English reads en-CA. Its suggestions +//! Each English has its own: American English reads en-US, Australian English en-AU (SCOWL's +//! Australian words: `billabong`, `brumby`; and `program`, `inquiry`), the other British-spelling +//! Englishes (en-GB, en-NZ, en-IE…) read en-GB and Canadian English reads en-CA. SCOWL has no +//! en-NZ: New Zealand English keeps `programme` and has few words of its own in SCOWL's en-AU +//! data, so it stays on en-GB. Its suggestions //! come from its own dictionary, and a word is right when its own dictionary knows it: the other //! dialects' dictionaries are never asked (en-US would take `colourblind` from en-GB's). SCOWL //! picks one spelling per dialect (en-GB `-ise` has no `organize`, en-US no `judgement`), and Word @@ -19,16 +22,17 @@ use std::sync::OnceLock; use crate::ProofLang; use crate::affix::AffixDict; -use crate::data::{EN_CA_AFF, EN_CA_SPELLING, EN_GB_AFF, EN_GB_SPELLING, EN_US_AFF, EN_US_SPELLING, SCOWL_NOTICE}; +use crate::data::{EN_AU_AFF, EN_AU_SPELLING, EN_CA_AFF, EN_CA_SPELLING, EN_GB_AFF, EN_GB_SPELLING, EN_US_AFF, EN_US_SPELLING, SCOWL_NOTICE}; -/// Per English (American, British, Canadian): the loaded dictionary, or why it isn't there. -static DICTS: [OnceLock>; 3] = [OnceLock::new(), OnceLock::new(), OnceLock::new()]; +/// Per English (American, British, Canadian, Australian): the loaded dictionary, or why it isn't there. +static DICTS: [OnceLock>; 4] = [OnceLock::new(), OnceLock::new(), OnceLock::new(), OnceLock::new()]; /// The slot, `.aff` and `.dic` text of `lang`'s dictionary. fn source(lang: ProofLang) -> (usize, &'static str, &'static str) { match lang { ProofLang::EnGb => (1, EN_GB_AFF, EN_GB_SPELLING), ProofLang::EnCa => (2, EN_CA_AFF, EN_CA_SPELLING), + ProofLang::EnAu => (3, EN_AU_AFF, EN_AU_SPELLING), _ => (0, EN_US_AFF, EN_US_SPELLING), } } @@ -50,9 +54,9 @@ pub(crate) fn dict(lang: ProofLang) -> Option<&'static AffixDict> { loaded.as_ref().ok() } -/// Does `lang` read the British `-ise` dictionary (en-GB, en-AU, en-NZ…)? +/// Does `lang` read a British `-ise` dictionary (en-GB, and en-NZ… through it; en-AU)? pub(crate) fn is_british(lang: ProofLang) -> bool { - source(lang).0 == 1 + matches!(source(lang).0, 1 | 3) } /// Why `lang`'s embedded SCOWL dictionary failed to load, once something tried to load it. From 87defe169f90289ce074aaa2fabd94e301f9a0f0 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Mon, 2 Nov 2026 00:04:07 +0000 Subject: [PATCH 03/40] =?UTF-8?q?FU.3:=20craft-proof=20pack=20languages=20?= =?UTF-8?q?=E2=80=94=20missing-dictionary=20list=20forgets=20its=20oldest,?= =?UTF-8?q?=20not=20all;=20a=20panic=20inside=20a=20check=20unpins=20its?= =?UTF-8?q?=20dictionaries;=20a=20word=20in=20another=20script=20than=20th?= =?UTF-8?q?e=20dictionary's=20isn't=20marked?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-ups from craft-libs#485: `CHECK_FROM` is reset by a drop guard; `remember_missing` drops the oldest of 256; a word none of whose letters the dictionary's words use (a Cyrillic word in a Dutch run) is left alone, as Word does. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FypgCX8LsYYE8fsj4byBm2 --- crates/proof/src/lib.rs | 24 +++++++ crates/proof/src/pack_lang.rs | 130 +++++++++++++++++++++++++++++----- 2 files changed, 137 insertions(+), 17 deletions(-) diff --git a/crates/proof/src/lib.rs b/crates/proof/src/lib.rs index d10708cf..4cfb9a5d 100644 --- a/crates/proof/src/lib.rs +++ b/crates/proof/src/lib.rs @@ -1709,6 +1709,30 @@ mod tests { let _ = std::fs::remove_dir_all(&root); } + /// Review follow-up (craft-libs#485): a word in another script than a pack language's + /// dictionary (a Russian name in a Dutch run, a Dutch word in a Greek one) isn't marked; a + /// misspelling in the dictionary's own script still is. + #[test] + fn pack_languages_leave_other_scripts_alone() { + crate::hold_dictionary_tests(); + set_dictionaries_folder(Some(shared_test_folder())); + let root = std::env::temp_dir().join(format!("wc-proof-other-script-packs-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&root); + write_pack(&root, "nl", &[("spelling.aff", b"SET UTF-8\n"), ("spelling.dic", "2\nhuis\nkat\n".as_bytes())]); + write_pack(&root, "el", &[("spelling.aff", b"SET UTF-8\n"), ("spelling.dic", "1\nσπίτι\n".as_bytes())]); + packs::set_packs_dir(Some(root.clone())); + let (nl, el) = (ProofLang::from_tag(Some("nl-NL")), ProofLang::from_tag(Some("el-GR"))); + assert!(is_correct_in("huis", nl) && !is_correct_in("huiss", nl)); + assert!(is_correct_in("Пушкин", nl) && is_correct_in("σπίτι", nl)); + assert!(is_correct_in("σπίτι", el) && !is_correct_in("σπίτιι", el) && is_correct_in("huis", el)); + let text = "Пушкин huiss"; + let issues = check_text(text, &[(0..text.len(), nl)]); + assert_eq!(issues.iter().map(|i| &text[i.start..i.end]).collect::>(), ["huiss"], "{issues:?}"); + packs::set_packs_dir(None); + set_dictionaries_folder(Some(shared_test_folder())); + let _ = std::fs::remove_dir_all(&root); + } + /// The `.dic` file `lang`'s spelling is read from with the installed packs and `system` as the /// system's only dictionary folder. fn chosen_dictionary(lang: ProofLang, system: &std::path::Path) -> Option { diff --git a/crates/proof/src/pack_lang.rs b/crates/proof/src/pack_lang.rs index 6c116653..56e8d090 100644 --- a/crates/proof/src/pack_lang.rs +++ b/crates/proof/src/pack_lang.rs @@ -6,6 +6,9 @@ //! files); the one used least recently goes first, but never one used during the current //! [`in_one_check`] call, so a paragraph in many languages loads each dictionary once. Locales //! with no dictionary are remembered apart and take no room. +//! +//! A word none of whose letters the dictionary's words use (a Cyrillic word in a Dutch run) isn't +//! this language's: it is never marked, as Word leaves a word in another script alone. use std::cell::Cell; use std::path::PathBuf; @@ -20,8 +23,10 @@ use crate::lang::PackLocale; pub(crate) const MAX_LOADED_BYTES: u64 = 256 << 20; /// Estimated memory per byte of `.aff` and `.dic` file. const BYTES_PER_FILE_BYTE: u64 = 6; -/// Most locales remembered as having no dictionary (the list is cleared when full). +/// Most locales remembered as having no dictionary (the oldest is forgotten first when full). const MAX_MISSING: usize = 256; +/// Most distinct letters kept in a dictionary's alphabet ([`alphabet`]). +const MAX_ALPHABET: usize = 4096; /// The budget in force (tests lower it). static BUDGET: AtomicU64 = AtomicU64::new(MAX_LOADED_BYTES); @@ -36,6 +41,8 @@ thread_local! { struct Loaded { locale: PackLocale, dict: Arc, + /// The lowercase letters its words use ([`alphabet`]), sorted. + alphabet: Arc<[char]>, /// The `.dic` it came from. file: Option, /// Estimated memory. @@ -66,15 +73,48 @@ pub(crate) fn reload() { /// over the budget. pub(crate) fn in_one_check(f: impl FnOnce() -> R) -> R { let outer = CHECK_FROM.with(Cell::get); - if outer.is_none() { + let _end = outer.is_none().then(|| { let now = CACHE.read().unwrap_or_else(PoisonError::into_inner).clock; CHECK_FROM.with(|c| c.set(Some(now.wrapping_add(1)))); - } - let r = f(); - if outer.is_none() { + EndCheck + }); + f() +} + +/// Ends the outermost [`in_one_check`] when dropped: also when `f` panics (`Session::run` catches +/// it), or every dictionary would stay pinned on this thread and the budget never hold again. +struct EndCheck; + +impl Drop for EndCheck { + fn drop(&mut self) { CHECK_FROM.with(|c| c.set(None)); } - r +} + +/// Remember that `locale` has no dictionary; when [`MAX_MISSING`] are remembered, the oldest is +/// forgotten (clearing them all would look every other one up again). +fn remember_missing(missing: &mut Vec, locale: PackLocale) { + if missing.len() >= MAX_MISSING { + missing.remove(0); + } + missing.push(locale); +} + +/// The lowercase letters the words of `d` use, sorted (at most [`MAX_ALPHABET`]). +fn alphabet(d: &AffixDict) -> Arc<[char]> { + let mut seen = std::collections::BTreeSet::new(); + for c in d.stems().flat_map(str::chars).filter(|c| c.is_alphabetic()).flat_map(char::to_lowercase) { + if seen.len() >= MAX_ALPHABET { + break; + } + seen.insert(c); + } + seen.into_iter().collect() +} + +/// Is `word` in another script than `alphabet`'s words: none of its letters is in it? +fn other_script(word: &str, alphabet: &[char]) -> bool { + !alphabet.is_empty() && !word.chars().filter(|c| c.is_alphabetic()).flat_map(char::to_lowercase).any(|c| alphabet.binary_search(&c).is_ok()) } #[cfg(test)] @@ -149,30 +189,34 @@ pub(crate) fn names(locale: PackLocale) -> Vec { /// `locale`'s dictionary, loaded on first use; an empty one when there is none. fn get(locale: PackLocale) -> (Arc, Option) { + let (d, file, _) = get_with_alphabet(locale); + (d, file) +} + +/// [`get`], with the dictionary's [`alphabet`]. +fn get_with_alphabet(locale: PackLocale) -> (Arc, Option, Arc<[char]>) { let reloads = { let mut c = CACHE.write().unwrap_or_else(PoisonError::into_inner); c.clock = c.clock.wrapping_add(1); let now = c.clock; if let Some(l) = c.loaded.iter_mut().find(|l| l.locale == locale) { l.used = now; - return (l.dict.clone(), l.file.clone()); + return (l.dict.clone(), l.file.clone(), l.alphabet.clone()); } if c.missing.contains(&locale) { - return (Arc::default(), None); + return (Arc::default(), None, Arc::from([])); } c.reloads }; let (dict, file, bytes) = load(locale); + let alphabet = alphabet(&dict); let mut c = CACHE.write().unwrap_or_else(PoisonError::into_inner); if c.reloads != reloads || c.loaded.iter().any(|l| l.locale == locale) || c.missing.contains(&locale) { - return (dict, file); + return (dict, file, alphabet); } if dict.is_empty() { - if c.missing.len() >= MAX_MISSING { - c.missing.clear(); - } - c.missing.push(locale); - return (dict, file); + remember_missing(&mut c.missing, locale); + return (dict, file, alphabet); } LOADS.fetch_add(1, Ordering::Relaxed); let budget = BUDGET.load(Ordering::Relaxed); @@ -187,8 +231,8 @@ fn get(locale: PackLocale) -> (Arc, Option) { total = total.saturating_sub(gone.bytes); } let used = c.clock; - c.loaded.push(Loaded { locale, dict: dict.clone(), file: file.clone(), bytes, used }); - (dict, file) + c.loaded.push(Loaded { locale, dict: dict.clone(), alphabet: alphabet.clone(), file: file.clone(), bytes, used }); + (dict, file, alphabet) } /// Load `locale`'s dictionary, with its estimated memory. @@ -229,11 +273,14 @@ pub(crate) fn load_error(locale: PackLocale) -> Option { /// Is `word` right in `locale`? Every word is when there is no dictionary. pub(crate) fn is_correct(word: &str, locale: PackLocale, opts: &crate::Options) -> bool { - let (d, _) = get(locale); + let (d, _, alphabet) = get_with_alphabet(locale); if d.is_empty() { return true; } let Some(w) = crate::spelling::word_to_check_in(&d, word, opts) else { return true }; + if other_script(w, &alphabet) { + return true; + } let known = |w: &str| d.check(w) || crate::in_user_dictionary(&w.to_lowercase()); if w.contains('-') && !known(w) { return w.split('-').all(|p| p.chars().count() <= 1 || known(p)); @@ -264,6 +311,55 @@ mod tests { PackLocale::new(t).unwrap() } + /// Review follow-up (craft-libs#485): a full list of locales with no dictionary forgets its + /// oldest one, not all of them, so a document in more languages doesn't look every one up again. + #[test] + fn missing_locales_forget_the_oldest_first() { + let tag = |i: usize| l(&format!("q{}{}", (b'a' + (i / 26 % 26) as u8) as char, (b'a' + (i % 26) as u8) as char)); + let mut missing = Vec::new(); + for i in 0..MAX_MISSING + 10 { + remember_missing(&mut missing, tag(i)); + assert!(missing.len() <= MAX_MISSING); + } + assert_eq!(missing.len(), MAX_MISSING); + assert!(!missing.contains(&tag(0)) && !missing.contains(&tag(9)), "the oldest went first"); + assert!(missing.contains(&tag(10)) && missing.contains(&tag(MAX_MISSING + 9)), "the rest stay"); + } + + /// Review follow-up (craft-libs#485): a panic inside a check (caught by the app) doesn't leave + /// this thread's dictionaries pinned past the budget for good. + #[test] + fn a_panic_inside_a_check_ends_it() { + let caught = std::panic::catch_unwind(|| in_one_check(|| panic!("a bug inside a check"))); + assert!(caught.is_err()); + assert_eq!(CHECK_FROM.with(Cell::get), None); + in_one_check(|| { + assert!(CHECK_FROM.with(Cell::get).is_some()); + in_one_check(|| ()); + assert!(CHECK_FROM.with(Cell::get).is_some(), "an inner check doesn't end the outer one"); + }); + assert_eq!(CHECK_FROM.with(Cell::get), None); + } + + /// Review follow-up (craft-libs#485): a word none of whose letters the dictionary uses is in + /// another script (a Cyrillic word in a Dutch run); a word sharing any letter is checked. + #[test] + fn words_in_another_script_are_not_the_dictionarys() { + let nl = AffixDict::from_hunspell("SET UTF-8\n", "3\nhuis\nkat\nÉén\n", &[]).unwrap(); + let a = alphabet(&nl); + assert_eq!( + &a[..], + ['a', 'h', 'i', 'k', 's', 't', 'u', 'é', 'n'].iter().copied().collect::>().into_iter().collect::>() + ); + for w in ["привет", "Привет", "αβγ", "שלום"] { + assert!(other_script(w, &a), "{w}"); + } + for w in ["huus", "kät", "huisп", "ÉÉN"] { + assert!(!other_script(w, &a), "{w}"); + } + assert!(!other_script("привет", &[]), "no alphabet: every word is checked"); + } + #[test] fn locales_and_file_names() { assert_eq!(l("nl-NL").as_str(), "nl-nl"); From 86345ebcf5cdff0c24e110e4592be9d82b8a95b9 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Mon, 2 Nov 2026 00:05:20 +0000 Subject: [PATCH 04/40] =?UTF-8?q?FU.4:=20craft-proof=20packs=20=E2=80=94?= =?UTF-8?q?=20remembered=20hyphenation=20patterns=20keep=20to=20a=20256=20?= =?UTF-8?q?MB=20budget=20of=20their=20own;=20a=20pack's=20spelling=20files?= =?UTF-8?q?=20are=20never=20candidates=20through=20a=20symbolic=20link?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-ups from craft-libs#485: up to 256 languages' pack patterns (64 MB files each) were kept by count only; `hunspell::ordered` checked pack files with `is_file`, which follows links, while the pack scan refuses them. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FypgCX8LsYYE8fsj4byBm2 --- crates/proof/src/hunspell.rs | 45 +++++++++++++++++++++----- crates/proof/src/packs.rs | 61 ++++++++++++++++++++++++++++++++---- 2 files changed, 93 insertions(+), 13 deletions(-) diff --git a/crates/proof/src/hunspell.rs b/crates/proof/src/hunspell.rs index 274af9eb..966ded72 100644 --- a/crates/proof/src/hunspell.rs +++ b/crates/proof/src/hunspell.rs @@ -196,19 +196,22 @@ pub(crate) fn ordered( let first = names.first().map(|n| norm(n)); let (region, rest): (Vec<&str>, Vec<&str>) = names.iter().copied().partition(|n| Some(norm(n)) == first); let pack: &[&str] = &[crate::packs::SPELLING]; + // (folder, names, a Language pack's: never read through a symbolic link, as the pack scan + // checks; a system folder's dictionaries often are links, `en_GB.dic` → `en_GB-ise.dic`). let tiers = dirs .iter() - .map(|d| (d, names)) - .chain(region_packs.iter().map(|d| (d, pack))) - .chain(system.iter().map(|d| (d, ®ion[..]))) - .chain(other_packs.iter().map(|d| (d, pack))) - .chain(system.iter().map(|d| (d, &rest[..]))); + .map(|d| (d, names, false)) + .chain(region_packs.iter().map(|d| (d, pack, true))) + .chain(system.iter().map(|d| (d, ®ion[..], false))) + .chain(other_packs.iter().map(|d| (d, pack, true))) + .chain(system.iter().map(|d| (d, &rest[..], false))); + let file = |p: &Path, pack: bool| if pack { std::fs::symlink_metadata(p).is_ok_and(|m| m.is_file()) } else { p.is_file() }; let mut out: Vec<(PathBuf, PathBuf)> = Vec::new(); - for (dir, names) in tiers { + for (dir, names, in_pack) in tiers { for n in names { let aff = dir.join(format!("{n}.aff")); let dic = dir.join(format!("{n}.dic")); - if aff.is_file() && dic.is_file() && !out.iter().any(|(_, d)| *d == dic) { + if file(&aff, in_pack) && file(&dic, in_pack) && !out.iter().any(|(_, d)| *d == dic) { out.push((aff, dic)); } } @@ -696,6 +699,34 @@ mod tests { } } + /// Review follow-up (craft-libs#485): a pack's `spelling.aff`/`.dic` that is a symbolic link + /// (swapped in after the pack scan) is never a candidate; one in a system folder still is. + #[cfg(unix)] + #[test] + fn pack_files_are_never_read_through_a_link() { + let root = std::env::temp_dir().join(format!("wc-proof-pack-links-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&root); + let (pack, system, elsewhere) = (root.join("packs").join("nl"), root.join("system"), root.join("elsewhere")); + for d in [&pack, &system, &elsewhere] { + std::fs::create_dir_all(d).unwrap(); + } + std::fs::write(elsewhere.join("x.aff"), "SET UTF-8\n").unwrap(); + std::fs::write(elsewhere.join("x.dic"), "1\nx\n").unwrap(); + std::fs::write(pack.join("spelling.aff"), "SET UTF-8\n").unwrap(); + std::os::unix::fs::symlink(elsewhere.join("x.dic"), pack.join("spelling.dic")).unwrap(); + std::os::unix::fs::symlink(elsewhere.join("x.aff"), system.join("nl.aff")).unwrap(); + std::os::unix::fs::symlink(elsewhere.join("x.dic"), system.join("nl.dic")).unwrap(); + let got: Vec = + ordered(&["nl"], &[], std::slice::from_ref(&pack), std::slice::from_ref(&system), &[]).into_iter().map(|(_, d)| d).collect(); + assert_eq!(got, [system.join("nl.dic")], "the system's link is followed, the pack's never"); + std::fs::remove_file(pack.join("spelling.dic")).unwrap(); + std::fs::write(pack.join("spelling.dic"), "1\nx\n").unwrap(); + let got: Vec = + ordered(&["nl"], &[], std::slice::from_ref(&pack), std::slice::from_ref(&system), &[]).into_iter().map(|(_, d)| d).collect(); + assert_eq!(got, [pack.join("spelling.dic"), system.join("nl.dic")]); + let _ = std::fs::remove_dir_all(&root); + } + /// WordCraft's own dictionaries folder under the user's application data is searched on every /// desktop system, after the setting and before the shared system locations (#1063). #[test] diff --git a/crates/proof/src/packs.rs b/crates/proof/src/packs.rs index 7424cd11..0d6cfbc8 100644 --- a/crates/proof/src/packs.rs +++ b/crates/proof/src/packs.rs @@ -34,6 +34,13 @@ pub(crate) const SPELLING: &str = "spelling"; const HYPHENATION: &str = "hyphenation.pat"; /// Most languages whose pack patterns are remembered (a document may tag thousands). const MAX_HYPHENATION_LOOKUPS: usize = 256; +/// The memory budget of the remembered pack patterns, in estimated bytes ([`pattern_bytes`]): +/// a pack's patterns may be as large as a dictionary (64 MB of file), and up to +/// [`MAX_HYPHENATION_LOOKUPS`] of them are kept, so they get a budget of their own, as the +/// dictionaries do (`pack_lang::MAX_LOADED_BYTES`). +const MAX_HYPHENATION_BYTES: u64 = 256 << 20; +/// Estimated memory per pattern (its packed key, its values and the map's own room). +const BYTES_PER_PATTERN: u64 = 64; /// A pack's description. const PACK_TOML: &str = "pack.toml"; /// A pack's licence texts. @@ -80,15 +87,18 @@ struct State { dir: Option, scan: Option>, /// Each language's pack patterns once looked up (`None`: no pack has them), at most - /// [`MAX_HYPHENATION_LOOKUPS`] languages (cleared when full). + /// [`MAX_HYPHENATION_LOOKUPS`] languages and [`MAX_HYPHENATION_BYTES`] (cleared when full). hyphenation: BTreeMap>>, + /// The estimated memory of the patterns in [`State::hyphenation`]. + hyphenation_bytes: u64, /// (locale, reason) of pack hyphenation files that failed to load. hyphenation_errors: Vec<(String, String)>, /// Counts changes, so a scan or load that started before one is not kept. changes: u64, } -static STATE: RwLock = RwLock::new(State { dir: None, scan: None, hyphenation: BTreeMap::new(), hyphenation_errors: Vec::new(), changes: 0 }); +static STATE: RwLock = + RwLock::new(State { dir: None, scan: None, hyphenation: BTreeMap::new(), hyphenation_bytes: 0, hyphenation_errors: Vec::new(), changes: 0 }); fn read() -> RwLockReadGuard<'static, State> { STATE.read().unwrap_or_else(PoisonError::into_inner) @@ -121,6 +131,7 @@ pub fn reload() { fn forget(s: &mut State) { s.scan = None; s.hyphenation.clear(); + s.hyphenation_bytes = 0; s.hyphenation_errors.clear(); s.changes = s.changes.wrapping_add(1); } @@ -566,10 +577,8 @@ pub(crate) fn hyphenation(lang: ProofLang) -> Option> { let mut s = write(); if s.changes == changes && !s.hyphenation.contains_key(&lang) { // ponytail: cleared when full, an LRU if many languages each need their patterns. - if s.hyphenation.len() >= MAX_HYPHENATION_LOOKUPS { - s.hyphenation.clear(); - } - s.hyphenation.insert(lang, found.clone()); + let bytes = found.as_deref().map_or(0, pattern_bytes); + remember_patterns(&mut s, lang, found.clone(), bytes, MAX_HYPHENATION_BYTES); for e in errors { if !s.hyphenation_errors.contains(&e) { s.hyphenation_errors.push(e); @@ -579,6 +588,23 @@ pub(crate) fn hyphenation(lang: ProofLang) -> Option> { found } +/// Estimated memory of `p`. +fn pattern_bytes(p: &Patterns) -> u64 { + (p.len() as u64).saturating_add(p.exception_count() as u64).saturating_mul(BYTES_PER_PATTERN) +} + +/// Remember `lang`'s pack patterns (`bytes` of them): over [`MAX_HYPHENATION_LOOKUPS`] languages +/// or `budget` bytes, the ones remembered so far are forgotten first. Patterns larger than the +/// whole budget are still kept alone, or they would be read from disk for every word. +fn remember_patterns(s: &mut State, lang: ProofLang, found: Option>, bytes: u64, budget: u64) { + if s.hyphenation.len() >= MAX_HYPHENATION_LOOKUPS || s.hyphenation_bytes.saturating_add(bytes) > budget { + s.hyphenation.clear(); + s.hyphenation_bytes = 0; + } + s.hyphenation.insert(lang, found); + s.hyphenation_bytes = s.hyphenation_bytes.saturating_add(bytes); +} + /// How many languages' pack patterns are remembered now. #[cfg(test)] pub(crate) fn hyphenation_lookups() -> usize { @@ -600,6 +626,29 @@ fn load_patterns(path: &Path) -> Result { mod tests { use super::*; + /// Review follow-up (craft-libs#485): the remembered pack patterns have a memory budget, not + /// only a count, so 256 languages of large patterns can't each stay in memory. + #[test] + fn remembered_patterns_keep_to_a_memory_budget() { + let mut s = State { dir: None, scan: None, hyphenation: BTreeMap::new(), hyphenation_bytes: 0, hyphenation_errors: Vec::new(), changes: 0 }; + let p = Arc::new(Patterns::parse("hy3ph he2n hena4 hen5at 1na n2at 1tio 2io o2n")); + let bytes = pattern_bytes(&p); + assert_eq!(bytes, 9 * BYTES_PER_PATTERN); + let lang = |t: &str| ProofLang::from_tag(Some(t)); + let budget = bytes * 2; + remember_patterns(&mut s, lang("nl"), Some(p.clone()), bytes, budget); + remember_patterns(&mut s, lang("fr"), Some(p.clone()), bytes, budget); + remember_patterns(&mut s, lang("sv"), None, 0, budget); + assert_eq!((s.hyphenation.len(), s.hyphenation_bytes), (3, budget), "a language with no patterns takes no room"); + remember_patterns(&mut s, lang("da"), Some(p.clone()), bytes, budget); + assert_eq!((s.hyphenation.len(), s.hyphenation_bytes), (1, bytes), "over the budget: the others are forgotten"); + assert!(s.hyphenation.contains_key(&lang("da"))); + remember_patterns(&mut s, lang("fi"), Some(p.clone()), budget * 4, budget); + assert_eq!((s.hyphenation.len(), s.hyphenation_bytes), (1, budget * 4), "patterns over the whole budget are kept alone"); + forget(&mut s); + assert_eq!((s.hyphenation.len(), s.hyphenation_bytes), (0, 0)); + } + /// Review of #485: a pack's name loses bidirectional and other format characters. #[test] fn pack_names_are_plain_text() { From 1f345ea80974844ffaec1c9b5bfeb5099d5cb34c Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Mon, 2 Nov 2026 00:06:36 +0000 Subject: [PATCH 05/40] =?UTF-8?q?FU.5:=20craft-proof=20id,=20ru=20?= =?UTF-8?q?=E2=80=94=20the=20cased-letter=20rule=20for=20words=20in=20capi?= =?UTF-8?q?tals=20(craft-libs#485):=20a=20caseless=20part=20is=20no=20abbr?= =?UTF-8?q?eviation;=20shared=20in=5Fcapitals=20/=20capitals=5Fpart?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FypgCX8LsYYE8fsj4byBm2 --- crates/proof/src/id/mod.rs | 15 ++++++++++++++- crates/proof/src/lib.rs | 2 +- crates/proof/src/ru/mod.rs | 6 ++++-- crates/proof/src/spelling.rs | 30 ++++++++++++++++++++++++++++-- 4 files changed, 47 insertions(+), 6 deletions(-) diff --git a/crates/proof/src/id/mod.rs b/crates/proof/src/id/mod.rs index c78e25d4..85dc825a 100644 --- a/crates/proof/src/id/mod.rs +++ b/crates/proof/src/id/mod.rs @@ -76,7 +76,7 @@ fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool { || w.split('-').all(|p| { p.chars().count() <= 1 || CLITICS.iter().any(|c| p.eq_ignore_ascii_case(c)) - || (opts.ignore_uppercase && p.chars().all(|c| !c.is_lowercase()) && p.chars().count() <= 6) + || (opts.ignore_uppercase && crate::spelling::capitals_part(p, 6)) || known(d, p) }); } @@ -141,6 +141,19 @@ mod tests { } } + /// Review follow-up (craft-libs#485's cased-letter rule): a part in a caseless script is no + /// abbreviation, so `buku-日本` is checked part by part; a part in capitals or without letters + /// still is one (`KTP-nya`, `COVID-19` with numbers checked). + #[test] + fn caseless_parts_are_not_abbreviations() { + test_dict::install(); + let d = dictionary(); + let defaults = crate::Options::default(); + assert!(!check(&d, "buku-日本", &defaults) && !check(&d, "buku-ไทย", &defaults)); + let numbers = crate::Options { ignore_numbers: false, ..defaults }; + assert!(check(&d, "KTP-nya", &defaults) && check(&d, "buku-19", &numbers)); + } + #[test] fn forbidden_words_are_rejected_in_every_case() { test_dict::install(); diff --git a/crates/proof/src/lib.rs b/crates/proof/src/lib.rs index 4cfb9a5d..561c3541 100644 --- a/crates/proof/src/lib.rs +++ b/crates/proof/src/lib.rs @@ -669,7 +669,7 @@ pub(crate) fn accepted_by_options(word: &str, opts: &Options) -> bool { || !w.chars().any(char::is_alphabetic) || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) // Only a word with a cased letter is in capitals: a Hebrew or Thai word has none. - || (opts.ignore_uppercase && w.chars().any(char::is_uppercase) && w.chars().all(|c| !c.is_lowercase())) + || (opts.ignore_uppercase && spelling::in_capitals(w)) // The whole word: the languages split a hyphenated word into its parts, and an added // `Unigranrio-RJ` is none of them (#2082). || in_user_dictionary(&w.to_lowercase()) diff --git a/crates/proof/src/ru/mod.rs b/crates/proof/src/ru/mod.rs index b88978af..6e9ee34d 100644 --- a/crates/proof/src/ru/mod.rs +++ b/crates/proof/src/ru/mod.rs @@ -105,7 +105,7 @@ fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool { if n <= 1 || (opts.ignore_numbers && w.chars().any(char::is_numeric)) || !w.chars().any(char::is_alphabetic) { return true; } - if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= ABBREVIATION_LETTERS { + if opts.ignore_uppercase && crate::spelling::in_capitals(w) && n <= ABBREVIATION_LETTERS { return true; } // Camel case (`ВКонтакте`, `МирКрасок`): a brand or an identifier, not a dictionary word. @@ -333,7 +333,7 @@ fn hyphenated(d: &AffixDict, w: &str, opts: &crate::Options) -> bool { let last = parts.len() - 1; parts.iter().enumerate().all(|(i, p)| { // An abbreviation part (`ИИ-консультант`, `SMS-рассылка`) or a known word. - if known(d, p) || (opts.ignore_uppercase && p.chars().all(|c| !c.is_lowercase()) && p.chars().count() <= ABBREVIATION_LETTERS) { + if known(d, p) || (opts.ignore_uppercase && crate::spelling::capitals_part(p, ABBREVIATION_LETTERS)) { return true; } // A first part with a linking vowel: the stem (`северо` → `север`) or its adjective @@ -485,5 +485,7 @@ mod tests { assert!(!crate::is_correct_in_with("м²", lang, &check_digits), "a superscript is a number too"); assert!(crate::is_correct_in_with("2026", lang, &check_digits), "numbers alone stay correct"); assert!(is_correct("ПРЕВЕТ"), "direct checks keep the defaults"); + // The cased-letter rule (craft-libs#485): capitals and letterless parts are abbreviations. + assert!(crate::is_correct_in_with("ИИ-2", lang, &check_digits) && crate::is_correct_in_with("ТУ-154М", lang, &check_digits)); } } diff --git a/crates/proof/src/spelling.rs b/crates/proof/src/spelling.rs index c80f42fe..1b42f5ff 100644 --- a/crates/proof/src/spelling.rs +++ b/crates/proof/src/spelling.rs @@ -32,17 +32,43 @@ pub(crate) fn word_to_check_in<'a>(d: &AffixDict, word: &'a str, opts: &Options) if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) { return None; } - // Only a word with a cased letter can be in capitals: Hebrew, Arabic, Thai… words have none. - if opts.ignore_uppercase && w.chars().any(char::is_uppercase) && w.chars().all(|c| !c.is_lowercase()) && n <= 6 { + if opts.ignore_uppercase && in_capitals(w) && n <= 6 { return None; } Some(w) } +/// Is `w` in capitals: a cased letter and no lower-case one? Only a word with a cased letter can +/// be: Hebrew, Arabic, Thai, Chinese… words have none, so they are never taken for acronyms. +pub(crate) fn in_capitals(w: &str) -> bool { + w.chars().any(char::is_uppercase) && w.chars().all(|c| !c.is_lowercase()) +} + +/// Is `part` of a hyphenated word an abbreviation Ignore words in UPPERCASE skips: at most `max` +/// characters, [`in_capitals`] or without letters (`SMS` in `SMS-ка`, `19` in `COVID-19`)? A +/// part in a caseless script is no abbreviation. +pub(crate) fn capitals_part(part: &str, max: usize) -> bool { + part.chars().count() <= max && (in_capitals(part) || !part.chars().any(char::is_alphabetic)) +} + #[cfg(test)] mod tests { use super::*; + /// craft-libs#485's cased-letter rule, shared by every language: a word or part is in + /// capitals only with a cased letter; a part without letters counts as an abbreviation. + #[test] + fn capitals_need_a_cased_letter() { + for w in ["PKO", "SMS", "ÉTÉ", "ВКП", "B2B"] { + assert!(in_capitals(w) && capitals_part(w, 6), "{w}"); + } + for w in ["日本", "שלום", "ไทย", "Pko", "pko", "123", ""] { + assert!(!in_capitals(w), "{w}"); + } + assert!(capitals_part("19", 6) && capitals_part("", 6)); + assert!(!capitals_part("日本", 6) && !capitals_part("ไทย", 6) && !capitals_part("ABCDEFG", 6)); + } + #[test] fn words_needing_no_lookup() { let d = AffixDict::from_hunspell("", "1\nkot\n", &[]).unwrap(); From 8712aa105a88619458b2081eeb569f089af0cb66 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Mon, 2 Nov 2026 00:09:16 +0000 Subject: [PATCH 06/40] =?UTF-8?q?FU.6:=20craft-proof=20affix=20=E2=80=94?= =?UTF-8?q?=20the=20stems'=20flags=20are=20budgeted=20while=20a=20dictiona?= =?UTF-8?q?ry=20is=20read=20(running=20counts=20of=20flag=20text=20and=20k?= =?UTF-8?q?eys),=20so=20a=20huge=20.dic=20fails=20early=20instead=20of=20a?= =?UTF-8?q?fter=20holding=20every=20entry's=20flags=20(#495=20P3)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FypgCX8LsYYE8fsj4byBm2 --- crates/proof/src/affix.rs | 103 +++++++++++++++++++++++++++++++------- 1 file changed, 86 insertions(+), 17 deletions(-) diff --git a/crates/proof/src/affix.rs b/crates/proof/src/affix.rs index 63d6f289..c3ac63ab 100644 --- a/crates/proof/src/affix.rs +++ b/crates/proof/src/affix.rs @@ -74,6 +74,14 @@ const MAX_ALIAS_BYTES: usize = 4 << 20; /// Most flags all distinct flag sets (stems' and continuation classes') may hold together: a lookup /// tests each distinct continuation set it meets at most once, so this bounds its cost. const MAX_SET_DATA: usize = 1 << 23; +/// Most flags all stems may carry together before their sets are merged and interned (the same +/// set on many stems counts each time; an `AF` alias counts once). Checked while reading, so a +/// huge `.dic` is refused before its flags fill memory, not after (#495: a 760 MB `.dic` once +/// held 3 GB before [`MAX_SET_DATA`] refused it). +const MAX_STEM_FLAGS: usize = 4 * MAX_SET_DATA; +/// Most bytes of flag text per flag of [`MAX_STEM_FLAGS`] the stem lines may hold in all while +/// they are read (`FLAG num`: `65535,` is six). +const FLAG_TEXT_PER_FLAG: usize = 6; /// Most distinct flags a dictionary may use: each gets a `u16` id. const MAX_DISTINCT_FLAGS: usize = 1 << 16; @@ -972,12 +980,19 @@ impl AffixDict { } fn from_hunspell_in(aff: &str, dic: &str, comments: &[&str], english: bool) -> Result { + AffixDict::from_hunspell_capped(aff, dic, comments, english, MAX_STEM_FLAGS) + } + + /// [`AffixDict::from_hunspell_in`] with at most `max_flags` stem flags in all + /// ([`MAX_STEM_FLAGS`]; tests lower it). + fn from_hunspell_capped(aff: &str, dic: &str, comments: &[&str], english: bool, max_flags: usize) -> Result { // Some dictionaries open with a UTF-8 BOM; it must not glue itself to the first // directive or turn the `.dic` count line into a stem. let aff = aff.strip_prefix('\u{feff}').unwrap_or(aff); let dic = dic.strip_prefix('\u{feff}').unwrap_or(dic); let rules = parse_aff(aff, english)?; let mut stems: Vec<(String, String)> = Vec::new(); + let mut flag_text = FlagText::new(max_flags); for (n, line) in dic.lines().enumerate() { // The entry may hold spaces (`ad hoc/B\tpo:adv`); it ends where Hunspell ends it. // Leading blanks belong to it, as in Hunspell (` lab` is no `lab`). @@ -1013,14 +1028,16 @@ impl AffixDict { } // With an `AF` table the field stays an alias number here: build() reads each alias // once, and every stem using it shares those flags. + flag_text.add(&flags)?; stems.push((word, flags)); } - AffixDict::build(rules, stems, comments.iter().map(ToString::to_string).collect(), MAX_FLAGS_PER_ENTRY) + AffixDict::build(rules, stems, comments.iter().map(ToString::to_string).collect(), MAX_FLAGS_PER_ENTRY, max_flags) } /// Build from parsed rules and stems whose flag fields carry at most `entry_cap` flags each /// ([`MAX_FLAGS_PER_ENTRY`] for `.dic` entries, [`MAX_FLAGS_PER_STEM`] for compact data). - fn build(rules: Rules, stems: Vec<(String, String)>, comments: Vec, entry_cap: usize) -> Result { + /// The stems' flags may number `max_flags` in all ([`MAX_STEM_FLAGS`]), counted as they are read. + fn build(rules: Rules, stems: Vec<(String, String)>, comments: Vec, entry_cap: usize, max_flags: usize) -> Result { let mode = rules.mode; // A `.dic` entry's malformed flag items read as Hunspell reads them ([`FlagMode`]); an // entry with more than MAX_FLAGS_PER_STEM flags, or compact data whose flags don't read, @@ -1049,20 +1066,24 @@ impl AffixDict { } Some((start, stem_keys.len(), word)) }; - let mut stems: Vec<(String, usize, usize, bool)> = stems - .into_iter() - .filter_map(|(w, f)| { - let range = match rules.alias_index(&f) { - Some(i) => *alias_ranges.entry(i).or_insert_with(|| read(rules.aliases.get(i).map_or("", String::as_str), &mut stem_keys)), - None if rules.aliases_on => Some((0, 0, true)), - None => read(&f, &mut stem_keys), - }; - if range.is_none_or(|r| !r.2) { - skipped += 1; - } - range.map(|(a, b, word)| (w, a, b, word)) - }) - .collect(); + let mut read_stems: Vec<(String, usize, usize, bool)> = Vec::with_capacity(stems.len()); + for (w, f) in stems { + let range = match rules.alias_index(&f) { + Some(i) => *alias_ranges.entry(i).or_insert_with(|| read(rules.aliases.get(i).map_or("", String::as_str), &mut stem_keys)), + None if rules.aliases_on => Some((0, 0, true)), + None => read(&f, &mut stem_keys), + }; + if stem_keys.len() > max_flags { + return Err(format!("the stems carry more than {max_flags} flags in all")); + } + if range.is_none_or(|r| !r.2) { + skipped += 1; + } + if let Some((a, b, word)) = range { + read_stems.push((w, a, b, word)); + } + } + let mut stems = read_stems; if skipped > 0 { log::debug!("affix dictionary: {skipped} entries with unreadable or too many flags skipped"); } @@ -2059,6 +2080,7 @@ impl AffixDict { } let body = raw.get(words_at + 7..).unwrap_or_default(); let mut stems: Vec<(String, String)> = Vec::new(); + let mut flag_text = FlagText::new(MAX_STEM_FLAGS); let mut cur: Vec = Vec::new(); let mut i = 0; while let Some(&p) = body.get(i) { @@ -2081,9 +2103,32 @@ impl AffixDict { if word.is_empty() || word.len() > MAX_WORD_BYTES { return Err("affix dictionary: bad stem".into()); } + flag_text.add(flags).map_err(|e| format!("affix dictionary: {e}"))?; stems.push((word.to_string(), flags.to_string())); } - AffixDict::build(rules, stems, comments, MAX_FLAGS_PER_STEM) + AffixDict::build(rules, stems, comments, MAX_FLAGS_PER_STEM, MAX_STEM_FLAGS) + } +} + +/// A running count of the stems' flag text while a dictionary is read: past +/// [`FLAG_TEXT_PER_FLAG`] bytes for each of `max_flags` flags, reading stops with an error, before +/// the rest of a huge `.dic` is held ([`MAX_STEM_FLAGS`]). +struct FlagText { + bytes: usize, + cap: usize, +} + +impl FlagText { + fn new(max_flags: usize) -> Self { + FlagText { bytes: 0, cap: max_flags.saturating_mul(FLAG_TEXT_PER_FLAG) } + } + + fn add(&mut self, flags: &str) -> Result<(), String> { + self.bytes = self.bytes.saturating_add(flags.len()); + if self.bytes > self.cap { + return Err(format!("the stems' flags take more than {} bytes", self.cap)); + } + Ok(()) } } @@ -2133,6 +2178,30 @@ pub(crate) mod tests { AffixDict::from_hunspell(AFF, DIC, &["test data"]).unwrap() } + /// #495 review: the stems' flags are counted as they are read, so a `.dic` whose flags pass + /// the budget is refused while reading (its flag text) or while its flags are read (the + /// keys), before they are all held; one within it loads. + #[test] + fn stem_flags_are_budgeted_while_reading() { + let aff = "SET UTF-8\nSFX A Y 1\nSFX A 0 s .\nSFX B Y 1\nSFX B 0 x .\n"; + let many: String = (0..100).map(|i| format!("w{i}/AB\n")).collect(); + let dic = format!("100\n{many}"); + // 200 flags: over a budget of 50 keys (whose flag text, 300 bytes, still fits). + let e = AffixDict::from_hunspell_capped(aff, &dic, &[], false, 50).unwrap_err(); + assert!(e.contains("more than 50 flags"), "{e}"); + let d = AffixDict::from_hunspell_capped(aff, &dic, &[], false, 200).unwrap(); + assert!(d.check("w7s") && d.check("w99x")); + // Flag text past the budget (entries over the per-entry cap hold no keys, but their text + // still counts): refused while reading the `.dic`. + let long = "A".repeat(MAX_FLAGS_PER_ENTRY + 1); + let dic = format!("3\na/{long}\nb/{long}\nc/{long}\n"); + let e = AffixDict::from_hunspell_capped(aff, &dic, &[], false, 2000).unwrap_err(); + assert!(e.contains("more than 12000 bytes"), "{e}"); + assert!(AffixDict::from_hunspell(aff, &dic, &[]).is_ok(), "within the real budget: skipped entries, no error"); + let mut text = FlagText::new(1); + assert!(text.add("ABCDEF").is_ok() && text.add("G").is_err()); + } + #[test] fn affix_headers_allow_comments_but_rule_conditions_stay_literal() { let aff = "SET UTF-8\nPFX X Y 1 # prefix comment\nPFX X 0 ne . # rule comment\nSFX y Y 1 # suffix comment\nSFX y 0 y . # rule comment\n"; From 65ce4f882a31c3b208287189c8cb16dfbbfbd640 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Mon, 2 Nov 2026 00:24:58 +0000 Subject: [PATCH 07/40] =?UTF-8?q?FU.7:=20craft-proof=20pack=20languages=20?= =?UTF-8?q?=E2=80=94=20another=20script=20is=20told=20by=20the=20letters'?= =?UTF-8?q?=20Unicode=20scripts,=20not=20the=20dictionary's=20letter=20inv?= =?UTF-8?q?entory,=20so=20a=20misspelling=20in=20the=20dictionary's=20own?= =?UTF-8?q?=20script=20(`zzzz`=20in=20Dutch)=20is=20always=20checked?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Found by a local CodeRabbit review of #496 (major). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FypgCX8LsYYE8fsj4byBm2 --- crates/proof/src/lib.rs | 1 + crates/proof/src/pack_lang.rs | 119 +++++++++++++++++++++++----------- 2 files changed, 81 insertions(+), 39 deletions(-) diff --git a/crates/proof/src/lib.rs b/crates/proof/src/lib.rs index 561c3541..b4b25f5e 100644 --- a/crates/proof/src/lib.rs +++ b/crates/proof/src/lib.rs @@ -1723,6 +1723,7 @@ mod tests { packs::set_packs_dir(Some(root.clone())); let (nl, el) = (ProofLang::from_tag(Some("nl-NL")), ProofLang::from_tag(Some("el-GR"))); assert!(is_correct_in("huis", nl) && !is_correct_in("huiss", nl)); + assert!(!is_correct_in("zzzz", nl), "a misspelling in the dictionary's script, whatever its letters"); assert!(is_correct_in("Пушкин", nl) && is_correct_in("σπίτι", nl)); assert!(is_correct_in("σπίτι", el) && !is_correct_in("σπίτιι", el) && is_correct_in("huis", el)); let text = "Пушкин huiss"; diff --git a/crates/proof/src/pack_lang.rs b/crates/proof/src/pack_lang.rs index 56e8d090..7b4d1931 100644 --- a/crates/proof/src/pack_lang.rs +++ b/crates/proof/src/pack_lang.rs @@ -7,8 +7,9 @@ //! [`in_one_check`] call, so a paragraph in many languages loads each dictionary once. Locales //! with no dictionary are remembered apart and take no room. //! -//! A word none of whose letters the dictionary's words use (a Cyrillic word in a Dutch run) isn't -//! this language's: it is never marked, as Word leaves a word in another script alone. +//! A word none of whose letters is in a script the dictionary's words are written in (a Cyrillic +//! word in a Dutch run) isn't this language's: it is never marked, as Word leaves a word in another +//! script alone. A word in the dictionary's script is always checked, whatever its letters. use std::cell::Cell; use std::path::PathBuf; @@ -25,8 +26,6 @@ pub(crate) const MAX_LOADED_BYTES: u64 = 256 << 20; const BYTES_PER_FILE_BYTE: u64 = 6; /// Most locales remembered as having no dictionary (the oldest is forgotten first when full). const MAX_MISSING: usize = 256; -/// Most distinct letters kept in a dictionary's alphabet ([`alphabet`]). -const MAX_ALPHABET: usize = 4096; /// The budget in force (tests lower it). static BUDGET: AtomicU64 = AtomicU64::new(MAX_LOADED_BYTES); @@ -41,8 +40,8 @@ thread_local! { struct Loaded { locale: PackLocale, dict: Arc, - /// The lowercase letters its words use ([`alphabet`]), sorted. - alphabet: Arc<[char]>, + /// The scripts its words are written in ([`scripts`]), sorted. + scripts: Arc<[u32]>, /// The `.dic` it came from. file: Option, /// Estimated memory. @@ -100,21 +99,61 @@ fn remember_missing(missing: &mut Vec, locale: PackLocale) { missing.push(locale); } -/// The lowercase letters the words of `d` use, sorted (at most [`MAX_ALPHABET`]). -fn alphabet(d: &AffixDict) -> Arc<[char]> { +/// The script of the letter `c`, as a number: the main alphabetic scripts by their Unicode blocks +/// (Latin with its extensions, Greek, Cyrillic, Armenian, Hebrew, Arabic, Georgian, Hangul, kana, +/// Han, Ethiopic); any other letter by its 128-character block. Only told apart, never named. +fn script_of(c: char) -> u32 { + const SCRIPTS: &[(u32, u32, u32)] = &[ + (0x0041, 0x024F, 1), + (0x0250, 0x02AF, 1), + (0x1E00, 0x1EFF, 1), + (0x2C60, 0x2C7F, 1), + (0xA720, 0xA7FF, 1), + (0xAB30, 0xAB6F, 1), + (0xFF21, 0xFF5A, 1), + (0x0370, 0x03FF, 2), + (0x1F00, 0x1FFF, 2), + (0x0400, 0x052F, 3), + (0x1C80, 0x1C8F, 3), + (0x2DE0, 0x2DFF, 3), + (0xA640, 0xA69F, 3), + (0x0530, 0x058F, 4), + (0xFB13, 0xFB17, 4), + (0x0590, 0x05FF, 5), + (0xFB1D, 0xFB4F, 5), + (0x0600, 0x06FF, 6), + (0x0750, 0x077F, 6), + (0x08A0, 0x08FF, 6), + (0xFB50, 0xFDFF, 6), + (0xFE70, 0xFEFF, 6), + (0x10A0, 0x10FF, 7), + (0x1C90, 0x1CBF, 7), + (0x2D00, 0x2D2F, 7), + (0x1100, 0x11FF, 8), + (0x3130, 0x318F, 8), + (0xAC00, 0xD7AF, 8), + (0x3040, 0x30FF, 9), + (0x3400, 0x4DBF, 10), + (0x4E00, 0x9FFF, 10), + (0x1200, 0x139F, 11), + ]; + let u = u32::from(c); + SCRIPTS.iter().find(|(lo, hi, _)| (*lo..=*hi).contains(&u)).map_or(0x100 + (u >> 7), |(_, _, s)| *s) +} + +/// The scripts the words of `d` are written in ([`script_of`] of their letters), sorted. +fn scripts(d: &AffixDict) -> Arc<[u32]> { let mut seen = std::collections::BTreeSet::new(); - for c in d.stems().flat_map(str::chars).filter(|c| c.is_alphabetic()).flat_map(char::to_lowercase) { - if seen.len() >= MAX_ALPHABET { - break; - } - seen.insert(c); + for c in d.stems().flat_map(str::chars).filter(|c| c.is_alphabetic()) { + seen.insert(script_of(c)); } seen.into_iter().collect() } -/// Is `word` in another script than `alphabet`'s words: none of its letters is in it? -fn other_script(word: &str, alphabet: &[char]) -> bool { - !alphabet.is_empty() && !word.chars().filter(|c| c.is_alphabetic()).flat_map(char::to_lowercase).any(|c| alphabet.binary_search(&c).is_ok()) +/// Is `word` in another script than the dictionary's (`scripts`): none of its letters is in one +/// of them? A word in the dictionary's script is never, whatever its letters (`zzzz` in Dutch). +fn other_script(word: &str, scripts: &[u32]) -> bool { + !scripts.is_empty() && !word.chars().filter(|c| c.is_alphabetic()).any(|c| scripts.binary_search(&script_of(c)).is_ok()) } #[cfg(test)] @@ -189,19 +228,19 @@ pub(crate) fn names(locale: PackLocale) -> Vec { /// `locale`'s dictionary, loaded on first use; an empty one when there is none. fn get(locale: PackLocale) -> (Arc, Option) { - let (d, file, _) = get_with_alphabet(locale); + let (d, file, _) = get_with_scripts(locale); (d, file) } -/// [`get`], with the dictionary's [`alphabet`]. -fn get_with_alphabet(locale: PackLocale) -> (Arc, Option, Arc<[char]>) { +/// [`get`], with the dictionary's [`scripts`]. +fn get_with_scripts(locale: PackLocale) -> (Arc, Option, Arc<[u32]>) { let reloads = { let mut c = CACHE.write().unwrap_or_else(PoisonError::into_inner); c.clock = c.clock.wrapping_add(1); let now = c.clock; if let Some(l) = c.loaded.iter_mut().find(|l| l.locale == locale) { l.used = now; - return (l.dict.clone(), l.file.clone(), l.alphabet.clone()); + return (l.dict.clone(), l.file.clone(), l.scripts.clone()); } if c.missing.contains(&locale) { return (Arc::default(), None, Arc::from([])); @@ -209,14 +248,14 @@ fn get_with_alphabet(locale: PackLocale) -> (Arc, Option, Ar c.reloads }; let (dict, file, bytes) = load(locale); - let alphabet = alphabet(&dict); + let scripts = scripts(&dict); let mut c = CACHE.write().unwrap_or_else(PoisonError::into_inner); if c.reloads != reloads || c.loaded.iter().any(|l| l.locale == locale) || c.missing.contains(&locale) { - return (dict, file, alphabet); + return (dict, file, scripts); } if dict.is_empty() { remember_missing(&mut c.missing, locale); - return (dict, file, alphabet); + return (dict, file, scripts); } LOADS.fetch_add(1, Ordering::Relaxed); let budget = BUDGET.load(Ordering::Relaxed); @@ -231,8 +270,8 @@ fn get_with_alphabet(locale: PackLocale) -> (Arc, Option, Ar total = total.saturating_sub(gone.bytes); } let used = c.clock; - c.loaded.push(Loaded { locale, dict: dict.clone(), alphabet: alphabet.clone(), file: file.clone(), bytes, used }); - (dict, file, alphabet) + c.loaded.push(Loaded { locale, dict: dict.clone(), scripts: scripts.clone(), file: file.clone(), bytes, used }); + (dict, file, scripts) } /// Load `locale`'s dictionary, with its estimated memory. @@ -273,12 +312,12 @@ pub(crate) fn load_error(locale: PackLocale) -> Option { /// Is `word` right in `locale`? Every word is when there is no dictionary. pub(crate) fn is_correct(word: &str, locale: PackLocale, opts: &crate::Options) -> bool { - let (d, _, alphabet) = get_with_alphabet(locale); + let (d, _, scripts) = get_with_scripts(locale); if d.is_empty() { return true; } let Some(w) = crate::spelling::word_to_check_in(&d, word, opts) else { return true }; - if other_script(w, &alphabet) { + if other_script(w, &scripts) { return true; } let known = |w: &str| d.check(w) || crate::in_user_dictionary(&w.to_lowercase()); @@ -341,23 +380,25 @@ mod tests { assert_eq!(CHECK_FROM.with(Cell::get), None); } - /// Review follow-up (craft-libs#485): a word none of whose letters the dictionary uses is in - /// another script (a Cyrillic word in a Dutch run); a word sharing any letter is checked. + /// Review follow-up (craft-libs#485): a word with no letter in the dictionary's scripts is + /// in another script (a Cyrillic word in a Dutch run); a word in its script is always checked, + /// even with letters no dictionary word uses (CodeRabbit on #496: `zzzz`). #[test] fn words_in_another_script_are_not_the_dictionarys() { let nl = AffixDict::from_hunspell("SET UTF-8\n", "3\nhuis\nkat\nÉén\n", &[]).unwrap(); - let a = alphabet(&nl); - assert_eq!( - &a[..], - ['a', 'h', 'i', 'k', 's', 't', 'u', 'é', 'n'].iter().copied().collect::>().into_iter().collect::>() - ); - for w in ["привет", "Привет", "αβγ", "שלום"] { - assert!(other_script(w, &a), "{w}"); + let latin = scripts(&nl); + assert_eq!(latin.len(), 1); + for w in ["привет", "Привет", "αβγ", "שלום", "日本"] { + assert!(other_script(w, &latin), "{w}"); } - for w in ["huus", "kät", "huisп", "ÉÉN"] { - assert!(!other_script(w, &a), "{w}"); + for w in ["huus", "kät", "huisп", "ÉÉN", "zzzz", "qxqw", "ŋʒ"] { + assert!(!other_script(w, &latin), "{w}"); } - assert!(!other_script("привет", &[]), "no alphabet: every word is checked"); + assert!(!other_script("привет", &[]), "no scripts: every word is checked"); + let ru = AffixDict::from_hunspell("SET UTF-8\n", "1\nдом\n", &[]).unwrap(); + assert!(!other_script("щщщ", &scripts(&ru)) && other_script("dom", &scripts(&ru))); + assert_ne!(script_of('ა'), script_of('a')); + assert_eq!(script_of('ă'), script_of('Ạ')); } #[test] From 87c7d1874a616f9f69485a36e55e1b5c8d1346bc Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Mon, 2 Nov 2026 01:21:43 +0000 Subject: [PATCH 08/40] CP7.1: craft-proof reads flag-based compounding (COMPOUNDFLAG/BEGIN/MIDDLE/END, PERMITFLAG, FORBIDFLAG, ROOT, WORDMAX, SYLLABLE, CHECKCOMPOUND*, SIMPLIFIEDTRIPLE, FORCEUCASE), as the hunspell CLI answers Black-box: every rule pinned to hunspell 1.7.5's answers on synthetic dictionaries and on the Language packs' own files. Bounded work per word (16 parts, 65,536 steps). Also: ONLYINCOMPOUND belongs to its homonym like NEEDAFFIX; COMPOUNDRULE reads the Swedish `)k` and the Mongolian `[..]`, `%?`, `.` as the CLI does; COMPOUNDMORESUFFIXES is refused only with flag-based compounding (Korean keeps loading). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FypgCX8LsYYE8fsj4byBm2 --- crates/proof/src/affix.rs | 214 +++- crates/proof/src/affix/compound.rs | 1491 ++++++++++++++++++++++++++++ 2 files changed, 1655 insertions(+), 50 deletions(-) create mode 100644 crates/proof/src/affix/compound.rs diff --git a/crates/proof/src/affix.rs b/crates/proof/src/affix.rs index 63d6f289..7bec2722 100644 --- a/crates/proof/src/affix.rs +++ b/crates/proof/src/affix.rs @@ -23,10 +23,11 @@ //! Compounds by `COMPOUNDRULE` (patterns of flags with `*` and `?`, as SCOWL's English //! dictionaries use for ordinals: `21st`, `112th`; with long or numeric flags each flag is written //! in parentheses, `(aa)*(bb)`), with `ONLYINCOMPOUND` and `COMPOUNDMIN`; the parts of such a -//! compound are stems, without affixes, as in Hunspell. -//! Morphological data (`AM` aliases, the fields after a stem) is skipped. Rejected: flag-based -//! compounding (`COMPOUNDFLAG`, `COMPOUNDBEGIN`…) and the other directives that change which -//! words are correct. +//! compound are stems, without affixes, as in Hunspell. Flag-based compounds (`COMPOUNDFLAG`, +//! `COMPOUNDBEGIN`/`MIDDLE`/`END`, `COMPOUNDPERMITFLAG`, `COMPOUNDFORBIDFLAG`, `COMPOUNDWORDMAX`, +//! the `CHECKCOMPOUND…` checks, `FORCEUCASE`…): see [`compound`]. +//! Morphological data (`AM` aliases, the fields after a stem) is skipped. Rejected: the other +//! directives that change which words are correct (`COMPLEXPREFIXES`, `COMPOUNDMORESUFFIXES`…). //! //! Each distinct flag gets a dense `u16` id; a stem's flags and a rule's continuation classes are //! a sorted set of ids, interned once in a flat table, so membership is a binary search in a few @@ -40,6 +41,8 @@ use std::hash::Hasher; use crate::patterns::{FxHasher, FxMap}; +mod compound; + const MAGIC: &str = "WCAFF1"; /// Most bytes the compact format inflates to, written or read. const MAX_INFLATED: usize = 64 << 20; @@ -79,17 +82,7 @@ const MAX_SET_DATA: usize = 1 << 23; const MAX_DISTINCT_FLAGS: usize = 1 << 16; /// Directives that change which words are correct in ways this engine doesn't implement. -const UNSUPPORTED: &[&str] = &[ - "COMPLEXPREFIXES", - "COMPOUNDBEGIN", - "COMPOUNDEND", - "COMPOUNDFIRST", - "COMPOUNDFLAG", - "COMPOUNDLAST", - "COMPOUNDMIDDLE", - "COMPOUNDPERMITFLAG", - "PSEUDOROOT", -]; +const UNSUPPORTED: &[&str] = &["COMPLEXPREFIXES", "PSEUDOROOT"]; /// Most `COMPOUNDRULE` patterns kept, and most tokens in one: a hostile `.aff` can't make a /// compound check cost more than a few thousand stem lookups per word. @@ -113,7 +106,29 @@ enum Repeat { fn parse_compound_rule(pattern: &str, mode: FlagMode) -> Result, String> { let mut out: Vec<(FlagKey, Repeat)> = Vec::new(); let mut chars = pattern.chars().filter(|&c| c != EMOJI_SELECTOR); + // With long or numeric flags, what isn't a flag in parentheses (the Mongolian dictionary + // writes such rules) is read as the CLI answers (black-box): a bracketed list (`[a0,a1]`) is + // skipped with its `*` or `?`, and a punctuation mark (`%?`, `.`) makes a rule that matches + // nothing (a token no stem carries). + let mut skipped = false; while let Some(c) = chars.next() { + if mode != FlagMode::Char { + let skip = match c { + '[' => { + if !chars.by_ref().any(|c| c == ']') { + return Err(format!("COMPOUNDRULE `{pattern}`: `[` is not closed")); + } + true + } + '*' | '?' => skipped, + c if !c.is_alphanumeric() && !matches!(c, '(' | ')') => return Ok(vec![(FlagKey::MAX, Repeat::Once)]), + _ => false, + }; + skipped = skip; + if skip { + continue; + } + } match c { '*' | '?' => { let Some(last) = out.last_mut().filter(|t| t.1 == Repeat::Once) else { @@ -133,6 +148,9 @@ fn parse_compound_rule(pattern: &str, mode: FlagMode) -> Result out.push((FlagKey::from(c), Repeat::Once)), ')' => return Err(format!("COMPOUNDRULE `{pattern}`: `)` closes no group")), c if mode == FlagMode::Char => out.push((FlagKey::from(c), Repeat::Once)), _ => return Err(format!("COMPOUNDRULE `{pattern}`: long and numeric flags are written in ( )")), @@ -257,6 +275,9 @@ struct SfxGroup { rules: Vec, /// The rules of `rules` with continuation classes: the only ones a chain can go through. cont_rules: Vec, + /// The rules of `rules` whose flag some continuation class holds: the only ones that can be + /// the outer suffix of two in a row. + outer_rules: Vec, } /// A stem list with affix rules (see the module docs). @@ -334,6 +355,12 @@ pub struct AffixDict { /// `COMPOUNDRULE`), so a word starting otherwise is no compound: SCOWL's ordinals start with /// a digit, and a misspelled word costs one lookup here instead of one per cut. compound_firsts: Vec, + /// Flag-based compounding (`COMPOUNDFLAG`…), if the dictionary compounds by flags. + cpd: Option>, + /// Appended string → the prefix rules adding it, in rule order. + pfx_index: FxMap, Vec>, + /// Longest appended prefix, in chars. + max_pfx_chars: usize, } /// An `ICONV` or `OCONV` table: `(from, to)` pairs in source order, found by the first character @@ -492,6 +519,8 @@ struct Rules { only_in_compound: Option, /// `COMPOUNDMIN`: the shortest part of a compound, in characters (Hunspell's default is 3). compound_min: Option, + /// Flag-based compounding directives. + cpd: compound::Raw, /// `AF` flag aliases: alias `n` (1-based) stands for `aliases[n - 1]`. aliases: Vec, /// The `AF` table opened with its count line, so flag fields are alias numbers. A table @@ -749,7 +778,7 @@ impl FlagMode { /// `SUBSTANDARD` (suggestions), `KEEPCASE` (case variants of a word) or `NEEDAFFIX` (a stem on /// its own). #[derive(Clone, Copy, Debug, Default)] -struct Excluded([Option; 5]); +struct Excluded([Option; 5], bool); impl Excluded { /// Does the flag set `set` carry an excluded flag? @@ -757,6 +786,16 @@ impl Excluded { self.0.iter().flatten().any(|f| has(set, *f)) } + /// These exclusions, and no flag-based compounds (`MAXCPDSUGS 0` for suggestions). + fn without_compounds(mut self) -> Excluded { + self.1 = true; + self + } + + fn no_compounds(self) -> bool { + self.1 + } + /// These exclusions and `flag` too. fn and(mut self, flag: Option) -> Excluded { if flag.is_some() @@ -809,16 +848,8 @@ fn intersects(a: &[u16], b: &[u16]) -> bool { small.iter().any(|f| has(large, *f)) } -/// Directives an English dictionary may carry that this engine doesn't implement but that don't -/// change which English words are right, ignored by [`AffixDict::from_hunspell_english`]: -/// flag-based compounding (English dictionaries compound only ordinals, by `COMPOUNDRULE`, which -/// is read). Output conversion (`OCONV`) is read, for English as for every language: suggestions -/// are written as the dictionary says (`don’t`). -const IGNORED_IN_ENGLISH: &[&str] = - &["COMPOUNDBEGIN", "COMPOUNDEND", "COMPOUNDFIRST", "COMPOUNDFLAG", "COMPOUNDLAST", "COMPOUNDMIDDLE", "COMPOUNDPERMITFLAG"]; - -/// Read Hunspell `.aff` text (already decoded to UTF-8). `english`: ignore the directives in -/// [`IGNORED_IN_ENGLISH`] instead of refusing them. +/// Read Hunspell `.aff` text (already decoded to UTF-8). `english`: ignore the +/// flag-based compounding directives (see [`compound::is_directive`]). fn parse_aff(aff: &str, english: bool) -> Result { let mut r = Rules::default(); let mut cross: HashMap<(bool, FlagKey), bool> = HashMap::new(); @@ -836,8 +867,8 @@ fn parse_aff(aff: &str, english: bool) -> Result { } let at = |k: usize| f.get(k).copied().unwrap_or(""); match dir { - "SET" | "LANG" | "NAME" | "VERSION" | "HOME" | "WORDCHARS" | "MAXNGRAMSUGS" | "MAXDIFF" | "ONLYMAXDIFF" | "NOSPLITSUGS" - | "SUGSWITHDOTS" | "WARN" | "FORBIDWARN" | "MAXCPDSUGS" => {} + "SET" | "NAME" | "VERSION" | "HOME" | "WORDCHARS" | "MAXNGRAMSUGS" | "MAXDIFF" | "ONLYMAXDIFF" | "SUGSWITHDOTS" | "WARN" + | "FORBIDWARN" => {} // Morphological aliases, and a flag that only matters for morphological generation: // the words it marks are correct. "AM" | "LEMMA_PRESENT" => {} @@ -935,9 +966,16 @@ fn parse_aff(aff: &str, english: bool) -> Result { r.prefixes.push(rule); } } - d if english && IGNORED_IN_ENGLISH.contains(&d) => {} + // English dictionaries compound only ordinals, by `COMPOUNDRULE` (read above): their + // flag-based compounding directives don't change which English words are right. + d if english && compound::is_directive(d) => {} d if UNSUPPORTED.contains(&d) => return Err(format!("line {}: {d} is not supported", n + 1)), - _ => {} + _ => { + let mut cpd = std::mem::take(&mut r.cpd); + let read = cpd.read(&f, &mut |x| r.one_flag(x)); + r.cpd = cpd; + read.map_err(|e| format!("line {}: {e}", n + 1))?; + } } } // Aliases may be declared after the rules that use them; resolve once all are read. @@ -1075,7 +1113,8 @@ impl AffixDict { // NEEDAFFIX and CIRCUMFIX flags may only ever appear in rules' continuation classes. let directives = [rules.forbidden, rules.no_suggest, rules.substandard, rules.keepcase, rules.only_in_compound, rules.need_affix, rules.circumfix]; - for k in all.chain(directives.into_iter().flatten()) { + let cpd_keys = rules.cpd.keys(mode)?; + for k in all.chain(directives.into_iter().flatten()).chain(cpd_keys) { distinct.entry(k).or_insert(0); if distinct.len() > MAX_DISTINCT_FLAGS { return Err(format!("more than {MAX_DISTINCT_FLAGS} distinct flags")); @@ -1122,6 +1161,7 @@ impl AffixDict { compound_text: rules.compound_rules, only_in_compound: rules.only_in_compound.and_then(id_of), compound_min: rules.compound_min.unwrap_or(3), + cpd: rules.cpd.build(mode, &id_of)?, ..AffixDict::default() }; d.offs.push(0); @@ -1142,6 +1182,10 @@ impl AffixDict { // NEEDAFFIX belongs to its homonym: the word stands alone if any homonym does // (`foo/NA` and `foo/B`: `foo` is right). let mut all_need = true; + // So does ONLYINCOMPOUND: a homonym with neither stands alone (German `schulen/DIXY` + // beside `schulen/Sozm`), but `buch/Sozm` beside `buch/hke` (NEEDAFFIX) doesn't, as + // in the CLI. + let mut alone = false; let mut j = i; while let Some((w, a, b, entry_is_word)) = stems.get(j) && w == word @@ -1159,6 +1203,7 @@ impl AffixDict { } is_word = true; all_need &= d.need_affix.is_some_and(|n| fresh.contains(&n)); + alone |= !d.only_in_compound.is_some_and(|o| fresh.contains(&o)) && !d.need_affix.is_some_and(|n| fresh.contains(&n)); fresh.sort_unstable(); fresh.dedup(); fresh.retain(|&f| !has(&merged, f)); @@ -1183,6 +1228,9 @@ impl AffixDict { if !all_need && let Some(n) = d.need_affix { merged.retain(|&f| f != n); } + if alone { + merged.retain(|&f| Some(f) != d.only_in_compound && Some(f) != d.need_affix); + } for &f in &carried { if has(&merged, f) { continue; @@ -1285,6 +1333,14 @@ impl AffixDict { } } self.sfx_by_flag = by_flag; + let mut pfx_index: FxMap, Vec> = FxMap::default(); + for (i, r) in self.prefixes.iter().enumerate() { + if let Ok(i) = u32::try_from(i) { + pfx_index.entry(r.add.clone()).or_default().push(i); + } + } + self.pfx_index = pfx_index; + self.max_pfx_chars = self.prefixes.iter().map(|r| r.add.chars().count()).max().unwrap_or(0); let mut cont_flag = vec![false; self.flag_keys.len()]; for r in self.prefixes.iter().chain(&self.suffixes) { for &f in self.set(r.cont) { @@ -1294,6 +1350,10 @@ impl AffixDict { } } self.has_cont = cont_flag.contains(&true); + for g in idx.values_mut().flatten() { + let outer = |i: &&u32| self.suffixes.get(**i as usize).is_some_and(|r| cont_flag.get(usize::from(r.flag)).copied().unwrap_or(false)); + g.outer_rules = g.rules.iter().filter(outer).copied().collect(); + } self.cont_flag = cont_flag; self.max_sfx_chars = self.suffixes.iter().map(|r| r.add.chars().count()).max().unwrap_or(0); self.sfx_index = idx; @@ -1366,7 +1426,12 @@ impl AffixDict { /// suggestions. fn excluded(&self, suggestible: bool) -> Excluded { let e = Excluded::default().and(self.forbidden); - if suggestible { e.and(self.no_suggest).and(self.substandard) } else { e } + if !suggestible { + return e; + } + let e = e.and(self.no_suggest).and(self.substandard); + // `MAXCPDSUGS 0`: compounds are right but never suggested. + if self.cpd.as_ref().is_some_and(|c| c.no_suggestions) { e.without_compounds() } else { e } } /// Flag set of a stem, if `w` is one. @@ -1382,8 +1447,19 @@ impl AffixDict { self.stem_set(w).filter(|set| !excluded.hits(set) && !self.only_in_compound.is_some_and(|f| has(set, f))) } + /// Is `w` listed with `FORBIDDENWORD`? Then it is no word, however else it could be built + /// (affixes, compounds), as in Hunspell. + fn is_forbidden(&self, w: &str) -> bool { + self.forbidden.is_some_and(|f| self.stem_set(w).is_some_and(|set| has(set, f))) + } + /// Flag set of the stem `w`, whatever flags it carries. fn stem_set(&self, w: &str) -> Option<&[u16]> { + self.stem_lookup(w).map(|(_, set)| set) + } + + /// Index and flag set of the stem `w`, whatever flags it carries. + fn stem_lookup(&self, w: &str) -> Option<(u32, &[u16])> { if self.table.is_empty() { return None; } @@ -1395,7 +1471,7 @@ impl AffixDict { return None; } if self.word(t as usize) == Some(w) { - return Some(self.set(*self.flags.get(t as usize)?)); + return Some((t, self.set(*self.flags.get(t as usize)?))); } slot = (slot + 1) & mask; } @@ -1444,7 +1520,7 @@ impl AffixDict { inner.clear(); for r in g.cont_rules.iter().filter_map(|&i| self.suffixes.get(i as usize)) { let meets = *seen.entry(r.cont).or_insert_with(|| probe.meets(self.set(r.cont))); - if meets && !self.cont_holds(r, self.circumfix) && r.cond_matches(&root, true) { + if meets && !self.cont_holds(r, self.circumfix) && !self.cont_holds(r, self.only_in_compound) && r.cond_matches(&root, true) { inner.push(r.flag); } } @@ -1489,7 +1565,20 @@ impl AffixDict { /// [`AffixDict::contains`] on normalized input, where stems carrying an `excluded` flag /// don't count (forbidden words; case variants of `KEEPCASE` stems). fn contains_except(&self, w: &str, excluded: Excluded) -> bool { - if w.is_empty() || w.len() > MAX_WORD_BYTES { + self.contains_in(w, excluded, matches!(casing(w), Casing::Initial | Casing::Upper)) + } + + /// [`AffixDict::contains_except`], where `capitalized` says whether the word as written + /// starts with a capital (`w` may be its lower-case form): a compound whose last part has + /// `FORCEUCASE` is right only then. + fn contains_in(&self, w: &str, excluded: Excluded, capitalized: bool) -> bool { + self.simple_word(w, excluded) || self.compound(w, excluded) || self.flag_compound(w, excluded, capitalized) + } + + /// Is `w` a stem or an affixed form of one (no compound)? Affixes that are only valid inside + /// a compound (`ONLYINCOMPOUND`) don't count. + fn simple_word(&self, w: &str, excluded: Excluded) -> bool { + if w.is_empty() || w.len() > MAX_WORD_BYTES || self.is_forbidden(w) { return false; } // A stem that needs an affix is no word on its own. @@ -1497,9 +1586,17 @@ impl AffixDict { return true; } let mut root = String::with_capacity(w.len() + 8); - for r in &self.prefixes { + // The prefix rules whose appended text starts `w`, found by that text (one hash lookup per + // length) rather than by trying every rule. + let ends = std::iter::once(0).chain(w.char_indices().map(|(i, c)| i + c.len_utf8())); + let rules = ends + .take(self.max_pfx_chars + 1) + .filter_map(|b| self.pfx_index.get(w.get(..b)?)) + .flatten() + .filter_map(|&i| self.prefixes.get(i as usize)); + for r in rules { let Some(rest) = w.strip_prefix(&*r.add) else { continue }; - if rest.is_empty() && !self.full_strip { + if (rest.is_empty() && !self.full_strip) || self.cont_holds(r, self.only_in_compound) { continue; } root.clear(); @@ -1524,7 +1621,7 @@ impl AffixDict { return true; } } - self.compound(w, excluded) + false } /// Is `w` a compound of two or more stems (no affixes) whose flags spell out one of the @@ -1536,7 +1633,7 @@ impl AffixDict { if self.compound_rules.is_empty() || w.len() > MAX_COMPOUND_BYTES || !w.chars().next().is_some_and(|c| self.compound_firsts.contains(&c)) - || self.forbidden.is_some_and(|f| self.stem_set(w).is_some_and(|set| has(set, f))) + || self.is_forbidden(w) { return false; } @@ -1573,7 +1670,11 @@ impl AffixDict { fn suffixed(&self, w: &str, need: Option, circ: bool, pfx_needs: bool, excluded: Excluded) -> bool { // Fits the suffix rule `r`: circumfix alike, and in need of another affix only when a // prefix that doesn't is there. - let fits = |r: &Rule| self.cont_holds(r, self.circumfix) == circ && (!self.cont_holds(r, self.need_affix) || (need.is_some() && !pfx_needs)); + let fits = |r: &Rule| { + self.cont_holds(r, self.circumfix) == circ + && (!self.cont_holds(r, self.need_affix) || (need.is_some() && !pfx_needs)) + && !self.cont_holds(r, self.only_in_compound) + }; let mut root = String::with_capacity(w.len() + 8); let mut flags: Vec = Vec::new(); // Each split point from the end: the appended string is `w[b..]`, at most the longest @@ -1609,7 +1710,11 @@ impl AffixDict { // Only a flag some continuation class holds can come from another rule. The // outer suffix of a chain has another affix under it, so its NEEDAFFIX is met, // and Hunspell ignores its CIRCUMFIX there (`workables` from `s/c`). - flags.extend(rules().filter(|r| self.is_cont_flag(r.flag) && r.cond_matches(&root, true)).map(|r| r.flag)); + flags.extend( + rules() + .filter(|r| self.is_cont_flag(r.flag) && !self.cont_holds(r, self.only_in_compound) && r.cond_matches(&root, true)) + .map(|r| r.flag), + ); flags.sort_unstable(); flags.dedup(); if !flags.is_empty() && self.form_has_flag(&root, &flags, INNER_CHAIN, excluded) { @@ -1656,12 +1761,12 @@ impl AffixDict { let lower = w.to_lowercase(); // CHECKSHARPS: a KEEPCASE word with `ß` may be capitalized (`Gruß` for `Gruß/k`). let ex = if self.check_sharps && lower.contains('ß') { excluded } else { keep }; - self.contains_except(&lower, ex) + self.contains_in(&lower, ex, true) } Casing::Upper => { let lower = w.to_lowercase(); self.contains_except(&capitalize(&lower), keep) - || self.contains_except(&lower, keep) + || self.contains_in(&lower, keep, true) || (self.check_sharps && self.sharps(&lower, excluded)) } } @@ -1712,7 +1817,7 @@ impl AffixDict { let mut queued: std::collections::HashSet<(String, u32)> = std::collections::HashSet::new(); let mut work = 0usize; for r in &self.suffixes { - if !has(f, r.flag) || !r.cond_matches(stem, true) { + if !has(f, r.flag) || !r.cond_matches(stem, true) || self.cont_holds(r, self.only_in_compound) { continue; } let Some(base) = stem.strip_suffix(&*r.strip) else { continue }; @@ -1750,7 +1855,7 @@ impl AffixDict { if work > MAX_FORMS_WORK { break 'chain; } - if !r.cond_matches(&form, true) { + if !r.cond_matches(&form, true) || self.cont_holds(r, self.only_in_compound) { continue; } let Some(base) = form.strip_suffix(&*r.strip) else { continue }; @@ -1770,7 +1875,7 @@ impl AffixDict { } } } - for r in &self.prefixes { + for r in self.prefixes.iter().filter(|r| !self.cont_holds(r, self.only_in_compound)) { let on_stem = has(f, r.flag); let circ = self.cont_holds(r, self.circumfix); let needs = self.cont_holds(r, self.need_affix); @@ -1933,7 +2038,7 @@ impl AffixDict { out.push('\n'); } }; - if self.compound_text.is_empty() { + if self.compound_text.is_empty() && self.cpd.is_none() { write_only_in_compound(&mut out); } else { out.push_str(&format!("COMPOUNDMIN {}\n", self.compound_min)); @@ -1942,6 +2047,9 @@ impl AffixDict { out.push_str(&format!("COMPOUNDRULE {p}\n")); } } + if let Some(c) = &self.cpd { + c.write(&mut out, &|k, out: &mut String| self.flag_mode.write(std::iter::once(k), out)); + } let z = |s: &str| if s.is_empty() { "0".to_string() } else { s.to_string() }; for (kind, rules) in [("PFX", &self.prefixes), ("SFX", &self.suffixes)] { for r in rules { @@ -2034,6 +2142,12 @@ impl AffixDict { } rules.compound_rules.push(at(1).to_string()); } + d if compound::is_directive(d) => { + let mut cpd = std::mem::take(&mut rules.cpd); + let read = cpd.read(&f, &mut |x| rules.one_flag(x)); + rules.cpd = cpd; + read.map_err(|e| format!("affix dictionary: bad `{line}`: {e}"))?; + } "MAP" => rules.map.push(at(1).chars().collect()), "REP" => rules.rep.push((at(1).replace('_', " "), at(2).replace('_', " "))), kind @ ("PFX" | "SFX") => { @@ -3029,7 +3143,7 @@ pub(crate) mod tests { /// craft-libs#482 review: an English dictionary with compounding and output conversion, as the /// usual en_US and en_GB ones have, loads for English: `COMPOUNDRULE` makes its ordinals, its /// `ONLYINCOMPOUND` stems are words only inside them, `COMPOUNDFLAG` is ignored (other - /// languages still refuse it) and `OCONV` is applied to suggestions. (A synthetic `.aff` + /// languages read it) and `OCONV` is applied to suggestions. (A synthetic `.aff` /// written for this test.) #[test] fn english_dictionaries_ignore_compounding_and_output_conversion() { @@ -3037,7 +3151,7 @@ pub(crate) mod tests { COMPOUNDMIN 1\nONLYINCOMPOUND c\nCOMPOUNDRULE 2\nCOMPOUNDRULE n*1t\nCOMPOUNDRULE n*mp\nCOMPOUNDFLAG Z\n\ WORDCHARS 0123456789\nSFX S Y 1\nSFX S 0 s [^sxzhy]\nSFX D Y 1\nSFX D 0 ed [^ey]\n"; let dic = "7\nhouse/S\nwalk/SD\n0th/pc\n1st/p\n1th/tc\n1/n1\nthe\n"; - assert!(AffixDict::from_hunspell(aff, dic, &[]).is_err(), "refused for other languages"); + assert!(AffixDict::from_hunspell(&format!("{aff}COMPOUNDMORESUFFIXES\n"), dic, &[]).is_err(), "refused with COMPOUNDFLAG"); let d = AffixDict::from_hunspell_english(aff, dic, &[]).unwrap(); for w in ["house", "houses", "walked", "walks", "the"] { assert!(d.check(w), "{w}"); @@ -3073,7 +3187,7 @@ pub(crate) mod tests { assert!(AffixDict::from_hunspell("FLAG long\n", DIC, &[]).is_ok()); assert!(AffixDict::from_hunspell("FLAG num\n", DIC, &[]).is_ok()); assert!(AffixDict::from_hunspell("FLAG bogus\n", DIC, &[]).is_err()); - assert!(AffixDict::from_hunspell("COMPOUNDFLAG X\n", DIC, &[]).is_err()); + assert!(AffixDict::from_hunspell("COMPOUNDFLAG X\nCHECKCOMPOUNDPATTERN 1\nCHECKCOMPOUNDPATTERN a b c\n", DIC, &[]).is_err()); assert!(AffixDict::from_hunspell("SFX a Y 1\nSFX a 0 y [ab\n", DIC, &[]).is_err()); assert!(AffixDict::from_bytes(b"not deflate").is_err()); let ok = dict().to_bytes().unwrap(); diff --git a/crates/proof/src/affix/compound.rs b/crates/proof/src/affix/compound.rs new file mode 100644 index 00000000..31be0c8b --- /dev/null +++ b/crates/proof/src/affix/compound.rs @@ -0,0 +1,1491 @@ +//! Flag-based compounding, as the hunspell(5) man page describes it and the `hunspell` 1.7.5 CLI +//! answers on synthetic dictionaries (black-box; every rule below has a test pinned to the CLI's +//! answer in the tests below). +//! +//! A compound is two or more parts, each at least `COMPOUNDMIN` characters, each a stem or an +//! affixed form of one whose flags (the stem's, and the continuation classes of its affixes) +//! allow its place: `COMPOUNDFLAG` anywhere, `COMPOUNDBEGIN` first, `COMPOUNDMIDDLE` inside, +//! `COMPOUNDEND` last. (`COMPOUNDFIRST` and `COMPOUNDLAST` are ignored, as the CLI does.) +//! +//! - A prefix may sit on any part but the last and a suffix only on the last, unless the affix +//! carries `COMPOUNDPERMITFLAG`. Two suffixes in a row (continuation classes) only on the last. +//! - An affix carrying `ONLYINCOMPOUND` is only valid inside a compound: a prefix anywhere, a +//! suffix anywhere but on the last part (the CLI's answers; the word alone is no word). +//! - `COMPOUNDFORBIDFLAG` on a stem or an affix: that part may only be last. +//! - A stem with `NEEDAFFIX` is a part only with an affix; `FORBIDDENWORD` stems are no part, and +//! a forbidden word is never rebuilt from parts. +//! - `COMPOUNDWORDMAX`: most parts (a `COMPOUNDROOT` stem counts as two). With `COMPOUNDSYLLABLE` +//! and `LANG hu`, more parts are allowed while the word has at most that many vowels; with +//! `COMPOUNDSYLLABLE` and another language the CLI applies no limit, and neither does this. +//! - `CHECKCOMPOUNDDUP`: the last two parts may not be the same stem (`foofoo`; `foofoobar` is +//! fine, as in the CLI). +//! - `CHECKCOMPOUNDCASE`: no capital on either side of a boundary, unless the other side is `-`. +//! - `CHECKCOMPOUNDTRIPLE`: no three equal letters across a boundary (`schiff|fahrt`). +//! `SIMPLIFIEDTRIPLE`: a part ending in a double letter may share its last letter with the next +//! part (`schiffahrt` = `schiff` + `fahrt`). +//! - `CHECKCOMPOUNDPATTERN end[/flag] begin[/flag]`: no boundary where the part before ends with +//! `end` and the part after begins with `begin` (and they carry the flags); `0` for `end` means +//! an unaffixed part. The optional replacement field is refused. +//! - `CHECKCOMPOUNDREP`: no compound that a `REP` replacement turns into a word (not a compound), +//! checked on every two neighbouring parts and on every run of parts that ends the word. +//! - A compound of three or more parts is rejected when one slip turns it into a word (not a +//! compound): see [`AffixDict::near_word`]. The CLI's answers, with or without +//! `CHECKCOMPOUNDREP`. +//! - `FORCEUCASE` on the last part: only the capitalized word (or the word in capitals) is right. +//! - `MAXCPDSUGS 0`: compounds are never suggested. +//! +//! Not read: `SYLLABLENUM` (Hungarian), which the man page doesn't describe; it is ignored. +//! +//! The work per word is bounded: words up to [`MAX_WORD_BYTES`](super::MAX_WORD_BYTES), at most +//! [`MAX_PARTS`] parts, each part's analyses computed once, and at most [`MAX_WORK`] steps (rules +//! tried and splits followed) per word; past that the word is no compound. + +use super::{AffixDict, Excluded, FlagKey, FlagMode, Rule, has}; + +/// Most parts a compound may have, whatever `COMPOUNDWORDMAX` says. +pub(super) const MAX_PARTS: usize = 16; +/// Most steps one compound check may take (affix rules tried, parts looked up, splits followed). +pub(super) const MAX_WORK: usize = 1 << 16; +/// Most `CHECKCOMPOUNDPATTERN` lines (the Dutch dictionary has 52). +const MAX_PATTERNS: usize = 512; +/// Longest text on either side of a `CHECKCOMPOUNDPATTERN`, in bytes. +const MAX_PATTERN_BYTES: usize = 64; +/// Most vowels a `COMPOUNDSYLLABLE` line may list. +const MAX_VOWELS: usize = 64; +/// Most `MAP` variants, `TRY` letters and `KEY` row letters the three-part check tries. +const MAX_MAP_VARIANTS: usize = 256; +const MAX_TRY: usize = 128; +const MAX_KEY_ROW: usize = 128; +/// The keyboard rows the CLI assumes when a dictionary has no `KEY`. +const DEFAULT_KEY: &str = "qwertyuiop|asdfghjkl|zxcvbnm"; + +/// The part may come first, in the middle, last. +const FIRST: u8 = 1; +const MIDDLE: u8 = 2; +const LAST: u8 = 4; + +/// The single-flag compounding directives, in the order [`Raw::flags`] keeps them. +const FLAG_DIRECTIVES: [&str; 8] = + ["COMPOUNDFLAG", "COMPOUNDBEGIN", "COMPOUNDMIDDLE", "COMPOUNDEND", "COMPOUNDPERMITFLAG", "COMPOUNDFORBIDFLAG", "COMPOUNDROOT", "FORCEUCASE"]; +const FLAG: usize = 0; +const BEGIN: usize = 1; +const MIDDLE_FLAG: usize = 2; +const END: usize = 3; +const PERMIT: usize = 4; +const FORBID: usize = 5; +const ROOT: usize = 6; +const FORCE_UCASE: usize = 7; + +/// The yes/no compounding directives, in the order [`Raw::switches`] keeps them. +const SWITCHES: [&str; 5] = ["CHECKCOMPOUNDDUP", "CHECKCOMPOUNDREP", "CHECKCOMPOUNDCASE", "CHECKCOMPOUNDTRIPLE", "SIMPLIFIEDTRIPLE"]; +const DUP: usize = 0; +const REP: usize = 1; +const CASE: usize = 2; +const TRIPLE: usize = 3; +const SIMPLIFIED: usize = 4; + +/// Compounding directives as read from the `.aff` or the compact format, before flags get ids. +#[derive(Clone, Debug, Default)] +pub(super) struct Raw { + flags: [Option; 8], + switches: [bool; 5], + word_max: Option, + /// `COMPOUNDSYLLABLE`: most vowels, and the vowels. + syllable: Option<(usize, String)>, + /// `LANG hu…`: the Hungarian syllable rule applies. + hungarian: bool, + /// `CHECKCOMPOUNDPATTERN` lines, as written (`end[/flag] begin[/flag]`). + patterns: Vec<(String, String)>, + pattern_header: bool, + /// `MAXCPDSUGS 0`. + no_suggestions: bool, + /// `NOSPLITSUGS`. + no_split_suggestions: bool, + /// `COMPOUNDMORESUFFIXES`: refused with flag-based compounding, which it would change; + /// ignored without (Korean has it beside `COMPOUNDRULE`s only). + more_suffixes: bool, +} + +impl Raw { + /// Read the compounding directive `dir` of the line `f` (its fields); `Ok(false)` when `dir` + /// is none. `one_flag` reads a flag in the dictionary's `FLAG` mode. + pub(super) fn read(&mut self, f: &[&str], one_flag: &mut impl FnMut(&str) -> Result) -> Result { + let Some(&dir) = f.first() else { return Ok(false) }; + let at = |k: usize| f.get(k).copied().unwrap_or(""); + if let Some(i) = FLAG_DIRECTIVES.iter().position(|&d| d == dir) { + let k = one_flag(at(1)).map_err(|e| format!("{dir} needs one flag: {e}"))?; + if let Some(slot) = self.flags.get_mut(i) { + *slot = Some(k); + } + return Ok(true); + } + if let Some(i) = SWITCHES.iter().position(|&d| d == dir) { + if let Some(s) = self.switches.get_mut(i) { + *s = true; + } + return Ok(true); + } + match dir { + // `COMPOUNDWORDMAX 2 y` (Hungarian): the number is the first field. + "COMPOUNDWORDMAX" => { + let n = at(1).parse::().map_err(|_| "COMPOUNDWORDMAX needs a number".to_string())?; + self.word_max = Some(n.clamp(1, MAX_PARTS)); + } + "COMPOUNDSYLLABLE" => { + let n = at(1).parse::().map_err(|_| "COMPOUNDSYLLABLE needs a number".to_string())?; + let vowels: String = at(2).chars().take(MAX_VOWELS).collect(); + self.syllable = Some((n.min(super::MAX_WORD_BYTES), vowels)); + } + "LANG" => self.hungarian = at(1) == "hu" || at(1).starts_with("hu_") || at(1).starts_with("hu-"), + "MAXCPDSUGS" => self.no_suggestions = at(1) == "0", + "NOSPLITSUGS" => self.no_split_suggestions = true, + "COMPOUNDMORESUFFIXES" => self.more_suffixes = true, + "CHECKCOMPOUNDPATTERN" => self.read_pattern(f)?, + _ => return Ok(false), + } + Ok(true) + } + + /// A `CHECKCOMPOUNDPATTERN` line: the count line, or `end[/flag] begin[/flag]` with an + /// optional comment (`# …`). A replacement field is refused: no Language pack uses one. + fn read_pattern(&mut self, f: &[&str]) -> Result<(), String> { + let fields: Vec<&str> = f.iter().skip(1).copied().take_while(|x| !x.starts_with('#')).collect(); + if !self.pattern_header && fields.len() == 1 && fields.first().is_some_and(|x| x.parse::().is_ok()) { + self.pattern_header = true; + return Ok(()); + } + let [end, begin] = fields.as_slice() else { + return Err(format!("CHECKCOMPOUNDPATTERN `{}`: only `end[/flag] begin[/flag]` is supported", fields.join(" "))); + }; + if end.len() > MAX_PATTERN_BYTES || begin.len() > MAX_PATTERN_BYTES || self.patterns.len() >= MAX_PATTERNS { + return Err(format!("CHECKCOMPOUNDPATTERN: over {MAX_PATTERNS} patterns or {MAX_PATTERN_BYTES} bytes")); + } + self.patterns.push(((*end).to_string(), (*begin).to_string())); + Ok(()) + } + + /// Whether any directive makes words compound by flags (the rest only refine that). + fn compounds(&self) -> bool { + [FLAG, BEGIN, MIDDLE_FLAG, END].iter().any(|&i| self.flags.get(i).is_some_and(Option::is_some)) + } + + /// Every flag the directives and patterns name, so that each gets an id. + pub(super) fn keys(&self, mode: FlagMode) -> Result, String> { + let mut keys: Vec = self.flags.iter().flatten().copied().collect(); + for (end, begin) in &self.patterns { + for side in [end, begin] { + if let Some((_, flag)) = side.split_once('/') { + keys.push(mode.parse_one(flag).map_err(|e| format!("CHECKCOMPOUNDPATTERN `{side}`: {e}"))?); + } + } + } + Ok(keys) + } + + /// The compounding of a dictionary whose flags got ids by `id_of`; `None` when it doesn't + /// compound by flags. + pub(super) fn build(&self, mode: FlagMode, id_of: &impl Fn(FlagKey) -> Option) -> Result>, String> { + if !self.compounds() { + return Ok(None); + } + if self.more_suffixes { + return Err("COMPOUNDMORESUFFIXES is not supported with flag-based compounding".into()); + } + let id = |i: usize| self.flags.get(i).copied().flatten().and_then(id_of); + let switch = |i: usize| self.switches.get(i).copied().unwrap_or(false); + let mut c = Compounding { + flag: id(FLAG), + begin: id(BEGIN), + middle: id(MIDDLE_FLAG), + end: id(END), + permit: id(PERMIT), + forbid: id(FORBID), + root: id(ROOT), + force_ucase: id(FORCE_UCASE), + dup: switch(DUP), + rep: switch(REP), + case: switch(CASE), + triple: switch(TRIPLE), + simplified: switch(SIMPLIFIED), + no_suggestions: self.no_suggestions, + two_words: !self.no_split_suggestions, + raw: self.clone(), + ..Compounding::default() + }; + // `COMPOUNDSYLLABLE` turns `COMPOUNDWORDMAX` into the Hungarian syllable rule, or (other + // languages) lifts it: the CLI's answers. + match (&self.syllable, self.hungarian) { + (None, _) => c.word_max = self.word_max, + (Some((n, vowels)), true) => { + c.word_max = self.word_max; + c.syllables = Some((*n, vowels.chars().collect())); + } + (Some(_), false) => {} + } + if let Some(f) = c.force_ucase { + c.marks.push(f); + } + for (end, begin) in &self.patterns { + let side = |text: &str, c: &mut Compounding| -> Result<(String, Need), String> { + let (chars, flag) = text.split_once('/').map_or((text, None), |(a, b)| (a, Some(b))); + let need = match flag { + None => Need::Any, + Some(flag) => match id_of(mode.parse_one(flag)?) { + Some(f) => Need::Mark(c.mark(f)?), + None => Need::Never, + }, + }; + Ok((chars.to_string(), need)) + }; + let (end_chars, end_need) = side(end, &mut c)?; + let (begin, begin_need) = side(begin, &mut c)?; + let unmodified = end_chars == "0"; + let end = if unmodified { String::new() } else { end_chars }; + c.patterns.push(Pattern { end: end.into(), end_need, begin: begin.into(), begin_need, unmodified }); + } + Ok(Some(Box::new(c))) + } +} + +/// What a side of a `CHECKCOMPOUNDPATTERN` asks of a part's flags. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Need { + Any, + /// The flag of [`Compounding::marks`] at this index. + Mark(u8), + /// A flag no stem or rule carries. + Never, +} + +impl Need { + fn met(self, marks: u64) -> bool { + match self { + Need::Any => true, + Need::Mark(i) => marks & (1 << i) != 0, + Need::Never => false, + } + } +} + +#[derive(Clone, Debug)] +struct Pattern { + end: Box, + end_need: Need, + begin: Box, + begin_need: Need, + /// `0` for `end`: the part before is a stem without affixes. + unmodified: bool, +} + +/// The flag-based compounding of a dictionary (see the module docs). +#[derive(Clone, Debug, Default)] +pub(super) struct Compounding { + flag: Option, + begin: Option, + middle: Option, + end: Option, + permit: Option, + forbid: Option, + root: Option, + force_ucase: Option, + word_max: Option, + /// The Hungarian rule: past `word_max` parts, at most this many vowels in the word. + syllables: Option<(usize, Box<[char]>)>, + dup: bool, + rep: bool, + case: bool, + triple: bool, + simplified: bool, + pub(super) no_suggestions: bool, + /// A compound of three or more parts that splits into two words is wrong (unless + /// `NOSPLITSUGS`: the CLI ties the two to each other). + two_words: bool, + patterns: Vec, + /// Flags tracked per part (`FORCEUCASE`, the patterns' flags): bit `i` of [`Info::marks`]. + marks: Vec, + /// The directives as read, written back by [`Compounding::write`]. + raw: Raw, +} + +impl Compounding { + /// The mark index of `flag`, adding it. + fn mark(&mut self, flag: u16) -> Result { + let i = match self.marks.iter().position(|&m| m == flag) { + Some(i) => i, + None => { + self.marks.push(flag); + self.marks.len() - 1 + } + }; + u8::try_from(i).ok().filter(|&i| i < 64).ok_or_else(|| "CHECKCOMPOUNDPATTERN: more than 63 distinct flags".to_string()) + } + + /// The directives as compact-format lines (flags written by `write_flag`). + pub(super) fn write(&self, out: &mut String, write_flag: &impl Fn(FlagKey, &mut String)) { + let raw = &self.raw; + if raw.hungarian { + out.push_str("LANG hu\n"); + } + for (name, key) in FLAG_DIRECTIVES.iter().zip(&raw.flags) { + if let Some(k) = key { + out.push_str(name); + out.push(' '); + write_flag(*k, out); + out.push('\n'); + } + } + for (name, on) in SWITCHES.iter().zip(&raw.switches) { + if *on { + out.push_str(name); + out.push('\n'); + } + } + if let Some(n) = raw.word_max { + out.push_str(&format!("COMPOUNDWORDMAX {n}\n")); + } + if let Some((n, vowels)) = &raw.syllable { + out.push_str(&format!("COMPOUNDSYLLABLE {n} {vowels}\n")); + } + if raw.no_suggestions { + out.push_str("MAXCPDSUGS 0\n"); + } + if raw.no_split_suggestions { + out.push_str("NOSPLITSUGS\n"); + } + for (end, begin) in &raw.patterns { + out.push_str(&format!("CHECKCOMPOUNDPATTERN {end} {begin}\n")); + } + } +} + +/// One way a piece of the word can be a compound part. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct Info { + /// Where it may sit: [`FIRST`], [`MIDDLE`], [`LAST`]. + pos: u8, + /// Which of [`Compounding::marks`] its flags hold. + marks: u64, + /// No affix. + unmodified: bool, + /// The stem's index (`CHECKCOMPOUNDDUP` compares stems). + stem: u32, + /// How many words it counts as (`COMPOUNDROOT`: two). + words: u8, +} + +/// A stem found for a part, and the affixes that make the part from it. +#[derive(Clone, Copy)] +struct Analysis<'a> { + stem: u32, + set: &'a [u16], + pfx: Option<&'a Rule>, + sfx: Option<&'a Rule>, + /// The inner suffix of two in a row (`sfx` is the outer one). + inner: Option<&'a Rule>, +} + +/// One compound check: the word, its parts' analyses (each piece looked up once), the parts +/// chosen so far and the work done. +struct Search<'a> { + c: &'a Compounding, + w: &'a str, + /// Byte offset of each char, and `w.len()`. + cuts: Vec, + chars: Vec, + excluded: Excluded, + capitalized: bool, + /// Per piece `(start, end)` in chars: its infos as a range of `infos`. + memo: Vec>, + infos: Vec, + /// The parts so far: start and end char, and the info. + path: Vec<(usize, usize, Info)>, + work: usize, + /// [`AffixDict::near_word`], once answered. + near: Option, +} + +impl AffixDict { + /// Is `w` a compound by the flag-based rules (see the module docs)? `capitalized`: the word + /// as the user wrote it starts with a capital (`FORCEUCASE`). + pub(super) fn flag_compound(&self, w: &str, excluded: Excluded, capitalized: bool) -> bool { + let Some(c) = self.cpd.as_deref() else { return false }; + if excluded.no_compounds() || w.is_empty() || w.len() > super::MAX_WORD_BYTES { + return false; + } + if self.is_forbidden(w) { + return false; + } + let cuts: Vec = w.char_indices().map(|(i, _)| i).chain(std::iter::once(w.len())).collect(); + let chars: Vec = w.chars().collect(); + let n = cuts.len(); + let mut s = Search { + c, + w, + cuts, + chars, + excluded, + capitalized, + memo: vec![None; n * n], + infos: Vec::new(), + path: Vec::with_capacity(MAX_PARTS), + work: 0, + near: None, + }; + self.parts_from(&mut s, 0) + } + + /// Can the rest of the word, from char `start`, be split into parts after `s.path`? + fn parts_from(&self, s: &mut Search, start: usize) -> bool { + let total = s.chars.len(); + let count = s.path.len(); + let min = self.compound_min.max(1); + for end in start.saturating_add(min)..=total { + s.work += 1; + if s.work > MAX_WORK { + return false; + } + let last = end == total; + if !last && count + 1 >= MAX_PARTS { + break; + } + let place = if last { + LAST + } else if count == 0 { + FIRST + } else { + MIDDLE + }; + let (a, b) = self.infos_of(s, start, end); + for k in a..b { + let Some(info) = s.infos.get(k as usize).copied() else { continue }; + if info.pos & place == 0 || !self.boundary_ok(s, start, &info) { + continue; + } + s.path.push((start, end, info)); + let ok = if last { self.whole_ok(s) } else { self.words_ok(s, false) && self.next_parts(s, end) }; + s.path.pop(); + if ok { + return true; + } + } + } + false + } + + /// After a part ending at char `end`: the next part starts there, or (`SIMPLIFIEDTRIPLE`) + /// one char earlier when the part ends in a double letter. + fn next_parts(&self, s: &mut Search, end: usize) -> bool { + if self.parts_from(s, end) { + return true; + } + let double = end >= 2 && s.chars.get(end - 1).is_some_and(|c| s.chars.get(end - 2) == Some(c)); + s.c.simplified && double && self.parts_from(s, end - 1) + } + + /// The analyses of the piece from char `a` to char `b`, computed once: a range of `s.infos`. + fn infos_of(&self, s: &mut Search, a: usize, b: usize) -> (u32, u32) { + let n = s.cuts.len(); + let slot = a.saturating_mul(n).saturating_add(b); + if let Some(Some(r)) = s.memo.get(slot) { + return *r; + } + let from = u32::try_from(s.infos.len()).unwrap_or(u32::MAX); + if let (Some(&x), Some(&y)) = (s.cuts.get(a), s.cuts.get(b)) + && let Some(piece) = s.w.get(x..y) + { + let mut found: Vec = Vec::new(); + let mut work = s.work; + self.analyses(piece, s.excluded, &mut work, &mut |an: Analysis| { + if let Some(info) = self.info(s.c, &an) + && !found.contains(&info) + { + found.push(info); + } + }); + s.work = work; + s.infos.extend(found); + } + let to = u32::try_from(s.infos.len()).unwrap_or(u32::MAX); + if let Some(m) = s.memo.get_mut(slot) { + *m = Some((from, to)); + } + (from, to) + } + + /// Where the analysis `an` lets its piece sit in a compound, if anywhere. + fn info(&self, c: &Compounding, an: &Analysis) -> Option { + let sets = [Some(an.set), an.pfx.map(|r| self.set(r.cont)), an.sfx.map(|r| self.set(r.cont)), an.inner.map(|r| self.set(r.cont))]; + let carries = |f: Option| f.is_some_and(|f| sets.iter().flatten().any(|s| has(s, f))); + let mut pos = 0; + if carries(c.flag) { + pos = FIRST | MIDDLE | LAST; + } + for (flag, p) in [(c.begin, FIRST), (c.middle, MIDDLE), (c.end, LAST)] { + if carries(flag) { + pos |= p; + } + } + // A prefix without COMPOUNDPERMITFLAG: any part but the last (`foo|unfoo|bar` is right + // in the CLI, `foo|unfoo` wrong). + if an.pfx.is_some_and(|r| !self.cont_holds(r, c.permit)) { + pos &= !LAST; + } + if let Some(r) = an.sfx { + if !self.cont_holds(r, c.permit) { + pos &= LAST; + } + if self.cont_holds(r, self.only_in_compound) { + pos &= !LAST; + } + } + if an.inner.is_some() { + pos &= LAST; + } + if carries(c.forbid) { + pos &= LAST; + } + if pos == 0 { + return None; + } + let mut marks = 0u64; + for (i, &m) in c.marks.iter().enumerate().take(64) { + if carries(Some(m)) { + marks |= 1 << i; + } + } + let words = if carries(c.root) { 2 } else { 1 }; + Some(Info { pos, marks, unmodified: an.pfx.is_none() && an.sfx.is_none(), stem: an.stem, words }) + } + + /// Every way `p` is a stem, or a stem with a prefix, one or two suffixes, or a prefix and a + /// suffix, as a compound part may be (`ONLYINCOMPOUND` affixes and stems included). + fn analyses<'a>(&'a self, p: &str, excluded: Excluded, work: &mut usize, emit: &mut impl FnMut(Analysis<'a>)) { + if let Some((stem, set)) = self.part_stem(p, excluded) + && !self.need_affix.is_some_and(|n| has(set, n)) + { + emit(Analysis { stem, set, pfx: None, sfx: None, inner: None }); + } + self.suffix_analyses(p, None, excluded, work, emit); + let mut root = String::with_capacity(p.len() + 8); + let ends = std::iter::once(0).chain(p.char_indices().map(|(i, c)| i + c.len_utf8())); + for b in ends.take(self.max_pfx_chars + 1) { + let (Some(add), Some(rest)) = (p.get(..b), p.get(b..)) else { continue }; + if rest.is_empty() && !self.full_strip { + continue; + } + let Some(ids) = self.pfx_index.get(add) else { continue }; + for r in ids.iter().filter_map(|&i| self.prefixes.get(i as usize)) { + *work += 1; + if *work > MAX_WORK { + return; + } + root.clear(); + root.push_str(&r.strip); + root.push_str(rest); + if !r.cond_matches(&root, false) { + continue; + } + let (circ, needs) = (self.cont_holds(r, self.circumfix), self.cont_holds(r, self.need_affix)); + if !circ + && !needs + && let Some((stem, set)) = self.part_stem(&root, excluded) + && has(set, r.flag) + { + emit(Analysis { stem, set, pfx: Some(r), sfx: None, inner: None }); + } + if r.cross { + self.suffix_analyses(&root, Some(r), excluded, work, emit); + } + } + } + } + + /// The analyses of `p` that end in a suffix: on a stem (after the prefix `pfx`, if any), or + /// (without a prefix) on a form of another suffix. + fn suffix_analyses<'a>(&'a self, p: &str, pfx: Option<&'a Rule>, excluded: Excluded, work: &mut usize, emit: &mut impl FnMut(Analysis<'a>)) { + let mut root = String::with_capacity(p.len() + 8); + let cuts = std::iter::once(p.len()).chain(p.char_indices().rev().map(|(i, _)| i).filter(|&i| i > 0 || self.full_strip)); + for b in cuts.take(self.max_sfx_chars + 1) { + let (Some(base), Some(add)) = (p.get(..b), p.get(b..)) else { continue }; + let Some(groups) = self.sfx_index.get(add) else { continue }; + for g in groups { + root.clear(); + root.push_str(base); + root.push_str(&g.strip); + let rules = || g.rules.iter().filter_map(|&i| self.suffixes.get(i as usize)); + if let Some((stem, set)) = self.part_stem(&root, excluded) { + for r in rules() { + *work += 1; + if *work > MAX_WORK { + return; + } + if self.suffix_fits(r, set, pfx) && r.cond_matches(&root, true) { + emit(Analysis { stem, set, pfx, sfx: Some(r), inner: None }); + } + } + } + // Two suffixes in a row: the outer ones that another suffix's continuation + // classes can license, looked for together under one inner scan. + if pfx.is_none() && !g.outer_rules.is_empty() { + let outers: Vec<&Rule> = g + .outer_rules + .iter() + .filter_map(|&i| self.suffixes.get(i as usize)) + .filter(|r| !self.cont_holds(r, self.only_in_compound) && r.cond_matches(&root, true)) + .collect(); + *work += outers.len(); + if !outers.is_empty() { + self.inner_analyses(&root, &outers, excluded, work, emit); + } + } + } + } + } + + /// Does the suffix `r` apply to a stem with the flags `set`, after the prefix `pfx` (if any)? + /// Circumfix alike; a suffix that needs another affix only after a prefix that doesn't. + fn suffix_fits(&self, r: &Rule, set: &[u16], pfx: Option<&Rule>) -> bool { + if !has(set, r.flag) { + return false; + } + let circ = self.cont_holds(r, self.circumfix); + let needs = self.cont_holds(r, self.need_affix); + match pfx { + None => !circ && !needs, + Some(p) => { + r.cross + && (has(set, p.flag) || self.cont_holds(r, Some(p.flag))) + && circ == self.cont_holds(p, self.circumfix) + && !(needs && self.cont_holds(p, self.need_affix)) + } + } + } + + /// `form` (what an outer suffix of `outers` was taken off) as a stem with an inner suffix + /// whose continuation classes hold that outer suffix's flag. + fn inner_analyses<'a>(&'a self, form: &str, outers: &[&'a Rule], excluded: Excluded, work: &mut usize, emit: &mut impl FnMut(Analysis<'a>)) { + let mut root = String::with_capacity(form.len() + 8); + let cuts = std::iter::once(form.len()).chain(form.char_indices().rev().map(|(i, _)| i).filter(|&i| i > 0 || self.full_strip)); + for b in cuts.take(self.max_sfx_chars + 1) { + let (Some(base), Some(add)) = (form.get(..b), form.get(b..)) else { continue }; + let Some(groups) = self.sfx_index.get(add) else { continue }; + for g in groups.iter().filter(|g| !g.cont_rules.is_empty()) { + root.clear(); + root.push_str(base); + root.push_str(&g.strip); + let Some((stem, set)) = self.part_stem(&root, excluded) else { continue }; + for r in g.cont_rules.iter().filter_map(|&i| self.suffixes.get(i as usize)) { + *work += 1; + if *work > MAX_WORK { + return; + } + if !has(set, r.flag) + || self.cont_holds(r, self.circumfix) + || self.cont_holds(r, self.only_in_compound) + || !r.cond_matches(&root, true) + { + continue; + } + for &outer in outers.iter().filter(|o| self.cont_holds(r, Some(o.flag))) { + *work += 1; + emit(Analysis { stem, set, pfx: None, sfx: Some(outer), inner: Some(r) }); + } + } + } + } + } + + /// The stem `w` as a compound part: its index and flags, unless it is forbidden or excluded. + fn part_stem(&self, w: &str, excluded: Excluded) -> Option<(u32, &[u16])> { + self.stem_lookup(w).filter(|(_, set)| !excluded.hits(set) && !self.forbidden.is_some_and(|f| has(set, f))) + } + + /// May a part with `info` start at char `start`, after the last part of `s.path` + /// (`CHECKCOMPOUNDCASE`, `CHECKCOMPOUNDTRIPLE`, `CHECKCOMPOUNDPATTERN`)? + fn boundary_ok(&self, s: &Search, start: usize, info: &Info) -> bool { + let Some(&(ps, pe, prev)) = s.path.last() else { return true }; + let c = s.c; + let (Some(&a), Some(&b)) = (s.chars.get(pe.saturating_sub(1)), s.chars.get(start)) else { return true }; + if c.case && (a.is_uppercase() || b.is_uppercase()) && a != '-' && b != '-' { + return false; + } + if c.triple && start == pe && a == b { + let before = pe >= 2 && s.chars.get(pe - 2) == Some(&a); + let after = s.chars.get(start + 1) == Some(&b); + if before || after { + return false; + } + } + if c.patterns.is_empty() { + return true; + } + let text = |x: usize, y: usize| s.cuts.get(x).zip(s.cuts.get(y)).and_then(|(&x, &y)| s.w.get(x..y)).unwrap_or_default(); + let (before, after) = (text(ps, pe), text(start, s.chars.len())); + !c.patterns.iter().any(|p| { + before.ends_with(&*p.end) + && after.starts_with(&*p.begin) + && (!p.unmodified || prev.unmodified) + && p.end_need.met(prev.marks) + && p.begin_need.met(info.marks) + }) + } + + /// Are the parts of `s.path` few enough (`COMPOUNDWORDMAX`, `COMPOUNDROOT`)? `whole`: the + /// word is complete (the Hungarian syllable rule counts its vowels). + fn words_ok(&self, s: &Search, whole: bool) -> bool { + let Some(max) = s.c.word_max else { return true }; + let words: usize = s.path.iter().map(|p| usize::from(p.2.words)).sum(); + if words <= max { + return true; + } + match &s.c.syllables { + None => false, + Some(_) if !whole => true, + Some((most, vowels)) => s.chars.iter().filter(|c| vowels.contains(c)).count() <= *most, + } + } + + /// The checks on a complete split: two parts at least, `COMPOUNDWORDMAX`, `CHECKCOMPOUNDDUP`, + /// `FORCEUCASE`, the three-part rule and `CHECKCOMPOUNDREP`. + fn whole_ok(&self, s: &mut Search) -> bool { + let n = s.path.len(); + if n < 2 || !self.words_ok(s, true) { + return false; + } + let (Some(&(_, _, last)), Some(&(_, _, prev))) = (s.path.last(), s.path.get(n - 2)) else { return false }; + if s.c.dup && last.stem == prev.stem { + return false; + } + if last.marks & 1 != 0 && s.c.force_ucase.is_some() && !s.capitalized { + return false; + } + let span = |s: &Search, i: usize, j: usize| -> String { + let (Some(&(a, ..)), Some(&(_, b, _))) = (s.path.get(i), s.path.get(j)) else { return String::new() }; + s.cuts.get(a).zip(s.cuts.get(b)).and_then(|(&x, &y)| s.w.get(x..y)).unwrap_or_default().to_string() + }; + if n >= 3 && self.near_word(s) { + return false; + } + if s.c.rep && !self.rep.is_empty() { + let pairs = (0..n - 1).map(|i| (i, i + 1)); + let tails = (0..n - 1).map(|i| (i, n - 1)); + let spans: Vec = pairs.chain(tails).map(|(i, j)| span(s, i, j)).collect(); + if spans.iter().any(|sp| self.rep_fault(sp, s)) { + return false; + } + } + true + } + + /// The check on a compound of three or more parts: is the word one slip away from a word + /// (not a compound)? Then it is taken for a misspelling of that word. The slips are the + /// CLI's (black-box, see the tests): a letter too many, a pair typed twice, two letters + /// swapped or one moved (up to 4 apart), a keyboard neighbour (`KEY`), a `TRY` letter, a + /// `REP` or `MAP` change, the word capitalized, and (without `NOSPLITSUGS`) two words. It + /// depends on the word only, so it is answered once per check. + fn near_word(&self, s: &mut Search) -> bool { + if let Some(near) = s.near { + return near; + } + let excluded = s.excluded; + let mut work = s.work; + let mut is_word = |cand: &[char]| { + work += 1; + work > MAX_WORK || self.simple_word(&cand.iter().collect::(), excluded) + }; + let near = self.near_miss(&s.chars, s.c.two_words, &mut is_word); + s.work = work; + s.near = Some(near); + near + } + + fn near_miss(&self, w: &[char], two_words: bool, is_word: &mut impl FnMut(&[char]) -> bool) -> bool { + let n = w.len(); + let mut cand: Vec = Vec::with_capacity(n + 8); + let mut first = w.iter().take(1).flat_map(|c| c.to_uppercase()).collect::>(); + first.extend(w.iter().skip(1)); + if is_word(&first) || is_word(&w.iter().flat_map(|c| c.to_uppercase()).collect::>()) { + return true; + } + for i in 0..n { + // A letter too many, and a pair typed twice (`abab` for `ab`). + cand.clear(); + cand.extend(w.iter().take(i).chain(w.iter().skip(i + 1))); + if is_word(&cand) { + return true; + } + if i + 3 < n && w.get(i) == w.get(i + 2) && w.get(i + 1) == w.get(i + 3) { + cand.clear(); + cand.extend(w.iter().take(i).chain(w.iter().skip(i + 2))); + if is_word(&cand) { + return true; + } + } + for d in 1..=4 { + if i + d >= n { + break; + } + // Swapped with a letter up to 4 further on. + cand.clear(); + cand.extend_from_slice(w); + cand.swap(i, i + d); + if cand != w && is_word(&cand) { + return true; + } + // Moved 2 to 4 places on or back (one place is a swap). + if d >= 2 { + cand.clear(); + cand.extend_from_slice(w); + for turn in [1, 2] { + if let Some(slice) = cand.get_mut(i..=i + d) { + // Left by one: `w[i]` moves on; then right by two: `w[i + d]` back. + if turn == 1 { slice.rotate_left(1) } else { slice.rotate_right(2) } + } + if is_word(&cand) { + return true; + } + } + } + } + } + // One letter for another: a keyboard neighbour, a TRY letter. + let key = if self.key.is_empty() { DEFAULT_KEY } else { self.key.as_str() }; + let neighbours = |c: char| { + key.split('|').flat_map(move |row| { + let row: Vec = row.chars().take(MAX_KEY_ROW).collect(); + let at: Vec = row.iter().enumerate().filter(|(_, k)| **k == c).map(|(i, _)| i).collect(); + at.into_iter() + .flat_map(move |i| [i.checked_sub(1).and_then(|j| row.get(j)).copied(), row.get(i + 1).copied()]) + .flatten() + .collect::>() + }) + }; + let tries: Vec = self.try_chars.chars().take(MAX_TRY).collect(); + for i in 0..n { + let Some(&c) = w.get(i) else { continue }; + for r in neighbours(c).chain(tries.iter().copied()) { + if r == c { + continue; + } + cand.clear(); + cand.extend_from_slice(w); + if let Some(x) = cand.get_mut(i) { + *x = r; + } + if is_word(&cand) { + return true; + } + } + } + // `REP` (here `^` and `$` tie to the word's start and end) and `MAP` changes. + let text: String = w.iter().collect(); + for (from, to) in &self.rep { + let (start, from) = from.strip_prefix('^').map_or((false, from.as_str()), |f| (true, f)); + let (end, from) = from.strip_suffix('$').map_or((false, from), |f| (true, f)); + if from.is_empty() || to.contains(' ') { + continue; + } + for (i, _) in text.match_indices(from) { + if (start && i != 0) || (end && i + from.len() != text.len()) { + continue; + } + let changed: Vec = text + .get(..i) + .unwrap_or_default() + .chars() + .chain(to.chars()) + .chain(text.get(i + from.len()..).unwrap_or_default().chars()) + .collect(); + if is_word(&changed) { + return true; + } + } + } + let mut budget = MAX_MAP_VARIANTS; + if !self.map.is_empty() && self.map_variant(w, 0, &mut w.to_vec(), &mut budget, is_word) { + return true; + } + two_words + && (1..n).any(|i| { + let (head, tail) = w.split_at(i); + is_word(head) && is_word(tail) + }) + } + + /// Does changing letters of `cand` from `i` on to others of their `MAP` group make a word? + fn map_variant(&self, w: &[char], i: usize, cand: &mut Vec, budget: &mut usize, is_word: &mut impl FnMut(&[char]) -> bool) -> bool { + let Some(&c) = w.get(i) else { return cand.as_slice() != w && is_word(cand) }; + if self.map_variant(w, i + 1, cand, budget, is_word) { + return true; + } + for group in self.map.iter().filter(|g| g.contains(&c)) { + for &r in group.iter().filter(|&&r| r != c) { + if *budget == 0 { + return false; + } + *budget -= 1; + if let Some(x) = cand.get_mut(i) { + *x = r; + } + let hit = self.map_variant(w, i + 1, cand, budget, is_word); + if let Some(x) = cand.get_mut(i) { + *x = c; + } + if hit { + return true; + } + } + } + false + } + + /// Does a `REP` replacement turn `text` into a word (not a compound)? + fn rep_fault(&self, text: &str, s: &mut Search) -> bool { + let mut cand = String::with_capacity(text.len() + 8); + for (from, to) in &self.rep { + // Replacements tied to a word's start or end (`^`, `$`) are for suggestions: the CLI + // never applies them here. + if from.is_empty() || from.starts_with('^') || from.ends_with('$') { + continue; + } + for (i, _) in text.match_indices(from.as_str()) { + s.work += 1; + if s.work > MAX_WORK { + return true; + } + cand.clear(); + cand.push_str(text.get(..i).unwrap_or_default()); + cand.push_str(to); + cand.push_str(text.get(i + from.len()..).unwrap_or_default()); + if self.simple_word(&cand, s.excluded) { + return true; + } + } + } + false + } +} + +/// The compounding directives a compact-format line may hold, read like the `.aff`'s. +pub(super) fn is_directive(dir: &str) -> bool { + FLAG_DIRECTIVES.contains(&dir) + || SWITCHES.contains(&dir) + || matches!(dir, "COMPOUNDWORDMAX" | "COMPOUNDSYLLABLE" | "LANG" | "MAXCPDSUGS" | "NOSPLITSUGS" | "CHECKCOMPOUNDPATTERN") +} + +#[cfg(test)] +mod tests { + use std::collections::HashSet; + + use crate::affix::AffixDict; + + /// A synthetic dictionary and the `hunspell` 1.7.5 CLI's answers on it (black-box: `hunspell + /// -a`), for every word of one to three `pieces` and the `extra` words. `listed` are the + /// right words when `right`, else the wrong ones; every other word is the opposite. + struct Pinned { + name: &'static str, + aff: &'static str, + dic: &'static str, + pieces: &'static str, + extra: &'static str, + right: bool, + listed: &'static str, + } + + fn load(aff: &str, dic: &str) -> AffixDict { + let entries: Vec<&str> = dic.split_whitespace().collect(); + let dic = format!("{}\n{}\n", entries.len(), entries.join("\n")); + AffixDict::from_hunspell(&format!("SET UTF-8\n{aff}\n"), &dic, &[]).unwrap() + } + + fn candidates(p: &Pinned) -> Vec { + let pieces: Vec<&str> = p.pieces.split_whitespace().collect(); + let mut out: Vec = pieces.iter().map(|a| (*a).to_string()).collect(); + for a in &pieces { + for b in &pieces { + out.push(format!("{a}{b}")); + } + } + for a in &pieces { + for b in &pieces { + for c in &pieces { + out.push(format!("{a}{b}{c}")); + } + } + } + out.extend(p.extra.split_whitespace().map(str::to_string)); + out + } + + /// One case per directive (or pair that only means something together), each word's answer + /// pinned to the CLI's, before and after the compact format. + #[test] + fn flag_compounds_answer_as_the_cli_does() { + let mut failed = Vec::new(); + for p in PINNED { + let d = load(p.aff, p.dic); + let back = AffixDict::from_bytes(&d.to_bytes().unwrap()).unwrap(); + let listed: HashSet<&str> = p.listed.split_whitespace().collect(); + for w in candidates(p) { + let want = listed.contains(w.as_str()) == p.right; + if d.check(&w) != want || back.check(&w) != want { + failed.push(format!("{}: {w} should be {}", p.name, if want { "right" } else { "wrong" })); + } + } + } + assert!(failed.is_empty(), "{failed:#?}"); + } + + /// Real dictionaries (the Language packs' `.aff`/`.dic`): words the CLI answers right and + /// wrong on the same files (black-box; mostly random compounds of the dictionary's forms). + /// Run when a craft-i18n checkout is in `CRAFT_I18N_DIR` and its harness has fetched the pack + /// files (`CRAFT_I18N_CACHE`, default `$CRAFT_I18N_DIR/target/language-data-cache`, files named + /// by their sha256); each language whose files are missing is skipped. + #[test] + fn pack_dictionaries_compound_as_the_cli_does() { + let Some(dir) = std::env::var_os("CRAFT_I18N_DIR").map(std::path::PathBuf::from) else { return }; + let Ok(manifest) = std::fs::read_to_string(dir.join("language-data/manifest.toml")) else { return }; + let cache = std::env::var_os("CRAFT_I18N_CACHE").map_or_else(|| dir.join("target/language-data-cache"), std::path::PathBuf::from); + let manifest: toml::Table = manifest.parse().unwrap(); + let files = manifest.get("file").and_then(toml::Value::as_array).cloned().unwrap_or_default(); + let read = |locale: &str, kind: &str| { + let f = files.iter().find(|f| { + f.get("locale").and_then(toml::Value::as_str) == Some(locale) && f.get("kind").and_then(toml::Value::as_str) == Some(kind) + })?; + std::fs::read(cache.join(f.get("sha256")?.as_str()?)).ok() + }; + for (locale, right, wrong) in REAL { + let (Some(aff), Some(dic)) = (read(locale, "spelling-aff"), read(locale, "spelling-dic")) else { + eprintln!("{locale}: pack files not fetched, skipped"); + continue; + }; + let (aff, dic) = crate::hunspell::decode(&aff, &dic).unwrap(); + let d = AffixDict::from_hunspell(&aff, &dic, &[]).unwrap_or_else(|e| panic!("{locale}: {e}")); + for w in right.split_whitespace() { + assert!(d.check(w), "{locale}: {w} is right"); + } + for w in wrong.split_whitespace() { + assert!(!d.check(w), "{locale}: {w} is wrong"); + } + } + } + + /// (locale, right words, wrong words), the CLI's answers. + const REAL: &[(&str, &str, &str)] = &[ + ( + "da", + "sugammadexgårdfacadens kraftudtagssmutterbuttetheden pebermakrellensudvalgsarbejde Helsinkifirbladen", + "arbejdslagernebegivende visdommensrustet farfarentraveport egyptologiskopladesvoldtoges", + ), + ( + "de", + "Südwestbeifallsturm Machtfeudalideologie Schleifhärtering Gesprächskontur schule", + "chilisoßeswitttenaus Magmabewegungaminosäurebasis ausgezählterzehntagefieber heerfolgetemperatechnik haus buch", + ), + ( + "de-at", + "Verträglichkeitsnackenleder Sichelmangrovenwäldern Wehrpflichtbeiruter Kühlerblockigelnothilfe", + "Jungenschulemakromere Dagösnestküken vogtlandbahnelektronenarmem qualitativmedwedew", + ), + ( + "de-ch", + "Königsstahlextensionsfähigkeit Barlachpregelinsel Euroasiendysprosiums Hexendiabasgesteine", + "eingeglittenswohlbeleibtesten plakatflächewürdigens durchzuwandernliebediener zurückerinnernskolonnadenform", + ), + ( + "eo", + "elipsoidolumŝirmila resonancosonarkivojn bazanumeralosukersciuraj vivspertoellistiĝkapablo", + "nekapneatsgvidrolosmitozon Alaholaŭpove kapbalanadajsaŭtomatiĝadajn filamentecukrematoriosilviajn", + ), + ( + "et", + "jahvakemgisesski talõššidekäunuvaikski kõhnunuimatessegikräunujaisse nekroosiupsusina", + "spinningtoismele kollektivismessüngimainagi elektrolüüsispilgutelebki fosforisesverääni", + ), + ("fa", "", "بنمایاندمنتراوی زرشکتبهترانملجاجتکارترینید پارامترمبرقصاندیم"), + ("hr", "truležne", "vaeašteka razuvjeriliuzdanajpsujućemu pilencadiparieovom časomicevoziti"), + ( + "hu", + "eurofelsálazásiabbjáéin", + "síposékéibahetvenegyszerezni BMsorbánocskáérakönigesedéknek antihősekorgalvántelepeitekig Grazcákompendiumnyijaimmal", + ), + ("mn", "Канейдианхалифат Балжойрд", "өехийлчхөгсудлагчдархуутайгүй зээнцрүүдийгээбачиг цуглавруудынхаатөвшинт"), + ("nb", "", "fellesomkostningasweeperrollen kremfløytenalkoholdebatter Elsepålestikka translatøreksamenerkeldgufsene"), + ( + "nl", + "pikjesboekbinderspersen samengingengeloofsverdeeldheid sleepliftenerfgoed stortputtenbehadeling", + "onbeleefdheidhortensiastraat verwindjuilee Lotharengersvolontair gechromeerdsrietpluimen", + ), + ("nn", "", "orstuvastavantgardistar påskemåndagarkjempekenguruar piggtrådmusikkarreknemaskinar stimreinselikniårekeiper"), + ( + "sv", + "målvärdenservername sunniglädjepirrets kedjelängdendöbattangerna basfärgmetropoliterna", + "ogenomskådligsbåtsmanstorps tännkarheiskanens stängningstaggensbehängas råseleedemonlägeouppvärmt", + ), + ( + "sv-fi", + "Ausåsciseleringar uppsatsämnesinsatsens pimpelfiskelagstiftningsarbete orderstocksglutenet", + "smitartäbyeors triialiseringarkronkoloni databasenslågstadiumcardiganen springformsknegarensronneby", + ), + ]; + + /// `COMPOUNDRULE` as the Swedish (`)k`) and Mongolian (`[a0,a1]`, `%?`, `.`) dictionaries + /// write it, read as the CLI answers on the same rules. + #[test] + fn compound_rules_with_odd_marks_answer_as_the_cli_does() { + let d = load("COMPOUNDMIN 1\nCOMPOUNDRULE 1\nCOMPOUNDRULE )k", "a/) b/k"); + let back = AffixDict::from_bytes(&d.to_bytes().unwrap()).unwrap(); + for d in [&d, &back] { + assert!(d.check("ab") && !d.check("abb") && !d.check("aab")); + } + let d = load("FLAG long\nCOMPOUNDMIN 1\nCOMPOUNDRULE 1\nCOMPOUNDRULE (aa)*[a0,a1](bb)", "x/aa y/bb"); + assert!(d.check("xy") && d.check("xxy") && !d.check("xyy") && !d.check("yx")); + for rule in ["(nn)*%?", "(nn)*.(nn)*%?", "(aa)*%?(bb)"] { + let d = load(&format!("FLAG long\nCOMPOUNDMIN 1\nCOMPOUNDRULE 1\nCOMPOUNDRULE {rule}"), "x/nn y/nn z/aa w/bb"); + let back = AffixDict::from_bytes(&d.to_bytes().unwrap()).unwrap(); + for d in [&d, &back] { + assert!(!d.check("xy") && !d.check("xxy") && !d.check("zw") && d.check("x"), "{rule}"); + } + } + } + + /// The worst dictionary for the search: every letter and pair of letters a part anywhere, + /// every check on, hundreds of `REP`s, a long `TRY`, `MAP` groups and chained suffixes that + /// permit compounding. Each word still answers within the work bound. + #[test] + fn hostile_compounding_stays_cheap() { + let letters: Vec = ('a'..='z').collect(); + let mut aff = String::from( + "COMPOUNDFLAG X\nCOMPOUNDMIN 1\nCOMPOUNDWORDMAX 99\nCOMPOUNDPERMITFLAG P\nONLYINCOMPOUND O\nCHECKCOMPOUNDDUP\nCHECKCOMPOUNDREP\n\ + CHECKCOMPOUNDCASE\nCHECKCOMPOUNDTRIPLE\nSIMPLIFIEDTRIPLE\nTRY abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ\n\ + MAP 3\nMAP aeiou\nMAP bcdfg\nMAP hjklm\nCHECKCOMPOUNDPATTERN 2\nCHECKCOMPOUNDPATTERN a/X b/X\nCHECKCOMPOUNDPATTERN 0/P c\n", + ); + aff.push_str(&format!("REP {}\n", letters.len() * letters.len())); + for a in &letters { + for b in &letters { + aff.push_str(&format!("REP {a} {b}{a}\n")); + } + } + for (i, a) in letters.iter().enumerate() { + aff.push_str(&format!("SFX {a} Y 1\nSFX {a} 0 {a}/XPO{} .\n", letters.get(i + 1).copied().unwrap_or('a'))); + } + let flags: String = letters.iter().collect(); + let mut dic: Vec = letters.iter().map(|a| format!("{a}/X{flags}")).collect(); + for a in &letters { + for b in &letters { + dic.push(format!("{a}{b}/X{flags}")); + } + } + let d = AffixDict::from_hunspell(&format!("SET UTF-8\n{aff}"), &format!("{}\n{}\n", dic.len(), dic.join("\n")), &[]).unwrap(); + let mut rng = 7u64; + let random: String = (0..120) + .map(|_| { + rng = rng.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407); + letters.get((rng >> 33) as usize % 26).copied().unwrap_or('a') + }) + .collect(); + let mut slowest = std::time::Duration::ZERO; + let mut words: Vec = vec!["a".repeat(120), "ab".repeat(60), random.clone()]; + for n in [8, 16, 24, 32, 40, 48, 64] { + words.extend([random.chars().take(n).collect(), "a".repeat(n), "ab".repeat(n / 2), "aab".repeat(n / 3)]); + } + for w in &words { + let t = std::time::Instant::now(); + let _ = d.check(w); + slowest = slowest.max(t.elapsed()); + } + eprintln!("hostile compounding: slowest word {slowest:?}"); + assert!(slowest.as_secs() < 5, "{slowest:?}"); + } + + const PINNED: &[Pinned] = &[ + Pinned { + name: "flag", + aff: "COMPOUNDFLAG X", + dic: "foo/X bar/X ba/X baz", + pieces: "foo bar ba baz", + extra: "", + right: true, + listed: "foo bar ba baz foofoo foobar barfoo barbar foofoofoo foofoobar foobarfoo foobarbar barfoofoo barfoobar \ + barbarfoo barbarbar", + }, + Pinned { name: "min", aff: "COMPOUNDFLAG X\nCOMPOUNDMIN 1", dic: "f/X bar/X", pieces: "f bar", extra: "", right: false, listed: "" }, + Pinned { + name: "affixes", + aff: "COMPOUNDFLAG X\nCOMPOUNDPERMITFLAG Z\nSFX S Y 1\nSFX S 0 s .\nPFX P Y 1\nPFX P 0 un .\nSFX T Y 1\nSFX T 0 t/Z .\nPFX Q Y 1\nPFX Q 0 re/Z .", + dic: "foo/XSPTQ bar/XSPTQ baz/S", + pieces: "foo foos unfoo foot refoo bar", + extra: "barunfoo unfoobar barfoot footbar refoobar barrefoo barfoos foosbar baz", + right: false, + listed: "foounfoo foosfoo foosfoos foosunfoo foosfoot foosrefoo foosbar unfoounfoo footunfoo refoounfoo barunfoo \ + foofoounfoo foofoosfoo foofoosfoos foofoosunfoo foofoosfoot foofoosrefoo foofoosbar foounfoounfoo foofootunfoo \ + foorefoounfoo foobarunfoo foosfoofoo foosfoofoos foosfoounfoo foosfoofoot foosfoorefoo foosfoobar foosfoosfoo \ + foosfoosfoos foosfoosunfoo foosfoosfoot foosfoosrefoo foosfoosbar foosunfoofoo foosunfoofoos foosunfoounfoo \ + foosunfoofoot foosunfoorefoo foosunfoobar foosfootfoo foosfootfoos foosfootunfoo foosfootfoot foosfootrefoo \ + foosfootbar foosrefoofoo foosrefoofoos foosrefoounfoo foosrefoofoot foosrefoorefoo foosrefoobar foosbarfoo \ + foosbarfoos foosbarunfoo foosbarfoot foosbarrefoo foosbarbar unfoofoounfoo unfoofoosfoo unfoofoosfoos \ + unfoofoosunfoo unfoofoosfoot unfoofoosrefoo unfoofoosbar unfoounfoounfoo unfoofootunfoo unfoorefoounfoo \ + unfoobarunfoo footfoounfoo footfoosfoo footfoosfoos footfoosunfoo footfoosfoot footfoosrefoo footfoosbar \ + footunfoounfoo footfootunfoo footrefoounfoo footbarunfoo refoofoounfoo refoofoosfoo refoofoosfoos \ + refoofoosunfoo refoofoosfoot refoofoosrefoo refoofoosbar refoounfoounfoo refoofootunfoo refoorefoounfoo \ + refoobarunfoo barfoounfoo barfoosfoo barfoosfoos barfoosunfoo barfoosfoot barfoosrefoo barfoosbar \ + barunfoounfoo barfootunfoo barrefoounfoo barbarunfoo", + }, + Pinned { + name: "affix_flags", + aff: "COMPOUNDFLAG X\nSFX A Y 1\nSFX A 0 s/X .\nPFX B Y 1\nPFX B 0 un/X .\nSFX C Y 1\nSFX C 0 en .", + dic: "arbeit/AC comp/XC haus/B", + pieces: "arbeit arbeits comp compen haus unhaus", + extra: "", + right: true, + listed: "arbeit arbeits comp compen haus unhaus comparbeits compcomp compcompen unhausarbeits unhauscomp unhauscompen \ + compcomparbeits compcompcomp compcompcompen compunhausarbeits compunhauscomp compunhauscompen \ + unhauscomparbeits unhauscompcomp unhauscompcompen unhausunhausarbeits unhausunhauscomp unhausunhauscompen", + }, + Pinned { + name: "positions", + aff: "COMPOUNDFLAG X\nCOMPOUNDBEGIN B\nCOMPOUNDMIDDLE M\nCOMPOUNDEND E", + dic: "beg/B mid/M end/E any/X all/BME", + pieces: "beg mid end any all", + extra: "", + right: true, + listed: "beg mid end any all begend begany begall anyend anyany anyall allend allany allall begmidend begmidany \ + begmidall beganyend beganyany beganyall begallend begallany begallall anymidend anymidany anymidall anyanyend \ + anyanyany anyanyall anyallend anyallany anyallall allmidend allmidany allmidall allanyend allanyany allanyall \ + allallend allallany allallall", + }, + Pinned { + name: "first_last", + aff: "COMPOUNDFIRST B\nCOMPOUNDLAST E", + dic: "beg/B end/E", + pieces: "beg end", + extra: "", + right: true, + listed: "beg end", + }, + Pinned { + name: "onlyincompound", + aff: "COMPOUNDFLAG X\nONLYINCOMPOUND O\nSFX A Y 1\nSFX A 0 er/B .\nSFX B Y 1\nSFX B 0 s .\nSFX F Y 1\nSFX F 0 s/O .", + dic: "foo/XAF bar/XAF zap/XO qux/A", + pieces: "foo foos fooers bar zap", + extra: "zap foos fooer fooers quxer quxers", + right: true, + listed: "foo fooers bar foofoo foofooers foobar foozap barfoo barfooers barbar barzap zapfoo zapfooers zapbar zapzap \ + foofoofoo foofoofooers foofoobar foofoozap foobarfoo foobarfooers foobarbar foobarzap foozapfoo foozapfooers \ + foozapbar foozapzap barfoofoo barfoofooers barfoobar barfoozap barbarfoo barbarfooers barbarbar barbarzap \ + barzapfoo barzapfooers barzapbar barzapzap zapfoofoo zapfoofooers zapfoobar zapfoozap zapbarfoo zapbarfooers \ + zapbarbar zapbarzap zapzapfoo zapzapfooers zapzapbar zapzapzap fooer quxer quxers", + }, + Pinned { + name: "onlyincompound_permit", + aff: "COMPOUNDFLAG X\nONLYINCOMPOUND O\nCOMPOUNDPERMITFLAG P\nSFX F Y 1\nSFX F 0 s/OP .\nSFX G Y 1\nSFX G 0 s/O .\nPFX H Y 1\nPFX H 0 un/O .\nPFX I Y 1\nPFX I 0 re/OP .", + dic: "foo/XFHI bar/XFHI baz/XG", + pieces: "foo foos unfoo refoo bazs", + extra: "foos unfoo refoo bazs baz", + right: true, + listed: "foo foofoo foorefoo foosfoo foosrefoo unfoofoo unfoorefoo refoofoo refoorefoo foofoofoo foofoorefoo foofoosfoo \ + foofoosrefoo foounfoofoo foounfoorefoo foorefoofoo foorefoorefoo foosfoofoo foosfoorefoo foosfoosfoo \ + foosfoosrefoo foosunfoofoo foosunfoorefoo foosrefoofoo foosrefoorefoo unfoofoofoo unfoofoorefoo unfoofoosfoo \ + unfoofoosrefoo unfoounfoofoo unfoounfoorefoo unfoorefoofoo unfoorefoorefoo refoofoofoo refoofoorefoo \ + refoofoosfoo refoofoosrefoo refoounfoofoo refoounfoorefoo refoorefoofoo refoorefoorefoo baz", + }, + Pinned { + name: "forbid_needaffix", + aff: "COMPOUNDFLAG X\nCOMPOUNDFORBIDFLAG Q\nCOMPOUNDPERMITFLAG P\nNEEDAFFIX N\nSFX S Y 1\nSFX S 0 s .\nSFX T Y 1\nSFX T 0 t/Q .\nSFX U Y 1\nSFX U 0 u/QP .\nSFX V Y 1\nSFX V 0 v/P .", + dic: "foo/XSTUV bar/XSTUV qoo/XQS nee/XNSV noo/NSV", + pieces: "foo foot foou qoo nee nees neev", + extra: "nee noo noos", + right: true, + listed: "foo foot foou qoo nees neev foofoo foofoot foofoou fooqoo foonees fooneev neevfoo neevfoot neevfoou neevqoo \ + neevnees neevneev foofoofoo foofoofoot foofoofoou foofooqoo foofoonees foofooneev fooneevfoo fooneevfoot \ + fooneevfoou fooneevqoo fooneevnees fooneevneev neevfoofoo neevfoofoot neevfoofoou neevfooqoo neevfoonees \ + neevfooneev neevneevfoo neevneevfoot neevneevfoou neevneevqoo neevneevnees neevneevneev noos", + }, + Pinned { + name: "dup", + aff: "COMPOUNDFLAG X\nCHECKCOMPOUNDDUP\nSFX S Y 1\nSFX S 0 s .", + dic: "foo/XS bar/XS foos/X", + pieces: "foo foos bar", + extra: "", + right: false, + listed: "foofoo barbar foofoofoo foobarbar foosfoofoo foosbarbar barfoofoo barbarbar", + }, + Pinned { + name: "case", + aff: "COMPOUNDFLAG X\nCHECKCOMPOUNDCASE", + dic: "foo/X Bar/X baz/X QUX/X", + pieces: "foo Bar baz QUX", + extra: "", + right: true, + listed: "foo Bar baz QUX foofoo foobaz Barfoo Barbaz bazfoo bazbaz foofoofoo foofoobaz foobazfoo foobazbaz Barfoofoo \ + Barfoobaz Barbazfoo Barbazbaz bazfoofoo bazfoobaz bazbazfoo bazbazbaz", + }, + Pinned { + name: "triple", + aff: "COMPOUNDFLAG X\nCHECKCOMPOUNDTRIPLE", + dic: "foo/X bar/X schiff/X fahrt/X ooze/X baa/X", + pieces: "foo schiff fahrt ooze baa", + extra: "", + right: false, + listed: "fooooze schifffoo schifffahrt foofooooze fooschifffoo fooschifffahrt foooozefoo foooozeschiff foooozefahrt \ + foooozeooze foooozebaa schifffoofoo schifffooschiff schifffoofahrt schifffooooze schifffoobaa schiffschifffoo \ + schiffschifffahrt schifffahrtfoo schifffahrtschiff schifffahrtfahrt schifffahrtooze schifffahrtbaa \ + fahrtfooooze fahrtschifffoo fahrtschifffahrt oozefooooze oozeschifffoo oozeschifffahrt baafooooze baaschifffoo \ + baaschifffahrt", + }, + Pinned { + name: "simplified_triple", + aff: "COMPOUNDFLAG X\nCHECKCOMPOUNDTRIPLE\nSIMPLIFIEDTRIPLE", + dic: "foo/X bar/X schiff/X fahrt/X ooze/X baa/X", + pieces: "foo schiff fahrt ooze baa", + extra: "schiffahrt", + right: false, + listed: "fooooze schifffoo schifffahrt foofooooze fooschifffoo fooschifffahrt foooozefoo foooozeschiff foooozefahrt \ + foooozeooze foooozebaa schifffoofoo schifffooschiff schifffoofahrt schifffooooze schifffoobaa schiffschifffoo \ + schiffschifffahrt schifffahrtfoo schifffahrtschiff schifffahrtfahrt schifffahrtooze schifffahrtbaa \ + fahrtfooooze fahrtschifffoo fahrtschifffahrt oozefooooze oozeschifffoo oozeschifffahrt baafooooze baaschifffoo \ + baaschifffahrt", + }, + Pinned { + name: "simplified_only", + aff: "COMPOUNDFLAG X\nSIMPLIFIEDTRIPLE", + dic: "foo/X schiff/X fahrt/X ooze/X", + pieces: "foo schiff fahrt ooze", + extra: "schiffahrt fooze", + right: false, + listed: "fooze", + }, + Pinned { + name: "rep", + aff: "COMPOUNDFLAG X\nCHECKCOMPOUNDREP\nREP 4\nREP oo o\nREP rb r\nREP ^ba x\nREP az$ x\nSFX S Y 1\nSFX S 0 s .", + dic: "foo/XS bar/XS baz/X fobar/S barb/X baraz/S fobaz bx", + pieces: "foo foos bar bars baz barb", + extra: "", + right: true, + listed: "foo foos bar bars baz barb foofoo foofoos foobarb barfoo barfoos barbar barbars barbarb bazfoo bazfoos bazbar \ + bazbars bazbaz bazbarb barbfoo barbfoos barbbar barbbars barbbaz barbbarb foofoofoo foofoofoos foofoobarb \ + foobarbfoo foobarbfoos foobarbbar foobarbbars foobarbbaz foobarbbarb barfoofoo barfoofoos barfoobarb barbarfoo \ + barbarfoos barbarbar barbarbars barbarbarb barbarbfoo barbarbfoos barbarbbar barbarbbars barbarbbaz \ + barbarbbarb bazfoofoo bazfoofoos bazfoobarb bazbarfoo bazbarfoos bazbarbar bazbarbars bazbarbarb bazbazfoo \ + bazbazfoos bazbazbar bazbazbars bazbazbaz bazbazbarb bazbarbfoo bazbarbfoos bazbarbbar bazbarbbars bazbarbbaz \ + bazbarbbarb barbfoofoo barbfoofoos barbfoobarb barbbarfoo barbbarfoos barbbarbar barbbarbars barbbarbarb \ + barbbazfoo barbbazfoos barbbazbar barbbazbars barbbazbaz barbbazbarb barbbarbfoo barbbarbfoos barbbarbbar \ + barbbarbbars barbbarbbaz barbbarbbarb", + }, + Pinned { + name: "rep_spans", + aff: "COMPOUNDFLAG X\nCHECKCOMPOUNDREP\nREP 1\nREP oo o", + dic: "foo/X bar/X baz/X qux/X fobarbaz quxfobar", + pieces: "foo bar baz qux", + extra: "", + right: false, + listed: "foobarbaz quxfoobar", + }, + Pinned { + name: "three_parts", + aff: "COMPOUNDFLAG X", + dic: "foo/X bar/X baz/X qux/X foobar fooba barbazqux", + pieces: "foo bar baz qux", + extra: "fooxbarbaz", + right: false, + listed: "foofoobar foobarfoo foobarbar foobarbaz foobarqux barfoobar bazfoobar bazbarqux quxfoobar fooxbarbaz", + }, + Pinned { + name: "three_parts_nosplitsugs", + aff: "COMPOUNDFLAG X\nNOSPLITSUGS", + dic: "foo/X bar/X baz/X qux/X foobar fooba", + pieces: "foo bar baz qux", + extra: "", + right: false, + listed: "", + }, + Pinned { + name: "pattern", + aff: "COMPOUNDFLAG X\nCOMPOUNDPERMITFLAG P\nCHECKCOMPOUNDPATTERN 3\nCHECKCOMPOUNDPATTERN oo ba # comment\nCHECKCOMPOUNDPATTERN /F /G\nCHECKCOMPOUNDPATTERN er /H\nSFX S Y 1\nSFX S 0 s/P .\nSFX T Y 1\nSFX T 0 t/PF .\nSFX E Y 1\nSFX E 0 er/P .", + dic: "foo/XST bar/XSG baz/XS qux/XF zed/XH hey/XSE heyer/XS", + pieces: "foo foot bar baz qux zed heyer hey", + extra: "", + right: false, + listed: "foobar foobaz footbar quxbar heyerzed foofoobar foofoobaz foofootbar foobarfoo foobarfoot foobarbar foobarbaz \ + foobarqux foobarzed foobarheyer foobarhey foobazfoo foobazfoot foobazbar foobazbaz foobazqux foobazzed \ + foobazheyer foobazhey fooquxbar fooheyerzed footfoobar footfoobaz footfootbar footbarfoo footbarfoot \ + footbarbar footbarbaz footbarqux footbarzed footbarheyer footbarhey footquxbar footheyerzed barfoobar \ + barfoobaz barfootbar barquxbar barheyerzed bazfoobar bazfoobaz bazfootbar bazquxbar bazheyerzed quxfoobar \ + quxfoobaz quxfootbar quxbarfoo quxbarfoot quxbarbar quxbarbaz quxbarqux quxbarzed quxbarheyer quxbarhey \ + quxquxbar quxheyerzed zedfoobar zedfoobaz zedfootbar zedquxbar zedheyerzed heyerfoobar heyerfoobaz \ + heyerfootbar heyerquxbar heyerzedfoo heyerzedfoot heyerzedbar heyerzedbaz heyerzedqux heyerzedzed \ + heyerzedheyer heyerzedhey heyerheyerzed heyfoobar heyfoobaz heyfootbar heyquxbar heyheyerzed", + }, + Pinned { + name: "pattern_unmodified_forceucase", + aff: "COMPOUNDFLAG X\nFORCEUCASE U\nCHECKCOMPOUNDPATTERN 1\nCHECKCOMPOUNDPATTERN 0/A b\nCOMPOUNDPERMITFLAG P\nSFX S Y 1\nSFX S 0 s .\nSFX R Y 1\nSFX R 0 r/P .", + dic: "foo/XSR straat/XUS bar/X zoo/XAR big/X", + pieces: "foo straat bar zoo zoor big", + extra: "Foostraat FOOSTRAAT Foostraats", + right: false, + listed: "foostraat straatstraat barstraat zoostraat zoobar zoobig zoorstraat bigstraat foofoostraat foostraatstraat \ + foobarstraat foozoostraat foozoobar foozoobig foozoorstraat foobigstraat straatfoostraat straatstraatstraat \ + straatbarstraat straatzoostraat straatzoobar straatzoobig straatzoorstraat straatbigstraat barfoostraat \ + barstraatstraat barbarstraat barzoostraat barzoobar barzoobig barzoorstraat barbigstraat zoofoostraat \ + zoostraatstraat zoobarfoo zoobarstraat zoobarbar zoobarzoo zoobarzoor zoobarbig zoozoostraat zoozoobar \ + zoozoobig zoozoorstraat zoobigfoo zoobigstraat zoobigbar zoobigzoo zoobigzoor zoobigbig zoorfoostraat \ + zoorstraatstraat zoorbarstraat zoorzoostraat zoorzoobar zoorzoobig zoorzoorstraat zoorbigstraat bigfoostraat \ + bigstraatstraat bigbarstraat bigzoostraat bigzoobar bigzoobig bigzoorstraat bigbigstraat", + }, + Pinned { + name: "wordmax_root", + aff: "COMPOUNDFLAG X\nCOMPOUNDWORDMAX 2\nCOMPOUNDROOT R", + dic: "foo/X bar/X foobaz/XR", + pieces: "foo bar foobaz", + extra: "", + right: true, + listed: "foo bar foobaz foofoo foobar barfoo barbar", + }, + Pinned { + name: "wordmax", + aff: "COMPOUNDFLAG X\nCOMPOUNDWORDMAX 3\nCOMPOUNDMIN 1", + dic: "fo/X ba/X", + pieces: "fo ba", + extra: "fobafoba fobafobafo", + right: false, + listed: "fobafoba fobafobafo", + }, + Pinned { + name: "syllable", + aff: "COMPOUNDFLAG X\nCOMPOUNDWORDMAX 2\nCOMPOUNDSYLLABLE 6 aeiou\nCOMPOUNDMIN 1", + dic: "ba/X be/X baaa/X", + pieces: "ba be baaa", + extra: "babababa", + right: false, + listed: "", + }, + Pinned { + name: "syllable_hu", + aff: "LANG hu_HU\nCOMPOUNDFLAG X\nCOMPOUNDWORDMAX 2 y\nCOMPOUNDSYLLABLE 6 aeiou\nCOMPOUNDMIN 1", + dic: "ba/X be/X baaa/X", + pieces: "ba be baaa", + extra: "babababa", + right: false, + listed: "babaaabaaa bebaaabaaa baaababaaa baaabebaaa baaabaaaba baaabaaabe baaabaaabaaa", + }, + Pinned { + name: "near_slips", + aff: "COMPOUNDFLAG X\nCOMPOUNDMIN 4\nTRY x\nREP 1\nREP ^ab yz\nMAP 1\nMAP eë", + dic: "abcd/X efgh/X ijkl/X mnop/X abcdefghikjl efghabcdjkli ijklabcdefgg xnopabcdefgh yzcdmnopijkl ëfghmnopabcd Ijklmnopabcd", + pieces: "abcd efgh ijkl mnop", + extra: "", + right: false, + listed: "abcdefghijkl abcdmnopijkl efghabcdijkl efghmnopabcd ijklabcdefgh ijklmnopabcd mnopabcdefgh", + }, + Pinned { + name: "forbidden", + aff: "COMPOUNDFLAG X\nFORBIDDENWORD F\nSFX S Y 1\nSFX S 0 en .", + dic: "foo/X brok/XS broken/F bar/XF foobar/F", + pieces: "foo brok broken bar", + extra: "foobar broken", + right: true, + listed: "foo brok foofoo foobrok foobroken brokfoo brokbrok brokbroken foofoofoo foofoobrok foofoobroken foobrokfoo \ + foobrokbrok foobrokbroken brokfoofoo brokfoobrok brokfoobroken brokbrokfoo brokbrokbrok brokbrokbroken", + }, + ]; +} From 9a442e70480ebe32d257ff27d4bb46dc7dbfab07 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Mon, 2 Nov 2026 01:21:43 +0000 Subject: [PATCH 09/40] =?UTF-8?q?CP7.2:=20affix=5Fcheck=20example=20?= =?UTF-8?q?=E2=80=94=20compare=20the=20affix=20engine=20with=20another=20c?= =?UTF-8?q?hecker's=20answers=20(black-box)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FypgCX8LsYYE8fsj4byBm2 --- crates/proof/examples/affix_check.rs | 126 +++++++++++++++++++++++++++ 1 file changed, 126 insertions(+) create mode 100644 crates/proof/examples/affix_check.rs diff --git a/crates/proof/examples/affix_check.rs b/crates/proof/examples/affix_check.rs new file mode 100644 index 00000000..31692618 --- /dev/null +++ b/crates/proof/examples/affix_check.rs @@ -0,0 +1,126 @@ +//! Compare the affix engine with another spell checker, black-box: load a Hunspell `.aff`/`.dic` +//! pair and answer for words, or make candidate compounds to ask both about. +//! +//! ```sh +//! # One word per line in, `word1` (right) or `word0` out, and the time taken on stderr. +//! cargo run --release -p craft-proof --example affix_check -- check de_DE.aff de_DE.dic < words.txt +//! # 2000 candidates (forms of random stems glued together, some with a letter changed), seed 7. +//! cargo run --release -p craft-proof --example affix_check -- gen de_DE.aff de_DE.dic 2000 7 > words.txt +//! ``` +//! +//! The answers are compared with `hunspell -d -l` on the same words +//! (the words it lists are the wrong ones). Nothing of the other checker is read but its answers. + +use std::io::{BufRead, Write}; + +use craft_proof::affix::AffixDict; + +fn load(aff: &str, dic: &str) -> Result { + let aff = std::fs::read(aff).map_err(|e| format!("{aff}: {e}"))?; + let dic = std::fs::read(dic).map_err(|e| format!("{dic}: {e}"))?; + let (aff, dic) = craft_proof::hunspell::decode(&aff, &dic)?; + AffixDict::from_hunspell(&aff, &dic, &[]) +} + +/// xorshift64*: a reproducible stream of numbers, enough to pick stems. +struct Rng(u64); + +impl Rng { + fn next(&mut self) -> u64 { + self.0 ^= self.0 >> 12; + self.0 ^= self.0 << 25; + self.0 ^= self.0 >> 27; + self.0.wrapping_mul(0x2545_f491_4f6c_dd1d) + } + + fn below(&mut self, n: usize) -> usize { + if n == 0 { 0 } else { (self.next() % n as u64) as usize } + } +} + +/// `w` with its first letter in lower case (a compound's later parts). +fn decapitalize(w: &str) -> String { + let mut c = w.chars(); + c.next().map(|f| f.to_lowercase().chain(c).collect()).unwrap_or_default() +} + +fn generate(d: &AffixDict, n: usize, seed: u64) -> Vec { + let stems: Vec<&str> = d.stems().filter(|s| s.chars().all(char::is_alphabetic)).collect(); + let mut rng = Rng(seed.max(1)); + let form = |rng: &mut Rng| -> String { + let s = stems.get(rng.below(stems.len())).copied().unwrap_or_default(); + let forms = d.forms(s); + forms.get(rng.below(forms.len())).cloned().unwrap_or_else(|| s.to_string()) + }; + let mut out = Vec::with_capacity(n); + while out.len() < n { + let parts = 2 + rng.below(2); + let mut w = form(&mut rng); + for _ in 1..parts { + if rng.below(4) == 0 { + w.push('s'); + } + w.push_str(&decapitalize(&form(&mut rng))); + } + if rng.below(3) == 0 { + // A letter dropped, doubled or changed: mostly wrong words. + let chars: Vec = w.chars().collect(); + let i = rng.below(chars.len()); + let mut m: String = chars.iter().take(i).collect(); + match rng.below(3) { + 0 => {} + 1 => m.extend(chars.get(i).into_iter().chain(chars.get(i))), + _ => m.push('e'), + } + m.extend(chars.iter().skip(i + 1)); + w = m; + } + // Letters only: the other checker's command line splits words at anything else. + if w.chars().count() >= 4 && w.len() <= 100 && w.chars().all(char::is_alphabetic) { + out.push(w); + } + } + out +} + +fn main() -> Result<(), String> { + let args: Vec = std::env::args().skip(1).collect(); + match args.as_slice() { + [cmd, aff, dic] if cmd == "check" => { + let d = load(aff, dic)?; + let words: Vec = std::io::stdin().lock().lines().map_while(Result::ok).collect(); + let t = std::time::Instant::now(); + let mut slowest: Vec<(std::time::Duration, &str)> = Vec::new(); + let answers: Vec = words + .iter() + .map(|w| { + let t = std::time::Instant::now(); + let ok = d.check(w); + slowest.push((t.elapsed(), w)); + ok + }) + .collect(); + let took = t.elapsed(); + slowest.sort_by_key(|s| std::cmp::Reverse(s.0)); + for (t, w) in slowest.iter().take(3) { + eprintln!("slow: {w} {t:?}"); + } + let mut out = std::io::stdout().lock(); + for (w, ok) in words.iter().zip(answers) { + writeln!(out, "{w}\t{}", u8::from(ok)).map_err(|e| e.to_string())?; + } + eprintln!("{} words in {took:?}", words.len()); + Ok(()) + } + [cmd, aff, dic, n, seed] if cmd == "gen" => { + let d = load(aff, dic)?; + let n: usize = n.parse().map_err(|_| "n is a number")?; + let seed: u64 = seed.parse().map_err(|_| "seed is a number")?; + for w in generate(&d, n, seed) { + println!("{w}"); + } + Ok(()) + } + _ => Err("usage: affix_check check < words | affix_check gen ".into()), + } +} From 3eaa276786df0e70712b28a3912269783abc8940 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Mon, 2 Nov 2026 02:08:35 +0000 Subject: [PATCH 10/40] =?UTF-8?q?CP7.3:=20craft-libs#497=20review=20?= =?UTF-8?q?=E2=80=94=20compounds=20as=20the=20hunspell=20CLI=20answers:=20?= =?UTF-8?q?ONLYINCOMPOUND=20homonyms=20keep=20their=20flags,=20triples=20a?= =?UTF-8?q?fter=20SIMPLIFIEDTRIPLE,=20KEEPCASE=20and=20capitals=20in=20par?= =?UTF-8?q?ts,=20CHECKCOMPOUNDPATTERN=200,=20a=20forbidden=20prefix,=20FOR?= =?UTF-8?q?CEUCASE;=20suggestions=20and=20many-way=20pieces=20stay=20cheap?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - ONLYINCOMPOUND homonyms beside others are kept apart (compound parts only), so their suffixes make no standalone words (de verfilmungs, sv norges); round-trips as a second stem line. - CHECKCOMPOUNDTRIPLE also checks a SIMPLIFIEDTRIPLE join (busssen, booob wrong). - KEEPCASE holds for the first part only; a capitalized stem is a part (FooQux). - CHECKCOMPOUNDPATTERN `0`: no suffix (a prefix doesn't count): unfoobar wrong. - COMPOUNDFORBIDFLAG on a prefix: no part anywhere (bardefoo wrong). - FORCEUCASE: from the stem only; wrong only without any capital (fooQuxbar right). - Suggestion candidates: 2,048 compound steps each, compounds for the first 128 only. - A piece's ways are hashed, capped at 256 and counted (with the patterns) against the budget. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FypgCX8LsYYE8fsj4byBm2 --- crates/proof/examples/affix_check.rs | 39 +++- crates/proof/src/affix.rs | 183 ++++++++++++--- crates/proof/src/affix/compound.rs | 318 +++++++++++++++++++++++---- crates/proof/src/affix_suggest.rs | 23 +- 4 files changed, 474 insertions(+), 89 deletions(-) diff --git a/crates/proof/examples/affix_check.rs b/crates/proof/examples/affix_check.rs index 31692618..329c4fcb 100644 --- a/crates/proof/examples/affix_check.rs +++ b/crates/proof/examples/affix_check.rs @@ -6,6 +6,8 @@ //! cargo run --release -p craft-proof --example affix_check -- check de_DE.aff de_DE.dic < words.txt //! # 2000 candidates (forms of random stems glued together, some with a letter changed), seed 7. //! cargo run --release -p craft-proof --example affix_check -- gen de_DE.aff de_DE.dic 2000 7 > words.txt +//! # Single words instead: forms of random stems, and stems with another stem's ending. +//! cargo run --release -p craft-proof --example affix_check -- single de_DE.aff de_DE.dic 2000 7 > words.txt //! ``` //! //! The answers are compared with `hunspell -d -l` on the same words @@ -83,6 +85,36 @@ fn generate(d: &AffixDict, n: usize, seed: u64) -> Vec { out } +/// Single words (no compounds): forms of random stems, and stems with another stem's ending +/// (mostly wrong, some right: an ending the stem's own homonym allows or not). +fn single(d: &AffixDict, n: usize, seed: u64) -> Vec { + let stems: Vec<&str> = d.stems().filter(|s| s.chars().all(char::is_alphabetic)).collect(); + let mut rng = Rng(seed.max(1)); + let mut endings: Vec = Vec::new(); + for _ in 0..200 { + let s = stems.get(rng.below(stems.len())).copied().unwrap_or_default(); + endings.extend(d.forms(s).iter().filter_map(|f| f.strip_prefix(s)).filter(|e| !e.is_empty()).map(str::to_string)); + } + endings.sort(); + endings.dedup(); + let mut out = Vec::with_capacity(n); + while out.len() < n { + let s = stems.get(rng.below(stems.len())).copied().unwrap_or_default(); + let forms = d.forms(s); + if let Some(f) = forms.get(rng.below(forms.len())) { + out.push(f.clone()); + } + if let Some(e) = endings.get(rng.below(endings.len())) { + out.push(format!("{s}{e}")); + } + out.retain(|w| w.chars().all(char::is_alphabetic)); + } + // Letters only: the other checker's command line splits words at anything else. + out.retain(|w| w.chars().all(char::is_alphabetic)); + out.truncate(n); + out +} + fn main() -> Result<(), String> { let args: Vec = std::env::args().skip(1).collect(); match args.as_slice() { @@ -112,15 +144,16 @@ fn main() -> Result<(), String> { eprintln!("{} words in {took:?}", words.len()); Ok(()) } - [cmd, aff, dic, n, seed] if cmd == "gen" => { + [cmd, aff, dic, n, seed] if cmd == "gen" || cmd == "single" => { let d = load(aff, dic)?; let n: usize = n.parse().map_err(|_| "n is a number")?; let seed: u64 = seed.parse().map_err(|_| "seed is a number")?; - for w in generate(&d, n, seed) { + let words = if cmd == "gen" { generate(&d, n, seed) } else { single(&d, n, seed) }; + for w in words { println!("{w}"); } Ok(()) } - _ => Err("usage: affix_check check < words | affix_check gen ".into()), + _ => Err("usage: affix_check check < words | affix_check gen|single ".into()), } } diff --git a/crates/proof/src/affix.rs b/crates/proof/src/affix.rs index 7bec2722..2eccbc29 100644 --- a/crates/proof/src/affix.rs +++ b/crates/proof/src/affix.rs @@ -43,6 +43,8 @@ use crate::patterns::{FxHasher, FxMap}; mod compound; +pub(crate) use compound::MAX_SUGGEST_COMPOUNDS; + const MAGIC: &str = "WCAFF1"; /// Most bytes the compact format inflates to, written or read. const MAX_INFLATED: usize = 64 << 20; @@ -357,6 +359,11 @@ pub struct AffixDict { compound_firsts: Vec, /// Flag-based compounding (`COMPOUNDFLAG`…), if the dictionary compounds by flags. cpd: Option>, + /// Stems with `ONLYINCOMPOUND` homonyms beside others: stem index → those homonyms' flags + /// (compound parts only), while `flags` holds the others'. + cpd_sets: FxMap, + /// Those stems' sets with all their homonyms' flags, for `COMPOUNDRULE` parts. + rule_sets: FxMap, /// Appended string → the prefix rules adding it, in rule order. pfx_index: FxMap, Vec>, /// Longest appended prefix, in chars. @@ -490,6 +497,12 @@ pub fn capitalize(w: &str) -> String { } } +/// `w` with its first letter in lower case. +fn decapitalize(w: &str) -> String { + let mut c = w.chars(); + c.next().map(|f| f.to_lowercase().chain(c).collect()).unwrap_or_default() +} + /// Parsed rules and tables, shared by the Hunspell reader and the compact format. #[derive(Default)] struct Rules { @@ -777,8 +790,10 @@ impl FlagMode { /// Stem flags that rule a lookup out: `FORBIDDENWORD` always, and per lookup `NOSUGGEST` and /// `SUBSTANDARD` (suggestions), `KEEPCASE` (case variants of a word) or `NEEDAFFIX` (a stem on /// its own). +/// The second field: no flag-based compounds; the third: a suggestion candidate, whose compound +/// search gets a small budget ([`compound::MAX_SUGGEST_WORK`]). #[derive(Clone, Copy, Debug, Default)] -struct Excluded([Option; 5], bool); +struct Excluded([Option; 5], bool, bool); impl Excluded { /// Does the flag set `set` carry an excluded flag? @@ -796,6 +811,20 @@ impl Excluded { self.1 } + fn suggesting(self) -> bool { + self.2 + } + + /// These exclusions without `flag`. + fn without(mut self, flag: Option) -> Excluded { + for slot in &mut self.0 { + if *slot == flag { + *slot = None; + } + } + self + } + /// These exclusions and `flag` too. fn and(mut self, flag: Option) -> Excluded { if flag.is_some() @@ -842,6 +871,44 @@ fn has(set: &[u16], flag: u16) -> bool { set.binary_search(&flag).is_ok() } +/// Add a homonym's sorted flags `fresh` to the sorted set `set`; `false` (and nothing added) when +/// that would take it past [`MAX_FLAGS_PER_STEM`]. +fn merge_homonym(set: &mut Vec, fresh: &[u16]) -> bool { + let before = set.len(); + let add: Vec = fresh.iter().copied().filter(|&f| !has(set, f)).collect(); + if add.is_empty() { + // A repeated homonym adds nothing: no sort (200,000 duplicate lines of a stem with + // thousands of flags would each pay one). + return true; + } + if before.saturating_add(add.len()) > MAX_FLAGS_PER_STEM { + return false; + } + set.extend(add); + // A sorted run and a sorted tail: the stable sort merges them in linear time. + set.sort(); + true +} + +/// Put the exclusion flags `carried` (FORBIDDENWORD, NOSUGGEST, SUBSTANDARD of any homonym) back +/// into the sorted set `set`, making room by dropping a flag that excludes nothing (there are at +/// most three exclusion flags, so a full set always has one). +fn carry_exclusions(set: &mut Vec, carried: &[u16], exclusions: &[u16]) { + for &f in carried { + if has(set, f) { + continue; + } + if set.len() >= MAX_FLAGS_PER_STEM + && let Some(at) = set.iter().rposition(|g| !exclusions.contains(g)) + { + set.remove(at); + } + if let Err(at) = set.binary_search(&f) { + set.insert(at, f); + } + } +} + /// Do two sorted flag sets share a flag? (A binary search per flag of the smaller one.) fn intersects(a: &[u16], b: &[u16]) -> bool { let (small, large) = if a.len() <= b.len() { (a, b) } else { (b, a) }; @@ -1172,20 +1239,25 @@ impl AffixDict { let mut i = 0; let mut dropped = 0usize; let mut merged: Vec = Vec::new(); + let mut solo: Vec = Vec::new(); + let mut only_set: Vec = Vec::new(); let mut fresh: Vec = Vec::new(); let mut carried: Vec = Vec::new(); while i < stems.len() { let Some((word, ..)) = stems.get(i) else { break }; merged.clear(); + solo.clear(); + only_set.clear(); carried.clear(); let mut is_word = false; // NEEDAFFIX belongs to its homonym: the word stands alone if any homonym does // (`foo/NA` and `foo/B`: `foo` is right). let mut all_need = true; - // So does ONLYINCOMPOUND: a homonym with neither stands alone (German `schulen/DIXY` - // beside `schulen/Sozm`), but `buch/Sozm` beside `buch/hke` (NEEDAFFIX) doesn't, as - // in the CLI. - let mut alone = false; + // So does ONLYINCOMPOUND, with its flags: beside homonyms without it, the homonyms + // with it are kept apart (`cpd_sets`), only for compound parts (German + // `schulen/DIXY` beside `schulen/Sozm`: `schulen` alone, but not `schulens`), as in + // the CLI. + let (mut solo_need, mut only_need, mut any_solo, mut any_only) = (true, true, false, false); let mut j = i; while let Some((w, a, b, entry_is_word)) = stems.get(j) && w == word @@ -1202,22 +1274,24 @@ impl AffixDict { continue; } is_word = true; - all_need &= d.need_affix.is_some_and(|n| fresh.contains(&n)); - alone |= !d.only_in_compound.is_some_and(|o| fresh.contains(&o)) && !d.need_affix.is_some_and(|n| fresh.contains(&n)); + let need = d.need_affix.is_some_and(|n| fresh.contains(&n)); + let only = d.only_in_compound.is_some_and(|o| fresh.contains(&o)); + all_need &= need; fresh.sort_unstable(); fresh.dedup(); - fresh.retain(|&f| !has(&merged, f)); + if only { + any_only = true; + only_need &= need; + merge_homonym(&mut only_set, &fresh); + } else { + any_solo = true; + solo_need &= need; + merge_homonym(&mut solo, &fresh); + } // Homonyms can add up to more flags than one entry holds; past MAX_FLAGS_PER_STEM // an entry's other flags are dropped (its exclusions are carried below), so the // compact data never holds a set its reader refuses. - if fresh.is_empty() { - // A repeated homonym adds nothing: no sort (200,000 duplicate lines of a - // stem with thousands of flags would each pay one). - } else if merged.len().saturating_add(fresh.len()) <= MAX_FLAGS_PER_STEM { - merged.extend_from_slice(&fresh); - // A sorted run and a sorted tail: the stable sort merges them in linear time. - merged.sort(); - } else { + if !merge_homonym(&mut merged, &fresh) { dropped += 1; } } @@ -1228,26 +1302,35 @@ impl AffixDict { if !all_need && let Some(n) = d.need_affix { merged.retain(|&f| f != n); } - if alone { - merged.retain(|&f| Some(f) != d.only_in_compound && Some(f) != d.need_affix); - } - for &f in &carried { - if has(&merged, f) { - continue; + let mixed = any_solo && any_only; + let cpd_id = if mixed { + for (set, all_need) in [(&mut solo, solo_need), (&mut only_set, only_need)] { + if !all_need && let Some(n) = d.need_affix { + set.retain(|&f| f != n); + } + carry_exclusions(set, &carried, &exclusions); } - // Make room by dropping a flag that excludes nothing (there are at most three - // exclusion flags, so a full set always has one). - if merged.len() >= MAX_FLAGS_PER_STEM - && let Some(at) = merged.iter().rposition(|g| !exclusions.contains(g)) - { - merged.remove(at); - } - if let Err(at) = merged.binary_search(&f) { - merged.insert(at, f); + // `COMPOUNDRULE` parts read all the homonyms' flags. + if !d.compound_rules.is_empty() { + carry_exclusions(&mut merged, &carried, &exclusions); + let all = d.intern(&mut set_ids, &mut merged)?; + if let Ok(at) = u32::try_from(d.flags.len()) { + d.rule_sets.insert(at, all); + } } + std::mem::swap(&mut merged, &mut solo); + Some(d.intern(&mut set_ids, &mut only_set)?) + } else { + carry_exclusions(&mut merged, &carried, &exclusions); + None + }; + if let Some(c) = cpd_id + && let Ok(at) = u32::try_from(d.flags.len()) + { + d.cpd_sets.insert(at, c); } let id = d.intern(&mut set_ids, &mut merged)?; - if merged.iter().any(|f| compound_flags.contains(f)) + if [Some(id), cpd_id].into_iter().flatten().any(|s| d.set(s).iter().any(|f| compound_flags.contains(f))) && let Some(c) = word.chars().next() && !d.compound_firsts.contains(&c) { @@ -1429,7 +1512,8 @@ impl AffixDict { if !suggestible { return e; } - let e = e.and(self.no_suggest).and(self.substandard); + let mut e = e.and(self.no_suggest).and(self.substandard); + e.2 = true; // `MAXCPDSUGS 0`: compounds are right but never suggested. if self.cpd.as_ref().is_some_and(|c| c.no_suggestions) { e.without_compounds() } else { e } } @@ -1546,6 +1630,18 @@ impl AffixDict { self.contains_except(&self.normalize(w), self.excluded(true)) } + /// [`AffixDict::contains_suggestible`], without flag-based compounds when `compounds` is + /// false (suggestion candidates past the first [`compound::MAX_SUGGEST_COMPOUNDS`]). + pub(crate) fn contains_suggestible_with(&self, w: &str, compounds: bool) -> bool { + let e = self.excluded(true); + self.contains_except(&self.normalize(w), if compounds { e } else { e.without_compounds() }) + } + + /// Whether the dictionary compounds by flags (suggestion candidates then cost more). + pub(crate) fn flag_compounds(&self) -> bool { + self.cpd.is_some() + } + /// `w` after the `ICONV` conversions (at each position the longest match wins) and without /// the `IGNORE` characters. Input longer than [`MAX_INPUT_BYTES`] is no word and comes back /// unchanged, so the length check after this rejects it without converting it. @@ -1655,7 +1751,10 @@ impl AffixDict { let slot = parts.get_mut(a * n + b)?; *slot.get_or_insert_with(|| { let (Some(&x), Some(&y)) = (cuts.get(a), cuts.get(b)) else { return None }; - w.get(x..y).and_then(|p| self.stem_set(p)).filter(|set| !excluded.hits(set) && bare(set)) + w.get(x..y) + .and_then(|p| self.stem_lookup(p)) + .map(|(stem, set)| self.rule_sets.get(&stem).map_or(set, |&s| self.set(s))) + .filter(|set| !excluded.hits(set) && bare(set)) }) }; self.compound_rules.iter().any(|rule| compound_matches(rule, &mut part, n)) @@ -1756,7 +1855,10 @@ impl AffixDict { } let keep = excluded.and(self.keepcase); match casing(&w) { - Casing::Lower | Casing::Mixed => false, + Casing::Lower => false, + // A compound may join a capitalized stem to others, and the word start with a + // capital (`FooQux` from `foo` and `Qux`: right in the CLI). + Casing::Mixed => w.chars().next().is_some_and(char::is_uppercase) && self.flag_compound(&decapitalize(&w), excluded, true), Casing::Initial => { let lower = w.to_lowercase(); // CHECKSHARPS: a KEEPCASE word with `ß` may be capitalized (`Gruß` for `Gruß/k`). @@ -1995,7 +2097,7 @@ impl AffixDict { // In one-character mode a directive flag that no stem or rule uses isn't written, as // before flag ids existed, so those dictionaries' compact data stays byte for byte the same. let used = |flag: u16| { - self.flags.iter().any(|&id| self.set(id).contains(&flag)) + self.flags.iter().chain(self.cpd_sets.values()).any(|&id| self.set(id).contains(&flag)) || self.prefixes.iter().chain(&self.suffixes).any(|r| r.flag == flag || self.set(r.cont).contains(&flag)) }; if self.full_strip { @@ -2082,6 +2184,15 @@ impl AffixDict { raw.extend_from_slice(flags.as_bytes()); } raw.push(b'\n'); + // The ONLYINCOMPOUND homonyms' flags: the same stem again, merged back on reading. + if let Some(&c) = u32::try_from(i).ok().and_then(|i| self.cpd_sets.get(&i)) { + raw.push(b.len().min(255) as u8); + raw.extend_from_slice(b.get(b.len().min(255)..).unwrap_or_default()); + let mut flags = String::from("/"); + self.write_flags(self.set(c), &mut flags); + raw.extend_from_slice(flags.as_bytes()); + raw.push(b'\n'); + } prev = b; } // What [`AffixDict::from_bytes`] would refuse to inflate is refused here. diff --git a/crates/proof/src/affix/compound.rs b/crates/proof/src/affix/compound.rs index 31be0c8b..bc5b513d 100644 --- a/crates/proof/src/affix/compound.rs +++ b/crates/proof/src/affix/compound.rs @@ -11,7 +11,7 @@ //! carries `COMPOUNDPERMITFLAG`. Two suffixes in a row (continuation classes) only on the last. //! - An affix carrying `ONLYINCOMPOUND` is only valid inside a compound: a prefix anywhere, a //! suffix anywhere but on the last part (the CLI's answers; the word alone is no word). -//! - `COMPOUNDFORBIDFLAG` on a stem or an affix: that part may only be last. +//! - `COMPOUNDFORBIDFLAG` on a stem or a suffix: that part may only be last; on a prefix: no part. //! - A stem with `NEEDAFFIX` is a part only with an affix; `FORBIDDENWORD` stems are no part, and //! a forbidden word is never rebuilt from parts. //! - `COMPOUNDWORDMAX`: most parts (a `COMPOUNDROOT` stem counts as two). With `COMPOUNDSYLLABLE` @@ -22,16 +22,20 @@ //! - `CHECKCOMPOUNDCASE`: no capital on either side of a boundary, unless the other side is `-`. //! - `CHECKCOMPOUNDTRIPLE`: no three equal letters across a boundary (`schiff|fahrt`). //! `SIMPLIFIEDTRIPLE`: a part ending in a double letter may share its last letter with the next -//! part (`schiffahrt` = `schiff` + `fahrt`). +//! part (`schiffahrt` = `schiff` + `fahrt`), but with `CHECKCOMPOUNDTRIPLE` not leave three +//! equal letters (`busssen`). //! - `CHECKCOMPOUNDPATTERN end[/flag] begin[/flag]`: no boundary where the part before ends with //! `end` and the part after begins with `begin` (and they carry the flags); `0` for `end` means -//! an unaffixed part. The optional replacement field is refused. +//! a part without a suffix. The optional replacement field is refused. //! - `CHECKCOMPOUNDREP`: no compound that a `REP` replacement turns into a word (not a compound), //! checked on every two neighbouring parts and on every run of parts that ends the word. //! - A compound of three or more parts is rejected when one slip turns it into a word (not a //! compound): see [`AffixDict::near_word`]. The CLI's answers, with or without //! `CHECKCOMPOUNDREP`. -//! - `FORCEUCASE` on the last part: only the capitalized word (or the word in capitals) is right. +//! - `FORCEUCASE` on the last part's stem: a word with no capital is wrong. +//! - Case: `KEEPCASE` holds for the first part only; a capitalized stem may be a part, and a word +//! with capitals inside may start with a capital (`FooQux`). +//! - `ONLYINCOMPOUND` homonyms beside others keep their own flags (compound parts only). //! - `MAXCPDSUGS 0`: compounds are never suggested. //! //! Not read: `SYLLABLENUM` (Hungarian), which the man page doesn't describe; it is ignored. @@ -46,12 +50,19 @@ use super::{AffixDict, Excluded, FlagKey, FlagMode, Rule, has}; pub(super) const MAX_PARTS: usize = 16; /// Most steps one compound check may take (affix rules tried, parts looked up, splits followed). pub(super) const MAX_WORK: usize = 1 << 16; +/// Most steps the compound search of one suggestion candidate may take. +pub(crate) const MAX_SUGGEST_WORK: usize = 1 << 11; +/// Most candidates of one suggestion call whose compound search runs; the later ones are +/// checked as simple words only, so a long garbled word's suggestions stay quick. +pub(crate) const MAX_SUGGEST_COMPOUNDS: usize = 128; /// Most `CHECKCOMPOUNDPATTERN` lines (the Dutch dictionary has 52). const MAX_PATTERNS: usize = 512; /// Longest text on either side of a `CHECKCOMPOUNDPATTERN`, in bytes. const MAX_PATTERN_BYTES: usize = 64; /// Most vowels a `COMPOUNDSYLLABLE` line may list. const MAX_VOWELS: usize = 64; +/// Most ways one piece of a word may be a part (distinct flags and places). +const MAX_INFOS: usize = 256; /// Most `MAP` variants, `TRY` letters and `KEY` row letters the three-part check tries. const MAX_MAP_VARIANTS: usize = 256; const MAX_TRY: usize = 128; @@ -359,14 +370,16 @@ impl Compounding { } /// One way a piece of the word can be a compound part. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] struct Info { /// Where it may sit: [`FIRST`], [`MIDDLE`], [`LAST`]. pos: u8, /// Which of [`Compounding::marks`] its flags hold. marks: u64, - /// No affix. + /// No suffix (`CHECKCOMPOUNDPATTERN`'s `0`). unmodified: bool, + /// The stem carries `FORCEUCASE`. + force: bool, /// The stem's index (`CHECKCOMPOUNDDUP` compares stems). stem: u32, /// How many words it counts as (`COMPOUNDROOT`: two). @@ -393,6 +406,8 @@ struct Search<'a> { cuts: Vec, chars: Vec, excluded: Excluded, + /// The first part's (KEEPCASE holds there). + first_excluded: Excluded, capitalized: bool, /// Per piece `(start, end)` in chars: its infos as a range of `infos`. memo: Vec>, @@ -415,6 +430,10 @@ impl AffixDict { if self.is_forbidden(w) { return false; } + // KEEPCASE holds for the first part only (from `baz/K`, `Foobaz` and `FOOBAZ` are right + // in the CLI, `Bazfoo` and `BAZFOO` wrong). + let first_excluded = excluded; + let excluded = excluded.without(self.keepcase); let cuts: Vec = w.char_indices().map(|(i, _)| i).chain(std::iter::once(w.len())).collect(); let chars: Vec = w.chars().collect(); let n = cuts.len(); @@ -424,11 +443,13 @@ impl AffixDict { cuts, chars, excluded, + first_excluded, capitalized, memo: vec![None; n * n], infos: Vec::new(), path: Vec::with_capacity(MAX_PARTS), - work: 0, + // A suggestion candidate's search gets MAX_SUGGEST_WORK steps of the budget. + work: if excluded.suggesting() { MAX_WORK - MAX_SUGGEST_WORK } else { 0 }, near: None, }; self.parts_from(&mut s, 0) @@ -457,6 +478,11 @@ impl AffixDict { }; let (a, b) = self.infos_of(s, start, end); for k in a..b { + // Each way of the piece costs a step, and a step per pattern at the boundary. + s.work += 1 + s.c.patterns.len(); + if s.work > MAX_WORK { + return false; + } let Some(info) = s.infos.get(k as usize).copied() else { continue }; if info.pos & place == 0 || !self.boundary_ok(s, start, &info) { continue; @@ -493,11 +519,16 @@ impl AffixDict { if let (Some(&x), Some(&y)) = (s.cuts.get(a), s.cuts.get(b)) && let Some(piece) = s.w.get(x..y) { + // Deduplicated by hashing, and at most MAX_INFOS per piece (each one is tried at + // every split, against every pattern). + let mut seen: std::collections::HashSet> = Default::default(); let mut found: Vec = Vec::new(); let mut work = s.work; - self.analyses(piece, s.excluded, &mut work, &mut |an: Analysis| { - if let Some(info) = self.info(s.c, &an) - && !found.contains(&info) + let excluded = if a == 0 { s.first_excluded } else { s.excluded }; + self.analyses(piece, excluded, &mut work, &mut |an: Analysis| { + if found.len() < MAX_INFOS + && let Some(info) = self.info(s.c, &an) + && seen.insert(info) { found.push(info); } @@ -516,6 +547,10 @@ impl AffixDict { fn info(&self, c: &Compounding, an: &Analysis) -> Option { let sets = [Some(an.set), an.pfx.map(|r| self.set(r.cont)), an.sfx.map(|r| self.set(r.cont)), an.inner.map(|r| self.set(r.cont))]; let carries = |f: Option| f.is_some_and(|f| sets.iter().flatten().any(|s| has(s, f))); + // COMPOUNDFORBIDFLAG on a prefix: no part anywhere (`bar|defoo` is wrong in the CLI). + if an.pfx.is_some_and(|r| self.cont_holds(r, c.forbid)) { + return None; + } let mut pos = 0; if carries(c.flag) { pos = FIRST | MIDDLE | LAST; @@ -554,16 +589,19 @@ impl AffixDict { } } let words = if carries(c.root) { 2 } else { 1 }; - Some(Info { pos, marks, unmodified: an.pfx.is_none() && an.sfx.is_none(), stem: an.stem, words }) + // FORCEUCASE counts on the stem only (a suffix's continuation class holding it is ignored); + // a pattern's `0` means no suffix (a prefix doesn't count): the CLI's answers. + let force = c.force_ucase.is_some_and(|f| has(an.set, f)); + Some(Info { pos, marks, unmodified: an.sfx.is_none(), force, stem: an.stem, words }) } /// Every way `p` is a stem, or a stem with a prefix, one or two suffixes, or a prefix and a /// suffix, as a compound part may be (`ONLYINCOMPOUND` affixes and stems included). fn analyses<'a>(&'a self, p: &str, excluded: Excluded, work: &mut usize, emit: &mut impl FnMut(Analysis<'a>)) { - if let Some((stem, set)) = self.part_stem(p, excluded) - && !self.need_affix.is_some_and(|n| has(set, n)) - { - emit(Analysis { stem, set, pfx: None, sfx: None, inner: None }); + for (stem, set) in self.part_stems(p, excluded) { + if !self.need_affix.is_some_and(|n| has(set, n)) { + emit(Analysis { stem, set, pfx: None, sfx: None, inner: None }); + } } self.suffix_analyses(p, None, excluded, work, emit); let mut root = String::with_capacity(p.len() + 8); @@ -586,12 +624,12 @@ impl AffixDict { continue; } let (circ, needs) = (self.cont_holds(r, self.circumfix), self.cont_holds(r, self.need_affix)); - if !circ - && !needs - && let Some((stem, set)) = self.part_stem(&root, excluded) - && has(set, r.flag) - { - emit(Analysis { stem, set, pfx: Some(r), sfx: None, inner: None }); + if !circ && !needs { + for (stem, set) in self.part_stems(&root, excluded) { + if has(set, r.flag) { + emit(Analysis { stem, set, pfx: Some(r), sfx: None, inner: None }); + } + } } if r.cross { self.suffix_analyses(&root, Some(r), excluded, work, emit); @@ -613,7 +651,7 @@ impl AffixDict { root.push_str(base); root.push_str(&g.strip); let rules = || g.rules.iter().filter_map(|&i| self.suffixes.get(i as usize)); - if let Some((stem, set)) = self.part_stem(&root, excluded) { + for (stem, set) in self.part_stems(&root, excluded) { for r in rules() { *work += 1; if *work > MAX_WORK { @@ -673,22 +711,23 @@ impl AffixDict { root.clear(); root.push_str(base); root.push_str(&g.strip); - let Some((stem, set)) = self.part_stem(&root, excluded) else { continue }; - for r in g.cont_rules.iter().filter_map(|&i| self.suffixes.get(i as usize)) { - *work += 1; - if *work > MAX_WORK { - return; - } - if !has(set, r.flag) - || self.cont_holds(r, self.circumfix) - || self.cont_holds(r, self.only_in_compound) - || !r.cond_matches(&root, true) - { - continue; - } - for &outer in outers.iter().filter(|o| self.cont_holds(r, Some(o.flag))) { + for (stem, set) in self.part_stems(&root, excluded) { + for r in g.cont_rules.iter().filter_map(|&i| self.suffixes.get(i as usize)) { *work += 1; - emit(Analysis { stem, set, pfx: None, sfx: Some(outer), inner: Some(r) }); + if *work > MAX_WORK { + return; + } + if !has(set, r.flag) + || self.cont_holds(r, self.circumfix) + || self.cont_holds(r, self.only_in_compound) + || !r.cond_matches(&root, true) + { + continue; + } + for &outer in outers.iter().filter(|o| self.cont_holds(r, Some(o.flag))) { + *work += 1; + emit(Analysis { stem, set, pfx: None, sfx: Some(outer), inner: Some(r) }); + } } } } @@ -696,8 +735,12 @@ impl AffixDict { } /// The stem `w` as a compound part: its index and flags, unless it is forbidden or excluded. - fn part_stem(&self, w: &str, excluded: Excluded) -> Option<(u32, &[u16])> { - self.stem_lookup(w).filter(|(_, set)| !excluded.hits(set) && !self.forbidden.is_some_and(|f| has(set, f))) + /// A stem with `ONLYINCOMPOUND` homonyms beside others is two: each group's flags apply on + /// their own (`foo/XOS` and `foo/T`: `foos` is a part, `foot` is not, as in the CLI). + fn part_stems(&self, w: &str, excluded: Excluded) -> impl Iterator { + let found = self.stem_lookup(w); + let only = found.and_then(|(stem, _)| self.cpd_sets.get(&stem).map(|&c| (stem, self.set(c)))); + found.into_iter().chain(only).filter(move |(_, set)| !excluded.hits(set) && !self.forbidden.is_some_and(|f| has(set, f))) } /// May a part with `info` start at char `start`, after the last part of `s.path` @@ -716,6 +759,11 @@ impl AffixDict { return false; } } + // A SIMPLIFIEDTRIPLE join (the parts share a letter) still may not leave three equal + // letters in the word (`buss|ssen` as `busssen`, `boo|oob` as `booob`). + if c.triple && start + 1 == pe && s.chars.get(pe) == Some(&a) { + return false; + } if c.patterns.is_empty() { return true; } @@ -756,7 +804,8 @@ impl AffixDict { if s.c.dup && last.stem == prev.stem { return false; } - if last.marks & 1 != 0 && s.c.force_ucase.is_some() && !s.capitalized { + // FORCEUCASE: a word with no capital at all is wrong (`fooQuxbar` is right in the CLI). + if last.force && !s.capitalized && !s.chars.iter().any(|c| c.is_uppercase()) { return false; } let span = |s: &Search, i: usize, j: usize| -> String { @@ -1067,6 +1116,13 @@ mod tests { for w in wrong.split_whitespace() { assert!(!d.check(w), "{locale}: {w} is wrong"); } + // A long garbled word (wrong words run together): its suggestions stay quick. + let garbled: String = wrong.chars().filter(|c| !c.is_whitespace()).take(38).collect(); + let t = std::time::Instant::now(); + let _ = crate::affix_suggest::suggest_in(&d, &garbled, 8, &crate::affix_suggest::Tuning::DEFAULT); + let took = t.elapsed(); + eprintln!("{locale}: suggestions for {garbled} in {took:?}"); + assert!(took.as_secs() < 5, "{locale}: {took:?}"); } } @@ -1079,8 +1135,8 @@ mod tests { ), ( "de", - "Südwestbeifallsturm Machtfeudalideologie Schleifhärtering Gesprächskontur schule", - "chilisoßeswitttenaus Magmabewegungaminosäurebasis ausgezählterzehntagefieber heerfolgetemperatechnik haus buch", + "Südwestbeifallsturm Machtfeudalideologie Schleifhärtering Gesprächskontur schule Verfilmungen Aufwärmungen", + "chilisoßeswitttenaus Magmabewegungaminosäurebasis ausgezählterzehntagefieber heerfolgetemperatechnik haus buch verfilmungs verfilmungen aufwärmungen", ), ( "de-at", @@ -1119,8 +1175,8 @@ mod tests { ("nn", "", "orstuvastavantgardistar påskemåndagarkjempekenguruar piggtrådmusikkarreknemaskinar stimreinselikniårekeiper"), ( "sv", - "målvärdenservername sunniglädjepirrets kedjelängdendöbattangerna basfärgmetropoliterna", - "ogenomskådligsbåtsmanstorps tännkarheiskanens stängningstaggensbehängas råseleedemonlägeouppvärmt", + "målvärdenservername sunniglädjepirrets kedjelängdendöbattangerna basfärgmetropoliterna Norges Sveriges norge", + "ogenomskådligsbåtsmanstorps tännkarheiskanens stängningstaggensbehängas råseleedemonlägeouppvärmt norges sveriges afrikas", ), ( "sv-fi", @@ -1129,6 +1185,36 @@ mod tests { ), ]; + /// craft-libs#497 review: hundreds of prefixes and suffixes that each add a part to the same + /// one-letter stem, with distinct pattern flags, make tens of thousands of ways for one + /// piece. They are capped and counted, so a word still answers at once. + #[test] + fn many_ways_for_one_piece_stay_cheap() { + let marks: Vec = (0..62).filter_map(|i| char::from_u32(0x3041 + i)).collect(); + let mut aff = String::from("SET UTF-8\nFLAG UTF-8\nCOMPOUNDFLAG X\nCOMPOUNDMIN 1\nCOMPOUNDPERMITFLAG P\nCOMPOUNDWORDMAX 99\n"); + let mut pats: Vec = marks.iter().map(|m| format!("zz/{m} zz/{m}")).collect(); + pats.push("b a".into()); + aff.push_str(&format!("CHECKCOMPOUNDPATTERN {}\n", pats.len())); + for p in &pats { + aff.push_str(&format!("CHECKCOMPOUNDPATTERN {p}\n")); + } + let mut flags = String::new(); + for (kind, base, shift) in [("PFX", 0x4e00, 0), ("SFX", 0x5e00, 31)] { + for i in 0..250u32 { + let Some(f) = char::from_u32(base + i) else { continue }; + let m: String = (0..31).filter(|b| (i >> b) & 1 == 1).filter_map(|b| marks.get(shift + b)).collect(); + aff.push_str(&format!("{kind} {f} Y 1\n{kind} {f} 0 a/XP{m} .\n")); + flags.push(f); + } + } + let d = AffixDict::from_hunspell(&aff, &format!("2\na/X{flags}\nb/X\n"), &[]).unwrap(); + let t = std::time::Instant::now(); + for w in ["baaa", "bbbbbbbbbbbbbbbbbbbbaaa", "ab"] { + let _ = d.check(w); + } + assert!(t.elapsed().as_millis() < 2000, "{:?}", t.elapsed()); + } + /// `COMPOUNDRULE` as the Swedish (`)k`) and Mongolian (`[a0,a1]`, `%?`, `.`) dictionaries /// write it, read as the CLI answers on the same rules. #[test] @@ -1194,7 +1280,11 @@ mod tests { let _ = d.check(w); slowest = slowest.max(t.elapsed()); } - eprintln!("hostile compounding: slowest word {slowest:?}"); + let t = std::time::Instant::now(); + let _ = crate::affix_suggest::suggest_in(&d, &"qwertyuiopasdfghjklzxcvbnm".repeat(2)[..38], 8, &crate::affix_suggest::Tuning::DEFAULT); + let suggest = t.elapsed(); + eprintln!("hostile compounding: slowest word {slowest:?}, suggestions {suggest:?}"); + assert!(suggest.as_secs() < 10, "{suggest:?}"); assert!(slowest.as_secs() < 5, "{slowest:?}"); } @@ -1477,6 +1567,144 @@ mod tests { right: false, listed: "abcdefghijkl abcdmnopijkl efghabcdijkl efghmnopabcd ijklabcdefgh ijklmnopabcd mnopabcdefgh", }, + Pinned { + name: "onlyincompound_homonyms", + aff: "COMPOUNDFLAG X\nONLYINCOMPOUND O\nSFX S Y 1\nSFX S 0 s .\nSFX T Y 1\nSFX T 0 t .", + dic: "foo/XOS foo/T bar/X baz/XO baz/XT", + pieces: "foo foos foot bar baz bazt bazs", + extra: "", + right: true, + listed: "foo foot bar baz bazt foofoo foofoos foobar foobaz foobazt barfoo barfoos barbar barbaz barbazt bazfoo bazfoos \ + bazbar bazbaz bazbazt foofoofoo foofoofoos foofoobar foofoobaz foofoobazt foobarfoo foobarfoos foobarbar \ + foobarbaz foobarbazt foobazfoo foobazfoos foobazbar foobazbaz foobazbazt barfoofoo barfoofoos barfoobar \ + barfoobaz barfoobazt barbarfoo barbarfoos barbarbar barbarbaz barbarbazt barbazfoo barbazfoos barbazbar \ + barbazbaz barbazbazt bazfoofoo bazfoofoos bazfoobar bazfoobaz bazfoobazt bazbarfoo bazbarfoos bazbarbar \ + bazbarbaz bazbarbazt bazbazfoo bazbazfoos bazbazbar bazbazbaz bazbazbazt", + }, + Pinned { + name: "onlyincompound_homonyms_alone", + aff: "ONLYINCOMPOUND O\nNEEDAFFIX N\nSFX S Y 1\nSFX S 0 s .\nPFX U Y 1\nPFX U 0 un .", + dic: "bar/OS bar/U qux/O qux/S zed/OS", + pieces: "", + extra: "bar bars unbar qux quxs unqux zed zeds", + right: true, + listed: "bar unbar qux quxs", + }, + Pinned { + name: "simplified_triple_checked", + aff: "COMPOUNDFLAG X\nCHECKCOMPOUNDTRIPLE\nSIMPLIFIEDTRIPLE", + dic: "boo/X oob/X ob/X xaa/X aay/X ay/X schiff/X fahrt/X ffahrt/X buss/X ssen/X fjell/X llama/X", + pieces: "", + extra: "booob boob boooob boobob xaaay xaay xaaaay schiffahrt schifffahrt schiffffahrt booobboo boobboo busssen bussen fjelllama fjellama", + right: true, + listed: "schiffahrt", + }, + Pinned { + name: "simplified_triple_min", + aff: "COMPOUNDFLAG X\nCOMPOUNDMIN 3\nCHECKCOMPOUNDTRIPLE\nSIMPLIFIEDTRIPLE", + dic: "foo/X ooa/X oo/X bb/X bbb/X abb/X bba/X", + pieces: "foo ooa oo abb bba bbb", + extra: "fooa abba abbba abbbba", + right: true, + listed: "foo ooa oo abb bba bbb foofoo fooabb foobba foobbb ooafoo ooaooa ooaabb ooabba ooabbb abbfoo abbooa abbabb \ + bbafoo bbaooa bbaabb bbabba bbabbb bbbfoo bbbooa bbbabb foofoofoo foofooabb foofoobba foofoobbb fooabbfoo \ + fooabbooa fooabbabb foobbafoo foobbaooa foobbaabb foobbabba foobbabbb foobbbfoo foobbbooa foobbbabb ooafoofoo \ + ooafooabb ooafoobba ooafoobbb ooaooafoo ooaooaooa ooaooaabb ooaooabba ooaooabbb ooaabbfoo ooaabbooa ooaabbabb \ + ooabbafoo ooabbaooa ooabbaabb ooabbabba ooabbabbb ooabbbfoo ooabbbooa ooabbbabb abbfoofoo abbfooabb abbfoobba \ + abbfoobbb abbooafoo abbooaooa abbooaabb abbooabba abbooabbb abbabbfoo abbabbooa abbabbabb bbafoofoo bbafooabb \ + bbafoobba bbafoobbb bbaooafoo bbaooaooa bbaooaabb bbaooabba bbaooabbb bbaabbfoo bbaabbooa bbaabbabb bbabbafoo \ + bbabbaooa bbabbaabb bbabbabba bbabbabbb bbabbbfoo bbabbbooa bbabbbabb bbbfoofoo bbbfooabb bbbfoobba bbbfoobbb \ + bbbooafoo bbbooaooa bbbooaabb bbbooabba bbbooabbb bbbabbfoo bbbabbooa bbbabbabb", + }, + Pinned { + name: "keepcase_parts", + aff: "COMPOUNDFLAG X\nKEEPCASE K", + dic: "foo/X baz/XK bar/X", + pieces: "foo baz bar Foo Baz BAZ FOO", + extra: "", + right: true, + listed: "foo baz bar Foo FOO foofoo foobaz foobar bazfoo bazbaz bazbar barfoo barbaz barbar Foofoo Foobaz Foobar FOOBAZ \ + FOOFOO foofoofoo foofoobaz foofoobar foobazfoo foobazbaz foobazbar foobarfoo foobarbaz foobarbar bazfoofoo \ + bazfoobaz bazfoobar bazbazfoo bazbazbaz bazbazbar bazbarfoo bazbarbaz bazbarbar barfoofoo barfoobaz barfoobar \ + barbazfoo barbazbaz barbazbar barbarfoo barbarbaz barbarbar Foofoofoo Foofoobaz Foofoobar Foobazfoo Foobazbaz \ + Foobazbar Foobarfoo Foobarbaz Foobarbar FOOBAZBAZ FOOBAZFOO FOOFOOBAZ FOOFOOFOO", + }, + Pinned { + name: "capitalized_stem_parts", + aff: "COMPOUNDFLAG X", + dic: "foo/X Qux/X bar/X", + pieces: "foo Qux qux bar Foo QUX", + extra: "", + right: true, + listed: "foo Qux bar Foo QUX foofoo fooQux foobar Quxfoo QuxQux Quxbar barfoo barQux barbar Foofoo FooQux Foobar \ + foofoofoo foofooQux foofoobar fooQuxfoo fooQuxQux fooQuxbar foobarfoo foobarQux foobarbar Quxfoofoo QuxfooQux \ + Quxfoobar QuxQuxfoo QuxQuxQux QuxQuxbar Quxbarfoo QuxbarQux Quxbarbar barfoofoo barfooQux barfoobar barQuxfoo \ + barQuxQux barQuxbar barbarfoo barbarQux barbarbar Foofoofoo FoofooQux Foofoobar FooQuxfoo FooQuxQux FooQuxbar \ + Foobarfoo FoobarQux Foobarbar", + }, + Pinned { + name: "forceucase_mixed", + aff: "COMPOUNDFLAG X\nFORCEUCASE U\nKEEPCASE K", + dic: "foo/X bar/XU baz/XK Qux/X", + pieces: "foo bar baz Foo Bar Qux", + extra: "FOOBAR FooBar fooBar FOOBAZ Foobaz QuxBar Quxbar QUXBAR", + right: true, + listed: "foo bar baz Foo Bar Qux foofoo foobaz fooQux barfoo barbaz barQux bazfoo bazbaz bazQux Foofoo Foobar Foobaz \ + FooQux Barfoo Barbar Barbaz BarQux Quxfoo Quxbar Quxbaz QuxQux foofoofoo foofoobaz foofooQux foobarfoo \ + foobarbaz foobarQux foobazfoo foobazbaz foobazQux fooQuxfoo fooQuxbar fooQuxbaz fooQuxQux barfoofoo barfoobaz \ + barfooQux barbarfoo barbarbaz barbarQux barbazfoo barbazbaz barbazQux barQuxfoo barQuxbar barQuxbaz barQuxQux \ + bazfoofoo bazfoobaz bazfooQux bazbarfoo bazbarbaz bazbarQux bazbazfoo bazbazbaz bazbazQux bazQuxfoo bazQuxbar \ + bazQuxbaz bazQuxQux Foofoofoo Foofoobar Foofoobaz FoofooQux Foobarfoo Foobarbar Foobarbaz FoobarQux Foobazfoo \ + Foobazbar Foobazbaz FoobazQux FooQuxfoo FooQuxbar FooQuxbaz FooQuxQux Barfoofoo Barfoobar Barfoobaz BarfooQux \ + Barbarfoo Barbarbar Barbarbaz BarbarQux Barbazfoo Barbazbar Barbazbaz BarbazQux BarQuxfoo BarQuxbar BarQuxbaz \ + BarQuxQux Quxfoofoo Quxfoobar Quxfoobaz QuxfooQux Quxbarfoo Quxbarbar Quxbarbaz QuxbarQux Quxbazfoo Quxbazbar \ + Quxbazbaz QuxbazQux QuxQuxfoo QuxQuxbar QuxQuxbaz QuxQuxQux FOOBAR FOOBAZ QUXBAR", + }, + Pinned { + name: "forceucase_suffix", + aff: "COMPOUNDFLAG X\nFORCEUCASE U\nSFX S Y 1\nSFX S 0 s/U .", + dic: "foo/X baz/XS", + pieces: "foo baz bazs", + extra: "Foobazs", + right: false, + listed: "bazsfoo bazsbaz bazsbazs foobazsfoo foobazsbaz foobazsbazs bazbazsfoo bazbazsbaz bazbazsbazs bazsfoofoo \ + bazsfoobaz bazsfoobazs bazsbazfoo bazsbazbaz bazsbazbazs bazsbazsfoo bazsbazsbaz bazsbazsbazs", + }, + Pinned { + name: "pattern_unsuffixed", + aff: "COMPOUNDFLAG X\nCOMPOUNDPERMITFLAG P\nSFX S Y 1\nSFX S 0 s/P .\nPFX U Y 1\nPFX U 0 un/P .\nCHECKCOMPOUNDPATTERN 2\nCHECKCOMPOUNDPATTERN 0 b\nCHECKCOMPOUNDPATTERN 0 un", + dic: "foo/XSU bar/XSU oof/X", + pieces: "foo foos unfoo bar bars unbar oof", + extra: "", + right: true, + listed: "foo foos unfoo bar bars unbar oof foofoo foofoos foooof foosfoo foosfoos foosunfoo foosbar foosbars foosunbar \ + foosoof unfoofoo unfoofoos unfoooof barfoo barfoos baroof barsfoo barsfoos barsunfoo barsbar barsbars \ + barsunbar barsoof unbarfoo unbarfoos unbaroof ooffoo ooffoos oofoof foofoofoo foofoofoos foofoooof foofoosfoo \ + foofoosfoos foofoosunfoo foofoosbar foofoosbars foofoosunbar foofoosoof fooooffoo fooooffoos foooofoof \ + foosfoofoo foosfoofoos foosfoooof foosfoosfoo foosfoosfoos foosfoosunfoo foosfoosbar foosfoosbars \ + foosfoosunbar foosfoosoof foosunfoofoo foosunfoofoos foosunfoooof foosbarfoo foosbarfoos foosbaroof \ + foosbarsfoo foosbarsfoos foosbarsunfoo foosbarsbar foosbarsbars foosbarsunbar foosbarsoof foosunbarfoo \ + foosunbarfoos foosunbaroof foosooffoo foosooffoos foosoofoof unfoofoofoo unfoofoofoos unfoofoooof unfoofoosfoo \ + unfoofoosfoos unfoofoosunfoo unfoofoosbar unfoofoosbars unfoofoosunbar unfoofoosoof unfooooffoo unfooooffoos \ + unfoooofoof barfoofoo barfoofoos barfoooof barfoosfoo barfoosfoos barfoosunfoo barfoosbar barfoosbars \ + barfoosunbar barfoosoof barooffoo barooffoos baroofoof barsfoofoo barsfoofoos barsfoooof barsfoosfoo \ + barsfoosfoos barsfoosunfoo barsfoosbar barsfoosbars barsfoosunbar barsfoosoof barsunfoofoo barsunfoofoos \ + barsunfoooof barsbarfoo barsbarfoos barsbaroof barsbarsfoo barsbarsfoos barsbarsunfoo barsbarsbar barsbarsbars \ + barsbarsunbar barsbarsoof barsunbarfoo barsunbarfoos barsunbaroof barsooffoo barsooffoos barsoofoof \ + unbarfoofoo unbarfoofoos unbarfoooof unbarfoosfoo unbarfoosfoos unbarfoosunfoo unbarfoosbar unbarfoosbars \ + unbarfoosunbar unbarfoosoof unbarooffoo unbarooffoos unbaroofoof ooffoofoo ooffoofoos ooffoooof ooffoosfoo \ + ooffoosfoos ooffoosunfoo ooffoosbar ooffoosbars ooffoosunbar ooffoosoof oofooffoo oofooffoos oofoofoof", + }, + Pinned { + name: "forbidflag_prefix", + aff: "COMPOUNDFLAG X\nCOMPOUNDPERMITFLAG P\nCOMPOUNDFORBIDFLAG Q\nPFX C Y 1\nPFX C 0 de/PQ .", + dic: "foo/XC bar/X", + pieces: "foo defoo bar", + extra: "", + right: true, + listed: "foo defoo bar foofoo foobar barfoo barbar foofoofoo foofoobar foobarfoo foobarbar barfoofoo barfoobar \ + barbarfoo barbarbar", + }, Pinned { name: "forbidden", aff: "COMPOUNDFLAG X\nFORBIDDENWORD F\nSFX S Y 1\nSFX S 0 en .", diff --git a/crates/proof/src/affix_suggest.rs b/crates/proof/src/affix_suggest.rs index 3c9b06bc..7d488209 100644 --- a/crates/proof/src/affix_suggest.rs +++ b/crates/proof/src/affix_suggest.rs @@ -125,6 +125,8 @@ struct Collector<'a> { names: bool, /// May half of a missing space be one ([`Tuning::strict_case`])? split_names: bool, + /// Candidates checked with compounds so far (at most `MAX_SUGGEST_COMPOUNDS`). + compound_checks: usize, } impl Collector<'_> { @@ -144,14 +146,16 @@ impl Collector<'_> { let mut words = Vec::new(); let mut cost = cost; let names = if cand.contains(' ') { self.split_names } else { self.names }; + let compounds = self.d.flag_compounds() && self.compound_checks < crate::affix::MAX_SUGGEST_COMPOUNDS; + self.compound_checks += usize::from(compounds); for part in cand.split(' ') { - if self.d.contains_suggestible(part) { + if self.d.contains_suggestible_with(part, compounds) { words.push(part.to_string()); } else if !names { return; } else { let c = capitalize(part); - if !self.d.contains_suggestible(&c) { + if !self.d.contains_suggestible_with(&c, compounds) { return; } words.push(c); @@ -308,7 +312,8 @@ pub(crate) fn suggest_in(d: &AffixDict, word: &str, max: usize, t: &Tuning) -> V let chars: Vec = input.chars().collect(); let names = !t.strict_case || case != Casing::Lower; let split_names = !t.strict_case; - let mut col = Collector { d, input: input.clone(), tested: HashMap::new(), found: HashMap::new(), seq: 0, names, split_names }; + let mut col = + Collector { d, input: input.clone(), tested: HashMap::new(), found: HashMap::new(), seq: 0, names, split_names, compound_checks: 0 }; add_case(&mut col, word, &input); let map = groups(d, t); add_map_slips(&mut col, &chars, &map, t); @@ -640,8 +645,16 @@ mod tests { #[test] fn a_cheaper_later_offer_wins() { let d = AffixDict::from_hunspell("", "1\ncafe\n", &[]).unwrap(); - let mut col = - Collector { d: &d, input: "cafè".into(), tested: HashMap::new(), found: HashMap::new(), seq: 0, names: true, split_names: true }; + let mut col = Collector { + d: &d, + input: "cafè".into(), + tested: HashMap::new(), + found: HashMap::new(), + seq: 0, + names: true, + split_names: true, + compound_checks: 0, + }; col.offer("cafe", COST_DROP_DIACRITIC); col.offer("cafe", COST_REP); col.offer("cafe", COST_SCAN); From b8efc5add9e2e8d69ac6a5939a35b713e73a35be Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Mon, 2 Nov 2026 03:18:31 +0000 Subject: [PATCH 11/40] =?UTF-8?q?CP7.4:=20craft-libs#497=20re-review=20?= =?UTF-8?q?=E2=80=94=20hu=20prefixed=20suffix=20chains,=20homonyms=20kept?= =?UTF-8?q?=20apart,=20all-caps=20mixed-case=20stems;=20compound=20suggest?= =?UTF-8?q?ions=20cheapest-first=20in=20one=20budget?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - A prefix with two chained suffixes (hu `meg`+`épít`+`és`+`e`, eo `enjugo`): every rule cross, the prefix flag from the stem or a continuation class (CLI answers). - Homonyms with different flags are kept apart (and written apart in the compact form): a prefix and a suffix, or a compound flag and a suffix, must come from one homonym (nl `afstopje`, `schuurpoederden` wrong in the CLI). nl now 100% in both comparisons. - Hungarian syllable rule leaves out a last suffix with no continuation classes; COMPOUNDROOT counts on the stem only. - Near-word check: a word listed as an ONLYINCOMPOUND/NEEDAFFIX stem counts only with a suffix, not a prefix (de `Fahrradverkehr` right). - A word in capitals may be a mixed-case stem (`KB`/`KBS` from `kB/S`, hu `DNY`, nl `BSC`). - Mixed-case retry keeps KEEPCASE for the first part; KEEPCASE kept apart from the other exclusions (NOSUGGEST sharing its flag is no longer cleared); overflow of the homonym groups counts as dropped. - Suggestions: candidates are checked as simple words first; the rest get the compound check cheapest first, those changed where the word stops being parts before others, in one step budget per call, with piece analyses shared between candidates. The suggestions' stem scan makes forms in one shared budget (mn). - affix_check example: `suggest` command; AffixDict::suggest. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FypgCX8LsYYE8fsj4byBm2 --- crates/proof/examples/affix_check.rs | 14 +- crates/proof/src/affix.rs | 343 ++++++++++++++++++++++----- crates/proof/src/affix/compound.rs | 303 +++++++++++++++++++++-- crates/proof/src/affix_suggest.rs | 97 +++++++- 4 files changed, 664 insertions(+), 93 deletions(-) diff --git a/crates/proof/examples/affix_check.rs b/crates/proof/examples/affix_check.rs index 329c4fcb..78c3f9bc 100644 --- a/crates/proof/examples/affix_check.rs +++ b/crates/proof/examples/affix_check.rs @@ -8,6 +8,8 @@ //! cargo run --release -p craft-proof --example affix_check -- gen de_DE.aff de_DE.dic 2000 7 > words.txt //! # Single words instead: forms of random stems, and stems with another stem's ending. //! cargo run --release -p craft-proof --example affix_check -- single de_DE.aff de_DE.dic 2000 7 > words.txt +//! # Suggestions: `wordmicrosecondssuggestion,suggestion…` per line. +//! cargo run --release -p craft-proof --example affix_check -- suggest de_DE.aff de_DE.dic < typos.txt //! ``` //! //! The answers are compared with `hunspell -d -l` on the same words @@ -144,6 +146,16 @@ fn main() -> Result<(), String> { eprintln!("{} words in {took:?}", words.len()); Ok(()) } + [cmd, aff, dic] if cmd == "suggest" => { + let d = load(aff, dic)?; + let mut out = std::io::stdout().lock(); + for w in std::io::stdin().lock().lines().map_while(Result::ok) { + let t = std::time::Instant::now(); + let s = d.suggest(&w, 8); + writeln!(out, "{w}\t{}\t{}", t.elapsed().as_micros(), s.join(",")).map_err(|e| e.to_string())?; + } + Ok(()) + } [cmd, aff, dic, n, seed] if cmd == "gen" || cmd == "single" => { let d = load(aff, dic)?; let n: usize = n.parse().map_err(|_| "n is a number")?; @@ -154,6 +166,6 @@ fn main() -> Result<(), String> { } Ok(()) } - _ => Err("usage: affix_check check < words | affix_check gen|single ".into()), + _ => Err("usage: affix_check check|suggest < words | affix_check gen|single ".into()), } } diff --git a/crates/proof/src/affix.rs b/crates/proof/src/affix.rs index 2eccbc29..cdc2ccc5 100644 --- a/crates/proof/src/affix.rs +++ b/crates/proof/src/affix.rs @@ -43,7 +43,7 @@ use crate::patterns::{FxHasher, FxMap}; mod compound; -pub(crate) use compound::MAX_SUGGEST_COMPOUNDS; +pub(crate) use compound::{MAX_SUGGEST_COMPOUND_WORK, Pieces}; const MAGIC: &str = "WCAFF1"; /// Most bytes the compact format inflates to, written or read. @@ -224,6 +224,8 @@ fn parse_cond(text: &str) -> Result, String> { Ok(out.into_boxed_slice()) } +/// Most homonyms of one stem kept apart ([`AffixDict::homonyms`]); past it their flags count merged. +const MAX_HOMONYMS: usize = 16; /// How many suffixes a word may carry in a row through continuation classes (`work` + /// `able/B` + `s` → `workables` is two). Hunspell strips at most two suffixes, so a third /// chained suffix is not a word; the bound also keeps hostile input from walking deep recursions. @@ -364,8 +366,16 @@ pub struct AffixDict { cpd_sets: FxMap, /// Those stems' sets with all their homonyms' flags, for `COMPOUNDRULE` parts. rule_sets: FxMap, + /// Stems listed more than once with different flags (not the `ONLYINCOMPOUND` homonyms + /// beside others): stem index → each homonym's flags. `flags` holds them merged, which + /// answers for one affix; a prefix with a suffix must come from one homonym (Dutch `stop/Pj…` + /// takes `af`, `stop/Ca…` takes `je`: `afstopje` is wrong in the CLI). + homonyms: FxMap>, /// Appended string → the prefix rules adding it, in rule order. pfx_index: FxMap, Vec>, + /// Stems in mixed case (`kB`, `MSc`) by their lower case: a word in capitals may be one + /// (`KB`, `KBS`), as in the CLI. + mixed: FxMap, Vec>, /// Longest appended prefix, in chars. max_pfx_chars: usize, } @@ -791,14 +801,15 @@ impl FlagMode { /// `SUBSTANDARD` (suggestions), `KEEPCASE` (case variants of a word) or `NEEDAFFIX` (a stem on /// its own). /// The second field: no flag-based compounds; the third: a suggestion candidate, whose compound -/// search gets a small budget ([`compound::MAX_SUGGEST_WORK`]). +/// search gets a small budget ([`compound::MAX_SUGGEST_WORK`]); the fourth: `KEEPCASE`, kept +/// apart since it holds for a compound's first part only (and may share its id with another). #[derive(Clone, Copy, Debug, Default)] -struct Excluded([Option; 5], bool, bool); +struct Excluded([Option; 5], bool, bool, Option); impl Excluded { /// Does the flag set `set` carry an excluded flag? fn hits(self, set: &[u16]) -> bool { - self.0.iter().flatten().any(|f| has(set, *f)) + self.0.iter().chain(std::iter::once(&self.3)).flatten().any(|f| has(set, *f)) } /// These exclusions, and no flag-based compounds (`MAXCPDSUGS 0` for suggestions). @@ -815,13 +826,15 @@ impl Excluded { self.2 } - /// These exclusions without `flag`. - fn without(mut self, flag: Option) -> Excluded { - for slot in &mut self.0 { - if *slot == flag { - *slot = None; - } - } + /// These exclusions and `KEEPCASE` (`flag`): a case variant of the word is being checked. + fn keeping(mut self, flag: Option) -> Excluded { + self.3 = flag; + self + } + + /// These exclusions without `KEEPCASE` (a compound part after the first). + fn without_keep(mut self) -> Excluded { + self.3 = None; self } @@ -871,6 +884,19 @@ fn has(set: &[u16], flag: u16) -> bool { set.binary_search(&flag).is_ok() } +/// Note a homonym's flags among its group's (`group`), unless an equal one is there; false past +/// [`MAX_HOMONYMS`] (then the merged flags answer: hostile input can't make it costly). +fn note_homonym(group: &mut Vec>, fresh: &[u16]) -> bool { + if group.iter().any(|g| g == fresh) { + return true; + } + if group.len() >= MAX_HOMONYMS { + return false; + } + group.push(fresh.to_vec()); + true +} + /// Add a homonym's sorted flags `fresh` to the sorted set `set`; `false` (and nothing added) when /// that would take it past [`MAX_FLAGS_PER_STEM`]. fn merge_homonym(set: &mut Vec, fresh: &[u16]) -> bool { @@ -1243,12 +1269,18 @@ impl AffixDict { let mut only_set: Vec = Vec::new(); let mut fresh: Vec = Vec::new(); let mut carried: Vec = Vec::new(); + // Each homonym's flags (ONLYINCOMPOUND ones apart), and whether all fit. + let mut each: Vec> = Vec::new(); + let mut each_only: Vec> = Vec::new(); while i < stems.len() { let Some((word, ..)) = stems.get(i) else { break }; merged.clear(); solo.clear(); only_set.clear(); carried.clear(); + each.clear(); + each_only.clear(); + let mut fits = true; let mut is_word = false; // NEEDAFFIX belongs to its homonym: the word stands alone if any homonym does // (`foo/NA` and `foo/B`: `foo` is right). @@ -1279,21 +1311,23 @@ impl AffixDict { all_need &= need; fresh.sort_unstable(); fresh.dedup(); - if only { + let group_fits = if only { any_only = true; only_need &= need; - merge_homonym(&mut only_set, &fresh); + merge_homonym(&mut only_set, &fresh) } else { any_solo = true; solo_need &= need; - merge_homonym(&mut solo, &fresh); - } + merge_homonym(&mut solo, &fresh) + }; // Homonyms can add up to more flags than one entry holds; past MAX_FLAGS_PER_STEM // an entry's other flags are dropped (its exclusions are carried below), so the // compact data never holds a set its reader refuses. - if !merge_homonym(&mut merged, &fresh) { + if !merge_homonym(&mut merged, &fresh) || !group_fits { dropped += 1; + fits = false; } + fits &= note_homonym(if only { &mut each_only } else { &mut each }, &fresh); } if !is_word { i = j; @@ -1303,6 +1337,11 @@ impl AffixDict { merged.retain(|&f| f != n); } let mixed = any_solo && any_only; + // The homonyms whose flags `flags` holds merged, if they differ. + let apart = if mixed { std::mem::take(&mut each) } else { each.drain(..).chain(each_only.drain(..)).collect() }; + if fits { + d.keep_apart(&mut set_ids, apart, &carried, &exclusions)?; + } let cpd_id = if mixed { for (set, all_need) in [(&mut solo, solo_need), (&mut only_set, only_need)] { if !all_need && let Some(n) = d.need_affix { @@ -1378,6 +1417,23 @@ impl AffixDict { Ok(id) } + /// Keep the homonyms `apart` of the stem about to be pushed, each with the exclusions + /// `carried` (see [`AffixDict::homonyms`]), when there are two or more. + fn keep_apart(&mut self, ids: &mut FxMap, u32>, mut apart: Vec>, carried: &[u16], exclusions: &[u16]) -> Result<(), String> { + if apart.len() < 2 { + return Ok(()); + } + let mut sets = Vec::with_capacity(apart.len()); + for set in &mut apart { + carry_exclusions(set, carried, exclusions); + sets.push(self.intern(ids, set)?); + } + if let Ok(at) = u32::try_from(self.flags.len()) { + self.homonyms.insert(at, sets.into()); + } + Ok(()) + } + /// The flag set `id`, sorted (empty when unknown). fn set(&self, id: u32) -> &[u16] { let i = id as usize; @@ -1423,6 +1479,15 @@ impl AffixDict { } } self.pfx_index = pfx_index; + let mut mixed: FxMap, Vec> = FxMap::default(); + for i in 0..self.len() { + if let (Some(w), Ok(i)) = (self.word(i), u32::try_from(i)) + && casing(w) == Casing::Mixed + { + mixed.entry(w.to_lowercase().into()).or_default().push(i); + } + } + self.mixed = mixed; self.max_pfx_chars = self.prefixes.iter().map(|r| r.add.chars().count()).max().unwrap_or(0); let mut cont_flag = vec![false; self.flag_keys.len()]; for r in self.prefixes.iter().chain(&self.suffixes) { @@ -1531,6 +1596,19 @@ impl AffixDict { self.stem_set(w).filter(|set| !excluded.hits(set) && !self.only_in_compound.is_some_and(|f| has(set, f))) } + /// Does one homonym of the stem `w` (excluded ones aside) have flags meeting `pred`? + fn homonym_meets(&self, w: &str, excluded: Excluded, pred: impl Fn(&[u16]) -> bool) -> bool { + let Some((at, set)) = self.stem_lookup(w) else { return false }; + let usable = |set: &[u16]| !excluded.hits(set) && !self.only_in_compound.is_some_and(|f| has(set, f)); + if !usable(set) { + return false; + } + match self.homonyms.get(&at) { + Some(sets) => sets.iter().map(|&h| self.set(h)).any(|h| usable(h) && pred(h)), + None => pred(set), + } + } + /// Is `w` listed with `FORBIDDENWORD`? Then it is no word, however else it could be built /// (affixes, compounds), as in Hunspell. fn is_forbidden(&self, w: &str) -> bool { @@ -1578,8 +1656,13 @@ impl AffixDict { /// The rules of a group that could lead to `w` are merged into one set of their flags (sorted) /// and followed once, not once per rule: the answer is the same, but thousands of rules of one /// flag cost one walk, not one each (#3613). - fn form_has_flag(&self, w: &str, flags: &[u16], depth: u8, excluded: Excluded) -> bool { - if self.stem_flags_except(w, excluded).is_some_and(|f| intersects(f, flags)) { + /// + /// With `pfx` set (a prefix was taken off), every rule on the way must allow cross products + /// and the prefix's flag must come from the stem or a continuation class on the way + /// (`meg` + `épít` + `és` + `e`). + fn form_has_flag(&self, w: &str, flags: &[u16], depth: u8, pfx: Option, excluded: Excluded) -> bool { + let fits = |f: &[u16]| intersects(f, flags) && pfx.is_none_or(|p| has(f, p)); + if if pfx.is_some() { self.homonym_meets(w, excluded, fits) } else { self.stem_flags_except(w, excluded).is_some_and(fits) } { return true; } // Without continuation classes no rule leads from a form to another. @@ -1588,6 +1671,8 @@ impl AffixDict { } let mut root = String::with_capacity(w.len() + 8); let mut inner: Vec = Vec::new(); + // Rules whose continuation classes give the prefix its flag: under them it is met. + let mut met: Vec = Vec::new(); // Does a continuation set hold one of `flags`? Answered once per distinct set (rules // share interned sets) and, for many flags, through a bitmap: a lookup costs at most the // total size of the distinct sets it meets, not rules × set size. @@ -1602,16 +1687,24 @@ impl AffixDict { root.push_str(base); root.push_str(&g.strip); inner.clear(); + met.clear(); for r in g.cont_rules.iter().filter_map(|&i| self.suffixes.get(i as usize)) { let meets = *seen.entry(r.cont).or_insert_with(|| probe.meets(self.set(r.cont))); - if meets && !self.cont_holds(r, self.circumfix) && !self.cont_holds(r, self.only_in_compound) && r.cond_matches(&root, true) { - inner.push(r.flag); + if meets + && (pfx.is_none() || r.cross) + && !self.cont_holds(r, self.circumfix) + && !self.cont_holds(r, self.only_in_compound) + && r.cond_matches(&root, true) + { + if pfx.is_some_and(|p| self.cont_holds(r, Some(p))) { met.push(r.flag) } else { inner.push(r.flag) } } } - inner.sort_unstable(); - inner.dedup(); - if !inner.is_empty() && self.form_has_flag(&root, &inner, depth - 1, excluded) { - return true; + for (list, p) in [(&mut inner, pfx), (&mut met, None)] { + list.sort_unstable(); + list.dedup(); + if !list.is_empty() && self.form_has_flag(&root, list, depth - 1, p, excluded) { + return true; + } } } } @@ -1631,12 +1724,33 @@ impl AffixDict { } /// [`AffixDict::contains_suggestible`], without flag-based compounds when `compounds` is - /// false (suggestion candidates past the first [`compound::MAX_SUGGEST_COMPOUNDS`]). + /// false (suggestion candidates past the compound check budget). pub(crate) fn contains_suggestible_with(&self, w: &str, compounds: bool) -> bool { let e = self.excluded(true); self.contains_except(&self.normalize(w), if compounds { e } else { e.without_compounds() }) } + /// Suggestions for `word`, best first, at most `max`, from the dictionary's own tables (as + /// for a Language pack). + pub fn suggest(&self, word: &str, max: usize) -> Vec { + crate::affix_suggest::suggest_in(self, word, max, &crate::affix_suggest::Tuning::DEFAULT) + } + + /// [`AffixDict::contains_suggestible`], where the flag-based compound search spends its steps + /// from `budget` (at most [`compound::MAX_SUGGEST_WORK`] for this word). + pub(crate) fn contains_suggestible_spending(&self, w: &str, budget: &mut usize, pieces: &mut compound::Pieces) -> bool { + let w = self.normalize(w); + let e = self.excluded(true); + if self.simple_word(&w, e) || self.compound(&w, e) { + return true; + } + let allowed = (*budget).min(compound::MAX_SUGGEST_WORK); + let mut left = allowed; + let found = self.flag_compound_spending(&w, e, matches!(casing(&w), Casing::Initial | Casing::Upper), &mut left, Some(pieces)); + *budget = budget.saturating_sub(allowed - left); + found + } + /// Whether the dictionary compounds by flags (suggestion candidates then cost more). pub(crate) fn flag_compounds(&self) -> bool { self.cpd.is_some() @@ -1674,6 +1788,11 @@ impl AffixDict { /// Is `w` a stem or an affixed form of one (no compound)? Affixes that are only valid inside /// a compound (`ONLYINCOMPOUND`) don't count. fn simple_word(&self, w: &str, excluded: Excluded) -> bool { + self.simple_word_with(w, excluded, true) + } + + /// [`AffixDict::simple_word`], with prefixes only when `prefixes` is true. + fn simple_word_with(&self, w: &str, excluded: Excluded, prefixes: bool) -> bool { if w.is_empty() || w.len() > MAX_WORD_BYTES || self.is_forbidden(w) { return false; } @@ -1681,6 +1800,9 @@ impl AffixDict { if self.stem_flags_except(w, excluded.and(self.need_affix)).is_some() || self.suffixed(w, None, false, false, excluded) { return true; } + if !prefixes { + return false; + } let mut root = String::with_capacity(w.len() + 8); // The prefix rules whose appended text starts `w`, found by that text (one hash lookup per // length) rather than by trying every rule. @@ -1709,7 +1831,7 @@ impl AffixDict { if self.stem_flags_except(&root, excluded).is_some_and(|f| has(f, r.flag)) { return true; } - if self.is_cont_flag(r.flag) && self.form_has_flag(&root, &[r.flag], INNER_CHAIN, excluded) { + if self.is_cont_flag(r.flag) && self.form_has_flag(&root, &[r.flag], INNER_CHAIN, None, excluded) { return true; } } @@ -1776,6 +1898,7 @@ impl AffixDict { }; let mut root = String::with_capacity(w.len() + 8); let mut flags: Vec = Vec::new(); + let mut met: Vec = Vec::new(); // Each split point from the end: the appended string is `w[b..]`, at most the longest // suffix, and something must stay before it. let cuts = std::iter::once(w.len()).chain(w.char_indices().rev().map(|(i, _)| i).filter(|&i| i > 0 || self.full_strip)); @@ -1789,35 +1912,48 @@ impl AffixDict { let rules = || g.rules.iter().filter_map(|&i| self.suffixes.get(i as usize)); // The stem path: `root` is a stem carrying the rule's flag (and the prefix's, // when a cross product is being checked). - if let Some(f) = self.stem_flags_except(&root, excluded) - && rules().any(|r| { + let takes = |f: &[u16]| { + rules().any(|r| { has(f, r.flag) && need.is_none_or(|n| has(f, n) || self.cont_holds(r, Some(n))) && (need.is_none() || r.cross) && fits(r) && r.cond_matches(&root, true) }) - { + }; + if if need.is_some() { + self.homonym_meets(&root, excluded, takes) + } else { + self.stem_flags_except(&root, excluded).is_some_and(takes) + } { return true; } // Continuation classes: `root` need not be a stem carrying this rule's flag — // it may be a form that gained the flag from another rule's continuation. The // matching rules' flags are looked for together, in one walk. Without continuation // classes that walk would only repeat the stem path above. - if need.is_none() && self.has_cont { + if self.has_cont && !circ { flags.clear(); + met.clear(); // Only a flag some continuation class holds can come from another rule. The // outer suffix of a chain has another affix under it, so its NEEDAFFIX is met, // and Hunspell ignores its CIRCUMFIX there (`workables` from `s/c`). - flags.extend( - rules() - .filter(|r| self.is_cont_flag(r.flag) && !self.cont_holds(r, self.only_in_compound) && r.cond_matches(&root, true)) - .map(|r| r.flag), - ); - flags.sort_unstable(); - flags.dedup(); - if !flags.is_empty() && self.form_has_flag(&root, &flags, INNER_CHAIN, excluded) { - return true; + // After a prefix every rule must allow cross products, and a rule whose + // continuation classes hold the prefix's flag meets it for the whole chain. + for r in rules().filter(|r| { + self.is_cont_flag(r.flag) + && (need.is_none() || r.cross) + && !self.cont_holds(r, self.only_in_compound) + && r.cond_matches(&root, true) + }) { + if need.is_some_and(|n| self.cont_holds(r, Some(n))) { met.push(r.flag) } else { flags.push(r.flag) } + } + for (list, p) in [(&mut flags, need), (&mut met, None)] { + list.sort_unstable(); + list.dedup(); + if !list.is_empty() && self.form_has_flag(&root, list, INNER_CHAIN, p, excluded) { + return true; + } } } } @@ -1841,7 +1977,7 @@ impl AffixDict { /// Whether `w` is a word only through a `KEEPCASE` stem, so it is correct in this letter case /// alone (`km` from `km/K`, which `Km` and `KM` are not). pub(crate) fn case_locked(&self, w: &str) -> bool { - self.keepcase.is_some() && self.contains(w) && !self.contains_except(&self.normalize(w), self.excluded(false).and(self.keepcase)) + self.keepcase.is_some() && self.contains(w) && !self.contains_except(&self.normalize(w), self.excluded(false).keeping(self.keepcase)) } fn check_except(&self, w: &str, excluded: Excluded) -> bool { @@ -1853,12 +1989,12 @@ impl AffixDict { if w.len() > MAX_INPUT_BYTES { return false; } - let keep = excluded.and(self.keepcase); + let keep = excluded.keeping(self.keepcase); match casing(&w) { Casing::Lower => false, // A compound may join a capitalized stem to others, and the word start with a // capital (`FooQux` from `foo` and `Qux`: right in the CLI). - Casing::Mixed => w.chars().next().is_some_and(char::is_uppercase) && self.flag_compound(&decapitalize(&w), excluded, true), + Casing::Mixed => w.chars().next().is_some_and(char::is_uppercase) && self.flag_compound(&decapitalize(&w), keep, true), Casing::Initial => { let lower = w.to_lowercase(); // CHECKSHARPS: a KEEPCASE word with `ß` may be capitalized (`Gruß` for `Gruß/k`). @@ -1869,11 +2005,24 @@ impl AffixDict { let lower = w.to_lowercase(); self.contains_except(&capitalize(&lower), keep) || self.contains_in(&lower, keep, true) + || self.mixed_case(&lower, keep) || (self.check_sharps && self.sharps(&lower, excluded)) } } } + /// Is `lower` (a word in capitals, lowercased) a stem in mixed case, or one with an ending + /// (`kbs` from `kB/S`)? + fn mixed_case(&self, lower: &str, excluded: Excluded) -> bool { + if self.mixed.is_empty() { + return false; + } + let ends = lower.char_indices().map(|(i, c)| i + c.len_utf8()); + ends.filter_map(|b| Some((self.mixed.get(lower.get(..b)?)?, lower.get(b..)?))).any(|(stems, rest)| { + stems.iter().take(MAX_HOMONYMS).filter_map(|&i| self.word(i as usize)).any(|m| self.contains_except(&format!("{m}{rest}"), excluded)) + }) + } + /// CHECKSHARPS: is `lower` (a word in capitals, lowercased) a word with some of its `ss` read /// as `ß`, in lower case or capitalized? Such a form may come from a `KEEPCASE` stem (in /// capitals a word with `ß` is written with `SS`). The first [`MAX_SHARPS`] `ss` pairs are @@ -1904,7 +2053,22 @@ impl AffixDict { /// cross products, so a dictionary with thousands of self-continuing rules can't make one stem /// cost millions of strings (#3613). pub fn forms(&self, stem: &str) -> Vec { + let mut budget = MAX_FORMS_WORK; + self.forms_spending(stem, &mut budget) + } + + /// [`AffixDict::forms`], with the rules it tries taken off `budget` (several stems' forms + /// share one budget: the suggestions' scan of 48 stems in a dictionary of 12,000 rules). + pub(crate) fn forms_spending(&self, stem: &str, budget: &mut usize) -> Vec { let Some(f) = self.stem_flags(stem) else { return Vec::new() }; + let start = *budget; + let out = self.forms_within(stem, f, (*budget).min(MAX_FORMS_WORK)); + *budget = start.saturating_sub(out.1); + out.0 + } + + /// The forms of `stem` (flags `f`) in at most `limit` rules tried, and the rules tried. + fn forms_within(&self, stem: &str, f: &[u16], limit: usize) -> (Vec, usize) { let mut out = Vec::new(); if !self.need_affix.is_some_and(|n| has(f, n)) { out.push(stem.to_string()); @@ -1919,7 +2083,14 @@ impl AffixDict { let mut queued: std::collections::HashSet<(String, u32)> = std::collections::HashSet::new(); let mut work = 0usize; for r in &self.suffixes { - if !has(f, r.flag) || !r.cond_matches(stem, true) || self.cont_holds(r, self.only_in_compound) { + if !has(f, r.flag) { + continue; + } + work += 1; + if work > limit { + return (out, work); + } + if !r.cond_matches(stem, true) || self.cont_holds(r, self.only_in_compound) { continue; } let Some(base) = stem.strip_suffix(&*r.strip) else { continue }; @@ -1941,7 +2112,7 @@ impl AffixDict { out.extend(non_empty(form)); } if out.len() >= MAX_FORMS { - return out; + return (out, work); } } // Continuation classes chain suffixes (`workable/B` → `workables`), up to [`MAX_CHAIN`] @@ -1954,7 +2125,7 @@ impl AffixDict { for (form, flags) in level { for r in self.suffixes_with(flags).into_iter().filter_map(|i| self.suffixes.get(i as usize)) { work += 1; - if work > MAX_FORMS_WORK { + if work > limit { break 'chain; } if !r.cond_matches(&form, true) || self.cont_holds(r, self.only_in_compound) { @@ -1972,7 +2143,7 @@ impl AffixDict { // and Hunspell ignores its CIRCUMFIX there (`workables` from `s/c`). out.extend(non_empty(next)); if out.len() >= MAX_FORMS { - return out; + return (out, work); } } } @@ -1988,8 +2159,8 @@ impl AffixDict { // Every suffix form looked at counts against the work budget, matching or not. for (s, s_circ, s_needs, s_cont) in &cross_sfx { work += 1; - if out.len() >= MAX_FORMS || work > MAX_FORMS_WORK { - return out; + if out.len() >= MAX_FORMS || work > limit { + return (out, work); } // The prefix's flag comes from the stem or the suffix's continuation classes; // circumfix alike; not both in need of another affix. @@ -2000,10 +2171,10 @@ impl AffixDict { } } if out.len() >= MAX_FORMS { - return out; + return (out, work); } } - out + (out, work) } /// `root` with the prefix rule `r` added, when the checker takes it back off: the rule's @@ -2177,20 +2348,23 @@ impl AffixDict { let p = prev.iter().zip(b).take_while(|(x, y)| x == y).count().min(255); raw.push(p as u8); raw.extend_from_slice(b.get(p..).unwrap_or_default()); - let set = self.flags.get(i).map_or(&[][..], |&id| self.set(id)); - if !set.is_empty() { - let mut flags = String::from("/"); - self.write_flags(set, &mut flags); - raw.extend_from_slice(flags.as_bytes()); - } - raw.push(b'\n'); - // The ONLYINCOMPOUND homonyms' flags: the same stem again, merged back on reading. - if let Some(&c) = u32::try_from(i).ok().and_then(|i| self.cpd_sets.get(&i)) { - raw.push(b.len().min(255) as u8); - raw.extend_from_slice(b.get(b.len().min(255)..).unwrap_or_default()); - let mut flags = String::from("/"); - self.write_flags(self.set(c), &mut flags); - raw.extend_from_slice(flags.as_bytes()); + // The homonyms kept apart each on a line, or their flags merged; then the + // ONLYINCOMPOUND homonyms' flags. The same stem again is merged back on reading. + let at = u32::try_from(i).ok(); + let main = self.flags.get(i).copied().unwrap_or_default(); + let apart = at.and_then(|i| self.homonyms.get(&i)).map_or(std::slice::from_ref(&main), |h| &h[..]); + let cpd = at.and_then(|i| self.cpd_sets.get(&i)); + for (k, &id) in apart.iter().chain(cpd).enumerate() { + if k > 0 { + raw.push(b.len().min(255) as u8); + raw.extend_from_slice(b.get(b.len().min(255)..).unwrap_or_default()); + } + let set = self.set(id); + if !set.is_empty() { + let mut flags = String::from("/"); + self.write_flags(set, &mut flags); + raw.extend_from_slice(flags.as_bytes()); + } raw.push(b'\n'); } prev = b; @@ -3219,6 +3393,47 @@ pub(crate) mod tests { assert!(d.check("neworkables")); } + #[test] + fn a_prefix_with_a_suffix_chain() { + // The CLI's answers (`meg` + `épít` + `és` + `e` in Hungarian): every rule must allow + // cross products, and the prefix's flag comes from the stem or a continuation class. + let rules = |p: char, a: char, b: char, a_cont: &str, b_cont: &str| { + format!("SET UTF-8\nPFX P {p} 1\nPFX P 0 un .\nSFX A {a} 1\nSFX A 0 a/B{a_cont} .\nSFX B {b} 1\nSFX B 0 b{b_cont} .\n") + }; + let d = |aff: &str, dic: &str| AffixDict::from_hunspell(aff, &format!("1\n{dic}\n"), &[]).unwrap(); + let yes = d(&rules('Y', 'Y', 'Y', "", ""), "foo/PA"); + assert!(yes.check("unfooab") && yes.check("unfooa") && yes.check("fooab")); + for (p, a, b) in [('Y', 'Y', 'N'), ('Y', 'N', 'Y'), ('N', 'Y', 'Y')] { + assert!(!d(&rules(p, a, b, "", ""), "foo/PA").check("unfooab"), "{p}{a}{b}"); + } + assert!(d(&rules('Y', 'Y', 'Y', "P", ""), "foo/A").check("unfooab"), "flag from the inner suffix"); + let outer = d(&rules('Y', 'Y', 'Y', "", "/P"), "foo/A"); + assert!(outer.check("unfooab") && !outer.check("unfooa"), "flag from the outer suffix"); + assert!(!d(&rules('Y', 'Y', 'Y', "", ""), "foo/A").check("unfooab"), "no prefix flag"); + } + + #[test] + fn a_prefix_and_a_suffix_come_from_one_homonym() { + // `stop/PA` and `stop/S`: `unstop` and `stops` are right, `unstops` wrong (the CLI's + // answers), also after a round trip through the compact form. + let aff = "SET UTF-8\nPFX P Y 1\nPFX P 0 un .\nSFX A Y 1\nSFX A 0 a/B .\nSFX B Y 1\nSFX B 0 b .\nSFX S Y 1\nSFX S 0 s .\n"; + let d = AffixDict::from_hunspell(aff, "3\nstop/PA\nstop/S\ngo/PS\n", &[]).unwrap(); + let again = AffixDict::from_bytes(&d.to_bytes().unwrap()).unwrap(); + for d in [d, again] { + assert!(d.check("unstop") && d.check("stops") && d.check("unstopa") && d.check("unstopab") && d.check("ungos")); + assert!(!d.check("unstops")); + } + } + + #[test] + fn a_word_in_capitals_may_be_a_mixed_case_stem() { + // The CLI's answers: `KB` and `KBS` from `kB/SU`, not `KBT`, `UNKB` or `Kb`. + let aff = "SET UTF-8\nSFX S Y 1\nSFX S 0 s .\nPFX U Y 1\nPFX U 0 un .\nSFX T Y 1\nSFX T 0 t .\n"; + let d = AffixDict::from_hunspell(aff, "2\nkB/SU\naBcD\n", &[]).unwrap(); + assert!(d.check("KB") && d.check("KBS") && d.check("kB") && d.check("ABCD")); + assert!(!d.check("KBT") && !d.check("UNKB") && !d.check("Kb") && !d.check("kb") && !d.check("Abcd")); + } + #[test] fn hostile_compound_rules_are_refused_or_cheap() { for bad in ["COMPOUNDRULE *a", "COMPOUNDRULE (ab)", "COMPOUNDRULE a**", "COMPOUNDMIN x", "ONLYINCOMPOUND ab"] { diff --git a/crates/proof/src/affix/compound.rs b/crates/proof/src/affix/compound.rs index bc5b513d..4edf3df8 100644 --- a/crates/proof/src/affix/compound.rs +++ b/crates/proof/src/affix/compound.rs @@ -51,10 +51,10 @@ pub(super) const MAX_PARTS: usize = 16; /// Most steps one compound check may take (affix rules tried, parts looked up, splits followed). pub(super) const MAX_WORK: usize = 1 << 16; /// Most steps the compound search of one suggestion candidate may take. -pub(crate) const MAX_SUGGEST_WORK: usize = 1 << 11; -/// Most candidates of one suggestion call whose compound search runs; the later ones are -/// checked as simple words only, so a long garbled word's suggestions stay quick. -pub(crate) const MAX_SUGGEST_COMPOUNDS: usize = 128; +pub(crate) const MAX_SUGGEST_WORK: usize = 1 << 13; +/// Most steps the compound searches of one suggestion call may take in all; the candidates +/// left are checked as simple words only, so a long garbled word's suggestions stay quick. +pub(crate) const MAX_SUGGEST_COMPOUND_WORK: usize = 1 << 17; /// Most `CHECKCOMPOUNDPATTERN` lines (the Dutch dictionary has 52). const MAX_PATTERNS: usize = 512; /// Longest text on either side of a `CHECKCOMPOUNDPATTERN`, in bytes. @@ -384,6 +384,9 @@ struct Info { stem: u32, /// How many words it counts as (`COMPOUNDROOT`: two). words: u8, + /// Syllables of its suffix the Hungarian rule leaves out (as the last part): a suffix with no + /// continuation classes doesn't count. + tail: u8, } /// A stem found for a part, and the affixes that make the part from it. @@ -397,6 +400,15 @@ struct Analysis<'a> { inner: Option<&'a Rule>, } +/// Most pieces a [`Pieces`] cache keeps. +const MAX_CACHED_PIECES: usize = 1 << 14; + +/// The infos of pieces, kept across the compound checks of one suggestion call: its candidates +/// differ from the word by a letter or two, so most of their pieces are the same. Indexed by +/// whether the piece starts the word (KEEPCASE holds there). +#[derive(Default)] +pub(crate) struct Pieces([crate::patterns::FxMap, Box<[Info]>>; 2], usize); + /// One compound check: the word, its parts' analyses (each piece looked up once), the parts /// chosen so far and the work done. struct Search<'a> { @@ -417,14 +429,31 @@ struct Search<'a> { work: usize, /// [`AffixDict::near_word`], once answered. near: Option, + /// Pieces analysed by earlier checks with the same exclusions. + cache: Option<&'a mut Pieces>, } impl AffixDict { /// Is `w` a compound by the flag-based rules (see the module docs)? `capitalized`: the word /// as the user wrote it starts with a capital (`FORCEUCASE`). pub(super) fn flag_compound(&self, w: &str, excluded: Excluded, capitalized: bool) -> bool { + // A suggestion candidate's search gets MAX_SUGGEST_WORK steps. + let mut budget = if excluded.suggesting() { MAX_SUGGEST_WORK } else { MAX_WORK }; + self.flag_compound_spending(w, excluded, capitalized, &mut budget, None) + } + + /// [`AffixDict::flag_compound`] in at most `budget` steps (and [`MAX_WORK`]), taking the + /// steps it spends off `budget`. `cache`: pieces analysed before, with the same `excluded`. + pub(super) fn flag_compound_spending( + &self, + w: &str, + excluded: Excluded, + capitalized: bool, + budget: &mut usize, + cache: Option<&mut Pieces>, + ) -> bool { let Some(c) = self.cpd.as_deref() else { return false }; - if excluded.no_compounds() || w.is_empty() || w.len() > super::MAX_WORD_BYTES { + if excluded.no_compounds() || w.is_empty() || w.len() > super::MAX_WORD_BYTES || *budget == 0 { return false; } if self.is_forbidden(w) { @@ -433,7 +462,7 @@ impl AffixDict { // KEEPCASE holds for the first part only (from `baz/K`, `Foobaz` and `FOOBAZ` are right // in the CLI, `Bazfoo` and `BAZFOO` wrong). let first_excluded = excluded; - let excluded = excluded.without(self.keepcase); + let excluded = excluded.without_keep(); let cuts: Vec = w.char_indices().map(|(i, _)| i).chain(std::iter::once(w.len())).collect(); let chars: Vec = w.chars().collect(); let n = cuts.len(); @@ -448,11 +477,83 @@ impl AffixDict { memo: vec![None; n * n], infos: Vec::new(), path: Vec::with_capacity(MAX_PARTS), - // A suggestion candidate's search gets MAX_SUGGEST_WORK steps of the budget. - work: if excluded.suggesting() { MAX_WORK - MAX_SUGGEST_WORK } else { 0 }, + work: MAX_WORK - (*budget).min(MAX_WORK), near: None, + cache, }; - self.parts_from(&mut s, 0) + let start = s.work; + let found = self.parts_from(&mut s, 0); + *budget = budget.saturating_sub(s.work.saturating_sub(start).max(1)); + found + } + + /// Where a slip in `w` (no compound) likely is, in chars: from the furthest point its start + /// reaches as compound parts to the nearest its end does, if anything reaches. A candidate + /// changed there is checked as a compound first. `capitalized`: the same with a capital + /// first letter counts too. + pub(crate) fn compound_gap(&self, w: &str, capitalized: bool, budget: &mut usize, pieces: &mut Pieces) -> Option<(usize, usize)> { + let c = self.cpd.as_deref()?; + if w.is_empty() || w.len() > super::MAX_WORD_BYTES || *budget == 0 { + return None; + } + let excluded = self.excluded(true); + let n = w.chars().count(); + let min = self.compound_min.max(1); + let mut reach = vec![false; n + 1]; + let mut back = vec![false; n + 1]; + let cap = capitalized.then(|| super::capitalize(w)); + for word in std::iter::once(w).chain(cap.as_deref()) { + let mut s = Search { + c, + w: word, + cuts: word.char_indices().map(|(i, _)| i).chain(std::iter::once(word.len())).collect(), + chars: word.chars().collect(), + excluded: excluded.without_keep(), + first_excluded: excluded, + capitalized, + memo: vec![None; (n + 1) * (n + 1)], + infos: Vec::new(), + path: Vec::new(), + work: MAX_WORK - (*budget).min(MAX_WORK), + near: None, + cache: Some(&mut *pieces), + }; + let start = s.work; + let fits = |s: &mut Search, a: usize, b: usize, place: u8| { + let (x, y) = self.infos_of(s, a, b); + s.infos.get(x as usize..y as usize).is_some_and(|infos| infos.iter().any(|i| i.pos & place != 0)) + }; + for a in 0..n { + if a > 0 && !reach.get(a).copied().unwrap_or(false) { + continue; + } + for b in a.saturating_add(min)..n { + if s.work <= MAX_WORK + && fits(&mut s, a, b, if a == 0 { FIRST } else { MIDDLE }) + && let Some(r) = reach.get_mut(b) + { + *r = true; + } + } + } + for b in (1..=n).rev() { + if b < n && !back.get(b).copied().unwrap_or(false) { + continue; + } + for a in 1..=b.saturating_sub(min) { + if s.work <= MAX_WORK + && fits(&mut s, a, b, if b == n { LAST } else { MIDDLE }) + && let Some(r) = back.get_mut(a) + { + *r = true; + } + } + } + *budget = budget.saturating_sub(s.work.saturating_sub(start)); + } + let from = reach.iter().rposition(|&r| r).unwrap_or(0); + let to = back.iter().position(|&r| r).unwrap_or(n); + Some((from.min(to), from.max(to))) } /// Can the rest of the word, from char `start`, be split into parts after `s.path`? @@ -516,7 +617,14 @@ impl AffixDict { return *r; } let from = u32::try_from(s.infos.len()).unwrap_or(u32::MAX); - if let (Some(&x), Some(&y)) = (s.cuts.get(a), s.cuts.get(b)) + let cached = s.cuts.get(a).zip(s.cuts.get(b)).and_then(|(&x, &y)| s.w.get(x..y)).and_then(|piece| { + let map = s.cache.as_deref()?.0.get(usize::from(a == 0))?; + map.get(piece).map(|v| v.to_vec()) + }); + if let Some(v) = cached { + s.work += 1; + s.infos.extend(v); + } else if let (Some(&x), Some(&y)) = (s.cuts.get(a), s.cuts.get(b)) && let Some(piece) = s.w.get(x..y) { // Deduplicated by hashing, and at most MAX_INFOS per piece (each one is tried at @@ -534,6 +642,15 @@ impl AffixDict { } }); s.work = work; + // Only a piece analysed in full is kept (one cut short by the budget isn't). + if work <= MAX_WORK + && let Some(c) = s.cache.as_deref_mut() + && c.1 < MAX_CACHED_PIECES + && let Some(map) = c.0.get_mut(usize::from(a == 0)) + { + map.insert(piece.into(), found.clone().into()); + c.1 += 1; + } s.infos.extend(found); } let to = u32::try_from(s.infos.len()).unwrap_or(u32::MAX); @@ -588,11 +705,19 @@ impl AffixDict { marks |= 1 << i; } } - let words = if carries(c.root) { 2 } else { 1 }; - // FORCEUCASE counts on the stem only (a suffix's continuation class holding it is ignored); + // COMPOUNDROOT and FORCEUCASE count on the stem only (an affix's continuation class + // holding them is ignored); // a pattern's `0` means no suffix (a prefix doesn't count): the CLI's answers. + let words = if c.root.is_some_and(|f| has(an.set, f)) { 2 } else { 1 }; let force = c.force_ucase.is_some_and(|f| has(an.set, f)); - Some(Info { pos, marks, unmodified: an.sfx.is_none(), force, stem: an.stem, words }) + // The Hungarian syllable rule leaves out a suffix with no continuation classes (the + // CLI's answers: `bab|bab|baata` from `ata` is right, from `ata/Z` wrong). + let tail = match (&c.syllables, an.sfx) { + (Some((_, vowels)), Some(r)) if self.set(r.cont).is_empty() => r.add.chars().filter(|ch| vowels.contains(ch)).count(), + _ => 0, + }; + let tail = u8::try_from(tail).unwrap_or(u8::MAX); + Some(Info { pos, marks, unmodified: an.sfx.is_none(), force, stem: an.stem, words, tail }) } /// Every way `p` is a stem, or a stem with a prefix, one or two suffixes, or a prefix and a @@ -737,10 +862,17 @@ impl AffixDict { /// The stem `w` as a compound part: its index and flags, unless it is forbidden or excluded. /// A stem with `ONLYINCOMPOUND` homonyms beside others is two: each group's flags apply on /// their own (`foo/XOS` and `foo/T`: `foos` is a part, `foot` is not, as in the CLI). + /// + /// Homonyms kept apart ([`AffixDict::homonyms`]) are each a part of their own: a suffix and + /// the compound flag must come from one (Dutch `poeder` the noun compounds, `poederden` the + /// verb form doesn't: `schuurpoederden` is wrong in the CLI). fn part_stems(&self, w: &str, excluded: Excluded) -> impl Iterator { let found = self.stem_lookup(w); + let apart = found.and_then(|(stem, _)| self.homonyms.get(&stem).map(|h| (stem, h))); + let main = found.filter(|_| apart.is_none()); + let each = apart.into_iter().flat_map(move |(stem, h)| h.iter().map(move |&id| (stem, self.set(id)))); let only = found.and_then(|(stem, _)| self.cpd_sets.get(&stem).map(|&c| (stem, self.set(c)))); - found.into_iter().chain(only).filter(move |(_, set)| !excluded.hits(set) && !self.forbidden.is_some_and(|f| has(set, f))) + main.into_iter().chain(each).chain(only).filter(move |(_, set)| !excluded.hits(set) && !self.forbidden.is_some_and(|f| has(set, f))) } /// May a part with `info` start at char `start`, after the last part of `s.path` @@ -789,7 +921,10 @@ impl AffixDict { match &s.c.syllables { None => false, Some(_) if !whole => true, - Some((most, vowels)) => s.chars.iter().filter(|c| vowels.contains(c)).count() <= *most, + Some((most, vowels)) => { + let tail = s.path.last().map_or(0, |p| usize::from(p.2.tail)); + s.chars.iter().filter(|c| vowels.contains(c)).count().saturating_sub(tail) <= *most + } } } @@ -838,9 +973,13 @@ impl AffixDict { } let excluded = s.excluded; let mut work = s.work; + // A word listed as a stem that can't stand alone (ONLYINCOMPOUND, NEEDAFFIX) counts here + // only with a suffix, not with a prefix (`ver` + `kehr` beside `verkehr/zo`: + // `Fahrradverkehr` is right in the CLI). let mut is_word = |cand: &[char]| { work += 1; - work > MAX_WORK || self.simple_word(&cand.iter().collect::(), excluded) + let cand: String = cand.iter().collect(); + work > MAX_WORK || self.simple_word_with(&cand, excluded, self.stem_set(&cand).is_none()) }; let near = self.near_miss(&s.chars, s.c.two_words, &mut is_word); s.work = work; @@ -1123,9 +1262,37 @@ mod tests { let took = t.elapsed(); eprintln!("{locale}: suggestions for {garbled} in {took:?}"); assert!(took.as_secs() < 5, "{locale}: {took:?}"); + // A one-letter slip of a long compound gets the compound back. + for (typo, meant) in TYPOS.iter().filter(|t| t.0 == *locale).flat_map(|t| t.1.iter()) { + let got = d.suggest(typo, 8); + assert!(got.iter().any(|g| g == meant), "{locale}: {typo} → {got:?}, not {meant}"); + } } } + /// (locale, [(typo, the compound meant)]): each one's suggestions hold it. + const TYPOS: &[(&str, &[(&str, &str)])] = &[ + ( + "de", + &[ + ("Donaudampfschff", "Donaudampfschiff"), + ("Arbetszimmer", "Arbeitszimmer"), + ("Kinerspielplatz", "Kinderspielplatz"), + ("Bundesverfassugsgericht", "Bundesverfassungsgericht"), + ], + ), + ( + "nl", + &[ + ("ziekenhusopname", "ziekenhuisopname"), + ("voetbawledstrijd", "voetbalwedstrijd"), + ("arbeidsongeschiktheidsverzeekring", "arbeidsongeschiktheidsverzekering"), + ("tandartsafsparak", "tandartsafspraak"), + ("belastingaagifte", "belastingaangifte"), + ], + ), + ]; + /// (locale, right words, wrong words), the CLI's answers. const REAL: &[(&str, &str, &str)] = &[ ( @@ -1715,5 +1882,109 @@ mod tests { listed: "foo brok foofoo foobrok foobroken brokfoo brokbrok brokbroken foofoofoo foofoobrok foofoobroken foobrokfoo \ foobrokbrok foobrokbroken brokfoofoo brokfoobrok brokfoobroken brokbrokfoo brokbrokbrok brokbrokbroken", }, + Pinned { + name: "mixed_keep", + aff: "COMPOUNDFLAG X\nKEEPCASE K", + dic: "foo/X baz/XK Qux/X bar/X", + pieces: "foo baz Qux Baz Foo BAZ QUX", + extra: "BazQux BazfooQux FooQux FooBaz fooBaz FOOQux BAZQux", + right: true, + listed: "foo baz Qux Foo QUX foofoo foobaz fooQux bazfoo bazbaz bazQux Quxfoo Quxbaz QuxQux Foofoo Foobaz FooQux QUXBAZ \ + foofoofoo foofoobaz foofooQux foobazfoo foobazbaz foobazQux fooQuxfoo fooQuxbaz fooQuxQux bazfoofoo bazfoobaz \ + bazfooQux bazbazfoo bazbazbaz bazbazQux bazQuxfoo bazQuxbaz bazQuxQux Quxfoofoo Quxfoobaz QuxfooQux Quxbazfoo \ + Quxbazbaz QuxbazQux QuxQuxfoo QuxQuxbaz QuxQuxQux Foofoofoo Foofoobaz FoofooQux Foobazfoo Foobazbaz FoobazQux \ + FooQuxfoo FooQuxbaz FooQuxQux QUXBAZBAZ", + }, + Pinned { + name: "mixed_case_chk", + aff: "COMPOUNDFLAG X\nCHECKCOMPOUNDCASE", + dic: "foo/X Qux/X bar/X", + pieces: "foo Qux Foo bar Bar", + extra: "", + right: true, + listed: "foo Qux Foo bar Bar foofoo foobar Quxfoo Quxbar Foofoo Foobar barfoo barbar Barfoo Barbar foofoofoo foofoobar \ + foobarfoo foobarbar Quxfoofoo Quxfoobar Quxbarfoo Quxbarbar Foofoofoo Foofoobar Foobarfoo Foobarbar barfoofoo \ + barfoobar barbarfoo barbarbar Barfoofoo Barfoobar Barbarfoo Barbarbar", + }, + Pinned { + name: "mixed_forbid", + aff: "COMPOUNDFLAG X\nFORBIDDENWORD F", + dic: "foo/X Qux/X fooQux/F bar/X", + pieces: "foo Qux Foo bar", + extra: "FooQux fooQux FOOQUX", + right: false, + listed: "fooQux fooFoo QuxFoo FooQux FooFoo barFoo foofooFoo fooQuxFoo fooFoofoo fooFooQux fooFooFoo fooFoobar \ + foobarFoo QuxfooFoo QuxQuxFoo QuxFoofoo QuxFooQux QuxFooFoo QuxFoobar QuxbarFoo FoofooFoo FooQuxFoo FooFoofoo \ + FooFooQux FooFooFoo FooFoobar FoobarFoo barfooFoo barQuxFoo barFoofoo barFooQux barFooFoo barFoobar barbarFoo \ + FOOQUX", + }, + Pinned { + name: "mixed_onlyc", + aff: "COMPOUNDFLAG X\nONLYINCOMPOUND O", + dic: "foo/XO Qux/X bar/X", + pieces: "foo Qux Foo bar Bar", + extra: "", + right: true, + listed: "Qux bar Bar foofoo fooQux foobar Quxfoo QuxQux Quxbar Foofoo FooQux Foobar barfoo barQux barbar Barfoo BarQux \ + Barbar foofoofoo foofooQux foofoobar fooQuxfoo fooQuxQux fooQuxbar foobarfoo foobarQux foobarbar Quxfoofoo \ + QuxfooQux Quxfoobar QuxQuxfoo QuxQuxQux QuxQuxbar Quxbarfoo QuxbarQux Quxbarbar Foofoofoo FoofooQux Foofoobar \ + FooQuxfoo FooQuxQux FooQuxbar Foobarfoo FoobarQux Foobarbar barfoofoo barfooQux barfoobar barQuxfoo barQuxQux \ + barQuxbar barbarfoo barbarQux barbarbar Barfoofoo BarfooQux Barfoobar BarQuxfoo BarQuxQux BarQuxbar Barbarfoo \ + BarbarQux Barbarbar", + }, + Pinned { + name: "mixed_force", + aff: "COMPOUNDFLAG X\nFORCEUCASE U", + dic: "foo/X Qux/XU bar/XU", + pieces: "foo Qux Foo bar Bar", + extra: "", + right: true, + listed: "foo Qux Foo bar Bar foofoo fooQux Quxfoo QuxQux Quxbar Foofoo FooQux Foobar barfoo barQux Barfoo BarQux Barbar \ + foofoofoo foofooQux fooQuxfoo fooQuxQux fooQuxbar foobarfoo foobarQux Quxfoofoo QuxfooQux Quxfoobar QuxQuxfoo \ + QuxQuxQux QuxQuxbar Quxbarfoo QuxbarQux Quxbarbar Foofoofoo FoofooQux Foofoobar FooQuxfoo FooQuxQux FooQuxbar \ + Foobarfoo FoobarQux Foobarbar barfoofoo barfooQux barQuxfoo barQuxQux barQuxbar barbarfoo barbarQux Barfoofoo \ + BarfooQux Barfoobar BarQuxfoo BarQuxQux BarQuxbar Barbarfoo BarbarQux Barbarbar", + }, + Pinned { + name: "keep_nosug", + aff: "COMPOUNDFLAG X\nKEEPCASE K\nNOSUGGEST K", + dic: "foo/X baz/XK", + pieces: "foo baz Foo Baz", + extra: "", + right: true, + listed: "foo baz Foo foofoo foobaz bazfoo bazbaz Foofoo Foobaz foofoofoo foofoobaz foobazfoo foobazbaz bazfoofoo \ + bazfoobaz bazbazfoo bazbazbaz Foofoofoo Foofoobaz Foobazfoo Foobazbaz", + }, + Pinned { + name: "root_affix", + aff: "COMPOUNDFLAG Y\nCOMPOUNDWORDMAX 2\nCOMPOUNDROOT R\nCOMPOUNDPERMITFLAG P\nSFX A Y 1\nSFX A 0 s/RP .\nPFX B Y 1\nPFX B 0 un/RP .", + dic: "foo/YAB bar/YAB baz/YR", + pieces: "foo foos unfoo bar bars baz", + extra: "", + right: true, + listed: "foo foos unfoo bar bars baz foofoo foofoos foounfoo foobar foobars foosfoo foosfoos foosunfoo foosbar foosbars \ + unfoofoo unfoofoos unfoounfoo unfoobar unfoobars barfoo barfoos barunfoo barbar barbars barsfoo barsfoos \ + barsunfoo barsbar barsbars", + }, + Pinned { + name: "syllable_hu_suffix", + aff: "LANG hu_HU\nCOMPOUNDMIN 1\nCOMPOUNDFLAG Y\nCOMPOUNDWORDMAX 2\nCOMPOUNDSYLLABLE 4 aeiou\nSFX C Y 1\nSFX C 0 ata .\nSFX D Y 1\nSFX D 0 ete/Z .\nSFX E Y 1\nSFX E 0 t/F .\nSFX F Y 1\nSFX F 0 ete .", + dic: "ba/Y bab/YCDE", + pieces: "ba bab babata babete babtete", + extra: "bababababata babababababete bababababtete", + right: true, + listed: "ba bab babata babete babtete baba babab bababata bababete bababtete babba babbab babbabata babbabete \ + babbabtete bababa bababab babababata babababtete bababba bababbab bababbabata bababbabtete babbaba babbabab \ + babbababata babbababtete babbabba babbabbab babbabbabata babbabbabtete bababababata bababababtete", + }, + Pinned { + name: "near_prefix_shadow", + aff: "COMPOUNDBEGIN x\nCOMPOUNDMIDDLE y\nCOMPOUNDEND z\nNEEDAFFIX h\nONLYINCOMPOUND o\nSFX S Y 1\nSFX S 0 ad .\nPFX P Y 1\nPFX P 0 fa .\nSFX T Y 1\nSFX T 0 hr .\nPFX V Y 1\nPFX V 0 ve .", + dic: "fahr/x rad/y verkehr/zo hrrad/P fahrrad/h rke/VT fahrr/S", + pieces: "fahr rad verkehr", + extra: "fahrradverkehr fahrradverkehrr fahrradverkehrad", + right: true, + listed: "fahr rad verkehr fahrrad fahrverkehr fahrradverkehr", + }, ]; } diff --git a/crates/proof/src/affix_suggest.rs b/crates/proof/src/affix_suggest.rs index 7d488209..5e49183e 100644 --- a/crates/proof/src/affix_suggest.rs +++ b/crates/proof/src/affix_suggest.rs @@ -25,8 +25,12 @@ use crate::affix::{AffixDict, Casing, capitalize, casing}; /// Longest word suggestions are looked for, in letters. pub(crate) const MAX_LETTERS: usize = 40; const MAX_MAP_VARIANTS: usize = 4096; +/// Most candidates kept waiting for the compound check in one call (the rest are dropped). +const MAX_DEFERRED: usize = 1 << 14; /// Stems the scan of round 6 expands. const SCAN_STEMS: usize = 48; +/// Most affix rules tried in all to make those stems' forms (Mongolian has 12,000 rules). +const SCAN_FORMS_WORK: usize = 1 << 16; const COST_CASE: u32 = 0; pub(crate) const COST_ADD_DIACRITIC: u32 = 4; @@ -125,8 +129,12 @@ struct Collector<'a> { names: bool, /// May half of a missing space be one ([`Tuning::strict_case`])? split_names: bool, - /// Candidates checked with compounds so far (at most `MAX_SUGGEST_COMPOUNDS`). - compound_checks: usize, + /// Candidates no simple word, left for the compound check: (cost, order, candidate). + deferred: Vec<(u32, usize, String)>, + /// Steps left for the compound searches of this call (`MAX_SUGGEST_COMPOUND_WORK` at first). + compound_work: usize, + /// The pieces those searches analysed, shared between them. + pieces: crate::affix::Pieces, } impl Collector<'_> { @@ -143,20 +151,68 @@ impl Collector<'_> { self.tested.insert(cand.to_string(), cost); } } + // A candidate is checked as simple words here; one of a single word that isn't one waits + // for the compound check, which the cheapest candidates get first ([`Self::compounds`]). + if !self.accept(cand, cost, false) && self.d.flag_compounds() && !cand.contains(' ') && self.deferred.len() < MAX_DEFERRED { + self.seq += 1; + self.deferred.push((cost, self.seq, cand.to_string())); + } + } + + /// Check the waiting candidates as compounds, cheapest (then earliest) first, in one budget + /// of steps per call: compound checks cost far more than lookups, and a one-letter slip of a + /// long compound is the likeliest intent. + /// + /// A candidate changed where the word stops being parts (`ziekenhusopname`: `zieken` and + /// `opname` are, `hus` isn't) goes first. + fn compounds(&mut self) { + let mut waiting = std::mem::take(&mut self.deferred); + if waiting.is_empty() { + return; + } + let gap = self.d.compound_gap(&self.input, self.names, &mut self.compound_work, &mut self.pieces); + let input: Vec = self.input.chars().collect(); + let outside = |cand: &str| { + let Some((from, to)) = gap else { return false }; + let at = cand.chars().zip(&input).position(|(a, b)| a != *b).unwrap_or(input.len().min(cand.chars().count())); + at + 1 < from || at > to + 1 + }; + let mut waiting: Vec<(bool, u32, usize, String)> = waiting.drain(..).map(|(cost, seq, cand)| (outside(&cand), cost, seq, cand)).collect(); + waiting.sort_unstable(); + for (_, cost, _, cand) in waiting { + if self.compound_work == 0 { + break; + } + // Offered again at a lower cost meanwhile: that offer is the one to check. + if self.tested.get(&cand).is_some_and(|&c| c < cost) || self.found.contains_key(&cand) { + continue; + } + self.accept(&cand, cost, true); + } + } + + /// Keep `cand` if each of its words is in the dictionary (`compounds`: flag-based compounds + /// count); whether it was. + fn accept(&mut self, cand: &str, cost: u32, compounds: bool) -> bool { let mut words = Vec::new(); let mut cost = cost; let names = if cand.contains(' ') { self.split_names } else { self.names }; - let compounds = self.d.flag_compounds() && self.compound_checks < crate::affix::MAX_SUGGEST_COMPOUNDS; - self.compound_checks += usize::from(compounds); + let d = self.d; + let mut work = self.compound_work; + let pieces = &mut self.pieces; + let mut is_word = + |w: &str| if compounds { d.contains_suggestible_spending(w, &mut work, pieces) } else { d.contains_suggestible_with(w, false) }; for part in cand.split(' ') { - if self.d.contains_suggestible_with(part, compounds) { + if is_word(part) { words.push(part.to_string()); } else if !names { - return; + self.compound_work = work; + return false; } else { let c = capitalize(part); - if !self.d.contains_suggestible_with(&c, compounds) { - return; + if !is_word(&c) { + self.compound_work = work; + return false; } words.push(c); // A name ranks after a common word at the same distance (`moze` → `może`, then @@ -164,7 +220,9 @@ impl Collector<'_> { cost += 1; } } + self.compound_work = work; self.keep(words.join(" "), cost); + true } /// Keep a suggestion known to be correct, unless the dictionary says never to offer it. @@ -312,8 +370,18 @@ pub(crate) fn suggest_in(d: &AffixDict, word: &str, max: usize, t: &Tuning) -> V let chars: Vec = input.chars().collect(); let names = !t.strict_case || case != Casing::Lower; let split_names = !t.strict_case; - let mut col = - Collector { d, input: input.clone(), tested: HashMap::new(), found: HashMap::new(), seq: 0, names, split_names, compound_checks: 0 }; + let mut col = Collector { + d, + input: input.clone(), + tested: HashMap::new(), + found: HashMap::new(), + seq: 0, + names, + split_names, + deferred: Vec::new(), + compound_work: crate::affix::MAX_SUGGEST_COMPOUND_WORK, + pieces: Default::default(), + }; add_case(&mut col, word, &input); let map = groups(d, t); add_map_slips(&mut col, &chars, &map, t); @@ -321,9 +389,11 @@ pub(crate) fn suggest_in(d: &AffixDict, word: &str, max: usize, t: &Tuning) -> V add_splits(&mut col, &input, &t.split); add_key_neighbours(&mut col, &chars); add_typos(&mut col, &chars, t); + col.compounds(); // 6. Further off: scan the stems that start like the word. if col.count_within(COST_MISTYPE) < max { scan(&mut col, &chars, &map); + col.compounds(); } let mut found: Vec<(String, (u32, usize))> = col.found.into_iter().collect(); // Among equal costs, the change nearest the word's start is the likelier intent @@ -526,8 +596,9 @@ fn scan(col: &mut Collector, chars: &[char], map: &[Vec]) { best.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(b.1))); let related = |a: char, b: char| fold(a, map) == fold(b, map); let limit = 4; + let mut budget = SCAN_FORMS_WORK; for (_, stem) in best { - for form in d.forms(stem) { + for form in d.forms_spending(stem, &mut budget) { let lf: Vec = form.to_lowercase().chars().collect(); if lf.len().abs_diff(chars.len()) > 2 { continue; @@ -653,7 +724,9 @@ mod tests { seq: 0, names: true, split_names: true, - compound_checks: 0, + deferred: Vec::new(), + compound_work: crate::affix::MAX_SUGGEST_COMPOUND_WORK, + pieces: Default::default(), }; col.offer("cafe", COST_DROP_DIACRITIC); col.offer("cafe", COST_REP); From 188c3154aed3f8bba5bd65023c852c21870e2882 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Mon, 2 Nov 2026 03:59:36 +0000 Subject: [PATCH 12/40] =?UTF-8?q?CP7.5:=20craft-libs#497=20re-review=20?= =?UTF-8?q?=E2=80=94=20prefixes=20leaving=20one=20root=20share=20one=20suf?= =?UTF-8?q?fix=20walk;=20all-caps=20mixed-case=20stems=20are=20no=20compou?= =?UTF-8?q?nds;=20homonym=20flags=20written;=20compact=20format=20WCAFF2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - simple_word groups the prefix rules that leave the same root (circumfix and NEEDAFFIX alike) and answers them in one stem lookup, one chain walk and one suffix walk with all their flags (FlagProbe): 1,000 or 16,000 prefixes adding nothing no longer cost one walk each (h1.aff: 1.6 s → 17 ms per two words; 16k prefixes 22 s → 19 ms). - Deferred suggestion candidates are not looked up again before their compound check. - A word in capitals matches a mixed-case stem as a simple word only (`JQUERYKABEL`, `MSCFOO` wrong, as in the CLI). - to_bytes writes the flags only the kept-apart homonyms use (NEEDAFFIX beside a compound flag survives a round trip). - MAGIC WCAFF2: compact data from before homonyms were kept apart is refused. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FypgCX8LsYYE8fsj4byBm2 --- crates/proof/src/affix.rs | 145 +++++++++++++++++++++--------- crates/proof/src/affix_suggest.rs | 2 +- 2 files changed, 106 insertions(+), 41 deletions(-) diff --git a/crates/proof/src/affix.rs b/crates/proof/src/affix.rs index cdc2ccc5..b990c443 100644 --- a/crates/proof/src/affix.rs +++ b/crates/proof/src/affix.rs @@ -45,7 +45,7 @@ mod compound; pub(crate) use compound::{MAX_SUGGEST_COMPOUND_WORK, Pieces}; -const MAGIC: &str = "WCAFF1"; +const MAGIC: &str = "WCAFF2"; /// Most bytes the compact format inflates to, written or read. const MAX_INFLATED: usize = 64 << 20; /// Longest word (in bytes) looked up; longer input is never a dictionary word. @@ -1657,11 +1657,11 @@ impl AffixDict { /// and followed once, not once per rule: the answer is the same, but thousands of rules of one /// flag cost one walk, not one each (#3613). /// - /// With `pfx` set (a prefix was taken off), every rule on the way must allow cross products - /// and the prefix's flag must come from the stem or a continuation class on the way - /// (`meg` + `épít` + `és` + `e`). - fn form_has_flag(&self, w: &str, flags: &[u16], depth: u8, pfx: Option, excluded: Excluded) -> bool { - let fits = |f: &[u16]| intersects(f, flags) && pfx.is_none_or(|p| has(f, p)); + /// With `pfx` set (a prefix was taken off: the flags of the prefixes that could be it), every + /// rule on the way must allow cross products and one prefix's flag must come from the stem or + /// a continuation class on the way (`meg` + `épít` + `és` + `e`). + fn form_has_flag(&self, w: &str, flags: &[u16], depth: u8, pfx: Option<&FlagProbe>, excluded: Excluded) -> bool { + let fits = |f: &[u16]| intersects(f, flags) && pfx.is_none_or(|p| p.meets(f)); if if pfx.is_some() { self.homonym_meets(w, excluded, fits) } else { self.stem_flags_except(w, excluded).is_some_and(fits) } { return true; } @@ -1696,7 +1696,7 @@ impl AffixDict { && !self.cont_holds(r, self.only_in_compound) && r.cond_matches(&root, true) { - if pfx.is_some_and(|p| self.cont_holds(r, Some(p))) { met.push(r.flag) } else { inner.push(r.flag) } + if pfx.is_some_and(|p| p.meets(self.set(r.cont))) { met.push(r.flag) } else { inner.push(r.flag) } } } for (list, p) in [(&mut inner, pfx), (&mut met, None)] { @@ -1736,14 +1736,12 @@ impl AffixDict { crate::affix_suggest::suggest_in(self, word, max, &crate::affix_suggest::Tuning::DEFAULT) } - /// [`AffixDict::contains_suggestible`], where the flag-based compound search spends its steps - /// from `budget` (at most [`compound::MAX_SUGGEST_WORK`] for this word). - pub(crate) fn contains_suggestible_spending(&self, w: &str, budget: &mut usize, pieces: &mut compound::Pieces) -> bool { + /// Is `w`, known to be no suggestible simple word or `COMPOUNDRULE` compound, a flag-based + /// compound? The search spends its steps from `budget` (at most + /// [`compound::MAX_SUGGEST_WORK`] for this word); the word isn't looked up again. + pub(crate) fn flag_compound_spending_suggestible(&self, w: &str, budget: &mut usize, pieces: &mut compound::Pieces) -> bool { let w = self.normalize(w); let e = self.excluded(true); - if self.simple_word(&w, e) || self.compound(&w, e) { - return true; - } let allowed = (*budget).min(compound::MAX_SUGGEST_WORK); let mut left = allowed; let found = self.flag_compound_spending(&w, e, matches!(casing(&w), Casing::Initial | Casing::Upper), &mut left, Some(pieces)); @@ -1803,7 +1801,6 @@ impl AffixDict { if !prefixes { return false; } - let mut root = String::with_capacity(w.len() + 8); // The prefix rules whose appended text starts `w`, found by that text (one hash lookup per // length) rather than by trying every rule. let ends = std::iter::once(0).chain(w.char_indices().map(|(i, c)| i + c.len_utf8())); @@ -1812,30 +1809,48 @@ impl AffixDict { .filter_map(|b| self.pfx_index.get(w.get(..b)?)) .flatten() .filter_map(|&i| self.prefixes.get(i as usize)); + // What each rule leaves: (root, circumfix, needs another affix, cross, flag). The rules + // that leave the same root alike are answered together, in one walk each. + let mut taken: Vec<(String, bool, bool, bool, u16)> = Vec::new(); for r in rules { let Some(rest) = w.strip_prefix(&*r.add) else { continue }; if (rest.is_empty() && !self.full_strip) || self.cont_holds(r, self.only_in_compound) { continue; } - root.clear(); - root.push_str(&r.strip); - root.push_str(rest); - if !r.cond_matches(&root, false) { - continue; + let root = format!("{}{rest}", r.strip); + if r.cond_matches(&root, false) { + // A circumfix prefix works only with its suffix, and one that needs another + // affix only with a suffix too. + taken.push((root, self.cont_holds(r, self.circumfix), self.cont_holds(r, self.need_affix), r.cross, r.flag)); } - // A circumfix prefix works only with its suffix, and one that needs another affix - // only with a suffix too. - let circ = self.cont_holds(r, self.circumfix); - let needs = self.cont_holds(r, self.need_affix); - if !circ && !needs { - if self.stem_flags_except(&root, excluded).is_some_and(|f| has(f, r.flag)) { + } + taken.sort_unstable(); + let mut plain: Vec = Vec::new(); + let mut crossed: Vec = Vec::new(); + for group in taken.chunk_by(|a, b| (&a.0, a.1, a.2) == (&b.0, b.1, b.2)) { + let Some((root, circ, needs, ..)) = group.first() else { continue }; + plain.clear(); + crossed.clear(); + for &(.., cross, flag) in group { + if !circ && !needs { + plain.push(flag); + } + if cross { + crossed.push(flag); + } + } + plain.dedup(); + crossed.dedup(); + if !plain.is_empty() { + if self.stem_flags_except(root, excluded).is_some_and(|f| intersects(f, &plain)) { return true; } - if self.is_cont_flag(r.flag) && self.form_has_flag(&root, &[r.flag], INNER_CHAIN, None, excluded) { + plain.retain(|&f| self.is_cont_flag(f)); + if !plain.is_empty() && self.form_has_flag(root, &plain, INNER_CHAIN, None, excluded) { return true; } } - if r.cross && self.suffixed(&root, Some(r.flag), circ, needs, excluded) { + if !crossed.is_empty() && self.suffixed(root, Some(&FlagProbe::new(&crossed)), *circ, *needs, excluded) { return true; } } @@ -1888,7 +1903,9 @@ impl AffixDict { /// one: the suffix must be one too, and otherwise must not be. `pfx_needs`: the prefix needs /// another affix (NEEDAFFIX), so the suffix may not need one too (Hunspell rejects `reworks` /// from `re/N` and `s/N`). - fn suffixed(&self, w: &str, need: Option, circ: bool, pfx_needs: bool, excluded: Excluded) -> bool { + /// `need` holds the flags of all the prefixes that could have been taken off (one walk for + /// them all: a thousand prefix rules adding nothing cost one walk, not a thousand). + fn suffixed(&self, w: &str, need: Option<&FlagProbe>, circ: bool, pfx_needs: bool, excluded: Excluded) -> bool { // Fits the suffix rule `r`: circumfix alike, and in need of another affix only when a // prefix that doesn't is there. let fits = |r: &Rule| { @@ -1915,7 +1932,7 @@ impl AffixDict { let takes = |f: &[u16]| { rules().any(|r| { has(f, r.flag) - && need.is_none_or(|n| has(f, n) || self.cont_holds(r, Some(n))) + && need.is_none_or(|n| n.meets(f) || n.meets(self.set(r.cont))) && (need.is_none() || r.cross) && fits(r) && r.cond_matches(&root, true) @@ -1946,7 +1963,7 @@ impl AffixDict { && !self.cont_holds(r, self.only_in_compound) && r.cond_matches(&root, true) }) { - if need.is_some_and(|n| self.cont_holds(r, Some(n))) { met.push(r.flag) } else { flags.push(r.flag) } + if need.is_some_and(|n| n.meets(self.set(r.cont))) { met.push(r.flag) } else { flags.push(r.flag) } } for (list, p) in [(&mut flags, need), (&mut met, None)] { list.sort_unstable(); @@ -2012,14 +2029,14 @@ impl AffixDict { } /// Is `lower` (a word in capitals, lowercased) a stem in mixed case, or one with an ending - /// (`kbs` from `kB/S`)? + /// (`kbs` from `kB/S`)? Not a compound of one (`JQUERYKABEL` is wrong in the CLI). fn mixed_case(&self, lower: &str, excluded: Excluded) -> bool { if self.mixed.is_empty() { return false; } let ends = lower.char_indices().map(|(i, c)| i + c.len_utf8()); ends.filter_map(|b| Some((self.mixed.get(lower.get(..b)?)?, lower.get(b..)?))).any(|(stems, rest)| { - stems.iter().take(MAX_HOMONYMS).filter_map(|&i| self.word(i as usize)).any(|m| self.contains_except(&format!("{m}{rest}"), excluded)) + stems.iter().take(MAX_HOMONYMS).filter_map(|&i| self.word(i as usize)).any(|m| self.simple_word(&format!("{m}{rest}"), excluded)) }) } @@ -2236,7 +2253,7 @@ impl AffixDict { &self.comments } - /// Serialize to WordCraft's compact format: raw-deflate of `WCAFF1`, a `FLAG` line when flags + /// Serialize to WordCraft's compact format: raw-deflate of `WCAFF2`, a `FLAG` line when flags /// aren't one character each, the comments, the tables and rules as text lines, a `WORDS` /// line, then one line per stem, front-coded against the previous one (a byte with the shared /// prefix length in bytes, the rest of the word, `/` and its flags if any). Flags are written @@ -2268,7 +2285,7 @@ impl AffixDict { // In one-character mode a directive flag that no stem or rule uses isn't written, as // before flag ids existed, so those dictionaries' compact data stays byte for byte the same. let used = |flag: u16| { - self.flags.iter().chain(self.cpd_sets.values()).any(|&id| self.set(id).contains(&flag)) + self.flags.iter().chain(self.cpd_sets.values()).chain(self.homonyms.values().flatten()).any(|&id| self.set(id).contains(&flag)) || self.prefixes.iter().chain(&self.suffixes).any(|r| r.flag == flag || self.set(r.cont).contains(&flag)) }; if self.full_strip { @@ -3434,6 +3451,51 @@ pub(crate) mod tests { assert!(!d.check("KBT") && !d.check("UNKB") && !d.check("Kb") && !d.check("kb") && !d.check("Abcd")); } + #[test] + fn a_word_in_capitals_is_no_compound_of_a_mixed_case_stem() { + // The CLI's answers: `MSCS` and `ABC` right, `MSCFOO` and `ABCFOO` wrong. + let d = AffixDict::from_hunspell("SET UTF-8\nCOMPOUNDFLAG X\nSFX S Y 1\nSFX S 0 s .\n", "3\nMSc/SX\naBc/X\nfoo/X\n", &[]).unwrap(); + assert!(d.check("MSCS") && d.check("MSC") && d.check("ABC") && d.check("MScfoo")); + assert!(!d.check("MSCFOO") && !d.check("ABCFOO") && !d.check("FOOABC")); + } + + #[test] + fn homonyms_kept_apart_survive_the_compact_form() { + // `foo/XN` beside `foo/S`: `foos` right, `foobar` wrong (the CLI's answers), also after + // a round trip (the NEEDAFFIX flag is written). + let aff = "SET UTF-8\nCOMPOUNDFLAG X\nNEEDAFFIX N\nCOMPOUNDMIN 1\nSFX S Y 1\nSFX S 0 s .\n"; + let d = AffixDict::from_hunspell(aff, "3\nfoo/XN\nfoo/S\nbar/X\n", &[]).unwrap(); + let again = AffixDict::from_bytes(&d.to_bytes().unwrap()).unwrap(); + for d in [d, again] { + assert!(d.check("foos") && d.check("foo")); + assert!(!d.check("foobar") && !d.check("foosbar") && !d.check("barfoo")); + } + } + + #[test] + fn many_prefixes_adding_nothing_stay_cheap() { + // A thousand prefix rules that add nothing, and 40 chained suffix lengths of 50 flags + // each: every prefix leaves the same root, so they share one walk. + let mut aff = String::from("SET UTF-8\nFLAG num\n"); + for f in 1..=1000 { + aff.push_str(&format!("PFX {f} Y 1\nPFX {f} 0 0 .\n")); + } + let conts: Vec = (0..50).map(|k| (5000 + k * 7).to_string()).collect(); + for len in 1..=40 { + for k in 0..50 { + let f = 5000 + (len - 1) * 50 + k; + aff.push_str(&format!("SFX {f} Y 1\nSFX {f} 0 {}/{} .\n", "a".repeat(len), conts.join(","))); + } + } + let d = AffixDict::from_hunspell(&aff, "2\nb/1,5000\nab/2\n", &[]).unwrap(); + let t = std::time::Instant::now(); + for n in [40, 119] { + let _ = d.check(&"a".repeat(n)); + } + let _ = d.suggest(&format!("{}x", "a".repeat(39)), 8); + assert!(t.elapsed().as_secs() < 5, "{:?}", t.elapsed()); + } + #[test] fn hostile_compound_rules_are_refused_or_cheap() { for bad in ["COMPOUNDRULE *a", "COMPOUNDRULE (ab)", "COMPOUNDRULE a**", "COMPOUNDMIN x", "ONLYINCOMPOUND ab"] { @@ -3518,7 +3580,7 @@ pub(crate) mod tests { assert!(AffixDict::from_bytes(b"not deflate").is_err()); let ok = dict().to_bytes().unwrap(); assert!(AffixDict::from_bytes(&ok[..ok.len() / 2]).is_err()); - let garbage = miniz_oxide::deflate::compress_to_vec(b"WCAFF1\nWORDS\n\x09abc\n", 6); + let garbage = miniz_oxide::deflate::compress_to_vec(b"WCAFF2\nWORDS\n\x09abc\n", 6); assert!(AffixDict::from_bytes(&garbage).is_err(), "prefix longer than the previous word"); let empty = AffixDict::default(); assert!(!empty.contains("kot") && !empty.check("Kot") && empty.forms("kot").is_empty()); @@ -3558,7 +3620,7 @@ pub(crate) mod tests { let d = AffixDict::from_hunspell(NUM_AFF, NUM_DIC, &["num test"]).unwrap(); let bytes = d.to_bytes().unwrap(); let raw = miniz_oxide::inflate::decompress_to_vec(&bytes).unwrap(); - assert!(raw.starts_with(b"WCAFF1\nFLAG num\n"), "the mode is written before any flag"); + assert!(raw.starts_with(b"WCAFF2\nFLAG num\n"), "the mode is written before any flag"); let text = String::from_utf8_lossy(&raw); assert!(text.contains("FORBIDDENWORD 900\n") && text.contains("SFX 12 Y 0 om/1503 .\n") && text.contains("grad/7,12\n")); let back = AffixDict::from_bytes(&bytes).unwrap(); @@ -3617,7 +3679,7 @@ pub(crate) mod tests { assert_eq!(d.len(), 5); assert_long_dict(&d); let bytes = d.to_bytes().unwrap(); - assert!(miniz_oxide::inflate::decompress_to_vec(&bytes).unwrap().starts_with(b"WCAFF1\nFLAG long\n")); + assert!(miniz_oxide::inflate::decompress_to_vec(&bytes).unwrap().starts_with(b"WCAFF2\nFLAG long\n")); let back = AffixDict::from_bytes(&bytes).unwrap(); assert_long_dict(&back); assert_eq!(back.to_bytes().unwrap(), bytes); @@ -3910,15 +3972,18 @@ pub(crate) mod tests { #[test] fn compact_data_without_a_flag_line_still_loads() { - // Data written before the FLAG line existed: one-character flags, no FLAG line. - let old = miniz_oxide::deflate::compress_to_vec(b"WCAFF1\n# old data\nFORBIDDENWORD F\nSFX a Y 0 y .\nWORDS\n\x00kot/a\n\x00zly/F\n", 6); + // Data from before homonyms were kept apart (WCAFF1) is refused, not read with old answers. + let v1 = miniz_oxide::deflate::compress_to_vec(b"WCAFF1\nWORDS\n\x00kot\n", 6); + assert!(AffixDict::from_bytes(&v1).is_err()); + // One-character flags need no FLAG line. + let old = miniz_oxide::deflate::compress_to_vec(b"WCAFF2\n# old data\nFORBIDDENWORD F\nSFX a Y 0 y .\nWORDS\n\x00kot/a\n\x00zly/F\n", 6); let d = AffixDict::from_bytes(&old).unwrap(); assert!(d.check("kot") && d.check("koty") && !d.check("zly")); // One-character dictionaries still write no FLAG line, so their data is unchanged. let raw = miniz_oxide::inflate::decompress_to_vec(&dict().to_bytes().unwrap()).unwrap(); assert!(!String::from_utf8_lossy(&raw).contains("\nFLAG")); // A FLAG line after a flag, or an unknown one, is bad data. - for head in ["WCAFF1\nSFX a Y 0 y .\nFLAG num\n", "WCAFF1\nFLAG bogus\n"] { + for head in ["WCAFF2\nSFX a Y 0 y .\nFLAG num\n", "WCAFF2\nFLAG bogus\n"] { let bytes = miniz_oxide::deflate::compress_to_vec(format!("{head}WORDS\n\x00kot\n").as_bytes(), 6); assert!(AffixDict::from_bytes(&bytes).is_err(), "{head:?}"); } diff --git a/crates/proof/src/affix_suggest.rs b/crates/proof/src/affix_suggest.rs index 5e49183e..a7c738f1 100644 --- a/crates/proof/src/affix_suggest.rs +++ b/crates/proof/src/affix_suggest.rs @@ -201,7 +201,7 @@ impl Collector<'_> { let mut work = self.compound_work; let pieces = &mut self.pieces; let mut is_word = - |w: &str| if compounds { d.contains_suggestible_spending(w, &mut work, pieces) } else { d.contains_suggestible_with(w, false) }; + |w: &str| if compounds { d.flag_compound_spending_suggestible(w, &mut work, pieces) } else { d.contains_suggestible_with(w, false) }; for part in cand.split(' ') { if is_word(part) { words.push(part.to_string()); From d0c341c88f4a46449fd11b8ee0152abe1051d3f0 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Mon, 2 Nov 2026 04:22:14 +0000 Subject: [PATCH 13/40] =?UTF-8?q?CP7.6:=20craft-libs#497=20re-review=20?= =?UTF-8?q?=E2=80=94=20prefix=20flags=20sorted=20before=20lookup;=20one=20?= =?UTF-8?q?step=20budget=20per=20simple-word=20check?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - simple_word sorts the grouped prefix flags (non-cross then cross flags were unsorted, so `redo`/`rework` from a no-cross prefix beside cross ones were missed). - One budget of 65,536 affix rules per simple-word check, shared by every prefix group and suffix walk; past it the word is taken for no word. Prefixes that each strip their own text (h2.aff: 1,000 roots) no longer cost a full walk each: 1.7 s → 1.1 ms per two words, suggest 14.9 s → 65 ms. The 77 fetched spelling packs answer as without a budget on 6,000 generated words each. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FypgCX8LsYYE8fsj4byBm2 --- crates/proof/src/affix.rs | 80 +++++++++++++++++++++++++++++++++++---- 1 file changed, 73 insertions(+), 7 deletions(-) diff --git a/crates/proof/src/affix.rs b/crates/proof/src/affix.rs index b990c443..3da15e6b 100644 --- a/crates/proof/src/affix.rs +++ b/crates/proof/src/affix.rs @@ -224,6 +224,8 @@ fn parse_cond(text: &str) -> Result, String> { Ok(out.into_boxed_slice()) } +/// Most affix rules one simple-word check tries, all prefixes and suffix chains together. +const MAX_SIMPLE_WORK: usize = 1 << 16; /// Most homonyms of one stem kept apart ([`AffixDict::homonyms`]); past it their flags count merged. const MAX_HOMONYMS: usize = 16; /// How many suffixes a word may carry in a row through continuation classes (`work` + @@ -879,6 +881,18 @@ impl<'a> FlagProbe<'a> { } } +/// Affix rules a word's check may still try ([`MAX_SIMPLE_WORK`] at first). +struct Steps(std::cell::Cell); + +impl Steps { + /// Take `n` steps; false when they ran out. + fn spend(&self, n: usize) -> bool { + let left = self.0.get(); + self.0.set(left.saturating_sub(n.max(1))); + left > 0 + } +} + /// Does the sorted flag set `set` hold `flag`? fn has(set: &[u16], flag: u16) -> bool { set.binary_search(&flag).is_ok() @@ -1660,7 +1674,7 @@ impl AffixDict { /// With `pfx` set (a prefix was taken off: the flags of the prefixes that could be it), every /// rule on the way must allow cross products and one prefix's flag must come from the stem or /// a continuation class on the way (`meg` + `épít` + `és` + `e`). - fn form_has_flag(&self, w: &str, flags: &[u16], depth: u8, pfx: Option<&FlagProbe>, excluded: Excluded) -> bool { + fn form_has_flag(&self, w: &str, flags: &[u16], depth: u8, pfx: Option<&FlagProbe>, excluded: Excluded, steps: &Steps) -> bool { let fits = |f: &[u16]| intersects(f, flags) && pfx.is_none_or(|p| p.meets(f)); if if pfx.is_some() { self.homonym_meets(w, excluded, fits) } else { self.stem_flags_except(w, excluded).is_some_and(fits) } { return true; @@ -1683,6 +1697,9 @@ impl AffixDict { let (Some(base), Some(add)) = (w.get(..b), w.get(b..)) else { continue }; let Some(groups) = self.sfx_index.get(add) else { continue }; for g in groups.iter().filter(|g| !g.cont_rules.is_empty()) { + if !steps.spend(g.cont_rules.len()) { + return false; + } root.clear(); root.push_str(base); root.push_str(&g.strip); @@ -1702,7 +1719,7 @@ impl AffixDict { for (list, p) in [(&mut inner, pfx), (&mut met, None)] { list.sort_unstable(); list.dedup(); - if !list.is_empty() && self.form_has_flag(&root, list, depth - 1, p, excluded) { + if !list.is_empty() && self.form_has_flag(&root, list, depth - 1, p, excluded, steps) { return true; } } @@ -1794,8 +1811,11 @@ impl AffixDict { if w.is_empty() || w.len() > MAX_WORD_BYTES || self.is_forbidden(w) { return false; } + // The affix rules tried for this word, all ways together: past MAX_SIMPLE_WORK it is + // taken for no word (hostile rules can't make one word cost seconds). + let steps = Steps(std::cell::Cell::new(MAX_SIMPLE_WORK)); // A stem that needs an affix is no word on its own. - if self.stem_flags_except(w, excluded.and(self.need_affix)).is_some() || self.suffixed(w, None, false, false, excluded) { + if self.stem_flags_except(w, excluded.and(self.need_affix)).is_some() || self.suffixed(w, None, false, false, excluded, &steps) { return true; } if !prefixes { @@ -1828,6 +1848,9 @@ impl AffixDict { let mut plain: Vec = Vec::new(); let mut crossed: Vec = Vec::new(); for group in taken.chunk_by(|a, b| (&a.0, a.1, a.2) == (&b.0, b.1, b.2)) { + if !steps.spend(group.len()) { + return false; + } let Some((root, circ, needs, ..)) = group.first() else { continue }; plain.clear(); crossed.clear(); @@ -1839,18 +1862,20 @@ impl AffixDict { crossed.push(flag); } } + plain.sort_unstable(); plain.dedup(); + crossed.sort_unstable(); crossed.dedup(); if !plain.is_empty() { if self.stem_flags_except(root, excluded).is_some_and(|f| intersects(f, &plain)) { return true; } plain.retain(|&f| self.is_cont_flag(f)); - if !plain.is_empty() && self.form_has_flag(root, &plain, INNER_CHAIN, None, excluded) { + if !plain.is_empty() && self.form_has_flag(root, &plain, INNER_CHAIN, None, excluded, &steps) { return true; } } - if !crossed.is_empty() && self.suffixed(root, Some(&FlagProbe::new(&crossed)), *circ, *needs, excluded) { + if !crossed.is_empty() && self.suffixed(root, Some(&FlagProbe::new(&crossed)), *circ, *needs, excluded, &steps) { return true; } } @@ -1905,7 +1930,7 @@ impl AffixDict { /// from `re/N` and `s/N`). /// `need` holds the flags of all the prefixes that could have been taken off (one walk for /// them all: a thousand prefix rules adding nothing cost one walk, not a thousand). - fn suffixed(&self, w: &str, need: Option<&FlagProbe>, circ: bool, pfx_needs: bool, excluded: Excluded) -> bool { + fn suffixed(&self, w: &str, need: Option<&FlagProbe>, circ: bool, pfx_needs: bool, excluded: Excluded, steps: &Steps) -> bool { // Fits the suffix rule `r`: circumfix alike, and in need of another affix only when a // prefix that doesn't is there. let fits = |r: &Rule| { @@ -1923,6 +1948,9 @@ impl AffixDict { let (Some(base), Some(add)) = (w.get(..b), w.get(b..)) else { continue }; let Some(groups) = self.sfx_index.get(add) else { continue }; for g in groups { + if !steps.spend(g.rules.len()) { + return false; + } root.clear(); root.push_str(base); root.push_str(&g.strip); @@ -1968,7 +1996,7 @@ impl AffixDict { for (list, p) in [(&mut flags, need), (&mut met, None)] { list.sort_unstable(); list.dedup(); - if !list.is_empty() && self.form_has_flag(&root, list, INNER_CHAIN, p, excluded) { + if !list.is_empty() && self.form_has_flag(&root, list, INNER_CHAIN, p, excluded, steps) { return true; } } @@ -3496,6 +3524,44 @@ pub(crate) mod tests { assert!(t.elapsed().as_secs() < 5, "{:?}", t.elapsed()); } + #[test] + fn prefixes_with_and_without_cross_products_on_one_root() { + // The CLI's answers: `redo` and `rework` right, from `c` (no cross products) beside `a` + // and `b` (cross products) adding the same `re`. + let aff = "SET UTF-8\nPFX a Y 1\nPFX a 0 re .\nPFX b Y 1\nPFX b 0 re .\nPFX c N 1\nPFX c 0 re .\nSFX S Y 1\nSFX S 0 s .\n"; + let d = AffixDict::from_hunspell(aff, "2\ndo/c\nwork/cS\n", &[]).unwrap(); + assert!(d.check("redo") && d.check("rework") && d.check("works")); + assert!(!d.check("reworks")); + let aff = "SET UTF-8\nFLAG num\nPFX 1 Y 1\nPFX 1 0 re .\nPFX 2 Y 1\nPFX 2 0 re .\nPFX 3 N 1\nPFX 3 0 re .\nPFX 4 N 1\nPFX 4 0 re .\nPFX 5 N 1\nPFX 5 0 re .\n"; + let d = AffixDict::from_hunspell(aff, "5\nfoo/3\nbar/4\nbaz/5\nqux/1\nquu/2\n", &[]).unwrap(); + assert!(["refoo", "rebar", "rebaz", "requx", "requu"].iter().all(|w| d.check(w))); + } + + #[test] + fn many_prefixes_with_their_own_roots_stay_cheap() { + // A thousand prefix rules each stripping its own letter (a thousand roots), and 40 + // chained suffix lengths of 50 flags each: one word's rules share one budget. + let mut aff = String::from("SET UTF-8\nFLAG num\n"); + for f in 1..=1000u32 { + let c = char::from_u32(0x4e00 + f).unwrap(); + aff.push_str(&format!("PFX {f} Y 1\nPFX {f} {c} 0 .\n")); + } + let conts: Vec = (0..50).map(|k| (5000 + k * 7).to_string()).collect(); + for len in 1..=40 { + for k in 0..50 { + let f = 5000 + (len - 1) * 50 + k; + aff.push_str(&format!("SFX {f} Y 1\nSFX {f} 0 {}/{} .\n", "a".repeat(len), conts.join(","))); + } + } + let d = AffixDict::from_hunspell(&aff, "2\nb/1,5000\nab/2\n", &[]).unwrap(); + let t = std::time::Instant::now(); + for n in [40, 119] { + let _ = d.check(&"a".repeat(n)); + } + let _ = d.suggest(&format!("{}x", "a".repeat(39)), 8); + assert!(t.elapsed().as_secs() < 10, "{:?}", t.elapsed()); + } + #[test] fn hostile_compound_rules_are_refused_or_cheap() { for bad in ["COMPOUNDRULE *a", "COMPOUNDRULE (ab)", "COMPOUNDRULE a**", "COMPOUNDMIN x", "ONLYINCOMPOUND ab"] { From c6c5386c6bcd7a6d24cb5854e82f4192d7c02f31 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Mon, 2 Nov 2026 04:56:18 +0000 Subject: [PATCH 14/40] CP7.7: one suggest call shares one simple-word budget Every TRY/KEY/typo candidate got a fresh 65,536-rule check, so a hostile .aff with a 128-letter TRY line took ~20 s per suggest call. Now a suggest call shares 2^23 affix rules across its simple-word checks, each capped at 2^11; prefix-rule collection counts against the budget and mixed-case lookups try at most MAX_HOMONYMS stems. Suggestion lists are unchanged on the 77 packs and the recall sets. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FypgCX8LsYYE8fsj4byBm2 --- crates/proof/src/affix.rs | 88 ++++++++++++++++++++++++++++--- crates/proof/src/affix_suggest.rs | 2 + 2 files changed, 82 insertions(+), 8 deletions(-) diff --git a/crates/proof/src/affix.rs b/crates/proof/src/affix.rs index 3da15e6b..32bc494b 100644 --- a/crates/proof/src/affix.rs +++ b/crates/proof/src/affix.rs @@ -226,6 +226,10 @@ fn parse_cond(text: &str) -> Result, String> { /// Most affix rules one simple-word check tries, all prefixes and suffix chains together. const MAX_SIMPLE_WORK: usize = 1 << 16; +/// Most affix rules one simple-word check of a suggestion candidate tries. +const MAX_SUGGEST_SIMPLE_WORK: usize = 1 << 11; +/// Most affix rules the simple-word checks of one suggestion call try in all. +const MAX_SUGGEST_STEPS: usize = 1 << 23; /// Most homonyms of one stem kept apart ([`AffixDict::homonyms`]); past it their flags count merged. const MAX_HOMONYMS: usize = 16; /// How many suffixes a word may carry in a row through continuation classes (`work` + @@ -881,6 +885,30 @@ impl<'a> FlagProbe<'a> { } } +// ponytail: a thread-local rather than a parameter through every check path; the guard below +// restores it, so nested or concurrent calls on other threads don't share it. +thread_local! { + /// Affix rules one suggestion call may still try in its simple-word checks, while it runs + /// ([`SuggestSteps`]); `None` outside one. + static SUGGEST_STEPS: std::cell::Cell> = const { std::cell::Cell::new(None) }; +} + +/// While alive, the simple-word checks on this thread share [`MAX_SUGGEST_STEPS`] affix rules +/// (one suggestion call: thousands of candidates can't each spend a full check's budget). +pub(crate) struct SuggestSteps(Option); + +impl SuggestSteps { + pub(crate) fn start() -> SuggestSteps { + SuggestSteps(SUGGEST_STEPS.with(|s| s.replace(Some(MAX_SUGGEST_STEPS)))) + } +} + +impl Drop for SuggestSteps { + fn drop(&mut self) { + SUGGEST_STEPS.with(|s| s.set(self.0)); + } +} + /// Affix rules a word's check may still try ([`MAX_SIMPLE_WORK`] at first). struct Steps(std::cell::Cell); @@ -1812,10 +1840,23 @@ impl AffixDict { return false; } // The affix rules tried for this word, all ways together: past MAX_SIMPLE_WORK it is - // taken for no word (hostile rules can't make one word cost seconds). - let steps = Steps(std::cell::Cell::new(MAX_SIMPLE_WORK)); + // taken for no word (hostile rules can't make one word cost seconds). Inside a + // suggestion call the word gets at most MAX_SUGGEST_SIMPLE_WORK of the call's budget. + let shared = SUGGEST_STEPS.with(std::cell::Cell::get); + let cap = shared.map_or(MAX_SIMPLE_WORK, |left| left.min(MAX_SUGGEST_SIMPLE_WORK)); + let steps = Steps(std::cell::Cell::new(cap)); + let found = self.simple_word_in(w, excluded, prefixes, &steps); + if let Some(left) = shared { + let spent = cap.saturating_sub(steps.0.get()).max(1); + SUGGEST_STEPS.with(|s| s.set(Some(left.saturating_sub(spent)))); + } + found + } + + /// [`AffixDict::simple_word_with`] in the affix rules `steps` allows. + fn simple_word_in(&self, w: &str, excluded: Excluded, prefixes: bool, steps: &Steps) -> bool { // A stem that needs an affix is no word on its own. - if self.stem_flags_except(w, excluded.and(self.need_affix)).is_some() || self.suffixed(w, None, false, false, excluded, &steps) { + if self.stem_flags_except(w, excluded.and(self.need_affix)).is_some() || self.suffixed(w, None, false, false, excluded, steps) { return true; } if !prefixes { @@ -1833,6 +1874,9 @@ impl AffixDict { // that leave the same root alike are answered together, in one walk each. let mut taken: Vec<(String, bool, bool, bool, u16)> = Vec::new(); for r in rules { + if !steps.spend(1) { + return false; + } let Some(rest) = w.strip_prefix(&*r.add) else { continue }; if (rest.is_empty() && !self.full_strip) || self.cont_holds(r, self.only_in_compound) { continue; @@ -1871,11 +1915,11 @@ impl AffixDict { return true; } plain.retain(|&f| self.is_cont_flag(f)); - if !plain.is_empty() && self.form_has_flag(root, &plain, INNER_CHAIN, None, excluded, &steps) { + if !plain.is_empty() && self.form_has_flag(root, &plain, INNER_CHAIN, None, excluded, steps) { return true; } } - if !crossed.is_empty() && self.suffixed(root, Some(&FlagProbe::new(&crossed)), *circ, *needs, excluded, &steps) { + if !crossed.is_empty() && self.suffixed(root, Some(&FlagProbe::new(&crossed)), *circ, *needs, excluded, steps) { return true; } } @@ -2063,9 +2107,11 @@ impl AffixDict { return false; } let ends = lower.char_indices().map(|(i, c)| i + c.len_utf8()); - ends.filter_map(|b| Some((self.mixed.get(lower.get(..b)?)?, lower.get(b..)?))).any(|(stems, rest)| { - stems.iter().take(MAX_HOMONYMS).filter_map(|&i| self.word(i as usize)).any(|m| self.simple_word(&format!("{m}{rest}"), excluded)) - }) + // At most MAX_HOMONYMS stems in all are tried, whatever the word's length. + ends.filter_map(|b| Some((self.mixed.get(lower.get(..b)?)?, lower.get(b..)?))) + .flat_map(|(stems, rest)| stems.iter().filter_map(|&i| self.word(i as usize)).map(move |m| (m, rest))) + .take(MAX_HOMONYMS) + .any(|(m, rest)| self.simple_word(&format!("{m}{rest}"), excluded)) } /// CHECKSHARPS: is `lower` (a word in capitals, lowercased) a word with some of its `ss` read @@ -3562,6 +3608,32 @@ pub(crate) mod tests { assert!(t.elapsed().as_secs() < 10, "{:?}", t.elapsed()); } + #[test] + fn suggesting_with_a_long_try_line_shares_one_budget() { + // craft-libs#497 review: a thousand one-letter prefixes, 40 chained suffix lengths and + // a 128-letter TRY line. Each TRY candidate is a simple-word check; one suggest call + // shares one budget between them (17 s in a release build before, 0.06 s after). + let mut aff = String::from("SET UTF-8\nFLAG num\nTRY "); + aff.extend(('a'..='z').chain((0x100..0x166).filter_map(char::from_u32))); + aff.push('\n'); + for f in 1..=1000u32 { + let c = char::from_u32(0x4e00 + f).unwrap(); + aff.push_str(&format!("PFX {f} Y 1\nPFX {f} {c} 0 .\n")); + } + let conts: Vec = (0..286).map(|k| (5000 + k * 7).to_string()).collect(); + for len in 1..=40 { + for k in 0..50 { + let f = 5000 + (len - 1) * 50 + k; + aff.push_str(&format!("SFX {f} Y 1\nSFX {f} 0 {}/{} .\n", "a".repeat(len), conts.join(","))); + } + } + assert_eq!(aff.lines().nth(2).map(|l| l.chars().count()), Some(4 + 128)); + let d = AffixDict::from_hunspell(&aff, "2\nb/1,5000\nab/2\n", &[]).unwrap(); + let t = std::time::Instant::now(); + let _ = d.suggest(&format!("{}x{}", "a".repeat(20), "a".repeat(19)), 8); + assert!(t.elapsed().as_secs() < 10, "{:?}", t.elapsed()); + } + #[test] fn hostile_compound_rules_are_refused_or_cheap() { for bad in ["COMPOUNDRULE *a", "COMPOUNDRULE (ab)", "COMPOUNDRULE a**", "COMPOUNDMIN x", "ONLYINCOMPOUND ab"] { diff --git a/crates/proof/src/affix_suggest.rs b/crates/proof/src/affix_suggest.rs index a7c738f1..3b498542 100644 --- a/crates/proof/src/affix_suggest.rs +++ b/crates/proof/src/affix_suggest.rs @@ -365,6 +365,8 @@ pub(crate) fn suggest_in(d: &AffixDict, word: &str, max: usize, t: &Tuning) -> V if n == 0 || n > MAX_LETTERS || max == 0 || d.is_empty() || word.contains(char::is_whitespace) { return Vec::new(); } + // The simple-word checks of this call share one budget of affix rules. + let _steps = crate::affix::SuggestSteps::start(); let case = casing(word); let input = word.to_lowercase(); let chars: Vec = input.chars().collect(); From 9b6ef1199a6d8e2e7065ce16220a89850c4b1a9a Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Mon, 2 Nov 2026 05:55:02 +0000 Subject: [PATCH 15/40] CP7.8: suggestion steps charge the work done; big homonym aliases load once A step now counts a rule, a condition class, a split's suffix lookup, 16 flags of a set looked through and a group visit, so hostile rules whose steps were cheap to count but costly to run hit the budget in time. Retuned: 2^13 steps per candidate, 2^25 per suggest call; the 81 packs' typo sets suggest exactly as at d0c341c8. Condition character sets are sorted and looked up by binary search. A stem's sets are worked out once per distinct entry list when its entries hold 256+ flags (16 kept-apart homonyms of 8,190 flags on 17,576 stems: 55 s load, now 57 ms). A near-word candidate whose steps ran out counts as a word, so suggest never offers a compound check() would refuse. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FypgCX8LsYYE8fsj4byBm2 --- crates/proof/src/affix.rs | 224 ++++++++++++++++++++++++----- crates/proof/src/affix/compound.rs | 4 +- 2 files changed, 191 insertions(+), 37 deletions(-) diff --git a/crates/proof/src/affix.rs b/crates/proof/src/affix.rs index 32bc494b..1c32affc 100644 --- a/crates/proof/src/affix.rs +++ b/crates/proof/src/affix.rs @@ -172,7 +172,11 @@ fn parse_compound_rule(pattern: &str, mode: FlagMode) -> Result }, + /// Sorted and deduplicated. + Set { + negated: bool, + chars: Box<[char]>, + }, } impl Cond { @@ -180,7 +184,7 @@ impl Cond { match self { Cond::Any => true, Cond::Char(x) => *x == c, - Cond::Set { negated, chars } => chars.contains(&c) != *negated, + Cond::Set { negated, chars } => chars.binary_search(&c).is_ok() != *negated, } } } @@ -215,6 +219,10 @@ fn parse_cond(text: &str) -> Result, String> { if !closed { return Err(format!("unclosed `[` in condition `{text}`")); } + // Sorted, so a character is looked up, not compared with each (a hostile + // condition can list thousands). + set.sort_unstable(); + set.dedup(); out.push(Cond::Set { negated, chars: set.into_boxed_slice() }); } ']' => return Err(format!("stray `]` in condition `{text}`")), @@ -224,12 +232,19 @@ fn parse_cond(text: &str) -> Result, String> { Ok(out.into_boxed_slice()) } -/// Most affix rules one simple-word check tries, all prefixes and suffix chains together. +/// Steps one simple-word check takes at most, all prefixes and suffix chains together. A step is +/// about one rule tried: a rule, a character class of its condition, [`FLAGS_PER_STEP`] flags +/// of a set looked through, and [`GROUP_STEPS`] for each group of rules a split of the word leads to. const MAX_SIMPLE_WORK: usize = 1 << 16; -/// Most affix rules one simple-word check of a suggestion candidate tries. -const MAX_SUGGEST_SIMPLE_WORK: usize = 1 << 11; -/// Most affix rules the simple-word checks of one suggestion call try in all. -const MAX_SUGGEST_STEPS: usize = 1 << 23; +/// Flags of a set looked through for one step. +const FLAGS_PER_STEP: usize = 16; +/// Steps a group of rules costs before its rules (the root it leaves is built and looked up). +const GROUP_STEPS: usize = 4; +/// Steps one simple-word check of a suggestion candidate takes at most. +const MAX_SUGGEST_SIMPLE_WORK: usize = 1 << 13; +/// Steps the simple-word checks of one suggestion call take in all (real packs' longest typos +/// need about 2^24.5; hostile rules stop here in under a second). +const MAX_SUGGEST_STEPS: usize = 1 << 25; /// Most homonyms of one stem kept apart ([`AffixDict::homonyms`]); past it their flags count merged. const MAX_HOMONYMS: usize = 16; /// How many suffixes a word may carry in a row through continuation classes (`work` + @@ -888,12 +903,12 @@ impl<'a> FlagProbe<'a> { // ponytail: a thread-local rather than a parameter through every check path; the guard below // restores it, so nested or concurrent calls on other threads don't share it. thread_local! { - /// Affix rules one suggestion call may still try in its simple-word checks, while it runs + /// Steps one suggestion call may still take in its simple-word checks, while it runs /// ([`SuggestSteps`]); `None` outside one. static SUGGEST_STEPS: std::cell::Cell> = const { std::cell::Cell::new(None) }; } -/// While alive, the simple-word checks on this thread share [`MAX_SUGGEST_STEPS`] affix rules +/// While alive, the simple-word checks on this thread share [`MAX_SUGGEST_STEPS`] steps /// (one suggestion call: thousands of candidates can't each spend a full check's budget). pub(crate) struct SuggestSteps(Option); @@ -909,10 +924,20 @@ impl Drop for SuggestSteps { } } -/// Affix rules a word's check may still try ([`MAX_SIMPLE_WORK`] at first). +/// Steps a word's check may still take ([`MAX_SIMPLE_WORK`] at first). struct Steps(std::cell::Cell); impl Steps { + /// Pay for looking through a flag set of `len` flags: about [`FLAGS_PER_STEP`] flags a step. + fn scan(&self, len: usize) -> bool { + self.spend(len / FLAGS_PER_STEP + 1) + } + + /// Pay for a rule's condition check (a step per character class). + fn cond(&self, r: &Rule) -> bool { + self.spend(r.cond.len()) + } + /// Take `n` steps; false when they ran out. fn spend(&self, n: usize) -> bool { let left = self.0.get(); @@ -921,6 +946,20 @@ impl Steps { } } +/// A stem's flag sets as [`AffixDict::build`] works them out from its entries. +struct StemSets { + id: u32, + cpd: Option, + rule: Option, + homonyms: Option>, + dropped: usize, + compound: bool, +} + +/// Flags a stem's entries hold in all from which its sets are worked out once per distinct +/// entry list ([`StemSets`]); real stems have a few dozen. +const MEMO_STEM_FLAGS: usize = 256; + /// Does the sorted flag set `set` hold `flag`? fn has(set: &[u16], flag: u16) -> bool { set.binary_search(&flag).is_ok() @@ -1314,8 +1353,25 @@ impl AffixDict { // Each homonym's flags (ONLYINCOMPOUND ones apart), and whether all fit. let mut each: Vec> = Vec::new(); let mut each_only: Vec> = Vec::new(); + // A stem's sets depend only on its entries' flag ranges: stems whose entries share big + // `AF` aliases are worked out once (16 homonyms of 8,000 flags on 17,576 stems loaded in + // 55 s when each stem merged them again). + let mut memo: FxMap, Option> = FxMap::default(); while i < stems.len() { let Some((word, ..)) = stems.get(i) else { break }; + let end = stems.get(i..).and_then(|rest| rest.iter().position(|s| s.0 != *word)).map_or(stems.len(), |k| i + k); + let entries: Vec<(usize, usize, bool)> = stems.get(i..end).unwrap_or_default().iter().map(|s| (s.1, s.2, s.3)).collect(); + let heavy = entries.iter().map(|e| e.1.saturating_sub(e.0)).sum::() >= MEMO_STEM_FLAGS; + let at = u32::try_from(d.flags.len()).ok(); + if heavy && let Some(known) = memo.get(&entries) { + if let Some(m) = known { + dropped += m.dropped; + d.push_known(word, m)?; + } + i = end; + continue; + } + let dropped_before = dropped; merged.clear(); solo.clear(); only_set.clear(); @@ -1372,6 +1428,9 @@ impl AffixDict { fits &= note_homonym(if only { &mut each_only } else { &mut each }, &fresh); } if !is_word { + if heavy { + memo.insert(entries, None); + } i = j; continue; } @@ -1411,11 +1470,12 @@ impl AffixDict { d.cpd_sets.insert(at, c); } let id = d.intern(&mut set_ids, &mut merged)?; - if [Some(id), cpd_id].into_iter().flatten().any(|s| d.set(s).iter().any(|f| compound_flags.contains(f))) - && let Some(c) = word.chars().next() - && !d.compound_firsts.contains(&c) - { - d.compound_firsts.push(c); + let compound = [Some(id), cpd_id].into_iter().flatten().any(|s| d.set(s).iter().any(|f| compound_flags.contains(f))); + d.note_first(word, compound); + if heavy && let Some(at) = at { + let homonyms = d.homonyms.get(&at).cloned(); + let rule = d.rule_sets.get(&at).copied(); + memo.insert(entries, Some(StemSets { id, cpd: cpd_id, rule, homonyms, dropped: dropped - dropped_before, compound })); } d.words.push_str(word); d.offs.push(u32::try_from(d.words.len()).map_err(|_| "stems too large".to_string())?); @@ -1442,6 +1502,36 @@ impl AffixDict { Ok(d) } + /// Note the first letter of `word` among the compound parts' when it is one (`compound`). + fn note_first(&mut self, word: &str, compound: bool) { + if compound + && let Some(c) = word.chars().next() + && !self.compound_firsts.contains(&c) + { + self.compound_firsts.push(c); + } + } + + /// Push the stem `word` with the sets `m` another stem with the same entries got. + fn push_known(&mut self, word: &str, m: &StemSets) -> Result<(), String> { + if let Ok(at) = u32::try_from(self.flags.len()) { + if let Some(h) = &m.homonyms { + self.homonyms.insert(at, h.clone()); + } + if let Some(c) = m.cpd { + self.cpd_sets.insert(at, c); + } + if let Some(r) = m.rule { + self.rule_sets.insert(at, r); + } + } + self.note_first(word, m.compound); + self.words.push_str(word); + self.offs.push(u32::try_from(self.words.len()).map_err(|_| "stems too large".to_string())?); + self.flags.push(m.id); + Ok(()) + } + /// The id of the flag set `flags` (sorted and deduplicated here), adding it if it is new. fn intern(&mut self, ids: &mut FxMap, u32>, flags: &mut Vec) -> Result { flags.sort_unstable(); @@ -1703,7 +1793,7 @@ impl AffixDict { /// rule on the way must allow cross products and one prefix's flag must come from the stem or /// a continuation class on the way (`meg` + `épít` + `és` + `e`). fn form_has_flag(&self, w: &str, flags: &[u16], depth: u8, pfx: Option<&FlagProbe>, excluded: Excluded, steps: &Steps) -> bool { - let fits = |f: &[u16]| intersects(f, flags) && pfx.is_none_or(|p| p.meets(f)); + let fits = |f: &[u16]| steps.scan(f.len()) && intersects(f, flags) && pfx.is_none_or(|p| p.meets(f)); if if pfx.is_some() { self.homonym_meets(w, excluded, fits) } else { self.stem_flags_except(w, excluded).is_some_and(fits) } { return true; } @@ -1718,14 +1808,21 @@ impl AffixDict { // Does a continuation set hold one of `flags`? Answered once per distinct set (rules // share interned sets) and, for many flags, through a bitmap: a lookup costs at most the // total size of the distinct sets it meets, not rules × set size. + if !steps.scan(flags.len()) { + return false; + } let probe = FlagProbe::new(flags); let mut seen: FxMap = FxMap::default(); let cuts = std::iter::once(w.len()).chain(w.char_indices().rev().map(|(i, _)| i).filter(|&i| i > 0 || self.full_strip)); for b in cuts.take(self.max_sfx_chars + 1) { let (Some(base), Some(add)) = (w.get(..b), w.get(b..)) else { continue }; + // Each split is a lookup of its appended text. + if !steps.scan(add.len()) { + return false; + } let Some(groups) = self.sfx_index.get(add) else { continue }; for g in groups.iter().filter(|g| !g.cont_rules.is_empty()) { - if !steps.spend(g.cont_rules.len()) { + if !steps.spend(g.cont_rules.len() + GROUP_STEPS) { return false; } root.clear(); @@ -1734,14 +1831,16 @@ impl AffixDict { inner.clear(); met.clear(); for r in g.cont_rules.iter().filter_map(|&i| self.suffixes.get(i as usize)) { - let meets = *seen.entry(r.cont).or_insert_with(|| probe.meets(self.set(r.cont))); + let cont = self.set(r.cont); + let meets = *seen.entry(r.cont).or_insert_with(|| steps.scan(cont.len()) && probe.meets(cont)); if meets && (pfx.is_none() || r.cross) && !self.cont_holds(r, self.circumfix) && !self.cont_holds(r, self.only_in_compound) + && steps.cond(r) && r.cond_matches(&root, true) { - if pfx.is_some_and(|p| p.meets(self.set(r.cont))) { met.push(r.flag) } else { inner.push(r.flag) } + if pfx.is_some_and(|p| steps.scan(cont.len()) && p.meets(cont)) { met.push(r.flag) } else { inner.push(r.flag) } } } for (list, p) in [(&mut inner, pfx), (&mut met, None)] { @@ -1836,10 +1935,15 @@ impl AffixDict { /// [`AffixDict::simple_word`], with prefixes only when `prefixes` is true. fn simple_word_with(&self, w: &str, excluded: Excluded, prefixes: bool) -> bool { + self.simple_word_budgeted(w, excluded, prefixes).unwrap_or(false) + } + + /// [`AffixDict::simple_word_with`], or `None` when its steps ran out before an answer. + fn simple_word_budgeted(&self, w: &str, excluded: Excluded, prefixes: bool) -> Option { if w.is_empty() || w.len() > MAX_WORD_BYTES || self.is_forbidden(w) { - return false; + return Some(false); } - // The affix rules tried for this word, all ways together: past MAX_SIMPLE_WORK it is + // The steps taken for this word, all ways together: past MAX_SIMPLE_WORK it is // taken for no word (hostile rules can't make one word cost seconds). Inside a // suggestion call the word gets at most MAX_SUGGEST_SIMPLE_WORK of the call's budget. let shared = SUGGEST_STEPS.with(std::cell::Cell::get); @@ -1850,10 +1954,10 @@ impl AffixDict { let spent = cap.saturating_sub(steps.0.get()).max(1); SUGGEST_STEPS.with(|s| s.set(Some(left.saturating_sub(spent)))); } - found + (found || steps.0.get() > 0).then_some(found) } - /// [`AffixDict::simple_word_with`] in the affix rules `steps` allows. + /// [`AffixDict::simple_word_with`] in the steps `steps` allows. fn simple_word_in(&self, w: &str, excluded: Excluded, prefixes: bool, steps: &Steps) -> bool { // A stem that needs an affix is no word on its own. if self.stem_flags_except(w, excluded.and(self.need_affix)).is_some() || self.suffixed(w, None, false, false, excluded, steps) { @@ -1874,7 +1978,7 @@ impl AffixDict { // that leave the same root alike are answered together, in one walk each. let mut taken: Vec<(String, bool, bool, bool, u16)> = Vec::new(); for r in rules { - if !steps.spend(1) { + if !steps.spend(1 + r.cond.len()) { return false; } let Some(rest) = w.strip_prefix(&*r.add) else { continue }; @@ -1892,7 +1996,7 @@ impl AffixDict { let mut plain: Vec = Vec::new(); let mut crossed: Vec = Vec::new(); for group in taken.chunk_by(|a, b| (&a.0, a.1, a.2) == (&b.0, b.1, b.2)) { - if !steps.spend(group.len()) { + if !steps.spend(group.len() + GROUP_STEPS) { return false; } let Some((root, circ, needs, ..)) = group.first() else { continue }; @@ -1911,7 +2015,7 @@ impl AffixDict { crossed.sort_unstable(); crossed.dedup(); if !plain.is_empty() { - if self.stem_flags_except(root, excluded).is_some_and(|f| intersects(f, &plain)) { + if self.stem_flags_except(root, excluded).is_some_and(|f| steps.scan(f.len()) && intersects(f, &plain)) { return true; } plain.retain(|&f| self.is_cont_flag(f)); @@ -1990,9 +2094,13 @@ impl AffixDict { let cuts = std::iter::once(w.len()).chain(w.char_indices().rev().map(|(i, _)| i).filter(|&i| i > 0 || self.full_strip)); for b in cuts.take(self.max_sfx_chars + 1) { let (Some(base), Some(add)) = (w.get(..b), w.get(b..)) else { continue }; + // Each split is a lookup of its appended text. + if !steps.scan(add.len()) { + return false; + } let Some(groups) = self.sfx_index.get(add) else { continue }; for g in groups { - if !steps.spend(g.rules.len()) { + if !steps.spend(g.rules.len() + GROUP_STEPS) { return false; } root.clear(); @@ -2002,13 +2110,18 @@ impl AffixDict { // The stem path: `root` is a stem carrying the rule's flag (and the prefix's, // when a cross product is being checked). let takes = |f: &[u16]| { - rules().any(|r| { - has(f, r.flag) - && need.is_none_or(|n| n.meets(f) || n.meets(self.set(r.cont))) - && (need.is_none() || r.cross) - && fits(r) - && r.cond_matches(&root, true) - }) + steps.spend(g.rules.len()) + && rules().any(|r| { + has(f, r.flag) + && need.is_none_or(|n| { + let cont = self.set(r.cont); + steps.scan(f.len() + cont.len()) && (n.meets(f) || n.meets(cont)) + }) + && (need.is_none() || r.cross) + && fits(r) + && steps.cond(r) + && r.cond_matches(&root, true) + }) }; if if need.is_some() { self.homonym_meets(&root, excluded, takes) @@ -2033,9 +2146,11 @@ impl AffixDict { self.is_cont_flag(r.flag) && (need.is_none() || r.cross) && !self.cont_holds(r, self.only_in_compound) + && steps.cond(r) && r.cond_matches(&root, true) }) { - if need.is_some_and(|n| n.meets(self.set(r.cont))) { met.push(r.flag) } else { flags.push(r.flag) } + let cont = self.set(r.cont); + if need.is_some_and(|n| steps.scan(cont.len()) && n.meets(cont)) { met.push(r.flag) } else { flags.push(r.flag) } } for (list, p) in [(&mut flags, need), (&mut met, None)] { list.sort_unstable(); @@ -3612,7 +3727,7 @@ pub(crate) mod tests { fn suggesting_with_a_long_try_line_shares_one_budget() { // craft-libs#497 review: a thousand one-letter prefixes, 40 chained suffix lengths and // a 128-letter TRY line. Each TRY candidate is a simple-word check; one suggest call - // shares one budget between them (17 s in a release build before, 0.06 s after). + // shares one budget between them (17 s in a release build before CP7.7). let mut aff = String::from("SET UTF-8\nFLAG num\nTRY "); aff.extend(('a'..='z').chain((0x100..0x166).filter_map(char::from_u32))); aff.push('\n'); @@ -3634,6 +3749,43 @@ pub(crate) mod tests { assert!(t.elapsed().as_secs() < 10, "{:?}", t.elapsed()); } + #[test] + fn a_check_out_of_steps_says_so() { + // craft-libs#497 review: the near-word check of a compound treats a candidate whose + // steps ran out as a word (the compound is refused), not as no word. + let d = AffixDict::from_hunspell("SET UTF-8\nSFX A Y 1\nSFX A 0 s .\n", "1\nkot/A\n", &[]).unwrap(); + let e = d.excluded(true); + assert_eq!(d.simple_word_budgeted("kots", e, true), Some(true)); + let _steps = SuggestSteps::start(); + SUGGEST_STEPS.with(|s| s.set(Some(0))); + assert_eq!(d.simple_word_budgeted("kots", e, true), None); + assert_eq!(d.simple_word_budgeted("kot", e, true), Some(true)); + } + + #[test] + fn many_stems_sharing_big_homonym_aliases_load_fast() { + // craft-libs#497 review: 16 homonyms per stem, each an `AF` alias of thousands of flags, + // on thousands of stems. Each stem's sets are worked out once per distinct entry list. + let n = 4000; + let mut aff = String::from("SET UTF-8\nFLAG num\nAF 16\n"); + for k in 0..16 { + let flags: Vec = (1..=n).filter(|&f| f != n - k).map(|f| f.to_string()).collect(); + aff.push_str(&format!("AF {}\n", flags.join(","))); + } + aff.push_str("SFX 1 Y 1\nSFX 1 0 s .\n"); + let stems: Vec = (0..2000).map(|i| format!("w{i}")).collect(); + let mut dic = format!("{}\n", stems.len() * 16); + for s in &stems { + for k in 1..=16 { + dic.push_str(&format!("{s}/{k}\n")); + } + } + let t = std::time::Instant::now(); + let d = AffixDict::from_hunspell(&aff, &dic, &[]).unwrap(); + assert!(t.elapsed().as_secs() < 10, "{:?}", t.elapsed()); + assert!(d.check("w7") && d.check("w1999s") && !d.check("w2000")); + } + #[test] fn hostile_compound_rules_are_refused_or_cheap() { for bad in ["COMPOUNDRULE *a", "COMPOUNDRULE (ab)", "COMPOUNDRULE a**", "COMPOUNDMIN x", "ONLYINCOMPOUND ab"] { diff --git a/crates/proof/src/affix/compound.rs b/crates/proof/src/affix/compound.rs index 4edf3df8..3530c095 100644 --- a/crates/proof/src/affix/compound.rs +++ b/crates/proof/src/affix/compound.rs @@ -979,7 +979,9 @@ impl AffixDict { let mut is_word = |cand: &[char]| { work += 1; let cand: String = cand.iter().collect(); - work > MAX_WORK || self.simple_word_with(&cand, excluded, self.stem_set(&cand).is_none()) + // Past its budget a candidate counts as a word: the compound is then rejected, as + // it is past MAX_WORK, so a suggestion never offers one the check would refuse. + work > MAX_WORK || self.simple_word_budgeted(&cand, excluded, self.stem_set(&cand).is_none()).unwrap_or(true) }; let near = self.near_miss(&s.chars, s.c.two_words, &mut is_word); s.work = work; From 9ef9185131b03245e2f61d3b129e24211728ab57 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Mon, 2 Nov 2026 21:43:25 +0000 Subject: [PATCH 16/40] =?UTF-8?q?FU.8:=20craft-proof=20cs=20=E2=80=94=20re?= =?UTF-8?q?gression=20test=20from=20wordcraft#4683:=20a=20letter=20that=20?= =?UTF-8?q?lowercases=20to=20two=20chars=20(=C4=B0)=20still=20gets=20its?= =?UTF-8?q?=20one-letter=20edits?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The shared suggestion code already edits across the whole lowercased word, so no code change is needed; the test keeps it that way. Co-Authored-By: Claude Opus 5.5 --- crates/proof/src/cs/suggest.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/crates/proof/src/cs/suggest.rs b/crates/proof/src/cs/suggest.rs index 05eba74a..1e31496e 100644 --- a/crates/proof/src/cs/suggest.rs +++ b/crates/proof/src/cs/suggest.rs @@ -75,4 +75,13 @@ mod tests { assert!(!suggest("město", 4).contains(&"město".into())); assert!(!suggest("zqzakázane", 8).contains(&"zqzakázané".into())); } + + #[test] + fn a_letter_that_lowercases_to_two_chars_is_still_suggested_for() { + // wordcraft#4683: "İx" lowercases to three chars ("i", U+0307, "x") but has two typed + // letters; the one-letter edits stay in range and dropping the dot still finds "ix". + let d = AffixDict::from_hunspell("", "1\nix\n", &[]).unwrap(); + let suggestions = suggest_in(&d, "İx", 4); + assert!(suggestions.contains(&"Ix".into()), "{suggestions:?}"); + } } From 7c3f45dd5ef91a6f75f09b99267dae6e30fb1044 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Mon, 2 Nov 2026 21:46:00 +0000 Subject: [PATCH 17/40] proof: skip a licence file that isn't UTF-8 instead of failing the build Co-Authored-By: Claude Opus 5.5 --- crates/proof/build.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/crates/proof/build.rs b/crates/proof/build.rs index 8ff77fc4..391de460 100644 --- a/crates/proof/build.rs +++ b/crates/proof/build.rs @@ -152,8 +152,14 @@ fn main() { files.sort(); for f in files { println!("cargo::rerun-if-changed={}", f.display()); + // `include_str!` fails the whole build on a file that isn't UTF-8 (a Latin-1 + // licence text) and can't name a path that isn't either: leave such a file out. + let (Some(path), true) = (f.to_str(), std::fs::read_to_string(&f).is_ok()) else { + println!("cargo::{level}=CRAFT_I18N_DIR: {} is not UTF-8; its licence text is not embedded", f.display()); + continue; + }; let title = format!("{locale}: {}", f.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default()); - let _ = write!(licences, "({title:?}, include_str!({:?})), ", f.display().to_string()); + let _ = write!(licences, "({title:?}, include_str!({path:?})), "); } } } From 83a4d73c69d290d0d9674e7ac0b3acb02cc4bbe1 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Mon, 2 Nov 2026 21:46:00 +0000 Subject: [PATCH 18/40] proof: affix_check generators give up on a dictionary with no usable stems Co-Authored-By: Claude Opus 5.5 --- crates/proof/examples/affix_check.rs | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/crates/proof/examples/affix_check.rs b/crates/proof/examples/affix_check.rs index 78c3f9bc..4319d506 100644 --- a/crates/proof/examples/affix_check.rs +++ b/crates/proof/examples/affix_check.rs @@ -48,6 +48,9 @@ fn decapitalize(w: &str) -> String { c.next().map(|f| f.to_lowercase().chain(c).collect()).unwrap_or_default() } +/// Attempts per requested word before a generator gives up (a dictionary that yields none). +const MAX_TRIES_PER_WORD: usize = 1000; + fn generate(d: &AffixDict, n: usize, seed: u64) -> Vec { let stems: Vec<&str> = d.stems().filter(|s| s.chars().all(char::is_alphabetic)).collect(); let mut rng = Rng(seed.max(1)); @@ -57,7 +60,10 @@ fn generate(d: &AffixDict, n: usize, seed: u64) -> Vec { forms.get(rng.below(forms.len())).cloned().unwrap_or_else(|| s.to_string()) }; let mut out = Vec::with_capacity(n); - while out.len() < n { + // A dictionary without alphabetic stems yields no word: give up instead of looping forever. + let mut tries = n.saturating_mul(MAX_TRIES_PER_WORD); + while out.len() < n && !stems.is_empty() && tries > 0 { + tries -= 1; let parts = 2 + rng.below(2); let mut w = form(&mut rng); for _ in 1..parts { @@ -100,7 +106,9 @@ fn single(d: &AffixDict, n: usize, seed: u64) -> Vec { endings.sort(); endings.dedup(); let mut out = Vec::with_capacity(n); - while out.len() < n { + let mut tries = n.saturating_mul(MAX_TRIES_PER_WORD); + while out.len() < n && !stems.is_empty() && tries > 0 { + tries -= 1; let s = stems.get(rng.below(stems.len())).copied().unwrap_or_default(); let forms = d.forms(s); if let Some(f) = forms.get(rng.below(forms.len())) { @@ -161,6 +169,9 @@ fn main() -> Result<(), String> { let n: usize = n.parse().map_err(|_| "n is a number")?; let seed: u64 = seed.parse().map_err(|_| "seed is a number")?; let words = if cmd == "gen" { generate(&d, n, seed) } else { single(&d, n, seed) }; + if words.len() < n { + return Err(format!("only {} of {n} words could be generated from this dictionary", words.len())); + } for w in words { println!("{w}"); } From d279c8a99c624dfeadc479fff4c223f1009b52a3 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Mon, 2 Nov 2026 21:46:00 +0000 Subject: [PATCH 19/40] proof: a pack's ru/bg/tr patterns need every letter of the word, else the built-in rules Co-Authored-By: Claude Opus 5.5 --- crates/proof/src/hyphen.rs | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/crates/proof/src/hyphen.rs b/crates/proof/src/hyphen.rs index 6bc804b2..7693a9b6 100644 --- a/crates/proof/src/hyphen.rs +++ b/crates/proof/src/hyphen.rs @@ -217,13 +217,15 @@ fn composed_chars(core: &[char]) -> (Vec, Vec>) { } /// Does a pack's `pats` hyphenate `core` in `lang`? Russian, Bulgarian and Turkish: a word in -/// their script (the guard their rules use). Polish, Portuguese, Italian and German: every word, -/// as their embedded patterns do. Any other language: a word whose letters the patterns know. +/// their script (the guard their rules use) whose letters the patterns know, so a word with a +/// letter the pack never mentions (`ё`) goes on to the built-in rules. Polish, Portuguese, Italian +/// and German: every word, as their embedded patterns do. Any other language: a word whose letters +/// the patterns know. fn pack_patterns_fit(lang: ProofLang, pats: &Patterns, core: &[char]) -> bool { match lang { - ProofLang::Ru => core.iter().all(|&c| crate::ru::is_cyrillic(c)), - ProofLang::Bg => core.iter().all(|&c| crate::bg::is_letter(c)), - ProofLang::Tr => core.iter().all(|&c| crate::tr::hyphen::is_letter(c)), + ProofLang::Ru => core.iter().all(|&c| crate::ru::is_cyrillic(c)) && pats.values(core).is_some(), + ProofLang::Bg => core.iter().all(|&c| crate::bg::is_letter(c)) && pats.values(core).is_some(), + ProofLang::Tr => core.iter().all(|&c| crate::tr::hyphen::is_letter(c)) && pats.values(core).is_some(), ProofLang::Pl | ProofLang::Pt | ProofLang::It | ProofLang::De => true, _ => pats.values(core).is_some(), } @@ -534,6 +536,19 @@ fn heuristic_points_pl(c: &[char]) -> Vec { mod tests { use super::*; + /// craft-libs#497 review: a Russian, Bulgarian or Turkish pack's patterns only take a word + /// whose letters they know; any other goes to the built-in rules, not the English heuristic. + #[test] + fn pack_patterns_for_ru_bg_tr_need_every_letter_of_the_word() { + let chars = |w: &str| w.chars().collect::>(); + let cases = [(ProofLang::Ru, "1на 1ка", "нака", "ёка"), (ProofLang::Bg, "1на 1ка", "нака", "жка"), (ProofLang::Tr, "a1b", "abab", "abğ")]; + for (lang, pat, known, unknown) in cases { + let p = Patterns::parse(pat); + assert!(pack_patterns_fit(lang, &p, &chars(known)), "{lang:?} {known}"); + assert!(!pack_patterns_fit(lang, &p, &chars(unknown)), "{lang:?} {unknown}"); + } + } + /// CodeRabbit on craft-libs#484: a Turkish word with a suffix after an apostrophe is /// hyphenated by the Turkish rules, part by part, never next to the apostrophe. #[test] From e91b2dbb97e9642f31b7628a796cd33db49b39a2 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Mon, 2 Nov 2026 21:46:00 +0000 Subject: [PATCH 20/40] proof: drop the unused binding in the pronoun-prefix arm Co-Authored-By: Claude Opus 5.5 --- crates/proof/src/ru/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/proof/src/ru/mod.rs b/crates/proof/src/ru/mod.rs index b88978af..0bed8dfc 100644 --- a/crates/proof/src/ru/mod.rs +++ b/crates/proof/src/ru/mod.rs @@ -321,7 +321,7 @@ fn hyphenated(d: &AffixDict, w: &str, opts: &crate::Options) -> bool { } match lp.as_slice() { // `кое-кто`, `кой-какой`. - [prefix, rest] if PRONOUN_PREFIXES.contains(prefix) => return parts.get(1).is_some_and(|r| known(d, r)), + [prefix, _] if PRONOUN_PREFIXES.contains(prefix) => return parts.get(1).is_some_and(|r| known(d, r)), // `по-русски` (from `русский`), `по-новому`, `по-моему`. ["по", rest] => { let Some(r) = parts.get(1) else { return false }; From 61105d293aa1347e730320dd8bf151da91dad2a4 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Mon, 2 Nov 2026 21:46:00 +0000 Subject: [PATCH 21/40] proof: no comma mark after ister or gerek (verb and noun uses) Co-Authored-By: Claude Opus 5.5 --- crates/proof/src/tr/grammar.rs | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/crates/proof/src/tr/grammar.rs b/crates/proof/src/tr/grammar.rs index 81ba93c2..b2391000 100644 --- a/crates/proof/src/tr/grammar.rs +++ b/crates/proof/src/tr/grammar.rs @@ -128,8 +128,10 @@ const ABBREVIATIONS: &[&str] = &[ /// Conjunctions that must not be preceded by a comma. const CONJUNCTIONS_NO_PRE_COMMA: &[&str] = &["ve", "veya", "yahut", "velev", "veyahut", "de", "da"]; -/// Conjunctions that must not be followed by a comma. -const CONJUNCTIONS_NO_POST_COMMA: &[&str] = &["ve", "veya", "yahut", "veyahut", "hem", "gerek", "ister"]; +/// Conjunctions that must not be followed by a comma. Not `ister` and `gerek`: they are +/// conjunctions only in `ister … ister` / `gerek … gerek`; otherwise `ister` is the verb +/// (`Kim ister, gelsin.`) and `gerek` a noun or adjective, and a comma after them is correct. +const CONJUNCTIONS_NO_POST_COMMA: &[&str] = &["ve", "veya", "yahut", "veyahut", "hem"]; /// Check grammar and punctuation in a Turkish text block. pub fn check(text: &str) -> Vec { @@ -411,7 +413,15 @@ mod tests { /// (craft-libs#488 review). #[test] fn a_comma_after_de_ne_or_ya_is_fine() { - for text in ["Ali de, Veli de geldi.", "Ne, sen mi geldin?", "Ya, bilmiyorum.", "Bu mesele de, haklı olan bizdik."] { + for text in [ + "Ali de, Veli de geldi.", + "Ne, sen mi geldin?", + "Ya, bilmiyorum.", + "Bu mesele de, haklı olan bizdik.", + "Kim ister, gelsin.", + "Ali ister, Veli istemez.", + "Gerek, yok.", + ] { assert!(check(text).is_empty(), "{text}: {:?}", check(text)); } } From 6e0351f5543ad18753890cca8e01af55789ac2fc Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Mon, 2 Nov 2026 21:46:08 +0000 Subject: [PATCH 22/40] proof: ru is not an embedded locale by default (only ru-extra.txt is) Co-Authored-By: Claude Opus 5.5 --- crates/proof/README.md | 2 +- crates/proof/src/packs.rs | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/crates/proof/README.md b/crates/proof/README.md index 49a16473..9ad76062 100644 --- a/crates/proof/README.md +++ b/crates/proof/README.md @@ -79,7 +79,7 @@ pub fn packs_dir() -> Option; // ＋ pub fn installed() -> Vec; // InstalledPack { locale, name, licence, path }, by locale pub fn licence_texts() -> Vec<(String, String)>; // (title, text): embedded English notices, then each pack's LICENCES/* pub fn load_errors() -> Vec<(String, String)>; // ＋ (locale or folder, reason): packs skipped, pattern files that failed -pub fn embedded_locales() -> &'static [&'static str]; // ＋ what this build embeds ("en-us", "de", "pl", "pt", "it", "ru" by default; "en-us" with CRAFT_PROOF_EMBED=en) +pub fn embedded_locales() -> &'static [&'static str]; // ＋ what this build embeds ("en-us", "de", "pl", "pt", "it" by default; "en-us" with CRAFT_PROOF_EMBED=en) // craft_proof (crate root) pub enum ProofLang { …, Pack(PackLocale), Other } // ＋ Pack: any language without tools of its own (nl, fr, sr-Latn…) diff --git a/crates/proof/src/packs.rs b/crates/proof/src/packs.rs index 2fb291e9..9d5ea1fa 100644 --- a/crates/proof/src/packs.rs +++ b/crates/proof/src/packs.rs @@ -190,8 +190,9 @@ const BUILT_IN_MOBY_NOTICE: &str = "The English word list built into craft-proof const EXTRA_EN_NOTICE: &str = "The English word supplement (en-extra.txt), common words the Moby list lacks, is written by \ WordCraft contributors and licensed MIT OR Apache-2.0."; -/// The locales whose Language data is embedded in this build: `en-us`, and by default `de`, -/// `pl`, `pt`, `it`, `ru` (until Language packs ship, CP4.4); a build with `CRAFT_PROOF_EMBED` +/// The locales whose upstream Language data is embedded in this build: `en-us`, and by default `de`, +/// `pl`, `pt`, `it` (until Language packs ship, CP4.4; our own `ru-extra.txt` is embedded but not +/// listed); a build with `CRAFT_PROOF_EMBED` /// embeds English and the ones it names (`en`: English only). pub fn embedded_locales() -> &'static [&'static str] { crate::data::EMBEDDED_LOCALES From 61030150dc71f0eacdef5708103d42d618cc71e7 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Mon, 2 Nov 2026 21:56:44 +0000 Subject: [PATCH 23/40] =?UTF-8?q?craft-proof=20cs,=20hr,=20it=20=E2=80=94?= =?UTF-8?q?=20a=20hyphenated=20word=20is=20looked=20up=20whole=20before=20?= =?UTF-8?q?its=20parts,=20and=20an=20elision=20(l'ex-amico)=20is=20read=20?= =?UTF-8?q?in=20each=20part=20(#496=20review)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- crates/proof/src/cs/mod.rs | 11 ++++++++++- crates/proof/src/hr/mod.rs | 11 ++++++++++- crates/proof/src/it/mod.rs | 18 +++++++++++++++++- 3 files changed, 37 insertions(+), 3 deletions(-) diff --git a/crates/proof/src/cs/mod.rs b/crates/proof/src/cs/mod.rs index 2be5e3f0..b1695140 100644 --- a/crates/proof/src/cs/mod.rs +++ b/crates/proof/src/cs/mod.rs @@ -63,7 +63,8 @@ fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool { return true; }; if w.contains('-') { - return w.split('-').all(|p| p.chars().count() <= 1 || known(d, p)); + // The whole word first, as Hunspell does: a hyphenated entry or user word. + return known(d, w) || w.split('-').all(|p| p.chars().count() <= 1 || known(d, p)); } known(d, w) } @@ -142,6 +143,14 @@ mod tests { } } + /// craft-libs#496 review: a hyphenated dictionary entry is a word even when its parts aren't. + #[test] + fn a_hyphenated_entry_is_looked_up_whole() { + let d = AffixDict::from_hunspell("SET UTF-8\n", "1\ntic-tac\n", &[]).unwrap(); + assert!(is_correct_with(&d, "tic-tac")); + assert!(!is_correct_with(&d, "tic-tacc")); + } + #[test] fn missing_dictionary_and_ignored_inputs_are_quiet() { let empty = AffixDict::default(); diff --git a/crates/proof/src/hr/mod.rs b/crates/proof/src/hr/mod.rs index 53edf067..e0bda809 100644 --- a/crates/proof/src/hr/mod.rs +++ b/crates/proof/src/hr/mod.rs @@ -65,7 +65,8 @@ fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool { return true; }; if w.contains('-') { - return w.split('-').all(|p| p.chars().count() <= 1 || known(d, p)); + // The whole word first, as Hunspell does: a hyphenated entry or user word. + return known(d, w) || w.split('-').all(|p| p.chars().count() <= 1 || known(d, p)); } known(d, w) } @@ -105,6 +106,14 @@ mod tests { } } + /// craft-libs#496 review: a hyphenated dictionary entry is a word even when its parts aren't. + #[test] + fn a_hyphenated_entry_is_looked_up_whole() { + let d = AffixDict::from_hunspell("SET UTF-8\n", "1\ntic-tac\n", &[]).unwrap(); + assert!(is_correct_with(&d, "tic-tac")); + assert!(!is_correct_with(&d, "tic-tacc")); + } + #[test] fn missing_dictionary_is_silent() { assert!(is_correct_with(&AffixDict::default(), "nepostojećariječ")); diff --git a/crates/proof/src/it/mod.rs b/crates/proof/src/it/mod.rs index a742eaa2..44550501 100644 --- a/crates/proof/src/it/mod.rs +++ b/crates/proof/src/it/mod.rs @@ -97,8 +97,14 @@ fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool { } // The whole word first, as Hunspell does: a hyphenated entry or user word. if w.contains('-') { - return known(d, w) || w.split('-').all(|p| p.chars().count() <= 1 || known(d, p)); + return known(d, w) || w.split('-').all(|p| p.chars().count() <= 1 || known_elided(d, p)); } + known_elided(d, w) +} + +/// [`known`] for a word that may start with an elided proclitic (`l'amico`, `l'ex`): the word after +/// the apostrophe is what gets checked. +fn known_elided(d: &AffixDict, w: &str) -> bool { if let Some((head, tail)) = w.split_once(['\'', '’']) && PROCLITICS.contains(&head.to_lowercase().as_str()) && !tail.is_empty() @@ -148,6 +154,16 @@ mod tests { assert!(!is_correct_with(&d, "tic-tacc")); } + /// craft-libs#496 review: an elision inside a hyphenated word (`l'ex-amico`) is read per part. + #[test] + fn an_elision_in_a_hyphenated_part_is_read() { + let d = AffixDict::from_hunspell("SET UTF-8\n", "2\nex\namico\n", &[]).unwrap(); + for w in ["l'ex-amico", "dell’ex-amico", "ex-amico"] { + assert!(is_correct_with(&d, w), "{w}"); + } + assert!(!is_correct_with(&d, "l'exx-amico")); + } + #[test] fn without_a_dictionary_nothing_is_flagged() { let none = AffixDict::default(); From 15d3c0d370e470f2782a186ad17a9897608d71d5 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Mon, 2 Nov 2026 21:56:44 +0000 Subject: [PATCH 24/40] =?UTF-8?q?craft-proof=20it,=20pt=20=E2=80=94=20narr?= =?UTF-8?q?ation=20after=20a=20quoted=20or=20dashed=20question=20or=20excl?= =?UTF-8?q?amation=20is=20not=20a=20new=20sentence=20(=C2=ABVai=3F=C2=BB?= =?UTF-8?q?=20disse=20lui)=20(#496=20review)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- crates/proof/src/it/grammar.rs | 22 ++++++++++++++++++++-- crates/proof/src/pt/grammar.rs | 11 ++++++++++- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/crates/proof/src/it/grammar.rs b/crates/proof/src/it/grammar.rs index 57c62e28..e90e324b 100644 --- a/crates/proof/src/it/grammar.rs +++ b/crates/proof/src/it/grammar.rs @@ -150,10 +150,12 @@ fn abbreviation_or_number(prev: &str) -> bool { || ABBREVIATIONS.contains(&prev.to_lowercase().as_str()) } +const CLOSING: [char; 8] = ['"', '”', '’', '\'', ')', '»', ']', '\u{1}']; + /// Does the gap between two words end a sentence: a `.`, `!` or `?` (not an ellipsis, not after /// an abbreviation), then space, with nothing but quotes and brackets around? fn ends_sentence(gap: &str, prev: &str) -> bool { - let t = gap.trim_start_matches(['"', '”', '’', '\'', ')', '»', ']', '\u{1}']); + let t = gap.trim_start_matches(CLOSING); let Some(end) = t.chars().next() else { return false }; if !matches!(end, '.' | '!' | '?') || t.starts_with("..") { return false; @@ -161,7 +163,13 @@ fn ends_sentence(gap: &str, prev: &str) -> bool { if end == '.' && abbreviation_or_number(prev) { return false; } - let rest = t.trim_start_matches(['.', '!', '?']).trim_start_matches(['"', '”', '’', '\'', ')', '»', ']']); + let after_marks = t.trim_start_matches(['.', '!', '?']); + let rest = after_marks.trim_start_matches(CLOSING); + // `«Vai?» disse lui.` and `—Vai? —disse lui.`: the narration after a quoted question or + // exclamation, or after a dialogue dash, goes on in lower case. + if rest.contains('—') || (end != '.' && after_marks.starts_with(CLOSING)) { + return false; + } rest.starts_with(char::is_whitespace) && rest.trim_start().chars().all(|c| matches!(c, '"' | '“' | '«' | '(' | '[' | '\'' | '—')) } @@ -403,6 +411,16 @@ mod tests { } } + /// craft-libs#496 review: narration after a quoted or dashed question or exclamation goes on in + /// lower case. + #[test] + fn narration_after_a_quoted_question_is_not_a_new_sentence() { + for t in ["«Vai?» disse lui.", "«Vieni!» gridò Anna.", "\"Vai?\" disse lui.", "—Vai? —disse lui.", "— Vai! — disse lui."] { + assert!(fixes(t).is_empty(), "{t}: {:?}", fixes(t)); + } + assert!(fixes("Vado a Roma? sì.").contains(&pair("sì", "Sì")), "a bare question still ends the sentence"); + } + #[test] fn po_takes_an_apostrophe() { assert_eq!(fixes("Dammi un pò di pane"), [pair("pò", "po'")]); diff --git a/crates/proof/src/pt/grammar.rs b/crates/proof/src/pt/grammar.rs index 303929de..be8325bd 100644 --- a/crates/proof/src/pt/grammar.rs +++ b/crates/proof/src/pt/grammar.rs @@ -62,7 +62,12 @@ fn ends_sentence(gap: &str, prev: &str) -> bool { if end == '.' && abbreviation_or_number(prev) { return false; } - let rest = t.trim_start_matches(['.', '!', '?']).trim_start_matches(CLOSING); + let after_marks = t.trim_start_matches(['.', '!', '?']); + let rest = after_marks.trim_start_matches(CLOSING); + // `"Vai?" disse ele.`: narration after a quoted question or exclamation goes on in lower case. + if end != '.' && after_marks.starts_with(CLOSING) { + return false; + } // A dialogue dash after the mark brings back the narration (`—Vai? —disse ele.`), which goes // on in lower case. rest.starts_with(char::is_whitespace) && !rest.contains('—') && rest.trim_start().chars().all(|c| OPENING.contains(&c)) @@ -225,6 +230,10 @@ mod tests { // craft-libs#488 review: no space after the dash. clean("—Vai? —disse ele."); clean("— Vai? — disse ele."); + // craft-libs#496 review: a closing quote after the mark. + clean("\"Vai?\" disse ele."); + clean("«Vai!» gritou ela."); + assert!(!messages("Vai? \"sim\".").is_empty(), "a bare question still ends the sentence"); clean("Ele disse: \"Volto amanhã\". E saiu."); // A list item may start in lower case. clean("maçãs,"); From bd3db196a56a545155c6c09ab535e5808e9b5ae2 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Mon, 2 Nov 2026 21:56:44 +0000 Subject: [PATCH 25/40] =?UTF-8?q?craft-proof=20es=20=E2=80=94=20a=20run=20?= =?UTF-8?q?of=20spaces=20is=20one=20Espacio=20doble=20issue=20over=20the?= =?UTF-8?q?=20whole=20run=20(#496=20review)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- crates/proof/src/es/grammar.rs | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/crates/proof/src/es/grammar.rs b/crates/proof/src/es/grammar.rs index 42eede50..687ecac3 100644 --- a/crates/proof/src/es/grammar.rs +++ b/crates/proof/src/es/grammar.rs @@ -75,11 +75,21 @@ fn spaces(text: &str, lang: ProofLang, v: &mut Vec) { v.push(issue(i, i + 1 + p.len_utf8(), "Espacio antes de un signo de puntuación".into(), vec![p.to_string()], lang)); } } - for (i, _) in text.match_indices(" ") { - if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) { + // Each run of two or more spaces once, over the whole run. + let bytes = text.as_bytes(); + let mut i = 0; + while i < bytes.len() { + if bytes.get(i) != Some(&b' ') { + i += 1; continue; } - v.push(issue(i, i + 2, "Espacio doble".into(), vec![" ".into()], lang)); + let start = i; + while bytes.get(i) == Some(&b' ') { + i += 1; + } + if i - start >= 2 { + v.push(issue(start, i, "Espacio doble".into(), vec![" ".into()], lang)); + } } } @@ -194,6 +204,16 @@ mod tests { .collect() } + /// craft-libs#496 review: a run of spaces is one issue over the whole run. + #[test] + fn a_run_of_spaces_is_reported_whole() { + for (text, run) in [("casa roja", " "), ("casa roja", " "), ("casa roja", " "), ("casa roja", " ")] { + let m: Vec<_> = messages(text).into_iter().filter(|(_, msg)| msg == "Espacio doble").collect(); + assert_eq!(m, [(run.to_string(), "Espacio doble".to_string())], "{text}"); + } + assert!(messages("casa roja").is_empty()); + } + #[test] fn repeated_words_and_spaces_have_spanish_messages() { let text = "Es es la casa , dijo."; From d2563a77d416eef2bf4a6ee7e0c3cd2ddbced749 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Mon, 2 Nov 2026 21:56:44 +0000 Subject: [PATCH 26/40] =?UTF-8?q?craft-proof=20tr=20=E2=80=94=20te=20and?= =?UTF-8?q?=20ta=20are=20the=20locative=20suffix,=20not=20a=20detached=20p?= =?UTF-8?q?article,=20so=20kitapte=20is=20not=20offered=20as=20kitap=20te?= =?UTF-8?q?=20(#496=20review)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- crates/proof/src/tr/suggest.rs | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/crates/proof/src/tr/suggest.rs b/crates/proof/src/tr/suggest.rs index 0bb705b8..9dbff6f1 100644 --- a/crates/proof/src/tr/suggest.rs +++ b/crates/proof/src/tr/suggest.rs @@ -97,8 +97,8 @@ pub fn suggest_with(d: &TrDict, word: &str, max: usize) -> Vec { } } - // 3. Detached suffix check: `-de`, `-da`, `-mi`, `-mı`, `-mu`, `-mü` - for suffix in ["de", "da", "te", "ta", "mi", "mı", "mu", "mü"] { + // 3. Detached particle check: `-de`, `-da`, `-mi`, `-mı`, `-mu`, `-mü` + for suffix in ["de", "da", "mi", "mı", "mu", "mü"] { if lower.ends_with(suffix) && lower.len() > suffix.len() + 1 { let stem = &lower[..lower.len() - suffix.len()]; if d.is_empty() || d.contains(stem) { @@ -333,6 +333,19 @@ mod tests { assert!(sugs.contains(&"geldin mi".to_string())); } + /// craft-libs#496 review: the particle is `de`/`da` whatever consonant comes before it. `te` + /// and `ta` are only the locative suffix after a voiceless consonant (`kitapta`), written + /// attached, so `kitapte` is never offered as `kitap te`. + #[test] + fn the_detached_particle_is_never_te_or_ta() { + let d = TrDict::from_hunspell("SET UTF-8\n", "2\nkitap\nsonra\n").unwrap(); + assert!(suggest_with(&d, "kitapde", 8).contains(&"kitap de".to_string())); + for w in ["kitapte", "kitapta"] { + let sugs = suggest_with(&d, w, 8); + assert!(!sugs.iter().any(|s| s == "kitap te" || s == "kitap ta"), "{w}: {sugs:?}"); + } + } + #[test] fn overlong_words_get_no_suggestions_promptly() { let d = TrDict::from_hunspell("SET UTF-8\nSFX 1 Y 1\nSFX 1 0 lar .\n", "2\nkitap/1\nsonra\n").unwrap(); From 466919a8a5182e61dc3a7fd7420163d8bd8cf121 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Mon, 2 Nov 2026 21:56:44 +0000 Subject: [PATCH 27/40] =?UTF-8?q?craft-proof=20english=20=E2=80=94=20with?= =?UTF-8?q?=20SCOWL=20embedded,=20a=20dictionaries-folder=20en=5FGB=20no?= =?UTF-8?q?=20longer=20outranks=20the=20embedded=20en-AU=20(#496=20review)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- crates/proof/src/english.rs | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/crates/proof/src/english.rs b/crates/proof/src/english.rs index 6617b1fb..cd214bac 100644 --- a/crates/proof/src/english.rs +++ b/crates/proof/src/english.rs @@ -98,6 +98,15 @@ fn names(lang: ProofLang) -> &'static [&'static str] { } } +/// The names [`names`] gives that outrank the dialect's own embedded SCOWL dictionary: its own pair +/// only, not Australian English's British fallback (a user's `en_GB` is for British text). +fn own_names(lang: ProofLang) -> &'static [&'static str] { + match lang { + ProofLang::EnAu => &["en_AU", "en-AU"], + _ => names(lang), + } +} + /// How many Englishes have a slot of their own ([`slot`]). const ENGLISHES: usize = 4; @@ -206,7 +215,7 @@ fn key(lang: ProofLang) -> &'static str { /// would rather use), for [`note_error`]. fn load(lang: ProofLang) -> (Loaded, Option<(PathBuf, String)>) { let candidates = if crate::scowl::dict(lang).is_some() { - crate::hunspell::user_candidates(names(lang)) + crate::hunspell::user_candidates(own_names(lang)) } else { crate::hunspell::candidates_exact(names(lang)) }; @@ -651,6 +660,14 @@ mod tests { std::fs::write(dir.join("en_AU.aff"), "SET UTF-8\n").unwrap(); std::fs::write(dir.join("en_AU.dic"), "1\nzzyzx\n").unwrap(); assert_eq!(found(names(ProofLang::EnAu)), ["en_AU", "en_GB"]); + // craft-libs#496 review: with SCOWL embedded, a user's en_GB must not outrank its en-AU. + assert_eq!(found(own_names(ProofLang::EnAu)), ["en_AU"]); + std::fs::remove_file(dir.join("en_AU.aff")).unwrap(); + std::fs::remove_file(dir.join("en_AU.dic")).unwrap(); + assert!(found(own_names(ProofLang::EnAu)).is_empty(), "an en_GB alone is not Australian English's own"); + for lang in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa] { + assert_eq!(own_names(lang), names(lang)); + } let _ = std::fs::remove_dir_all(&dir); } From 54ac5fbc4c94acb728f80f995f020173b915be2d Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Mon, 2 Nov 2026 21:56:44 +0000 Subject: [PATCH 28/40] =?UTF-8?q?craft-proof=20=E2=80=94=20README:=20embed?= =?UTF-8?q?ded=5Flocales()=20and=20the=20SCOWL=20byte=20count=20match=20bu?= =?UTF-8?q?ild.rs=20with=20en-AU;=20ru:=20drop=20an=20unused=20binding=20(?= =?UTF-8?q?#496=20review)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- crates/proof/README.md | 4 ++-- crates/proof/src/ru/mod.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/proof/README.md b/crates/proof/README.md index 4aeb677b..13ab9347 100644 --- a/crates/proof/README.md +++ b/crates/proof/README.md @@ -79,7 +79,7 @@ pub fn packs_dir() -> Option; // ＋ pub fn installed() -> Vec; // InstalledPack { locale, name, licence, path }, by locale pub fn licence_texts() -> Vec<(String, String)>; // (title, text): embedded English notices, then each pack's LICENCES/* pub fn load_errors() -> Vec<(String, String)>; // ＋ (locale or folder, reason): packs skipped, pattern files that failed -pub fn embedded_locales() -> &'static [&'static str]; // ＋ what this build embeds ("en-us", "de", "pl", "pt", "it", "ru" by default; "en-us" with CRAFT_PROOF_EMBED=en) +pub fn embedded_locales() -> &'static [&'static str]; // ＋ locales with embedded upstream data, not our supplements ("en-us", "en-gb", "en-ca", "en-au", "pl", "pt", "it", "de" by default; the four English ones with CRAFT_PROOF_EMBED=en) // craft_proof (crate root) pub enum ProofLang { …, Pack(PackLocale), Other } // ＋ Pack: any language without tools of its own (nl, fr, sr-Latn…) @@ -200,7 +200,7 @@ craft-i18n's copy. The SCOWL notice (SCOWL's licence, the Ispell BSD licence of the affix files and Benjamin Titze's notice) is embedded with the dictionaries: `craft_proof::scowl::notice()`, for an About window. -Embedding SCOWL adds 1,542,396 bytes of Language data (the en-US, en-GB and en-CA `.aff`/`.dic` +Embedding SCOWL adds 2,056,533 bytes of Language data (the en-US, en-GB, en-CA and en-AU `.aff`/`.dic` and the notice); the Moby list (432,905 bytes) stays for hyphenation. ## Check diff --git a/crates/proof/src/ru/mod.rs b/crates/proof/src/ru/mod.rs index 6e9ee34d..4a7cd3e0 100644 --- a/crates/proof/src/ru/mod.rs +++ b/crates/proof/src/ru/mod.rs @@ -321,7 +321,7 @@ fn hyphenated(d: &AffixDict, w: &str, opts: &crate::Options) -> bool { } match lp.as_slice() { // `кое-кто`, `кой-какой`. - [prefix, rest] if PRONOUN_PREFIXES.contains(prefix) => return parts.get(1).is_some_and(|r| known(d, r)), + [prefix, _] if PRONOUN_PREFIXES.contains(prefix) => return parts.get(1).is_some_and(|r| known(d, r)), // `по-русски` (from `русский`), `по-новому`, `по-моему`. ["по", rest] => { let Some(r) = parts.get(1) else { return false }; From 5eb1710914d007a5a03f1b824886819206b2b4eb Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Mon, 2 Nov 2026 21:59:48 +0000 Subject: [PATCH 29/40] proof: a pack's exception word satisfies the ru/bg/tr letter check Co-Authored-By: Claude Opus 5.5 --- crates/proof/src/hyphen.rs | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/crates/proof/src/hyphen.rs b/crates/proof/src/hyphen.rs index 7693a9b6..cc9d4fbd 100644 --- a/crates/proof/src/hyphen.rs +++ b/crates/proof/src/hyphen.rs @@ -223,14 +223,19 @@ fn composed_chars(core: &[char]) -> (Vec, Vec>) { /// the patterns know. fn pack_patterns_fit(lang: ProofLang, pats: &Patterns, core: &[char]) -> bool { match lang { - ProofLang::Ru => core.iter().all(|&c| crate::ru::is_cyrillic(c)) && pats.values(core).is_some(), - ProofLang::Bg => core.iter().all(|&c| crate::bg::is_letter(c)) && pats.values(core).is_some(), - ProofLang::Tr => core.iter().all(|&c| crate::tr::hyphen::is_letter(c)) && pats.values(core).is_some(), + ProofLang::Ru => core.iter().all(|&c| crate::ru::is_cyrillic(c)) && pack_knows(pats, core), + ProofLang::Bg => core.iter().all(|&c| crate::bg::is_letter(c)) && pack_knows(pats, core), + ProofLang::Tr => core.iter().all(|&c| crate::tr::hyphen::is_letter(c)) && pack_knows(pats, core), ProofLang::Pl | ProofLang::Pt | ProofLang::It | ProofLang::De => true, _ => pats.values(core).is_some(), } } +/// Do `pats` know `core`: an exception word, or a word whose letters the patterns mention? +fn pack_knows(pats: &Patterns, core: &[char]) -> bool { + pats.exception(&core.iter().collect::()).is_some() || pats.values(core).is_some() +} + /// Polish: exception words, then the patterns, then the Polish syllable heuristic. fn core_breaks_pl(core: &[char]) -> (Vec, Source) { core_breaks_pl_with(Patterns::pl(), core) @@ -549,6 +554,17 @@ mod tests { } } + /// An exception word of a pack keeps its breaks even with a letter the patterns never mention. + #[test] + fn pack_exception_words_need_no_pattern_letters() { + let chars = |w: &str| w.chars().collect::>(); + let p = Patterns::parse("1на 1ка ё-ка"); + let w = chars("ёка"); + assert!(pack_patterns_fit(ProofLang::Ru, &p, &w)); + assert_eq!(core_breaks_patterns(&p, &w), (vec![1], Source::Dictionary)); + assert!(!pack_patterns_fit(ProofLang::Ru, &p, &chars("ёна"))); + } + /// CodeRabbit on craft-libs#484: a Turkish word with a suffix after an apostrophe is /// hyphenated by the Turkish rules, part by part, never next to the apostrophe. #[test] From 7b64b78cb016704a97480d07410064b1b40f5492 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Mon, 2 Nov 2026 23:23:09 +0000 Subject: [PATCH 30/40] =?UTF-8?q?craft-proof=20english=20=E2=80=94=20Austr?= =?UTF-8?q?alian=20English=20also=20takes=20a=20word=20in=20SCOWL's=20en-G?= =?UTF-8?q?B=20list,=20and=20without=20SCOWL=20every=20en=5FAU=20source=20?= =?UTF-8?q?is=20read=20before=20any=20en=5FGB=20one=20(#496=20review)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- crates/proof/src/english.rs | 63 ++++++++++++++++++++++++++++++++----- crates/proof/src/lib.rs | 20 ++++++++++++ 2 files changed, 75 insertions(+), 8 deletions(-) diff --git a/crates/proof/src/english.rs b/crates/proof/src/english.rs index cd214bac..c27f01a9 100644 --- a/crates/proof/src/english.rs +++ b/crates/proof/src/english.rs @@ -91,13 +91,29 @@ fn names(lang: ProofLang) -> &'static [&'static str] { match lang { ProofLang::EnGb => &["en_GB", "en-GB"], ProofLang::EnCa => &["en_CA", "en-CA"], - // Australian English's own, else a British one: the British-spelling dictionary it read - // before it had its own (a system en_GB on a build without SCOWL). + // Australian English's own, else a British one (a system en_GB on a build without SCOWL): + // [`exact_candidates`] reads every en_AU source before any en_GB one. ProofLang::EnAu => &["en_AU", "en-AU", "en_GB", "en-GB"], _ => &["en_US", "en-US"], } } +/// The Hunspell pairs of `lang` without SCOWL, from `find` (a folder search for some names, +/// [`crate::hunspell::candidates_exact`]): its own names first, and only then Australian English's +/// British fallback, so every en_AU source (a dictionaries folder, a pack, the system) comes before +/// any en_GB one. +fn exact_candidates(lang: ProofLang, find: impl Fn(&[&str]) -> Vec<(PathBuf, PathBuf)>) -> Vec<(PathBuf, PathBuf)> { + let mut out = find(own_names(lang)); + if lang == ProofLang::EnAu { + for c in find(&names(lang)[own_names(lang).len()..]) { + if !out.iter().any(|(_, d)| *d == c.1) { + out.push(c); + } + } + } + out +} + /// The names [`names`] gives that outrank the dialect's own embedded SCOWL dictionary: its own pair /// only, not Australian English's British fallback (a user's `en_GB` is for British text). fn own_names(lang: ProofLang) -> &'static [&'static str] { @@ -217,7 +233,7 @@ fn load(lang: ProofLang) -> (Loaded, Option<(PathBuf, String)>) { let candidates = if crate::scowl::dict(lang).is_some() { crate::hunspell::user_candidates(own_names(lang)) } else { - crate::hunspell::candidates_exact(names(lang)) + exact_candidates(lang, crate::hunspell::candidates_exact) }; if candidates.is_empty() { log::info!("English spelling: no Hunspell {} dictionary to read first; using SCOWL or the built-in word list", key(lang)); @@ -300,10 +316,16 @@ fn pick(found: Loaded, lang: ProofLang) -> Speller { if let Some((d, dic)) = found { return Speller::Hunspell(d, dic); } - match crate::scowl::dict(lang) { - Some(d) => Speller::Scowl(vec![d], crate::scowl::is_british(lang)), - None => Speller::List, + let Some(d) = crate::scowl::dict(lang) else { return Speller::List }; + let mut ds = vec![d]; + // SCOWL's en-AU list keeps one spelling per word (`enquiry`, not `inquiry`); the British list + // has the rest, which Australian English takes. Suggestions come from the first, en-AU. + if lang == ProofLang::EnAu + && let Some(gb) = crate::scowl::dict(ProofLang::EnGb) + { + ds.push(gb); } + Speller::Scowl(ds, crate::scowl::is_british(lang)) } /// [`speller`] for English with no dialect ([`crate::is_correct`]): a SCOWL word of any English @@ -631,11 +653,11 @@ mod tests { let fallback = pick(None, lang); if crate::scowl::dict(lang).is_some() { assert!( - matches!(&fallback, Speller::Scowl(ds, _) if ds.len() == 1 && std::ptr::eq(ds[0], crate::scowl::dict(lang).unwrap())), + matches!(&fallback, Speller::Scowl(ds, _) if ds.len() == if lang == ProofLang::EnAu { 2 } else { 1 } && std::ptr::eq(ds[0], crate::scowl::dict(lang).unwrap())), "{lang:?}" ); assert!(fallback.knows("21st") && !fallback.knows("becuase"), "{lang:?}"); - assert_eq!(fallback.knows("colour"), lang != ProofLang::En, "{lang:?}: its own dictionary only"); + assert_eq!(fallback.knows("colour"), lang != ProofLang::En, "{lang:?}: its own dictionary (en-AU: and en-GB's)"); } else { assert!(matches!(fallback, Speller::List), "{lang:?}"); } @@ -671,6 +693,31 @@ mod tests { let _ = std::fs::remove_dir_all(&dir); } + /// Without SCOWL, every en_AU source comes before any en_GB one (a dictionaries folder's, an + /// installed pack's, the system's): a user's en_GB doesn't outrank a system en_AU. + #[test] + fn australian_english_reads_every_en_au_before_any_en_gb() { + let dir = std::env::temp_dir().join(format!("wc-proof-en-au-order-{}", std::process::id())); + for sub in ["user", "system"] { + std::fs::create_dir_all(dir.join(sub)).unwrap(); + } + let write = |sub: &str, name: &str| { + std::fs::write(dir.join(sub).join(format!("{name}.aff")), "SET UTF-8\n").unwrap(); + std::fs::write(dir.join(sub).join(format!("{name}.dic")), "1\nzzyzx\n").unwrap(); + }; + write("user", "en_GB"); + write("system", "en_AU"); + let (user, system) = (vec![dir.join("user")], vec![dir.join("system")]); + let find = |n: &[&str]| crate::hunspell::ordered(n, &user, &[], &system, &[]); + let stems = |v: Vec<(PathBuf, PathBuf)>| -> Vec { + v.iter().filter_map(|(_, d)| d.file_stem().map(|s| s.to_string_lossy().into_owned())).collect() + }; + assert_eq!(stems(exact_candidates(ProofLang::EnAu, find)), ["en_AU", "en_GB"]); + assert_eq!(stems(exact_candidates(ProofLang::EnGb, find)), ["en_GB"]); + assert_eq!(stems(exact_candidates(ProofLang::En, find)), Vec::::new()); + let _ = std::fs::remove_dir_all(&dir); + } + /// craft-libs#482 review: a load that started before a reload (the dictionaries folder /// changed) serves its own check but isn't kept, so the next check reads the new folder; a /// load after it is kept, and shared with the Englishes that read the same file. diff --git a/crates/proof/src/lib.rs b/crates/proof/src/lib.rs index b4b25f5e..679a9a72 100644 --- a/crates/proof/src/lib.rs +++ b/crates/proof/src/lib.rs @@ -2571,6 +2571,26 @@ mod tests { assert!(check_spelling_in(text, gb).is_empty(), "{:?}", check_spelling_in(text, gb)); } + /// Review follow-up: SCOWL's en-AU list keeps one spelling per word, so Australian English also + /// takes a word the en-GB list has (`enquiry`, `guerrilla`, `liquorice`); American-only + /// spellings, wrong in both, stay wrong. Suggestions still come from the en-AU list. + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] + #[test] + fn australian_english_also_takes_british_words() { + crate::hold_dictionary_tests(); + let (au, gb) = (ProofLang::EnAu, ProofLang::EnGb); + for w in ["enquiry", "enquire", "guerrilla", "guerrillas", "euthanise", "liveable", "liquorice", "veranda", "burka", "Anthea"] { + assert!(is_correct_in(w, gb), "en-GB: {w}"); + assert!(is_correct_in(w, au), "en-AU: {w}"); + } + for w in ["analyze", "catalyze", "color", "center", "defense", "zzyzxq"] { + assert!(!is_correct_in(w, gb), "en-GB: {w}"); + assert!(!is_correct_in(w, au), "en-AU: {w}"); + } + assert!(is_correct_in("billabong", au) && !is_correct_in("billabong", gb), "en-AU still has its own"); + assert!(suggest_in("billabon", 5, au).iter().any(|s| s == "billabong"), "{:?}", suggest_in("billabon", 5, au)); + } + /// English spelling reads SCOWL 2026 through the affix engine: one dictionary per English, /// its notice embedded, and its ordinals (`COMPOUNDRULE`) understood. #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] From cb82e9142862e2921bbbc825bc4f95f9163747f6 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Mon, 2 Nov 2026 23:23:15 +0000 Subject: [PATCH 31/40] =?UTF-8?q?craft-proof=20=E2=80=94=20a=20run=20of=20?= =?UTF-8?q?spaces=20is=20one=20double-space=20issue=20over=20the=20whole?= =?UTF-8?q?=20run=20in=20every=20language,=20through=20one=20shared=20spac?= =?UTF-8?q?e=5Fruns=20helper=20(#496=20review)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- crates/proof/src/cs/grammar.rs | 7 ++-- crates/proof/src/es/grammar.rs | 17 ++-------- crates/proof/src/hr/grammar.rs | 7 ++-- crates/proof/src/id/grammar.rs | 7 ++-- crates/proof/src/it/grammar.rs | 7 ++-- crates/proof/src/lib.rs | 60 ++++++++++++++++++++++++++++++---- crates/proof/src/pl/grammar.rs | 7 ++-- crates/proof/src/pt/grammar.rs | 7 ++-- crates/proof/src/ru/grammar.rs | 7 ++-- 9 files changed, 70 insertions(+), 56 deletions(-) diff --git a/crates/proof/src/cs/grammar.rs b/crates/proof/src/cs/grammar.rs index 8a2590e6..e2d29954 100644 --- a/crates/proof/src/cs/grammar.rs +++ b/crates/proof/src/cs/grammar.rs @@ -62,11 +62,8 @@ fn spaces(text: &str, v: &mut Vec) { v.push(issue(i, i + 1 + p.len_utf8(), "Mezera před interpunkcí".into(), vec![p.to_string()])); } } - for (i, _) in text.match_indices(" ") { - if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) { - continue; - } - v.push(issue(i, i + 2, "Dvojitá mezera".into(), vec![" ".into()])); + for run in crate::space_runs(text) { + v.push(issue(run.start, run.end, "Dvojitá mezera".into(), vec![" ".into()])); } } diff --git a/crates/proof/src/es/grammar.rs b/crates/proof/src/es/grammar.rs index 687ecac3..00b20789 100644 --- a/crates/proof/src/es/grammar.rs +++ b/crates/proof/src/es/grammar.rs @@ -75,21 +75,8 @@ fn spaces(text: &str, lang: ProofLang, v: &mut Vec) { v.push(issue(i, i + 1 + p.len_utf8(), "Espacio antes de un signo de puntuación".into(), vec![p.to_string()], lang)); } } - // Each run of two or more spaces once, over the whole run. - let bytes = text.as_bytes(); - let mut i = 0; - while i < bytes.len() { - if bytes.get(i) != Some(&b' ') { - i += 1; - continue; - } - let start = i; - while bytes.get(i) == Some(&b' ') { - i += 1; - } - if i - start >= 2 { - v.push(issue(start, i, "Espacio doble".into(), vec![" ".into()], lang)); - } + for run in crate::space_runs(text) { + v.push(issue(run.start, run.end, "Espacio doble".into(), vec![" ".into()], lang)); } } diff --git a/crates/proof/src/hr/grammar.rs b/crates/proof/src/hr/grammar.rs index 2605ae88..f4997e9d 100644 --- a/crates/proof/src/hr/grammar.rs +++ b/crates/proof/src/hr/grammar.rs @@ -105,11 +105,8 @@ fn spaces(text: &str, v: &mut Vec) { v.push(issue(i, i + 1 + p.len_utf8(), "Razmak ispred interpunkcijskog znaka".into(), vec![p.to_string()])); } } - for (i, _) in text.match_indices(" ") { - if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) { - continue; - } - v.push(issue(i, i + 2, "Dvostruki razmak".into(), vec![" ".into()])); + for run in crate::space_runs(text) { + v.push(issue(run.start, run.end, "Dvostruki razmak".into(), vec![" ".into()])); } } diff --git a/crates/proof/src/id/grammar.rs b/crates/proof/src/id/grammar.rs index 72ca10c9..6c13b7f1 100644 --- a/crates/proof/src/id/grammar.rs +++ b/crates/proof/src/id/grammar.rs @@ -56,11 +56,8 @@ fn spaces(text: &str, v: &mut Vec) { v.push(issue(i, i + 1 + p.len_utf8(), "Spasi sebelum tanda baca".into(), vec![p.to_string()])); } } - for (i, _) in text.match_indices(" ") { - if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) { - continue; - } - v.push(issue(i, i + 2, "Spasi ganda".into(), vec![" ".into()])); + for run in crate::space_runs(text) { + v.push(issue(run.start, run.end, "Spasi ganda".into(), vec![" ".into()])); } } diff --git a/crates/proof/src/it/grammar.rs b/crates/proof/src/it/grammar.rs index e90e324b..cad690c4 100644 --- a/crates/proof/src/it/grammar.rs +++ b/crates/proof/src/it/grammar.rs @@ -211,11 +211,8 @@ fn spaces(text: &str, v: &mut Vec) { v.push(issue(i, i + 1 + p.len_utf8(), "Spazio prima della punteggiatura".into(), vec![p.to_string()])); } } - for (i, _) in text.match_indices(" ") { - if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) { - continue; - } - v.push(issue(i, i + 2, "Spazio doppio".into(), vec![" ".into()])); + for run in crate::space_runs(text) { + v.push(issue(run.start, run.end, "Spazio doppio".into(), vec![" ".into()])); } } diff --git a/crates/proof/src/lib.rs b/crates/proof/src/lib.rs index 679a9a72..62df1b7e 100644 --- a/crates/proof/src/lib.rs +++ b/crates/proof/src/lib.rs @@ -152,6 +152,28 @@ fn dictionary_key(w: &str) -> String { composed(&w.to_lowercase()).into_owned() } +/// Each run of two or more spaces in `text`, once, over the whole run (a fix replaces it with one +/// space). A run of three is one issue over three, not a pair and a leftover. +pub(crate) fn space_runs(text: &str) -> Vec> { + let bytes = text.as_bytes(); + let mut runs = Vec::new(); + let mut i = 0; + while i < bytes.len() { + if bytes.get(i) != Some(&b' ') { + i += 1; + continue; + } + let start = i; + while bytes.get(i) == Some(&b' ') { + i += 1; + } + if i - start >= 2 { + runs.push(start..i); + } + } + runs +} + /// Add a word to the user dictionary ("Add to Dictionary"). It applies to every language. pub fn add_word(w: &str) { if user_words().write().unwrap_or_else(std::sync::PoisonError::into_inner).insert(dictionary_key(w)) { @@ -1098,13 +1120,10 @@ fn grammar_en(text: &str, capital_first: bool) -> Vec { lang: ProofLang::En, }); } - for (i, _) in text.match_indices(" ") { - if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) { - continue; - } + for run in space_runs(text) { v.push(Issue { - start: i, - end: i + 2, + start: run.start, + end: run.end, kind: IssueKind::Grammar, message: "Extra space".into(), suggestions: vec![" ".into()], @@ -2591,6 +2610,35 @@ mod tests { assert!(suggest_in("billabon", 5, au).iter().any(|s| s == "billabong"), "{:?}", suggest_in("billabon", 5, au)); } + /// Review follow-up: a run of spaces is one issue over the whole run, in every language with a + /// double-space rule (a run of three used to leave two after the fix, four three). + #[test] + fn a_run_of_spaces_is_one_issue_in_every_language() { + assert_eq!(space_runs("a b c d "), [3..5, 6..9, 10..14]); + assert!(space_runs("a b c").is_empty() && space_runs("").is_empty() && space_runs(" ").is_empty()); + let langs = [ + ProofLang::En, + ProofLang::EnGb, + ProofLang::Es, + ProofLang::It, + ProofLang::Pt, + ProofLang::Cs, + ProofLang::Hr, + ProofLang::Id, + ProofLang::Pl, + ProofLang::Ru, + ]; + for lang in langs { + for n in 2..=5 { + let text = format!("Aaa{}bbb", " ".repeat(n)); + let hits: Vec<_> = + check_grammar_in(&text, lang).into_iter().filter(|i| i.start <= 3 && i.end > 3 && i.suggestions == [" "]).collect(); + assert_eq!(hits.len(), 1, "{lang:?} {n}: {hits:?}"); + assert_eq!((hits[0].start, hits[0].end), (3, 3 + n), "{lang:?} {n}"); + } + } + } + /// English spelling reads SCOWL 2026 through the affix engine: one dictionary per English, /// its notice embedded, and its ordinals (`COMPOUNDRULE`) understood. #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] diff --git a/crates/proof/src/pl/grammar.rs b/crates/proof/src/pl/grammar.rs index d514429e..2fd93f86 100644 --- a/crates/proof/src/pl/grammar.rs +++ b/crates/proof/src/pl/grammar.rs @@ -304,11 +304,8 @@ fn spaces(text: &str, v: &mut Vec) { v.push(issue(i, i + 1 + p.len_utf8(), "Zbędna spacja przed znakiem interpunkcyjnym".into(), vec![p.to_string()])); } } - for (i, _) in text.match_indices(" ") { - if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) { - continue; - } - v.push(issue(i, i + 2, "Podwójna spacja".into(), vec![" ".into()])); + for run in crate::space_runs(text) { + v.push(issue(run.start, run.end, "Podwójna spacja".into(), vec![" ".into()])); } } diff --git a/crates/proof/src/pt/grammar.rs b/crates/proof/src/pt/grammar.rs index be8325bd..fc658e22 100644 --- a/crates/proof/src/pt/grammar.rs +++ b/crates/proof/src/pt/grammar.rs @@ -129,11 +129,8 @@ fn spaces(text: &str, v: &mut Vec) { v.push(issue(i, i + 1 + p.len_utf8(), "Espaço antes da pontuação".into(), vec![p.to_string()])); } } - for (i, _) in text.match_indices(" ") { - if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) { - continue; - } - v.push(issue(i, i + 2, "Espaço duplo".into(), vec![" ".into()])); + for run in crate::space_runs(text) { + v.push(issue(run.start, run.end, "Espaço duplo".into(), vec![" ".into()])); } } diff --git a/crates/proof/src/ru/grammar.rs b/crates/proof/src/ru/grammar.rs index ff8711fb..4d1ce3c2 100644 --- a/crates/proof/src/ru/grammar.rs +++ b/crates/proof/src/ru/grammar.rs @@ -503,11 +503,8 @@ fn spaces(text: &str, v: &mut Vec) { v.push(issue(i, i + 1 + p.len_utf8(), "Лишний пробел перед знаком препинания".into(), vec![p.to_string()])); } } - for (i, _) in text.match_indices(" ") { - if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) { - continue; - } - v.push(issue(i, i + 2, "Двойной пробел".into(), vec![" ".into()])); + for run in crate::space_runs(text) { + v.push(issue(run.start, run.end, "Двойной пробел".into(), vec![" ".into()])); } } From a8b3bb74c8e51fdca1c4b156129c26a6e1b1385d Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Mon, 2 Nov 2026 23:23:15 +0000 Subject: [PATCH 32/40] =?UTF-8?q?craft-proof=20it,=20pt=20=E2=80=94=20the?= =?UTF-8?q?=20object=20placeholder=20is=20not=20a=20closing=20quote,=20so?= =?UTF-8?q?=20Chi=20era=3F\u{1}=20era=20Marco.=20still=20gets=20its=20capi?= =?UTF-8?q?tal=20mark=20(#496=20review)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- crates/proof/src/it/grammar.rs | 5 ++++- crates/proof/src/pt/grammar.rs | 6 ++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/crates/proof/src/it/grammar.rs b/crates/proof/src/it/grammar.rs index cad690c4..620e284a 100644 --- a/crates/proof/src/it/grammar.rs +++ b/crates/proof/src/it/grammar.rs @@ -165,9 +165,10 @@ fn ends_sentence(gap: &str, prev: &str) -> bool { } let after_marks = t.trim_start_matches(['.', '!', '?']); let rest = after_marks.trim_start_matches(CLOSING); + // (Not the object placeholder `\u{1}`: `Chi era?\u{1} era Marco.` still starts a sentence.) // `«Vai?» disse lui.` and `—Vai? —disse lui.`: the narration after a quoted question or // exclamation, or after a dialogue dash, goes on in lower case. - if rest.contains('—') || (end != '.' && after_marks.starts_with(CLOSING)) { + if rest.contains('—') || (end != '.' && after_marks.starts_with(|c| c != '\u{1}' && CLOSING.contains(&c))) { return false; } rest.starts_with(char::is_whitespace) && rest.trim_start().chars().all(|c| matches!(c, '"' | '“' | '«' | '(' | '[' | '\'' | '—')) @@ -416,6 +417,8 @@ mod tests { assert!(fixes(t).is_empty(), "{t}: {:?}", fixes(t)); } assert!(fixes("Vado a Roma? sì.").contains(&pair("sì", "Sì")), "a bare question still ends the sentence"); + // An inline object (`\u{1}`) is not a closing quote. + assert!(fixes("Chi era?\u{1} era Marco.").contains(&pair("era", "Era")), "{:?}", fixes("Chi era?\u{1} era Marco.")); } #[test] diff --git a/crates/proof/src/pt/grammar.rs b/crates/proof/src/pt/grammar.rs index fc658e22..23891238 100644 --- a/crates/proof/src/pt/grammar.rs +++ b/crates/proof/src/pt/grammar.rs @@ -64,8 +64,8 @@ fn ends_sentence(gap: &str, prev: &str) -> bool { } let after_marks = t.trim_start_matches(['.', '!', '?']); let rest = after_marks.trim_start_matches(CLOSING); - // `"Vai?" disse ele.`: narration after a quoted question or exclamation goes on in lower case. - if end != '.' && after_marks.starts_with(CLOSING) { + // `"Vai?" disse ele.` (not the object placeholder `\u{1}`): narration after a quoted question or exclamation goes on in lower case. + if end != '.' && after_marks.starts_with(|c| c != '\u{1}' && CLOSING.contains(&c)) { return false; } // A dialogue dash after the mark brings back the narration (`—Vai? —disse ele.`), which goes @@ -231,6 +231,8 @@ mod tests { clean("\"Vai?\" disse ele."); clean("«Vai!» gritou ela."); assert!(!messages("Vai? \"sim\".").is_empty(), "a bare question still ends the sentence"); + // An inline object (`\u{1}`) is not a closing quote. + assert!(!messages("Quem era?\u{1} era Marco.").is_empty()); clean("Ele disse: \"Volto amanhã\". E saiu."); // A list item may start in lower case. clean("maçãs,"); From f3d411638d2b76cd9a7fd23fad0d9820329657de Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Mon, 2 Nov 2026 23:47:08 +0000 Subject: [PATCH 33/40] =?UTF-8?q?craft-proof=20it=20=E2=80=94=20the=20dial?= =?UTF-8?q?ogue-dash=20exception=20applies=20after=20=3F=20and=20!=20only,?= =?UTF-8?q?=20so=20Fin=C3=AC.=20=E2=80=94domani=20partiamo.=20gets=20its?= =?UTF-8?q?=20capital=20mark=20again=20(#496=20review)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- crates/proof/src/it/grammar.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/proof/src/it/grammar.rs b/crates/proof/src/it/grammar.rs index 620e284a..fd0799f7 100644 --- a/crates/proof/src/it/grammar.rs +++ b/crates/proof/src/it/grammar.rs @@ -167,8 +167,8 @@ fn ends_sentence(gap: &str, prev: &str) -> bool { let rest = after_marks.trim_start_matches(CLOSING); // (Not the object placeholder `\u{1}`: `Chi era?\u{1} era Marco.` still starts a sentence.) // `«Vai?» disse lui.` and `—Vai? —disse lui.`: the narration after a quoted question or - // exclamation, or after a dialogue dash, goes on in lower case. - if rest.contains('—') || (end != '.' && after_marks.starts_with(|c| c != '\u{1}' && CLOSING.contains(&c))) { + // exclamation (not after a period: `Finì. —domani` starts a sentence), or after a dialogue dash, goes on in lower case. + if end != '.' && (rest.contains('—') || after_marks.starts_with(|c| c != '\u{1}' && CLOSING.contains(&c))) { return false; } rest.starts_with(char::is_whitespace) && rest.trim_start().chars().all(|c| matches!(c, '"' | '“' | '«' | '(' | '[' | '\'' | '—')) @@ -417,6 +417,8 @@ mod tests { assert!(fixes(t).is_empty(), "{t}: {:?}", fixes(t)); } assert!(fixes("Vado a Roma? sì.").contains(&pair("sì", "Sì")), "a bare question still ends the sentence"); + // After a period the dash opens the next line of speech: a sentence starts. + assert!(fixes("Finì. —domani partiamo.").contains(&pair("domani", "Domani")), "{:?}", fixes("Finì. —domani partiamo.")); // An inline object (`\u{1}`) is not a closing quote. assert!(fixes("Chi era?\u{1} era Marco.").contains(&pair("era", "Era")), "{:?}", fixes("Chi era?\u{1} era Marco.")); } From 4c8cb404ab16f275d93dff1baae3ef9c2491251b Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Mon, 2 Nov 2026 23:48:04 +0000 Subject: [PATCH 34/40] =?UTF-8?q?craft-proof=20it=20=E2=80=94=20a=20space?= =?UTF-8?q?=20after=20the=20dialogue=20dash=20no=20longer=20hides=20the=20?= =?UTF-8?q?capital=20mark=20(Fin=C3=AC.=20=E2=80=94=20domani=20partiamo.)?= =?UTF-8?q?=20(#496=20review)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- crates/proof/src/it/grammar.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/proof/src/it/grammar.rs b/crates/proof/src/it/grammar.rs index fd0799f7..15078e72 100644 --- a/crates/proof/src/it/grammar.rs +++ b/crates/proof/src/it/grammar.rs @@ -171,7 +171,8 @@ fn ends_sentence(gap: &str, prev: &str) -> bool { if end != '.' && (rest.contains('—') || after_marks.starts_with(|c| c != '\u{1}' && CLOSING.contains(&c))) { return false; } - rest.starts_with(char::is_whitespace) && rest.trim_start().chars().all(|c| matches!(c, '"' | '“' | '«' | '(' | '[' | '\'' | '—')) + rest.starts_with(char::is_whitespace) + && rest.trim_start().chars().all(|c| c.is_whitespace() || matches!(c, '"' | '“' | '«' | '(' | '[' | '\'' | '—')) } fn repeated_words(text: &str, ws: &[(usize, usize)], v: &mut Vec) { @@ -419,6 +420,7 @@ mod tests { assert!(fixes("Vado a Roma? sì.").contains(&pair("sì", "Sì")), "a bare question still ends the sentence"); // After a period the dash opens the next line of speech: a sentence starts. assert!(fixes("Finì. —domani partiamo.").contains(&pair("domani", "Domani")), "{:?}", fixes("Finì. —domani partiamo.")); + assert!(fixes("Finì. — domani partiamo.").contains(&pair("domani", "Domani")), "{:?}", fixes("Finì. — domani partiamo.")); // An inline object (`\u{1}`) is not a closing quote. assert!(fixes("Chi era?\u{1} era Marco.").contains(&pair("era", "Era")), "{:?}", fixes("Chi era?\u{1} era Marco.")); } From 3e37fbcd7ef18a2bec10910f5318e4abf34a1242 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Mon, 2 Nov 2026 23:56:09 +0000 Subject: [PATCH 35/40] =?UTF-8?q?craft-proof=20it,=20pt=20=E2=80=94=20an?= =?UTF-8?q?=20object=20placeholder=20between=20the=20mark=20and=20a=20real?= =?UTF-8?q?=20closing=20quote=20keeps=20the=20narration=20exemption=20(?= =?UTF-8?q?=C2=ABVai=3F\u{1}=C2=BB=20disse=20lui.)=20(#496=20review)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- crates/proof/src/it/grammar.rs | 4 +++- crates/proof/src/pt/grammar.rs | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/crates/proof/src/it/grammar.rs b/crates/proof/src/it/grammar.rs index 15078e72..9019218c 100644 --- a/crates/proof/src/it/grammar.rs +++ b/crates/proof/src/it/grammar.rs @@ -168,7 +168,7 @@ fn ends_sentence(gap: &str, prev: &str) -> bool { // (Not the object placeholder `\u{1}`: `Chi era?\u{1} era Marco.` still starts a sentence.) // `«Vai?» disse lui.` and `—Vai? —disse lui.`: the narration after a quoted question or // exclamation (not after a period: `Finì. —domani` starts a sentence), or after a dialogue dash, goes on in lower case. - if end != '.' && (rest.contains('—') || after_marks.starts_with(|c| c != '\u{1}' && CLOSING.contains(&c))) { + if end != '.' && (rest.contains('—') || after_marks.trim_start_matches('\u{1}').starts_with(CLOSING)) { return false; } rest.starts_with(char::is_whitespace) @@ -421,6 +421,8 @@ mod tests { // After a period the dash opens the next line of speech: a sentence starts. assert!(fixes("Finì. —domani partiamo.").contains(&pair("domani", "Domani")), "{:?}", fixes("Finì. —domani partiamo.")); assert!(fixes("Finì. — domani partiamo.").contains(&pair("domani", "Domani")), "{:?}", fixes("Finì. — domani partiamo.")); + // An inline object (`\u{1}`) before a real closing quote is skipped. + assert!(fixes("«Vai?\u{1}» disse lui.").is_empty(), "{:?}", fixes("«Vai?\u{1}» disse lui.")); // An inline object (`\u{1}`) is not a closing quote. assert!(fixes("Chi era?\u{1} era Marco.").contains(&pair("era", "Era")), "{:?}", fixes("Chi era?\u{1} era Marco.")); } diff --git a/crates/proof/src/pt/grammar.rs b/crates/proof/src/pt/grammar.rs index 23891238..eb0f89a6 100644 --- a/crates/proof/src/pt/grammar.rs +++ b/crates/proof/src/pt/grammar.rs @@ -65,7 +65,7 @@ fn ends_sentence(gap: &str, prev: &str) -> bool { let after_marks = t.trim_start_matches(['.', '!', '?']); let rest = after_marks.trim_start_matches(CLOSING); // `"Vai?" disse ele.` (not the object placeholder `\u{1}`): narration after a quoted question or exclamation goes on in lower case. - if end != '.' && after_marks.starts_with(|c| c != '\u{1}' && CLOSING.contains(&c)) { + if end != '.' && after_marks.trim_start_matches('\u{1}').starts_with(CLOSING) { return false; } // A dialogue dash after the mark brings back the narration (`—Vai? —disse ele.`), which goes @@ -231,6 +231,8 @@ mod tests { clean("\"Vai?\" disse ele."); clean("«Vai!» gritou ela."); assert!(!messages("Vai? \"sim\".").is_empty(), "a bare question still ends the sentence"); + // An inline object (`\u{1}`) before a real closing quote is skipped. + clean("\"Vai?\u{1}\" disse ele."); // An inline object (`\u{1}`) is not a closing quote. assert!(!messages("Quem era?\u{1} era Marco.").is_empty()); clean("Ele disse: \"Volto amanhã\". E saiu."); From aa3730352f3e7a7216d86bef3613b652b843579a Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Mon, 2 Nov 2026 23:56:09 +0000 Subject: [PATCH 36/40] =?UTF-8?q?craft-proof=20=E2=80=94=20en-AU=20comment?= =?UTF-8?q?=20and=20docs:=20it=20also=20takes=20en-GB's=20words=20(enquiry?= =?UTF-8?q?=20is=20only=20in=20the=20en-GB=20list),=20suggestions=20from?= =?UTF-8?q?=20en-AU=20(#496=20review)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- crates/proof/README.md | 5 +++-- crates/proof/src/english.rs | 7 ++++--- crates/proof/src/scowl.rs | 4 +++- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/crates/proof/README.md b/crates/proof/README.md index 13ab9347..97067014 100644 --- a/crates/proof/README.md +++ b/crates/proof/README.md @@ -156,8 +156,9 @@ English spelling uses the first of these that is there; `english_source(lang)` s (Australian words such as `billabong`, with en-GB's spelling policy); en-NZ, en-IE and the other British-spelling Englishes read en-GB; Canadian English reads en-CA. A word is right when the English's own dictionary knows it (or `en-extra.txt` does), never because another English's does - (`colourblind` is wrong in en-US, `colorblind` in en-GB); suggestions come from the same - dictionary. SCOWL's size-60 lists keep one spelling per dialect, and Word accepts more, so + (`colourblind` is wrong in en-US, `colorblind` in en-GB); the one exception is Australian + English, which also takes en-GB's words (`enquiry`: en-AU's list keeps `inquiry` only). + Suggestions come from the English's own dictionary (en-AU's for Australian). SCOWL's size-60 lists keep one spelling per dialect, and Word accepts more, so British English also takes the `-ize` spellings of its `-ise` words that American or Canadian English spell so (`organize`, `realization`; never `wize` or `advize`, and never `-yze`: `reanalyze` stays American), and `en-variants.txt` flags the other dialect's spellings and accepts this dialect's other spellings of known words (`judgement` in en-US). English with no dialect diff --git a/crates/proof/src/english.rs b/crates/proof/src/english.rs index c27f01a9..e821e801 100644 --- a/crates/proof/src/english.rs +++ b/crates/proof/src/english.rs @@ -7,7 +7,8 @@ //! 3. **Embedded SCOWL**: the run's English's SCOWL dictionary (en-US, en-GB, en-AU or en-CA) embedded //! from craft-i18n at build time ([`crate::scowl`]); a word it knows (British English: also //! the `-ize` spelling of one it knows, [`ise_spelling`]), or the supplement ([`EXTRA`]), is -//! right; never one only another English knows. When it is there, the system locations aren't +//! right; never one only another English knows, except that Australian English also takes +//! en-GB's words (its suggestions still come from en-AU). When it is there, the system locations aren't //! searched. //! 4. **System Hunspell**: a pair for the run's own English in the system locations (none on the //! web). `COMPOUNDRULE` (ordinals) and `OCONV` are read; flag-based compounding is ignored @@ -318,8 +319,8 @@ fn pick(found: Loaded, lang: ProofLang) -> Speller { } let Some(d) = crate::scowl::dict(lang) else { return Speller::List }; let mut ds = vec![d]; - // SCOWL's en-AU list keeps one spelling per word (`enquiry`, not `inquiry`); the British list - // has the rest, which Australian English takes. Suggestions come from the first, en-AU. + // SCOWL's en-AU list keeps one spelling per word (`inquiry`; `enquiry` is only in the en-GB + // list), and Australian English takes both. Suggestions come from the first, en-AU. if lang == ProofLang::EnAu && let Some(gb) = crate::scowl::dict(ProofLang::EnGb) { diff --git a/crates/proof/src/scowl.rs b/crates/proof/src/scowl.rs index e9f41b9e..5e84ea48 100644 --- a/crates/proof/src/scowl.rs +++ b/crates/proof/src/scowl.rs @@ -8,7 +8,9 @@ //! en-NZ: New Zealand English keeps `programme` and has few words of its own in SCOWL's en-AU //! data, so it stays on en-GB. Its suggestions //! come from its own dictionary, and a word is right when its own dictionary knows it: the other -//! dialects' dictionaries are never asked (en-US would take `colourblind` from en-GB's). SCOWL +//! dialects' dictionaries are never asked (en-US would take `colourblind` from en-GB's), except +//! that en-AU's list keeps one spelling per word, so Australian English also takes en-GB's words +//! (`enquiry`; its suggestions still come from en-AU). SCOWL //! picks one spelling per dialect (en-GB `-ise` has no `organize`, en-US no `judgement`), and Word //! accepts more, so British English also takes the `-ize` spellings of its `-ise` words that //! en-US or en-CA spells so, and `en-variants.txt` widens each dialect by its pairs From 3aca2642c5a3cb039a70a3819eb0e30fabf7ba6a Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Mon, 2 Nov 2026 23:56:09 +0000 Subject: [PATCH 37/40] =?UTF-8?q?craft-proof=20tr=20=E2=80=94=20the=20doub?= =?UTF-8?q?le-space=20rule=20uses=20space=5Fruns;=20the=20all-languages=20?= =?UTF-8?q?test=20adds=20Tr,=20EnAu,=20EnCa=20and=20pt-PT=20(#496=20review?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- crates/proof/src/lib.rs | 7 ++++++- crates/proof/src/tr/grammar.rs | 26 ++++++++++---------------- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/crates/proof/src/lib.rs b/crates/proof/src/lib.rs index 62df1b7e..2e804097 100644 --- a/crates/proof/src/lib.rs +++ b/crates/proof/src/lib.rs @@ -2627,7 +2627,12 @@ mod tests { ProofLang::Id, ProofLang::Pl, ProofLang::Ru, - ]; + ProofLang::Tr, + ProofLang::EnAu, + ProofLang::EnCa, + ] + .into_iter() + .chain(PackLocale::new("pt-PT").map(ProofLang::Pack)); for lang in langs { for n in 2..=5 { let text = format!("Aaa{}bbb", " ".repeat(n)); diff --git a/crates/proof/src/tr/grammar.rs b/crates/proof/src/tr/grammar.rs index 81ba93c2..9daf5bfc 100644 --- a/crates/proof/src/tr/grammar.rs +++ b/crates/proof/src/tr/grammar.rs @@ -209,23 +209,17 @@ pub fn check_with(text: &str, capital_first: bool) -> Vec { }); } } - // Double space - if c == ' ' && i + 1 < chars.len() && chars[i + 1].1 == ' ' && (i == 0 || chars[i - 1].1 != ' ') { - let mut end = idx + 1; - while end < text.len() && text.as_bytes().get(end) == Some(&b' ') { - end += 1; - } - v.push(Issue { - start: idx, - end, - kind: IssueKind::Grammar, - message: "Fazladan boşluk".into(), - suggestions: vec![" ".into()], - lang: ProofLang::Tr, - }); - } } - + for run in crate::space_runs(text) { + v.push(Issue { + start: run.start, + end: run.end, + kind: IssueKind::Grammar, + message: "Fazladan boşluk".into(), + suggestions: vec![" ".into()], + lang: ProofLang::Tr, + }); + } // 3. Comma before or after conjunctions (`ali, ve veli` or `ve, veli`) for (i, &(w_start, w_end)) in ws.iter().enumerate() { let word = match text.get(w_start..w_end) { From 14cf444fcfcf34ad94e9a2134139b9233e4d5e85 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Mon, 2 Nov 2026 23:56:09 +0000 Subject: [PATCH 38/40] =?UTF-8?q?craft-proof=20scowl=20=E2=80=94=20load=5F?= =?UTF-8?q?error(en-AU)=20also=20reports=20a=20failed=20en-GB=20list,=20wh?= =?UTF-8?q?ich=20en-AU=20reads=20too=20(#496=20review)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- crates/proof/src/scowl.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/proof/src/scowl.rs b/crates/proof/src/scowl.rs index 5e84ea48..cb101cd8 100644 --- a/crates/proof/src/scowl.rs +++ b/crates/proof/src/scowl.rs @@ -64,7 +64,9 @@ pub(crate) fn is_british(lang: ProofLang) -> bool { /// Why `lang`'s embedded SCOWL dictionary failed to load, once something tried to load it. pub(crate) fn load_error(lang: ProofLang) -> Option { let (i, ..) = source(lang); - DICTS.get(i)?.get()?.as_ref().err().map(|e| format!("embedded SCOWL dictionary: {e}")) + let own = DICTS.get(i).and_then(|d| d.get()).and_then(|r| r.as_ref().err()).map(|e| format!("embedded SCOWL dictionary: {e}")); + // Australian English also reads the en-GB list (`crate::english`): its failure is reported too. + own.or_else(|| if lang == ProofLang::EnAu { load_error(ProofLang::EnGb).map(|e| format!("{e} (en-GB, also read for en-AU)")) } else { None }) } /// The SCOWL copyright and licence notices (the upstream README, with the Ispell BSD licence of From 3012d73c8261eb497aba38be041e7b55f607b465 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Tue, 3 Nov 2026 00:47:28 +0000 Subject: [PATCH 39/40] proof: AffixDict::build reads the stems' flags in its own function Co-Authored-By: Claude Opus 5.5 --- crates/proof/src/affix.rs | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/crates/proof/src/affix.rs b/crates/proof/src/affix.rs index 20ce6fda..10f89180 100644 --- a/crates/proof/src/affix.rs +++ b/crates/proof/src/affix.rs @@ -1176,6 +1176,9 @@ fn parse_aff(aff: &str, english: bool) -> Result { Ok(r) } +/// A dictionary's stems as `(word, start, end, is a word)` ranges of one buffer of flag keys. +type StemFlags = (Vec<(String, usize, usize, bool)>, Vec); + impl AffixDict { /// Build from Hunspell `.aff` and `.dic` text (decoded to UTF-8; the `.dic` count line and /// morphological fields are ignored). `comments` (provenance, licence) are kept with the data. @@ -1246,10 +1249,9 @@ impl AffixDict { AffixDict::build(rules, stems, comments.iter().map(ToString::to_string).collect(), MAX_FLAGS_PER_ENTRY, max_flags) } - /// Build from parsed rules and stems whose flag fields carry at most `entry_cap` flags each - /// ([`MAX_FLAGS_PER_ENTRY`] for `.dic` entries, [`MAX_FLAGS_PER_STEM`] for compact data). - /// The stems' flags may number `max_flags` in all ([`MAX_STEM_FLAGS`]), counted as they are read. - fn build(rules: Rules, stems: Vec<(String, String)>, comments: Vec, entry_cap: usize, max_flags: usize) -> Result { + /// Read each stem's flag field ([`AffixDict::build`] explains the rules) into ranges of one key + /// buffer, within the `max_flags` budget: the stems as `(word, start, end, is a word)`, and the keys. + fn read_stem_flags(rules: &Rules, stems: Vec<(String, String)>, entry_cap: usize, max_flags: usize) -> Result { let mode = rules.mode; // A `.dic` entry's malformed flag items read as Hunspell reads them ([`FlagMode`]); an // entry with more than MAX_FLAGS_PER_STEM flags, or compact data whose flags don't read, @@ -1295,10 +1297,19 @@ impl AffixDict { read_stems.push((w, a, b, word)); } } - let mut stems = read_stems; if skipped > 0 { log::debug!("affix dictionary: {skipped} entries with unreadable or too many flags skipped"); } + Ok((read_stems, stem_keys)) + } + + /// Build from parsed rules and stems whose flag fields carry at most `entry_cap` flags each + /// ([`MAX_FLAGS_PER_ENTRY`] for `.dic` entries, [`MAX_FLAGS_PER_STEM`] for compact data). + /// The stems' flags may number `max_flags` in all ([`MAX_STEM_FLAGS`]), counted as they are read. + fn build(rules: Rules, stems: Vec<(String, String)>, comments: Vec, entry_cap: usize, max_flags: usize) -> Result { + let mode = rules.mode; + let exclusion_keys = [rules.forbidden, rules.no_suggest, rules.substandard]; + let (mut stems, stem_keys) = AffixDict::read_stem_flags(&rules, stems, entry_cap, max_flags)?; let conts = |rs: &[RawRule]| rs.iter().map(|r| mode.parse(&r.4)).collect::, _>>(); let (pfx_cont, sfx_cont) = (conts(&rules.prefixes)?, conts(&rules.suffixes)?); // Flags: every one a rule, a stem or a directive uses gets an id, in key order. From 22144c823a79258d5ff1f5a43ac6b85958d0eceb Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Tue, 3 Nov 2026 00:47:39 +0000 Subject: [PATCH 40/40] proof: licence warnings report the actual error; machine-independent English test; SuggestSteps drop note Co-Authored-By: Claude Opus 5.5 --- crates/proof/build.rs | 8 ++++++-- crates/proof/src/affix.rs | 2 ++ crates/proof/src/lib.rs | 3 ++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/crates/proof/build.rs b/crates/proof/build.rs index 2922baa0..fd34c22d 100644 --- a/crates/proof/build.rs +++ b/crates/proof/build.rs @@ -155,10 +155,14 @@ fn main() { println!("cargo::rerun-if-changed={}", f.display()); // `include_str!` fails the whole build on a file that isn't UTF-8 (a Latin-1 // licence text) and can't name a path that isn't either: leave such a file out. - let (Some(path), true) = (f.to_str(), std::fs::read_to_string(&f).is_ok()) else { - println!("cargo::{level}=CRAFT_I18N_DIR: {} is not UTF-8; its licence text is not embedded", f.display()); + let Some(path) = f.to_str() else { + println!("cargo::{level}=CRAFT_I18N_DIR: the path {} is not UTF-8; its licence text is not embedded", f.display()); continue; }; + if let Err(e) = std::fs::read_to_string(&f) { + println!("cargo::{level}=CRAFT_I18N_DIR: {path} can't be read as text ({e}); its licence text is not embedded"); + continue; + } let title = format!("{locale}: {}", f.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default()); let _ = write!(licences, "({title:?}, include_str!({path:?})), "); } diff --git a/crates/proof/src/affix.rs b/crates/proof/src/affix.rs index 10f89180..0df68665 100644 --- a/crates/proof/src/affix.rs +++ b/crates/proof/src/affix.rs @@ -926,6 +926,8 @@ impl SuggestSteps { } } +// Dropping restores the outer budget as it was: the steps a nested call took are not charged to +// it. Harmless while suggestion calls don't nest; a nested call would have to charge the outer one. impl Drop for SuggestSteps { fn drop(&mut self) { SUGGEST_STEPS.with(|s| s.set(self.0)); diff --git a/crates/proof/src/lib.rs b/crates/proof/src/lib.rs index 2e804097..3df6cd01 100644 --- a/crates/proof/src/lib.rs +++ b/crates/proof/src/lib.rs @@ -1340,7 +1340,8 @@ mod tests { } // Text in a language without proofing is never checked with English. assert!(check_spelling_in("hous teh zxqv", ProofLang::Other).is_empty()); - assert!(check_text("hous teh", &[(0..8, ProofLang::De)]).is_empty()); + // `Other` has no dictionary on any machine (German may: a system `de_DE` Hunspell). + assert!(check_text("hous teh", &[(0..8, ProofLang::Other)]).is_empty()); assert_eq!(english_source(ProofLang::Pl), None); drop_english_files(&dir, english); assert_ne!(english_source(ProofLang::En), Some(EnglishSource::Hunspell(dir.join("en_US.dic"))));