From 51090dd6e5e6ea1a27fc11f3525323839ae4ff28 Mon Sep 17 00:00:00 2001 From: dotCooCoo Date: Sat, 24 Oct 2026 01:47:38 -0700 Subject: [PATCH 01/16] Proofing: web addresses after a no-break space are not marked as misspelled The spelling check found the start of the white-space-delimited token around a word one byte after the last white-space character. After a no-break space (two bytes in UTF-8) or an ideographic space (three bytes) that offset is inside the character, so the token could not be read and was never classified as a web or e-mail address. With Ignore web and file addresses on, the words of such an address were still marked as misspelled. The token now starts after the whole white-space character. A proofing test covers a web address, a www. address and an e-mail address after the two spaces, with the option on and off. --- crates/proof/src/lib.rs | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/crates/proof/src/lib.rs b/crates/proof/src/lib.rs index de596782..65190cab 100644 --- a/crates/proof/src/lib.rs +++ b/crates/proof/src/lib.rs @@ -693,7 +693,9 @@ fn spelling_issues(text: &str, lang: impl Fn(usize) -> ProofLang, opts: &Options /// The whitespace-delimited token enclosing the word at `a..b` of `text`: its byte range, and /// whether it looks like a web or e-mail address. fn enclosing_token(text: &str, a: usize, b: usize) -> (usize, usize, bool) { - let ts = text.get(..a).and_then(|t| t.rfind(char::is_whitespace)).map(|i| i + 1).unwrap_or(0); + // The token starts after the last white-space character, whatever its width (a no-break + // space is two bytes and an ideographic space is three). + let ts = text.get(..a).and_then(|t| t.char_indices().rev().find(|(_, c)| c.is_whitespace())).map(|(i, c)| i + c.len_utf8()).unwrap_or(0); let te = text.get(b..).and_then(|t| t.find(char::is_whitespace)).map(|i| b + i).unwrap_or(text.len()); let token = text.get(ts..te).unwrap_or(""); (ts, te, token.contains("://") || token.contains('@') || token.starts_with("www.")) @@ -1431,6 +1433,18 @@ mod tests { assert_eq!(bad.last(), Some(&"wugwug"), "{bad:?}"); } + #[test] + fn address_after_a_wide_space_is_an_internet_token() { + // A web or e-mail address after a no-break space or an ideographic space is skipped, + // as it is after a plain space. + let t = "see\u{a0}https://xyzq.example/blorp and\u{3000}www.qwirpy.example or\u{a0}zxq@mail.example wugwug"; + let bad: Vec<&str> = check_spelling(t).iter().map(|i| &t[i.start..i.end]).collect(); + assert_eq!(bad, vec!["wugwug"]); + let opts = Options { ignore_internet: false, ..Options::default() }; + let bad: Vec<&str> = check_spelling_with(t, &opts).iter().map(|i| &t[i.start..i.end]).collect(); + assert!(["xyzq", "blorp", "qwirpy", "zxq", "wugwug"].iter().all(|w| bad.contains(w)), "{bad:?}"); + } + #[test] fn spelling_scales_over_one_long_token() { // #1335: thousands of punctuation-separated words inside a single From 330502fd0b0bbc07d938b77d8cf5349afcbd7272 Mon Sep 17 00:00:00 2001 From: VoiceLessQ Date: Sun, 25 Oct 2026 01:19:28 -0200 Subject: [PATCH 02/16] Method paths for 18 closures that only call a method (SonarCloud S1612) SonarCloud's analysis of main at b4da59f0c lists 23 closures such as `|v| v.as_slice()` that hand their argument straight to one method. This passes the method itself: `Vec::as_slice`, `str::to_ascii_lowercase`, `f64::total_cmp` and the like, all forms the repo already uses elsewhere. Behaviour is unchanged. Five stay. The four in crates/proof/src/es/mod.rs sit in code that storytold/wordcraft#4191 replaces, and its replacement already uses `PoisonError::into_inner`. In crates/engine/src/io/safe_save.rs the path would be `::is::`, which reads worse than `|inner| inner.is::()`. --- crates/proof/src/affix.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/proof/src/affix.rs b/crates/proof/src/affix.rs index 1839a103..c2580bca 100644 --- a/crates/proof/src/affix.rs +++ b/crates/proof/src/affix.rs @@ -419,7 +419,7 @@ impl AffixDict { } stems.push((word, flags)); } - AffixDict::build(rules, stems, comments.iter().map(|s| s.to_string()).collect()) + AffixDict::build(rules, stems, comments.iter().map(ToString::to_string).collect()) } fn build(rules: Rules, mut stems: Vec<(String, String)>, comments: Vec) -> Result { From cdd1b413e983a5f436b1339041959e795ff80c2a Mon Sep 17 00:00:00 2001 From: VoiceLessQ Date: Sun, 25 Oct 2026 01:01:41 -0200 Subject: [PATCH 03/16] proof: remove the unused Patterns::from_parts It came in with the first proofing commit (bcbe09df8) and has never had a caller; `#[allow(dead_code)]` kept the warning quiet. Because it is `pub(crate)`, nothing outside wordcraft-proof can reach it. --- crates/proof/src/patterns.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/crates/proof/src/patterns.rs b/crates/proof/src/patterns.rs index a91235e4..cb5855d4 100644 --- a/crates/proof/src/patterns.rs +++ b/crates/proof/src/patterns.rs @@ -217,11 +217,6 @@ impl Patterns { self.exceptions.len() } - #[allow(dead_code)] - pub(crate) fn from_parts(alpha: Alphabet, map: FxMap>, max_len: usize) -> Patterns { - Patterns { alpha, map, max_len, exceptions: HashMap::new() } - } - /// Make `letters` part of the alphabet even if no pattern mentions them. pub fn add_letters(&mut self, letters: impl IntoIterator) { for c in letters { From cb0f20361d99f87747a73b7dd0ea0e114c95cc1f Mon Sep 17 00:00:00 2001 From: skylar Date: Sun, 25 Oct 2026 15:34:24 -0600 Subject: [PATCH 04/16] chore(deps): refresh safe dependency bumps serde_json 1.0.154, proptest 1.12, base64 0.23, arboard 3.7, open 5.4.5, winresource 0.1.34, pollster 1.0, miniz_oxide 0.9, sha2 0.11.1, libc 0.2.191. --- crates/proof/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/proof/Cargo.toml b/crates/proof/Cargo.toml index 234684a9..4c1a5b11 100644 --- a/crates/proof/Cargo.toml +++ b/crates/proof/Cargo.toml @@ -10,7 +10,7 @@ workspace = true [dependencies] log = { workspace = true } -miniz_oxide = "0.8" +miniz_oxide = "0.9" unicode-normalization = "0.1" [dev-dependencies] From 9d2444375406c3a2f657d721c11838565c6848a5 Mon Sep 17 00:00:00 2001 From: skylar Date: Sun, 25 Oct 2026 16:53:12 -0600 Subject: [PATCH 05/16] fix(deps): keep miniz_oxide at 0.8 --- crates/proof/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/proof/Cargo.toml b/crates/proof/Cargo.toml index 4c1a5b11..234684a9 100644 --- a/crates/proof/Cargo.toml +++ b/crates/proof/Cargo.toml @@ -10,7 +10,7 @@ workspace = true [dependencies] log = { workspace = true } -miniz_oxide = "0.9" +miniz_oxide = "0.8" unicode-normalization = "0.1" [dev-dependencies] From 8dd9441fa9d83c537db6ced851b11d8eeac71746 Mon Sep 17 00:00:00 2001 From: skylar Date: Sun, 25 Oct 2026 16:57:11 -0600 Subject: [PATCH 06/16] chore(deps): bump miniz_oxide to 0.9 --- crates/proof/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/proof/Cargo.toml b/crates/proof/Cargo.toml index 234684a9..4c1a5b11 100644 --- a/crates/proof/Cargo.toml +++ b/crates/proof/Cargo.toml @@ -10,7 +10,7 @@ workspace = true [dependencies] log = { workspace = true } -miniz_oxide = "0.8" +miniz_oxide = "0.9" unicode-normalization = "0.1" [dev-dependencies] From 5f7ccde9fb24eb7e430d9d37b88d64b7245d529d Mon Sep 17 00:00:00 2001 From: skylar Date: Sun, 25 Oct 2026 16:57:14 -0600 Subject: [PATCH 07/16] fix(proof): keep inflate error output out of messages --- crates/proof/src/affix.rs | 4 ++-- crates/proof/src/dict.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/proof/src/affix.rs b/crates/proof/src/affix.rs index c2580bca..7fb7d968 100644 --- a/crates/proof/src/affix.rs +++ b/crates/proof/src/affix.rs @@ -877,8 +877,8 @@ impl AffixDict { /// Read data written by [`AffixDict::to_bytes`]. pub fn from_bytes(data: &[u8]) -> Result { - let raw = - miniz_oxide::inflate::decompress_to_vec_with_limit(data, 64 << 20).map_err(|e| format!("affix dictionary: inflate failed: {e:?}"))?; + let raw = miniz_oxide::inflate::decompress_to_vec_with_limit(data, 64 << 20) + .map_err(|e| format!("affix dictionary: inflate failed: {:?}", e.status))?; let words_at = raw.windows(7).position(|w| w == b"\nWORDS\n").ok_or("affix dictionary: no WORDS section")?; let head = std::str::from_utf8(raw.get(..words_at).unwrap_or_default()).map_err(|_| "affix dictionary: bad utf-8 in the rules")?; let mut lines = head.lines(); diff --git a/crates/proof/src/dict.rs b/crates/proof/src/dict.rs index 83a54d44..789fb6a8 100644 --- a/crates/proof/src/dict.rs +++ b/crates/proof/src/dict.rs @@ -78,7 +78,7 @@ impl Dictionary { /// Deserialize data written by [`Dictionary::to_bytes`]. pub fn from_bytes(data: &[u8]) -> Result { - let raw = miniz_oxide::inflate::decompress_to_vec(data).map_err(|e| format!("hyphenation dictionary: inflate failed: {e:?}"))?; + let raw = miniz_oxide::inflate::decompress_to_vec(data).map_err(|e| format!("hyphenation dictionary: inflate failed: {:?}", e.status))?; let body = raw.strip_prefix(MAGIC).ok_or("hyphenation dictionary: bad header")?; let mut d = Dictionary { words: String::with_capacity(body.len()), offs: vec![0], masks: Vec::new() }; let mut cur: Vec = Vec::new(); From 8135718dac7109c9e4caf45022c91c83c321274a Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 15:12:03 +0100 Subject: [PATCH 08/16] =?UTF-8?q?storytold/wordcraft#3613:=20affix=20rules?= =?UTF-8?q?=20=E2=80=94=20checker=20follows=20merged=20flag=20masks,=20for?= =?UTF-8?q?ms=20bounded,=20suffix=20index=20hashed?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Part of storytold/wordcraft#3613 (audit findings 2 and 6). - Checker: `suffixed` and `form_has_flag` OR the flags of a group's matching rules into one mask and recurse once, not once per rule. Same answer (the walk is a disjunction over flags); 16k self-continuing empty-append rules: 6 s -> <1 ms per check (debug). - Finding 2: `forms` expands each (form, continuation) pair once, walks continuation rules through a per-flag rule list, and stops at MAX_FORMS (4096) forms / MAX_FORMS_WORK rule visits, so a dictionary of self-continuing rules can't allocate R^2 strings per stem. - Finding 6: the suffix index finds a rule's (append, strip) group by hash, not by scanning the append's groups: building with many strips per append is linear. Not done here (rest of storytold/wordcraft#3613): a work budget for distinct-strip chains in the checker, rule caps in parse_aff, proofing off the UI thread. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/proof/src/affix.rs | 164 +++++++++++++++++++++++++++++++++----- 1 file changed, 144 insertions(+), 20 deletions(-) diff --git a/crates/proof/src/affix.rs b/crates/proof/src/affix.rs index 7fb7d968..fefc7f6a 100644 --- a/crates/proof/src/affix.rs +++ b/crates/proof/src/affix.rs @@ -120,6 +120,11 @@ fn parse_cond(text: &str) -> Result, String> { const MAX_CHAIN: u8 = 2; /// How many suffixes may sit under the outermost one: the depth `form_has_flag` walks. const INNER_CHAIN: u8 = MAX_CHAIN - 1; +/// Most forms [`AffixDict::forms`] lists for one stem (real stems have a few hundred at most). +pub const MAX_FORMS: usize = 4096; +/// Most rules [`AffixDict::forms`] tries along continuation chains and prefix–suffix cross +/// products for one stem: real dictionaries need a few thousand, hostile ones are cut off. +const MAX_FORMS_WORK: usize = 1 << 18; /// One prefix or suffix rule. #[derive(Clone, Debug)] @@ -182,6 +187,8 @@ pub struct AffixDict { suffixes: Vec, /// Appended string → groups of suffix rules by stripped string. sfx_index: FxMap, Vec>, + /// Per flag bit: the suffix rules of that flag, in rule order. + sfx_by_flag: Vec>, /// Longest appended suffix, in chars. max_sfx_chars: usize, try_chars: String, @@ -484,14 +491,27 @@ impl AffixDict { /// Build the suffix index and the stem hash table. fn index(&mut self) { let mut idx: FxMap, Vec> = FxMap::default(); + // Each (appended, stripped) pair's group, found by hash rather than by scanning the + // appended string's groups: a dictionary with many strips per suffix stays linear. + let mut group_of: FxMap<(&str, &str), usize> = FxMap::default(); + let mut by_flag: Vec> = vec![Vec::new(); 128]; for (i, r) in self.suffixes.iter().enumerate() { let groups = idx.entry(r.add.clone()).or_default(); let Ok(i) = u32::try_from(i) else { break }; - match groups.iter_mut().find(|g| g.strip == r.strip) { + match group_of.get(&(&*r.add, &*r.strip)).and_then(|&g| groups.get_mut(g)) { Some(g) => g.rules.push(i), - None => groups.push(SfxGroup { strip: r.strip.clone(), rules: vec![i] }), + None => { + group_of.insert((&r.add, &r.strip), groups.len()); + groups.push(SfxGroup { strip: r.strip.clone(), rules: vec![i] }); + } + } + if r.bit != 0 + && let Some(list) = by_flag.get_mut(r.bit.trailing_zeros() as usize) + { + list.push(i); } } + self.sfx_by_flag = by_flag; self.max_sfx_chars = self.suffixes.iter().map(|r| r.add.chars().count()).max().unwrap_or(0); self.sfx_index = idx; let n = self.len(); @@ -596,10 +616,14 @@ impl AffixDict { None } - /// Does `w` carry flag `bit` — as a stem, or as the form of a suffix rule whose - /// continuation classes include `bit` (chained suffixation, depth-bounded)? - fn form_has_flag(&self, w: &str, bit: u128, depth: u8, excluded: u128) -> bool { - if self.stem_flags_excluding(w, excluded).is_some_and(|f| f & bit != 0) { + /// Does `w` carry any flag of `flags` — as a stem, or as the form of a suffix rule whose + /// continuation classes include one (chained suffixation, depth-bounded)? + /// + /// The rules of a group that could lead to `w` are merged into one mask of their flags and + /// followed once, not once per rule: the answer is the same, but thousands of rules of one + /// flag cost one walk, not one each (#3613). + fn form_has_flag(&self, w: &str, flags: u128, depth: u8, excluded: u128) -> bool { + if self.stem_flags_excluding(w, excluded).is_some_and(|f| f & flags != 0) { return true; } if depth == 0 { @@ -614,10 +638,14 @@ impl AffixDict { root.clear(); root.push_str(base); root.push_str(&g.strip); - for r in g.rules.iter().filter_map(|&i| self.suffixes.get(i as usize)) { - if r.cont & bit != 0 && r.cond_matches(&root, true) && self.form_has_flag(&root, r.bit, depth - 1, excluded) { - return true; - } + let inner = g + .rules + .iter() + .filter_map(|&i| self.suffixes.get(i as usize)) + .filter(|r| r.cont & flags != 0 && r.cond_matches(&root, true)) + .fold(0u128, |m, r| m | r.bit); + if inner != 0 && self.form_has_flag(&root, inner, depth - 1, excluded) { + return true; } } } @@ -690,9 +718,13 @@ impl AffixDict { return true; } // Continuation classes: `root` need not be a stem carrying this rule's flag — - // it may be a form that gained the flag from another rule's continuation. - if need == 0 && rules().any(|r| r.cond_matches(&root, true) && self.form_has_flag(&root, r.bit, INNER_CHAIN, excluded)) { - return true; + // it may be a form that gained the flag from another rule's continuation. The + // matching rules' flags are looked for together, in one walk. + if need == 0 { + let flags = rules().filter(|r| r.cond_matches(&root, true)).fold(0u128, |m, r| m | r.bit); + if flags != 0 && self.form_has_flag(&root, flags, INNER_CHAIN, excluded) { + return true; + } } } } @@ -725,13 +757,19 @@ impl AffixDict { } } - /// Every form of the stem `stem` (itself included), unsorted, possibly with repeats. + /// Every form of the stem `stem` (itself included), unsorted, possibly with repeats; at most + /// [`MAX_FORMS`], and at most [`MAX_FORMS_WORK`] rules tried along the continuation chains and + /// cross products, so a dictionary with thousands of self-continuing rules can't make one stem + /// cost millions of strings (#3613). pub fn forms(&self, stem: &str) -> Vec { let Some(f) = self.stem_flags(stem) else { return Vec::new() }; let mut out = vec![stem.to_string()]; let mut cross_sfx = Vec::new(); - // Suffix forms queued with the continuation classes they carry, for chained expansion. + // Suffix forms queued with the continuation classes they carry, for chained expansion; + // each (form, classes) pair is expanded once. let mut chain: Vec<(String, u128)> = Vec::new(); + let mut queued: std::collections::HashSet<(String, u128)> = std::collections::HashSet::new(); + let mut work = 0usize; for r in &self.suffixes { if f & r.bit == 0 || !r.cond_matches(stem, true) { continue; @@ -741,29 +779,39 @@ impl AffixDict { if r.cross { cross_sfx.push(form.clone()); } - if r.cont != 0 { + if r.cont != 0 && queued.insert((form.clone(), r.cont)) { chain.push((form.clone(), r.cont)); } out.push(form); + if out.len() >= MAX_FORMS { + return out; + } } // Continuation classes chain suffixes (`workable/B` → `workables`), up to [`MAX_CHAIN`] // suffixes in all — the same bound the checker walks. - for _ in 0..INNER_CHAIN { + 'chain: for _ in 0..INNER_CHAIN { let level = std::mem::take(&mut chain); if level.is_empty() { break; } for (form, flags) in level { - for r in &self.suffixes { - if flags & r.bit == 0 || !r.cond_matches(&form, true) { + for r in self.suffixes_with(flags).into_iter().filter_map(|i| self.suffixes.get(i as usize)) { + work += 1; + if work > MAX_FORMS_WORK { + break 'chain; + } + if !r.cond_matches(&form, true) { continue; } let Some(base) = form.strip_suffix(&*r.strip) else { continue }; let next = format!("{base}{}", r.add); - if r.cont != 0 { + if r.cont != 0 && queued.insert((next.clone(), r.cont)) { chain.push((next.clone(), r.cont)); } out.push(next); + if out.len() >= MAX_FORMS { + return out; + } } } } @@ -775,15 +823,38 @@ impl AffixDict { out.push(format!("{}{rest}", r.add)); if r.cross { for s in &cross_sfx { + work += 1; + if out.len() >= MAX_FORMS || work > MAX_FORMS_WORK { + return out; + } if let Some(rest) = s.strip_prefix(&*r.strip) { out.push(format!("{}{rest}", r.add)); } } } + if out.len() >= MAX_FORMS { + return out; + } } out } + /// The suffix rules carrying any flag of `mask`, in rule order. + fn suffixes_with(&self, mask: u128) -> Vec { + let mut rules: Vec = Vec::new(); + let mut lists = 0; + for (bit, list) in self.sfx_by_flag.iter().enumerate() { + if bit < 128 && mask & (1u128 << bit) != 0 && !list.is_empty() { + rules.extend_from_slice(list); + lists += 1; + } + } + if lists > 1 { + rules.sort_unstable(); + } + rules + } + /// The `TRY` letters, most frequent first (suggestions try these). pub fn try_chars(&self) -> &str { &self.try_chars @@ -1216,6 +1287,59 @@ pub(crate) mod tests { } } + #[test] + fn continuation_classes_merge_the_flags_of_a_group() { + // `s` under B and under C share one group (same append, same strip): a form carrying + // either flag takes it, and the stem carrying neither doesn't. + let aff = "SET UTF-8\nSFX A N 1\nSFX A 0 able/C .\nSFX D N 1\nSFX D 0 er/B .\nSFX B N 1\nSFX B 0 s .\nSFX C N 1\nSFX C 0 s [e]\n"; + let d = AffixDict::from_hunspell(aff, "2\nwork/A\nplay/D\n", &[]).unwrap(); + assert!(d.contains("workables") && d.contains("players")); + assert!(!d.contains("works") && !d.contains("plays") && !d.contains("workabless")); + assert!(d.forms("work").iter().any(|f| f == "workables") && d.forms("play").iter().any(|f| f == "players")); + } + + /// `n` suffix rules of flag A on one stem, `rule(i)` giving each rule's strip, append and condition. + fn many_rules(n: usize, rule: impl Fn(usize) -> String) -> AffixDict { + let mut aff = format!("SET UTF-8\nSFX A Y {n}\n"); + for i in 0..n { + aff.push_str(&format!("SFX A {}\n", rule(i))); + } + AffixDict::from_hunspell(&aff, "1\nkot/A\n", &[]).unwrap() + } + + #[test] + fn self_continuing_rules_check_in_linear_time() { + // Thousands of empty-append rules that continue with their own flag: each check used to + // follow every rule from every rule, millions of steps a word (#3613). + let d = many_rules(16_000, |_| "0 0/A .".into()); + let t = std::time::Instant::now(); + assert!(d.check("kot") && !d.check("pies") && !d.check("kotx")); + assert!(t.elapsed().as_secs_f64() < 2.0, "{:?}", t.elapsed()); + } + + #[test] + fn forms_of_self_continuing_rules_are_bounded() { + // Every rule's form continues with every rule: millions of strings for one stem (#3613). + let same = many_rules(3000, |_| "0 0/A .".into()); + let distinct = many_rules(3000, |i| format!("0 x{i}/A .")); + let t = std::time::Instant::now(); + let (a, b) = (same.forms("kot"), distinct.forms("kot")); + assert!(t.elapsed().as_secs_f64() < 2.0, "{:?}", t.elapsed()); + assert!(a.len() <= MAX_FORMS && b.len() <= MAX_FORMS, "{} {}", a.len(), b.len()); + assert!(b.iter().any(|f| f == "kotx0x1"), "chained forms are still listed"); + } + + #[test] + fn many_strips_per_suffix_index_in_linear_time() { + // One append with thousands of strips: the index used to scan the append's groups for + // each rule. + let t = std::time::Instant::now(); + let d = many_rules(30_000, |i| format!("k{i} y .")); + assert!(t.elapsed().as_secs_f64() < 2.0, "{:?}", t.elapsed()); + assert_eq!(d.sfx_index.get("y").map(Vec::len), Some(30_000)); + assert!(!d.contains("koty")); + } + #[test] fn compact_format_round_trips_continuations_and_forbidden() { let aff = "SET UTF-8\nFLAG UTF-8\nFORBIDDENWORD ý\nSFX A N 1\nSFX A 0 able/B .\nSFX B N 1\nSFX B 0 s .\n"; From f0561b7fb0a6f36fc3ddd8cf80019ee475a4f44b Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 15:13:52 +0100 Subject: [PATCH 09/16] =?UTF-8?q?storytold/wordcraft#3661:=20bounded=20dic?= =?UTF-8?q?tionary=20reads=20=E2=80=94=20hyphenation=20inflate=20limit,=20?= =?UTF-8?q?capped=20Hunspell=20and=20Turkish=20reads?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Audit finding 4, and the rest of storytold/wordcraft#3661. - dict.rs: `Dictionary::from_bytes` inflates with `decompress_to_vec_with_limit` (64 MiB; the bundled en-US one is ~1 MB) and caps the words it keeps (bytes and count): front coding lets a 2-byte line spell out a 255-byte word, so the inflated size alone doesn't bound storage. - hunspell.rs: one `read_capped` reader — regular files only, size checked on the open handle and the read itself bounded by `take(MAX_FILE + 1)`, so no TOCTOU between metadata and read and no endless read from a FIFO or device. - tr: the Turkish loader reads through `read_capped` and decodes with `hunspell::decode` (it read both files whole, with no cap at all). Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/proof/src/dict.rs | 48 +++++++++++++++++++++++++++++-- crates/proof/src/hunspell.rs | 56 ++++++++++++++++++++++++++++++------ crates/proof/src/tr/mod.rs | 23 +++++++-------- 3 files changed, 105 insertions(+), 22 deletions(-) diff --git a/crates/proof/src/dict.rs b/crates/proof/src/dict.rs index 789fb6a8..81eb0de1 100644 --- a/crates/proof/src/dict.rs +++ b/crates/proof/src/dict.rs @@ -14,6 +14,11 @@ pub struct Dictionary { } const MAGIC: &[u8] = b"DCHYPH1\n"; +/// Largest inflated dictionary, and most bytes of words kept from one (the bundled US English one +/// is about 1 MB). +const MAX_BYTES: usize = 64 << 20; +/// Most words read from one dictionary (the bundled one has about 160k). +const MAX_ENTRIES: usize = 4_000_000; impl Dictionary { /// Build from `(lowercase word, break mask)` pairs (sorted and de-duplicated here; the first @@ -76,9 +81,16 @@ impl Dictionary { miniz_oxide::deflate::compress_to_vec(&raw, 10) } - /// Deserialize data written by [`Dictionary::to_bytes`]. + /// Deserialize data written by [`Dictionary::to_bytes`]. Data that inflates past + /// [`MAX_BYTES`], or holds more than [`MAX_ENTRIES`] words or [`MAX_BYTES`] of them, is refused. pub fn from_bytes(data: &[u8]) -> Result { - let raw = miniz_oxide::inflate::decompress_to_vec(data).map_err(|e| format!("hyphenation dictionary: inflate failed: {:?}", e.status))?; + Dictionary::from_bytes_within(data, MAX_BYTES, MAX_ENTRIES) + } + + /// [`Dictionary::from_bytes`] with the limits given. + fn from_bytes_within(data: &[u8], max_bytes: usize, max_entries: usize) -> Result { + let raw = miniz_oxide::inflate::decompress_to_vec_with_limit(data, max_bytes) + .map_err(|e| format!("hyphenation dictionary: inflate failed: {:?}", e.status))?; let body = raw.strip_prefix(MAGIC).ok_or("hyphenation dictionary: bad header")?; let mut d = Dictionary { words: String::with_capacity(body.len()), offs: vec![0], masks: Vec::new() }; let mut cur: Vec = Vec::new(); @@ -94,6 +106,11 @@ impl Dictionary { i = end + 1; let s = std::str::from_utf8(&cur).map_err(|_| "hyphenation dictionary: bad utf-8")?; let (w, m) = parse_hyphenated(s); + // Front coding lets a short line repeat a long prefix: bound what is kept, not only + // what is read. + if d.masks.len() >= max_entries || d.words.len() + w.len() > max_bytes { + return Err("hyphenation dictionary: too large".into()); + } d.words.push_str(&w); d.offs.push(d.words.len() as u32); d.masks.push(m); @@ -147,4 +164,31 @@ mod tests { assert_eq!(back.get("naïveté").map(|m| hyphenated("naïveté", m)), Some("naïve-té".to_string())); assert!(back.get("dog").is_none()); } + + #[test] + fn oversized_data_is_refused() { + const LIMIT: usize = 1 << 16; + let within = |raw: &[u8]| Dictionary::from_bytes_within(&miniz_oxide::deflate::compress_to_vec(raw, 6), LIMIT, 1000); + // Data that inflates past the limit is refused before anything is decoded. + let mut raw = MAGIC.to_vec(); + raw.resize(LIMIT + 1, b'a'); + assert!(within(&raw).unwrap_err().contains("inflate failed")); + // Short lines that each repeat a long prefix: small to inflate, but the words they spell + // out would be ~100x larger. Kept storage is bounded too. + let mut raw = MAGIC.to_vec(); + raw.push(0); + raw.extend_from_slice(&[b'a'; 255]); + raw.push(b'\n'); + for _ in 0..(LIMIT / 255 + 2) { + raw.extend_from_slice(&[255, b'\n']); + } + assert!(raw.len() < LIMIT); + assert!(within(&raw).unwrap_err().contains("too large")); + // So is the number of words. + let words: Vec<(String, u64)> = (0..1001).map(|i| (format!("w{i}"), 0)).collect(); + let raw = miniz_oxide::inflate::decompress_to_vec(&Dictionary::from_entries(words).to_bytes()).unwrap(); + assert!(within(&raw).unwrap_err().contains("too large")); + // The real limits leave the bundled dictionary whole. + assert!(Dictionary::en_us().len() > 100_000); + } } diff --git a/crates/proof/src/hunspell.rs b/crates/proof/src/hunspell.rs index 0e4303be..a5b41214 100644 --- a/crates/proof/src/hunspell.rs +++ b/crates/proof/src/hunspell.rs @@ -140,16 +140,31 @@ pub fn find_in(dir: &Path, names: &[&str]) -> Option<(PathBuf, PathBuf)> { }) } +/// Read a dictionary file: a regular file of at most [`MAX_FILE`] bytes. The read itself is +/// bounded, so a file that grows after its size is checked (or a filesystem that reports a wrong +/// size) can't make it read more (#3661). +pub(crate) fn read_capped(p: &Path) -> Result, String> { + use std::io::Read; + let err = |e: std::io::Error| format!("{}: {e}", p.display()); + let f = std::fs::File::open(p).map_err(err)?; + let meta = f.metadata().map_err(err)?; + if !meta.is_file() { + return Err(format!("{}: not a regular file", p.display())); + } + if meta.len() > MAX_FILE { + return Err(format!("{}: too large ({} bytes)", p.display(), meta.len())); + } + let mut buf = Vec::new(); + f.take(MAX_FILE + 1).read_to_end(&mut buf).map_err(err)?; + if buf.len() as u64 > MAX_FILE { + return Err(format!("{}: too large", p.display())); + } + Ok(buf) +} + /// Read a Hunspell pair from disk (size-capped), decoded per the `.aff`'s `SET` line. pub fn load(aff: &Path, dic: &Path) -> Result { - let read = |p: &Path| -> Result, String> { - let len = std::fs::metadata(p).map_err(|e| format!("{}: {e}", p.display()))?.len(); - if len > MAX_FILE { - return Err(format!("{}: too large ({len} bytes)", p.display())); - } - std::fs::read(p).map_err(|e| format!("{}: {e}", p.display())) - }; - let (aff_text, dic_text) = decode(&read(aff)?, &read(dic)?)?; + let (aff_text, dic_text) = decode(&read_capped(aff)?, &read_capped(dic)?)?; let source = format!("Hunspell dictionary {}", dic.display()); AffixDict::from_hunspell(&aff_text, &dic_text, &[source.as_str()]) } @@ -283,4 +298,29 @@ mod tests { assert!(load(&dir.join("missing.aff"), &dic).is_err()); let _ = std::fs::remove_dir_all(&dir); } + + #[test] + fn reads_are_bounded_and_regular_files_only() { + let dir = std::env::temp_dir().join(format!("wordcraft-hunspell-cap-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let aff = dir.join("xx.aff"); + std::fs::write(&aff, "SET UTF-8\n").unwrap(); + // One byte over the cap (sparse: nothing is written) is refused without reading it. + let big = dir.join("big.dic"); + std::fs::File::create(&big).unwrap().set_len(MAX_FILE + 1).unwrap(); + let t = std::time::Instant::now(); + assert!(load(&aff, &big).unwrap_err().contains("too large")); + // A directory, and on Unix an endless device, are not dictionary files. + assert!(load(&aff, &dir).unwrap_err().contains("not a regular file")); + #[cfg(unix)] + assert!(load(&aff, Path::new("/dev/zero")).unwrap_err().contains("not a regular file")); + assert!(t.elapsed().as_secs_f64() < 1.0, "{:?}", t.elapsed()); + // Turkish reads through the same reader. + let (tr_aff, tr_dic) = (dir.join("tr_TR.aff"), dir.join("tr_TR.dic")); + std::fs::write(&tr_aff, "SET UTF-8\nSFX 1 Y 1\nSFX 1 0 lar .\n").unwrap(); + std::fs::write(&tr_dic, "1\nkitap/1\n").unwrap(); + assert!(crate::tr::load(&tr_aff, &tr_dic).unwrap().contains("kitaplar")); + assert!(crate::tr::load(&tr_aff, &big).unwrap_err().contains("too large")); + let _ = std::fs::remove_dir_all(&dir); + } } diff --git a/crates/proof/src/tr/mod.rs b/crates/proof/src/tr/mod.rs index bcab1546..25ee424c 100644 --- a/crates/proof/src/tr/mod.rs +++ b/crates/proof/src/tr/mod.rs @@ -44,18 +44,17 @@ fn load_installed() -> TrDict { log::info!("Turkish spelling: no Hunspell tr_TR dictionary installed"); return TrDict::default(); }; - let read = |p: &std::path::Path| std::fs::read_to_string(p); - match (read(&aff), read(&dic)) { - (Ok(aff_text), Ok(dic_text)) => crate::hunspell::note_load(NAMES[0], &dic, TrDict::from_hunspell(&aff_text, &dic_text)).unwrap_or_else(|e| { - log::warn!("Turkish spelling: {e}"); - TrDict::default() - }), - (a, d) => { - let reason = a.err().or(d.err()).map_or_else(String::new, |e| e.to_string()); - let _ = crate::hunspell::note_load::<()>(NAMES[0], &dic, Err(reason)); - TrDict::default() - } - } + crate::hunspell::note_load(NAMES[0], &dic, load(&aff, &dic)).unwrap_or_else(|e| { + log::warn!("Turkish spelling: {e}"); + TrDict::default() + }) +} + +/// Read a Turkish Hunspell pair from disk: size-capped and decoded per the `.aff`'s `SET` line, +/// like the other languages' dictionaries (#3661). +pub(crate) fn load(aff: &std::path::Path, dic: &std::path::Path) -> Result { + let (aff_text, dic_text) = crate::hunspell::decode(&crate::hunspell::read_capped(aff)?, &crate::hunspell::read_capped(dic)?)?; + TrDict::from_hunspell(&aff_text, &dic_text) } /// Use this Turkish dictionary from now on. From eb37158db4f44440780c2e447a10a8ec398908d7 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 15:15:27 +0100 Subject: [PATCH 10/16] storytold/wordcraft#3657: Russian hyphenation and hyphenated output linear in word length Audit findings 10 (storytold/wordcraft#3657) and 15 (storytold/wordcraft#3692). - ru::hyphen::points: the cuts come in increasing order, so a repeat can only be the last one; compare with `out.last()` instead of `out.contains` per syllable (O(V^2) on a long Cyrillic word, during layout). - hyphenate_word_in: walk the sorted break points alongside the chars instead of searching the point list for every char. Same output. Test: russian_hyphenation_and_hyphenated_output_are_linear (8x the letters costs ~8x; the old code measured 31x and 30x). Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/proof/src/hyphen.rs | 30 +++++++++++++++++++++++++++--- crates/proof/src/ru/hyphen.rs | 4 +++- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/crates/proof/src/hyphen.rs b/crates/proof/src/hyphen.rs index ed9f77f9..b203ff1a 100644 --- a/crates/proof/src/hyphen.rs +++ b/crates/proof/src/hyphen.rs @@ -320,10 +320,14 @@ pub fn hyphenate_word(word: &str, lim: &Limits) -> String { /// `word`, a word in `lang`, with `-` at every hyphenation point (tests and diagnostics). pub fn hyphenate_word_in(word: &str, lim: &Limits, lang: ProofLang) -> String { - let pts = hyphen_points_in(word, lim, lang); - let mut s = String::with_capacity(word.len() + pts.len()); + let mut pts = hyphen_points_in(word, lim, lang); + pts.sort_unstable(); + // Walk the points alongside the chars rather than searching them for each char. + let mut next = pts.into_iter().peekable(); + let mut s = String::with_capacity(word.len() + 8); for (i, ch) in word.chars().enumerate() { - if pts.contains(&i) { + if next.next_if_eq(&i).is_some() { + while next.next_if_eq(&i).is_some() {} s.push('-'); } s.push(ch); @@ -706,6 +710,26 @@ mod tests { } } + #[test] + fn russian_hyphenation_and_hyphenated_output_are_linear() { + // A Russian word breaks at every syllable: each break used to be checked against all the + // earlier ones (#3657), and the hyphenated output searched every break for each char. + let word = |k: usize| "мама".repeat(k); + assert_eq!(word_breaks_in(&word(2), ProofLang::Ru).0, [2, 4, 6]); + assert_eq!(hyphenate_word_in(&word(2), &Limits::default(), ProofLang::Ru), "ма-ма-ма-ма"); + // The breaks' rescan is a cheap one per break: it takes longer words to stand out. + for (what, k, repeat, f) in [ + ("points", 1000, 3, (|w: &str| drop(hyphen_points_in(w, &Limits::default(), ProofLang::Ru))) as fn(&str)), + ("hyphenated", 400, 20, |w: &str| drop(hyphenate_word_in(w, &Limits::default(), ProofLang::Ru))), + ] { + let (small, big) = (word(k), word(8 * k)); + let time = |w: &str| (0..repeat).for_each(|_| f(w)); + let (t1, t8, ratio) = cost_ratio(8, || time(&small), || time(&big)); + eprintln!("{what}: t(n) {t1:.4} s, t(8n) {t8:.4} s, ratio {ratio:.1}"); + assert!(ratio < 24.0, "8x the letters, not linear: {what} {ratio:.1}"); + } + } + #[test] fn heuristic_fallback() { let c: Vec = "happen".chars().collect(); diff --git a/crates/proof/src/ru/hyphen.rs b/crates/proof/src/ru/hyphen.rs index c02aa127..33d19b77 100644 --- a/crates/proof/src/ru/hyphen.rs +++ b/crates/proof/src/ru/hyphen.rs @@ -57,7 +57,9 @@ pub fn points(c: &[char]) -> Vec { while c.get(cut).is_some_and(|&x| sticks_left(x)) && cut < end { cut += 1; } - if cut >= 2 && n - cut >= 2 && !out.contains(&cut) { + // Cuts come in increasing order (each lies after the previous pair's vowel), so a repeat + // can only be the last one: no scan of the whole list per syllable (#3657). + if cut >= 2 && n - cut >= 2 && out.last().is_none_or(|&l| l < cut) { out.push(cut); } } From a3983c4789e26710b3134906242119970661fe95 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 22 Oct 2026 15:15:56 +0100 Subject: [PATCH 11/16] storytold/wordcraft#3625: Turkish suggestions and lookups have a word-length cap Audit finding 20. - tr::suggest_with refuses words over 64 letters (and words with spaces), like the other languages' suggesters (40 there; Turkish agglutinated words run past 40). Each candidate edit copies the whole word, so a long word cost O(n^2) per suggestion request on the UI thread. - TrDict::contains returns false past MAX_WORD_BYTES (120, the affix engine's bound) instead of collecting the char indices of the whole word. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/proof/src/tr/dict.rs | 3 ++- crates/proof/src/tr/suggest.rs | 22 +++++++++++++++++++++- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/crates/proof/src/tr/dict.rs b/crates/proof/src/tr/dict.rs index ed57cbca..4b59ccc2 100644 --- a/crates/proof/src/tr/dict.rs +++ b/crates/proof/src/tr/dict.rs @@ -87,7 +87,8 @@ impl TrDict { /// Check if word is correct (stem or affixed). pub fn contains(&self, word: &str) -> bool { - if self.is_empty() || word.is_empty() { + // Longer input is never a dictionary word, and isn't scanned (#3625). + if self.is_empty() || word.is_empty() || word.len() > crate::affix::MAX_WORD_BYTES { return false; } diff --git a/crates/proof/src/tr/suggest.rs b/crates/proof/src/tr/suggest.rs index dedc1bb2..4f4f3d66 100644 --- a/crates/proof/src/tr/suggest.rs +++ b/crates/proof/src/tr/suggest.rs @@ -46,6 +46,12 @@ const COMMON_REPLACEMENTS: &[(&str, &str)] = &[ ("deyil", "değil"), ]; +/// Longest word (in letters) given suggestions. Every candidate is a fresh copy of the word, so +/// the cost grows with the square of its length (#3625); Turkish words run longer than most +/// (`Çekoslovakyalılaştıramadıklarımızdanmışsınız` is 43 letters), so the cap is above the other +/// languages' 40. +const MAX_LETTERS: usize = 64; + /// Produce spelling suggestions for a misspelled Turkish word. pub fn suggest(word: &str, max: usize) -> Vec { let dict = super::dictionary(); @@ -54,7 +60,8 @@ pub fn suggest(word: &str, max: usize) -> Vec { /// Suggest with a given dictionary. pub fn suggest_with(d: &TrDict, word: &str, max: usize) -> Vec { - if word.is_empty() || max == 0 { + let n = word.chars().count(); + if n == 0 || n > MAX_LETTERS || max == 0 || word.contains(char::is_whitespace) { return Vec::new(); } @@ -277,6 +284,19 @@ mod tests { assert!(sugs.contains(&"geldin mi".to_string())); } + #[test] + fn overlong_words_get_no_suggestions_promptly() { + let d = TrDict::from_hunspell("SET UTF-8\nSFX 1 Y 1\nSFX 1 0 lar .\n", "2\nkitap/1\nsonra\n").unwrap(); + assert!(suggest_with(&d, "sorna", 3).contains(&"sonra".to_string())); + assert!(suggest_with(&d, "kitaplr", 3).contains(&"kitaplar".to_string())); + let t = std::time::Instant::now(); + let long = "a".repeat(100_000); + assert!(suggest_with(&d, &long, 8).is_empty()); + assert!(!d.contains(&long) && !d.contains(&format!("kitap{long}"))); + assert!(!super::super::is_correct_with(&d, &long)); + assert!(t.elapsed().as_secs_f64() < 1.0, "{:?}", t.elapsed()); + } + #[test] fn turkish_to_lower_handles_i() { assert_eq!(turkish_to_lower("IŞIK"), "ışık"); From a86631131f53d77380ec48d14c29a0832856fc54 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Fri, 23 Oct 2026 02:45:26 +0100 Subject: [PATCH 12/16] =?UTF-8?q?storytold/wordcraft#3625:=20review=20?= =?UTF-8?q?=E2=80=94=20"bir=20ka=C3=A7"=20still=20gets=20"birka=C3=A7":=20?= =?UTF-8?q?the=20replacement=20table=20is=20checked=20before=20multi-word?= =?UTF-8?q?=20input=20is=20refused?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01CX1hMKX28764Lrx5moxs1o --- crates/proof/src/tr/suggest.rs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/crates/proof/src/tr/suggest.rs b/crates/proof/src/tr/suggest.rs index 4f4f3d66..38020ded 100644 --- a/crates/proof/src/tr/suggest.rs +++ b/crates/proof/src/tr/suggest.rs @@ -61,7 +61,7 @@ pub fn suggest(word: &str, max: usize) -> Vec { /// Suggest with a given dictionary. pub fn suggest_with(d: &TrDict, word: &str, max: usize) -> Vec { let n = word.chars().count(); - if n == 0 || n > MAX_LETTERS || max == 0 || word.contains(char::is_whitespace) { + if n == 0 || n > MAX_LETTERS || max == 0 { return Vec::new(); } @@ -79,6 +79,10 @@ pub fn suggest_with(d: &TrDict, word: &str, max: usize) -> Vec { } } } + // Only the table takes more than one word (`bir kaç`). + if word.contains(char::is_whitespace) { + return suggestions; + } // 2. Diacritic substitutions (`c` -> `ç`, `g` -> `ğ`, `ı` -> `i`, `o` -> `ö`, `s` -> `ş`, `u` -> `ü`) if !d.is_empty() { @@ -297,6 +301,14 @@ mod tests { assert!(t.elapsed().as_secs_f64() < 1.0, "{:?}", t.elapsed()); } + #[test] + fn two_words_get_only_the_tables_replacement() { + let d = TrDict::from_hunspell("SET UTF-8\n", "2\nbirkaç\nsonra\n").unwrap(); + assert_eq!(suggest_with(&d, "bir kaç", 3), vec!["birkaç".to_string()]); + assert_eq!(suggest_with(&d, "Bir kaç", 3), vec!["birkaç".to_string()]); + assert!(suggest_with(&d, "sorna sorna", 3).is_empty()); + } + #[test] fn turkish_to_lower_handles_i() { assert_eq!(turkish_to_lower("IŞIK"), "ışık"); From 935c10d3d06b89402217b8bd9cbb1ae7ae4157ae Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Fri, 23 Oct 2026 11:27:52 +0100 Subject: [PATCH 13/16] =?UTF-8?q?storytold/wordcraft#3661:=20review=20?= =?UTF-8?q?=E2=80=94=20ISO=208859-9=20Turkish=20dictionaries=20load=20thro?= =?UTF-8?q?ugh=20the=20capped=20reader?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Asked to confirm a non-UTF-8 Turkish dictionary still loads: before this PR tr_TR was read with read_to_string, so a non-UTF-8 file never loaded; now it goes through read_capped and decode, which didn't know ISO 8859-9 (Latin-5, the encoding of older tr_TR dictionaries) and refused it as unsupported. decode now reads `SET ISO8859-9`, and a test writes an ISO 8859-9 .aff/.dic to disk and finds words with ğ ı ş İ ç ö ü, as stems and with suffixes. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01CX1hMKX28764Lrx5moxs1o --- crates/proof/src/hunspell.rs | 47 +++++++++++++++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/crates/proof/src/hunspell.rs b/crates/proof/src/hunspell.rs index a5b41214..e86c7f68 100644 --- a/crates/proof/src/hunspell.rs +++ b/crates/proof/src/hunspell.rs @@ -174,6 +174,23 @@ fn decode_latin1(bytes: &[u8]) -> String { bytes.iter().map(|&b| char::from(b)).collect() } +/// ISO 8859-9 (Latin-5, Turkish): ISO 8859-1 with Ğ İ Ş ğ ı ş in place of Ð Ý Þ ð ý þ. The +/// encoding of older Turkish dictionaries (`SET ISO8859-9`). +fn decode_latin5(bytes: &[u8]) -> String { + bytes + .iter() + .map(|&b| match b { + 0xd0 => 'Ğ', + 0xdd => 'İ', + 0xde => 'Ş', + 0xf0 => 'ğ', + 0xfd => 'ı', + 0xfe => 'ş', + _ => char::from(b), + }) + .collect() +} + /// ISO 8859-2 code points 0xA0..=0xFF (0x00..=0x9F map to themselves). const ISO_8859_2_HIGH: &str = "\u{a0}Ą˘Ł¤ĽŚ§¨ŠŞŤŹ\u{ad}ŽŻ°ą˛ł´ľśˇ¸šşťź˝žżŔÁÂĂÄĹĆÇČÉĘËĚÍÎĎĐŃŇÓÔŐÖ×ŘŮÚŰÜÝŢßŕáâăäĺćçčéęëěíîďđńňóôőö÷řůúűüýţ˙"; @@ -199,7 +216,8 @@ fn decode_high(bytes: &[u8], high: &str) -> String { /// Decode an `.aff`/`.dic` pair to UTF-8 from the encoding the `.aff`'s `SET` line names /// (UTF-8 when there is none; ISO 8859-2 is the other one Polish dictionaries use, ISO 8859-1 the -/// one the original Portuguese VERO uses, KOI8-R and Windows-1251 the ones older Russian dictionaries use). +/// one the original Portuguese VERO uses, ISO 8859-9 the one older Turkish dictionaries use, KOI8-R +/// and Windows-1251 the ones older Russian dictionaries use). pub fn decode(aff: &[u8], dic: &[u8]) -> Result<(String, String), String> { let head = String::from_utf8_lossy(aff.get(..aff.len().min(8192)).unwrap_or_default()).to_string(); let set = head.lines().find_map(|l| l.trim_start_matches('\u{feff}').strip_prefix("SET ")).map(str::trim).unwrap_or("UTF-8").to_ascii_uppercase(); @@ -210,6 +228,7 @@ pub fn decode(aff: &[u8], dic: &[u8]) -> Result<(String, String), String> { } "ISO8859-1" | "ISO-8859-1" => Ok((decode_latin1(aff), decode_latin1(dic))), "ISO8859-2" | "ISO-8859-2" => Ok((decode_latin2(aff), decode_latin2(dic))), + "ISO8859-9" | "ISO-8859-9" => Ok((decode_latin5(aff), decode_latin5(dic))), "KOI8-R" | "KOI8R" => Ok((decode_high(aff, KOI8_R_HIGH), decode_high(dic, KOI8_R_HIGH))), "CP1251" | "WINDOWS-1251" | "MICROSOFT-CP1251" => Ok((decode_high(aff, CP1251_HIGH), decode_high(dic, CP1251_HIGH))), other => Err(format!("unsupported encoding {other}")), @@ -228,6 +247,8 @@ mod tests { assert_eq!(d, "2\nżółw\n"); let (_, d) = decode("SET UTF-8\n".as_bytes(), "żółw".as_bytes()).unwrap(); assert_eq!(d, "żółw"); + let (_, d) = decode(b"SET ISO8859-9\n", b"\xd0\xdd\xde\xf0\xfd\xfe\xd6\xe7").unwrap(); + assert_eq!(d, "ĞİŞğışÖç"); assert!(decode(b"SET SHIFT_JIS\n", b"").is_err()); assert!(decode(b"SET UTF-8\n", b"\xff\xfe").is_err()); } @@ -323,4 +344,28 @@ mod tests { assert!(crate::tr::load(&tr_aff, &big).unwrap_err().contains("too large")); let _ = std::fs::remove_dir_all(&dir); } + + /// A Turkish dictionary in ISO 8859-9, the encoding older tr_TR dictionaries use, read from + /// disk through the capped reader and decoded per its `SET` line: its words with Turkish + /// letters (ğ ı ş İ, and ç ö ü from Latin-1) are found, as stems and with a suffix. Before the + /// Turkish dictionary went through this reader it was read as UTF-8, which these bytes are not. + #[test] + fn a_turkish_dictionary_in_iso_8859_9_loads_from_disk() { + let dir = std::env::temp_dir().join(format!("wordcraft-hunspell-latin5-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let (aff, dic) = (dir.join("tr_TR.aff"), dir.join("tr_TR.dic")); + std::fs::write(&aff, b"SET ISO8859-9\nSFX 1 Y 2\nSFX 1 0 lar .\nSFX 1 0 \xfd .\nSFX 2 Y 1\nSFX 2 0 ler .\n").unwrap(); + // ağaç/1 (-lar, -ı), ışık, şişe/2 (-ler), İzmir, öğrenci/2, güneş. + let words: &[u8] = b"6\na\xf0a\xe7/1\n\xfd\xfe\xfdk\n\xfei\xfee/2\n\xddzmir\n\xf6\xf0renci/2\ng\xfcne\xfe\n"; + std::fs::write(&dic, words).unwrap(); + assert!(String::from_utf8(words.to_vec()).is_err(), "the .dic is not UTF-8"); + let d = crate::tr::load(&aff, &dic).unwrap(); + for w in ["ağaç", "ağaçlar", "ağaçı", "ışık", "şişe", "şişeler", "İzmir", "öğrenci", "öğrenciler", "güneş"] { + assert!(d.contains(w), "{w}"); + } + // The six letters ISO 8859-9 puts in place of ISO 8859-1's aren't read as those. + assert!(!d.contains("aðaç") && !d.contains("ýþýk") && !d.contains("Ýzmir")); + assert!(!d.contains("ağaçs") && !d.contains("ışıklar") && !d.contains("şişelar"), "only the rules' suffixes, on the stems that take them"); + let _ = std::fs::remove_dir_all(&dir); + } } From 06af26ed67032fd2d8673fa4bd283ca9b0783117 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Fri, 23 Oct 2026 14:20:40 +0100 Subject: [PATCH 14/16] =?UTF-8?q?storytold/wordcraft#3692:=20review=20?= =?UTF-8?q?=E2=80=94=20proof=20timing=20tests=20get=20a=20margin=20a=20bus?= =?UTF-8?q?y=20CI=20machine=20can't=20break?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wall-clock bounds of 1-2 s failed on loaded runners. Each bound is now 10 s, at least 10x the fixed code's time and well under the old code's on the same input (measured by running the pre-fix code): - self-continuing rules: 40,000 rules (was 16,000); ~1 ms now, old checker 14 s at 16,000. - many strips per suffix: 200,000 rules (was 30,000); 0.4 s now, old index 3 s at 30,000 and 11 min at 300,000. - forms of self-continuing rules: the MAX_FORMS count catches the old code (9 million forms); the time bound is a backstop. - overlong Turkish words: ~20 ms now; without the caps it ran past 5 minutes. - capped Hunspell reads: no clock; the "not a regular file" error is the proof (the old reader never returned from /dev/zero). Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01CX1hMKX28764Lrx5moxs1o --- crates/proof/src/affix.rs | 22 ++++++++++++++-------- crates/proof/src/hunspell.rs | 5 ++--- crates/proof/src/tr/suggest.rs | 3 ++- 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/crates/proof/src/affix.rs b/crates/proof/src/affix.rs index fefc7f6a..03fc27a6 100644 --- a/crates/proof/src/affix.rs +++ b/crates/proof/src/affix.rs @@ -1310,11 +1310,13 @@ pub(crate) mod tests { #[test] fn self_continuing_rules_check_in_linear_time() { // Thousands of empty-append rules that continue with their own flag: each check used to - // follow every rule from every rule, millions of steps a word (#3613). - let d = many_rules(16_000, |_| "0 0/A .".into()); + // follow every rule from every rule, millions of steps a word (#3613). The checks take + // about a millisecond; the old checker took 14 s for two of them at 16,000 rules, so at + // 40,000 (quadratic: minutes) the 10 s bound leaves a busy machine a wide margin. + let d = many_rules(40_000, |_| "0 0/A .".into()); let t = std::time::Instant::now(); assert!(d.check("kot") && !d.check("pies") && !d.check("kotx")); - assert!(t.elapsed().as_secs_f64() < 2.0, "{:?}", t.elapsed()); + assert!(t.elapsed().as_secs_f64() < 10.0, "{:?}", t.elapsed()); } #[test] @@ -1324,7 +1326,9 @@ pub(crate) mod tests { let distinct = many_rules(3000, |i| format!("0 x{i}/A .")); let t = std::time::Instant::now(); let (a, b) = (same.forms("kot"), distinct.forms("kot")); - assert!(t.elapsed().as_secs_f64() < 2.0, "{:?}", t.elapsed()); + // The cap on the count is what tells the old code (9 million forms each) apart; the + // generous time bound (tens of ms here) only backs up the rule-visit cap. + assert!(t.elapsed().as_secs_f64() < 10.0, "{:?}", t.elapsed()); assert!(a.len() <= MAX_FORMS && b.len() <= MAX_FORMS, "{} {}", a.len(), b.len()); assert!(b.iter().any(|f| f == "kotx0x1"), "chained forms are still listed"); } @@ -1332,11 +1336,13 @@ pub(crate) mod tests { #[test] fn many_strips_per_suffix_index_in_linear_time() { // One append with thousands of strips: the index used to scan the append's groups for - // each rule. + // each rule. Building 200,000 rules takes well under a second; the old index took 3 s at + // 30,000 and over 10 minutes at 300,000. + let n = 200_000; let t = std::time::Instant::now(); - let d = many_rules(30_000, |i| format!("k{i} y .")); - assert!(t.elapsed().as_secs_f64() < 2.0, "{:?}", t.elapsed()); - assert_eq!(d.sfx_index.get("y").map(Vec::len), Some(30_000)); + let d = many_rules(n, |i| format!("k{i} y .")); + assert!(t.elapsed().as_secs_f64() < 10.0, "{:?}", t.elapsed()); + assert_eq!(d.sfx_index.get("y").map(Vec::len), Some(n)); assert!(!d.contains("koty")); } diff --git a/crates/proof/src/hunspell.rs b/crates/proof/src/hunspell.rs index e86c7f68..3449c8ca 100644 --- a/crates/proof/src/hunspell.rs +++ b/crates/proof/src/hunspell.rs @@ -329,13 +329,12 @@ mod tests { // One byte over the cap (sparse: nothing is written) is refused without reading it. let big = dir.join("big.dic"); std::fs::File::create(&big).unwrap().set_len(MAX_FILE + 1).unwrap(); - let t = std::time::Instant::now(); assert!(load(&aff, &big).unwrap_err().contains("too large")); - // A directory, and on Unix an endless device, are not dictionary files. + // A directory, and on Unix an endless device, are not dictionary files. (No clock: a + // reader that read `/dev/zero` would never return, so the error itself is the proof.) assert!(load(&aff, &dir).unwrap_err().contains("not a regular file")); #[cfg(unix)] assert!(load(&aff, Path::new("/dev/zero")).unwrap_err().contains("not a regular file")); - assert!(t.elapsed().as_secs_f64() < 1.0, "{:?}", t.elapsed()); // Turkish reads through the same reader. let (tr_aff, tr_dic) = (dir.join("tr_TR.aff"), dir.join("tr_TR.dic")); std::fs::write(&tr_aff, "SET UTF-8\nSFX 1 Y 1\nSFX 1 0 lar .\n").unwrap(); diff --git a/crates/proof/src/tr/suggest.rs b/crates/proof/src/tr/suggest.rs index 38020ded..c5f4db89 100644 --- a/crates/proof/src/tr/suggest.rs +++ b/crates/proof/src/tr/suggest.rs @@ -298,7 +298,8 @@ mod tests { assert!(suggest_with(&d, &long, 8).is_empty()); assert!(!d.contains(&long) && !d.contains(&format!("kitap{long}"))); assert!(!super::super::is_correct_with(&d, &long)); - assert!(t.elapsed().as_secs_f64() < 1.0, "{:?}", t.elapsed()); + // Tens of milliseconds with the caps; without them this ran for over 5 minutes. + assert!(t.elapsed().as_secs_f64() < 10.0, "{:?}", t.elapsed()); } #[test] From e8c1c4757afae7a17c22876506b622675a02916a Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sun, 25 Oct 2026 21:28:59 -0400 Subject: [PATCH 15/16] Sonar: use PoisonError::into_inner, explicit test imports, [[ in import-cert - proof: pass PoisonError::into_inner instead of an identity closure (S1612). - layout tests: replace glob super::* imports with explicit names (S2208). - packaging/macos/import-cert.sh: use [[ for the status test (S7688). --- crates/proof/src/es/mod.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/proof/src/es/mod.rs b/crates/proof/src/es/mod.rs index 94c6511a..351a69b0 100644 --- a/crates/proof/src/es/mod.rs +++ b/crates/proof/src/es/mod.rs @@ -16,7 +16,7 @@ mod suggest; pub(crate) mod test_dict; pub mod voseo; -use std::sync::{Arc, RwLock}; +use std::sync::{Arc, PoisonError, RwLock}; use crate::ProofLang; use crate::affix::AffixDict; @@ -62,11 +62,11 @@ pub fn dictionary_name(lang: ProofLang) -> &'static str { /// see [`has_dictionary`]). pub fn dictionary(lang: ProofLang) -> Arc { let Some(slot) = DICTS.get(variety(lang).0) else { return Arc::new(AffixDict::default()) }; - if let Some(d) = slot.read().unwrap_or_else(|e| e.into_inner()).as_ref() { + if let Some(d) = slot.read().unwrap_or_else(PoisonError::into_inner).as_ref() { return d.clone(); } let loaded = Arc::new(load_installed(lang)); - let mut slot = slot.write().unwrap_or_else(|e| e.into_inner()); + let mut slot = slot.write().unwrap_or_else(PoisonError::into_inner); // Another thread (or an install) may have filled it meanwhile: that one wins. slot.get_or_insert(loaded).clone() } @@ -90,7 +90,7 @@ fn load_installed(lang: ProofLang) -> AffixDict { pub fn install_dictionary(lang: ProofLang, aff: &str, dic: &str) -> Result<(), String> { let d = AffixDict::from_hunspell(aff, dic, &[])?; if let Some(slot) = DICTS.get(variety(lang).0) { - *slot.write().unwrap_or_else(|e| e.into_inner()) = Some(Arc::new(d)); + *slot.write().unwrap_or_else(PoisonError::into_inner) = Some(Arc::new(d)); } Ok(()) } @@ -99,7 +99,7 @@ pub fn install_dictionary(lang: ProofLang, aff: &str, dic: &str) -> Result<(), S /// (after [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed). pub fn reload_dictionary() { for slot in &DICTS { - *slot.write().unwrap_or_else(|e| e.into_inner()) = None; + *slot.write().unwrap_or_else(PoisonError::into_inner) = None; } } From 901f8cad59f96207e8a5bcebcd9575b9e5e1f702 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Mon, 26 Oct 2026 16:01:36 -0400 Subject: [PATCH 16/16] Sonar: reduce cognitive complexity in highlight, review and outline commands --- crates/proof/src/affix.rs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/crates/proof/src/affix.rs b/crates/proof/src/affix.rs index 03fc27a6..4ec06846 100644 --- a/crates/proof/src/affix.rs +++ b/crates/proof/src/affix.rs @@ -314,13 +314,12 @@ fn parse_aff(aff: &str) -> Result { match dir { "SET" | "LANG" | "NAME" | "VERSION" | "HOME" | "WORDCHARS" | "MAXNGRAMSUGS" | "MAXDIFF" | "ONLYMAXDIFF" | "NOSPLITSUGS" | "SUGSWITHDOTS" | "WARN" | "FORBIDWARN" | "MAXCPDSUGS" => {} - "FLAG" => { - // Per-character flag modes (`FLAG UTF-8`, as VERO's pt_BR dictionary uses) work - // with the one-character parsing here; only the multi-character encodings differ. - if !matches!(at(1), "UTF-8" | "DEFAULT" | "utf-8" | "default" | "UTF8" | "") { - return Err(format!("line {}: FLAG {} is not supported (one-character flags only)", n + 1, at(1))); - } + // Per-character flag modes (`FLAG UTF-8`, as VERO's pt_BR dictionary uses) work + // with the one-character parsing here; only the multi-character encodings differ. + "FLAG" if !matches!(at(1), "UTF-8" | "DEFAULT" | "utf-8" | "default" | "UTF8" | "") => { + return Err(format!("line {}: FLAG {} is not supported (one-character flags only)", n + 1, at(1))); } + "FLAG" => {} directive @ ("FORBIDDENWORD" | "NOSUGGEST") => { let mut it = at(1).chars().filter(|&c| c != EMOJI_SELECTOR); match (it.next(), it.next()) {