From b732f2abc7ccc10ca825d4548447d6f81e5f7778 Mon Sep 17 00:00:00 2001 From: Naxvog Date: Tue, 3 Nov 2026 06:44:55 +0100 Subject: [PATCH] proof: cap .aff REP replacements at parse time (storytold/wordcraft#3655) Both readers kept every REP line: an .aff with hundreds of thousands of entries, or megabyte-long patterns and replacements, made each suggestion round hang and allocate hundreds of MB. push_rep keeps at most 4096 entries, drops patterns longer than MAX_LETTERS letters (no suggester takes a longer word, so they never match) and replacements longer than MAX_WORD_BYTES. Ported from storytold/wordcraft#4632, which can't land now that proofing lives here. Co-Authored-By: Claude Opus 5.5 --- crates/proof/src/affix.rs | 19 ++++++++++++++-- crates/proof/src/affix/rep_tests.rs | 34 +++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 2 deletions(-) create mode 100644 crates/proof/src/affix/rep_tests.rs diff --git a/crates/proof/src/affix.rs b/crates/proof/src/affix.rs index 0df68665..53af2c31 100644 --- a/crates/proof/src/affix.rs +++ b/crates/proof/src/affix.rs @@ -42,6 +42,8 @@ use std::hash::Hasher; use crate::patterns::{FxHasher, FxMap}; mod compound; +#[cfg(test)] +mod rep_tests; pub(crate) use compound::{MAX_SUGGEST_COMPOUND_WORK, Pieces}; @@ -677,6 +679,19 @@ fn cap_try(s: &str) -> String { out } +/// Most `REP` entries kept; every suggestion round tries each one at every place it matches. +const MAX_REP_ENTRIES: usize = 4096; + +/// Adds a `REP` entry (`_` stands for a space) unless the table is full, `from` is empty or longer +/// than any word a suggester takes (it could never match), or `to` is longer than a dictionary word. +fn push_rep(rep: &mut Vec<(String, String)>, from: &str, to: &str) { + // Letters, not bytes: a 40-letter word in a four-byte script is 160 bytes long. + let fits = from.chars().nth(crate::affix_suggest::MAX_LETTERS).is_none(); + if rep.len() < MAX_REP_ENTRIES && !from.is_empty() && fits && to.len() <= MAX_WORD_BYTES { + rep.push((from.replace('_', " "), to.replace('_', " "))); + } +} + /// `form`, unless it is empty: no word is, and `FULLSTRIP` can strip a stem down to nothing. fn non_empty(form: String) -> Option { Some(form).filter(|f| !f.is_empty()) @@ -1120,7 +1135,7 @@ fn parse_aff(aff: &str, english: bool) -> Result { if !rep_header && at(1).parse::().is_ok() { rep_header = true; } else if f.len() >= 3 { - r.rep.push((at(1).replace('_', " "), at(2).replace('_', " "))); + push_rep(&mut r.rep, at(1), at(2)); } } "PFX" | "SFX" => { @@ -2674,7 +2689,7 @@ impl AffixDict { read.map_err(|e| format!("affix dictionary: bad `{line}`: {e}"))?; } "MAP" => rules.map.push(at(1).chars().collect()), - "REP" => rules.rep.push((at(1).replace('_', " "), at(2).replace('_', " "))), + "REP" => push_rep(&mut rules.rep, at(1), at(2)), kind @ ("PFX" | "SFX") => { let flag = rules.one_flag(at(1)).map_err(|e| format!("affix dictionary: bad flag in `{line}`: {e}"))?; if f.len() != 6 { diff --git a/crates/proof/src/affix/rep_tests.rs b/crates/proof/src/affix/rep_tests.rs new file mode 100644 index 00000000..652ca0f8 --- /dev/null +++ b/crates/proof/src/affix/rep_tests.rs @@ -0,0 +1,34 @@ +//! `REP` table limits (storytold/wordcraft#3655). + +use super::*; + +/// storytold/wordcraft#3655: a hostile `.aff` can't grow the `REP` table past its cap or fill it +/// with patterns and replacements longer than a word. +#[test] +fn rep_table_is_capped_and_drops_oversized_entries() { + let mut aff = String::from("REP 200000\nREP rz ż\n"); + // Bytes, not letters, bound `to`: 31 four-byte letters are 124 bytes. + let wide = "\u{1D51E}".repeat(MAX_WORD_BYTES / 4 + 1); + aff += &format!("REP q {}\nREP {} q\nREP q {wide}\n", "z".repeat(1 << 20), "z".repeat(MAX_WORD_BYTES + 1)); + for i in 0..200_000 { + aff += &format!("REP q z{i}\n"); + } + let d = AffixDict::from_hunspell(&aff, "1\nkot\n", &[]).unwrap(); + let check = |d: &AffixDict| { + assert_eq!(d.rep().len(), MAX_REP_ENTRIES); + assert_eq!(d.rep().first(), Some(&("rz".to_string(), "ż".to_string()))); + assert!(d.rep().iter().all(|(a, b)| a.chars().count() <= 40 && b.len() <= MAX_WORD_BYTES)); + }; + check(&d); + // A `from` of 40 four-byte letters (160 bytes) can match a word and is kept; 41 can't. + let long = "\u{1D51E}".repeat(40); + let d40 = AffixDict::from_hunspell(&format!("REP 2\nREP {long} a\nREP {long}\u{1D51E} a\n"), "1\nkot\n", &[]).unwrap(); + assert_eq!(d40.rep(), [(long, "a".to_string())]); + check(&AffixDict::from_bytes(&d.to_bytes().unwrap()).unwrap()); + + // Oversized REP lines fed straight to the compact reader are dropped as well. + // A bare `REP` reaches `push_rep` with an empty `from` here, and is dropped. + let raw = format!("{MAGIC}\nREP q {}\nREP\nREP a_b c\nWORDS\n\0kot\n", "z".repeat(1 << 20)); + let bytes = miniz_oxide::deflate::compress_to_vec(raw.as_bytes(), 6); + assert_eq!(AffixDict::from_bytes(&bytes).unwrap().rep(), &[("a b".to_string(), "c".to_string())]); +}