Merge remote-tracking branch 'upstream/main' into fix/proof-shared-language-engine

# Conflicts:
#	crates/proof/src/es/mod.rs
#	crates/proof/src/tr/mod.rs

crates/proof/Cargo.toml

@@ -10,7 +10,7 @@ workspace = true
 
 [dependencies]
 log = { workspace = true }
-miniz_oxide = "0.8"
+miniz_oxide = "0.9"
 unicode-normalization = "0.1"
 
 [dev-dependencies]

crates/proof/src/affix.rs

@@ -120,6 +120,11 @@ fn parse_cond(text: &str) -> Result<Box<[Cond]>, String> {
 const MAX_CHAIN: u8 = 2;
 /// How many suffixes may sit under the outermost one: the depth `form_has_flag` walks.
 const INNER_CHAIN: u8 = MAX_CHAIN - 1;
+/// Most forms [`AffixDict::forms`] lists for one stem (real stems have a few hundred at most).
+pub const MAX_FORMS: usize = 4096;
+/// Most rules [`AffixDict::forms`] tries along continuation chains and prefix–suffix cross
+/// products for one stem: real dictionaries need a few thousand, hostile ones are cut off.
+const MAX_FORMS_WORK: usize = 1 << 18;
 
 /// One prefix or suffix rule.
 #[derive(Clone, Debug)]
@@ -182,6 +187,8 @@ pub struct AffixDict {
     suffixes: Vec<Rule>,
     /// Appended string → groups of suffix rules by stripped string.
     sfx_index: FxMap<Box<str>, Vec<SfxGroup>>,
+    /// Per flag bit: the suffix rules of that flag, in rule order.
+    sfx_by_flag: Vec<Vec<u32>>,
     /// Longest appended suffix, in chars.
     max_sfx_chars: usize,
     try_chars: String,
@@ -307,13 +314,12 @@ fn parse_aff(aff: &str) -> Result<Rules, String> {
         match dir {
             "SET" | "LANG" | "NAME" | "VERSION" | "HOME" | "WORDCHARS" | "MAXNGRAMSUGS" | "MAXDIFF" | "ONLYMAXDIFF" | "NOSPLITSUGS"
             | "SUGSWITHDOTS" | "WARN" | "FORBIDWARN" | "MAXCPDSUGS" => {}
-            "FLAG" => {
-                // Per-character flag modes (`FLAG UTF-8`, as VERO's pt_BR dictionary uses) work
-                // with the one-character parsing here; only the multi-character encodings differ.
-                if !matches!(at(1), "UTF-8" | "DEFAULT" | "utf-8" | "default" | "UTF8" | "") {
-                    return Err(format!("line {}: FLAG {} is not supported (one-character flags only)", n + 1, at(1)));
-                }
+            // Per-character flag modes (`FLAG UTF-8`, as VERO's pt_BR dictionary uses) work
+            // with the one-character parsing here; only the multi-character encodings differ.
+            "FLAG" if !matches!(at(1), "UTF-8" | "DEFAULT" | "utf-8" | "default" | "UTF8" | "") => {
+                return Err(format!("line {}: FLAG {} is not supported (one-character flags only)", n + 1, at(1)));
             }
+            "FLAG" => {}
             directive @ ("FORBIDDENWORD" | "NOSUGGEST") => {
                 let mut it = at(1).chars().filter(|&c| c != EMOJI_SELECTOR);
                 match (it.next(), it.next()) {
@@ -419,7 +425,7 @@ impl AffixDict {
             }
             stems.push((word, flags));
         }
-        AffixDict::build(rules, stems, comments.iter().map(|s| s.to_string()).collect())
+        AffixDict::build(rules, stems, comments.iter().map(ToString::to_string).collect())
     }
 
     fn build(rules: Rules, mut stems: Vec<(String, String)>, comments: Vec<String>) -> Result<AffixDict, String> {
@@ -484,14 +490,27 @@ impl AffixDict {
     /// Build the suffix index and the stem hash table.
     fn index(&mut self) {
         let mut idx: FxMap<Box<str>, Vec<SfxGroup>> = FxMap::default();
+        // Each (appended, stripped) pair's group, found by hash rather than by scanning the
+        // appended string's groups: a dictionary with many strips per suffix stays linear.
+        let mut group_of: FxMap<(&str, &str), usize> = FxMap::default();
+        let mut by_flag: Vec<Vec<u32>> = vec![Vec::new(); 128];
         for (i, r) in self.suffixes.iter().enumerate() {
             let groups = idx.entry(r.add.clone()).or_default();
             let Ok(i) = u32::try_from(i) else { break };
-            match groups.iter_mut().find(|g| g.strip == r.strip) {
+            match group_of.get(&(&*r.add, &*r.strip)).and_then(|&g| groups.get_mut(g)) {
                 Some(g) => g.rules.push(i),
-                None => groups.push(SfxGroup { strip: r.strip.clone(), rules: vec![i] }),
+                None => {
+                    group_of.insert((&r.add, &r.strip), groups.len());
+                    groups.push(SfxGroup { strip: r.strip.clone(), rules: vec![i] });
+                }
+            }
+            if r.bit != 0
+                && let Some(list) = by_flag.get_mut(r.bit.trailing_zeros() as usize)
+            {
+                list.push(i);
             }
         }
+        self.sfx_by_flag = by_flag;
         self.max_sfx_chars = self.suffixes.iter().map(|r| r.add.chars().count()).max().unwrap_or(0);
         self.sfx_index = idx;
         let n = self.len();
@@ -596,10 +615,14 @@ impl AffixDict {
         None
     }
 
-    /// Does `w` carry flag `bit` — as a stem, or as the form of a suffix rule whose
-    /// continuation classes include `bit` (chained suffixation, depth-bounded)?
-    fn form_has_flag(&self, w: &str, bit: u128, depth: u8, excluded: u128) -> bool {
-        if self.stem_flags_excluding(w, excluded).is_some_and(|f| f & bit != 0) {
+    /// Does `w` carry any flag of `flags` — as a stem, or as the form of a suffix rule whose
+    /// continuation classes include one (chained suffixation, depth-bounded)?
+    ///
+    /// The rules of a group that could lead to `w` are merged into one mask of their flags and
+    /// followed once, not once per rule: the answer is the same, but thousands of rules of one
+    /// flag cost one walk, not one each (#3613).
+    fn form_has_flag(&self, w: &str, flags: u128, depth: u8, excluded: u128) -> bool {
+        if self.stem_flags_excluding(w, excluded).is_some_and(|f| f & flags != 0) {
             return true;
         }
         if depth == 0 {
@@ -614,10 +637,14 @@ impl AffixDict {
                 root.clear();
                 root.push_str(base);
                 root.push_str(&g.strip);
-                for r in g.rules.iter().filter_map(|&i| self.suffixes.get(i as usize)) {
-                    if r.cont & bit != 0 && r.cond_matches(&root, true) && self.form_has_flag(&root, r.bit, depth - 1, excluded) {
-                        return true;
-                    }
+                let inner = g
+                    .rules
+                    .iter()
+                    .filter_map(|&i| self.suffixes.get(i as usize))
+                    .filter(|r| r.cont & flags != 0 && r.cond_matches(&root, true))
+                    .fold(0u128, |m, r| m | r.bit);
+                if inner != 0 && self.form_has_flag(&root, inner, depth - 1, excluded) {
+                    return true;
                 }
             }
         }
@@ -690,9 +717,13 @@ impl AffixDict {
                     return true;
                 }
                 // Continuation classes: `root` need not be a stem carrying this rule's flag —
-                // it may be a form that gained the flag from another rule's continuation.
-                if need == 0 && rules().any(|r| r.cond_matches(&root, true) && self.form_has_flag(&root, r.bit, INNER_CHAIN, excluded)) {
-                    return true;
+                // it may be a form that gained the flag from another rule's continuation. The
+                // matching rules' flags are looked for together, in one walk.
+                if need == 0 {
+                    let flags = rules().filter(|r| r.cond_matches(&root, true)).fold(0u128, |m, r| m | r.bit);
+                    if flags != 0 && self.form_has_flag(&root, flags, INNER_CHAIN, excluded) {
+                        return true;
+                    }
                 }
             }
         }
@@ -725,13 +756,19 @@ impl AffixDict {
         }
     }
 
-    /// Every form of the stem `stem` (itself included), unsorted, possibly with repeats.
+    /// Every form of the stem `stem` (itself included), unsorted, possibly with repeats; at most
+    /// [`MAX_FORMS`], and at most [`MAX_FORMS_WORK`] rules tried along the continuation chains and
+    /// cross products, so a dictionary with thousands of self-continuing rules can't make one stem
+    /// cost millions of strings (#3613).
     pub fn forms(&self, stem: &str) -> Vec<String> {
         let Some(f) = self.stem_flags(stem) else { return Vec::new() };
         let mut out = vec![stem.to_string()];
         let mut cross_sfx = Vec::new();
-        // Suffix forms queued with the continuation classes they carry, for chained expansion.
+        // Suffix forms queued with the continuation classes they carry, for chained expansion;
+        // each (form, classes) pair is expanded once.
         let mut chain: Vec<(String, u128)> = Vec::new();
+        let mut queued: std::collections::HashSet<(String, u128)> = std::collections::HashSet::new();
+        let mut work = 0usize;
         for r in &self.suffixes {
             if f & r.bit == 0 || !r.cond_matches(stem, true) {
                 continue;
@@ -741,29 +778,39 @@ impl AffixDict {
             if r.cross {
                 cross_sfx.push(form.clone());
             }
-            if r.cont != 0 {
+            if r.cont != 0 && queued.insert((form.clone(), r.cont)) {
                 chain.push((form.clone(), r.cont));
             }
             out.push(form);
+            if out.len() >= MAX_FORMS {
+                return out;
+            }
         }
         // Continuation classes chain suffixes (`workable/B` → `workables`), up to [`MAX_CHAIN`]
         // suffixes in all — the same bound the checker walks.
-        for _ in 0..INNER_CHAIN {
+        'chain: for _ in 0..INNER_CHAIN {
             let level = std::mem::take(&mut chain);
             if level.is_empty() {
                 break;
             }
             for (form, flags) in level {
-                for r in &self.suffixes {
-                    if flags & r.bit == 0 || !r.cond_matches(&form, true) {
+                for r in self.suffixes_with(flags).into_iter().filter_map(|i| self.suffixes.get(i as usize)) {
+                    work += 1;
+                    if work > MAX_FORMS_WORK {
+                        break 'chain;
+                    }
+                    if !r.cond_matches(&form, true) {
                         continue;
                     }
                     let Some(base) = form.strip_suffix(&*r.strip) else { continue };
                     let next = format!("{base}{}", r.add);
-                    if r.cont != 0 {
+                    if r.cont != 0 && queued.insert((next.clone(), r.cont)) {
                         chain.push((next.clone(), r.cont));
                     }
                     out.push(next);
+                    if out.len() >= MAX_FORMS {
+                        return out;
+                    }
                 }
             }
         }
@@ -775,15 +822,38 @@ impl AffixDict {
             out.push(format!("{}{rest}", r.add));
             if r.cross {
                 for s in &cross_sfx {
+                    work += 1;
+                    if out.len() >= MAX_FORMS || work > MAX_FORMS_WORK {
+                        return out;
+                    }
                     if let Some(rest) = s.strip_prefix(&*r.strip) {
                         out.push(format!("{}{rest}", r.add));
                     }
                 }
             }
+            if out.len() >= MAX_FORMS {
+                return out;
+            }
         }
         out
     }
 
+    /// The suffix rules carrying any flag of `mask`, in rule order.
+    fn suffixes_with(&self, mask: u128) -> Vec<u32> {
+        let mut rules: Vec<u32> = Vec::new();
+        let mut lists = 0;
+        for (bit, list) in self.sfx_by_flag.iter().enumerate() {
+            if bit < 128 && mask & (1u128 << bit) != 0 && !list.is_empty() {
+                rules.extend_from_slice(list);
+                lists += 1;
+            }
+        }
+        if lists > 1 {
+            rules.sort_unstable();
+        }
+        rules
+    }
+
     /// The `TRY` letters, most frequent first (suggestions try these).
     pub fn try_chars(&self) -> &str {
         &self.try_chars
@@ -877,8 +947,8 @@ impl AffixDict {
 
     /// Read data written by [`AffixDict::to_bytes`].
     pub fn from_bytes(data: &[u8]) -> Result<AffixDict, String> {
-        let raw =
-            miniz_oxide::inflate::decompress_to_vec_with_limit(data, 64 << 20).map_err(|e| format!("affix dictionary: inflate failed: {e:?}"))?;
+        let raw = miniz_oxide::inflate::decompress_to_vec_with_limit(data, 64 << 20)
+            .map_err(|e| format!("affix dictionary: inflate failed: {:?}", e.status))?;
         let words_at = raw.windows(7).position(|w| w == b"\nWORDS\n").ok_or("affix dictionary: no WORDS section")?;
         let head = std::str::from_utf8(raw.get(..words_at).unwrap_or_default()).map_err(|_| "affix dictionary: bad utf-8 in the rules")?;
         let mut lines = head.lines();
@@ -1216,6 +1286,65 @@ pub(crate) mod tests {
         }
     }
 
+    #[test]
+    fn continuation_classes_merge_the_flags_of_a_group() {
+        // `s` under B and under C share one group (same append, same strip): a form carrying
+        // either flag takes it, and the stem carrying neither doesn't.
+        let aff = "SET UTF-8\nSFX A N 1\nSFX A 0 able/C .\nSFX D N 1\nSFX D 0 er/B .\nSFX B N 1\nSFX B 0 s .\nSFX C N 1\nSFX C 0 s [e]\n";
+        let d = AffixDict::from_hunspell(aff, "2\nwork/A\nplay/D\n", &[]).unwrap();
+        assert!(d.contains("workables") && d.contains("players"));
+        assert!(!d.contains("works") && !d.contains("plays") && !d.contains("workabless"));
+        assert!(d.forms("work").iter().any(|f| f == "workables") && d.forms("play").iter().any(|f| f == "players"));
+    }
+
+    /// `n` suffix rules of flag A on one stem, `rule(i)` giving each rule's strip, append and condition.
+    fn many_rules(n: usize, rule: impl Fn(usize) -> String) -> AffixDict {
+        let mut aff = format!("SET UTF-8\nSFX A Y {n}\n");
+        for i in 0..n {
+            aff.push_str(&format!("SFX A {}\n", rule(i)));
+        }
+        AffixDict::from_hunspell(&aff, "1\nkot/A\n", &[]).unwrap()
+    }
+
+    #[test]
+    fn self_continuing_rules_check_in_linear_time() {
+        // Thousands of empty-append rules that continue with their own flag: each check used to
+        // follow every rule from every rule, millions of steps a word (#3613). The checks take
+        // about a millisecond; the old checker took 14 s for two of them at 16,000 rules, so at
+        // 40,000 (quadratic: minutes) the 10 s bound leaves a busy machine a wide margin.
+        let d = many_rules(40_000, |_| "0 0/A .".into());
+        let t = std::time::Instant::now();
+        assert!(d.check("kot") && !d.check("pies") && !d.check("kotx"));
+        assert!(t.elapsed().as_secs_f64() < 10.0, "{:?}", t.elapsed());
+    }
+
+    #[test]
+    fn forms_of_self_continuing_rules_are_bounded() {
+        // Every rule's form continues with every rule: millions of strings for one stem (#3613).
+        let same = many_rules(3000, |_| "0 0/A .".into());
+        let distinct = many_rules(3000, |i| format!("0 x{i}/A ."));
+        let t = std::time::Instant::now();
+        let (a, b) = (same.forms("kot"), distinct.forms("kot"));
+        // The cap on the count is what tells the old code (9 million forms each) apart; the
+        // generous time bound (tens of ms here) only backs up the rule-visit cap.
+        assert!(t.elapsed().as_secs_f64() < 10.0, "{:?}", t.elapsed());
+        assert!(a.len() <= MAX_FORMS && b.len() <= MAX_FORMS, "{} {}", a.len(), b.len());
+        assert!(b.iter().any(|f| f == "kotx0x1"), "chained forms are still listed");
+    }
+
+    #[test]
+    fn many_strips_per_suffix_index_in_linear_time() {
+        // One append with thousands of strips: the index used to scan the append's groups for
+        // each rule. Building 200,000 rules takes well under a second; the old index took 3 s at
+        // 30,000 and over 10 minutes at 300,000.
+        let n = 200_000;
+        let t = std::time::Instant::now();
+        let d = many_rules(n, |i| format!("k{i} y ."));
+        assert!(t.elapsed().as_secs_f64() < 10.0, "{:?}", t.elapsed());
+        assert_eq!(d.sfx_index.get("y").map(Vec::len), Some(n));
+        assert!(!d.contains("koty"));
+    }
+
     #[test]
     fn compact_format_round_trips_continuations_and_forbidden() {
         let aff = "SET UTF-8\nFLAG UTF-8\nFORBIDDENWORD ý\nSFX A N 1\nSFX A   0   able/B   .\nSFX B N 1\nSFX B   0   s   .\n";

crates/proof/src/dict.rs

@@ -14,6 +14,11 @@ pub struct Dictionary {
 }
 
 const MAGIC: &[u8] = b"DCHYPH1\n";
+/// Largest inflated dictionary, and most bytes of words kept from one (the bundled US English one
+/// is about 1 MB).
+const MAX_BYTES: usize = 64 << 20;
+/// Most words read from one dictionary (the bundled one has about 160k).
+const MAX_ENTRIES: usize = 4_000_000;
 
 impl Dictionary {
     /// Build from `(lowercase word, break mask)` pairs (sorted and de-duplicated here; the first
@@ -76,9 +81,16 @@ impl Dictionary {
         miniz_oxide::deflate::compress_to_vec(&raw, 10)
     }
 
-    /// Deserialize data written by [`Dictionary::to_bytes`].
+    /// Deserialize data written by [`Dictionary::to_bytes`]. Data that inflates past
+    /// [`MAX_BYTES`], or holds more than [`MAX_ENTRIES`] words or [`MAX_BYTES`] of them, is refused.
     pub fn from_bytes(data: &[u8]) -> Result<Dictionary, String> {
-        let raw = miniz_oxide::inflate::decompress_to_vec(data).map_err(|e| format!("hyphenation dictionary: inflate failed: {e:?}"))?;
+        Dictionary::from_bytes_within(data, MAX_BYTES, MAX_ENTRIES)
+    }
+
+    /// [`Dictionary::from_bytes`] with the limits given.
+    fn from_bytes_within(data: &[u8], max_bytes: usize, max_entries: usize) -> Result<Dictionary, String> {
+        let raw = miniz_oxide::inflate::decompress_to_vec_with_limit(data, max_bytes)
+            .map_err(|e| format!("hyphenation dictionary: inflate failed: {:?}", e.status))?;
         let body = raw.strip_prefix(MAGIC).ok_or("hyphenation dictionary: bad header")?;
         let mut d = Dictionary { words: String::with_capacity(body.len()), offs: vec![0], masks: Vec::new() };
         let mut cur: Vec<u8> = Vec::new();
@@ -94,6 +106,11 @@ impl Dictionary {
             i = end + 1;
             let s = std::str::from_utf8(&cur).map_err(|_| "hyphenation dictionary: bad utf-8")?;
             let (w, m) = parse_hyphenated(s);
+            // Front coding lets a short line repeat a long prefix: bound what is kept, not only
+            // what is read.
+            if d.masks.len() >= max_entries || d.words.len() + w.len() > max_bytes {
+                return Err("hyphenation dictionary: too large".into());
+            }
             d.words.push_str(&w);
             d.offs.push(d.words.len() as u32);
             d.masks.push(m);
@@ -147,4 +164,31 @@ mod tests {
         assert_eq!(back.get("naïveté").map(|m| hyphenated("naïveté", m)), Some("naïve-té".to_string()));
         assert!(back.get("dog").is_none());
     }
+
+    #[test]
+    fn oversized_data_is_refused() {
+        const LIMIT: usize = 1 << 16;
+        let within = |raw: &[u8]| Dictionary::from_bytes_within(&miniz_oxide::deflate::compress_to_vec(raw, 6), LIMIT, 1000);
+        // Data that inflates past the limit is refused before anything is decoded.
+        let mut raw = MAGIC.to_vec();
+        raw.resize(LIMIT + 1, b'a');
+        assert!(within(&raw).unwrap_err().contains("inflate failed"));
+        // Short lines that each repeat a long prefix: small to inflate, but the words they spell
+        // out would be ~100x larger. Kept storage is bounded too.
+        let mut raw = MAGIC.to_vec();
+        raw.push(0);
+        raw.extend_from_slice(&[b'a'; 255]);
+        raw.push(b'\n');
+        for _ in 0..(LIMIT / 255 + 2) {
+            raw.extend_from_slice(&[255, b'\n']);
+        }
+        assert!(raw.len() < LIMIT);
+        assert!(within(&raw).unwrap_err().contains("too large"));
+        // So is the number of words.
+        let words: Vec<(String, u64)> = (0..1001).map(|i| (format!("w{i}"), 0)).collect();
+        let raw = miniz_oxide::inflate::decompress_to_vec(&Dictionary::from_entries(words).to_bytes()).unwrap();
+        assert!(within(&raw).unwrap_err().contains("too large"));
+        // The real limits leave the bundled dictionary whole.
+        assert!(Dictionary::en_us().len() > 100_000);
+    }
 }

crates/proof/src/hunspell.rs

@@ -140,16 +140,31 @@ pub fn find_in(dir: &Path, names: &[&str]) -> Option<(PathBuf, PathBuf)> {
     })
 }
 
+/// Read a dictionary file: a regular file of at most [`MAX_FILE`] bytes. The read itself is
+/// bounded, so a file that grows after its size is checked (or a filesystem that reports a wrong
+/// size) can't make it read more (#3661).
+pub(crate) fn read_capped(p: &Path) -> Result<Vec<u8>, String> {
+    use std::io::Read;
+    let err = |e: std::io::Error| format!("{}: {e}", p.display());
+    let f = std::fs::File::open(p).map_err(err)?;
+    let meta = f.metadata().map_err(err)?;
+    if !meta.is_file() {
+        return Err(format!("{}: not a regular file", p.display()));
+    }
+    if meta.len() > MAX_FILE {
+        return Err(format!("{}: too large ({} bytes)", p.display(), meta.len()));
+    }
+    let mut buf = Vec::new();
+    f.take(MAX_FILE + 1).read_to_end(&mut buf).map_err(err)?;
+    if buf.len() as u64 > MAX_FILE {
+        return Err(format!("{}: too large", p.display()));
+    }
+    Ok(buf)
+}
+
 /// Read a Hunspell pair from disk (size-capped), decoded per the `.aff`'s `SET` line.
 pub fn load(aff: &Path, dic: &Path) -> Result<AffixDict, String> {
-    let read = |p: &Path| -> Result<Vec<u8>, String> {
-        let len = std::fs::metadata(p).map_err(|e| format!("{}: {e}", p.display()))?.len();
-        if len > MAX_FILE {
-            return Err(format!("{}: too large ({len} bytes)", p.display()));
-        }
-        std::fs::read(p).map_err(|e| format!("{}: {e}", p.display()))
-    };
-    let (aff_text, dic_text) = decode(&read(aff)?, &read(dic)?)?;
+    let (aff_text, dic_text) = decode(&read_capped(aff)?, &read_capped(dic)?)?;
     let source = format!("Hunspell dictionary {}", dic.display());
     AffixDict::from_hunspell(&aff_text, &dic_text, &[source.as_str()])
 }
@@ -159,6 +174,23 @@ fn decode_latin1(bytes: &[u8]) -> String {
     bytes.iter().map(|&b| char::from(b)).collect()
 }
 
+/// ISO 8859-9 (Latin-5, Turkish): ISO 8859-1 with Ğ İ Ş ğ ı ş in place of Ð Ý Þ ð ý þ. The
+/// encoding of older Turkish dictionaries (`SET ISO8859-9`).
+fn decode_latin5(bytes: &[u8]) -> String {
+    bytes
+        .iter()
+        .map(|&b| match b {
+            0xd0 => 'Ğ',
+            0xdd => 'İ',
+            0xde => 'Ş',
+            0xf0 => 'ğ',
+            0xfd => 'ı',
+            0xfe => 'ş',
+            _ => char::from(b),
+        })
+        .collect()
+}
+
 /// ISO 8859-2 code points 0xA0..=0xFF (0x00..=0x9F map to themselves).
 const ISO_8859_2_HIGH: &str = "\u{a0}Ą˘Ł¤ĽŚ§¨ŠŞŤŹ\u{ad}ŽŻ°ą˛ł´ľśˇ¸šşťź˝žżŔÁÂĂÄĹĆÇČÉĘËĚÍÎĎĐŃŇÓÔŐÖ×ŘŮÚŰÜÝŢßŕáâăäĺćçčéęëěíîďđńňóôőö÷řůúűüýţ˙";
 
@@ -184,7 +216,8 @@ fn decode_high(bytes: &[u8], high: &str) -> String {
 
 /// Decode an `.aff`/`.dic` pair to UTF-8 from the encoding the `.aff`'s `SET` line names
 /// (UTF-8 when there is none; ISO 8859-2 is the other one Polish dictionaries use, ISO 8859-1 the
-/// one the original Portuguese VERO uses, KOI8-R and Windows-1251 the ones older Russian dictionaries use).
+/// one the original Portuguese VERO uses, ISO 8859-9 the one older Turkish dictionaries use, KOI8-R
+/// and Windows-1251 the ones older Russian dictionaries use).
 pub fn decode(aff: &[u8], dic: &[u8]) -> Result<(String, String), String> {
     let head = String::from_utf8_lossy(aff.get(..aff.len().min(8192)).unwrap_or_default()).to_string();
     let set = head.lines().find_map(|l| l.trim_start_matches('\u{feff}').strip_prefix("SET ")).map(str::trim).unwrap_or("UTF-8").to_ascii_uppercase();
@@ -195,6 +228,7 @@ pub fn decode(aff: &[u8], dic: &[u8]) -> Result<(String, String), String> {
         }
         "ISO8859-1" | "ISO-8859-1" => Ok((decode_latin1(aff), decode_latin1(dic))),
         "ISO8859-2" | "ISO-8859-2" => Ok((decode_latin2(aff), decode_latin2(dic))),
+        "ISO8859-9" | "ISO-8859-9" => Ok((decode_latin5(aff), decode_latin5(dic))),
         "KOI8-R" | "KOI8R" => Ok((decode_high(aff, KOI8_R_HIGH), decode_high(dic, KOI8_R_HIGH))),
         "CP1251" | "WINDOWS-1251" | "MICROSOFT-CP1251" => Ok((decode_high(aff, CP1251_HIGH), decode_high(dic, CP1251_HIGH))),
         other => Err(format!("unsupported encoding {other}")),
@@ -213,6 +247,8 @@ mod tests {
         assert_eq!(d, "2\nżółw\n");
         let (_, d) = decode("SET UTF-8\n".as_bytes(), "żółw".as_bytes()).unwrap();
         assert_eq!(d, "żółw");
+        let (_, d) = decode(b"SET ISO8859-9\n", b"\xd0\xdd\xde\xf0\xfd\xfe\xd6\xe7").unwrap();
+        assert_eq!(d, "ĞİŞğışÖç");
         assert!(decode(b"SET SHIFT_JIS\n", b"").is_err());
         assert!(decode(b"SET UTF-8\n", b"\xff\xfe").is_err());
     }
@@ -283,4 +319,52 @@ mod tests {
         assert!(load(&dir.join("missing.aff"), &dic).is_err());
         let _ = std::fs::remove_dir_all(&dir);
     }
+
+    #[test]
+    fn reads_are_bounded_and_regular_files_only() {
+        let dir = std::env::temp_dir().join(format!("wordcraft-hunspell-cap-{}", std::process::id()));
+        std::fs::create_dir_all(&dir).unwrap();
+        let aff = dir.join("xx.aff");
+        std::fs::write(&aff, "SET UTF-8\n").unwrap();
+        // One byte over the cap (sparse: nothing is written) is refused without reading it.
+        let big = dir.join("big.dic");
+        std::fs::File::create(&big).unwrap().set_len(MAX_FILE + 1).unwrap();
+        assert!(load(&aff, &big).unwrap_err().contains("too large"));
+        // A directory, and on Unix an endless device, are not dictionary files. (No clock: a
+        // reader that read `/dev/zero` would never return, so the error itself is the proof.)
+        assert!(load(&aff, &dir).unwrap_err().contains("not a regular file"));
+        #[cfg(unix)]
+        assert!(load(&aff, Path::new("/dev/zero")).unwrap_err().contains("not a regular file"));
+        // Turkish reads through the same reader.
+        let (tr_aff, tr_dic) = (dir.join("tr_TR.aff"), dir.join("tr_TR.dic"));
+        std::fs::write(&tr_aff, "SET UTF-8\nSFX 1 Y 1\nSFX 1 0 lar .\n").unwrap();
+        std::fs::write(&tr_dic, "1\nkitap/1\n").unwrap();
+        assert!(crate::tr::load(&tr_aff, &tr_dic).unwrap().contains("kitaplar"));
+        assert!(crate::tr::load(&tr_aff, &big).unwrap_err().contains("too large"));
+        let _ = std::fs::remove_dir_all(&dir);
+    }
+
+    /// A Turkish dictionary in ISO 8859-9, the encoding older tr_TR dictionaries use, read from
+    /// disk through the capped reader and decoded per its `SET` line: its words with Turkish
+    /// letters (ğ ı ş İ, and ç ö ü from Latin-1) are found, as stems and with a suffix. Before the
+    /// Turkish dictionary went through this reader it was read as UTF-8, which these bytes are not.
+    #[test]
+    fn a_turkish_dictionary_in_iso_8859_9_loads_from_disk() {
+        let dir = std::env::temp_dir().join(format!("wordcraft-hunspell-latin5-{}", std::process::id()));
+        std::fs::create_dir_all(&dir).unwrap();
+        let (aff, dic) = (dir.join("tr_TR.aff"), dir.join("tr_TR.dic"));
+        std::fs::write(&aff, b"SET ISO8859-9\nSFX 1 Y 2\nSFX 1 0 lar .\nSFX 1 0 \xfd .\nSFX 2 Y 1\nSFX 2 0 ler .\n").unwrap();
+        // ağaç/1 (-lar, -ı), ışık, şişe/2 (-ler), İzmir, öğrenci/2, güneş.
+        let words: &[u8] = b"6\na\xf0a\xe7/1\n\xfd\xfe\xfdk\n\xfei\xfee/2\n\xddzmir\n\xf6\xf0renci/2\ng\xfcne\xfe\n";
+        std::fs::write(&dic, words).unwrap();
+        assert!(String::from_utf8(words.to_vec()).is_err(), "the .dic is not UTF-8");
+        let d = crate::tr::load(&aff, &dic).unwrap();
+        for w in ["ağaç", "ağaçlar", "ağaçı", "ışık", "şişe", "şişeler", "İzmir", "öğrenci", "öğrenciler", "güneş"] {
+            assert!(d.contains(w), "{w}");
+        }
+        // The six letters ISO 8859-9 puts in place of ISO 8859-1's aren't read as those.
+        assert!(!d.contains("aðaç") && !d.contains("ýþýk") && !d.contains("Ýzmir"));
+        assert!(!d.contains("ağaçs") && !d.contains("ışıklar") && !d.contains("şişelar"), "only the rules' suffixes, on the stems that take them");
+        let _ = std::fs::remove_dir_all(&dir);
+    }
 }

crates/proof/src/hyphen.rs

@@ -320,10 +320,14 @@ pub fn hyphenate_word(word: &str, lim: &Limits) -> String {
 
 /// `word`, a word in `lang`, with `-` at every hyphenation point (tests and diagnostics).
 pub fn hyphenate_word_in(word: &str, lim: &Limits, lang: ProofLang) -> String {
-    let pts = hyphen_points_in(word, lim, lang);
-    let mut s = String::with_capacity(word.len() + pts.len());
+    let mut pts = hyphen_points_in(word, lim, lang);
+    pts.sort_unstable();
+    // Walk the points alongside the chars rather than searching them for each char.
+    let mut next = pts.into_iter().peekable();
+    let mut s = String::with_capacity(word.len() + 8);
     for (i, ch) in word.chars().enumerate() {
-        if pts.contains(&i) {
+        if next.next_if_eq(&i).is_some() {
+            while next.next_if_eq(&i).is_some() {}
             s.push('-');
         }
         s.push(ch);
@@ -706,6 +710,26 @@ mod tests {
         }
     }
 
+    #[test]
+    fn russian_hyphenation_and_hyphenated_output_are_linear() {
+        // A Russian word breaks at every syllable: each break used to be checked against all the
+        // earlier ones (#3657), and the hyphenated output searched every break for each char.
+        let word = |k: usize| "мама".repeat(k);
+        assert_eq!(word_breaks_in(&word(2), ProofLang::Ru).0, [2, 4, 6]);
+        assert_eq!(hyphenate_word_in(&word(2), &Limits::default(), ProofLang::Ru), "ма-ма-ма-ма");
+        // The breaks' rescan is a cheap one per break: it takes longer words to stand out.
+        for (what, k, repeat, f) in [
+            ("points", 1000, 3, (|w: &str| drop(hyphen_points_in(w, &Limits::default(), ProofLang::Ru))) as fn(&str)),
+            ("hyphenated", 400, 20, |w: &str| drop(hyphenate_word_in(w, &Limits::default(), ProofLang::Ru))),
+        ] {
+            let (small, big) = (word(k), word(8 * k));
+            let time = |w: &str| (0..repeat).for_each(|_| f(w));
+            let (t1, t8, ratio) = cost_ratio(8, || time(&small), || time(&big));
+            eprintln!("{what}: t(n) {t1:.4} s, t(8n) {t8:.4} s, ratio {ratio:.1}");
+            assert!(ratio < 24.0, "8x the letters, not linear: {what} {ratio:.1}");
+        }
+    }
+
     #[test]
     fn heuristic_fallback() {
         let c: Vec<char> = "happen".chars().collect();

crates/proof/src/lib.rs

@@ -790,7 +790,9 @@ fn spelling_issues(text: &str, lang: impl Fn(usize) -> ProofLang, opts: &Options
 /// The whitespace-delimited token enclosing the word at `a..b` of `text`: its byte range, and
 /// whether it looks like a web or e-mail address.
 fn enclosing_token(text: &str, a: usize, b: usize) -> (usize, usize, bool) {
-    let ts = text.get(..a).and_then(|t| t.rfind(char::is_whitespace)).map(|i| i + 1).unwrap_or(0);
+    // The token starts after the last white-space character, whatever its width (a no-break
+    // space is two bytes and an ideographic space is three).
+    let ts = text.get(..a).and_then(|t| t.char_indices().rev().find(|(_, c)| c.is_whitespace())).map(|(i, c)| i + c.len_utf8()).unwrap_or(0);
     let te = text.get(b..).and_then(|t| t.find(char::is_whitespace)).map(|i| b + i).unwrap_or(text.len());
     (ts, te, text.get(ts..te).is_some_and(is_address))
 }
@@ -1568,6 +1570,18 @@ mod tests {
         assert_eq!(bad.last(), Some(&"wugwug"), "{bad:?}");
     }
 
+    #[test]
+    fn address_after_a_wide_space_is_an_internet_token() {
+        // A web or e-mail address after a no-break space or an ideographic space is skipped,
+        // as it is after a plain space.
+        let t = "see\u{a0}https://xyzq.example/blorp and\u{3000}www.qwirpy.example or\u{a0}zxq@mail.example wugwug";
+        let bad: Vec<&str> = check_spelling(t).iter().map(|i| &t[i.start..i.end]).collect();
+        assert_eq!(bad, vec!["wugwug"]);
+        let opts = Options { ignore_internet: false, ..Options::default() };
+        let bad: Vec<&str> = check_spelling_with(t, &opts).iter().map(|i| &t[i.start..i.end]).collect();
+        assert!(["xyzq", "blorp", "qwirpy", "zxq", "wugwug"].iter().all(|w| bad.contains(w)), "{bad:?}");
+    }
+
     #[test]
     fn spelling_scales_over_one_long_token() {
         // #1335: thousands of punctuation-separated words inside a single

crates/proof/src/patterns.rs

@@ -217,11 +217,6 @@ impl Patterns {
         self.exceptions.len()
     }
 
-    #[allow(dead_code)]
-    pub(crate) fn from_parts(alpha: Alphabet, map: FxMap<u128, Box<[u8]>>, max_len: usize) -> Patterns {
-        Patterns { alpha, map, max_len, exceptions: HashMap::new() }
-    }
-
     /// Make `letters` part of the alphabet even if no pattern mentions them.
     pub fn add_letters(&mut self, letters: impl IntoIterator<Item = char>) {
         for c in letters {

crates/proof/src/ru/hyphen.rs

@@ -57,7 +57,9 @@ pub fn points(c: &[char]) -> Vec<usize> {
         while c.get(cut).is_some_and(|&x| sticks_left(x)) && cut < end {
             cut += 1;
         }
-        if cut >= 2 && n - cut >= 2 && !out.contains(&cut) {
+        // Cuts come in increasing order (each lies after the previous pair's vowel), so a repeat
+        // can only be the last one: no scan of the whole list per syllable (#3657).
+        if cut >= 2 && n - cut >= 2 && out.last().is_none_or(|&l| l < cut) {
             out.push(cut);
         }
     }

crates/proof/src/tr/dict.rs

@@ -28,8 +28,7 @@ pub struct TrDict {
 
 impl crate::slot::Dictionary for TrDict {
     fn load(aff: &std::path::Path, dic: &std::path::Path) -> Result<Self, String> {
-        let read = |p: &std::path::Path| std::fs::read_to_string(p).map_err(|e| e.to_string());
-        TrDict::from_hunspell(&read(aff)?, &read(dic)?)
+        super::load(aff, dic)
     }
 
     fn parse(aff: &str, dic: &str) -> Result<Self, String> {
@@ -102,7 +101,8 @@ impl TrDict {
 
     /// Check if word is correct (stem or affixed).
     pub fn contains(&self, word: &str) -> bool {
-        if self.is_empty() || word.is_empty() {
+        // Longer input is never a dictionary word, and isn't scanned (#3625).
+        if self.is_empty() || word.is_empty() || word.len() > crate::affix::MAX_WORD_BYTES {
             return false;
         }
 

crates/proof/src/tr/mod.rs

@@ -37,6 +37,13 @@ pub fn dictionary() -> Arc<TrDict> {
     DICT.get()
 }
 
+/// Read a Turkish Hunspell pair from disk: size-capped and decoded per the `.aff`'s `SET` line,
+/// like the other languages' dictionaries (#3661).
+pub(crate) fn load(aff: &std::path::Path, dic: &std::path::Path) -> Result<TrDict, String> {
+    let (aff_text, dic_text) = crate::hunspell::decode(&crate::hunspell::read_capped(aff)?, &crate::hunspell::read_capped(dic)?)?;
+    TrDict::from_hunspell(&aff_text, &dic_text)
+}
+
 /// Use this Turkish dictionary from now on (Hunspell `.aff` and `.dic` text). On error the
 /// current dictionary stays.
 pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> {

crates/proof/src/tr/suggest.rs

@@ -46,6 +46,12 @@ const COMMON_REPLACEMENTS: &[(&str, &str)] = &[
     ("deyil", "değil"),
 ];
 
+/// Longest word (in letters) given suggestions. Every candidate is a fresh copy of the word, so
+/// the cost grows with the square of its length (#3625); Turkish words run longer than most
+/// (`Çekoslovakyalılaştıramadıklarımızdanmışsınız` is 43 letters), so the cap is above the other
+/// languages' 40.
+const MAX_LETTERS: usize = 64;
+
 /// Produce spelling suggestions for a misspelled Turkish word.
 pub fn suggest(word: &str, max: usize) -> Vec<String> {
     let dict = super::dictionary();
@@ -54,7 +60,8 @@ pub fn suggest(word: &str, max: usize) -> Vec<String> {
 
 /// Suggest with a given dictionary.
 pub fn suggest_with(d: &TrDict, word: &str, max: usize) -> Vec<String> {
-    if word.is_empty() || max == 0 {
+    let n = word.chars().count();
+    if n == 0 || n > MAX_LETTERS || max == 0 {
         return Vec::new();
     }
 
@@ -72,6 +79,10 @@ pub fn suggest_with(d: &TrDict, word: &str, max: usize) -> Vec<String> {
             }
         }
     }
+    // Only the table takes more than one word (`bir kaç`).
+    if word.contains(char::is_whitespace) {
+        return suggestions;
+    }
 
     // 2. Diacritic substitutions (`c` -> `ç`, `g` -> `ğ`, `ı` -> `i`, `o` -> `ö`, `s` -> `ş`, `u` -> `ü`)
     if !d.is_empty() {
@@ -277,6 +288,28 @@ mod tests {
         assert!(sugs.contains(&"geldin mi".to_string()));
     }
 
+    #[test]
+    fn overlong_words_get_no_suggestions_promptly() {
+        let d = TrDict::from_hunspell("SET UTF-8\nSFX 1 Y 1\nSFX 1 0 lar .\n", "2\nkitap/1\nsonra\n").unwrap();
+        assert!(suggest_with(&d, "sorna", 3).contains(&"sonra".to_string()));
+        assert!(suggest_with(&d, "kitaplr", 3).contains(&"kitaplar".to_string()));
+        let t = std::time::Instant::now();
+        let long = "a".repeat(100_000);
+        assert!(suggest_with(&d, &long, 8).is_empty());
+        assert!(!d.contains(&long) && !d.contains(&format!("kitap{long}")));
+        assert!(!super::super::is_correct_with(&d, &long));
+        // Tens of milliseconds with the caps; without them this ran for over 5 minutes.
+        assert!(t.elapsed().as_secs_f64() < 10.0, "{:?}", t.elapsed());
+    }
+
+    #[test]
+    fn two_words_get_only_the_tables_replacement() {
+        let d = TrDict::from_hunspell("SET UTF-8\n", "2\nbirkaç\nsonra\n").unwrap();
+        assert_eq!(suggest_with(&d, "bir kaç", 3), vec!["birkaç".to_string()]);
+        assert_eq!(suggest_with(&d, "Bir kaç", 3), vec!["birkaç".to_string()]);
+        assert!(suggest_with(&d, "sorna sorna", 3).is_empty());
+    }
+
     #[test]
     fn turkish_to_lower_handles_i() {
         assert_eq!(turkish_to_lower("IŞIK"), "ışık");