diff --git a/crates/proof/Cargo.toml b/crates/proof/Cargo.toml index 234684a9..4c1a5b11 100644 --- a/crates/proof/Cargo.toml +++ b/crates/proof/Cargo.toml @@ -10,7 +10,7 @@ workspace = true [dependencies] log = { workspace = true } -miniz_oxide = "0.8" +miniz_oxide = "0.9" unicode-normalization = "0.1" [dev-dependencies] diff --git a/crates/proof/src/affix.rs b/crates/proof/src/affix.rs index 1839a103..4ec06846 100644 --- a/crates/proof/src/affix.rs +++ b/crates/proof/src/affix.rs @@ -120,6 +120,11 @@ fn parse_cond(text: &str) -> Result, String> { const MAX_CHAIN: u8 = 2; /// How many suffixes may sit under the outermost one: the depth `form_has_flag` walks. const INNER_CHAIN: u8 = MAX_CHAIN - 1; +/// Most forms [`AffixDict::forms`] lists for one stem (real stems have a few hundred at most). +pub const MAX_FORMS: usize = 4096; +/// Most rules [`AffixDict::forms`] tries along continuation chains and prefix–suffix cross +/// products for one stem: real dictionaries need a few thousand, hostile ones are cut off. +const MAX_FORMS_WORK: usize = 1 << 18; /// One prefix or suffix rule. #[derive(Clone, Debug)] @@ -182,6 +187,8 @@ pub struct AffixDict { suffixes: Vec, /// Appended string → groups of suffix rules by stripped string. sfx_index: FxMap, Vec>, + /// Per flag bit: the suffix rules of that flag, in rule order. + sfx_by_flag: Vec>, /// Longest appended suffix, in chars. max_sfx_chars: usize, try_chars: String, @@ -307,13 +314,12 @@ fn parse_aff(aff: &str) -> Result { match dir { "SET" | "LANG" | "NAME" | "VERSION" | "HOME" | "WORDCHARS" | "MAXNGRAMSUGS" | "MAXDIFF" | "ONLYMAXDIFF" | "NOSPLITSUGS" | "SUGSWITHDOTS" | "WARN" | "FORBIDWARN" | "MAXCPDSUGS" => {} - "FLAG" => { - // Per-character flag modes (`FLAG UTF-8`, as VERO's pt_BR dictionary uses) work - // with the one-character parsing here; only the multi-character encodings differ. - if !matches!(at(1), "UTF-8" | "DEFAULT" | "utf-8" | "default" | "UTF8" | "") { - return Err(format!("line {}: FLAG {} is not supported (one-character flags only)", n + 1, at(1))); - } + // Per-character flag modes (`FLAG UTF-8`, as VERO's pt_BR dictionary uses) work + // with the one-character parsing here; only the multi-character encodings differ. + "FLAG" if !matches!(at(1), "UTF-8" | "DEFAULT" | "utf-8" | "default" | "UTF8" | "") => { + return Err(format!("line {}: FLAG {} is not supported (one-character flags only)", n + 1, at(1))); } + "FLAG" => {} directive @ ("FORBIDDENWORD" | "NOSUGGEST") => { let mut it = at(1).chars().filter(|&c| c != EMOJI_SELECTOR); match (it.next(), it.next()) { @@ -419,7 +425,7 @@ impl AffixDict { } stems.push((word, flags)); } - AffixDict::build(rules, stems, comments.iter().map(|s| s.to_string()).collect()) + AffixDict::build(rules, stems, comments.iter().map(ToString::to_string).collect()) } fn build(rules: Rules, mut stems: Vec<(String, String)>, comments: Vec) -> Result { @@ -484,14 +490,27 @@ impl AffixDict { /// Build the suffix index and the stem hash table. fn index(&mut self) { let mut idx: FxMap, Vec> = FxMap::default(); + // Each (appended, stripped) pair's group, found by hash rather than by scanning the + // appended string's groups: a dictionary with many strips per suffix stays linear. + let mut group_of: FxMap<(&str, &str), usize> = FxMap::default(); + let mut by_flag: Vec> = vec![Vec::new(); 128]; for (i, r) in self.suffixes.iter().enumerate() { let groups = idx.entry(r.add.clone()).or_default(); let Ok(i) = u32::try_from(i) else { break }; - match groups.iter_mut().find(|g| g.strip == r.strip) { + match group_of.get(&(&*r.add, &*r.strip)).and_then(|&g| groups.get_mut(g)) { Some(g) => g.rules.push(i), - None => groups.push(SfxGroup { strip: r.strip.clone(), rules: vec![i] }), + None => { + group_of.insert((&r.add, &r.strip), groups.len()); + groups.push(SfxGroup { strip: r.strip.clone(), rules: vec![i] }); + } + } + if r.bit != 0 + && let Some(list) = by_flag.get_mut(r.bit.trailing_zeros() as usize) + { + list.push(i); } } + self.sfx_by_flag = by_flag; self.max_sfx_chars = self.suffixes.iter().map(|r| r.add.chars().count()).max().unwrap_or(0); self.sfx_index = idx; let n = self.len(); @@ -596,10 +615,14 @@ impl AffixDict { None } - /// Does `w` carry flag `bit` — as a stem, or as the form of a suffix rule whose - /// continuation classes include `bit` (chained suffixation, depth-bounded)? - fn form_has_flag(&self, w: &str, bit: u128, depth: u8, excluded: u128) -> bool { - if self.stem_flags_excluding(w, excluded).is_some_and(|f| f & bit != 0) { + /// Does `w` carry any flag of `flags` — as a stem, or as the form of a suffix rule whose + /// continuation classes include one (chained suffixation, depth-bounded)? + /// + /// The rules of a group that could lead to `w` are merged into one mask of their flags and + /// followed once, not once per rule: the answer is the same, but thousands of rules of one + /// flag cost one walk, not one each (#3613). + fn form_has_flag(&self, w: &str, flags: u128, depth: u8, excluded: u128) -> bool { + if self.stem_flags_excluding(w, excluded).is_some_and(|f| f & flags != 0) { return true; } if depth == 0 { @@ -614,10 +637,14 @@ impl AffixDict { root.clear(); root.push_str(base); root.push_str(&g.strip); - for r in g.rules.iter().filter_map(|&i| self.suffixes.get(i as usize)) { - if r.cont & bit != 0 && r.cond_matches(&root, true) && self.form_has_flag(&root, r.bit, depth - 1, excluded) { - return true; - } + let inner = g + .rules + .iter() + .filter_map(|&i| self.suffixes.get(i as usize)) + .filter(|r| r.cont & flags != 0 && r.cond_matches(&root, true)) + .fold(0u128, |m, r| m | r.bit); + if inner != 0 && self.form_has_flag(&root, inner, depth - 1, excluded) { + return true; } } } @@ -690,9 +717,13 @@ impl AffixDict { return true; } // Continuation classes: `root` need not be a stem carrying this rule's flag — - // it may be a form that gained the flag from another rule's continuation. - if need == 0 && rules().any(|r| r.cond_matches(&root, true) && self.form_has_flag(&root, r.bit, INNER_CHAIN, excluded)) { - return true; + // it may be a form that gained the flag from another rule's continuation. The + // matching rules' flags are looked for together, in one walk. + if need == 0 { + let flags = rules().filter(|r| r.cond_matches(&root, true)).fold(0u128, |m, r| m | r.bit); + if flags != 0 && self.form_has_flag(&root, flags, INNER_CHAIN, excluded) { + return true; + } } } } @@ -725,13 +756,19 @@ impl AffixDict { } } - /// Every form of the stem `stem` (itself included), unsorted, possibly with repeats. + /// Every form of the stem `stem` (itself included), unsorted, possibly with repeats; at most + /// [`MAX_FORMS`], and at most [`MAX_FORMS_WORK`] rules tried along the continuation chains and + /// cross products, so a dictionary with thousands of self-continuing rules can't make one stem + /// cost millions of strings (#3613). pub fn forms(&self, stem: &str) -> Vec { let Some(f) = self.stem_flags(stem) else { return Vec::new() }; let mut out = vec![stem.to_string()]; let mut cross_sfx = Vec::new(); - // Suffix forms queued with the continuation classes they carry, for chained expansion. + // Suffix forms queued with the continuation classes they carry, for chained expansion; + // each (form, classes) pair is expanded once. let mut chain: Vec<(String, u128)> = Vec::new(); + let mut queued: std::collections::HashSet<(String, u128)> = std::collections::HashSet::new(); + let mut work = 0usize; for r in &self.suffixes { if f & r.bit == 0 || !r.cond_matches(stem, true) { continue; @@ -741,29 +778,39 @@ impl AffixDict { if r.cross { cross_sfx.push(form.clone()); } - if r.cont != 0 { + if r.cont != 0 && queued.insert((form.clone(), r.cont)) { chain.push((form.clone(), r.cont)); } out.push(form); + if out.len() >= MAX_FORMS { + return out; + } } // Continuation classes chain suffixes (`workable/B` → `workables`), up to [`MAX_CHAIN`] // suffixes in all — the same bound the checker walks. - for _ in 0..INNER_CHAIN { + 'chain: for _ in 0..INNER_CHAIN { let level = std::mem::take(&mut chain); if level.is_empty() { break; } for (form, flags) in level { - for r in &self.suffixes { - if flags & r.bit == 0 || !r.cond_matches(&form, true) { + for r in self.suffixes_with(flags).into_iter().filter_map(|i| self.suffixes.get(i as usize)) { + work += 1; + if work > MAX_FORMS_WORK { + break 'chain; + } + if !r.cond_matches(&form, true) { continue; } let Some(base) = form.strip_suffix(&*r.strip) else { continue }; let next = format!("{base}{}", r.add); - if r.cont != 0 { + if r.cont != 0 && queued.insert((next.clone(), r.cont)) { chain.push((next.clone(), r.cont)); } out.push(next); + if out.len() >= MAX_FORMS { + return out; + } } } } @@ -775,15 +822,38 @@ impl AffixDict { out.push(format!("{}{rest}", r.add)); if r.cross { for s in &cross_sfx { + work += 1; + if out.len() >= MAX_FORMS || work > MAX_FORMS_WORK { + return out; + } if let Some(rest) = s.strip_prefix(&*r.strip) { out.push(format!("{}{rest}", r.add)); } } } + if out.len() >= MAX_FORMS { + return out; + } } out } + /// The suffix rules carrying any flag of `mask`, in rule order. + fn suffixes_with(&self, mask: u128) -> Vec { + let mut rules: Vec = Vec::new(); + let mut lists = 0; + for (bit, list) in self.sfx_by_flag.iter().enumerate() { + if bit < 128 && mask & (1u128 << bit) != 0 && !list.is_empty() { + rules.extend_from_slice(list); + lists += 1; + } + } + if lists > 1 { + rules.sort_unstable(); + } + rules + } + /// The `TRY` letters, most frequent first (suggestions try these). pub fn try_chars(&self) -> &str { &self.try_chars @@ -877,8 +947,8 @@ impl AffixDict { /// Read data written by [`AffixDict::to_bytes`]. pub fn from_bytes(data: &[u8]) -> Result { - let raw = - miniz_oxide::inflate::decompress_to_vec_with_limit(data, 64 << 20).map_err(|e| format!("affix dictionary: inflate failed: {e:?}"))?; + let raw = miniz_oxide::inflate::decompress_to_vec_with_limit(data, 64 << 20) + .map_err(|e| format!("affix dictionary: inflate failed: {:?}", e.status))?; let words_at = raw.windows(7).position(|w| w == b"\nWORDS\n").ok_or("affix dictionary: no WORDS section")?; let head = std::str::from_utf8(raw.get(..words_at).unwrap_or_default()).map_err(|_| "affix dictionary: bad utf-8 in the rules")?; let mut lines = head.lines(); @@ -1216,6 +1286,65 @@ pub(crate) mod tests { } } + #[test] + fn continuation_classes_merge_the_flags_of_a_group() { + // `s` under B and under C share one group (same append, same strip): a form carrying + // either flag takes it, and the stem carrying neither doesn't. + let aff = "SET UTF-8\nSFX A N 1\nSFX A 0 able/C .\nSFX D N 1\nSFX D 0 er/B .\nSFX B N 1\nSFX B 0 s .\nSFX C N 1\nSFX C 0 s [e]\n"; + let d = AffixDict::from_hunspell(aff, "2\nwork/A\nplay/D\n", &[]).unwrap(); + assert!(d.contains("workables") && d.contains("players")); + assert!(!d.contains("works") && !d.contains("plays") && !d.contains("workabless")); + assert!(d.forms("work").iter().any(|f| f == "workables") && d.forms("play").iter().any(|f| f == "players")); + } + + /// `n` suffix rules of flag A on one stem, `rule(i)` giving each rule's strip, append and condition. + fn many_rules(n: usize, rule: impl Fn(usize) -> String) -> AffixDict { + let mut aff = format!("SET UTF-8\nSFX A Y {n}\n"); + for i in 0..n { + aff.push_str(&format!("SFX A {}\n", rule(i))); + } + AffixDict::from_hunspell(&aff, "1\nkot/A\n", &[]).unwrap() + } + + #[test] + fn self_continuing_rules_check_in_linear_time() { + // Thousands of empty-append rules that continue with their own flag: each check used to + // follow every rule from every rule, millions of steps a word (#3613). The checks take + // about a millisecond; the old checker took 14 s for two of them at 16,000 rules, so at + // 40,000 (quadratic: minutes) the 10 s bound leaves a busy machine a wide margin. + let d = many_rules(40_000, |_| "0 0/A .".into()); + let t = std::time::Instant::now(); + assert!(d.check("kot") && !d.check("pies") && !d.check("kotx")); + assert!(t.elapsed().as_secs_f64() < 10.0, "{:?}", t.elapsed()); + } + + #[test] + fn forms_of_self_continuing_rules_are_bounded() { + // Every rule's form continues with every rule: millions of strings for one stem (#3613). + let same = many_rules(3000, |_| "0 0/A .".into()); + let distinct = many_rules(3000, |i| format!("0 x{i}/A .")); + let t = std::time::Instant::now(); + let (a, b) = (same.forms("kot"), distinct.forms("kot")); + // The cap on the count is what tells the old code (9 million forms each) apart; the + // generous time bound (tens of ms here) only backs up the rule-visit cap. + assert!(t.elapsed().as_secs_f64() < 10.0, "{:?}", t.elapsed()); + assert!(a.len() <= MAX_FORMS && b.len() <= MAX_FORMS, "{} {}", a.len(), b.len()); + assert!(b.iter().any(|f| f == "kotx0x1"), "chained forms are still listed"); + } + + #[test] + fn many_strips_per_suffix_index_in_linear_time() { + // One append with thousands of strips: the index used to scan the append's groups for + // each rule. Building 200,000 rules takes well under a second; the old index took 3 s at + // 30,000 and over 10 minutes at 300,000. + let n = 200_000; + let t = std::time::Instant::now(); + let d = many_rules(n, |i| format!("k{i} y .")); + assert!(t.elapsed().as_secs_f64() < 10.0, "{:?}", t.elapsed()); + assert_eq!(d.sfx_index.get("y").map(Vec::len), Some(n)); + assert!(!d.contains("koty")); + } + #[test] fn compact_format_round_trips_continuations_and_forbidden() { let aff = "SET UTF-8\nFLAG UTF-8\nFORBIDDENWORD ý\nSFX A N 1\nSFX A 0 able/B .\nSFX B N 1\nSFX B 0 s .\n"; diff --git a/crates/proof/src/dict.rs b/crates/proof/src/dict.rs index 83a54d44..81eb0de1 100644 --- a/crates/proof/src/dict.rs +++ b/crates/proof/src/dict.rs @@ -14,6 +14,11 @@ pub struct Dictionary { } const MAGIC: &[u8] = b"DCHYPH1\n"; +/// Largest inflated dictionary, and most bytes of words kept from one (the bundled US English one +/// is about 1 MB). +const MAX_BYTES: usize = 64 << 20; +/// Most words read from one dictionary (the bundled one has about 160k). +const MAX_ENTRIES: usize = 4_000_000; impl Dictionary { /// Build from `(lowercase word, break mask)` pairs (sorted and de-duplicated here; the first @@ -76,9 +81,16 @@ impl Dictionary { miniz_oxide::deflate::compress_to_vec(&raw, 10) } - /// Deserialize data written by [`Dictionary::to_bytes`]. + /// Deserialize data written by [`Dictionary::to_bytes`]. Data that inflates past + /// [`MAX_BYTES`], or holds more than [`MAX_ENTRIES`] words or [`MAX_BYTES`] of them, is refused. pub fn from_bytes(data: &[u8]) -> Result { - let raw = miniz_oxide::inflate::decompress_to_vec(data).map_err(|e| format!("hyphenation dictionary: inflate failed: {e:?}"))?; + Dictionary::from_bytes_within(data, MAX_BYTES, MAX_ENTRIES) + } + + /// [`Dictionary::from_bytes`] with the limits given. + fn from_bytes_within(data: &[u8], max_bytes: usize, max_entries: usize) -> Result { + let raw = miniz_oxide::inflate::decompress_to_vec_with_limit(data, max_bytes) + .map_err(|e| format!("hyphenation dictionary: inflate failed: {:?}", e.status))?; let body = raw.strip_prefix(MAGIC).ok_or("hyphenation dictionary: bad header")?; let mut d = Dictionary { words: String::with_capacity(body.len()), offs: vec![0], masks: Vec::new() }; let mut cur: Vec = Vec::new(); @@ -94,6 +106,11 @@ impl Dictionary { i = end + 1; let s = std::str::from_utf8(&cur).map_err(|_| "hyphenation dictionary: bad utf-8")?; let (w, m) = parse_hyphenated(s); + // Front coding lets a short line repeat a long prefix: bound what is kept, not only + // what is read. + if d.masks.len() >= max_entries || d.words.len() + w.len() > max_bytes { + return Err("hyphenation dictionary: too large".into()); + } d.words.push_str(&w); d.offs.push(d.words.len() as u32); d.masks.push(m); @@ -147,4 +164,31 @@ mod tests { assert_eq!(back.get("naïveté").map(|m| hyphenated("naïveté", m)), Some("naïve-té".to_string())); assert!(back.get("dog").is_none()); } + + #[test] + fn oversized_data_is_refused() { + const LIMIT: usize = 1 << 16; + let within = |raw: &[u8]| Dictionary::from_bytes_within(&miniz_oxide::deflate::compress_to_vec(raw, 6), LIMIT, 1000); + // Data that inflates past the limit is refused before anything is decoded. + let mut raw = MAGIC.to_vec(); + raw.resize(LIMIT + 1, b'a'); + assert!(within(&raw).unwrap_err().contains("inflate failed")); + // Short lines that each repeat a long prefix: small to inflate, but the words they spell + // out would be ~100x larger. Kept storage is bounded too. + let mut raw = MAGIC.to_vec(); + raw.push(0); + raw.extend_from_slice(&[b'a'; 255]); + raw.push(b'\n'); + for _ in 0..(LIMIT / 255 + 2) { + raw.extend_from_slice(&[255, b'\n']); + } + assert!(raw.len() < LIMIT); + assert!(within(&raw).unwrap_err().contains("too large")); + // So is the number of words. + let words: Vec<(String, u64)> = (0..1001).map(|i| (format!("w{i}"), 0)).collect(); + let raw = miniz_oxide::inflate::decompress_to_vec(&Dictionary::from_entries(words).to_bytes()).unwrap(); + assert!(within(&raw).unwrap_err().contains("too large")); + // The real limits leave the bundled dictionary whole. + assert!(Dictionary::en_us().len() > 100_000); + } } diff --git a/crates/proof/src/hunspell.rs b/crates/proof/src/hunspell.rs index 0e4303be..3449c8ca 100644 --- a/crates/proof/src/hunspell.rs +++ b/crates/proof/src/hunspell.rs @@ -140,16 +140,31 @@ pub fn find_in(dir: &Path, names: &[&str]) -> Option<(PathBuf, PathBuf)> { }) } +/// Read a dictionary file: a regular file of at most [`MAX_FILE`] bytes. The read itself is +/// bounded, so a file that grows after its size is checked (or a filesystem that reports a wrong +/// size) can't make it read more (#3661). +pub(crate) fn read_capped(p: &Path) -> Result, String> { + use std::io::Read; + let err = |e: std::io::Error| format!("{}: {e}", p.display()); + let f = std::fs::File::open(p).map_err(err)?; + let meta = f.metadata().map_err(err)?; + if !meta.is_file() { + return Err(format!("{}: not a regular file", p.display())); + } + if meta.len() > MAX_FILE { + return Err(format!("{}: too large ({} bytes)", p.display(), meta.len())); + } + let mut buf = Vec::new(); + f.take(MAX_FILE + 1).read_to_end(&mut buf).map_err(err)?; + if buf.len() as u64 > MAX_FILE { + return Err(format!("{}: too large", p.display())); + } + Ok(buf) +} + /// Read a Hunspell pair from disk (size-capped), decoded per the `.aff`'s `SET` line. pub fn load(aff: &Path, dic: &Path) -> Result { - let read = |p: &Path| -> Result, String> { - let len = std::fs::metadata(p).map_err(|e| format!("{}: {e}", p.display()))?.len(); - if len > MAX_FILE { - return Err(format!("{}: too large ({len} bytes)", p.display())); - } - std::fs::read(p).map_err(|e| format!("{}: {e}", p.display())) - }; - let (aff_text, dic_text) = decode(&read(aff)?, &read(dic)?)?; + let (aff_text, dic_text) = decode(&read_capped(aff)?, &read_capped(dic)?)?; let source = format!("Hunspell dictionary {}", dic.display()); AffixDict::from_hunspell(&aff_text, &dic_text, &[source.as_str()]) } @@ -159,6 +174,23 @@ fn decode_latin1(bytes: &[u8]) -> String { bytes.iter().map(|&b| char::from(b)).collect() } +/// ISO 8859-9 (Latin-5, Turkish): ISO 8859-1 with Ğ İ Ş ğ ı ş in place of Ð Ý Þ ð ý þ. The +/// encoding of older Turkish dictionaries (`SET ISO8859-9`). +fn decode_latin5(bytes: &[u8]) -> String { + bytes + .iter() + .map(|&b| match b { + 0xd0 => 'Ğ', + 0xdd => 'İ', + 0xde => 'Ş', + 0xf0 => 'ğ', + 0xfd => 'ı', + 0xfe => 'ş', + _ => char::from(b), + }) + .collect() +} + /// ISO 8859-2 code points 0xA0..=0xFF (0x00..=0x9F map to themselves). const ISO_8859_2_HIGH: &str = "\u{a0}Ą˘Ł¤ĽŚ§¨ŠŞŤŹ\u{ad}ŽŻ°ą˛ł´ľśˇ¸šşťź˝žżŔÁÂĂÄĹĆÇČÉĘËĚÍÎĎĐŃŇÓÔŐÖ×ŘŮÚŰÜÝŢßŕáâăäĺćçčéęëěíîďđńňóôőö÷řůúűüýţ˙"; @@ -184,7 +216,8 @@ fn decode_high(bytes: &[u8], high: &str) -> String { /// Decode an `.aff`/`.dic` pair to UTF-8 from the encoding the `.aff`'s `SET` line names /// (UTF-8 when there is none; ISO 8859-2 is the other one Polish dictionaries use, ISO 8859-1 the -/// one the original Portuguese VERO uses, KOI8-R and Windows-1251 the ones older Russian dictionaries use). +/// one the original Portuguese VERO uses, ISO 8859-9 the one older Turkish dictionaries use, KOI8-R +/// and Windows-1251 the ones older Russian dictionaries use). pub fn decode(aff: &[u8], dic: &[u8]) -> Result<(String, String), String> { let head = String::from_utf8_lossy(aff.get(..aff.len().min(8192)).unwrap_or_default()).to_string(); let set = head.lines().find_map(|l| l.trim_start_matches('\u{feff}').strip_prefix("SET ")).map(str::trim).unwrap_or("UTF-8").to_ascii_uppercase(); @@ -195,6 +228,7 @@ pub fn decode(aff: &[u8], dic: &[u8]) -> Result<(String, String), String> { } "ISO8859-1" | "ISO-8859-1" => Ok((decode_latin1(aff), decode_latin1(dic))), "ISO8859-2" | "ISO-8859-2" => Ok((decode_latin2(aff), decode_latin2(dic))), + "ISO8859-9" | "ISO-8859-9" => Ok((decode_latin5(aff), decode_latin5(dic))), "KOI8-R" | "KOI8R" => Ok((decode_high(aff, KOI8_R_HIGH), decode_high(dic, KOI8_R_HIGH))), "CP1251" | "WINDOWS-1251" | "MICROSOFT-CP1251" => Ok((decode_high(aff, CP1251_HIGH), decode_high(dic, CP1251_HIGH))), other => Err(format!("unsupported encoding {other}")), @@ -213,6 +247,8 @@ mod tests { assert_eq!(d, "2\nżółw\n"); let (_, d) = decode("SET UTF-8\n".as_bytes(), "żółw".as_bytes()).unwrap(); assert_eq!(d, "żółw"); + let (_, d) = decode(b"SET ISO8859-9\n", b"\xd0\xdd\xde\xf0\xfd\xfe\xd6\xe7").unwrap(); + assert_eq!(d, "ĞİŞğışÖç"); assert!(decode(b"SET SHIFT_JIS\n", b"").is_err()); assert!(decode(b"SET UTF-8\n", b"\xff\xfe").is_err()); } @@ -283,4 +319,52 @@ mod tests { assert!(load(&dir.join("missing.aff"), &dic).is_err()); let _ = std::fs::remove_dir_all(&dir); } + + #[test] + fn reads_are_bounded_and_regular_files_only() { + let dir = std::env::temp_dir().join(format!("wordcraft-hunspell-cap-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let aff = dir.join("xx.aff"); + std::fs::write(&aff, "SET UTF-8\n").unwrap(); + // One byte over the cap (sparse: nothing is written) is refused without reading it. + let big = dir.join("big.dic"); + std::fs::File::create(&big).unwrap().set_len(MAX_FILE + 1).unwrap(); + assert!(load(&aff, &big).unwrap_err().contains("too large")); + // A directory, and on Unix an endless device, are not dictionary files. (No clock: a + // reader that read `/dev/zero` would never return, so the error itself is the proof.) + assert!(load(&aff, &dir).unwrap_err().contains("not a regular file")); + #[cfg(unix)] + assert!(load(&aff, Path::new("/dev/zero")).unwrap_err().contains("not a regular file")); + // Turkish reads through the same reader. + let (tr_aff, tr_dic) = (dir.join("tr_TR.aff"), dir.join("tr_TR.dic")); + std::fs::write(&tr_aff, "SET UTF-8\nSFX 1 Y 1\nSFX 1 0 lar .\n").unwrap(); + std::fs::write(&tr_dic, "1\nkitap/1\n").unwrap(); + assert!(crate::tr::load(&tr_aff, &tr_dic).unwrap().contains("kitaplar")); + assert!(crate::tr::load(&tr_aff, &big).unwrap_err().contains("too large")); + let _ = std::fs::remove_dir_all(&dir); + } + + /// A Turkish dictionary in ISO 8859-9, the encoding older tr_TR dictionaries use, read from + /// disk through the capped reader and decoded per its `SET` line: its words with Turkish + /// letters (ğ ı ş İ, and ç ö ü from Latin-1) are found, as stems and with a suffix. Before the + /// Turkish dictionary went through this reader it was read as UTF-8, which these bytes are not. + #[test] + fn a_turkish_dictionary_in_iso_8859_9_loads_from_disk() { + let dir = std::env::temp_dir().join(format!("wordcraft-hunspell-latin5-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let (aff, dic) = (dir.join("tr_TR.aff"), dir.join("tr_TR.dic")); + std::fs::write(&aff, b"SET ISO8859-9\nSFX 1 Y 2\nSFX 1 0 lar .\nSFX 1 0 \xfd .\nSFX 2 Y 1\nSFX 2 0 ler .\n").unwrap(); + // ağaç/1 (-lar, -ı), ışık, şişe/2 (-ler), İzmir, öğrenci/2, güneş. + let words: &[u8] = b"6\na\xf0a\xe7/1\n\xfd\xfe\xfdk\n\xfei\xfee/2\n\xddzmir\n\xf6\xf0renci/2\ng\xfcne\xfe\n"; + std::fs::write(&dic, words).unwrap(); + assert!(String::from_utf8(words.to_vec()).is_err(), "the .dic is not UTF-8"); + let d = crate::tr::load(&aff, &dic).unwrap(); + for w in ["ağaç", "ağaçlar", "ağaçı", "ışık", "şişe", "şişeler", "İzmir", "öğrenci", "öğrenciler", "güneş"] { + assert!(d.contains(w), "{w}"); + } + // The six letters ISO 8859-9 puts in place of ISO 8859-1's aren't read as those. + assert!(!d.contains("aðaç") && !d.contains("ýþýk") && !d.contains("Ýzmir")); + assert!(!d.contains("ağaçs") && !d.contains("ışıklar") && !d.contains("şişelar"), "only the rules' suffixes, on the stems that take them"); + let _ = std::fs::remove_dir_all(&dir); + } } diff --git a/crates/proof/src/hyphen.rs b/crates/proof/src/hyphen.rs index ed9f77f9..b203ff1a 100644 --- a/crates/proof/src/hyphen.rs +++ b/crates/proof/src/hyphen.rs @@ -320,10 +320,14 @@ pub fn hyphenate_word(word: &str, lim: &Limits) -> String { /// `word`, a word in `lang`, with `-` at every hyphenation point (tests and diagnostics). pub fn hyphenate_word_in(word: &str, lim: &Limits, lang: ProofLang) -> String { - let pts = hyphen_points_in(word, lim, lang); - let mut s = String::with_capacity(word.len() + pts.len()); + let mut pts = hyphen_points_in(word, lim, lang); + pts.sort_unstable(); + // Walk the points alongside the chars rather than searching them for each char. + let mut next = pts.into_iter().peekable(); + let mut s = String::with_capacity(word.len() + 8); for (i, ch) in word.chars().enumerate() { - if pts.contains(&i) { + if next.next_if_eq(&i).is_some() { + while next.next_if_eq(&i).is_some() {} s.push('-'); } s.push(ch); @@ -706,6 +710,26 @@ mod tests { } } + #[test] + fn russian_hyphenation_and_hyphenated_output_are_linear() { + // A Russian word breaks at every syllable: each break used to be checked against all the + // earlier ones (#3657), and the hyphenated output searched every break for each char. + let word = |k: usize| "мама".repeat(k); + assert_eq!(word_breaks_in(&word(2), ProofLang::Ru).0, [2, 4, 6]); + assert_eq!(hyphenate_word_in(&word(2), &Limits::default(), ProofLang::Ru), "ма-ма-ма-ма"); + // The breaks' rescan is a cheap one per break: it takes longer words to stand out. + for (what, k, repeat, f) in [ + ("points", 1000, 3, (|w: &str| drop(hyphen_points_in(w, &Limits::default(), ProofLang::Ru))) as fn(&str)), + ("hyphenated", 400, 20, |w: &str| drop(hyphenate_word_in(w, &Limits::default(), ProofLang::Ru))), + ] { + let (small, big) = (word(k), word(8 * k)); + let time = |w: &str| (0..repeat).for_each(|_| f(w)); + let (t1, t8, ratio) = cost_ratio(8, || time(&small), || time(&big)); + eprintln!("{what}: t(n) {t1:.4} s, t(8n) {t8:.4} s, ratio {ratio:.1}"); + assert!(ratio < 24.0, "8x the letters, not linear: {what} {ratio:.1}"); + } + } + #[test] fn heuristic_fallback() { let c: Vec = "happen".chars().collect(); diff --git a/crates/proof/src/lib.rs b/crates/proof/src/lib.rs index 365229d7..473cd582 100644 --- a/crates/proof/src/lib.rs +++ b/crates/proof/src/lib.rs @@ -790,7 +790,9 @@ fn spelling_issues(text: &str, lang: impl Fn(usize) -> ProofLang, opts: &Options /// The whitespace-delimited token enclosing the word at `a..b` of `text`: its byte range, and /// whether it looks like a web or e-mail address. fn enclosing_token(text: &str, a: usize, b: usize) -> (usize, usize, bool) { - let ts = text.get(..a).and_then(|t| t.rfind(char::is_whitespace)).map(|i| i + 1).unwrap_or(0); + // The token starts after the last white-space character, whatever its width (a no-break + // space is two bytes and an ideographic space is three). + let ts = text.get(..a).and_then(|t| t.char_indices().rev().find(|(_, c)| c.is_whitespace())).map(|(i, c)| i + c.len_utf8()).unwrap_or(0); let te = text.get(b..).and_then(|t| t.find(char::is_whitespace)).map(|i| b + i).unwrap_or(text.len()); (ts, te, text.get(ts..te).is_some_and(is_address)) } @@ -1568,6 +1570,18 @@ mod tests { assert_eq!(bad.last(), Some(&"wugwug"), "{bad:?}"); } + #[test] + fn address_after_a_wide_space_is_an_internet_token() { + // A web or e-mail address after a no-break space or an ideographic space is skipped, + // as it is after a plain space. + let t = "see\u{a0}https://xyzq.example/blorp and\u{3000}www.qwirpy.example or\u{a0}zxq@mail.example wugwug"; + let bad: Vec<&str> = check_spelling(t).iter().map(|i| &t[i.start..i.end]).collect(); + assert_eq!(bad, vec!["wugwug"]); + let opts = Options { ignore_internet: false, ..Options::default() }; + let bad: Vec<&str> = check_spelling_with(t, &opts).iter().map(|i| &t[i.start..i.end]).collect(); + assert!(["xyzq", "blorp", "qwirpy", "zxq", "wugwug"].iter().all(|w| bad.contains(w)), "{bad:?}"); + } + #[test] fn spelling_scales_over_one_long_token() { // #1335: thousands of punctuation-separated words inside a single diff --git a/crates/proof/src/patterns.rs b/crates/proof/src/patterns.rs index a91235e4..cb5855d4 100644 --- a/crates/proof/src/patterns.rs +++ b/crates/proof/src/patterns.rs @@ -217,11 +217,6 @@ impl Patterns { self.exceptions.len() } - #[allow(dead_code)] - pub(crate) fn from_parts(alpha: Alphabet, map: FxMap>, max_len: usize) -> Patterns { - Patterns { alpha, map, max_len, exceptions: HashMap::new() } - } - /// Make `letters` part of the alphabet even if no pattern mentions them. pub fn add_letters(&mut self, letters: impl IntoIterator) { for c in letters { diff --git a/crates/proof/src/ru/hyphen.rs b/crates/proof/src/ru/hyphen.rs index c02aa127..33d19b77 100644 --- a/crates/proof/src/ru/hyphen.rs +++ b/crates/proof/src/ru/hyphen.rs @@ -57,7 +57,9 @@ pub fn points(c: &[char]) -> Vec { while c.get(cut).is_some_and(|&x| sticks_left(x)) && cut < end { cut += 1; } - if cut >= 2 && n - cut >= 2 && !out.contains(&cut) { + // Cuts come in increasing order (each lies after the previous pair's vowel), so a repeat + // can only be the last one: no scan of the whole list per syllable (#3657). + if cut >= 2 && n - cut >= 2 && out.last().is_none_or(|&l| l < cut) { out.push(cut); } } diff --git a/crates/proof/src/tr/dict.rs b/crates/proof/src/tr/dict.rs index b47ba7c1..8cbacf02 100644 --- a/crates/proof/src/tr/dict.rs +++ b/crates/proof/src/tr/dict.rs @@ -28,8 +28,7 @@ pub struct TrDict { impl crate::slot::Dictionary for TrDict { fn load(aff: &std::path::Path, dic: &std::path::Path) -> Result { - let read = |p: &std::path::Path| std::fs::read_to_string(p).map_err(|e| e.to_string()); - TrDict::from_hunspell(&read(aff)?, &read(dic)?) + super::load(aff, dic) } fn parse(aff: &str, dic: &str) -> Result { @@ -102,7 +101,8 @@ impl TrDict { /// Check if word is correct (stem or affixed). pub fn contains(&self, word: &str) -> bool { - if self.is_empty() || word.is_empty() { + // Longer input is never a dictionary word, and isn't scanned (#3625). + if self.is_empty() || word.is_empty() || word.len() > crate::affix::MAX_WORD_BYTES { return false; } diff --git a/crates/proof/src/tr/mod.rs b/crates/proof/src/tr/mod.rs index f247336b..967148d8 100644 --- a/crates/proof/src/tr/mod.rs +++ b/crates/proof/src/tr/mod.rs @@ -37,6 +37,13 @@ pub fn dictionary() -> Arc { DICT.get() } +/// Read a Turkish Hunspell pair from disk: size-capped and decoded per the `.aff`'s `SET` line, +/// like the other languages' dictionaries (#3661). +pub(crate) fn load(aff: &std::path::Path, dic: &std::path::Path) -> Result { + let (aff_text, dic_text) = crate::hunspell::decode(&crate::hunspell::read_capped(aff)?, &crate::hunspell::read_capped(dic)?)?; + TrDict::from_hunspell(&aff_text, &dic_text) +} + /// Use this Turkish dictionary from now on (Hunspell `.aff` and `.dic` text). On error the /// current dictionary stays. pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> { diff --git a/crates/proof/src/tr/suggest.rs b/crates/proof/src/tr/suggest.rs index f91f8e3c..08874420 100644 --- a/crates/proof/src/tr/suggest.rs +++ b/crates/proof/src/tr/suggest.rs @@ -46,6 +46,12 @@ const COMMON_REPLACEMENTS: &[(&str, &str)] = &[ ("deyil", "değil"), ]; +/// Longest word (in letters) given suggestions. Every candidate is a fresh copy of the word, so +/// the cost grows with the square of its length (#3625); Turkish words run longer than most +/// (`Çekoslovakyalılaştıramadıklarımızdanmışsınız` is 43 letters), so the cap is above the other +/// languages' 40. +const MAX_LETTERS: usize = 64; + /// Produce spelling suggestions for a misspelled Turkish word. pub fn suggest(word: &str, max: usize) -> Vec { let dict = super::dictionary(); @@ -54,7 +60,8 @@ pub fn suggest(word: &str, max: usize) -> Vec { /// Suggest with a given dictionary. pub fn suggest_with(d: &TrDict, word: &str, max: usize) -> Vec { - if word.is_empty() || max == 0 { + let n = word.chars().count(); + if n == 0 || n > MAX_LETTERS || max == 0 { return Vec::new(); } @@ -72,6 +79,10 @@ pub fn suggest_with(d: &TrDict, word: &str, max: usize) -> Vec { } } } + // Only the table takes more than one word (`bir kaç`). + if word.contains(char::is_whitespace) { + return suggestions; + } // 2. Diacritic substitutions (`c` -> `ç`, `g` -> `ğ`, `ı` -> `i`, `o` -> `ö`, `s` -> `ş`, `u` -> `ü`) if !d.is_empty() { @@ -277,6 +288,28 @@ mod tests { assert!(sugs.contains(&"geldin mi".to_string())); } + #[test] + fn overlong_words_get_no_suggestions_promptly() { + let d = TrDict::from_hunspell("SET UTF-8\nSFX 1 Y 1\nSFX 1 0 lar .\n", "2\nkitap/1\nsonra\n").unwrap(); + assert!(suggest_with(&d, "sorna", 3).contains(&"sonra".to_string())); + assert!(suggest_with(&d, "kitaplr", 3).contains(&"kitaplar".to_string())); + let t = std::time::Instant::now(); + let long = "a".repeat(100_000); + assert!(suggest_with(&d, &long, 8).is_empty()); + assert!(!d.contains(&long) && !d.contains(&format!("kitap{long}"))); + assert!(!super::super::is_correct_with(&d, &long)); + // Tens of milliseconds with the caps; without them this ran for over 5 minutes. + assert!(t.elapsed().as_secs_f64() < 10.0, "{:?}", t.elapsed()); + } + + #[test] + fn two_words_get_only_the_tables_replacement() { + let d = TrDict::from_hunspell("SET UTF-8\n", "2\nbirkaç\nsonra\n").unwrap(); + assert_eq!(suggest_with(&d, "bir kaç", 3), vec!["birkaç".to_string()]); + assert_eq!(suggest_with(&d, "Bir kaç", 3), vec!["birkaç".to_string()]); + assert!(suggest_with(&d, "sorna sorna", 3).is_empty()); + } + #[test] fn turkish_to_lower_handles_i() { assert_eq!(turkish_to_lower("IŞIK"), "ışık");