diff --git a/crates/proof/README.md b/crates/proof/README.md index 5d38164a..de847ae9 100644 --- a/crates/proof/README.md +++ b/crates/proof/README.md @@ -86,8 +86,9 @@ English spelling uses the first of these that is there; `english_source(lang)` s the built-in list, which has both spellings. Each name is looked for in every folder before the next name; a file that can't be read is skipped for the next name or folder; one file is read once for every English that uses it. `COMPOUNDRULE`, `ONLYINCOMPOUND` and `COMPOUNDMIN` - (ordinals such as `21st`) are read; flag-based compounding (`COMPOUNDFLAG`…) and `OCONV` are - ignored for English rather than refused. None on the web. + (ordinals such as `21st`) are read; flag-based compounding (`COMPOUNDFLAG`…) is ignored for + English rather than refused; `OCONV` writes suggestions as the dictionary says (`don’t`), as for + every language. None on the web. 5. **Built-in list:** `data/en-words.dic`, the whole Moby word list (about 165,000 words, 344 KB compressed, words only) built into the crate, with `data/en-extra.txt` (WordCraft's supplement of modern words, a copy of craft-i18n's) and the inflection rules. A word it doesn't know is diff --git a/crates/proof/src/affix.rs b/crates/proof/src/affix.rs index 2c129488..ba0229ea 100644 --- a/crates/proof/src/affix.rs +++ b/crates/proof/src/affix.rs @@ -18,7 +18,8 @@ //! products, continuation classes on suffixes (at most two suffixes in a row, as in Hunspell), //! `FORBIDDENWORD`, `NOSUGGEST`, `KEEPCASE` (the stem is a word only in its own letter case), //! `NEEDAFFIX`, `CIRCUMFIX`, `FULLSTRIP`, input conversion (`ICONV`: ligatures such as `fi` read as -//! `fi`) and ignored characters (`IGNORE`), and the suggestion tables `TRY`, `KEY`, `MAP` and `REP`. +//! `fi`), output conversion of suggestions (`OCONV`) and ignored characters (`IGNORE`), and the +//! suggestion tables `TRY`, `KEY`, `MAP` and `REP`. //! Compounds by `COMPOUNDRULE` (patterns of flags with `*` and `?`, as SCOWL's English //! dictionaries use for ordinals: `21st`, `112th`; with long or numeric flags each flag is written //! in parentheses, `(aa)*(bb)`), with `ONLYINCOMPOUND` and `COMPOUNDMIN`; the parts of such a @@ -46,6 +47,14 @@ const MAX_INFLATED: usize = 64 << 20; pub const MAX_WORD_BYTES: usize = 120; /// Empty slot in the stem hash table. const EMPTY: u32 = u32::MAX; +/// Longest input converted (`ICONV`) and stripped of `IGNORE` characters before a lookup: longer +/// input can't come down to [`MAX_WORD_BYTES`] in a real dictionary, so it is no word and is +/// never converted (the conversion would cost time on hostile input for nothing). +const MAX_INPUT_BYTES: usize = 4 * MAX_WORD_BYTES; +/// Most pairs an `ICONV` or `OCONV` table may hold (the Korean dictionary's hold 11,172 each). +const MAX_CONV_PAIRS: usize = 1 << 16; +/// Longest text on either side of an `ICONV` or `OCONV` pair, in bytes. +const MAX_CONV_BYTES: usize = 64; /// Most flags one `.dic` entry or one continuation class field may carry. Real dictionaries use a /// few hundred at most (Belarusian: 264); a longer field is hostile and is refused (`.aff`) or /// skipped (`.dic`, though its `FORBIDDENWORD` and `NOSUGGEST` flags still apply to the word). @@ -77,7 +86,6 @@ const UNSUPPORTED: &[&str] = &[ "COMPOUNDLAST", "COMPOUNDMIDDLE", "COMPOUNDPERMITFLAG", - "OCONV", "PSEUDOROOT", ]; @@ -297,8 +305,10 @@ pub struct AffixDict { circumfix: Option, /// `FULLSTRIP`: an affix may strip (and so replace) a whole stem. full_strip: bool, - /// `ICONV`: input conversions applied before a lookup, `(from, to)`. - iconv: Vec<(String, String)>, + /// `ICONV`: input conversions applied before a lookup. + iconv: Conv, + /// `OCONV`: output conversions applied to suggestions. + oconv: Conv, /// `IGNORE`: characters removed from input words (and from the stems and appended texts). ignore: Box<[char]>, /// Provenance and licence lines, kept through [`AffixDict::to_bytes`]. @@ -318,6 +328,75 @@ pub struct AffixDict { compound_firsts: Vec, } +/// An `ICONV` or `OCONV` table: `(from, to)` pairs in source order, found by the first character +/// of `from` so that converting a word costs a look at the few pairs that could start at each +/// character, not at the whole table (the Korean tables hold 11,172 pairs each). +#[derive(Clone, Debug, Default)] +struct Conv { + pairs: Vec<(String, String)>, + /// `from` → index into `pairs` (of equal ones, the later pair: it is the one that wins). + by_from: FxMap, + /// Longest `from`, in bytes. + longest: usize, +} + +impl Conv { + /// The table of `pairs` (checked by [`Rules::push_conv`]: non-empty `from`, bounded sizes). + fn new(pairs: Vec<(String, String)>) -> Conv { + let mut by_from: FxMap = FxMap::default(); + for (i, (from, _)) in pairs.iter().enumerate() { + if let Ok(i) = u32::try_from(i) { + by_from.insert(from.clone(), i); + } + } + let longest = pairs.iter().map(|p| p.0.len()).max().unwrap_or(0); + Conv { pairs, by_from, longest } + } + + fn is_empty(&self) -> bool { + self.pairs.is_empty() + } + + /// The pair to apply at the start of `rest`: the longest `from` that starts it. One hash + /// lookup per prefix of `rest` up to the longest `from` ([`MAX_CONV_BYTES`]), however many + /// pairs there are (craft-libs#489 review: 65,536 pairs on one first character). + fn at(&self, rest: &str) -> Option<&(String, String)> { + let ends = rest.char_indices().map(|(i, _)| i).skip(1).chain(std::iter::once(rest.len())); + let ends: Vec = ends.take_while(|&e| e <= self.longest).collect(); + ends.iter().rev().find_map(|&e| self.by_from.get(rest.get(..e)?)).and_then(|&i| self.pairs.get(i as usize)) + } + + /// `w` converted: at each position the longest matching `from` is replaced by its `to`, and + /// characters in `drop` are removed (outside the replacements too). Borrowed when nothing + /// changes. + fn apply<'a>(&self, w: &'a str, drop: &[char]) -> std::borrow::Cow<'a, str> { + if self.is_empty() && drop.is_empty() { + return std::borrow::Cow::Borrowed(w); + } + // One pass: the input is copied only from the first change on. + let mut out: Option = None; + let mut rest = w; + while let Some(c) = rest.chars().next() { + let done = w.get(..w.len() - rest.len()).unwrap_or_default(); + match self.at(rest) { + Some((from, to)) => { + out.get_or_insert_with(|| done.to_string()).extend(to.chars().filter(|c| !drop.contains(c))); + rest = rest.get(from.len()..).unwrap_or_default(); + } + None => { + if drop.contains(&c) { + out.get_or_insert_with(|| done.to_string()); + } else if let Some(o) = out.as_mut() { + o.push(c); + } + rest = rest.get(c.len_utf8()..).unwrap_or_default(); + } + } + } + out.map_or(std::borrow::Cow::Borrowed(w), std::borrow::Cow::Owned) + } +} + fn hash_str(s: &str) -> u64 { let mut h = FxHasher::default(); h.write(s.as_bytes()); @@ -389,6 +468,7 @@ struct Rules { circumfix: Option, full_strip: bool, iconv: Vec<(String, String)>, + oconv: Vec<(String, String)>, ignore: Vec, /// `COMPOUNDRULE` patterns, as written (`n*1t`); read in the `FLAG` mode by /// [`AffixDict::build`], since the `FLAG` line may come after them. @@ -423,6 +503,22 @@ impl Rules { Ok(()) } + /// Add a pair to the `ICONV` (`output` false) or `OCONV` table: at most [`MAX_CONV_PAIRS`] + /// pairs of at most [`MAX_CONV_BYTES`] each side. A pair with nothing to convert is skipped. + fn push_conv(&mut self, output: bool, from: &str, to: &str) -> Result<(), String> { + let (name, table) = if output { ("OCONV", &mut self.oconv) } else { ("ICONV", &mut self.iconv) }; + if from.len() > MAX_CONV_BYTES || to.len() > MAX_CONV_BYTES { + return Err(format!("{name} pair longer than {MAX_CONV_BYTES} bytes")); + } + if table.len() >= MAX_CONV_PAIRS { + return Err(format!("{name} table larger than {MAX_CONV_PAIRS} pairs")); + } + if !from.is_empty() { + table.push((from.to_string(), to.to_string())); + } + Ok(()) + } + /// The one flag of a rule header or a single-flag directive. fn one_flag(&mut self, field: &str) -> Result { self.flag_read = true; @@ -701,9 +797,10 @@ fn intersects(a: &[u16], b: &[u16]) -> bool { /// Directives an English dictionary may carry that this engine doesn't implement but that don't /// change which English words are right, ignored by [`AffixDict::from_hunspell_english`]: /// flag-based compounding (English dictionaries compound only ordinals, by `COMPOUNDRULE`, which -/// is read) and output conversion (`OCONV`: how suggestions are written, such as `'` as `’`). +/// is read). Output conversion (`OCONV`) is read, for English as for every language: suggestions +/// are written as the dictionary says (`don’t`). const IGNORED_IN_ENGLISH: &[&str] = - &["COMPOUNDBEGIN", "COMPOUNDEND", "COMPOUNDFIRST", "COMPOUNDFLAG", "COMPOUNDLAST", "COMPOUNDMIDDLE", "COMPOUNDPERMITFLAG", "OCONV"]; + &["COMPOUNDBEGIN", "COMPOUNDEND", "COMPOUNDFIRST", "COMPOUNDFLAG", "COMPOUNDLAST", "COMPOUNDMIDDLE", "COMPOUNDPERMITFLAG"]; /// Read Hunspell `.aff` text (already decoded to UTF-8). `english`: ignore the directives in /// [`IGNORED_IN_ENGLISH`] instead of refusing them. @@ -715,6 +812,7 @@ fn parse_aff(aff: &str, english: bool) -> Result { let mut af_header = false; let mut iconv_header = false; let mut compound_header = false; + let mut oconv_header = false; for (n, line) in aff.lines().enumerate() { let f: Vec<&str> = split_fields(line).collect(); let Some(&dir) = f.first() else { continue }; @@ -729,7 +827,7 @@ fn parse_aff(aff: &str, english: bool) -> Result { // the words it marks are correct. "AM" | "LEMMA_PRESENT" => {} "SUBSTANDARD" => r.substandard = Some(r.one_flag(at(1)).map_err(|e| format!("line {}: {dir} needs one flag: {e}", n + 1))?), - // The first `AF` or `ICONV` line is the table's count, as in Hunspell, even with a + // The first `AF`, `ICONV` or `OCONV` line is the table's count, as in Hunspell, even with a // trailing comment (`AF 1559 # aliases`). "AF" => { if !af_header { @@ -744,11 +842,12 @@ fn parse_aff(aff: &str, english: bool) -> Result { r.aliases.push(at(1).to_string()); } } - "ICONV" => { - if !iconv_header && at(1).parse::().is_ok() { - iconv_header = true; + "ICONV" | "OCONV" => { + let header = if dir == "OCONV" { &mut oconv_header } else { &mut iconv_header }; + if !*header && at(1).parse::().is_ok() { + *header = true; } else if f.len() >= 3 { - r.iconv.push((at(1).to_string(), at(2).to_string())); + r.push_conv(dir == "OCONV", at(1), at(2)).map_err(|e| format!("line {}: {e}", n + 1))?; } } "IGNORE" => r.ignore = at(1).chars().collect(), @@ -848,10 +947,10 @@ impl AffixDict { AffixDict::from_hunspell_in(aff, dic, comments, false) } - /// [`AffixDict::from_hunspell`] for an English dictionary: flag-based compounding and output - /// conversion (`COMPOUNDFLAG`, `OCONV`…), which this engine doesn't implement and English - /// spelling doesn't need, are ignored rather than refused. `COMPOUNDRULE`, `ONLYINCOMPOUND` - /// and `COMPOUNDMIN` (ordinals: `21st`) are read, as for every language. + /// [`AffixDict::from_hunspell`] for an English dictionary: flag-based compounding + /// (`COMPOUNDFLAG`…), which this engine doesn't implement and English spelling doesn't need, + /// is ignored rather than refused. `COMPOUNDRULE`, `ONLYINCOMPOUND` and `COMPOUNDMIN` + /// (ordinals: `21st`) and `OCONV` are read, as for every language. pub fn from_hunspell_english(aff: &str, dic: &str, comments: &[&str]) -> Result { AffixDict::from_hunspell_in(aff, dic, comments, true) } @@ -989,7 +1088,8 @@ impl AffixDict { circumfix: rules.circumfix.and_then(id_of), full_strip: rules.full_strip, flag_mode: mode, - iconv: rules.iconv, + iconv: Conv::new(rules.iconv), + oconv: Conv::new(rules.oconv), ignore: rules.ignore.into_boxed_slice(), try_chars: rules.try_chars, key: rules.key, @@ -1354,33 +1454,19 @@ impl AffixDict { } /// `w` after the `ICONV` conversions (at each position the longest match wins) and without - /// the `IGNORE` characters. + /// the `IGNORE` characters. Input longer than [`MAX_INPUT_BYTES`] is no word and comes back + /// unchanged, so the length check after this rejects it without converting it. fn normalize<'a>(&self, w: &'a str) -> std::borrow::Cow<'a, str> { - if self.iconv.is_empty() && self.ignore.is_empty() { + if w.len() > MAX_INPUT_BYTES { return std::borrow::Cow::Borrowed(w); } - // One pass: the input is copied only from the first change on. - let mut out: Option = None; - let mut rest = w; - while let Some(c) = rest.chars().next() { - let done = w.get(..w.len() - rest.len()).unwrap_or_default(); - let hit = self.iconv.iter().filter(|(from, _)| !from.is_empty() && rest.starts_with(from.as_str())).max_by_key(|(from, _)| from.len()); - match hit { - Some((from, to)) => { - out.get_or_insert_with(|| done.to_string()).extend(to.chars().filter(|c| !self.ignore.contains(c))); - rest = rest.get(from.len()..).unwrap_or_default(); - } - None => { - if self.ignore.contains(&c) { - out.get_or_insert_with(|| done.to_string()); - } else if let Some(o) = out.as_mut() { - o.push(c); - } - rest = rest.get(c.len_utf8()..).unwrap_or_default(); - } - } - } - out.map_or(std::borrow::Cow::Borrowed(w), std::borrow::Cow::Owned) + self.iconv.apply(w, &self.ignore) + } + + /// A suggestion as it is shown: `w` after the `OCONV` conversions (the French dictionary + /// writes `’` for the `'` its stems use). + pub fn output<'a>(&self, w: &'a str) -> std::borrow::Cow<'a, str> { + self.oconv.apply(w, &[]) } /// [`AffixDict::contains`] on normalized input, where stems carrying an `excluded` flag @@ -1767,8 +1853,10 @@ impl AffixDict { out.push('\n'); } } - for (a, b) in &self.iconv { - out.push_str(&format!("ICONV {a} {b}\n")); + for (name, table) in [("ICONV", &self.iconv), ("OCONV", &self.oconv)] { + for (a, b) in &table.pairs { + out.push_str(&format!("{name} {a} {b}\n")); + } } if !self.ignore.is_empty() { out.push_str(&format!("IGNORE {}\n", self.ignore.iter().collect::())); @@ -1870,7 +1958,9 @@ impl AffixDict { _ => rules.circumfix = Some(c), } } - "ICONV" if f.len() == 3 => rules.iconv.push((at(1).to_string(), at(2).to_string())), + conv @ ("ICONV" | "OCONV") if f.len() == 3 => { + rules.push_conv(conv == "OCONV", at(1), at(2)).map_err(|e| format!("affix dictionary: {e}"))?; + } "IGNORE" => rules.ignore = at(1).chars().collect(), "COMPOUNDMIN" => { let min = at(1).parse::().map_err(|_| format!("affix dictionary: bad `{line}`"))?; @@ -2415,6 +2505,58 @@ pub(crate) mod tests { } } + #[test] + fn oconv_converts_suggestions_only() { + // Answers as Hunspell 1.7 gives them on the same files: `kbt` → `k’t`. + let aff = "SET UTF-8\nTRY ab\nOCONV 1\nOCONV a ’\n"; + let d = AffixDict::from_hunspell(aff, "1\nkat\n", &[]).unwrap(); + assert!(d.check("kat") && !d.check("k’t"), "OCONV doesn't change what is correct"); + assert_eq!(d.output("kat"), "k’t"); + assert!(matches!(d.output("kot"), std::borrow::Cow::Borrowed("kot"))); + assert_eq!(crate::affix_suggest::suggest_in(&d, "kbt", 4, &crate::affix_suggest::Tuning::DEFAULT), ["k’t"]); + let back = AffixDict::from_bytes(&d.to_bytes().unwrap()).unwrap(); + assert_eq!(back.output("kat"), "k’t"); + // The count line may carry a comment, as for ICONV. + let d = AffixDict::from_hunspell("OCONV 1 # apostrophes\nOCONV ' ’\n", "1\nl'eau\n", &[]).unwrap(); + assert_eq!(d.output("l'eau"), "l’eau"); + } + + #[test] + fn conversions_take_the_longest_match_and_the_later_of_equal_ones() { + let aff = "ICONV 3\nICONV a x\nICONV ab y\nICONV a z\n"; + let d = AffixDict::from_hunspell(aff, "2\ny\nzc\n", &[]).unwrap(); + assert_eq!(d.normalize("ab"), "y"); + assert_eq!(d.normalize("ac"), "zc"); + assert!(d.check("ab") && d.check("ac")); + } + + #[test] + fn conversion_tables_are_capped() { + let pairs = |n: usize| (0..n).map(|i| format!("ICONV q{i} r\n")).collect::(); + let ok = AffixDict::from_hunspell(&format!("ICONV {MAX_CONV_PAIRS}\n{}", pairs(MAX_CONV_PAIRS)), "1\nr\n", &[]).unwrap(); + assert!(ok.check("q65535") && !ok.check("q65536")); + let err = AffixDict::from_hunspell(&format!("ICONV 1\n{}", pairs(MAX_CONV_PAIRS + 1)), "1\nr\n", &[]).unwrap_err(); + assert!(err.contains("ICONV table larger"), "{err}"); + let long = "x".repeat(MAX_CONV_BYTES + 1); + for aff in [format!("ICONV 1\nICONV {long} y\n"), format!("OCONV 1\nOCONV y {long}\n")] { + assert!(AffixDict::from_hunspell(&aff, "1\nr\n", &[]).is_err(), "{aff}"); + } + let raw = format!("{MAGIC}\nOCONV {long} y\nWORDS\n\0kot\n"); + assert!(AffixDict::from_bytes(&miniz_oxide::deflate::compress_to_vec(raw.as_bytes(), 6)).is_err()); + } + + #[test] + fn long_input_is_rejected_before_it_is_converted() { + let d = AffixDict::from_hunspell("ICONV 1\nICONV fi fi\nIGNORE -\n", "1\nfine\n", &[]).unwrap(); + // Ignored characters may take a word past MAX_WORD_BYTES and still leave a word. + let spaced = "f-i-n-e".to_string() + &"-".repeat(MAX_WORD_BYTES); + assert!(spaced.len() > MAX_WORD_BYTES && spaced.len() <= MAX_INPUT_BYTES && d.check(&spaced)); + // Past MAX_INPUT_BYTES the input is no word and is returned as it is, unconverted. + let long = "fi".repeat(MAX_INPUT_BYTES); + assert!(matches!(d.normalize(&long), std::borrow::Cow::Borrowed(_))); + assert!(!d.check(&long) && !d.contains(&long) && !d.can_suggest(&long)); + } + #[test] fn normalizing_borrows_when_nothing_changes() { let d = AffixDict::from_hunspell("SET UTF-8\nICONV 1\nICONV fi fi\nIGNORE ()\n", "1\nfine\n", &[]).unwrap(); @@ -2780,8 +2922,9 @@ pub(crate) mod tests { /// craft-libs#482 review: an English dictionary with compounding and output conversion, as the /// usual en_US and en_GB ones have, loads for English: `COMPOUNDRULE` makes its ordinals, its - /// `ONLYINCOMPOUND` stems are words only inside them, and `COMPOUNDFLAG` and `OCONV` are - /// ignored. Other languages still refuse those two. (A synthetic `.aff` written for this test.) + /// `ONLYINCOMPOUND` stems are words only inside them, `COMPOUNDFLAG` is ignored (other + /// languages still refuse it) and `OCONV` is applied to suggestions. (A synthetic `.aff` + /// written for this test.) #[test] fn english_dictionaries_ignore_compounding_and_output_conversion() { let aff = "SET UTF-8\nTRY esianrtolcdugmphbyfvkwz'\nICONV 1\nICONV ’ '\nOCONV 1\nOCONV ' ’\nNOSUGGEST !\n\ @@ -2796,6 +2939,11 @@ pub(crate) mod tests { assert!(!d.check("0th") && !d.check("1th"), "parts of compounds only"); assert!(d.check("1st") && d.check("11th") && d.check("111th") && !d.check("11st"), "ordinals by COMPOUNDRULE"); assert!(!d.check("housed") && !d.check("hous")); + // OCONV is read and applied to suggestions, as Hunspell does: `'` is shown as `’`. + let d = AffixDict::from_hunspell_english(aff, "2\ndon't\nwalk/S\n", &[]).unwrap(); + assert_eq!(d.output("don't"), "don’t"); + assert!(d.check("don't") && d.check("don’t"), "ICONV reads it back"); + assert!(crate::affix_suggest::suggest_in(&d, "donn't", 4, &crate::affix_suggest::Tuning::DEFAULT).contains(&"don’t".to_string())); // The directives that change which words are right are still refused. assert!(AffixDict::from_hunspell_english("PSEUDOROOT X\n", dic, &[]).is_err(), "PSEUDOROOT"); } diff --git a/crates/proof/src/affix_suggest.rs b/crates/proof/src/affix_suggest.rs index 5029c310..adf35265 100644 --- a/crates/proof/src/affix_suggest.rs +++ b/crates/proof/src/affix_suggest.rs @@ -327,6 +327,8 @@ pub(crate) fn suggest_in(d: &AffixDict, word: &str, max: usize, t: &Tuning) -> V }; // A KEEPCASE word only exists in its dictionary case: `Km` would be flagged, so offer `km`. let t = if recased != text && text.split(' ').any(|p| d.case_locked(p)) { text } else { recased }; + // Shown as the dictionary's OCONV table writes it (`'` → `’` in French). + let t = d.output(&t).into_owned(); if t != word && !out.contains(&t) { out.push(t); } @@ -517,6 +519,50 @@ fn scan(col: &mut Collector, chars: &[char], map: &[Vec]) { mod tests { use super::*; + /// craft-libs#489 review repros: OCONV applies to a suggestion after its case is set, case + /// sensitively, as the hunspell 1.7.5 CLI answers on the same files (`KBT` → `KAT`, not + /// `K’T`), and every suggestion is right. + #[test] + fn oconv_suggestions_match_hunspell() { + type Case = (&'static str, &'static str, &'static [(&'static str, &'static str)]); + let cases: &[Case] = &[ + ("SET UTF-8\nTRY ab\nOCONV 1\nOCONV a ’\n", "1\nkat\n", &[("kbt", "k’t"), ("Kbt", "K’t"), ("KBT", "KAT"), ("kot", "k’t")]), + ("SET UTF-8\nTRY ab\nOCONV 1\nOCONV k q\n", "1\nkat\n", &[("kbt", "qat"), ("Kbt", "Kat"), ("KBT", "KAT"), ("kot", "qat")]), + ( + "SET UTF-8\nTRY esianrtolcdugmphbyfvkwz'\nICONV 1\nICONV ’ '\nOCONV 1\nOCONV ' ’\nSFX S Y 1\nSFX S 0 's .\n", + "2\ncolour/S\ndon't\n", + &[("dont", "don’t"), ("DONT", "DON’T"), ("colours", "colour’s")], + ), + ]; + for (aff, dic, words) in cases { + let d = AffixDict::from_hunspell(aff, dic, &[]).unwrap(); + for (word, want) in *words { + let got = suggest_in(&d, word, 8, &Tuning::DEFAULT); + assert!(got.iter().any(|g| g == want), "{word}: {got:?}, want {want}"); + for g in &got { + assert!(d.check(g) || !aff.contains("ICONV"), "{word} → {g} is not right"); + } + } + } + } + + /// craft-libs#489 review: 65,536 ICONV pairs on one first character cost one hash lookup per + /// prefix, not a scan of the table, per position of a long word and per suggestion. + #[test] + fn a_huge_conversion_table_on_one_character_stays_fast() { + let mut aff = String::from("SET UTF-8\nTRY abc\nICONV 65536\n"); + let a60 = "a".repeat(60); + for c in (0x100u32..).filter_map(char::from_u32).take(65_536) { + aff.push_str(&format!("ICONV {a60}{c} x\n")); + } + let d = AffixDict::from_hunspell(&aff, "1\nabc\n", &[]).unwrap(); + let t = std::time::Instant::now(); + assert!(!d.check(&"a".repeat(470))); + let _ = suggest_in(&d, "abd", 4, &Tuning::DEFAULT); + let _ = suggest_in(&d, &"a".repeat(40), 4, &Tuning::DEFAULT); + assert!(t.elapsed().as_secs() < 5, "{:?}", t.elapsed()); + } + #[test] fn an_extra_trailing_letter_is_dropped_before_a_short_word_split() { let d = AffixDict::from_hunspell("", "2\npočítač\nz\n", &[]).unwrap(); diff --git a/crates/proof/src/english.rs b/crates/proof/src/english.rs index a18f060f..d3dc5d74 100644 --- a/crates/proof/src/english.rs +++ b/crates/proof/src/english.rs @@ -10,7 +10,8 @@ //! right; never one only another English knows. When it is there, the system locations aren't //! searched. //! 4. **System Hunspell**: a pair for the run's own English in the system locations (none on the -//! web). +//! web). `COMPOUNDRULE` (ordinals) and `OCONV` are read; flag-based compounding is ignored +//! ([`crate::hunspell::load_english`]). //! 5. **Built-in word list**: [`WORDS`], the whole public-domain Moby word list compiled into //! the crate (without its break points), with WordCraft's own supplement of modern words //! ([`EXTRA`]) and the usual inflections. A word it doesn't know is misspelled. (The Moby list @@ -21,8 +22,8 @@ //! folder before the next name. Another English's dictionary is never used: it would mark this //! English's spellings (`colour` in an `en_US` one). A file that can't be read is skipped for the //! next name or folder, and one file is read once for every English that uses it. `COMPOUNDRULE` -//! (ordinals) is read; flag-based compounding and `OCONV` are ignored -//! ([`crate::hunspell::load_english`]). +//! (ordinals) and `OCONV` (suggestions are written as the dictionary says: `don’t`) are read; +//! flag-based compounding is ignored ([`crate::hunspell::load_english`]). //! //! Whichever it is, the dialect policy of `en-variants.txt` ([`crate::en_variants`], when it is //! embedded) flags the other dialect's spellings and accepts this dialect's spellings of known diff --git a/crates/proof/src/hunspell.rs b/crates/proof/src/hunspell.rs index 6032e1c3..7e02e737 100644 --- a/crates/proof/src/hunspell.rs +++ b/crates/proof/src/hunspell.rs @@ -226,9 +226,9 @@ pub fn load(aff: &Path, dic: &Path) -> Result { AffixDict::from_hunspell(&aff_text, &dic_text, &[source.as_str()]) } -/// [`load`] for an English dictionary: directives that only matter for other languages -/// (compounding, output conversion) are ignored rather than refused -/// ([`AffixDict::from_hunspell_english`]). +/// [`load`] for an English dictionary: flag-based compounding (`COMPOUNDFLAG`…), which this engine +/// does not implement, is ignored rather than refused ([`AffixDict::from_hunspell_english`]); `OCONV` +/// is applied to suggestions, as for every language. pub fn load_english(aff: &Path, dic: &Path) -> Result { let (aff_text, dic_text) = decode(&read_capped(aff)?, &read_capped(dic)?)?; let source = format!("Hunspell dictionary {}", dic.display()); diff --git a/crates/proof/src/lib.rs b/crates/proof/src/lib.rs index ff384792..3577fbf8 100644 --- a/crates/proof/src/lib.rs +++ b/crates/proof/src/lib.rs @@ -675,8 +675,10 @@ fn suggest_english(word: &str, max: usize, lang: ProofLang) -> Vec { } let shared = english::suggest(word, max.saturating_add(2), lang); for s in shared { - let s = english_preferred(&s.to_lowercase(), lang).map_or(s, |p| match_case(word, p)); - if !out.contains(&s) { + let s = in_dialect(&s, word, lang); + // `color's` and `color’s` are one suggestion (a dictionary's OCONV writes the second). + let key = s.replace('’', "'"); + if !out.iter().any(|o| o.replace('’', "'") == key) { out.push(s); } } @@ -684,6 +686,20 @@ fn suggest_english(word: &str, max: usize, lang: ProofLang) -> Vec { out } +/// A suggestion `s` for `word` in `lang`'s spelling. Suggestions come as the dictionary's OCONV +/// writes them (`colour’s`); the dialect lists use `'`, so the mapping reads `'` and the result +/// keeps the dictionary's apostrophe (`color’s`). +fn in_dialect(s: &str, word: &str, lang: ProofLang) -> String { + let curly = s.contains('’'); + match english_preferred(&s.to_lowercase().replace('’', "'"), lang) { + Some(p) => { + let p = match_case(word, p); + if curly { p.replace('\'', "’") } else { p } + } + None => s.to_string(), + } +} + /// Words in `text`: (byte start, byte end). Combining marks belong to the word they follow, so a /// word typed decomposed (`tambie\u{301}n`, NFD) is one word, as its precomposed form is. pub fn words(text: &str) -> Vec<(usize, usize)> { @@ -1067,6 +1083,18 @@ mod tests { use super::*; + /// craft-libs#489 review: a suggestion an OCONV table wrote with `’` is still mapped to this + /// dialect's spelling (`colour’s` → `color’s` in American English), and keeps its apostrophe. + #[test] + fn curly_suggestions_are_mapped_to_the_dialect() { + if english_preferred("colour's", ProofLang::En).is_none() { + return; // no dialect lists without CRAFT_I18N_DIR + } + assert_eq!(in_dialect("colour’s", "colours", ProofLang::En), "color’s"); + assert_eq!(in_dialect("colour's", "colours", ProofLang::En), "color's"); + assert_eq!(in_dialect("colour’s", "colours", ProofLang::EnGb), "colour’s"); + } + /// The non-English test dictionaries every dictionaries-folder test puts in its folder, so /// tests running meanwhile see the same words whichever of those folders is set. fn write_test_dictionaries(dir: &std::path::Path) {