Merge pull request #489 from benandez/p6-oconv

craft-proof: OCONV, and capped, indexed ICONV/OCONV tables

crates/proof/README.md

@@ -86,8 +86,9 @@ English spelling uses the first of these that is there; `english_source(lang)` s
    the built-in list, which has both spellings. Each name is looked for in every folder before the
    next name; a file that can't be read is skipped for the next name or folder; one file is read
    once for every English that uses it. `COMPOUNDRULE`, `ONLYINCOMPOUND` and `COMPOUNDMIN`
-   (ordinals such as `21st`) are read; flag-based compounding (`COMPOUNDFLAG`…) and `OCONV` are
-   ignored for English rather than refused. None on the web.
+   (ordinals such as `21st`) are read; flag-based compounding (`COMPOUNDFLAG`…) is ignored for
+   English rather than refused; `OCONV` writes suggestions as the dictionary says (`don’t`), as for
+   every language. None on the web.
 5. **Built-in list:** `data/en-words.dic`, the whole Moby word list (about 165,000 words, 344 KB
    compressed, words only) built into the crate, with `data/en-extra.txt` (WordCraft's supplement
    of modern words, a copy of craft-i18n's) and the inflection rules. A word it doesn't know is

crates/proof/src/affix.rs

@@ -18,7 +18,8 @@
 //! products, continuation classes on suffixes (at most two suffixes in a row, as in Hunspell),
 //! `FORBIDDENWORD`, `NOSUGGEST`, `KEEPCASE` (the stem is a word only in its own letter case),
 //! `NEEDAFFIX`, `CIRCUMFIX`, `FULLSTRIP`, input conversion (`ICONV`: ligatures such as `fi` read as
-//! `fi`) and ignored characters (`IGNORE`), and the suggestion tables `TRY`, `KEY`, `MAP` and `REP`.
+//! `fi`), output conversion of suggestions (`OCONV`) and ignored characters (`IGNORE`), and the
+//! suggestion tables `TRY`, `KEY`, `MAP` and `REP`.
 //! Compounds by `COMPOUNDRULE` (patterns of flags with `*` and `?`, as SCOWL's English
 //! dictionaries use for ordinals: `21st`, `112th`; with long or numeric flags each flag is written
 //! in parentheses, `(aa)*(bb)`), with `ONLYINCOMPOUND` and `COMPOUNDMIN`; the parts of such a
@@ -46,6 +47,14 @@ const MAX_INFLATED: usize = 64 << 20;
 pub const MAX_WORD_BYTES: usize = 120;
 /// Empty slot in the stem hash table.
 const EMPTY: u32 = u32::MAX;
+/// Longest input converted (`ICONV`) and stripped of `IGNORE` characters before a lookup: longer
+/// input can't come down to [`MAX_WORD_BYTES`] in a real dictionary, so it is no word and is
+/// never converted (the conversion would cost time on hostile input for nothing).
+const MAX_INPUT_BYTES: usize = 4 * MAX_WORD_BYTES;
+/// Most pairs an `ICONV` or `OCONV` table may hold (the Korean dictionary's hold 11,172 each).
+const MAX_CONV_PAIRS: usize = 1 << 16;
+/// Longest text on either side of an `ICONV` or `OCONV` pair, in bytes.
+const MAX_CONV_BYTES: usize = 64;
 /// Most flags one `.dic` entry or one continuation class field may carry. Real dictionaries use a
 /// few hundred at most (Belarusian: 264); a longer field is hostile and is refused (`.aff`) or
 /// skipped (`.dic`, though its `FORBIDDENWORD` and `NOSUGGEST` flags still apply to the word).
@@ -77,7 +86,6 @@ const UNSUPPORTED: &[&str] = &[
     "COMPOUNDLAST",
     "COMPOUNDMIDDLE",
     "COMPOUNDPERMITFLAG",
-    "OCONV",
     "PSEUDOROOT",
 ];
 
@@ -297,8 +305,10 @@ pub struct AffixDict {
     circumfix: Option<u16>,
     /// `FULLSTRIP`: an affix may strip (and so replace) a whole stem.
     full_strip: bool,
-    /// `ICONV`: input conversions applied before a lookup, `(from, to)`.
-    iconv: Vec<(String, String)>,
+    /// `ICONV`: input conversions applied before a lookup.
+    iconv: Conv,
+    /// `OCONV`: output conversions applied to suggestions.
+    oconv: Conv,
     /// `IGNORE`: characters removed from input words (and from the stems and appended texts).
     ignore: Box<[char]>,
     /// Provenance and licence lines, kept through [`AffixDict::to_bytes`].
@@ -318,6 +328,75 @@ pub struct AffixDict {
     compound_firsts: Vec<char>,
 }
 
+/// An `ICONV` or `OCONV` table: `(from, to)` pairs in source order, found by the first character
+/// of `from` so that converting a word costs a look at the few pairs that could start at each
+/// character, not at the whole table (the Korean tables hold 11,172 pairs each).
+#[derive(Clone, Debug, Default)]
+struct Conv {
+    pairs: Vec<(String, String)>,
+    /// `from` → index into `pairs` (of equal ones, the later pair: it is the one that wins).
+    by_from: FxMap<String, u32>,
+    /// Longest `from`, in bytes.
+    longest: usize,
+}
+
+impl Conv {
+    /// The table of `pairs` (checked by [`Rules::push_conv`]: non-empty `from`, bounded sizes).
+    fn new(pairs: Vec<(String, String)>) -> Conv {
+        let mut by_from: FxMap<String, u32> = FxMap::default();
+        for (i, (from, _)) in pairs.iter().enumerate() {
+            if let Ok(i) = u32::try_from(i) {
+                by_from.insert(from.clone(), i);
+            }
+        }
+        let longest = pairs.iter().map(|p| p.0.len()).max().unwrap_or(0);
+        Conv { pairs, by_from, longest }
+    }
+
+    fn is_empty(&self) -> bool {
+        self.pairs.is_empty()
+    }
+
+    /// The pair to apply at the start of `rest`: the longest `from` that starts it. One hash
+    /// lookup per prefix of `rest` up to the longest `from` ([`MAX_CONV_BYTES`]), however many
+    /// pairs there are (craft-libs#489 review: 65,536 pairs on one first character).
+    fn at(&self, rest: &str) -> Option<&(String, String)> {
+        let ends = rest.char_indices().map(|(i, _)| i).skip(1).chain(std::iter::once(rest.len()));
+        let ends: Vec<usize> = ends.take_while(|&e| e <= self.longest).collect();
+        ends.iter().rev().find_map(|&e| self.by_from.get(rest.get(..e)?)).and_then(|&i| self.pairs.get(i as usize))
+    }
+
+    /// `w` converted: at each position the longest matching `from` is replaced by its `to`, and
+    /// characters in `drop` are removed (outside the replacements too). Borrowed when nothing
+    /// changes.
+    fn apply<'a>(&self, w: &'a str, drop: &[char]) -> std::borrow::Cow<'a, str> {
+        if self.is_empty() && drop.is_empty() {
+            return std::borrow::Cow::Borrowed(w);
+        }
+        // One pass: the input is copied only from the first change on.
+        let mut out: Option<String> = None;
+        let mut rest = w;
+        while let Some(c) = rest.chars().next() {
+            let done = w.get(..w.len() - rest.len()).unwrap_or_default();
+            match self.at(rest) {
+                Some((from, to)) => {
+                    out.get_or_insert_with(|| done.to_string()).extend(to.chars().filter(|c| !drop.contains(c)));
+                    rest = rest.get(from.len()..).unwrap_or_default();
+                }
+                None => {
+                    if drop.contains(&c) {
+                        out.get_or_insert_with(|| done.to_string());
+                    } else if let Some(o) = out.as_mut() {
+                        o.push(c);
+                    }
+                    rest = rest.get(c.len_utf8()..).unwrap_or_default();
+                }
+            }
+        }
+        out.map_or(std::borrow::Cow::Borrowed(w), std::borrow::Cow::Owned)
+    }
+}
+
 fn hash_str(s: &str) -> u64 {
     let mut h = FxHasher::default();
     h.write(s.as_bytes());
@@ -389,6 +468,7 @@ struct Rules {
     circumfix: Option<FlagKey>,
     full_strip: bool,
     iconv: Vec<(String, String)>,
+    oconv: Vec<(String, String)>,
     ignore: Vec<char>,
     /// `COMPOUNDRULE` patterns, as written (`n*1t`); read in the `FLAG` mode by
     /// [`AffixDict::build`], since the `FLAG` line may come after them.
@@ -423,6 +503,22 @@ impl Rules {
         Ok(())
     }
 
+    /// Add a pair to the `ICONV` (`output` false) or `OCONV` table: at most [`MAX_CONV_PAIRS`]
+    /// pairs of at most [`MAX_CONV_BYTES`] each side. A pair with nothing to convert is skipped.
+    fn push_conv(&mut self, output: bool, from: &str, to: &str) -> Result<(), String> {
+        let (name, table) = if output { ("OCONV", &mut self.oconv) } else { ("ICONV", &mut self.iconv) };
+        if from.len() > MAX_CONV_BYTES || to.len() > MAX_CONV_BYTES {
+            return Err(format!("{name} pair longer than {MAX_CONV_BYTES} bytes"));
+        }
+        if table.len() >= MAX_CONV_PAIRS {
+            return Err(format!("{name} table larger than {MAX_CONV_PAIRS} pairs"));
+        }
+        if !from.is_empty() {
+            table.push((from.to_string(), to.to_string()));
+        }
+        Ok(())
+    }
+
     /// The one flag of a rule header or a single-flag directive.
     fn one_flag(&mut self, field: &str) -> Result<FlagKey, String> {
         self.flag_read = true;
@@ -701,9 +797,10 @@ fn intersects(a: &[u16], b: &[u16]) -> bool {
 /// Directives an English dictionary may carry that this engine doesn't implement but that don't
 /// change which English words are right, ignored by [`AffixDict::from_hunspell_english`]:
 /// flag-based compounding (English dictionaries compound only ordinals, by `COMPOUNDRULE`, which
-/// is read) and output conversion (`OCONV`: how suggestions are written, such as `'` as `’`).
+/// is read). Output conversion (`OCONV`) is read, for English as for every language: suggestions
+/// are written as the dictionary says (`don’t`).
 const IGNORED_IN_ENGLISH: &[&str] =
-    &["COMPOUNDBEGIN", "COMPOUNDEND", "COMPOUNDFIRST", "COMPOUNDFLAG", "COMPOUNDLAST", "COMPOUNDMIDDLE", "COMPOUNDPERMITFLAG", "OCONV"];
+    &["COMPOUNDBEGIN", "COMPOUNDEND", "COMPOUNDFIRST", "COMPOUNDFLAG", "COMPOUNDLAST", "COMPOUNDMIDDLE", "COMPOUNDPERMITFLAG"];
 
 /// Read Hunspell `.aff` text (already decoded to UTF-8). `english`: ignore the directives in
 /// [`IGNORED_IN_ENGLISH`] instead of refusing them.
@@ -715,6 +812,7 @@ fn parse_aff(aff: &str, english: bool) -> Result<Rules, String> {
     let mut af_header = false;
     let mut iconv_header = false;
     let mut compound_header = false;
+    let mut oconv_header = false;
     for (n, line) in aff.lines().enumerate() {
         let f: Vec<&str> = split_fields(line).collect();
         let Some(&dir) = f.first() else { continue };
@@ -729,7 +827,7 @@ fn parse_aff(aff: &str, english: bool) -> Result<Rules, String> {
             // the words it marks are correct.
             "AM" | "LEMMA_PRESENT" => {}
             "SUBSTANDARD" => r.substandard = Some(r.one_flag(at(1)).map_err(|e| format!("line {}: {dir} needs one flag: {e}", n + 1))?),
-            // The first `AF` or `ICONV` line is the table's count, as in Hunspell, even with a
+            // The first `AF`, `ICONV` or `OCONV` line is the table's count, as in Hunspell, even with a
             // trailing comment (`AF 1559 # aliases`).
             "AF" => {
                 if !af_header {
@@ -744,11 +842,12 @@ fn parse_aff(aff: &str, english: bool) -> Result<Rules, String> {
                     r.aliases.push(at(1).to_string());
                 }
             }
-            "ICONV" => {
-                if !iconv_header && at(1).parse::<usize>().is_ok() {
-                    iconv_header = true;
+            "ICONV" | "OCONV" => {
+                let header = if dir == "OCONV" { &mut oconv_header } else { &mut iconv_header };
+                if !*header && at(1).parse::<usize>().is_ok() {
+                    *header = true;
                 } else if f.len() >= 3 {
-                    r.iconv.push((at(1).to_string(), at(2).to_string()));
+                    r.push_conv(dir == "OCONV", at(1), at(2)).map_err(|e| format!("line {}: {e}", n + 1))?;
                 }
             }
             "IGNORE" => r.ignore = at(1).chars().collect(),
@@ -848,10 +947,10 @@ impl AffixDict {
         AffixDict::from_hunspell_in(aff, dic, comments, false)
     }
 
-    /// [`AffixDict::from_hunspell`] for an English dictionary: flag-based compounding and output
-    /// conversion (`COMPOUNDFLAG`, `OCONV`…), which this engine doesn't implement and English
-    /// spelling doesn't need, are ignored rather than refused. `COMPOUNDRULE`, `ONLYINCOMPOUND`
-    /// and `COMPOUNDMIN` (ordinals: `21st`) are read, as for every language.
+    /// [`AffixDict::from_hunspell`] for an English dictionary: flag-based compounding
+    /// (`COMPOUNDFLAG`…), which this engine doesn't implement and English spelling doesn't need,
+    /// is ignored rather than refused. `COMPOUNDRULE`, `ONLYINCOMPOUND` and `COMPOUNDMIN`
+    /// (ordinals: `21st`) and `OCONV` are read, as for every language.
     pub fn from_hunspell_english(aff: &str, dic: &str, comments: &[&str]) -> Result<AffixDict, String> {
         AffixDict::from_hunspell_in(aff, dic, comments, true)
     }
@@ -989,7 +1088,8 @@ impl AffixDict {
             circumfix: rules.circumfix.and_then(id_of),
             full_strip: rules.full_strip,
             flag_mode: mode,
-            iconv: rules.iconv,
+            iconv: Conv::new(rules.iconv),
+            oconv: Conv::new(rules.oconv),
             ignore: rules.ignore.into_boxed_slice(),
             try_chars: rules.try_chars,
             key: rules.key,
@@ -1354,33 +1454,19 @@ impl AffixDict {
     }
 
     /// `w` after the `ICONV` conversions (at each position the longest match wins) and without
-    /// the `IGNORE` characters.
+    /// the `IGNORE` characters. Input longer than [`MAX_INPUT_BYTES`] is no word and comes back
+    /// unchanged, so the length check after this rejects it without converting it.
     fn normalize<'a>(&self, w: &'a str) -> std::borrow::Cow<'a, str> {
-        if self.iconv.is_empty() && self.ignore.is_empty() {
+        if w.len() > MAX_INPUT_BYTES {
             return std::borrow::Cow::Borrowed(w);
         }
-        // One pass: the input is copied only from the first change on.
-        let mut out: Option<String> = None;
-        let mut rest = w;
-        while let Some(c) = rest.chars().next() {
-            let done = w.get(..w.len() - rest.len()).unwrap_or_default();
-            let hit = self.iconv.iter().filter(|(from, _)| !from.is_empty() && rest.starts_with(from.as_str())).max_by_key(|(from, _)| from.len());
-            match hit {
-                Some((from, to)) => {
-                    out.get_or_insert_with(|| done.to_string()).extend(to.chars().filter(|c| !self.ignore.contains(c)));
-                    rest = rest.get(from.len()..).unwrap_or_default();
-                }
-                None => {
-                    if self.ignore.contains(&c) {
-                        out.get_or_insert_with(|| done.to_string());
-                    } else if let Some(o) = out.as_mut() {
-                        o.push(c);
-                    }
-                    rest = rest.get(c.len_utf8()..).unwrap_or_default();
-                }
-            }
-        }
-        out.map_or(std::borrow::Cow::Borrowed(w), std::borrow::Cow::Owned)
+        self.iconv.apply(w, &self.ignore)
+    }
+
+    /// A suggestion as it is shown: `w` after the `OCONV` conversions (the French dictionary
+    /// writes `’` for the `'` its stems use).
+    pub fn output<'a>(&self, w: &'a str) -> std::borrow::Cow<'a, str> {
+        self.oconv.apply(w, &[])
     }
 
     /// [`AffixDict::contains`] on normalized input, where stems carrying an `excluded` flag
@@ -1767,8 +1853,10 @@ impl AffixDict {
                 out.push('\n');
             }
         }
-        for (a, b) in &self.iconv {
-            out.push_str(&format!("ICONV {a} {b}\n"));
+        for (name, table) in [("ICONV", &self.iconv), ("OCONV", &self.oconv)] {
+            for (a, b) in &table.pairs {
+                out.push_str(&format!("{name} {a} {b}\n"));
+            }
         }
         if !self.ignore.is_empty() {
             out.push_str(&format!("IGNORE {}\n", self.ignore.iter().collect::<String>()));
@@ -1870,7 +1958,9 @@ impl AffixDict {
                         _ => rules.circumfix = Some(c),
                     }
                 }
-                "ICONV" if f.len() == 3 => rules.iconv.push((at(1).to_string(), at(2).to_string())),
+                conv @ ("ICONV" | "OCONV") if f.len() == 3 => {
+                    rules.push_conv(conv == "OCONV", at(1), at(2)).map_err(|e| format!("affix dictionary: {e}"))?;
+                }
                 "IGNORE" => rules.ignore = at(1).chars().collect(),
                 "COMPOUNDMIN" => {
                     let min = at(1).parse::<usize>().map_err(|_| format!("affix dictionary: bad `{line}`"))?;
@@ -2415,6 +2505,58 @@ pub(crate) mod tests {
         }
     }
 
+    #[test]
+    fn oconv_converts_suggestions_only() {
+        // Answers as Hunspell 1.7 gives them on the same files: `kbt` → `k’t`.
+        let aff = "SET UTF-8\nTRY ab\nOCONV 1\nOCONV a ’\n";
+        let d = AffixDict::from_hunspell(aff, "1\nkat\n", &[]).unwrap();
+        assert!(d.check("kat") && !d.check("k’t"), "OCONV doesn't change what is correct");
+        assert_eq!(d.output("kat"), "k’t");
+        assert!(matches!(d.output("kot"), std::borrow::Cow::Borrowed("kot")));
+        assert_eq!(crate::affix_suggest::suggest_in(&d, "kbt", 4, &crate::affix_suggest::Tuning::DEFAULT), ["k’t"]);
+        let back = AffixDict::from_bytes(&d.to_bytes().unwrap()).unwrap();
+        assert_eq!(back.output("kat"), "k’t");
+        // The count line may carry a comment, as for ICONV.
+        let d = AffixDict::from_hunspell("OCONV 1 # apostrophes\nOCONV ' ’\n", "1\nl'eau\n", &[]).unwrap();
+        assert_eq!(d.output("l'eau"), "l’eau");
+    }
+
+    #[test]
+    fn conversions_take_the_longest_match_and_the_later_of_equal_ones() {
+        let aff = "ICONV 3\nICONV a x\nICONV ab y\nICONV a z\n";
+        let d = AffixDict::from_hunspell(aff, "2\ny\nzc\n", &[]).unwrap();
+        assert_eq!(d.normalize("ab"), "y");
+        assert_eq!(d.normalize("ac"), "zc");
+        assert!(d.check("ab") && d.check("ac"));
+    }
+
+    #[test]
+    fn conversion_tables_are_capped() {
+        let pairs = |n: usize| (0..n).map(|i| format!("ICONV q{i} r\n")).collect::<String>();
+        let ok = AffixDict::from_hunspell(&format!("ICONV {MAX_CONV_PAIRS}\n{}", pairs(MAX_CONV_PAIRS)), "1\nr\n", &[]).unwrap();
+        assert!(ok.check("q65535") && !ok.check("q65536"));
+        let err = AffixDict::from_hunspell(&format!("ICONV 1\n{}", pairs(MAX_CONV_PAIRS + 1)), "1\nr\n", &[]).unwrap_err();
+        assert!(err.contains("ICONV table larger"), "{err}");
+        let long = "x".repeat(MAX_CONV_BYTES + 1);
+        for aff in [format!("ICONV 1\nICONV {long} y\n"), format!("OCONV 1\nOCONV y {long}\n")] {
+            assert!(AffixDict::from_hunspell(&aff, "1\nr\n", &[]).is_err(), "{aff}");
+        }
+        let raw = format!("{MAGIC}\nOCONV {long} y\nWORDS\n\0kot\n");
+        assert!(AffixDict::from_bytes(&miniz_oxide::deflate::compress_to_vec(raw.as_bytes(), 6)).is_err());
+    }
+
+    #[test]
+    fn long_input_is_rejected_before_it_is_converted() {
+        let d = AffixDict::from_hunspell("ICONV 1\nICONV fi fi\nIGNORE -\n", "1\nfine\n", &[]).unwrap();
+        // Ignored characters may take a word past MAX_WORD_BYTES and still leave a word.
+        let spaced = "f-i-n-e".to_string() + &"-".repeat(MAX_WORD_BYTES);
+        assert!(spaced.len() > MAX_WORD_BYTES && spaced.len() <= MAX_INPUT_BYTES && d.check(&spaced));
+        // Past MAX_INPUT_BYTES the input is no word and is returned as it is, unconverted.
+        let long = "fi".repeat(MAX_INPUT_BYTES);
+        assert!(matches!(d.normalize(&long), std::borrow::Cow::Borrowed(_)));
+        assert!(!d.check(&long) && !d.contains(&long) && !d.can_suggest(&long));
+    }
+
     #[test]
     fn normalizing_borrows_when_nothing_changes() {
         let d = AffixDict::from_hunspell("SET UTF-8\nICONV 1\nICONV fi fi\nIGNORE ()\n", "1\nfine\n", &[]).unwrap();
@@ -2780,8 +2922,9 @@ pub(crate) mod tests {
 
     /// craft-libs#482 review: an English dictionary with compounding and output conversion, as the
     /// usual en_US and en_GB ones have, loads for English: `COMPOUNDRULE` makes its ordinals, its
-    /// `ONLYINCOMPOUND` stems are words only inside them, and `COMPOUNDFLAG` and `OCONV` are
-    /// ignored. Other languages still refuse those two. (A synthetic `.aff` written for this test.)
+    /// `ONLYINCOMPOUND` stems are words only inside them, `COMPOUNDFLAG` is ignored (other
+    /// languages still refuse it) and `OCONV` is applied to suggestions. (A synthetic `.aff`
+    /// written for this test.)
     #[test]
     fn english_dictionaries_ignore_compounding_and_output_conversion() {
         let aff = "SET UTF-8\nTRY esianrtolcdugmphbyfvkwz'\nICONV 1\nICONV ’ '\nOCONV 1\nOCONV ' ’\nNOSUGGEST !\n\
@@ -2796,6 +2939,11 @@ pub(crate) mod tests {
         assert!(!d.check("0th") && !d.check("1th"), "parts of compounds only");
         assert!(d.check("1st") && d.check("11th") && d.check("111th") && !d.check("11st"), "ordinals by COMPOUNDRULE");
         assert!(!d.check("housed") && !d.check("hous"));
+        // OCONV is read and applied to suggestions, as Hunspell does: `'` is shown as `’`.
+        let d = AffixDict::from_hunspell_english(aff, "2\ndon't\nwalk/S\n", &[]).unwrap();
+        assert_eq!(d.output("don't"), "don’t");
+        assert!(d.check("don't") && d.check("don’t"), "ICONV reads it back");
+        assert!(crate::affix_suggest::suggest_in(&d, "donn't", 4, &crate::affix_suggest::Tuning::DEFAULT).contains(&"don’t".to_string()));
         // The directives that change which words are right are still refused.
         assert!(AffixDict::from_hunspell_english("PSEUDOROOT X\n", dic, &[]).is_err(), "PSEUDOROOT");
     }

crates/proof/src/affix_suggest.rs

@@ -327,6 +327,8 @@ pub(crate) fn suggest_in(d: &AffixDict, word: &str, max: usize, t: &Tuning) -> V
         };
         // A KEEPCASE word only exists in its dictionary case: `Km` would be flagged, so offer `km`.
         let t = if recased != text && text.split(' ').any(|p| d.case_locked(p)) { text } else { recased };
+        // Shown as the dictionary's OCONV table writes it (`'` → `’` in French).
+        let t = d.output(&t).into_owned();
         if t != word && !out.contains(&t) {
             out.push(t);
         }
@@ -517,6 +519,50 @@ fn scan(col: &mut Collector, chars: &[char], map: &[Vec<char>]) {
 mod tests {
     use super::*;
 
+    /// craft-libs#489 review repros: OCONV applies to a suggestion after its case is set, case
+    /// sensitively, as the hunspell 1.7.5 CLI answers on the same files (`KBT` → `KAT`, not
+    /// `K’T`), and every suggestion is right.
+    #[test]
+    fn oconv_suggestions_match_hunspell() {
+        type Case = (&'static str, &'static str, &'static [(&'static str, &'static str)]);
+        let cases: &[Case] = &[
+            ("SET UTF-8\nTRY ab\nOCONV 1\nOCONV a ’\n", "1\nkat\n", &[("kbt", "k’t"), ("Kbt", "K’t"), ("KBT", "KAT"), ("kot", "k’t")]),
+            ("SET UTF-8\nTRY ab\nOCONV 1\nOCONV k q\n", "1\nkat\n", &[("kbt", "qat"), ("Kbt", "Kat"), ("KBT", "KAT"), ("kot", "qat")]),
+            (
+                "SET UTF-8\nTRY esianrtolcdugmphbyfvkwz'\nICONV 1\nICONV ’ '\nOCONV 1\nOCONV ' ’\nSFX S Y 1\nSFX S 0 's .\n",
+                "2\ncolour/S\ndon't\n",
+                &[("dont", "don’t"), ("DONT", "DON’T"), ("colours", "colour’s")],
+            ),
+        ];
+        for (aff, dic, words) in cases {
+            let d = AffixDict::from_hunspell(aff, dic, &[]).unwrap();
+            for (word, want) in *words {
+                let got = suggest_in(&d, word, 8, &Tuning::DEFAULT);
+                assert!(got.iter().any(|g| g == want), "{word}: {got:?}, want {want}");
+                for g in &got {
+                    assert!(d.check(g) || !aff.contains("ICONV"), "{word} → {g} is not right");
+                }
+            }
+        }
+    }
+
+    /// craft-libs#489 review: 65,536 ICONV pairs on one first character cost one hash lookup per
+    /// prefix, not a scan of the table, per position of a long word and per suggestion.
+    #[test]
+    fn a_huge_conversion_table_on_one_character_stays_fast() {
+        let mut aff = String::from("SET UTF-8\nTRY abc\nICONV 65536\n");
+        let a60 = "a".repeat(60);
+        for c in (0x100u32..).filter_map(char::from_u32).take(65_536) {
+            aff.push_str(&format!("ICONV {a60}{c} x\n"));
+        }
+        let d = AffixDict::from_hunspell(&aff, "1\nabc\n", &[]).unwrap();
+        let t = std::time::Instant::now();
+        assert!(!d.check(&"a".repeat(470)));
+        let _ = suggest_in(&d, "abd", 4, &Tuning::DEFAULT);
+        let _ = suggest_in(&d, &"a".repeat(40), 4, &Tuning::DEFAULT);
+        assert!(t.elapsed().as_secs() < 5, "{:?}", t.elapsed());
+    }
+
     #[test]
     fn an_extra_trailing_letter_is_dropped_before_a_short_word_split() {
         let d = AffixDict::from_hunspell("", "2\npočítač\nz\n", &[]).unwrap();

crates/proof/src/english.rs

@@ -10,7 +10,8 @@
 //!    right; never one only another English knows. When it is there, the system locations aren't
 //!    searched.
 //! 4. **System Hunspell**: a pair for the run's own English in the system locations (none on the
-//!    web).
+//!    web). `COMPOUNDRULE` (ordinals) and `OCONV` are read; flag-based compounding is ignored
+//!    ([`crate::hunspell::load_english`]).
 //! 5. **Built-in word list**: [`WORDS`], the whole public-domain Moby word list compiled into
 //!    the crate (without its break points), with WordCraft's own supplement of modern words
 //!    ([`EXTRA`]) and the usual inflections. A word it doesn't know is misspelled. (The Moby list
@@ -21,8 +22,8 @@
 //! folder before the next name. Another English's dictionary is never used: it would mark this
 //! English's spellings (`colour` in an `en_US` one). A file that can't be read is skipped for the
 //! next name or folder, and one file is read once for every English that uses it. `COMPOUNDRULE`
-//! (ordinals) is read; flag-based compounding and `OCONV` are ignored
-//! ([`crate::hunspell::load_english`]).
+//! (ordinals) and `OCONV` (suggestions are written as the dictionary says: `don’t`) are read;
+//! flag-based compounding is ignored ([`crate::hunspell::load_english`]).
 //!
 //! Whichever it is, the dialect policy of `en-variants.txt` ([`crate::en_variants`], when it is
 //! embedded) flags the other dialect's spellings and accepts this dialect's spellings of known

crates/proof/src/hunspell.rs

@@ -226,9 +226,9 @@ pub fn load(aff: &Path, dic: &Path) -> Result<AffixDict, String> {
     AffixDict::from_hunspell(&aff_text, &dic_text, &[source.as_str()])
 }
 
-/// [`load`] for an English dictionary: directives that only matter for other languages
-/// (compounding, output conversion) are ignored rather than refused
-/// ([`AffixDict::from_hunspell_english`]).
+/// [`load`] for an English dictionary: flag-based compounding (`COMPOUNDFLAG`…), which this engine
+/// does not implement, is ignored rather than refused ([`AffixDict::from_hunspell_english`]); `OCONV`
+/// is applied to suggestions, as for every language.
 pub fn load_english(aff: &Path, dic: &Path) -> Result<AffixDict, String> {
     let (aff_text, dic_text) = decode(&read_capped(aff)?, &read_capped(dic)?)?;
     let source = format!("Hunspell dictionary {}", dic.display());

crates/proof/src/lib.rs

@@ -675,8 +675,10 @@ fn suggest_english(word: &str, max: usize, lang: ProofLang) -> Vec<String> {
     }
     let shared = english::suggest(word, max.saturating_add(2), lang);
     for s in shared {
-        let s = english_preferred(&s.to_lowercase(), lang).map_or(s, |p| match_case(word, p));
-        if !out.contains(&s) {
+        let s = in_dialect(&s, word, lang);
+        // `color's` and `color’s` are one suggestion (a dictionary's OCONV writes the second).
+        let key = s.replace('’', "'");
+        if !out.iter().any(|o| o.replace('’', "'") == key) {
             out.push(s);
         }
     }
@@ -684,6 +686,20 @@ fn suggest_english(word: &str, max: usize, lang: ProofLang) -> Vec<String> {
     out
 }
 
+/// A suggestion `s` for `word` in `lang`'s spelling. Suggestions come as the dictionary's OCONV
+/// writes them (`colour’s`); the dialect lists use `'`, so the mapping reads `'` and the result
+/// keeps the dictionary's apostrophe (`color’s`).
+fn in_dialect(s: &str, word: &str, lang: ProofLang) -> String {
+    let curly = s.contains('’');
+    match english_preferred(&s.to_lowercase().replace('’', "'"), lang) {
+        Some(p) => {
+            let p = match_case(word, p);
+            if curly { p.replace('\'', "’") } else { p }
+        }
+        None => s.to_string(),
+    }
+}
+
 /// Words in `text`: (byte start, byte end). Combining marks belong to the word they follow, so a
 /// word typed decomposed (`tambie\u{301}n`, NFD) is one word, as its precomposed form is.
 pub fn words(text: &str) -> Vec<(usize, usize)> {
@@ -1067,6 +1083,18 @@ mod tests {
 
     use super::*;
 
+    /// craft-libs#489 review: a suggestion an OCONV table wrote with `’` is still mapped to this
+    /// dialect's spelling (`colour’s` → `color’s` in American English), and keeps its apostrophe.
+    #[test]
+    fn curly_suggestions_are_mapped_to_the_dialect() {
+        if english_preferred("colour's", ProofLang::En).is_none() {
+            return; // no dialect lists without CRAFT_I18N_DIR
+        }
+        assert_eq!(in_dialect("colour’s", "colours", ProofLang::En), "color’s");
+        assert_eq!(in_dialect("colour's", "colours", ProofLang::En), "color's");
+        assert_eq!(in_dialect("colour’s", "colours", ProofLang::EnGb), "colour’s");
+    }
+
     /// The non-English test dictionaries every dictionaries-folder test puts in its folder, so
     /// tests running meanwhile see the same words whichever of those folders is set.
     fn write_test_dictionaries(dir: &std::path::Path) {