proof: .dic lines and alias numbers read as Hunspell reads them (storytold/wordcraft#735)

Checked against the Hunspell 1.7 command line:
- Without a tab or a morphological field the whole line is the entry, so
  a phrase with single-byte flags (tök jó/396 in hu_HU) no longer fails
  the load or loses its alias; leading blanks belong to the entry.
- With an AF table a flag field is the alias its leading digits number
  (fal/2x is alias 2, lab/b has none); an undefined alias, in a .dic
  entry or a continuation class, reads as no flags instead of failing
  the dictionary; a table without its count line is ignored.
- A REP count line may carry a comment, like AF and ICONV.

normalize copies the input only from its first change on, and SET and
FLAG are found in one pass over the .aff.

crates/proof/src/affix.rs

@@ -343,28 +343,27 @@ struct Rules {
     ignore: Vec<char>,
     /// `AF` flag aliases: alias `n` (1-based) stands for `aliases[n - 1]`.
     aliases: Vec<String>,
+    /// The `AF` table opened with its count line, so flag fields are alias numbers. A table
+    /// without one is rejected, as in Hunspell: its lines are ignored.
+    aliases_on: bool,
 }
 
 impl Rules {
-    /// Flags as written in a `.dic` entry or a continuation class: an `AF` alias number when the
-    /// dictionary declares aliases, otherwise the flag characters themselves.
-    fn flags<'a>(&'a self, written: &'a str) -> Result<&'a str, String> {
-        if self.aliases.is_empty() || written.is_empty() || !written.bytes().all(|b| b.is_ascii_digit()) {
-            return Ok(written);
+    /// Flags as written in a `.dic` entry or a continuation class: the flag characters
+    /// themselves, or with an `AF` table the alias its leading digits number, as Hunspell reads
+    /// them (`2x` is alias 2). A field without digits or an undefined alias reads as no flags:
+    /// the stem is still a word, the rule still applies, without the flags (Hunspell warns).
+    fn flags<'a>(&'a self, written: &'a str) -> &'a str {
+        if !self.aliases_on {
+            return written;
         }
+        let digits = written.bytes().take_while(u8::is_ascii_digit).count();
         written
-            .parse::<usize>()
-            .ok()
+            .get(..digits)
+            .and_then(|d| d.parse::<usize>().ok())
             .and_then(|n| n.checked_sub(1))
             .and_then(|i| self.aliases.get(i))
-            .map(String::as_str)
-            .ok_or_else(|| format!("flag alias {written} is not defined"))
-    }
-
-    /// The flags of a `.dic` entry. An undefined alias reads as no flags, as in Hunspell: the
-    /// stem is still a word, without affixes.
-    fn dic_flags<'a>(&'a self, written: &'a str) -> &'a str {
-        self.flags(written).unwrap_or("")
+            .map_or("", String::as_str)
     }
 
     /// `w` without the `IGNORE` characters.
@@ -446,9 +445,10 @@ fn parse_aff(aff: &str) -> Result<Rules, String> {
             // The first `AF` or `ICONV` line is the table's count, as in Hunspell, even with a
             // trailing comment (`AF 1559 # aliases`).
             "AF" => {
-                if !af_header && at(1).parse::<usize>().is_ok() {
+                if !af_header {
                     af_header = true;
-                } else {
+                    r.aliases_on = at(1).parse::<usize>().is_ok();
+                } else if r.aliases_on {
                     r.aliases.push(at(1).to_string());
                 }
             }
@@ -479,7 +479,7 @@ fn parse_aff(aff: &str) -> Result<Rules, String> {
                 }
             }
             "REP" => {
-                if !rep_header && f.len() == 2 && at(1).parse::<usize>().is_ok() {
+                if !rep_header && at(1).parse::<usize>().is_ok() {
                     rep_header = true;
                 } else if f.len() >= 3 {
                     r.rep.push((at(1).replace('_', " "), at(2).replace('_', " ")));
@@ -522,16 +522,15 @@ fn parse_aff(aff: &str) -> Result<Rules, String> {
     // Aliases may be declared after the rules that use them; resolve once all are read.
     // Hunspell removes `IGNORE` characters from the appended text only: a stripped text that
     // holds one never matches a stem (stems lose them too), so the rule never applies.
-    let resolve = |rules: &mut Vec<RawRule>, r: &Rules| -> Result<(), String> {
+    let resolve = |rules: &mut Vec<RawRule>, r: &Rules| {
         for rule in rules.iter_mut() {
-            rule.4 = r.flags(&rule.4)?.to_string();
+            rule.4 = r.flags(&rule.4).to_string();
             rule.3 = r.strip_ignored(&rule.3);
         }
-        Ok(())
     };
     let (mut pfx, mut sfx) = (std::mem::take(&mut r.prefixes), std::mem::take(&mut r.suffixes));
-    resolve(&mut pfx, &r)?;
-    resolve(&mut sfx, &r)?;
+    resolve(&mut pfx, &r);
+    resolve(&mut sfx, &r);
     (r.prefixes, r.suffixes) = (pfx, sfx);
     Ok(r)
 }
@@ -548,7 +547,7 @@ impl AffixDict {
         let mut stems: Vec<(String, String)> = Vec::new();
         for (n, line) in dic.lines().enumerate() {
             // The entry may hold spaces (`ad hoc/B\tpo:adv`); it ends where Hunspell ends it.
-            let line = line.trim_start_matches([' ', '\t']);
+            // Leading blanks belong to it, as in Hunspell (`  lab` is no `lab`).
             let tok = line.get(..crate::hunspell::dic_entry_end(line.as_bytes())).unwrap_or_default();
             if tok.is_empty() {
                 continue;
@@ -579,7 +578,7 @@ impl AffixDict {
             if word.is_empty() || word.len() > MAX_WORD_BYTES || word.contains(['\n', '/']) {
                 continue;
             }
-            let flags = rules.dic_flags(&flags).to_string();
+            let flags = rules.flags(&flags).to_string();
             stems.push((word, flags));
         }
         AffixDict::build(rules, stems, comments.iter().map(ToString::to_string).collect())
@@ -828,28 +827,31 @@ impl AffixDict {
     /// `w` after the `ICONV` conversions (at each position the longest match wins) and without
     /// the `IGNORE` characters.
     fn normalize<'a>(&self, w: &'a str) -> std::borrow::Cow<'a, str> {
-        let converts = |rest: &str| self.iconv.iter().any(|(from, _)| !from.is_empty() && rest.contains(from.as_str()));
-        if !w.contains(&*self.ignore) && !converts(w) {
+        if self.iconv.is_empty() && self.ignore.is_empty() {
             return std::borrow::Cow::Borrowed(w);
         }
-        let mut out = String::with_capacity(w.len());
+        // One pass: the input is copied only from the first change on.
+        let mut out: Option<String> = None;
         let mut rest = w;
         while let Some(c) = rest.chars().next() {
+            let done = w.get(..w.len() - rest.len()).unwrap_or_default();
             let hit = self.iconv.iter().filter(|(from, _)| !from.is_empty() && rest.starts_with(from.as_str())).max_by_key(|(from, _)| from.len());
             match hit {
                 Some((from, to)) => {
-                    out.extend(to.chars().filter(|c| !self.ignore.contains(c)));
+                    out.get_or_insert_with(|| done.to_string()).extend(to.chars().filter(|c| !self.ignore.contains(c)));
                     rest = rest.get(from.len()..).unwrap_or_default();
                 }
                 None => {
-                    if !self.ignore.contains(&c) {
-                        out.push(c);
+                    if self.ignore.contains(&c) {
+                        out.get_or_insert_with(|| done.to_string());
+                    } else if let Some(o) = out.as_mut() {
+                        o.push(c);
                     }
                     rest = rest.get(c.len_utf8()..).unwrap_or_default();
                 }
             }
         }
-        std::borrow::Cow::Owned(out)
+        out.map_or(std::borrow::Cow::Borrowed(w), std::borrow::Cow::Owned)
     }
 
     /// [`AffixDict::contains`] on normalized input, where stems carrying a flag in `deny` don't
@@ -1612,6 +1614,28 @@ pub(crate) mod tests {
         assert!(back.check("fines") && back.check("waffle") && back.check("waf(fle)"));
     }
 
+    #[test]
+    fn alias_numbers_are_read_as_hunspell_reads_them() {
+        // Checked with the Hunspell 1.7 command line. With an `AF` table a flag field is the alias
+        // its leading digits number: `fal/2x` is alias 2, `lab/b` has no flags.
+        let aff = "SET UTF-8\nAF 2\nAF B\nAF C\nSFX B Y 1\nSFX B 0 s .\nSFX C Y 1\nSFX C 0 x .\nSFX b Y 1\nSFX b 0 q .\n";
+        let d = AffixDict::from_hunspell(aff, "3\nlab/b\nfal/2x\nkut/1\n", &[]).unwrap();
+        assert!(d.check("lab") && !d.check("labq") && !d.check("labs"));
+        assert!(d.check("falx") && !d.check("fals") && d.check("kuts"));
+        // A table without its count line is rejected: no alias applies.
+        let broken = AffixDict::from_hunspell("SET UTF-8\nAF B\nAF C\nSFX B Y 1\nSFX B 0 s .\n", "2\nlab/1\nfal/2\n", &[]).unwrap();
+        assert!(broken.check("lab") && !broken.check("labs") && !broken.check("fals"));
+        // An undefined alias in a continuation class leaves the rule in force, without it.
+        let cont = AffixDict::from_hunspell("SET UTF-8\nAF 1\nAF a\nSFX a Y 1\nSFX a 0 ok/7 .\n", "1\nhaz/1\n", &[]).unwrap();
+        assert!(cont.check("haz") && cont.check("hazok"));
+    }
+
+    #[test]
+    fn a_rep_count_line_may_carry_a_comment() {
+        let d = AffixDict::from_hunspell("SET UTF-8\nREP 1 # confusions\nREP x y\n", "1\nay\n", &[]).unwrap();
+        assert_eq!(d.rep(), [("x".to_string(), "y".to_string())]);
+    }
+
     #[test]
     fn count_lines_may_carry_a_comment() {
         // Hunspell reads the first `AF` and `ICONV` line as the count, comment and all: `AF 2 #`
@@ -1630,10 +1654,14 @@ pub(crate) mod tests {
     fn dic_entries_may_hold_spaces() {
         // `ad hoc/B` is one entry, as in Hunspell; `ad` alone is no word.
         let aff = "SET UTF-8\nSFX B Y 1\nSFX B 0 s .\n";
-        let d = AffixDict::from_hunspell(aff, "3\nad hoc/B\tpo:adv\nword/B po:noun\n  kot/B 12\n", &[]).unwrap();
-        assert!(d.contains("ad hoc") && d.contains("ad hocs"));
-        assert!(!d.contains("ad") && !d.contains("hoc") && !d.contains("ads"));
+        // Checked with the Hunspell 1.7 command line: without a tab or a morphological field the
+        // whole line is the entry (`kot/B 12` has the flag `B`), and leading blanks belong to it
+        // (`  lab` is no `lab`).
+        let d = AffixDict::from_hunspell(aff, "5\nad hoc/B\tpo:adv\nword/B po:noun\nkot/B 12\n  lab/B\nés így/B\n", &[]).unwrap();
+        assert!(d.contains("ad hoc") && d.contains("ad hocs") && d.contains("és így") && d.contains("és ígys"));
+        assert!(!d.contains("ad") && !d.contains("hoc") && !d.contains("ads") && !d.contains("így"));
         assert!(d.contains("words") && d.contains("kots") && !d.contains("kot/B"));
+        assert!(!d.contains("lab") && !d.contains("labs"));
     }
 
     #[test]
@@ -1691,8 +1719,9 @@ pub(crate) mod tests {
         for dir in UNSUPPORTED {
             assert!(AffixDict::from_hunspell(&format!("SET UTF-8\n{dir} X\n"), DIC, &[]).is_err(), "{dir} should be refused");
         }
-        // The directives this engine reads, with no other change to the dictionary.
-        for line in ["AF 1\nAF ab", "AM 1\nAM po:noun", "ICONV 1\nICONV fi fi", "IGNORE ()", "KEEPCASE k", "SUBSTANDARD x", "LEMMA_PRESENT l"] {
+        // The directives this engine reads, with no other change to the dictionary (`AF` would
+        // turn the flags of `DIC` into alias numbers; it has tests of its own).
+        for line in ["AM 1\nAM po:noun", "ICONV 1\nICONV fi fi", "IGNORE ()", "KEEPCASE k", "SUBSTANDARD x", "LEMMA_PRESENT l"] {
             let d = AffixDict::from_hunspell(&format!("{AFF}{line}\n"), DIC, &[]).unwrap_or_else(|e| panic!("{line}: {e}"));
             assert!(d.check("koty") && d.check("niekoty") && !d.check("kotx"), "{line}");
         }

crates/proof/src/hunspell.rs

@@ -224,22 +224,11 @@ fn decode_high(bytes: &[u8], high: &str) -> String {
 /// UTF-8. Those fields are read byte by byte (each byte becomes U+0000..U+00FF, one flag per
 /// byte); the words, affixes and conditions around them are UTF-8.
 pub fn decode(aff: &[u8], dic: &[u8]) -> Result<(String, String), String> {
-    // The whole file is searched: a `FLAG UTF-8` after the rules still decides how the `.dic`
-    // flags are read, as in Hunspell.
-    let directive = |name: &str| {
-        aff.split(|&b| b == b'\n').find_map(|l| {
-            let l = l.strip_prefix("\u{feff}".as_bytes()).unwrap_or(l);
-            // Only the first argument counts, as in Hunspell: `FLAG UTF-8 # Unicode flags` is UTF-8.
-            l.strip_prefix(name.as_bytes())
-                .filter(|rest| rest.first().is_some_and(|b| matches!(b, b' ' | b'\t')))
-                .and_then(|rest| rest.split(|b| matches!(b, b' ' | b'\t' | b'\r')).find(|f| !f.is_empty()))
-                .map(|f| String::from_utf8_lossy(f).to_ascii_uppercase())
-        })
-    };
-    let set = directive("SET").unwrap_or_else(|| "UTF-8".into());
+    let (set, flag) = directives(aff);
+    let set = set.unwrap_or_else(|| "UTF-8".into());
     match set.as_str() {
         "UTF-8" | "UTF8" => {
-            if matches!(directive("FLAG").as_deref(), Some("UTF-8" | "UTF8")) {
+            if matches!(flag.as_deref(), Some("UTF-8" | "UTF8")) {
                 let text = |b: &[u8], what: &str| String::from_utf8(b.to_vec()).map_err(|e| format!("{what}: not UTF-8: {e}"));
                 return Ok((text(aff, ".aff")?, text(dic, ".dic")?));
             }
@@ -254,6 +243,32 @@ pub fn decode(aff: &[u8], dic: &[u8]) -> Result<(String, String), String> {
     }
 }
 
+/// The first arguments of the `.aff`'s first `SET` and `FLAG` lines, in capitals, found in one pass
+/// over the whole file: a `FLAG UTF-8` after the rules still decides how the `.dic` flags are read,
+/// as in Hunspell. Only the first argument counts: `FLAG UTF-8 # Unicode flags` is UTF-8.
+fn directives(aff: &[u8]) -> (Option<String>, Option<String>) {
+    let (mut set, mut flag) = (None, None);
+    for l in aff.split(|&b| b == b'\n') {
+        if set.is_some() && flag.is_some() {
+            break;
+        }
+        let l = l.strip_prefix("\u{feff}".as_bytes()).unwrap_or(l);
+        let slot = match l.first() {
+            Some(b'S') if set.is_none() => (&mut set, &b"SET"[..]),
+            Some(b'F') if flag.is_none() => (&mut flag, &b"FLAG"[..]),
+            _ => continue,
+        };
+        let arg = l
+            .strip_prefix(slot.1)
+            .filter(|rest| rest.first().is_some_and(|b| matches!(b, b' ' | b'\t')))
+            .and_then(|rest| rest.split(|b| matches!(b, b' ' | b'\t' | b'\r')).find(|f| !f.is_empty()));
+        if let Some(a) = arg {
+            *slot.0 = Some(String::from_utf8_lossy(a).to_ascii_uppercase());
+        }
+    }
+    (set, flag)
+}
+
 /// Directives whose one argument is a flag.
 const FLAG_DIRECTIVES: &[&str] = &[
     "CIRCUMFIX",
@@ -406,7 +421,8 @@ fn comment_field(dir: &str, field: impl Fn(usize) -> Option<String>) -> usize {
 
 /// Where the `word/flags` entry of a `.dic` line ends, as Hunspell reads it: at a tab or at the
 /// first morphological field (a space or tab, two characters and `:`, as in ` po:noun`), whichever
-/// comes first, so an entry may hold spaces (`ad hoc`); with neither, at the first space.
+/// comes first, so an entry may hold spaces (`ad hoc`); with neither, at the end of the line
+/// (`kot/B 12` has the flags `B 12`, as in Hunspell).
 pub(crate) fn dic_entry_end(line: &[u8]) -> usize {
     let blank = |b: &u8| matches!(b, b' ' | b'\t');
     let morph = line.windows(4).position(|w| matches!(w, [a, b, c, b':'] if blank(a) && !blank(b) && !blank(c)));
@@ -415,10 +431,10 @@ pub(crate) fn dic_entry_end(line: &[u8]) -> usize {
         (Some(m), Some(t)) => m.min(t),
         (m, t) => match m.or(t) {
             Some(e) => e,
-            None => line.iter().position(|&b| b == b' ').unwrap_or(line.len()),
+            None => line.len(),
         },
     };
-    // Spaces before the morphological field aren't part of the entry.
+    // Blanks before the morphological field or the line end aren't part of the entry.
     line.get(..end).map_or(end, |e| e.iter().rposition(|b| !blank(b)).map_or(0, |k| k + 1))
 }
 
@@ -495,10 +511,10 @@ mod tests {
     fn utf8_dictionaries_keep_single_byte_flags() {
         // The Hungarian dictionary is `SET UTF-8` but writes flags as single bytes above 0x7F,
         // which aren't UTF-8; each byte is one flag. Its `NAME` line is ISO 8859-2.
-        let aff = b"SET UTF-8\nNAME Magyar helyes\xedr\xe1si\nAF 1\nAF \xd5a # 1\nSFX \xd5 Y 1\nSFX \xd5 0 ok/\xe9 . 7\nSFX \xe9 Y 1\nSFX \xe9 0 ban .\nSFX a Y 1\nSFX a 0 at .\n";
+        let aff = b"SET UTF-8\nNAME Magyar helyes\xedr\xe1si\nAF 2\nAF \xd5a # 1\nAF \xe9 # 2\nSFX \xd5 Y 1\nSFX \xd5 0 ok/2 . 7\nSFX \xe9 Y 1\nSFX \xe9 0 ban .\nSFX a Y 1\nSFX a 0 at .\n";
         let dic = "2\nház/1\t1\nkert/\u{e9}\n".as_bytes();
         let (a, d) = decode(aff, dic).unwrap();
-        assert!(a.contains("AF \u{d5}a # 1") && a.contains("SFX \u{d5} 0 ok/\u{e9} . 7"));
+        assert!(a.contains("AF \u{d5}a # 1") && a.contains("AF \u{e9} # 2") && a.contains("SFX \u{d5} 0 ok/2 . 7"));
         assert!(d.contains("kert/\u{c3}\u{a9}"), "a UTF-8 é in the flags is two one-byte flags");
         let dict = crate::affix::AffixDict::from_hunspell(&a, &d, &[]).unwrap();
         assert!(dict.check("ház") && dict.check("házok") && dict.check("házokban") && dict.check("házat"));
@@ -578,12 +594,17 @@ mod tests {
         assert_eq!(end("word/A po:noun\tts:x"), "word/A", "the morphological field comes before the tab");
         assert_eq!(end("ad hoc/B\tpo:adv"), "ad hoc/B", "a tab ends a phrase");
         assert_eq!(end("ad hoc/B po:adv"), "ad hoc/B");
-        assert_eq!(end("word/A 123"), "word/A", "neither: the first space");
+        assert_eq!(end("word/A 123"), "word/A 123", "neither: the whole line, as in Hunspell");
+        assert_eq!(end("ad hoc/B  "), "ad hoc/B");
         assert_eq!(end("word"), "word");
         assert_eq!(end("a:b/A"), "a:b/A", "a colon inside the word isn't a field");
         // Single-byte flags of a phrase entry are read byte by byte, then the UTF-8 morphology.
         let (_, d) = decode(b"SET UTF-8\n", b"1\nad hoc/\xe9\tpo:adv\n").unwrap();
         assert_eq!(d, "1\nad hoc/\u{e9}\tpo:adv\n");
+        // Without a tab or a morphological field the whole line is the entry: its byte flags
+        // don't fail the load (the Hungarian dictionary has such lines: `tök jó/396`).
+        let (_, d) = decode(b"SET UTF-8\n", b"1\nad hoc/\xe9\n").unwrap();
+        assert_eq!(d, "1\nad hoc/\u{e9}\n");
     }
 
     #[test]