diff --git a/crates/proof/src/affix.rs b/crates/proof/src/affix.rs index 2d3a7717..3fbb756f 100644 --- a/crates/proof/src/affix.rs +++ b/crates/proof/src/affix.rs @@ -343,28 +343,27 @@ struct Rules { ignore: Vec, /// `AF` flag aliases: alias `n` (1-based) stands for `aliases[n - 1]`. aliases: Vec, + /// The `AF` table opened with its count line, so flag fields are alias numbers. A table + /// without one is rejected, as in Hunspell: its lines are ignored. + aliases_on: bool, } impl Rules { - /// Flags as written in a `.dic` entry or a continuation class: an `AF` alias number when the - /// dictionary declares aliases, otherwise the flag characters themselves. - fn flags<'a>(&'a self, written: &'a str) -> Result<&'a str, String> { - if self.aliases.is_empty() || written.is_empty() || !written.bytes().all(|b| b.is_ascii_digit()) { - return Ok(written); + /// Flags as written in a `.dic` entry or a continuation class: the flag characters + /// themselves, or with an `AF` table the alias its leading digits number, as Hunspell reads + /// them (`2x` is alias 2). A field without digits or an undefined alias reads as no flags: + /// the stem is still a word, the rule still applies, without the flags (Hunspell warns). + fn flags<'a>(&'a self, written: &'a str) -> &'a str { + if !self.aliases_on { + return written; } + let digits = written.bytes().take_while(u8::is_ascii_digit).count(); written - .parse::() - .ok() + .get(..digits) + .and_then(|d| d.parse::().ok()) .and_then(|n| n.checked_sub(1)) .and_then(|i| self.aliases.get(i)) - .map(String::as_str) - .ok_or_else(|| format!("flag alias {written} is not defined")) - } - - /// The flags of a `.dic` entry. An undefined alias reads as no flags, as in Hunspell: the - /// stem is still a word, without affixes. - fn dic_flags<'a>(&'a self, written: &'a str) -> &'a str { - self.flags(written).unwrap_or("") + .map_or("", String::as_str) } /// `w` without the `IGNORE` characters. @@ -446,9 +445,10 @@ fn parse_aff(aff: &str) -> Result { // The first `AF` or `ICONV` line is the table's count, as in Hunspell, even with a // trailing comment (`AF 1559 # aliases`). "AF" => { - if !af_header && at(1).parse::().is_ok() { + if !af_header { af_header = true; - } else { + r.aliases_on = at(1).parse::().is_ok(); + } else if r.aliases_on { r.aliases.push(at(1).to_string()); } } @@ -479,7 +479,7 @@ fn parse_aff(aff: &str) -> Result { } } "REP" => { - if !rep_header && f.len() == 2 && at(1).parse::().is_ok() { + if !rep_header && at(1).parse::().is_ok() { rep_header = true; } else if f.len() >= 3 { r.rep.push((at(1).replace('_', " "), at(2).replace('_', " "))); @@ -522,16 +522,15 @@ fn parse_aff(aff: &str) -> Result { // Aliases may be declared after the rules that use them; resolve once all are read. // Hunspell removes `IGNORE` characters from the appended text only: a stripped text that // holds one never matches a stem (stems lose them too), so the rule never applies. - let resolve = |rules: &mut Vec, r: &Rules| -> Result<(), String> { + let resolve = |rules: &mut Vec, r: &Rules| { for rule in rules.iter_mut() { - rule.4 = r.flags(&rule.4)?.to_string(); + rule.4 = r.flags(&rule.4).to_string(); rule.3 = r.strip_ignored(&rule.3); } - Ok(()) }; let (mut pfx, mut sfx) = (std::mem::take(&mut r.prefixes), std::mem::take(&mut r.suffixes)); - resolve(&mut pfx, &r)?; - resolve(&mut sfx, &r)?; + resolve(&mut pfx, &r); + resolve(&mut sfx, &r); (r.prefixes, r.suffixes) = (pfx, sfx); Ok(r) } @@ -548,7 +547,7 @@ impl AffixDict { let mut stems: Vec<(String, String)> = Vec::new(); for (n, line) in dic.lines().enumerate() { // The entry may hold spaces (`ad hoc/B\tpo:adv`); it ends where Hunspell ends it. - let line = line.trim_start_matches([' ', '\t']); + // Leading blanks belong to it, as in Hunspell (` lab` is no `lab`). let tok = line.get(..crate::hunspell::dic_entry_end(line.as_bytes())).unwrap_or_default(); if tok.is_empty() { continue; @@ -579,7 +578,7 @@ impl AffixDict { if word.is_empty() || word.len() > MAX_WORD_BYTES || word.contains(['\n', '/']) { continue; } - let flags = rules.dic_flags(&flags).to_string(); + let flags = rules.flags(&flags).to_string(); stems.push((word, flags)); } AffixDict::build(rules, stems, comments.iter().map(ToString::to_string).collect()) @@ -828,28 +827,31 @@ impl AffixDict { /// `w` after the `ICONV` conversions (at each position the longest match wins) and without /// the `IGNORE` characters. fn normalize<'a>(&self, w: &'a str) -> std::borrow::Cow<'a, str> { - let converts = |rest: &str| self.iconv.iter().any(|(from, _)| !from.is_empty() && rest.contains(from.as_str())); - if !w.contains(&*self.ignore) && !converts(w) { + if self.iconv.is_empty() && self.ignore.is_empty() { return std::borrow::Cow::Borrowed(w); } - let mut out = String::with_capacity(w.len()); + // One pass: the input is copied only from the first change on. + let mut out: Option = None; let mut rest = w; while let Some(c) = rest.chars().next() { + let done = w.get(..w.len() - rest.len()).unwrap_or_default(); let hit = self.iconv.iter().filter(|(from, _)| !from.is_empty() && rest.starts_with(from.as_str())).max_by_key(|(from, _)| from.len()); match hit { Some((from, to)) => { - out.extend(to.chars().filter(|c| !self.ignore.contains(c))); + out.get_or_insert_with(|| done.to_string()).extend(to.chars().filter(|c| !self.ignore.contains(c))); rest = rest.get(from.len()..).unwrap_or_default(); } None => { - if !self.ignore.contains(&c) { - out.push(c); + if self.ignore.contains(&c) { + out.get_or_insert_with(|| done.to_string()); + } else if let Some(o) = out.as_mut() { + o.push(c); } rest = rest.get(c.len_utf8()..).unwrap_or_default(); } } } - std::borrow::Cow::Owned(out) + out.map_or(std::borrow::Cow::Borrowed(w), std::borrow::Cow::Owned) } /// [`AffixDict::contains`] on normalized input, where stems carrying a flag in `deny` don't @@ -1612,6 +1614,28 @@ pub(crate) mod tests { assert!(back.check("fines") && back.check("waffle") && back.check("waf(fle)")); } + #[test] + fn alias_numbers_are_read_as_hunspell_reads_them() { + // Checked with the Hunspell 1.7 command line. With an `AF` table a flag field is the alias + // its leading digits number: `fal/2x` is alias 2, `lab/b` has no flags. + let aff = "SET UTF-8\nAF 2\nAF B\nAF C\nSFX B Y 1\nSFX B 0 s .\nSFX C Y 1\nSFX C 0 x .\nSFX b Y 1\nSFX b 0 q .\n"; + let d = AffixDict::from_hunspell(aff, "3\nlab/b\nfal/2x\nkut/1\n", &[]).unwrap(); + assert!(d.check("lab") && !d.check("labq") && !d.check("labs")); + assert!(d.check("falx") && !d.check("fals") && d.check("kuts")); + // A table without its count line is rejected: no alias applies. + let broken = AffixDict::from_hunspell("SET UTF-8\nAF B\nAF C\nSFX B Y 1\nSFX B 0 s .\n", "2\nlab/1\nfal/2\n", &[]).unwrap(); + assert!(broken.check("lab") && !broken.check("labs") && !broken.check("fals")); + // An undefined alias in a continuation class leaves the rule in force, without it. + let cont = AffixDict::from_hunspell("SET UTF-8\nAF 1\nAF a\nSFX a Y 1\nSFX a 0 ok/7 .\n", "1\nhaz/1\n", &[]).unwrap(); + assert!(cont.check("haz") && cont.check("hazok")); + } + + #[test] + fn a_rep_count_line_may_carry_a_comment() { + let d = AffixDict::from_hunspell("SET UTF-8\nREP 1 # confusions\nREP x y\n", "1\nay\n", &[]).unwrap(); + assert_eq!(d.rep(), [("x".to_string(), "y".to_string())]); + } + #[test] fn count_lines_may_carry_a_comment() { // Hunspell reads the first `AF` and `ICONV` line as the count, comment and all: `AF 2 #` @@ -1630,10 +1654,14 @@ pub(crate) mod tests { fn dic_entries_may_hold_spaces() { // `ad hoc/B` is one entry, as in Hunspell; `ad` alone is no word. let aff = "SET UTF-8\nSFX B Y 1\nSFX B 0 s .\n"; - let d = AffixDict::from_hunspell(aff, "3\nad hoc/B\tpo:adv\nword/B po:noun\n kot/B 12\n", &[]).unwrap(); - assert!(d.contains("ad hoc") && d.contains("ad hocs")); - assert!(!d.contains("ad") && !d.contains("hoc") && !d.contains("ads")); + // Checked with the Hunspell 1.7 command line: without a tab or a morphological field the + // whole line is the entry (`kot/B 12` has the flag `B`), and leading blanks belong to it + // (` lab` is no `lab`). + let d = AffixDict::from_hunspell(aff, "5\nad hoc/B\tpo:adv\nword/B po:noun\nkot/B 12\n lab/B\nés így/B\n", &[]).unwrap(); + assert!(d.contains("ad hoc") && d.contains("ad hocs") && d.contains("és így") && d.contains("és ígys")); + assert!(!d.contains("ad") && !d.contains("hoc") && !d.contains("ads") && !d.contains("így")); assert!(d.contains("words") && d.contains("kots") && !d.contains("kot/B")); + assert!(!d.contains("lab") && !d.contains("labs")); } #[test] @@ -1691,8 +1719,9 @@ pub(crate) mod tests { for dir in UNSUPPORTED { assert!(AffixDict::from_hunspell(&format!("SET UTF-8\n{dir} X\n"), DIC, &[]).is_err(), "{dir} should be refused"); } - // The directives this engine reads, with no other change to the dictionary. - for line in ["AF 1\nAF ab", "AM 1\nAM po:noun", "ICONV 1\nICONV fi fi", "IGNORE ()", "KEEPCASE k", "SUBSTANDARD x", "LEMMA_PRESENT l"] { + // The directives this engine reads, with no other change to the dictionary (`AF` would + // turn the flags of `DIC` into alias numbers; it has tests of its own). + for line in ["AM 1\nAM po:noun", "ICONV 1\nICONV fi fi", "IGNORE ()", "KEEPCASE k", "SUBSTANDARD x", "LEMMA_PRESENT l"] { let d = AffixDict::from_hunspell(&format!("{AFF}{line}\n"), DIC, &[]).unwrap_or_else(|e| panic!("{line}: {e}")); assert!(d.check("koty") && d.check("niekoty") && !d.check("kotx"), "{line}"); } diff --git a/crates/proof/src/hunspell.rs b/crates/proof/src/hunspell.rs index 7152f7a4..d2d26416 100644 --- a/crates/proof/src/hunspell.rs +++ b/crates/proof/src/hunspell.rs @@ -224,22 +224,11 @@ fn decode_high(bytes: &[u8], high: &str) -> String { /// UTF-8. Those fields are read byte by byte (each byte becomes U+0000..U+00FF, one flag per /// byte); the words, affixes and conditions around them are UTF-8. pub fn decode(aff: &[u8], dic: &[u8]) -> Result<(String, String), String> { - // The whole file is searched: a `FLAG UTF-8` after the rules still decides how the `.dic` - // flags are read, as in Hunspell. - let directive = |name: &str| { - aff.split(|&b| b == b'\n').find_map(|l| { - let l = l.strip_prefix("\u{feff}".as_bytes()).unwrap_or(l); - // Only the first argument counts, as in Hunspell: `FLAG UTF-8 # Unicode flags` is UTF-8. - l.strip_prefix(name.as_bytes()) - .filter(|rest| rest.first().is_some_and(|b| matches!(b, b' ' | b'\t'))) - .and_then(|rest| rest.split(|b| matches!(b, b' ' | b'\t' | b'\r')).find(|f| !f.is_empty())) - .map(|f| String::from_utf8_lossy(f).to_ascii_uppercase()) - }) - }; - let set = directive("SET").unwrap_or_else(|| "UTF-8".into()); + let (set, flag) = directives(aff); + let set = set.unwrap_or_else(|| "UTF-8".into()); match set.as_str() { "UTF-8" | "UTF8" => { - if matches!(directive("FLAG").as_deref(), Some("UTF-8" | "UTF8")) { + if matches!(flag.as_deref(), Some("UTF-8" | "UTF8")) { let text = |b: &[u8], what: &str| String::from_utf8(b.to_vec()).map_err(|e| format!("{what}: not UTF-8: {e}")); return Ok((text(aff, ".aff")?, text(dic, ".dic")?)); } @@ -254,6 +243,32 @@ pub fn decode(aff: &[u8], dic: &[u8]) -> Result<(String, String), String> { } } +/// The first arguments of the `.aff`'s first `SET` and `FLAG` lines, in capitals, found in one pass +/// over the whole file: a `FLAG UTF-8` after the rules still decides how the `.dic` flags are read, +/// as in Hunspell. Only the first argument counts: `FLAG UTF-8 # Unicode flags` is UTF-8. +fn directives(aff: &[u8]) -> (Option, Option) { + let (mut set, mut flag) = (None, None); + for l in aff.split(|&b| b == b'\n') { + if set.is_some() && flag.is_some() { + break; + } + let l = l.strip_prefix("\u{feff}".as_bytes()).unwrap_or(l); + let slot = match l.first() { + Some(b'S') if set.is_none() => (&mut set, &b"SET"[..]), + Some(b'F') if flag.is_none() => (&mut flag, &b"FLAG"[..]), + _ => continue, + }; + let arg = l + .strip_prefix(slot.1) + .filter(|rest| rest.first().is_some_and(|b| matches!(b, b' ' | b'\t'))) + .and_then(|rest| rest.split(|b| matches!(b, b' ' | b'\t' | b'\r')).find(|f| !f.is_empty())); + if let Some(a) = arg { + *slot.0 = Some(String::from_utf8_lossy(a).to_ascii_uppercase()); + } + } + (set, flag) +} + /// Directives whose one argument is a flag. const FLAG_DIRECTIVES: &[&str] = &[ "CIRCUMFIX", @@ -406,7 +421,8 @@ fn comment_field(dir: &str, field: impl Fn(usize) -> Option) -> usize { /// Where the `word/flags` entry of a `.dic` line ends, as Hunspell reads it: at a tab or at the /// first morphological field (a space or tab, two characters and `:`, as in ` po:noun`), whichever -/// comes first, so an entry may hold spaces (`ad hoc`); with neither, at the first space. +/// comes first, so an entry may hold spaces (`ad hoc`); with neither, at the end of the line +/// (`kot/B 12` has the flags `B 12`, as in Hunspell). pub(crate) fn dic_entry_end(line: &[u8]) -> usize { let blank = |b: &u8| matches!(b, b' ' | b'\t'); let morph = line.windows(4).position(|w| matches!(w, [a, b, c, b':'] if blank(a) && !blank(b) && !blank(c))); @@ -415,10 +431,10 @@ pub(crate) fn dic_entry_end(line: &[u8]) -> usize { (Some(m), Some(t)) => m.min(t), (m, t) => match m.or(t) { Some(e) => e, - None => line.iter().position(|&b| b == b' ').unwrap_or(line.len()), + None => line.len(), }, }; - // Spaces before the morphological field aren't part of the entry. + // Blanks before the morphological field or the line end aren't part of the entry. line.get(..end).map_or(end, |e| e.iter().rposition(|b| !blank(b)).map_or(0, |k| k + 1)) } @@ -495,10 +511,10 @@ mod tests { fn utf8_dictionaries_keep_single_byte_flags() { // The Hungarian dictionary is `SET UTF-8` but writes flags as single bytes above 0x7F, // which aren't UTF-8; each byte is one flag. Its `NAME` line is ISO 8859-2. - let aff = b"SET UTF-8\nNAME Magyar helyes\xedr\xe1si\nAF 1\nAF \xd5a # 1\nSFX \xd5 Y 1\nSFX \xd5 0 ok/\xe9 . 7\nSFX \xe9 Y 1\nSFX \xe9 0 ban .\nSFX a Y 1\nSFX a 0 at .\n"; + let aff = b"SET UTF-8\nNAME Magyar helyes\xedr\xe1si\nAF 2\nAF \xd5a # 1\nAF \xe9 # 2\nSFX \xd5 Y 1\nSFX \xd5 0 ok/2 . 7\nSFX \xe9 Y 1\nSFX \xe9 0 ban .\nSFX a Y 1\nSFX a 0 at .\n"; let dic = "2\nház/1\t1\nkert/\u{e9}\n".as_bytes(); let (a, d) = decode(aff, dic).unwrap(); - assert!(a.contains("AF \u{d5}a # 1") && a.contains("SFX \u{d5} 0 ok/\u{e9} . 7")); + assert!(a.contains("AF \u{d5}a # 1") && a.contains("AF \u{e9} # 2") && a.contains("SFX \u{d5} 0 ok/2 . 7")); assert!(d.contains("kert/\u{c3}\u{a9}"), "a UTF-8 é in the flags is two one-byte flags"); let dict = crate::affix::AffixDict::from_hunspell(&a, &d, &[]).unwrap(); assert!(dict.check("ház") && dict.check("házok") && dict.check("házokban") && dict.check("házat")); @@ -578,12 +594,17 @@ mod tests { assert_eq!(end("word/A po:noun\tts:x"), "word/A", "the morphological field comes before the tab"); assert_eq!(end("ad hoc/B\tpo:adv"), "ad hoc/B", "a tab ends a phrase"); assert_eq!(end("ad hoc/B po:adv"), "ad hoc/B"); - assert_eq!(end("word/A 123"), "word/A", "neither: the first space"); + assert_eq!(end("word/A 123"), "word/A 123", "neither: the whole line, as in Hunspell"); + assert_eq!(end("ad hoc/B "), "ad hoc/B"); assert_eq!(end("word"), "word"); assert_eq!(end("a:b/A"), "a:b/A", "a colon inside the word isn't a field"); // Single-byte flags of a phrase entry are read byte by byte, then the UTF-8 morphology. let (_, d) = decode(b"SET UTF-8\n", b"1\nad hoc/\xe9\tpo:adv\n").unwrap(); assert_eq!(d, "1\nad hoc/\u{e9}\tpo:adv\n"); + // Without a tab or a morphological field the whole line is the entry: its byte flags + // don't fail the load (the Hungarian dictionary has such lines: `tök jó/396`). + let (_, d) = decode(b"SET UTF-8\n", b"1\nad hoc/\xe9\n").unwrap(); + assert_eq!(d, "1\nad hoc/\u{e9}\n"); } #[test]