daniel543x/craft-libs
1
0 Fork 0
mirror of https://github.com/storytold/craft-libs.git synced 2026-11-08 19:42:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge storytold/wordcraft#4575

Browse Source
This commit is contained in:
Martin Ball committed 2026-10-30 21:16:12 +00:00
2 parents 34f31b9a4c 1d2fa3c66d
commit 12c0bd61e3
2 files changed
+835 -113
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

crates/proof/src/affix.rs
+495 -109
View File
@@ -11,10 +11,14 @@
//! also be stored in WordCraft's compact format ([`AffixDict::to_bytes`]): the rules as text
//! lines and the stems front-coded, raw-deflated together (the Polish dictionary is under 1 MB).
//!
//! Supported: one-character flags (at most 128 distinct, any Unicode character — `FLAG UTF-8`),
//! `PFX`/`SFX` with strip, append and condition (`.`, letters, `[abc]`, `[^abc]`), cross
//! products, continuation classes on suffixes (at most two suffixes in a row, as in Hunspell),
//! `FORBIDDENWORD`, `NOSUGGEST`, and the suggestion tables `TRY`, `KEY`, `MAP` and `REP`. Rejected: compounding,
//! Supported: one-character flags (at most 256 distinct, any Unicode character — `FLAG UTF-8`),
//! flag aliases (`AF`: a number standing for a set of flags), `PFX`/`SFX` with strip, append and
//! condition (`.`, letters, `[abc]`, `[^abc]`), cross products, continuation classes on suffixes
//! (at most two suffixes in a row, as in Hunspell), `FORBIDDENWORD`, `NOSUGGEST`, `KEEPCASE` (the
//! stem is a word only in its own letter case), input conversion (`ICONV`: ligatures such as `fi`
//! read as `fi`) and ignored characters (`IGNORE`), and the suggestion tables `TRY`, `KEY`, `MAP`
//! and `REP`.
//! Morphological data (`AM` aliases, the fields after a stem) is skipped. Rejected: compounding,
//! long or numeric flags, `NEEDAFFIX` and the other directives that change which words are correct.
//!
//! Lookups never panic: stems sit in one string, found through an open-addressing hash table, and
@@ -33,30 +37,91 @@ const EMPTY: u32 = u32::MAX;
/// Directives that change which words are correct in ways this engine doesn't implement.
const UNSUPPORTED: &[&str] = &[
"AF",
"AM",
"CHECKSHARPS",
"CIRCUMFIX",
"COMPLEXPREFIXES",
"COMPOUNDBEGIN",
"COMPOUNDEND",
"COMPOUNDFIRST",
"COMPOUNDFLAG",
"COMPOUNDLAST",
"COMPOUNDMIDDLE",
"COMPOUNDMIN",
"COMPOUNDPERMITFLAG",
"COMPOUNDRULE",
"FULLSTRIP",
"ICONV",
"IGNORE",
"KEEPCASE",
"LEMMA_PRESENT",
"NEEDAFFIX",
"OCONV",
"ONLYINCOMPOUND",
"PSEUDOROOT",
"SUBSTANDARD",
];
/// Most distinct flags a dictionary may use: every single-byte flag (the Hungarian dictionary
/// uses 158).
const MAX_FLAGS: usize = 256;
/// A set of flags, one bit per distinct flag character of the dictionary.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)]
struct Mask([u64; MAX_FLAGS / 64]);
impl Mask {
const EMPTY: Mask = Mask([0; MAX_FLAGS / 64]);
/// The set holding flag number `i` alone (empty when `i` is out of range).
fn bit(i: usize) -> Mask {
let mut m = Mask::EMPTY;
if let Some(w) = m.0.get_mut(i / 64) {
*w = 1 << (i % 64);
}
m
}
/// The number of the lowest flag in the set, if any.
fn first(self) -> Option<usize> {
self.0.iter().enumerate().find(|(_, w)| **w != 0).map(|(i, w)| i * 64 + w.trailing_zeros() as usize)
}
fn is_empty(self) -> bool {
self.0.iter().all(|&w| w == 0)
}
/// Do the two sets share a flag?
fn meets(self, other: Mask) -> bool {
!(self & other).is_empty()
}
/// Does this set hold every flag of `other`?
fn covers(self, other: Mask) -> bool {
(self & other) == other
}
}
impl std::ops::BitAnd for Mask {
type Output = Mask;
fn bitand(mut self, o: Mask) -> Mask {
for (a, b) in self.0.iter_mut().zip(o.0) {
*a &= b;
}
self
}
}
impl std::ops::BitOr for Mask {
type Output = Mask;
fn bitor(mut self, o: Mask) -> Mask {
self |= o;
self
}
}
impl std::ops::BitOrAssign for Mask {
fn bitor_assign(&mut self, o: Mask) {
for (a, b) in self.0.iter_mut().zip(o.0) {
*a |= b;
}
}
}
/// One character class of a rule's condition.
#[derive(Clone, Debug, PartialEq, Eq)]
enum Cond {
@@ -130,14 +195,14 @@ const MAX_FORMS_WORK: usize = 1 << 18;
#[derive(Clone, Debug)]
struct Rule {
/// Bit of the rule's flag in the stems' flag masks.
bit: u128,
bit: Mask,
flag: char,
cross: bool,
strip: Box<str>,
add: Box<str>,
/// Continuation classes (`SFX A 0 able/B`): flags the derived form carries, licensing
/// further affixes. Chained application is bounded by [`MAX_CHAIN`].
cont: u128,
cont: Mask,
cond: Box<[Cond]>,
cond_text: Box<str>,
}
@@ -174,11 +239,13 @@ pub struct AffixDict {
/// Per stem: index into `sets`.
flags: Vec<u16>,
/// Distinct flag masks.
sets: Vec<u128>,
sets: Vec<Mask>,
/// The `FORBIDDENWORD` flag's bit; stems carrying it are not words (0: none declared).
forbidden: u128,
forbidden: Mask,
/// Stems with NOSUGGEST are correct, but none of their forms are suggested.
no_suggest: u128,
no_suggest: Mask,
/// Stems with SUBSTANDARD are correct too, and likewise never suggested.
substandard: Mask,
/// Open-addressing hash table of stem indices (`EMPTY` = free).
table: Vec<u32>,
/// Flag char of each bit.
@@ -195,6 +262,12 @@ pub struct AffixDict {
key: String,
map: Vec<Vec<char>>,
rep: Vec<(String, String)>,
/// The `KEEPCASE` flag's bit: such stems are words only in their own letter case (0: none).
keepcase: Mask,
/// `ICONV`: input conversions applied before a lookup, `(from, to)`.
iconv: Vec<(String, String)>,
/// `IGNORE`: characters removed from input words (and from the stems and appended texts).
ignore: Box<[char]>,
/// Provenance and licence lines, kept through [`AffixDict::to_bytes`].
comments: Vec<String>,
}
@@ -263,6 +336,57 @@ struct Rules {
/// The `FORBIDDENWORD` flag: stems carrying it are listed but never correct.
forbidden: Option<char>,
no_suggest: Option<char>,
substandard: Option<char>,
/// The `KEEPCASE` flag.
keepcase: Option<char>,
iconv: Vec<(String, String)>,
ignore: Vec<char>,
/// `AF` flag aliases: alias `n` (1-based) stands for `aliases[n - 1]`.
aliases: Vec<String>,
/// The `AF` table opened with its count line, so flag fields are alias numbers. A table
/// without one is rejected, as in Hunspell: its lines are ignored.
aliases_on: bool,
}
impl Rules {
/// Flags as written in a `.dic` entry or a continuation class: the flag characters
/// themselves, or with an `AF` table the alias its leading digits number, as Hunspell reads
/// them (`2x` is alias 2). A field without digits or an undefined alias reads as no flags:
/// the stem is still a word, the rule still applies, without the flags (Hunspell warns).
fn flags<'a>(&'a self, written: &'a str) -> &'a str {
if !self.aliases_on {
return written;
}
let digits = written.bytes().take_while(u8::is_ascii_digit).count();
written
.get(..digits)
.and_then(|d| d.parse::<usize>().ok())
.and_then(|n| n.checked_sub(1))
.and_then(|i| self.aliases.get(i))
.map_or("", String::as_str)
}
/// `w` without the `IGNORE` characters.
fn strip_ignored(&self, w: &str) -> String {
if self.ignore.is_empty() { w.to_string() } else { w.chars().filter(|c| !self.ignore.contains(c)).collect() }
}
}
/// The fields of an `.aff`/`.dic` line: Hunspell separates them with spaces and tabs only, so a
/// flag byte read as U+0085 or U+00A0 (see [`crate::hunspell::decode`]) stays a flag.
fn split_fields(line: &str) -> impl Iterator<Item = &str> {
line.split([' ', '\t']).filter(|f| !f.is_empty())
}
/// The one flag character a directive (`FORBIDDENWORD w`) names.
fn one_flag(arg: &str, dir: &str, n: usize) -> Result<char, String> {
let mut it = arg.chars().filter(|&c| c != EMOJI_SELECTOR);
match (it.next(), it.next()) {
// Only the field separators of `split_fields` can't be a flag: a flag byte read as
// U+0085 or U+00A0 is one.
(Some(c), None) if !matches!(c, ' ' | '\t') => Ok(c),
_ => Err(format!("line {}: {dir} needs one flag character", n + 1)),
}
}
/// A rule as parsed from text, before flags become bits: `(flag, cross, strip, add, cont, cond)`.
@@ -304,8 +428,10 @@ fn parse_aff(aff: &str) -> Result<Rules, String> {
let mut cross: HashMap<(bool, char), bool> = HashMap::new();
let mut map_header = false;
let mut rep_header = false;
let mut af_header = false;
let mut iconv_header = false;
for (n, line) in aff.lines().enumerate() {
let f: Vec<&str> = line.split_whitespace().collect();
let f: Vec<&str> = split_fields(line).collect();
let Some(&dir) = f.first() else { continue };
if dir.starts_with('#') {
continue;
@@ -314,25 +440,38 @@ fn parse_aff(aff: &str) -> Result<Rules, String> {
match dir {
"SET" | "LANG" | "NAME" | "VERSION" | "HOME" | "WORDCHARS" | "MAXNGRAMSUGS" | "MAXDIFF" | "ONLYMAXDIFF" | "NOSPLITSUGS"
| "SUGSWITHDOTS" | "WARN" | "FORBIDWARN" | "MAXCPDSUGS" => {}
// Morphological aliases, and a flag that only matters for morphological generation:
// the words it marks are correct.
"AM" | "LEMMA_PRESENT" => {}
"SUBSTANDARD" => r.substandard = Some(one_flag(at(1), dir, n)?),
// The first `AF` or `ICONV` line is the table's count, as in Hunspell, even with a
// trailing comment (`AF 1559 # aliases`).
"AF" => {
if !af_header {
af_header = true;
// A count below 1 rejects the table too: flags stay literal (`AF 0`).
r.aliases_on = at(1).parse::<usize>().is_ok_and(|n| n > 0);
} else if r.aliases_on {
r.aliases.push(at(1).to_string());
}
}
"ICONV" => {
if !iconv_header && at(1).parse::<usize>().is_ok() {
iconv_header = true;
} else if f.len() >= 3 {
r.iconv.push((at(1).to_string(), at(2).to_string()));
}
}
"IGNORE" => r.ignore = at(1).chars().collect(),
"KEEPCASE" => r.keepcase = Some(one_flag(at(1), dir, n)?),
// Per-character flag modes (`FLAG UTF-8`, as VERO's pt_BR dictionary uses) work
// with the one-character parsing here; only the multi-character encodings differ.
"FLAG" if !matches!(at(1), "UTF-8" | "DEFAULT" | "utf-8" | "default" | "UTF8" | "") => {
return Err(format!("line {}: FLAG {} is not supported (one-character flags only)", n + 1, at(1)));
}
"FLAG" => {}
directive @ ("FORBIDDENWORD" | "NOSUGGEST") => {
let mut it = at(1).chars().filter(|&c| c != EMOJI_SELECTOR);
match (it.next(), it.next()) {
(Some(c), None) if !c.is_whitespace() => {
if directive == "FORBIDDENWORD" {
r.forbidden = Some(c);
} else {
r.no_suggest = Some(c);
}
}
_ => return Err(format!("line {}: {directive} needs one flag character", n + 1)),
}
}
"FORBIDDENWORD" => r.forbidden = Some(one_flag(at(1), dir, n)?),
"NOSUGGEST" => r.no_suggest = Some(one_flag(at(1), dir, n)?),
"TRY" => r.try_chars = cap_try(at(1)),
"KEY" => r.key = at(1).to_string(),
"MAP" => {
@@ -343,7 +482,7 @@ fn parse_aff(aff: &str) -> Result<Rules, String> {
}
}
"REP" => {
if !rep_header && f.len() == 2 && at(1).parse::<usize>().is_ok() {
if !rep_header && at(1).parse::<usize>().is_ok() {
rep_header = true;
} else if f.len() >= 3 {
r.rep.push((at(1).replace('_', " "), at(2).replace('_', " ")));
@@ -383,6 +522,19 @@ fn parse_aff(aff: &str) -> Result<Rules, String> {
_ => {}
}
}
// Aliases may be declared after the rules that use them; resolve once all are read.
// Hunspell removes `IGNORE` characters from the appended text only: a stripped text that
// holds one never matches a stem (stems lose them too), so the rule never applies.
let resolve = |rules: &mut Vec<RawRule>, r: &Rules| {
for rule in rules.iter_mut() {
rule.4 = r.flags(&rule.4).to_string();
rule.3 = r.strip_ignored(&rule.3);
}
};
let (mut pfx, mut sfx) = (std::mem::take(&mut r.prefixes), std::mem::take(&mut r.suffixes));
resolve(&mut pfx, &r);
resolve(&mut sfx, &r);
(r.prefixes, r.suffixes) = (pfx, sfx);
Ok(r)
}
@@ -397,7 +549,12 @@ impl AffixDict {
let rules = parse_aff(aff)?;
let mut stems: Vec<(String, String)> = Vec::new();
for (n, line) in dic.lines().enumerate() {
let Some(tok) = line.split_whitespace().next() else { continue };
// The entry may hold spaces (`ad hoc/B\tpo:adv`); it ends where Hunspell ends it.
// Leading blanks belong to it, as in Hunspell (` lab` is no `lab`).
let tok = line.get(..crate::hunspell::dic_entry_end(line.as_bytes())).unwrap_or_default();
if tok.is_empty() {
continue;
}
if n == 0 && tok.parse::<usize>().is_ok() {
continue;
}
@@ -420,9 +577,11 @@ impl AffixDict {
word.push(c);
}
}
let word = rules.strip_ignored(&word);
if word.is_empty() || word.len() > MAX_WORD_BYTES || word.contains(['\n', '/']) {
continue;
}
let flags = rules.flags(&flags).to_string();
stems.push((word, flags));
}
AffixDict::build(rules, stems, comments.iter().map(ToString::to_string).collect())
@@ -434,11 +593,12 @@ impl AffixDict {
flag_chars.extend(stems.iter().flat_map(|s| s.1.chars()));
flag_chars.sort_unstable();
flag_chars.dedup();
if flag_chars.len() > 128 {
return Err(format!("{} distinct flags; at most 128 are supported", flag_chars.len()));
if flag_chars.len() > MAX_FLAGS {
return Err(format!("{} distinct flags; at most {MAX_FLAGS} are supported", flag_chars.len()));
}
let bit_of = |c: char| flag_chars.iter().position(|&x| x == c).map_or(0, |i| 1u128 << i);
let forbidden = rules.forbidden.map_or(0, bit_of);
let bit_of = |c: char| flag_chars.iter().position(|&x| x == c).map_or(Mask::EMPTY, Mask::bit);
let forbidden = rules.forbidden.map_or(Mask::EMPTY, bit_of);
let keepcase = rules.keepcase.map_or(Mask::EMPTY, bit_of);
// Stems sorted by bytes; homonyms (the same word twice) are merged.
stems.sort_by(|a, b| a.0.as_bytes().cmp(b.0.as_bytes()));
let mut d = AffixDict {
@@ -446,7 +606,11 @@ impl AffixDict {
offs: Vec::with_capacity(stems.len() + 1),
flag_chars: flag_chars.clone(),
forbidden,
no_suggest: rules.no_suggest.map_or(0, bit_of),
no_suggest: rules.no_suggest.map_or(Mask::EMPTY, bit_of),
substandard: rules.substandard.map_or(Mask::EMPTY, bit_of),
keepcase,
iconv: rules.iconv,
ignore: rules.ignore.into_boxed_slice(),
try_chars: rules.try_chars,
key: rules.key,
map: rules.map,
@@ -455,16 +619,16 @@ impl AffixDict {
..AffixDict::default()
};
d.offs.push(0);
let mut set_ids: HashMap<u128, u16> = HashMap::new();
let mut set_ids: HashMap<Mask, u16> = HashMap::new();
let mut i = 0;
while i < stems.len() {
let Some((word, _)) = stems.get(i) else { break };
let mut mask = 0u128;
let mut mask = Mask::EMPTY;
let mut j = i;
while let Some((w, f)) = stems.get(j)
&& w == word
{
mask |= f.chars().map(bit_of).fold(0, |a, b| a | b);
mask |= f.chars().map(bit_of).fold(Mask::EMPTY, |a, b| a | b);
j += 1;
}
let next_id = u16::try_from(d.sets.len()).map_err(|_| "too many distinct flag sets".to_string())?;
@@ -478,7 +642,7 @@ impl AffixDict {
i = j;
}
let mk = |(flag, cross, strip, add, cont, cond): RawRule| -> Result<Rule, String> {
let cont = cont.chars().fold(0u128, |m, c| m | bit_of(c));
let cont = cont.chars().fold(Mask::EMPTY, |m, c| m | bit_of(c));
Ok(Rule { bit: bit_of(flag), flag, cross, strip: strip.into(), add: add.into(), cont, cond: parse_cond(&cond)?, cond_text: cond.into() })
};
d.prefixes = rules.prefixes.into_iter().map(mk).collect::<Result<_, _>>()?;
@@ -493,7 +657,7 @@ impl AffixDict {
// Each (appended, stripped) pair's group, found by hash rather than by scanning the
// appended string's groups: a dictionary with many strips per suffix stays linear.
let mut group_of: FxMap<(&str, &str), usize> = FxMap::default();
let mut by_flag: Vec<Vec<u32>> = vec![Vec::new(); 128];
let mut by_flag: Vec<Vec<u32>> = vec![Vec::new(); MAX_FLAGS];
for (i, r) in self.suffixes.iter().enumerate() {
let groups = idx.entry(r.add.clone()).or_default();
let Ok(i) = u32::try_from(i) else { break };
@@ -504,9 +668,7 @@ impl AffixDict {
groups.push(SfxGroup { strip: r.strip.clone(), rules: vec![i] });
}
}
if r.bit != 0
&& let Some(list) = by_flag.get_mut(r.bit.trailing_zeros() as usize)
{
if let Some(list) = r.bit.first().and_then(|b| by_flag.get_mut(b)) {
list.push(i);
}
}
@@ -551,12 +713,12 @@ impl AffixDict {
}
/// The flag characters of `mask`, in flag order (how the compact format stores flags).
fn flag_chars_for(&self, mask: u128) -> String {
self.flag_chars.iter().enumerate().filter(|&(i, _)| mask & (1u128 << i) != 0).map(|(_, c)| c).collect()
fn flag_chars_for(&self, mask: Mask) -> String {
self.flag_chars.iter().enumerate().filter(|&(i, _)| mask.meets(Mask::bit(i))).map(|(_, c)| c).collect()
}
/// The character of `mask` when it holds exactly one flag.
fn flag_char(&self, mask: u128) -> Option<char> {
fn flag_char(&self, mask: Mask) -> Option<char> {
let s = self.flag_chars_for(mask);
let mut it = s.chars();
match (it.next(), it.next()) {
@@ -589,11 +751,12 @@ impl AffixDict {
}
/// Flag mask of a stem, if `w` is one.
fn stem_flags(&self, w: &str) -> Option<u128> {
self.stem_flags_excluding(w, self.forbidden)
fn stem_flags(&self, w: &str) -> Option<Mask> {
self.stem_flags_except(w, Mask::EMPTY)
}
fn stem_flags_excluding(&self, w: &str, excluded: u128) -> Option<u128> {
/// Flag mask of a stem, if `w` is one and carries none of the flags in `deny`.
fn stem_flags_except(&self, w: &str, deny: Mask) -> Option<Mask> {
if self.table.is_empty() {
return None;
}
@@ -607,8 +770,9 @@ impl AffixDict {
if self.word(t as usize) == Some(w) {
let id = *self.flags.get(t as usize)?;
let mask = self.sets.get(usize::from(id)).copied();
// All derivations use this lookup: exclusions apply to stems and their forms.
return mask.filter(|m| m & excluded == 0);
// A stem carrying FORBIDDENWORD is listed but is not a word: no direct hit,
// and (this is the only stem lookup) no affixed form is built on it either.
return mask.filter(|m| !m.meets(self.forbidden | deny));
}
slot = (slot + 1) & mask;
}
@@ -621,8 +785,8 @@ impl AffixDict {
/// The rules of a group that could lead to `w` are merged into one mask of their flags and
/// followed once, not once per rule: the answer is the same, but thousands of rules of one
/// flag cost one walk, not one each (#3613).
fn form_has_flag(&self, w: &str, flags: u128, depth: u8, excluded: u128) -> bool {
if self.stem_flags_excluding(w, excluded).is_some_and(|f| f & flags != 0) {
fn form_has_flag(&self, w: &str, flags: Mask, depth: u8, deny: Mask) -> bool {
if self.stem_flags_except(w, deny).is_some_and(|f| f.meets(flags)) {
return true;
}
if depth == 0 {
@@ -641,9 +805,9 @@ impl AffixDict {
.rules
.iter()
.filter_map(|&i| self.suffixes.get(i as usize))
.filter(|r| r.cont & flags != 0 && r.cond_matches(&root, true))
.fold(0u128, |m, r| m | r.bit);
if inner != 0 && self.form_has_flag(&root, inner, depth - 1, excluded) {
.filter(|r| r.cont.meets(flags) && r.cond_matches(&root, true))
.fold(Mask::EMPTY, |m, r| m | r.bit);
if !inner.is_empty() && self.form_has_flag(&root, inner, depth - 1, deny) {
return true;
}
}
@@ -651,21 +815,55 @@ impl AffixDict {
false
}
/// Is `w` exactly (case and all) a stem or an affixed form of one?
/// Is `w` exactly (case and all) a stem or an affixed form of one? `ICONV` and `IGNORE`
/// apply first.
pub fn contains(&self, w: &str) -> bool {
self.contains_excluding(w, self.forbidden)
self.contains_except(&self.normalize(w), Mask::EMPTY)
}
/// Exactly spelled dictionary words eligible for suggestions (NOSUGGEST stems excluded).
/// Exactly spelled dictionary words eligible for suggestions (NOSUGGEST and SUBSTANDARD
/// stems excluded).
pub fn contains_suggestible(&self, w: &str) -> bool {
self.contains_excluding(w, self.forbidden | self.no_suggest)
self.contains_except(&self.normalize(w), self.no_suggest | self.substandard)
}
fn contains_excluding(&self, w: &str, excluded: u128) -> bool {
/// `w` after the `ICONV` conversions (at each position the longest match wins) and without
/// the `IGNORE` characters.
fn normalize<'a>(&self, w: &'a str) -> std::borrow::Cow<'a, str> {
if self.iconv.is_empty() && self.ignore.is_empty() {
return std::borrow::Cow::Borrowed(w);
}
// One pass: the input is copied only from the first change on.
let mut out: Option<String> = None;
let mut rest = w;
while let Some(c) = rest.chars().next() {
let done = w.get(..w.len() - rest.len()).unwrap_or_default();
let hit = self.iconv.iter().filter(|(from, _)| !from.is_empty() && rest.starts_with(from.as_str())).max_by_key(|(from, _)| from.len());
match hit {
Some((from, to)) => {
out.get_or_insert_with(|| done.to_string()).extend(to.chars().filter(|c| !self.ignore.contains(c)));
rest = rest.get(from.len()..).unwrap_or_default();
}
None => {
if self.ignore.contains(&c) {
out.get_or_insert_with(|| done.to_string());
} else if let Some(o) = out.as_mut() {
o.push(c);
}
rest = rest.get(c.len_utf8()..).unwrap_or_default();
}
}
}
out.map_or(std::borrow::Cow::Borrowed(w), std::borrow::Cow::Owned)
}
/// [`AffixDict::contains`] on normalized input, where stems carrying a flag in `deny` don't
/// count (case variants of `KEEPCASE` stems).
fn contains_except(&self, w: &str, deny: Mask) -> bool {
if w.is_empty() || w.len() > MAX_WORD_BYTES {
return false;
}
if self.stem_flags_excluding(w, excluded).is_some() || self.suffixed(w, 0, excluded) {
if self.stem_flags_except(w, deny).is_some() || self.suffixed(w, Mask::EMPTY, deny) {
return true;
}
let mut root = String::with_capacity(w.len() + 8);
@@ -680,13 +878,13 @@ impl AffixDict {
if !r.cond_matches(&root, false) {
continue;
}
if self.stem_flags_excluding(&root, excluded).is_some_and(|f| f & r.bit != 0) {
if self.stem_flags_except(&root, deny).is_some_and(|f| f.meets(r.bit)) {
return true;
}
if self.form_has_flag(&root, r.bit, INNER_CHAIN, excluded) {
if self.form_has_flag(&root, r.bit, INNER_CHAIN, deny) {
return true;
}
if r.cross && self.suffixed(&root, r.bit, excluded) {
if r.cross && self.suffixed(&root, r.bit, deny) {
return true;
}
}
@@ -695,7 +893,7 @@ impl AffixDict {
/// Is `w` a stem with one suffix? With `need` set (a prefix was taken off), the stem must
/// carry those flags too and the suffix must allow cross products.
fn suffixed(&self, w: &str, need: u128, excluded: u128) -> bool {
fn suffixed(&self, w: &str, need: Mask, deny: Mask) -> bool {
let mut root = String::with_capacity(w.len() + 8);
// Each split point from the end: the appended string is `w[b..]`, at most the longest
// suffix, and something must stay before it.
@@ -710,18 +908,18 @@ impl AffixDict {
let rules = || g.rules.iter().filter_map(|&i| self.suffixes.get(i as usize));
// The stem path: `root` is a stem carrying the rule's flag (and the prefix's,
// when a cross product is being checked).
if let Some(f) = self.stem_flags_excluding(&root, excluded)
&& f & need == need
&& rules().any(|r| f & r.bit != 0 && (need == 0 || r.cross) && r.cond_matches(&root, true))
if let Some(f) = self.stem_flags_except(&root, deny)
&& f.covers(need)
&& rules().any(|r| f.meets(r.bit) && (need.is_empty() || r.cross) && r.cond_matches(&root, true))
{
return true;
}
// Continuation classes: `root` need not be a stem carrying this rule's flag —
// it may be a form that gained the flag from another rule's continuation. The
// matching rules' flags are looked for together, in one walk.
if need == 0 {
let flags = rules().filter(|r| r.cond_matches(&root, true)).fold(0u128, |m, r| m | r.bit);
if flags != 0 && self.form_has_flag(&root, flags, INNER_CHAIN, excluded) {
if need.is_empty() {
let flags = rules().filter(|r| r.cond_matches(&root, true)).fold(Mask::EMPTY, |m, r| m | r.bit);
if !flags.is_empty() && self.form_has_flag(&root, flags, INNER_CHAIN, deny) {
return true;
}
}
@@ -732,26 +930,29 @@ impl AffixDict {
/// Is `w` correct, Hunspell style: as written, or — for a capitalized word or one in capitals
/// — in lower case (and capitalized), since a sentence may start with any word and headings
/// may be in capitals. Lower-case input never matches a capitalized stem (`warszawa`).
/// may be in capitals. Lower-case input never matches a capitalized stem (`warszawa`), and a
/// `KEEPCASE` stem matches only as written.
pub fn check(&self, w: &str) -> bool {
self.check_excluding(w, self.forbidden)
self.check_except(w, Mask::EMPTY)
}
/// Whether a suggestion is correct in its displayed case and has an eligible stem.
pub fn can_suggest(&self, w: &str) -> bool {
self.check_excluding(w, self.forbidden | self.no_suggest)
self.check_except(w, self.no_suggest | self.substandard)
}
fn check_excluding(&self, w: &str, excluded: u128) -> bool {
if self.contains_excluding(w, excluded) {
fn check_except(&self, w: &str, deny: Mask) -> bool {
let w = self.normalize(w);
if self.contains_except(&w, deny) {
return true;
}
match casing(w) {
let deny = deny | self.keepcase;
match casing(&w) {
Casing::Lower | Casing::Mixed => false,
Casing::Initial => self.contains_excluding(&w.to_lowercase(), excluded),
Casing::Initial => self.contains_except(&w.to_lowercase(), deny),
Casing::Upper => {
let lower = w.to_lowercase();
self.contains_excluding(&capitalize(&lower), excluded) || self.contains_excluding(&lower, excluded)
self.contains_except(&capitalize(&lower), deny) || self.contains_except(&lower, deny)
}
}
}
@@ -766,11 +967,11 @@ impl AffixDict {
let mut cross_sfx = Vec::new();
// Suffix forms queued with the continuation classes they carry, for chained expansion;
// each (form, classes) pair is expanded once.
let mut chain: Vec<(String, u128)> = Vec::new();
let mut queued: std::collections::HashSet<(String, u128)> = std::collections::HashSet::new();
let mut chain: Vec<(String, Mask)> = Vec::new();
let mut queued: std::collections::HashSet<(String, Mask)> = std::collections::HashSet::new();
let mut work = 0usize;
for r in &self.suffixes {
if f & r.bit == 0 || !r.cond_matches(stem, true) {
if !f.meets(r.bit) || !r.cond_matches(stem, true) {
continue;
}
let Some(base) = stem.strip_suffix(&*r.strip) else { continue };
@@ -778,7 +979,7 @@ impl AffixDict {
if r.cross {
cross_sfx.push(form.clone());
}
if r.cont != 0 && queued.insert((form.clone(), r.cont)) {
if !r.cont.is_empty() && queued.insert((form.clone(), r.cont)) {
chain.push((form.clone(), r.cont));
}
out.push(form);
@@ -804,7 +1005,7 @@ impl AffixDict {
}
let Some(base) = form.strip_suffix(&*r.strip) else { continue };
let next = format!("{base}{}", r.add);
if r.cont != 0 && queued.insert((next.clone(), r.cont)) {
if !r.cont.is_empty() && queued.insert((next.clone(), r.cont)) {
chain.push((next.clone(), r.cont));
}
out.push(next);
@@ -815,7 +1016,7 @@ impl AffixDict {
}
}
for r in &self.prefixes {
if f & r.bit == 0 || !r.cond_matches(stem, false) {
if !f.meets(r.bit) || !r.cond_matches(stem, false) {
continue;
}
let Some(rest) = stem.strip_prefix(&*r.strip) else { continue };
@@ -839,11 +1040,11 @@ impl AffixDict {
}
/// The suffix rules carrying any flag of `mask`, in rule order.
fn suffixes_with(&self, mask: u128) -> Vec<u32> {
fn suffixes_with(&self, mask: Mask) -> Vec<u32> {
let mut rules: Vec<u32> = Vec::new();
let mut lists = 0;
for (bit, list) in self.sfx_by_flag.iter().enumerate() {
if bit < 128 && mask & (1u128 << bit) != 0 && !list.is_empty() {
if mask.meets(Mask::bit(bit)) && !list.is_empty() {
rules.extend_from_slice(list);
lists += 1;
}
@@ -904,13 +1105,21 @@ impl AffixDict {
for (a, b) in &self.rep {
out.push_str(&format!("REP {} {}\n", a.replace(' ', "_"), b.replace(' ', "_")));
}
for (directive, mask) in [("FORBIDDENWORD", self.forbidden), ("NOSUGGEST", self.no_suggest)] {
if mask != 0
for (directive, mask) in
[("FORBIDDENWORD", self.forbidden), ("NOSUGGEST", self.no_suggest), ("SUBSTANDARD", self.substandard), ("KEEPCASE", self.keepcase)]
{
if !mask.is_empty()
&& let Some(c) = self.flag_char(mask)
{
out.push_str(&format!("{directive} {c}\n"));
}
}
for (a, b) in &self.iconv {
out.push_str(&format!("ICONV {a} {b}\n"));
}
if !self.ignore.is_empty() {
out.push_str(&format!("IGNORE {}\n", self.ignore.iter().collect::<String>()));
}
let z = |s: &str| if s.is_empty() { "0".to_string() } else { s.to_string() };
for (kind, rules) in [("PFX", &self.prefixes), ("SFX", &self.suffixes)] {
for r in rules {
@@ -934,8 +1143,8 @@ impl AffixDict {
let p = prev.iter().zip(b).take_while(|(x, y)| x == y).count().min(255);
raw.push(p as u8);
raw.extend_from_slice(b.get(p..).unwrap_or_default());
let mask = self.flags.get(i).and_then(|&id| self.sets.get(usize::from(id))).copied().unwrap_or(0);
if mask != 0 {
let mask = self.flags.get(i).and_then(|&id| self.sets.get(usize::from(id))).copied().unwrap_or_default();
if !mask.is_empty() {
raw.push(b'/');
raw.extend_from_slice(self.flag_chars_for(mask).as_bytes());
}
@@ -962,26 +1171,24 @@ impl AffixDict {
comments.push(c.to_string());
continue;
}
let f: Vec<&str> = line.split_whitespace().collect();
let f: Vec<&str> = split_fields(line).collect();
let at = |k: usize| f.get(k).copied().unwrap_or("");
match at(0) {
"TRY" => rules.try_chars = cap_try(at(1)),
"KEY" => rules.key = at(1).to_string(),
"MAP" => rules.map.push(at(1).chars().collect()),
"REP" => rules.rep.push((at(1).replace('_', " "), at(2).replace('_', " "))),
directive @ ("FORBIDDENWORD" | "NOSUGGEST") => {
let mut it = at(1).chars();
match (it.next(), it.next()) {
(Some(c), None) if !c.is_whitespace() => {
if directive == "FORBIDDENWORD" {
rules.forbidden = Some(c);
} else {
rules.no_suggest = Some(c);
}
}
_ => return Err(format!("affix dictionary: bad {directive} in `{line}`")),
directive @ ("FORBIDDENWORD" | "NOSUGGEST" | "SUBSTANDARD" | "KEEPCASE") => {
let c = one_flag(at(1), directive, 0).map_err(|_| format!("affix dictionary: bad {directive} in `{line}`"))?;
match directive {
"FORBIDDENWORD" => rules.forbidden = Some(c),
"NOSUGGEST" => rules.no_suggest = Some(c),
"SUBSTANDARD" => rules.substandard = Some(c),
_ => rules.keepcase = Some(c),
}
}
"ICONV" if f.len() == 3 => rules.iconv.push((at(1).to_string(), at(2).to_string())),
"IGNORE" => rules.ignore = at(1).chars().collect(),
"MAP" => rules.map.push(at(1).chars().collect()),
"REP" => rules.rep.push((at(1).replace('_', " "), at(2).replace('_', " "))),
kind @ ("PFX" | "SFX") => {
let mut fc = at(1).chars();
let (Some(flag), None) = (fc.next(), fc.next()) else { return Err(format!("affix dictionary: bad flag in `{line}`")) };
@@ -1223,7 +1430,7 @@ pub(crate) mod tests {
#[test]
fn more_than_64_distinct_flags_load() {
// VERO pt_BR uses 106 distinct flags (FLAG UTF-8). Build one with every basic Latin
// letter, the digits and eight accented letters: 70 flags, 64 < n <= 128.
// letter, the digits and eight accented letters: 70 flags, more than 64.
let mut flags = String::new();
let mut aff = String::from("SET UTF-8\nFLAG UTF-8\n");
let chars: Vec<char> = ('a'..='z').chain('A'..='Z').chain('0'..='9').chain(['á', 'é', 'í', 'ó', 'ú', 'ç', 'ã', 'õ']).collect();
@@ -1239,6 +1446,24 @@ pub(crate) mod tests {
assert!(!d.contains("verboß"), "ß is not a flag and not a stem");
}
#[test]
fn more_than_128_distinct_flags_load() {
// The Hungarian dictionary uses 158 single-byte flags: 200 flags here, U+0080 up.
let flags: String = (0x80u32..0x80 + 200).filter_map(char::from_u32).collect();
let mut aff = String::from("SET UTF-8\n");
for (i, c) in flags.chars().enumerate() {
aff.push_str(&format!("SFX {c} N 1\nSFX {c} 0 x{i} .\n"));
}
let half: String = flags.chars().step_by(2).collect();
let d = AffixDict::from_hunspell(&aff, &format!("1\nszó/{half}\n"), &[]).unwrap();
assert!(d.contains("szóx0") && d.contains("szóx198") && d.contains("szóx130"));
assert!(!d.contains("szóx1") && !d.contains("szóx199") && !d.contains("szóx131"));
let back = AffixDict::from_bytes(&d.to_bytes()).unwrap();
assert!(back.contains("szóx198") && !back.contains("szóx199"));
let too_many: String = (0x100u32..0x100 + 257).filter_map(char::from_u32).collect();
assert!(AffixDict::from_hunspell("SET UTF-8\n", &format!("1\nszó/{too_many}\n"), &[]).is_err());
}
#[test]
fn forbidden_words_are_rejected() {
// VERO pt_BR marks spellings it must reject (flag ý under FLAG UTF-8) as stems carrying
@@ -1364,6 +1589,167 @@ pub(crate) mod tests {
assert!(d.contains("kot") && d.contains("koty"));
}
#[test]
fn flag_aliases_stand_for_flag_sets() {
// `AF`: `/1` in the `.dic` and in a continuation class means the first alias's flags.
// Morphological aliases (`AM`) and the fields after a stem are skipped.
let aff = "SET UTF-8\nAF 2\nAF ab # 1\nAF c # 2\nAM 1\nAM po:noun\nSFX a Y 1\nSFX a 0 ok/2 . 1\nSFX b Y 1\nSFX b 0 at . 1\nSFX c Y 1\nSFX c 0 ban . 1\n";
let d = AffixDict::from_hunspell(aff, "1\nház/1\t1\n", &[]).unwrap();
assert!(d.contains("ház") && d.contains("házok") && d.contains("házat") && d.contains("házokban"));
assert!(!d.contains("házban"), "the stem has a and b, not c");
// An undefined alias reads as no flags, as in Hunspell: the stem loads, without affixes.
let undefined = AffixDict::from_hunspell(aff, "2\nház/3\nkert/1\n", &[]).unwrap();
assert!(undefined.check("ház") && !undefined.check("házok"), "alias 3 is not defined");
assert!(undefined.check("kertok"), "the other entries keep their aliases");
let back = AffixDict::from_bytes(&d.to_bytes()).unwrap();
assert!(back.contains("házokban") && !back.contains("házban"), "aliases are expanded in the compact format");
}
#[test]
fn iconv_and_ignore_normalize_input() {
// Ligatures read as their letters; `IGNORE` characters vanish from input and stems.
let aff = "SET UTF-8\nICONV 2\nICONV fi fi\nICONV ffl ffl\nIGNORE ()\nSFX a Y 1\nSFX a 0 s .\n";
let d = AffixDict::from_hunspell(aff, "2\nfine/a\nwaf(fle)/a\n", &[]).unwrap();
assert!(d.check("fine") && d.check("fines") && d.check("Fine") && d.contains("fine"));
assert!(d.check("waffle") && d.check("waffles") && d.check("wa(ffle)"));
assert!(!d.check("finer"));
let back = AffixDict::from_bytes(&d.to_bytes()).unwrap();
assert!(back.check("fines") && back.check("waffle") && back.check("waf(fle)"));
}
#[test]
fn alias_numbers_are_read_as_hunspell_reads_them() {
// Checked with the Hunspell 1.7 command line. With an `AF` table a flag field is the alias
// its leading digits number: `fal/2x` is alias 2, `lab/b` has no flags.
let aff = "SET UTF-8\nAF 2\nAF B\nAF C\nSFX B Y 1\nSFX B 0 s .\nSFX C Y 1\nSFX C 0 x .\nSFX b Y 1\nSFX b 0 q .\n";
let d = AffixDict::from_hunspell(aff, "3\nlab/b\nfal/2x\nkut/1\n", &[]).unwrap();
assert!(d.check("lab") && !d.check("labq") && !d.check("labs"));
assert!(d.check("falx") && !d.check("fals") && d.check("kuts"));
// A table without its count line is rejected: no alias applies.
let broken = AffixDict::from_hunspell("SET UTF-8\nAF B\nAF C\nSFX B Y 1\nSFX B 0 s .\n", "2\nlab/1\nfal/2\n", &[]).unwrap();
assert!(broken.check("lab") && !broken.check("labs") && !broken.check("fals"));
// `AF 0` is rejected the same way: `lab/B` keeps its literal flag.
let zero = AffixDict::from_hunspell("SET UTF-8\nAF 0\nSFX B Y 1\nSFX B 0 s .\n", "1\nlab/B\n", &[]).unwrap();
assert!(zero.check("lab") && zero.check("labs"));
// An undefined alias in a continuation class leaves the rule in force, without it.
let cont = AffixDict::from_hunspell("SET UTF-8\nAF 1\nAF a\nSFX a Y 1\nSFX a 0 ok/7 .\n", "1\nhaz/1\n", &[]).unwrap();
assert!(cont.check("haz") && cont.check("hazok"));
}
#[test]
fn a_rep_count_line_may_carry_a_comment() {
let d = AffixDict::from_hunspell("SET UTF-8\nREP 1 # confusions\nREP x y\n", "1\nay\n", &[]).unwrap();
assert_eq!(d.rep(), [("x".to_string(), "y".to_string())]);
}
#[test]
fn count_lines_may_carry_a_comment() {
// Hunspell reads the first `AF` and `ICONV` line as the count, comment and all: `AF 2 #`
// is no alias, and `ICONV 1 # …` doesn't turn `1` into `#`.
let aff = "SET UTF-8\nAF 2 # aliases\nAF a # 1\nAF b # 2\nICONV 1 # ligatures\nICONV q y\nSFX a Y 1\nSFX a 0 ok .\nSFX b Y 1\nSFX b 0 at .\n";
let d = AffixDict::from_hunspell(aff, "3\nhaz/1\nkert/2\na1\n", &[]).unwrap();
assert!(d.check("hazok") && !d.check("hazat") && d.check("kertat") && !d.check("kertok"), "alias 1 is `a`");
assert!(d.check("a1") && !d.check("a#"));
// The UTF-8 reader keeps such a line as it is, so both readers agree on it.
let (a, dic) = crate::hunspell::decode(aff.as_bytes(), "1\nhaz/1\n".as_bytes()).unwrap();
assert!(a.contains("AF 2 # aliases") && a.contains("ICONV 1 # ligatures"), "{a}");
assert!(AffixDict::from_hunspell(&a, &dic, &[]).unwrap().check("hazok"));
}
#[test]
fn dic_entries_may_hold_spaces() {
// `ad hoc/B` is one entry, as in Hunspell; `ad` alone is no word.
let aff = "SET UTF-8\nSFX B Y 1\nSFX B 0 s .\n";
// Checked with the Hunspell 1.7 command line: without a tab or a morphological field the
// whole line is the entry (`kot/B 12` has the flag `B`), and leading blanks belong to it
// (` lab` is no `lab`).
let d = AffixDict::from_hunspell(aff, "5\nad hoc/B\tpo:adv\nword/B po:noun\nkot/B 12\n lab/B\nés így/B\n", &[]).unwrap();
assert!(d.contains("ad hoc") && d.contains("ad hocs") && d.contains("és így") && d.contains("és ígys"));
assert!(!d.contains("ad") && !d.contains("hoc") && !d.contains("ads") && !d.contains("így"));
assert!(d.contains("words") && d.contains("kots") && !d.contains("kot/B"));
assert!(!d.contains("lab") && !d.contains("labs"));
}
#[test]
fn ignored_characters_leave_the_stripped_text_alone() {
// Hunspell removes `IGNORE` characters from the appended text, not from the stripped
// one: `SFX a bx c` never matches the stem `abx` (stored as `ab`), so `ac` is wrong,
// while `SFX a b cx` appends `c` (checked with the Hunspell 1.7 command line).
let strip = AffixDict::from_hunspell("SET UTF-8\nIGNORE x\nSFX a Y 1\nSFX a bx c .\n", "1\nabx/a\n", &[]).unwrap();
assert!(strip.check("ab") && strip.check("abx") && !strip.check("ac"));
let add = AffixDict::from_hunspell("SET UTF-8\nIGNORE x\nSFX a Y 1\nSFX a b cx .\n", "1\nab/a\n", &[]).unwrap();
assert!(add.check("ac") && add.check("acx"));
for d in [&strip, &add] {
let back = AffixDict::from_bytes(&d.to_bytes()).unwrap();
assert_eq!(back.check("ac"), d.check("ac"));
}
}
#[test]
fn normalizing_borrows_when_nothing_changes() {
let d = AffixDict::from_hunspell("SET UTF-8\nICONV 1\nICONV fi fi\nIGNORE ()\n", "1\nfine\n", &[]).unwrap();
assert!(matches!(d.normalize("fine"), std::borrow::Cow::Borrowed("fine")));
assert_eq!(d.normalize("(fi)ne"), "fine");
assert!(matches!(dict().normalize("kot"), std::borrow::Cow::Borrowed("kot")));
}
#[test]
fn keepcase_stems_match_only_as_written() {
let aff = "SET UTF-8\nKEEPCASE k\nSFX a Y 1\nSFX a 0 s .\n";
let d = AffixDict::from_hunspell(aff, "3\npH/k\nkm/ka\nkot/a\n", &[]).unwrap();
assert!(d.check("pH") && d.check("km") && d.check("kms"));
assert!(!d.check("PH") && !d.check("Km") && !d.check("KM") && !d.check("Kms"));
assert!(d.check("Kot") && d.check("KOTS"), "other stems still follow the sentence and heading case");
let back = AffixDict::from_bytes(&d.to_bytes()).unwrap();
assert!(back.check("km") && !back.check("Km"));
}
#[test]
fn substandard_and_lemma_present_words_are_correct() {
// `SUBSTANDARD` and `LEMMA_PRESENT` steer morphological generation, and a SUBSTANDARD
// stem is never suggested; the stems and rules they mark are still correct.
let aff = "SET UTF-8\nSUBSTANDARD x\nLEMMA_PRESENT l\nSFX a Y 2\nSFX a 0 y .\nSFX a 0 em/x .\n";
let d = AffixDict::from_hunspell(aff, "2\nkot/a\npies/axl\n", &[]).unwrap();
for dict in [&d, &AffixDict::from_bytes(&d.to_bytes()).unwrap()] {
assert!(dict.check("kot") && dict.check("koty") && dict.check("kotem"));
assert!(dict.check("pies") && dict.check("piesy") && dict.check("Piesem"));
assert!(dict.can_suggest("koty") && dict.can_suggest("kotem") && dict.contains_suggestible("kot"));
assert!(!dict.can_suggest("piesy") && !dict.can_suggest("Pies") && !dict.contains_suggestible("pies"));
}
}
#[test]
fn unsupported_directives_are_refused_and_the_rest_load() {
// A dictionary that needs a directive outside this engine fails to load rather than
// accepting the wrong words.
for dir in UNSUPPORTED {
assert!(AffixDict::from_hunspell(&format!("SET UTF-8\n{dir} X\n"), DIC, &[]).is_err(), "{dir} should be refused");
}
// The directives this engine reads, with no other change to the dictionary (`AF` would
// turn the flags of `DIC` into alias numbers; it has tests of its own).
for line in ["AM 1\nAM po:noun", "ICONV 1\nICONV fi fi", "IGNORE ()", "KEEPCASE k", "SUBSTANDARD x", "LEMMA_PRESENT l"] {
let d = AffixDict::from_hunspell(&format!("{AFF}{line}\n"), DIC, &[]).unwrap_or_else(|e| panic!("{line}: {e}"));
assert!(d.check("koty") && d.check("niekoty") && !d.check("kotx"), "{line}");
}
}
#[test]
fn fields_split_on_spaces_and_tabs_only() {
// A flag byte read as U+00A0 (no-break space) or U+0085 is a flag, not a separator.
let aff = "SET UTF-8\nSFX \u{a0} Y 1\nSFX \u{a0} 0 y .\nSFX \u{85} Y 1\nSFX \u{85} 0 em .\n";
let d = AffixDict::from_hunspell(aff, "1\nkot/\u{a0}\u{85}\n", &[]).unwrap();
assert!(d.contains("koty") && d.contains("kotem"));
let back = AffixDict::from_bytes(&d.to_bytes()).unwrap();
assert!(back.contains("koty") && back.contains("kotem"));
// The same bytes as one-flag directives, as Hunspell reads them.
let aff = "SET UTF-8\nFORBIDDENWORD \u{a0}\nKEEPCASE \u{85}\nSFX a Y 1\nSFX a 0 s .\n";
let d = AffixDict::from_hunspell(aff, "3\nkot/a\nrossz/\u{a0}\npH/\u{85}\n", &[]).unwrap();
for dict in [&d, &AffixDict::from_bytes(&d.to_bytes()).unwrap()] {
assert!(dict.check("kots") && !dict.check("rossz"));
assert!(dict.check("pH") && !dict.check("PH"));
}
}
#[test]
fn unsupported_and_broken_input_is_an_error() {
assert!(AffixDict::from_hunspell("FLAG long\n", DIC, &[]).is_err());
crates/proof/src/hunspell.rs
+340 -4
View File
@@ -218,13 +218,23 @@ fn decode_high(bytes: &[u8], high: &str) -> String {
/// (UTF-8 when there is none; ISO 8859-2 is the other one Polish dictionaries use, ISO 8859-1 the
/// one the original Portuguese VERO uses, ISO 8859-9 the one older Turkish dictionaries use, KOI8-R
/// and Windows-1251 the ones older Russian dictionaries use).
///
/// Under a UTF-8 `SET` with single-byte flags (no `FLAG` line, or `FLAG DEFAULT`), flags are bytes,
/// as in Hunspell: the Hungarian dictionary writes its affix flags and flag aliases as bytes above
/// 0x7F that aren't UTF-8. Those fields are read byte by byte (each byte becomes U+0000..U+00FF,
/// one flag per byte); the words, affixes and conditions around them are UTF-8. With any other
/// `FLAG` (`UTF-8`, `num`, `long`) flags are text and the whole file is UTF-8: `SFX 1234` keeps
/// its number (the Turkish dictionary's `FLAG num`).
pub fn decode(aff: &[u8], dic: &[u8]) -> Result<(String, String), String> {
let head = String::from_utf8_lossy(aff.get(..aff.len().min(8192)).unwrap_or_default()).to_string();
let set = head.lines().find_map(|l| l.trim_start_matches('\u{feff}').strip_prefix("SET ")).map(str::trim).unwrap_or("UTF-8").to_ascii_uppercase();
let (set, flag) = directives(aff);
let set = set.unwrap_or_else(|| "UTF-8".into());
match set.as_str() {
"UTF-8" | "UTF8" => {
let text = |b: &[u8], what: &str| String::from_utf8(b.to_vec()).map_err(|e| format!("{what}: not UTF-8: {e}"));
Ok((text(aff, ".aff")?, text(dic, ".dic")?))
if flag.as_deref().is_some_and(|f| f != "DEFAULT") {
let text = |b: &[u8], what: &str| String::from_utf8(b.to_vec()).map_err(|e| format!("{what}: not UTF-8: {e}"));
return Ok((text(aff, ".aff")?, text(dic, ".dic")?));
}
Ok((decode_utf8_aff(aff)?, decode_utf8_dic(dic)?))
}
"ISO8859-1" | "ISO-8859-1" => Ok((decode_latin1(aff), decode_latin1(dic))),
"ISO8859-2" | "ISO-8859-2" => Ok((decode_latin2(aff), decode_latin2(dic))),
@@ -235,6 +245,225 @@ pub fn decode(aff: &[u8], dic: &[u8]) -> Result<(String, String), String> {
}
}
/// The first arguments of the `.aff`'s first `SET` and `FLAG` lines, in capitals, found in one pass
/// over the whole file: a `FLAG UTF-8` after the rules still decides how the `.dic` flags are read,
/// as in Hunspell. Only the first argument counts: `FLAG UTF-8 # Unicode flags` is UTF-8.
fn directives(aff: &[u8]) -> (Option<String>, Option<String>) {
let (mut set, mut flag) = (None, None);
for l in aff.split(|&b| b == b'\n') {
if set.is_some() && flag.is_some() {
break;
}
let l = l.strip_prefix("\u{feff}".as_bytes()).unwrap_or(l);
let slot = match l.first() {
Some(b'S') if set.is_none() => (&mut set, &b"SET"[..]),
Some(b'F') if flag.is_none() => (&mut flag, &b"FLAG"[..]),
_ => continue,
};
let arg = l
.strip_prefix(slot.1)
.filter(|rest| rest.first().is_some_and(|b| matches!(b, b' ' | b'\t')))
.and_then(|rest| rest.split(|b| matches!(b, b' ' | b'\t' | b'\r')).find(|f| !f.is_empty()));
if let Some(a) = arg {
*slot.0 = Some(String::from_utf8_lossy(a).to_ascii_uppercase());
}
}
(set, flag)
}
/// Directives whose one argument is a flag.
const FLAG_DIRECTIVES: &[&str] = &[
"CIRCUMFIX",
"COMPOUNDBEGIN",
"COMPOUNDEND",
"COMPOUNDFLAG",
"COMPOUNDFORBIDFLAG",
"COMPOUNDLAST",
"COMPOUNDFIRST",
"COMPOUNDMIDDLE",
"COMPOUNDPERMITFLAG",
"COMPOUNDROOT",
"FORBIDDENWORD",
"FORCEUCASE",
"GENERATE",
"HU_KOTOHANGZO",
"KEEPCASE",
"LEMMA_PRESENT",
"NEEDAFFIX",
"NOSUGGEST",
"ONLYINCOMPOUND",
"ONLYROOT",
"PSEUDOROOT",
"SUBSTANDARD",
"WARN",
];
/// Directives that only describe the dictionary; a non-UTF-8 byte in them (the Hungarian
/// dictionary's `NAME` is ISO 8859-2) is replaced instead of failing the load.
const METADATA_DIRECTIVES: &[&str] = &["NAME", "HOME", "VERSION", "LANG", "AM"];
/// `text/flags`: the text as UTF-8 and the flags byte by byte (`\/` is a literal slash). `what`
/// names the line in an error.
fn text_and_flags(field: &[u8], what: impl Fn() -> String) -> Result<String, String> {
let mut slash = None;
let mut i = 0;
while let Some(&b) = field.get(i) {
if b == b'\\' && field.get(i + 1) == Some(&b'/') {
i += 2;
continue;
}
if b == b'/' {
slash = Some(i);
break;
}
i += 1;
}
let (text, flags) = match slash {
Some(k) => (field.get(..k).unwrap_or_default(), field.get(k..).unwrap_or_default()),
None => (field, &[][..]),
};
let mut out = std::str::from_utf8(text).map_err(|e| format!("{}: not UTF-8: {e}", what()))?.to_string();
out.push_str(&decode_latin1(flags));
Ok(out)
}
/// Split a line into fields on spaces and tabs, keeping the separators (so a line rebuilt from
/// its parts is the original).
fn fields(line: &[u8]) -> Vec<&[u8]> {
let mut out = Vec::new();
let mut start = 0;
let mut in_sep = line.first().is_some_and(|b| matches!(b, b' ' | b'\t'));
for (i, b) in line.iter().enumerate() {
let sep = matches!(b, b' ' | b'\t');
if sep != in_sep {
out.push(line.get(start..i).unwrap_or_default());
start = i;
in_sep = sep;
}
}
out.push(line.get(start..).unwrap_or_default());
out
}
/// A UTF-8 `.aff` with single-byte flags (see [`decode`]).
fn decode_utf8_aff(aff: &[u8]) -> Result<String, String> {
let mut out = String::with_capacity(aff.len());
for (n, line) in aff.split(|&b| b == b'\n').enumerate() {
if n > 0 {
out.push('\n');
}
if let Ok(s) = std::str::from_utf8(line)
&& s.is_ascii()
{
out.push_str(s);
continue;
}
let parts = fields(line);
// Field index among the non-separator parts.
let words: Vec<usize> =
parts.iter().enumerate().filter(|(_, p)| !p.first().is_some_and(|b| matches!(b, b' ' | b'\t'))).map(|(i, _)| i).collect();
let dir = words
.first()
.and_then(|&i| parts.get(i))
.map(|p| String::from_utf8_lossy(p).trim_start_matches('\u{feff}').to_string())
.unwrap_or_default();
let what = || format!(".aff line {}", n + 1);
if dir.starts_with('#') || METADATA_DIRECTIVES.contains(&dir.as_str()) {
out.push_str(&String::from_utf8_lossy(line));
continue;
}
// A trailing `# comment` (the Hungarian `AF` lines number themselves) is never a word or
// a flag; from there on the line is kept as text. It can only start after the directive's
// own fields, since a field such as an affix condition may itself begin with `#`.
let field_text = |k: usize| words.get(k).and_then(|&w| parts.get(w)).map(|p| String::from_utf8_lossy(p).into_owned());
let first_free = comment_field(&dir, field_text);
let comment_at = words.iter().skip(first_free).position(|&w| parts.get(w).is_some_and(|f| f.first() == Some(&b'#'))).map(|k| k + first_free);
// The field index of the next non-separator part.
let mut next_field = 0;
for (i, p) in parts.iter().enumerate() {
let field = (!p.first().is_some_and(|b| matches!(b, b' ' | b'\t'))).then(|| {
next_field += 1;
next_field - 1
});
let in_comment = comment_at.is_some_and(|c| words.get(c).is_some_and(|&w| i >= w));
let decoded = match (dir.as_str(), field) {
_ if in_comment => String::from_utf8_lossy(p).into_owned(),
("AF" | "COMPOUNDRULE" | "SYLLABLENUM", Some(1)) => decode_latin1(p),
("PFX" | "SFX", Some(3)) | ("CHECKCOMPOUNDPATTERN", Some(1 | 2)) => text_and_flags(p, what)?,
// A field that names one flag keeps its first byte, as Hunspell's `decode_flag`
// does: `SFX é` is the flag 0xC3, the first of the two flags `kert/é` carries.
("PFX" | "SFX", Some(1)) => decode_latin1(p.get(..1).unwrap_or_default()),
(d, Some(1)) if FLAG_DIRECTIVES.contains(&d) => decode_latin1(p.get(..1).unwrap_or_default()),
_ => std::str::from_utf8(p).map_err(|e| format!("{}: not UTF-8: {e}", what()))?.to_string(),
};
out.push_str(&decoded);
}
}
Ok(out)
}
/// The first field of an `.aff` line where a trailing `# comment` may start: after the
/// directive's own fields. A table's count line (`REP 12`, `PFX A Y 3`) has fewer fields than its
/// rows (`REP a b`, `PFX A 0 x .`). `field(k)` is the line's `k`-th field.
fn comment_field(dir: &str, field: impl Fn(usize) -> Option<String>) -> usize {
let number = |k: usize| field(k).is_some_and(|f| f.parse::<usize>().is_ok());
match dir {
"PFX" | "SFX" if field(2).is_some_and(|f| f == "Y" || f == "N") && number(3) => 4,
"PFX" | "SFX" => 5,
"REP" | "ICONV" | "OCONV" | "PHONE" | "MAP" | "BREAK" | "AF" | "AM" | "COMPOUNDRULE" | "CHECKCOMPOUNDPATTERN"
if number(1) && field(2).is_none_or(|f| f.starts_with('#')) =>
{
2
}
"REP" | "ICONV" | "OCONV" | "PHONE" => 3,
"CHECKCOMPOUNDPATTERN" => 4,
_ => 2,
}
}
/// Where the `word/flags` entry of a `.dic` line ends, as Hunspell reads it: at a tab or at the
/// first morphological field (a space or tab, two characters and `:`, as in ` po:noun`), whichever
/// comes first, so an entry may hold spaces (`ad hoc`); with neither, at the end of the line
/// (`kot/B 12` has the flags `B 12`, as in Hunspell).
pub(crate) fn dic_entry_end(line: &[u8]) -> usize {
let blank = |b: &u8| matches!(b, b' ' | b'\t');
let morph = line.windows(4).position(|w| matches!(w, [a, b, c, b':'] if blank(a) && !blank(b) && !blank(c)));
let tab = line.iter().position(|&b| b == b'\t');
let end = match (morph, tab) {
(Some(m), Some(t)) => m.min(t),
(m, t) => match m.or(t) {
Some(e) => e,
None => line.len(),
},
};
// Blanks before the morphological field or the line end aren't part of the entry.
line.get(..end).map_or(end, |e| e.iter().rposition(|b| !blank(b)).map_or(0, |k| k + 1))
}
/// A UTF-8 `.dic` with single-byte flags: the word is UTF-8, the flags after `/` are bytes.
fn decode_utf8_dic(dic: &[u8]) -> Result<String, String> {
let mut out = String::with_capacity(dic.len());
for (n, line) in dic.split(|&b| b == b'\n').enumerate() {
if n > 0 {
out.push('\n');
}
if let Ok(s) = std::str::from_utf8(line)
&& s.split_once('/').is_none_or(|(_, flags)| flags.is_ascii())
{
out.push_str(s);
continue;
}
let what = || format!(".dic line {}", n + 1);
// The first field is `word/flags` (it may hold spaces, see [`dic_entry_end`]);
// morphological fields after it are UTF-8 text.
let end = dic_entry_end(line);
out.push_str(&text_and_flags(line.get(..end).unwrap_or_default(), what)?);
let rest = line.get(end..).unwrap_or_default();
out.push_str(std::str::from_utf8(rest).map_err(|e| format!("{}: not UTF-8: {e}", what()))?);
}
Ok(out)
}
#[cfg(test)]
mod tests {
use super::*;
@@ -280,6 +509,78 @@ mod tests {
assert!(decode(b"SET ISO-8859-1\n", b"\xe7").is_ok());
}
#[test]
fn utf8_dictionaries_keep_single_byte_flags() {
// The Hungarian dictionary is `SET UTF-8` but writes flags as single bytes above 0x7F,
// which aren't UTF-8; each byte is one flag. Its `NAME` line is ISO 8859-2.
let aff = b"SET UTF-8\nNAME Magyar helyes\xedr\xe1si\nAF 2\nAF \xd5a # 1\nAF \xe9 # 2\nSFX \xd5 Y 1\nSFX \xd5 0 ok/2 . 7\nSFX \xe9 Y 1\nSFX \xe9 0 ban .\nSFX a Y 1\nSFX a 0 at .\n";
let dic = "2\nház/1\t1\nkert/\u{e9}\n".as_bytes();
let (a, d) = decode(aff, dic).unwrap();
assert!(a.contains("AF \u{d5}a # 1") && a.contains("AF \u{e9} # 2") && a.contains("SFX \u{d5} 0 ok/2 . 7"));
assert!(d.contains("kert/\u{c3}\u{a9}"), "a UTF-8 é in the flags is two one-byte flags");
let dict = crate::affix::AffixDict::from_hunspell(&a, &d, &[]).unwrap();
assert!(dict.check("ház") && dict.check("házok") && dict.check("házokban") && dict.check("házat"));
assert!(!dict.check("házban"));
// `FLAG UTF-8` keeps the whole file UTF-8, and words must still be UTF-8.
let (_, d) = decode(b"SET UTF-8\nFLAG UTF-8\n", "kert/\u{e9}\n".as_bytes()).unwrap();
assert_eq!(d, "kert/\u{e9}\n");
assert!(decode(b"SET UTF-8\nSFX a Y 1\nSFX a 0 \xe9 .\n", b"").is_err(), "an affix must be UTF-8");
assert!(decode(b"SET UTF-8\n", b"1\nh\xe1z/a\n").is_err(), "a stem must be UTF-8");
}
#[test]
fn single_flag_fields_keep_their_first_byte() {
// Without `FLAG UTF-8`, a UTF-8 `é` (0xC3 0xA9) in a field that names one flag is the
// flag 0xC3; in a `.dic` flag list it is the two flags 0xC3 and 0xA9.
let aff = "SET UTF-8\nKEEPCASE é\nSFX é Y 1\nSFX é 0 ok .\nSYLLABLENUM é\n".as_bytes();
let (a, d) = decode(aff, "2\nkert/é\nNATO/é\n".as_bytes()).unwrap();
assert!(a.contains("KEEPCASE \u{c3}\n") && a.contains("SFX \u{c3} Y 1") && a.contains("SFX \u{c3} 0 ok ."), "{a}");
assert!(a.contains("SYLLABLENUM \u{c3}\u{a9}"), "SYLLABLENUM lists flags: {a}");
assert!(d.contains("kert/\u{c3}\u{a9}"));
let dict = crate::affix::AffixDict::from_hunspell(&a, &d, &[]).unwrap();
assert!(dict.check("kert") && dict.check("kertok"));
assert!(dict.check("NATO") && !dict.check("Nato"), "KEEPCASE é reaches NATO/é");
}
#[test]
fn numeric_and_long_flags_stay_text() {
// `FLAG num` (the Turkish dictionary) and `FLAG long`: no byte-wise flag reading, so a
// number with a non-ASCII rule or comment keeps every digit.
let aff = "SET UTF-8\nFLAG num\nFORBIDDENWORD 6000 # tiltott ő\nSFX 1234 Y 1\nSFX 1234 0 ları .\n";
let (a, d) = decode(aff.as_bytes(), "1\nkitap/1234\n".as_bytes()).unwrap();
assert_eq!(a, aff);
assert_eq!(d, "1\nkitap/1234\n");
let tr = crate::tr::dict::TrDict::from_hunspell(&a, &d).unwrap();
assert!(tr.contains("kitapları") && tr.contains("kitap"));
let long = "SET UTF-8\nFLAG long\nSFX Aa Y 1\nSFX Aa 0 ős .\n";
assert_eq!(decode(long.as_bytes(), "1\nkő/Aa\n".as_bytes()).unwrap().0, long);
assert!(decode(b"SET UTF-8\nFLAG num\n", b"1\nh\xe1z/1\n").is_err(), "the .dic must be UTF-8");
// `FLAG DEFAULT` is the single-byte default.
let (_, d) = decode(b"SET UTF-8\nFLAG default\n", b"1\nkert/\xe9\n").unwrap();
assert_eq!(d, "1\nkert/\u{e9}\n");
}
#[test]
fn flag_utf8_counts_anywhere_in_the_aff() {
// A `FLAG UTF-8` after a long rule table still makes the `.dic` flags Unicode.
let mut aff = b"SET UTF-8\nSFX a Y 400\n".to_vec();
for _ in 0..400 {
aff.extend_from_slice(b"SFX a 0 aaaaaaaaaaaaaaaaaaaaaaaa .\n");
}
aff.extend_from_slice(b"FLAG UTF-8\r\n");
assert!(aff.len() > 8192);
let (_, d) = decode(&aff, "kert/\u{e9}\n".as_bytes()).unwrap();
assert_eq!(d, "kert/\u{e9}\n");
}
#[test]
fn a_comment_after_flag_utf8_keeps_unicode_flags() {
let (_, d) = decode(b"SET UTF-8\nFLAG UTF-8 # Unicode flags\n", "kert/\u{e9}\n".as_bytes()).unwrap();
assert_eq!(d, "kert/\u{e9}\n");
let (_, d) = decode(b"SET\tUTF-8 # comment\nFLAG\t\tUTF-8\n", "kert/\u{e9}\n".as_bytes()).unwrap();
assert_eq!(d, "kert/\u{e9}\n", "repeated spaces and tabs separate the argument");
}
#[test]
fn decodes_russian_single_byte_encodings() {
// `ёлка` in KOI8-R (0xA3 0xCC 0xCB 0xC1) and in Windows-1251 (0xB8 0xEB 0xEA 0xE0).
@@ -291,6 +592,41 @@ mod tests {
assert_eq!(decode_high(&[0x98], CP1251_HIGH), "\u{fffd}");
}
#[test]
fn comments_start_after_the_directive_fields() {
let f = |line: &'static str| move |k: usize| line.split_whitespace().nth(k).map(str::to_string);
assert_eq!(comment_field("PFX", f("PFX A Y 3 # count")), 4);
assert_eq!(comment_field("SFX", f("SFX A 0 # .")), 5, "a rule's condition may be `#`");
assert_eq!(comment_field("AF", f("AF 1234 # aliases")), 2);
assert_eq!(comment_field("AF", f("AF Aa # 1")), 2);
assert_eq!(comment_field("REP", f("REP 12")), 2);
assert_eq!(comment_field("REP", f("REP # x")), 3);
assert_eq!(comment_field("CHECKCOMPOUNDPATTERN", f("CHECKCOMPOUNDPATTERN a b c")), 4);
assert_eq!(comment_field("KEEPCASE", f("KEEPCASE k")), 2);
// A byte flag after an append text that begins with `#` stays a flag, not a comment.
let (a, _) = decode(b"SET UTF-8\nSFX \xd5 Y 1 # header\nSFX \xd5 0 #/\xe9 . # rule\n", b"").unwrap();
assert!(a.contains("SFX \u{d5} 0 #/\u{e9} . # rule"), "{a}");
}
#[test]
fn dic_entries_end_where_hunspell_ends_them() {
let end = |l: &str| l.get(..dic_entry_end(l.as_bytes())).unwrap().to_string();
assert_eq!(end("word/A po:noun\tts:x"), "word/A", "the morphological field comes before the tab");
assert_eq!(end("ad hoc/B\tpo:adv"), "ad hoc/B", "a tab ends a phrase");
assert_eq!(end("ad hoc/B po:adv"), "ad hoc/B");
assert_eq!(end("word/A 123"), "word/A 123", "neither: the whole line, as in Hunspell");
assert_eq!(end("ad hoc/B "), "ad hoc/B");
assert_eq!(end("word"), "word");
assert_eq!(end("a:b/A"), "a:b/A", "a colon inside the word isn't a field");
// Single-byte flags of a phrase entry are read byte by byte, then the UTF-8 morphology.
let (_, d) = decode(b"SET UTF-8\n", b"1\nad hoc/\xe9\tpo:adv\n").unwrap();
assert_eq!(d, "1\nad hoc/\u{e9}\tpo:adv\n");
// Without a tab or a morphological field the whole line is the entry: its byte flags
// don't fail the load (the Hungarian dictionary has such lines: `tök jó/396`).
let (_, d) = decode(b"SET UTF-8\n", b"1\nad hoc/\xe9\n").unwrap();
assert_eq!(d, "1\nad hoc/\u{e9}\n");
}
#[test]
fn libreoffice_extension_dirs_follow_the_language() {
// LibreOffice's IDs are per dictionary: Polish is `dict-pl`, Brazilian Portuguese