From e4182149a5388d3993b92194dcf880837bcd1a1f Mon Sep 17 00:00:00 2001 From: Harrison Smith Date: Sat, 31 Oct 2026 20:10:25 -0600 Subject: [PATCH] proof: read FLAG num (and long) affix dictionaries (storytold/wordcraft#4967) The affix engine read one-character flags only, at most 128 of them, as bits of a u128. It now reads `FLAG num` (comma-separated numbers 1-65535, leading digits as in Hunspell) and `FLAG long` (two characters), with up to 65,536 distinct flags (storytold/wordcraft#4966): - One reader (`FlagMode`) parses and writes every flag field: .dic flags, PFX/SFX flags and continuation classes, FORBIDDENWORD and NOSUGGEST. - Each flag gets a dense u16 id; a stem's flags and a rule's continuation classes are sorted id sets interned in one flat table (set 0 is empty). - Hostile input: at most 1,024 flags per entry (Belarusian uses 264), a bad .aff flag is an error, a .dic entry whose flags don't read is skipped, and a FLAG line after flags were read is refused. - Lookups skip the continuation walk for groups and flags no continuation class can reach, so checks are as fast or faster with the Polish, Portuguese, Czech, Italian and Russian dictionaries (same answers). - The compact format writes a `FLAG num`/`FLAG long` line only for those modes, so one-character data is byte-for-byte unchanged and old data still loads. LibreOffice's Serbian (Cyrillic and Latin), Belarusian, Sinhala, Kannada and Nepali dictionaries now load; Danish, Galician, Icelandic and Korean still stop at COMPOUNDMIN, NEEDAFFIX, FULLSTRIP and ICONV. Ported from storytold/wordcraft#5142 (f5778bb76) onto craft-proof. craft-proof already had storytold/wordcraft#735 (AF aliases, SUBSTANDARD, KEEPCASE, ICONV, IGNORE, 256 one-character flags as a bitmask), which this commit predates; conflicts resolved by carrying those through the flag ids: AF aliases resolve to flag text read in the FLAG mode, SUBSTANDARD and KEEPCASE are flag ids like FORBIDDENWORD and NOSUGGEST, and lookups exclude up to four flags (Excluded). The FLAG long test lines in the language modules and slot.rs now use an unknown mode. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FypgCX8LsYYE8fsj4byBm2 --- crates/proof/src/affix.rs | 1029 +++++++++++++++++++++++++----------- crates/proof/src/cs/mod.rs | 2 +- crates/proof/src/es/mod.rs | 2 +- crates/proof/src/id/mod.rs | 2 +- crates/proof/src/it/mod.rs | 2 +- crates/proof/src/pl/mod.rs | 2 +- crates/proof/src/pt/mod.rs | 2 +- crates/proof/src/slot.rs | 2 +- 8 files changed, 737 insertions(+), 306 deletions(-) diff --git a/crates/proof/src/affix.rs b/crates/proof/src/affix.rs index 064339638..4b8894413 100644 --- a/crates/proof/src/affix.rs +++ b/crates/proof/src/affix.rs @@ -11,18 +11,24 @@ //! also be stored in WordCraft's compact format ([`AffixDict::to_bytes`]): the rules as text //! lines and the stems front-coded, raw-deflated together (the Polish dictionary is under 1 MB). //! -//! Supported: one-character flags (at most 256 distinct, any Unicode character — `FLAG UTF-8`), -//! flag aliases (`AF`: a number standing for a set of flags), `PFX`/`SFX` with strip, append and +//! Supported: one-character flags (any Unicode character — `FLAG UTF-8`), two-character flags +//! (`FLAG long`) and numeric flags (`FLAG num`: comma-separated numbers from 1 to 65,535), up to +//! 65,536 distinct flags in all ([`FlagMode`]), flag aliases (`AF`: a number standing for a set of flags), `PFX`/`SFX` with strip, append and //! condition (`.`, letters, `[abc]`, `[^abc]`), cross products, continuation classes on suffixes //! (at most two suffixes in a row, as in Hunspell), `FORBIDDENWORD`, `NOSUGGEST`, `KEEPCASE` (the //! stem is a word only in its own letter case), input conversion (`ICONV`: ligatures such as `fi` //! read as `fi`) and ignored characters (`IGNORE`), and the suggestion tables `TRY`, `KEY`, `MAP` -//! and `REP`. Compounds by `COMPOUNDRULE` (patterns of one-character flags with `*` and `?`, as -//! SCOWL's English dictionaries use for ordinals: `21st`, `112th`), with `ONLYINCOMPOUND` and -//! `COMPOUNDMIN`; the parts of such a compound are stems, without affixes, as in Hunspell. +//! and `REP`. Compounds by `COMPOUNDRULE` (patterns of flags with `*` and `?`, as SCOWL's English +//! dictionaries use for ordinals: `21st`, `112th`; with long or numeric flags each flag is written +//! in parentheses, `(aa)*(bb)`), with `ONLYINCOMPOUND` and `COMPOUNDMIN`; the parts of such a +//! compound are stems, without affixes, as in Hunspell. //! Morphological data (`AM` aliases, the fields after a stem) is skipped. Rejected: flag-based -//! compounding (`COMPOUNDFLAG`, `COMPOUNDBEGIN`…), long or numeric flags, `NEEDAFFIX` and the -//! other directives that change which words are correct. +//! compounding (`COMPOUNDFLAG`, `COMPOUNDBEGIN`…), `NEEDAFFIX` and the other directives that +//! change which words are correct. +//! +//! Each distinct flag gets a dense `u16` id; a stem's flags and a rule's continuation classes are +//! a sorted set of ids, interned once in a flat table, so membership is a binary search in a few +//! ids and a dictionary with thousands of flags (Serbian: 1,903) costs no more per lookup. //! //! Lookups never panic: stems sit in one string, found through an open-addressing hash table, and //! a probe allocates only small scratch strings for the candidate roots it tries. @@ -37,6 +43,12 @@ const MAGIC: &str = "WCAFF1"; pub const MAX_WORD_BYTES: usize = 120; /// Empty slot in the stem hash table. const EMPTY: u32 = u32::MAX; +/// Most flags one `.dic` entry or one continuation class field may carry. Real dictionaries use a +/// few hundred at most (Belarusian: 264); a longer field is hostile and is refused (`.aff`) or +/// skipped (`.dic`). +pub const MAX_FLAGS_PER_ENTRY: usize = 1024; +/// Most distinct flags a dictionary may use: each gets a `u16` id. +const MAX_DISTINCT_FLAGS: usize = 1 << 16; /// Directives that change which words are correct in ways this engine doesn't implement. const UNSUPPORTED: &[&str] = &[ @@ -73,10 +85,12 @@ enum Repeat { Any, } -/// A `COMPOUNDRULE` pattern as written (`n*1t`): its tokens, each a flag and how often it repeats. -fn parse_compound_rule(pattern: &str) -> Result, String> { - let mut out: Vec<(char, Repeat)> = Vec::new(); - for c in pattern.chars().filter(|&c| c != EMOJI_SELECTOR) { +/// A `COMPOUNDRULE` pattern as written (`n*1t`, or `(aa)*(bb)` with long or numeric flags): its +/// tokens, each a flag and how often it repeats. +fn parse_compound_rule(pattern: &str, mode: FlagMode) -> Result, String> { + let mut out: Vec<(FlagKey, Repeat)> = Vec::new(); + let mut chars = pattern.chars().filter(|&c| c != EMOJI_SELECTOR); + while let Some(c) = chars.next() { match c { '*' | '?' => { let Some(last) = out.last_mut().filter(|t| t.1 == Repeat::Once) else { @@ -84,8 +98,21 @@ fn parse_compound_rule(pattern: &str) -> Result, String> { }; last.1 = if c == '*' { Repeat::Any } else { Repeat::Optional }; } - '(' | ')' => return Err(format!("COMPOUNDRULE `{pattern}`: ( ) groups (long flags) are not supported")), - c => out.push((c, Repeat::Once)), + '(' => { + let mut inner = String::new(); + loop { + match chars.next() { + Some(')') => break, + Some(c) if inner.len() < MAX_COMPOUND_BYTES => inner.push(c), + _ => return Err(format!("COMPOUNDRULE `{pattern}`: `(` is not closed")), + } + } + let key = mode.parse_one(&inner).map_err(|e| format!("COMPOUNDRULE `{pattern}`: {e}"))?; + out.push((key, Repeat::Once)); + } + ')' => return Err(format!("COMPOUNDRULE `{pattern}`: `)` closes no group")), + c if mode == FlagMode::Char => out.push((FlagKey::from(c), Repeat::Once)), + _ => return Err(format!("COMPOUNDRULE `{pattern}`: long and numeric flags are written in ( )")), } if out.len() > MAX_COMPOUND_TOKENS { return Err(format!("COMPOUNDRULE `{pattern}`: more than {MAX_COMPOUND_TOKENS} flags")); @@ -97,72 +124,6 @@ fn parse_compound_rule(pattern: &str) -> Result, String> { Ok(out) } -/// Most distinct flags a dictionary may use: every single-byte flag (the Hungarian dictionary -/// uses 158). -const MAX_FLAGS: usize = 256; - -/// A set of flags, one bit per distinct flag character of the dictionary. -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] -struct Mask([u64; MAX_FLAGS / 64]); - -impl Mask { - const EMPTY: Mask = Mask([0; MAX_FLAGS / 64]); - - /// The set holding flag number `i` alone (empty when `i` is out of range). - fn bit(i: usize) -> Mask { - let mut m = Mask::EMPTY; - if let Some(w) = m.0.get_mut(i / 64) { - *w = 1 << (i % 64); - } - m - } - - /// The number of the lowest flag in the set, if any. - fn first(self) -> Option { - self.0.iter().enumerate().find(|(_, w)| **w != 0).map(|(i, w)| i * 64 + w.trailing_zeros() as usize) - } - - fn is_empty(self) -> bool { - self.0.iter().all(|&w| w == 0) - } - - /// Do the two sets share a flag? - fn meets(self, other: Mask) -> bool { - !(self & other).is_empty() - } - - /// Does this set hold every flag of `other`? - fn covers(self, other: Mask) -> bool { - (self & other) == other - } -} - -impl std::ops::BitAnd for Mask { - type Output = Mask; - fn bitand(mut self, o: Mask) -> Mask { - for (a, b) in self.0.iter_mut().zip(o.0) { - *a &= b; - } - self - } -} - -impl std::ops::BitOr for Mask { - type Output = Mask; - fn bitor(mut self, o: Mask) -> Mask { - self |= o; - self - } -} - -impl std::ops::BitOrAssign for Mask { - fn bitor_assign(&mut self, o: Mask) { - for (a, b) in self.0.iter_mut().zip(o.0) { - *a |= b; - } - } -} - /// One character class of a rule's condition. #[derive(Clone, Debug, PartialEq, Eq)] enum Cond { @@ -235,15 +196,14 @@ const MAX_FORMS_WORK: usize = 1 << 18; /// One prefix or suffix rule. #[derive(Clone, Debug)] struct Rule { - /// Bit of the rule's flag in the stems' flag masks. - bit: Mask, - flag: char, + /// The rule's flag (a dense id, see [`AffixDict::flag_keys`]). + flag: u16, cross: bool, strip: Box, add: Box, /// Continuation classes (`SFX A 0 able/B`): flags the derived form carries, licensing - /// further affixes. Chained application is bounded by [`MAX_CHAIN`]. - cont: Mask, + /// further affixes, as a flag set id (0: none). Chained application is bounded by [`MAX_CHAIN`]. + cont: u32, cond: Box<[Cond]>, cond_text: Box, } @@ -269,6 +229,8 @@ impl Rule { struct SfxGroup { strip: Box, rules: Vec, + /// The rules of `rules` with continuation classes: the only ones a chain can go through. + cont_rules: Vec, } /// A stem list with affix rules (see the module docs). @@ -277,25 +239,34 @@ pub struct AffixDict { /// All stems, concatenated, sorted by bytes. words: String, offs: Vec, - /// Per stem: index into `sets`. - flags: Vec, - /// Distinct flag masks. - sets: Vec, - /// The `FORBIDDENWORD` flag's bit; stems carrying it are not words (0: none declared). - forbidden: Mask, + /// Per stem: its flag set id. + flags: Vec, + /// Distinct flag sets, each a sorted run of flag ids in `set_data`: set `i` is + /// `set_data[set_offs[i]..set_offs[i + 1]]`. Set 0 is the empty set. + set_offs: Vec, + set_data: Vec, + /// The `FORBIDDENWORD` flag; stems carrying it are not words. + forbidden: Option, /// Stems with NOSUGGEST are correct, but none of their forms are suggested. - no_suggest: Mask, + no_suggest: Option, /// Stems with SUBSTANDARD are correct too, and likewise never suggested. - substandard: Mask, + substandard: Option, /// Open-addressing hash table of stem indices (`EMPTY` = free). table: Vec, - /// Flag char of each bit. - flag_chars: Vec, + /// How the source wrote flags (kept for the compact format). + flag_mode: FlagMode, + /// Each flag id's flag as the source wrote it, sorted (the id is the index). + flag_keys: Vec, + /// Whether any rule has continuation classes (lookups skip the chain walk otherwise). + has_cont: bool, + /// Per flag id: whether some rule's continuation classes hold it, so that a form (not only + /// a stem) can carry it. + cont_flag: Vec, prefixes: Vec, suffixes: Vec, /// Appended string → groups of suffix rules by stripped string. sfx_index: FxMap, Vec>, - /// Per flag bit: the suffix rules of that flag, in rule order. + /// Per flag id: the suffix rules of that flag, in rule order. sfx_by_flag: Vec>, /// Longest appended suffix, in chars. max_sfx_chars: usize, @@ -303,20 +274,21 @@ pub struct AffixDict { key: String, map: Vec>, rep: Vec<(String, String)>, - /// The `KEEPCASE` flag's bit: such stems are words only in their own letter case (0: none). - keepcase: Mask, + /// The `KEEPCASE` flag: such stems are words only in their own letter case. + keepcase: Option, /// `ICONV`: input conversions applied before a lookup, `(from, to)`. iconv: Vec<(String, String)>, /// `IGNORE`: characters removed from input words (and from the stems and appended texts). ignore: Box<[char]>, /// Provenance and licence lines, kept through [`AffixDict::to_bytes`]. comments: Vec, - /// `COMPOUNDRULE` patterns: per token, the flag's bit and how often it may repeat. - compound_rules: Vec>, + /// `COMPOUNDRULE` patterns: per token, the flag's id (`None`: a flag no stem carries, which + /// matches no part) and how often it may repeat. + compound_rules: Vec, Repeat)>>, /// The same patterns as written, for [`AffixDict::to_bytes`]. compound_text: Vec, - /// The `ONLYINCOMPOUND` flag's bit: such stems are words only as parts of a compound. - only_in_compound: Mask, + /// The `ONLYINCOMPOUND` flag: such stems are words only as parts of a compound. + only_in_compound: Option, /// `COMPOUNDMIN`, in characters. compound_min: usize, /// The first characters of the stems a compound part can be (their flags are in a @@ -387,17 +359,18 @@ struct Rules { map: Vec>, rep: Vec<(String, String)>, /// The `FORBIDDENWORD` flag: stems carrying it are listed but never correct. - forbidden: Option, - no_suggest: Option, - substandard: Option, + forbidden: Option, + no_suggest: Option, + substandard: Option, /// The `KEEPCASE` flag. - keepcase: Option, + keepcase: Option, iconv: Vec<(String, String)>, ignore: Vec, - /// `COMPOUNDRULE` patterns, as written (`n*1t`). + /// `COMPOUNDRULE` patterns, as written (`n*1t`); read in the `FLAG` mode by + /// [`AffixDict::build`], since the `FLAG` line may come after them. compound_rules: Vec, /// The `ONLYINCOMPOUND` flag. - only_in_compound: Option, + only_in_compound: Option, /// `COMPOUNDMIN`: the shortest part of a compound, in characters (Hunspell's default is 3). compound_min: Option, /// `AF` flag aliases: alias `n` (1-based) stands for `aliases[n - 1]`. @@ -405,9 +378,29 @@ struct Rules { /// The `AF` table opened with its count line, so flag fields are alias numbers. A table /// without one is rejected, as in Hunspell: its lines are ignored. aliases_on: bool, + /// How flags are written, from the `FLAG` line. + mode: FlagMode, + /// Whether a flag has been read: a later `FLAG` line can't change how the earlier ones read. + flag_read: bool, } impl Rules { + /// A `FLAG` line. + fn set_mode(&mut self, name: &str) -> Result<(), String> { + let mode = FlagMode::from_name(name).ok_or_else(|| format!("FLAG {name} is not supported"))?; + if self.flag_read && mode != self.mode { + return Err(format!("FLAG {name} after flags were read")); + } + self.mode = mode; + Ok(()) + } + + /// The one flag of a rule header or a single-flag directive. + fn one_flag(&mut self, field: &str) -> Result { + self.flag_read = true; + self.mode.parse_one(field) + } + /// Flags as written in a `.dic` entry or a continuation class: the flag characters /// themselves, or with an `AF` table the alias its leading digits number, as Hunspell reads /// them (`2x` is alias 2). A field without digits or an undefined alias reads as no flags: @@ -437,19 +430,8 @@ fn split_fields(line: &str) -> impl Iterator { line.split([' ', '\t']).filter(|f| !f.is_empty()) } -/// The one flag character a directive (`FORBIDDENWORD w`) names. -fn one_flag(arg: &str, dir: &str, n: usize) -> Result { - let mut it = arg.chars().filter(|&c| c != EMOJI_SELECTOR); - match (it.next(), it.next()) { - // Only the field separators of `split_fields` can't be a flag: a flag byte read as - // U+0085 or U+00A0 is one. - (Some(c), None) if !matches!(c, ' ' | '\t') => Ok(c), - _ => Err(format!("line {}: {dir} needs one flag character", n + 1)), - } -} - -/// A rule as parsed from text, before flags become bits: `(flag, cross, strip, add, cont, cond)`. -type RawRule = (char, bool, String, String, String, String); +/// A rule as parsed from text, before flags get ids: `(flag, cross, strip, add, cont, cond)`. +type RawRule = (FlagKey, bool, String, String, String, String); /// Most distinct `TRY` characters kept; a hostile `.aff` could otherwise make every suggestion /// round cost time and memory proportional to the file size. @@ -481,6 +463,147 @@ fn zero_empty(s: &str) -> String { /// but it is not a flag of its own: it is dropped wherever flags are read. const EMOJI_SELECTOR: char = '\u{fe0f}'; +/// A flag as written in the source, before it gets a dense id: a character (one-character mode), +/// two characters (`FLAG long`, the first in the high 32 bits) or a number (`FLAG num`). +type FlagKey = u64; + +/// How flags are written: the `.aff`'s `FLAG` line, also kept in the compact format. Every flag +/// field (`.dic` entries, `PFX`/`SFX` flags and continuation classes, `FORBIDDENWORD`…) is read +/// and written here, and nowhere else. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum FlagMode { + /// One character per flag (no `FLAG` line, `FLAG UTF-8`, `FLAG DEFAULT`). + #[default] + Char, + /// Two characters per flag (`FLAG long`): `Aa`, `B1`. + Long, + /// Decimal numbers from 1 to 65,535 separated by commas (`FLAG num`): `12,104`. As in + /// Hunspell, each number is the item's leading digits: Nepali writes `0/17X` for flag 17. + Num, +} + +impl FlagMode { + /// The mode a `FLAG` line names, if this engine reads it. + fn from_name(name: &str) -> Option { + match name { + "UTF-8" | "DEFAULT" | "utf-8" | "default" | "UTF8" | "" => Some(FlagMode::Char), + "long" | "LONG" => Some(FlagMode::Long), + "num" | "NUM" => Some(FlagMode::Num), + _ => None, + } + } + + /// The `FLAG` value the compact format writes (one-character mode writes no line). + fn name(self) -> Option<&'static str> { + match self { + FlagMode::Char => None, + FlagMode::Long => Some("long"), + FlagMode::Num => Some("num"), + } + } + + /// The flags of a field (`.dic` flags, continuation classes), in source order, at most + /// [`MAX_FLAGS_PER_ENTRY`]. An empty field has none. + fn parse(self, field: &str) -> Result, String> { + let mut out = Vec::new(); + let mut push = |k: FlagKey| { + if out.len() >= MAX_FLAGS_PER_ENTRY { + return Err(format!("more than {MAX_FLAGS_PER_ENTRY} flags in `{}`", field.chars().take(40).collect::())); + } + out.push(k); + Ok(()) + }; + match self { + FlagMode::Char => { + for c in field.chars().filter(|&c| c != EMOJI_SELECTOR) { + push(FlagKey::from(c))?; + } + } + FlagMode::Long => { + let mut chars = field.chars(); + while let Some(a) = chars.next() { + let b = chars.next().ok_or_else(|| format!("long flags `{field}` have an odd number of characters"))?; + push((FlagKey::from(a) << 32) | FlagKey::from(b))?; + } + } + FlagMode::Num if field.is_empty() => {} + FlagMode::Num => { + for item in field.split(',') { + // The leading digits (no sign), 1 to 65,535: at most five digits. + let digits = item.bytes().take_while(u8::is_ascii_digit).count(); + let n = if (1..=5).contains(&digits) { item.get(..digits).and_then(|d| d.parse::().ok()) } else { None }; + match n { + Some(n @ 1..=0xffff) => push(FlagKey::from(n))?, + _ => return Err(format!("`{}` is not a flag number (1 to 65535)", item.chars().take(20).collect::())), + } + } + } + } + Ok(out) + } + + /// The one flag of a field (`PFX`/`SFX` flags, `FORBIDDENWORD`, `NOSUGGEST`…). + fn parse_one(self, field: &str) -> Result { + match self.parse(field)?.as_slice() { + [k] => Ok(*k), + _ => Err(format!("`{field}` is not one flag")), + } + } + + /// Append the flags `keys` to `out` as the source writes them. + fn write(self, keys: impl Iterator, out: &mut String) { + let ch = |k: FlagKey| u32::try_from(k).ok().and_then(char::from_u32); + for (i, k) in keys.enumerate() { + match self { + FlagMode::Char => out.extend(ch(k)), + FlagMode::Long => { + out.extend(ch(k >> 32)); + out.extend(ch(k & 0xffff_ffff)); + } + FlagMode::Num => { + if i > 0 { + out.push(','); + } + out.push_str(&k.to_string()); + } + } + } + } +} + +/// Stem flags that rule a lookup out: `FORBIDDENWORD` always, and per lookup `NOSUGGEST` and +/// `SUBSTANDARD` (suggestions) or `KEEPCASE` (case variants of a word). +#[derive(Clone, Copy, Debug, Default)] +struct Excluded([Option; 4]); + +impl Excluded { + /// Does the flag set `set` carry an excluded flag? + fn hits(self, set: &[u16]) -> bool { + self.0.iter().flatten().any(|f| has(set, *f)) + } + + /// These exclusions and `flag` too. + fn and(mut self, flag: Option) -> Excluded { + if flag.is_some() + && let Some(slot) = self.0.iter_mut().find(|s| s.is_none()) + { + *slot = flag; + } + self + } +} + +/// Does the sorted flag set `set` hold `flag`? +fn has(set: &[u16], flag: u16) -> bool { + set.binary_search(&flag).is_ok() +} + +/// Do two sorted flag sets share a flag? (A binary search per flag of the smaller one.) +fn intersects(a: &[u16], b: &[u16]) -> bool { + let (small, large) = if a.len() <= b.len() { (a, b) } else { (b, a) }; + small.iter().any(|f| has(large, *f)) +} + /// Directives an English dictionary may carry that this engine doesn't implement but that don't /// change which English words are right, ignored by [`AffixDict::from_hunspell_english`]: /// flag-based compounding (English dictionaries compound only ordinals, by `COMPOUNDRULE`, which @@ -492,7 +615,7 @@ const IGNORED_IN_ENGLISH: &[&str] = /// [`IGNORED_IN_ENGLISH`] instead of refusing them. fn parse_aff(aff: &str, english: bool) -> Result { let mut r = Rules::default(); - let mut cross: HashMap<(bool, char), bool> = HashMap::new(); + let mut cross: HashMap<(bool, FlagKey), bool> = HashMap::new(); let mut map_header = false; let mut rep_header = false; let mut af_header = false; @@ -511,7 +634,7 @@ fn parse_aff(aff: &str, english: bool) -> Result { // Morphological aliases, and a flag that only matters for morphological generation: // the words it marks are correct. "AM" | "LEMMA_PRESENT" => {} - "SUBSTANDARD" => r.substandard = Some(one_flag(at(1), dir, n)?), + "SUBSTANDARD" => r.substandard = Some(r.one_flag(at(1)).map_err(|e| format!("line {}: {dir} needs one flag: {e}", n + 1))?), // The first `AF` or `ICONV` line is the table's count, as in Hunspell, even with a // trailing comment (`AF 1559 # aliases`). "AF" => { @@ -531,32 +654,28 @@ fn parse_aff(aff: &str, english: bool) -> Result { } } "IGNORE" => r.ignore = at(1).chars().collect(), - "KEEPCASE" => r.keepcase = Some(one_flag(at(1), dir, n)?), + "KEEPCASE" => r.keepcase = Some(r.one_flag(at(1)).map_err(|e| format!("line {}: {dir} needs one flag: {e}", n + 1))?), // The first `COMPOUNDRULE` line is the table's count, as in Hunspell. "COMPOUNDRULE" => { if !compound_header && at(1).parse::().is_ok() { compound_header = true; } else { - parse_compound_rule(at(1)).map_err(|e| format!("line {}: {e}", n + 1))?; if r.compound_rules.len() >= MAX_COMPOUND_RULES { return Err(format!("line {}: more than {MAX_COMPOUND_RULES} COMPOUNDRULE patterns", n + 1)); } r.compound_rules.push(at(1).to_string()); } } - "ONLYINCOMPOUND" => r.only_in_compound = Some(one_flag(at(1), dir, n)?), + "ONLYINCOMPOUND" => { + r.only_in_compound = Some(r.one_flag(at(1)).map_err(|e| format!("line {}: {dir} needs one flag: {e}", n + 1))?); + } "COMPOUNDMIN" => { let min = at(1).parse::().map_err(|_| format!("line {}: COMPOUNDMIN needs a number", n + 1))?; r.compound_min = Some(min.clamp(1, MAX_COMPOUND_BYTES)); } - // Per-character flag modes (`FLAG UTF-8`, as VERO's pt_BR dictionary uses) work - // with the one-character parsing here; only the multi-character encodings differ. - "FLAG" if !matches!(at(1), "UTF-8" | "DEFAULT" | "utf-8" | "default" | "UTF8" | "") => { - return Err(format!("line {}: FLAG {} is not supported (one-character flags only)", n + 1, at(1))); - } - "FLAG" => {} - "FORBIDDENWORD" => r.forbidden = Some(one_flag(at(1), dir, n)?), - "NOSUGGEST" => r.no_suggest = Some(one_flag(at(1), dir, n)?), + "FLAG" => r.set_mode(at(1)).map_err(|e| format!("line {}: {e}", n + 1))?, + "FORBIDDENWORD" => r.forbidden = Some(r.one_flag(at(1)).map_err(|e| format!("line {}: {dir} needs one flag: {e}", n + 1))?), + "NOSUGGEST" => r.no_suggest = Some(r.one_flag(at(1)).map_err(|e| format!("line {}: {dir} needs one flag: {e}", n + 1))?), "TRY" => r.try_chars = cap_try(at(1)), "KEY" => r.key = at(1).to_string(), "MAP" => { @@ -575,10 +694,7 @@ fn parse_aff(aff: &str, english: bool) -> Result { } "PFX" | "SFX" => { let suffix = dir == "SFX"; - let mut flag_chars = at(1).chars().filter(|&c| c != EMOJI_SELECTOR); - let (Some(flag), None) = (flag_chars.next(), flag_chars.next()) else { - return Err(format!("line {}: flag `{}` is not one character", n + 1, at(1))); - }; + let flag = r.one_flag(at(1)).map_err(|e| format!("line {}: {e}", n + 1))?; // Only a header ignores trailing fields; a rule condition beginning with # is literal. let is_header = f.len() >= 4 && matches!(at(2), "Y" | "N") && at(3).parse::().is_ok() && !cross.contains_key(&(suffix, flag)); if is_header { @@ -685,29 +801,62 @@ impl AffixDict { AffixDict::build(rules, stems, comments.iter().map(ToString::to_string).collect()) } - fn build(rules: Rules, mut stems: Vec<(String, String)>, comments: Vec) -> Result { - // Flags: every one a rule or a stem uses gets a bit. - let mut flag_chars: Vec = rules.prefixes.iter().chain(&rules.suffixes).map(|r| r.0).collect(); - flag_chars.extend(stems.iter().flat_map(|s| s.1.chars())); - // A compound rule's flag no stem carries gets no bit: its token matches no part. - flag_chars.sort_unstable(); - flag_chars.dedup(); - if flag_chars.len() > MAX_FLAGS { - return Err(format!("{} distinct flags; at most {MAX_FLAGS} are supported", flag_chars.len())); + fn build(rules: Rules, stems: Vec<(String, String)>, comments: Vec) -> Result { + let mode = rules.mode; + // A `.dic` entry whose flags don't read (a bad number, too many flags) is skipped, as a + // word Hunspell would read with the wrong flags: better unknown than wrongly accepted. + let mut skipped = 0usize; + // Each stem's flags are a range of `stem_keys`: one buffer, not one allocation per stem. + let mut stem_keys: Vec = Vec::new(); + let mut stems: Vec<(String, usize, usize)> = stems + .into_iter() + .filter_map(|(w, f)| match mode.parse(&f) { + Ok(keys) => { + let start = stem_keys.len(); + stem_keys.extend(keys); + Some((w, start, stem_keys.len())) + } + Err(_) => { + skipped += 1; + None + } + }) + .collect(); + if skipped > 0 { + log::debug!("affix dictionary: {skipped} entries with unreadable flags skipped"); } - let bit_of = |c: char| flag_chars.iter().position(|&x| x == c).map_or(Mask::EMPTY, Mask::bit); - let forbidden = rules.forbidden.map_or(Mask::EMPTY, bit_of); - let keepcase = rules.keepcase.map_or(Mask::EMPTY, bit_of); + let conts = |rs: &[RawRule]| rs.iter().map(|r| mode.parse(&r.4)).collect::, _>>(); + let (pfx_cont, sfx_cont) = (conts(&rules.prefixes)?, conts(&rules.suffixes)?); + // Flags: every one a rule, a stem or a directive uses gets an id, in key order. + let mut distinct: FxMap = FxMap::default(); + let all = rules.prefixes.iter().chain(&rules.suffixes).map(|r| r.0); + let all = all.chain(pfx_cont.iter().chain(&sfx_cont).flatten().copied()).chain(stem_keys.iter().copied()); + for k in all.chain(rules.forbidden).chain(rules.no_suggest).chain(rules.substandard).chain(rules.keepcase).chain(rules.only_in_compound) { + distinct.entry(k).or_insert(0); + if distinct.len() > MAX_DISTINCT_FLAGS { + return Err(format!("more than {MAX_DISTINCT_FLAGS} distinct flags")); + } + } + let mut flag_keys: Vec = distinct.keys().copied().collect(); + flag_keys.sort_unstable(); + // At most 2^16 keys, so every index fits a u16. + for (i, k) in flag_keys.iter().enumerate() { + if let (Some(id), Ok(i)) = (distinct.get_mut(k), u16::try_from(i)) { + *id = i; + } + } + let id_of = |k: FlagKey| distinct.get(&k).copied(); // Stems sorted by bytes; homonyms (the same word twice) are merged. stems.sort_by(|a, b| a.0.as_bytes().cmp(b.0.as_bytes())); let mut d = AffixDict { words: String::with_capacity(stems.iter().map(|s| s.0.len()).sum()), offs: Vec::with_capacity(stems.len() + 1), - flag_chars: flag_chars.clone(), - forbidden, - no_suggest: rules.no_suggest.map_or(Mask::EMPTY, bit_of), - substandard: rules.substandard.map_or(Mask::EMPTY, bit_of), - keepcase, + set_offs: vec![0, 0], + forbidden: rules.forbidden.and_then(id_of), + no_suggest: rules.no_suggest.and_then(id_of), + substandard: rules.substandard.and_then(id_of), + keepcase: rules.keepcase.and_then(id_of), + flag_mode: mode, iconv: rules.iconv, ignore: rules.ignore.into_boxed_slice(), try_chars: rules.try_chars, @@ -715,36 +864,35 @@ impl AffixDict { map: rules.map, rep: rules.rep, comments, + // A compound rule's flag no stem carries gets no id: its token matches no part. compound_rules: rules .compound_rules .iter() - .map(|p| parse_compound_rule(p).map(|t| t.into_iter().map(|(c, rep)| (bit_of(c), rep)).collect())) + .map(|p| parse_compound_rule(p, mode).map(|t| t.into_iter().map(|(k, rep)| (id_of(k), rep)).collect())) .collect::>()?, compound_text: rules.compound_rules, - only_in_compound: rules.only_in_compound.map_or(Mask::EMPTY, bit_of), + only_in_compound: rules.only_in_compound.and_then(id_of), compound_min: rules.compound_min.unwrap_or(3), ..AffixDict::default() }; d.offs.push(0); - let compound_flags = d.compound_rules.iter().flatten().fold(Mask::EMPTY, |m, (bit, _)| m | *bit); - let mut set_ids: HashMap = HashMap::new(); + let compound_flags: Vec = d.compound_rules.iter().flatten().filter_map(|(f, _)| *f).collect(); + let mut set_ids: FxMap, u32> = FxMap::default(); + set_ids.insert(Vec::new(), 0); let mut i = 0; + let mut merged: Vec = Vec::new(); while i < stems.len() { - let Some((word, _)) = stems.get(i) else { break }; - let mut mask = Mask::EMPTY; + let Some((word, ..)) = stems.get(i) else { break }; + merged.clear(); let mut j = i; - while let Some((w, f)) = stems.get(j) + while let Some((w, a, b)) = stems.get(j) && w == word { - mask |= f.chars().map(bit_of).fold(Mask::EMPTY, |a, b| a | b); + merged.extend(stem_keys.get(*a..*b).unwrap_or_default().iter().filter_map(|&k| id_of(k))); j += 1; } - let next_id = u16::try_from(d.sets.len()).map_err(|_| "too many distinct flag sets".to_string())?; - let id = *set_ids.entry(mask).or_insert(next_id); - if usize::from(id) == d.sets.len() { - d.sets.push(mask); - } - if mask.meets(compound_flags) + let id = d.intern(&mut set_ids, &mut merged)?; + if merged.iter().any(|f| compound_flags.contains(f)) && let Some(c) = word.chars().next() && !d.compound_firsts.contains(&c) { @@ -755,38 +903,85 @@ impl AffixDict { d.flags.push(id); i = j; } - let mk = |(flag, cross, strip, add, cont, cond): RawRule| -> Result { - let cont = cont.chars().fold(Mask::EMPTY, |m, c| m | bit_of(c)); - Ok(Rule { bit: bit_of(flag), flag, cross, strip: strip.into(), add: add.into(), cont, cond: parse_cond(&cond)?, cond_text: cond.into() }) + let mut build_rules = |raw: Vec, conts: Vec>, d: &mut AffixDict| -> Result, String> { + raw.into_iter() + .zip(conts) + .map(|((flag, cross, strip, add, _, cond), cont)| { + let flag = id_of(flag).ok_or("affix flag without an id")?; + let cont = d.intern(&mut set_ids, &mut cont.iter().filter_map(|&k| id_of(k)).collect())?; + Ok(Rule { flag, cross, strip: strip.into(), add: add.into(), cont, cond: parse_cond(&cond)?, cond_text: cond.into() }) + }) + .collect() }; - d.prefixes = rules.prefixes.into_iter().map(mk).collect::>()?; - d.suffixes = rules.suffixes.into_iter().map(mk).collect::>()?; + d.prefixes = build_rules(rules.prefixes, pfx_cont, &mut d)?; + d.suffixes = build_rules(rules.suffixes, sfx_cont, &mut d)?; + d.flag_keys = flag_keys; d.index(); Ok(d) } + /// The id of the flag set `flags` (sorted and deduplicated here), adding it if it is new. + fn intern(&mut self, ids: &mut FxMap, u32>, flags: &mut Vec) -> Result { + flags.sort_unstable(); + flags.dedup(); + if let Some(&id) = ids.get(flags.as_slice()) { + return Ok(id); + } + let id = u32::try_from(self.set_offs.len().saturating_sub(1)).map_err(|_| "too many distinct flag sets".to_string())?; + self.set_data.extend_from_slice(flags); + self.set_offs.push(u32::try_from(self.set_data.len()).map_err(|_| "flag sets too large".to_string())?); + ids.insert(flags.clone(), id); + Ok(id) + } + + /// The flag set `id`, sorted (empty when unknown). + fn set(&self, id: u32) -> &[u16] { + let i = id as usize; + match (self.set_offs.get(i), self.set_offs.get(i + 1)) { + (Some(&a), Some(&b)) => self.set_data.get(a as usize..b as usize).unwrap_or_default(), + _ => &[], + } + } + /// Build the suffix index and the stem hash table. fn index(&mut self) { let mut idx: FxMap, Vec> = FxMap::default(); // Each (appended, stripped) pair's group, found by hash rather than by scanning the // appended string's groups: a dictionary with many strips per suffix stays linear. let mut group_of: FxMap<(&str, &str), usize> = FxMap::default(); - let mut by_flag: Vec> = vec![Vec::new(); MAX_FLAGS]; + let mut by_flag: Vec> = vec![Vec::new(); self.flag_keys.len()]; for (i, r) in self.suffixes.iter().enumerate() { let groups = idx.entry(r.add.clone()).or_default(); let Ok(i) = u32::try_from(i) else { break }; - match group_of.get(&(&*r.add, &*r.strip)).and_then(|&g| groups.get_mut(g)) { - Some(g) => g.rules.push(i), + let g = match group_of.get(&(&*r.add, &*r.strip)).copied() { + Some(g) => g, None => { group_of.insert((&r.add, &r.strip), groups.len()); - groups.push(SfxGroup { strip: r.strip.clone(), rules: vec![i] }); + groups.push(SfxGroup { strip: r.strip.clone(), ..SfxGroup::default() }); + groups.len() - 1 + } + }; + if let Some(g) = groups.get_mut(g) { + g.rules.push(i); + if r.cont != 0 { + g.cont_rules.push(i); } } - if let Some(list) = r.bit.first().and_then(|b| by_flag.get_mut(b)) { + if let Some(list) = by_flag.get_mut(usize::from(r.flag)) { list.push(i); } } self.sfx_by_flag = by_flag; + let mut cont_flag = vec![false; self.flag_keys.len()]; + for r in self.prefixes.iter().chain(&self.suffixes) { + for &f in self.set(r.cont) { + if let Some(c) = cont_flag.get_mut(usize::from(f)) { + *c = true; + } + } + } + self.has_cont = cont_flag.contains(&true); + self.cont_flag = cont_flag; self.max_sfx_chars = self.suffixes.iter().map(|r| r.add.chars().count()).max().unwrap_or(0); self.sfx_index = idx; let n = self.len(); @@ -826,19 +1021,9 @@ impl AffixDict { self.words.get(a..b) } - /// The flag characters of `mask`, in flag order (how the compact format stores flags). - fn flag_chars_for(&self, mask: Mask) -> String { - self.flag_chars.iter().enumerate().filter(|&(i, _)| mask.meets(Mask::bit(i))).map(|(_, c)| c).collect() - } - - /// The character of `mask` when it holds exactly one flag. - fn flag_char(&self, mask: Mask) -> Option { - let s = self.flag_chars_for(mask); - let mut it = s.chars(); - match (it.next(), it.next()) { - (Some(c), None) => Some(c), - _ => None, - } + /// Append the flags of the set `ids` (or one flag) to `out` as the source wrote them. + fn write_flags(&self, ids: &[u16], out: &mut String) { + self.flag_mode.write(ids.iter().filter_map(|&i| self.flag_keys.get(usize::from(i)).copied()), out); } /// The stems, in byte order. @@ -864,18 +1049,28 @@ impl AffixDict { (first..self.len()).map_while(move |i| self.word(i).filter(|w| w.starts_with(prefix.as_str()))) } - /// Flag mask of a stem, if `w` is one. - fn stem_flags(&self, w: &str) -> Option { - self.stem_flags_except(w, Mask::EMPTY) + /// What lookups exclude: forbidden words, and NOSUGGEST and SUBSTANDARD stems too for + /// suggestions. + fn excluded(&self, suggestible: bool) -> Excluded { + let e = Excluded::default().and(self.forbidden); + if suggestible { e.and(self.no_suggest).and(self.substandard) } else { e } } - /// Flag mask of a stem, if `w` is one and carries none of the flags in `deny`. - fn stem_flags_except(&self, w: &str, deny: Mask) -> Option { - self.stem_mask(w).filter(|m| !m.meets(self.forbidden | self.only_in_compound | deny)) + /// Flag set of a stem, if `w` is one. + fn stem_flags(&self, w: &str) -> Option<&[u16]> { + self.stem_flags_except(w, self.excluded(false)) } - /// Flag mask of the stem `w`, whatever flags it carries. - fn stem_mask(&self, w: &str) -> Option { + /// Flag set of a stem, if `w` is one and carries none of the `excluded` flags. A stem + /// carrying FORBIDDENWORD (listed, not a word) or ONLYINCOMPOUND (a word only inside a + /// compound, [`Self::compound`]) is no hit, and (this is the only stem lookup outside + /// compounds) no affixed form is built on it either. + fn stem_flags_except(&self, w: &str, excluded: Excluded) -> Option<&[u16]> { + self.stem_set(w).filter(|set| !excluded.hits(set) && !self.only_in_compound.is_some_and(|f| has(set, f))) + } + + /// Flag set of the stem `w`, whatever flags it carries. + fn stem_set(&self, w: &str) -> Option<&[u16]> { if self.table.is_empty() { return None; } @@ -887,46 +1082,53 @@ impl AffixDict { return None; } if self.word(t as usize) == Some(w) { - let id = *self.flags.get(t as usize)?; - let mask = self.sets.get(usize::from(id)).copied(); - // [`Self::stem_flags_except`] drops FORBIDDENWORD stems (listed, not words) and - // ONLYINCOMPOUND ones (words only inside a compound, [`Self::compound`]). - return mask; + return Some(self.set(*self.flags.get(t as usize)?)); } slot = (slot + 1) & mask; } None } + /// Can a derived form carry `flag` (does some rule's continuation classes hold it)? + fn is_cont_flag(&self, flag: u16) -> bool { + self.cont_flag.get(usize::from(flag)).copied().unwrap_or(false) + } + /// Does `w` carry any flag of `flags` — as a stem, or as the form of a suffix rule whose /// continuation classes include one (chained suffixation, depth-bounded)? /// - /// The rules of a group that could lead to `w` are merged into one mask of their flags and - /// followed once, not once per rule: the answer is the same, but thousands of rules of one + /// The rules of a group that could lead to `w` are merged into one set of their flags (sorted) + /// and followed once, not once per rule: the answer is the same, but thousands of rules of one /// flag cost one walk, not one each (#3613). - fn form_has_flag(&self, w: &str, flags: Mask, depth: u8, deny: Mask) -> bool { - if self.stem_flags_except(w, deny).is_some_and(|f| f.meets(flags)) { + fn form_has_flag(&self, w: &str, flags: &[u16], depth: u8, excluded: Excluded) -> bool { + if self.stem_flags_except(w, excluded).is_some_and(|f| intersects(f, flags)) { return true; } - if depth == 0 { + // Without continuation classes no rule leads from a form to another. + if depth == 0 || !self.has_cont { return false; } let mut root = String::with_capacity(w.len() + 8); + let mut inner: Vec = Vec::new(); let cuts = std::iter::once(w.len()).chain(w.char_indices().rev().map(|(i, _)| i).filter(|&i| i > 0)); for b in cuts.take(self.max_sfx_chars + 1) { let (Some(base), Some(add)) = (w.get(..b), w.get(b..)) else { continue }; let Some(groups) = self.sfx_index.get(add) else { continue }; - for g in groups { + for g in groups.iter().filter(|g| !g.cont_rules.is_empty()) { root.clear(); root.push_str(base); root.push_str(&g.strip); - let inner = g - .rules - .iter() - .filter_map(|&i| self.suffixes.get(i as usize)) - .filter(|r| r.cont.meets(flags) && r.cond_matches(&root, true)) - .fold(Mask::EMPTY, |m, r| m | r.bit); - if !inner.is_empty() && self.form_has_flag(&root, inner, depth - 1, deny) { + inner.clear(); + inner.extend( + g.cont_rules + .iter() + .filter_map(|&i| self.suffixes.get(i as usize)) + .filter(|r| intersects(self.set(r.cont), flags) && r.cond_matches(&root, true)) + .map(|r| r.flag), + ); + inner.sort_unstable(); + inner.dedup(); + if !inner.is_empty() && self.form_has_flag(&root, &inner, depth - 1, excluded) { return true; } } @@ -937,13 +1139,13 @@ impl AffixDict { /// Is `w` exactly (case and all) a stem or an affixed form of one? `ICONV` and `IGNORE` /// apply first. pub fn contains(&self, w: &str) -> bool { - self.contains_except(&self.normalize(w), Mask::EMPTY) + self.contains_except(&self.normalize(w), self.excluded(false)) } /// Exactly spelled dictionary words eligible for suggestions (NOSUGGEST and SUBSTANDARD /// stems excluded). pub fn contains_suggestible(&self, w: &str) -> bool { - self.contains_except(&self.normalize(w), self.no_suggest | self.substandard) + self.contains_except(&self.normalize(w), self.excluded(true)) } /// `w` after the `ICONV` conversions (at each position the longest match wins) and without @@ -976,13 +1178,13 @@ impl AffixDict { out.map_or(std::borrow::Cow::Borrowed(w), std::borrow::Cow::Owned) } - /// [`AffixDict::contains`] on normalized input, where stems carrying a flag in `deny` don't - /// count (case variants of `KEEPCASE` stems). - fn contains_except(&self, w: &str, deny: Mask) -> bool { + /// [`AffixDict::contains`] on normalized input, where stems carrying an `excluded` flag + /// don't count (forbidden words; case variants of `KEEPCASE` stems). + fn contains_except(&self, w: &str, excluded: Excluded) -> bool { if w.is_empty() || w.len() > MAX_WORD_BYTES { return false; } - if self.stem_flags_except(w, deny).is_some() || self.suffixed(w, Mask::EMPTY, deny) { + if self.stem_flags_except(w, excluded).is_some() || self.suffixed(w, None, excluded) { return true; } let mut root = String::with_capacity(w.len() + 8); @@ -997,17 +1199,17 @@ impl AffixDict { if !r.cond_matches(&root, false) { continue; } - if self.stem_flags_except(&root, deny).is_some_and(|f| f.meets(r.bit)) { + if self.stem_flags_except(&root, excluded).is_some_and(|f| has(f, r.flag)) { return true; } - if self.form_has_flag(&root, r.bit, INNER_CHAIN, deny) { + if self.is_cont_flag(r.flag) && self.form_has_flag(&root, &[r.flag], INNER_CHAIN, excluded) { return true; } - if r.cross && self.suffixed(&root, r.bit, deny) { + if r.cross && self.suffixed(&root, Some(r.flag), excluded) { return true; } } - self.compound(w, deny) + self.compound(w, excluded) } /// Is `w` a compound of two or more stems (no affixes) whose flags spell out one of the @@ -1015,39 +1217,41 @@ impl AffixDict { /// `1st/p` under `n*mp`. A dynamic programme over (byte offset, token, parts so far), so a /// hostile pattern costs at most a few thousand lookups. A forbidden word is never rebuilt /// from its parts. - fn compound(&self, w: &str, deny: Mask) -> bool { + fn compound(&self, w: &str, excluded: Excluded) -> bool { if self.compound_rules.is_empty() || w.len() > MAX_COMPOUND_BYTES || !w.chars().next().is_some_and(|c| self.compound_firsts.contains(&c)) - || self.stem_mask(w).is_some_and(|m| m.meets(self.forbidden)) + || self.forbidden.is_some_and(|f| self.stem_set(w).is_some_and(|set| has(set, f))) { return false; } + let excluded = excluded.and(self.forbidden); // Char boundaries of `w`, `0` and `w.len()` included. let cuts: Vec = w.char_indices().map(|(i, _)| i).chain(std::iter::once(w.len())).collect(); let n = cuts.len(); - // Flag mask of each part `w[cuts[a]..cuts[b]]`, by (a, b), looked up when a rule first + // Flag set of each part `w[cuts[a]..cuts[b]]`, by (a, b), looked up when a rule first // reaches cut `a` (`None`: not looked up yet; `Some(None)`: not a stem). Only the parts // after a cut a rule reaches are looked up, so a word with no first part costs `n`. - let mut parts: Vec>> = vec![None; n * n]; + let mut parts: Vec>> = vec![None; n * n]; let min = self.compound_min; - let mut part = |a: usize, b: usize| -> Option { + let mut part = |a: usize, b: usize| -> Option<&[u16]> { if b < a.saturating_add(min) { return None; } let slot = parts.get_mut(a * n + b)?; *slot.get_or_insert_with(|| { let (Some(&x), Some(&y)) = (cuts.get(a), cuts.get(b)) else { return None }; - w.get(x..y).and_then(|p| self.stem_mask(p)).filter(|m| !m.meets(self.forbidden | deny)) + w.get(x..y).and_then(|p| self.stem_set(p)).filter(|set| !excluded.hits(set)) }) }; self.compound_rules.iter().any(|rule| compound_matches(rule, &mut part, n)) } /// Is `w` a stem with one suffix? With `need` set (a prefix was taken off), the stem must - /// carry those flags too and the suffix must allow cross products. - fn suffixed(&self, w: &str, need: Mask, deny: Mask) -> bool { + /// carry that flag too and the suffix must allow cross products. + fn suffixed(&self, w: &str, need: Option, excluded: Excluded) -> bool { let mut root = String::with_capacity(w.len() + 8); + let mut flags: Vec = Vec::new(); // Each split point from the end: the appended string is `w[b..]`, at most the longest // suffix, and something must stay before it. let cuts = std::iter::once(w.len()).chain(w.char_indices().rev().map(|(i, _)| i).filter(|&i| i > 0)); @@ -1061,18 +1265,23 @@ impl AffixDict { let rules = || g.rules.iter().filter_map(|&i| self.suffixes.get(i as usize)); // The stem path: `root` is a stem carrying the rule's flag (and the prefix's, // when a cross product is being checked). - if let Some(f) = self.stem_flags_except(&root, deny) - && f.covers(need) - && rules().any(|r| f.meets(r.bit) && (need.is_empty() || r.cross) && r.cond_matches(&root, true)) + if let Some(f) = self.stem_flags_except(&root, excluded) + && need.is_none_or(|n| has(f, n)) + && rules().any(|r| has(f, r.flag) && (need.is_none() || r.cross) && r.cond_matches(&root, true)) { return true; } // Continuation classes: `root` need not be a stem carrying this rule's flag — // it may be a form that gained the flag from another rule's continuation. The - // matching rules' flags are looked for together, in one walk. - if need.is_empty() { - let flags = rules().filter(|r| r.cond_matches(&root, true)).fold(Mask::EMPTY, |m, r| m | r.bit); - if !flags.is_empty() && self.form_has_flag(&root, flags, INNER_CHAIN, deny) { + // matching rules' flags are looked for together, in one walk. Without continuation + // classes that walk would only repeat the stem path above. + if need.is_none() && self.has_cont { + flags.clear(); + // Only a flag some continuation class holds can come from another rule. + flags.extend(rules().filter(|r| self.is_cont_flag(r.flag) && r.cond_matches(&root, true)).map(|r| r.flag)); + flags.sort_unstable(); + flags.dedup(); + if !flags.is_empty() && self.form_has_flag(&root, &flags, INNER_CHAIN, excluded) { return true; } } @@ -1086,32 +1295,32 @@ impl AffixDict { /// may be in capitals. Lower-case input never matches a capitalized stem (`warszawa`), and a /// `KEEPCASE` stem matches only as written. pub fn check(&self, w: &str) -> bool { - self.check_except(w, Mask::EMPTY) + self.check_except(w, self.excluded(false)) } /// Whether a suggestion is correct in its displayed case and has an eligible stem. pub fn can_suggest(&self, w: &str) -> bool { - self.check_except(w, self.no_suggest | self.substandard) + self.check_except(w, self.excluded(true)) } /// Whether `w` is a word only through a `KEEPCASE` stem, so it is correct in this letter case /// alone (`km` from `km/K`, which `Km` and `KM` are not). pub(crate) fn case_locked(&self, w: &str) -> bool { - self.keepcase != Mask::EMPTY && self.contains(w) && !self.contains_except(&self.normalize(w), self.keepcase) + self.keepcase.is_some() && self.contains(w) && !self.contains_except(&self.normalize(w), self.excluded(false).and(self.keepcase)) } - fn check_except(&self, w: &str, deny: Mask) -> bool { + fn check_except(&self, w: &str, excluded: Excluded) -> bool { let w = self.normalize(w); - if self.contains_except(&w, deny) { + if self.contains_except(&w, excluded) { return true; } - let deny = deny | self.keepcase; + let excluded = excluded.and(self.keepcase); match casing(&w) { Casing::Lower | Casing::Mixed => false, - Casing::Initial => self.contains_except(&w.to_lowercase(), deny), + Casing::Initial => self.contains_except(&w.to_lowercase(), excluded), Casing::Upper => { let lower = w.to_lowercase(); - self.contains_except(&capitalize(&lower), deny) || self.contains_except(&lower, deny) + self.contains_except(&capitalize(&lower), excluded) || self.contains_except(&lower, excluded) } } } @@ -1126,11 +1335,11 @@ impl AffixDict { let mut cross_sfx = Vec::new(); // Suffix forms queued with the continuation classes they carry, for chained expansion; // each (form, classes) pair is expanded once. - let mut chain: Vec<(String, Mask)> = Vec::new(); - let mut queued: std::collections::HashSet<(String, Mask)> = std::collections::HashSet::new(); + let mut chain: Vec<(String, u32)> = Vec::new(); + let mut queued: std::collections::HashSet<(String, u32)> = std::collections::HashSet::new(); let mut work = 0usize; for r in &self.suffixes { - if !f.meets(r.bit) || !r.cond_matches(stem, true) { + if !has(f, r.flag) || !r.cond_matches(stem, true) { continue; } let Some(base) = stem.strip_suffix(&*r.strip) else { continue }; @@ -1138,7 +1347,7 @@ impl AffixDict { if r.cross { cross_sfx.push(form.clone()); } - if !r.cont.is_empty() && queued.insert((form.clone(), r.cont)) { + if r.cont != 0 && queued.insert((form.clone(), r.cont)) { chain.push((form.clone(), r.cont)); } out.push(form); @@ -1164,7 +1373,7 @@ impl AffixDict { } let Some(base) = form.strip_suffix(&*r.strip) else { continue }; let next = format!("{base}{}", r.add); - if !r.cont.is_empty() && queued.insert((next.clone(), r.cont)) { + if r.cont != 0 && queued.insert((next.clone(), r.cont)) { chain.push((next.clone(), r.cont)); } out.push(next); @@ -1175,7 +1384,7 @@ impl AffixDict { } } for r in &self.prefixes { - if !f.meets(r.bit) || !r.cond_matches(stem, false) { + if !has(f, r.flag) || !r.cond_matches(stem, false) { continue; } let Some(rest) = stem.strip_prefix(&*r.strip) else { continue }; @@ -1198,12 +1407,14 @@ impl AffixDict { out } - /// The suffix rules carrying any flag of `mask`, in rule order. - fn suffixes_with(&self, mask: Mask) -> Vec { + /// The suffix rules carrying any flag of the set `set`, in rule order. + fn suffixes_with(&self, set: u32) -> Vec { let mut rules: Vec = Vec::new(); let mut lists = 0; - for (bit, list) in self.sfx_by_flag.iter().enumerate() { - if mask.meets(Mask::bit(bit)) && !list.is_empty() { + for &flag in self.set(set) { + if let Some(list) = self.sfx_by_flag.get(usize::from(flag)) + && !list.is_empty() + { rules.extend_from_slice(list); lists += 1; } @@ -1239,14 +1450,18 @@ impl AffixDict { &self.comments } - /// Serialize to WordCraft's compact format: raw-deflate of `WCAFF1`, the comments, the - /// tables and rules as text lines, a `WORDS` line, then one line per stem, front-coded - /// against the previous one (a byte with the shared prefix length in bytes, the rest of the - /// word, `/` and its flags if any). + /// Serialize to WordCraft's compact format: raw-deflate of `WCAFF1`, a `FLAG` line when flags + /// aren't one character each, the comments, the tables and rules as text lines, a `WORDS` + /// line, then one line per stem, front-coded against the previous one (a byte with the shared + /// prefix length in bytes, the rest of the word, `/` and its flags if any). Flags are written + /// as the `.aff` wrote them, so one-character data is the same as before `FLAG` lines existed. pub fn to_bytes(&self) -> Vec { let mut out = String::new(); out.push_str(MAGIC); out.push('\n'); + if let Some(name) = self.flag_mode.name() { + out.push_str(&format!("FLAG {name}\n")); + } for c in &self.comments { out.push_str("# "); out.push_str(c.trim()); @@ -1264,13 +1479,14 @@ impl AffixDict { for (a, b) in &self.rep { out.push_str(&format!("REP {} {}\n", a.replace(' ', "_"), b.replace(' ', "_"))); } - for (directive, mask) in - [("FORBIDDENWORD", self.forbidden), ("NOSUGGEST", self.no_suggest), ("SUBSTANDARD", self.substandard), ("KEEPCASE", self.keepcase)] + for (directive, flag) in + [("FORBIDDENWORD", self.forbidden), ("NOSUGGEST", self.no_suggest), ("SUBSTANDARD", self.substandard), ("KEEPCASE", self.keepcase), ("ONLYINCOMPOUND", self.only_in_compound)] { - if !mask.is_empty() - && let Some(c) = self.flag_char(mask) - { - out.push_str(&format!("{directive} {c}\n")); + if let Some(flag) = flag { + out.push_str(directive); + out.push(' '); + self.write_flags(&[flag], &mut out); + out.push('\n'); } } for (a, b) in &self.iconv { @@ -1279,11 +1495,6 @@ impl AffixDict { if !self.ignore.is_empty() { out.push_str(&format!("IGNORE {}\n", self.ignore.iter().collect::())); } - if !self.only_in_compound.is_empty() - && let Some(c) = self.flag_char(self.only_in_compound) - { - out.push_str(&format!("ONLYINCOMPOUND {c}\n")); - } if !self.compound_text.is_empty() { out.push_str(&format!("COMPOUNDMIN {}\n", self.compound_min)); for p in &self.compound_text { @@ -1296,12 +1507,14 @@ impl AffixDict { let y = if r.cross { "Y" } else { "N" }; // Continuation classes ride on the appended field, as in the `.aff` source. let mut add = z(&r.add); - let cont = self.flag_chars_for(r.cont); + let cont = self.set(r.cont); if !cont.is_empty() { add.push('/'); - add.push_str(&cont); + self.write_flags(cont, &mut add); } - out.push_str(&format!("{kind} {} {y} {} {} {}\n", r.flag, z(&r.strip), add, r.cond_text)); + let mut flag = String::new(); + self.write_flags(&[r.flag], &mut flag); + out.push_str(&format!("{kind} {flag} {y} {} {} {}\n", z(&r.strip), add, r.cond_text)); } } out.push_str("WORDS\n"); @@ -1313,10 +1526,11 @@ impl AffixDict { let p = prev.iter().zip(b).take_while(|(x, y)| x == y).count().min(255); raw.push(p as u8); raw.extend_from_slice(b.get(p..).unwrap_or_default()); - let mask = self.flags.get(i).and_then(|&id| self.sets.get(usize::from(id))).copied().unwrap_or_default(); - if !mask.is_empty() { - raw.push(b'/'); - raw.extend_from_slice(self.flag_chars_for(mask).as_bytes()); + let set = self.flags.get(i).map_or(&[][..], |&id| self.set(id)); + if !set.is_empty() { + let mut flags = String::from("/"); + self.write_flags(set, &mut flags); + raw.extend_from_slice(flags.as_bytes()); } raw.push(b'\n'); prev = b; @@ -1344,14 +1558,16 @@ impl AffixDict { let f: Vec<&str> = split_fields(line).collect(); let at = |k: usize| f.get(k).copied().unwrap_or(""); match at(0) { + "FLAG" => rules.set_mode(at(1)).map_err(|e| format!("affix dictionary: {e}"))?, "TRY" => rules.try_chars = cap_try(at(1)), "KEY" => rules.key = at(1).to_string(), - directive @ ("FORBIDDENWORD" | "NOSUGGEST" | "SUBSTANDARD" | "KEEPCASE") => { - let c = one_flag(at(1), directive, 0).map_err(|_| format!("affix dictionary: bad {directive} in `{line}`"))?; + directive @ ("FORBIDDENWORD" | "NOSUGGEST" | "SUBSTANDARD" | "KEEPCASE" | "ONLYINCOMPOUND") => { + let c = rules.one_flag(at(1)).map_err(|e| format!("affix dictionary: bad {directive} in `{line}`: {e}"))?; match directive { "FORBIDDENWORD" => rules.forbidden = Some(c), "NOSUGGEST" => rules.no_suggest = Some(c), "SUBSTANDARD" => rules.substandard = Some(c), + "ONLYINCOMPOUND" => rules.only_in_compound = Some(c), _ => rules.keepcase = Some(c), } } @@ -1361,11 +1577,7 @@ impl AffixDict { let min = at(1).parse::().map_err(|_| format!("affix dictionary: bad `{line}`"))?; rules.compound_min = Some(min.clamp(1, MAX_COMPOUND_BYTES)); } - "ONLYINCOMPOUND" => { - rules.only_in_compound = Some(one_flag(at(1), "ONLYINCOMPOUND", 0).map_err(|_| format!("affix dictionary: bad `{line}`"))?); - } "COMPOUNDRULE" => { - parse_compound_rule(at(1)).map_err(|e| format!("affix dictionary: {e}"))?; if rules.compound_rules.len() >= MAX_COMPOUND_RULES { return Err("affix dictionary: too many COMPOUNDRULE patterns".into()); } @@ -1374,8 +1586,7 @@ impl AffixDict { "MAP" => rules.map.push(at(1).chars().collect()), "REP" => rules.rep.push((at(1).replace('_', " "), at(2).replace('_', " "))), kind @ ("PFX" | "SFX") => { - let mut fc = at(1).chars(); - let (Some(flag), None) = (fc.next(), fc.next()) else { return Err(format!("affix dictionary: bad flag in `{line}`")) }; + let flag = rules.one_flag(at(1)).map_err(|e| format!("affix dictionary: bad flag in `{line}`: {e}"))?; if f.len() != 6 { return Err(format!("affix dictionary: bad rule `{line}`")); } @@ -1428,7 +1639,7 @@ impl AffixDict { /// Does a compound rule's token list match a split of the word into two or more parts? `part(a, /// b)` is the flags of the stem between char cuts `a` and `b` (`None`: not a stem). States are /// (cut, token, parts so far capped at 2), each visited once. -fn compound_matches(rule: &[(Mask, Repeat)], part: &mut impl FnMut(usize, usize) -> Option, n: usize) -> bool { +fn compound_matches<'a>(rule: &[(Option, Repeat)], part: &mut impl FnMut(usize, usize) -> Option<&'a [u16]>, n: usize) -> bool { let t = rule.len(); let idx = |cut: usize, tok: usize, k: usize| (cut * (t + 1) + tok) * 3 + k; let mut seen = vec![false; n * (t + 1) * 3]; @@ -1444,12 +1655,14 @@ fn compound_matches(rule: &[(Mask, Repeat)], part: &mut impl FnMut(usize, usize) } continue; } - let Some(&(bit, repeat)) = rule.get(tok) else { continue }; + let Some(&(flag, repeat)) = rule.get(tok) else { continue }; if repeat != Repeat::Once { stack.push((cut, tok + 1, k)); } for next in cut + 1..n { - if part(cut, next).is_some_and(|m| m.meets(bit)) { + if let Some(flag) = flag + && part(cut, next).is_some_and(|set| has(set, flag)) + { let k = (k + 1).min(2); stack.push((next, if repeat == Repeat::Any { tok } else { tok + 1 }, k)); } @@ -1614,7 +1827,7 @@ pub(crate) mod tests { #[test] fn flag_utf8_and_default_modes_are_accepted() { // `FLAG UTF-8` (VERO pt_BR) and `FLAG DEFAULT` are per-character flag modes: the same - // mode the reader already assumes. Only `long` and `num` stay unsupported. + // mode the reader assumes without a FLAG line. for mode in ["UTF-8", "DEFAULT"] { let aff = format!("SET UTF-8\nFLAG {mode}\nSFX a Y 1\nSFX a 0 y [^y]\n"); let d = AffixDict::from_hunspell(&aff, "1\nkot/a\n", &[]).unwrap(); @@ -1632,7 +1845,7 @@ pub(crate) mod tests { assert!(d.contains("banco") && d.contains("bancos")); assert!(d.contains("telebanco") && d.contains("telebancos"), "the ☎ prefix applies"); assert!(d.contains("basuras") && !d.contains("telebasura"), "only to stems that carry it"); - assert!(!d.flag_chars.contains(&EMOJI_SELECTOR), "the selector is not a flag"); + assert!(!d.flag_keys.contains(&FlagKey::from(EMOJI_SELECTOR)), "the selector is not a flag"); } #[test] @@ -1677,8 +1890,9 @@ pub(crate) mod tests { assert!(!d.contains("szóx1") && !d.contains("szóx199") && !d.contains("szóx131")); let back = AffixDict::from_bytes(&d.to_bytes()).unwrap(); assert!(back.contains("szóx198") && !back.contains("szóx199")); - let too_many: String = (0x100u32..0x100 + 257).filter_map(char::from_u32).collect(); - assert!(AffixDict::from_hunspell("SET UTF-8\n", &format!("1\nszó/{too_many}\n"), &[]).is_err()); + // 257 distinct flags load too now (each flag gets a u16 id). + let many: String = (0x100u32..0x100 + 257).filter_map(char::from_u32).collect(); + assert!(AffixDict::from_hunspell("SET UTF-8\n", &format!("1\nszó/{many}\n"), &[]).is_ok()); } #[test] @@ -2002,6 +2216,26 @@ pub(crate) mod tests { assert!(back.check("foo") && !back.check("qux"), "a part-only stem stays a part"); } + /// With long or numeric flags a `COMPOUNDRULE` writes each flag in parentheses, as Hunspell + /// reads them (checked black-box against the `hunspell` CLI); a bare flag is refused. + #[test] + fn compound_rules_read_long_and_numeric_flags() { + let aff = "SET UTF-8\nFLAG long\nCOMPOUNDMIN 1\nONLYINCOMPOUND cc\nCOMPOUNDRULE 1\nCOMPOUNDRULE (nn)*(mm)(pp)\n"; + let d = AffixDict::from_hunspell(aff, "4\n1/nnmm\n2/nnmm\n1st/pp\n1th/ppcc\n", &[]).unwrap(); + for w in ["1st", "21st", "11st", "221st", "21th"] { + assert!(d.check(w), "{w}"); + } + assert!(!d.check("1th") && !d.check("12")); + let back = AffixDict::from_bytes(&d.to_bytes()).unwrap(); + assert!(back.check("221st") && !back.check("1th")); + let aff = "SET UTF-8\nFLAG num\nCOMPOUNDMIN 1\nONLYINCOMPOUND 9\nCOMPOUNDRULE 1\nCOMPOUNDRULE (1)*(2)(3)\n"; + let d = AffixDict::from_hunspell(aff, "3\n1/1,2\n2/1,2\n1st/3\n", &[]).unwrap(); + assert!(d.check("21st") && d.check("221st") && !d.check("12st")); + for bad in ["FLAG long\nCOMPOUNDRULE nn*", "FLAG long\nCOMPOUNDRULE (n)", "FLAG num\nCOMPOUNDRULE (1", "FLAG num\nCOMPOUNDRULE (x)"] { + assert!(AffixDict::from_hunspell(&format!("SET UTF-8\n{bad}\n"), "1\nx\n", &[]).is_err(), "{bad}"); + } + } + #[test] fn hostile_compound_rules_are_refused_or_cheap() { for bad in ["COMPOUNDRULE *a", "COMPOUNDRULE (ab)", "COMPOUNDRULE a**", "COMPOUNDMIN x", "ONLYINCOMPOUND ab"] { @@ -2060,8 +2294,10 @@ pub(crate) mod tests { #[test] fn unsupported_and_broken_input_is_an_error() { - assert!(AffixDict::from_hunspell("FLAG long\n", DIC, &[]).is_err()); - assert!(AffixDict::from_hunspell("FLAG num\n", DIC, &[]).is_err()); + // `FLAG long` and `FLAG num` load now (see the tests below); an unknown mode doesn't. + assert!(AffixDict::from_hunspell("FLAG long\n", DIC, &[]).is_ok()); + assert!(AffixDict::from_hunspell("FLAG num\n", DIC, &[]).is_ok()); + assert!(AffixDict::from_hunspell("FLAG bogus\n", DIC, &[]).is_err()); assert!(AffixDict::from_hunspell("COMPOUNDFLAG X\n", DIC, &[]).is_err()); assert!(AffixDict::from_hunspell("SFX a Y 1\nSFX a 0 y [ab\n", DIC, &[]).is_err()); assert!(AffixDict::from_bytes(b"not deflate").is_err()); @@ -2073,6 +2309,177 @@ pub(crate) mod tests { assert!(!empty.contains("kot") && !empty.check("Kot") && empty.forms("kot").is_empty()); } + /// A small `FLAG num` dictionary (made-up words): prefixes, suffixes, a continuation class, + /// a cross product, the largest flag number, and numeric FORBIDDENWORD and NOSUGGEST flags. + const NUM_AFF: &str = "SET UTF-8\nFLAG num\nFORBIDDENWORD 900\nNOSUGGEST 901\nPFX 7 Y 1\nPFX 7 0 ne .\nSFX 12 Y 2\nSFX 12 0 a .\nSFX 12 0 om/1503 .\nSFX 1503 N 1\nSFX 1503 0 e .\nSFX 65535 N 1\nSFX 65535 0 i .\n"; + const NUM_DIC: &str = "5\ngrad/12,7\nmost/65535\nkuca/12,901\ngradd/900\nlist/1503\n"; + + fn assert_num_dict(d: &AffixDict) { + for w in ["grad", "grada", "gradom", "gradome", "negrad", "negrada", "negradom", "most", "mosti", "list", "liste", "Grada", "MOSTI"] { + assert!(d.check(w), "{w}"); + } + // 1503 reaches `grad` only through `om`; 12 and 65535 belong to other stems; 7 is a prefix. + for w in ["grade", "gradi", "mosta", "nemost", "lista", "listom", "gradomi", "7grad", "grad12"] { + assert!(!d.check(w), "{w}"); + } + assert!(!d.check("gradd") && !d.check("Gradd") && d.forms("gradd").is_empty(), "numeric FORBIDDENWORD"); + assert!(d.check("kuca") && d.check("kucaa") && !d.can_suggest("kuca") && !d.can_suggest("kucaom"), "numeric NOSUGGEST"); + assert!(d.can_suggest("gradome")); + let mut forms = d.forms("grad"); + forms.sort(); + forms.dedup(); + assert_eq!(forms, ["grad", "grada", "gradom", "gradome", "negrad", "negrada", "negradom"]); + } + + #[test] + fn flag_num_dictionaries_load_and_check() { + let d = AffixDict::from_hunspell(NUM_AFF, NUM_DIC, &[]).unwrap(); + assert_eq!((d.len(), d.rule_count()), (5, 5)); + assert_num_dict(&d); + } + + #[test] + fn flag_num_dictionaries_round_trip_the_compact_format() { + let d = AffixDict::from_hunspell(NUM_AFF, NUM_DIC, &["num test"]).unwrap(); + let bytes = d.to_bytes(); + let raw = miniz_oxide::inflate::decompress_to_vec(&bytes).unwrap(); + assert!(raw.starts_with(b"WCAFF1\nFLAG num\n"), "the mode is written before any flag"); + let text = String::from_utf8_lossy(&raw); + assert!(text.contains("FORBIDDENWORD 900\n") && text.contains("SFX 12 Y 0 om/1503 .\n") && text.contains("grad/7,12\n")); + let back = AffixDict::from_bytes(&bytes).unwrap(); + assert_num_dict(&back); + assert_eq!(back.comments(), ["num test"]); + assert_eq!(back.to_bytes(), bytes, "a second round trip writes the same data"); + } + + /// A small `FLAG long` dictionary: two-character flags that share characters with each other. + const LONG_AFF: &str = "SET UTF-8\nFLAG long\nFORBIDDENWORD !!\nSFX Aa Y 1\nSFX Aa 0 s/Bb .\nSFX Bb N 1\nSFX Bb 0 ly .\nPFX Cc Y 1\nPFX Cc 0 un .\nSFX aA N 1\nSFX aA 0 er .\n"; + const LONG_DIC: &str = "5\nkind/AaCc\nhope/Bb\nbadd/!!\nodd/Aab\nwarm/aA\n"; + + fn assert_long_dict(d: &AffixDict) { + for w in ["kind", "kinds", "kindsly", "unkind", "unkinds", "hope", "hopely", "warm", "warmer"] { + assert!(d.check(w), "{w}"); + } + for w in ["kindly", "kinder", "hopes", "unhope", "warms", "unwarm", "badd", "odd", "odds"] { + assert!(!d.check(w), "{w}"); + } + } + + #[test] + fn flag_long_dictionaries_load_check_and_round_trip() { + let d = AffixDict::from_hunspell(LONG_AFF, LONG_DIC, &[]).unwrap(); + // `odd/Aab` has an odd number of flag characters: the entry is skipped. + assert_eq!(d.len(), 4); + assert_long_dict(&d); + let bytes = d.to_bytes(); + assert!(miniz_oxide::inflate::decompress_to_vec(&bytes).unwrap().starts_with(b"WCAFF1\nFLAG long\n")); + let back = AffixDict::from_bytes(&bytes).unwrap(); + assert_long_dict(&back); + assert_eq!(back.to_bytes(), bytes); + assert!(AffixDict::from_hunspell("FLAG long\nSFX A Y 1\nSFX A 0 s .\n", "0\n", &[]).is_err(), "a one-character long flag"); + } + + #[test] + fn malformed_flag_numbers_are_refused_in_the_aff_and_skipped_in_the_dic() { + for rule in ["x", "0", "65536", "99999999999999999999999", "+5", "-1", "1,2", " "] { + let aff = format!("FLAG num\nSFX {rule} Y 1\nSFX {rule} 0 a .\n"); + assert!(AffixDict::from_hunspell(&aff, "1\nw/1\n", &[]).is_err(), "rule flag `{rule}`"); + } + for cont in ["2,,3", "2,", ",2", "a", "0", "70000", "x2"] { + let aff = format!("FLAG num\nSFX 1 Y 1\nSFX 1 0 a/{cont} .\n"); + assert!(AffixDict::from_hunspell(&aff, "1\nw/1\n", &[]).is_err(), "continuation `{cont}`"); + } + for directive in ["FORBIDDENWORD 1,2", "FORBIDDENWORD abc", "NOSUGGEST 0", "NOSUGGEST", "FORBIDDENWORD 65536"] { + assert!(AffixDict::from_hunspell(&format!("FLAG num\n{directive}\n"), "0\n", &[]).is_err(), "{directive}"); + } + assert!(AffixDict::from_hunspell("SFX a Y 1\nSFX a 0 y .\nFLAG num\n", "0\n", &[]).is_err(), "FLAG after flags"); + assert!(AffixDict::from_hunspell("FLAG num\nFLAG num\nSFX 1 Y 1\nSFX 1 0 a .\n", "0\n", &[]).is_ok()); + // The .dic: entries whose flags don't read are left out; the rest of the file loads. + let dic = "8\nok/1\nzero/0\nbig/70000\nhuge/123456789012345678901234567890\nsign/-1\ncomma/1,\nletters/a1\nplain\n"; + let d = AffixDict::from_hunspell("FLAG num\nSFX 1 Y 1\nSFX 1 0 a .\n", dic, &[]).unwrap(); + assert!(d.check("ok") && d.check("oka") && d.check("plain")); + for w in ["zero", "big", "huge", "sign", "comma", "commaa", "letters"] { + assert!(!d.check(w), "{w}"); + } + assert_eq!(d.len(), 2); + // Text after a number's digits is ignored, as Hunspell does (Nepali's `0/17X`). + let d = AffixDict::from_hunspell("FLAG num\nSFX 1 Y 1\nSFX 1 0 a/2X .\nSFX 2 N 1\nSFX 2 0 i .\n", "1\nw/1x,7\n", &[]).unwrap(); + assert!(d.check("wa") && d.check("wai") && !d.check("wi")); + } + + #[test] + fn thousands_of_flags_load() { + // Serbian uses 1,903 numeric flags, Belarusian and Sinhala 5,000, each stem one or two: + // 3,000 flags here, all appending `a` (one suffix group of 3,000 rules), plus flags up to + // the largest number. + let mut aff = String::from("SET UTF-8\nFLAG num\n"); + let mut dic = String::from("3002\n"); + for n in 1..=3000 { + aff.push_str(&format!("SFX {n} N 1\nSFX {n} 0 a [^q]\n")); + dic.push_str(&format!("w{n}/{n}\n")); + } + aff.push_str("SFX 65000 N 1\nSFX 65000 0 i .\n"); + dic.push_str("dva/3000,65000\nq/1\n"); + let d = AffixDict::from_hunspell(&aff, &dic, &[]).unwrap(); + assert_eq!(d.rule_count(), 3001); + for w in ["w1a", "w1500a", "w3000a", "dva", "dvaa", "dvai", "q"] { + assert!(d.check(w), "{w}"); + } + assert!(!d.check("qa") && !d.check("w1i") && d.check("w1")); + let back = AffixDict::from_bytes(&d.to_bytes()).unwrap(); + assert!(back.check("w2999a") && back.check("dvai") && !back.check("w7i")); + + // More than 128 one-character flags (#4966): 300 letters, each its own suffix. + let letters: Vec = (0x100u32..0x100 + 300).filter_map(char::from_u32).collect(); + let mut aff = String::from("SET UTF-8\nFLAG UTF-8\n"); + for c in &letters { + aff.push_str(&format!("SFX {c} N 1\nSFX {c} 0 {c} .\n")); + } + let all: String = letters.iter().take(256).collect(); + let d = AffixDict::from_hunspell(&aff, &format!("2\nbase/{all}\nlast/{}\n", letters[299]), &[]).unwrap(); + assert!(d.check(&format!("base{}", letters[0])) && d.check(&format!("base{}", letters[255]))); + assert!(!d.check(&format!("base{}", letters[299])) && d.check(&format!("last{}", letters[299]))); + } + + #[test] + fn flag_counts_are_capped() { + let field = |n: usize| (1..=n).map(|i| i.to_string()).collect::>().join(","); + let aff = "FLAG num\nSFX 1 Y 1\nSFX 1 0 a .\n"; + // At most MAX_FLAGS_PER_ENTRY flags per .dic entry: one more and the entry is skipped. + let dic = format!("2\nfull/{}\nover/{}\n", field(MAX_FLAGS_PER_ENTRY), field(MAX_FLAGS_PER_ENTRY + 1)); + let d = AffixDict::from_hunspell(aff, &dic, &[]).unwrap(); + assert!(d.check("fulla") && !d.check("over") && !d.check("overa")); + let cont = format!("FLAG num\nSFX 1 Y 1\nSFX 1 0 a/{} .\n", field(MAX_FLAGS_PER_ENTRY + 1)); + assert!(AffixDict::from_hunspell(&cont, "0\n", &[]).is_err(), "too many continuation classes"); + // More distinct flags than a u16 id holds (only one-character flags can get there): an + // error, not a panic or a wrapped id. + let chars: Vec = + (0x4e00u32..0xa000).chain(0xac00..0xd7a4).chain(0x20000..0x2a6e0).filter_map(char::from_u32).take(MAX_DISTINCT_FLAGS + 1).collect(); + assert_eq!(chars.len(), MAX_DISTINCT_FLAGS + 1); + let mut dic = String::new(); + for (i, chunk) in chars.chunks(MAX_FLAGS_PER_ENTRY).enumerate() { + dic.push_str(&format!("w{i}/{}\n", chunk.iter().collect::())); + } + let err = AffixDict::from_hunspell("", &dic, &[]).unwrap_err(); + assert!(err.contains("distinct flags"), "{err}"); + } + + #[test] + fn compact_data_without_a_flag_line_still_loads() { + // Data written before the FLAG line existed: one-character flags, no FLAG line. + let old = miniz_oxide::deflate::compress_to_vec(b"WCAFF1\n# old data\nFORBIDDENWORD F\nSFX a Y 0 y .\nWORDS\n\x00kot/a\n\x00zly/F\n", 6); + let d = AffixDict::from_bytes(&old).unwrap(); + assert!(d.check("kot") && d.check("koty") && !d.check("zly")); + // One-character dictionaries still write no FLAG line, so their data is unchanged. + let raw = miniz_oxide::inflate::decompress_to_vec(&dict().to_bytes()).unwrap(); + assert!(!String::from_utf8_lossy(&raw).contains("\nFLAG")); + // A FLAG line after a flag, or an unknown one, is bad data. + for head in ["WCAFF1\nSFX a Y 0 y .\nFLAG num\n", "WCAFF1\nFLAG bogus\n"] { + let bytes = miniz_oxide::deflate::compress_to_vec(format!("{head}WORDS\n\x00kot\n").as_bytes(), 6); + assert!(AffixDict::from_bytes(&bytes).is_err(), "{head:?}"); + } + } + proptest::proptest! { #[test] fn lookups_never_panic(s in "\\PC{0,40}", bytes in proptest::collection::vec(proptest::num::u8::ANY, 0..200)) { @@ -2082,5 +2489,29 @@ pub(crate) mod tests { let _ = AffixDict::from_bytes(&bytes); let _ = AffixDict::from_bytes(&miniz_oxide::deflate::compress_to_vec(&bytes, 1)); } + + #[test] + fn any_flag_field_reads_without_panicking(field in "[0-9,a-zA-Z!\u{fe0f}\u{1f6ab}]{0,24}", word in "\\PC{0,12}") { + for mode in ["num", "long", "UTF-8"] { + let aff = format!("FLAG {mode} +FORBIDDENWORD {field} +SFX {field} Y 1 +SFX {field} 0 a/{field} . +"); + for aff in [aff.as_str(), &format!("FLAG {mode} +SFX 12 Y 1 +SFX 12 0 a . +")] { + if let Ok(d) = AffixDict::from_hunspell(aff, &format!("2 +w/{field} +x/12 +"), &[]) { + let _ = (d.check(&word), d.check("wa"), d.forms("w"), d.forms("x")); + let back = AffixDict::from_bytes(&d.to_bytes()); + proptest::prop_assert!(back.is_ok(), "{mode} {field:?}: {back:?}"); + } + } + } + } } } diff --git a/crates/proof/src/cs/mod.rs b/crates/proof/src/cs/mod.rs index 9f8bbd0a7..3f6e4a248 100644 --- a/crates/proof/src/cs/mod.rs +++ b/crates/proof/src/cs/mod.rs @@ -146,7 +146,7 @@ mod tests { fn missing_dictionary_and_ignored_inputs_are_quiet() { let empty = AffixDict::default(); assert!(is_correct_with(&empty, "pocitac")); - assert!(install_dictionary("FLAG long\n", "1\nstrom\n").is_err()); + assert!(install_dictionary("FLAG bogus\n", "1\nstrom\n").is_err()); let d = AffixDict::from_hunspell(test_dict::AFF, test_dict::DIC, &[]).unwrap(); for w in ["", "a", "2026", "ČR", "článek2", "日本語"] { assert!(is_correct_with(&d, w), "{w}"); diff --git a/crates/proof/src/es/mod.rs b/crates/proof/src/es/mod.rs index 5263bd1d9..4d19462df 100644 --- a/crates/proof/src/es/mod.rs +++ b/crates/proof/src/es/mod.rs @@ -272,7 +272,7 @@ mod tests { fn missing_dictionary_and_ignored_inputs_are_quiet() { let empty = AffixDict::default(); assert!(is_correct_with(&empty, "prueva", ProofLang::Es)); - assert!(install_dictionary(ProofLang::Es, "FLAG long\n", "1\ncasa\n").is_err()); + assert!(install_dictionary(ProofLang::Es, "FLAG bogus\n", "1\ncasa\n").is_err()); let d = AffixDict::from_hunspell(test_dict::AFF, test_dict::DIC, &[]).unwrap(); for w in ["", "a", "2026", "EE", "1.º", "日本語", "-", "--"] { assert!(is_correct_with(&d, w, ProofLang::EsUy), "{w}"); diff --git a/crates/proof/src/id/mod.rs b/crates/proof/src/id/mod.rs index 04248d638..53ba16ec4 100644 --- a/crates/proof/src/id/mod.rs +++ b/crates/proof/src/id/mod.rs @@ -153,7 +153,7 @@ mod tests { fn missing_dictionary_and_ignored_inputs_are_quiet() { let empty = AffixDict::default(); assert!(is_correct_with(&empty, "rumha")); - assert!(install_dictionary("FLAG long\n", "1\nrumah\n").is_err()); + assert!(install_dictionary("FLAG bogus\n", "1\nrumah\n").is_err()); let d = AffixDict::from_hunspell(test_dict::AFF, test_dict::DIC, &[]).unwrap(); for w in ["", "a", "2026", "RI", "ke-2", "日本語", "-", "--"] { assert!(is_correct_with(&d, w), "{w}"); diff --git a/crates/proof/src/it/mod.rs b/crates/proof/src/it/mod.rs index ee5aaaee3..ecc4aad67 100644 --- a/crates/proof/src/it/mod.rs +++ b/crates/proof/src/it/mod.rs @@ -145,7 +145,7 @@ mod tests { for w in ["caffè", "perché", "xqzv", ""] { assert!(is_correct_with(&none, w), "{w}"); } - assert!(install_dictionary("FLAG long\n", "1\nkot\n").is_err(), "unsupported files are refused"); + assert!(install_dictionary("FLAG bogus\n", "1\nkot\n").is_err(), "unsupported files are refused"); } #[test] diff --git a/crates/proof/src/pl/mod.rs b/crates/proof/src/pl/mod.rs index 7cde9d0f9..83d629c45 100644 --- a/crates/proof/src/pl/mod.rs +++ b/crates/proof/src/pl/mod.rs @@ -116,7 +116,7 @@ mod tests { assert!(is_correct_with(&none, w), "{w}"); } assert!(suggest::suggest_in(&none, "żeka", 5).is_empty()); - assert!(install_dictionary("FLAG long\n", "1\nkot\n").is_err(), "unsupported files are refused"); + assert!(install_dictionary("FLAG bogus\n", "1\nkot\n").is_err(), "unsupported files are refused"); } #[test] diff --git a/crates/proof/src/pt/mod.rs b/crates/proof/src/pt/mod.rs index ea91dbdb1..661bb9987 100644 --- a/crates/proof/src/pt/mod.rs +++ b/crates/proof/src/pt/mod.rs @@ -102,7 +102,7 @@ mod tests { for w in ["coraçao", "saude", "xqzv", ""] { assert!(is_correct_with(&none, w), "{w}"); } - assert!(install_dictionary("FLAG long\n", "1\nkot\n").is_err(), "unsupported files are refused"); + assert!(install_dictionary("FLAG bogus\n", "1\nkot\n").is_err(), "unsupported files are refused"); } #[test] diff --git a/crates/proof/src/slot.rs b/crates/proof/src/slot.rs index 6255ae5a5..a2200aca0 100644 --- a/crates/proof/src/slot.rs +++ b/crates/proof/src/slot.rs @@ -185,7 +185,7 @@ mod tests { #[test] fn an_installed_dictionary_has_no_file_until_reloaded() { assert!(!SLOT.has_dictionary(), "nothing installed under this name"); - assert!(SLOT.install("FLAG long\n", "1\nkot\n").is_err(), "unsupported files are refused"); + assert!(SLOT.install("FLAG bogus\n", "1\nkot\n").is_err(), "unsupported files are refused"); assert!(!SLOT.has_dictionary(), "a refused install keeps the current one"); SLOT.install("", "1\nkot\n").unwrap(); assert!(SLOT.has_dictionary() && SLOT.get().check("kot"));