daniel543x/craft-libs
1
0 Fork 0
mirror of https://github.com/storytold/craft-libs.git synced 2026-11-09 20:12:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

proof: flag aliases, single-byte flags, ICONV, IGNORE and KEEPCASE in affix dictionaries (storytold/wordcraft#735)

Browse Source 
The Hungarian Hunspell dictionary declares SET UTF-8 but writes its flags as single bytes, uses AF aliases and 158 distinct flags. Read flag fields byte by byte unless FLAG UTF-8 is set, expand AF aliases, skip AM, apply ICONV and IGNORE, honour KEEPCASE, and widen flag masks to 256 bits. Compounding and NEEDAFFIX stay unsupported for now.
This commit is contained in:
Skaywalker committed 2026-10-27 13:09:33 +01:00
1 parent 901f8cad59
commit 4e20a2d871
2 files changed
+530 -110
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

crates/proof/src/affix.rs
+347 -107
View File
@@ -11,10 +11,14 @@
//! also be stored in WordCraft's compact format ([`AffixDict::to_bytes`]): the rules as text
//! lines and the stems front-coded, raw-deflated together (the Polish dictionary is under 1 MB).
//!
//! Supported: one-character flags (at most 128 distinct, any Unicode character — `FLAG UTF-8`),
//! `PFX`/`SFX` with strip, append and condition (`.`, letters, `[abc]`, `[^abc]`), cross
//! products, continuation classes on suffixes (at most two suffixes in a row, as in Hunspell),
//! `FORBIDDENWORD`, `NOSUGGEST`, and the suggestion tables `TRY`, `KEY`, `MAP` and `REP`. Rejected: compounding,
//! Supported: one-character flags (at most 256 distinct, any Unicode character — `FLAG UTF-8`),
//! flag aliases (`AF`: a number standing for a set of flags), `PFX`/`SFX` with strip, append and
//! condition (`.`, letters, `[abc]`, `[^abc]`), cross products, continuation classes on suffixes
//! (at most two suffixes in a row, as in Hunspell), `FORBIDDENWORD`, `NOSUGGEST`, `KEEPCASE` (the
//! stem is a word only in its own letter case), input conversion (`ICONV`: ligatures such as `fi`
//! read as `fi`) and ignored characters (`IGNORE`), and the suggestion tables `TRY`, `KEY`, `MAP`
//! and `REP`.
//! Morphological data (`AM` aliases, the fields after a stem) is skipped. Rejected: compounding,
//! long or numeric flags, `NEEDAFFIX` and the other directives that change which words are correct.
//!
//! Lookups never panic: stems sit in one string, found through an open-addressing hash table, and
@@ -33,8 +37,6 @@ const EMPTY: u32 = u32::MAX;
/// Directives that change which words are correct in ways this engine doesn't implement.
const UNSUPPORTED: &[&str] = &[
"AF",
"AM",
"CHECKSHARPS",
"CIRCUMFIX",
"COMPLEXPREFIXES",
@@ -46,17 +48,78 @@ const UNSUPPORTED: &[&str] = &[
"COMPOUNDPERMITFLAG",
"COMPOUNDRULE",
"FULLSTRIP",
"ICONV",
"IGNORE",
"KEEPCASE",
"LEMMA_PRESENT",
"NEEDAFFIX",
"OCONV",
"ONLYINCOMPOUND",
"PSEUDOROOT",
"SUBSTANDARD",
];
/// Most distinct flags a dictionary may use: every single-byte flag (the Hungarian dictionary
/// uses 158).
const MAX_FLAGS: usize = 256;
/// A set of flags, one bit per distinct flag character of the dictionary.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)]
struct Mask([u64; MAX_FLAGS / 64]);
impl Mask {
const EMPTY: Mask = Mask([0; MAX_FLAGS / 64]);
/// The set holding flag number `i` alone (empty when `i` is out of range).
fn bit(i: usize) -> Mask {
let mut m = Mask::EMPTY;
if let Some(w) = m.0.get_mut(i / 64) {
*w = 1 << (i % 64);
}
m
}
/// The number of the lowest flag in the set, if any.
fn first(self) -> Option<usize> {
self.0.iter().enumerate().find(|(_, w)| **w != 0).map(|(i, w)| i * 64 + w.trailing_zeros() as usize)
}
fn is_empty(self) -> bool {
self.0.iter().all(|&w| w == 0)
}
/// Do the two sets share a flag?
fn meets(self, other: Mask) -> bool {
!(self & other).is_empty()
}
/// Does this set hold every flag of `other`?
fn covers(self, other: Mask) -> bool {
(self & other) == other
}
}
impl std::ops::BitAnd for Mask {
type Output = Mask;
fn bitand(mut self, o: Mask) -> Mask {
for (a, b) in self.0.iter_mut().zip(o.0) {
*a &= b;
}
self
}
}
impl std::ops::BitOr for Mask {
type Output = Mask;
fn bitor(mut self, o: Mask) -> Mask {
self |= o;
self
}
}
impl std::ops::BitOrAssign for Mask {
fn bitor_assign(&mut self, o: Mask) {
for (a, b) in self.0.iter_mut().zip(o.0) {
*a |= b;
}
}
}
/// One character class of a rule's condition.
#[derive(Clone, Debug, PartialEq, Eq)]
enum Cond {
@@ -130,14 +193,14 @@ const MAX_FORMS_WORK: usize = 1 << 18;
#[derive(Clone, Debug)]
struct Rule {
/// Bit of the rule's flag in the stems' flag masks.
bit: u128,
bit: Mask,
flag: char,
cross: bool,
strip: Box<str>,
add: Box<str>,
/// Continuation classes (`SFX A 0 able/B`): flags the derived form carries, licensing
/// further affixes. Chained application is bounded by [`MAX_CHAIN`].
cont: u128,
cont: Mask,
cond: Box<[Cond]>,
cond_text: Box<str>,
}
@@ -174,11 +237,11 @@ pub struct AffixDict {
/// Per stem: index into `sets`.
flags: Vec<u16>,
/// Distinct flag masks.
sets: Vec<u128>,
sets: Vec<Mask>,
/// The `FORBIDDENWORD` flag's bit; stems carrying it are not words (0: none declared).
forbidden: u128,
forbidden: Mask,
/// Stems with NOSUGGEST are correct, but none of their forms are suggested.
no_suggest: u128,
no_suggest: Mask,
/// Open-addressing hash table of stem indices (`EMPTY` = free).
table: Vec<u32>,
/// Flag char of each bit.
@@ -195,6 +258,12 @@ pub struct AffixDict {
key: String,
map: Vec<Vec<char>>,
rep: Vec<(String, String)>,
/// The `KEEPCASE` flag's bit: such stems are words only in their own letter case (0: none).
keepcase: Mask,
/// `ICONV`: input conversions applied before a lookup, `(from, to)`.
iconv: Vec<(String, String)>,
/// `IGNORE`: characters removed from input words (and from the stems and affixes).
ignore: Box<[char]>,
/// Provenance and licence lines, kept through [`AffixDict::to_bytes`].
comments: Vec<String>,
}
@@ -263,6 +332,49 @@ struct Rules {
/// The `FORBIDDENWORD` flag: stems carrying it are listed but never correct.
forbidden: Option<char>,
no_suggest: Option<char>,
/// The `KEEPCASE` flag.
keepcase: Option<char>,
iconv: Vec<(String, String)>,
ignore: Vec<char>,
/// `AF` flag aliases: alias `n` (1-based) stands for `aliases[n - 1]`.
aliases: Vec<String>,
}
impl Rules {
/// Flags as written in a `.dic` entry or a continuation class: an `AF` alias number when the
/// dictionary declares aliases, otherwise the flag characters themselves.
fn flags<'a>(&'a self, written: &'a str) -> Result<&'a str, String> {
if self.aliases.is_empty() || written.is_empty() || !written.bytes().all(|b| b.is_ascii_digit()) {
return Ok(written);
}
written
.parse::<usize>()
.ok()
.and_then(|n| n.checked_sub(1))
.and_then(|i| self.aliases.get(i))
.map(String::as_str)
.ok_or_else(|| format!("flag alias {written} is not defined"))
}
/// `w` without the `IGNORE` characters.
fn strip_ignored(&self, w: &str) -> String {
if self.ignore.is_empty() { w.to_string() } else { w.chars().filter(|c| !self.ignore.contains(c)).collect() }
}
}
/// The fields of an `.aff`/`.dic` line: Hunspell separates them with spaces and tabs only, so a
/// flag byte read as U+0085 or U+00A0 (see [`crate::hunspell::decode`]) stays a flag.
fn split_fields(line: &str) -> impl Iterator<Item = &str> {
line.split([' ', '\t']).filter(|f| !f.is_empty())
}
/// The one flag character a directive (`FORBIDDENWORD w`) names.
fn one_flag(arg: &str, dir: &str, n: usize) -> Result<char, String> {
let mut it = arg.chars().filter(|&c| c != EMOJI_SELECTOR);
match (it.next(), it.next()) {
(Some(c), None) if !c.is_whitespace() => Ok(c),
_ => Err(format!("line {}: {dir} needs one flag character", n + 1)),
}
}
/// A rule as parsed from text, before flags become bits: `(flag, cross, strip, add, cont, cond)`.
@@ -304,8 +416,10 @@ fn parse_aff(aff: &str) -> Result<Rules, String> {
let mut cross: HashMap<(bool, char), bool> = HashMap::new();
let mut map_header = false;
let mut rep_header = false;
let mut af_header = false;
let mut iconv_header = false;
for (n, line) in aff.lines().enumerate() {
let f: Vec<&str> = line.split_whitespace().collect();
let f: Vec<&str> = split_fields(line).collect();
let Some(&dir) = f.first() else { continue };
if dir.starts_with('#') {
continue;
@@ -314,25 +428,33 @@ fn parse_aff(aff: &str) -> Result<Rules, String> {
match dir {
"SET" | "LANG" | "NAME" | "VERSION" | "HOME" | "WORDCHARS" | "MAXNGRAMSUGS" | "MAXDIFF" | "ONLYMAXDIFF" | "NOSPLITSUGS"
| "SUGSWITHDOTS" | "WARN" | "FORBIDWARN" | "MAXCPDSUGS" => {}
// Morphological aliases, and flags that only matter for morphological generation
// and suggestions: the words they mark are correct.
"AM" | "SUBSTANDARD" | "LEMMA_PRESENT" => {}
"AF" => {
if !af_header && f.len() == 2 && at(1).parse::<usize>().is_ok() {
af_header = true;
} else {
r.aliases.push(at(1).to_string());
}
}
"ICONV" => {
if !iconv_header && f.len() == 2 && at(1).parse::<usize>().is_ok() {
iconv_header = true;
} else if f.len() >= 3 {
r.iconv.push((at(1).to_string(), at(2).to_string()));
}
}
"IGNORE" => r.ignore = at(1).chars().collect(),
"KEEPCASE" => r.keepcase = Some(one_flag(at(1), dir, n)?),
// Per-character flag modes (`FLAG UTF-8`, as VERO's pt_BR dictionary uses) work
// with the one-character parsing here; only the multi-character encodings differ.
"FLAG" if !matches!(at(1), "UTF-8" | "DEFAULT" | "utf-8" | "default" | "UTF8" | "") => {
return Err(format!("line {}: FLAG {} is not supported (one-character flags only)", n + 1, at(1)));
}
"FLAG" => {}
directive @ ("FORBIDDENWORD" | "NOSUGGEST") => {
let mut it = at(1).chars().filter(|&c| c != EMOJI_SELECTOR);
match (it.next(), it.next()) {
(Some(c), None) if !c.is_whitespace() => {
if directive == "FORBIDDENWORD" {
r.forbidden = Some(c);
} else {
r.no_suggest = Some(c);
}
}
_ => return Err(format!("line {}: {directive} needs one flag character", n + 1)),
}
}
"FORBIDDENWORD" => r.forbidden = Some(one_flag(at(1), dir, n)?),
"NOSUGGEST" => r.no_suggest = Some(one_flag(at(1), dir, n)?),
"TRY" => r.try_chars = cap_try(at(1)),
"KEY" => r.key = at(1).to_string(),
"MAP" => {
@@ -383,6 +505,19 @@ fn parse_aff(aff: &str) -> Result<Rules, String> {
_ => {}
}
}
// Aliases may be declared after the rules that use them; resolve once all are read.
let resolve = |rules: &mut Vec<RawRule>, r: &Rules| -> Result<(), String> {
for rule in rules.iter_mut() {
rule.4 = r.flags(&rule.4)?.to_string();
rule.2 = r.strip_ignored(&rule.2);
rule.3 = r.strip_ignored(&rule.3);
}
Ok(())
};
let (mut pfx, mut sfx) = (std::mem::take(&mut r.prefixes), std::mem::take(&mut r.suffixes));
resolve(&mut pfx, &r)?;
resolve(&mut sfx, &r)?;
(r.prefixes, r.suffixes) = (pfx, sfx);
Ok(r)
}
@@ -397,7 +532,7 @@ impl AffixDict {
let rules = parse_aff(aff)?;
let mut stems: Vec<(String, String)> = Vec::new();
for (n, line) in dic.lines().enumerate() {
let Some(tok) = line.split_whitespace().next() else { continue };
let Some(tok) = split_fields(line).next() else { continue };
if n == 0 && tok.parse::<usize>().is_ok() {
continue;
}
@@ -420,9 +555,11 @@ impl AffixDict {
word.push(c);
}
}
let word = rules.strip_ignored(&word);
if word.is_empty() || word.len() > MAX_WORD_BYTES || word.contains(['\n', '/']) {
continue;
}
let flags = rules.flags(&flags).map_err(|e| format!(".dic line {}: {e}", n + 1))?.to_string();
stems.push((word, flags));
}
AffixDict::build(rules, stems, comments.iter().map(ToString::to_string).collect())
@@ -434,11 +571,12 @@ impl AffixDict {
flag_chars.extend(stems.iter().flat_map(|s| s.1.chars()));
flag_chars.sort_unstable();
flag_chars.dedup();
if flag_chars.len() > 128 {
return Err(format!("{} distinct flags; at most 128 are supported", flag_chars.len()));
if flag_chars.len() > MAX_FLAGS {
return Err(format!("{} distinct flags; at most {MAX_FLAGS} are supported", flag_chars.len()));
}
let bit_of = |c: char| flag_chars.iter().position(|&x| x == c).map_or(0, |i| 1u128 << i);
let forbidden = rules.forbidden.map_or(0, bit_of);
let bit_of = |c: char| flag_chars.iter().position(|&x| x == c).map_or(Mask::EMPTY, Mask::bit);
let forbidden = rules.forbidden.map_or(Mask::EMPTY, bit_of);
let keepcase = rules.keepcase.map_or(Mask::EMPTY, bit_of);
// Stems sorted by bytes; homonyms (the same word twice) are merged.
stems.sort_by(|a, b| a.0.as_bytes().cmp(b.0.as_bytes()));
let mut d = AffixDict {
@@ -446,7 +584,10 @@ impl AffixDict {
offs: Vec::with_capacity(stems.len() + 1),
flag_chars: flag_chars.clone(),
forbidden,
no_suggest: rules.no_suggest.map_or(0, bit_of),
no_suggest: rules.no_suggest.map_or(Mask::EMPTY, bit_of),
keepcase,
iconv: rules.iconv,
ignore: rules.ignore.into_boxed_slice(),
try_chars: rules.try_chars,
key: rules.key,
map: rules.map,
@@ -455,16 +596,16 @@ impl AffixDict {
..AffixDict::default()
};
d.offs.push(0);
let mut set_ids: HashMap<u128, u16> = HashMap::new();
let mut set_ids: HashMap<Mask, u16> = HashMap::new();
let mut i = 0;
while i < stems.len() {
let Some((word, _)) = stems.get(i) else { break };
let mut mask = 0u128;
let mut mask = Mask::EMPTY;
let mut j = i;
while let Some((w, f)) = stems.get(j)
&& w == word
{
mask |= f.chars().map(bit_of).fold(0, |a, b| a | b);
mask |= f.chars().map(bit_of).fold(Mask::EMPTY, |a, b| a | b);
j += 1;
}
let next_id = u16::try_from(d.sets.len()).map_err(|_| "too many distinct flag sets".to_string())?;
@@ -478,7 +619,7 @@ impl AffixDict {
i = j;
}
let mk = |(flag, cross, strip, add, cont, cond): RawRule| -> Result<Rule, String> {
let cont = cont.chars().fold(0u128, |m, c| m | bit_of(c));
let cont = cont.chars().fold(Mask::EMPTY, |m, c| m | bit_of(c));
Ok(Rule { bit: bit_of(flag), flag, cross, strip: strip.into(), add: add.into(), cont, cond: parse_cond(&cond)?, cond_text: cond.into() })
};
d.prefixes = rules.prefixes.into_iter().map(mk).collect::<Result<_, _>>()?;
@@ -493,7 +634,7 @@ impl AffixDict {
// Each (appended, stripped) pair's group, found by hash rather than by scanning the
// appended string's groups: a dictionary with many strips per suffix stays linear.
let mut group_of: FxMap<(&str, &str), usize> = FxMap::default();
let mut by_flag: Vec<Vec<u32>> = vec![Vec::new(); 128];
let mut by_flag: Vec<Vec<u32>> = vec![Vec::new(); MAX_FLAGS];
for (i, r) in self.suffixes.iter().enumerate() {
let groups = idx.entry(r.add.clone()).or_default();
let Ok(i) = u32::try_from(i) else { break };
@@ -504,9 +645,7 @@ impl AffixDict {
groups.push(SfxGroup { strip: r.strip.clone(), rules: vec![i] });
}
}
if r.bit != 0
&& let Some(list) = by_flag.get_mut(r.bit.trailing_zeros() as usize)
{
if let Some(list) = r.bit.first().and_then(|b| by_flag.get_mut(b)) {
list.push(i);
}
}
@@ -551,12 +690,12 @@ impl AffixDict {
}
/// The flag characters of `mask`, in flag order (how the compact format stores flags).
fn flag_chars_for(&self, mask: u128) -> String {
self.flag_chars.iter().enumerate().filter(|&(i, _)| mask & (1u128 << i) != 0).map(|(_, c)| c).collect()
fn flag_chars_for(&self, mask: Mask) -> String {
self.flag_chars.iter().enumerate().filter(|&(i, _)| mask.meets(Mask::bit(i))).map(|(_, c)| c).collect()
}
/// The character of `mask` when it holds exactly one flag.
fn flag_char(&self, mask: u128) -> Option<char> {
fn flag_char(&self, mask: Mask) -> Option<char> {
let s = self.flag_chars_for(mask);
let mut it = s.chars();
match (it.next(), it.next()) {
@@ -589,11 +728,12 @@ impl AffixDict {
}
/// Flag mask of a stem, if `w` is one.
fn stem_flags(&self, w: &str) -> Option<u128> {
self.stem_flags_excluding(w, self.forbidden)
fn stem_flags(&self, w: &str) -> Option<Mask> {
self.stem_flags_except(w, Mask::EMPTY)
}
fn stem_flags_excluding(&self, w: &str, excluded: u128) -> Option<u128> {
/// Flag mask of a stem, if `w` is one and carries none of the flags in `deny`.
fn stem_flags_except(&self, w: &str, deny: Mask) -> Option<Mask> {
if self.table.is_empty() {
return None;
}
@@ -607,8 +747,9 @@ impl AffixDict {
if self.word(t as usize) == Some(w) {
let id = *self.flags.get(t as usize)?;
let mask = self.sets.get(usize::from(id)).copied();
// All derivations use this lookup: exclusions apply to stems and their forms.
return mask.filter(|m| m & excluded == 0);
// A stem carrying FORBIDDENWORD is listed but is not a word: no direct hit,
// and (this is the only stem lookup) no affixed form is built on it either.
return mask.filter(|m| !m.meets(self.forbidden | deny));
}
slot = (slot + 1) & mask;
}
@@ -621,8 +762,8 @@ impl AffixDict {
/// The rules of a group that could lead to `w` are merged into one mask of their flags and
/// followed once, not once per rule: the answer is the same, but thousands of rules of one
/// flag cost one walk, not one each (#3613).
fn form_has_flag(&self, w: &str, flags: u128, depth: u8, excluded: u128) -> bool {
if self.stem_flags_excluding(w, excluded).is_some_and(|f| f & flags != 0) {
fn form_has_flag(&self, w: &str, flags: Mask, depth: u8, deny: Mask) -> bool {
if self.stem_flags_except(w, deny).is_some_and(|f| f.meets(flags)) {
return true;
}
if depth == 0 {
@@ -641,9 +782,9 @@ impl AffixDict {
.rules
.iter()
.filter_map(|&i| self.suffixes.get(i as usize))
.filter(|r| r.cont & flags != 0 && r.cond_matches(&root, true))
.fold(0u128, |m, r| m | r.bit);
if inner != 0 && self.form_has_flag(&root, inner, depth - 1, excluded) {
.filter(|r| r.cont.meets(flags) && r.cond_matches(&root, true))
.fold(Mask::EMPTY, |m, r| m | r.bit);
if !inner.is_empty() && self.form_has_flag(&root, inner, depth - 1, deny) {
return true;
}
}
@@ -651,21 +792,50 @@ impl AffixDict {
false
}
/// Is `w` exactly (case and all) a stem or an affixed form of one?
/// Is `w` exactly (case and all) a stem or an affixed form of one? `ICONV` and `IGNORE`
/// apply first.
pub fn contains(&self, w: &str) -> bool {
self.contains_excluding(w, self.forbidden)
self.contains_except(&self.normalize(w), Mask::EMPTY)
}
/// Exactly spelled dictionary words eligible for suggestions (NOSUGGEST stems excluded).
pub fn contains_suggestible(&self, w: &str) -> bool {
self.contains_excluding(w, self.forbidden | self.no_suggest)
self.contains_except(&self.normalize(w), self.no_suggest)
}
fn contains_excluding(&self, w: &str, excluded: u128) -> bool {
/// `w` after the `ICONV` conversions (at each position the longest match wins) and without
/// the `IGNORE` characters.
fn normalize<'a>(&self, w: &'a str) -> std::borrow::Cow<'a, str> {
if self.iconv.is_empty() && self.ignore.is_empty() {
return std::borrow::Cow::Borrowed(w);
}
let mut out = String::with_capacity(w.len());
let mut rest = w;
while let Some(c) = rest.chars().next() {
let hit = self.iconv.iter().filter(|(from, _)| !from.is_empty() && rest.starts_with(from.as_str())).max_by_key(|(from, _)| from.len());
match hit {
Some((from, to)) => {
out.extend(to.chars().filter(|c| !self.ignore.contains(c)));
rest = rest.get(from.len()..).unwrap_or_default();
}
None => {
if !self.ignore.contains(&c) {
out.push(c);
}
rest = rest.get(c.len_utf8()..).unwrap_or_default();
}
}
}
std::borrow::Cow::Owned(out)
}
/// [`AffixDict::contains`] on normalized input, where stems carrying a flag in `deny` don't
/// count (case variants of `KEEPCASE` stems).
fn contains_except(&self, w: &str, deny: Mask) -> bool {
if w.is_empty() || w.len() > MAX_WORD_BYTES {
return false;
}
if self.stem_flags_excluding(w, excluded).is_some() || self.suffixed(w, 0, excluded) {
if self.stem_flags_except(w, deny).is_some() || self.suffixed(w, Mask::EMPTY, deny) {
return true;
}
let mut root = String::with_capacity(w.len() + 8);
@@ -680,13 +850,13 @@ impl AffixDict {
if !r.cond_matches(&root, false) {
continue;
}
if self.stem_flags_excluding(&root, excluded).is_some_and(|f| f & r.bit != 0) {
if self.stem_flags_except(&root, deny).is_some_and(|f| f.meets(r.bit)) {
return true;
}
if self.form_has_flag(&root, r.bit, INNER_CHAIN, excluded) {
if self.form_has_flag(&root, r.bit, INNER_CHAIN, deny) {
return true;
}
if r.cross && self.suffixed(&root, r.bit, excluded) {
if r.cross && self.suffixed(&root, r.bit, deny) {
return true;
}
}
@@ -695,7 +865,7 @@ impl AffixDict {
/// Is `w` a stem with one suffix? With `need` set (a prefix was taken off), the stem must
/// carry those flags too and the suffix must allow cross products.
fn suffixed(&self, w: &str, need: u128, excluded: u128) -> bool {
fn suffixed(&self, w: &str, need: Mask, deny: Mask) -> bool {
let mut root = String::with_capacity(w.len() + 8);
// Each split point from the end: the appended string is `w[b..]`, at most the longest
// suffix, and something must stay before it.
@@ -710,18 +880,18 @@ impl AffixDict {
let rules = || g.rules.iter().filter_map(|&i| self.suffixes.get(i as usize));
// The stem path: `root` is a stem carrying the rule's flag (and the prefix's,
// when a cross product is being checked).
if let Some(f) = self.stem_flags_excluding(&root, excluded)
&& f & need == need
&& rules().any(|r| f & r.bit != 0 && (need == 0 || r.cross) && r.cond_matches(&root, true))
if let Some(f) = self.stem_flags_except(&root, deny)
&& f.covers(need)
&& rules().any(|r| f.meets(r.bit) && (need.is_empty() || r.cross) && r.cond_matches(&root, true))
{
return true;
}
// Continuation classes: `root` need not be a stem carrying this rule's flag —
// it may be a form that gained the flag from another rule's continuation. The
// matching rules' flags are looked for together, in one walk.
if need == 0 {
let flags = rules().filter(|r| r.cond_matches(&root, true)).fold(0u128, |m, r| m | r.bit);
if flags != 0 && self.form_has_flag(&root, flags, INNER_CHAIN, excluded) {
if need.is_empty() {
let flags = rules().filter(|r| r.cond_matches(&root, true)).fold(Mask::EMPTY, |m, r| m | r.bit);
if !flags.is_empty() && self.form_has_flag(&root, flags, INNER_CHAIN, deny) {
return true;
}
}
@@ -732,26 +902,29 @@ impl AffixDict {
/// Is `w` correct, Hunspell style: as written, or — for a capitalized word or one in capitals
/// — in lower case (and capitalized), since a sentence may start with any word and headings
/// may be in capitals. Lower-case input never matches a capitalized stem (`warszawa`).
/// may be in capitals. Lower-case input never matches a capitalized stem (`warszawa`), and a
/// `KEEPCASE` stem matches only as written.
pub fn check(&self, w: &str) -> bool {
self.check_excluding(w, self.forbidden)
self.check_except(w, Mask::EMPTY)
}
/// Whether a suggestion is correct in its displayed case and has an eligible stem.
pub fn can_suggest(&self, w: &str) -> bool {
self.check_excluding(w, self.forbidden | self.no_suggest)
self.check_except(w, self.no_suggest)
}
fn check_excluding(&self, w: &str, excluded: u128) -> bool {
if self.contains_excluding(w, excluded) {
fn check_except(&self, w: &str, deny: Mask) -> bool {
let w = self.normalize(w);
if self.contains_except(&w, deny) {
return true;
}
match casing(w) {
let deny = deny | self.keepcase;
match casing(&w) {
Casing::Lower | Casing::Mixed => false,
Casing::Initial => self.contains_excluding(&w.to_lowercase(), excluded),
Casing::Initial => self.contains_except(&w.to_lowercase(), deny),
Casing::Upper => {
let lower = w.to_lowercase();
self.contains_excluding(&capitalize(&lower), excluded) || self.contains_excluding(&lower, excluded)
self.contains_except(&capitalize(&lower), deny) || self.contains_except(&lower, deny)
}
}
}
@@ -766,11 +939,11 @@ impl AffixDict {
let mut cross_sfx = Vec::new();
// Suffix forms queued with the continuation classes they carry, for chained expansion;
// each (form, classes) pair is expanded once.
let mut chain: Vec<(String, u128)> = Vec::new();
let mut queued: std::collections::HashSet<(String, u128)> = std::collections::HashSet::new();
let mut chain: Vec<(String, Mask)> = Vec::new();
let mut queued: std::collections::HashSet<(String, Mask)> = std::collections::HashSet::new();
let mut work = 0usize;
for r in &self.suffixes {
if f & r.bit == 0 || !r.cond_matches(stem, true) {
if !f.meets(r.bit) || !r.cond_matches(stem, true) {
continue;
}
let Some(base) = stem.strip_suffix(&*r.strip) else { continue };
@@ -778,7 +951,7 @@ impl AffixDict {
if r.cross {
cross_sfx.push(form.clone());
}
if r.cont != 0 && queued.insert((form.clone(), r.cont)) {
if !r.cont.is_empty() && queued.insert((form.clone(), r.cont)) {
chain.push((form.clone(), r.cont));
}
out.push(form);
@@ -804,7 +977,7 @@ impl AffixDict {
}
let Some(base) = form.strip_suffix(&*r.strip) else { continue };
let next = format!("{base}{}", r.add);
if r.cont != 0 && queued.insert((next.clone(), r.cont)) {
if !r.cont.is_empty() && queued.insert((next.clone(), r.cont)) {
chain.push((next.clone(), r.cont));
}
out.push(next);
@@ -815,7 +988,7 @@ impl AffixDict {
}
}
for r in &self.prefixes {
if f & r.bit == 0 || !r.cond_matches(stem, false) {
if !f.meets(r.bit) || !r.cond_matches(stem, false) {
continue;
}
let Some(rest) = stem.strip_prefix(&*r.strip) else { continue };
@@ -839,11 +1012,11 @@ impl AffixDict {
}
/// The suffix rules carrying any flag of `mask`, in rule order.
fn suffixes_with(&self, mask: u128) -> Vec<u32> {
fn suffixes_with(&self, mask: Mask) -> Vec<u32> {
let mut rules: Vec<u32> = Vec::new();
let mut lists = 0;
for (bit, list) in self.sfx_by_flag.iter().enumerate() {
if bit < 128 && mask & (1u128 << bit) != 0 && !list.is_empty() {
if mask.meets(Mask::bit(bit)) && !list.is_empty() {
rules.extend_from_slice(list);
lists += 1;
}
@@ -904,13 +1077,19 @@ impl AffixDict {
for (a, b) in &self.rep {
out.push_str(&format!("REP {} {}\n", a.replace(' ', "_"), b.replace(' ', "_")));
}
for (directive, mask) in [("FORBIDDENWORD", self.forbidden), ("NOSUGGEST", self.no_suggest)] {
if mask != 0
for (directive, mask) in [("FORBIDDENWORD", self.forbidden), ("NOSUGGEST", self.no_suggest), ("KEEPCASE", self.keepcase)] {
if !mask.is_empty()
&& let Some(c) = self.flag_char(mask)
{
out.push_str(&format!("{directive} {c}\n"));
}
}
for (a, b) in &self.iconv {
out.push_str(&format!("ICONV {a} {b}\n"));
}
if !self.ignore.is_empty() {
out.push_str(&format!("IGNORE {}\n", self.ignore.iter().collect::<String>()));
}
let z = |s: &str| if s.is_empty() { "0".to_string() } else { s.to_string() };
for (kind, rules) in [("PFX", &self.prefixes), ("SFX", &self.suffixes)] {
for r in rules {
@@ -934,8 +1113,8 @@ impl AffixDict {
let p = prev.iter().zip(b).take_while(|(x, y)| x == y).count().min(255);
raw.push(p as u8);
raw.extend_from_slice(b.get(p..).unwrap_or_default());
let mask = self.flags.get(i).and_then(|&id| self.sets.get(usize::from(id))).copied().unwrap_or(0);
if mask != 0 {
let mask = self.flags.get(i).and_then(|&id| self.sets.get(usize::from(id))).copied().unwrap_or_default();
if !mask.is_empty() {
raw.push(b'/');
raw.extend_from_slice(self.flag_chars_for(mask).as_bytes());
}
@@ -962,26 +1141,23 @@ impl AffixDict {
comments.push(c.to_string());
continue;
}
let f: Vec<&str> = line.split_whitespace().collect();
let f: Vec<&str> = split_fields(line).collect();
let at = |k: usize| f.get(k).copied().unwrap_or("");
match at(0) {
"TRY" => rules.try_chars = cap_try(at(1)),
"KEY" => rules.key = at(1).to_string(),
"MAP" => rules.map.push(at(1).chars().collect()),
"REP" => rules.rep.push((at(1).replace('_', " "), at(2).replace('_', " "))),
directive @ ("FORBIDDENWORD" | "NOSUGGEST") => {
let mut it = at(1).chars();
match (it.next(), it.next()) {
(Some(c), None) if !c.is_whitespace() => {
if directive == "FORBIDDENWORD" {
rules.forbidden = Some(c);
} else {
rules.no_suggest = Some(c);
}
}
_ => return Err(format!("affix dictionary: bad {directive} in `{line}`")),
directive @ ("FORBIDDENWORD" | "NOSUGGEST" | "KEEPCASE") => {
let c = one_flag(at(1), directive, 0).map_err(|_| format!("affix dictionary: bad {directive} in `{line}`"))?;
match directive {
"FORBIDDENWORD" => rules.forbidden = Some(c),
"NOSUGGEST" => rules.no_suggest = Some(c),
_ => rules.keepcase = Some(c),
}
}
"ICONV" if f.len() == 3 => rules.iconv.push((at(1).to_string(), at(2).to_string())),
"IGNORE" => rules.ignore = at(1).chars().collect(),
"MAP" => rules.map.push(at(1).chars().collect()),
"REP" => rules.rep.push((at(1).replace('_', " "), at(2).replace('_', " "))),
kind @ ("PFX" | "SFX") => {
let mut fc = at(1).chars();
let (Some(flag), None) = (fc.next(), fc.next()) else { return Err(format!("affix dictionary: bad flag in `{line}`")) };
@@ -1223,7 +1399,7 @@ pub(crate) mod tests {
#[test]
fn more_than_64_distinct_flags_load() {
// VERO pt_BR uses 106 distinct flags (FLAG UTF-8). Build one with every basic Latin
// letter, the digits and eight accented letters: 70 flags, 64 < n <= 128.
// letter, the digits and eight accented letters: 70 flags, more than 64.
let mut flags = String::new();
let mut aff = String::from("SET UTF-8\nFLAG UTF-8\n");
let chars: Vec<char> = ('a'..='z').chain('A'..='Z').chain('0'..='9').chain(['á', 'é', 'í', 'ó', 'ú', 'ç', 'ã', 'õ']).collect();
@@ -1239,6 +1415,24 @@ pub(crate) mod tests {
assert!(!d.contains("verboß"), "ß is not a flag and not a stem");
}
#[test]
fn more_than_128_distinct_flags_load() {
// The Hungarian dictionary uses 158 single-byte flags: 200 flags here, U+0080 up.
let flags: String = (0x80u32..0x80 + 200).filter_map(char::from_u32).collect();
let mut aff = String::from("SET UTF-8\n");
for (i, c) in flags.chars().enumerate() {
aff.push_str(&format!("SFX {c} N 1\nSFX {c} 0 x{i} .\n"));
}
let half: String = flags.chars().step_by(2).collect();
let d = AffixDict::from_hunspell(&aff, &format!("1\nszó/{half}\n"), &[]).unwrap();
assert!(d.contains("szóx0") && d.contains("szóx198") && d.contains("szóx130"));
assert!(!d.contains("szóx1") && !d.contains("szóx199") && !d.contains("szóx131"));
let back = AffixDict::from_bytes(&d.to_bytes()).unwrap();
assert!(back.contains("szóx198") && !back.contains("szóx199"));
let too_many: String = (0x100u32..0x100 + 257).filter_map(char::from_u32).collect();
assert!(AffixDict::from_hunspell("SET UTF-8\n", &format!("1\nszó/{too_many}\n"), &[]).is_err());
}
#[test]
fn forbidden_words_are_rejected() {
// VERO pt_BR marks spellings it must reject (flag ý under FLAG UTF-8) as stems carrying
@@ -1364,6 +1558,52 @@ pub(crate) mod tests {
assert!(d.contains("kot") && d.contains("koty"));
}
#[test]
fn flag_aliases_stand_for_flag_sets() {
// `AF`: `/1` in the `.dic` and in a continuation class means the first alias's flags.
// Morphological aliases (`AM`) and the fields after a stem are skipped.
let aff = "SET UTF-8\nAF 2\nAF ab # 1\nAF c # 2\nAM 1\nAM po:noun\nSFX a Y 1\nSFX a 0 ok/2 . 1\nSFX b Y 1\nSFX b 0 at . 1\nSFX c Y 1\nSFX c 0 ban . 1\n";
let d = AffixDict::from_hunspell(aff, "1\nház/1\t1\n", &[]).unwrap();
assert!(d.contains("ház") && d.contains("házok") && d.contains("házat") && d.contains("házokban"));
assert!(!d.contains("házban"), "the stem has a and b, not c");
assert!(AffixDict::from_hunspell(aff, "1\nház/3\n", &[]).is_err(), "alias 3 is not defined");
let back = AffixDict::from_bytes(&d.to_bytes()).unwrap();
assert!(back.contains("házokban") && !back.contains("házban"), "aliases are expanded in the compact format");
}
#[test]
fn iconv_and_ignore_normalize_input() {
// Ligatures read as their letters; `IGNORE` characters vanish from input and stems.
let aff = "SET UTF-8\nICONV 2\nICONV fi fi\nICONV ffl ffl\nIGNORE ()\nSFX a Y 1\nSFX a 0 s .\n";
let d = AffixDict::from_hunspell(aff, "2\nfine/a\nwaf(fle)/a\n", &[]).unwrap();
assert!(d.check("fine") && d.check("fines") && d.check("Fine") && d.contains("fine"));
assert!(d.check("waffle") && d.check("waffles") && d.check("wa(ffle)"));
assert!(!d.check("finer"));
let back = AffixDict::from_bytes(&d.to_bytes()).unwrap();
assert!(back.check("fines") && back.check("waffle") && back.check("waf(fle)"));
}
#[test]
fn keepcase_stems_match_only_as_written() {
let aff = "SET UTF-8\nKEEPCASE k\nSFX a Y 1\nSFX a 0 s .\n";
let d = AffixDict::from_hunspell(aff, "3\npH/k\nkm/ka\nkot/a\n", &[]).unwrap();
assert!(d.check("pH") && d.check("km") && d.check("kms"));
assert!(!d.check("PH") && !d.check("Km") && !d.check("KM") && !d.check("Kms"));
assert!(d.check("Kot") && d.check("KOTS"), "other stems still follow the sentence and heading case");
let back = AffixDict::from_bytes(&d.to_bytes()).unwrap();
assert!(back.check("km") && !back.check("Km"));
}
#[test]
fn fields_split_on_spaces_and_tabs_only() {
// A flag byte read as U+00A0 (no-break space) or U+0085 is a flag, not a separator.
let aff = "SET UTF-8\nSFX \u{a0} Y 1\nSFX \u{a0} 0 y .\nSFX \u{85} Y 1\nSFX \u{85} 0 em .\n";
let d = AffixDict::from_hunspell(aff, "1\nkot/\u{a0}\u{85}\n", &[]).unwrap();
assert!(d.contains("koty") && d.contains("kotem"));
let back = AffixDict::from_bytes(&d.to_bytes()).unwrap();
assert!(back.contains("koty") && back.contains("kotem"));
}
#[test]
fn unsupported_and_broken_input_is_an_error() {
assert!(AffixDict::from_hunspell("FLAG long\n", DIC, &[]).is_err());
crates/proof/src/hunspell.rs
+183 -3
View File
@@ -218,13 +218,27 @@ fn decode_high(bytes: &[u8], high: &str) -> String {
/// (UTF-8 when there is none; ISO 8859-2 is the other one Polish dictionaries use, ISO 8859-1 the
/// one the original Portuguese VERO uses, ISO 8859-9 the one older Turkish dictionaries use, KOI8-R
/// and Windows-1251 the ones older Russian dictionaries use).
///
/// Under a UTF-8 `SET` without `FLAG UTF-8`, flags are still single bytes, as in Hunspell: the
/// Hungarian dictionary writes its affix flags and flag aliases as bytes above 0x7F that aren't
/// UTF-8. Those fields are read byte by byte (each byte becomes U+0000..U+00FF, one flag per
/// byte); the words, affixes and conditions around them are UTF-8.
pub fn decode(aff: &[u8], dic: &[u8]) -> Result<(String, String), String> {
let head = String::from_utf8_lossy(aff.get(..aff.len().min(8192)).unwrap_or_default()).to_string();
let set = head.lines().find_map(|l| l.trim_start_matches('\u{feff}').strip_prefix("SET ")).map(str::trim).unwrap_or("UTF-8").to_ascii_uppercase();
let directive = |name: &str| {
head.lines().find_map(|l| {
let l = l.trim_start_matches('\u{feff}');
l.strip_prefix(name).filter(|rest| rest.starts_with([' ', '\t'])).map(|rest| rest.trim().to_ascii_uppercase())
})
};
let set = directive("SET").unwrap_or_else(|| "UTF-8".into());
match set.as_str() {
"UTF-8" | "UTF8" => {
let text = |b: &[u8], what: &str| String::from_utf8(b.to_vec()).map_err(|e| format!("{what}: not UTF-8: {e}"));
Ok((text(aff, ".aff")?, text(dic, ".dic")?))
if matches!(directive("FLAG").as_deref(), Some("UTF-8" | "UTF8")) {
let text = |b: &[u8], what: &str| String::from_utf8(b.to_vec()).map_err(|e| format!("{what}: not UTF-8: {e}"));
return Ok((text(aff, ".aff")?, text(dic, ".dic")?));
}
Ok((decode_utf8_aff(aff)?, decode_utf8_dic(dic)?))
}
"ISO8859-1" | "ISO-8859-1" => Ok((decode_latin1(aff), decode_latin1(dic))),
"ISO8859-2" | "ISO-8859-2" => Ok((decode_latin2(aff), decode_latin2(dic))),
@@ -235,6 +249,153 @@ pub fn decode(aff: &[u8], dic: &[u8]) -> Result<(String, String), String> {
}
}
/// Directives whose one argument is a flag.
const FLAG_DIRECTIVES: &[&str] = &[
"CIRCUMFIX",
"COMPOUNDBEGIN",
"COMPOUNDEND",
"COMPOUNDFLAG",
"COMPOUNDFORBIDFLAG",
"COMPOUNDLAST",
"COMPOUNDFIRST",
"COMPOUNDMIDDLE",
"COMPOUNDPERMITFLAG",
"COMPOUNDROOT",
"FORBIDDENWORD",
"FORCEUCASE",
"GENERATE",
"HU_KOTOHANGZO",
"KEEPCASE",
"LEMMA_PRESENT",
"NEEDAFFIX",
"NOSUGGEST",
"ONLYINCOMPOUND",
"ONLYROOT",
"PSEUDOROOT",
"SUBSTANDARD",
"WARN",
];
/// Directives that only describe the dictionary; a non-UTF-8 byte in them (the Hungarian
/// dictionary's `NAME` is ISO 8859-2) is replaced instead of failing the load.
const METADATA_DIRECTIVES: &[&str] = &["NAME", "HOME", "VERSION", "LANG", "AM"];
/// One byte per char: a flag field read as single-byte flags.
fn bytes_as_flags(b: &[u8]) -> String {
b.iter().map(|&x| char::from(x)).collect()
}
/// `text/flags`: the text as UTF-8 and the flags byte by byte (`\/` is a literal slash).
fn text_and_flags(field: &[u8], what: &str) -> Result<String, String> {
let mut slash = None;
let mut i = 0;
while let Some(&b) = field.get(i) {
if b == b'\\' && field.get(i + 1) == Some(&b'/') {
i += 2;
continue;
}
if b == b'/' {
slash = Some(i);
break;
}
i += 1;
}
let (text, flags) = match slash {
Some(k) => (field.get(..k).unwrap_or_default(), field.get(k..).unwrap_or_default()),
None => (field, &[][..]),
};
let mut out = std::str::from_utf8(text).map_err(|e| format!("{what}: not UTF-8: {e}"))?.to_string();
out.push_str(&bytes_as_flags(flags));
Ok(out)
}
/// Split a line into fields on spaces and tabs, keeping the separators (so a line rebuilt from
/// its parts is the original).
fn fields(line: &[u8]) -> Vec<&[u8]> {
let mut out = Vec::new();
let mut start = 0;
let mut in_sep = line.first().is_some_and(|b| matches!(b, b' ' | b'\t'));
for (i, b) in line.iter().enumerate() {
let sep = matches!(b, b' ' | b'\t');
if sep != in_sep {
out.push(line.get(start..i).unwrap_or_default());
start = i;
in_sep = sep;
}
}
out.push(line.get(start..).unwrap_or_default());
out
}
/// A UTF-8 `.aff` with single-byte flags (see [`decode`]).
fn decode_utf8_aff(aff: &[u8]) -> Result<String, String> {
let mut out = String::with_capacity(aff.len());
for (n, line) in aff.split(|&b| b == b'\n').enumerate() {
if n > 0 {
out.push('\n');
}
if let Ok(s) = std::str::from_utf8(line)
&& s.is_ascii()
{
out.push_str(s);
continue;
}
let parts = fields(line);
// Field index among the non-separator parts.
let words: Vec<usize> =
parts.iter().enumerate().filter(|(_, p)| !p.first().is_some_and(|b| matches!(b, b' ' | b'\t'))).map(|(i, _)| i).collect();
let dir = words
.first()
.and_then(|&i| parts.get(i))
.map(|p| String::from_utf8_lossy(p).trim_start_matches('\u{feff}').to_string())
.unwrap_or_default();
let what = format!(".aff line {}", n + 1);
if dir.starts_with('#') || METADATA_DIRECTIVES.contains(&dir.as_str()) {
out.push_str(&String::from_utf8_lossy(line));
continue;
}
// A trailing `# comment` (the Hungarian `AF` lines number themselves) is never a word or
// a flag; from there on the line is kept as text.
let comment_at = words.iter().skip(1).position(|&w| parts.get(w).is_some_and(|f| f.first() == Some(&b'#'))).map(|k| k + 1);
for (i, p) in parts.iter().enumerate() {
let field = words.iter().position(|&w| w == i);
let in_comment = comment_at.is_some_and(|c| words.get(c).is_some_and(|&w| i >= w));
let decoded = match (dir.as_str(), field) {
_ if in_comment => String::from_utf8_lossy(p).into_owned(),
("AF" | "PFX" | "SFX" | "COMPOUNDRULE", Some(1)) => bytes_as_flags(p),
("PFX" | "SFX", Some(3)) | ("CHECKCOMPOUNDPATTERN", Some(1 | 2)) => text_and_flags(p, &what)?,
(d, Some(1)) if FLAG_DIRECTIVES.contains(&d) => bytes_as_flags(p),
_ => std::str::from_utf8(p).map_err(|e| format!("{what}: not UTF-8: {e}"))?.to_string(),
};
out.push_str(&decoded);
}
}
Ok(out)
}
/// A UTF-8 `.dic` with single-byte flags: the word is UTF-8, the flags after `/` are bytes.
fn decode_utf8_dic(dic: &[u8]) -> Result<String, String> {
let mut out = String::with_capacity(dic.len());
for (n, line) in dic.split(|&b| b == b'\n').enumerate() {
if n > 0 {
out.push('\n');
}
if let Ok(s) = std::str::from_utf8(line)
&& s.split_once('/').is_none_or(|(_, flags)| flags.is_ascii())
{
out.push_str(s);
continue;
}
let what = format!(".dic line {}", n + 1);
// The first field is `word/flags`; morphological fields after it are UTF-8 text.
let end = line.iter().position(|b| matches!(b, b' ' | b'\t')).unwrap_or(line.len());
out.push_str(&text_and_flags(line.get(..end).unwrap_or_default(), &what)?);
let rest = line.get(end..).unwrap_or_default();
out.push_str(std::str::from_utf8(rest).map_err(|e| format!("{what}: not UTF-8: {e}"))?);
}
Ok(out)
}
#[cfg(test)]
mod tests {
use super::*;
@@ -280,6 +441,25 @@ mod tests {
assert!(decode(b"SET ISO-8859-1\n", b"\xe7").is_ok());
}
#[test]
fn utf8_dictionaries_keep_single_byte_flags() {
// The Hungarian dictionary is `SET UTF-8` but writes flags as single bytes above 0x7F,
// which aren't UTF-8; each byte is one flag. Its `NAME` line is ISO 8859-2.
let aff = b"SET UTF-8\nNAME Magyar helyes\xedr\xe1si\nAF 1\nAF \xd5a # 1\nSFX \xd5 Y 1\nSFX \xd5 0 ok/\xe9 . 7\nSFX \xe9 Y 1\nSFX \xe9 0 ban .\nSFX a Y 1\nSFX a 0 at .\n";
let dic = "2\nház/1\t1\nkert/\u{e9}\n".as_bytes();
let (a, d) = decode(aff, dic).unwrap();
assert!(a.contains("AF \u{d5}a # 1") && a.contains("SFX \u{d5} 0 ok/\u{e9} . 7"));
assert!(d.contains("kert/\u{c3}\u{a9}"), "a UTF-8 é in the flags is two one-byte flags");
let dict = crate::affix::AffixDict::from_hunspell(&a, &d, &[]).unwrap();
assert!(dict.check("ház") && dict.check("házok") && dict.check("házokban") && dict.check("házat"));
assert!(!dict.check("házban"));
// `FLAG UTF-8` keeps the whole file UTF-8, and words must still be UTF-8.
let (_, d) = decode(b"SET UTF-8\nFLAG UTF-8\n", "kert/\u{e9}\n".as_bytes()).unwrap();
assert_eq!(d, "kert/\u{e9}\n");
assert!(decode(b"SET UTF-8\nSFX a Y 1\nSFX a 0 \xe9 .\n", b"").is_err(), "an affix must be UTF-8");
assert!(decode(b"SET UTF-8\n", b"1\nh\xe1z/a\n").is_err(), "a stem must be UTF-8");
}
#[test]
fn decodes_russian_single_byte_encodings() {
// `ёлка` in KOI8-R (0xA3 0xCC 0xCB 0xC1) and in Windows-1251 (0xB8 0xEB 0xEA 0xE0).