diff --git a/crates/proof/src/affix.rs b/crates/proof/src/affix.rs index ba0229ea..c9b8d339 100644 --- a/crates/proof/src/affix.rs +++ b/crates/proof/src/affix.rs @@ -17,7 +17,7 @@ //! flags), `PFX`/`SFX` with strip, append and condition (`.`, letters, `[abc]`, `[^abc]`), cross //! products, continuation classes on suffixes (at most two suffixes in a row, as in Hunspell), //! `FORBIDDENWORD`, `NOSUGGEST`, `KEEPCASE` (the stem is a word only in its own letter case), -//! `NEEDAFFIX`, `CIRCUMFIX`, `FULLSTRIP`, input conversion (`ICONV`: ligatures such as `fi` read as +//! `NEEDAFFIX`, `CIRCUMFIX`, `FULLSTRIP`, `CHECKSHARPS` (German `SS` for `ß` in capitals), input conversion (`ICONV`: ligatures such as `fi` read as //! `fi`), output conversion of suggestions (`OCONV`) and ignored characters (`IGNORE`), and the //! suggestion tables `TRY`, `KEY`, `MAP` and `REP`. //! Compounds by `COMPOUNDRULE` (patterns of flags with `*` and `?`, as SCOWL's English @@ -77,7 +77,6 @@ const MAX_DISTINCT_FLAGS: usize = 1 << 16; /// Directives that change which words are correct in ways this engine doesn't implement. const UNSUPPORTED: &[&str] = &[ - "CHECKSHARPS", "COMPLEXPREFIXES", "COMPOUNDBEGIN", "COMPOUNDEND", @@ -208,6 +207,9 @@ fn parse_cond(text: &str) -> Result, String> { const MAX_CHAIN: u8 = 2; /// How many suffixes may sit under the outermost one: the depth `form_has_flag` walks. const INNER_CHAIN: u8 = MAX_CHAIN - 1; +/// Most `ss` pairs of a word in capitals that CHECKSHARPS reads as `ß` (each combination is tried: +/// at most 31 variants). +const MAX_SHARPS: usize = 5; /// Most forms [`AffixDict::forms`] lists for one stem (real stems have a few hundred at most). pub const MAX_FORMS: usize = 4096; /// Most rules [`AffixDict::forms`] tries along continuation chains and prefix–suffix cross @@ -305,6 +307,9 @@ pub struct AffixDict { circumfix: Option, /// `FULLSTRIP`: an affix may strip (and so replace) a whole stem. full_strip: bool, + /// `CHECKSHARPS`: in a word in capitals, `SS` may stand for `ß` (`STRASSE` for `Straße`), and + /// a `KEEPCASE` word with `ß` may be capitalized and written in capitals with `SS`. + check_sharps: bool, /// `ICONV`: input conversions applied before a lookup. iconv: Conv, /// `OCONV`: output conversions applied to suggestions. @@ -417,9 +422,15 @@ pub enum Casing { } /// The casing of `w`. +/// +/// `ß` has no capital of its own in Hunspell's tables and counts as neither case, as there: +/// `STRAßE` is in capitals. pub fn casing(w: &str) -> Casing { let (mut upper, mut lower, mut first_upper) = (0usize, 0usize, false); for (i, c) in w.chars().enumerate() { + if c == 'ß' { + continue; + } if c.is_uppercase() { upper += 1; if i == 0 { @@ -467,6 +478,7 @@ struct Rules { need_affix: Option, circumfix: Option, full_strip: bool, + check_sharps: bool, iconv: Vec<(String, String)>, oconv: Vec<(String, String)>, ignore: Vec, @@ -873,6 +885,7 @@ fn parse_aff(aff: &str, english: bool) -> Result { "NEEDAFFIX" => r.need_affix = Some(r.one_flag(at(1)).map_err(|e| format!("line {}: {dir} needs one flag: {e}", n + 1))?), "CIRCUMFIX" => r.circumfix = Some(r.one_flag(at(1)).map_err(|e| format!("line {}: {dir} needs one flag: {e}", n + 1))?), "FULLSTRIP" => r.full_strip = true, + "CHECKSHARPS" => r.check_sharps = true, "FLAG" => r.set_mode(at(1)).map_err(|e| format!("line {}: {e}", n + 1))?, "FORBIDDENWORD" => r.forbidden = Some(r.one_flag(at(1)).map_err(|e| format!("line {}: {dir} needs one flag: {e}", n + 1))?), "NOSUGGEST" => r.no_suggest = Some(r.one_flag(at(1)).map_err(|e| format!("line {}: {dir} needs one flag: {e}", n + 1))?), @@ -1087,6 +1100,7 @@ impl AffixDict { need_affix: rules.need_affix.and_then(id_of), circumfix: rules.circumfix.and_then(id_of), full_strip: rules.full_strip, + check_sharps: rules.check_sharps, flag_mode: mode, iconv: Conv::new(rules.iconv), oconv: Conv::new(rules.oconv), @@ -1628,17 +1642,53 @@ impl AffixDict { if self.contains_except(&w, excluded) { return true; } - let excluded = excluded.and(self.keepcase); + // Over this no case variant can be a word either. + if w.len() > MAX_INPUT_BYTES { + return false; + } + let keep = excluded.and(self.keepcase); match casing(&w) { Casing::Lower | Casing::Mixed => false, - Casing::Initial => self.contains_except(&w.to_lowercase(), excluded), + Casing::Initial => { + let lower = w.to_lowercase(); + // CHECKSHARPS: a KEEPCASE word with `ß` may be capitalized (`Gruß` for `Gruß/k`). + let ex = if self.check_sharps && lower.contains('ß') { excluded } else { keep }; + self.contains_except(&lower, ex) + } Casing::Upper => { let lower = w.to_lowercase(); - self.contains_except(&capitalize(&lower), excluded) || self.contains_except(&lower, excluded) + self.contains_except(&capitalize(&lower), keep) + || self.contains_except(&lower, keep) + || (self.check_sharps && self.sharps(&lower, excluded)) } } } + /// CHECKSHARPS: is `lower` (a word in capitals, lowercased) a word with some of its `ss` read + /// as `ß`, in lower case or capitalized? Such a form may come from a `KEEPCASE` stem (in + /// capitals a word with `ß` is written with `SS`). The first [`MAX_SHARPS`] `ss` pairs are + /// tried, in every combination. + fn sharps(&self, lower: &str, excluded: Excluded) -> bool { + let at: Vec = lower.match_indices("ss").map(|(i, _)| i).take(MAX_SHARPS).collect(); + let mut variant = String::with_capacity(lower.len()); + for pick in 1u32..(1 << at.len()) { + variant.clear(); + let mut from = 0; + for (k, &i) in at.iter().enumerate() { + if pick & (1 << k) != 0 { + variant.push_str(lower.get(from..i).unwrap_or_default()); + variant.push('ß'); + from = i + 2; + } + } + variant.push_str(lower.get(from..).unwrap_or_default()); + if self.contains_except(&variant, excluded) || self.contains_except(&capitalize(&variant), excluded) { + return true; + } + } + false + } + /// Every form of the stem `stem` (itself included), unsorted, possibly with repeats; at most /// [`MAX_FORMS`], and at most [`MAX_FORMS_WORK`] rules tried along the continuation chains and /// cross products, so a dictionary with thousands of self-continuing rules can't make one stem @@ -1775,6 +1825,11 @@ impl AffixDict { rules } + /// Whether the dictionary sets `CHECKSHARPS` (`SS` may stand for `ß` in capitals). + pub(crate) fn check_sharps(&self) -> bool { + self.check_sharps + } + /// The `TRY` letters, most frequent first (suggestions try these). pub fn try_chars(&self) -> &str { &self.try_chars @@ -1838,6 +1893,9 @@ impl AffixDict { if self.full_strip { out.push_str("FULLSTRIP\n"); } + if self.check_sharps { + out.push_str("CHECKSHARPS\n"); + } for (directive, flag) in [ ("FORBIDDENWORD", self.forbidden), ("NOSUGGEST", self.no_suggest), @@ -1946,6 +2004,7 @@ impl AffixDict { "TRY" => rules.try_chars = cap_try(at(1)), "KEY" => rules.key = at(1).to_string(), "FULLSTRIP" => rules.full_strip = true, + "CHECKSHARPS" => rules.check_sharps = true, directive @ ("FORBIDDENWORD" | "NOSUGGEST" | "SUBSTANDARD" | "KEEPCASE" | "ONLYINCOMPOUND" | "NEEDAFFIX" | "CIRCUMFIX") => { let c = rules.one_flag(at(1)).map_err(|e| format!("affix dictionary: bad {directive} in `{line}`: {e}"))?; match directive { @@ -2505,6 +2564,50 @@ pub(crate) mod tests { } } + /// CHECKSHARPS answers as Hunspell 1.7 gives them on the same files (`*` correct, `&` not). + #[test] + fn checksharps_reads_ss_as_sharp_s_in_capitals() { + let dic = "6\nstraße\nMaße\nfußball/k\nmesse\nGruß/k\nSchloss\n"; + let words = [ + ("straße", true, true), + ("Straße", true, true), + ("STRASSE", false, true), + ("STRAßE", true, true), + ("STRAẞE", true, true), + ("Strasse", false, false), + ("strasse", false, false), + ("MASSE", false, true), + ("MAßE", true, true), + ("maße", false, false), + ("FUSSBALL", false, true), + ("fußball", true, true), + ("Fußball", false, true), + ("FUßBALL", false, false), + ("MESSE", true, true), + ("MEßE", false, false), + ("GRUSS", false, true), + ("Gruß", true, true), + ("gruß", false, false), + ("GRUß", false, false), + ("SCHLOSS", true, true), + ("SCHLOß", false, false), + ]; + for (sharps, aff) in [(false, "SET UTF-8\nKEEPCASE k\n"), (true, "SET UTF-8\nCHECKSHARPS\nKEEPCASE k\n")] { + let d = AffixDict::from_hunspell(aff, dic, &[]).unwrap(); + let back = AffixDict::from_bytes(&d.to_bytes().unwrap()).unwrap(); + for (w, without, with) in words { + let want = if sharps { with } else { without }; + assert_eq!(d.check(w), want, "{w}, CHECKSHARPS {sharps}"); + assert_eq!(back.check(w), want, "{w}, CHECKSHARPS {sharps}, compact"); + } + } + assert_eq!(casing("STRAßE"), Casing::Upper); + assert_eq!(casing("Maß"), Casing::Initial); + // Many `SS` pairs: only the first MAX_SHARPS are varied, and a long word costs nothing. + let d = AffixDict::from_hunspell("CHECKSHARPS\n", "1\naßaßaßaßaß\n", &[]).unwrap(); + assert!(d.check("ASSASSASSASSASS") && !d.check(&"SS".repeat(60))); + } + #[test] fn oconv_converts_suggestions_only() { // Answers as Hunspell 1.7 gives them on the same files: `kbt` → `k’t`. diff --git a/crates/proof/src/affix_suggest.rs b/crates/proof/src/affix_suggest.rs index adf35265..b20dfb93 100644 --- a/crates/proof/src/affix_suggest.rs +++ b/crates/proof/src/affix_suggest.rs @@ -319,21 +319,37 @@ pub(crate) fn suggest_in(d: &AffixDict, word: &str, max: usize, t: &Tuning) -> V let diff_at = |text: &str| chars.iter().zip(text.chars()).position(|(a, b)| *a != b).unwrap_or(n); found.sort_by_key(|(text, (cost, seq))| (*cost, diff_at(text), *seq)); let mut out: Vec = Vec::new(); - for (text, _) in found { - let recased = match case { - Casing::Initial => capitalize(&text), - Casing::Upper if n > 1 => text.to_uppercase(), - _ => text.clone(), - }; + 'found: for (text, _) in found { // A KEEPCASE word only exists in its dictionary case: `Km` would be flagged, so offer `km`. - let t = if recased != text && text.split(' ').any(|p| d.case_locked(p)) { text } else { recased }; - // Shown as the dictionary's OCONV table writes it (`'` → `’` in French). - let t = d.output(&t).into_owned(); - if t != word && !out.contains(&t) { - out.push(t); - } - if out.len() >= max { - break; + // Under CHECKSHARPS one with `ß` may still be capitalized, or written in capitals with SS + // (`Fußball`, `FUSSBALL`), as hunspell 1.7.5 offers. + let locked = text.split(' ').any(|p| d.case_locked(p)); + let sharp = d.check_sharps() && text.contains('ß'); + let recased: Vec = match case { + Casing::Initial if !locked || sharp => vec![capitalize(&text)], + // In capitals `ß` stays `ß` (`STRAßE` is right with or without CHECKSHARPS); under + // CHECKSHARPS the SS spelling comes first, as Hunspell offers it. + Casing::Upper if n > 1 => { + let ss = text.to_uppercase(); + let kept: String = text.split('ß').map(str::to_uppercase).collect::>().join("ß"); + match (locked, sharp) { + (false, false) => vec![kept], + (false, true) => vec![ss, kept], + (true, true) => vec![ss, text.clone()], + (true, false) => vec![text.clone()], + } + } + _ => vec![text.clone()], + }; + for t in recased { + // Shown as the dictionary's OCONV table writes it (`'` → `’` in French). + let t = d.output(&t).into_owned(); + if t != word && !out.contains(&t) { + out.push(t); + } + if out.len() >= max { + break 'found; + } } } out @@ -519,6 +535,31 @@ fn scan(col: &mut Collector, chars: &[char], map: &[Vec]) { mod tests { use super::*; + /// craft-libs#490 review: a suggestion recased to capitals keeps `ß` (`STRAßE`, right with or + /// without CHECKSHARPS); under CHECKSHARPS the SS form comes first, and a KEEPCASE `ß` word may + /// be capitalized or written in capitals with SS. Expected lists are the hunspell 1.7.5 CLI's + /// on the same files; every suggestion is right. + #[test] + fn sharp_s_suggestions_match_hunspell() { + let try_ = "TRY esianrtolcdugmphbyfvkwzß\n"; + let cases: &[(String, &str, &str, &[&str])] = &[ + (format!("SET UTF-8\n{try_}"), "1\nstraße\n", "STRAßEE", &["STRAßE"]), + (format!("SET UTF-8\nCHECKSHARPS\n{try_}"), "1\nstraße\n", "STRAßEE", &["STRASSE", "STRAßE"]), + (format!("SET UTF-8\nCHECKSHARPS\n{try_}"), "1\nstraße\n", "Straßee", &["Straße"]), + (format!("SET UTF-8\nCHECKSHARPS\nKEEPCASE k\n{try_}"), "2\nfußball/k\nGruß/k\n", "FUSBALL", &["FUSSBALL", "fußball"]), + (format!("SET UTF-8\nCHECKSHARPS\nKEEPCASE k\n{try_}"), "2\nfußball/k\nGruß/k\n", "Fusball", &["Fußball"]), + (format!("SET UTF-8\nCHECKSHARPS\nKEEPCASE k\n{try_}"), "2\nfußball/k\nGruß/k\n", "GRUS", &["GRUSS", "Gruß"]), + ]; + for (aff, dic, word, want) in cases { + let d = AffixDict::from_hunspell(aff, dic, &[]).unwrap(); + let got = suggest_in(&d, word, 8, &Tuning::DEFAULT); + assert_eq!(&got, want, "{word} with\n{aff}"); + for g in &got { + assert!(d.check(g), "{word} → {g} is not right"); + } + } + } + /// craft-libs#489 review repros: OCONV applies to a suggestion after its case is set, case /// sensitively, as the hunspell 1.7.5 CLI answers on the same files (`KBT` → `KAT`, not /// `K’T`), and every suggestion is right.