diff --git a/crates/proof/src/affix.rs b/crates/proof/src/affix.rs index fb362c76..1cbf3d1d 100644 --- a/crates/proof/src/affix.rs +++ b/crates/proof/src/affix.rs @@ -14,7 +14,7 @@ //! Supported: one-character flags (at most 128 distinct, any Unicode character — `FLAG UTF-8`), //! `PFX`/`SFX` with strip, append and condition (`.`, letters, `[abc]`, `[^abc]`), cross //! products, continuation classes on suffixes (at most two suffixes in a row, as in Hunspell), -//! `FORBIDDENWORD`, and the suggestion tables `TRY`, `MAP` and `REP`. Rejected: compounding, +//! `FORBIDDENWORD`, `NOSUGGEST`, and the suggestion tables `TRY`, `KEY`, `MAP` and `REP`. Rejected: compounding, //! long or numeric flags, `NEEDAFFIX` and the other directives that change which words are correct. //! //! Lookups never panic: stems sit in one string, found through an open-addressing hash table, and @@ -172,6 +172,8 @@ pub struct AffixDict { sets: Vec, /// The `FORBIDDENWORD` flag's bit; stems carrying it are not words (0: none declared). forbidden: u128, + /// Stems with NOSUGGEST are correct, but none of their forms are suggested. + no_suggest: u128, /// Open-addressing hash table of stem indices (`EMPTY` = free). table: Vec, /// Flag char of each bit. @@ -183,6 +185,7 @@ pub struct AffixDict { /// Longest appended suffix, in chars. max_sfx_chars: usize, try_chars: String, + key: String, map: Vec>, rep: Vec<(String, String)>, /// Provenance and licence lines, kept through [`AffixDict::to_bytes`]. @@ -247,10 +250,12 @@ struct Rules { prefixes: Vec, suffixes: Vec, try_chars: String, + key: String, map: Vec>, rep: Vec<(String, String)>, /// The `FORBIDDENWORD` flag: stems carrying it are listed but never correct. forbidden: Option, + no_suggest: Option, } /// A rule as parsed from text, before flags become bits: `(flag, cross, strip, add, cont, cond)`. @@ -275,8 +280,8 @@ fn parse_aff(aff: &str) -> Result { } let at = |k: usize| f.get(k).copied().unwrap_or(""); match dir { - "SET" | "LANG" | "NAME" | "VERSION" | "HOME" | "WORDCHARS" | "KEY" | "MAXNGRAMSUGS" | "MAXDIFF" | "ONLYMAXDIFF" | "NOSPLITSUGS" - | "SUGSWITHDOTS" | "NOSUGGEST" | "WARN" | "FORBIDWARN" | "MAXCPDSUGS" => {} + "SET" | "LANG" | "NAME" | "VERSION" | "HOME" | "WORDCHARS" | "MAXNGRAMSUGS" | "MAXDIFF" | "ONLYMAXDIFF" | "NOSPLITSUGS" + | "SUGSWITHDOTS" | "WARN" | "FORBIDWARN" | "MAXCPDSUGS" => {} "FLAG" => { // Per-character flag modes (`FLAG UTF-8`, as VERO's pt_BR dictionary uses) work // with the one-character parsing here; only the multi-character encodings differ. @@ -284,14 +289,21 @@ fn parse_aff(aff: &str) -> Result { return Err(format!("line {}: FLAG {} is not supported (one-character flags only)", n + 1, at(1))); } } - "FORBIDDENWORD" => { + directive @ ("FORBIDDENWORD" | "NOSUGGEST") => { let mut it = at(1).chars(); match (it.next(), it.next()) { - (Some(c), None) if !c.is_whitespace() => r.forbidden = Some(c), - _ => return Err(format!("line {}: FORBIDDENWORD needs one flag character", n + 1)), + (Some(c), None) if !c.is_whitespace() => { + if directive == "FORBIDDENWORD" { + r.forbidden = Some(c); + } else { + r.no_suggest = Some(c); + } + } + _ => return Err(format!("line {}: {directive} needs one flag character", n + 1)), } } "TRY" => r.try_chars = at(1).to_string(), + "KEY" => r.key = at(1).to_string(), "MAP" => { if !map_header && at(1).parse::().is_ok() { map_header = true; @@ -312,7 +324,8 @@ fn parse_aff(aff: &str) -> Result { let (Some(flag), None) = (flag_chars.next(), flag_chars.next()) else { return Err(format!("line {}: flag `{}` is not one character", n + 1, at(1))); }; - let is_header = f.len() == 4 && matches!(at(2), "Y" | "N") && at(3).parse::().is_ok() && !cross.contains_key(&(suffix, flag)); + // Only a header ignores trailing fields; a rule condition beginning with # is literal. + let is_header = f.len() >= 4 && matches!(at(2), "Y" | "N") && at(3).parse::().is_ok() && !cross.contains_key(&(suffix, flag)); if is_header { cross.insert((suffix, flag), at(2) == "Y"); continue; @@ -400,7 +413,9 @@ impl AffixDict { offs: Vec::with_capacity(stems.len() + 1), flag_chars: flag_chars.clone(), forbidden, + no_suggest: rules.no_suggest.map_or(0, bit_of), try_chars: rules.try_chars, + key: rules.key, map: rules.map, rep: rules.rep, comments, @@ -529,6 +544,10 @@ impl AffixDict { /// Flag mask of a stem, if `w` is one. fn stem_flags(&self, w: &str) -> Option { + self.stem_flags_excluding(w, self.forbidden) + } + + fn stem_flags_excluding(&self, w: &str, excluded: u128) -> Option { if self.table.is_empty() { return None; } @@ -542,9 +561,8 @@ impl AffixDict { if self.word(t as usize) == Some(w) { let id = *self.flags.get(t as usize)?; let mask = self.sets.get(usize::from(id)).copied(); - // A stem carrying FORBIDDENWORD is listed but is not a word: no direct hit, - // and (this is the only stem lookup) no affixed form is built on it either. - return mask.filter(|m| m & self.forbidden == 0); + // All derivations use this lookup: exclusions apply to stems and their forms. + return mask.filter(|m| m & excluded == 0); } slot = (slot + 1) & mask; } @@ -553,8 +571,8 @@ impl AffixDict { /// Does `w` carry flag `bit` — as a stem, or as the form of a suffix rule whose /// continuation classes include `bit` (chained suffixation, depth-bounded)? - fn form_has_flag(&self, w: &str, bit: u128, depth: u8) -> bool { - if self.stem_flags(w).is_some_and(|f| f & bit != 0) { + fn form_has_flag(&self, w: &str, bit: u128, depth: u8, excluded: u128) -> bool { + if self.stem_flags_excluding(w, excluded).is_some_and(|f| f & bit != 0) { return true; } if depth == 0 { @@ -570,7 +588,7 @@ impl AffixDict { root.push_str(base); root.push_str(&g.strip); for r in g.rules.iter().filter_map(|&i| self.suffixes.get(i as usize)) { - if r.cont & bit != 0 && r.cond_matches(&root, true) && self.form_has_flag(&root, r.bit, depth - 1) { + if r.cont & bit != 0 && r.cond_matches(&root, true) && self.form_has_flag(&root, r.bit, depth - 1, excluded) { return true; } } @@ -581,10 +599,19 @@ impl AffixDict { /// Is `w` exactly (case and all) a stem or an affixed form of one? pub fn contains(&self, w: &str) -> bool { + self.contains_excluding(w, self.forbidden) + } + + /// Exactly spelled dictionary words eligible for suggestions (NOSUGGEST stems excluded). + pub fn contains_suggestible(&self, w: &str) -> bool { + self.contains_excluding(w, self.forbidden | self.no_suggest) + } + + fn contains_excluding(&self, w: &str, excluded: u128) -> bool { if w.is_empty() || w.len() > MAX_WORD_BYTES { return false; } - if self.stem_flags(w).is_some() || self.suffixed(w, 0) { + if self.stem_flags_excluding(w, excluded).is_some() || self.suffixed(w, 0, excluded) { return true; } let mut root = String::with_capacity(w.len() + 8); @@ -599,13 +626,13 @@ impl AffixDict { if !r.cond_matches(&root, false) { continue; } - if self.stem_flags(&root).is_some_and(|f| f & r.bit != 0) { + if self.stem_flags_excluding(&root, excluded).is_some_and(|f| f & r.bit != 0) { return true; } - if self.form_has_flag(&root, r.bit, INNER_CHAIN) { + if self.form_has_flag(&root, r.bit, INNER_CHAIN, excluded) { return true; } - if r.cross && self.suffixed(&root, r.bit) { + if r.cross && self.suffixed(&root, r.bit, excluded) { return true; } } @@ -614,7 +641,7 @@ impl AffixDict { /// Is `w` a stem with one suffix? With `need` set (a prefix was taken off), the stem must /// carry those flags too and the suffix must allow cross products. - fn suffixed(&self, w: &str, need: u128) -> bool { + fn suffixed(&self, w: &str, need: u128, excluded: u128) -> bool { let mut root = String::with_capacity(w.len() + 8); // Each split point from the end: the appended string is `w[b..]`, at most the longest // suffix, and something must stay before it. @@ -629,7 +656,7 @@ impl AffixDict { let rules = || g.rules.iter().filter_map(|&i| self.suffixes.get(i as usize)); // The stem path: `root` is a stem carrying the rule's flag (and the prefix's, // when a cross product is being checked). - if let Some(f) = self.stem_flags(&root) + if let Some(f) = self.stem_flags_excluding(&root, excluded) && f & need == need && rules().any(|r| f & r.bit != 0 && (need == 0 || r.cross) && r.cond_matches(&root, true)) { @@ -637,7 +664,7 @@ impl AffixDict { } // Continuation classes: `root` need not be a stem carrying this rule's flag — // it may be a form that gained the flag from another rule's continuation. - if need == 0 && rules().any(|r| r.cond_matches(&root, true) && self.form_has_flag(&root, r.bit, INNER_CHAIN)) { + if need == 0 && rules().any(|r| r.cond_matches(&root, true) && self.form_has_flag(&root, r.bit, INNER_CHAIN, excluded)) { return true; } } @@ -649,15 +676,24 @@ impl AffixDict { /// — in lower case (and capitalized), since a sentence may start with any word and headings /// may be in capitals. Lower-case input never matches a capitalized stem (`warszawa`). pub fn check(&self, w: &str) -> bool { - if self.contains(w) { + self.check_excluding(w, self.forbidden) + } + + /// Whether a suggestion is correct in its displayed case and has an eligible stem. + pub fn can_suggest(&self, w: &str) -> bool { + self.check_excluding(w, self.forbidden | self.no_suggest) + } + + fn check_excluding(&self, w: &str, excluded: u128) -> bool { + if self.contains_excluding(w, excluded) { return true; } match casing(w) { Casing::Lower | Casing::Mixed => false, - Casing::Initial => self.contains(&w.to_lowercase()), + Casing::Initial => self.contains_excluding(&w.to_lowercase(), excluded), Casing::Upper => { let lower = w.to_lowercase(); - self.contains(&capitalize(&lower)) || self.contains(&lower) + self.contains_excluding(&capitalize(&lower), excluded) || self.contains_excluding(&lower, excluded) } } } @@ -726,6 +762,11 @@ impl AffixDict { &self.try_chars } + /// The `KEY` keyboard rows, separated by `|` (adjacent letters are likely mistypes). + pub fn key(&self) -> &str { + &self.key + } + /// The `MAP` groups: letters often confused with each other (`aą`, `zżź`, `oóu`…). pub fn map(&self) -> &[Vec] { &self.map @@ -757,16 +798,21 @@ impl AffixDict { if !self.try_chars.is_empty() { out.push_str(&format!("TRY {}\n", self.try_chars)); } + if !self.key.is_empty() { + out.push_str(&format!("KEY {}\n", self.key)); + } for m in &self.map { out.push_str(&format!("MAP {}\n", m.iter().collect::())); } for (a, b) in &self.rep { out.push_str(&format!("REP {} {}\n", a.replace(' ', "_"), b.replace(' ', "_"))); } - if self.forbidden != 0 - && let Some(c) = self.flag_char(self.forbidden) - { - out.push_str(&format!("FORBIDDENWORD {c}\n")); + for (directive, mask) in [("FORBIDDENWORD", self.forbidden), ("NOSUGGEST", self.no_suggest)] { + if mask != 0 + && let Some(c) = self.flag_char(mask) + { + out.push_str(&format!("{directive} {c}\n")); + } } let z = |s: &str| if s.is_empty() { "0".to_string() } else { s.to_string() }; for (kind, rules) in [("PFX", &self.prefixes), ("SFX", &self.suffixes)] { @@ -823,13 +869,20 @@ impl AffixDict { let at = |k: usize| f.get(k).copied().unwrap_or(""); match at(0) { "TRY" => rules.try_chars = at(1).to_string(), + "KEY" => rules.key = at(1).to_string(), "MAP" => rules.map.push(at(1).chars().collect()), "REP" => rules.rep.push((at(1).replace('_', " "), at(2).replace('_', " "))), - "FORBIDDENWORD" => { + directive @ ("FORBIDDENWORD" | "NOSUGGEST") => { let mut it = at(1).chars(); match (it.next(), it.next()) { - (Some(c), None) if !c.is_whitespace() => rules.forbidden = Some(c), - _ => return Err(format!("affix dictionary: bad FORBIDDENWORD in `{line}`")), + (Some(c), None) if !c.is_whitespace() => { + if directive == "FORBIDDENWORD" { + rules.forbidden = Some(c); + } else { + rules.no_suggest = Some(c); + } + } + _ => return Err(format!("affix dictionary: bad {directive} in `{line}`")), } } kind @ ("PFX" | "SFX") => { @@ -885,7 +938,7 @@ impl AffixDict { } #[cfg(test)] -mod tests { +pub(crate) mod tests { use super::*; const AFF: &str = "SET UTF-8\nTRY aeoiułkr\n\nPFX b Y 1\nPFX b 0 nie .\n\nSFX a Y 3\nSFX a 0 y [^y]\nSFX a ek ka ek\nSFX a 0 em .\n\nSFX c N 1\nSFX c a ą a\n\nMAP 2\nMAP aą\nMAP oóu\nREP 2\nREP ż rz\nREP u ó\n"; @@ -895,6 +948,69 @@ mod tests { AffixDict::from_hunspell(AFF, DIC, &["test data"]).unwrap() } + #[test] + fn affix_headers_allow_comments_but_rule_conditions_stay_literal() { + let aff = "SET UTF-8\nPFX X Y 1 # prefix comment\nPFX X 0 ne . # rule comment\nSFX y Y 1 # suffix comment\nSFX y 0 y . # rule comment\n"; + let d = AffixDict::from_hunspell(aff, "1\nstrom/Xy\n", &[]).unwrap(); + assert_eq!(d.rule_count(), 2, "headers must not become affix rules"); + for w in ["strom", "stromy", "nestrom", "nestromy"] { + assert!(d.contains(w), "{w}"); + } + for w in ["stromY1", "stromN1", "X", "y"] { + assert!(!d.contains(w), "{w}"); + } + let hash = AffixDict::from_hunspell("SFX H Y 1\nSFX H 0 # .\nTRY a#b\nKEY a#b\n", "1\nstrom/H\n", &[]).unwrap(); + assert!(hash.contains("strom#"), "a hash in an affix field is literal"); + let literal = AffixDict::from_hunspell( + "PFX P Y 1 # header\nPFX P 0 pre #\nSFX S Y 1 # header\nSFX S 0 post #\n", + "3\n#stem/P\nstem#/S\nstem/PS\n", + &[], + ) + .unwrap(); + assert!(literal.contains("pre#stem") && literal.contains("stem#post")); + assert!(!literal.contains("prestem") && !literal.contains("stempost"), "# must remain a literal condition"); + + assert_eq!(hash.try_chars(), "a#b"); + assert_eq!(hash.key(), "a#b"); + assert_eq!(AffixDict::from_bytes(&hash.to_bytes()).unwrap().key(), "a#b"); + } + + /// Shared hand-written fixture: collectors must reject every path to NOSUGGEST forms. + pub(crate) fn assert_nosuggest_is_honoured(suggest: impl Fn(&AffixDict, &str, usize) -> Vec) { + let aff = "SET UTF-8\nNOSUGGEST N\nFORBIDDENWORD F\nTRY abeklmnostxy\nMAP 1\nMAP ae\nKEY ak\nREP 2\nREP xy toka\nREP zz normal\nPFX X Y 1\nPFX X 0 pre .\nSFX A Y 1\nSFX A 0 ble/B .\nSFX B Y 1\nSFX B 0 s .\n"; + let d = AffixDict::from_hunspell(aff, "4\ntoka/NXA\nnormal/XA\nTkname/N\nbanned/F\n", &[]).unwrap(); + for word in d.forms("toka").into_iter().chain(["TOKA".into(), "Toka".into(), "Tkname".into(), "TKNAME".into()]) { + assert!(d.check(&word), "NOSUGGEST is still correct: {word}"); + assert!(!d.can_suggest(&word), "NOSUGGEST form: {word}"); + } + for input in ["tokax", "tokablesx", "pretokax", "pretokablesx", "toka", "TOKA", "tkname", "teka", "xy", "normaltoka", "toak", "taak"] { + let results = suggest(&d, input, 32); + assert!(results.iter().all(|word| word.split(' ').all(|part| d.can_suggest(part))), "{input}: {results:?}"); + assert!(results.iter().all(|word| !word.to_lowercase().contains("toka") && word.to_lowercase() != "tkname"), "{input}: {results:?}"); + } + assert!(suggest(&d, "norrmal", 8).contains(&"normal".into())); + assert!(suggest(&d, "zz", 8).contains(&"normal".into())); + // Filtering happens before display casing: ordinary mixed-case stems retain the + // existing all-caps suggestion behaviour, with and without a NOSUGGEST declaration. + for aff in ["", "NOSUGGEST N\n"] { + let ordinary = AffixDict::from_hunspell(aff, "2\nmixCase\nhidden/N\n", &[]).unwrap(); + assert!(suggest(&ordinary, "MIXCASX", 8).contains(&"MIXCASE".into())); + } + } + + #[test] + fn nosuggest_is_correct_but_not_suggestible_and_survives_compact_format() { + let d = AffixDict::from_hunspell("NOSUGGEST N\nSFX S Y 1\nSFX S 0 s .\n", "2\nsecret/NS\nnormal/S\n", &[]).unwrap(); + for dict in [&d, &AffixDict::from_bytes(&d.to_bytes()).unwrap()] { + for word in ["secret", "secrets", "Secret", "SECRETS"] { + assert!(dict.check(word), "{word}"); + assert!(!dict.can_suggest(word), "{word}"); + } + assert!(dict.contains_suggestible("normal") && dict.can_suggest("NORMALS")); + } + assert!(AffixDict::from_hunspell("NOSUGGEST NN\n", "1\nword\n", &[]).is_err()); + } + #[test] fn stems_suffixes_prefixes_and_cross_products() { let d = dict(); diff --git a/crates/proof/src/cs/grammar.rs b/crates/proof/src/cs/grammar.rs new file mode 100644 index 00000000..26a14fcd --- /dev/null +++ b/crates/proof/src/cs/grammar.rs @@ -0,0 +1,103 @@ +//! Generic repeated-word and spacing checks with Czech messages. +//! No Czech-specific grammar, comma or sentence-capital rules are inferred. + +use crate::{Issue, IssueKind, ProofLang, words}; + +fn issue(start: usize, end: usize, message: String, suggestions: Vec) -> Issue { + Issue { start, end, kind: IssueKind::Grammar, message, suggestions, lang: ProofLang::Cs } +} + +pub fn check(text: &str) -> Vec { + let mut issues = Vec::new(); + repeated_words(text, &words(text), &mut issues); + spaces(text, &mut issues); + issues.sort_by_key(|i| (i.start, i.end)); + issues +} + +fn repeated_words(text: &str, ws: &[(usize, usize)], v: &mut Vec) { + for pair in ws.windows(2) { + let [(a0, b0), (a1, b1)] = pair else { continue }; + let (Some(w0), Some(w1), Some(gap)) = (text.get(*a0..*b0), text.get(*a1..*b1), text.get(*b0..*a1)) else { continue }; + if !gap.is_empty() && gap.chars().all(char::is_whitespace) && w0.chars().all(char::is_alphabetic) && w0.to_lowercase() == w1.to_lowercase() { + v.push(issue(*a0, *b1, format!("Opakované slovo: „{w1}“"), vec![w0.to_string()])); + } + } +} + +fn spaces(text: &str, v: &mut Vec) { + for (i, c) in text.char_indices() { + if c != ' ' { + continue; + } + let rest = text.get(i + 1..).unwrap_or(""); + let mut it = rest.chars(); + let Some(p) = it.next() else { continue }; + let after = it.next(); + // A ratio/score keeps the spaces in digit-space-colon-space-digit. + let ratio = p == ':' + && text.get(..i).and_then(|before| before.chars().next_back()).is_some_and(|c| c.is_ascii_digit()) + && after == Some(' ') + && it.next().is_some_and(|c| c.is_ascii_digit()); + let flag = !ratio + && match p { + ',' => true, + // `;)` and `;-)` are emoticons. + ';' => !matches!(after, Some(')' | '(' | '-' | 'P' | 'D')), + // ` ...` is an ellipsis, ` .5` a number. + '.' => !matches!(after, Some('.')) && !after.is_some_and(|a| a.is_ascii_digit()), + // ` :)`, ` :-)`, ` :D` and similar are emoticons; ` ?!` is still a stray space. + ':' | '!' | '?' => after.is_none_or(|a| a.is_whitespace() || matches!(a, '"' | '”' | '’' | '!' | '?')), + _ => false, + }; + if flag { + v.push(issue(i, i + 1 + p.len_utf8(), "Mezera před interpunkcí".into(), vec![p.to_string()])); + } + } + for (i, _) in text.match_indices(" ") { + if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) { + continue; + } + v.push(issue(i, i + 2, "Dvojitá mezera".into(), vec![" ".into()])); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn repeated_words_and_spaces_have_czech_messages() { + let text = "Město město , škola."; + let issues = check(text); + assert_eq!(issues.len(), 3, "{issues:?}"); + assert_eq!(issues[0].message, "Opakované slovo: „město“"); + assert_eq!(issues[0].suggestions, ["Město"]); + assert!(issues.iter().any(|i| i.message == "Mezera před interpunkcí" && &text[i.start..i.end] == " ,")); + assert!(issues.iter().any(|i| i.message == "Dvojitá mezera")); + assert!(issues.iter().all(|i| i.lang == ProofLang::Cs && i.kind == IssueKind::Grammar)); + for text in ["město. škola", "Dr. Novák", "A apple", "Město, město!", "Úsměv :) Číslo .5 a tečky ..."] { + assert!(check(text).is_empty(), "{text}"); + } + } + + #[test] + fn ratios_and_scores_keep_their_spaces() { + for text in ["3 : 1", "1 : 2", "Skóre 12 : 10.", "Poměr 1 : 100."] { + assert!(check(text).is_empty(), "{text}: {:?}", check(text)); + } + for text in ["Slovo : další", "3 : slovo", "slovo : 1", "3 :"] { + assert!(check(text).iter().any(|i| i.message == "Mezera před interpunkcí"), "{text}"); + } + } + + proptest::proptest! { + #[test] + fn never_panics_and_ranges_are_valid(s in "\\PC{0,80}") { + for issue in check(&s) { + proptest::prop_assert!(issue.start <= issue.end && issue.end <= s.len()); + proptest::prop_assert!(s.is_char_boundary(issue.start) && s.is_char_boundary(issue.end)); + } + } + } +} diff --git a/crates/proof/src/cs/mod.rs b/crates/proof/src/cs/mod.rs new file mode 100644 index 00000000..25253d2a --- /dev/null +++ b/crates/proof/src/cs/mod.rs @@ -0,0 +1,212 @@ +//! Czech proofing using the installed Hunspell `cs_CZ` dictionary (never bundled). +//! Suggestions use its runtime MAP/REP/TRY/KEY tables. Generic grammar checks have Czech +//! messages; hyphenation keeps the existing fallback, without Czech pattern data. + +pub mod grammar; +mod suggest; +#[cfg(test)] +pub(crate) mod test_dict; + +use std::sync::{Arc, RwLock}; + +use crate::affix::AffixDict; + +pub use suggest::suggest; + +/// The Czech dictionary: `None` until first use, then the installed one (or an empty one). +static DICT: RwLock>> = RwLock::new(None); + +/// File names of a Czech Hunspell dictionary. +const NAMES: &[&str] = &["cs_CZ", "cs-CZ", "cs"]; + +/// The Czech dictionary: the installed Hunspell `cs_CZ` (read on first use), one given with +/// [`install_dictionary`], or an empty one when none is available (Czech spelling is off +/// then; see [`has_dictionary`]). +pub fn dictionary() -> Arc { + if let Some(d) = DICT.read().unwrap_or_else(|e| e.into_inner()).as_ref() { + return d.clone(); + } + let loaded = Arc::new(load_installed()); + let mut slot = DICT.write().unwrap_or_else(|e| e.into_inner()); + // Another thread (or an install) may have filled it meanwhile: that one wins. + slot.get_or_insert(loaded).clone() +} + +fn load_installed() -> AffixDict { + let Some((aff, dic)) = crate::hunspell::find(NAMES) else { + log::info!("Czech spelling: no Hunspell cs_CZ dictionary installed"); + return AffixDict::default(); + }; + crate::hunspell::load(&aff, &dic).unwrap_or_else(|e| { + log::warn!("Czech spelling: {e}"); + AffixDict::default() + }) +} + +/// Use this Czech dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to +/// UTF-8), for example one the user picked. On error the current dictionary stays. +pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> { + let d = AffixDict::from_hunspell(aff, dic, &[])?; + *DICT.write().unwrap_or_else(|e| e.into_inner()) = Some(Arc::new(d)); + Ok(()) +} + +/// Forget the loaded dictionary, so the next check looks for an installed one again (after +/// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed). +pub fn reload_dictionary() { + *DICT.write().unwrap_or_else(|e| e.into_inner()) = None; +} + +/// Is a Czech dictionary available (spelling is checked)? +pub fn has_dictionary() -> bool { + !dictionary().is_empty() +} + +/// Is `c` a letter of the Latin script (the only one the Czech dictionary covers)? +fn is_latin_letter(c: char) -> bool { + c.is_alphabetic() && (c < '\u{0250}' || ('\u{1E00}'..='\u{1EFF}').contains(&c)) +} + +/// Is `word` spelled correctly in Czech? Numbers, single letters, short words in capitals +/// (acronyms: `ČR`, `EU`), words with digits and words in other scripts are accepted, and so +/// is every word when no Czech dictionary is installed. +pub fn is_correct(word: &str) -> bool { + is_correct_with(&dictionary(), word) +} + +/// [`is_correct`] against a given dictionary (an empty one accepts everything). +pub fn is_correct_with(d: &AffixDict, word: &str) -> bool { + if d.is_empty() { + return true; // no Czech dictionary installed: no spelling marks + } + let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’'); + let w = w.trim_matches(['\'', '’']); + let n = w.chars().count(); + if n <= 1 || w.chars().any(|c| c.is_ascii_digit()) || !w.chars().any(char::is_alphabetic) { + return true; + } + if w.chars().all(|c| !c.is_lowercase()) && n <= 6 { + return true; + } + if !w.chars().any(is_latin_letter) { + return true; // other scripts: no dictionary + } + if w.contains('-') { + return w.split('-').all(|p| p.chars().count() <= 1 || known(d, p)); + } + known(d, w) +} + +/// Is one word (no hyphens) in the dictionary or the user dictionary? +fn known(d: &AffixDict, w: &str) -> bool { + d.check(w) || crate::in_user_dictionary(&w.to_lowercase()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn dictionary_lookup_accepts_czech_file_names() { + let dir = std::env::temp_dir().join(format!("wordcraft-cs-aliases-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + for name in ["cs_CZ", "cs-CZ", "cs"] { + std::fs::write(dir.join(format!("{name}.aff")), test_dict::AFF).unwrap(); + std::fs::write(dir.join(format!("{name}.dic")), test_dict::DIC).unwrap(); + let (aff, dic) = crate::hunspell::find_in(&dir, NAMES).unwrap(); + let d = crate::hunspell::load(&aff, &dic).unwrap(); + assert!(is_correct_with(&d, "městě")); + std::fs::remove_file(aff).unwrap(); + std::fs::remove_file(dic).unwrap(); + } + std::fs::remove_dir(dir).unwrap(); + } + + #[test] + fn czech_words_affixes_chains_case_and_diacritics() { + test_dict::install(); + let d = dictionary(); + assert!(has_dictionary()); + assert_eq!(d.len(), 20); + assert_eq!(d.rule_count(), 8); + for w in [ + "město", + "města", + "městě", + "neměsta", + "rychle", + "rychlejší", + "učitel", + "učitelé", + "Praha", + "Praze", + "Město", + "POČÍTAČ", + "příliš", + "žluťoučký", + "kůň", + "úpěl", + "ďábelské", + "ódy", + "děkujeme", + "zpráva", + "zprava", + "škola-počítač", + ] { + assert!(is_correct(w), "{w}"); + } + for w in ["městu", "městoě", "rychleejší", "učiteléé", "praha", "mĚsto", "pocitac", "dekujem", "škola-pocitac"] { + assert!(!is_correct(w), "{w}"); + } + for c in "ěščřžýáíéůú".chars() { + assert!(test_dict::DIC.contains(c), "UTF-8 fixture covers {c}"); + } + } + + #[test] + fn forbidden_words_are_rejected_in_every_case() { + test_dict::install(); + for w in ["zqzakázané", "Zqzakázané", "ZQZAKÁZANÉ"] { + assert!(!is_correct(w), "{w}"); + } + } + + #[test] + fn missing_dictionary_and_ignored_inputs_are_quiet() { + let empty = AffixDict::default(); + assert!(is_correct_with(&empty, "pocitac")); + assert!(install_dictionary("FLAG long\n", "1\nstrom\n").is_err()); + let d = AffixDict::from_hunspell(test_dict::AFF, test_dict::DIC, &[]).unwrap(); + for w in ["", "a", "2026", "ČR", "článek2", "日本語"] { + assert!(is_correct_with(&d, w), "{w}"); + } + } + + #[test] + fn user_dictionary_applies() { + test_dict::install(); + assert!(!is_correct("zqčeskévlastníslovo")); + crate::add_word("Zqčeskévlastníslovo"); + assert!(is_correct("zqčeskévlastníslovo")); + } + + #[test] + fn czech_dispatch_messages_options_and_mixed_spans() { + test_dict::install(); + assert!(crate::has_dictionary(Some("cs-CZ"))); + assert!(crate::is_correct_for("městě", Some("cze"))); + let text = "Město pocitac"; + let issues = crate::check_text(text, &[(0..text.len(), crate::ProofLang::Cs)]); + assert_eq!(issues.len(), 1, "{issues:?}"); + assert_eq!(&text[issues[0].start..issues[0].end], "pocitac"); + assert_eq!(issues[0].message, "Možná pravopisná chyba"); + assert_eq!(issues[0].lang, crate::ProofLang::Cs); + assert!(crate::suggest_in("pocitac", 8, crate::ProofLang::Cs).contains(&"počítač".into())); + let text = "Město this is is wrong"; + let issues = crate::check_text(text, &[(0..6, crate::ProofLang::Cs), (7..text.len(), crate::ProofLang::En)]); + assert!(issues.iter().any(|i| i.lang == crate::ProofLang::En && i.message.starts_with("Repeated word"))); + assert!(issues.iter().all(|i| i.start >= 7)); + let opts = crate::Options { ignore_uppercase: false, ..Default::default() }; + assert!(!crate::is_correct_in_with("ZQZAKÁZANÉ", crate::ProofLang::Cs, &opts)); + } +} diff --git a/crates/proof/src/cs/suggest.rs b/crates/proof/src/cs/suggest.rs new file mode 100644 index 00000000..6c600091 --- /dev/null +++ b/crates/proof/src/cs/suggest.rs @@ -0,0 +1,424 @@ +//! Czech suggestions: bounded MAP variants, REP replacements, KEY neighbours, TRY edits +//! and a stem scan. All linguistic tables are read from the installed dictionary. + +use std::collections::{HashMap, HashSet}; + +use crate::affix::{AffixDict, Casing, capitalize, casing}; + +const MAX_LETTERS: usize = 40; +const MAX_MAP_CHANGES: usize = 4; +const MAX_MAP_VARIANTS: usize = 4096; +/// Stems the scan of round 6 expands. +const SCAN_STEMS: usize = 48; + +const COST_CASE: u32 = 0; +const COST_ADD_DIACRITIC: u32 = 4; +const COST_OTHER_MAP: u32 = 7; +const COST_DROP_DIACRITIC: u32 = 11; +const COST_REP: u32 = 10; +const COST_SPLIT: u32 = 12; +const COST_SPLIT_RARE: u32 = 17; +const COST_KEY: u32 = 13; +const COST_SWAP: u32 = 14; +const COST_DROP_OR_ADD: u32 = 15; +const COST_MISTYPE: u32 = 16; +const COST_SCAN: u32 = 20; + +struct Collector<'a> { + d: &'a AffixDict, + /// The word, lower case. + input: String, + tested: HashSet, + /// Suggestion → (cost, order found). + found: HashMap, + seq: usize, +} + +impl Collector<'_> { + /// Check a lower-case candidate (a word, or two separated by a space) and keep it if every + /// word is in the dictionary, as written or capitalized (names). + fn offer(&mut self, cand: &str, cost: u32) { + if cand.is_empty() || cand == self.input || !self.tested.insert(cand.to_string()) { + return; + } + let mut words = Vec::new(); + let mut cost = cost; + for part in cand.split(' ') { + if self.d.contains_suggestible(part) { + words.push(part.to_string()); + } else { + let c = capitalize(part); + if !self.d.contains_suggestible(&c) { + return; + } + words.push(c); + // A name ranks after a common word at the same distance (a capitalized name). + cost += 1; + } + } + self.keep(words.join(" "), cost); + } + + /// Keep a suggestion known to be correct. + fn keep(&mut self, text: String, cost: u32) { + if !text.split(' ').all(|part| self.d.can_suggest(part)) { + return; + } + self.seq += 1; + let seq = self.seq; + let e = self.found.entry(text).or_insert((cost, seq)); + if cost < e.0 { + *e = (cost, seq); + } + } + + fn count_within(&self, cost: u32) -> usize { + self.found.values().filter(|(c, _)| *c <= cost).count() + } +} + +/// The MAP group of each letter (its other members are its likely confusions). +fn alternatives(c: char, map: &[Vec]) -> Vec { + map.iter().filter(|g| g.contains(&c)).flat_map(|g| g.iter().copied()).filter(|&x| x != c).collect() +} + +/// The cost of changing letter `from` to its MAP relative `to`. Adding a missing accent is the +/// commonest slip (typing without diacritics); an accent the writer did type +/// is likelier right, so taking one away costs more than a known confusion from the REP table. +fn map_cost(from: char, to: char) -> u32 { + match (from.is_ascii(), to.is_ascii()) { + (true, false) => COST_ADD_DIACRITIC, + (false, true) => COST_DROP_DIACRITIC, + _ => COST_OTHER_MAP, + } +} + +/// Every way of changing up to [`MAX_MAP_CHANGES`] letters to a MAP relative, with its cost. +fn map_variants(chars: &[char], alts: &[Vec], i: usize, changes: usize, cost: u32, buf: &mut Vec, out: &mut Vec<(String, u32)>) { + if out.len() >= MAX_MAP_VARIANTS { + return; + } + let Some(&c) = chars.get(i) else { + if changes > 0 { + out.push((buf.iter().collect(), cost)); + } + return; + }; + buf.push(c); + map_variants(chars, alts, i + 1, changes, cost, buf, out); + buf.pop(); + if changes < MAX_MAP_CHANGES { + for &a in alts.get(i).map(Vec::as_slice).unwrap_or_default() { + buf.push(a); + map_variants(chars, alts, i + 1, changes + 1, cost + map_cost(c, a), buf, out); + buf.pop(); + } + } +} + +/// A letter's MAP representative (the first of its group), lower case: as defined by the installed table. +fn fold(c: char, map: &[Vec]) -> char { + let l = c.to_lowercase().next().unwrap_or(c); + map.iter().find(|g| g.contains(&l)).and_then(|g| g.first().copied()).unwrap_or(l) +} + +/// Optimal string alignment distance where a MAP slip (dictionary-defined relatives) costs 1 and any other +/// edit 2; `limit + 1` once it is certainly above `limit`. +fn weighted_distance(a: &[char], b: &[char], related: &dyn Fn(char, char) -> bool, limit: u32) -> u32 { + let m = b.len(); + let over = limit + 1; + let mut prev2 = vec![0u32; m + 1]; + let mut prev: Vec = (0..=m as u32).map(|j| j * 2).collect(); + let mut cur = vec![0u32; m + 1]; + for (i, &ai) in a.iter().enumerate() { + if let Some(c) = cur.get_mut(0) { + *c = (i as u32 + 1) * 2; + } + let mut row_min = u32::MAX; + for (j, &bj) in b.iter().enumerate() { + let sub = if ai == bj { + 0 + } else if related(ai, bj) { + 1 + } else { + 2 + }; + let at = |v: &[u32], k: usize| v.get(k).copied().unwrap_or(u32::MAX / 4); + let mut v = (at(&prev, j + 1) + 2).min(at(&cur, j) + 2).min(at(&prev, j) + sub); + if i > 0 && j > 0 && a.get(i - 1) == Some(&bj) && b.get(j - 1) == Some(&ai) { + v = v.min(at(&prev2, j - 1) + 2); + } + if let Some(c) = cur.get_mut(j + 1) { + *c = v; + } + row_min = row_min.min(v); + } + if m > 0 && row_min > limit { + return over; + } + std::mem::swap(&mut prev2, &mut prev); + std::mem::swap(&mut prev, &mut cur); + } + prev.get(m).copied().unwrap_or(over).min(over) +} + +/// Spelling suggestions for a Czech word, best first, at most `max`, in the word's letter case. +pub fn suggest(word: &str, max: usize) -> Vec { + suggest_in(&super::dictionary(), word, max) +} + +/// [`suggest`] against a given dictionary. +pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec { + let word = word.trim(); + let n = word.chars().count(); + if n == 0 || n > MAX_LETTERS || max == 0 || d.is_empty() || word.contains(char::is_whitespace) { + return Vec::new(); + } + let case = casing(word); + let input = word.to_lowercase(); + let chars: Vec = input.chars().collect(); + let mut col = Collector { d, input: input.clone(), tested: HashSet::new(), found: HashMap::new(), seq: 0 }; + // 1. Letter case. + if d.contains(&input) && word != input { + col.keep(input.clone(), COST_CASE); + } else if d.contains(&capitalize(&input)) && word != capitalize(&input) { + col.keep(capitalize(&input), COST_CASE); + } + // 2. Accents and other MAP slips. + let map = d.map(); + let alts: Vec> = chars.iter().map(|&c| alternatives(c, map)).collect(); + let mut variants = Vec::new(); + map_variants(&chars, &alts, 0, 0, 0, &mut Vec::with_capacity(n), &mut variants); + variants.sort_by_key(|v| v.1); + for (v, cost) in &variants { + col.offer(v, *cost); + } + // 3. Common confusions. + for (from, to) in d.rep() { + if from.is_empty() { + continue; + } + for (i, _) in input.match_indices(from.as_str()) { + if let (Some(a), Some(b)) = (input.get(..i), input.get(i + from.len()..)) { + col.offer(&format!("{a}{to}{b}"), COST_REP); + } + } + } + // 4. A missing space; both halves must be dictionary words. + for (i, _) in input.char_indices().skip(1) { + let (Some(a), Some(b)) = (input.get(..i), input.get(i..)) else { continue }; + let long = |part: &str| part.chars().count() >= 3; + let cost = if long(a) && long(b) { COST_SPLIT } else { COST_SPLIT_RARE }; + col.offer(&format!("{a} {b}"), cost); + } + // 5. Keyboard neighbours from the installed dictionary's KEY rows. + for row in d.key().split('|').take(32) { + let keys: Vec = row.chars().take(128).collect(); + for pair in keys.windows(2) { + let [a, b] = pair else { continue }; + for (i, &c) in chars.iter().enumerate() { + let replacement = if c == *a { + Some(*b) + } else if c == *b { + Some(*a) + } else { + None + }; + if let Some(replacement) = replacement { + let mut v = chars.clone(); + if let Some(x) = v.get_mut(i) { + *x = replacement; + } + col.offer(&v.iter().collect::(), COST_KEY); + } + } + } + } + // One-letter edits use only the runtime TRY alphabet, with bounded work. + let mut letters: Vec = d.try_chars().chars().filter(|c| c.is_lowercase()).take(128).collect(); + letters.sort_unstable(); + letters.dedup(); + let s = |v: &[char]| v.iter().collect::(); + for i in 0..n.saturating_sub(1) { + if chars.get(i) != chars.get(i + 1) { + let mut v = chars.clone(); + v.swap(i, i + 1); + col.offer(&s(&v), COST_SWAP); + } + } + for i in 0..n { + let mut v = chars.clone(); + v.remove(i); + col.offer(&s(&v), COST_DROP_OR_ADD); + } + for i in 0..=n { + for &c in &letters { + let mut v = chars.clone(); + v.insert(i, c); + col.offer(&s(&v), COST_DROP_OR_ADD); + } + } + for i in 0..n { + for &c in &letters { + if chars.get(i) != Some(&c) { + let mut v = chars.clone(); + if let Some(x) = v.get_mut(i) { + *x = c; + } + col.offer(&s(&v), COST_MISTYPE); + } + } + } + // 6. Further off: scan the stems that start like the word. + if col.count_within(COST_MISTYPE) < max { + scan(&mut col, &chars); + } + let mut found: Vec<(String, (u32, usize))> = col.found.into_iter().collect(); + // Among equal costs, the change nearest the word's start is the likelier intent + // (the first changed letter breaks ties). + let diff_at = |text: &str| chars.iter().zip(text.chars()).position(|(a, b)| *a != b).unwrap_or(n); + found.sort_by_key(|(text, (cost, seq))| (*cost, diff_at(text), *seq)); + let mut out: Vec = Vec::new(); + for (text, _) in found { + let t = match case { + Casing::Initial => capitalize(&text), + Casing::Upper if n > 1 => text.to_uppercase(), + _ => text, + }; + if t != word && !out.contains(&t) { + out.push(t); + } + if out.len() >= max { + break; + } + } + out +} + +/// Round 6: rank the stems starting with the word's first letter (or a MAP relative of it, either +/// case) by shared letter pairs, then keep the forms of the best within two typos. +fn scan(col: &mut Collector, chars: &[char]) { + let d = col.d; + let map = d.map(); + let Some(&first) = chars.first() else { return }; + let folded: Vec = chars.iter().map(|&c| fold(c, map)).collect(); + let pairs: HashSet<(char, char)> = std::iter::once(' ').chain(folded.iter().copied()).zip(folded.iter().copied()).collect(); + let mut firsts: Vec = std::iter::once(first).chain(alternatives(first, map)).collect(); + let upper: Vec = firsts.iter().flat_map(|c| c.to_uppercase()).collect(); + firsts.extend(upper); + firsts.sort_unstable(); + firsts.dedup(); + let n = folded.len(); + let mut best: Vec<(u32, &str)> = Vec::new(); + for f in firsts { + let prefix = f.to_string(); + for stem in d.stems_with_prefix(&prefix) { + let sf: Vec = stem.chars().map(|c| fold(c, map)).collect(); + let shared = std::iter::once(' ').chain(sf.iter().copied()).zip(sf.iter().copied()).filter(|p| pairs.contains(p)).count(); + if shared < 2 { + continue; + } + // Shared pairs, less the stem's letters the word doesn't explain, less a length gap + // beyond what an ending explains. + let extra = sf.len().saturating_sub(shared); + let gap = sf.len().abs_diff(n).saturating_sub(4); + let score = (shared * 4).saturating_sub(extra * 3 + gap * 2) as u32; + if score == 0 { + continue; + } + if best.len() < SCAN_STEMS { + best.push((score, stem)); + } else if let Some(min) = best.iter_mut().min_by_key(|b| b.0) + && score > min.0 + { + *min = (score, stem); + } + } + } + best.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(b.1))); + let related = |a: char, b: char| fold(a, map) == fold(b, map); + let limit = 4; + for (_, stem) in best { + for form in d.forms(stem) { + let lf: Vec = form.to_lowercase().chars().collect(); + if lf.len().abs_diff(chars.len()) > 2 { + continue; + } + let dist = weighted_distance(&lf, chars, &related, limit); + if dist <= limit && lf.as_slice() != chars { + col.keep(form, COST_SCAN + dist * 3); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn an_extra_trailing_letter_is_dropped_before_a_short_word_split() { + let d = AffixDict::from_hunspell("", "2\npočítač\nz\n", &[]).unwrap(); + let suggestions = suggest_in(&d, "počítačz", 8); + assert_eq!(suggestions.first().map(String::as_str), Some("počítač"), "{suggestions:?}"); + assert!(suggestions.contains(&"počítač z".into()), "{suggestions:?}"); + } + + #[test] + fn suggestions_are_fast_enough() { + crate::cs::test_dict::install(); + let _ = suggest("město", 1); + let t = std::time::Instant::now(); + for word in ["zqčeskévelmidlouhéslovo", "pocitacz", "nejrychllejší"] { + let _ = suggest(word, 6); + } + let ms = t.elapsed().as_secs_f64() * 1000.0; + eprintln!("3 Czech suggestion calls: {ms:.0} ms"); + assert!(ms < 20_000.0, "{ms} ms"); + } + + #[test] + fn nosuggest_stems_and_forms_are_never_offered() { + crate::affix::tests::assert_nosuggest_is_honoured(suggest_in); + } + + #[test] + fn czech_suggestions_use_runtime_tables_and_case() { + crate::cs::test_dict::install(); + for (word, want) in [ + ("mesto", "město"), + ("mňe", "mě"), + ("xůž", "nůž"), + ("škloa", "škola"), + ("pocitac", "počítač"), + ("Mesto", "Město"), + ("praha", "Praha"), + ("toškola", "to škola"), + ] { + assert!(suggest(word, 8).contains(&want.into()), "{word}: {:?}", suggest(word, 8)); + } + // Each table changes the answer without a hard-coded Czech alphabet or confusion list. + for (table, wrong, right) in + [("MAP 1\nMAP cč\n", "caj", "čaj"), ("REP 1\nREP xy město\n", "xy", "město"), ("TRY š\n", "kola", "škola"), ("KEY xn\n", "xůž", "nůž")] + { + let dic = format!("1\n{right}\n"); + let d = AffixDict::from_hunspell(table, &dic, &[]).unwrap(); + assert!(suggest_in(&d, wrong, 4).contains(&right.into()), "{table}"); + let without = AffixDict::from_hunspell("", &dic, &[]).unwrap(); + assert!(!suggest_in(&without, wrong, 4).contains(&right.into()), "must depend on {table}"); + } + } + + #[test] + fn suggestions_stay_bounded_and_skip_forbidden_words() { + crate::cs::test_dict::install(); + for w in ["", "dvě slova", &"ě".repeat(41)] { + assert!(suggest(w, 4).is_empty()); + } + assert!(suggest("mesto", 0).is_empty()); + assert!(suggest_in(&AffixDict::default(), "mesto", 4).is_empty()); + assert!(suggest("město", 4).len() <= 4); + assert!(!suggest("město", 4).contains(&"město".into())); + assert!(!suggest("zqzakázane", 8).contains(&"zqzakázané".into())); + } +} diff --git a/crates/proof/src/cs/test_dict.rs b/crates/proof/src/cs/test_dict.rs new file mode 100644 index 00000000..6cc59759 --- /dev/null +++ b/crates/proof/src/cs/test_dict.rs @@ -0,0 +1,59 @@ +//! Hand-written synthetic Czech test data. No words, rules or suggestion tables were copied +//! from a system dictionary. This is a small fixture, not a redistributable Czech dictionary. + +pub const AFF: &str = "SET UTF-8 +FLAG UTF-8 +FORBIDDENWORD F +TRY aeiouěščřžýáíéůúntklmp +KEY xn|qř +MAP 5 +MAP eěé +MAP cč +MAP rř +MAP uůú +MAP ií +REP 1 +REP mňe mě +SFX A Y 2 # synthetic declension +SFX A o a o +SFX A o ě o +SFX B N 1 +SFX B 0 e/C . +SFX C N 1 +SFX C e ejší e +SFX D N 1 +SFX D 0 itel/E . +SFX E N 1 +SFX E 0 é . +SFX P Y 1 +SFX P ha ze ha +PFX X Y 1 +PFX X 0 ne . +"; + +pub const DIC: &str = "20 +město/AX +rychl/B +uč/D +Praha/P +škola +počítač +příliš +žluťoučký +kůň +úpěl +ďábelské +ódy +zpráva +zprava +děkujeme +mě +nůž +si +to +zqzakázané/F +"; + +pub fn install() { + super::install_dictionary(AFF, DIC).unwrap(); +} diff --git a/crates/proof/src/hunspell.rs b/crates/proof/src/hunspell.rs index 4a42e3ec..d4b6f07b 100644 --- a/crates/proof/src/hunspell.rs +++ b/crates/proof/src/hunspell.rs @@ -200,6 +200,7 @@ mod tests { fn libreoffice_extension_dirs_follow_the_language() { // LibreOffice's IDs are per dictionary: Polish is `dict-pl`, Brazilian Portuguese // `dict-pt-BR`, so both the primary-language folder and the dashed full name are tried. + assert_eq!(super::lo_ext_dirs("cs_CZ"), ["dict-cs", "dict-cs-CZ"]); assert_eq!(super::lo_ext_dirs("pl_PL"), ["dict-pl", "dict-pl-PL"]); assert_eq!(super::lo_ext_dirs("pt_BR"), ["dict-pt", "dict-pt-BR"]); assert_eq!(super::lo_ext_dirs("de-DE"), ["dict-de", "dict-de-DE"]); diff --git a/crates/proof/src/hyphen.rs b/crates/proof/src/hyphen.rs index f4096c63..d0eae0c4 100644 --- a/crates/proof/src/hyphen.rs +++ b/crates/proof/src/hyphen.rs @@ -856,7 +856,9 @@ mod tests { #[test] fn languages_without_patterns_keep_the_english_ones() { for w in ["hyphenation", "typography", "internationalization"] { - assert_eq!(hyphen_points_in(w, &Limits::default(), ProofLang::Other), hyphen_points(w, &Limits::default()), "{w}"); + for lang in [ProofLang::Other, ProofLang::Cs] { + assert_eq!(hyphen_points_in(w, &Limits::default(), lang), hyphen_points(w, &Limits::default()), "{w}"); + } } } diff --git a/crates/proof/src/lang.rs b/crates/proof/src/lang.rs index 61dfc753..e6000c48 100644 --- a/crates/proof/src/lang.rs +++ b/crates/proof/src/lang.rs @@ -32,6 +32,9 @@ pub enum ProofLang { /// Russian (`ru`, `ru-RU`): the installed Hunspell dictionary (when there is one), the Russian /// grammar rules and the Russian word-division rules. Ru, + /// Czech (`cs`, `cs-CZ`): the installed Hunspell dictionary, dictionary-driven suggestions + /// and generic punctuation checks. Hyphenation keeps the fallback English patterns. + Cs, /// A language without proofing tools here (`fr-FR`, `ja-JP`, `x-none`…). Like Word without /// the language's proofing tools: no spelling or grammar marks, so its words are never /// flagged as English misspellings. Hyphenation keeps using the English patterns, as before @@ -41,7 +44,7 @@ pub enum ProofLang { impl ProofLang { /// Every language with a spelling dictionary, in display order. - pub const PROOFED: [ProofLang; 7] = [ProofLang::En, ProofLang::Pl, ProofLang::Pt, ProofLang::It, ProofLang::Hr, ProofLang::Tr, ProofLang::Ru]; + pub const PROOFED: [ProofLang; 8] = [ProofLang::En, ProofLang::Pl, ProofLang::Pt, ProofLang::It, ProofLang::Hr, ProofLang::Tr, ProofLang::Ru, ProofLang::Cs]; /// The proofing language of a BCP 47 tag (`pl-PL`, `en_GB`, `PL`…). Only the primary subtag /// counts. No tag (or an empty one) is English, WordCraft's default editing language. @@ -64,12 +67,14 @@ impl ProofLang { ProofLang::Tr } else if primary.eq_ignore_ascii_case("ru") || primary.eq_ignore_ascii_case("rus") { ProofLang::Ru + } else if primary.eq_ignore_ascii_case("cs") || primary.eq_ignore_ascii_case("ces") || primary.eq_ignore_ascii_case("cze") { + ProofLang::Cs } else { ProofLang::Other } } - /// The primary language subtag (`en`, `pl`, `bg`, `tr`, `ru`); empty for [`ProofLang::Other`]. + /// The primary language subtag (`en`, `pl`, `bg`, `tr`, `ru`, `cs`); empty for [`ProofLang::Other`]. pub fn code(self) -> &'static str { match self { ProofLang::En => "en", @@ -80,6 +85,7 @@ impl ProofLang { ProofLang::Hr => "hr", ProofLang::Tr => "tr", ProofLang::Ru => "ru", + ProofLang::Cs => "cs", ProofLang::Other => "", } } @@ -159,6 +165,11 @@ mod tests { for t in ["it", "it-IT", "IT_it", "it-CH", "it-CH.UTF-8", " it-IT ", "ita"] { assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::It, "{t}"); } + for t in ["cs", "cs-CZ", "CS-cz", "cs_CZ", " ces ", "CES", "cze", "CZE"] { + assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::Cs, "{t}"); + } + assert_eq!(ProofLang::Cs.code(), "cs"); + assert!(ProofLang::Cs.is_proofed() && ProofLang::PROOFED.contains(&ProofLang::Cs)); for t in ["en", "en-US", "en-GB", "EN_au", "en-Latn-US"] { assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::En, "{t}"); } @@ -168,7 +179,7 @@ mod tests { for t in ["tr", "tr-TR", "tr-CY", "TR_tr", "tur"] { assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::Tr, "{t}"); } - for t in ["fr-FR", "de", "x-none", "ja-JP", "plx", "bgx", "trx", "e", "-"] { + for t in ["fr-FR", "de", "x-none", "ja-JP", "plx", "bgx", "trx", "csx", "e", "-"] { assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::Other, "{t}"); } assert_eq!(ProofLang::from_tag(None), ProofLang::En); diff --git a/crates/proof/src/lib.rs b/crates/proof/src/lib.rs index a0099c66..114097d4 100644 --- a/crates/proof/src/lib.rs +++ b/crates/proof/src/lib.rs @@ -28,6 +28,9 @@ //! words (`кто-то`, `по-русски`, `северо-западный`) understood, suggestions tuned for Russian //! (`е`/`ё`, `жи`/`ши`, `тся`/`ться`, voiced and voiceless pairs, missing spaces), Russian //! grammar rules with Russian messages, and Russian word division. +//! - **Czech** ([`ProofLang::Cs`], module [`cs`]): the installed Hunspell `cs_CZ` dictionary +//! (not bundled), dictionary-driven suggestions and generic checks with Czech messages. +//! Hyphenation uses the existing English fallback; no Czech patterns are bundled. //! - **Other languages** ([`ProofLang::Other`]): no spelling or grammar marks, like Word without //! the language's proofing tools. //! - **Hyphenation** ([`hyphen`]): dictionary, Liang patterns (English, Polish, Portuguese), @@ -38,6 +41,7 @@ pub mod affix; pub mod bg; +pub mod cs; pub mod dict; pub mod hr; pub mod hunspell; @@ -593,6 +597,7 @@ fn spelling_issue(text: &str, a: usize, b: usize, lang: ProofLang, opts: &Option ProofLang::It => "Possibile errore di ortografia", ProofLang::Tr => "Olası yazım hatası", ProofLang::Ru => "Возможна орфографическая ошибка", + ProofLang::Cs => "Možná pravopisná chyba", _ => "Possible spelling mistake", }; Some(Issue { start: a, end: b, kind: IssueKind::Spelling, message: message.into(), suggestions: Vec::new(), lang }) @@ -613,6 +618,7 @@ pub fn is_correct_in_with(word: &str, lang: ProofLang, opts: &Options) -> bool { ProofLang::It => accepted_by_options(word, opts) || it::is_correct(word), ProofLang::Tr => accepted_by_options(word, opts) || tr::is_correct(word), ProofLang::Ru => accepted_by_options(word, opts) || ru::is_correct(word), + ProofLang::Cs => accepted_by_options(word, opts) || cs::is_correct(word), ProofLang::Bg | ProofLang::Other => true, } } @@ -627,13 +633,14 @@ pub fn suggest_in(word: &str, max: usize, lang: ProofLang) -> Vec { ProofLang::It => it::suggest(word, max), ProofLang::Tr => tr::suggest(word, max), ProofLang::Ru => ru::suggest(word, max), + ProofLang::Cs => cs::suggest(word, max), ProofLang::Bg | ProofLang::Other => Vec::new(), } } /// Grammar issues in a paragraph of text in `lang`: the English rules, the Polish ones /// ([`pl::grammar`]), the Portuguese ones ([`pt::grammar`]), the Italian ones ([`it::grammar`]), -/// the Turkish ones ([`tr::grammar`]), or none for a language WordCraft doesn't proof. +/// the Turkish ones ([`tr::grammar`]), Czech generic checks ([`cs::grammar`]), or none for a language WordCraft doesn't proof. pub fn check_grammar_in(text: &str, lang: ProofLang) -> Vec { match lang { ProofLang::En => check_grammar(text), @@ -643,6 +650,7 @@ pub fn check_grammar_in(text: &str, lang: ProofLang) -> Vec { ProofLang::It => it::grammar::check(text), ProofLang::Tr => tr::grammar::check(text), ProofLang::Ru => ru::grammar::check(text), + ProofLang::Cs => cs::grammar::check(text), ProofLang::Bg | ProofLang::Other => Vec::new(), } } @@ -1290,7 +1298,7 @@ mod tests { let _ = has_dictionary(Some(&s)); let _ = is_correct_for(&s, Some(&s)); let _ = hyphen::hyphen_points(&s, &hyphen::Limits::default()); - for lang in [ProofLang::En, ProofLang::Pl, ProofLang::Pt, ProofLang::It, ProofLang::Hr, ProofLang::Tr, ProofLang::Ru, ProofLang::Other] { + for lang in [ProofLang::En, ProofLang::Pl, ProofLang::Pt, ProofLang::It, ProofLang::Hr, ProofLang::Tr, ProofLang::Ru, ProofLang::Cs, ProofLang::Other] { let _ = check_spelling_in(&s, lang); let _ = check_grammar_in(&s, lang); let _ = is_correct_in(&s, lang); @@ -1321,8 +1329,14 @@ mod tests { } #[test] - fn mixed_spans_never_panic(s in "\\PC{0,60}", cuts in proptest::collection::vec((0usize..80, 0usize..80, 0u8..7), 0..6)) { - let langs = [ProofLang::En, ProofLang::Pl, ProofLang::Pt, ProofLang::It, ProofLang::Tr, ProofLang::Ru, ProofLang::Other]; + fn czech_suggestions_never_panic(s in "[a-zěščřžýáíéůúĚŠČŘŽÝÁÍÉŮÚ'’ -]{0,14}") { + cs::test_dict::install(); + let _ = suggest_in(&s, 4, ProofLang::Cs); + } + + #[test] + fn mixed_spans_never_panic(s in "\\PC{0,60}", cuts in proptest::collection::vec((0usize..80, 0usize..80, 0u8..8), 0..6)) { + let langs = [ProofLang::En, ProofLang::Pl, ProofLang::Pt, ProofLang::It, ProofLang::Tr, ProofLang::Ru, ProofLang::Cs, ProofLang::Other]; let spans: Vec<(Range, ProofLang)> = cuts.iter().map(|&(a, b, l)| (a.min(b)..a.max(b), langs[l as usize % langs.len()])).collect(); for i in check_text(&s, &spans) { proptest::prop_assert!(i.start <= i.end && i.end <= s.len()); diff --git a/crates/proof/src/pl/suggest.rs b/crates/proof/src/pl/suggest.rs index 10cfc076..0e32a1ea 100644 --- a/crates/proof/src/pl/suggest.rs +++ b/crates/proof/src/pl/suggest.rs @@ -70,11 +70,11 @@ impl Collector<'_> { let mut words = Vec::new(); let mut cost = cost; for part in cand.split(' ') { - if self.d.contains(part) { + if self.d.contains_suggestible(part) { words.push(part.to_string()); } else { let c = capitalize(part); - if !self.d.contains(&c) { + if !self.d.contains_suggestible(&c) { return; } words.push(c); @@ -88,6 +88,9 @@ impl Collector<'_> { /// Keep a suggestion known to be correct. fn keep(&mut self, text: String, cost: u32) { + if !text.split(' ').all(|part| self.d.can_suggest(part)) { + return; + } self.seq += 1; let seq = self.seq; let e = self.found.entry(text).or_insert((cost, seq)); @@ -371,6 +374,11 @@ mod tests { assert!(s.iter().any(|x| x == want), "{word} → {s:?}, want {want}"); } + #[test] + fn nosuggest_stems_and_forms_are_never_offered() { + crate::affix::tests::assert_nosuggest_is_honoured(suggest_in); + } + #[test] fn polish_misspellings_get_the_right_suggestion() { crate::pl::test_dict::install(); diff --git a/crates/proof/src/pt/suggest.rs b/crates/proof/src/pt/suggest.rs index 314ef39b..36f1c7b0 100644 --- a/crates/proof/src/pt/suggest.rs +++ b/crates/proof/src/pt/suggest.rs @@ -73,11 +73,11 @@ impl Collector<'_> { let mut words = Vec::new(); let mut cost = cost; for part in cand.split(' ') { - if self.d.contains(part) { + if self.d.contains_suggestible(part) { words.push(part.to_string()); } else { let c = capitalize(part); - if !self.d.contains(&c) { + if !self.d.contains_suggestible(&c) { return; } words.push(c); @@ -90,6 +90,9 @@ impl Collector<'_> { /// Keep a suggestion known to be correct. fn keep(&mut self, text: String, cost: u32) { + if !text.split(' ').all(|part| self.d.can_suggest(part)) { + return; + } self.seq += 1; let seq = self.seq; let e = self.found.entry(text).or_insert((cost, seq)); @@ -373,6 +376,11 @@ mod tests { assert!(s.iter().any(|x| x == want), "{word} → {s:?}, want {want}"); } + #[test] + fn nosuggest_stems_and_forms_are_never_offered() { + crate::affix::tests::assert_nosuggest_is_honoured(suggest_in); + } + #[test] fn portuguese_misspellings_get_the_right_suggestion() { crate::pt::test_dict::install();