diff --git a/crates/proof/Cargo.toml b/crates/proof/Cargo.toml index ff3d96cc..abee1907 100644 --- a/crates/proof/Cargo.toml +++ b/crates/proof/Cargo.toml @@ -3,7 +3,7 @@ name = "wordcraft-proof" version.workspace = true edition.workspace = true license.workspace = true -description = "WordCraft proofing: spelling (public-domain word list), suggestions, grammar checks, hyphenation" +description = "WordCraft proofing per language: spelling, suggestions, grammar checks and hyphenation for English (public-domain word list) and Polish (installed Hunspell dictionary, TeX patterns)" [lints] workspace = true diff --git a/crates/proof/src/affix.rs b/crates/proof/src/affix.rs new file mode 100644 index 00000000..eeef432e --- /dev/null +++ b/crates/proof/src/affix.rs @@ -0,0 +1,845 @@ +//! Hunspell-style affix dictionaries — the subset the SJP.PL Polish dictionary (`pl_PL`) uses. +//! +//! A word is correct when it is a stem, a stem with one suffix, a stem with one prefix, or a stem +//! with a prefix and a suffix that both allow cross products; each rule applies only to stems +//! carrying its flag and ending (suffixes) or starting (prefixes) as its condition says. Highly +//! inflected languages store millions of forms this way in a few hundred thousand stems: the +//! Polish dictionary's ~350k stems and ~7,500 rules accept ~4.5 million forms. +//! +//! The rules and stems come from a Hunspell `.aff`/`.dic` pair ([`AffixDict::from_hunspell`]), +//! read at runtime from the dictionaries installed on the system ([`crate::hunspell`]). They can +//! also be stored in WordCraft's compact format ([`AffixDict::to_bytes`]): the rules as text +//! lines and the stems front-coded, raw-deflated together (the Polish dictionary is under 1 MB). +//! +//! Supported: one-character flags (at most 64 distinct), `PFX`/`SFX` with strip, append and +//! condition (`.`, letters, `[abc]`, `[^abc]`), cross products, and the suggestion tables `TRY`, +//! `MAP` and `REP`. Rejected: compounding, continuation classes, long or numeric flags, +//! `NEEDAFFIX`, `FORBIDDENWORD` and the other directives that change which words are correct. +//! +//! Lookups never panic and never allocate per probe: stems sit in one string, found through an +//! open-addressing hash table. + +use std::collections::HashMap; +use std::hash::Hasher; + +use crate::patterns::{FxHasher, FxMap}; + +const MAGIC: &str = "WCAFF1"; +/// Longest word (in bytes) looked up; longer input is never a dictionary word. +pub const MAX_WORD_BYTES: usize = 120; +/// Empty slot in the stem hash table. +const EMPTY: u32 = u32::MAX; + +/// Directives that change which words are correct in ways this engine doesn't implement. +const UNSUPPORTED: &[&str] = &[ + "AF", + "AM", + "CHECKSHARPS", + "CIRCUMFIX", + "COMPLEXPREFIXES", + "COMPOUNDBEGIN", + "COMPOUNDEND", + "COMPOUNDFLAG", + "COMPOUNDMIDDLE", + "COMPOUNDMIN", + "COMPOUNDPERMITFLAG", + "COMPOUNDRULE", + "FORBIDDENWORD", + "FULLSTRIP", + "ICONV", + "IGNORE", + "KEEPCASE", + "LEMMA_PRESENT", + "NEEDAFFIX", + "OCONV", + "ONLYINCOMPOUND", + "PSEUDOROOT", + "SUBSTANDARD", +]; + +/// One character class of a rule's condition. +#[derive(Clone, Debug, PartialEq, Eq)] +enum Cond { + Any, + Char(char), + Set { negated: bool, chars: Box<[char]> }, +} + +impl Cond { + fn matches(&self, c: char) -> bool { + match self { + Cond::Any => true, + Cond::Char(x) => *x == c, + Cond::Set { negated, chars } => chars.contains(&c) != *negated, + } + } +} + +/// Parse a Hunspell condition (`.`, `[^i]e`, `[km]anie`…). +fn parse_cond(text: &str) -> Result, String> { + if text == "." { + return Ok(Box::new([])); + } + let mut out = Vec::new(); + let mut chars = text.chars(); + while let Some(c) = chars.next() { + match c { + '.' => out.push(Cond::Any), + '[' => { + let mut set = Vec::new(); + let mut negated = false; + let mut closed = false; + let mut first = true; + for d in chars.by_ref() { + if d == ']' { + closed = true; + break; + } + if first && d == '^' { + negated = true; + } else { + set.push(d); + } + first = false; + } + if !closed { + return Err(format!("unclosed `[` in condition `{text}`")); + } + out.push(Cond::Set { negated, chars: set.into_boxed_slice() }); + } + ']' => return Err(format!("stray `]` in condition `{text}`")), + c => out.push(Cond::Char(c)), + } + } + Ok(out.into_boxed_slice()) +} + +/// One prefix or suffix rule. +#[derive(Clone, Debug)] +struct Rule { + /// Bit of the rule's flag in the stems' flag masks. + bit: u64, + flag: char, + cross: bool, + strip: Box, + add: Box, + cond: Box<[Cond]>, + cond_text: Box, +} + +impl Rule { + /// Does `root` end (suffix rule) or start (prefix rule) as the condition says? + fn cond_matches(&self, root: &str, suffix: bool) -> bool { + if self.cond.is_empty() { + return true; + } + if suffix { + let mut it = root.chars().rev(); + self.cond.iter().rev().all(|c| it.next().is_some_and(|x| c.matches(x))) + } else { + let mut it = root.chars(); + self.cond.iter().all(|c| it.next().is_some_and(|x| c.matches(x))) + } + } +} + +/// Suffix rules sharing an appended string and a stripped one: one stem lookup serves them all. +#[derive(Clone, Debug, Default)] +struct SfxGroup { + strip: Box, + rules: Vec, +} + +/// A stem list with affix rules (see the module docs). +#[derive(Clone, Debug, Default)] +pub struct AffixDict { + /// All stems, concatenated, sorted by bytes. + words: String, + offs: Vec, + /// Per stem: index into `sets`. + flags: Vec, + /// Distinct flag masks. + sets: Vec, + /// Open-addressing hash table of stem indices (`EMPTY` = free). + table: Vec, + /// Flag char of each bit. + flag_chars: Vec, + prefixes: Vec, + suffixes: Vec, + /// Appended string → groups of suffix rules by stripped string. + sfx_index: FxMap, Vec>, + /// Longest appended suffix, in chars. + max_sfx_chars: usize, + try_chars: String, + map: Vec>, + rep: Vec<(String, String)>, + /// Provenance and licence lines, kept through [`AffixDict::to_bytes`]. + comments: Vec, +} + +fn hash_str(s: &str) -> u64 { + let mut h = FxHasher::default(); + h.write(s.as_bytes()); + h.finish() +} + +/// Letter case of a word. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Casing { + /// No capitals (`kot`). + Lower, + /// First letter capital, no other (`Kot`). + Initial, + /// Every cased letter a capital, at least two (`KOT`). + Upper, + /// Anything else (`McDonald`, `kOT`). + Mixed, +} + +/// The casing of `w`. +pub fn casing(w: &str) -> Casing { + let (mut upper, mut lower, mut first_upper) = (0usize, 0usize, false); + for (i, c) in w.chars().enumerate() { + if c.is_uppercase() { + upper += 1; + if i == 0 { + first_upper = true; + } + } else if c.is_lowercase() { + lower += 1; + } + } + if upper == 0 { + Casing::Lower + } else if upper == 1 && first_upper { + Casing::Initial + } else if lower == 0 { + Casing::Upper + } else { + Casing::Mixed + } +} + +/// `w` with its first letter a capital. +pub fn capitalize(w: &str) -> String { + let mut c = w.chars(); + match c.next() { + Some(f) => f.to_uppercase().chain(c).collect(), + None => String::new(), + } +} + +/// Parsed rules and tables, shared by the Hunspell reader and the compact format. +#[derive(Default)] +struct Rules { + prefixes: Vec<(char, bool, String, String, String)>, + suffixes: Vec<(char, bool, String, String, String)>, + try_chars: String, + map: Vec>, + rep: Vec<(String, String)>, +} + +/// `0` means empty in Hunspell's strip and append fields. +fn zero_empty(s: &str) -> String { + if s == "0" { String::new() } else { s.to_string() } +} + +/// Read Hunspell `.aff` text (already decoded to UTF-8). +fn parse_aff(aff: &str) -> Result { + let mut r = Rules::default(); + let mut cross: HashMap<(bool, char), bool> = HashMap::new(); + let mut map_header = false; + let mut rep_header = false; + for (n, line) in aff.lines().enumerate() { + let f: Vec<&str> = line.split_whitespace().collect(); + let Some(&dir) = f.first() else { continue }; + if dir.starts_with('#') { + continue; + } + let at = |k: usize| f.get(k).copied().unwrap_or(""); + match dir { + "SET" | "LANG" | "NAME" | "VERSION" | "HOME" | "WORDCHARS" | "KEY" | "MAXNGRAMSUGS" | "MAXDIFF" | "ONLYMAXDIFF" | "NOSPLITSUGS" + | "SUGSWITHDOTS" | "NOSUGGEST" | "WARN" | "FORBIDWARN" | "MAXCPDSUGS" => {} + "FLAG" => return Err(format!("line {}: FLAG {} is not supported (one-character flags only)", n + 1, at(1))), + "TRY" => r.try_chars = at(1).to_string(), + "MAP" => { + if !map_header && at(1).parse::().is_ok() { + map_header = true; + } else if !at(1).contains('(') { + r.map.push(at(1).chars().collect()); + } + } + "REP" => { + if !rep_header && f.len() == 2 && at(1).parse::().is_ok() { + rep_header = true; + } else if f.len() >= 3 { + r.rep.push((at(1).replace('_', " "), at(2).replace('_', " "))); + } + } + "PFX" | "SFX" => { + let suffix = dir == "SFX"; + let mut flag_chars = at(1).chars(); + let (Some(flag), None) = (flag_chars.next(), flag_chars.next()) else { + return Err(format!("line {}: flag `{}` is not one character", n + 1, at(1))); + }; + let is_header = f.len() == 4 && matches!(at(2), "Y" | "N") && at(3).parse::().is_ok() && !cross.contains_key(&(suffix, flag)); + if is_header { + cross.insert((suffix, flag), at(2) == "Y"); + continue; + } + if f.len() < 4 { + return Err(format!("line {}: short affix rule", n + 1)); + } + let add = at(3); + if add.contains('/') { + return Err(format!("line {}: continuation classes are not supported", n + 1)); + } + let cond = if f.len() >= 5 { at(4) } else { "." }; + parse_cond(cond).map_err(|e| format!("line {}: {e}", n + 1))?; + let c = cross.get(&(suffix, flag)).copied().unwrap_or(false); + let rule = (flag, c, zero_empty(at(2)), zero_empty(add), cond.to_string()); + if suffix { + r.suffixes.push(rule); + } else { + r.prefixes.push(rule); + } + } + d if UNSUPPORTED.contains(&d) => return Err(format!("line {}: {d} is not supported", n + 1)), + _ => {} + } + } + Ok(r) +} + +impl AffixDict { + /// Build from Hunspell `.aff` and `.dic` text (decoded to UTF-8; the `.dic` count line and + /// morphological fields are ignored). `comments` (provenance, licence) are kept with the data. + pub fn from_hunspell(aff: &str, dic: &str, comments: &[&str]) -> Result { + let rules = parse_aff(aff)?; + let mut stems: Vec<(String, String)> = Vec::new(); + for (n, line) in dic.lines().enumerate() { + let Some(tok) = line.split_whitespace().next() else { continue }; + if n == 0 && tok.parse::().is_ok() { + continue; + } + // `\/` is a literal slash in a word; the first plain `/` starts the flags. + let mut word = String::new(); + let mut flags = String::new(); + let mut chars = tok.chars().peekable(); + let mut in_flags = false; + while let Some(c) = chars.next() { + if in_flags { + flags.push(c); + } else if c == '\\' && chars.peek() == Some(&'/') { + word.push('/'); + chars.next(); + } else if c == '/' { + in_flags = true; + } else { + word.push(c); + } + } + if word.is_empty() || word.len() > MAX_WORD_BYTES || word.contains(['\n', '/']) { + continue; + } + stems.push((word, flags)); + } + AffixDict::build(rules, stems, comments.iter().map(|s| s.to_string()).collect()) + } + + fn build(rules: Rules, mut stems: Vec<(String, String)>, comments: Vec) -> Result { + // Flags: every one a rule or a stem uses gets a bit. + let mut flag_chars: Vec = rules.prefixes.iter().chain(&rules.suffixes).map(|r| r.0).collect(); + flag_chars.extend(stems.iter().flat_map(|s| s.1.chars())); + flag_chars.sort_unstable(); + flag_chars.dedup(); + if flag_chars.len() > 64 { + return Err(format!("{} distinct flags; at most 64 are supported", flag_chars.len())); + } + let bit_of = |c: char| flag_chars.iter().position(|&x| x == c).map_or(0, |i| 1u64 << i); + // Stems sorted by bytes; homonyms (the same word twice) are merged. + stems.sort_by(|a, b| a.0.as_bytes().cmp(b.0.as_bytes())); + let mut d = AffixDict { + words: String::with_capacity(stems.iter().map(|s| s.0.len()).sum()), + offs: Vec::with_capacity(stems.len() + 1), + flag_chars: flag_chars.clone(), + try_chars: rules.try_chars, + map: rules.map, + rep: rules.rep, + comments, + ..AffixDict::default() + }; + d.offs.push(0); + let mut set_ids: HashMap = HashMap::new(); + let mut i = 0; + while i < stems.len() { + let Some((word, _)) = stems.get(i) else { break }; + let mut mask = 0u64; + let mut j = i; + while let Some((w, f)) = stems.get(j) + && w == word + { + mask |= f.chars().map(bit_of).fold(0, |a, b| a | b); + j += 1; + } + let next_id = u16::try_from(d.sets.len()).map_err(|_| "too many distinct flag sets".to_string())?; + let id = *set_ids.entry(mask).or_insert(next_id); + if usize::from(id) == d.sets.len() { + d.sets.push(mask); + } + d.words.push_str(word); + d.offs.push(u32::try_from(d.words.len()).map_err(|_| "stems too large".to_string())?); + d.flags.push(id); + i = j; + } + let mk = |(flag, cross, strip, add, cond): (char, bool, String, String, String)| -> Result { + Ok(Rule { bit: bit_of(flag), flag, cross, strip: strip.into(), add: add.into(), cond: parse_cond(&cond)?, cond_text: cond.into() }) + }; + d.prefixes = rules.prefixes.into_iter().map(mk).collect::>()?; + d.suffixes = rules.suffixes.into_iter().map(mk).collect::>()?; + d.index(); + Ok(d) + } + + /// Build the suffix index and the stem hash table. + fn index(&mut self) { + let mut idx: FxMap, Vec> = FxMap::default(); + for (i, r) in self.suffixes.iter().enumerate() { + let groups = idx.entry(r.add.clone()).or_default(); + let Ok(i) = u32::try_from(i) else { break }; + match groups.iter_mut().find(|g| g.strip == r.strip) { + Some(g) => g.rules.push(i), + None => groups.push(SfxGroup { strip: r.strip.clone(), rules: vec![i] }), + } + } + self.max_sfx_chars = self.suffixes.iter().map(|r| r.add.chars().count()).max().unwrap_or(0); + self.sfx_index = idx; + let n = self.len(); + let size = (n.max(1) * 2).next_power_of_two(); + let mut table = vec![EMPTY; size]; + let mask = size - 1; + for i in 0..n { + let Some(w) = self.word(i) else { continue }; + let mut slot = hash_str(w) as usize & mask; + while table.get(slot).is_some_and(|&t| t != EMPTY) { + slot = (slot + 1) & mask; + } + if let (Some(t), Ok(i)) = (table.get_mut(slot), u32::try_from(i)) { + *t = i; + } + } + self.table = table; + } + + /// Number of stems. + pub fn len(&self) -> usize { + self.flags.len() + } + + pub fn is_empty(&self) -> bool { + self.flags.is_empty() + } + + /// Number of prefix and suffix rules. + pub fn rule_count(&self) -> usize { + self.prefixes.len() + self.suffixes.len() + } + + fn word(&self, i: usize) -> Option<&str> { + let a = *self.offs.get(i)? as usize; + let b = *self.offs.get(i + 1)? as usize; + self.words.get(a..b) + } + + /// The stems, in byte order. + pub fn stems(&self) -> impl Iterator + '_ { + (0..self.len()).filter_map(|i| self.word(i)) + } + + /// The stems starting with `prefix`, in byte order (a binary search, then a scan). + pub fn stems_with_prefix<'a>(&'a self, prefix: &str) -> impl Iterator + use<'a> { + let prefix = prefix.to_string(); + let first = { + let (mut lo, mut hi) = (0usize, self.len()); + while lo < hi { + let mid = lo + (hi - lo) / 2; + if self.word(mid).is_some_and(|w| w.as_bytes() < prefix.as_bytes()) { + lo = mid + 1; + } else { + hi = mid; + } + } + lo + }; + (first..self.len()).map_while(move |i| self.word(i).filter(|w| w.starts_with(prefix.as_str()))) + } + + /// Flag mask of a stem, if `w` is one. + fn stem_flags(&self, w: &str) -> Option { + if self.table.is_empty() { + return None; + } + let mask = self.table.len() - 1; + let mut slot = hash_str(w) as usize & mask; + for _ in 0..self.table.len() { + let t = *self.table.get(slot)?; + if t == EMPTY { + return None; + } + if self.word(t as usize) == Some(w) { + let id = *self.flags.get(t as usize)?; + return self.sets.get(usize::from(id)).copied(); + } + slot = (slot + 1) & mask; + } + None + } + + /// Is `w` exactly (case and all) a stem or an affixed form of one? + pub fn contains(&self, w: &str) -> bool { + if w.is_empty() || w.len() > MAX_WORD_BYTES { + return false; + } + if self.stem_flags(w).is_some() || self.suffixed(w, 0) { + return true; + } + let mut root = String::with_capacity(w.len() + 8); + for r in &self.prefixes { + let Some(rest) = w.strip_prefix(&*r.add) else { continue }; + if rest.is_empty() { + continue; + } + root.clear(); + root.push_str(&r.strip); + root.push_str(rest); + if !r.cond_matches(&root, false) { + continue; + } + if self.stem_flags(&root).is_some_and(|f| f & r.bit != 0) { + return true; + } + if r.cross && self.suffixed(&root, r.bit) { + return true; + } + } + false + } + + /// Is `w` a stem with one suffix? With `need` set (a prefix was taken off), the stem must + /// carry those flags too and the suffix must allow cross products. + fn suffixed(&self, w: &str, need: u64) -> bool { + let mut root = String::with_capacity(w.len() + 8); + // Each split point from the end: the appended string is `w[b..]`, at most the longest + // suffix, and something must stay before it. + let cuts = std::iter::once(w.len()).chain(w.char_indices().rev().map(|(i, _)| i).filter(|&i| i > 0)); + for b in cuts.take(self.max_sfx_chars + 1) { + let (Some(base), Some(add)) = (w.get(..b), w.get(b..)) else { continue }; + let Some(groups) = self.sfx_index.get(add) else { continue }; + for g in groups { + root.clear(); + root.push_str(base); + root.push_str(&g.strip); + let Some(f) = self.stem_flags(&root) else { continue }; + if f & need != need { + continue; + } + let ok = g + .rules + .iter() + .filter_map(|&i| self.suffixes.get(i as usize)) + .any(|r| f & r.bit != 0 && (need == 0 || r.cross) && r.cond_matches(&root, true)); + if ok { + return true; + } + } + } + false + } + + /// Is `w` correct, Hunspell style: as written, or — for a capitalized word or one in capitals + /// — in lower case (and capitalized), since a sentence may start with any word and headings + /// may be in capitals. Lower-case input never matches a capitalized stem (`warszawa`). + pub fn check(&self, w: &str) -> bool { + if self.contains(w) { + return true; + } + match casing(w) { + Casing::Lower | Casing::Mixed => false, + Casing::Initial => self.contains(&w.to_lowercase()), + Casing::Upper => { + let lower = w.to_lowercase(); + self.contains(&capitalize(&lower)) || self.contains(&lower) + } + } + } + + /// Every form of the stem `stem` (itself included), unsorted, possibly with repeats. + pub fn forms(&self, stem: &str) -> Vec { + let Some(f) = self.stem_flags(stem) else { return Vec::new() }; + let mut out = vec![stem.to_string()]; + let mut cross_sfx = Vec::new(); + for r in &self.suffixes { + if f & r.bit == 0 || !r.cond_matches(stem, true) { + continue; + } + let Some(base) = stem.strip_suffix(&*r.strip) else { continue }; + let form = format!("{base}{}", r.add); + if r.cross { + cross_sfx.push(form.clone()); + } + out.push(form); + } + for r in &self.prefixes { + if f & r.bit == 0 || !r.cond_matches(stem, false) { + continue; + } + let Some(rest) = stem.strip_prefix(&*r.strip) else { continue }; + out.push(format!("{}{rest}", r.add)); + if r.cross { + for s in &cross_sfx { + if let Some(rest) = s.strip_prefix(&*r.strip) { + out.push(format!("{}{rest}", r.add)); + } + } + } + } + out + } + + /// The `TRY` letters, most frequent first (suggestions try these). + pub fn try_chars(&self) -> &str { + &self.try_chars + } + + /// The `MAP` groups: letters often confused with each other (`aą`, `zżź`, `oóu`…). + pub fn map(&self) -> &[Vec] { + &self.map + } + + /// The `REP` table: common misspellings, `(wrong, right)`. + pub fn rep(&self) -> &[(String, String)] { + &self.rep + } + + /// The provenance and licence lines. + pub fn comments(&self) -> &[String] { + &self.comments + } + + /// Serialize to WordCraft's compact format: raw-deflate of `WCAFF1`, the comments, the + /// tables and rules as text lines, a `WORDS` line, then one line per stem, front-coded + /// against the previous one (a byte with the shared prefix length in bytes, the rest of the + /// word, `/` and its flags if any). + pub fn to_bytes(&self) -> Vec { + let mut out = String::new(); + out.push_str(MAGIC); + out.push('\n'); + for c in &self.comments { + out.push_str("# "); + out.push_str(c.trim()); + out.push('\n'); + } + if !self.try_chars.is_empty() { + out.push_str(&format!("TRY {}\n", self.try_chars)); + } + for m in &self.map { + out.push_str(&format!("MAP {}\n", m.iter().collect::())); + } + for (a, b) in &self.rep { + out.push_str(&format!("REP {} {}\n", a.replace(' ', "_"), b.replace(' ', "_"))); + } + let z = |s: &str| if s.is_empty() { "0".to_string() } else { s.to_string() }; + for (kind, rules) in [("PFX", &self.prefixes), ("SFX", &self.suffixes)] { + for r in rules { + let y = if r.cross { "Y" } else { "N" }; + out.push_str(&format!("{kind} {} {y} {} {} {}\n", r.flag, z(&r.strip), z(&r.add), r.cond_text)); + } + } + out.push_str("WORDS\n"); + let mut raw = out.into_bytes(); + let mut prev: &[u8] = &[]; + for i in 0..self.len() { + let Some(w) = self.word(i) else { continue }; + let b = w.as_bytes(); + let p = prev.iter().zip(b).take_while(|(x, y)| x == y).count().min(255); + raw.push(p as u8); + raw.extend_from_slice(b.get(p..).unwrap_or_default()); + let mask = self.flags.get(i).and_then(|&id| self.sets.get(usize::from(id))).copied().unwrap_or(0); + if mask != 0 { + raw.push(b'/'); + for (bit, c) in self.flag_chars.iter().enumerate() { + if mask & (1 << bit) != 0 { + let mut buf = [0u8; 4]; + raw.extend_from_slice(c.encode_utf8(&mut buf).as_bytes()); + } + } + } + raw.push(b'\n'); + prev = b; + } + miniz_oxide::deflate::compress_to_vec(&raw, 10) + } + + /// Read data written by [`AffixDict::to_bytes`]. + pub fn from_bytes(data: &[u8]) -> Result { + let raw = + miniz_oxide::inflate::decompress_to_vec_with_limit(data, 64 << 20).map_err(|e| format!("affix dictionary: inflate failed: {e:?}"))?; + let words_at = raw.windows(7).position(|w| w == b"\nWORDS\n").ok_or("affix dictionary: no WORDS section")?; + let head = std::str::from_utf8(raw.get(..words_at).unwrap_or_default()).map_err(|_| "affix dictionary: bad utf-8 in the rules")?; + let mut lines = head.lines(); + if lines.next() != Some(MAGIC) { + return Err("affix dictionary: bad header".into()); + } + let mut rules = Rules::default(); + let mut comments = Vec::new(); + for line in lines { + if let Some(c) = line.strip_prefix("# ") { + comments.push(c.to_string()); + continue; + } + let f: Vec<&str> = line.split_whitespace().collect(); + let at = |k: usize| f.get(k).copied().unwrap_or(""); + match at(0) { + "TRY" => rules.try_chars = at(1).to_string(), + "MAP" => rules.map.push(at(1).chars().collect()), + "REP" => rules.rep.push((at(1).replace('_', " "), at(2).replace('_', " "))), + kind @ ("PFX" | "SFX") => { + let mut fc = at(1).chars(); + let (Some(flag), None) = (fc.next(), fc.next()) else { return Err(format!("affix dictionary: bad flag in `{line}`")) }; + if f.len() != 6 { + return Err(format!("affix dictionary: bad rule `{line}`")); + } + let rule = (flag, at(2) == "Y", zero_empty(at(3)), zero_empty(at(4)), at(5).to_string()); + if kind == "SFX" { + rules.suffixes.push(rule); + } else { + rules.prefixes.push(rule); + } + } + "" => {} + other => return Err(format!("affix dictionary: unknown line `{other}`")), + } + } + let body = raw.get(words_at + 7..).unwrap_or_default(); + let mut stems: Vec<(String, String)> = Vec::new(); + let mut cur: Vec = Vec::new(); + let mut i = 0; + while let Some(&p) = body.get(i) { + let rest = body.get(i + 1..).unwrap_or_default(); + let len = rest.iter().position(|&b| b == b'\n').ok_or("affix dictionary: truncated")?; + let line = rest.get(..len).unwrap_or_default(); + i += len + 2; + let p = p as usize; + if p > cur.len() { + return Err("affix dictionary: bad prefix".into()); + } + let (tail, flags) = match line.iter().position(|&b| b == b'/') { + Some(k) => (line.get(..k).unwrap_or_default(), line.get(k + 1..).unwrap_or_default()), + None => (line, &[][..]), + }; + cur.truncate(p); + cur.extend_from_slice(tail); + let word = std::str::from_utf8(&cur).map_err(|_| "affix dictionary: bad utf-8 in a stem")?; + let flags = std::str::from_utf8(flags).map_err(|_| "affix dictionary: bad utf-8 in flags")?; + if word.is_empty() || word.len() > MAX_WORD_BYTES { + return Err("affix dictionary: bad stem".into()); + } + stems.push((word.to_string(), flags.to_string())); + } + AffixDict::build(rules, stems, comments) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const AFF: &str = "SET UTF-8\nTRY aeoiułkr\n\nPFX b Y 1\nPFX b 0 nie .\n\nSFX a Y 3\nSFX a 0 y [^y]\nSFX a ek ka ek\nSFX a 0 em .\n\nSFX c N 1\nSFX c a ą a\n\nMAP 2\nMAP aą\nMAP oóu\nREP 2\nREP ż rz\nREP u ó\n"; + const DIC: &str = "4\nkot/ab\npiesek/a\nkrowa/c\nWarszawa/c\n"; + + fn dict() -> AffixDict { + AffixDict::from_hunspell(AFF, DIC, &["test data"]).unwrap() + } + + #[test] + fn stems_suffixes_prefixes_and_cross_products() { + let d = dict(); + assert_eq!(d.len(), 4); + for w in ["kot", "koty", "kotem", "piesek", "pieska", "piesekem", "krowa", "krową", "niekot", "niekoty", "niekotem", "Warszawa", "Warszawą"] + { + assert!(d.contains(w), "{w}"); + } + for w in ["kotą", "pieseka", "pieskem", "niepiesek", "niekrową", "krowy", "warszawa", "", "nie", "y", "ka"] { + assert!(!d.contains(w), "{w}"); + } + } + + #[test] + fn case_variants() { + let d = dict(); + assert!(d.check("Kot") && d.check("KOT") && d.check("Koty") && d.check("WARSZAWĄ") && d.check("Warszawą")); + assert!(!d.check("warszawa") && !d.check("kOT") && !d.check("KoT")); + assert_eq!(casing("Kot"), Casing::Initial); + assert_eq!(casing("KOT"), Casing::Upper); + assert_eq!(casing("K"), Casing::Initial); + assert_eq!(casing("kot"), Casing::Lower); + assert_eq!(casing("McDonald"), Casing::Mixed); + assert_eq!(capitalize("żółw"), "Żółw"); + } + + #[test] + fn forms_are_all_accepted() { + let d = dict(); + let mut forms = d.forms("kot"); + forms.sort(); + forms.dedup(); + assert_eq!(forms, ["kot", "kotem", "koty", "niekot", "niekotem", "niekoty"]); + for stem in d.stems().map(str::to_string).collect::>() { + for f in d.forms(&stem) { + assert!(d.contains(&f), "{stem} → {f}"); + } + } + } + + #[test] + fn compact_format_round_trips() { + let d = dict(); + let back = AffixDict::from_bytes(&d.to_bytes()).unwrap(); + assert_eq!(back.len(), d.len()); + assert_eq!(back.rule_count(), d.rule_count()); + assert_eq!(back.comments(), ["test data"]); + assert_eq!(back.rep(), d.rep()); + assert_eq!(back.map(), d.map()); + assert_eq!(back.try_chars(), "aeoiułkr"); + for w in ["kotem", "niekoty", "pieska", "krową", "Warszawą"] { + assert!(back.contains(w), "{w}"); + } + assert!(!back.contains("kotą")); + } + + #[test] + fn unsupported_and_broken_input_is_an_error() { + assert!(AffixDict::from_hunspell("FLAG long\n", DIC, &[]).is_err()); + assert!(AffixDict::from_hunspell("COMPOUNDFLAG X\n", DIC, &[]).is_err()); + assert!(AffixDict::from_hunspell("SFX a Y 1\nSFX a 0 y/b .\n", DIC, &[]).is_err()); + assert!(AffixDict::from_hunspell("SFX a Y 1\nSFX a 0 y [ab\n", DIC, &[]).is_err()); + assert!(AffixDict::from_bytes(b"not deflate").is_err()); + let ok = dict().to_bytes(); + assert!(AffixDict::from_bytes(&ok[..ok.len() / 2]).is_err()); + let garbage = miniz_oxide::deflate::compress_to_vec(b"WCAFF1\nWORDS\n\x09abc\n", 6); + assert!(AffixDict::from_bytes(&garbage).is_err(), "prefix longer than the previous word"); + let empty = AffixDict::default(); + assert!(!empty.contains("kot") && !empty.check("Kot") && empty.forms("kot").is_empty()); + } + + proptest::proptest! { + #[test] + fn lookups_never_panic(s in "\\PC{0,40}", bytes in proptest::collection::vec(proptest::num::u8::ANY, 0..200)) { + let d = dict(); + let _ = d.check(&s); + let _ = d.forms(&s); + let _ = AffixDict::from_bytes(&bytes); + let _ = AffixDict::from_bytes(&miniz_oxide::deflate::compress_to_vec(&bytes, 1)); + } + } +} diff --git a/crates/proof/src/hunspell.rs b/crates/proof/src/hunspell.rs new file mode 100644 index 00000000..2c09b11d --- /dev/null +++ b/crates/proof/src/hunspell.rs @@ -0,0 +1,163 @@ +//! Hunspell dictionaries installed on the system, read at runtime. +//! +//! WordCraft doesn't bundle large spelling dictionaries. For languages with an affix dictionary +//! (Polish), it reads the standard Hunspell `.aff`/`.dic` pair installed by the operating system, +//! a package manager or LibreOffice, with [`crate::affix::AffixDict::from_hunspell`]. When none is +//! installed the language has no spelling check (no marks, no errors), like Word without the +//! language's proofing tools. +//! +//! Search order: the directory in `WORDCRAFT_DICTIONARIES` (or one set with +//! [`set_dictionary_dir`]), then the usual system locations. On the web there are none. + +use std::path::{Path, PathBuf}; +use std::sync::RwLock; + +use crate::affix::AffixDict; + +/// Largest `.aff` or `.dic` file read (real ones are a few MB). +const MAX_FILE: u64 = 64 << 20; + +/// An extra directory searched first (a setting), set by the front end. +static EXTRA_DIR: RwLock> = RwLock::new(None); + +/// Search `dir` before the system locations (a "dictionaries folder" setting); `None` clears it. +/// Takes effect for dictionaries not loaded yet. +pub fn set_dictionary_dir(dir: Option) { + *EXTRA_DIR.write().unwrap_or_else(|e| e.into_inner()) = dir; +} + +/// Directories that may hold Hunspell dictionaries, most specific first. +pub fn search_dirs() -> Vec { + let mut dirs: Vec = Vec::new(); + if let Some(d) = EXTRA_DIR.read().unwrap_or_else(|e| e.into_inner()).clone() { + dirs.push(d); + } + if cfg!(target_arch = "wasm32") { + return dirs; + } + if let Some(d) = std::env::var_os("WORDCRAFT_DICTIONARIES").filter(|v| !v.is_empty()) { + dirs.push(PathBuf::from(d)); + } + let home = std::env::var_os("HOME").filter(|v| !v.is_empty()).map(PathBuf::from); + if cfg!(windows) { + for var in ["LOCALAPPDATA", "APPDATA"] { + if let Some(base) = std::env::var_os(var).filter(|v| !v.is_empty()).map(PathBuf::from) { + dirs.push(base.join("WordCraft").join("dictionaries")); + dirs.push(base.join("hunspell")); + } + } + for var in ["ProgramFiles", "ProgramFiles(x86)"] { + if let Some(base) = std::env::var_os(var).filter(|v| !v.is_empty()).map(PathBuf::from) { + dirs.push(base.join("LibreOffice").join("share").join("extensions").join("dict-pl")); + } + } + } else if cfg!(target_os = "macos") { + if let Some(h) = &home { + dirs.push(h.join("Library/Spelling")); + } + dirs.push(PathBuf::from("/Library/Spelling")); + dirs.push(PathBuf::from("/opt/homebrew/share/hunspell")); + dirs.push(PathBuf::from("/usr/local/share/hunspell")); + dirs.push(PathBuf::from("/Applications/LibreOffice.app/Contents/Resources/extensions/dict-pl")); + } else { + if let Some(h) = &home { + dirs.push(h.join(".local/share/hunspell")); + } + for d in [ + "/usr/share/hunspell", + "/usr/share/myspell", + "/usr/share/myspell/dicts", + "/usr/local/share/hunspell", + "/usr/local/share/myspell", + "/usr/lib/libreoffice/share/extensions/dict-pl", + "/opt/libreoffice/share/extensions/dict-pl", + ] { + dirs.push(PathBuf::from(d)); + } + } + dirs +} + +/// The first `.aff`/`.dic` pair found for any of `names` (`pl_PL`, `pl`), searching +/// [`search_dirs`]. +pub fn find(names: &[&str]) -> Option<(PathBuf, PathBuf)> { + search_dirs().into_iter().find_map(|dir| find_in(&dir, names)) +} + +/// The first `.aff`/`.dic` pair for any of `names` in `dir`. +pub fn find_in(dir: &Path, names: &[&str]) -> Option<(PathBuf, PathBuf)> { + names.iter().find_map(|n| { + let aff = dir.join(format!("{n}.aff")); + let dic = dir.join(format!("{n}.dic")); + (aff.is_file() && dic.is_file()).then_some((aff, dic)) + }) +} + +/// Read a Hunspell pair from disk (size-capped), decoded per the `.aff`'s `SET` line. +pub fn load(aff: &Path, dic: &Path) -> Result { + let read = |p: &Path| -> Result, String> { + let len = std::fs::metadata(p).map_err(|e| format!("{}: {e}", p.display()))?.len(); + if len > MAX_FILE { + return Err(format!("{}: too large ({len} bytes)", p.display())); + } + std::fs::read(p).map_err(|e| format!("{}: {e}", p.display())) + }; + let (aff_text, dic_text) = decode(&read(aff)?, &read(dic)?)?; + let source = format!("Hunspell dictionary {}", dic.display()); + AffixDict::from_hunspell(&aff_text, &dic_text, &[source.as_str()]) +} + +/// ISO 8859-2 code points 0xA0..=0xFF (0x00..=0x9F map to themselves). +const ISO_8859_2_HIGH: &str = "\u{a0}Ą˘Ł¤ĽŚ§¨ŠŞŤŹ\u{ad}ŽŻ°ą˛ł´ľśˇ¸šşťź˝žżŔÁÂĂÄĹĆÇČÉĘËĚÍÎĎĐŃŇÓÔŐÖ×ŘŮÚŰÜÝŢßŕáâăäĺćçčéęëěíîďđńňóôőö÷řůúűüýţ˙"; + +fn decode_latin2(bytes: &[u8]) -> String { + let high: Vec = ISO_8859_2_HIGH.chars().collect(); + bytes.iter().map(|&b| if b < 0xa0 { char::from(b) } else { high.get(usize::from(b - 0xa0)).copied().unwrap_or('\u{fffd}') }).collect() +} + +/// Decode an `.aff`/`.dic` pair to UTF-8 from the encoding the `.aff`'s `SET` line names +/// (UTF-8 when there is none; ISO 8859-2 is the other one Polish dictionaries use). +pub fn decode(aff: &[u8], dic: &[u8]) -> Result<(String, String), String> { + let head = String::from_utf8_lossy(aff.get(..aff.len().min(8192)).unwrap_or_default()).to_string(); + let set = head.lines().find_map(|l| l.trim_start_matches('\u{feff}').strip_prefix("SET ")).map(str::trim).unwrap_or("UTF-8").to_ascii_uppercase(); + match set.as_str() { + "UTF-8" | "UTF8" => { + let text = |b: &[u8], what: &str| String::from_utf8(b.to_vec()).map_err(|e| format!("{what}: not UTF-8: {e}")); + Ok((text(aff, ".aff")?, text(dic, ".dic")?)) + } + "ISO8859-2" | "ISO-8859-2" => Ok((decode_latin2(aff), decode_latin2(dic))), + other => Err(format!("unsupported encoding {other}")), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn decodes_latin2_and_utf8() { + let aff = b"SET ISO8859-2\nTRY a\n"; + let (a, d) = decode(aff, b"2\n\xbf\xf3\xb3w\n").unwrap(); + assert!(a.contains("TRY a")); + assert_eq!(d, "2\nżółw\n"); + let (_, d) = decode("SET UTF-8\n".as_bytes(), "żółw".as_bytes()).unwrap(); + assert_eq!(d, "żółw"); + assert!(decode(b"SET KOI8-R\n", b"").is_err()); + assert!(decode(b"SET UTF-8\n", b"\xff\xfe").is_err()); + } + + #[test] + fn finds_and_loads_a_pair_in_a_directory() { + let dir = std::env::temp_dir().join(format!("wordcraft-hunspell-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + assert!(find_in(&dir, &["xx_XX"]).is_none(), "no files: nothing found"); + std::fs::write(dir.join("xx_XX.aff"), "SET UTF-8\nSFX A Y 1\nSFX A 0 y .\n").unwrap(); + assert!(find_in(&dir, &["xx_XX"]).is_none(), "an .aff alone isn't a dictionary"); + std::fs::write(dir.join("xx_XX.dic"), "1\nkot/A\n").unwrap(); + let (aff, dic) = find_in(&dir, &["yy", "xx_XX"]).unwrap(); + let d = load(&aff, &dic).unwrap(); + assert!(d.check("kot") && d.check("koty") && !d.check("kotx")); + assert!(load(&dir.join("missing.aff"), &dic).is_err()); + let _ = std::fs::remove_dir_all(&dir); + } +} diff --git a/crates/proof/src/hyphen.rs b/crates/proof/src/hyphen.rs index 358502e9..6462938b 100644 --- a/crates/proof/src/hyphen.rs +++ b/crates/proof/src/hyphen.rs @@ -1,6 +1,7 @@ -//! English hyphenation. +//! Hyphenation, per proofing language ([`ProofLang`]). //! -//! Lookup order for each word: +//! **English** (and languages without patterns of their own, as before languages were told +//! apart), lookup order for each word: //! 1. the **dictionary** ([`Dictionary::en_us`]) — the public-domain Moby Hyphenator II word list //! by Grady Ward (~160k words), stored compactly in `assets/hyphenation/en-us.dic`; simple //! inflections (`'s`, `-s`, `-es`, `-ed`, `-d`) inherit the breaks of their stem; @@ -8,6 +9,10 @@ //! set, trained from the same list by DesignCraft's patgen (generated by DesignCraft's `hyphgen` example); //! 3. a small syllable **heuristic** for words outside the pattern alphabet. //! +//! **Polish**: the exception words, then the TeX Polish patterns ([`Patterns::pl`], by Hanna +//! Kołodziejska, Bogusław Jackowski and Marek Ryćko, MIT licence option), then a Polish syllable +//! heuristic for words with letters outside the Polish alphabet (`é`, `ü`…). +//! //! Then the paragraph's limits ([`Limits`]) are applied. pub use crate::dict; @@ -16,11 +21,14 @@ pub use crate::patterns; use std::sync::OnceLock; +pub use crate::lang::ProofLang; pub use dict::Dictionary; pub use patterns::Patterns; static EN_US_DIC: &[u8] = include_bytes!("../../../assets/hyphenation/en-us.dic"); static EN_US_PAT: &str = include_str!("../../../assets/hyphenation/en-us.pat"); +/// hyph-utf8's `hyph-pl.tex`, unmodified (its licence header is in the file). +static PL_PAT: &str = include_str!("../../../assets/hyphenation/pl.pat"); impl Dictionary { /// The bundled US English dictionary (decoded on first use). @@ -44,6 +52,12 @@ impl Patterns { static P: OnceLock = OnceLock::new(); P.get_or_init(|| Patterns::parse(EN_US_PAT)) } + + /// The bundled Polish patterns and exception words (parsed on first use). + pub fn pl() -> &'static Patterns { + static P: OnceLock = OnceLock::new(); + P.get_or_init(|| Patterns::parse(PL_PAT)) + } } /// Hyphenation limits from the paragraph's Hyphenation settings. @@ -79,11 +93,14 @@ fn is_word_char(c: char) -> bool { } /// Unrestricted break points of one lowercase core (letters and `'` only, no leading/trailing `'`). -fn core_breaks(core: &[char]) -> (Vec, Source) { +fn core_breaks(core: &[char], lang: ProofLang) -> (Vec, Source) { let n = core.len(); if n < 2 { return (vec![], Source::None); } + if lang == ProofLang::Pl { + return core_breaks_pl(core); + } let dict = Dictionary::en_us(); let s: String = core.iter().collect(); let from_mask = |m: u64, len: usize| (1..len.min(64)).filter(|&i| m & (1 << i) != 0).collect::>(); @@ -108,10 +125,30 @@ fn core_breaks(core: &[char]) -> (Vec, Source) { (heuristic_points(core), Source::Heuristic) } -/// Unrestricted break points of `word` (char indices; a hyphen goes before that char), handling -/// case, apostrophes (`don't`, `’s`), surrounding punctuation and compounds (each letter run is -/// hyphenated separately). No break is ever placed next to an apostrophe or non-letter. +/// Polish: exception words, then the patterns, then the Polish syllable heuristic. +fn core_breaks_pl(core: &[char]) -> (Vec, Source) { + let pats = Patterns::pl(); + let s: String = core.iter().collect(); + if let Some(p) = pats.exception(&s) { + return (p.to_vec(), Source::Dictionary); + } + if !pats.is_empty() + && let Some(p) = pats.points(core) + { + return (p, Source::Patterns); + } + (heuristic_points_pl(core), Source::Heuristic) +} + +/// Unrestricted break points of an English `word` ([`word_breaks_in`] with [`ProofLang::En`]). pub fn word_breaks(word: &str) -> (Vec, Source) { + word_breaks_in(word, ProofLang::En) +} + +/// Unrestricted break points of `word` in `lang` (char indices; a hyphen goes before that char), +/// handling case, apostrophes (`don't`, `’s`), surrounding punctuation and compounds (each letter +/// run is hyphenated separately). No break is ever placed next to an apostrophe or non-letter. +pub fn word_breaks_in(word: &str, lang: ProofLang) -> (Vec, Source) { let chars: Vec = word.chars().collect(); let mut out = Vec::new(); let mut src = Source::None; @@ -133,7 +170,7 @@ pub fn word_breaks(word: &str) -> (Vec, Source) { b -= 1; } let core: Vec = chars[a..b].iter().map(|&c| if is_apostrophe(c) { '\'' } else { c.to_lowercase().next().unwrap_or(c) }).collect(); - let (pts, s) = core_breaks(&core); + let (pts, s) = core_breaks(&core, lang); if src == Source::None { src = s; } @@ -143,11 +180,16 @@ pub fn word_breaks(word: &str) -> (Vec, Source) { (out, src) } -/// Allowed hyphenation points in `word` as char indices (a hyphen goes before that char), with -/// the paragraph limits applied to each letter run. +/// Allowed hyphenation points in an English `word` ([`hyphen_points_in`] with [`ProofLang::En`]). pub fn hyphen_points(word: &str, lim: &Limits) -> Vec { + hyphen_points_in(word, lim, ProofLang::En) +} + +/// Allowed hyphenation points in `word`, a word in `lang`, as char indices (a hyphen goes before +/// that char), with the paragraph limits applied to each letter run. +pub fn hyphen_points_in(word: &str, lim: &Limits, lang: ProofLang) -> Vec { let chars: Vec = word.chars().collect(); - let (pts, _) = word_breaks(word); + let (pts, _) = word_breaks_in(word, lang); if pts.is_empty() { return pts; } @@ -208,7 +250,12 @@ pub fn hyphen_points(word: &str, lim: &Limits) -> Vec { /// `word` with `-` at every hyphenation point (tests and diagnostics). pub fn hyphenate_word(word: &str, lim: &Limits) -> String { - let pts = hyphen_points(word, lim); + hyphenate_word_in(word, lim, ProofLang::En) +} + +/// `word`, a word in `lang`, with `-` at every hyphenation point (tests and diagnostics). +pub fn hyphenate_word_in(word: &str, lim: &Limits, lang: ProofLang) -> String { + let pts = hyphen_points_in(word, lim, lang); let mut s = String::with_capacity(word.len() + pts.len()); for (i, ch) in word.chars().enumerate() { if pts.contains(&i) { @@ -221,8 +268,14 @@ pub fn hyphenate_word(word: &str, lim: &Limits) -> String { // ---------- heuristic fallback (words outside the pattern alphabet) ---------- +/// Letters with a diacritic that are consonants (`ł`, `ż`, `ç`, `ñ`…); any other non-ASCII letter +/// counts as a vowel (`é`, `ø`, `ą`…), and so do letters of other scripts, which keeps the +/// heuristic from breaking them. +const NON_ASCII_CONSONANTS: &str = "łńśćźżçčďđğģķļľĺňñņřŕšşșťţțžßŋðþ"; + fn is_vowel(c: char) -> bool { - matches!(c.to_ascii_lowercase(), 'a' | 'e' | 'i' | 'o' | 'u' | 'y') || (c.is_alphabetic() && !c.is_ascii()) + let l = c.to_lowercase().next().unwrap_or(c); + if l.is_ascii() { matches!(l, 'a' | 'e' | 'i' | 'o' | 'u' | 'y') } else { l.is_alphabetic() && !NON_ASCII_CONSONANTS.contains(l) } } fn onset_pair(a: char, b: char) -> bool { @@ -286,6 +339,52 @@ fn heuristic_points(c: &[char]) -> Vec { out } +/// Polish digraphs, one sound each: never split. +fn is_digraph_pl(a: char, b: char) -> bool { + matches!((a, b), ('c', 'h' | 'z') | ('d', 'z' | 'ź' | 'ż') | ('r' | 's', 'z')) +} + +/// Polish syllable heuristic, for words with letters outside the Polish pattern alphabet (`é`, +/// `ü`…): one break between each pair of syllable nuclei — after a leading sonorant (`kon-ty`, +/// `Pol-ska`) or inside a doubled consonant (`Pöp-pel`), else before the consonant cluster +/// (`sio-stra`). Digraphs (`ch`, `cz`, `dz`, `dź`, `dż`, `rz`, `sz`) stay whole, and an `i` that +/// only softens the consonant before a vowel (`nia`, `sio`) stays with that consonant. +fn heuristic_points_pl(c: &[char]) -> Vec { + let n = c.len(); + if n < 4 || !c.iter().all(|c| c.is_alphabetic()) { + return vec![]; + } + let v: Vec = c.iter().map(|&x| is_vowel(x)).collect(); + let at = |i: usize| v.get(i).copied().unwrap_or(false); + // A syllable nucleus: a vowel, except a softening `i` (after a consonant, before a vowel). + let nucleus: Vec = (0..n).map(|i| at(i) && !(c.get(i) == Some(&'i') && i > 0 && !at(i - 1) && at(i + 1))).collect(); + let nuclei: Vec = (0..n).filter(|&i| nucleus.get(i).copied().unwrap_or(false)).collect(); + let mut out = vec![]; + for pair in nuclei.windows(2) { + let [a, b] = pair else { continue }; + let (start, end) = (a + 1, *b); + if start >= end { + continue; // adjacent vowels (na-u-ka): no break inside a vowel pair here + } + // Consonant units of the cluster: digraphs and a softening `i` join their consonant. + let mut units: Vec = Vec::new(); + let mut i = start; + while i < end { + units.push(i); + i += if i + 1 < end && c.get(i).zip(c.get(i + 1)).is_some_and(|(x, y)| is_digraph_pl(*x, *y)) { 2 } else { 1 }; + if i < end && c.get(i) == Some(&'i') { + i += 1; + } + } + let sonorant = |i: usize| matches!(c.get(i), Some('l' | 'ł' | 'm' | 'n' | 'ń' | 'r' | 'j')); + let first_is_single = units.get(1).is_some_and(|&second| second == start + 1); + // A doubled consonant splits (`lek-ki`, `Pöp-pel`), as does a cluster after a sonorant. + let doubled = c.get(start).is_some_and(|x| c.get(start + 1) == Some(x)); + out.push(if units.len() >= 2 && first_is_single && (sonorant(start) || doubled) { start + 1 } else { start }); + } + out +} + #[cfg(test)] mod tests { use super::*; @@ -527,4 +626,76 @@ mod tests { let c: Vec = "happen".chars().collect(); assert_eq!(heuristic_points(&c), vec![3]); } + + #[test] + fn heuristic_knows_consonants_with_diacritics() { + // `ł`, `ż`, `ç` are consonants (every non-ASCII letter used to count as a vowel). + assert!(!is_vowel('ł') && !is_vowel('Ż') && !is_vowel('ç') && is_vowel('é') && is_vowel('ą') && is_vowel('ø')); + let c: Vec = "façade".chars().collect(); + assert_eq!(heuristic_points(&c), vec![2]); + } + + fn pl(w: &str) -> String { + hyphenate_word_in(w, &Limits::default(), ProofLang::Pl) + } + + #[test] + fn polish_patterns_load() { + let p = Patterns::pl(); + assert!(p.len() > 4000, "{}", p.len()); + assert_eq!(p.exception_count(), 20); + assert_eq!(p.exception("nigdy"), Some(&[2][..])); + } + + #[test] + fn polish_words_hyphenate_by_polish_rules() { + for (w, want) in [ + ("konstytucja", "kon-sty-tu-cja"), + ("konstytucyjnego", "kon-sty-tu-cyj-ne-go"), + ("Rzeczpospolita", "Rzecz-po-spo-li-ta"), + ("przedsiębiorstwo", "przed-się-bior-stwo"), + ("pięćdziesięciu", "pięć-dzie-się-ciu"), + ("dziewczyna", "dziew-czy-na"), + ("książka", "książ-ka"), + ("Warszawa", "War-sza-wa"), + ("szczęście", "szczę-ście"), + ("rzeczywistość", "rze-czy-wi-stość"), + ("najpiękniejszy", "naj-pięk-niej-szy"), + ("wyobraźnia", "wy-obraź-nia"), + ("gżegżółka", "gżeg-żół-ka"), + // One syllable, or a single letter left over: no break. + ("chrząszcz", "chrząszcz"), + ("źdźbło", "źdźbło"), + ("obiad", "obiad"), + // The pattern file's exception words. + ("nigdy", "ni-gdy"), + ("przynajmniej", "przy-naj-mniej"), + ("skądinąd", "skąd-inąd"), + // Compounds: each part, never next to the hyphen. + ("biało-czerwony", "bia-ło-czer-wo-ny"), + ] { + assert_eq!(pl(w), want, "{w}"); + } + assert_eq!(word_breaks_in("nigdy", ProofLang::Pl).1, Source::Dictionary); + assert_eq!(word_breaks_in("konstytucja", ProofLang::Pl).1, Source::Patterns); + } + + #[test] + fn polish_heuristic_for_foreign_letters() { + // Letters outside the Polish alphabet: the Polish syllable heuristic, digraphs kept whole. + assert_eq!(word_breaks_in("Pöppelmann", ProofLang::Pl).1, Source::Heuristic); + assert_eq!(pl("Pöppelmann"), "Pöp-pel-mann"); + assert_eq!(pl("Müllerowi"), "Mül-le-ro-wi"); + let c: Vec = "kawiarnié".chars().collect(); + assert_eq!(heuristic_points_pl(&c), vec![2, 6], "ka-wiar-nié: `wi` and `ni` soften, `rn` splits after the sonorant"); + let c: Vec = "szczérze".chars().collect(); + assert_eq!(heuristic_points_pl(&c), vec![5], "szczé-rze: `rz` stays whole"); + } + + #[test] + fn languages_without_patterns_keep_the_english_ones() { + for w in ["hyphenation", "typography", "internationalization"] { + assert_eq!(hyphen_points_in(w, &Limits::default(), ProofLang::Other), hyphen_points(w, &Limits::default()), "{w}"); + } + } } diff --git a/crates/proof/src/lang.rs b/crates/proof/src/lang.rs new file mode 100644 index 00000000..bfd0bea3 --- /dev/null +++ b/crates/proof/src/lang.rs @@ -0,0 +1,131 @@ +//! Proofing languages: which dictionary, grammar rules and hyphenation patterns apply to a run of +//! text. A run's language is its BCP 47 tag (`w:lang` in .docx, `CharProps::lang` in the model), +//! resolved with [`ProofLang::from_tag`]. + +/// A language WordCraft can proof, or [`ProofLang::Other`] for one it can't. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub enum ProofLang { + /// English, any region (`en`, `en-US`, `en-GB`…): the public-domain Moby word list, the + /// English grammar rules and the English hyphenation dictionary and patterns. Also text with + /// no language at all (WordCraft's default editing language is US English, like Word's). + #[default] + En, + /// Polish (`pl`, `pl-PL`): the installed Hunspell dictionary (when there is one), the Polish grammar rules and the Polish + /// hyphenation patterns. + Pl, + /// A language without proofing tools here (`fr-FR`, `ja-JP`, `x-none`…). Like Word without + /// the language's proofing tools: no spelling or grammar marks, so its words are never + /// flagged as English misspellings. Hyphenation keeps using the English patterns, as before + /// languages were told apart. + Other, +} + +impl ProofLang { + /// Every language with a spelling dictionary, in display order. + pub const PROOFED: [ProofLang; 2] = [ProofLang::En, ProofLang::Pl]; + + /// The proofing language of a BCP 47 tag (`pl-PL`, `en_GB`, `PL`…). Only the primary subtag + /// counts. No tag (or an empty one) is English, WordCraft's default editing language. + pub fn from_tag(tag: Option<&str>) -> ProofLang { + let Some(tag) = tag.map(str::trim).filter(|t| !t.is_empty()) else { return ProofLang::En }; + let primary = tag.split(['-', '_']).next().unwrap_or(""); + if primary.eq_ignore_ascii_case("en") || primary.eq_ignore_ascii_case("eng") { + ProofLang::En + } else if primary.eq_ignore_ascii_case("pl") || primary.eq_ignore_ascii_case("pol") { + ProofLang::Pl + } else { + ProofLang::Other + } + } + + /// The primary language subtag (`en`, `pl`); empty for [`ProofLang::Other`]. + pub fn code(self) -> &'static str { + match self { + ProofLang::En => "en", + ProofLang::Pl => "pl", + ProofLang::Other => "", + } + } + + /// Does WordCraft check spelling and grammar in this language? + pub fn is_proofed(self) -> bool { + self != ProofLang::Other + } +} + +/// A BCP 47 tag in canonical case (`pl-PL`, `en-GB`, `zh-Hant-TW`, `x-none`), or `None` when +/// `tag` isn't shaped like one: 1–35 ASCII letters, digits and `-`/`_` separators, starting with +/// a 2–8 letter language (or `x`/`i` for private and grandfathered tags). `_` becomes `-`. +pub fn normalize_tag(tag: &str) -> Option { + let tag = tag.trim(); + if tag.is_empty() || tag.len() > 35 || !tag.chars().all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_') { + return None; + } + let parts: Vec<&str> = tag.split(['-', '_']).collect(); + if parts.iter().any(|p| p.is_empty() || p.len() > 8) { + return None; + } + let first = parts.first()?; + let language = (2..=8).contains(&first.len()) && first.chars().all(|c| c.is_ascii_alphabetic()); + if !language && !first.eq_ignore_ascii_case("x") && !first.eq_ignore_ascii_case("i") { + return None; + } + let letters = |p: &str| p.chars().all(|c| c.is_ascii_alphabetic()); + // After a singleton (`x-…` private use, `u-…` extensions) subtags are lower case. + let singleton = parts.iter().position(|p| p.len() == 1).unwrap_or(usize::MAX); + let out: Vec = parts + .iter() + .enumerate() + .map(|(i, p)| { + if i >= singleton { + p.to_ascii_lowercase() + } else if i > 0 && p.len() == 2 && letters(p) { + p.to_ascii_uppercase() // region + } else if i > 0 && p.len() == 4 && letters(p) { + let lower = p.to_ascii_lowercase(); + let mut c = lower.chars(); + c.next().map(|f| f.to_ascii_uppercase().to_string() + c.as_str()).unwrap_or_default() // script + } else { + p.to_ascii_lowercase() + } + }) + .collect(); + Some(out.join("-")) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn tags_normalize() { + assert_eq!(normalize_tag("PL-pl").as_deref(), Some("pl-PL")); + assert_eq!(normalize_tag(" pl_pl ").as_deref(), Some("pl-PL")); + assert_eq!(normalize_tag("pl").as_deref(), Some("pl")); + assert_eq!(normalize_tag("zh-hant-tw").as_deref(), Some("zh-Hant-TW")); + assert_eq!(normalize_tag("es-419").as_deref(), Some("es-419")); + assert_eq!(normalize_tag("x-none").as_deref(), Some("x-none")); + assert_eq!(normalize_tag("de-DE-u-co-PHONEBK").as_deref(), Some("de-DE-u-co-phonebk")); + for bad in ["", " ", "p", "1pl", "pl--PL", "pl PL", "pl-PL;", "polski język", "-pl", "pl-", "abcdefghi", &"a".repeat(40)] { + assert_eq!(normalize_tag(bad), None, "{bad:?}"); + } + } + + #[test] + fn tags_resolve() { + for t in ["pl", "pl-PL", "PL-pl", "pl_PL", "pl_PL.UTF-8", " pl-PL ", "pol"] { + assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::Pl, "{t}"); + } + for t in ["en", "en-US", "en-GB", "EN_au", "en-Latn-US"] { + assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::En, "{t}"); + } + for t in ["fr-FR", "de", "x-none", "ja-JP", "plx", "e", "-"] { + assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::Other, "{t}"); + } + assert_eq!(ProofLang::from_tag(None), ProofLang::En); + assert_eq!(ProofLang::from_tag(Some("")), ProofLang::En); + assert_eq!(ProofLang::from_tag(Some(" ")), ProofLang::En); + assert!(ProofLang::Pl.is_proofed() && !ProofLang::Other.is_proofed()); + assert_eq!(ProofLang::Pl.code(), "pl"); + } +} diff --git a/crates/proof/src/lib.rs b/crates/proof/src/lib.rs index 6651ea67..925ab7ec 100644 --- a/crates/proof/src/lib.rs +++ b/crates/proof/src/lib.rs @@ -1,20 +1,34 @@ -//! WordCraft proofing. +//! WordCraft proofing, per language. A run's language (its BCP 47 tag) picks the tools through +//! [`ProofLang`]; the `*_in` functions take it, the older English-only ones remain. //! -//! - **Spelling** against the public-domain Moby word list (~160k words, the same list that -//! drives hyphenation) plus our supplement of modern words it lacks -//! (`assets/spelling/en-extra.txt`: `desktop`, `email`, `download`…), with inflections (`-s`, -//! `-es`, `-ed`, `-ing`, `-ly`, `'s`…), a user dictionary and ignore lists; suggestions by edit -//! distance. -//! - **Grammar** checks that are cheap and reliable: repeated words, `a`/`an`, capital letter -//! after a sentence end, spaces before punctuation, doubled spaces. -//! - **Hyphenation** ([`hyphen`]): dictionary, Liang patterns, heuristic. +//! - **English** ([`ProofLang::En`]; also text with no language): spelling against the +//! public-domain Moby word list (~160k words, the same list that drives hyphenation) plus our +//! supplement of modern words it lacks (`assets/spelling/en-extra.txt`: `desktop`, `email`, +//! `download`…), with inflections (`-s`, `-es`, `-ed`, `-ing`, `-ly`, `'s`…), a user dictionary and ignore lists; +//! suggestions by edit distance. Grammar checks that are cheap and reliable: repeated words, +//! `a`/`an`, capital letter after a sentence end, spaces before punctuation, doubled spaces. +//! - **Polish** ([`ProofLang::Pl`], module [`pl`]): spelling against the Hunspell `pl_PL` +//! dictionary installed on the system ([`hunspell`]; not bundled, so without it Polish has no +//! spelling marks), read by the affix engine ([`affix`]), suggestions tuned for Polish +//! (diacritics, `ż`/`rz`, `u`/`ó`, missing spaces), and Polish grammar rules with Polish +//! messages. +//! - **Other languages** ([`ProofLang::Other`]): no spelling or grammar marks, like Word without +//! the language's proofing tools. +//! - **Hyphenation** ([`hyphen`]): dictionary, Liang patterns (English, Polish), heuristic. +//! +//! [`check_text`] checks a paragraph whose runs are in different languages. #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +pub mod affix; pub mod dict; +pub mod hunspell; pub mod hyphen; +pub mod lang; pub mod patterns; +pub mod pl; use std::collections::HashSet; +use std::ops::Range; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{OnceLock, RwLock}; @@ -35,14 +49,20 @@ impl Default for Options { } } +pub use lang::ProofLang; + /// A problem found in text: byte range, kind, message and suggestions. #[derive(Clone, Debug, PartialEq)] pub struct Issue { pub start: usize, pub end: usize, pub kind: IssueKind, + /// In the language of the text it was found in (Polish rules explain in Polish). pub message: String, pub suggestions: Vec, + /// The language the text was checked in: spelling suggestions come from its dictionary + /// ([`suggest_in`]). + pub lang: ProofLang, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -59,7 +79,7 @@ fn user_words() -> &'static RwLock> { /// Bumped whenever the user dictionary changes, so cached proofing results can be dropped. static DICTIONARY_GENERATION: AtomicU64 = AtomicU64::new(0); -/// Add a word to the user dictionary ("Add to Dictionary"). +/// Add a word to the user dictionary ("Add to Dictionary"). It applies to every language. pub fn add_word(w: &str) { if user_words().write().unwrap_or_else(|e| e.into_inner()).insert(w.to_lowercase()) { DICTIONARY_GENERATION.fetch_add(1, Ordering::Relaxed); @@ -80,6 +100,11 @@ pub fn dictionary_generation() -> u64 { DICTIONARY_GENERATION.load(Ordering::Relaxed) } +/// Is `lower` (a lower-case word) in the user dictionary? +pub(crate) fn in_user_dictionary(lower: &str) -> bool { + user_words().read().unwrap_or_else(|e| e.into_inner()).contains(lower) +} + /// Words in the user dictionary. pub fn user_dictionary() -> Vec { let mut v: Vec = user_words().read().unwrap_or_else(|e| e.into_inner()).iter().cloned().collect(); @@ -161,24 +186,28 @@ fn known_core(w: &str) -> bool { false } -/// Is `word` spelled correctly? Numbers, single letters, words in capitals and words with -/// digits are accepted (Word's default [`Options`]). +/// Is `word` spelled correctly? Numbers, single letters, ALL-CAPS acronyms, URLs and words with +/// digits are accepted. pub fn is_correct(word: &str) -> bool { is_correct_with(word, &Options::default()) } +/// The words every language's check accepts under `opts`: single letters, words without +/// letters, words with digits (Ignore words that contain numbers) and words in capitals +/// (Ignore words in UPPERCASE). +fn accepted_by_options(word: &str, opts: &Options) -> bool { + let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'').trim_matches('\''); + w.chars().count() <= 1 + || !w.chars().any(char::is_alphabetic) + || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) + || (opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase())) +} + /// [`is_correct`] with the File › Options › Proofing choices. pub fn is_correct_with(word: &str, opts: &Options) -> bool { let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\''); let w = w.trim_matches('\''); - if w.chars().count() <= 1 || !w.chars().any(char::is_alphabetic) { - return true; - } - let digits = w.chars().any(|c| c.is_ascii_digit()); - if digits && opts.ignore_numbers { - return true; - } - if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) { + if accepted_by_options(w, opts) { return true; } if !w.is_ascii() && !w.chars().any(|c| c.is_ascii_alphabetic()) { @@ -299,31 +328,129 @@ pub fn words(text: &str) -> Vec<(usize, usize)> { .collect() } -/// Spelling issues in a paragraph of text (no suggestions; ask [`suggest`] on demand). +/// Spelling issues in a paragraph of English text (no suggestions; ask [`suggest`] on demand). pub fn check_spelling(text: &str) -> Vec { check_spelling_with(text, &Options::default()) } /// [`check_spelling`] with the File › Options › Proofing choices. pub fn check_spelling_with(text: &str, opts: &Options) -> Vec { - let mut v = Vec::new(); - for (a, b) in words(text) { - let Some(w) = text.get(a..b) else { continue }; - // Skip URLs/emails. - let ts = text.get(..a).and_then(|t| t.rfind(char::is_whitespace)).map(|i| i + 1).unwrap_or(0); - let te = text.get(b..).and_then(|t| t.find(char::is_whitespace)).map(|i| b + i).unwrap_or(text.len()); - let token = text.get(ts..te).unwrap_or(""); - if opts.ignore_internet && (token.contains("://") || token.contains('@') || token.starts_with("www.")) { - continue; - } - if !is_correct_with(w, opts) { - v.push(Issue { start: a, end: b, kind: IssueKind::Spelling, message: "Possible spelling mistake".into(), suggestions: Vec::new() }); + words(text).into_iter().filter_map(|(a, b)| spelling_issue(text, a, b, ProofLang::En, opts)).collect() +} + +/// Spelling issues in a paragraph of text in `lang` (none for a language without a dictionary; +/// no suggestions: ask [`suggest_in`] on demand). +pub fn check_spelling_in(text: &str, lang: ProofLang) -> Vec { + if !lang.is_proofed() { + return Vec::new(); + } + words(text).into_iter().filter_map(|(a, b)| spelling_issue(text, a, b, lang, &Options::default())).collect() +} + +/// The spelling issue of the word at `a..b` of `text`, if it is misspelled in `lang`. Words in +/// web and e-mail addresses are skipped (unless `opts` says otherwise). +fn spelling_issue(text: &str, a: usize, b: usize, lang: ProofLang, opts: &Options) -> Option { + let w = text.get(a..b)?; + let ts = text.get(..a).and_then(|t| t.rfind(char::is_whitespace)).map(|i| i + 1).unwrap_or(0); + let te = text.get(b..).and_then(|t| t.find(char::is_whitespace)).map(|i| b + i).unwrap_or(text.len()); + let token = text.get(ts..te).unwrap_or(""); + let internet = token.contains("://") || token.contains('@') || token.starts_with("www."); + if (opts.ignore_internet && internet) || is_correct_in_with(w, lang, opts) { + return None; + } + let message = match lang { + ProofLang::Pl => "Możliwy błąd pisowni", + _ => "Possible spelling mistake", + }; + Some(Issue { start: a, end: b, kind: IssueKind::Spelling, message: message.into(), suggestions: Vec::new(), lang }) +} + +/// Is `word` spelled correctly in `lang`? Always true in a language without a dictionary. +pub fn is_correct_in(word: &str, lang: ProofLang) -> bool { + is_correct_in_with(word, lang, &Options::default()) +} + +/// [`is_correct_in`] with the File › Options › Proofing choices. +pub fn is_correct_in_with(word: &str, lang: ProofLang, opts: &Options) -> bool { + match lang { + ProofLang::En => is_correct_with(word, opts), + ProofLang::Pl => accepted_by_options(word, opts) || pl::is_correct(word), + ProofLang::Other => true, + } +} + +/// Spelling suggestions for a word misspelled in `lang` (best first, at most `max`). +pub fn suggest_in(word: &str, max: usize, lang: ProofLang) -> Vec { + match lang { + ProofLang::En => suggest(word, max), + ProofLang::Pl => pl::suggest(word, max), + ProofLang::Other => Vec::new(), + } +} + +/// Grammar issues in a paragraph of text in `lang`: the English rules, the Polish ones +/// ([`pl::grammar`]), or none for a language WordCraft doesn't proof. +pub fn check_grammar_in(text: &str, lang: ProofLang) -> Vec { + match lang { + ProofLang::En => check_grammar(text), + ProofLang::Pl => pl::grammar::check(text), + ProofLang::Other => Vec::new(), + } +} + +/// Spelling and grammar issues in a paragraph whose text is in several languages, in order. +/// `spans` gives the language of byte ranges of `text` (in order, not overlapping); text outside +/// them is English. Each word is spelled in the language of its first letter. Each language's +/// grammar rules read the whole paragraph (so sentence boundaries are seen) but report only +/// inside that language's spans: an English quotation in a Polish paragraph gets English +/// spelling and is never told a Polish rule. +pub fn check_text(text: &str, spans: &[(Range, ProofLang)]) -> Vec { + check_text_with(text, spans, &Options::default(), true) +} + +/// [`check_text`] with the File › Options › Proofing choices: what spelling skips, and whether +/// grammar is checked at all. +pub fn check_text_with(text: &str, spans: &[(Range, ProofLang)], opts: &Options, grammar: bool) -> Vec { + // Sorted, empty ones dropped, neighbours of one language merged. + let mut sorted: Vec<&(Range, ProofLang)> = spans.iter().filter(|(r, _)| r.start < r.end).collect(); + sorted.sort_by_key(|s| s.0.start); + let mut merged: Vec<(Range, ProofLang)> = Vec::new(); + for (r, l) in sorted { + match merged.last_mut() { + Some((m, ml)) if *ml == *l && m.end == r.start => m.end = r.end, + _ => merged.push((r.clone(), *l)), } } + let lang_at = |i: usize| -> ProofLang { + let k = merged.partition_point(|(r, _)| r.end <= i); + merged.get(k).filter(|(r, _)| r.start <= i).map_or(ProofLang::En, |s| s.1) + }; + let mut v: Vec = words(text).into_iter().filter_map(|(a, b)| spelling_issue(text, a, b, lang_at(a), opts)).collect(); + if !grammar { + v.sort_by_key(|i| (i.start, i.end)); + return v; + } + let mut langs: Vec = merged.iter().map(|s| s.1).collect(); + let mut covered = 0; + for (r, _) in &merged { + if r.start > covered { + break; + } + covered = covered.max(r.end); + } + if covered < text.len() || merged.is_empty() { + langs.push(ProofLang::En); + } + langs.sort(); + langs.dedup(); + for lang in langs { + v.extend(check_grammar_in(text, lang).into_iter().filter(|i| lang_at(i.start) == lang)); + } + v.sort_by_key(|i| (i.start, i.end)); v } -/// Grammar issues in a paragraph of text. +/// Grammar issues in a paragraph of English text. pub fn check_grammar(text: &str) -> Vec { let mut v = Vec::new(); let ws = words(text); @@ -341,6 +468,7 @@ pub fn check_grammar(text: &str) -> Vec { kind: IssueKind::Grammar, message: format!("Repeated word: \"{w1}\""), suggestions: vec![w0.to_string()], + lang: ProofLang::En, }); } if gap == " " && (w0 == "a" || w0 == "A") && w1.chars().next().is_some_and(|c| "aeiouAEIOU".contains(c)) && !starts_consonant_sound(w1) { @@ -350,6 +478,7 @@ pub fn check_grammar(text: &str) -> Vec { kind: IssueKind::Grammar, message: "Use \"an\" before a vowel sound".into(), suggestions: vec![if w0 == "A" { "An" } else { "an" }.into()], + lang: ProofLang::En, }); } if gap == " " @@ -363,6 +492,7 @@ pub fn check_grammar(text: &str) -> Vec { kind: IssueKind::Grammar, message: "Use \"a\" before a consonant sound".into(), suggestions: vec![if w0 == "An" { "A" } else { "a" }.into()], + lang: ProofLang::En, }); } } @@ -377,13 +507,21 @@ pub fn check_grammar(text: &str) -> Vec { kind: IssueKind::Grammar, message: "Remove the space before the punctuation".into(), suggestions: vec![text.get(i + 1..i + 2).unwrap_or("").to_string()], + lang: ProofLang::En, }); } for (i, _) in text.match_indices(" ") { if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) { continue; } - v.push(Issue { start: i, end: i + 2, kind: IssueKind::Grammar, message: "Extra space".into(), suggestions: vec![" ".into()] }); + v.push(Issue { + start: i, + end: i + 2, + kind: IssueKind::Grammar, + message: "Extra space".into(), + suggestions: vec![" ".into()], + lang: ProofLang::En, + }); } // Sentence start capital. let mut after_end = true; @@ -397,6 +535,7 @@ pub fn check_grammar(text: &str) -> Vec { kind: IssueKind::Grammar, message: "Capitalize the first word of a sentence".into(), suggestions: vec![fixed], + lang: ProofLang::En, }); } let tail = text.get(*b..).unwrap_or(""); @@ -541,6 +680,51 @@ mod tests { assert!(hyphen::hyphenate_word("hyphenation", &l).matches('-').count() >= 2); } + fn flagged(t: &str, v: &[Issue]) -> Vec { + v.iter().filter(|i| i.kind == IssueKind::Spelling).map(|i| t[i.start..i.end].to_string()).collect() + } + + #[test] + fn each_language_uses_its_own_tools() { + pl::test_dict::install(); + let pl = "Zażółć gęślą jaźń, chrząszcz brzmi w trzcinie."; + assert!(check_spelling_in(pl, ProofLang::Pl).is_empty(), "{:?}", check_spelling_in(pl, ProofLang::Pl)); + assert!(check_spelling_in(pl, ProofLang::En).len() >= 3, "Polish words are English misspellings"); + assert!(check_spelling_in("Nous écrivons en français.", ProofLang::Other).is_empty()); + assert!(check_grammar_in("this is is wrong", ProofLang::Other).is_empty()); + assert!(is_correct_in("anything", ProofLang::Other) && suggest_in("xyzzy", 5, ProofLang::Other).is_empty()); + let v = check_spelling_in("Płynie żeka.", ProofLang::Pl); + assert_eq!(flagged("Płynie żeka.", &v), ["żeka"]); + assert_eq!((v[0].lang, v[0].message.as_str()), (ProofLang::Pl, "Możliwy błąd pisowni")); + assert!(suggest_in("żeka", 5, ProofLang::Pl).contains(&"rzeka".to_string())); + assert!(suggest_in("recieve", 5, ProofLang::En).contains(&"receive".to_string())); + // Polish grammar speaks Polish and has no English a/an rule. + let g = check_grammar_in("Wiem że to jest a apple.", ProofLang::Pl); + assert!(g.iter().any(|i| i.message == "Brak przecinka przed „że”"), "{g:?}"); + assert!(!g.iter().any(|i| i.message.contains("\"an\"")), "{g:?}"); + } + + #[test] + fn mixed_language_paragraphs_check_each_run_in_its_language() { + pl::test_dict::install(); + // A Polish paragraph quoting English: English spelling and rules inside the quote only. + let t = "Napisał „the quick brown fox is is here” i poszedł spać, bo żeka wylała."; + let (a, b) = (t.find("the").unwrap(), t.find('”').unwrap()); + let spans = [(0..a, ProofLang::Pl), (a..b, ProofLang::En), (b..t.len(), ProofLang::Pl)]; + let v = check_text(t, &spans); + assert_eq!(flagged(t, &v), ["żeka"], "{v:?}"); + let rep = v.iter().find(|i| i.kind == IssueKind::Grammar).unwrap(); + assert_eq!((rep.lang, rep.message.as_str()), (ProofLang::En, "Repeated word: \"is\""), "{v:?}"); + // The English rules don't run over the Polish text (no "Capitalize" for the lower-case + // Polish words), and the Polish rules don't run over the English quote. + assert!(v.iter().all(|i| i.lang == ProofLang::Pl || (a..b).contains(&i.start)), "{v:?}"); + // Text outside every span is English; no spans at all is English. + assert_eq!(flagged("helo świat", &check_text("helo świat", &[])), ["helo", "świat"]); + assert_eq!(flagged("helo świat", &check_text("helo świat", &[(5..12, ProofLang::Pl)])), ["helo"]); + // A language without tools: nothing flagged there. + assert!(check_text("Bonjour mes amis", &[(0..16, ProofLang::Other)]).is_empty()); + } + proptest::proptest! { #[test] fn never_panics(s in "\\PC{0,80}") { @@ -548,6 +732,28 @@ mod tests { let _ = check_grammar(&s); let _ = suggest(&s, 3); let _ = hyphen::hyphen_points(&s, &hyphen::Limits::default()); + for lang in [ProofLang::En, ProofLang::Pl, ProofLang::Other] { + let _ = check_spelling_in(&s, lang); + let _ = check_grammar_in(&s, lang); + let _ = is_correct_in(&s, lang); + let _ = hyphen::hyphen_points_in(&s, &hyphen::Limits::default(), lang); + } + let _ = lang::normalize_tag(&s); + let _ = ProofLang::from_tag(Some(&s)); + } + + #[test] + fn polish_suggestions_never_panic(s in "[a-ząćęłńóśźżA-ZĄĆĘŁŃÓŚŹŻ'’ -]{0,14}") { + let _ = suggest_in(&s, 4, ProofLang::Pl); + } + + #[test] + fn mixed_spans_never_panic(s in "\\PC{0,60}", cuts in proptest::collection::vec((0usize..80, 0usize..80, 0u8..3), 0..6)) { + let langs = [ProofLang::En, ProofLang::Pl, ProofLang::Other]; + let spans: Vec<(Range, ProofLang)> = cuts.iter().map(|&(a, b, l)| (a.min(b)..a.max(b), langs[l as usize % 3])).collect(); + for i in check_text(&s, &spans) { + proptest::prop_assert!(i.start <= i.end && i.end <= s.len()); + } } } } diff --git a/crates/proof/src/patterns.rs b/crates/proof/src/patterns.rs index d3e40600..50dd7444 100644 --- a/crates/proof/src/patterns.rs +++ b/crates/proof/src/patterns.rs @@ -1,18 +1,22 @@ //! Liang hyphenation patterns (TeX notation, e.g. `.ab1c`, `4t1ion`): parsing and application. //! //! The bundled English set is *ours*: trained from the public-domain Moby Hyphenator word list by -//! DesignCraft's patgen (see `examples/hyphgen.rs`). Patterns are stored in a hash map keyed by the -//! packed letter string, so applying them to a word costs `O(len × max_pattern_len)` lookups. +//! DesignCraft's patgen (see `examples/hyphgen.rs`). The Polish set is the TeX one by Hanna +//! Kołodziejska, Bogusław Jackowski and Marek Ryćko (hyph-utf8 `hyph-pl.tex`, used under its MIT +//! licence option), bundled unmodified: [`Patterns::parse`] reads TeX's `\patterns{…}` and +//! `\hyphenation{…}` blocks directly. Patterns are stored in a hash map keyed by the packed letter +//! string, so applying them to a word costs `O(len × max_pattern_len)` lookups. use std::collections::HashMap; use std::hash::{BuildHasherDefault, Hasher}; /// Word-boundary marker in patterns. pub const DOT: char = '.'; -/// Longest pattern the packed key supports. -pub const MAX_PATTERN_LEN: usize = 9; +/// Longest pattern the packed key supports (6 bits a letter in a `u128`; the Polish set's longest +/// is 14 including the dots). +pub const MAX_PATTERN_LEN: usize = 20; -/// Small, fast multiplicative hasher for packed `u64` keys (FxHash-style). +/// Small, fast multiplicative hasher for packed integer keys and short strings (FxHash-style). #[derive(Default, Clone, Copy)] pub struct FxHasher(u64); @@ -21,13 +25,27 @@ impl Hasher for FxHasher { self.0 } fn write(&mut self, bytes: &[u8]) { - for &b in bytes { + let (chunks, rest) = bytes.as_chunks::<8>(); + for c in chunks { + self.write_u64(u64::from_le_bytes(*c)); + } + for &b in rest { self.write_u64(b as u64); } } + fn write_u8(&mut self, i: u8) { + self.write_u64(i as u64); + } + fn write_usize(&mut self, i: usize) { + self.write_u64(i as u64); + } fn write_u64(&mut self, i: u64) { self.0 = (self.0.rotate_left(5) ^ i).wrapping_mul(0x51_7c_c1_b7_27_22_0a_95); } + fn write_u128(&mut self, i: u128) { + self.write_u64(i as u64); + self.write_u64((i >> 64) as u64); + } } pub type FxMap = HashMap>; @@ -72,36 +90,67 @@ impl Alphabet { } } -/// Packed key of a code string (≤ [`MAX_PATTERN_LEN`] codes). +/// Packed key of a code string (≤ [`MAX_PATTERN_LEN`] codes): 6 bits a code, the length on top. #[inline] -pub fn pack(codes: &[u8]) -> u64 { - let mut k = 0u64; +pub fn pack(codes: &[u8]) -> u128 { + let mut k = 0u128; for &c in codes { - k = (k << 6) | c as u64; + k = (k << 6) | c as u128; } - k | ((codes.len() as u64) << 58) + k | ((codes.len() as u128) << 120) } -/// A compiled pattern set. +/// A compiled pattern set, with its exception words (TeX's `\hyphenation{…}`). #[derive(Clone, Debug, Default)] pub struct Patterns { alpha: Alphabet, /// Packed letters → inter-letter values (`len + 1` entries). - map: FxMap>, + map: FxMap>, max_len: usize, + /// Lowercase word → its break points (char indices), overriding the patterns. + exceptions: HashMap>, +} + +/// Which TeX block [`Patterns::parse`] is reading. +#[derive(Clone, Copy, PartialEq, Eq)] +enum Block { + Patterns, + Exceptions, } impl Patterns { - /// Parse TeX-style patterns: whitespace-separated, `%`/`#` start a comment line. + /// Parse patterns: whitespace-separated, `%` comments to the end of the line (`#` too at the + /// start of a line). A whole TeX pattern file works as is: `\patterns{…}` holds patterns and + /// `\hyphenation{…}` exception words written with their hyphens (`ni-gdy`); outside any + /// block, a token with a hyphen is an exception and anything else a pattern. pub fn parse(text: &str) -> Patterns { let mut p = Patterns::default(); + let mut block = Block::Patterns; for line in text.lines() { let line = line.trim(); - if line.is_empty() || line.starts_with('%') || line.starts_with('#') { + if line.starts_with('#') { continue; } + let line = line.split('%').next().unwrap_or(""); for tok in line.split_whitespace() { - p.insert(tok); + let mut tok = tok; + if let Some(cmd) = tok.strip_prefix('\\') { + let (name, rest) = cmd.split_once('{').unwrap_or((cmd, "")); + block = if name == "hyphenation" { Block::Exceptions } else { Block::Patterns }; + tok = rest; + } + let closes = tok.ends_with('}'); + let tok = tok.trim_end_matches('}'); + if !tok.is_empty() { + if block == Block::Exceptions || tok.contains('-') { + p.insert_exception(tok); + } else { + p.insert(tok); + } + } + if closes { + block = Block::Patterns; + } } } p @@ -128,9 +177,40 @@ impl Patterns { true } + /// Add an exception word written with its hyphens (`ni-gdy`; no hyphen: never broken). + pub fn insert_exception(&mut self, word: &str) { + let mut plain = String::with_capacity(word.len()); + let mut points = Vec::new(); + let mut n = 0usize; + for c in word.chars() { + if c == '-' { + if n > 0 && points.last() != Some(&n) { + points.push(n); + } + } else { + plain.extend(c.to_lowercase()); + n += 1; + } + } + points.retain(|&i| i < n); + if n > 0 { + self.exceptions.insert(plain, points.into_boxed_slice()); + } + } + + /// Break points of an exception word (lowercase), if it is one. + pub fn exception(&self, word: &str) -> Option<&[usize]> { + self.exceptions.get(word).map(|b| &b[..]) + } + + /// Number of exception words. + pub fn exception_count(&self) -> usize { + self.exceptions.len() + } + #[allow(dead_code)] - pub(crate) fn from_parts(alpha: Alphabet, map: FxMap>, max_len: usize) -> Patterns { - Patterns { alpha, map, max_len } + pub(crate) fn from_parts(alpha: Alphabet, map: FxMap>, max_len: usize) -> Patterns { + Patterns { alpha, map, max_len, exceptions: HashMap::new() } } /// Make `letters` part of the alphabet even if no pattern mentions them. @@ -164,13 +244,13 @@ impl Patterns { let mut vals = vec![0u8; codes.len() + 1]; apply(&self.map, self.max_len, &codes, &mut vals); // Padded position p sits before padded char p; word char i is padded char i + 1. - Some(vals[1..codes.len()].to_vec()) + vals.get(1..codes.len()).map(<[u8]>::to_vec) } /// Break points (char indices, a hyphen goes before that char) of a lowercase word. pub fn points(&self, word: &[char]) -> Option> { let v = self.values(word)?; - Some((1..word.len()).filter(|&i| v[i] % 2 == 1).collect()) + Some((1..word.len()).filter(|&i| v.get(i).is_some_and(|x| x % 2 == 1)).collect()) } /// The patterns in TeX notation (sorted by letters), e.g. for writing a pattern file. @@ -183,17 +263,17 @@ impl Patterns { .map .iter() .map(|(&key, vals)| { - let n = (key >> 58) as usize; + let n = ((key >> 120) as usize).min(MAX_PATTERN_LEN); let letters: Vec = (0..n).map(|i| ch(((key >> (6 * (n - 1 - i))) & 63) as u8)).collect(); let mut s = String::new(); for (i, l) in letters.iter().enumerate() { - if vals[i] > 0 { - s.push(char::from(b'0' + vals[i])); + if let Some(&v) = vals.get(i).filter(|v| **v > 0) { + s.push(char::from(b'0' + v.min(9))); } s.push(*l); } - if vals[n] > 0 { - s.push(char::from(b'0' + vals[n])); + if let Some(&v) = vals.get(n).filter(|v| **v > 0) { + s.push(char::from(b'0' + v.min(9))); } (letters.into_iter().collect(), s) }) @@ -203,18 +283,20 @@ impl Patterns { } } -/// Apply `map` to padded `codes`, raising `vals` (len = codes.len() + 1). +/// Apply `map` to padded `codes`, raising `vals` (len = codes.len() + 1; shorter is tolerated). #[inline] -pub fn apply(map: &FxMap>, max_len: usize, codes: &[u8], vals: &mut [u8]) { +pub fn apply(map: &FxMap>, max_len: usize, codes: &[u8], vals: &mut [u8]) { let n = codes.len(); for s in 0..n { - let mut k = 0u64; - for len in 1..=max_len.min(n - s) { - k = (k << 6) | codes[s + len - 1] as u64; - if let Some(v) = map.get(&(k | ((len as u64) << 58))) { + let mut k = 0u128; + for (len, &c) in codes.iter().skip(s).take(max_len.min(MAX_PATTERN_LEN)).enumerate() { + k = (k << 6) | c as u128; + if let Some(v) = map.get(&(k | (((len + 1) as u128) << 120))) { for (j, &x) in v.iter().enumerate() { - if x > vals[s + j] { - vals[s + j] = x; + if let Some(slot) = vals.get_mut(s + j) + && x > *slot + { + *slot = x; } } } @@ -237,4 +319,28 @@ mod tests { assert!(tex.contains(&"hen5at".to_string())); assert!(p.values(&['h', 'é']).is_none()); } + + #[test] + fn tex_files_parse_with_blocks_comments_and_exceptions() { + let p = Patterns::parse( + "% a comment\n\\patterns{\n1na n2at % trailing comment\n.ab4cdefghijklm5n\n}\n\\hyphenation{\nni-gdy\nna-dal\nzawsze\n}\nhy-phen-ation\n", + ); + assert_eq!(p.len(), 3); + assert_eq!(p.exception("nigdy"), Some(&[2][..])); + assert_eq!(p.exception("nadal"), Some(&[2][..])); + assert_eq!(p.exception("zawsze"), Some(&[][..])); + assert_eq!(p.exception("hyphenation"), Some(&[2, 6][..])); + assert_eq!(p.exception_count(), 4); + // A pattern longer than the old 9-letter limit applies. + let w: Vec = "abcdefghijklmn".chars().collect(); + assert_eq!(p.values(&w).unwrap()[13], 5); + assert!(p.to_tex().contains(&".ab4cdefghijklm5n".to_string())); + } + + #[test] + fn apply_tolerates_short_value_buffers() { + let p = Patterns::parse("a1b"); + let mut vals = [0u8; 1]; + apply(&p.map, p.max_len, &[1, 2, 3, 1], &mut vals); + } } diff --git a/crates/proof/src/pl/grammar.rs b/crates/proof/src/pl/grammar.rs new file mode 100644 index 00000000..ec9d6771 --- /dev/null +++ b/crates/proof/src/pl/grammar.rs @@ -0,0 +1,542 @@ +//! Polish grammar and punctuation checks: cheap, rule-based and tuned to stay quiet when unsure +//! (a missed comma costs less than a wrong mark on correct text). Messages are in Polish. +//! +//! - **Repeated word** (`się się`), except interjections and words Polish doubles on purpose +//! (`to to`, `tak tak`, `ha ha`). +//! - **Space before punctuation** (`tak ,`) and **double spaces**; emoticons (` :)`) and +//! ellipses are left alone. +//! - **Capital letter at the start of a sentence**, knowing Polish abbreviations (`np.`, `itd.`, +//! `r.`, `m.in.`), initials, ordinal numbers (`5. miejsce`), ellipses and dialogue dashes +//! (`– Tak? – zapytał`), after which lower case is correct. +//! - **Missing comma** before subordinating conjunctions and relative pronouns that always open a +//! clause: `że`, `ponieważ`, `gdyż`, `aby`, `żeby`, `jeśli`, `jeżeli`, `gdyby`, `lecz` and the +//! forms of `który` — unless a conjunction or particle comes first (`mimo że`, `chyba że`, +//! `tak aby`, `i który`) or, for `który`, a preposition (`w którym`: the comma goes before it). +//! - **`we`/`ze`** where the next word's consonant cluster requires them (`we wtorek`, +//! `ze szkoły`, `ze mną`). +//! - **Doubled punctuation** (`,,` — often a typed opening quote `„` — `..`, `;;`). +//! +//! No English rule (such as `a`/`an`) applies to Polish text. + +use crate::{Issue, IssueKind, ProofLang, words}; + +/// Words Polish repeats on purpose: interjections, onomatopoeia and emphatic doubling. +const ALLOWED_REPEATS: &[&str] = &[ + "to", "tak", "nie", "no", "ha", "he", "hi", "hej", "oj", "aj", "ej", "ach", "och", "ech", "puk", "hop", "bum", "cyk", "tik", "pa", "fiu", "la", + "bardzo", "dawno", "daleko", "długo", +]; + +/// Abbreviations written with a final period after which a sentence goes on in lower case +/// (lower case, without the period). +const ABBREVIATIONS: &[&str] = &[ + "np", "itd", "itp", "tzn", "tj", "tzw", "ok", "ul", "al", "pl", "os", "in", "wg", "zob", "por", "str", "ss", "nr", "pkt", "ust", "art", "par", + "rozdz", "tab", "rys", "il", "przyp", "red", "wyd", "oprac", "tłum", "cyt", "jw", "ds", "im", "ks", "św", "bł", "prof", "dr", "doc", "hab", + "inż", "mgr", "lek", "płk", "gen", "mjr", "kpt", "ppłk", "sierż", "szer", "kmdr", "adm", "abp", "bp", "br", "ub", "bm", "pn", "płd", "płn", + "wsch", "zach", "gł", "godz", "min", "sek", "tys", "mln", "mld", "zł", "gr", "kg", "dag", "mies", "tyg", "pon", "wt", "śr", "czw", "pt", "sob", + "niedz", "sty", "lut", "mar", "kwi", "cze", "lip", "sie", "wrz", "paź", "lis", "gru", "ang", "niem", "franc", "łac", "ros", "wł", "hiszp", "pol", + "lit", "dosł", "przen", "pot", "zdrob", "fot", "ryc", "tel", "kom", "ew", "ob", "obyw", "pp", "zm", "ur", "ca", "vs", "etc", "cdn", "dot", "wz", + "zw", "zał", "cz", "rr", "ww", "jęz", "jun", "sen", "mec", "red", "dyr", "kier", "proc", "poj", "lp", "lm", "zaw", "woj", "pow", "gm", "wsp", + "nast", "poprz", "ang", "fr", +]; + +/// Subordinating conjunctions and relative pronouns that open a clause: a comma comes before +/// them. +const CLAUSE_OPENERS: &[&str] = &[ + "że", + "ponieważ", + "gdyż", + "aby", + "żeby", + "jeśli", + "jeżeli", + "gdyby", + "lecz", + "który", + "która", + "które", + "którego", + "której", + "któremu", + "którą", + "którym", + "których", + "którymi", + "którzy", +]; + +/// Words after which a clause opener takes no comma of its own: conjunctions (`i że`), particles +/// (`tylko jeśli`, `zwłaszcza że`) and the first halves of compound conjunctions (`mimo że`, +/// `chyba że`, `tak aby`, `jak gdyby`, `na wypadek gdyby`, `z tym że`). +const NO_COMMA_AFTER: &[&str] = &[ + "i", + "a", + "oraz", + "lub", + "albo", + "ani", + "czy", + "bądź", + "ale", + "lecz", + "bo", + "że", + "aby", + "żeby", + "by", + "niż", + "jak", + "jako", + "niby", + "tylko", + "nawet", + "właśnie", + "zwłaszcza", + "szczególnie", + "głównie", + "częściowo", + "jedynie", + "wyłącznie", + "chyba", + "mianowicie", + "przynajmniej", + "również", + "także", + "też", + "dopiero", + "jeszcze", + "już", + "zaś", + "więc", + "zatem", + "toteż", + "przecież", + "chociaż", + "choć", + "choćby", + "gdy", + "kiedy", + "jeśli", + "jeżeli", + "gdyby", + "skoro", + "zanim", + "dopóki", + "póki", + "aż", + "co", + "tak", + "mimo", + "pomimo", + "wtedy", + "teraz", + "dziś", + "dzisiaj", + "podczas", + "tym", + "bardziej", + "warunkiem", + "wypadek", + "zamiast", + "byle", + "lada", + "wiadomo", + "obojętnie", + "jedno", + "wszystkim", + "może", + "pewnie", + "prawdopodobnie", + "zapewne", + "dlatego", + "tyle", + "dość", + "ponieważ", + "gdyż", + "lecz", + "albowiem", + "oto", + "no", +]; + +/// Prepositions: a relative pronoun after one takes the comma before the preposition instead +/// (`dom, w którym`). +const PREPOSITIONS: &[&str] = &[ + "w", + "we", + "z", + "ze", + "na", + "do", + "od", + "ode", + "o", + "u", + "po", + "za", + "przed", + "przede", + "przez", + "przeze", + "przy", + "nad", + "nade", + "pod", + "pode", + "dla", + "bez", + "beze", + "ku", + "między", + "pomiędzy", + "wśród", + "spośród", + "spod", + "spoza", + "sprzed", + "znad", + "zza", + "poza", + "ponad", + "poprzez", + "dzięki", + "według", + "wobec", + "oprócz", + "prócz", + "obok", + "koło", + "około", + "wokół", + "dookoła", + "naprzeciw", + "naprzeciwko", + "wzdłuż", + "wewnątrz", + "zewnątrz", + "względem", + "przeciw", + "przeciwko", + "wbrew", + "wskutek", + "śród", + "wedle", + "pośród", + "ponad", + "niedaleko", + "blisko", +]; + +fn issue(start: usize, end: usize, message: String, suggestions: Vec) -> Issue { + Issue { start, end, kind: IssueKind::Grammar, message, suggestions, lang: ProofLang::Pl } +} + +fn capitalize(w: &str) -> String { + crate::affix::capitalize(w) +} + +/// Polish vowels (and the accented vowels of loanwords). +fn is_vowel(c: char) -> bool { + matches!(c.to_lowercase().next().unwrap_or(c), 'a' | 'ą' | 'e' | 'ę' | 'i' | 'o' | 'ó' | 'u' | 'y' | 'é' | 'á' | 'ö' | 'ü' | 'í' | 'ú') +} + +fn is_consonant(c: char) -> bool { + c.is_alphabetic() && !is_vowel(c) +} + +/// Grammar issues in a paragraph of Polish text, in order. +pub fn check(text: &str) -> Vec { + let ws: Vec<(usize, usize)> = words(text); + let mut v = Vec::new(); + repeated_words(text, &ws, &mut v); + spaces(text, &mut v); + sentence_capitals(text, &ws, &mut v); + missing_commas(text, &ws, &mut v); + we_ze(text, &ws, &mut v); + doubled_punctuation(text, &mut v); + v.sort_by_key(|i| (i.start, i.end)); + v +} + +fn repeated_words(text: &str, ws: &[(usize, usize)], v: &mut Vec) { + for pair in ws.windows(2) { + let [(a0, b0), (a1, b1)] = pair else { continue }; + let (Some(w0), Some(w1), Some(gap)) = (text.get(*a0..*b0), text.get(*a1..*b1), text.get(*b0..*a1)) else { continue }; + if !gap.is_empty() + && gap.chars().all(char::is_whitespace) + && w0.chars().all(char::is_alphabetic) + && w0.to_lowercase() == w1.to_lowercase() + && !ALLOWED_REPEATS.contains(&w0.to_lowercase().as_str()) + { + v.push(issue(*a0, *b1, format!("Powtórzone słowo: „{w1}”"), vec![w0.to_string()])); + } + } +} + +fn spaces(text: &str, v: &mut Vec) { + for (i, c) in text.char_indices() { + if c != ' ' { + continue; + } + let rest = text.get(i + 1..).unwrap_or(""); + let mut it = rest.chars(); + let Some(p) = it.next() else { continue }; + let after = it.next(); + let flag = match p { + ',' => true, + // `;)` and `;-)` are emoticons. + ';' => !matches!(after, Some(')' | '(' | '-' | 'P' | 'D')), + // ` ...` is an ellipsis, ` .5` a number. + '.' => !matches!(after, Some('.')) && !after.is_some_and(|a| a.is_ascii_digit()), + // ` :)`, ` :-)`, ` :D` and similar are emoticons; ` ?!` is still a stray space. + ':' | '!' | '?' => after.is_none_or(|a| a.is_whitespace() || matches!(a, '"' | '”' | '»' | '’' | '!' | '?')), + _ => false, + }; + if flag { + v.push(issue(i, i + 1 + p.len_utf8(), "Zbędna spacja przed znakiem interpunkcyjnym".into(), vec![p.to_string()])); + } + } + for (i, _) in text.match_indices(" ") { + if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) { + continue; + } + v.push(issue(i, i + 2, "Podwójna spacja".into(), vec![" ".into()])); + } +} + +/// Is `prev`, before a period, an abbreviation, an initial, a number or a Roman numeral (after +/// which a lower-case word doesn't start a sentence)? +fn abbreviation_or_number(prev: &str) -> bool { + prev.chars().count() <= 1 + || prev.chars().any(|c| c.is_ascii_digit()) + || (prev.chars().all(|c| matches!(c, 'I' | 'V' | 'X' | 'L' | 'C' | 'D' | 'M'))) + || ABBREVIATIONS.contains(&prev.to_lowercase().as_str()) +} + +const CLOSING: [char; 8] = ['"', '”', '’', '\'', ')', '»', ']', '\u{1}']; +const OPENING: [char; 8] = ['„', '"', '“', '«', '(', '[', '\'', '‚']; + +/// Does the gap between two words end a sentence: a `.`, `!` or `?` (not an ellipsis, not after +/// an abbreviation), then space, with nothing but quotes and brackets around? +fn ends_sentence(gap: &str, prev: &str) -> bool { + let t = gap.trim_start_matches(CLOSING); + let Some(end) = t.chars().next() else { return false }; + if !matches!(end, '.' | '!' | '?') || t.starts_with("..") { + return false; + } + if end == '.' && abbreviation_or_number(prev) { + return false; + } + let rest = t.trim_start_matches(['.', '!', '?']).trim_start_matches(CLOSING); + rest.starts_with(char::is_whitespace) && rest.trim_start().chars().all(|c| OPENING.contains(&c)) +} + +fn sentence_capitals(text: &str, ws: &[(usize, usize)], v: &mut Vec) { + // List items (`– jabłka,`) may start in lower case. + let list_item = text.trim_end().ends_with([',', ';']); + for (k, &(a, b)) in ws.iter().enumerate() { + let Some(w) = text.get(a..b) else { continue }; + let starts = if k == 0 { + let before = text.get(..a).unwrap_or(""); + !list_item && before.chars().all(|c| c.is_whitespace() || OPENING.contains(&c)) + } else { + let Some(&(pa, pb)) = ws.get(k - 1) else { continue }; + ends_sentence(text.get(pb..a).unwrap_or(""), text.get(pa..pb).unwrap_or("")) + }; + if starts && w.chars().next().is_some_and(char::is_lowercase) && w.chars().all(char::is_alphabetic) { + v.push(issue(a, b, "Wielka litera na początku zdania".into(), vec![capitalize(w)])); + } + } +} + +fn missing_commas(text: &str, ws: &[(usize, usize)], v: &mut Vec) { + for k in 1..ws.len() { + let (Some(&(pa, pb)), Some(&(a, b))) = (ws.get(k - 1), ws.get(k)) else { continue }; + let (Some(prev), Some(w), Some(gap)) = (text.get(pa..pb), text.get(a..b), text.get(pb..a)) else { continue }; + if !CLAUSE_OPENERS.contains(&w) || gap.is_empty() || !gap.chars().all(|c| c == ' ' || c == '\u{a0}') { + continue; + } + if !prev.chars().all(char::is_alphabetic) { + continue; + } + let p = prev.to_lowercase(); + let p = p.as_str(); + if NO_COMMA_AFTER.contains(&p) { + continue; + } + if w.starts_with("któr") && (PREPOSITIONS.contains(&p) || p == "to") { + continue; + } + v.push(issue(pb, b, format!("Brak przecinka przed „{w}”"), vec![format!(", {w}")])); + } +} + +fn we_ze(text: &str, ws: &[(usize, usize)], v: &mut Vec) { + for pair in ws.windows(2) { + let [(a0, b0), (a1, b1)] = pair else { continue }; + let (Some(w0), Some(w1), Some(gap)) = (text.get(*a0..*b0), text.get(*a1..*b1), text.get(*b0..*a1)) else { continue }; + if gap != " " && gap != "\u{a0}" { + continue; + } + let next = w1.to_lowercase(); + let c: Vec = next.chars().collect(); + let at = |i: usize| c.get(i).copied(); + let (long, wanted) = match w0 { + "w" | "W" => ("we", (at(0) == Some('w') && at(1).is_some_and(is_consonant)) || next == "mnie"), + "z" | "Z" => { + // The first sound: `sz` is one consonant (`z szafy`, but `ze szkoły`). + let after = if at(0) == Some('s') && at(1) == Some('z') { at(2) } else { at(1) }; + let sibilant = matches!(at(0), Some('s' | 'z' | 'ś' | 'ź' | 'ż')); + ("ze", (sibilant && after.is_some_and(is_consonant)) || next == "mną") + } + _ => continue, + }; + if wanted { + let fixed = if w0.starts_with(char::is_uppercase) { capitalize(long) } else { long.to_string() }; + v.push(issue(*a0, *b0, format!("Przed tym słowem piszemy „{long}”"), vec![fixed])); + } + } +} + +fn doubled_punctuation(text: &str, v: &mut Vec) { + let bytes = text.as_bytes(); + let mut i = 0; + while let Some(&b) = bytes.get(i) { + if !matches!(b, b',' | b'.' | b';') { + i += 1; + continue; + } + let run = bytes.get(i..).map_or(0, |r| r.iter().take_while(|&&x| x == b).count()); + let flag = match b { + b'.' => run == 2, + _ => run >= 2, + }; + if flag { + let end = i + run; + let suggestions: Vec = match b { + // `,,` before a word, after a space: a typed opening quote. + b',' if text.get(end..).and_then(|t| t.chars().next()).is_some_and(char::is_alphabetic) + && (i == 0 || text.get(..i).is_some_and(|t| t.ends_with(char::is_whitespace))) => + { + vec!["„".into()] + } + b'.' => vec![".".into(), "…".into()], + _ => vec![char::from(b).to_string()], + }; + v.push(issue(i, end, "Powtórzony znak interpunkcyjny".into(), suggestions)); + } + i += run.max(1); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn messages(t: &str) -> Vec { + check(t).into_iter().map(|i| format!("{}@{}", i.message, &t[i.start..i.end])).collect() + } + + fn clean(t: &str) { + let m = messages(t); + assert!(m.is_empty(), "{t:?}: {m:?}"); + } + + #[test] + fn repeated_words() { + let v = check("Ala ma ma kota i się się cieszy."); + assert_eq!(v.len(), 2, "{v:?}"); + assert_eq!(v[0].message, "Powtórzone słowo: „ma”"); + assert_eq!(v[0].suggestions, ["ma"]); + assert!(v.iter().all(|i| i.lang == ProofLang::Pl && i.kind == IssueKind::Grammar)); + clean("Tak tak, to to samo. Ha ha!"); + } + + #[test] + fn spaces_and_punctuation() { + let m = messages("Ala ma kota , a kot ma Alę . Czy na pewno ?"); + assert!(m.contains(&"Zbędna spacja przed znakiem interpunkcyjnym@ ,".to_string()), "{m:?}"); + assert!(m.contains(&"Zbędna spacja przed znakiem interpunkcyjnym@ .".to_string()), "{m:?}"); + assert!(m.contains(&"Zbędna spacja przed znakiem interpunkcyjnym@ ?".to_string()), "{m:?}"); + assert!(m.contains(&"Podwójna spacja@ ".to_string()), "{m:?}"); + clean("Fajnie :) Naprawdę :-) Albo :D Wartość .5 i wielokropek ..."); + let m = messages("Powiedział ,,tak” i poszedł.. Potem;; koniec."); + assert_eq!(m.iter().filter(|x| x.starts_with("Powtórzony znak")).count(), 3, "{m:?}"); + assert_eq!(check("Powiedział ,,tak”.").iter().find(|i| i.message.starts_with("Powtórzony")).unwrap().suggestions, ["„"]); + clean("Czekał… i czekał... aż do rana."); + } + + #[test] + fn sentence_start_capitals() { + let m = messages("to jest zdanie. a to drugie! czy trzecie? tak."); + assert_eq!(m.iter().filter(|x| x.starts_with("Wielka litera")).count(), 4, "{m:?}"); + assert_eq!(check("ala ma kota.")[0].suggestions, ["Ala"]); + // Abbreviations, initials, ordinals, dates, ellipses and dialogue dashes. + clean("Kupiłem owoce, np. jabłka, gruszki itp. oraz warzywa."); + clean("W 1990 r. powstała firma, m.in. dzięki dotacji."); + clean("Spotkałem J. Kowalskiego i prof. Nowaka przy ul. Długiej."); + clean("Zajął 5. miejsce w XX w. sporcie."); + clean("Czekałem... aż w końcu przyszedł."); + clean("– Naprawdę? – zapytał. – Tak – odpowiedziała."); + clean("Powiedział: „Wracam”. Potem wyszedł."); + // A list item may start in lower case. + clean("jabłka,"); + } + + #[test] + fn missing_commas() { + let m = messages("Wiem że przyjdzie. Mam kota który śpi. Zostałem bo musiałem. Idę aby zdążyć. Nie przyszedł ponieważ padało."); + for want in [ + "Brak przecinka przed „że”@ że", + "Brak przecinka przed „który”@ który", + "Brak przecinka przed „aby”@ aby", + "Brak przecinka przed „ponieważ”@ ponieważ", + ] { + assert!(m.contains(&want.to_string()), "{want}: {m:?}"); + } + assert_eq!(check("Wiem że tak.")[0].suggestions, [", że"]); + // Compound conjunctions, particles, conjunctions and prepositions take no comma there. + clean("Przyszedł, mimo że padało, chyba że nie."); + clean("Zrobię to, tylko jeśli zdążę, i to tak aby nikt nie widział."); + clean("To dom, w którym mieszkam, i ogród, przez który idę."); + clean("Wiem, że przyjdzie, i że zostanie."); + clean("Myślę, że gdyby przyszedł, byłoby dobrze."); + clean("Zachowywał się, jak gdyby nigdy nic."); + clean("Który z nich? To który wybierasz?"); + clean("Po raz nie wiadomo który."); + clean("Dlatego że padało, zostaliśmy."); + } + + #[test] + fn we_and_ze() { + let m = messages("Spotkamy się w wtorek w Wrocławiu z szkoły z mną."); + assert_eq!(m.iter().filter(|x| x.starts_with("Przed tym słowem")).count(), 4, "{m:?}"); + let v = check("Z szkoły wracam."); + assert_eq!(v[0].suggestions, ["Ze"]); + clean("Byłem w wodzie, w wieku dziesięciu lat, z szafą, z siostrą i z zamkiem."); + clean("Wracam ze szkoły we wtorek ze mną."); + } + + #[test] + fn correct_polish_text_is_clean() { + clean( + "Zażółć gęślą jaźń. Chrząszcz brzmi w trzcinie, a źdźbło trawy się kołysze. \ + Przyjdę jutro, chyba że będzie padać. Książka, którą czytam, jest ciekawa.", + ); + clean("Pięćdziesięciu uczniów przyszło, ponieważ ogłoszono konkurs."); + } + + proptest::proptest! { + #[test] + fn never_panics(s in "\\PC{0,80}") { + let _ = check(&s); + } + + #[test] + fn never_panics_on_polish_like_text(s in "[a-ząćęłńóśźżA-Z ,.;:!?„”…\\-]{0,60}") { + for i in check(&s) { + proptest::prop_assert!(i.start <= i.end && i.end <= s.len()); + proptest::prop_assert!(s.is_char_boundary(i.start) && s.is_char_boundary(i.end)); + } + } + } +} diff --git a/crates/proof/src/pl/mod.rs b/crates/proof/src/pl/mod.rs new file mode 100644 index 00000000..74d8b8d2 --- /dev/null +++ b/crates/proof/src/pl/mod.rs @@ -0,0 +1,204 @@ +//! Polish proofing. +//! +//! - **Spelling** against the SJP.PL Hunspell dictionary installed on the system +//! ([`dictionary`], found by [`crate::hunspell`]: `pl_PL.aff`/`pl_PL.dic` from the OS, a +//! package manager or LibreOffice; it isn't bundled). Without it Polish text gets no spelling +//! marks. Capitalized stems are names (`Warszawa`); a sentence may start with any word and +//! headings may be in capitals. Hyphenated compounds are checked part by part +//! (`biało-czerwony`), foreign names inflected with an apostrophe are accepted +//! (`Kennedy'ego`), and the user dictionary applies as in English. +//! - **Suggestions** tuned for Polish ([`suggest()`]): missing diacritics first (`zolw` → `żółw`), then +//! the common spelling confusions (`ż`/`rz`, `u`/`ó`, `h`/`ch`…), a missing space (`napewno` → +//! `na pewno`), one-letter typos, and a dictionary scan for anything further off. +//! - **Grammar** ([`grammar`]): cheap, reliable rules with Polish messages. + +pub mod grammar; +mod suggest; +#[cfg(test)] +pub(crate) mod test_dict; + +use std::sync::{Arc, RwLock}; + +use crate::affix::AffixDict; + +pub use suggest::suggest; + +/// The Polish dictionary: `None` until first use, then the installed one (or an empty one). +static DICT: RwLock>> = RwLock::new(None); + +/// File names of a Polish Hunspell dictionary. +const NAMES: &[&str] = &["pl_PL", "pl-PL", "pl"]; + +/// The Polish dictionary: the installed Hunspell `pl_PL` (read on first use), one given with +/// [`install_dictionary`], or an empty one when none is available (Polish spelling is off then; +/// see [`has_dictionary`]). +pub fn dictionary() -> Arc { + if let Some(d) = DICT.read().unwrap_or_else(|e| e.into_inner()).as_ref() { + return d.clone(); + } + let loaded = Arc::new(load_installed()); + let mut slot = DICT.write().unwrap_or_else(|e| e.into_inner()); + // Another thread (or an install) may have filled it meanwhile: that one wins. + slot.get_or_insert(loaded).clone() +} + +fn load_installed() -> AffixDict { + let Some((aff, dic)) = crate::hunspell::find(NAMES) else { + log::info!("Polish spelling: no Hunspell pl_PL dictionary installed"); + return AffixDict::default(); + }; + crate::hunspell::load(&aff, &dic).unwrap_or_else(|e| { + log::warn!("Polish spelling: {e}"); + AffixDict::default() + }) +} + +/// Use this Polish dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to UTF-8), +/// for example one the user picked. On error the current dictionary stays. +pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> { + let d = AffixDict::from_hunspell(aff, dic, &[])?; + *DICT.write().unwrap_or_else(|e| e.into_inner()) = Some(Arc::new(d)); + Ok(()) +} + +/// Forget the loaded dictionary, so the next check looks for an installed one again (after +/// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed). +pub fn reload_dictionary() { + *DICT.write().unwrap_or_else(|e| e.into_inner()) = None; +} + +/// Is a Polish dictionary available (spelling is checked)? +pub fn has_dictionary() -> bool { + !dictionary().is_empty() +} + +/// Endings Polish adds after an apostrophe to foreign names whose last letter isn't pronounced +/// (`Kennedy'ego`, `Wilde'a`, `Shakespeare'owi`). +const APOSTROPHE_ENDINGS: &[&str] = &["a", "ach", "ami", "e", "ego", "em", "emu", "i", "m", "om", "owi", "owie", "u", "y", "ów", "ą", "ę"]; + +/// Is `c` a letter of the Latin script (the only one the Polish dictionary covers)? +fn is_latin_letter(c: char) -> bool { + c.is_alphabetic() && (c < '\u{0250}' || ('\u{1E00}'..='\u{1EFF}').contains(&c)) +} + +/// Is `word` spelled correctly in Polish? Numbers, single letters, short words in capitals +/// (acronyms: `PKO`, `ZUS`), words with digits and words in other scripts are accepted, and so +/// is every word when no Polish dictionary is installed. +pub fn is_correct(word: &str) -> bool { + is_correct_with(&dictionary(), word) +} + +/// [`is_correct`] against a given dictionary (an empty one accepts everything). +pub fn is_correct_with(d: &AffixDict, word: &str) -> bool { + if d.is_empty() { + return true; // no Polish dictionary installed: no spelling marks + } + let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’'); + let w = w.trim_matches(['\'', '’']); + let n = w.chars().count(); + if n <= 1 || w.chars().any(|c| c.is_ascii_digit()) || !w.chars().any(char::is_alphabetic) { + return true; + } + if w.chars().all(|c| !c.is_lowercase()) && n <= 6 { + return true; + } + if !w.chars().any(is_latin_letter) { + return true; // other scripts: no dictionary + } + let w = w.replace('’', "'"); + if w.contains('-') { + return w.split('-').all(|p| p.chars().count() <= 1 || known(d, p)); + } + known(d, &w) +} + +/// Is one word (no hyphens) in the dictionary, the user dictionary, or a foreign name with a +/// Polish ending after an apostrophe? +fn known(d: &AffixDict, w: &str) -> bool { + if d.check(w) || crate::in_user_dictionary(&w.to_lowercase()) { + return true; + } + if let Some((stem, ending)) = w.split_once('\'') { + return stem.chars().next().is_some_and(char::is_uppercase) && stem.chars().count() >= 2 && APOSTROPHE_ENDINGS.contains(&ending); + } + false +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_dictionary_loads() { + test_dict::install(); + let d = dictionary(); + assert!(has_dictionary()); + assert!(d.len() > 50, "{}", d.len()); + assert_eq!(d.rule_count(), 5, "the test dictionary's affix rules"); + assert!(!d.rep().is_empty() && !d.map().is_empty() && !d.try_chars().is_empty()); + } + + #[test] + fn without_a_dictionary_nothing_is_flagged_and_nothing_suggested() { + let none = AffixDict::default(); + for w in ["żeka", "kturzy", "xqzv", ""] { + assert!(is_correct_with(&none, w), "{w}"); + } + assert!(suggest::suggest_in(&none, "żeka", 5).is_empty()); + assert!(install_dictionary("FLAG long\n", "1\nkot\n").is_err(), "unsupported files are refused"); + } + + #[test] + fn polish_words_are_correct() { + test_dict::install(); + for w in [ + "źdźbło", + "gżegżółka", + "chrząszcz", + "pięćdziesięciu", + "rzeka", + "którzy", + "żółw", + "Zażółć", + "gęślą", + "jaźń", + "konstytucja", + "konstytucyjnego", + "niebieskiego", + "nieładnie", + "przyszliśmy", + "zrobilibyśmy", + "Warszawa", + "Warszawie", + "WARSZAWA", + "Kot", + "biało-czerwony", + "polsko-niemieckiej", + "Kennedy'ego", + "Shakespeare’a", + "PKO", + "2026", + "e-mail", + "w", + "się", + ] { + assert!(is_correct(w), "{w}"); + } + } + + #[test] + fn misspellings_are_flagged() { + test_dict::install(); + for w in ["żeka", "kturzy", "napewno", "wogóle", "zolw", "chuśtawka", "bżoza", "warszawa", "niewiem", "pszyjaciel", "Kennedy'xyz"] { + assert!(!is_correct(w), "{w}"); + } + } + + #[test] + fn user_dictionary_applies() { + test_dict::install(); + assert!(!is_correct("zxqpolskisłowo")); + crate::add_word("Zxqpolskisłowo"); + assert!(is_correct("zxqpolskisłowo")); + } +} diff --git a/crates/proof/src/pl/suggest.rs b/crates/proof/src/pl/suggest.rs new file mode 100644 index 00000000..101952cc --- /dev/null +++ b/crates/proof/src/pl/suggest.rs @@ -0,0 +1,437 @@ +//! Polish spelling suggestions, best first. +//! +//! Candidates come in rounds of rising cost, each checked against the dictionary: +//! 1. letter case (`warszawa` → `Warszawa`); +//! 2. missing or wrong diacritics, from the dictionary's `MAP` groups (`zolw` → `żółw`, `kturzy` +//! → `którzy`), up to four letters changed — Polish typed without diacritics is the most +//! common slip, so adding them ranks first, while taking away a diacritic the writer typed +//! ranks after the known confusions; +//! 3. the dictionary's `REP` table of common confusions (`żeka` → `rzeka`, `chuśtawka` → +//! `huśtawka`, `pszyjaciel` → `przyjaciel`); +//! 4. a missing space (`napewno` → `na pewno`, `wogóle` → `w ogóle`, `kochamcię` → `kocham +//! cię`), cheaper when one half is a short function word; +//! 5. one typo: a letter left out, added, swapped with the next or mistyped (`TRY` letters); +//! 6. when that finds fewer than asked for: the stems that start like the word (diacritics +//! folded) and share the most letter pairs with it, whose forms are ranked by an edit distance +//! in which a diacritic slip costs half (two typos at most). +//! +//! The input is bounded (40 letters), and so is every round, so a call costs a few thousand +//! dictionary lookups at most. + +use std::collections::{HashMap, HashSet}; + +use crate::affix::{AffixDict, Casing, capitalize, casing}; + +/// Short words often run into the next one (`na pewno`, `w ogóle`, `nie wiem`, `przede +/// wszystkim`). +const LEADING_WORDS: &[&str] = &[ + "a", "i", "o", "u", "w", "z", "we", "ze", "na", "po", "do", "od", "za", "co", "to", "nie", "jak", "tak", "przy", "bez", "pod", "nad", "dla", + "że", "by", "czy", "już", "ani", "przed", "przez", "przede", "ku", "aż", "bo", "ten", "ta", "te", "tym", "tej", "też", "jest", "ode", "beze", + "nade", +]; +/// Short words often run into the one before (`kocham cię`, `daj mi`, `bałem się`). +const TRAILING_WORDS: &[&str] = + &["się", "cię", "mi", "mu", "ci", "go", "ją", "je", "nam", "wam", "im", "by", "że", "no", "mnie", "tobie", "jej", "ich"]; +const MAX_LETTERS: usize = 40; +const MAX_MAP_CHANGES: usize = 4; +const MAX_MAP_VARIANTS: usize = 4096; +/// Stems the scan of round 6 expands. +const SCAN_STEMS: usize = 48; + +const COST_CASE: u32 = 0; +const COST_ADD_DIACRITIC: u32 = 4; +const COST_OTHER_MAP: u32 = 7; +const COST_DROP_DIACRITIC: u32 = 11; +const COST_REP: u32 = 10; +const COST_SPLIT: u32 = 12; +const COST_SPLIT_RARE: u32 = 18; +const COST_SWAP: u32 = 14; +const COST_DROP_OR_ADD: u32 = 15; +const COST_MISTYPE: u32 = 16; +const COST_SCAN: u32 = 20; + +struct Collector<'a> { + d: &'a AffixDict, + /// The word, lower case. + input: String, + tested: HashSet, + /// Suggestion → (cost, order found). + found: HashMap, + seq: usize, +} + +impl Collector<'_> { + /// Check a lower-case candidate (a word, or two separated by a space) and keep it if every + /// word is in the dictionary, as written or capitalized (names). + fn offer(&mut self, cand: &str, cost: u32) { + if cand.is_empty() || cand == self.input || !self.tested.insert(cand.to_string()) { + return; + } + let mut words = Vec::new(); + let mut cost = cost; + for part in cand.split(' ') { + if self.d.contains(part) { + words.push(part.to_string()); + } else { + let c = capitalize(part); + if !self.d.contains(&c) { + return; + } + words.push(c); + // A name ranks after a common word at the same distance (`moze` → `może`, then + // `Mozę`). + cost += 1; + } + } + self.keep(words.join(" "), cost); + } + + /// Keep a suggestion known to be correct. + fn keep(&mut self, text: String, cost: u32) { + self.seq += 1; + let seq = self.seq; + let e = self.found.entry(text).or_insert((cost, seq)); + if cost < e.0 { + *e = (cost, seq); + } + } + + fn count_within(&self, cost: u32) -> usize { + self.found.values().filter(|(c, _)| *c <= cost).count() + } +} + +/// The MAP group of each letter (its other members are its likely confusions). +fn alternatives(c: char, map: &[Vec]) -> Vec { + map.iter().filter(|g| g.contains(&c)).flat_map(|g| g.iter().copied()).filter(|&x| x != c).collect() +} + +/// The cost of changing letter `from` to its MAP relative `to`. Adding a missing diacritic is the +/// commonest slip (Polish typed on a keyboard without Polish letters: `zolw`); a diacritic the +/// writer did type is likelier right, so taking one away costs more than a known confusion from +/// the REP table (`żeka` → `rzeka` before `zeka`). +fn map_cost(from: char, to: char) -> u32 { + match (from.is_ascii(), to.is_ascii()) { + (true, false) => COST_ADD_DIACRITIC, + (false, true) => COST_DROP_DIACRITIC, + _ => COST_OTHER_MAP, + } +} + +/// Every way of changing up to [`MAX_MAP_CHANGES`] letters to a MAP relative, with its cost. +fn map_variants(chars: &[char], alts: &[Vec], i: usize, changes: usize, cost: u32, buf: &mut Vec, out: &mut Vec<(String, u32)>) { + if out.len() >= MAX_MAP_VARIANTS { + return; + } + let Some(&c) = chars.get(i) else { + if changes > 0 { + out.push((buf.iter().collect(), cost)); + } + return; + }; + buf.push(c); + map_variants(chars, alts, i + 1, changes, cost, buf, out); + buf.pop(); + if changes < MAX_MAP_CHANGES { + for &a in alts.get(i).map(Vec::as_slice).unwrap_or_default() { + buf.push(a); + map_variants(chars, alts, i + 1, changes + 1, cost + map_cost(c, a), buf, out); + buf.pop(); + } + } +} + +/// A letter's MAP representative (the first of its group), lower case: `ż` → `z`, `ó` → `o`. +fn fold(c: char, map: &[Vec]) -> char { + let l = c.to_lowercase().next().unwrap_or(c); + map.iter().find(|g| g.contains(&l)).and_then(|g| g.first().copied()).unwrap_or(l) +} + +/// Optimal string alignment distance where a MAP slip (`a`/`ą`, `u`/`ó`) costs 1 and any other +/// edit 2; `limit + 1` once it is certainly above `limit`. +fn weighted_distance(a: &[char], b: &[char], related: &dyn Fn(char, char) -> bool, limit: u32) -> u32 { + let m = b.len(); + let over = limit + 1; + let mut prev2 = vec![0u32; m + 1]; + let mut prev: Vec = (0..=m as u32).map(|j| j * 2).collect(); + let mut cur = vec![0u32; m + 1]; + for (i, &ai) in a.iter().enumerate() { + if let Some(c) = cur.get_mut(0) { + *c = (i as u32 + 1) * 2; + } + let mut row_min = u32::MAX; + for (j, &bj) in b.iter().enumerate() { + let sub = if ai == bj { + 0 + } else if related(ai, bj) { + 1 + } else { + 2 + }; + let at = |v: &[u32], k: usize| v.get(k).copied().unwrap_or(u32::MAX / 4); + let mut v = (at(&prev, j + 1) + 2).min(at(&cur, j) + 2).min(at(&prev, j) + sub); + if i > 0 && j > 0 && a.get(i - 1) == Some(&bj) && b.get(j - 1) == Some(&ai) { + v = v.min(at(&prev2, j - 1) + 2); + } + if let Some(c) = cur.get_mut(j + 1) { + *c = v; + } + row_min = row_min.min(v); + } + if m > 0 && row_min > limit { + return over; + } + std::mem::swap(&mut prev2, &mut prev); + std::mem::swap(&mut prev, &mut cur); + } + prev.get(m).copied().unwrap_or(over).min(over) +} + +/// Spelling suggestions for a Polish word, best first, at most `max`, in the word's letter case. +pub fn suggest(word: &str, max: usize) -> Vec { + suggest_in(&super::dictionary(), word, max) +} + +/// [`suggest`] against a given dictionary. +pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec { + let word = word.trim(); + let n = word.chars().count(); + if n == 0 || n > MAX_LETTERS || max == 0 || d.is_empty() || word.contains(char::is_whitespace) { + return Vec::new(); + } + let case = casing(word); + let input = word.to_lowercase(); + let chars: Vec = input.chars().collect(); + let mut col = Collector { d, input: input.clone(), tested: HashSet::new(), found: HashMap::new(), seq: 0 }; + // 1. Letter case. + if d.contains(&input) && word != input { + col.keep(input.clone(), COST_CASE); + } else if d.contains(&capitalize(&input)) && word != capitalize(&input) { + col.keep(capitalize(&input), COST_CASE); + } + // 2. Diacritics and other MAP slips. + let map = d.map(); + let alts: Vec> = chars.iter().map(|&c| alternatives(c, map)).collect(); + let mut variants = Vec::new(); + map_variants(&chars, &alts, 0, 0, 0, &mut Vec::with_capacity(n), &mut variants); + variants.sort_by_key(|v| v.1); + for (v, cost) in &variants { + col.offer(v, *cost); + } + // 3. Common confusions. + for (from, to) in d.rep() { + if from.is_empty() { + continue; + } + for (i, _) in input.match_indices(from.as_str()) { + if let (Some(a), Some(b)) = (input.get(..i), input.get(i + from.len()..)) { + col.offer(&format!("{a}{to}{b}"), COST_REP); + } + } + } + // 4. A missing space: cheap after a short leading word or before a short trailing one, + // dearer between two longer words, never leaving a stray short half (`że ka`). + for (i, _) in input.char_indices().skip(1) { + let (Some(a), Some(b)) = (input.get(..i), input.get(i..)) else { continue }; + let long = |p: &str| p.chars().count() >= 3; + let cost = if (LEADING_WORDS.contains(&a) && long(b)) || (long(a) && TRAILING_WORDS.contains(&b)) { + COST_SPLIT + } else if long(a) && long(b) { + COST_SPLIT_RARE + } else { + continue; + }; + col.offer(&format!("{a} {b}"), cost); + } + // 5. One typo. + let mut letters: Vec = d.try_chars().chars().filter(|c| c.is_lowercase()).collect(); + if letters.is_empty() { + letters = "aioeznrwcysptkmdłuljągbhęśćóżfńźvqx".chars().collect(); + } + let s = |v: &[char]| v.iter().collect::(); + for i in 0..n.saturating_sub(1) { + if chars.get(i) != chars.get(i + 1) { + let mut v = chars.clone(); + v.swap(i, i + 1); + col.offer(&s(&v), COST_SWAP); + } + } + for i in 0..n { + let mut v = chars.clone(); + v.remove(i); + col.offer(&s(&v), COST_DROP_OR_ADD); + } + for i in 0..=n { + for &c in &letters { + let mut v = chars.clone(); + v.insert(i, c); + col.offer(&s(&v), COST_DROP_OR_ADD); + } + } + for i in 0..n { + for &c in &letters { + if chars.get(i) != Some(&c) { + let mut v = chars.clone(); + if let Some(x) = v.get_mut(i) { + *x = c; + } + col.offer(&s(&v), COST_MISTYPE); + } + } + } + // 6. Further off: scan the stems that start like the word. + if col.count_within(COST_MISTYPE) < max { + scan(&mut col, &chars); + } + let mut found: Vec<(String, (u32, usize))> = col.found.into_iter().collect(); + found.sort_by_key(|(_, k)| *k); + let mut out: Vec = Vec::new(); + for (text, _) in found { + let t = match case { + Casing::Initial => capitalize(&text), + Casing::Upper if n > 1 => text.to_uppercase(), + _ => text, + }; + if t != word && !out.contains(&t) { + out.push(t); + } + if out.len() >= max { + break; + } + } + out +} + +/// Round 6: rank the stems starting with the word's first letter (or a MAP relative of it, either +/// case) by shared letter pairs, then keep the forms of the best within two typos. +fn scan(col: &mut Collector, chars: &[char]) { + let d = col.d; + let map = d.map(); + let Some(&first) = chars.first() else { return }; + let folded: Vec = chars.iter().map(|&c| fold(c, map)).collect(); + let pairs: HashSet<(char, char)> = std::iter::once(' ').chain(folded.iter().copied()).zip(folded.iter().copied()).collect(); + let mut firsts: Vec = std::iter::once(first).chain(alternatives(first, map)).collect(); + let upper: Vec = firsts.iter().flat_map(|c| c.to_uppercase()).collect(); + firsts.extend(upper); + firsts.sort_unstable(); + firsts.dedup(); + let n = folded.len(); + let mut best: Vec<(u32, &str)> = Vec::new(); + for f in firsts { + let prefix = f.to_string(); + for stem in d.stems_with_prefix(&prefix) { + let sf: Vec = stem.chars().map(|c| fold(c, map)).collect(); + let shared = std::iter::once(' ').chain(sf.iter().copied()).zip(sf.iter().copied()).filter(|p| pairs.contains(p)).count(); + if shared < 2 { + continue; + } + // Shared pairs, less the stem's letters the word doesn't explain, less a length gap + // beyond what an ending explains. + let extra = sf.len().saturating_sub(shared); + let gap = sf.len().abs_diff(n).saturating_sub(4); + let score = (shared * 4).saturating_sub(extra * 3 + gap * 2) as u32; + if score == 0 { + continue; + } + if best.len() < SCAN_STEMS { + best.push((score, stem)); + } else if let Some(min) = best.iter_mut().min_by_key(|b| b.0) + && score > min.0 + { + *min = (score, stem); + } + } + } + best.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(b.1))); + let related = |a: char, b: char| fold(a, map) == fold(b, map); + let limit = 4; + for (_, stem) in best { + for form in d.forms(stem) { + let lf: Vec = form.to_lowercase().chars().collect(); + if lf.len().abs_diff(chars.len()) > 2 { + continue; + } + let dist = weighted_distance(&lf, chars, &related, limit); + if dist <= limit && lf.as_slice() != chars { + col.keep(form, COST_SCAN + dist * 3); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn has(word: &str, want: &str) { + let s = suggest(word, 8); + assert!(s.iter().any(|x| x == want), "{word} → {s:?}, want {want}"); + } + + #[test] + fn polish_misspellings_get_the_right_suggestion() { + crate::pl::test_dict::install(); + has("żeka", "rzeka"); + has("kturzy", "którzy"); + has("napewno", "na pewno"); + has("wogóle", "w ogóle"); + has("niewiem", "nie wiem"); + has("zolw", "żółw"); + has("gzegzolka", "gżegżółka"); + has("chuśtawka", "huśtawka"); + has("bżoza", "brzoza"); + has("pszyjaciel", "przyjaciel"); + has("warszawa", "Warszawa"); + has("Żeka", "Rzeka"); + has("ZOLW", "ŻÓŁW"); + has("konstytucija", "konstytucja"); + has("wziąść", "wziąć"); + has("dziewczyan", "dziewczyna"); + } + + #[test] + fn the_likeliest_fix_comes_first() { + crate::pl::test_dict::install(); + assert_eq!(suggest("kturzy", 3).first().map(String::as_str), Some("którzy")); + assert_eq!(suggest("zolw", 3).first().map(String::as_str), Some("żółw")); + assert_eq!(suggest("napewno", 3).first().map(String::as_str), Some("na pewno")); + assert_eq!(suggest("żeka", 3).first().map(String::as_str), Some("rzeka")); + } + + #[test] + fn edge_cases_stay_empty_or_bounded() { + crate::pl::test_dict::install(); + assert!(suggest("", 5).is_empty()); + assert!(suggest("kot", 0).is_empty()); + assert!(suggest(&"a".repeat(41), 5).is_empty()); + assert!(suggest("dwa słowa", 5).is_empty()); + assert!(suggest("żółw", 5).len() <= 5); + assert!(!suggest("żółw", 5).contains(&"żółw".to_string())); + } + + #[test] + fn distance_weighs_diacritics_half() { + let map = vec![vec!['a', 'ą'], vec!['o', 'ó', 'u']]; + let related = |a: char, b: char| fold(a, &map) == fold(b, &map); + let c = |s: &str| s.chars().collect::>(); + assert_eq!(weighted_distance(&c("ktury"), &c("który"), &related, 9), 1); + assert_eq!(weighted_distance(&c("kot"), &c("kto"), &related, 9), 2); + assert_eq!(weighted_distance(&c("kot"), &c("kotek"), &related, 9), 4); + assert_eq!(weighted_distance(&c("kot"), &c("pies"), &related, 2), 3); + assert_eq!(weighted_distance(&c(""), &c("ab"), &related, 9), 4); + } + + #[test] + fn suggestions_are_fast_enough() { + crate::pl::test_dict::install(); + // Warm the dictionary, then time a worst case (a long word nothing near): the scan round. + let _ = suggest("kot", 1); + let t = std::time::Instant::now(); + for w in ["xqzvwpolskiegoslowa", "pszyjacielowi", "zażółćgęśląjaźń"] { + let _ = suggest(w, 6); + } + let ms = t.elapsed().as_secs_f64() * 1000.0; + eprintln!("3 suggestion calls: {ms:.0} ms"); + assert!(ms < 20_000.0, "{ms} ms"); + } +} diff --git a/crates/proof/src/pl/test_dict.rs b/crates/proof/src/pl/test_dict.rs new file mode 100644 index 00000000..4bd16d4c --- /dev/null +++ b/crates/proof/src/pl/test_dict.rs @@ -0,0 +1,108 @@ +//! A tiny synthetic Polish Hunspell dictionary for tests (written for these tests; not taken from +//! any real dictionary). Real dictionaries are installed on the system, never bundled. + +/// Affix rules: a few suffixes and one prefix, plus the suggestion tables the Polish +/// suggestions use (`TRY`, `MAP` for diacritics, `REP` for common confusions). +pub const AFF: &str = "SET UTF-8 +TRY aioeznrwcysżtkpdmłjlubąęgchśóźćń +MAP 9 +MAP aą +MAP cć +MAP eę +MAP lł +MAP nń +MAP oó +MAP uó +MAP sś +MAP zżź +REP 6 +REP ż rz +REP rz ż +REP ch h +REP h ch +REP sz rz +REP ść ć +SFX A Y 1 +SFX A a ie a +SFX B Y 3 +SFX B y ego y +SFX B i iego i +SFX B i iej i +PFX N Y 1 +PFX N 0 nie . +"; + +/// Stems (the count line first, as in Hunspell files). +pub const DIC: &str = "63 +źdźbło +gżegżółka +chrząszcz +pięćdziesięciu +rzeka +którzy +żółw +zażółć +gęślą +jaźń +konstytucja +konstytucyjny/B +niebieski/B +niemiecki/B +ładnie/N +przyszliśmy +zrobilibyśmy +Warszawa/A +kot +biało +czerwony +polsko +mail +w +się +huśtawka +brzoza +przyjaciel +wziąć +dziewczyna +na +pewno +ogóle +nie +wiem +płynie +a +i +o +to +że +tak +jest +brzmi +trzcina/A +napisał +poszedł +spać +bo +wylała +świat +dom +ma +ala +kota +dzień +dobry +jutro +bardzo +dobrze +mamy +czas +ten +"; + +/// Make [`AFF`]/[`DIC`] the Polish dictionary (every test that needs one calls this; they all +/// install the same one, so tests running in parallel agree). +pub fn install() { + if let Err(e) = super::install_dictionary(AFF, DIC) { + panic!("test dictionary: {e}"); + } +}