diff --git a/crates/proof/examples/affix_check.rs b/crates/proof/examples/affix_check.rs new file mode 100644 index 00000000..31692618 --- /dev/null +++ b/crates/proof/examples/affix_check.rs @@ -0,0 +1,126 @@ +//! Compare the affix engine with another spell checker, black-box: load a Hunspell `.aff`/`.dic` +//! pair and answer for words, or make candidate compounds to ask both about. +//! +//! ```sh +//! # One word per line in, `word1` (right) or `word0` out, and the time taken on stderr. +//! cargo run --release -p craft-proof --example affix_check -- check de_DE.aff de_DE.dic < words.txt +//! # 2000 candidates (forms of random stems glued together, some with a letter changed), seed 7. +//! cargo run --release -p craft-proof --example affix_check -- gen de_DE.aff de_DE.dic 2000 7 > words.txt +//! ``` +//! +//! The answers are compared with `hunspell -d -l` on the same words +//! (the words it lists are the wrong ones). Nothing of the other checker is read but its answers. + +use std::io::{BufRead, Write}; + +use craft_proof::affix::AffixDict; + +fn load(aff: &str, dic: &str) -> Result { + let aff = std::fs::read(aff).map_err(|e| format!("{aff}: {e}"))?; + let dic = std::fs::read(dic).map_err(|e| format!("{dic}: {e}"))?; + let (aff, dic) = craft_proof::hunspell::decode(&aff, &dic)?; + AffixDict::from_hunspell(&aff, &dic, &[]) +} + +/// xorshift64*: a reproducible stream of numbers, enough to pick stems. +struct Rng(u64); + +impl Rng { + fn next(&mut self) -> u64 { + self.0 ^= self.0 >> 12; + self.0 ^= self.0 << 25; + self.0 ^= self.0 >> 27; + self.0.wrapping_mul(0x2545_f491_4f6c_dd1d) + } + + fn below(&mut self, n: usize) -> usize { + if n == 0 { 0 } else { (self.next() % n as u64) as usize } + } +} + +/// `w` with its first letter in lower case (a compound's later parts). +fn decapitalize(w: &str) -> String { + let mut c = w.chars(); + c.next().map(|f| f.to_lowercase().chain(c).collect()).unwrap_or_default() +} + +fn generate(d: &AffixDict, n: usize, seed: u64) -> Vec { + let stems: Vec<&str> = d.stems().filter(|s| s.chars().all(char::is_alphabetic)).collect(); + let mut rng = Rng(seed.max(1)); + let form = |rng: &mut Rng| -> String { + let s = stems.get(rng.below(stems.len())).copied().unwrap_or_default(); + let forms = d.forms(s); + forms.get(rng.below(forms.len())).cloned().unwrap_or_else(|| s.to_string()) + }; + let mut out = Vec::with_capacity(n); + while out.len() < n { + let parts = 2 + rng.below(2); + let mut w = form(&mut rng); + for _ in 1..parts { + if rng.below(4) == 0 { + w.push('s'); + } + w.push_str(&decapitalize(&form(&mut rng))); + } + if rng.below(3) == 0 { + // A letter dropped, doubled or changed: mostly wrong words. + let chars: Vec = w.chars().collect(); + let i = rng.below(chars.len()); + let mut m: String = chars.iter().take(i).collect(); + match rng.below(3) { + 0 => {} + 1 => m.extend(chars.get(i).into_iter().chain(chars.get(i))), + _ => m.push('e'), + } + m.extend(chars.iter().skip(i + 1)); + w = m; + } + // Letters only: the other checker's command line splits words at anything else. + if w.chars().count() >= 4 && w.len() <= 100 && w.chars().all(char::is_alphabetic) { + out.push(w); + } + } + out +} + +fn main() -> Result<(), String> { + let args: Vec = std::env::args().skip(1).collect(); + match args.as_slice() { + [cmd, aff, dic] if cmd == "check" => { + let d = load(aff, dic)?; + let words: Vec = std::io::stdin().lock().lines().map_while(Result::ok).collect(); + let t = std::time::Instant::now(); + let mut slowest: Vec<(std::time::Duration, &str)> = Vec::new(); + let answers: Vec = words + .iter() + .map(|w| { + let t = std::time::Instant::now(); + let ok = d.check(w); + slowest.push((t.elapsed(), w)); + ok + }) + .collect(); + let took = t.elapsed(); + slowest.sort_by_key(|s| std::cmp::Reverse(s.0)); + for (t, w) in slowest.iter().take(3) { + eprintln!("slow: {w} {t:?}"); + } + let mut out = std::io::stdout().lock(); + for (w, ok) in words.iter().zip(answers) { + writeln!(out, "{w}\t{}", u8::from(ok)).map_err(|e| e.to_string())?; + } + eprintln!("{} words in {took:?}", words.len()); + Ok(()) + } + [cmd, aff, dic, n, seed] if cmd == "gen" => { + let d = load(aff, dic)?; + let n: usize = n.parse().map_err(|_| "n is a number")?; + let seed: u64 = seed.parse().map_err(|_| "seed is a number")?; + for w in generate(&d, n, seed) { + println!("{w}"); + } + Ok(()) + } + _ => Err("usage: affix_check check < words | affix_check gen ".into()), + } +}