CP7.2: affix_check example — compare the affix engine with another checker's answers (black-box)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01FypgCX8LsYYE8fsj4byBm2

crates/proof/examples/affix_check.rs

@@ -0,0 +1,126 @@
+//! Compare the affix engine with another spell checker, black-box: load a Hunspell `.aff`/`.dic`
+//! pair and answer for words, or make candidate compounds to ask both about.
+//!
+//! ```sh
+//! # One word per line in, `word<TAB>1` (right) or `word<TAB>0` out, and the time taken on stderr.
+//! cargo run --release -p craft-proof --example affix_check -- check de_DE.aff de_DE.dic < words.txt
+//! # 2000 candidates (forms of random stems glued together, some with a letter changed), seed 7.
+//! cargo run --release -p craft-proof --example affix_check -- gen de_DE.aff de_DE.dic 2000 7 > words.txt
+//! ```
+//!
+//! The answers are compared with `hunspell -d <path without extension> -l` on the same words
+//! (the words it lists are the wrong ones). Nothing of the other checker is read but its answers.
+
+use std::io::{BufRead, Write};
+
+use craft_proof::affix::AffixDict;
+
+fn load(aff: &str, dic: &str) -> Result<AffixDict, String> {
+    let aff = std::fs::read(aff).map_err(|e| format!("{aff}: {e}"))?;
+    let dic = std::fs::read(dic).map_err(|e| format!("{dic}: {e}"))?;
+    let (aff, dic) = craft_proof::hunspell::decode(&aff, &dic)?;
+    AffixDict::from_hunspell(&aff, &dic, &[])
+}
+
+/// xorshift64*: a reproducible stream of numbers, enough to pick stems.
+struct Rng(u64);
+
+impl Rng {
+    fn next(&mut self) -> u64 {
+        self.0 ^= self.0 >> 12;
+        self.0 ^= self.0 << 25;
+        self.0 ^= self.0 >> 27;
+        self.0.wrapping_mul(0x2545_f491_4f6c_dd1d)
+    }
+
+    fn below(&mut self, n: usize) -> usize {
+        if n == 0 { 0 } else { (self.next() % n as u64) as usize }
+    }
+}
+
+/// `w` with its first letter in lower case (a compound's later parts).
+fn decapitalize(w: &str) -> String {
+    let mut c = w.chars();
+    c.next().map(|f| f.to_lowercase().chain(c).collect()).unwrap_or_default()
+}
+
+fn generate(d: &AffixDict, n: usize, seed: u64) -> Vec<String> {
+    let stems: Vec<&str> = d.stems().filter(|s| s.chars().all(char::is_alphabetic)).collect();
+    let mut rng = Rng(seed.max(1));
+    let form = |rng: &mut Rng| -> String {
+        let s = stems.get(rng.below(stems.len())).copied().unwrap_or_default();
+        let forms = d.forms(s);
+        forms.get(rng.below(forms.len())).cloned().unwrap_or_else(|| s.to_string())
+    };
+    let mut out = Vec::with_capacity(n);
+    while out.len() < n {
+        let parts = 2 + rng.below(2);
+        let mut w = form(&mut rng);
+        for _ in 1..parts {
+            if rng.below(4) == 0 {
+                w.push('s');
+            }
+            w.push_str(&decapitalize(&form(&mut rng)));
+        }
+        if rng.below(3) == 0 {
+            // A letter dropped, doubled or changed: mostly wrong words.
+            let chars: Vec<char> = w.chars().collect();
+            let i = rng.below(chars.len());
+            let mut m: String = chars.iter().take(i).collect();
+            match rng.below(3) {
+                0 => {}
+                1 => m.extend(chars.get(i).into_iter().chain(chars.get(i))),
+                _ => m.push('e'),
+            }
+            m.extend(chars.iter().skip(i + 1));
+            w = m;
+        }
+        // Letters only: the other checker's command line splits words at anything else.
+        if w.chars().count() >= 4 && w.len() <= 100 && w.chars().all(char::is_alphabetic) {
+            out.push(w);
+        }
+    }
+    out
+}
+
+fn main() -> Result<(), String> {
+    let args: Vec<String> = std::env::args().skip(1).collect();
+    match args.as_slice() {
+        [cmd, aff, dic] if cmd == "check" => {
+            let d = load(aff, dic)?;
+            let words: Vec<String> = std::io::stdin().lock().lines().map_while(Result::ok).collect();
+            let t = std::time::Instant::now();
+            let mut slowest: Vec<(std::time::Duration, &str)> = Vec::new();
+            let answers: Vec<bool> = words
+                .iter()
+                .map(|w| {
+                    let t = std::time::Instant::now();
+                    let ok = d.check(w);
+                    slowest.push((t.elapsed(), w));
+                    ok
+                })
+                .collect();
+            let took = t.elapsed();
+            slowest.sort_by_key(|s| std::cmp::Reverse(s.0));
+            for (t, w) in slowest.iter().take(3) {
+                eprintln!("slow: {w} {t:?}");
+            }
+            let mut out = std::io::stdout().lock();
+            for (w, ok) in words.iter().zip(answers) {
+                writeln!(out, "{w}\t{}", u8::from(ok)).map_err(|e| e.to_string())?;
+            }
+            eprintln!("{} words in {took:?}", words.len());
+            Ok(())
+        }
+        [cmd, aff, dic, n, seed] if cmd == "gen" => {
+            let d = load(aff, dic)?;
+            let n: usize = n.parse().map_err(|_| "n is a number")?;
+            let seed: u64 = seed.parse().map_err(|_| "seed is a number")?;
+            for w in generate(&d, n, seed) {
+                println!("{w}");
+            }
+            Ok(())
+        }
+        _ => Err("usage: affix_check check <aff> <dic> < words | affix_check gen <aff> <dic> <n> <seed>".into()),
+    }
+}