craft-proof: decode KOI8-R and Windows-1251 dictionaries (from wordcraft#894)

Older Russian Hunspell dictionaries are in KOI8-R or Windows-1251. The
decoder and its test come from xmaks82's Russian proofing PR,
storytold/wordcraft#894, so that support lands in the shared loader as
asked in storytold/craft-libs#30.

Co-Authored-By: xmaks82 <200731976+xmaks82@users.noreply.github.com>
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

crates/proof/src/hunspell.rs

@@ -137,8 +137,24 @@ fn decode_latin2(bytes: &[u8]) -> String {
     bytes.iter().map(|&b| if b < 0xa0 { char::from(b) } else { high.get(usize::from(b - 0xa0)).copied().unwrap_or('\u{fffd}') }).collect()
 }
 
+/// KOI8-R code points 0x80..=0xFF (0x00..=0x7F are ASCII): the encoding of older Russian
+/// dictionaries.
+const KOI8_R_HIGH: &str =
+    "─│┌┐└┘├┤┬┴┼▀▄█▌▐░▒▓⌠■∙√≈≤≥\u{a0}⌡°²·÷═║╒ё╓╔╕╖╗╘╙╚╛╜╝╞╟╠╡Ё╢╣╤╥╦╧╨╩╪╫╬©юабцдефгхийклмнопярстужвьызшэщчъЮАБЦДЕФГХИЙКЛМНОПЯРСТУЖВЬЫЗШЭЩЧЪ";
+
+/// Windows-1251 code points 0x80..=0xFF (0x98 is unassigned): the other one Russian dictionaries
+/// use.
+const CP1251_HIGH: &str = "ЂЃ‚ѓ„…†‡€‰Љ‹ЊЌЋЏђ‘’“”•–—\u{fffd}™љ›њќћџ\u{a0}ЎўЈ¤Ґ¦§Ё©Є«¬\u{ad}®Ї°±Ііґµ¶·ё№є»јЅѕїАБВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯабвгдежзийклмнопрстуфхцчшщъыьэюя";
+
+/// Decode a single-byte encoding whose bytes 0x80..=0xFF are `high` (ASCII below).
+fn decode_high(bytes: &[u8], high: &str) -> String {
+    let high: Vec<char> = high.chars().collect();
+    bytes.iter().map(|&b| if b < 0x80 { char::from(b) } else { high.get(usize::from(b - 0x80)).copied().unwrap_or('\u{fffd}') }).collect()
+}
+
 /// Decode an `.aff`/`.dic` pair to UTF-8 from the encoding the `.aff`'s `SET` line names
-/// (UTF-8 when there is none; ISO 8859-2 is the other one Polish dictionaries use).
+/// (UTF-8 when there is none; ISO 8859-2 is the other one Polish dictionaries use, KOI8-R and
+/// Windows-1251 the ones older Russian dictionaries use).
 pub fn decode(aff: &[u8], dic: &[u8]) -> Result<(String, String), String> {
     let head = String::from_utf8_lossy(aff.get(..aff.len().min(8192)).unwrap_or_default()).to_string();
     let set = head.lines().find_map(|l| l.trim_start_matches('\u{feff}').strip_prefix("SET ")).map(str::trim).unwrap_or("UTF-8").to_ascii_uppercase();
@@ -148,6 +164,8 @@ pub fn decode(aff: &[u8], dic: &[u8]) -> Result<(String, String), String> {
             Ok((text(aff, ".aff")?, text(dic, ".dic")?))
         }
         "ISO8859-2" | "ISO-8859-2" => Ok((decode_latin2(aff), decode_latin2(dic))),
+        "KOI8-R" | "KOI8R" => Ok((decode_high(aff, KOI8_R_HIGH), decode_high(dic, KOI8_R_HIGH))),
+        "CP1251" | "WINDOWS-1251" | "MICROSOFT-CP1251" => Ok((decode_high(aff, CP1251_HIGH), decode_high(dic, CP1251_HIGH))),
         other => Err(format!("unsupported encoding {other}")),
     }
 }
@@ -164,10 +182,21 @@ mod tests {
         assert_eq!(d, "2\nżółw\n");
         let (_, d) = decode("SET UTF-8\n".as_bytes(), "żółw".as_bytes()).unwrap();
         assert_eq!(d, "żółw");
-        assert!(decode(b"SET KOI8-R\n", b"").is_err());
+        assert!(decode(b"SET SHIFT_JIS\n", b"").is_err());
         assert!(decode(b"SET UTF-8\n", b"\xff\xfe").is_err());
     }
 
+    #[test]
+    fn decodes_russian_single_byte_encodings() {
+        // `ёлка` in KOI8-R (0xA3 0xCC 0xCB 0xC1) and in Windows-1251 (0xB8 0xEB 0xEA 0xE0).
+        let (a, d) = decode(b"SET KOI8-R\nTRY \xcf\xc5\n", b"1\n\xa3\xcc\xcb\xc1/A\n").unwrap();
+        assert!(a.contains("TRY ое"), "{a}");
+        assert_eq!(d, "1\nёлка/A\n");
+        let (_, d) = decode(b"SET microsoft-cp1251\n", b"1\n\xb8\xeb\xea\xe0 \xc4\xee\xec\n").unwrap();
+        assert_eq!(d, "1\nёлка Дом\n");
+        assert_eq!(decode_high(&[0x98], CP1251_HIGH), "\u{fffd}");
+    }
+
     #[test]
     fn libreoffice_extension_dirs_follow_the_language() {
         assert_eq!(lo_ext_dirs("pl_PL"), ["dict-pl", "dict-pl-PL"]);

crates/proof/src/lib.rs

@@ -5,7 +5,7 @@
 //!   pair, word checks, generated forms, the suggestion tables (`TRY`, `MAP`, `REP`) and a
 //!   compact serialized form ([`affix::AffixDict::to_bytes`]) for fast loading.
 //! - [`hunspell`]: finding an installed pair per app ([`hunspell::Search`]), decoding it from
-//!   UTF-8 or ISO 8859-2 and loading it.
+//!   UTF-8, ISO 8859-2, KOI8-R or Windows-1251 and loading it.
 //! - [`tag`]: canonical BCP 47 tags and primary subtags, for mapping a text's language to a
 //!   dictionary name.
 //!