diff --git a/crates/proof/src/hunspell.rs b/crates/proof/src/hunspell.rs index ab5ecc03..245836dc 100644 --- a/crates/proof/src/hunspell.rs +++ b/crates/proof/src/hunspell.rs @@ -137,8 +137,24 @@ fn decode_latin2(bytes: &[u8]) -> String { bytes.iter().map(|&b| if b < 0xa0 { char::from(b) } else { high.get(usize::from(b - 0xa0)).copied().unwrap_or('\u{fffd}') }).collect() } +/// KOI8-R code points 0x80..=0xFF (0x00..=0x7F are ASCII): the encoding of older Russian +/// dictionaries. +const KOI8_R_HIGH: &str = + "─│┌┐└┘├┤┬┴┼▀▄█▌▐░▒▓⌠■∙√≈≤≥\u{a0}⌡°²·÷═║╒ё╓╔╕╖╗╘╙╚╛╜╝╞╟╠╡Ё╢╣╤╥╦╧╨╩╪╫╬©юабцдефгхийклмнопярстужвьызшэщчъЮАБЦДЕФГХИЙКЛМНОПЯРСТУЖВЬЫЗШЭЩЧЪ"; + +/// Windows-1251 code points 0x80..=0xFF (0x98 is unassigned): the other one Russian dictionaries +/// use. +const CP1251_HIGH: &str = "ЂЃ‚ѓ„…†‡€‰Љ‹ЊЌЋЏђ‘’“”•–—\u{fffd}™љ›њќћџ\u{a0}ЎўЈ¤Ґ¦§Ё©Є«¬\u{ad}®Ї°±Ііґµ¶·ё№є»јЅѕїАБВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯабвгдежзийклмнопрстуфхцчшщъыьэюя"; + +/// Decode a single-byte encoding whose bytes 0x80..=0xFF are `high` (ASCII below). +fn decode_high(bytes: &[u8], high: &str) -> String { + let high: Vec = high.chars().collect(); + bytes.iter().map(|&b| if b < 0x80 { char::from(b) } else { high.get(usize::from(b - 0x80)).copied().unwrap_or('\u{fffd}') }).collect() +} + /// Decode an `.aff`/`.dic` pair to UTF-8 from the encoding the `.aff`'s `SET` line names -/// (UTF-8 when there is none; ISO 8859-2 is the other one Polish dictionaries use). +/// (UTF-8 when there is none; ISO 8859-2 is the other one Polish dictionaries use, KOI8-R and +/// Windows-1251 the ones older Russian dictionaries use). pub fn decode(aff: &[u8], dic: &[u8]) -> Result<(String, String), String> { let head = String::from_utf8_lossy(aff.get(..aff.len().min(8192)).unwrap_or_default()).to_string(); let set = head.lines().find_map(|l| l.trim_start_matches('\u{feff}').strip_prefix("SET ")).map(str::trim).unwrap_or("UTF-8").to_ascii_uppercase(); @@ -148,6 +164,8 @@ pub fn decode(aff: &[u8], dic: &[u8]) -> Result<(String, String), String> { Ok((text(aff, ".aff")?, text(dic, ".dic")?)) } "ISO8859-2" | "ISO-8859-2" => Ok((decode_latin2(aff), decode_latin2(dic))), + "KOI8-R" | "KOI8R" => Ok((decode_high(aff, KOI8_R_HIGH), decode_high(dic, KOI8_R_HIGH))), + "CP1251" | "WINDOWS-1251" | "MICROSOFT-CP1251" => Ok((decode_high(aff, CP1251_HIGH), decode_high(dic, CP1251_HIGH))), other => Err(format!("unsupported encoding {other}")), } } @@ -164,10 +182,21 @@ mod tests { assert_eq!(d, "2\nżółw\n"); let (_, d) = decode("SET UTF-8\n".as_bytes(), "żółw".as_bytes()).unwrap(); assert_eq!(d, "żółw"); - assert!(decode(b"SET KOI8-R\n", b"").is_err()); + assert!(decode(b"SET SHIFT_JIS\n", b"").is_err()); assert!(decode(b"SET UTF-8\n", b"\xff\xfe").is_err()); } + #[test] + fn decodes_russian_single_byte_encodings() { + // `ёлка` in KOI8-R (0xA3 0xCC 0xCB 0xC1) and in Windows-1251 (0xB8 0xEB 0xEA 0xE0). + let (a, d) = decode(b"SET KOI8-R\nTRY \xcf\xc5\n", b"1\n\xa3\xcc\xcb\xc1/A\n").unwrap(); + assert!(a.contains("TRY ое"), "{a}"); + assert_eq!(d, "1\nёлка/A\n"); + let (_, d) = decode(b"SET microsoft-cp1251\n", b"1\n\xb8\xeb\xea\xe0 \xc4\xee\xec\n").unwrap(); + assert_eq!(d, "1\nёлка Дом\n"); + assert_eq!(decode_high(&[0x98], CP1251_HIGH), "\u{fffd}"); + } + #[test] fn libreoffice_extension_dirs_follow_the_language() { assert_eq!(lo_ext_dirs("pl_PL"), ["dict-pl", "dict-pl-PL"]); diff --git a/crates/proof/src/lib.rs b/crates/proof/src/lib.rs index 379bae2a..76baf4b5 100644 --- a/crates/proof/src/lib.rs +++ b/crates/proof/src/lib.rs @@ -5,7 +5,7 @@ //! pair, word checks, generated forms, the suggestion tables (`TRY`, `MAP`, `REP`) and a //! compact serialized form ([`affix::AffixDict::to_bytes`]) for fast loading. //! - [`hunspell`]: finding an installed pair per app ([`hunspell::Search`]), decoding it from -//! UTF-8 or ISO 8859-2 and loading it. +//! UTF-8, ISO 8859-2, KOI8-R or Windows-1251 and loading it. //! - [`tag`]: canonical BCP 47 tags and primary subtags, for mapping a text's language to a //! dictionary name. //!