mirror of
https://github.com/storytold/craft-libs.git
synced 2026-11-08 19:42:12 +00:00
CP6.1: decode SET ISO8859-7 and ISO8859-15 (Greek and Estonian dictionaries)
Tables from the Unicode Consortium's 8859-7.TXT (2003, with € ₯ ͺ) and 8859-15.TXT mapping files; unassigned bytes become U+FFFD. These are the only manifest SETs craft-proof couldn't decode. el now loads and passes its check words; et loads and next stops at COMPOUNDFLAG (compounding, storytold/wordcraft#2070). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01FypgCX8LsYYE8fsj4byBm2
This commit is contained in:
1 parent
bb859671cc
commit
16e4ed3115
1 file changed
+47
-4
@@ -260,8 +260,21 @@ fn decode_latin5(bytes: &[u8]) -> String {
|
||||
/// ISO 8859-2 code points 0xA0..=0xFF (0x00..=0x9F map to themselves).
|
||||
const ISO_8859_2_HIGH: &str = "\u{a0}Ą˘Ł¤ĽŚ§¨ŠŞŤŹ\u{ad}ŽŻ°ą˛ł´ľśˇ¸šşťź˝žżŔÁÂĂÄĹĆÇČÉĘËĚÍÎĎĐŃŇÓÔŐÖ×ŘŮÚŰÜÝŢßŕáâăäĺćçčéęëěíîďđńňóôőö÷řůúűüýţ˙";
|
||||
|
||||
fn decode_latin2(bytes: &[u8]) -> String {
|
||||
let high: Vec<char> = ISO_8859_2_HIGH.chars().collect();
|
||||
/// ISO 8859-7 (Greek) code points 0xA0..=0xFF; 0xAE, 0xD2 and 0xFF are unassigned and all read
|
||||
/// as U+FFFD. From the
|
||||
/// Unicode Consortium's mapping file `MAPPINGS/ISO8859/8859-7.TXT` (2003 edition, with € ₯ ͺ).
|
||||
/// The encoding of the Greek dictionary (`SET ISO8859-7`).
|
||||
const ISO_8859_7_HIGH: &str =
|
||||
"\u{a0}‘’£€₯¦§¨©ͺ«¬\u{ad}\u{fffd}―°±²³΄΅Ά·ΈΉΊ»Ό½ΎΏΐΑΒΓΔΕΖΗΘΙΚΛΜΝΞΟΠΡ\u{fffd}ΣΤΥΦΧΨΩΪΫάέήίΰαβγδεζηθικλμνξοπρςστυφχψωϊϋόύώ\u{fffd}";
|
||||
|
||||
/// ISO 8859-15 (Latin-9) code points 0xA0..=0xFF: ISO 8859-1 with € Š š Ž ž Œ œ Ÿ in eight
|
||||
/// places. From the Unicode Consortium's mapping file `MAPPINGS/ISO8859/8859-15.TXT`. The
|
||||
/// encoding of the Estonian dictionary (`SET ISO8859-15`).
|
||||
const ISO_8859_15_HIGH: &str = "\u{a0}¡¢£€¥Š§š©ª«¬\u{ad}®¯°±²³Žµ¶·ž¹º»ŒœŸ¿ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖרÙÚÛÜÝÞßàáâãäåæçèéêëìíîïðñòóôõö÷øùúûüýþÿ";
|
||||
|
||||
/// Decode an ISO 8859 part whose bytes 0xA0..=0xFF are `high` (0x00..=0x9F map to themselves).
|
||||
fn decode_iso(bytes: &[u8], high: &str) -> String {
|
||||
let high: Vec<char> = high.chars().collect();
|
||||
bytes.iter().map(|&b| if b < 0xa0 { char::from(b) } else { high.get(usize::from(b - 0xa0)).copied().unwrap_or('\u{fffd}') }).collect()
|
||||
}
|
||||
|
||||
@@ -282,7 +295,8 @@ fn decode_high(bytes: &[u8], high: &str) -> String {
|
||||
|
||||
/// Decode an `.aff`/`.dic` pair to UTF-8 from the encoding the `.aff`'s `SET` line names
|
||||
/// (UTF-8 when there is none; ISO 8859-2 is the other one Polish dictionaries use, ISO 8859-1 the
|
||||
/// one the original Portuguese VERO uses, ISO 8859-9 the one older Turkish dictionaries use, KOI8-R
|
||||
/// one the original Portuguese VERO uses, ISO 8859-7 the Greek one, ISO 8859-15 the Estonian one,
|
||||
/// ISO 8859-9 the one older Turkish dictionaries use, KOI8-R
|
||||
/// and Windows-1251 the ones older Russian dictionaries use).
|
||||
///
|
||||
/// Under a UTF-8 `SET` with single-byte flags (no `FLAG` line, or `FLAG DEFAULT`), flags are bytes,
|
||||
@@ -303,7 +317,9 @@ pub fn decode(aff: &[u8], dic: &[u8]) -> Result<(String, String), String> {
|
||||
Ok((decode_utf8_aff(aff)?, decode_utf8_dic(dic)?))
|
||||
}
|
||||
"ISO8859-1" | "ISO-8859-1" => Ok((decode_latin1(aff), decode_latin1(dic))),
|
||||
"ISO8859-2" | "ISO-8859-2" => Ok((decode_latin2(aff), decode_latin2(dic))),
|
||||
"ISO8859-2" | "ISO-8859-2" => Ok((decode_iso(aff, ISO_8859_2_HIGH), decode_iso(dic, ISO_8859_2_HIGH))),
|
||||
"ISO8859-7" | "ISO-8859-7" => Ok((decode_iso(aff, ISO_8859_7_HIGH), decode_iso(dic, ISO_8859_7_HIGH))),
|
||||
"ISO8859-15" | "ISO-8859-15" => Ok((decode_iso(aff, ISO_8859_15_HIGH), decode_iso(dic, ISO_8859_15_HIGH))),
|
||||
"ISO8859-9" | "ISO-8859-9" => Ok((decode_latin5(aff), decode_latin5(dic))),
|
||||
"KOI8-R" | "KOI8R" => Ok((decode_high(aff, KOI8_R_HIGH), decode_high(dic, KOI8_R_HIGH))),
|
||||
"CP1251" | "WINDOWS-1251" | "MICROSOFT-CP1251" => Ok((decode_high(aff, CP1251_HIGH), decode_high(dic, CP1251_HIGH))),
|
||||
@@ -602,6 +618,33 @@ mod tests {
|
||||
assert_eq!(load_error(key), None);
|
||||
}
|
||||
|
||||
/// Every table covers its 96 (ISO 8859) or 128 (KOI8-R, Windows-1251) high bytes.
|
||||
#[test]
|
||||
fn tables_are_complete() {
|
||||
for t in [ISO_8859_2_HIGH, ISO_8859_7_HIGH, ISO_8859_15_HIGH] {
|
||||
assert_eq!(t.chars().count(), 96, "{t}");
|
||||
}
|
||||
for t in [KOI8_R_HIGH, CP1251_HIGH] {
|
||||
assert_eq!(t.chars().count(), 128, "{t}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decodes_greek_and_latin9() {
|
||||
// `σπίτι νερό` in ISO 8859-7, and a flag after the slash stays ASCII.
|
||||
let (a, d) = decode(b"SET ISO8859-7\nTRY \xe1\n", b"2\n\xf3\xf0\xdf\xf4\xe9/A\n\xed\xe5\xf1\xfc\n").unwrap();
|
||||
assert!(a.contains("TRY α"), "{a}");
|
||||
assert_eq!(d, "2\nσπίτι/A\nνερό\n");
|
||||
// Unassigned bytes and the 2003 additions.
|
||||
let (_, d) = decode(b"SET ISO-8859-7\n", b"\xae\xd2\xff\xa4\xa5\xaa\xa2").unwrap();
|
||||
assert_eq!(d, "\u{fffd}\u{fffd}\u{fffd}€₯ͺ’");
|
||||
// `šokolaad õun` in ISO 8859-15, and its eight differences from ISO 8859-1.
|
||||
let (_, d) = decode(b"SET ISO8859-15\n", b"\xa8okolaad \xf5un").unwrap();
|
||||
assert_eq!(d, "šokolaad õun");
|
||||
let (_, d) = decode(b"SET ISO-8859-15\n", b"\xa4\xa6\xa8\xb4\xb8\xbc\xbd\xbe\xe4").unwrap();
|
||||
assert_eq!(d, "€ŠšŽžŒœŸä");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decodes_latin1() {
|
||||
// `orçamento esperávamos` in ISO 8859-1: ç = 0xE7, á = 0xE1.
|
||||
|
||||
Reference in new issue
Block a user