mirror of
https://github.com/storytold/craft-libs.git
synced 2026-11-14 06:22:24 +00:00
Dylan M. TaylorandSimon Koch-Sultan
9ab7768771
* Add craft-rtf: RTF tokenizer, text decoding, and a reader and writer for a small model The tokenizing rules, code pages, multibyte decoding, dates and escaping come from WordCraft's RTF reader and writer. The model (paragraphs with outline level and indent, character formatting, tables) covers what DesignCraft's text import, DeckCraft's outline import and export and PdfCraft's export use. An unpaired UTF-16 surrogate in \u reads as U+FFFD and a high surrogate pairs only with the very next \u, as storytold/wordcraft#6105 describes for Word. * craft-rtf: a symbol font's bytes are its own codes A font with \fcharset2 (Symbol, Wingdings) has no letters: a byte from 0x80 reads as U+F000 + the byte instead of a letter of the document's code page. The symbol page applies to body text only, a symbol default font's page does not apply to text in a font chosen with \f, \plain makes the default font's page current again, and a code page an entry names with \cpg is kept. Ported from storytold/wordcraft#6014. * craft-rtf: alignment, indents, spacing, superscript and subscript; NUL is not text The model and both directions gain what DesignCraft's RTF export writes: \ql \qc \qr \qj, \ri, \fi, \sb, \sa, \super, \sub, \nosupersub. A \u0 or \'00 no longer puts a NUL into the text. --------- Co-authored-by: Simon Koch-Sultan <126534658+scks1@users.noreply.github.com>
441 lines
30 KiB
Rust
441 lines
30 KiB
Rust
//! The code pages of 8-bit RTF text: Windows-1252 and the other single-byte Windows
|
|
//! code pages RTF files name with `\ansicpg` or a font's `\fcharset` (Cyrillic 1251, Central
|
|
//! European 1250, Greek, Turkish, Hebrew, Arabic, Baltic, Vietnamese, Thai), Mac OS Roman
|
|
//! (10000, RTF `\mac`), and the East Asian multibyte code pages (932 Shift-JIS, 936 GBK, 949 UHC,
|
|
//! 950 Big5), whose bytes decode as runs rather than one at a time.
|
|
|
|
/// Windows-1252 code points for 0x80..=0x9F.
|
|
const CP1252: [u16; 32] = [
|
|
0x20AC, 0x0081, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, 0x02C6, 0x2030, 0x0160, 0x2039, 0x0152, 0x008D, 0x017D, 0x008F, 0x0090, 0x2018,
|
|
0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0x02DC, 0x2122, 0x0161, 0x203A, 0x0153, 0x009D, 0x017E, 0x0178,
|
|
];
|
|
|
|
/// One Windows-1252 byte as a char.
|
|
pub fn cp1252(b: u8) -> char {
|
|
if (0x80..0xA0).contains(&b) {
|
|
CP1252.get((b - 0x80) as usize).and_then(|u| char::from_u32(u32::from(*u))).unwrap_or('\u{FFFD}')
|
|
} else {
|
|
b as char
|
|
}
|
|
}
|
|
|
|
/// Windows-874 code points for 0x80..=0xFF (U+FFFD where the byte is unassigned).
|
|
const CP874: [u16; 128] = [
|
|
0x20AC, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0x2026, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0x2018,
|
|
0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0x00A0, 0x0E01, 0x0E02, 0x0E03,
|
|
0x0E04, 0x0E05, 0x0E06, 0x0E07, 0x0E08, 0x0E09, 0x0E0A, 0x0E0B, 0x0E0C, 0x0E0D, 0x0E0E, 0x0E0F, 0x0E10, 0x0E11, 0x0E12, 0x0E13, 0x0E14, 0x0E15,
|
|
0x0E16, 0x0E17, 0x0E18, 0x0E19, 0x0E1A, 0x0E1B, 0x0E1C, 0x0E1D, 0x0E1E, 0x0E1F, 0x0E20, 0x0E21, 0x0E22, 0x0E23, 0x0E24, 0x0E25, 0x0E26, 0x0E27,
|
|
0x0E28, 0x0E29, 0x0E2A, 0x0E2B, 0x0E2C, 0x0E2D, 0x0E2E, 0x0E2F, 0x0E30, 0x0E31, 0x0E32, 0x0E33, 0x0E34, 0x0E35, 0x0E36, 0x0E37, 0x0E38, 0x0E39,
|
|
0x0E3A, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0x0E3F, 0x0E40, 0x0E41, 0x0E42, 0x0E43, 0x0E44, 0x0E45, 0x0E46, 0x0E47, 0x0E48, 0x0E49, 0x0E4A, 0x0E4B,
|
|
0x0E4C, 0x0E4D, 0x0E4E, 0x0E4F, 0x0E50, 0x0E51, 0x0E52, 0x0E53, 0x0E54, 0x0E55, 0x0E56, 0x0E57, 0x0E58, 0x0E59, 0x0E5A, 0x0E5B, 0xFFFD, 0xFFFD,
|
|
0xFFFD, 0xFFFD,
|
|
];
|
|
|
|
/// Windows-1250 code points for 0x80..=0xFF (U+FFFD where the byte is unassigned).
|
|
const CP1250: [u16; 128] = [
|
|
0x20AC, 0xFFFD, 0x201A, 0xFFFD, 0x201E, 0x2026, 0x2020, 0x2021, 0xFFFD, 0x2030, 0x0160, 0x2039, 0x015A, 0x0164, 0x017D, 0x0179, 0xFFFD, 0x2018,
|
|
0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0xFFFD, 0x2122, 0x0161, 0x203A, 0x015B, 0x0165, 0x017E, 0x017A, 0x00A0, 0x02C7, 0x02D8, 0x0141,
|
|
0x00A4, 0x0104, 0x00A6, 0x00A7, 0x00A8, 0x00A9, 0x015E, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x017B, 0x00B0, 0x00B1, 0x02DB, 0x0142, 0x00B4, 0x00B5,
|
|
0x00B6, 0x00B7, 0x00B8, 0x0105, 0x015F, 0x00BB, 0x013D, 0x02DD, 0x013E, 0x017C, 0x0154, 0x00C1, 0x00C2, 0x0102, 0x00C4, 0x0139, 0x0106, 0x00C7,
|
|
0x010C, 0x00C9, 0x0118, 0x00CB, 0x011A, 0x00CD, 0x00CE, 0x010E, 0x0110, 0x0143, 0x0147, 0x00D3, 0x00D4, 0x0150, 0x00D6, 0x00D7, 0x0158, 0x016E,
|
|
0x00DA, 0x0170, 0x00DC, 0x00DD, 0x0162, 0x00DF, 0x0155, 0x00E1, 0x00E2, 0x0103, 0x00E4, 0x013A, 0x0107, 0x00E7, 0x010D, 0x00E9, 0x0119, 0x00EB,
|
|
0x011B, 0x00ED, 0x00EE, 0x010F, 0x0111, 0x0144, 0x0148, 0x00F3, 0x00F4, 0x0151, 0x00F6, 0x00F7, 0x0159, 0x016F, 0x00FA, 0x0171, 0x00FC, 0x00FD,
|
|
0x0163, 0x02D9,
|
|
];
|
|
|
|
/// Windows-1251 code points for 0x80..=0xFF (U+FFFD where the byte is unassigned).
|
|
const CP1251: [u16; 128] = [
|
|
0x0402, 0x0403, 0x201A, 0x0453, 0x201E, 0x2026, 0x2020, 0x2021, 0x20AC, 0x2030, 0x0409, 0x2039, 0x040A, 0x040C, 0x040B, 0x040F, 0x0452, 0x2018,
|
|
0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0xFFFD, 0x2122, 0x0459, 0x203A, 0x045A, 0x045C, 0x045B, 0x045F, 0x00A0, 0x040E, 0x045E, 0x0408,
|
|
0x00A4, 0x0490, 0x00A6, 0x00A7, 0x0401, 0x00A9, 0x0404, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x0407, 0x00B0, 0x00B1, 0x0406, 0x0456, 0x0491, 0x00B5,
|
|
0x00B6, 0x00B7, 0x0451, 0x2116, 0x0454, 0x00BB, 0x0458, 0x0405, 0x0455, 0x0457, 0x0410, 0x0411, 0x0412, 0x0413, 0x0414, 0x0415, 0x0416, 0x0417,
|
|
0x0418, 0x0419, 0x041A, 0x041B, 0x041C, 0x041D, 0x041E, 0x041F, 0x0420, 0x0421, 0x0422, 0x0423, 0x0424, 0x0425, 0x0426, 0x0427, 0x0428, 0x0429,
|
|
0x042A, 0x042B, 0x042C, 0x042D, 0x042E, 0x042F, 0x0430, 0x0431, 0x0432, 0x0433, 0x0434, 0x0435, 0x0436, 0x0437, 0x0438, 0x0439, 0x043A, 0x043B,
|
|
0x043C, 0x043D, 0x043E, 0x043F, 0x0440, 0x0441, 0x0442, 0x0443, 0x0444, 0x0445, 0x0446, 0x0447, 0x0448, 0x0449, 0x044A, 0x044B, 0x044C, 0x044D,
|
|
0x044E, 0x044F,
|
|
];
|
|
|
|
/// Windows-1253 code points for 0x80..=0xFF (U+FFFD where the byte is unassigned).
|
|
const CP1253: [u16; 128] = [
|
|
0x20AC, 0xFFFD, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, 0xFFFD, 0x2030, 0xFFFD, 0x2039, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0x2018,
|
|
0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0xFFFD, 0x2122, 0xFFFD, 0x203A, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0x00A0, 0x0385, 0x0386, 0x00A3,
|
|
0x00A4, 0x00A5, 0x00A6, 0x00A7, 0x00A8, 0x00A9, 0xFFFD, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x2015, 0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x0384, 0x00B5,
|
|
0x00B6, 0x00B7, 0x0388, 0x0389, 0x038A, 0x00BB, 0x038C, 0x00BD, 0x038E, 0x038F, 0x0390, 0x0391, 0x0392, 0x0393, 0x0394, 0x0395, 0x0396, 0x0397,
|
|
0x0398, 0x0399, 0x039A, 0x039B, 0x039C, 0x039D, 0x039E, 0x039F, 0x03A0, 0x03A1, 0xFFFD, 0x03A3, 0x03A4, 0x03A5, 0x03A6, 0x03A7, 0x03A8, 0x03A9,
|
|
0x03AA, 0x03AB, 0x03AC, 0x03AD, 0x03AE, 0x03AF, 0x03B0, 0x03B1, 0x03B2, 0x03B3, 0x03B4, 0x03B5, 0x03B6, 0x03B7, 0x03B8, 0x03B9, 0x03BA, 0x03BB,
|
|
0x03BC, 0x03BD, 0x03BE, 0x03BF, 0x03C0, 0x03C1, 0x03C2, 0x03C3, 0x03C4, 0x03C5, 0x03C6, 0x03C7, 0x03C8, 0x03C9, 0x03CA, 0x03CB, 0x03CC, 0x03CD,
|
|
0x03CE, 0xFFFD,
|
|
];
|
|
|
|
/// Windows-1254 code points for 0x80..=0xFF (U+FFFD where the byte is unassigned).
|
|
const CP1254: [u16; 128] = [
|
|
0x20AC, 0xFFFD, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, 0x02C6, 0x2030, 0x0160, 0x2039, 0x0152, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0x2018,
|
|
0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0x02DC, 0x2122, 0x0161, 0x203A, 0x0153, 0xFFFD, 0xFFFD, 0x0178, 0x00A0, 0x00A1, 0x00A2, 0x00A3,
|
|
0x00A4, 0x00A5, 0x00A6, 0x00A7, 0x00A8, 0x00A9, 0x00AA, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x00AF, 0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x00B4, 0x00B5,
|
|
0x00B6, 0x00B7, 0x00B8, 0x00B9, 0x00BA, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0x00BF, 0x00C0, 0x00C1, 0x00C2, 0x00C3, 0x00C4, 0x00C5, 0x00C6, 0x00C7,
|
|
0x00C8, 0x00C9, 0x00CA, 0x00CB, 0x00CC, 0x00CD, 0x00CE, 0x00CF, 0x011E, 0x00D1, 0x00D2, 0x00D3, 0x00D4, 0x00D5, 0x00D6, 0x00D7, 0x00D8, 0x00D9,
|
|
0x00DA, 0x00DB, 0x00DC, 0x0130, 0x015E, 0x00DF, 0x00E0, 0x00E1, 0x00E2, 0x00E3, 0x00E4, 0x00E5, 0x00E6, 0x00E7, 0x00E8, 0x00E9, 0x00EA, 0x00EB,
|
|
0x00EC, 0x00ED, 0x00EE, 0x00EF, 0x011F, 0x00F1, 0x00F2, 0x00F3, 0x00F4, 0x00F5, 0x00F6, 0x00F7, 0x00F8, 0x00F9, 0x00FA, 0x00FB, 0x00FC, 0x0131,
|
|
0x015F, 0x00FF,
|
|
];
|
|
|
|
/// Windows-1255 code points for 0x80..=0xFF (U+FFFD where the byte is unassigned).
|
|
const CP1255: [u16; 128] = [
|
|
0x20AC, 0xFFFD, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, 0x02C6, 0x2030, 0xFFFD, 0x2039, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0x2018,
|
|
0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0x02DC, 0x2122, 0xFFFD, 0x203A, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0x00A0, 0x00A1, 0x00A2, 0x00A3,
|
|
0x20AA, 0x00A5, 0x00A6, 0x00A7, 0x00A8, 0x00A9, 0x00D7, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x00AF, 0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x00B4, 0x00B5,
|
|
0x00B6, 0x00B7, 0x00B8, 0x00B9, 0x00F7, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0x00BF, 0x05B0, 0x05B1, 0x05B2, 0x05B3, 0x05B4, 0x05B5, 0x05B6, 0x05B7,
|
|
0x05B8, 0x05B9, 0x05BA, 0x05BB, 0x05BC, 0x05BD, 0x05BE, 0x05BF, 0x05C0, 0x05C1, 0x05C2, 0x05C3, 0x05F0, 0x05F1, 0x05F2, 0x05F3, 0x05F4, 0xFFFD,
|
|
0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0x05D0, 0x05D1, 0x05D2, 0x05D3, 0x05D4, 0x05D5, 0x05D6, 0x05D7, 0x05D8, 0x05D9, 0x05DA, 0x05DB,
|
|
0x05DC, 0x05DD, 0x05DE, 0x05DF, 0x05E0, 0x05E1, 0x05E2, 0x05E3, 0x05E4, 0x05E5, 0x05E6, 0x05E7, 0x05E8, 0x05E9, 0x05EA, 0xFFFD, 0xFFFD, 0x200E,
|
|
0x200F, 0xFFFD,
|
|
];
|
|
|
|
/// Windows-1256 code points for 0x80..=0xFF (U+FFFD where the byte is unassigned).
|
|
const CP1256: [u16; 128] = [
|
|
0x20AC, 0x067E, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, 0x02C6, 0x2030, 0x0679, 0x2039, 0x0152, 0x0686, 0x0698, 0x0688, 0x06AF, 0x2018,
|
|
0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0x06A9, 0x2122, 0x0691, 0x203A, 0x0153, 0x200C, 0x200D, 0x06BA, 0x00A0, 0x060C, 0x00A2, 0x00A3,
|
|
0x00A4, 0x00A5, 0x00A6, 0x00A7, 0x00A8, 0x00A9, 0x06BE, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x00AF, 0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x00B4, 0x00B5,
|
|
0x00B6, 0x00B7, 0x00B8, 0x00B9, 0x061B, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0x061F, 0x06C1, 0x0621, 0x0622, 0x0623, 0x0624, 0x0625, 0x0626, 0x0627,
|
|
0x0628, 0x0629, 0x062A, 0x062B, 0x062C, 0x062D, 0x062E, 0x062F, 0x0630, 0x0631, 0x0632, 0x0633, 0x0634, 0x0635, 0x0636, 0x00D7, 0x0637, 0x0638,
|
|
0x0639, 0x063A, 0x0640, 0x0641, 0x0642, 0x0643, 0x00E0, 0x0644, 0x00E2, 0x0645, 0x0646, 0x0647, 0x0648, 0x00E7, 0x00E8, 0x00E9, 0x00EA, 0x00EB,
|
|
0x0649, 0x064A, 0x00EE, 0x00EF, 0x064B, 0x064C, 0x064D, 0x064E, 0x00F4, 0x064F, 0x0650, 0x00F7, 0x0651, 0x00F9, 0x0652, 0x00FB, 0x00FC, 0x200E,
|
|
0x200F, 0x06D2,
|
|
];
|
|
|
|
/// Windows-1257 code points for 0x80..=0xFF (U+FFFD where the byte is unassigned).
|
|
const CP1257: [u16; 128] = [
|
|
0x20AC, 0xFFFD, 0x201A, 0xFFFD, 0x201E, 0x2026, 0x2020, 0x2021, 0xFFFD, 0x2030, 0xFFFD, 0x2039, 0xFFFD, 0x00A8, 0x02C7, 0x00B8, 0xFFFD, 0x2018,
|
|
0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0xFFFD, 0x2122, 0xFFFD, 0x203A, 0xFFFD, 0x00AF, 0x02DB, 0xFFFD, 0x00A0, 0xFFFD, 0x00A2, 0x00A3,
|
|
0x00A4, 0xFFFD, 0x00A6, 0x00A7, 0x00D8, 0x00A9, 0x0156, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x00C6, 0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x00B4, 0x00B5,
|
|
0x00B6, 0x00B7, 0x00F8, 0x00B9, 0x0157, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0x00E6, 0x0104, 0x012E, 0x0100, 0x0106, 0x00C4, 0x00C5, 0x0118, 0x0112,
|
|
0x010C, 0x00C9, 0x0179, 0x0116, 0x0122, 0x0136, 0x012A, 0x013B, 0x0160, 0x0143, 0x0145, 0x00D3, 0x014C, 0x00D5, 0x00D6, 0x00D7, 0x0172, 0x0141,
|
|
0x015A, 0x016A, 0x00DC, 0x017B, 0x017D, 0x00DF, 0x0105, 0x012F, 0x0101, 0x0107, 0x00E4, 0x00E5, 0x0119, 0x0113, 0x010D, 0x00E9, 0x017A, 0x0117,
|
|
0x0123, 0x0137, 0x012B, 0x013C, 0x0161, 0x0144, 0x0146, 0x00F3, 0x014D, 0x00F5, 0x00F6, 0x00F7, 0x0173, 0x0142, 0x015B, 0x016B, 0x00FC, 0x017C,
|
|
0x017E, 0x02D9,
|
|
];
|
|
|
|
/// Windows-1258 code points for 0x80..=0xFF (U+FFFD where the byte is unassigned).
|
|
const CP1258: [u16; 128] = [
|
|
0x20AC, 0xFFFD, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, 0x02C6, 0x2030, 0xFFFD, 0x2039, 0x0152, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0x2018,
|
|
0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0x02DC, 0x2122, 0xFFFD, 0x203A, 0x0153, 0xFFFD, 0xFFFD, 0x0178, 0x00A0, 0x00A1, 0x00A2, 0x00A3,
|
|
0x00A4, 0x00A5, 0x00A6, 0x00A7, 0x00A8, 0x00A9, 0x00AA, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x00AF, 0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x00B4, 0x00B5,
|
|
0x00B6, 0x00B7, 0x00B8, 0x00B9, 0x00BA, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0x00BF, 0x00C0, 0x00C1, 0x00C2, 0x0102, 0x00C4, 0x00C5, 0x00C6, 0x00C7,
|
|
0x00C8, 0x00C9, 0x00CA, 0x00CB, 0x0300, 0x00CD, 0x00CE, 0x00CF, 0x0110, 0x00D1, 0x0309, 0x00D3, 0x00D4, 0x01A0, 0x00D6, 0x00D7, 0x00D8, 0x00D9,
|
|
0x00DA, 0x00DB, 0x00DC, 0x01AF, 0x0303, 0x00DF, 0x00E0, 0x00E1, 0x00E2, 0x0103, 0x00E4, 0x00E5, 0x00E6, 0x00E7, 0x00E8, 0x00E9, 0x00EA, 0x00EB,
|
|
0x0301, 0x00ED, 0x00EE, 0x00EF, 0x0111, 0x00F1, 0x0323, 0x00F3, 0x00F4, 0x01A1, 0x00F6, 0x00F7, 0x00F8, 0x00F9, 0x00FA, 0x00FB, 0x00FC, 0x01B0,
|
|
0x20AB, 0x00FF,
|
|
];
|
|
|
|
/// Mac OS Roman (code page 10000, RTF `\mac`) code points for 0x80..=0xFF, with the euro sign at
|
|
/// 0xDB (Mac OS 8.5 and later).
|
|
const MAC_ROMAN: [u16; 128] = [
|
|
0x00C4, 0x00C5, 0x00C7, 0x00C9, 0x00D1, 0x00D6, 0x00DC, 0x00E1, 0x00E0, 0x00E2, 0x00E4, 0x00E3, 0x00E5, 0x00E7, 0x00E9, 0x00E8, 0x00EA, 0x00EB,
|
|
0x00ED, 0x00EC, 0x00EE, 0x00EF, 0x00F1, 0x00F3, 0x00F2, 0x00F4, 0x00F6, 0x00F5, 0x00FA, 0x00F9, 0x00FB, 0x00FC, 0x2020, 0x00B0, 0x00A2, 0x00A3,
|
|
0x00A7, 0x2022, 0x00B6, 0x00DF, 0x00AE, 0x00A9, 0x2122, 0x00B4, 0x00A8, 0x2260, 0x00C6, 0x00D8, 0x221E, 0x00B1, 0x2264, 0x2265, 0x00A5, 0x00B5,
|
|
0x2202, 0x2211, 0x220F, 0x03C0, 0x222B, 0x00AA, 0x00BA, 0x03A9, 0x00E6, 0x00F8, 0x00BF, 0x00A1, 0x00AC, 0x221A, 0x0192, 0x2248, 0x2206, 0x00AB,
|
|
0x00BB, 0x2026, 0x00A0, 0x00C0, 0x00C3, 0x00D5, 0x0152, 0x0153, 0x2013, 0x2014, 0x201C, 0x201D, 0x2018, 0x2019, 0x00F7, 0x25CA, 0x00FF, 0x0178,
|
|
0x2044, 0x20AC, 0x2039, 0x203A, 0xFB01, 0xFB02, 0x2021, 0x00B7, 0x201A, 0x201E, 0x2030, 0x00C2, 0x00CA, 0x00C1, 0x00CB, 0x00C8, 0x00CD, 0x00CE,
|
|
0x00CF, 0x00CC, 0x00D3, 0x00D4, 0xF8FF, 0x00D2, 0x00DA, 0x00DB, 0x00D9, 0x0131, 0x02C6, 0x02DC, 0x00AF, 0x02D8, 0x02D9, 0x02DA, 0x00B8, 0x02DD,
|
|
0x02DB, 0x02C7,
|
|
];
|
|
/// IBM PC (OEM) code page 437 code points for 0x80..=0xFF (the Unicode Consortium's CP437
|
|
/// mapping file).
|
|
const CP437: [u16; 128] = [
|
|
0x00C7, 0x00FC, 0x00E9, 0x00E2, 0x00E4, 0x00E0, 0x00E5, 0x00E7, 0x00EA, 0x00EB, 0x00E8, 0x00EF, 0x00EE, 0x00EC, 0x00C4, 0x00C5, 0x00C9, 0x00E6,
|
|
0x00C6, 0x00F4, 0x00F6, 0x00F2, 0x00FB, 0x00F9, 0x00FF, 0x00D6, 0x00DC, 0x00A2, 0x00A3, 0x00A5, 0x20A7, 0x0192, 0x00E1, 0x00ED, 0x00F3, 0x00FA,
|
|
0x00F1, 0x00D1, 0x00AA, 0x00BA, 0x00BF, 0x2310, 0x00AC, 0x00BD, 0x00BC, 0x00A1, 0x00AB, 0x00BB, 0x2591, 0x2592, 0x2593, 0x2502, 0x2524, 0x2561,
|
|
0x2562, 0x2556, 0x2555, 0x2563, 0x2551, 0x2557, 0x255D, 0x255C, 0x255B, 0x2510, 0x2514, 0x2534, 0x252C, 0x251C, 0x2500, 0x253C, 0x255E, 0x255F,
|
|
0x255A, 0x2554, 0x2569, 0x2566, 0x2560, 0x2550, 0x256C, 0x2567, 0x2568, 0x2564, 0x2565, 0x2559, 0x2558, 0x2552, 0x2553, 0x256B, 0x256A, 0x2518,
|
|
0x250C, 0x2588, 0x2584, 0x258C, 0x2590, 0x2580, 0x03B1, 0x00DF, 0x0393, 0x03C0, 0x03A3, 0x03C3, 0x00B5, 0x03C4, 0x03A6, 0x0398, 0x03A9, 0x03B4,
|
|
0x221E, 0x03C6, 0x03B5, 0x2229, 0x2261, 0x00B1, 0x2265, 0x2264, 0x2320, 0x2321, 0x00F7, 0x2248, 0x00B0, 0x2219, 0x00B7, 0x221A, 0x207F, 0x00B2,
|
|
0x25A0, 0x00A0,
|
|
];
|
|
|
|
/// OEM code page 850 (DOS Latin-1, Western European) code points for 0x80..=0xFF (the Unicode
|
|
/// Consortium's CP850 mapping file).
|
|
const CP850: [u16; 128] = [
|
|
0x00C7, 0x00FC, 0x00E9, 0x00E2, 0x00E4, 0x00E0, 0x00E5, 0x00E7, 0x00EA, 0x00EB, 0x00E8, 0x00EF, 0x00EE, 0x00EC, 0x00C4, 0x00C5, 0x00C9, 0x00E6,
|
|
0x00C6, 0x00F4, 0x00F6, 0x00F2, 0x00FB, 0x00F9, 0x00FF, 0x00D6, 0x00DC, 0x00F8, 0x00A3, 0x00D8, 0x00D7, 0x0192, 0x00E1, 0x00ED, 0x00F3, 0x00FA,
|
|
0x00F1, 0x00D1, 0x00AA, 0x00BA, 0x00BF, 0x00AE, 0x00AC, 0x00BD, 0x00BC, 0x00A1, 0x00AB, 0x00BB, 0x2591, 0x2592, 0x2593, 0x2502, 0x2524, 0x00C1,
|
|
0x00C2, 0x00C0, 0x00A9, 0x2563, 0x2551, 0x2557, 0x255D, 0x00A2, 0x00A5, 0x2510, 0x2514, 0x2534, 0x252C, 0x251C, 0x2500, 0x253C, 0x00E3, 0x00C3,
|
|
0x255A, 0x2554, 0x2569, 0x2566, 0x2560, 0x2550, 0x256C, 0x00A4, 0x00F0, 0x00D0, 0x00CA, 0x00CB, 0x00C8, 0x0131, 0x00CD, 0x00CE, 0x00CF, 0x2518,
|
|
0x250C, 0x2588, 0x2584, 0x00A6, 0x00CC, 0x2580, 0x00D3, 0x00DF, 0x00D4, 0x00D2, 0x00F5, 0x00D5, 0x00B5, 0x00FE, 0x00DE, 0x00DA, 0x00DB, 0x00D9,
|
|
0x00FD, 0x00DD, 0x00AF, 0x00B4, 0x00AD, 0x00B1, 0x2017, 0x00BE, 0x00B6, 0x00A7, 0x00F7, 0x00B8, 0x00B0, 0x00A8, 0x00B7, 0x00B9, 0x00B3, 0x00B2,
|
|
0x25A0, 0x00A0,
|
|
];
|
|
|
|
/// OEM code page 852 (DOS Latin-2, Central European) code points for 0x80..=0xFF (the Unicode Consortium's CP852
|
|
/// mapping file).
|
|
const CP852: [u16; 128] = [
|
|
0x00C7, 0x00FC, 0x00E9, 0x00E2, 0x00E4, 0x016F, 0x0107, 0x00E7, 0x0142, 0x00EB, 0x0150, 0x0151, 0x00EE, 0x0179, 0x00C4, 0x0106, 0x00C9, 0x0139,
|
|
0x013A, 0x00F4, 0x00F6, 0x013D, 0x013E, 0x015A, 0x015B, 0x00D6, 0x00DC, 0x0164, 0x0165, 0x0141, 0x00D7, 0x010D, 0x00E1, 0x00ED, 0x00F3, 0x00FA,
|
|
0x0104, 0x0105, 0x017D, 0x017E, 0x0118, 0x0119, 0x00AC, 0x017A, 0x010C, 0x015F, 0x00AB, 0x00BB, 0x2591, 0x2592, 0x2593, 0x2502, 0x2524, 0x00C1,
|
|
0x00C2, 0x011A, 0x015E, 0x2563, 0x2551, 0x2557, 0x255D, 0x017B, 0x017C, 0x2510, 0x2514, 0x2534, 0x252C, 0x251C, 0x2500, 0x253C, 0x0102, 0x0103,
|
|
0x255A, 0x2554, 0x2569, 0x2566, 0x2560, 0x2550, 0x256C, 0x00A4, 0x0111, 0x0110, 0x010E, 0x00CB, 0x010F, 0x0147, 0x00CD, 0x00CE, 0x011B, 0x2518,
|
|
0x250C, 0x2588, 0x2584, 0x0162, 0x016E, 0x2580, 0x00D3, 0x00DF, 0x00D4, 0x0143, 0x0144, 0x0148, 0x0160, 0x0161, 0x0154, 0x00DA, 0x0155, 0x0170,
|
|
0x00FD, 0x00DD, 0x0163, 0x00B4, 0x00AD, 0x02DD, 0x02DB, 0x02C7, 0x02D8, 0x00A7, 0x00F7, 0x00B8, 0x00B0, 0x00A8, 0x02D9, 0x0171, 0x0158, 0x0159,
|
|
0x25A0, 0x00A0,
|
|
];
|
|
|
|
/// OEM code page 866 (DOS Cyrillic) code points for 0x80..=0xFF (the Unicode Consortium's CP866
|
|
/// mapping file).
|
|
const CP866: [u16; 128] = [
|
|
0x0410, 0x0411, 0x0412, 0x0413, 0x0414, 0x0415, 0x0416, 0x0417, 0x0418, 0x0419, 0x041A, 0x041B, 0x041C, 0x041D, 0x041E, 0x041F, 0x0420, 0x0421,
|
|
0x0422, 0x0423, 0x0424, 0x0425, 0x0426, 0x0427, 0x0428, 0x0429, 0x042A, 0x042B, 0x042C, 0x042D, 0x042E, 0x042F, 0x0430, 0x0431, 0x0432, 0x0433,
|
|
0x0434, 0x0435, 0x0436, 0x0437, 0x0438, 0x0439, 0x043A, 0x043B, 0x043C, 0x043D, 0x043E, 0x043F, 0x2591, 0x2592, 0x2593, 0x2502, 0x2524, 0x2561,
|
|
0x2562, 0x2556, 0x2555, 0x2563, 0x2551, 0x2557, 0x255D, 0x255C, 0x255B, 0x2510, 0x2514, 0x2534, 0x252C, 0x251C, 0x2500, 0x253C, 0x255E, 0x255F,
|
|
0x255A, 0x2554, 0x2569, 0x2566, 0x2560, 0x2550, 0x256C, 0x2567, 0x2568, 0x2564, 0x2565, 0x2559, 0x2558, 0x2552, 0x2553, 0x256B, 0x256A, 0x2518,
|
|
0x250C, 0x2588, 0x2584, 0x258C, 0x2590, 0x2580, 0x0440, 0x0441, 0x0442, 0x0443, 0x0444, 0x0445, 0x0446, 0x0447, 0x0448, 0x0449, 0x044A, 0x044B,
|
|
0x044C, 0x044D, 0x044E, 0x044F, 0x0401, 0x0451, 0x0404, 0x0454, 0x0407, 0x0457, 0x040E, 0x045E, 0x00B0, 0x2219, 0x00B7, 0x221A, 0x2116, 0x00A4,
|
|
0x25A0, 0x00A0,
|
|
];
|
|
|
|
/// OEM code page 737 (DOS Greek) code points for 0x80..=0xFF (the Unicode Consortium's CP737
|
|
/// mapping file).
|
|
const CP737: [u16; 128] = [
|
|
0x0391, 0x0392, 0x0393, 0x0394, 0x0395, 0x0396, 0x0397, 0x0398, 0x0399, 0x039A, 0x039B, 0x039C, 0x039D, 0x039E, 0x039F, 0x03A0, 0x03A1, 0x03A3,
|
|
0x03A4, 0x03A5, 0x03A6, 0x03A7, 0x03A8, 0x03A9, 0x03B1, 0x03B2, 0x03B3, 0x03B4, 0x03B5, 0x03B6, 0x03B7, 0x03B8, 0x03B9, 0x03BA, 0x03BB, 0x03BC,
|
|
0x03BD, 0x03BE, 0x03BF, 0x03C0, 0x03C1, 0x03C3, 0x03C2, 0x03C4, 0x03C5, 0x03C6, 0x03C7, 0x03C8, 0x2591, 0x2592, 0x2593, 0x2502, 0x2524, 0x2561,
|
|
0x2562, 0x2556, 0x2555, 0x2563, 0x2551, 0x2557, 0x255D, 0x255C, 0x255B, 0x2510, 0x2514, 0x2534, 0x252C, 0x251C, 0x2500, 0x253C, 0x255E, 0x255F,
|
|
0x255A, 0x2554, 0x2569, 0x2566, 0x2560, 0x2550, 0x256C, 0x2567, 0x2568, 0x2564, 0x2565, 0x2559, 0x2558, 0x2552, 0x2553, 0x256B, 0x256A, 0x2518,
|
|
0x250C, 0x2588, 0x2584, 0x258C, 0x2590, 0x2580, 0x03C9, 0x03AC, 0x03AD, 0x03AE, 0x03CA, 0x03AF, 0x03CC, 0x03CD, 0x03CB, 0x03CE, 0x0386, 0x0388,
|
|
0x0389, 0x038A, 0x038C, 0x038E, 0x038F, 0x00B1, 0x2265, 0x2264, 0x03AA, 0x03AB, 0x00F7, 0x2248, 0x00B0, 0x2219, 0x00B7, 0x221A, 0x207F, 0x00B2,
|
|
0x25A0, 0x00A0,
|
|
];
|
|
|
|
/// OEM code page 857 (DOS Turkish; U+FFFD where the byte is unassigned) code points for 0x80..=0xFF (the Unicode Consortium's CP857
|
|
/// mapping file).
|
|
const CP857: [u16; 128] = [
|
|
0x00C7, 0x00FC, 0x00E9, 0x00E2, 0x00E4, 0x00E0, 0x00E5, 0x00E7, 0x00EA, 0x00EB, 0x00E8, 0x00EF, 0x00EE, 0x0131, 0x00C4, 0x00C5, 0x00C9, 0x00E6,
|
|
0x00C6, 0x00F4, 0x00F6, 0x00F2, 0x00FB, 0x00F9, 0x0130, 0x00D6, 0x00DC, 0x00F8, 0x00A3, 0x00D8, 0x015E, 0x015F, 0x00E1, 0x00ED, 0x00F3, 0x00FA,
|
|
0x00F1, 0x00D1, 0x011E, 0x011F, 0x00BF, 0x00AE, 0x00AC, 0x00BD, 0x00BC, 0x00A1, 0x00AB, 0x00BB, 0x2591, 0x2592, 0x2593, 0x2502, 0x2524, 0x00C1,
|
|
0x00C2, 0x00C0, 0x00A9, 0x2563, 0x2551, 0x2557, 0x255D, 0x00A2, 0x00A5, 0x2510, 0x2514, 0x2534, 0x252C, 0x251C, 0x2500, 0x253C, 0x00E3, 0x00C3,
|
|
0x255A, 0x2554, 0x2569, 0x2566, 0x2560, 0x2550, 0x256C, 0x00A4, 0x00BA, 0x00AA, 0x00CA, 0x00CB, 0x00C8, 0xFFFD, 0x00CD, 0x00CE, 0x00CF, 0x2518,
|
|
0x250C, 0x2588, 0x2584, 0x00A6, 0x00CC, 0x2580, 0x00D3, 0x00DF, 0x00D4, 0x00D2, 0x00F5, 0x00D5, 0x00B5, 0xFFFD, 0x00D7, 0x00DA, 0x00DB, 0x00D9,
|
|
0x00EC, 0x00FF, 0x00AF, 0x00B4, 0x00AD, 0x00B1, 0xFFFD, 0x00BE, 0x00B6, 0x00A7, 0x00F7, 0x00B8, 0x00B0, 0x00A8, 0x00B7, 0x00B9, 0x00B3, 0x00B2,
|
|
0x25A0, 0x00A0,
|
|
];
|
|
|
|
/// The glyphs the OEM code pages show for 0x01..=0x1F (smileys, card suits, arrows), as
|
|
/// Windows' Alt+numpad entry produces them rather than the control characters.
|
|
const OEM_GLYPHS: [u16; 31] = [
|
|
0x263A, 0x263B, 0x2665, 0x2666, 0x2663, 0x2660, 0x2022, 0x25D8, 0x25CB, 0x25D9, 0x2642, 0x2640, 0x266A, 0x266B, 0x263C, 0x25BA, 0x25C4, 0x2195,
|
|
0x203C, 0x00B6, 0x00A7, 0x25AC, 0x21A8, 0x2191, 0x2193, 0x2192, 0x2190, 0x221F, 0x2194, 0x25B2, 0x25BC,
|
|
];
|
|
|
|
/// The OEM code pages [`oem_byte`] decodes.
|
|
pub const OEM_CODE_PAGES: [u16; 6] = [437, 737, 850, 852, 857, 866];
|
|
|
|
/// One byte of an OEM code page in [`OEM_CODE_PAGES`] (any other code page reads as 437) as a
|
|
/// char, with the graphic glyphs for 0x01..=0x1F and 0x7F (⌂); 0x00 stays NUL.
|
|
pub fn oem_byte(code_page: u16, b: u8) -> char {
|
|
let table: &[u16; 128] = match code_page {
|
|
737 => &CP737,
|
|
850 => &CP850,
|
|
852 => &CP852,
|
|
857 => &CP857,
|
|
866 => &CP866,
|
|
_ => &CP437,
|
|
};
|
|
let u = match b {
|
|
0x01..=0x1F => OEM_GLYPHS.get(usize::from(b - 1)).copied(),
|
|
0x7F => Some(0x2302),
|
|
0x80..=0xFF => table.get(usize::from(b - 0x80)).copied(),
|
|
_ => Some(u16::from(b)),
|
|
};
|
|
u.and_then(|u| char::from_u32(u32::from(u))).unwrap_or('\u{FFFD}')
|
|
}
|
|
|
|
/// The single-byte Windows code pages [`windows_byte`] decodes (1252 included).
|
|
pub const SINGLE_BYTE_CODE_PAGES: [u16; 10] = [874, 1250, 1251, 1252, 1253, 1254, 1255, 1256, 1257, 1258];
|
|
|
|
/// Mac OS Roman, RTF's `\mac` character set.
|
|
pub const MAC_ROMAN_CODE_PAGE: u16 = 10000;
|
|
|
|
/// One byte of a single-byte code page as a char: a Windows one in [`SINGLE_BYTE_CODE_PAGES`] or
|
|
/// Mac OS Roman ([`MAC_ROMAN_CODE_PAGE`]); any other code page reads as Windows-1252.
|
|
pub fn windows_byte(code_page: u16, b: u8) -> char {
|
|
let table: &[u16; 128] = match code_page {
|
|
874 => &CP874,
|
|
1250 => &CP1250,
|
|
1251 => &CP1251,
|
|
1253 => &CP1253,
|
|
1254 => &CP1254,
|
|
1255 => &CP1255,
|
|
1256 => &CP1256,
|
|
1257 => &CP1257,
|
|
1258 => &CP1258,
|
|
MAC_ROMAN_CODE_PAGE => &MAC_ROMAN,
|
|
_ => return cp1252(b),
|
|
};
|
|
if b < 0x80 {
|
|
return b as char;
|
|
}
|
|
table.get(usize::from(b - 0x80)).and_then(|u| char::from_u32(u32::from(*u))).unwrap_or('\u{FFFD}')
|
|
}
|
|
|
|
/// Whether [`code_page_byte`] decodes `code_page`: a Windows one ([`SINGLE_BYTE_CODE_PAGES`]), an
|
|
/// OEM (DOS) one ([`OEM_CODE_PAGES`]) or Mac OS Roman. Document formats (RTF) name all three.
|
|
pub fn is_byte_code_page(code_page: u16) -> bool {
|
|
SINGLE_BYTE_CODE_PAGES.contains(&code_page) || OEM_CODE_PAGES.contains(&code_page) || code_page == MAC_ROMAN_CODE_PAGE
|
|
}
|
|
|
|
/// Windows' symbol code page (`CP_SYMBOL`), the charset of symbol fonts (`\fcharset2`): a byte is
|
|
/// the font's own code, not a letter. [`code_page_byte`] reads one from 0x80 as U+F000 + the byte,
|
|
/// the form a Word symbol character takes (`w:sym w:char="F0D8"`).
|
|
pub const SYMBOL_CODE_PAGE: u16 = 42;
|
|
|
|
/// A byte of 8-bit document text in `code_page` (see [`is_byte_code_page`], and
|
|
/// [`SYMBOL_CODE_PAGE`] for a symbol font) as a char. Below 0x80 it is ASCII in every code page
|
|
/// (text, so an OEM page's glyphs for control codes don't apply).
|
|
pub fn code_page_byte(code_page: u16, b: u8) -> char {
|
|
if b >= 0x80 && code_page == SYMBOL_CODE_PAGE {
|
|
return char::from_u32(0xF000 + u32::from(b)).unwrap_or('\u{FFFD}');
|
|
}
|
|
if b >= 0x80 && OEM_CODE_PAGES.contains(&code_page) { oem_byte(code_page, b) } else { windows_byte(code_page, b) }
|
|
}
|
|
|
|
/// The East Asian multibyte code pages a document format names with `\ansicpg` or a font's `\cpg`:
|
|
/// 932 Shift-JIS (Japanese), 936 GBK (Simplified Chinese), 949 UHC (Korean) and 950 Big5
|
|
/// (Traditional Chinese).
|
|
pub const MULTIBYTE_CODE_PAGES: [u16; 4] = [932, 936, 949, 950];
|
|
|
|
/// Whether `code_page` is one of the East Asian multibyte code pages ([`MULTIBYTE_CODE_PAGES`]):
|
|
/// its bytes decode as a run, a lead byte together with its trail byte.
|
|
pub fn is_multibyte_code_page(code_page: u16) -> bool {
|
|
multibyte_encoding_name(code_page).is_some()
|
|
}
|
|
|
|
/// The WHATWG encoding name of an East Asian multibyte code page (see [`MULTIBYTE_CODE_PAGES`]),
|
|
/// for `encoding_rs::Encoding::for_label`; `None` for the single-byte code pages, whose bytes
|
|
/// decode one at a time, and anything else.
|
|
pub fn multibyte_encoding_name(code_page: u16) -> Option<&'static str> {
|
|
Some(match code_page {
|
|
932 => "shift_jis",
|
|
936 => "gbk",
|
|
949 => "euc-kr",
|
|
950 => "big5",
|
|
_ => return None,
|
|
})
|
|
}
|
|
|
|
/// Whether a document format can name `code_page` for its text: a single-byte Windows, DOS or Mac
|
|
/// one ([`is_byte_code_page`]) or an East Asian multibyte one ([`is_multibyte_code_page`]).
|
|
pub fn is_text_code_page(code_page: u16) -> bool {
|
|
is_byte_code_page(code_page) || is_multibyte_code_page(code_page)
|
|
}
|
|
|
|
/// The code page of an RTF or Word font charset (`\fcharset`): 204 Russian → 1251, 238 Eastern
|
|
/// European → 1250, 77 Mac → 10000, 254 PC / 255 OEM → 437, 128 Japanese → 932, 129 Korean → 949,
|
|
/// 134 Simplified Chinese → 936, 136 Traditional Chinese → 950, 2 Symbol → [`SYMBOL_CODE_PAGE`]…
|
|
/// `None` for the ANSI and default charsets (the document's code page applies).
|
|
pub fn charset_code_page(charset: i32) -> Option<u16> {
|
|
Some(match charset {
|
|
2 => SYMBOL_CODE_PAGE,
|
|
77 => MAC_ROMAN_CODE_PAGE,
|
|
128 => 932,
|
|
129 => 949,
|
|
134 => 936,
|
|
136 => 950,
|
|
161 => 1253,
|
|
162 => 1254,
|
|
163 => 1258,
|
|
177 => 1255,
|
|
178 => 1256,
|
|
186 => 1257,
|
|
204 => 1251,
|
|
222 => 874,
|
|
238 => 1250,
|
|
254 | 255 => 437,
|
|
_ => return None,
|
|
})
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn single_byte_code_pages() {
|
|
let decode = |cp: u16, bytes: &[u8]| bytes.iter().map(|&b| windows_byte(cp, b)).collect::<String>();
|
|
assert_eq!(decode(1251, &[0xD1, 0xF7, 0xE5, 0xF2, 0x20, 0xB9]), "Счет №");
|
|
assert_eq!(decode(1251, &[0xA8, 0xB8]), "Ёё");
|
|
assert_eq!(decode(1250, &[0xA3, 0xF3, 0x9F]), "Łóź");
|
|
assert_eq!(decode(1253, &[0xC1, 0xE2]), "Αβ");
|
|
assert_eq!(decode(1252, &[0x80, 0xE9]), "ێ");
|
|
assert_eq!(decode(9999, &[0xE9]), "é");
|
|
// Mac OS Roman: accents, the euro sign at 0xDB, typographic quotes.
|
|
assert_eq!(decode(10000, &[0x80, 0x8E, 0xDB, 0xF1, 0xF2, 0xF3, 0xD2, 0xD3, 0x41]), "Äé€ÒÚÛ“”A");
|
|
assert_eq!(charset_code_page(77), Some(10000));
|
|
assert_eq!(charset_code_page(254), Some(437));
|
|
assert_eq!(charset_code_page(255), Some(437));
|
|
assert_eq!(charset_code_page(204), Some(1251));
|
|
assert_eq!(charset_code_page(0), None);
|
|
for cp in SINGLE_BYTE_CODE_PAGES {
|
|
for b in 0..=255u8 {
|
|
let _ = windows_byte(cp, b);
|
|
}
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn windows_1255_holam_haser_for_vav() {
|
|
assert_eq!(windows_byte(1255, 0xCA), '\u{05BA}');
|
|
}
|
|
|
|
/// One decoder for document text in any code page RTF names: Windows, DOS or Mac.
|
|
#[test]
|
|
fn document_text_in_windows_dos_and_mac_code_pages() {
|
|
let decode = |cp: u16, bytes: &[u8]| bytes.iter().map(|&b| code_page_byte(cp, b)).collect::<String>();
|
|
assert_eq!(decode(437, &[0x82, 0xA0, 0x41, 0x01]), "éáA\u{1}", "DOS text; ASCII stays ASCII");
|
|
assert_eq!(decode(850, &[0x9D, 0xE9]), "ØÚ");
|
|
assert_eq!(decode(10000, &[0x8E]), "é");
|
|
assert_eq!(decode(1251, &[0xC4]), "Д");
|
|
// A symbol font's bytes are its own codes, from 0x80 the private-use form; ASCII stays ASCII.
|
|
assert_eq!(charset_code_page(2), Some(SYMBOL_CODE_PAGE));
|
|
assert_eq!(decode(SYMBOL_CODE_PAGE, &[0x4A, 0x93, 0xD8, 0xFC]), "J\u{F093}\u{F0D8}\u{F0FC}");
|
|
for cp in [437, 850, 866, 1250, 1252, 10000] {
|
|
assert!(is_byte_code_page(cp), "{cp}");
|
|
}
|
|
assert!(!is_byte_code_page(932) && !is_byte_code_page(0) && !SINGLE_BYTE_CODE_PAGES.contains(&10000));
|
|
}
|
|
|
|
/// The East Asian charsets and code pages RTF names are multibyte: their bytes decode
|
|
/// as runs (a lead byte and its trail byte together), not one at a time.
|
|
#[test]
|
|
fn east_asian_code_pages_are_multibyte() {
|
|
assert_eq!(charset_code_page(128), Some(932), "Japanese");
|
|
assert_eq!(charset_code_page(129), Some(949), "Korean");
|
|
assert_eq!(charset_code_page(134), Some(936), "Simplified Chinese");
|
|
assert_eq!(charset_code_page(136), Some(950), "Traditional Chinese");
|
|
assert_eq!(charset_code_page(255), Some(437), "OEM is single-byte");
|
|
assert_eq!(MULTIBYTE_CODE_PAGES, [932, 936, 949, 950]);
|
|
for cp in MULTIBYTE_CODE_PAGES {
|
|
assert!(is_multibyte_code_page(cp) && is_text_code_page(cp), "{cp}");
|
|
assert!(!is_byte_code_page(cp) && !SINGLE_BYTE_CODE_PAGES.contains(&cp), "{cp}");
|
|
assert!(multibyte_encoding_name(cp).is_some(), "{cp}");
|
|
}
|
|
assert!(is_text_code_page(1251) && is_text_code_page(437) && is_text_code_page(10000));
|
|
assert!(!is_text_code_page(0) && !is_text_code_page(9999));
|
|
}
|
|
|
|
#[test]
|
|
fn oem_code_pages() {
|
|
let decode = |cp: u16, bytes: &[u8]| bytes.iter().map(|&b| oem_byte(cp, b)).collect::<String>();
|
|
assert_eq!(decode(437, &[0x82, 0xE9, 0x01, 0x7F]), "éΘ☺⌂");
|
|
assert_eq!(decode(850, &[0x9B, 0xE9]), "øÚ");
|
|
assert_eq!(decode(852, &[0xA5, 0x88, 0xBE, 0xA9]), "ąłżę");
|
|
assert_eq!(decode(866, &[0x8F, 0xE0, 0xA8, 0xA2, 0xA5, 0xE2, 0xF0, 0xFC]), "ПриветЁ№");
|
|
assert_eq!(decode(737, &[0x80, 0x99, 0xE0]), "Αβω");
|
|
assert_eq!(decode(857, &[0x8D, 0x98, 0xA6, 0x9F]), "ıİĞş");
|
|
assert_eq!(oem_byte(857, 0xD5), '\u{FFFD}', "unassigned in 857");
|
|
assert_eq!(decode(9999, &[0xE9]), "Θ", "unknown pages read as 437");
|
|
for cp in OEM_CODE_PAGES {
|
|
for b in 0..=255u8 {
|
|
let _ = oem_byte(cp, b);
|
|
}
|
|
}
|
|
}
|
|
}
|