diff --git a/Cargo.lock b/Cargo.lock index 210f9e52c..aa8c292ef 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1293,6 +1293,13 @@ dependencies = [ "zune-jpegxl", ] +[[package]] +name = "craft-rtf" +version = "0.1.0" +dependencies = [ + "encoding_rs", +] + [[package]] name = "craft-settings" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 31174ab1c..e64b06d1d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -63,6 +63,7 @@ craft-print = { path = "crates/print" } craft-proof = { path = "crates/proof" } craft-psd = { path = "crates/psd" } craft-quit = { path = "crates/quit" } +craft-rtf = { path = "crates/rtf" } craft-raw = { path = "crates/raw" } craft-settings = { path = "crates/settings" } craft-silhouette-studio = { path = "crates/silhouette-studio" } diff --git a/README.md b/README.md index d592ad3d0..1d23a3afd 100644 --- a/README.md +++ b/README.md @@ -173,6 +173,9 @@ craft- = { git = "https://github.com/storytold/craft-libs", rev = "> }`, `Cell { right, paras }`. + +In `Run::text`, a tab is `\t`, and `LINE_BREAK` (`\n`), `PAGE_BREAK` (U+000C) and +`COLUMN_BREAK` (U+000E) are `\line`, `\page` and `\column`. Lengths are in twips and sizes in +half-points, as the file has them. + +`read` leaves out headers, footers, footnotes, comments, field instructions (a field's result +is kept), list labels (`\listtext`, `\pntext`), the stylesheet, pictures and objects (counted), +and every destination marked `\*`. Paragraph properties end with their group, as character +properties do. A trailing `\par` does not leave an empty last paragraph; empty paragraphs +elsewhere are kept. `\sect` ends a paragraph. + +### Limits + +| Limit | Value | Past it | +| --- | --- | --- | +| `MAX_GROUPS` | 512 nested groups | deeper groups are read in the state of the group around them | +| `MAX_FONTS` | 4096 font table entries | later fonts are not kept | +| `MAX_COLORS` | 4096 colour table entries | later colours are not kept | +| `MAX_FONT_NAME` | 256 bytes | the font is not kept | + +The reader does not recurse, and what it returns is never larger than a small multiple of the +file: each character of text costs the file at least one byte. + +## Not here yet + +WordCraft's full reader and writer (styles, lists, fields, bookmarks, pictures, notes, +comments, tracked changes, sections, headers and footers) are written against WordCraft's own +document model and stay there until that model has a neutral form. They can already sit on +`Tokens`, `TextDecoder`, `esc` and the date functions. diff --git a/crates/rtf/src/codepage.rs b/crates/rtf/src/codepage.rs new file mode 100644 index 000000000..7966a2105 --- /dev/null +++ b/crates/rtf/src/codepage.rs @@ -0,0 +1,440 @@ +//! The code pages of 8-bit RTF text: Windows-1252 and the other single-byte Windows +//! code pages RTF files name with `\ansicpg` or a font's `\fcharset` (Cyrillic 1251, Central +//! European 1250, Greek, Turkish, Hebrew, Arabic, Baltic, Vietnamese, Thai), Mac OS Roman +//! (10000, RTF `\mac`), and the East Asian multibyte code pages (932 Shift-JIS, 936 GBK, 949 UHC, +//! 950 Big5), whose bytes decode as runs rather than one at a time. + +/// Windows-1252 code points for 0x80..=0x9F. +const CP1252: [u16; 32] = [ + 0x20AC, 0x0081, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, 0x02C6, 0x2030, 0x0160, 0x2039, 0x0152, 0x008D, 0x017D, 0x008F, 0x0090, 0x2018, + 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0x02DC, 0x2122, 0x0161, 0x203A, 0x0153, 0x009D, 0x017E, 0x0178, +]; + +/// One Windows-1252 byte as a char. +pub fn cp1252(b: u8) -> char { + if (0x80..0xA0).contains(&b) { + CP1252.get((b - 0x80) as usize).and_then(|u| char::from_u32(u32::from(*u))).unwrap_or('\u{FFFD}') + } else { + b as char + } +} + +/// Windows-874 code points for 0x80..=0xFF (U+FFFD where the byte is unassigned). +const CP874: [u16; 128] = [ + 0x20AC, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0x2026, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0x2018, + 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0x00A0, 0x0E01, 0x0E02, 0x0E03, + 0x0E04, 0x0E05, 0x0E06, 0x0E07, 0x0E08, 0x0E09, 0x0E0A, 0x0E0B, 0x0E0C, 0x0E0D, 0x0E0E, 0x0E0F, 0x0E10, 0x0E11, 0x0E12, 0x0E13, 0x0E14, 0x0E15, + 0x0E16, 0x0E17, 0x0E18, 0x0E19, 0x0E1A, 0x0E1B, 0x0E1C, 0x0E1D, 0x0E1E, 0x0E1F, 0x0E20, 0x0E21, 0x0E22, 0x0E23, 0x0E24, 0x0E25, 0x0E26, 0x0E27, + 0x0E28, 0x0E29, 0x0E2A, 0x0E2B, 0x0E2C, 0x0E2D, 0x0E2E, 0x0E2F, 0x0E30, 0x0E31, 0x0E32, 0x0E33, 0x0E34, 0x0E35, 0x0E36, 0x0E37, 0x0E38, 0x0E39, + 0x0E3A, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0x0E3F, 0x0E40, 0x0E41, 0x0E42, 0x0E43, 0x0E44, 0x0E45, 0x0E46, 0x0E47, 0x0E48, 0x0E49, 0x0E4A, 0x0E4B, + 0x0E4C, 0x0E4D, 0x0E4E, 0x0E4F, 0x0E50, 0x0E51, 0x0E52, 0x0E53, 0x0E54, 0x0E55, 0x0E56, 0x0E57, 0x0E58, 0x0E59, 0x0E5A, 0x0E5B, 0xFFFD, 0xFFFD, + 0xFFFD, 0xFFFD, +]; + +/// Windows-1250 code points for 0x80..=0xFF (U+FFFD where the byte is unassigned). +const CP1250: [u16; 128] = [ + 0x20AC, 0xFFFD, 0x201A, 0xFFFD, 0x201E, 0x2026, 0x2020, 0x2021, 0xFFFD, 0x2030, 0x0160, 0x2039, 0x015A, 0x0164, 0x017D, 0x0179, 0xFFFD, 0x2018, + 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0xFFFD, 0x2122, 0x0161, 0x203A, 0x015B, 0x0165, 0x017E, 0x017A, 0x00A0, 0x02C7, 0x02D8, 0x0141, + 0x00A4, 0x0104, 0x00A6, 0x00A7, 0x00A8, 0x00A9, 0x015E, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x017B, 0x00B0, 0x00B1, 0x02DB, 0x0142, 0x00B4, 0x00B5, + 0x00B6, 0x00B7, 0x00B8, 0x0105, 0x015F, 0x00BB, 0x013D, 0x02DD, 0x013E, 0x017C, 0x0154, 0x00C1, 0x00C2, 0x0102, 0x00C4, 0x0139, 0x0106, 0x00C7, + 0x010C, 0x00C9, 0x0118, 0x00CB, 0x011A, 0x00CD, 0x00CE, 0x010E, 0x0110, 0x0143, 0x0147, 0x00D3, 0x00D4, 0x0150, 0x00D6, 0x00D7, 0x0158, 0x016E, + 0x00DA, 0x0170, 0x00DC, 0x00DD, 0x0162, 0x00DF, 0x0155, 0x00E1, 0x00E2, 0x0103, 0x00E4, 0x013A, 0x0107, 0x00E7, 0x010D, 0x00E9, 0x0119, 0x00EB, + 0x011B, 0x00ED, 0x00EE, 0x010F, 0x0111, 0x0144, 0x0148, 0x00F3, 0x00F4, 0x0151, 0x00F6, 0x00F7, 0x0159, 0x016F, 0x00FA, 0x0171, 0x00FC, 0x00FD, + 0x0163, 0x02D9, +]; + +/// Windows-1251 code points for 0x80..=0xFF (U+FFFD where the byte is unassigned). +const CP1251: [u16; 128] = [ + 0x0402, 0x0403, 0x201A, 0x0453, 0x201E, 0x2026, 0x2020, 0x2021, 0x20AC, 0x2030, 0x0409, 0x2039, 0x040A, 0x040C, 0x040B, 0x040F, 0x0452, 0x2018, + 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0xFFFD, 0x2122, 0x0459, 0x203A, 0x045A, 0x045C, 0x045B, 0x045F, 0x00A0, 0x040E, 0x045E, 0x0408, + 0x00A4, 0x0490, 0x00A6, 0x00A7, 0x0401, 0x00A9, 0x0404, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x0407, 0x00B0, 0x00B1, 0x0406, 0x0456, 0x0491, 0x00B5, + 0x00B6, 0x00B7, 0x0451, 0x2116, 0x0454, 0x00BB, 0x0458, 0x0405, 0x0455, 0x0457, 0x0410, 0x0411, 0x0412, 0x0413, 0x0414, 0x0415, 0x0416, 0x0417, + 0x0418, 0x0419, 0x041A, 0x041B, 0x041C, 0x041D, 0x041E, 0x041F, 0x0420, 0x0421, 0x0422, 0x0423, 0x0424, 0x0425, 0x0426, 0x0427, 0x0428, 0x0429, + 0x042A, 0x042B, 0x042C, 0x042D, 0x042E, 0x042F, 0x0430, 0x0431, 0x0432, 0x0433, 0x0434, 0x0435, 0x0436, 0x0437, 0x0438, 0x0439, 0x043A, 0x043B, + 0x043C, 0x043D, 0x043E, 0x043F, 0x0440, 0x0441, 0x0442, 0x0443, 0x0444, 0x0445, 0x0446, 0x0447, 0x0448, 0x0449, 0x044A, 0x044B, 0x044C, 0x044D, + 0x044E, 0x044F, +]; + +/// Windows-1253 code points for 0x80..=0xFF (U+FFFD where the byte is unassigned). +const CP1253: [u16; 128] = [ + 0x20AC, 0xFFFD, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, 0xFFFD, 0x2030, 0xFFFD, 0x2039, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0x2018, + 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0xFFFD, 0x2122, 0xFFFD, 0x203A, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0x00A0, 0x0385, 0x0386, 0x00A3, + 0x00A4, 0x00A5, 0x00A6, 0x00A7, 0x00A8, 0x00A9, 0xFFFD, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x2015, 0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x0384, 0x00B5, + 0x00B6, 0x00B7, 0x0388, 0x0389, 0x038A, 0x00BB, 0x038C, 0x00BD, 0x038E, 0x038F, 0x0390, 0x0391, 0x0392, 0x0393, 0x0394, 0x0395, 0x0396, 0x0397, + 0x0398, 0x0399, 0x039A, 0x039B, 0x039C, 0x039D, 0x039E, 0x039F, 0x03A0, 0x03A1, 0xFFFD, 0x03A3, 0x03A4, 0x03A5, 0x03A6, 0x03A7, 0x03A8, 0x03A9, + 0x03AA, 0x03AB, 0x03AC, 0x03AD, 0x03AE, 0x03AF, 0x03B0, 0x03B1, 0x03B2, 0x03B3, 0x03B4, 0x03B5, 0x03B6, 0x03B7, 0x03B8, 0x03B9, 0x03BA, 0x03BB, + 0x03BC, 0x03BD, 0x03BE, 0x03BF, 0x03C0, 0x03C1, 0x03C2, 0x03C3, 0x03C4, 0x03C5, 0x03C6, 0x03C7, 0x03C8, 0x03C9, 0x03CA, 0x03CB, 0x03CC, 0x03CD, + 0x03CE, 0xFFFD, +]; + +/// Windows-1254 code points for 0x80..=0xFF (U+FFFD where the byte is unassigned). +const CP1254: [u16; 128] = [ + 0x20AC, 0xFFFD, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, 0x02C6, 0x2030, 0x0160, 0x2039, 0x0152, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0x2018, + 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0x02DC, 0x2122, 0x0161, 0x203A, 0x0153, 0xFFFD, 0xFFFD, 0x0178, 0x00A0, 0x00A1, 0x00A2, 0x00A3, + 0x00A4, 0x00A5, 0x00A6, 0x00A7, 0x00A8, 0x00A9, 0x00AA, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x00AF, 0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x00B4, 0x00B5, + 0x00B6, 0x00B7, 0x00B8, 0x00B9, 0x00BA, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0x00BF, 0x00C0, 0x00C1, 0x00C2, 0x00C3, 0x00C4, 0x00C5, 0x00C6, 0x00C7, + 0x00C8, 0x00C9, 0x00CA, 0x00CB, 0x00CC, 0x00CD, 0x00CE, 0x00CF, 0x011E, 0x00D1, 0x00D2, 0x00D3, 0x00D4, 0x00D5, 0x00D6, 0x00D7, 0x00D8, 0x00D9, + 0x00DA, 0x00DB, 0x00DC, 0x0130, 0x015E, 0x00DF, 0x00E0, 0x00E1, 0x00E2, 0x00E3, 0x00E4, 0x00E5, 0x00E6, 0x00E7, 0x00E8, 0x00E9, 0x00EA, 0x00EB, + 0x00EC, 0x00ED, 0x00EE, 0x00EF, 0x011F, 0x00F1, 0x00F2, 0x00F3, 0x00F4, 0x00F5, 0x00F6, 0x00F7, 0x00F8, 0x00F9, 0x00FA, 0x00FB, 0x00FC, 0x0131, + 0x015F, 0x00FF, +]; + +/// Windows-1255 code points for 0x80..=0xFF (U+FFFD where the byte is unassigned). +const CP1255: [u16; 128] = [ + 0x20AC, 0xFFFD, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, 0x02C6, 0x2030, 0xFFFD, 0x2039, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0x2018, + 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0x02DC, 0x2122, 0xFFFD, 0x203A, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0x00A0, 0x00A1, 0x00A2, 0x00A3, + 0x20AA, 0x00A5, 0x00A6, 0x00A7, 0x00A8, 0x00A9, 0x00D7, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x00AF, 0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x00B4, 0x00B5, + 0x00B6, 0x00B7, 0x00B8, 0x00B9, 0x00F7, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0x00BF, 0x05B0, 0x05B1, 0x05B2, 0x05B3, 0x05B4, 0x05B5, 0x05B6, 0x05B7, + 0x05B8, 0x05B9, 0x05BA, 0x05BB, 0x05BC, 0x05BD, 0x05BE, 0x05BF, 0x05C0, 0x05C1, 0x05C2, 0x05C3, 0x05F0, 0x05F1, 0x05F2, 0x05F3, 0x05F4, 0xFFFD, + 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0x05D0, 0x05D1, 0x05D2, 0x05D3, 0x05D4, 0x05D5, 0x05D6, 0x05D7, 0x05D8, 0x05D9, 0x05DA, 0x05DB, + 0x05DC, 0x05DD, 0x05DE, 0x05DF, 0x05E0, 0x05E1, 0x05E2, 0x05E3, 0x05E4, 0x05E5, 0x05E6, 0x05E7, 0x05E8, 0x05E9, 0x05EA, 0xFFFD, 0xFFFD, 0x200E, + 0x200F, 0xFFFD, +]; + +/// Windows-1256 code points for 0x80..=0xFF (U+FFFD where the byte is unassigned). +const CP1256: [u16; 128] = [ + 0x20AC, 0x067E, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, 0x02C6, 0x2030, 0x0679, 0x2039, 0x0152, 0x0686, 0x0698, 0x0688, 0x06AF, 0x2018, + 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0x06A9, 0x2122, 0x0691, 0x203A, 0x0153, 0x200C, 0x200D, 0x06BA, 0x00A0, 0x060C, 0x00A2, 0x00A3, + 0x00A4, 0x00A5, 0x00A6, 0x00A7, 0x00A8, 0x00A9, 0x06BE, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x00AF, 0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x00B4, 0x00B5, + 0x00B6, 0x00B7, 0x00B8, 0x00B9, 0x061B, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0x061F, 0x06C1, 0x0621, 0x0622, 0x0623, 0x0624, 0x0625, 0x0626, 0x0627, + 0x0628, 0x0629, 0x062A, 0x062B, 0x062C, 0x062D, 0x062E, 0x062F, 0x0630, 0x0631, 0x0632, 0x0633, 0x0634, 0x0635, 0x0636, 0x00D7, 0x0637, 0x0638, + 0x0639, 0x063A, 0x0640, 0x0641, 0x0642, 0x0643, 0x00E0, 0x0644, 0x00E2, 0x0645, 0x0646, 0x0647, 0x0648, 0x00E7, 0x00E8, 0x00E9, 0x00EA, 0x00EB, + 0x0649, 0x064A, 0x00EE, 0x00EF, 0x064B, 0x064C, 0x064D, 0x064E, 0x00F4, 0x064F, 0x0650, 0x00F7, 0x0651, 0x00F9, 0x0652, 0x00FB, 0x00FC, 0x200E, + 0x200F, 0x06D2, +]; + +/// Windows-1257 code points for 0x80..=0xFF (U+FFFD where the byte is unassigned). +const CP1257: [u16; 128] = [ + 0x20AC, 0xFFFD, 0x201A, 0xFFFD, 0x201E, 0x2026, 0x2020, 0x2021, 0xFFFD, 0x2030, 0xFFFD, 0x2039, 0xFFFD, 0x00A8, 0x02C7, 0x00B8, 0xFFFD, 0x2018, + 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0xFFFD, 0x2122, 0xFFFD, 0x203A, 0xFFFD, 0x00AF, 0x02DB, 0xFFFD, 0x00A0, 0xFFFD, 0x00A2, 0x00A3, + 0x00A4, 0xFFFD, 0x00A6, 0x00A7, 0x00D8, 0x00A9, 0x0156, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x00C6, 0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x00B4, 0x00B5, + 0x00B6, 0x00B7, 0x00F8, 0x00B9, 0x0157, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0x00E6, 0x0104, 0x012E, 0x0100, 0x0106, 0x00C4, 0x00C5, 0x0118, 0x0112, + 0x010C, 0x00C9, 0x0179, 0x0116, 0x0122, 0x0136, 0x012A, 0x013B, 0x0160, 0x0143, 0x0145, 0x00D3, 0x014C, 0x00D5, 0x00D6, 0x00D7, 0x0172, 0x0141, + 0x015A, 0x016A, 0x00DC, 0x017B, 0x017D, 0x00DF, 0x0105, 0x012F, 0x0101, 0x0107, 0x00E4, 0x00E5, 0x0119, 0x0113, 0x010D, 0x00E9, 0x017A, 0x0117, + 0x0123, 0x0137, 0x012B, 0x013C, 0x0161, 0x0144, 0x0146, 0x00F3, 0x014D, 0x00F5, 0x00F6, 0x00F7, 0x0173, 0x0142, 0x015B, 0x016B, 0x00FC, 0x017C, + 0x017E, 0x02D9, +]; + +/// Windows-1258 code points for 0x80..=0xFF (U+FFFD where the byte is unassigned). +const CP1258: [u16; 128] = [ + 0x20AC, 0xFFFD, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, 0x02C6, 0x2030, 0xFFFD, 0x2039, 0x0152, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0x2018, + 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0x02DC, 0x2122, 0xFFFD, 0x203A, 0x0153, 0xFFFD, 0xFFFD, 0x0178, 0x00A0, 0x00A1, 0x00A2, 0x00A3, + 0x00A4, 0x00A5, 0x00A6, 0x00A7, 0x00A8, 0x00A9, 0x00AA, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x00AF, 0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x00B4, 0x00B5, + 0x00B6, 0x00B7, 0x00B8, 0x00B9, 0x00BA, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0x00BF, 0x00C0, 0x00C1, 0x00C2, 0x0102, 0x00C4, 0x00C5, 0x00C6, 0x00C7, + 0x00C8, 0x00C9, 0x00CA, 0x00CB, 0x0300, 0x00CD, 0x00CE, 0x00CF, 0x0110, 0x00D1, 0x0309, 0x00D3, 0x00D4, 0x01A0, 0x00D6, 0x00D7, 0x00D8, 0x00D9, + 0x00DA, 0x00DB, 0x00DC, 0x01AF, 0x0303, 0x00DF, 0x00E0, 0x00E1, 0x00E2, 0x0103, 0x00E4, 0x00E5, 0x00E6, 0x00E7, 0x00E8, 0x00E9, 0x00EA, 0x00EB, + 0x0301, 0x00ED, 0x00EE, 0x00EF, 0x0111, 0x00F1, 0x0323, 0x00F3, 0x00F4, 0x01A1, 0x00F6, 0x00F7, 0x00F8, 0x00F9, 0x00FA, 0x00FB, 0x00FC, 0x01B0, + 0x20AB, 0x00FF, +]; + +/// Mac OS Roman (code page 10000, RTF `\mac`) code points for 0x80..=0xFF, with the euro sign at +/// 0xDB (Mac OS 8.5 and later). +const MAC_ROMAN: [u16; 128] = [ + 0x00C4, 0x00C5, 0x00C7, 0x00C9, 0x00D1, 0x00D6, 0x00DC, 0x00E1, 0x00E0, 0x00E2, 0x00E4, 0x00E3, 0x00E5, 0x00E7, 0x00E9, 0x00E8, 0x00EA, 0x00EB, + 0x00ED, 0x00EC, 0x00EE, 0x00EF, 0x00F1, 0x00F3, 0x00F2, 0x00F4, 0x00F6, 0x00F5, 0x00FA, 0x00F9, 0x00FB, 0x00FC, 0x2020, 0x00B0, 0x00A2, 0x00A3, + 0x00A7, 0x2022, 0x00B6, 0x00DF, 0x00AE, 0x00A9, 0x2122, 0x00B4, 0x00A8, 0x2260, 0x00C6, 0x00D8, 0x221E, 0x00B1, 0x2264, 0x2265, 0x00A5, 0x00B5, + 0x2202, 0x2211, 0x220F, 0x03C0, 0x222B, 0x00AA, 0x00BA, 0x03A9, 0x00E6, 0x00F8, 0x00BF, 0x00A1, 0x00AC, 0x221A, 0x0192, 0x2248, 0x2206, 0x00AB, + 0x00BB, 0x2026, 0x00A0, 0x00C0, 0x00C3, 0x00D5, 0x0152, 0x0153, 0x2013, 0x2014, 0x201C, 0x201D, 0x2018, 0x2019, 0x00F7, 0x25CA, 0x00FF, 0x0178, + 0x2044, 0x20AC, 0x2039, 0x203A, 0xFB01, 0xFB02, 0x2021, 0x00B7, 0x201A, 0x201E, 0x2030, 0x00C2, 0x00CA, 0x00C1, 0x00CB, 0x00C8, 0x00CD, 0x00CE, + 0x00CF, 0x00CC, 0x00D3, 0x00D4, 0xF8FF, 0x00D2, 0x00DA, 0x00DB, 0x00D9, 0x0131, 0x02C6, 0x02DC, 0x00AF, 0x02D8, 0x02D9, 0x02DA, 0x00B8, 0x02DD, + 0x02DB, 0x02C7, +]; +/// IBM PC (OEM) code page 437 code points for 0x80..=0xFF (the Unicode Consortium's CP437 +/// mapping file). +const CP437: [u16; 128] = [ + 0x00C7, 0x00FC, 0x00E9, 0x00E2, 0x00E4, 0x00E0, 0x00E5, 0x00E7, 0x00EA, 0x00EB, 0x00E8, 0x00EF, 0x00EE, 0x00EC, 0x00C4, 0x00C5, 0x00C9, 0x00E6, + 0x00C6, 0x00F4, 0x00F6, 0x00F2, 0x00FB, 0x00F9, 0x00FF, 0x00D6, 0x00DC, 0x00A2, 0x00A3, 0x00A5, 0x20A7, 0x0192, 0x00E1, 0x00ED, 0x00F3, 0x00FA, + 0x00F1, 0x00D1, 0x00AA, 0x00BA, 0x00BF, 0x2310, 0x00AC, 0x00BD, 0x00BC, 0x00A1, 0x00AB, 0x00BB, 0x2591, 0x2592, 0x2593, 0x2502, 0x2524, 0x2561, + 0x2562, 0x2556, 0x2555, 0x2563, 0x2551, 0x2557, 0x255D, 0x255C, 0x255B, 0x2510, 0x2514, 0x2534, 0x252C, 0x251C, 0x2500, 0x253C, 0x255E, 0x255F, + 0x255A, 0x2554, 0x2569, 0x2566, 0x2560, 0x2550, 0x256C, 0x2567, 0x2568, 0x2564, 0x2565, 0x2559, 0x2558, 0x2552, 0x2553, 0x256B, 0x256A, 0x2518, + 0x250C, 0x2588, 0x2584, 0x258C, 0x2590, 0x2580, 0x03B1, 0x00DF, 0x0393, 0x03C0, 0x03A3, 0x03C3, 0x00B5, 0x03C4, 0x03A6, 0x0398, 0x03A9, 0x03B4, + 0x221E, 0x03C6, 0x03B5, 0x2229, 0x2261, 0x00B1, 0x2265, 0x2264, 0x2320, 0x2321, 0x00F7, 0x2248, 0x00B0, 0x2219, 0x00B7, 0x221A, 0x207F, 0x00B2, + 0x25A0, 0x00A0, +]; + +/// OEM code page 850 (DOS Latin-1, Western European) code points for 0x80..=0xFF (the Unicode +/// Consortium's CP850 mapping file). +const CP850: [u16; 128] = [ + 0x00C7, 0x00FC, 0x00E9, 0x00E2, 0x00E4, 0x00E0, 0x00E5, 0x00E7, 0x00EA, 0x00EB, 0x00E8, 0x00EF, 0x00EE, 0x00EC, 0x00C4, 0x00C5, 0x00C9, 0x00E6, + 0x00C6, 0x00F4, 0x00F6, 0x00F2, 0x00FB, 0x00F9, 0x00FF, 0x00D6, 0x00DC, 0x00F8, 0x00A3, 0x00D8, 0x00D7, 0x0192, 0x00E1, 0x00ED, 0x00F3, 0x00FA, + 0x00F1, 0x00D1, 0x00AA, 0x00BA, 0x00BF, 0x00AE, 0x00AC, 0x00BD, 0x00BC, 0x00A1, 0x00AB, 0x00BB, 0x2591, 0x2592, 0x2593, 0x2502, 0x2524, 0x00C1, + 0x00C2, 0x00C0, 0x00A9, 0x2563, 0x2551, 0x2557, 0x255D, 0x00A2, 0x00A5, 0x2510, 0x2514, 0x2534, 0x252C, 0x251C, 0x2500, 0x253C, 0x00E3, 0x00C3, + 0x255A, 0x2554, 0x2569, 0x2566, 0x2560, 0x2550, 0x256C, 0x00A4, 0x00F0, 0x00D0, 0x00CA, 0x00CB, 0x00C8, 0x0131, 0x00CD, 0x00CE, 0x00CF, 0x2518, + 0x250C, 0x2588, 0x2584, 0x00A6, 0x00CC, 0x2580, 0x00D3, 0x00DF, 0x00D4, 0x00D2, 0x00F5, 0x00D5, 0x00B5, 0x00FE, 0x00DE, 0x00DA, 0x00DB, 0x00D9, + 0x00FD, 0x00DD, 0x00AF, 0x00B4, 0x00AD, 0x00B1, 0x2017, 0x00BE, 0x00B6, 0x00A7, 0x00F7, 0x00B8, 0x00B0, 0x00A8, 0x00B7, 0x00B9, 0x00B3, 0x00B2, + 0x25A0, 0x00A0, +]; + +/// OEM code page 852 (DOS Latin-2, Central European) code points for 0x80..=0xFF (the Unicode Consortium's CP852 +/// mapping file). +const CP852: [u16; 128] = [ + 0x00C7, 0x00FC, 0x00E9, 0x00E2, 0x00E4, 0x016F, 0x0107, 0x00E7, 0x0142, 0x00EB, 0x0150, 0x0151, 0x00EE, 0x0179, 0x00C4, 0x0106, 0x00C9, 0x0139, + 0x013A, 0x00F4, 0x00F6, 0x013D, 0x013E, 0x015A, 0x015B, 0x00D6, 0x00DC, 0x0164, 0x0165, 0x0141, 0x00D7, 0x010D, 0x00E1, 0x00ED, 0x00F3, 0x00FA, + 0x0104, 0x0105, 0x017D, 0x017E, 0x0118, 0x0119, 0x00AC, 0x017A, 0x010C, 0x015F, 0x00AB, 0x00BB, 0x2591, 0x2592, 0x2593, 0x2502, 0x2524, 0x00C1, + 0x00C2, 0x011A, 0x015E, 0x2563, 0x2551, 0x2557, 0x255D, 0x017B, 0x017C, 0x2510, 0x2514, 0x2534, 0x252C, 0x251C, 0x2500, 0x253C, 0x0102, 0x0103, + 0x255A, 0x2554, 0x2569, 0x2566, 0x2560, 0x2550, 0x256C, 0x00A4, 0x0111, 0x0110, 0x010E, 0x00CB, 0x010F, 0x0147, 0x00CD, 0x00CE, 0x011B, 0x2518, + 0x250C, 0x2588, 0x2584, 0x0162, 0x016E, 0x2580, 0x00D3, 0x00DF, 0x00D4, 0x0143, 0x0144, 0x0148, 0x0160, 0x0161, 0x0154, 0x00DA, 0x0155, 0x0170, + 0x00FD, 0x00DD, 0x0163, 0x00B4, 0x00AD, 0x02DD, 0x02DB, 0x02C7, 0x02D8, 0x00A7, 0x00F7, 0x00B8, 0x00B0, 0x00A8, 0x02D9, 0x0171, 0x0158, 0x0159, + 0x25A0, 0x00A0, +]; + +/// OEM code page 866 (DOS Cyrillic) code points for 0x80..=0xFF (the Unicode Consortium's CP866 +/// mapping file). +const CP866: [u16; 128] = [ + 0x0410, 0x0411, 0x0412, 0x0413, 0x0414, 0x0415, 0x0416, 0x0417, 0x0418, 0x0419, 0x041A, 0x041B, 0x041C, 0x041D, 0x041E, 0x041F, 0x0420, 0x0421, + 0x0422, 0x0423, 0x0424, 0x0425, 0x0426, 0x0427, 0x0428, 0x0429, 0x042A, 0x042B, 0x042C, 0x042D, 0x042E, 0x042F, 0x0430, 0x0431, 0x0432, 0x0433, + 0x0434, 0x0435, 0x0436, 0x0437, 0x0438, 0x0439, 0x043A, 0x043B, 0x043C, 0x043D, 0x043E, 0x043F, 0x2591, 0x2592, 0x2593, 0x2502, 0x2524, 0x2561, + 0x2562, 0x2556, 0x2555, 0x2563, 0x2551, 0x2557, 0x255D, 0x255C, 0x255B, 0x2510, 0x2514, 0x2534, 0x252C, 0x251C, 0x2500, 0x253C, 0x255E, 0x255F, + 0x255A, 0x2554, 0x2569, 0x2566, 0x2560, 0x2550, 0x256C, 0x2567, 0x2568, 0x2564, 0x2565, 0x2559, 0x2558, 0x2552, 0x2553, 0x256B, 0x256A, 0x2518, + 0x250C, 0x2588, 0x2584, 0x258C, 0x2590, 0x2580, 0x0440, 0x0441, 0x0442, 0x0443, 0x0444, 0x0445, 0x0446, 0x0447, 0x0448, 0x0449, 0x044A, 0x044B, + 0x044C, 0x044D, 0x044E, 0x044F, 0x0401, 0x0451, 0x0404, 0x0454, 0x0407, 0x0457, 0x040E, 0x045E, 0x00B0, 0x2219, 0x00B7, 0x221A, 0x2116, 0x00A4, + 0x25A0, 0x00A0, +]; + +/// OEM code page 737 (DOS Greek) code points for 0x80..=0xFF (the Unicode Consortium's CP737 +/// mapping file). +const CP737: [u16; 128] = [ + 0x0391, 0x0392, 0x0393, 0x0394, 0x0395, 0x0396, 0x0397, 0x0398, 0x0399, 0x039A, 0x039B, 0x039C, 0x039D, 0x039E, 0x039F, 0x03A0, 0x03A1, 0x03A3, + 0x03A4, 0x03A5, 0x03A6, 0x03A7, 0x03A8, 0x03A9, 0x03B1, 0x03B2, 0x03B3, 0x03B4, 0x03B5, 0x03B6, 0x03B7, 0x03B8, 0x03B9, 0x03BA, 0x03BB, 0x03BC, + 0x03BD, 0x03BE, 0x03BF, 0x03C0, 0x03C1, 0x03C3, 0x03C2, 0x03C4, 0x03C5, 0x03C6, 0x03C7, 0x03C8, 0x2591, 0x2592, 0x2593, 0x2502, 0x2524, 0x2561, + 0x2562, 0x2556, 0x2555, 0x2563, 0x2551, 0x2557, 0x255D, 0x255C, 0x255B, 0x2510, 0x2514, 0x2534, 0x252C, 0x251C, 0x2500, 0x253C, 0x255E, 0x255F, + 0x255A, 0x2554, 0x2569, 0x2566, 0x2560, 0x2550, 0x256C, 0x2567, 0x2568, 0x2564, 0x2565, 0x2559, 0x2558, 0x2552, 0x2553, 0x256B, 0x256A, 0x2518, + 0x250C, 0x2588, 0x2584, 0x258C, 0x2590, 0x2580, 0x03C9, 0x03AC, 0x03AD, 0x03AE, 0x03CA, 0x03AF, 0x03CC, 0x03CD, 0x03CB, 0x03CE, 0x0386, 0x0388, + 0x0389, 0x038A, 0x038C, 0x038E, 0x038F, 0x00B1, 0x2265, 0x2264, 0x03AA, 0x03AB, 0x00F7, 0x2248, 0x00B0, 0x2219, 0x00B7, 0x221A, 0x207F, 0x00B2, + 0x25A0, 0x00A0, +]; + +/// OEM code page 857 (DOS Turkish; U+FFFD where the byte is unassigned) code points for 0x80..=0xFF (the Unicode Consortium's CP857 +/// mapping file). +const CP857: [u16; 128] = [ + 0x00C7, 0x00FC, 0x00E9, 0x00E2, 0x00E4, 0x00E0, 0x00E5, 0x00E7, 0x00EA, 0x00EB, 0x00E8, 0x00EF, 0x00EE, 0x0131, 0x00C4, 0x00C5, 0x00C9, 0x00E6, + 0x00C6, 0x00F4, 0x00F6, 0x00F2, 0x00FB, 0x00F9, 0x0130, 0x00D6, 0x00DC, 0x00F8, 0x00A3, 0x00D8, 0x015E, 0x015F, 0x00E1, 0x00ED, 0x00F3, 0x00FA, + 0x00F1, 0x00D1, 0x011E, 0x011F, 0x00BF, 0x00AE, 0x00AC, 0x00BD, 0x00BC, 0x00A1, 0x00AB, 0x00BB, 0x2591, 0x2592, 0x2593, 0x2502, 0x2524, 0x00C1, + 0x00C2, 0x00C0, 0x00A9, 0x2563, 0x2551, 0x2557, 0x255D, 0x00A2, 0x00A5, 0x2510, 0x2514, 0x2534, 0x252C, 0x251C, 0x2500, 0x253C, 0x00E3, 0x00C3, + 0x255A, 0x2554, 0x2569, 0x2566, 0x2560, 0x2550, 0x256C, 0x00A4, 0x00BA, 0x00AA, 0x00CA, 0x00CB, 0x00C8, 0xFFFD, 0x00CD, 0x00CE, 0x00CF, 0x2518, + 0x250C, 0x2588, 0x2584, 0x00A6, 0x00CC, 0x2580, 0x00D3, 0x00DF, 0x00D4, 0x00D2, 0x00F5, 0x00D5, 0x00B5, 0xFFFD, 0x00D7, 0x00DA, 0x00DB, 0x00D9, + 0x00EC, 0x00FF, 0x00AF, 0x00B4, 0x00AD, 0x00B1, 0xFFFD, 0x00BE, 0x00B6, 0x00A7, 0x00F7, 0x00B8, 0x00B0, 0x00A8, 0x00B7, 0x00B9, 0x00B3, 0x00B2, + 0x25A0, 0x00A0, +]; + +/// The glyphs the OEM code pages show for 0x01..=0x1F (smileys, card suits, arrows), as +/// Windows' Alt+numpad entry produces them rather than the control characters. +const OEM_GLYPHS: [u16; 31] = [ + 0x263A, 0x263B, 0x2665, 0x2666, 0x2663, 0x2660, 0x2022, 0x25D8, 0x25CB, 0x25D9, 0x2642, 0x2640, 0x266A, 0x266B, 0x263C, 0x25BA, 0x25C4, 0x2195, + 0x203C, 0x00B6, 0x00A7, 0x25AC, 0x21A8, 0x2191, 0x2193, 0x2192, 0x2190, 0x221F, 0x2194, 0x25B2, 0x25BC, +]; + +/// The OEM code pages [`oem_byte`] decodes. +pub const OEM_CODE_PAGES: [u16; 6] = [437, 737, 850, 852, 857, 866]; + +/// One byte of an OEM code page in [`OEM_CODE_PAGES`] (any other code page reads as 437) as a +/// char, with the graphic glyphs for 0x01..=0x1F and 0x7F (⌂); 0x00 stays NUL. +pub fn oem_byte(code_page: u16, b: u8) -> char { + let table: &[u16; 128] = match code_page { + 737 => &CP737, + 850 => &CP850, + 852 => &CP852, + 857 => &CP857, + 866 => &CP866, + _ => &CP437, + }; + let u = match b { + 0x01..=0x1F => OEM_GLYPHS.get(usize::from(b - 1)).copied(), + 0x7F => Some(0x2302), + 0x80..=0xFF => table.get(usize::from(b - 0x80)).copied(), + _ => Some(u16::from(b)), + }; + u.and_then(|u| char::from_u32(u32::from(u))).unwrap_or('\u{FFFD}') +} + +/// The single-byte Windows code pages [`windows_byte`] decodes (1252 included). +pub const SINGLE_BYTE_CODE_PAGES: [u16; 10] = [874, 1250, 1251, 1252, 1253, 1254, 1255, 1256, 1257, 1258]; + +/// Mac OS Roman, RTF's `\mac` character set. +pub const MAC_ROMAN_CODE_PAGE: u16 = 10000; + +/// One byte of a single-byte code page as a char: a Windows one in [`SINGLE_BYTE_CODE_PAGES`] or +/// Mac OS Roman ([`MAC_ROMAN_CODE_PAGE`]); any other code page reads as Windows-1252. +pub fn windows_byte(code_page: u16, b: u8) -> char { + let table: &[u16; 128] = match code_page { + 874 => &CP874, + 1250 => &CP1250, + 1251 => &CP1251, + 1253 => &CP1253, + 1254 => &CP1254, + 1255 => &CP1255, + 1256 => &CP1256, + 1257 => &CP1257, + 1258 => &CP1258, + MAC_ROMAN_CODE_PAGE => &MAC_ROMAN, + _ => return cp1252(b), + }; + if b < 0x80 { + return b as char; + } + table.get(usize::from(b - 0x80)).and_then(|u| char::from_u32(u32::from(*u))).unwrap_or('\u{FFFD}') +} + +/// Whether [`code_page_byte`] decodes `code_page`: a Windows one ([`SINGLE_BYTE_CODE_PAGES`]), an +/// OEM (DOS) one ([`OEM_CODE_PAGES`]) or Mac OS Roman. Document formats (RTF) name all three. +pub fn is_byte_code_page(code_page: u16) -> bool { + SINGLE_BYTE_CODE_PAGES.contains(&code_page) || OEM_CODE_PAGES.contains(&code_page) || code_page == MAC_ROMAN_CODE_PAGE +} + +/// Windows' symbol code page (`CP_SYMBOL`), the charset of symbol fonts (`\fcharset2`): a byte is +/// the font's own code, not a letter. [`code_page_byte`] reads one from 0x80 as U+F000 + the byte, +/// the form a Word symbol character takes (`w:sym w:char="F0D8"`). +pub const SYMBOL_CODE_PAGE: u16 = 42; + +/// A byte of 8-bit document text in `code_page` (see [`is_byte_code_page`], and +/// [`SYMBOL_CODE_PAGE`] for a symbol font) as a char. Below 0x80 it is ASCII in every code page +/// (text, so an OEM page's glyphs for control codes don't apply). +pub fn code_page_byte(code_page: u16, b: u8) -> char { + if b >= 0x80 && code_page == SYMBOL_CODE_PAGE { + return char::from_u32(0xF000 + u32::from(b)).unwrap_or('\u{FFFD}'); + } + if b >= 0x80 && OEM_CODE_PAGES.contains(&code_page) { oem_byte(code_page, b) } else { windows_byte(code_page, b) } +} + +/// The East Asian multibyte code pages a document format names with `\ansicpg` or a font's `\cpg`: +/// 932 Shift-JIS (Japanese), 936 GBK (Simplified Chinese), 949 UHC (Korean) and 950 Big5 +/// (Traditional Chinese). +pub const MULTIBYTE_CODE_PAGES: [u16; 4] = [932, 936, 949, 950]; + +/// Whether `code_page` is one of the East Asian multibyte code pages ([`MULTIBYTE_CODE_PAGES`]): +/// its bytes decode as a run, a lead byte together with its trail byte. +pub fn is_multibyte_code_page(code_page: u16) -> bool { + multibyte_encoding_name(code_page).is_some() +} + +/// The WHATWG encoding name of an East Asian multibyte code page (see [`MULTIBYTE_CODE_PAGES`]), +/// for `encoding_rs::Encoding::for_label`; `None` for the single-byte code pages, whose bytes +/// decode one at a time, and anything else. +pub fn multibyte_encoding_name(code_page: u16) -> Option<&'static str> { + Some(match code_page { + 932 => "shift_jis", + 936 => "gbk", + 949 => "euc-kr", + 950 => "big5", + _ => return None, + }) +} + +/// Whether a document format can name `code_page` for its text: a single-byte Windows, DOS or Mac +/// one ([`is_byte_code_page`]) or an East Asian multibyte one ([`is_multibyte_code_page`]). +pub fn is_text_code_page(code_page: u16) -> bool { + is_byte_code_page(code_page) || is_multibyte_code_page(code_page) +} + +/// The code page of an RTF or Word font charset (`\fcharset`): 204 Russian → 1251, 238 Eastern +/// European → 1250, 77 Mac → 10000, 254 PC / 255 OEM → 437, 128 Japanese → 932, 129 Korean → 949, +/// 134 Simplified Chinese → 936, 136 Traditional Chinese → 950, 2 Symbol → [`SYMBOL_CODE_PAGE`]… +/// `None` for the ANSI and default charsets (the document's code page applies). +pub fn charset_code_page(charset: i32) -> Option { + Some(match charset { + 2 => SYMBOL_CODE_PAGE, + 77 => MAC_ROMAN_CODE_PAGE, + 128 => 932, + 129 => 949, + 134 => 936, + 136 => 950, + 161 => 1253, + 162 => 1254, + 163 => 1258, + 177 => 1255, + 178 => 1256, + 186 => 1257, + 204 => 1251, + 222 => 874, + 238 => 1250, + 254 | 255 => 437, + _ => return None, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn single_byte_code_pages() { + let decode = |cp: u16, bytes: &[u8]| bytes.iter().map(|&b| windows_byte(cp, b)).collect::(); + assert_eq!(decode(1251, &[0xD1, 0xF7, 0xE5, 0xF2, 0x20, 0xB9]), "Счет №"); + assert_eq!(decode(1251, &[0xA8, 0xB8]), "Ёё"); + assert_eq!(decode(1250, &[0xA3, 0xF3, 0x9F]), "Łóź"); + assert_eq!(decode(1253, &[0xC1, 0xE2]), "Αβ"); + assert_eq!(decode(1252, &[0x80, 0xE9]), "€é"); + assert_eq!(decode(9999, &[0xE9]), "é"); + // Mac OS Roman: accents, the euro sign at 0xDB, typographic quotes. + assert_eq!(decode(10000, &[0x80, 0x8E, 0xDB, 0xF1, 0xF2, 0xF3, 0xD2, 0xD3, 0x41]), "Äé€ÒÚÛ“”A"); + assert_eq!(charset_code_page(77), Some(10000)); + assert_eq!(charset_code_page(254), Some(437)); + assert_eq!(charset_code_page(255), Some(437)); + assert_eq!(charset_code_page(204), Some(1251)); + assert_eq!(charset_code_page(0), None); + for cp in SINGLE_BYTE_CODE_PAGES { + for b in 0..=255u8 { + let _ = windows_byte(cp, b); + } + } + } + + #[test] + fn windows_1255_holam_haser_for_vav() { + assert_eq!(windows_byte(1255, 0xCA), '\u{05BA}'); + } + + /// One decoder for document text in any code page RTF names: Windows, DOS or Mac. + #[test] + fn document_text_in_windows_dos_and_mac_code_pages() { + let decode = |cp: u16, bytes: &[u8]| bytes.iter().map(|&b| code_page_byte(cp, b)).collect::(); + assert_eq!(decode(437, &[0x82, 0xA0, 0x41, 0x01]), "éáA\u{1}", "DOS text; ASCII stays ASCII"); + assert_eq!(decode(850, &[0x9D, 0xE9]), "ØÚ"); + assert_eq!(decode(10000, &[0x8E]), "é"); + assert_eq!(decode(1251, &[0xC4]), "Д"); + // A symbol font's bytes are its own codes, from 0x80 the private-use form; ASCII stays ASCII. + assert_eq!(charset_code_page(2), Some(SYMBOL_CODE_PAGE)); + assert_eq!(decode(SYMBOL_CODE_PAGE, &[0x4A, 0x93, 0xD8, 0xFC]), "J\u{F093}\u{F0D8}\u{F0FC}"); + for cp in [437, 850, 866, 1250, 1252, 10000] { + assert!(is_byte_code_page(cp), "{cp}"); + } + assert!(!is_byte_code_page(932) && !is_byte_code_page(0) && !SINGLE_BYTE_CODE_PAGES.contains(&10000)); + } + + /// The East Asian charsets and code pages RTF names are multibyte: their bytes decode + /// as runs (a lead byte and its trail byte together), not one at a time. + #[test] + fn east_asian_code_pages_are_multibyte() { + assert_eq!(charset_code_page(128), Some(932), "Japanese"); + assert_eq!(charset_code_page(129), Some(949), "Korean"); + assert_eq!(charset_code_page(134), Some(936), "Simplified Chinese"); + assert_eq!(charset_code_page(136), Some(950), "Traditional Chinese"); + assert_eq!(charset_code_page(255), Some(437), "OEM is single-byte"); + assert_eq!(MULTIBYTE_CODE_PAGES, [932, 936, 949, 950]); + for cp in MULTIBYTE_CODE_PAGES { + assert!(is_multibyte_code_page(cp) && is_text_code_page(cp), "{cp}"); + assert!(!is_byte_code_page(cp) && !SINGLE_BYTE_CODE_PAGES.contains(&cp), "{cp}"); + assert!(multibyte_encoding_name(cp).is_some(), "{cp}"); + } + assert!(is_text_code_page(1251) && is_text_code_page(437) && is_text_code_page(10000)); + assert!(!is_text_code_page(0) && !is_text_code_page(9999)); + } + + #[test] + fn oem_code_pages() { + let decode = |cp: u16, bytes: &[u8]| bytes.iter().map(|&b| oem_byte(cp, b)).collect::(); + assert_eq!(decode(437, &[0x82, 0xE9, 0x01, 0x7F]), "éΘ☺⌂"); + assert_eq!(decode(850, &[0x9B, 0xE9]), "øÚ"); + assert_eq!(decode(852, &[0xA5, 0x88, 0xBE, 0xA9]), "ąłżę"); + assert_eq!(decode(866, &[0x8F, 0xE0, 0xA8, 0xA2, 0xA5, 0xE2, 0xF0, 0xFC]), "ПриветЁ№"); + assert_eq!(decode(737, &[0x80, 0x99, 0xE0]), "Αβω"); + assert_eq!(decode(857, &[0x8D, 0x98, 0xA6, 0x9F]), "ıİĞş"); + assert_eq!(oem_byte(857, 0xD5), '\u{FFFD}', "unassigned in 857"); + assert_eq!(decode(9999, &[0xE9]), "Θ", "unknown pages read as 437"); + for cp in OEM_CODE_PAGES { + for b in 0..=255u8 { + let _ = oem_byte(cp, b); + } + } + } +} diff --git a/crates/rtf/src/date.rs b/crates/rtf/src/date.rs new file mode 100644 index 000000000..e0c58ab12 --- /dev/null +++ b/crates/rtf/src/date.rs @@ -0,0 +1,102 @@ +//! RTF dates: the DTTM bit pattern of revision marks, and the `\yr\mo\dy\hr\min` words of the +//! document information group. + +/// An ISO 8601 date-time from year, month, day, hour and minute (the parameters of an `\info` +/// date's `\yr \mo \dy \hr \min`), each clamped into range. +pub fn iso_date((y, mo, d, h, mi): (u32, u32, u32, u32, u32)) -> String { + format!("{y:04}-{:02}-{:02}T{:02}:{:02}:00Z", mo.clamp(1, 12), d.clamp(1, 31), h.min(23), mi.min(59)) +} + +/// An ISO 8601 date (`YYYY-MM-DD`, optionally `THH:MM`) as the words of an `\info` date: +/// `\yr2024\mo3\dy15\hr10\min30`. +pub fn date_words(s: &str) -> Option { + let num = |a: usize, b: usize| s.get(a..b).and_then(|x| x.parse::().ok()); + let (y, mo, d) = (num(0, 4)?, num(5, 7)?, num(8, 10)?); + let (h, mi) = (num(11, 13).unwrap_or(0), num(14, 16).unwrap_or(0)); + Some(format!("\\yr{y}\\mo{mo}\\dy{d}\\hr{h}\\min{mi}")) +} + +fn days_in_month(y: u32, m: u32) -> u32 { + match m { + 1 | 3 | 5 | 7 | 8 | 10 | 12 => 31, + 4 | 6 | 9 | 11 => 30, + 2 if (y.is_multiple_of(4) && !y.is_multiple_of(100)) || y.is_multiple_of(400) => 29, + 2 => 28, + _ => 0, + } +} + +/// Day of the week, 0 = Sunday (Sakamoto's method). +fn weekday(y: u32, m: u32, d: u32) -> u32 { + const T: [u32; 12] = [0, 3, 2, 5, 0, 3, 5, 1, 4, 6, 2, 4]; + let y = if m < 3 { y.saturating_sub(1) } else { y }; + (y + y / 4 - y / 100 + y / 400 + T.get(m.saturating_sub(1) as usize).copied().unwrap_or(0) + d) % 7 +} + +/// An RTF revision date (DTTM, RTF 1.9.1 "Revision Marks": minute in bits 0–5, hour 6–10, day +/// 11–15, month 16–19, year − 1900 20–28, weekday 29–31) as ISO 8601 (`2024-03-15T10:30:00Z`). +/// Zero or an impossible date is no date. DTTM has no time zone or seconds: times are read as +/// UTC (`Z`). +pub fn dttm_to_iso(v: u32) -> Option { + let (mi, h, d, mo, y) = (v & 0x3F, (v >> 6) & 0x1F, (v >> 11) & 0x1F, (v >> 16) & 0xF, 1900 + ((v >> 20) & 0x1FF)); + if v == 0 || mi > 59 || h > 23 || d == 0 || d > days_in_month(y, mo) { + return None; + } + Some(format!("{y:04}-{mo:02}-{d:02}T{h:02}:{mi:02}:00Z")) +} + +/// An ISO 8601 date (`YYYY-MM-DD`, optionally `THH:MM…`; seconds and zone ignored) as an RTF +/// DTTM; `None` when it isn't one or falls outside DTTM's years (1900–2411). DTTM has no time +/// zone or seconds: the clock time is written as given (a `Z` time stays the UTC clock time), and seconds +/// and any offset are dropped. +pub fn iso_to_dttm(s: &str) -> Option { + let num = |a: usize, b: usize| s.get(a..b).filter(|t| t.bytes().all(|c| c.is_ascii_digit())).and_then(|t| t.parse::().ok()); + let sep = |i: usize, c: &[&str]| s.get(i..i + 1).is_some_and(|t| c.contains(&t)); + let (y, mo, d) = (num(0, 4)?, num(5, 7)?, num(8, 10)?); + if !sep(4, &["-"]) || !sep(7, &["-"]) { + return None; + } + let (h, mi) = if s.len() > 10 { + if !sep(10, &["T", " "]) || !sep(13, &[":"]) { + return None; + } + (num(11, 13)?, num(14, 16)?) + } else { + (0, 0) + }; + if !(1900..=2411).contains(&y) || mi > 59 || h > 23 || d == 0 || d > days_in_month(y, mo) { + return None; + } + Some(mi | (h << 6) | (d << 11) | (mo << 16) | ((y - 1900) << 20) | (weekday(y, mo, d) << 29)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn dttm_round_trips_and_refuses_impossible_dates() { + let v = iso_to_dttm("2024-03-15T10:30:59+02:00").unwrap(); + assert_eq!(dttm_to_iso(v).as_deref(), Some("2024-03-15T10:30:00Z")); + assert_eq!(v >> 29, 5, "a Friday"); + assert_eq!(iso_to_dttm("2024-02-29").and_then(dttm_to_iso).as_deref(), Some("2024-02-29T00:00:00Z")); + for bad in ["2023-02-29", "1899-01-01", "2412-01-01", "2024-13-01", "2024-03-15T24:00", "2024/03/15", "2024-03-15x10:30", "", "+024-03-15"] { + assert_eq!(iso_to_dttm(bad), None, "{bad}"); + } + assert_eq!(dttm_to_iso(0), None); + assert_eq!(dttm_to_iso(u32::MAX), None); + // Day 31 of a 30-day month, hour 24, minute 60. + assert_eq!(dttm_to_iso((31 << 11) | (4 << 16) | (124 << 20)), None); + assert_eq!(dttm_to_iso((24 << 6) | (1 << 11) | (1 << 16) | (124 << 20)), None); + assert_eq!(dttm_to_iso(60 | (1 << 11) | (1 << 16) | (124 << 20)), None); + } + + #[test] + fn info_dates() { + assert_eq!(date_words("2024-03-15T10:30:00Z").as_deref(), Some("\\yr2024\\mo3\\dy15\\hr10\\min30")); + assert_eq!(date_words("2024-03-15").as_deref(), Some("\\yr2024\\mo3\\dy15\\hr0\\min0")); + assert_eq!(date_words("soon"), None); + assert_eq!(iso_date((2024, 3, 15, 10, 30)), "2024-03-15T10:30:00Z"); + assert_eq!(iso_date((2024, 0, 99, 99, 99)), "2024-01-31T23:59:00Z", "clamped"); + } +} diff --git a/crates/rtf/src/lex.rs b/crates/rtf/src/lex.rs new file mode 100644 index 000000000..348c9e6bb --- /dev/null +++ b/crates/rtf/src/lex.rs @@ -0,0 +1,231 @@ +//! The tokens of an RTF file. Nothing is interpreted here: a token is what the bytes say. + +/// Is this an RTF file? Leading white space, then `{\rtf`. +pub fn is_rtf(bytes: &[u8]) -> bool { + bytes.trim_ascii_start().starts_with(b"{\\rtf") +} + +/// One token. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Token<'a> { + /// `{` + Open, + /// `}` + Close, + /// A control word: `\par`, `\fs24`, `\li-720`. The parameter is clamped into `i32`. + Word { name: &'a str, param: Option }, + /// `\*`: the destination that follows may be ignored by a reader that does not know it. + Star, + /// `\'hh`: one byte of text in the current code page. + Hex(u8), + /// Any other control symbol, by the byte after the backslash: `\~`, `\-`, `\_`, `\\`, + /// `\{`, `\}`, and `\` before a line end (which is `\par`). + Symbol(u8), + /// The data of `\binN`: the `N` bytes after the word's delimiter, as they are. + Bin(&'a [u8]), + /// A run of text bytes in the current code page: no `{`, `}`, `\`, line end or NUL. + Text(&'a [u8]), +} + +/// The tokens of `bytes`, in order. Never fails: a malformed escape is dropped and whatever +/// follows it is read as the structure it is. +/// +/// - A control word is at most 32 letters and its parameter at most 10 digits; what follows is +/// text. A `-` starts a parameter only when a digit follows it. +/// - One space after a control word is its delimiter and not text. +/// - Line ends and NUL bytes between tokens are not tokens. +/// - `\'` without two hex digits gives no token: it and the hex digits it does have are +/// skipped, and a `}` or `\` after it is read as itself. +/// - A date word (`\revdttm`, `\revdttmdel`, `\crdate`, `\prdate`) holds a 32-bit pattern that +/// some writers put unsigned: it is read as that pattern, not clamped. +#[derive(Clone, Debug)] +pub struct Tokens<'a> { + bytes: &'a [u8], + at: usize, +} + +impl<'a> Tokens<'a> { + pub fn new(bytes: &'a [u8]) -> Tokens<'a> { + Tokens { bytes, at: 0 } + } + + /// The offset of the next token. + pub fn offset(&self) -> usize { + self.at + } + + fn word(&mut self) -> Token<'a> { + let b = self.bytes; + let start = self.at + 1; + let mut j = start; + while b.get(j).is_some_and(u8::is_ascii_alphabetic) && j - start < 32 { + j += 1; + } + let name = b.get(start..j).and_then(|w| std::str::from_utf8(w).ok()).unwrap_or(""); + let neg = b.get(j) == Some(&b'-'); + let digits = j + usize::from(neg); + let mut k = digits; + while b.get(k).is_some_and(u8::is_ascii_digit) && k - digits < 10 { + k += 1; + } + let param = if k > digits { + // At most 10 digits, so it fits an i64. + let v = b.get(digits..k).and_then(|d| std::str::from_utf8(d).ok()).and_then(|d| d.parse::().ok()).unwrap_or(0); + let v = if neg { -v } else { v }; + let dttm = matches!(name, "revdttm" | "revdttmdel" | "crdate" | "prdate"); + let v = if dttm && v > i64::from(i32::MAX) && v <= i64::from(u32::MAX) { v - (1i64 << 32) } else { v }; + Some(v.clamp(i64::from(i32::MIN), i64::from(i32::MAX)) as i32) + } else { + k = j; + None + }; + if b.get(k) == Some(&b' ') { + k += 1; + } + self.at = k; + if name == "bin" { + let n = usize::try_from(param.unwrap_or(0)).unwrap_or(0); + let end = k.saturating_add(n).min(b.len()); + self.at = end; + return Token::Bin(b.get(k..end).unwrap_or_default()); + } + Token::Word { name, param } + } +} + +impl<'a> Iterator for Tokens<'a> { + type Item = Token<'a>; + + fn next(&mut self) -> Option> { + let b = self.bytes; + loop { + let c = *b.get(self.at)?; + match c { + b'{' | b'}' => { + self.at += 1; + return Some(if c == b'{' { Token::Open } else { Token::Close }); + } + b'\r' | b'\n' | 0 => self.at += 1, + b'\\' => { + let n = *b.get(self.at + 1)?; + if n.is_ascii_alphabetic() { + return Some(self.word()); + } + self.at += 2; + match n { + b'\'' => { + let hex = b.get(self.at..self.at + 2).filter(|h| h.iter().all(u8::is_ascii_hexdigit)); + if let Some(v) = hex.and_then(|h| std::str::from_utf8(h).ok()).and_then(|h| u8::from_str_radix(h, 16).ok()) { + self.at += 2; + return Some(Token::Hex(v)); + } + // Malformed: skip the hex digits of a truncated pair, nothing else. + let mut n = 0; + while n < 2 && b.get(self.at).is_some_and(u8::is_ascii_hexdigit) { + self.at += 1; + n += 1; + } + } + b'*' => return Some(Token::Star), + b'\r' => return Some(Token::Symbol(b'\n')), + n => return Some(Token::Symbol(n)), + } + } + _ => { + let start = self.at; + while b.get(self.at).is_some_and(|c| !matches!(c, b'{' | b'}' | b'\\' | b'\r' | b'\n' | 0)) { + self.at += 1; + } + return Some(Token::Text(b.get(start..self.at).unwrap_or_default())); + } + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use Token::{Bin, Close, Hex, Open, Star, Symbol, Text}; + + fn w(name: &str, param: Option) -> Token<'_> { + Token::Word { name, param } + } + + fn lex(s: &[u8]) -> Vec> { + Tokens::new(s).collect() + } + + #[test] + fn groups_words_symbols_and_text() { + assert_eq!( + lex(br"{\rtf1\ansi {\*\generator X;}\b Hi\b0 there\par\~\'e9}"), + vec![ + Open, + w("rtf", Some(1)), + w("ansi", None), + Open, + Star, + w("generator", None), + Text(b"X;"), + Close, + w("b", None), + Text(b"Hi"), + w("b", Some(0)), + Text(b" there"), + w("par", None), + Symbol(b'~'), + Hex(0xE9), + Close + ] + ); + assert!(is_rtf(b" \r\n{\\rtf1}") && !is_rtf(b"{rtf1}") && !is_rtf(b"")); + } + + #[test] + fn a_hyphen_is_a_sign_only_before_a_digit() { + assert_eq!( + lex(br"\li-720 a\emdash-roll\tab- x"), + vec![w("li", Some(-720)), Text(b"a"), w("emdash", None), Text(b"-roll"), w("tab", None), Text(b"- x")] + ); + } + + #[test] + fn long_words_and_parameters_are_cut_not_scanned_without_end() { + let long = format!("\\{}1", "a".repeat(40)); + assert_eq!(lex(long.as_bytes()), vec![w(&"a".repeat(32), None), Text(b"aaaaaaaa1")]); + assert_eq!(lex(br"\fs123456789012 x"), vec![w("fs", Some(1_234_567_890)), Text(b"12 x")]); + assert_eq!(lex(br"\fs9999999999\li-9999999999"), vec![w("fs", Some(i32::MAX)), w("li", Some(i32::MIN))]); + } + + #[test] + fn dates_keep_their_bit_pattern() { + assert_eq!( + lex(br"\revdttm4294967295\crdate2147483648\fs4294967295"), + vec![w("revdttm", Some(-1)), w("crdate", Some(i32::MIN)), w("fs", Some(i32::MAX))] + ); + } + + #[test] + fn malformed_hex_escapes_leave_the_structure_alone() { + assert_eq!(lex(br"{\b A\'4}B"), vec![Open, w("b", None), Text(b"A"), Close, Text(b"B")]); + assert_eq!(lex(br"a\'+fb\'\par"), vec![Text(b"a"), Text(b"+fb"), w("par", None)]); + assert_eq!(lex(br"\'zz\'4"), vec![Text(b"zz")]); + assert_eq!(lex(b"a\\"), vec![Text(b"a")], "a backslash at the end is dropped"); + } + + #[test] + fn binary_data_is_not_scanned() { + assert_eq!(lex(b"{\\pict\\bin5 {\\}\0\\x}"), vec![Open, w("pict", None), Bin(b"{\\}\0\\"), Text(b"x"), Close]); + assert_eq!(lex(b"\\bin99 ab"), vec![Bin(b"ab")], "cut at the end of the file"); + assert_eq!(lex(b"\\bin-4 ab\\bin"), vec![Bin(b""), Text(b"ab"), Bin(b"")]); + } + + #[test] + fn line_ends_and_nul_are_not_text() { + assert_eq!(lex(b"a\r\nb\0c\\\nd\\\r\ne"), vec![Text(b"a"), Text(b"b"), Text(b"c"), Symbol(b'\n'), Text(b"d"), Symbol(b'\n'), Text(b"e")]); + assert_eq!(lex(b"\\\xe9x"), vec![Symbol(0xE9), Text(b"x")]); + let mut t = Tokens::new(b"ab{"); + assert_eq!((t.next(), t.offset()), (Some(Text(b"ab")), 2)); + } +} diff --git a/crates/rtf/src/lib.rs b/crates/rtf/src/lib.rs new file mode 100644 index 000000000..c6bba82ec --- /dev/null +++ b/crates/rtf/src/lib.rs @@ -0,0 +1,45 @@ +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +//! Rich Text Format. +//! +//! Three layers, each usable alone: +//! +//! - [`Tokens`]: the tokens of a file (groups, control words, `\'hh`, text runs, `\bin` data). +//! - [`TextDecoder`] and [`codepage`]: text in the file's code pages, `\uN` with its fallback +//! characters and surrogate pairs. [`esc`] goes the other way. [`dttm_to_iso`], +//! [`iso_to_dttm`], [`date_words`] and [`iso_date`] are RTF's two date forms. +//! - [`read`] and [`write`]: a file to and from a small neutral model ([`Document`]): +//! paragraphs with outline level, alignment, indents and spacing, runs with bold, italic, +//! underline, strikethrough, superscript and subscript, size, colour and font, and tables. +//! +//! A reader that keeps more than the model holds (styles, lists, fields, notes, pictures) is +//! built on the first two layers. +//! +//! ``` +//! use craft_rtf::{Block, read, write}; +//! +//! let doc = read(br"{\rtf1\ansi\pard\outlinelevel0 Caf\'e9 {\b \u8364?5}\par}")?; +//! let Some(Block::Para(p)) = doc.blocks.first() else { return Ok(()) }; +//! assert_eq!((p.text().as_str(), p.outline_level), ("Café €5", Some(0))); +//! assert!(p.runs[1].fmt.bold); +//! assert_eq!(read(write(&doc, "Helvetica").as_bytes())?, doc); +//! # Ok::<(), craft_rtf::NotRtf>(()) +//! ``` + +#[cfg(doctest)] +#[doc = include_str!("../README.md")] +struct ReadmeExamples; + +pub mod codepage; +mod date; +mod lex; +mod model; +mod read; +mod text; +mod write; + +pub use date::{date_words, dttm_to_iso, iso_date, iso_to_dttm}; +pub use lex::{Token, Tokens, is_rtf}; +pub use model::{Align, Block, COLUMN_BREAK, Cell, CharFmt, Document, LINE_BREAK, PAGE_BREAK, Para, Run, Table, VertAlign}; +pub use read::{MAX_COLORS, MAX_FONT_NAME, MAX_FONTS, MAX_GROUPS, NotRtf, read}; +pub use text::TextDecoder; +pub use write::{esc, esc_font, push_esc, write}; diff --git a/crates/rtf/src/model.rs b/crates/rtf/src/model.rs new file mode 100644 index 000000000..acbb3c7ee --- /dev/null +++ b/crates/rtf/src/model.rs @@ -0,0 +1,124 @@ +//! A small neutral model of formatted text: what the reader gives and the writer takes. + +/// A line break inside a paragraph (`\line`), in [`Run::text`]. +pub const LINE_BREAK: char = '\n'; +/// A page break (`\page`), in [`Run::text`]. +pub const PAGE_BREAK: char = '\u{000C}'; +/// A column break (`\column`), in [`Run::text`]. +pub const COLUMN_BREAK: char = '\u{000E}'; + +/// Where a run sits on the line. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum VertAlign { + #[default] + Baseline, + /// `\super` + Superscript, + /// `\sub` + Subscript, +} + +/// How a paragraph's lines are set. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Align { + /// `\ql` + Left, + /// `\qc` + Center, + /// `\qr` + Right, + /// `\qj` + Justified, +} + +/// Character formatting. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct CharFmt { + pub bold: bool, + pub italic: bool, + pub underline: bool, + pub strike: bool, + pub vert_align: VertAlign, + /// `\fsN`: the size in half-points (24 is 12 pt). `None`: the reader's default. + pub half_points: Option, + /// `\cfN` through the colour table, as red, green, blue. `None`: the default colour. + pub color: Option<[u8; 3]>, + /// `\fN` through the font table. `None`: the default font. + pub font: Option, +} + +/// Text in one formatting. Tabs are `\t`; breaks are [`LINE_BREAK`], [`PAGE_BREAK`] and +/// [`COLUMN_BREAK`]. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct Run { + pub text: String, + pub fmt: CharFmt, +} + +/// A paragraph. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct Para { + pub runs: Vec, + /// `\outlinelevelN`, 0 to 8: a heading of level N + 1. `None`: body text. + pub outline_level: Option, + /// `\liN`: the left indent in twips (1/1440 inch). + pub left_indent: i32, + /// `\riN`: the right indent in twips. + pub right_indent: i32, + /// `\fiN`: the first line's indent in twips, on top of the left indent; negative for a + /// hanging indent. + pub first_line_indent: i32, + /// `\sbN` and `\saN`: the space before and after in twips. + pub space_before: i32, + pub space_after: i32, + /// `\ql`, `\qc`, `\qr`, `\qj`. `None`: the file does not say. + pub align: Option, +} + +impl Para { + /// The paragraph's text without its formatting. + pub fn text(&self) -> String { + self.runs.iter().map(|r| r.text.as_str()).collect() + } +} + +/// A table cell. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct Cell { + /// `\cellxN`: the cell's right edge from the table's left edge, in twips. A merged cell + /// ends at the right edge of the last column it spans. + pub right: i32, + pub paras: Vec, +} + +/// A table: rows of cells. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct Table { + pub rows: Vec>, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Block { + Para(Para), + Table(Table), +} + +/// A document: the body's paragraphs and tables, in order. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct Document { + pub blocks: Vec, + /// Pictures the reader left out (`\pict`, `\object`). + pub pictures_left_out: usize, +} + +impl Document { + /// Every paragraph in reading order, a table's cells row by row. + pub fn paras(&self) -> impl Iterator { + self.blocks.iter().flat_map(|b| -> Box + '_> { + match b { + Block::Para(p) => Box::new(std::iter::once(p)), + Block::Table(t) => Box::new(t.rows.iter().flatten().flat_map(|c| &c.paras)), + } + }) + } +} diff --git a/crates/rtf/src/read.rs b/crates/rtf/src/read.rs new file mode 100644 index 000000000..46007dd9c --- /dev/null +++ b/crates/rtf/src/read.rs @@ -0,0 +1,517 @@ +//! RTF into the model. + +use std::collections::HashMap; + +use crate::codepage::{MAC_ROMAN_CODE_PAGE, SYMBOL_CODE_PAGE, charset_code_page, is_text_code_page}; +use crate::lex::{Token, Tokens, is_rtf}; +use crate::model::{Align, Block, COLUMN_BREAK, Cell, CharFmt, Document, LINE_BREAK, PAGE_BREAK, Para, Run, Table, VertAlign}; +use crate::text::TextDecoder; + +/// Deepest group nesting followed. Deeper groups are read in the state of the group around +/// them. +pub const MAX_GROUPS: usize = 512; +/// Most fonts kept from the font table. +pub const MAX_FONTS: usize = 4096; +/// Most colours kept from the colour table. +pub const MAX_COLORS: usize = 4096; +/// Longest font name kept, in bytes. A longer one is no font. +pub const MAX_FONT_NAME: usize = 256; + +/// The file does not start with `{\rtf`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct NotRtf; + +impl std::fmt::Display for NotRtf { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("not an RTF document (it doesn't start with {\\rtf)") + } +} + +impl std::error::Error for NotRtf {} + +/// Destinations whose text is not body text. Any destination marked `\*` is skipped too. +const SKIP: [&str; 30] = [ + "stylesheet", + "info", + "header", + "headerl", + "headerr", + "headerf", + "footer", + "footerl", + "footerr", + "footerf", + "footnote", + "listtable", + "listoverridetable", + "rsidtbl", + "generator", + "themedata", + "colorschememapping", + "latentstyles", + "datastore", + "xmlnstbl", + "fldinst", + "revtbl", + "filetbl", + "annotation", + "bkmkstart", + "bkmkend", + // A list item's rendered bullet or number, written before its text (Word: listtext; older + // writers: pntext and its parts). Its `\pard` must not reset the paragraph's own indent. + "listtext", + "pntext", + "pntxta", + "pntxtb", +]; + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +enum Dest { + #[default] + Body, + Skip, + FontTbl, + ColorTbl, +} + +/// What a group saves and restores. +#[derive(Clone, Debug)] +struct St { + dest: Dest, + fmt: CharFmt, + /// The code page of the current font, when its charset names one. + cp: Option, + /// A font was selected (`\f`) in this group or an enclosing one, and no `\plain` has since + /// made the default font current again. + font_set: bool, + uc: usize, + /// The paragraph properties in effect; their `runs` stay empty. + para: Para, + in_table: bool, +} + +impl Default for St { + fn default() -> St { + St { dest: Dest::Body, fmt: CharFmt::default(), cp: None, font_set: false, uc: 1, para: Para::default(), in_table: false } + } +} + +#[derive(Default)] +struct Reader { + st: St, + stack: Vec, + /// Groups open beyond [`MAX_GROUPS`]. + overflow: usize, + decoder: TextDecoder, + /// `\ansicpg`, `\mac`, `\pc`, `\pca`; 0 until the file names one (Windows-1252 then). + ansicpg: u16, + deff: i32, + fonts: HashMap, + font_cps: HashMap, + font_cur: i32, + font_cp_cur: Option, + /// The font table entry being read named its code page with `\cpg`. + font_cpg_named: bool, + font_name: String, + /// Where the text of a destination that keeps none goes. + scrap: String, + colors: Vec>, + color_cur: ([u8; 3], bool), + doc: Document, + para: Para, + text: String, + rows: Vec>, + row: Vec, + cell: Vec, + cellx: Vec, +} + +/// Reads an RTF file. +/// +/// Kept: body text in the file's code pages (`\ansicpg`, `\mac`, `\pc`, `\pca`, a font's +/// `\fcharset` or `\cpg`) and `\uN`; paragraphs (`\par`, `\sect`) with `\outlinelevel` and +/// `\li`, `\ri`, `\fi`, `\sb`, `\sa` and alignment; bold, italic, underline, strikethrough, +/// superscript and subscript, size, colour and font; tabs, line, page and +/// column breaks, the quote, dash and bullet words; tables (`\cellx`, `\cell`, `\row`). +/// +/// Left out: headers, footers, footnotes, comments, field instructions (a field's result is +/// kept), list labels, the stylesheet, pictures and objects (counted in +/// [`Document::pictures_left_out`]), and every destination marked `\*`. A table nested in a +/// cell reads as that cell's text. +/// +/// A trailing `\par` does not leave an empty last paragraph. Never fails on a file that starts +/// with `{\rtf`: what cannot be read is skipped. +pub fn read(bytes: &[u8]) -> Result { + if !is_rtf(bytes) { + return Err(NotRtf); + } + let mut r = Reader::default(); + let mut starred = false; + for token in Tokens::new(bytes) { + let star = std::mem::take(&mut starred); + match token { + Token::Open => r.open(), + Token::Close => r.close(), + Token::Star => starred = true, + Token::Word { name, param } => r.word(name, param, star), + Token::Hex(b) => r.hex(b), + Token::Symbol(s) => r.symbol(s), + Token::Text(t) => r.bytes(t), + Token::Bin(_) => {} + } + } + Ok(r.finish()) +} + +impl Reader { + /// The decoder, and where text goes in the current destination. + fn sink(&mut self) -> (&mut TextDecoder, &mut String) { + self.scrap.clear(); + let out = match self.st.dest { + Dest::Body => &mut self.text, + Dest::FontTbl => &mut self.font_name, + Dest::Skip | Dest::ColorTbl => &mut self.scrap, + }; + (&mut self.decoder, out) + } + + fn open(&mut self) { + let (decoder, out) = self.sink(); + decoder.group(out); + self.flush_run(); + if self.stack.len() >= MAX_GROUPS { + self.overflow += 1; + return; + } + self.stack.push(self.st.clone()); + } + + fn close(&mut self) { + let (decoder, out) = self.sink(); + decoder.group(out); + if self.overflow > 0 { + self.overflow -= 1; + return; + } + if self.st.dest == Dest::FontTbl { + self.commit_font(); + } + self.flush_run(); + if let Some(outer) = self.stack.pop() { + self.st = outer; + } + } + + /// The code page 8-bit text is read in: the current font's, else the default font's, else + /// the document's (Windows-1252 when none is named). The symbol page is for body text set in + /// a symbol font, so it never applies outside the body (names in the font table), and a + /// symbol default font's page does not apply to text in a font selected with `\f`. + fn code_page(&self) -> u16 { + let document = if self.ansicpg == 0 { 1252 } else { self.ansicpg }; + let body = self.st.dest == Dest::Body; + let current = self.st.cp.filter(|&cp| cp != SYMBOL_CODE_PAGE || body); + let default = self.font_cps.get(&self.deff).copied().filter(|&cp| cp != SYMBOL_CODE_PAGE || (body && !self.st.font_set)); + current.or(default).unwrap_or(document) + } + + fn bytes(&mut self, chunk: &[u8]) { + let cp = self.code_page(); + match self.st.dest { + Dest::Body => self.decoder.bytes(chunk, cp, &mut self.text), + Dest::FontTbl => { + // `;` ends an entry; a name may be split around it by formatting. + for (i, part) in chunk.split(|b| *b == b';').enumerate() { + if i > 0 { + self.commit_font(); + } + self.decoder.bytes(part, cp, &mut self.font_name); + } + } + Dest::ColorTbl => { + for _ in chunk.iter().filter(|b| **b == b';') { + self.commit_color(); + } + } + Dest::Skip => {} + } + } + + fn hex(&mut self, b: u8) { + let cp = self.code_page(); + // In a destination that keeps no text the byte still uses up a fallback position. + let (decoder, out) = self.sink(); + decoder.byte(b, cp, out); + } + + fn ch(&mut self, c: char) { + let (decoder, out) = self.sink(); + decoder.char(c, out); + } + + fn symbol(&mut self, s: u8) { + if self.decoder.skip() { + return; + } + match s { + b'~' => self.ch('\u{00A0}'), + b'-' => self.ch('\u{00AD}'), + b'_' => self.ch('\u{2011}'), + b'\\' | b'{' | b'}' => self.ch(s as char), + b'\n' if self.st.dest == Dest::Body => self.end_para(), + _ => {} + } + } + + fn commit_font(&mut self) { + let name = std::mem::take(&mut self.font_name); + let name = name.trim(); + let cp = self.font_cp_cur.take(); + if name.is_empty() || name.len() > MAX_FONT_NAME || (self.fonts.len() >= MAX_FONTS && !self.fonts.contains_key(&self.font_cur)) { + return; + } + self.fonts.insert(self.font_cur, name.to_string()); + if let Some(cp) = cp { + self.font_cps.insert(self.font_cur, cp); + } + } + + fn commit_color(&mut self) { + let (rgb, set) = std::mem::take(&mut self.color_cur); + if self.colors.len() < MAX_COLORS { + self.colors.push(set.then_some(rgb)); + } + } + + /// Ends the run being read: its text takes the formatting in effect now. + fn flush_run(&mut self) { + if self.text.is_empty() { + return; + } + let mut text = std::mem::take(&mut self.text); + // `\u0` and `\'00` are not text. + text.retain(|c| c != '\0'); + if text.is_empty() { + return; + } + match self.para.runs.last_mut() { + Some(last) if last.fmt == self.st.fmt => last.text.push_str(&text), + _ => self.para.runs.push(Run { text, fmt: self.st.fmt.clone() }), + } + } + + fn take_para(&mut self) -> Para { + self.decoder.finish(&mut self.text); + self.flush_run(); + let runs = std::mem::take(&mut self.para.runs); + Para { runs, ..self.st.para.clone() } + } + + /// The rows read so far become a table: something that is not a row follows them. + fn end_table(&mut self) { + if !self.rows.is_empty() { + self.doc.blocks.push(Block::Table(Table { rows: std::mem::take(&mut self.rows) })); + } + } + + fn end_para(&mut self) { + let para = self.take_para(); + if self.st.in_table || !self.cell.is_empty() { + self.cell.push(para); + } else { + self.end_table(); + self.doc.blocks.push(Block::Para(para)); + } + } + + fn end_cell(&mut self) { + let para = self.take_para(); + if !para.runs.is_empty() || self.cell.is_empty() { + self.cell.push(para); + } + self.row.push(Cell { right: 0, paras: std::mem::take(&mut self.cell) }); + } + + fn end_row(&mut self) { + let mut row = std::mem::take(&mut self.row); + for (cell, right) in row.iter_mut().zip(&self.cellx) { + cell.right = *right; + } + if !row.is_empty() { + self.rows.push(row); + } + } + + /// Changes the character formatting: the text read so far keeps the old one. + fn restyle(&mut self, f: impl FnOnce(&mut CharFmt)) { + self.flush_run(); + f(&mut self.st.fmt); + } + + fn word(&mut self, name: &str, param: Option, starred: bool) { + if name == "u" { + if let Some(n) = param { + let uc = self.st.uc; + let (decoder, out) = self.sink(); + decoder.unicode(n, uc, out); + } + return; + } + if self.decoder.skip() { + return; + } + if self.destination(name, starred) { + return; + } + let pv = param.unwrap_or(0); + match self.st.dest { + Dest::Skip => {} + Dest::ColorTbl => { + let v = pv.clamp(0, 255) as u8; + match name { + "red" => self.color_cur = ([v, self.color_cur.0[1], self.color_cur.0[2]], true), + "green" => self.color_cur = ([self.color_cur.0[0], v, self.color_cur.0[2]], true), + "blue" => self.color_cur = ([self.color_cur.0[0], self.color_cur.0[1], v], true), + _ => {} + } + } + Dest::FontTbl => match name { + "f" => { + self.font_cur = pv; + self.font_cpg_named = false; + } + "fcharset" => { + // The symbol charset doesn't replace a code page the entry named with `\cpg`. + let cp = charset_code_page(pv); + if !(self.font_cpg_named && cp == Some(SYMBOL_CODE_PAGE)) { + self.font_cp_cur = cp; + } + } + "cpg" => { + if let Some(cp) = u16::try_from(pv).ok().filter(|cp| is_text_code_page(*cp)) { + self.font_cp_cur = Some(cp); + self.font_cpg_named = true; + } + } + _ => {} + }, + Dest::Body => self.body_word(name, param), + } + } + + /// Does `name` start a destination? Then the group's text goes there. + fn destination(&mut self, name: &str, starred: bool) -> bool { + let dest = match name { + "fonttbl" => Dest::FontTbl, + "colortbl" => Dest::ColorTbl, + "pict" | "object" => { + if self.st.dest != Dest::Skip { + self.doc.pictures_left_out += 1; + } + Dest::Skip + } + // A field's result is its text; `\*\fldrslt` is not a destination to drop. + "fldrslt" => return true, + _ if SKIP.contains(&name) => Dest::Skip, + _ if starred => Dest::Skip, + _ => return false, + }; + self.flush_run(); + // Nothing inside a skipped destination comes back out of it. + if self.st.dest != Dest::Skip { + self.st.dest = dest; + } + true + } + + fn body_word(&mut self, name: &str, param: Option) { + let pv = param.unwrap_or(0); + let on = param != Some(0); + match name { + "par" | "sect" => self.end_para(), + "cell" => self.end_cell(), + "row" => self.end_row(), + "trowd" => self.cellx.clear(), + "cellx" => self.cellx.push(pv), + "intbl" => self.st.in_table = true, + "pard" => (self.st.para, self.st.in_table) = (Para::default(), false), + "outlinelevel" => self.st.para.outline_level = u8::try_from(pv).ok().filter(|l| *l <= 8), + "li" => self.st.para.left_indent = pv, + "ri" => self.st.para.right_indent = pv, + "fi" => self.st.para.first_line_indent = pv, + "sb" => self.st.para.space_before = pv, + "sa" => self.st.para.space_after = pv, + "ql" => self.st.para.align = Some(Align::Left), + "qc" => self.st.para.align = Some(Align::Center), + "qr" => self.st.para.align = Some(Align::Right), + "qj" => self.st.para.align = Some(Align::Justified), + "line" => self.ch(LINE_BREAK), + "tab" => self.ch('\t'), + "page" => self.ch(PAGE_BREAK), + "column" => self.ch(COLUMN_BREAK), + "emdash" => self.ch('\u{2014}'), + "endash" => self.ch('\u{2013}'), + "bullet" => self.ch('\u{2022}'), + "lquote" => self.ch('\u{2018}'), + "rquote" => self.ch('\u{2019}'), + "ldblquote" => self.ch('\u{201C}'), + "rdblquote" => self.ch('\u{201D}'), + "emspace" => self.ch('\u{2003}'), + "enspace" => self.ch('\u{2002}'), + "b" => self.restyle(|f| f.bold = on), + "i" => self.restyle(|f| f.italic = on), + "ul" => self.restyle(|f| f.underline = on), + "ulnone" => self.restyle(|f| f.underline = false), + "strike" => self.restyle(|f| f.strike = on), + "super" => self.restyle(|f| f.vert_align = if on { VertAlign::Superscript } else { VertAlign::Baseline }), + "sub" => self.restyle(|f| f.vert_align = if on { VertAlign::Subscript } else { VertAlign::Baseline }), + "nosupersub" => self.restyle(|f| f.vert_align = VertAlign::Baseline), + "fs" => self.restyle(|f| f.half_points = u32::try_from(pv).ok().filter(|s| *s > 0)), + "cf" => { + let color = usize::try_from(pv).ok().and_then(|i| self.colors.get(i).copied().flatten()); + self.restyle(|f| f.color = color); + } + "f" => { + let font = self.fonts.get(&pv).cloned(); + self.st.cp = self.font_cps.get(&pv).copied(); + self.st.font_set = true; + self.restyle(|f| f.font = font); + } + "plain" => { + // The default font is current again, and its code page with it. + self.st.cp = None; + self.st.font_set = false; + self.restyle(|f| *f = CharFmt::default()); + } + "uc" => self.st.uc = usize::try_from(pv).unwrap_or(0), + "ansicpg" => { + if let Some(cp) = u16::try_from(pv).ok().filter(|cp| is_text_code_page(*cp)) { + self.ansicpg = cp; + } + } + "mac" if self.ansicpg == 0 => self.ansicpg = MAC_ROMAN_CODE_PAGE, + "pc" if self.ansicpg == 0 => self.ansicpg = 437, + "pca" if self.ansicpg == 0 => self.ansicpg = 850, + "deff" => self.deff = pv, + _ => {} + } + } + + fn finish(mut self) -> Document { + let last = self.take_para(); + // A file cut inside a row keeps what the row had. + if !self.row.is_empty() || !self.cell.is_empty() { + if !last.runs.is_empty() || !self.cell.is_empty() { + self.cell.push(last); + self.row.push(Cell { right: 0, paras: std::mem::take(&mut self.cell) }); + } + self.end_row(); + self.end_table(); + } else { + self.end_table(); + if !last.runs.is_empty() { + self.doc.blocks.push(Block::Para(last)); + } + } + self.doc + } +} diff --git a/crates/rtf/src/text.rs b/crates/rtf/src/text.rs new file mode 100644 index 000000000..b1411842e --- /dev/null +++ b/crates/rtf/src/text.rs @@ -0,0 +1,252 @@ +//! The text of an RTF file: `\uN` with its fallback characters, and 8-bit text in a code page. + +use crate::codepage::{code_page_byte, multibyte_encoding_name}; + +fn multibyte(code_page: u16) -> Option<&'static encoding_rs::Encoding> { + encoding_rs::Encoding::for_label(multibyte_encoding_name(code_page)?.as_bytes()) +} + +/// Decodes the text tokens of one RTF file into `String`s the caller supplies. +/// +/// The caller says which code page applies to each piece of 8-bit text (the current font's, +/// else the default font's, else the document's) and how many fallback characters follow a +/// `\uN` (`\ucN`, 1 when not set). +/// +/// - `\uN` is one UTF-16 code unit. A high surrogate pairs only with a low surrogate in the +/// very next `\u`; every surrogate that does not pair reads as U+FFFD, as in Word. +/// - After a `\uN`, the next `\ucN` characters are its fallback and are skipped: a text byte, +/// a `\'hh`, a control word and a control symbol each count as one ([`TextDecoder::skip`]). +/// The skipping ends with the group ([`TextDecoder::group`]). +/// - A single-byte code page maps each byte. A multibyte one (932, 936, 949, 950) streams +/// through one decoder, so a lead byte and its trail byte decode together even when a +/// `\'hh`, a text run or a formatting word falls between them, and a run that happens to +/// start `EF BB BF` is not taken for a UTF-8 byte order mark. A lead byte without its trail +/// byte reads as U+FFFD. +#[derive(Default)] +pub struct TextDecoder { + skip: usize, + high: Option, + mb: Option, + mb_cp: u16, +} + +impl TextDecoder { + pub fn new() -> TextDecoder { + TextDecoder::default() + } + + /// A held high surrogate that nothing paired with. + fn flush_high(&mut self, out: &mut String) { + if self.high.take().is_some() { + out.push('\u{FFFD}'); + } + } + + /// `\uN` with `uc` fallback characters to skip after it. + pub fn unicode(&mut self, param: i32, uc: usize, out: &mut String) { + self.finish_multibyte(out); + let u = i64::from(param).rem_euclid(65_536) as u16; + if (0xD800..0xDC00).contains(&u) { + self.flush_high(out); + self.high = Some(u); + } else { + match self.high.take() { + Some(h) if (0xDC00..0xE000).contains(&u) => out.push(char::decode_utf16([h, u]).next().and_then(Result::ok).unwrap_or('\u{FFFD}')), + Some(_) => { + out.push('\u{FFFD}'); + out.push(char::from_u32(u32::from(u)).unwrap_or('\u{FFFD}')); + } + None => out.push(char::from_u32(u32::from(u)).unwrap_or('\u{FFFD}')), + } + } + self.skip = uc; + } + + /// Is the control word or symbol just read a fallback character of the last `\uN`? Then it + /// is used up and the caller ignores it. + pub fn skip(&mut self) -> bool { + let skipped = self.skip > 0; + self.skip = self.skip.saturating_sub(1); + skipped + } + + /// A character a control word or symbol stands for (`\emdash`, `\~`). + pub fn char(&mut self, c: char, out: &mut String) { + self.finish_multibyte(out); + self.flush_high(out); + out.push(c); + } + + /// `\'hh`: one byte in `code_page`. + pub fn byte(&mut self, b: u8, code_page: u16, out: &mut String) { + if !self.skip() { + self.decode(&[b], code_page, out); + } + } + + /// A run of text bytes in `code_page`. + pub fn bytes(&mut self, chunk: &[u8], code_page: u16, out: &mut String) { + let skip = self.skip.min(chunk.len()); + self.skip -= skip; + let chunk = chunk.get(skip..).unwrap_or_default(); + if !chunk.is_empty() { + self.decode(chunk, code_page, out); + } + } + + fn decode(&mut self, chunk: &[u8], cp: u16, out: &mut String) { + self.flush_high(out); + if self.mb.is_none() && chunk.is_ascii() { + // ASCII is itself in every code page here; with no lead byte pending, as it is. + out.push_str(std::str::from_utf8(chunk).unwrap_or_default()); + return; + } + let Some(encoding) = multibyte(cp) else { + self.finish_multibyte(out); + out.extend(chunk.iter().map(|b| code_page_byte(cp, *b))); + return; + }; + // A code page switch ends the previous run before the new decoder starts. + if self.mb_cp != cp { + self.finish_multibyte(out); + } + self.mb_cp = cp; + let decoder = self.mb.get_or_insert_with(|| encoding.new_decoder_without_bom_handling()); + let mut rest = chunk; + loop { + out.reserve(rest.len().saturating_mul(3).saturating_add(16)); + let (result, read, _) = decoder.decode_to_string(rest, out, false); + rest = rest.get(read..).unwrap_or_default(); + if !matches!(result, encoding_rs::CoderResult::OutputFull) { + break; + } + } + } + + /// Ends the current multibyte run: a lead byte still waiting for its trail byte becomes + /// U+FFFD. + fn finish_multibyte(&mut self, out: &mut String) { + if let Some(mut decoder) = self.mb.take() { + out.reserve(16); + let _ = decoder.decode_to_string(&[], out, true); + } + } + + /// A group opened or closed: the fallback of a `\uN` ends with its group, and so does a + /// multibyte run. + pub fn group(&mut self, out: &mut String) { + self.finish_multibyte(out); + self.skip = 0; + } + + /// The text ends here (a paragraph end, a cell end, the end of the file): nothing waits + /// for a partner any longer. + pub fn finish(&mut self, out: &mut String) { + self.finish_multibyte(out); + self.flush_high(out); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn units(ns: &[i32], uc: usize) -> String { + let (mut d, mut out) = (TextDecoder::new(), String::new()); + for n in ns { + d.unicode(*n, uc, &mut out); + } + d.finish(&mut out); + out + } + + #[test] + fn surrogates_pair_only_with_the_next_unit() { + assert_eq!(units(&[-10179, -8704], 0), "\u{1F600}"); + assert_eq!(units(&[0xD83D, 0xDE00], 0), "\u{1F600}", "written unsigned"); + assert_eq!(units(&[8364], 0), "€"); + assert_eq!(units(&[-10179], 0), "\u{FFFD}", "a lone high surrogate at the end"); + assert_eq!(units(&[-8704], 0), "\u{FFFD}", "a lone low surrogate"); + assert_eq!(units(&[-8704, -10179], 0), "\u{FFFD}\u{FFFD}", "low then high"); + assert_eq!(units(&[-10179, -10179, -8704], 0), "\u{FFFD}\u{1F600}", "a high surrogate before a pair"); + assert_eq!(units(&[-10179, 65, -8704], 0), "\u{FFFD}A\u{FFFD}"); + assert_eq!(units(&[65536 + 65, -65536 + 66], 0), "AB", "taken modulo 65536"); + } + + #[test] + fn a_held_high_surrogate_does_not_pair_across_text() { + let (mut d, mut out) = (TextDecoder::new(), String::new()); + d.unicode(-10179, 0, &mut out); + d.bytes(b"x", 1252, &mut out); + d.unicode(-8704, 0, &mut out); + d.unicode(-10179, 0, &mut out); + d.char('\u{2014}', &mut out); + d.unicode(-10179, 1, &mut out); + // The fallback and a formatting group do not come between a pair. + d.bytes(b"?", 1252, &mut out); + d.group(&mut out); + d.unicode(-8704, 1, &mut out); + d.finish(&mut out); + assert_eq!(out, "\u{FFFD}x\u{FFFD}\u{FFFD}\u{2014}\u{1F600}"); + } + + #[test] + fn fallback_characters_are_skipped() { + let (mut d, mut out) = (TextDecoder::new(), String::new()); + d.unicode(8364, 1, &mut out); + d.bytes(b"? price", 1252, &mut out); + d.unicode(8220, 2, &mut out); + d.byte(0x93, 1252, &mut out); + assert!(d.skip(), "a control word is one fallback character"); + assert!(!d.skip()); + d.unicode(233, 3, &mut out); + d.bytes(b"e", 1252, &mut out); + d.group(&mut out); + d.bytes(b"AB", 1252, &mut out); + assert_eq!(out, "\u{20AC} price\u{201C}\u{E9}AB", "the skip ends with the group"); + } + + #[test] + fn single_byte_code_pages() { + let (mut d, mut out) = (TextDecoder::new(), String::new()); + d.bytes(&[b'C', b'a', b'f', 0xE9, b' '], 1252, &mut out); + d.byte(0xC4, 1251, &mut out); + d.byte(0x8E, 10000, &mut out); + d.byte(0x82, 437, &mut out); + d.byte(0xE9, 9999, &mut out); + assert_eq!(out, "Café Дééé"); + } + + #[test] + fn a_lead_byte_and_its_trail_byte_decode_together() { + // 日本 in Shift-JIS is 93 FA 96 7B: the first character as two `\'hh`, the second as a + // `\'hh` lead byte and a raw trail byte (`{`'s neighbour 0x7B is a text byte here). + let (mut d, mut out) = (TextDecoder::new(), String::new()); + d.byte(0x93, 932, &mut out); + d.byte(0xFA, 932, &mut out); + d.byte(0x96, 932, &mut out); + d.bytes(b"\x7bA", 932, &mut out); + assert_eq!(out, "日本A"); + // A lead byte left at a group end, a code page switch or the end is U+FFFD. + for end in 0..3 { + let (mut d, mut out) = (TextDecoder::new(), String::new()); + d.byte(0x93, 932, &mut out); + match end { + 0 => d.group(&mut out), + 1 => d.bytes(&[0xE9], 1252, &mut out), + _ => d.finish(&mut out), + } + assert!(out.starts_with('\u{FFFD}'), "{end}: {out:?}"); + } + // GBK text that starts EF BB BF is not a UTF-8 byte order mark. + let (mut d, mut out) = (TextDecoder::new(), String::new()); + d.bytes(&[0xEF, 0xBB, 0xBF, 0xC4, 0xE3], 936, &mut out); + d.finish(&mut out); + assert_eq!(out.chars().count(), 3, "{out:?}"); + assert!(out.ends_with('\u{FFFD}') && !out.starts_with('\u{FEFF}'), "{out:?}"); + // A long run decodes whole. + let (mut d, mut out) = (TextDecoder::new(), String::new()); + d.bytes(&[0x93, 0xFA].repeat(10_000), 932, &mut out); + assert_eq!(out.chars().count(), 10_000); + } +} diff --git a/crates/rtf/src/write.rs b/crates/rtf/src/write.rs new file mode 100644 index 000000000..0b8f1dc8d --- /dev/null +++ b/crates/rtf/src/write.rs @@ -0,0 +1,198 @@ +//! The model as RTF, and the escaping every RTF writer needs. + +use std::fmt::Write; + +use crate::model::{Align, Block, COLUMN_BREAK, CharFmt, Document, LINE_BREAK, PAGE_BREAK, Para, VertAlign}; + +/// RTF-escapes text: `\`, `{` and `}` escaped; tab, line, page and column breaks, the +/// non-breaking space, the soft hyphen and the non-breaking hyphen as their control words; +/// everything past ASCII as `\uN?` (UTF-16 code units as signed 16-bit numbers, with a `?` +/// fallback, so the file needs `\uc1`, the default). Other ASCII control characters are +/// dropped. U+000B and U+2028 are line breaks too. +pub fn esc(s: &str) -> String { + let mut o = String::with_capacity(s.len() + 8); + push_esc(&mut o, s); + o +} + +/// [`esc`], appended to `o`. +pub fn push_esc(o: &mut String, s: &str) { + for c in s.chars() { + match c { + '\\' => o.push_str("\\\\"), + '{' => o.push_str("\\{"), + '}' => o.push_str("\\}"), + '\t' => o.push_str("\\tab "), + LINE_BREAK | '\u{000B}' | '\u{2028}' => o.push_str("\\line "), + PAGE_BREAK => o.push_str("\\page "), + COLUMN_BREAK => o.push_str("\\column "), + '\u{00A0}' => o.push_str("\\~"), + '\u{00AD}' => o.push_str("\\-"), + '\u{2011}' => o.push_str("\\_"), + c if c.is_ascii_control() => {} + c if c.is_ascii() => o.push(c), + c => { + let mut buf = [0u16; 2]; + for u in c.encode_utf16(&mut buf) { + // Writing to a String never fails. + let _ = write!(o, "\\u{}?", *u as i16); + } + } + } + } +} + +/// [`esc`] for a `\fonttbl` name: `;` would end the entry, so it is written `\'3b`, and control +/// characters, which [`esc`] writes as `\line` or `\tab`, are dropped. +pub fn esc_font(name: &str) -> String { + esc(&name.replace(char::is_control, "")).replace(';', "\\'3b") +} + +/// The fonts and colours a document uses, in the order of their tables. Font 0 is the default +/// font; colour 0 is the default (auto) colour, so the first real colour is `\cf1`. +struct Tables { + fonts: Vec, + colors: Vec<[u8; 3]>, +} + +impl Tables { + fn of(doc: &Document, default_font: &str) -> Tables { + let mut t = Tables { fonts: vec![default_font.to_string()], colors: Vec::new() }; + for fmt in doc.paras().flat_map(|p| &p.runs).map(|r| &r.fmt) { + if let Some(f) = fmt.font.as_deref().filter(|f| t.font(f).is_none()) { + t.fonts.push(f.to_string()); + } + if let Some(c) = fmt.color.filter(|c| t.color(*c).is_none()) { + t.colors.push(c); + } + } + t + } + fn font(&self, name: &str) -> Option { + self.fonts.iter().position(|f| f.eq_ignore_ascii_case(name)) + } + fn color(&self, c: [u8; 3]) -> Option { + self.colors.iter().position(|k| *k == c).map(|i| i + 1) + } + + fn fmt_words(&self, f: &CharFmt) -> String { + let mut w = String::new(); + for (on, word) in [(f.bold, "\\b"), (f.italic, "\\i"), (f.underline, "\\ul"), (f.strike, "\\strike")] { + if on { + w.push_str(word); + } + } + match f.vert_align { + VertAlign::Superscript => w.push_str("\\super"), + VertAlign::Subscript => w.push_str("\\sub"), + VertAlign::Baseline => {} + } + // Writing to a String never fails. + if let Some(i) = f.font.as_deref().and_then(|n| self.font(n)).filter(|i| *i > 0) { + let _ = write!(w, "\\f{i}"); + } + if let Some(s) = f.half_points { + let _ = write!(w, "\\fs{s}"); + } + if let Some(i) = f.color.and_then(|c| self.color(c)) { + let _ = write!(w, "\\cf{i}"); + } + w + } + + /// A paragraph's runs: a formatted run is a group, so its formatting ends with it. + fn runs(&self, p: &Para, o: &mut String) { + for r in &p.runs { + let words = self.fmt_words(&r.fmt); + if words.is_empty() { + push_esc(o, &r.text); + } else { + o.push('{'); + o.push_str(&words); + o.push(' '); + push_esc(o, &r.text); + o.push('}'); + } + } + } + + fn para_words(p: &Para, o: &mut String) { + if let Some(l) = p.outline_level.filter(|l| *l <= 8) { + let _ = write!(o, "\\outlinelevel{l}"); + } + match p.align { + Some(Align::Left) => o.push_str("\\ql"), + Some(Align::Center) => o.push_str("\\qc"), + Some(Align::Right) => o.push_str("\\qr"), + Some(Align::Justified) => o.push_str("\\qj"), + None => {} + } + for (word, twips) in + [("li", p.left_indent), ("ri", p.right_indent), ("fi", p.first_line_indent), ("sb", p.space_before), ("sa", p.space_after)] + { + if twips != 0 { + let _ = write!(o, "\\{word}{twips}"); + } + } + } +} + +/// Writes a document as RTF. `default_font` is the font of text that names none. +/// +/// The file is 7-bit: `\ansi\ansicpg1252\uc1`, a font table, a colour table when a run has a +/// colour, then one `\pard … \par` per paragraph and `\trowd … \row` per table row. What +/// [`read`](crate::read) reads from it is `doc` again, except that +/// [`Document::pictures_left_out`] is 0, a font equal to `default_font` reads as no font, +/// neighbouring runs of one formatting are one run, and empty runs, an empty last paragraph of a +/// cell and rows without cells are gone. +pub fn write(doc: &Document, default_font: &str) -> String { + let t = Tables::of(doc, default_font); + let mut o = String::from("{\\rtf1\\ansi\\ansicpg1252\\deff0\\uc1{\\fonttbl"); + for (i, f) in t.fonts.iter().enumerate() { + let _ = write!(o, "{{\\f{i}\\fnil {};}}", esc_font(f)); + } + o.push_str("}\n"); + if !t.colors.is_empty() { + o.push_str("{\\colortbl;"); + for [r, g, b] in &t.colors { + let _ = write!(o, "\\red{r}\\green{g}\\blue{b};"); + } + o.push_str("}\n"); + } + for block in &doc.blocks { + match block { + Block::Para(p) => { + o.push_str("\\pard"); + Tables::para_words(p, &mut o); + o.push(' '); + t.runs(p, &mut o); + o.push_str("\\par\n"); + } + Block::Table(table) => { + for row in table.rows.iter().filter(|r| !r.is_empty()) { + o.push_str("\\trowd\\trgaph108"); + for cell in row { + let _ = write!(o, "\\cellx{}", cell.right); + } + for cell in row { + for (i, p) in cell.paras.iter().enumerate() { + if i > 0 { + o.push_str("\\par"); + } + o.push_str("\\pard\\intbl"); + Tables::para_words(p, &mut o); + o.push(' '); + t.runs(p, &mut o); + } + o.push_str("\\cell"); + } + o.push_str("\\row\n"); + } + // The paragraph after a table is not in it. + o.push_str("\\pard"); + } + } + } + o.push('}'); + o +} diff --git a/crates/rtf/tests/read_write.rs b/crates/rtf/tests/read_write.rs new file mode 100644 index 000000000..9ef8fbfb7 --- /dev/null +++ b/crates/rtf/tests/read_write.rs @@ -0,0 +1,444 @@ +//! The reader and writer against the cases the apps' RTF code was tested with: DesignCraft's +//! text import, DeckCraft's outline import and export, PdfCraft's export, and WordCraft's +//! reader for code pages and `\u`. + +use craft_rtf::{ + Align, Block, COLUMN_BREAK, Cell, CharFmt, Document, LINE_BREAK, MAX_COLORS, MAX_FONT_NAME, MAX_FONTS, MAX_GROUPS, NotRtf, PAGE_BREAK, Para, Run, + Table, VertAlign, read, write, +}; + +fn text(rtf: &[u8]) -> String { + read(rtf).unwrap().paras().map(Para::text).collect::>().join("\n") +} + +fn paras(rtf: &[u8]) -> Vec { + read(rtf).unwrap().paras().cloned().collect() +} + +/// The formatting of the run that holds `needle`. +fn fmt_of(doc: &Document, needle: &str) -> CharFmt { + doc.paras().flat_map(|p| &p.runs).find(|r| r.text.contains(needle)).map(|r| r.fmt.clone()).unwrap() +} + +fn para(s: &str) -> Para { + Para { runs: vec![Run { text: s.into(), fmt: CharFmt::default() }], ..Para::default() } +} + +#[test] +fn not_rtf_is_refused() { + assert_eq!(read(b"PK\x03\x04"), Err(NotRtf)); + assert_eq!(read(b""), Err(NotRtf)); + assert!(read(b" \n{\\rtf1 a}").is_ok()); + assert!(NotRtf.to_string().contains("{\\rtf")); +} + +#[test] +fn text_and_formatting() { + let rtf = br"{\rtf1\ansi{\fonttbl{\f0 Times;}}{\colortbl;\red0\green0\blue0;}{\*\generator Test;} +\f0\fs24 Hello {\b bold} and {\i italic\i0 plain}\par +Caf\'e9 \u8364? price\tab tab\line next\par +}"; + let doc = read(rtf).unwrap(); + assert_eq!(text(rtf), "Hello bold and italic plain\nCafé € price\ttab\nnext"); + assert_eq!(doc.blocks.len(), 2, "a trailing \\par leaves no empty paragraph"); + assert!(fmt_of(&doc, "bold").bold && !fmt_of(&doc, "bold").italic); + assert!(fmt_of(&doc, "italic").italic); + let plain = fmt_of(&doc, " plain"); + assert!(!plain.italic && !plain.bold); + assert_eq!(fmt_of(&doc, "Hello").half_points, Some(24)); + assert_eq!(fmt_of(&doc, "Hello").font.as_deref(), Some("Times")); + assert!(!text(rtf).contains("Times") && !text(rtf).contains("Test"), "tables and starred destinations are not text"); +} + +#[test] +fn character_words() { + let doc = read(br"{\rtf1{\colortbl;\red255\green0\blue0;\red0\green0\blue999;}\ul a\ulnone b\strike c\strike0 d\cf1 e\cf2 f\cf0 g\cf9 h\b\i\plain i\fs0 j\fs-8 k}").unwrap(); + assert!(fmt_of(&doc, "a").underline && !fmt_of(&doc, "b").underline); + assert!(fmt_of(&doc, "c").strike && !fmt_of(&doc, "d").strike); + assert_eq!(fmt_of(&doc, "e").color, Some([255, 0, 0])); + assert_eq!(fmt_of(&doc, "f").color, Some([0, 0, 255]), "a component is clamped"); + assert_eq!(fmt_of(&doc, "g").color, None, "colour 0 is the default colour"); + assert_eq!(fmt_of(&doc, "h").color, None, "a colour the table does not have"); + assert_eq!(fmt_of(&doc, "i"), CharFmt::default(), "\\plain"); + assert_eq!((fmt_of(&doc, "j").half_points, fmt_of(&doc, "k").half_points), (None, None)); +} + +#[test] +fn special_characters_and_breaks() { + assert_eq!( + text(br"{\rtf1 a\~b\-c\_d\\e\{f\}g\emdash h\endash i\bullet j\lquote k\rquote l\ldblquote m\rdblquote}"), + "a\u{A0}b\u{AD}c\u{2011}d\\e{f}g—h–i•j‘k’l“m”" + ); + let breaks = format!("a{PAGE_BREAK}b{COLUMN_BREAK}c{LINE_BREAK}d\te"); + assert_eq!(text(br"{\rtf1 a\page b\column c\line d\tab e}"), breaks); + // A backslash before a line end is a paragraph end; a bare line end is nothing. + assert_eq!(text(b"{\\rtf1 a\\\nb\r\nc\\sect d}"), "a\nbc\nd"); +} + +#[test] +fn header_and_footer_variants_and_notes_are_skipped() { + for w in ["header", "headerl", "headerr", "headerf", "footer", "footerl", "footerr", "footerf", "footnote", "annotation", "stylesheet", "info"] { + let rtf = format!(r"{{\rtf1\ansi {{\{w} Page head\par}}Body text\par}}"); + assert_eq!(text(rtf.as_bytes()), "Body text", "{w}"); + } + // Nothing inside a skipped destination comes back out of it, whatever it nests. + assert_eq!(text(br"{\rtf1 {\header {\b {\fonttbl x} Head}}{\*\unknown {\field{\fldrslt hidden}}}Body}"), "Body"); +} + +#[test] +fn a_fields_result_is_its_text() { + assert_eq!(text(br#"{\rtf1 See {\field{\*\fldinst HYPERLINK "https://example.com"}{\fldrslt {\ul the site}}}.}"#), "See the site."); + assert_eq!(text(br"{\rtf1 {\field{\*\fldinst PAGE}{\*\fldrslt 3}} of 9}"), "3 of 9"); +} + +#[test] +fn pictures_and_objects_are_counted_not_read() { + let doc = read( + br"{\rtf1 a{\pict\pngblip 89504e47}b{\object\objemb{\*\objdata 0105}}{\*\shppict{\pict\pngblip 00}}{\nonshppict{\pict\wmetafile8 00}}c}", + ) + .unwrap(); + assert_eq!(doc.pictures_left_out, 3, "a picture inside a skipped destination is not counted again"); + assert_eq!(doc.paras().map(Para::text).collect::(), "abc"); + assert_eq!(text(b"{\\rtf1 a{\\pict\\bin4 }}{\\x}b}"), "ab", "binary data is not scanned for braces"); +} + +#[test] +fn out_of_range_unicode_wraps_and_hyphens_are_text() { + assert_eq!(text(br"{\rtf1 a\u4294967361?b}"), "a\u{FFFF}b", "clamped to i32, then modulo 65536"); + assert_eq!(text(br"{\rtf1\ansi Rock\emdash-roll, x\tab-y, a\li-720 b}"), "Rock\u{2014}-roll, x\t-y, ab"); + assert_eq!(text(br"{\rtf1 {\b A\'4}B}"), "AB"); + assert_eq!(text(br"{\rtf1 a\'+fb}"), "a+fb"); +} + +#[test] +fn unicode_fallbacks() { + assert_eq!(text(br"{\rtf1 \u8220\ldblquote Hi\u8221\rdblquote x\u8212\emdash y}"), "\u{201C}Hi\u{201D} x\u{2014}y"); + assert_eq!(text(br"{\rtf1 \u8220\{Hi}"), "\u{201C}Hi", "a control symbol is one fallback character"); + assert_eq!(text(br"{\rtf1 \u8220\'93Hi}"), "\u{201C}Hi"); + assert_eq!(text(br"{\rtf1 \u8220{\b Hi}}"), "\u{201C}Hi", "a group is not a character"); + assert_eq!(text(br"{\rtf1 {\b \u8364} AB}"), "\u{20AC} AB", "the skip ends with the group"); + assert_eq!(text(br"{\rtf1 {\uc2\b \u8364} ABC}"), "\u{20AC} ABC"); + assert_eq!(text(br"{\rtf1\uc0 \u8364 5\uc2 \u8364xy5}"), "\u{20AC}5\u{20AC}5"); + assert_eq!(text(br"{\rtf1 \u8364\u8364?5}"), "\u{20AC}\u{20AC}5", "a \\u is never a fallback character"); + assert_eq!(text(br"{\rtf1 {\header \u8364}?x}"), "?x", "a \\u in a skipped destination leaves nothing to skip outside it"); +} + +#[test] +fn nul_is_not_text() { + assert_eq!(text(br"{\rtf1 \u0?abc\par a\'00b\par {\b\'00}}"), "abc\nab"); + assert_eq!(text(b"{\\rtf1 a\0b\\par}\0\0"), "ab"); +} + +#[test] +fn surrogate_pairs() { + assert_eq!(text(br"{\rtf1\ansi\uc1 A\u-10179?\u-8704?B \u-10174?\u-8265?\u30000?}"), "A\u{1F600}B \u{20BB7}\u{7530}"); + assert_eq!(text(br"{\rtf1 A\u-10179?B\u-8704?C}"), "A\u{FFFD}B\u{FFFD}C", "half a pair is U+FFFD, as in Word"); + assert_eq!(text(br"{\rtf1 A\u-10179?\par\u-8704?C}"), "A\u{FFFD}\n\u{FFFD}C", "a pair does not span a paragraph end"); + assert_eq!(text(br"{\rtf1 A\u-10179?}"), "A\u{FFFD}"); +} + +#[test] +fn code_pages() { + assert_eq!(text(br"{\rtf1\ansi\ansicpg1251 \'cf\'f0\'e8\'e2\'e5\'f2}"), "Привет"); + assert_eq!(text(b"{\\rtf1\\ansi\\ansicpg1251 \xcf\xf0\xe8\xe2\xe5\xf2}"), "Привет", "raw bytes are text in the code page"); + assert_eq!(text(br"{\rtf1\mac Caf\'8e}"), "Café"); + assert_eq!(text(br"{\rtf1\pc Caf\'82}"), "Café"); + assert_eq!(text(br"{\rtf1\pca \'9d}"), "Ø"); + assert_eq!(text(br"{\rtf1\ansi Caf\'e9 \'80}"), "Café €", "Windows-1252 when the file names none"); + assert_eq!(text(br"{\rtf1\ansi\ansicpg9999 Caf\'e9}"), "Café", "an unknown code page is not taken"); + // The font's charset decides, then the default font's, then the document's. + let rtf = br"{\rtf1\ansi\ansicpg1252\deff1{\fonttbl{\f0\fcharset204 Arial Cyr;}{\f1\fcharset161 Greek;}{\f2\cpg1250 CE;}{\f3 Plain;}}\'e1 \f0\'e1 \f2\'b3 \f3\'e1 \plain\'e1}"; + assert_eq!(text(rtf), "α б ł α α"); +} + +/// A symbol font (`\fcharset2`: Symbol, Wingdings) has no letters: a byte from 0x80 is the font's +/// own code, U+F000 + the byte, whatever the document's code page. +#[test] +fn a_symbol_fonts_bytes_are_its_own_codes() { + for cpg in [1252, 1251] { + let rtf = format!(r"{{\rtf1\ansi\ansicpg{cpg}{{\fonttbl{{\f0 Arial;}}{{\f1\fcharset2 Wingdings;}}}}\f1 J\'93\'d8\f0 \'93\f1\'fc\plain\'93}}"); + assert_eq!(text(rtf.as_bytes()), "J\u{F093}\u{F0D8}\u{201C}\u{F0FC}\u{201C}", "{cpg}"); + } + // The symbol page is not for the font's own name in the table. + let doc = read(br"{\rtf1{\fonttbl{\f0\fcharset2 Sym\'e9bol;}}\f0 x}").unwrap(); + assert_eq!(fmt_of(&doc, "x").font.as_deref(), Some("Symébol")); + // A symbol default font's page is its own text's: not the text of a font chosen with \f, + // and its again after \plain. + let rtf = br"{\rtf1\ansi\deff0{\fonttbl{\f0\fcharset2 Symbol;}{\f1 Arial;}}\'b7 \f1\'b7 \plain\'b7}"; + assert_eq!(text(rtf), "\u{F0B7} \u{B7} \u{F0B7}"); + // A code page the entry names with \cpg is not replaced by the symbol charset. + let rtf = br"{\rtf1{\fonttbl{\f0\cpg1251\fcharset2 Odd;}{\f1\fcharset2\cpg1251 Odd2;}}\f0\'c4\f1\'c4}"; + assert_eq!(text(rtf), "\u{414}\u{414}"); +} + +#[test] +fn east_asian_text_decodes_across_tokens() { + // 日本語 in Shift-JIS: 93 FA 96 7B 8C EA. 0x7B is `{` as a byte, so it must be `\'7b`. + assert_eq!(text(br"{\rtf1\ansi\ansicpg932 \'93\'fa\'96\'7b\'8c\'ea}"), "日本語"); + assert_eq!(text(b"{\\rtf1\\ansi\\ansicpg932 \x93\xfa\\'96\\'7b\x8c\xea ok}"), "日本語 ok"); + assert_eq!(text(br"{\rtf1{\fonttbl{\f0\fcharset134 SimSun;}}\f0\'c4\'e3\'ba\'c3}"), "你好"); + assert_eq!(text(br"{\rtf1\ansicpg932 a\'93}b}"), "a\u{FFFD}b", "a lead byte left at a group end"); + assert_eq!(text(br"{\rtf1\ansicpg932 a\'93\par b}"), "a\u{FFFD}\nb"); +} + +#[test] +fn font_table_forms() { + // Entries as groups or bare, `\'3b` for a `;` in a name, an alternative name in a starred + // destination, an entry without its `;`. + let rtf = + br"{\rtf1{\fonttbl\f0\froman Times New Roman;\f1 A\'3bB;{\f2\fnil{\*\falt Other}DejaVu Sans;}{\f3 NoSemicolon}}\f0 a\f1 b\f2 c\f3 d\f9 e}"; + let doc = read(rtf).unwrap(); + let font = |s: &str| fmt_of(&doc, s).font; + assert_eq!(font("a").as_deref(), Some("Times New Roman")); + assert_eq!(font("b").as_deref(), Some("A;B")); + assert_eq!(font("c").as_deref(), Some("DejaVu Sans")); + assert_eq!(font("d").as_deref(), Some("NoSemicolon")); + assert_eq!(font("e"), None, "a font the table does not have"); +} + +#[test] +fn outline_levels_and_indents() { + let rtf = br"{\rtf1\ansi{\fonttbl{\f0 Arial;}}{\*\generator X;}\pard\outlinelevel0\b Title one\b0\par\pard\li720 Point \'93a\'94\par\pard\outlinelevel2\li1440 Sub\par\outlinelevel9 Deep\par\pard\li-720 Back\par}"; + let got: Vec<(String, Option, i32)> = paras(rtf).iter().map(|p| (p.text(), p.outline_level, p.left_indent)).collect(); + assert_eq!( + got, + [ + ("Title one".to_string(), Some(0), 0), + ("Point “a”".to_string(), None, 720), + ("Sub".to_string(), Some(2), 1440), + ("Deep".to_string(), None, 1440), + ("Back".to_string(), None, -720) + ] + ); + let huge = paras(br"{\rtf1\pard\li9223372036854775807 deep\par\pard\li-9223372036854775808 flat\par}"); + assert!(huge[0].left_indent > 1_000_000_000 && huge[1].left_indent <= 0, "{huge:?}"); +} + +/// Word writes a list item's bullet as `{\listtext\pard\plain …\'b7\tab}` (older writers: +/// `{\pntext …}`) after the paragraph's own `\li`. That group is the rendered bullet, not text. +#[test] +fn list_labels_are_not_text_and_keep_the_indent() { + let rtf = br"{\rtf1\ansi\pard Title\par\pard\fi-360\li720{\listtext\pard\plain\f1\'b7\tab}Point\par\pard\li720{\pntext\f1\'b7\tab}Old\par}"; + let got: Vec<(String, i32)> = paras(rtf).iter().map(|p| (p.text(), p.left_indent)).collect(); + assert_eq!(got, [("Title".to_string(), 0), ("Point".to_string(), 720), ("Old".to_string(), 720)]); +} + +#[test] +fn alignment_indents_spacing_and_position() { + let rtf = br"{\rtf1\pard\qc\li720\ri360\fi-360\sb120\sa240 x\super 2\sub i\nosupersub y\super z\super0 w\par\pard\qr r\par\pard\ql l\par\pard\qj j\par\pard n\par}"; + let doc = read(rtf).unwrap(); + let ps: Vec<&Para> = doc.paras().collect(); + assert_eq!((ps[0].left_indent, ps[0].right_indent, ps[0].first_line_indent, ps[0].space_before, ps[0].space_after), (720, 360, -360, 120, 240)); + assert_eq!( + ps.iter().map(|p| p.align).collect::>(), + [Some(Align::Center), Some(Align::Right), Some(Align::Left), Some(Align::Justified), None] + ); + assert_eq!((ps[4].left_indent, ps[4].space_after), (0, 0), "\\pard resets them"); + let va = |s: &str| fmt_of(&doc, s).vert_align; + assert_eq!( + (va("x"), va("2"), va("i"), va("y"), va("z"), va("w")), + (VertAlign::Baseline, VertAlign::Superscript, VertAlign::Subscript, VertAlign::Baseline, VertAlign::Superscript, VertAlign::Baseline) + ); +} + +#[test] +fn paragraph_properties_end_with_their_group() { + let got: Vec = paras(br"{\rtf1 {\li720 a\par}b\par{\pard\li360 c}\par}").iter().map(|p| p.left_indent).collect(); + assert_eq!(got, [720, 0, 0]); +} + +#[test] +fn empty_paragraphs_are_kept_except_the_last() { + let doc = read(br"{\rtf1 a\par\par b\par\par}").unwrap(); + assert_eq!(doc.blocks.len(), 4); + assert_eq!(read(br"{\rtf1 }").unwrap().blocks.len(), 0); + assert_eq!(read(br"{\rtf1 a}").unwrap().blocks.len(), 1); +} + +#[test] +fn tables() { + let rtf = br"{\rtf1 Before\par\trowd\trgaph108\cellx1440\cellx4320{\intbl\fs20 A1}\cell{\intbl\b B1}\cell\row +\trowd\cellx1440\cellx2880\cellx4320\pard\intbl A2\par second\cell\cell C2\cell\row\pard After\par\trowd\cellx100 X\cell\row}"; + let doc = read(rtf).unwrap(); + let [Block::Para(before), Block::Table(t), Block::Para(after), Block::Table(last)] = doc.blocks.as_slice() else { panic!("{doc:?}") }; + assert_eq!((before.text(), after.text()), ("Before".to_string(), "After".to_string())); + let cells = |row: &[Cell]| row.iter().map(|c| (c.right, c.paras.iter().map(Para::text).collect::>().join("|"))).collect::>(); + assert_eq!(cells(&t.rows[0]), [(1440, "A1".to_string()), (4320, "B1".to_string())]); + assert_eq!(cells(&t.rows[1]), [(1440, "A2|second".to_string()), (2880, String::new()), (4320, "C2".to_string())]); + assert!(t.rows[0][1].paras[0].runs[0].fmt.bold); + assert_eq!(cells(&last.rows[0]), [(100, "X".to_string())]); + // Reading order. + assert_eq!(doc.paras().map(Para::text).collect::>().join(" "), "Before A1 B1 A2 second C2 After X"); + // A file cut inside a row keeps what the row had. + let cut = read(br"{\rtf1\trowd\cellx100\intbl A\cell B").unwrap(); + let [Block::Table(t)] = cut.blocks.as_slice() else { panic!("{cut:?}") }; + assert_eq!(cells(&t.rows[0]), [(100, "A".to_string()), (0, "B".to_string())]); +} + +#[test] +fn groups_nested_past_the_limit_are_read_flat() { + let deep = MAX_GROUPS * 4; + let mut rtf = String::from("{\\rtf1 "); + rtf.push_str(&"{\\b ".repeat(deep)); + rtf.push_str("deep"); + rtf.push_str(&"}".repeat(deep)); + rtf.push_str(" after}"); + let doc = read(rtf.as_bytes()).unwrap(); + assert_eq!(doc.paras().map(Para::text).collect::(), "deep after"); + assert!(fmt_of(&doc, "deep").bold && !fmt_of(&doc, "after").bold, "the groups within the limit still close in step"); + // A skipped destination deeper than the limit is still skipped. + let mut rtf = String::from("{\\rtf1 "); + rtf.push_str(&"{".repeat(MAX_GROUPS - 3)); + rtf.push_str("{\\info {{{{secret}}}}}shown"); + assert_eq!(text(rtf.as_bytes()), "shown"); + // Unbalanced closes are ignored. + assert_eq!(text(b"{\\rtf1 a}}}}b{c"), "abc"); +} + +#[test] +fn font_and_colour_tables_are_capped() { + let mut rtf = String::from("{\\rtf1{\\fonttbl"); + for i in 0..MAX_FONTS + 10 { + rtf.push_str(&format!("{{\\f{i} F{i};}}")); + } + rtf.push_str(&format!("{{\\f9000 {};}}{{\\f9001 {};}}", "n".repeat(MAX_FONT_NAME), "n".repeat(MAX_FONT_NAME + 1))); + rtf.push_str("}{\\colortbl;"); + for i in 0..MAX_COLORS + 10 { + rtf.push_str(&format!("\\red{}\\green1\\blue1;", i % 256)); + } + rtf.push_str(&format!("}}\\f{} a\\f{} b\\f9000 x\\f9001 y\\cf{} c\\cf{} d}}", MAX_FONTS - 1, MAX_FONTS, MAX_COLORS - 1, MAX_COLORS)); + let doc = read(rtf.as_bytes()).unwrap(); + assert_eq!(fmt_of(&doc, "a").font, Some(format!("F{}", MAX_FONTS - 1))); + assert_eq!(fmt_of(&doc, "b").font, None, "past the font limit"); + assert_eq!(fmt_of(&doc, "x").font, None, "the table was full before the 256-byte name"); + assert_eq!(fmt_of(&doc, "y").font, None); + assert_eq!(fmt_of(&doc, "c").color, Some([((MAX_COLORS - 2) % 256) as u8, 1, 1])); + assert_eq!(fmt_of(&doc, "d").color, None, "past the colour limit"); + // The name limit on its own. + let rtf = format!("{{\\rtf1{{\\fonttbl{{\\f0 {};}}{{\\f1 {};}}}}\\f0 a\\f1 b}}", "n".repeat(MAX_FONT_NAME), "n".repeat(MAX_FONT_NAME + 1)); + let doc = read(rtf.as_bytes()).unwrap(); + assert_eq!(fmt_of(&doc, "a").font.map(|f| f.len()), Some(MAX_FONT_NAME)); + assert_eq!(fmt_of(&doc, "b").font, None); +} + +fn sample() -> Document { + let fmt = |f: fn(&mut CharFmt)| { + let mut c = CharFmt::default(); + f(&mut c); + c + }; + let run = |text: &str, fmt: CharFmt| Run { text: text.into(), fmt }; + let cell = |right, paras| Cell { right, paras }; + Document { + blocks: vec![ + Block::Para(Para { runs: vec![run("Plan {Q3} \\ 100%", fmt(|c| c.bold = true))], outline_level: Some(0), ..Para::default() }), + Block::Para(Para { + runs: vec![ + run(" Grow — fast 😀 ", CharFmt::default()), + run("Café\u{A0}résumé", fmt(|c| (c.italic, c.underline, c.half_points) = (true, true, Some(21)))), + run("\ttab\nline", fmt(|c| (c.strike, c.color) = (true, Some([255, 0, 128])))), + run("123 numbers first", fmt(|c| c.font = Some("Deja;Vu".into()))), + run("2", fmt(|c| c.vert_align = VertAlign::Superscript)), + run("i", fmt(|c| c.vert_align = VertAlign::Subscript)), + ], + outline_level: Some(8), + left_indent: 720, + right_indent: 360, + first_line_indent: -360, + space_before: 120, + space_after: 240, + align: Some(Align::Justified), + }), + Block::Para(Para::default()), + Block::Table(Table { + rows: vec![ + vec![cell(1440, vec![para("A1")]), cell(4320, vec![para("B1"), Para::default(), para("B1 third")])], + vec![ + cell(1440, vec![Para::default()]), + cell(2880, vec![para("日本語")]), + cell(4320, vec![Para { left_indent: -360, ..para("C2") }]), + ], + ], + }), + Block::Para(Para { left_indent: -100, ..para(&format!("after{PAGE_BREAK}page{COLUMN_BREAK}column")) }), + Block::Table(Table { rows: vec![vec![cell(50, vec![para("last")])]] }), + ], + pictures_left_out: 0, + } +} + +#[test] +fn what_is_written_reads_back() { + let doc = sample(); + let rtf = write(&doc, "Helvetica"); + assert!(rtf.is_ascii(), "7-bit"); + assert_eq!(read(rtf.as_bytes()).unwrap(), doc, "{rtf}"); + assert_eq!(write(&read(rtf.as_bytes()).unwrap(), "Helvetica"), rtf, "and writes the same file again"); + assert_eq!(read(write(&Document::default(), "x").as_bytes()).unwrap(), Document::default()); +} + +#[test] +fn the_written_file() { + let doc = Document { + blocks: vec![ + Block::Para(Para { + runs: vec![Run { text: "Title {1}".into(), fmt: CharFmt { bold: true, half_points: Some(32), ..CharFmt::default() } }], + outline_level: Some(0), + ..Para::default() + }), + Block::Para(Para { left_indent: 360, ..para("Body é") }), + Block::Table(Table { rows: vec![vec![Cell { right: 1440, paras: vec![para("A")] }, Cell { right: 2880, paras: vec![] }], vec![]] }), + ], + pictures_left_out: 7, + }; + let red = CharFmt { color: Some([255, 0, 0]), font: Some("Courier New".into()), ..CharFmt::default() }; + let mut colored = doc.clone(); + colored.blocks.push(Block::Para(Para { runs: vec![Run { text: "red".into(), fmt: red }], ..Para::default() })); + assert_eq!( + write(&doc, "Liberation Sans"), + "{\\rtf1\\ansi\\ansicpg1252\\deff0\\uc1{\\fonttbl{\\f0\\fnil Liberation Sans;}}\n\\pard\\outlinelevel0 {\\b\\fs32 Title \\{1\\}}\\par\n\\pard\\li360 Body \\u233?\\par\n\\trowd\\trgaph108\\cellx1440\\cellx2880\\pard\\intbl A\\cell\\cell\\row\n\\pard}" + ); + let rtf = write(&colored, "Liberation Sans"); + assert!(rtf.contains("{\\fonttbl{\\f0\\fnil Liberation Sans;}{\\f1\\fnil Courier New;}}\n{\\colortbl;\\red255\\green0\\blue0;}\n"), "{rtf}"); + assert!(rtf.ends_with("\\pard\\pard {\\f1\\cf1 red}\\par\n}"), "{rtf}"); + // The default font named on a run needs no word. + let named = Document { + blocks: vec![Block::Para(Para { + runs: vec![Run { text: "x".into(), fmt: CharFmt { font: Some("liberation sans".into()), ..CharFmt::default() } }], + ..Para::default() + })], + pictures_left_out: 0, + }; + assert!(write(&named, "Liberation Sans").contains("\\pard x\\par")); +} + +#[test] +fn escaping() { + assert_eq!(craft_rtf::esc("a\\b{c}d\te\nf\u{B}g\u{2028}h"), "a\\\\b\\{c\\}d\\tab e\\line f\\line g\\line h"); + assert_eq!(craft_rtf::esc("\u{A0}\u{AD}\u{2011}\u{C}\u{E}\u{1}\r"), "\\~\\-\\_\\page \\column "); + assert_eq!(craft_rtf::esc("é€😀"), "\\u233?\\u8364?\\u-10179?\\u-8704?"); + assert_eq!(craft_rtf::esc_font("A;B\n{C}"), "A\\'3bB\\{C\\}"); + let mut s = String::from("x"); + craft_rtf::push_esc(&mut s, "{"); + assert_eq!(s, "x\\{"); +} + +/// Every prefix of a file, and the file with each byte changed, reads without a panic. +#[test] +fn damaged_files_do_not_panic() { + let rtf = write(&sample(), "Helvetica"); + let rtf = format!( + "{rtf}{{\\rtf1\\ansicpg932{{\\fonttbl{{\\f0\\fcharset128 MS Mincho;}}}}{{\\info{{\\title x}}}}\\'93\\'fa\\u-10179?\\bin3 abc{{\\pict\\'00}}\\trowd\\cellx9 a\\cell\\row}}" + ); + let bytes = rtf.as_bytes(); + for end in 0..=bytes.len() { + let _ = read(&bytes[..end]); + } + for at in 0..bytes.len() { + for b in [b'{', b'}', b'\\', b'\'', 0xFF, 0, b'9', b'-', b';'] { + let mut damaged = bytes.to_vec(); + damaged[at] = b; + let _ = read(&damaged); + } + } +}