mirror of
https://github.com/storytold/craft-libs.git
synced 2026-11-14 06:22:24 +00:00
Add craft-rtf: RTF tokenizer, code pages, and a reader and writer for a small model (#890)
* Add craft-rtf: RTF tokenizer, text decoding, and a reader and writer for a small model The tokenizing rules, code pages, multibyte decoding, dates and escaping come from WordCraft's RTF reader and writer. The model (paragraphs with outline level and indent, character formatting, tables) covers what DesignCraft's text import, DeckCraft's outline import and export and PdfCraft's export use. An unpaired UTF-16 surrogate in \u reads as U+FFFD and a high surrogate pairs only with the very next \u, as storytold/wordcraft#6105 describes for Word. * craft-rtf: a symbol font's bytes are its own codes A font with \fcharset2 (Symbol, Wingdings) has no letters: a byte from 0x80 reads as U+F000 + the byte instead of a letter of the document's code page. The symbol page applies to body text only, a symbol default font's page does not apply to text in a font chosen with \f, \plain makes the default font's page current again, and a code page an entry names with \cpg is kept. Ported from storytold/wordcraft#6014. * craft-rtf: alignment, indents, spacing, superscript and subscript; NUL is not text The model and both directions gain what DesignCraft's RTF export writes: \ql \qc \qr \qj, \ri, \fi, \sb, \sa, \super, \sub, \nosupersub. A \u0 or \'00 no longer puts a NUL into the text. --------- Co-authored-by: Simon Koch-Sultan <126534658+scks1@users.noreply.github.com>
This commit is contained in:
Dylan M. TaylorandSimon Koch-Sultan authored and 
GitHub committed
1 parent
1c46124314
commit
9ab7768771
14 files changed
+2466
No files matched your search
Generated
+7
@@ -1293,6 +1293,13 @@ dependencies = [
|
||||
"zune-jpegxl",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "craft-rtf"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"encoding_rs",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "craft-settings"
|
||||
version = "0.1.0"
|
||||
|
||||
@@ -63,6 +63,7 @@ craft-print = { path = "crates/print" }
|
||||
craft-proof = { path = "crates/proof" }
|
||||
craft-psd = { path = "crates/psd" }
|
||||
craft-quit = { path = "crates/quit" }
|
||||
craft-rtf = { path = "crates/rtf" }
|
||||
craft-raw = { path = "crates/raw" }
|
||||
craft-settings = { path = "crates/settings" }
|
||||
craft-silhouette-studio = { path = "crates/silhouette-studio" }
|
||||
|
||||
@@ -173,6 +173,9 @@ craft-<name> = { git = "https://github.com/storytold/craft-libs", rev = "<commit
|
||||
- [`craft-quit`](crates/quit/src/lib.rs): native macOS quit, egui close handling and an optional
|
||||
app menu. Unsaved-document decisions and save prompts stay in each app.
|
||||
|
||||
- [`craft-rtf`](crates/rtf/README.md): Rich Text Format: a tokenizer, text decoding in the file's code pages
|
||||
(`\uN`, `\'hh`, single-byte and East Asian multibyte pages), escaping and dates, and a reader and writer
|
||||
for a small neutral model (paragraphs, character formatting, outline levels, tables).
|
||||
- [`craft-settings`](crates/settings/README.md): `config.toml` settings storage for every app: platform folders, portable mode and overrides, atomic saves while running, backup, damaged-file recovery and schema migrations. Policy in [docs/settings-storage.md](docs/settings-storage.md).
|
||||
- [`craft-silhouette-studio`](crates/silhouette-studio/src/lib.rs): bounded, fail-closed reader for Silhouette Studio `.studio` and `.studio3` documents: outlines, line colours and page size in millimetres. Protected designs are refused.
|
||||
- [`craft-svg`](crates/svg/src/lib.rs): SVG. `sniff` recognises SVG and SVGZ data; `svgz` unpacks
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
[package]
|
||||
name = "craft-rtf"
|
||||
description = "Rich Text Format: a tokenizer, text decoding in the file's code pages, and a reader and writer for a small neutral model (paragraphs, character formatting, outline levels, tables)."
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
rust-version.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
# The East Asian multibyte code pages (932, 936, 949, 950).
|
||||
encoding_rs = { workspace = true }
|
||||
@@ -0,0 +1,87 @@
|
||||
# craft-rtf
|
||||
|
||||
Rich Text Format, read and written. It replaces the RTF code each Craft app carried: the
|
||||
tokenizer, code pages, `\u` handling, dates and escaping come from WordCraft's reader and
|
||||
writer; the small model covers what DesignCraft's text import, DeckCraft's outline import and
|
||||
export and PdfCraft's export read and write.
|
||||
|
||||
```rust
|
||||
use craft_rtf::{Block, CharFmt, Document, Para, Run, read, write};
|
||||
|
||||
let bold = CharFmt { bold: true, ..CharFmt::default() };
|
||||
let doc = Document {
|
||||
blocks: vec![Block::Para(Para {
|
||||
runs: vec![Run { text: "Plan {Q3} ".into(), fmt: bold }, Run { text: "café 😀".into(), fmt: CharFmt::default() }],
|
||||
outline_level: Some(0),
|
||||
..Para::default()
|
||||
})],
|
||||
pictures_left_out: 0,
|
||||
};
|
||||
let rtf = write(&doc, "Liberation Sans");
|
||||
assert!(rtf.contains(r"\outlinelevel0 {\b Plan \{Q3\} }caf\u233?"));
|
||||
assert_eq!(read(rtf.as_bytes()).unwrap(), doc);
|
||||
```
|
||||
|
||||
## Layers
|
||||
|
||||
| Layer | What it is | For |
|
||||
| --- | --- | --- |
|
||||
| `Tokens` | The tokens of a file: `{`, `}`, control words with their parameter, `\*`, `\'hh`, control symbols, text runs, `\bin` data. | Any reader. |
|
||||
| `TextDecoder`, `codepage` | 8-bit text in a code page, `\uN` and its fallback characters. | Any reader. |
|
||||
| `esc`, `esc_font`, `date_words`, `iso_to_dttm` | Escaping and dates. | Any writer. |
|
||||
| `read`, `write`, `Document` | A file to and from a small model. | An app that needs text with basic formatting. |
|
||||
|
||||
A reader that keeps more than the model holds (styles, lists, fields, notes, tracked changes,
|
||||
pictures) uses the first two layers and its own interpretation of the control words.
|
||||
|
||||
## Text
|
||||
|
||||
- **Code pages.** 8-bit text is in the current font's code page (`\fcharset`, `\cpg`), else
|
||||
the default font's (`\deff`), else the document's (`\ansicpg`, `\mac`, `\pc`, `\pca`), else
|
||||
Windows-1252. Single-byte: Windows 874 and 1250 to 1258, DOS 437, 737, 850, 852, 857, 866,
|
||||
Mac OS Roman. Multibyte: 932 (Shift-JIS), 936 (GBK), 949 (UHC), 950 (Big5), streamed through
|
||||
one decoder so a lead byte and its trail byte decode together whatever falls between them.
|
||||
Raw bytes above 0x7F are text in the code page, as `\'hh` is. They are not read as UTF-8.
|
||||
- **`\uN`.** One UTF-16 code unit, taken modulo 65,536. A high surrogate pairs only with a low
|
||||
surrogate in the very next `\u`; a surrogate that does not pair is U+FFFD, as in Word. The
|
||||
`\ucN` characters after a `\u` are its fallback and are skipped: a text byte, a `\'hh`, a
|
||||
control word and a control symbol each count as one, a following `\u` does not, and the
|
||||
skipping ends with the group.
|
||||
- **Malformed input.** `\'` without two hex digits is dropped with the hex digits it has; a
|
||||
`}` or `\` after it keeps its meaning. A control word is at most 32 letters and a parameter
|
||||
at most 10 digits. A parameter outside `i32` is clamped.
|
||||
|
||||
## The model
|
||||
|
||||
`Document { blocks, pictures_left_out }`, `Block::{Para, Table}`, `Para { runs, outline_level,
|
||||
left_indent, right_indent, first_line_indent, space_before, space_after, align }`, `Run { text, fmt }`, `CharFmt { bold, italic, underline, strike,
|
||||
vert_align, half_points, color, font }`, `Table { rows: Vec<Vec<Cell>> }`, `Cell { right, paras }`.
|
||||
|
||||
In `Run::text`, a tab is `\t`, and `LINE_BREAK` (`\n`), `PAGE_BREAK` (U+000C) and
|
||||
`COLUMN_BREAK` (U+000E) are `\line`, `\page` and `\column`. Lengths are in twips and sizes in
|
||||
half-points, as the file has them.
|
||||
|
||||
`read` leaves out headers, footers, footnotes, comments, field instructions (a field's result
|
||||
is kept), list labels (`\listtext`, `\pntext`), the stylesheet, pictures and objects (counted),
|
||||
and every destination marked `\*`. Paragraph properties end with their group, as character
|
||||
properties do. A trailing `\par` does not leave an empty last paragraph; empty paragraphs
|
||||
elsewhere are kept. `\sect` ends a paragraph.
|
||||
|
||||
### Limits
|
||||
|
||||
| Limit | Value | Past it |
|
||||
| --- | --- | --- |
|
||||
| `MAX_GROUPS` | 512 nested groups | deeper groups are read in the state of the group around them |
|
||||
| `MAX_FONTS` | 4096 font table entries | later fonts are not kept |
|
||||
| `MAX_COLORS` | 4096 colour table entries | later colours are not kept |
|
||||
| `MAX_FONT_NAME` | 256 bytes | the font is not kept |
|
||||
|
||||
The reader does not recurse, and what it returns is never larger than a small multiple of the
|
||||
file: each character of text costs the file at least one byte.
|
||||
|
||||
## Not here yet
|
||||
|
||||
WordCraft's full reader and writer (styles, lists, fields, bookmarks, pictures, notes,
|
||||
comments, tracked changes, sections, headers and footers) are written against WordCraft's own
|
||||
document model and stay there until that model has a neutral form. They can already sit on
|
||||
`Tokens`, `TextDecoder`, `esc` and the date functions.
|
||||
@@ -0,0 +1,440 @@
|
||||
//! The code pages of 8-bit RTF text: Windows-1252 and the other single-byte Windows
|
||||
//! code pages RTF files name with `\ansicpg` or a font's `\fcharset` (Cyrillic 1251, Central
|
||||
//! European 1250, Greek, Turkish, Hebrew, Arabic, Baltic, Vietnamese, Thai), Mac OS Roman
|
||||
//! (10000, RTF `\mac`), and the East Asian multibyte code pages (932 Shift-JIS, 936 GBK, 949 UHC,
|
||||
//! 950 Big5), whose bytes decode as runs rather than one at a time.
|
||||
|
||||
/// Windows-1252 code points for 0x80..=0x9F.
|
||||
const CP1252: [u16; 32] = [
|
||||
0x20AC, 0x0081, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, 0x02C6, 0x2030, 0x0160, 0x2039, 0x0152, 0x008D, 0x017D, 0x008F, 0x0090, 0x2018,
|
||||
0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0x02DC, 0x2122, 0x0161, 0x203A, 0x0153, 0x009D, 0x017E, 0x0178,
|
||||
];
|
||||
|
||||
/// One Windows-1252 byte as a char.
|
||||
pub fn cp1252(b: u8) -> char {
|
||||
if (0x80..0xA0).contains(&b) {
|
||||
CP1252.get((b - 0x80) as usize).and_then(|u| char::from_u32(u32::from(*u))).unwrap_or('\u{FFFD}')
|
||||
} else {
|
||||
b as char
|
||||
}
|
||||
}
|
||||
|
||||
/// Windows-874 code points for 0x80..=0xFF (U+FFFD where the byte is unassigned).
|
||||
const CP874: [u16; 128] = [
|
||||
0x20AC, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0x2026, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0x2018,
|
||||
0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0x00A0, 0x0E01, 0x0E02, 0x0E03,
|
||||
0x0E04, 0x0E05, 0x0E06, 0x0E07, 0x0E08, 0x0E09, 0x0E0A, 0x0E0B, 0x0E0C, 0x0E0D, 0x0E0E, 0x0E0F, 0x0E10, 0x0E11, 0x0E12, 0x0E13, 0x0E14, 0x0E15,
|
||||
0x0E16, 0x0E17, 0x0E18, 0x0E19, 0x0E1A, 0x0E1B, 0x0E1C, 0x0E1D, 0x0E1E, 0x0E1F, 0x0E20, 0x0E21, 0x0E22, 0x0E23, 0x0E24, 0x0E25, 0x0E26, 0x0E27,
|
||||
0x0E28, 0x0E29, 0x0E2A, 0x0E2B, 0x0E2C, 0x0E2D, 0x0E2E, 0x0E2F, 0x0E30, 0x0E31, 0x0E32, 0x0E33, 0x0E34, 0x0E35, 0x0E36, 0x0E37, 0x0E38, 0x0E39,
|
||||
0x0E3A, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0x0E3F, 0x0E40, 0x0E41, 0x0E42, 0x0E43, 0x0E44, 0x0E45, 0x0E46, 0x0E47, 0x0E48, 0x0E49, 0x0E4A, 0x0E4B,
|
||||
0x0E4C, 0x0E4D, 0x0E4E, 0x0E4F, 0x0E50, 0x0E51, 0x0E52, 0x0E53, 0x0E54, 0x0E55, 0x0E56, 0x0E57, 0x0E58, 0x0E59, 0x0E5A, 0x0E5B, 0xFFFD, 0xFFFD,
|
||||
0xFFFD, 0xFFFD,
|
||||
];
|
||||
|
||||
/// Windows-1250 code points for 0x80..=0xFF (U+FFFD where the byte is unassigned).
|
||||
const CP1250: [u16; 128] = [
|
||||
0x20AC, 0xFFFD, 0x201A, 0xFFFD, 0x201E, 0x2026, 0x2020, 0x2021, 0xFFFD, 0x2030, 0x0160, 0x2039, 0x015A, 0x0164, 0x017D, 0x0179, 0xFFFD, 0x2018,
|
||||
0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0xFFFD, 0x2122, 0x0161, 0x203A, 0x015B, 0x0165, 0x017E, 0x017A, 0x00A0, 0x02C7, 0x02D8, 0x0141,
|
||||
0x00A4, 0x0104, 0x00A6, 0x00A7, 0x00A8, 0x00A9, 0x015E, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x017B, 0x00B0, 0x00B1, 0x02DB, 0x0142, 0x00B4, 0x00B5,
|
||||
0x00B6, 0x00B7, 0x00B8, 0x0105, 0x015F, 0x00BB, 0x013D, 0x02DD, 0x013E, 0x017C, 0x0154, 0x00C1, 0x00C2, 0x0102, 0x00C4, 0x0139, 0x0106, 0x00C7,
|
||||
0x010C, 0x00C9, 0x0118, 0x00CB, 0x011A, 0x00CD, 0x00CE, 0x010E, 0x0110, 0x0143, 0x0147, 0x00D3, 0x00D4, 0x0150, 0x00D6, 0x00D7, 0x0158, 0x016E,
|
||||
0x00DA, 0x0170, 0x00DC, 0x00DD, 0x0162, 0x00DF, 0x0155, 0x00E1, 0x00E2, 0x0103, 0x00E4, 0x013A, 0x0107, 0x00E7, 0x010D, 0x00E9, 0x0119, 0x00EB,
|
||||
0x011B, 0x00ED, 0x00EE, 0x010F, 0x0111, 0x0144, 0x0148, 0x00F3, 0x00F4, 0x0151, 0x00F6, 0x00F7, 0x0159, 0x016F, 0x00FA, 0x0171, 0x00FC, 0x00FD,
|
||||
0x0163, 0x02D9,
|
||||
];
|
||||
|
||||
/// Windows-1251 code points for 0x80..=0xFF (U+FFFD where the byte is unassigned).
|
||||
const CP1251: [u16; 128] = [
|
||||
0x0402, 0x0403, 0x201A, 0x0453, 0x201E, 0x2026, 0x2020, 0x2021, 0x20AC, 0x2030, 0x0409, 0x2039, 0x040A, 0x040C, 0x040B, 0x040F, 0x0452, 0x2018,
|
||||
0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0xFFFD, 0x2122, 0x0459, 0x203A, 0x045A, 0x045C, 0x045B, 0x045F, 0x00A0, 0x040E, 0x045E, 0x0408,
|
||||
0x00A4, 0x0490, 0x00A6, 0x00A7, 0x0401, 0x00A9, 0x0404, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x0407, 0x00B0, 0x00B1, 0x0406, 0x0456, 0x0491, 0x00B5,
|
||||
0x00B6, 0x00B7, 0x0451, 0x2116, 0x0454, 0x00BB, 0x0458, 0x0405, 0x0455, 0x0457, 0x0410, 0x0411, 0x0412, 0x0413, 0x0414, 0x0415, 0x0416, 0x0417,
|
||||
0x0418, 0x0419, 0x041A, 0x041B, 0x041C, 0x041D, 0x041E, 0x041F, 0x0420, 0x0421, 0x0422, 0x0423, 0x0424, 0x0425, 0x0426, 0x0427, 0x0428, 0x0429,
|
||||
0x042A, 0x042B, 0x042C, 0x042D, 0x042E, 0x042F, 0x0430, 0x0431, 0x0432, 0x0433, 0x0434, 0x0435, 0x0436, 0x0437, 0x0438, 0x0439, 0x043A, 0x043B,
|
||||
0x043C, 0x043D, 0x043E, 0x043F, 0x0440, 0x0441, 0x0442, 0x0443, 0x0444, 0x0445, 0x0446, 0x0447, 0x0448, 0x0449, 0x044A, 0x044B, 0x044C, 0x044D,
|
||||
0x044E, 0x044F,
|
||||
];
|
||||
|
||||
/// Windows-1253 code points for 0x80..=0xFF (U+FFFD where the byte is unassigned).
|
||||
const CP1253: [u16; 128] = [
|
||||
0x20AC, 0xFFFD, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, 0xFFFD, 0x2030, 0xFFFD, 0x2039, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0x2018,
|
||||
0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0xFFFD, 0x2122, 0xFFFD, 0x203A, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0x00A0, 0x0385, 0x0386, 0x00A3,
|
||||
0x00A4, 0x00A5, 0x00A6, 0x00A7, 0x00A8, 0x00A9, 0xFFFD, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x2015, 0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x0384, 0x00B5,
|
||||
0x00B6, 0x00B7, 0x0388, 0x0389, 0x038A, 0x00BB, 0x038C, 0x00BD, 0x038E, 0x038F, 0x0390, 0x0391, 0x0392, 0x0393, 0x0394, 0x0395, 0x0396, 0x0397,
|
||||
0x0398, 0x0399, 0x039A, 0x039B, 0x039C, 0x039D, 0x039E, 0x039F, 0x03A0, 0x03A1, 0xFFFD, 0x03A3, 0x03A4, 0x03A5, 0x03A6, 0x03A7, 0x03A8, 0x03A9,
|
||||
0x03AA, 0x03AB, 0x03AC, 0x03AD, 0x03AE, 0x03AF, 0x03B0, 0x03B1, 0x03B2, 0x03B3, 0x03B4, 0x03B5, 0x03B6, 0x03B7, 0x03B8, 0x03B9, 0x03BA, 0x03BB,
|
||||
0x03BC, 0x03BD, 0x03BE, 0x03BF, 0x03C0, 0x03C1, 0x03C2, 0x03C3, 0x03C4, 0x03C5, 0x03C6, 0x03C7, 0x03C8, 0x03C9, 0x03CA, 0x03CB, 0x03CC, 0x03CD,
|
||||
0x03CE, 0xFFFD,
|
||||
];
|
||||
|
||||
/// Windows-1254 code points for 0x80..=0xFF (U+FFFD where the byte is unassigned).
|
||||
const CP1254: [u16; 128] = [
|
||||
0x20AC, 0xFFFD, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, 0x02C6, 0x2030, 0x0160, 0x2039, 0x0152, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0x2018,
|
||||
0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0x02DC, 0x2122, 0x0161, 0x203A, 0x0153, 0xFFFD, 0xFFFD, 0x0178, 0x00A0, 0x00A1, 0x00A2, 0x00A3,
|
||||
0x00A4, 0x00A5, 0x00A6, 0x00A7, 0x00A8, 0x00A9, 0x00AA, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x00AF, 0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x00B4, 0x00B5,
|
||||
0x00B6, 0x00B7, 0x00B8, 0x00B9, 0x00BA, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0x00BF, 0x00C0, 0x00C1, 0x00C2, 0x00C3, 0x00C4, 0x00C5, 0x00C6, 0x00C7,
|
||||
0x00C8, 0x00C9, 0x00CA, 0x00CB, 0x00CC, 0x00CD, 0x00CE, 0x00CF, 0x011E, 0x00D1, 0x00D2, 0x00D3, 0x00D4, 0x00D5, 0x00D6, 0x00D7, 0x00D8, 0x00D9,
|
||||
0x00DA, 0x00DB, 0x00DC, 0x0130, 0x015E, 0x00DF, 0x00E0, 0x00E1, 0x00E2, 0x00E3, 0x00E4, 0x00E5, 0x00E6, 0x00E7, 0x00E8, 0x00E9, 0x00EA, 0x00EB,
|
||||
0x00EC, 0x00ED, 0x00EE, 0x00EF, 0x011F, 0x00F1, 0x00F2, 0x00F3, 0x00F4, 0x00F5, 0x00F6, 0x00F7, 0x00F8, 0x00F9, 0x00FA, 0x00FB, 0x00FC, 0x0131,
|
||||
0x015F, 0x00FF,
|
||||
];
|
||||
|
||||
/// Windows-1255 code points for 0x80..=0xFF (U+FFFD where the byte is unassigned).
|
||||
const CP1255: [u16; 128] = [
|
||||
0x20AC, 0xFFFD, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, 0x02C6, 0x2030, 0xFFFD, 0x2039, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0x2018,
|
||||
0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0x02DC, 0x2122, 0xFFFD, 0x203A, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0x00A0, 0x00A1, 0x00A2, 0x00A3,
|
||||
0x20AA, 0x00A5, 0x00A6, 0x00A7, 0x00A8, 0x00A9, 0x00D7, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x00AF, 0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x00B4, 0x00B5,
|
||||
0x00B6, 0x00B7, 0x00B8, 0x00B9, 0x00F7, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0x00BF, 0x05B0, 0x05B1, 0x05B2, 0x05B3, 0x05B4, 0x05B5, 0x05B6, 0x05B7,
|
||||
0x05B8, 0x05B9, 0x05BA, 0x05BB, 0x05BC, 0x05BD, 0x05BE, 0x05BF, 0x05C0, 0x05C1, 0x05C2, 0x05C3, 0x05F0, 0x05F1, 0x05F2, 0x05F3, 0x05F4, 0xFFFD,
|
||||
0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0x05D0, 0x05D1, 0x05D2, 0x05D3, 0x05D4, 0x05D5, 0x05D6, 0x05D7, 0x05D8, 0x05D9, 0x05DA, 0x05DB,
|
||||
0x05DC, 0x05DD, 0x05DE, 0x05DF, 0x05E0, 0x05E1, 0x05E2, 0x05E3, 0x05E4, 0x05E5, 0x05E6, 0x05E7, 0x05E8, 0x05E9, 0x05EA, 0xFFFD, 0xFFFD, 0x200E,
|
||||
0x200F, 0xFFFD,
|
||||
];
|
||||
|
||||
/// Windows-1256 code points for 0x80..=0xFF (U+FFFD where the byte is unassigned).
|
||||
const CP1256: [u16; 128] = [
|
||||
0x20AC, 0x067E, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, 0x02C6, 0x2030, 0x0679, 0x2039, 0x0152, 0x0686, 0x0698, 0x0688, 0x06AF, 0x2018,
|
||||
0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0x06A9, 0x2122, 0x0691, 0x203A, 0x0153, 0x200C, 0x200D, 0x06BA, 0x00A0, 0x060C, 0x00A2, 0x00A3,
|
||||
0x00A4, 0x00A5, 0x00A6, 0x00A7, 0x00A8, 0x00A9, 0x06BE, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x00AF, 0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x00B4, 0x00B5,
|
||||
0x00B6, 0x00B7, 0x00B8, 0x00B9, 0x061B, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0x061F, 0x06C1, 0x0621, 0x0622, 0x0623, 0x0624, 0x0625, 0x0626, 0x0627,
|
||||
0x0628, 0x0629, 0x062A, 0x062B, 0x062C, 0x062D, 0x062E, 0x062F, 0x0630, 0x0631, 0x0632, 0x0633, 0x0634, 0x0635, 0x0636, 0x00D7, 0x0637, 0x0638,
|
||||
0x0639, 0x063A, 0x0640, 0x0641, 0x0642, 0x0643, 0x00E0, 0x0644, 0x00E2, 0x0645, 0x0646, 0x0647, 0x0648, 0x00E7, 0x00E8, 0x00E9, 0x00EA, 0x00EB,
|
||||
0x0649, 0x064A, 0x00EE, 0x00EF, 0x064B, 0x064C, 0x064D, 0x064E, 0x00F4, 0x064F, 0x0650, 0x00F7, 0x0651, 0x00F9, 0x0652, 0x00FB, 0x00FC, 0x200E,
|
||||
0x200F, 0x06D2,
|
||||
];
|
||||
|
||||
/// Windows-1257 code points for 0x80..=0xFF (U+FFFD where the byte is unassigned).
|
||||
const CP1257: [u16; 128] = [
|
||||
0x20AC, 0xFFFD, 0x201A, 0xFFFD, 0x201E, 0x2026, 0x2020, 0x2021, 0xFFFD, 0x2030, 0xFFFD, 0x2039, 0xFFFD, 0x00A8, 0x02C7, 0x00B8, 0xFFFD, 0x2018,
|
||||
0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0xFFFD, 0x2122, 0xFFFD, 0x203A, 0xFFFD, 0x00AF, 0x02DB, 0xFFFD, 0x00A0, 0xFFFD, 0x00A2, 0x00A3,
|
||||
0x00A4, 0xFFFD, 0x00A6, 0x00A7, 0x00D8, 0x00A9, 0x0156, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x00C6, 0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x00B4, 0x00B5,
|
||||
0x00B6, 0x00B7, 0x00F8, 0x00B9, 0x0157, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0x00E6, 0x0104, 0x012E, 0x0100, 0x0106, 0x00C4, 0x00C5, 0x0118, 0x0112,
|
||||
0x010C, 0x00C9, 0x0179, 0x0116, 0x0122, 0x0136, 0x012A, 0x013B, 0x0160, 0x0143, 0x0145, 0x00D3, 0x014C, 0x00D5, 0x00D6, 0x00D7, 0x0172, 0x0141,
|
||||
0x015A, 0x016A, 0x00DC, 0x017B, 0x017D, 0x00DF, 0x0105, 0x012F, 0x0101, 0x0107, 0x00E4, 0x00E5, 0x0119, 0x0113, 0x010D, 0x00E9, 0x017A, 0x0117,
|
||||
0x0123, 0x0137, 0x012B, 0x013C, 0x0161, 0x0144, 0x0146, 0x00F3, 0x014D, 0x00F5, 0x00F6, 0x00F7, 0x0173, 0x0142, 0x015B, 0x016B, 0x00FC, 0x017C,
|
||||
0x017E, 0x02D9,
|
||||
];
|
||||
|
||||
/// Windows-1258 code points for 0x80..=0xFF (U+FFFD where the byte is unassigned).
|
||||
const CP1258: [u16; 128] = [
|
||||
0x20AC, 0xFFFD, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, 0x02C6, 0x2030, 0xFFFD, 0x2039, 0x0152, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0x2018,
|
||||
0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0x02DC, 0x2122, 0xFFFD, 0x203A, 0x0153, 0xFFFD, 0xFFFD, 0x0178, 0x00A0, 0x00A1, 0x00A2, 0x00A3,
|
||||
0x00A4, 0x00A5, 0x00A6, 0x00A7, 0x00A8, 0x00A9, 0x00AA, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x00AF, 0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x00B4, 0x00B5,
|
||||
0x00B6, 0x00B7, 0x00B8, 0x00B9, 0x00BA, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0x00BF, 0x00C0, 0x00C1, 0x00C2, 0x0102, 0x00C4, 0x00C5, 0x00C6, 0x00C7,
|
||||
0x00C8, 0x00C9, 0x00CA, 0x00CB, 0x0300, 0x00CD, 0x00CE, 0x00CF, 0x0110, 0x00D1, 0x0309, 0x00D3, 0x00D4, 0x01A0, 0x00D6, 0x00D7, 0x00D8, 0x00D9,
|
||||
0x00DA, 0x00DB, 0x00DC, 0x01AF, 0x0303, 0x00DF, 0x00E0, 0x00E1, 0x00E2, 0x0103, 0x00E4, 0x00E5, 0x00E6, 0x00E7, 0x00E8, 0x00E9, 0x00EA, 0x00EB,
|
||||
0x0301, 0x00ED, 0x00EE, 0x00EF, 0x0111, 0x00F1, 0x0323, 0x00F3, 0x00F4, 0x01A1, 0x00F6, 0x00F7, 0x00F8, 0x00F9, 0x00FA, 0x00FB, 0x00FC, 0x01B0,
|
||||
0x20AB, 0x00FF,
|
||||
];
|
||||
|
||||
/// Mac OS Roman (code page 10000, RTF `\mac`) code points for 0x80..=0xFF, with the euro sign at
|
||||
/// 0xDB (Mac OS 8.5 and later).
|
||||
const MAC_ROMAN: [u16; 128] = [
|
||||
0x00C4, 0x00C5, 0x00C7, 0x00C9, 0x00D1, 0x00D6, 0x00DC, 0x00E1, 0x00E0, 0x00E2, 0x00E4, 0x00E3, 0x00E5, 0x00E7, 0x00E9, 0x00E8, 0x00EA, 0x00EB,
|
||||
0x00ED, 0x00EC, 0x00EE, 0x00EF, 0x00F1, 0x00F3, 0x00F2, 0x00F4, 0x00F6, 0x00F5, 0x00FA, 0x00F9, 0x00FB, 0x00FC, 0x2020, 0x00B0, 0x00A2, 0x00A3,
|
||||
0x00A7, 0x2022, 0x00B6, 0x00DF, 0x00AE, 0x00A9, 0x2122, 0x00B4, 0x00A8, 0x2260, 0x00C6, 0x00D8, 0x221E, 0x00B1, 0x2264, 0x2265, 0x00A5, 0x00B5,
|
||||
0x2202, 0x2211, 0x220F, 0x03C0, 0x222B, 0x00AA, 0x00BA, 0x03A9, 0x00E6, 0x00F8, 0x00BF, 0x00A1, 0x00AC, 0x221A, 0x0192, 0x2248, 0x2206, 0x00AB,
|
||||
0x00BB, 0x2026, 0x00A0, 0x00C0, 0x00C3, 0x00D5, 0x0152, 0x0153, 0x2013, 0x2014, 0x201C, 0x201D, 0x2018, 0x2019, 0x00F7, 0x25CA, 0x00FF, 0x0178,
|
||||
0x2044, 0x20AC, 0x2039, 0x203A, 0xFB01, 0xFB02, 0x2021, 0x00B7, 0x201A, 0x201E, 0x2030, 0x00C2, 0x00CA, 0x00C1, 0x00CB, 0x00C8, 0x00CD, 0x00CE,
|
||||
0x00CF, 0x00CC, 0x00D3, 0x00D4, 0xF8FF, 0x00D2, 0x00DA, 0x00DB, 0x00D9, 0x0131, 0x02C6, 0x02DC, 0x00AF, 0x02D8, 0x02D9, 0x02DA, 0x00B8, 0x02DD,
|
||||
0x02DB, 0x02C7,
|
||||
];
|
||||
/// IBM PC (OEM) code page 437 code points for 0x80..=0xFF (the Unicode Consortium's CP437
|
||||
/// mapping file).
|
||||
const CP437: [u16; 128] = [
|
||||
0x00C7, 0x00FC, 0x00E9, 0x00E2, 0x00E4, 0x00E0, 0x00E5, 0x00E7, 0x00EA, 0x00EB, 0x00E8, 0x00EF, 0x00EE, 0x00EC, 0x00C4, 0x00C5, 0x00C9, 0x00E6,
|
||||
0x00C6, 0x00F4, 0x00F6, 0x00F2, 0x00FB, 0x00F9, 0x00FF, 0x00D6, 0x00DC, 0x00A2, 0x00A3, 0x00A5, 0x20A7, 0x0192, 0x00E1, 0x00ED, 0x00F3, 0x00FA,
|
||||
0x00F1, 0x00D1, 0x00AA, 0x00BA, 0x00BF, 0x2310, 0x00AC, 0x00BD, 0x00BC, 0x00A1, 0x00AB, 0x00BB, 0x2591, 0x2592, 0x2593, 0x2502, 0x2524, 0x2561,
|
||||
0x2562, 0x2556, 0x2555, 0x2563, 0x2551, 0x2557, 0x255D, 0x255C, 0x255B, 0x2510, 0x2514, 0x2534, 0x252C, 0x251C, 0x2500, 0x253C, 0x255E, 0x255F,
|
||||
0x255A, 0x2554, 0x2569, 0x2566, 0x2560, 0x2550, 0x256C, 0x2567, 0x2568, 0x2564, 0x2565, 0x2559, 0x2558, 0x2552, 0x2553, 0x256B, 0x256A, 0x2518,
|
||||
0x250C, 0x2588, 0x2584, 0x258C, 0x2590, 0x2580, 0x03B1, 0x00DF, 0x0393, 0x03C0, 0x03A3, 0x03C3, 0x00B5, 0x03C4, 0x03A6, 0x0398, 0x03A9, 0x03B4,
|
||||
0x221E, 0x03C6, 0x03B5, 0x2229, 0x2261, 0x00B1, 0x2265, 0x2264, 0x2320, 0x2321, 0x00F7, 0x2248, 0x00B0, 0x2219, 0x00B7, 0x221A, 0x207F, 0x00B2,
|
||||
0x25A0, 0x00A0,
|
||||
];
|
||||
|
||||
/// OEM code page 850 (DOS Latin-1, Western European) code points for 0x80..=0xFF (the Unicode
|
||||
/// Consortium's CP850 mapping file).
|
||||
const CP850: [u16; 128] = [
|
||||
0x00C7, 0x00FC, 0x00E9, 0x00E2, 0x00E4, 0x00E0, 0x00E5, 0x00E7, 0x00EA, 0x00EB, 0x00E8, 0x00EF, 0x00EE, 0x00EC, 0x00C4, 0x00C5, 0x00C9, 0x00E6,
|
||||
0x00C6, 0x00F4, 0x00F6, 0x00F2, 0x00FB, 0x00F9, 0x00FF, 0x00D6, 0x00DC, 0x00F8, 0x00A3, 0x00D8, 0x00D7, 0x0192, 0x00E1, 0x00ED, 0x00F3, 0x00FA,
|
||||
0x00F1, 0x00D1, 0x00AA, 0x00BA, 0x00BF, 0x00AE, 0x00AC, 0x00BD, 0x00BC, 0x00A1, 0x00AB, 0x00BB, 0x2591, 0x2592, 0x2593, 0x2502, 0x2524, 0x00C1,
|
||||
0x00C2, 0x00C0, 0x00A9, 0x2563, 0x2551, 0x2557, 0x255D, 0x00A2, 0x00A5, 0x2510, 0x2514, 0x2534, 0x252C, 0x251C, 0x2500, 0x253C, 0x00E3, 0x00C3,
|
||||
0x255A, 0x2554, 0x2569, 0x2566, 0x2560, 0x2550, 0x256C, 0x00A4, 0x00F0, 0x00D0, 0x00CA, 0x00CB, 0x00C8, 0x0131, 0x00CD, 0x00CE, 0x00CF, 0x2518,
|
||||
0x250C, 0x2588, 0x2584, 0x00A6, 0x00CC, 0x2580, 0x00D3, 0x00DF, 0x00D4, 0x00D2, 0x00F5, 0x00D5, 0x00B5, 0x00FE, 0x00DE, 0x00DA, 0x00DB, 0x00D9,
|
||||
0x00FD, 0x00DD, 0x00AF, 0x00B4, 0x00AD, 0x00B1, 0x2017, 0x00BE, 0x00B6, 0x00A7, 0x00F7, 0x00B8, 0x00B0, 0x00A8, 0x00B7, 0x00B9, 0x00B3, 0x00B2,
|
||||
0x25A0, 0x00A0,
|
||||
];
|
||||
|
||||
/// OEM code page 852 (DOS Latin-2, Central European) code points for 0x80..=0xFF (the Unicode Consortium's CP852
|
||||
/// mapping file).
|
||||
const CP852: [u16; 128] = [
|
||||
0x00C7, 0x00FC, 0x00E9, 0x00E2, 0x00E4, 0x016F, 0x0107, 0x00E7, 0x0142, 0x00EB, 0x0150, 0x0151, 0x00EE, 0x0179, 0x00C4, 0x0106, 0x00C9, 0x0139,
|
||||
0x013A, 0x00F4, 0x00F6, 0x013D, 0x013E, 0x015A, 0x015B, 0x00D6, 0x00DC, 0x0164, 0x0165, 0x0141, 0x00D7, 0x010D, 0x00E1, 0x00ED, 0x00F3, 0x00FA,
|
||||
0x0104, 0x0105, 0x017D, 0x017E, 0x0118, 0x0119, 0x00AC, 0x017A, 0x010C, 0x015F, 0x00AB, 0x00BB, 0x2591, 0x2592, 0x2593, 0x2502, 0x2524, 0x00C1,
|
||||
0x00C2, 0x011A, 0x015E, 0x2563, 0x2551, 0x2557, 0x255D, 0x017B, 0x017C, 0x2510, 0x2514, 0x2534, 0x252C, 0x251C, 0x2500, 0x253C, 0x0102, 0x0103,
|
||||
0x255A, 0x2554, 0x2569, 0x2566, 0x2560, 0x2550, 0x256C, 0x00A4, 0x0111, 0x0110, 0x010E, 0x00CB, 0x010F, 0x0147, 0x00CD, 0x00CE, 0x011B, 0x2518,
|
||||
0x250C, 0x2588, 0x2584, 0x0162, 0x016E, 0x2580, 0x00D3, 0x00DF, 0x00D4, 0x0143, 0x0144, 0x0148, 0x0160, 0x0161, 0x0154, 0x00DA, 0x0155, 0x0170,
|
||||
0x00FD, 0x00DD, 0x0163, 0x00B4, 0x00AD, 0x02DD, 0x02DB, 0x02C7, 0x02D8, 0x00A7, 0x00F7, 0x00B8, 0x00B0, 0x00A8, 0x02D9, 0x0171, 0x0158, 0x0159,
|
||||
0x25A0, 0x00A0,
|
||||
];
|
||||
|
||||
/// OEM code page 866 (DOS Cyrillic) code points for 0x80..=0xFF (the Unicode Consortium's CP866
|
||||
/// mapping file).
|
||||
const CP866: [u16; 128] = [
|
||||
0x0410, 0x0411, 0x0412, 0x0413, 0x0414, 0x0415, 0x0416, 0x0417, 0x0418, 0x0419, 0x041A, 0x041B, 0x041C, 0x041D, 0x041E, 0x041F, 0x0420, 0x0421,
|
||||
0x0422, 0x0423, 0x0424, 0x0425, 0x0426, 0x0427, 0x0428, 0x0429, 0x042A, 0x042B, 0x042C, 0x042D, 0x042E, 0x042F, 0x0430, 0x0431, 0x0432, 0x0433,
|
||||
0x0434, 0x0435, 0x0436, 0x0437, 0x0438, 0x0439, 0x043A, 0x043B, 0x043C, 0x043D, 0x043E, 0x043F, 0x2591, 0x2592, 0x2593, 0x2502, 0x2524, 0x2561,
|
||||
0x2562, 0x2556, 0x2555, 0x2563, 0x2551, 0x2557, 0x255D, 0x255C, 0x255B, 0x2510, 0x2514, 0x2534, 0x252C, 0x251C, 0x2500, 0x253C, 0x255E, 0x255F,
|
||||
0x255A, 0x2554, 0x2569, 0x2566, 0x2560, 0x2550, 0x256C, 0x2567, 0x2568, 0x2564, 0x2565, 0x2559, 0x2558, 0x2552, 0x2553, 0x256B, 0x256A, 0x2518,
|
||||
0x250C, 0x2588, 0x2584, 0x258C, 0x2590, 0x2580, 0x0440, 0x0441, 0x0442, 0x0443, 0x0444, 0x0445, 0x0446, 0x0447, 0x0448, 0x0449, 0x044A, 0x044B,
|
||||
0x044C, 0x044D, 0x044E, 0x044F, 0x0401, 0x0451, 0x0404, 0x0454, 0x0407, 0x0457, 0x040E, 0x045E, 0x00B0, 0x2219, 0x00B7, 0x221A, 0x2116, 0x00A4,
|
||||
0x25A0, 0x00A0,
|
||||
];
|
||||
|
||||
/// OEM code page 737 (DOS Greek) code points for 0x80..=0xFF (the Unicode Consortium's CP737
|
||||
/// mapping file).
|
||||
const CP737: [u16; 128] = [
|
||||
0x0391, 0x0392, 0x0393, 0x0394, 0x0395, 0x0396, 0x0397, 0x0398, 0x0399, 0x039A, 0x039B, 0x039C, 0x039D, 0x039E, 0x039F, 0x03A0, 0x03A1, 0x03A3,
|
||||
0x03A4, 0x03A5, 0x03A6, 0x03A7, 0x03A8, 0x03A9, 0x03B1, 0x03B2, 0x03B3, 0x03B4, 0x03B5, 0x03B6, 0x03B7, 0x03B8, 0x03B9, 0x03BA, 0x03BB, 0x03BC,
|
||||
0x03BD, 0x03BE, 0x03BF, 0x03C0, 0x03C1, 0x03C3, 0x03C2, 0x03C4, 0x03C5, 0x03C6, 0x03C7, 0x03C8, 0x2591, 0x2592, 0x2593, 0x2502, 0x2524, 0x2561,
|
||||
0x2562, 0x2556, 0x2555, 0x2563, 0x2551, 0x2557, 0x255D, 0x255C, 0x255B, 0x2510, 0x2514, 0x2534, 0x252C, 0x251C, 0x2500, 0x253C, 0x255E, 0x255F,
|
||||
0x255A, 0x2554, 0x2569, 0x2566, 0x2560, 0x2550, 0x256C, 0x2567, 0x2568, 0x2564, 0x2565, 0x2559, 0x2558, 0x2552, 0x2553, 0x256B, 0x256A, 0x2518,
|
||||
0x250C, 0x2588, 0x2584, 0x258C, 0x2590, 0x2580, 0x03C9, 0x03AC, 0x03AD, 0x03AE, 0x03CA, 0x03AF, 0x03CC, 0x03CD, 0x03CB, 0x03CE, 0x0386, 0x0388,
|
||||
0x0389, 0x038A, 0x038C, 0x038E, 0x038F, 0x00B1, 0x2265, 0x2264, 0x03AA, 0x03AB, 0x00F7, 0x2248, 0x00B0, 0x2219, 0x00B7, 0x221A, 0x207F, 0x00B2,
|
||||
0x25A0, 0x00A0,
|
||||
];
|
||||
|
||||
/// OEM code page 857 (DOS Turkish; U+FFFD where the byte is unassigned) code points for 0x80..=0xFF (the Unicode Consortium's CP857
|
||||
/// mapping file).
|
||||
const CP857: [u16; 128] = [
|
||||
0x00C7, 0x00FC, 0x00E9, 0x00E2, 0x00E4, 0x00E0, 0x00E5, 0x00E7, 0x00EA, 0x00EB, 0x00E8, 0x00EF, 0x00EE, 0x0131, 0x00C4, 0x00C5, 0x00C9, 0x00E6,
|
||||
0x00C6, 0x00F4, 0x00F6, 0x00F2, 0x00FB, 0x00F9, 0x0130, 0x00D6, 0x00DC, 0x00F8, 0x00A3, 0x00D8, 0x015E, 0x015F, 0x00E1, 0x00ED, 0x00F3, 0x00FA,
|
||||
0x00F1, 0x00D1, 0x011E, 0x011F, 0x00BF, 0x00AE, 0x00AC, 0x00BD, 0x00BC, 0x00A1, 0x00AB, 0x00BB, 0x2591, 0x2592, 0x2593, 0x2502, 0x2524, 0x00C1,
|
||||
0x00C2, 0x00C0, 0x00A9, 0x2563, 0x2551, 0x2557, 0x255D, 0x00A2, 0x00A5, 0x2510, 0x2514, 0x2534, 0x252C, 0x251C, 0x2500, 0x253C, 0x00E3, 0x00C3,
|
||||
0x255A, 0x2554, 0x2569, 0x2566, 0x2560, 0x2550, 0x256C, 0x00A4, 0x00BA, 0x00AA, 0x00CA, 0x00CB, 0x00C8, 0xFFFD, 0x00CD, 0x00CE, 0x00CF, 0x2518,
|
||||
0x250C, 0x2588, 0x2584, 0x00A6, 0x00CC, 0x2580, 0x00D3, 0x00DF, 0x00D4, 0x00D2, 0x00F5, 0x00D5, 0x00B5, 0xFFFD, 0x00D7, 0x00DA, 0x00DB, 0x00D9,
|
||||
0x00EC, 0x00FF, 0x00AF, 0x00B4, 0x00AD, 0x00B1, 0xFFFD, 0x00BE, 0x00B6, 0x00A7, 0x00F7, 0x00B8, 0x00B0, 0x00A8, 0x00B7, 0x00B9, 0x00B3, 0x00B2,
|
||||
0x25A0, 0x00A0,
|
||||
];
|
||||
|
||||
/// The glyphs the OEM code pages show for 0x01..=0x1F (smileys, card suits, arrows), as
|
||||
/// Windows' Alt+numpad entry produces them rather than the control characters.
|
||||
const OEM_GLYPHS: [u16; 31] = [
|
||||
0x263A, 0x263B, 0x2665, 0x2666, 0x2663, 0x2660, 0x2022, 0x25D8, 0x25CB, 0x25D9, 0x2642, 0x2640, 0x266A, 0x266B, 0x263C, 0x25BA, 0x25C4, 0x2195,
|
||||
0x203C, 0x00B6, 0x00A7, 0x25AC, 0x21A8, 0x2191, 0x2193, 0x2192, 0x2190, 0x221F, 0x2194, 0x25B2, 0x25BC,
|
||||
];
|
||||
|
||||
/// The OEM code pages [`oem_byte`] decodes.
|
||||
pub const OEM_CODE_PAGES: [u16; 6] = [437, 737, 850, 852, 857, 866];
|
||||
|
||||
/// One byte of an OEM code page in [`OEM_CODE_PAGES`] (any other code page reads as 437) as a
|
||||
/// char, with the graphic glyphs for 0x01..=0x1F and 0x7F (⌂); 0x00 stays NUL.
|
||||
pub fn oem_byte(code_page: u16, b: u8) -> char {
|
||||
let table: &[u16; 128] = match code_page {
|
||||
737 => &CP737,
|
||||
850 => &CP850,
|
||||
852 => &CP852,
|
||||
857 => &CP857,
|
||||
866 => &CP866,
|
||||
_ => &CP437,
|
||||
};
|
||||
let u = match b {
|
||||
0x01..=0x1F => OEM_GLYPHS.get(usize::from(b - 1)).copied(),
|
||||
0x7F => Some(0x2302),
|
||||
0x80..=0xFF => table.get(usize::from(b - 0x80)).copied(),
|
||||
_ => Some(u16::from(b)),
|
||||
};
|
||||
u.and_then(|u| char::from_u32(u32::from(u))).unwrap_or('\u{FFFD}')
|
||||
}
|
||||
|
||||
/// The single-byte Windows code pages [`windows_byte`] decodes (1252 included).
|
||||
pub const SINGLE_BYTE_CODE_PAGES: [u16; 10] = [874, 1250, 1251, 1252, 1253, 1254, 1255, 1256, 1257, 1258];
|
||||
|
||||
/// Mac OS Roman, RTF's `\mac` character set.
|
||||
pub const MAC_ROMAN_CODE_PAGE: u16 = 10000;
|
||||
|
||||
/// One byte of a single-byte code page as a char: a Windows one in [`SINGLE_BYTE_CODE_PAGES`] or
|
||||
/// Mac OS Roman ([`MAC_ROMAN_CODE_PAGE`]); any other code page reads as Windows-1252.
|
||||
pub fn windows_byte(code_page: u16, b: u8) -> char {
|
||||
let table: &[u16; 128] = match code_page {
|
||||
874 => &CP874,
|
||||
1250 => &CP1250,
|
||||
1251 => &CP1251,
|
||||
1253 => &CP1253,
|
||||
1254 => &CP1254,
|
||||
1255 => &CP1255,
|
||||
1256 => &CP1256,
|
||||
1257 => &CP1257,
|
||||
1258 => &CP1258,
|
||||
MAC_ROMAN_CODE_PAGE => &MAC_ROMAN,
|
||||
_ => return cp1252(b),
|
||||
};
|
||||
if b < 0x80 {
|
||||
return b as char;
|
||||
}
|
||||
table.get(usize::from(b - 0x80)).and_then(|u| char::from_u32(u32::from(*u))).unwrap_or('\u{FFFD}')
|
||||
}
|
||||
|
||||
/// Whether [`code_page_byte`] decodes `code_page`: a Windows one ([`SINGLE_BYTE_CODE_PAGES`]), an
|
||||
/// OEM (DOS) one ([`OEM_CODE_PAGES`]) or Mac OS Roman. Document formats (RTF) name all three.
|
||||
pub fn is_byte_code_page(code_page: u16) -> bool {
|
||||
SINGLE_BYTE_CODE_PAGES.contains(&code_page) || OEM_CODE_PAGES.contains(&code_page) || code_page == MAC_ROMAN_CODE_PAGE
|
||||
}
|
||||
|
||||
/// Windows' symbol code page (`CP_SYMBOL`), the charset of symbol fonts (`\fcharset2`): a byte is
|
||||
/// the font's own code, not a letter. [`code_page_byte`] reads one from 0x80 as U+F000 + the byte,
|
||||
/// the form a Word symbol character takes (`w:sym w:char="F0D8"`).
|
||||
pub const SYMBOL_CODE_PAGE: u16 = 42;
|
||||
|
||||
/// A byte of 8-bit document text in `code_page` (see [`is_byte_code_page`], and
|
||||
/// [`SYMBOL_CODE_PAGE`] for a symbol font) as a char. Below 0x80 it is ASCII in every code page
|
||||
/// (text, so an OEM page's glyphs for control codes don't apply).
|
||||
pub fn code_page_byte(code_page: u16, b: u8) -> char {
|
||||
if b >= 0x80 && code_page == SYMBOL_CODE_PAGE {
|
||||
return char::from_u32(0xF000 + u32::from(b)).unwrap_or('\u{FFFD}');
|
||||
}
|
||||
if b >= 0x80 && OEM_CODE_PAGES.contains(&code_page) { oem_byte(code_page, b) } else { windows_byte(code_page, b) }
|
||||
}
|
||||
|
||||
/// The East Asian multibyte code pages a document format names with `\ansicpg` or a font's `\cpg`:
|
||||
/// 932 Shift-JIS (Japanese), 936 GBK (Simplified Chinese), 949 UHC (Korean) and 950 Big5
|
||||
/// (Traditional Chinese).
|
||||
pub const MULTIBYTE_CODE_PAGES: [u16; 4] = [932, 936, 949, 950];
|
||||
|
||||
/// Whether `code_page` is one of the East Asian multibyte code pages ([`MULTIBYTE_CODE_PAGES`]):
|
||||
/// its bytes decode as a run, a lead byte together with its trail byte.
|
||||
pub fn is_multibyte_code_page(code_page: u16) -> bool {
|
||||
multibyte_encoding_name(code_page).is_some()
|
||||
}
|
||||
|
||||
/// The WHATWG encoding name of an East Asian multibyte code page (see [`MULTIBYTE_CODE_PAGES`]),
|
||||
/// for `encoding_rs::Encoding::for_label`; `None` for the single-byte code pages, whose bytes
|
||||
/// decode one at a time, and anything else.
|
||||
pub fn multibyte_encoding_name(code_page: u16) -> Option<&'static str> {
|
||||
Some(match code_page {
|
||||
932 => "shift_jis",
|
||||
936 => "gbk",
|
||||
949 => "euc-kr",
|
||||
950 => "big5",
|
||||
_ => return None,
|
||||
})
|
||||
}
|
||||
|
||||
/// Whether a document format can name `code_page` for its text: a single-byte Windows, DOS or Mac
|
||||
/// one ([`is_byte_code_page`]) or an East Asian multibyte one ([`is_multibyte_code_page`]).
|
||||
pub fn is_text_code_page(code_page: u16) -> bool {
|
||||
is_byte_code_page(code_page) || is_multibyte_code_page(code_page)
|
||||
}
|
||||
|
||||
/// The code page of an RTF or Word font charset (`\fcharset`): 204 Russian → 1251, 238 Eastern
|
||||
/// European → 1250, 77 Mac → 10000, 254 PC / 255 OEM → 437, 128 Japanese → 932, 129 Korean → 949,
|
||||
/// 134 Simplified Chinese → 936, 136 Traditional Chinese → 950, 2 Symbol → [`SYMBOL_CODE_PAGE`]…
|
||||
/// `None` for the ANSI and default charsets (the document's code page applies).
|
||||
pub fn charset_code_page(charset: i32) -> Option<u16> {
|
||||
Some(match charset {
|
||||
2 => SYMBOL_CODE_PAGE,
|
||||
77 => MAC_ROMAN_CODE_PAGE,
|
||||
128 => 932,
|
||||
129 => 949,
|
||||
134 => 936,
|
||||
136 => 950,
|
||||
161 => 1253,
|
||||
162 => 1254,
|
||||
163 => 1258,
|
||||
177 => 1255,
|
||||
178 => 1256,
|
||||
186 => 1257,
|
||||
204 => 1251,
|
||||
222 => 874,
|
||||
238 => 1250,
|
||||
254 | 255 => 437,
|
||||
_ => return None,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn single_byte_code_pages() {
|
||||
let decode = |cp: u16, bytes: &[u8]| bytes.iter().map(|&b| windows_byte(cp, b)).collect::<String>();
|
||||
assert_eq!(decode(1251, &[0xD1, 0xF7, 0xE5, 0xF2, 0x20, 0xB9]), "Счет №");
|
||||
assert_eq!(decode(1251, &[0xA8, 0xB8]), "Ёё");
|
||||
assert_eq!(decode(1250, &[0xA3, 0xF3, 0x9F]), "Łóź");
|
||||
assert_eq!(decode(1253, &[0xC1, 0xE2]), "Αβ");
|
||||
assert_eq!(decode(1252, &[0x80, 0xE9]), "ێ");
|
||||
assert_eq!(decode(9999, &[0xE9]), "é");
|
||||
// Mac OS Roman: accents, the euro sign at 0xDB, typographic quotes.
|
||||
assert_eq!(decode(10000, &[0x80, 0x8E, 0xDB, 0xF1, 0xF2, 0xF3, 0xD2, 0xD3, 0x41]), "Äé€ÒÚÛ“”A");
|
||||
assert_eq!(charset_code_page(77), Some(10000));
|
||||
assert_eq!(charset_code_page(254), Some(437));
|
||||
assert_eq!(charset_code_page(255), Some(437));
|
||||
assert_eq!(charset_code_page(204), Some(1251));
|
||||
assert_eq!(charset_code_page(0), None);
|
||||
for cp in SINGLE_BYTE_CODE_PAGES {
|
||||
for b in 0..=255u8 {
|
||||
let _ = windows_byte(cp, b);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn windows_1255_holam_haser_for_vav() {
|
||||
assert_eq!(windows_byte(1255, 0xCA), '\u{05BA}');
|
||||
}
|
||||
|
||||
/// One decoder for document text in any code page RTF names: Windows, DOS or Mac.
|
||||
#[test]
|
||||
fn document_text_in_windows_dos_and_mac_code_pages() {
|
||||
let decode = |cp: u16, bytes: &[u8]| bytes.iter().map(|&b| code_page_byte(cp, b)).collect::<String>();
|
||||
assert_eq!(decode(437, &[0x82, 0xA0, 0x41, 0x01]), "éáA\u{1}", "DOS text; ASCII stays ASCII");
|
||||
assert_eq!(decode(850, &[0x9D, 0xE9]), "ØÚ");
|
||||
assert_eq!(decode(10000, &[0x8E]), "é");
|
||||
assert_eq!(decode(1251, &[0xC4]), "Д");
|
||||
// A symbol font's bytes are its own codes, from 0x80 the private-use form; ASCII stays ASCII.
|
||||
assert_eq!(charset_code_page(2), Some(SYMBOL_CODE_PAGE));
|
||||
assert_eq!(decode(SYMBOL_CODE_PAGE, &[0x4A, 0x93, 0xD8, 0xFC]), "J\u{F093}\u{F0D8}\u{F0FC}");
|
||||
for cp in [437, 850, 866, 1250, 1252, 10000] {
|
||||
assert!(is_byte_code_page(cp), "{cp}");
|
||||
}
|
||||
assert!(!is_byte_code_page(932) && !is_byte_code_page(0) && !SINGLE_BYTE_CODE_PAGES.contains(&10000));
|
||||
}
|
||||
|
||||
/// The East Asian charsets and code pages RTF names are multibyte: their bytes decode
|
||||
/// as runs (a lead byte and its trail byte together), not one at a time.
|
||||
#[test]
|
||||
fn east_asian_code_pages_are_multibyte() {
|
||||
assert_eq!(charset_code_page(128), Some(932), "Japanese");
|
||||
assert_eq!(charset_code_page(129), Some(949), "Korean");
|
||||
assert_eq!(charset_code_page(134), Some(936), "Simplified Chinese");
|
||||
assert_eq!(charset_code_page(136), Some(950), "Traditional Chinese");
|
||||
assert_eq!(charset_code_page(255), Some(437), "OEM is single-byte");
|
||||
assert_eq!(MULTIBYTE_CODE_PAGES, [932, 936, 949, 950]);
|
||||
for cp in MULTIBYTE_CODE_PAGES {
|
||||
assert!(is_multibyte_code_page(cp) && is_text_code_page(cp), "{cp}");
|
||||
assert!(!is_byte_code_page(cp) && !SINGLE_BYTE_CODE_PAGES.contains(&cp), "{cp}");
|
||||
assert!(multibyte_encoding_name(cp).is_some(), "{cp}");
|
||||
}
|
||||
assert!(is_text_code_page(1251) && is_text_code_page(437) && is_text_code_page(10000));
|
||||
assert!(!is_text_code_page(0) && !is_text_code_page(9999));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn oem_code_pages() {
|
||||
let decode = |cp: u16, bytes: &[u8]| bytes.iter().map(|&b| oem_byte(cp, b)).collect::<String>();
|
||||
assert_eq!(decode(437, &[0x82, 0xE9, 0x01, 0x7F]), "éΘ☺⌂");
|
||||
assert_eq!(decode(850, &[0x9B, 0xE9]), "øÚ");
|
||||
assert_eq!(decode(852, &[0xA5, 0x88, 0xBE, 0xA9]), "ąłżę");
|
||||
assert_eq!(decode(866, &[0x8F, 0xE0, 0xA8, 0xA2, 0xA5, 0xE2, 0xF0, 0xFC]), "ПриветЁ№");
|
||||
assert_eq!(decode(737, &[0x80, 0x99, 0xE0]), "Αβω");
|
||||
assert_eq!(decode(857, &[0x8D, 0x98, 0xA6, 0x9F]), "ıİĞş");
|
||||
assert_eq!(oem_byte(857, 0xD5), '\u{FFFD}', "unassigned in 857");
|
||||
assert_eq!(decode(9999, &[0xE9]), "Θ", "unknown pages read as 437");
|
||||
for cp in OEM_CODE_PAGES {
|
||||
for b in 0..=255u8 {
|
||||
let _ = oem_byte(cp, b);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
//! RTF dates: the DTTM bit pattern of revision marks, and the `\yr\mo\dy\hr\min` words of the
|
||||
//! document information group.
|
||||
|
||||
/// An ISO 8601 date-time from year, month, day, hour and minute (the parameters of an `\info`
|
||||
/// date's `\yr \mo \dy \hr \min`), each clamped into range.
|
||||
pub fn iso_date((y, mo, d, h, mi): (u32, u32, u32, u32, u32)) -> String {
|
||||
format!("{y:04}-{:02}-{:02}T{:02}:{:02}:00Z", mo.clamp(1, 12), d.clamp(1, 31), h.min(23), mi.min(59))
|
||||
}
|
||||
|
||||
/// An ISO 8601 date (`YYYY-MM-DD`, optionally `THH:MM`) as the words of an `\info` date:
|
||||
/// `\yr2024\mo3\dy15\hr10\min30`.
|
||||
pub fn date_words(s: &str) -> Option<String> {
|
||||
let num = |a: usize, b: usize| s.get(a..b).and_then(|x| x.parse::<u32>().ok());
|
||||
let (y, mo, d) = (num(0, 4)?, num(5, 7)?, num(8, 10)?);
|
||||
let (h, mi) = (num(11, 13).unwrap_or(0), num(14, 16).unwrap_or(0));
|
||||
Some(format!("\\yr{y}\\mo{mo}\\dy{d}\\hr{h}\\min{mi}"))
|
||||
}
|
||||
|
||||
fn days_in_month(y: u32, m: u32) -> u32 {
|
||||
match m {
|
||||
1 | 3 | 5 | 7 | 8 | 10 | 12 => 31,
|
||||
4 | 6 | 9 | 11 => 30,
|
||||
2 if (y.is_multiple_of(4) && !y.is_multiple_of(100)) || y.is_multiple_of(400) => 29,
|
||||
2 => 28,
|
||||
_ => 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// Day of the week, 0 = Sunday (Sakamoto's method).
|
||||
fn weekday(y: u32, m: u32, d: u32) -> u32 {
|
||||
const T: [u32; 12] = [0, 3, 2, 5, 0, 3, 5, 1, 4, 6, 2, 4];
|
||||
let y = if m < 3 { y.saturating_sub(1) } else { y };
|
||||
(y + y / 4 - y / 100 + y / 400 + T.get(m.saturating_sub(1) as usize).copied().unwrap_or(0) + d) % 7
|
||||
}
|
||||
|
||||
/// An RTF revision date (DTTM, RTF 1.9.1 "Revision Marks": minute in bits 0–5, hour 6–10, day
|
||||
/// 11–15, month 16–19, year − 1900 20–28, weekday 29–31) as ISO 8601 (`2024-03-15T10:30:00Z`).
|
||||
/// Zero or an impossible date is no date. DTTM has no time zone or seconds: times are read as
|
||||
/// UTC (`Z`).
|
||||
pub fn dttm_to_iso(v: u32) -> Option<String> {
|
||||
let (mi, h, d, mo, y) = (v & 0x3F, (v >> 6) & 0x1F, (v >> 11) & 0x1F, (v >> 16) & 0xF, 1900 + ((v >> 20) & 0x1FF));
|
||||
if v == 0 || mi > 59 || h > 23 || d == 0 || d > days_in_month(y, mo) {
|
||||
return None;
|
||||
}
|
||||
Some(format!("{y:04}-{mo:02}-{d:02}T{h:02}:{mi:02}:00Z"))
|
||||
}
|
||||
|
||||
/// An ISO 8601 date (`YYYY-MM-DD`, optionally `THH:MM…`; seconds and zone ignored) as an RTF
|
||||
/// DTTM; `None` when it isn't one or falls outside DTTM's years (1900–2411). DTTM has no time
|
||||
/// zone or seconds: the clock time is written as given (a `Z` time stays the UTC clock time), and seconds
|
||||
/// and any offset are dropped.
|
||||
pub fn iso_to_dttm(s: &str) -> Option<u32> {
|
||||
let num = |a: usize, b: usize| s.get(a..b).filter(|t| t.bytes().all(|c| c.is_ascii_digit())).and_then(|t| t.parse::<u32>().ok());
|
||||
let sep = |i: usize, c: &[&str]| s.get(i..i + 1).is_some_and(|t| c.contains(&t));
|
||||
let (y, mo, d) = (num(0, 4)?, num(5, 7)?, num(8, 10)?);
|
||||
if !sep(4, &["-"]) || !sep(7, &["-"]) {
|
||||
return None;
|
||||
}
|
||||
let (h, mi) = if s.len() > 10 {
|
||||
if !sep(10, &["T", " "]) || !sep(13, &[":"]) {
|
||||
return None;
|
||||
}
|
||||
(num(11, 13)?, num(14, 16)?)
|
||||
} else {
|
||||
(0, 0)
|
||||
};
|
||||
if !(1900..=2411).contains(&y) || mi > 59 || h > 23 || d == 0 || d > days_in_month(y, mo) {
|
||||
return None;
|
||||
}
|
||||
Some(mi | (h << 6) | (d << 11) | (mo << 16) | ((y - 1900) << 20) | (weekday(y, mo, d) << 29))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn dttm_round_trips_and_refuses_impossible_dates() {
|
||||
let v = iso_to_dttm("2024-03-15T10:30:59+02:00").unwrap();
|
||||
assert_eq!(dttm_to_iso(v).as_deref(), Some("2024-03-15T10:30:00Z"));
|
||||
assert_eq!(v >> 29, 5, "a Friday");
|
||||
assert_eq!(iso_to_dttm("2024-02-29").and_then(dttm_to_iso).as_deref(), Some("2024-02-29T00:00:00Z"));
|
||||
for bad in ["2023-02-29", "1899-01-01", "2412-01-01", "2024-13-01", "2024-03-15T24:00", "2024/03/15", "2024-03-15x10:30", "", "+024-03-15"] {
|
||||
assert_eq!(iso_to_dttm(bad), None, "{bad}");
|
||||
}
|
||||
assert_eq!(dttm_to_iso(0), None);
|
||||
assert_eq!(dttm_to_iso(u32::MAX), None);
|
||||
// Day 31 of a 30-day month, hour 24, minute 60.
|
||||
assert_eq!(dttm_to_iso((31 << 11) | (4 << 16) | (124 << 20)), None);
|
||||
assert_eq!(dttm_to_iso((24 << 6) | (1 << 11) | (1 << 16) | (124 << 20)), None);
|
||||
assert_eq!(dttm_to_iso(60 | (1 << 11) | (1 << 16) | (124 << 20)), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn info_dates() {
|
||||
assert_eq!(date_words("2024-03-15T10:30:00Z").as_deref(), Some("\\yr2024\\mo3\\dy15\\hr10\\min30"));
|
||||
assert_eq!(date_words("2024-03-15").as_deref(), Some("\\yr2024\\mo3\\dy15\\hr0\\min0"));
|
||||
assert_eq!(date_words("soon"), None);
|
||||
assert_eq!(iso_date((2024, 3, 15, 10, 30)), "2024-03-15T10:30:00Z");
|
||||
assert_eq!(iso_date((2024, 0, 99, 99, 99)), "2024-01-31T23:59:00Z", "clamped");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,231 @@
|
||||
//! The tokens of an RTF file. Nothing is interpreted here: a token is what the bytes say.
|
||||
|
||||
/// Is this an RTF file? Leading white space, then `{\rtf`.
|
||||
pub fn is_rtf(bytes: &[u8]) -> bool {
|
||||
bytes.trim_ascii_start().starts_with(b"{\\rtf")
|
||||
}
|
||||
|
||||
/// One token.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum Token<'a> {
|
||||
/// `{`
|
||||
Open,
|
||||
/// `}`
|
||||
Close,
|
||||
/// A control word: `\par`, `\fs24`, `\li-720`. The parameter is clamped into `i32`.
|
||||
Word { name: &'a str, param: Option<i32> },
|
||||
/// `\*`: the destination that follows may be ignored by a reader that does not know it.
|
||||
Star,
|
||||
/// `\'hh`: one byte of text in the current code page.
|
||||
Hex(u8),
|
||||
/// Any other control symbol, by the byte after the backslash: `\~`, `\-`, `\_`, `\\`,
|
||||
/// `\{`, `\}`, and `\` before a line end (which is `\par`).
|
||||
Symbol(u8),
|
||||
/// The data of `\binN`: the `N` bytes after the word's delimiter, as they are.
|
||||
Bin(&'a [u8]),
|
||||
/// A run of text bytes in the current code page: no `{`, `}`, `\`, line end or NUL.
|
||||
Text(&'a [u8]),
|
||||
}
|
||||
|
||||
/// The tokens of `bytes`, in order. Never fails: a malformed escape is dropped and whatever
|
||||
/// follows it is read as the structure it is.
|
||||
///
|
||||
/// - A control word is at most 32 letters and its parameter at most 10 digits; what follows is
|
||||
/// text. A `-` starts a parameter only when a digit follows it.
|
||||
/// - One space after a control word is its delimiter and not text.
|
||||
/// - Line ends and NUL bytes between tokens are not tokens.
|
||||
/// - `\'` without two hex digits gives no token: it and the hex digits it does have are
|
||||
/// skipped, and a `}` or `\` after it is read as itself.
|
||||
/// - A date word (`\revdttm`, `\revdttmdel`, `\crdate`, `\prdate`) holds a 32-bit pattern that
|
||||
/// some writers put unsigned: it is read as that pattern, not clamped.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct Tokens<'a> {
|
||||
bytes: &'a [u8],
|
||||
at: usize,
|
||||
}
|
||||
|
||||
impl<'a> Tokens<'a> {
|
||||
pub fn new(bytes: &'a [u8]) -> Tokens<'a> {
|
||||
Tokens { bytes, at: 0 }
|
||||
}
|
||||
|
||||
/// The offset of the next token.
|
||||
pub fn offset(&self) -> usize {
|
||||
self.at
|
||||
}
|
||||
|
||||
fn word(&mut self) -> Token<'a> {
|
||||
let b = self.bytes;
|
||||
let start = self.at + 1;
|
||||
let mut j = start;
|
||||
while b.get(j).is_some_and(u8::is_ascii_alphabetic) && j - start < 32 {
|
||||
j += 1;
|
||||
}
|
||||
let name = b.get(start..j).and_then(|w| std::str::from_utf8(w).ok()).unwrap_or("");
|
||||
let neg = b.get(j) == Some(&b'-');
|
||||
let digits = j + usize::from(neg);
|
||||
let mut k = digits;
|
||||
while b.get(k).is_some_and(u8::is_ascii_digit) && k - digits < 10 {
|
||||
k += 1;
|
||||
}
|
||||
let param = if k > digits {
|
||||
// At most 10 digits, so it fits an i64.
|
||||
let v = b.get(digits..k).and_then(|d| std::str::from_utf8(d).ok()).and_then(|d| d.parse::<i64>().ok()).unwrap_or(0);
|
||||
let v = if neg { -v } else { v };
|
||||
let dttm = matches!(name, "revdttm" | "revdttmdel" | "crdate" | "prdate");
|
||||
let v = if dttm && v > i64::from(i32::MAX) && v <= i64::from(u32::MAX) { v - (1i64 << 32) } else { v };
|
||||
Some(v.clamp(i64::from(i32::MIN), i64::from(i32::MAX)) as i32)
|
||||
} else {
|
||||
k = j;
|
||||
None
|
||||
};
|
||||
if b.get(k) == Some(&b' ') {
|
||||
k += 1;
|
||||
}
|
||||
self.at = k;
|
||||
if name == "bin" {
|
||||
let n = usize::try_from(param.unwrap_or(0)).unwrap_or(0);
|
||||
let end = k.saturating_add(n).min(b.len());
|
||||
self.at = end;
|
||||
return Token::Bin(b.get(k..end).unwrap_or_default());
|
||||
}
|
||||
Token::Word { name, param }
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> Iterator for Tokens<'a> {
|
||||
type Item = Token<'a>;
|
||||
|
||||
fn next(&mut self) -> Option<Token<'a>> {
|
||||
let b = self.bytes;
|
||||
loop {
|
||||
let c = *b.get(self.at)?;
|
||||
match c {
|
||||
b'{' | b'}' => {
|
||||
self.at += 1;
|
||||
return Some(if c == b'{' { Token::Open } else { Token::Close });
|
||||
}
|
||||
b'\r' | b'\n' | 0 => self.at += 1,
|
||||
b'\\' => {
|
||||
let n = *b.get(self.at + 1)?;
|
||||
if n.is_ascii_alphabetic() {
|
||||
return Some(self.word());
|
||||
}
|
||||
self.at += 2;
|
||||
match n {
|
||||
b'\'' => {
|
||||
let hex = b.get(self.at..self.at + 2).filter(|h| h.iter().all(u8::is_ascii_hexdigit));
|
||||
if let Some(v) = hex.and_then(|h| std::str::from_utf8(h).ok()).and_then(|h| u8::from_str_radix(h, 16).ok()) {
|
||||
self.at += 2;
|
||||
return Some(Token::Hex(v));
|
||||
}
|
||||
// Malformed: skip the hex digits of a truncated pair, nothing else.
|
||||
let mut n = 0;
|
||||
while n < 2 && b.get(self.at).is_some_and(u8::is_ascii_hexdigit) {
|
||||
self.at += 1;
|
||||
n += 1;
|
||||
}
|
||||
}
|
||||
b'*' => return Some(Token::Star),
|
||||
b'\r' => return Some(Token::Symbol(b'\n')),
|
||||
n => return Some(Token::Symbol(n)),
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
let start = self.at;
|
||||
while b.get(self.at).is_some_and(|c| !matches!(c, b'{' | b'}' | b'\\' | b'\r' | b'\n' | 0)) {
|
||||
self.at += 1;
|
||||
}
|
||||
return Some(Token::Text(b.get(start..self.at).unwrap_or_default()));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use Token::{Bin, Close, Hex, Open, Star, Symbol, Text};
|
||||
|
||||
fn w(name: &str, param: Option<i32>) -> Token<'_> {
|
||||
Token::Word { name, param }
|
||||
}
|
||||
|
||||
fn lex(s: &[u8]) -> Vec<Token<'_>> {
|
||||
Tokens::new(s).collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn groups_words_symbols_and_text() {
|
||||
assert_eq!(
|
||||
lex(br"{\rtf1\ansi {\*\generator X;}\b Hi\b0 there\par\~\'e9}"),
|
||||
vec![
|
||||
Open,
|
||||
w("rtf", Some(1)),
|
||||
w("ansi", None),
|
||||
Open,
|
||||
Star,
|
||||
w("generator", None),
|
||||
Text(b"X;"),
|
||||
Close,
|
||||
w("b", None),
|
||||
Text(b"Hi"),
|
||||
w("b", Some(0)),
|
||||
Text(b" there"),
|
||||
w("par", None),
|
||||
Symbol(b'~'),
|
||||
Hex(0xE9),
|
||||
Close
|
||||
]
|
||||
);
|
||||
assert!(is_rtf(b" \r\n{\\rtf1}") && !is_rtf(b"{rtf1}") && !is_rtf(b""));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_hyphen_is_a_sign_only_before_a_digit() {
|
||||
assert_eq!(
|
||||
lex(br"\li-720 a\emdash-roll\tab- x"),
|
||||
vec![w("li", Some(-720)), Text(b"a"), w("emdash", None), Text(b"-roll"), w("tab", None), Text(b"- x")]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn long_words_and_parameters_are_cut_not_scanned_without_end() {
|
||||
let long = format!("\\{}1", "a".repeat(40));
|
||||
assert_eq!(lex(long.as_bytes()), vec![w(&"a".repeat(32), None), Text(b"aaaaaaaa1")]);
|
||||
assert_eq!(lex(br"\fs123456789012 x"), vec![w("fs", Some(1_234_567_890)), Text(b"12 x")]);
|
||||
assert_eq!(lex(br"\fs9999999999\li-9999999999"), vec![w("fs", Some(i32::MAX)), w("li", Some(i32::MIN))]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dates_keep_their_bit_pattern() {
|
||||
assert_eq!(
|
||||
lex(br"\revdttm4294967295\crdate2147483648\fs4294967295"),
|
||||
vec![w("revdttm", Some(-1)), w("crdate", Some(i32::MIN)), w("fs", Some(i32::MAX))]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn malformed_hex_escapes_leave_the_structure_alone() {
|
||||
assert_eq!(lex(br"{\b A\'4}B"), vec![Open, w("b", None), Text(b"A"), Close, Text(b"B")]);
|
||||
assert_eq!(lex(br"a\'+fb\'\par"), vec![Text(b"a"), Text(b"+fb"), w("par", None)]);
|
||||
assert_eq!(lex(br"\'zz\'4"), vec![Text(b"zz")]);
|
||||
assert_eq!(lex(b"a\\"), vec![Text(b"a")], "a backslash at the end is dropped");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn binary_data_is_not_scanned() {
|
||||
assert_eq!(lex(b"{\\pict\\bin5 {\\}\0\\x}"), vec![Open, w("pict", None), Bin(b"{\\}\0\\"), Text(b"x"), Close]);
|
||||
assert_eq!(lex(b"\\bin99 ab"), vec![Bin(b"ab")], "cut at the end of the file");
|
||||
assert_eq!(lex(b"\\bin-4 ab\\bin"), vec![Bin(b""), Text(b"ab"), Bin(b"")]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn line_ends_and_nul_are_not_text() {
|
||||
assert_eq!(lex(b"a\r\nb\0c\\\nd\\\r\ne"), vec![Text(b"a"), Text(b"b"), Text(b"c"), Symbol(b'\n'), Text(b"d"), Symbol(b'\n'), Text(b"e")]);
|
||||
assert_eq!(lex(b"\\\xe9x"), vec![Symbol(0xE9), Text(b"x")]);
|
||||
let mut t = Tokens::new(b"ab{");
|
||||
assert_eq!((t.next(), t.offset()), (Some(Text(b"ab")), 2));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
|
||||
//! Rich Text Format.
|
||||
//!
|
||||
//! Three layers, each usable alone:
|
||||
//!
|
||||
//! - [`Tokens`]: the tokens of a file (groups, control words, `\'hh`, text runs, `\bin` data).
|
||||
//! - [`TextDecoder`] and [`codepage`]: text in the file's code pages, `\uN` with its fallback
|
||||
//! characters and surrogate pairs. [`esc`] goes the other way. [`dttm_to_iso`],
|
||||
//! [`iso_to_dttm`], [`date_words`] and [`iso_date`] are RTF's two date forms.
|
||||
//! - [`read`] and [`write`]: a file to and from a small neutral model ([`Document`]):
|
||||
//! paragraphs with outline level, alignment, indents and spacing, runs with bold, italic,
|
||||
//! underline, strikethrough, superscript and subscript, size, colour and font, and tables.
|
||||
//!
|
||||
//! A reader that keeps more than the model holds (styles, lists, fields, notes, pictures) is
|
||||
//! built on the first two layers.
|
||||
//!
|
||||
//! ```
|
||||
//! use craft_rtf::{Block, read, write};
|
||||
//!
|
||||
//! let doc = read(br"{\rtf1\ansi\pard\outlinelevel0 Caf\'e9 {\b \u8364?5}\par}")?;
|
||||
//! let Some(Block::Para(p)) = doc.blocks.first() else { return Ok(()) };
|
||||
//! assert_eq!((p.text().as_str(), p.outline_level), ("Café €5", Some(0)));
|
||||
//! assert!(p.runs[1].fmt.bold);
|
||||
//! assert_eq!(read(write(&doc, "Helvetica").as_bytes())?, doc);
|
||||
//! # Ok::<(), craft_rtf::NotRtf>(())
|
||||
//! ```
|
||||
|
||||
#[cfg(doctest)]
|
||||
#[doc = include_str!("../README.md")]
|
||||
struct ReadmeExamples;
|
||||
|
||||
pub mod codepage;
|
||||
mod date;
|
||||
mod lex;
|
||||
mod model;
|
||||
mod read;
|
||||
mod text;
|
||||
mod write;
|
||||
|
||||
pub use date::{date_words, dttm_to_iso, iso_date, iso_to_dttm};
|
||||
pub use lex::{Token, Tokens, is_rtf};
|
||||
pub use model::{Align, Block, COLUMN_BREAK, Cell, CharFmt, Document, LINE_BREAK, PAGE_BREAK, Para, Run, Table, VertAlign};
|
||||
pub use read::{MAX_COLORS, MAX_FONT_NAME, MAX_FONTS, MAX_GROUPS, NotRtf, read};
|
||||
pub use text::TextDecoder;
|
||||
pub use write::{esc, esc_font, push_esc, write};
|
||||
@@ -0,0 +1,124 @@
|
||||
//! A small neutral model of formatted text: what the reader gives and the writer takes.
|
||||
|
||||
/// A line break inside a paragraph (`\line`), in [`Run::text`].
|
||||
pub const LINE_BREAK: char = '\n';
|
||||
/// A page break (`\page`), in [`Run::text`].
|
||||
pub const PAGE_BREAK: char = '\u{000C}';
|
||||
/// A column break (`\column`), in [`Run::text`].
|
||||
pub const COLUMN_BREAK: char = '\u{000E}';
|
||||
|
||||
/// Where a run sits on the line.
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
||||
pub enum VertAlign {
|
||||
#[default]
|
||||
Baseline,
|
||||
/// `\super`
|
||||
Superscript,
|
||||
/// `\sub`
|
||||
Subscript,
|
||||
}
|
||||
|
||||
/// How a paragraph's lines are set.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum Align {
|
||||
/// `\ql`
|
||||
Left,
|
||||
/// `\qc`
|
||||
Center,
|
||||
/// `\qr`
|
||||
Right,
|
||||
/// `\qj`
|
||||
Justified,
|
||||
}
|
||||
|
||||
/// Character formatting.
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
||||
pub struct CharFmt {
|
||||
pub bold: bool,
|
||||
pub italic: bool,
|
||||
pub underline: bool,
|
||||
pub strike: bool,
|
||||
pub vert_align: VertAlign,
|
||||
/// `\fsN`: the size in half-points (24 is 12 pt). `None`: the reader's default.
|
||||
pub half_points: Option<u32>,
|
||||
/// `\cfN` through the colour table, as red, green, blue. `None`: the default colour.
|
||||
pub color: Option<[u8; 3]>,
|
||||
/// `\fN` through the font table. `None`: the default font.
|
||||
pub font: Option<String>,
|
||||
}
|
||||
|
||||
/// Text in one formatting. Tabs are `\t`; breaks are [`LINE_BREAK`], [`PAGE_BREAK`] and
|
||||
/// [`COLUMN_BREAK`].
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
||||
pub struct Run {
|
||||
pub text: String,
|
||||
pub fmt: CharFmt,
|
||||
}
|
||||
|
||||
/// A paragraph.
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
||||
pub struct Para {
|
||||
pub runs: Vec<Run>,
|
||||
/// `\outlinelevelN`, 0 to 8: a heading of level N + 1. `None`: body text.
|
||||
pub outline_level: Option<u8>,
|
||||
/// `\liN`: the left indent in twips (1/1440 inch).
|
||||
pub left_indent: i32,
|
||||
/// `\riN`: the right indent in twips.
|
||||
pub right_indent: i32,
|
||||
/// `\fiN`: the first line's indent in twips, on top of the left indent; negative for a
|
||||
/// hanging indent.
|
||||
pub first_line_indent: i32,
|
||||
/// `\sbN` and `\saN`: the space before and after in twips.
|
||||
pub space_before: i32,
|
||||
pub space_after: i32,
|
||||
/// `\ql`, `\qc`, `\qr`, `\qj`. `None`: the file does not say.
|
||||
pub align: Option<Align>,
|
||||
}
|
||||
|
||||
impl Para {
|
||||
/// The paragraph's text without its formatting.
|
||||
pub fn text(&self) -> String {
|
||||
self.runs.iter().map(|r| r.text.as_str()).collect()
|
||||
}
|
||||
}
|
||||
|
||||
/// A table cell.
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
||||
pub struct Cell {
|
||||
/// `\cellxN`: the cell's right edge from the table's left edge, in twips. A merged cell
|
||||
/// ends at the right edge of the last column it spans.
|
||||
pub right: i32,
|
||||
pub paras: Vec<Para>,
|
||||
}
|
||||
|
||||
/// A table: rows of cells.
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
||||
pub struct Table {
|
||||
pub rows: Vec<Vec<Cell>>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub enum Block {
|
||||
Para(Para),
|
||||
Table(Table),
|
||||
}
|
||||
|
||||
/// A document: the body's paragraphs and tables, in order.
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
||||
pub struct Document {
|
||||
pub blocks: Vec<Block>,
|
||||
/// Pictures the reader left out (`\pict`, `\object`).
|
||||
pub pictures_left_out: usize,
|
||||
}
|
||||
|
||||
impl Document {
|
||||
/// Every paragraph in reading order, a table's cells row by row.
|
||||
pub fn paras(&self) -> impl Iterator<Item = &Para> {
|
||||
self.blocks.iter().flat_map(|b| -> Box<dyn Iterator<Item = &Para> + '_> {
|
||||
match b {
|
||||
Block::Para(p) => Box::new(std::iter::once(p)),
|
||||
Block::Table(t) => Box::new(t.rows.iter().flatten().flat_map(|c| &c.paras)),
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,517 @@
|
||||
//! RTF into the model.
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use crate::codepage::{MAC_ROMAN_CODE_PAGE, SYMBOL_CODE_PAGE, charset_code_page, is_text_code_page};
|
||||
use crate::lex::{Token, Tokens, is_rtf};
|
||||
use crate::model::{Align, Block, COLUMN_BREAK, Cell, CharFmt, Document, LINE_BREAK, PAGE_BREAK, Para, Run, Table, VertAlign};
|
||||
use crate::text::TextDecoder;
|
||||
|
||||
/// Deepest group nesting followed. Deeper groups are read in the state of the group around
|
||||
/// them.
|
||||
pub const MAX_GROUPS: usize = 512;
|
||||
/// Most fonts kept from the font table.
|
||||
pub const MAX_FONTS: usize = 4096;
|
||||
/// Most colours kept from the colour table.
|
||||
pub const MAX_COLORS: usize = 4096;
|
||||
/// Longest font name kept, in bytes. A longer one is no font.
|
||||
pub const MAX_FONT_NAME: usize = 256;
|
||||
|
||||
/// The file does not start with `{\rtf`.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct NotRtf;
|
||||
|
||||
impl std::fmt::Display for NotRtf {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.write_str("not an RTF document (it doesn't start with {\\rtf)")
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for NotRtf {}
|
||||
|
||||
/// Destinations whose text is not body text. Any destination marked `\*` is skipped too.
|
||||
const SKIP: [&str; 30] = [
|
||||
"stylesheet",
|
||||
"info",
|
||||
"header",
|
||||
"headerl",
|
||||
"headerr",
|
||||
"headerf",
|
||||
"footer",
|
||||
"footerl",
|
||||
"footerr",
|
||||
"footerf",
|
||||
"footnote",
|
||||
"listtable",
|
||||
"listoverridetable",
|
||||
"rsidtbl",
|
||||
"generator",
|
||||
"themedata",
|
||||
"colorschememapping",
|
||||
"latentstyles",
|
||||
"datastore",
|
||||
"xmlnstbl",
|
||||
"fldinst",
|
||||
"revtbl",
|
||||
"filetbl",
|
||||
"annotation",
|
||||
"bkmkstart",
|
||||
"bkmkend",
|
||||
// A list item's rendered bullet or number, written before its text (Word: listtext; older
|
||||
// writers: pntext and its parts). Its `\pard` must not reset the paragraph's own indent.
|
||||
"listtext",
|
||||
"pntext",
|
||||
"pntxta",
|
||||
"pntxtb",
|
||||
];
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
||||
enum Dest {
|
||||
#[default]
|
||||
Body,
|
||||
Skip,
|
||||
FontTbl,
|
||||
ColorTbl,
|
||||
}
|
||||
|
||||
/// What a group saves and restores.
|
||||
#[derive(Clone, Debug)]
|
||||
struct St {
|
||||
dest: Dest,
|
||||
fmt: CharFmt,
|
||||
/// The code page of the current font, when its charset names one.
|
||||
cp: Option<u16>,
|
||||
/// A font was selected (`\f`) in this group or an enclosing one, and no `\plain` has since
|
||||
/// made the default font current again.
|
||||
font_set: bool,
|
||||
uc: usize,
|
||||
/// The paragraph properties in effect; their `runs` stay empty.
|
||||
para: Para,
|
||||
in_table: bool,
|
||||
}
|
||||
|
||||
impl Default for St {
|
||||
fn default() -> St {
|
||||
St { dest: Dest::Body, fmt: CharFmt::default(), cp: None, font_set: false, uc: 1, para: Para::default(), in_table: false }
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct Reader {
|
||||
st: St,
|
||||
stack: Vec<St>,
|
||||
/// Groups open beyond [`MAX_GROUPS`].
|
||||
overflow: usize,
|
||||
decoder: TextDecoder,
|
||||
/// `\ansicpg`, `\mac`, `\pc`, `\pca`; 0 until the file names one (Windows-1252 then).
|
||||
ansicpg: u16,
|
||||
deff: i32,
|
||||
fonts: HashMap<i32, String>,
|
||||
font_cps: HashMap<i32, u16>,
|
||||
font_cur: i32,
|
||||
font_cp_cur: Option<u16>,
|
||||
/// The font table entry being read named its code page with `\cpg`.
|
||||
font_cpg_named: bool,
|
||||
font_name: String,
|
||||
/// Where the text of a destination that keeps none goes.
|
||||
scrap: String,
|
||||
colors: Vec<Option<[u8; 3]>>,
|
||||
color_cur: ([u8; 3], bool),
|
||||
doc: Document,
|
||||
para: Para,
|
||||
text: String,
|
||||
rows: Vec<Vec<Cell>>,
|
||||
row: Vec<Cell>,
|
||||
cell: Vec<Para>,
|
||||
cellx: Vec<i32>,
|
||||
}
|
||||
|
||||
/// Reads an RTF file.
|
||||
///
|
||||
/// Kept: body text in the file's code pages (`\ansicpg`, `\mac`, `\pc`, `\pca`, a font's
|
||||
/// `\fcharset` or `\cpg`) and `\uN`; paragraphs (`\par`, `\sect`) with `\outlinelevel` and
|
||||
/// `\li`, `\ri`, `\fi`, `\sb`, `\sa` and alignment; bold, italic, underline, strikethrough,
|
||||
/// superscript and subscript, size, colour and font; tabs, line, page and
|
||||
/// column breaks, the quote, dash and bullet words; tables (`\cellx`, `\cell`, `\row`).
|
||||
///
|
||||
/// Left out: headers, footers, footnotes, comments, field instructions (a field's result is
|
||||
/// kept), list labels, the stylesheet, pictures and objects (counted in
|
||||
/// [`Document::pictures_left_out`]), and every destination marked `\*`. A table nested in a
|
||||
/// cell reads as that cell's text.
|
||||
///
|
||||
/// A trailing `\par` does not leave an empty last paragraph. Never fails on a file that starts
|
||||
/// with `{\rtf`: what cannot be read is skipped.
|
||||
pub fn read(bytes: &[u8]) -> Result<Document, NotRtf> {
|
||||
if !is_rtf(bytes) {
|
||||
return Err(NotRtf);
|
||||
}
|
||||
let mut r = Reader::default();
|
||||
let mut starred = false;
|
||||
for token in Tokens::new(bytes) {
|
||||
let star = std::mem::take(&mut starred);
|
||||
match token {
|
||||
Token::Open => r.open(),
|
||||
Token::Close => r.close(),
|
||||
Token::Star => starred = true,
|
||||
Token::Word { name, param } => r.word(name, param, star),
|
||||
Token::Hex(b) => r.hex(b),
|
||||
Token::Symbol(s) => r.symbol(s),
|
||||
Token::Text(t) => r.bytes(t),
|
||||
Token::Bin(_) => {}
|
||||
}
|
||||
}
|
||||
Ok(r.finish())
|
||||
}
|
||||
|
||||
impl Reader {
|
||||
/// The decoder, and where text goes in the current destination.
|
||||
fn sink(&mut self) -> (&mut TextDecoder, &mut String) {
|
||||
self.scrap.clear();
|
||||
let out = match self.st.dest {
|
||||
Dest::Body => &mut self.text,
|
||||
Dest::FontTbl => &mut self.font_name,
|
||||
Dest::Skip | Dest::ColorTbl => &mut self.scrap,
|
||||
};
|
||||
(&mut self.decoder, out)
|
||||
}
|
||||
|
||||
fn open(&mut self) {
|
||||
let (decoder, out) = self.sink();
|
||||
decoder.group(out);
|
||||
self.flush_run();
|
||||
if self.stack.len() >= MAX_GROUPS {
|
||||
self.overflow += 1;
|
||||
return;
|
||||
}
|
||||
self.stack.push(self.st.clone());
|
||||
}
|
||||
|
||||
fn close(&mut self) {
|
||||
let (decoder, out) = self.sink();
|
||||
decoder.group(out);
|
||||
if self.overflow > 0 {
|
||||
self.overflow -= 1;
|
||||
return;
|
||||
}
|
||||
if self.st.dest == Dest::FontTbl {
|
||||
self.commit_font();
|
||||
}
|
||||
self.flush_run();
|
||||
if let Some(outer) = self.stack.pop() {
|
||||
self.st = outer;
|
||||
}
|
||||
}
|
||||
|
||||
/// The code page 8-bit text is read in: the current font's, else the default font's, else
|
||||
/// the document's (Windows-1252 when none is named). The symbol page is for body text set in
|
||||
/// a symbol font, so it never applies outside the body (names in the font table), and a
|
||||
/// symbol default font's page does not apply to text in a font selected with `\f`.
|
||||
fn code_page(&self) -> u16 {
|
||||
let document = if self.ansicpg == 0 { 1252 } else { self.ansicpg };
|
||||
let body = self.st.dest == Dest::Body;
|
||||
let current = self.st.cp.filter(|&cp| cp != SYMBOL_CODE_PAGE || body);
|
||||
let default = self.font_cps.get(&self.deff).copied().filter(|&cp| cp != SYMBOL_CODE_PAGE || (body && !self.st.font_set));
|
||||
current.or(default).unwrap_or(document)
|
||||
}
|
||||
|
||||
fn bytes(&mut self, chunk: &[u8]) {
|
||||
let cp = self.code_page();
|
||||
match self.st.dest {
|
||||
Dest::Body => self.decoder.bytes(chunk, cp, &mut self.text),
|
||||
Dest::FontTbl => {
|
||||
// `;` ends an entry; a name may be split around it by formatting.
|
||||
for (i, part) in chunk.split(|b| *b == b';').enumerate() {
|
||||
if i > 0 {
|
||||
self.commit_font();
|
||||
}
|
||||
self.decoder.bytes(part, cp, &mut self.font_name);
|
||||
}
|
||||
}
|
||||
Dest::ColorTbl => {
|
||||
for _ in chunk.iter().filter(|b| **b == b';') {
|
||||
self.commit_color();
|
||||
}
|
||||
}
|
||||
Dest::Skip => {}
|
||||
}
|
||||
}
|
||||
|
||||
fn hex(&mut self, b: u8) {
|
||||
let cp = self.code_page();
|
||||
// In a destination that keeps no text the byte still uses up a fallback position.
|
||||
let (decoder, out) = self.sink();
|
||||
decoder.byte(b, cp, out);
|
||||
}
|
||||
|
||||
fn ch(&mut self, c: char) {
|
||||
let (decoder, out) = self.sink();
|
||||
decoder.char(c, out);
|
||||
}
|
||||
|
||||
fn symbol(&mut self, s: u8) {
|
||||
if self.decoder.skip() {
|
||||
return;
|
||||
}
|
||||
match s {
|
||||
b'~' => self.ch('\u{00A0}'),
|
||||
b'-' => self.ch('\u{00AD}'),
|
||||
b'_' => self.ch('\u{2011}'),
|
||||
b'\\' | b'{' | b'}' => self.ch(s as char),
|
||||
b'\n' if self.st.dest == Dest::Body => self.end_para(),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
fn commit_font(&mut self) {
|
||||
let name = std::mem::take(&mut self.font_name);
|
||||
let name = name.trim();
|
||||
let cp = self.font_cp_cur.take();
|
||||
if name.is_empty() || name.len() > MAX_FONT_NAME || (self.fonts.len() >= MAX_FONTS && !self.fonts.contains_key(&self.font_cur)) {
|
||||
return;
|
||||
}
|
||||
self.fonts.insert(self.font_cur, name.to_string());
|
||||
if let Some(cp) = cp {
|
||||
self.font_cps.insert(self.font_cur, cp);
|
||||
}
|
||||
}
|
||||
|
||||
fn commit_color(&mut self) {
|
||||
let (rgb, set) = std::mem::take(&mut self.color_cur);
|
||||
if self.colors.len() < MAX_COLORS {
|
||||
self.colors.push(set.then_some(rgb));
|
||||
}
|
||||
}
|
||||
|
||||
/// Ends the run being read: its text takes the formatting in effect now.
|
||||
fn flush_run(&mut self) {
|
||||
if self.text.is_empty() {
|
||||
return;
|
||||
}
|
||||
let mut text = std::mem::take(&mut self.text);
|
||||
// `\u0` and `\'00` are not text.
|
||||
text.retain(|c| c != '\0');
|
||||
if text.is_empty() {
|
||||
return;
|
||||
}
|
||||
match self.para.runs.last_mut() {
|
||||
Some(last) if last.fmt == self.st.fmt => last.text.push_str(&text),
|
||||
_ => self.para.runs.push(Run { text, fmt: self.st.fmt.clone() }),
|
||||
}
|
||||
}
|
||||
|
||||
fn take_para(&mut self) -> Para {
|
||||
self.decoder.finish(&mut self.text);
|
||||
self.flush_run();
|
||||
let runs = std::mem::take(&mut self.para.runs);
|
||||
Para { runs, ..self.st.para.clone() }
|
||||
}
|
||||
|
||||
/// The rows read so far become a table: something that is not a row follows them.
|
||||
fn end_table(&mut self) {
|
||||
if !self.rows.is_empty() {
|
||||
self.doc.blocks.push(Block::Table(Table { rows: std::mem::take(&mut self.rows) }));
|
||||
}
|
||||
}
|
||||
|
||||
fn end_para(&mut self) {
|
||||
let para = self.take_para();
|
||||
if self.st.in_table || !self.cell.is_empty() {
|
||||
self.cell.push(para);
|
||||
} else {
|
||||
self.end_table();
|
||||
self.doc.blocks.push(Block::Para(para));
|
||||
}
|
||||
}
|
||||
|
||||
fn end_cell(&mut self) {
|
||||
let para = self.take_para();
|
||||
if !para.runs.is_empty() || self.cell.is_empty() {
|
||||
self.cell.push(para);
|
||||
}
|
||||
self.row.push(Cell { right: 0, paras: std::mem::take(&mut self.cell) });
|
||||
}
|
||||
|
||||
fn end_row(&mut self) {
|
||||
let mut row = std::mem::take(&mut self.row);
|
||||
for (cell, right) in row.iter_mut().zip(&self.cellx) {
|
||||
cell.right = *right;
|
||||
}
|
||||
if !row.is_empty() {
|
||||
self.rows.push(row);
|
||||
}
|
||||
}
|
||||
|
||||
/// Changes the character formatting: the text read so far keeps the old one.
|
||||
fn restyle(&mut self, f: impl FnOnce(&mut CharFmt)) {
|
||||
self.flush_run();
|
||||
f(&mut self.st.fmt);
|
||||
}
|
||||
|
||||
fn word(&mut self, name: &str, param: Option<i32>, starred: bool) {
|
||||
if name == "u" {
|
||||
if let Some(n) = param {
|
||||
let uc = self.st.uc;
|
||||
let (decoder, out) = self.sink();
|
||||
decoder.unicode(n, uc, out);
|
||||
}
|
||||
return;
|
||||
}
|
||||
if self.decoder.skip() {
|
||||
return;
|
||||
}
|
||||
if self.destination(name, starred) {
|
||||
return;
|
||||
}
|
||||
let pv = param.unwrap_or(0);
|
||||
match self.st.dest {
|
||||
Dest::Skip => {}
|
||||
Dest::ColorTbl => {
|
||||
let v = pv.clamp(0, 255) as u8;
|
||||
match name {
|
||||
"red" => self.color_cur = ([v, self.color_cur.0[1], self.color_cur.0[2]], true),
|
||||
"green" => self.color_cur = ([self.color_cur.0[0], v, self.color_cur.0[2]], true),
|
||||
"blue" => self.color_cur = ([self.color_cur.0[0], self.color_cur.0[1], v], true),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
Dest::FontTbl => match name {
|
||||
"f" => {
|
||||
self.font_cur = pv;
|
||||
self.font_cpg_named = false;
|
||||
}
|
||||
"fcharset" => {
|
||||
// The symbol charset doesn't replace a code page the entry named with `\cpg`.
|
||||
let cp = charset_code_page(pv);
|
||||
if !(self.font_cpg_named && cp == Some(SYMBOL_CODE_PAGE)) {
|
||||
self.font_cp_cur = cp;
|
||||
}
|
||||
}
|
||||
"cpg" => {
|
||||
if let Some(cp) = u16::try_from(pv).ok().filter(|cp| is_text_code_page(*cp)) {
|
||||
self.font_cp_cur = Some(cp);
|
||||
self.font_cpg_named = true;
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
},
|
||||
Dest::Body => self.body_word(name, param),
|
||||
}
|
||||
}
|
||||
|
||||
/// Does `name` start a destination? Then the group's text goes there.
|
||||
fn destination(&mut self, name: &str, starred: bool) -> bool {
|
||||
let dest = match name {
|
||||
"fonttbl" => Dest::FontTbl,
|
||||
"colortbl" => Dest::ColorTbl,
|
||||
"pict" | "object" => {
|
||||
if self.st.dest != Dest::Skip {
|
||||
self.doc.pictures_left_out += 1;
|
||||
}
|
||||
Dest::Skip
|
||||
}
|
||||
// A field's result is its text; `\*\fldrslt` is not a destination to drop.
|
||||
"fldrslt" => return true,
|
||||
_ if SKIP.contains(&name) => Dest::Skip,
|
||||
_ if starred => Dest::Skip,
|
||||
_ => return false,
|
||||
};
|
||||
self.flush_run();
|
||||
// Nothing inside a skipped destination comes back out of it.
|
||||
if self.st.dest != Dest::Skip {
|
||||
self.st.dest = dest;
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
fn body_word(&mut self, name: &str, param: Option<i32>) {
|
||||
let pv = param.unwrap_or(0);
|
||||
let on = param != Some(0);
|
||||
match name {
|
||||
"par" | "sect" => self.end_para(),
|
||||
"cell" => self.end_cell(),
|
||||
"row" => self.end_row(),
|
||||
"trowd" => self.cellx.clear(),
|
||||
"cellx" => self.cellx.push(pv),
|
||||
"intbl" => self.st.in_table = true,
|
||||
"pard" => (self.st.para, self.st.in_table) = (Para::default(), false),
|
||||
"outlinelevel" => self.st.para.outline_level = u8::try_from(pv).ok().filter(|l| *l <= 8),
|
||||
"li" => self.st.para.left_indent = pv,
|
||||
"ri" => self.st.para.right_indent = pv,
|
||||
"fi" => self.st.para.first_line_indent = pv,
|
||||
"sb" => self.st.para.space_before = pv,
|
||||
"sa" => self.st.para.space_after = pv,
|
||||
"ql" => self.st.para.align = Some(Align::Left),
|
||||
"qc" => self.st.para.align = Some(Align::Center),
|
||||
"qr" => self.st.para.align = Some(Align::Right),
|
||||
"qj" => self.st.para.align = Some(Align::Justified),
|
||||
"line" => self.ch(LINE_BREAK),
|
||||
"tab" => self.ch('\t'),
|
||||
"page" => self.ch(PAGE_BREAK),
|
||||
"column" => self.ch(COLUMN_BREAK),
|
||||
"emdash" => self.ch('\u{2014}'),
|
||||
"endash" => self.ch('\u{2013}'),
|
||||
"bullet" => self.ch('\u{2022}'),
|
||||
"lquote" => self.ch('\u{2018}'),
|
||||
"rquote" => self.ch('\u{2019}'),
|
||||
"ldblquote" => self.ch('\u{201C}'),
|
||||
"rdblquote" => self.ch('\u{201D}'),
|
||||
"emspace" => self.ch('\u{2003}'),
|
||||
"enspace" => self.ch('\u{2002}'),
|
||||
"b" => self.restyle(|f| f.bold = on),
|
||||
"i" => self.restyle(|f| f.italic = on),
|
||||
"ul" => self.restyle(|f| f.underline = on),
|
||||
"ulnone" => self.restyle(|f| f.underline = false),
|
||||
"strike" => self.restyle(|f| f.strike = on),
|
||||
"super" => self.restyle(|f| f.vert_align = if on { VertAlign::Superscript } else { VertAlign::Baseline }),
|
||||
"sub" => self.restyle(|f| f.vert_align = if on { VertAlign::Subscript } else { VertAlign::Baseline }),
|
||||
"nosupersub" => self.restyle(|f| f.vert_align = VertAlign::Baseline),
|
||||
"fs" => self.restyle(|f| f.half_points = u32::try_from(pv).ok().filter(|s| *s > 0)),
|
||||
"cf" => {
|
||||
let color = usize::try_from(pv).ok().and_then(|i| self.colors.get(i).copied().flatten());
|
||||
self.restyle(|f| f.color = color);
|
||||
}
|
||||
"f" => {
|
||||
let font = self.fonts.get(&pv).cloned();
|
||||
self.st.cp = self.font_cps.get(&pv).copied();
|
||||
self.st.font_set = true;
|
||||
self.restyle(|f| f.font = font);
|
||||
}
|
||||
"plain" => {
|
||||
// The default font is current again, and its code page with it.
|
||||
self.st.cp = None;
|
||||
self.st.font_set = false;
|
||||
self.restyle(|f| *f = CharFmt::default());
|
||||
}
|
||||
"uc" => self.st.uc = usize::try_from(pv).unwrap_or(0),
|
||||
"ansicpg" => {
|
||||
if let Some(cp) = u16::try_from(pv).ok().filter(|cp| is_text_code_page(*cp)) {
|
||||
self.ansicpg = cp;
|
||||
}
|
||||
}
|
||||
"mac" if self.ansicpg == 0 => self.ansicpg = MAC_ROMAN_CODE_PAGE,
|
||||
"pc" if self.ansicpg == 0 => self.ansicpg = 437,
|
||||
"pca" if self.ansicpg == 0 => self.ansicpg = 850,
|
||||
"deff" => self.deff = pv,
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
fn finish(mut self) -> Document {
|
||||
let last = self.take_para();
|
||||
// A file cut inside a row keeps what the row had.
|
||||
if !self.row.is_empty() || !self.cell.is_empty() {
|
||||
if !last.runs.is_empty() || !self.cell.is_empty() {
|
||||
self.cell.push(last);
|
||||
self.row.push(Cell { right: 0, paras: std::mem::take(&mut self.cell) });
|
||||
}
|
||||
self.end_row();
|
||||
self.end_table();
|
||||
} else {
|
||||
self.end_table();
|
||||
if !last.runs.is_empty() {
|
||||
self.doc.blocks.push(Block::Para(last));
|
||||
}
|
||||
}
|
||||
self.doc
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,252 @@
|
||||
//! The text of an RTF file: `\uN` with its fallback characters, and 8-bit text in a code page.
|
||||
|
||||
use crate::codepage::{code_page_byte, multibyte_encoding_name};
|
||||
|
||||
fn multibyte(code_page: u16) -> Option<&'static encoding_rs::Encoding> {
|
||||
encoding_rs::Encoding::for_label(multibyte_encoding_name(code_page)?.as_bytes())
|
||||
}
|
||||
|
||||
/// Decodes the text tokens of one RTF file into `String`s the caller supplies.
|
||||
///
|
||||
/// The caller says which code page applies to each piece of 8-bit text (the current font's,
|
||||
/// else the default font's, else the document's) and how many fallback characters follow a
|
||||
/// `\uN` (`\ucN`, 1 when not set).
|
||||
///
|
||||
/// - `\uN` is one UTF-16 code unit. A high surrogate pairs only with a low surrogate in the
|
||||
/// very next `\u`; every surrogate that does not pair reads as U+FFFD, as in Word.
|
||||
/// - After a `\uN`, the next `\ucN` characters are its fallback and are skipped: a text byte,
|
||||
/// a `\'hh`, a control word and a control symbol each count as one ([`TextDecoder::skip`]).
|
||||
/// The skipping ends with the group ([`TextDecoder::group`]).
|
||||
/// - A single-byte code page maps each byte. A multibyte one (932, 936, 949, 950) streams
|
||||
/// through one decoder, so a lead byte and its trail byte decode together even when a
|
||||
/// `\'hh`, a text run or a formatting word falls between them, and a run that happens to
|
||||
/// start `EF BB BF` is not taken for a UTF-8 byte order mark. A lead byte without its trail
|
||||
/// byte reads as U+FFFD.
|
||||
#[derive(Default)]
|
||||
pub struct TextDecoder {
|
||||
skip: usize,
|
||||
high: Option<u16>,
|
||||
mb: Option<encoding_rs::Decoder>,
|
||||
mb_cp: u16,
|
||||
}
|
||||
|
||||
impl TextDecoder {
|
||||
pub fn new() -> TextDecoder {
|
||||
TextDecoder::default()
|
||||
}
|
||||
|
||||
/// A held high surrogate that nothing paired with.
|
||||
fn flush_high(&mut self, out: &mut String) {
|
||||
if self.high.take().is_some() {
|
||||
out.push('\u{FFFD}');
|
||||
}
|
||||
}
|
||||
|
||||
/// `\uN` with `uc` fallback characters to skip after it.
|
||||
pub fn unicode(&mut self, param: i32, uc: usize, out: &mut String) {
|
||||
self.finish_multibyte(out);
|
||||
let u = i64::from(param).rem_euclid(65_536) as u16;
|
||||
if (0xD800..0xDC00).contains(&u) {
|
||||
self.flush_high(out);
|
||||
self.high = Some(u);
|
||||
} else {
|
||||
match self.high.take() {
|
||||
Some(h) if (0xDC00..0xE000).contains(&u) => out.push(char::decode_utf16([h, u]).next().and_then(Result::ok).unwrap_or('\u{FFFD}')),
|
||||
Some(_) => {
|
||||
out.push('\u{FFFD}');
|
||||
out.push(char::from_u32(u32::from(u)).unwrap_or('\u{FFFD}'));
|
||||
}
|
||||
None => out.push(char::from_u32(u32::from(u)).unwrap_or('\u{FFFD}')),
|
||||
}
|
||||
}
|
||||
self.skip = uc;
|
||||
}
|
||||
|
||||
/// Is the control word or symbol just read a fallback character of the last `\uN`? Then it
|
||||
/// is used up and the caller ignores it.
|
||||
pub fn skip(&mut self) -> bool {
|
||||
let skipped = self.skip > 0;
|
||||
self.skip = self.skip.saturating_sub(1);
|
||||
skipped
|
||||
}
|
||||
|
||||
/// A character a control word or symbol stands for (`\emdash`, `\~`).
|
||||
pub fn char(&mut self, c: char, out: &mut String) {
|
||||
self.finish_multibyte(out);
|
||||
self.flush_high(out);
|
||||
out.push(c);
|
||||
}
|
||||
|
||||
/// `\'hh`: one byte in `code_page`.
|
||||
pub fn byte(&mut self, b: u8, code_page: u16, out: &mut String) {
|
||||
if !self.skip() {
|
||||
self.decode(&[b], code_page, out);
|
||||
}
|
||||
}
|
||||
|
||||
/// A run of text bytes in `code_page`.
|
||||
pub fn bytes(&mut self, chunk: &[u8], code_page: u16, out: &mut String) {
|
||||
let skip = self.skip.min(chunk.len());
|
||||
self.skip -= skip;
|
||||
let chunk = chunk.get(skip..).unwrap_or_default();
|
||||
if !chunk.is_empty() {
|
||||
self.decode(chunk, code_page, out);
|
||||
}
|
||||
}
|
||||
|
||||
fn decode(&mut self, chunk: &[u8], cp: u16, out: &mut String) {
|
||||
self.flush_high(out);
|
||||
if self.mb.is_none() && chunk.is_ascii() {
|
||||
// ASCII is itself in every code page here; with no lead byte pending, as it is.
|
||||
out.push_str(std::str::from_utf8(chunk).unwrap_or_default());
|
||||
return;
|
||||
}
|
||||
let Some(encoding) = multibyte(cp) else {
|
||||
self.finish_multibyte(out);
|
||||
out.extend(chunk.iter().map(|b| code_page_byte(cp, *b)));
|
||||
return;
|
||||
};
|
||||
// A code page switch ends the previous run before the new decoder starts.
|
||||
if self.mb_cp != cp {
|
||||
self.finish_multibyte(out);
|
||||
}
|
||||
self.mb_cp = cp;
|
||||
let decoder = self.mb.get_or_insert_with(|| encoding.new_decoder_without_bom_handling());
|
||||
let mut rest = chunk;
|
||||
loop {
|
||||
out.reserve(rest.len().saturating_mul(3).saturating_add(16));
|
||||
let (result, read, _) = decoder.decode_to_string(rest, out, false);
|
||||
rest = rest.get(read..).unwrap_or_default();
|
||||
if !matches!(result, encoding_rs::CoderResult::OutputFull) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Ends the current multibyte run: a lead byte still waiting for its trail byte becomes
|
||||
/// U+FFFD.
|
||||
fn finish_multibyte(&mut self, out: &mut String) {
|
||||
if let Some(mut decoder) = self.mb.take() {
|
||||
out.reserve(16);
|
||||
let _ = decoder.decode_to_string(&[], out, true);
|
||||
}
|
||||
}
|
||||
|
||||
/// A group opened or closed: the fallback of a `\uN` ends with its group, and so does a
|
||||
/// multibyte run.
|
||||
pub fn group(&mut self, out: &mut String) {
|
||||
self.finish_multibyte(out);
|
||||
self.skip = 0;
|
||||
}
|
||||
|
||||
/// The text ends here (a paragraph end, a cell end, the end of the file): nothing waits
|
||||
/// for a partner any longer.
|
||||
pub fn finish(&mut self, out: &mut String) {
|
||||
self.finish_multibyte(out);
|
||||
self.flush_high(out);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn units(ns: &[i32], uc: usize) -> String {
|
||||
let (mut d, mut out) = (TextDecoder::new(), String::new());
|
||||
for n in ns {
|
||||
d.unicode(*n, uc, &mut out);
|
||||
}
|
||||
d.finish(&mut out);
|
||||
out
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn surrogates_pair_only_with_the_next_unit() {
|
||||
assert_eq!(units(&[-10179, -8704], 0), "\u{1F600}");
|
||||
assert_eq!(units(&[0xD83D, 0xDE00], 0), "\u{1F600}", "written unsigned");
|
||||
assert_eq!(units(&[8364], 0), "€");
|
||||
assert_eq!(units(&[-10179], 0), "\u{FFFD}", "a lone high surrogate at the end");
|
||||
assert_eq!(units(&[-8704], 0), "\u{FFFD}", "a lone low surrogate");
|
||||
assert_eq!(units(&[-8704, -10179], 0), "\u{FFFD}\u{FFFD}", "low then high");
|
||||
assert_eq!(units(&[-10179, -10179, -8704], 0), "\u{FFFD}\u{1F600}", "a high surrogate before a pair");
|
||||
assert_eq!(units(&[-10179, 65, -8704], 0), "\u{FFFD}A\u{FFFD}");
|
||||
assert_eq!(units(&[65536 + 65, -65536 + 66], 0), "AB", "taken modulo 65536");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_held_high_surrogate_does_not_pair_across_text() {
|
||||
let (mut d, mut out) = (TextDecoder::new(), String::new());
|
||||
d.unicode(-10179, 0, &mut out);
|
||||
d.bytes(b"x", 1252, &mut out);
|
||||
d.unicode(-8704, 0, &mut out);
|
||||
d.unicode(-10179, 0, &mut out);
|
||||
d.char('\u{2014}', &mut out);
|
||||
d.unicode(-10179, 1, &mut out);
|
||||
// The fallback and a formatting group do not come between a pair.
|
||||
d.bytes(b"?", 1252, &mut out);
|
||||
d.group(&mut out);
|
||||
d.unicode(-8704, 1, &mut out);
|
||||
d.finish(&mut out);
|
||||
assert_eq!(out, "\u{FFFD}x\u{FFFD}\u{FFFD}\u{2014}\u{1F600}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fallback_characters_are_skipped() {
|
||||
let (mut d, mut out) = (TextDecoder::new(), String::new());
|
||||
d.unicode(8364, 1, &mut out);
|
||||
d.bytes(b"? price", 1252, &mut out);
|
||||
d.unicode(8220, 2, &mut out);
|
||||
d.byte(0x93, 1252, &mut out);
|
||||
assert!(d.skip(), "a control word is one fallback character");
|
||||
assert!(!d.skip());
|
||||
d.unicode(233, 3, &mut out);
|
||||
d.bytes(b"e", 1252, &mut out);
|
||||
d.group(&mut out);
|
||||
d.bytes(b"AB", 1252, &mut out);
|
||||
assert_eq!(out, "\u{20AC} price\u{201C}\u{E9}AB", "the skip ends with the group");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn single_byte_code_pages() {
|
||||
let (mut d, mut out) = (TextDecoder::new(), String::new());
|
||||
d.bytes(&[b'C', b'a', b'f', 0xE9, b' '], 1252, &mut out);
|
||||
d.byte(0xC4, 1251, &mut out);
|
||||
d.byte(0x8E, 10000, &mut out);
|
||||
d.byte(0x82, 437, &mut out);
|
||||
d.byte(0xE9, 9999, &mut out);
|
||||
assert_eq!(out, "Café Дééé");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_lead_byte_and_its_trail_byte_decode_together() {
|
||||
// 日本 in Shift-JIS is 93 FA 96 7B: the first character as two `\'hh`, the second as a
|
||||
// `\'hh` lead byte and a raw trail byte (`{`'s neighbour 0x7B is a text byte here).
|
||||
let (mut d, mut out) = (TextDecoder::new(), String::new());
|
||||
d.byte(0x93, 932, &mut out);
|
||||
d.byte(0xFA, 932, &mut out);
|
||||
d.byte(0x96, 932, &mut out);
|
||||
d.bytes(b"\x7bA", 932, &mut out);
|
||||
assert_eq!(out, "日本A");
|
||||
// A lead byte left at a group end, a code page switch or the end is U+FFFD.
|
||||
for end in 0..3 {
|
||||
let (mut d, mut out) = (TextDecoder::new(), String::new());
|
||||
d.byte(0x93, 932, &mut out);
|
||||
match end {
|
||||
0 => d.group(&mut out),
|
||||
1 => d.bytes(&[0xE9], 1252, &mut out),
|
||||
_ => d.finish(&mut out),
|
||||
}
|
||||
assert!(out.starts_with('\u{FFFD}'), "{end}: {out:?}");
|
||||
}
|
||||
// GBK text that starts EF BB BF is not a UTF-8 byte order mark.
|
||||
let (mut d, mut out) = (TextDecoder::new(), String::new());
|
||||
d.bytes(&[0xEF, 0xBB, 0xBF, 0xC4, 0xE3], 936, &mut out);
|
||||
d.finish(&mut out);
|
||||
assert_eq!(out.chars().count(), 3, "{out:?}");
|
||||
assert!(out.ends_with('\u{FFFD}') && !out.starts_with('\u{FEFF}'), "{out:?}");
|
||||
// A long run decodes whole.
|
||||
let (mut d, mut out) = (TextDecoder::new(), String::new());
|
||||
d.bytes(&[0x93, 0xFA].repeat(10_000), 932, &mut out);
|
||||
assert_eq!(out.chars().count(), 10_000);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,198 @@
|
||||
//! The model as RTF, and the escaping every RTF writer needs.
|
||||
|
||||
use std::fmt::Write;
|
||||
|
||||
use crate::model::{Align, Block, COLUMN_BREAK, CharFmt, Document, LINE_BREAK, PAGE_BREAK, Para, VertAlign};
|
||||
|
||||
/// RTF-escapes text: `\`, `{` and `}` escaped; tab, line, page and column breaks, the
|
||||
/// non-breaking space, the soft hyphen and the non-breaking hyphen as their control words;
|
||||
/// everything past ASCII as `\uN?` (UTF-16 code units as signed 16-bit numbers, with a `?`
|
||||
/// fallback, so the file needs `\uc1`, the default). Other ASCII control characters are
|
||||
/// dropped. U+000B and U+2028 are line breaks too.
|
||||
pub fn esc(s: &str) -> String {
|
||||
let mut o = String::with_capacity(s.len() + 8);
|
||||
push_esc(&mut o, s);
|
||||
o
|
||||
}
|
||||
|
||||
/// [`esc`], appended to `o`.
|
||||
pub fn push_esc(o: &mut String, s: &str) {
|
||||
for c in s.chars() {
|
||||
match c {
|
||||
'\\' => o.push_str("\\\\"),
|
||||
'{' => o.push_str("\\{"),
|
||||
'}' => o.push_str("\\}"),
|
||||
'\t' => o.push_str("\\tab "),
|
||||
LINE_BREAK | '\u{000B}' | '\u{2028}' => o.push_str("\\line "),
|
||||
PAGE_BREAK => o.push_str("\\page "),
|
||||
COLUMN_BREAK => o.push_str("\\column "),
|
||||
'\u{00A0}' => o.push_str("\\~"),
|
||||
'\u{00AD}' => o.push_str("\\-"),
|
||||
'\u{2011}' => o.push_str("\\_"),
|
||||
c if c.is_ascii_control() => {}
|
||||
c if c.is_ascii() => o.push(c),
|
||||
c => {
|
||||
let mut buf = [0u16; 2];
|
||||
for u in c.encode_utf16(&mut buf) {
|
||||
// Writing to a String never fails.
|
||||
let _ = write!(o, "\\u{}?", *u as i16);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// [`esc`] for a `\fonttbl` name: `;` would end the entry, so it is written `\'3b`, and control
|
||||
/// characters, which [`esc`] writes as `\line` or `\tab`, are dropped.
|
||||
pub fn esc_font(name: &str) -> String {
|
||||
esc(&name.replace(char::is_control, "")).replace(';', "\\'3b")
|
||||
}
|
||||
|
||||
/// The fonts and colours a document uses, in the order of their tables. Font 0 is the default
|
||||
/// font; colour 0 is the default (auto) colour, so the first real colour is `\cf1`.
|
||||
struct Tables {
|
||||
fonts: Vec<String>,
|
||||
colors: Vec<[u8; 3]>,
|
||||
}
|
||||
|
||||
impl Tables {
|
||||
fn of(doc: &Document, default_font: &str) -> Tables {
|
||||
let mut t = Tables { fonts: vec![default_font.to_string()], colors: Vec::new() };
|
||||
for fmt in doc.paras().flat_map(|p| &p.runs).map(|r| &r.fmt) {
|
||||
if let Some(f) = fmt.font.as_deref().filter(|f| t.font(f).is_none()) {
|
||||
t.fonts.push(f.to_string());
|
||||
}
|
||||
if let Some(c) = fmt.color.filter(|c| t.color(*c).is_none()) {
|
||||
t.colors.push(c);
|
||||
}
|
||||
}
|
||||
t
|
||||
}
|
||||
fn font(&self, name: &str) -> Option<usize> {
|
||||
self.fonts.iter().position(|f| f.eq_ignore_ascii_case(name))
|
||||
}
|
||||
fn color(&self, c: [u8; 3]) -> Option<usize> {
|
||||
self.colors.iter().position(|k| *k == c).map(|i| i + 1)
|
||||
}
|
||||
|
||||
fn fmt_words(&self, f: &CharFmt) -> String {
|
||||
let mut w = String::new();
|
||||
for (on, word) in [(f.bold, "\\b"), (f.italic, "\\i"), (f.underline, "\\ul"), (f.strike, "\\strike")] {
|
||||
if on {
|
||||
w.push_str(word);
|
||||
}
|
||||
}
|
||||
match f.vert_align {
|
||||
VertAlign::Superscript => w.push_str("\\super"),
|
||||
VertAlign::Subscript => w.push_str("\\sub"),
|
||||
VertAlign::Baseline => {}
|
||||
}
|
||||
// Writing to a String never fails.
|
||||
if let Some(i) = f.font.as_deref().and_then(|n| self.font(n)).filter(|i| *i > 0) {
|
||||
let _ = write!(w, "\\f{i}");
|
||||
}
|
||||
if let Some(s) = f.half_points {
|
||||
let _ = write!(w, "\\fs{s}");
|
||||
}
|
||||
if let Some(i) = f.color.and_then(|c| self.color(c)) {
|
||||
let _ = write!(w, "\\cf{i}");
|
||||
}
|
||||
w
|
||||
}
|
||||
|
||||
/// A paragraph's runs: a formatted run is a group, so its formatting ends with it.
|
||||
fn runs(&self, p: &Para, o: &mut String) {
|
||||
for r in &p.runs {
|
||||
let words = self.fmt_words(&r.fmt);
|
||||
if words.is_empty() {
|
||||
push_esc(o, &r.text);
|
||||
} else {
|
||||
o.push('{');
|
||||
o.push_str(&words);
|
||||
o.push(' ');
|
||||
push_esc(o, &r.text);
|
||||
o.push('}');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn para_words(p: &Para, o: &mut String) {
|
||||
if let Some(l) = p.outline_level.filter(|l| *l <= 8) {
|
||||
let _ = write!(o, "\\outlinelevel{l}");
|
||||
}
|
||||
match p.align {
|
||||
Some(Align::Left) => o.push_str("\\ql"),
|
||||
Some(Align::Center) => o.push_str("\\qc"),
|
||||
Some(Align::Right) => o.push_str("\\qr"),
|
||||
Some(Align::Justified) => o.push_str("\\qj"),
|
||||
None => {}
|
||||
}
|
||||
for (word, twips) in
|
||||
[("li", p.left_indent), ("ri", p.right_indent), ("fi", p.first_line_indent), ("sb", p.space_before), ("sa", p.space_after)]
|
||||
{
|
||||
if twips != 0 {
|
||||
let _ = write!(o, "\\{word}{twips}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Writes a document as RTF. `default_font` is the font of text that names none.
|
||||
///
|
||||
/// The file is 7-bit: `\ansi\ansicpg1252\uc1`, a font table, a colour table when a run has a
|
||||
/// colour, then one `\pard … \par` per paragraph and `\trowd … \row` per table row. What
|
||||
/// [`read`](crate::read) reads from it is `doc` again, except that
|
||||
/// [`Document::pictures_left_out`] is 0, a font equal to `default_font` reads as no font,
|
||||
/// neighbouring runs of one formatting are one run, and empty runs, an empty last paragraph of a
|
||||
/// cell and rows without cells are gone.
|
||||
pub fn write(doc: &Document, default_font: &str) -> String {
|
||||
let t = Tables::of(doc, default_font);
|
||||
let mut o = String::from("{\\rtf1\\ansi\\ansicpg1252\\deff0\\uc1{\\fonttbl");
|
||||
for (i, f) in t.fonts.iter().enumerate() {
|
||||
let _ = write!(o, "{{\\f{i}\\fnil {};}}", esc_font(f));
|
||||
}
|
||||
o.push_str("}\n");
|
||||
if !t.colors.is_empty() {
|
||||
o.push_str("{\\colortbl;");
|
||||
for [r, g, b] in &t.colors {
|
||||
let _ = write!(o, "\\red{r}\\green{g}\\blue{b};");
|
||||
}
|
||||
o.push_str("}\n");
|
||||
}
|
||||
for block in &doc.blocks {
|
||||
match block {
|
||||
Block::Para(p) => {
|
||||
o.push_str("\\pard");
|
||||
Tables::para_words(p, &mut o);
|
||||
o.push(' ');
|
||||
t.runs(p, &mut o);
|
||||
o.push_str("\\par\n");
|
||||
}
|
||||
Block::Table(table) => {
|
||||
for row in table.rows.iter().filter(|r| !r.is_empty()) {
|
||||
o.push_str("\\trowd\\trgaph108");
|
||||
for cell in row {
|
||||
let _ = write!(o, "\\cellx{}", cell.right);
|
||||
}
|
||||
for cell in row {
|
||||
for (i, p) in cell.paras.iter().enumerate() {
|
||||
if i > 0 {
|
||||
o.push_str("\\par");
|
||||
}
|
||||
o.push_str("\\pard\\intbl");
|
||||
Tables::para_words(p, &mut o);
|
||||
o.push(' ');
|
||||
t.runs(p, &mut o);
|
||||
}
|
||||
o.push_str("\\cell");
|
||||
}
|
||||
o.push_str("\\row\n");
|
||||
}
|
||||
// The paragraph after a table is not in it.
|
||||
o.push_str("\\pard");
|
||||
}
|
||||
}
|
||||
}
|
||||
o.push('}');
|
||||
o
|
||||
}
|
||||
@@ -0,0 +1,444 @@
|
||||
//! The reader and writer against the cases the apps' RTF code was tested with: DesignCraft's
|
||||
//! text import, DeckCraft's outline import and export, PdfCraft's export, and WordCraft's
|
||||
//! reader for code pages and `\u`.
|
||||
|
||||
use craft_rtf::{
|
||||
Align, Block, COLUMN_BREAK, Cell, CharFmt, Document, LINE_BREAK, MAX_COLORS, MAX_FONT_NAME, MAX_FONTS, MAX_GROUPS, NotRtf, PAGE_BREAK, Para, Run,
|
||||
Table, VertAlign, read, write,
|
||||
};
|
||||
|
||||
fn text(rtf: &[u8]) -> String {
|
||||
read(rtf).unwrap().paras().map(Para::text).collect::<Vec<_>>().join("\n")
|
||||
}
|
||||
|
||||
fn paras(rtf: &[u8]) -> Vec<Para> {
|
||||
read(rtf).unwrap().paras().cloned().collect()
|
||||
}
|
||||
|
||||
/// The formatting of the run that holds `needle`.
|
||||
fn fmt_of(doc: &Document, needle: &str) -> CharFmt {
|
||||
doc.paras().flat_map(|p| &p.runs).find(|r| r.text.contains(needle)).map(|r| r.fmt.clone()).unwrap()
|
||||
}
|
||||
|
||||
fn para(s: &str) -> Para {
|
||||
Para { runs: vec![Run { text: s.into(), fmt: CharFmt::default() }], ..Para::default() }
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn not_rtf_is_refused() {
|
||||
assert_eq!(read(b"PK\x03\x04"), Err(NotRtf));
|
||||
assert_eq!(read(b""), Err(NotRtf));
|
||||
assert!(read(b" \n{\\rtf1 a}").is_ok());
|
||||
assert!(NotRtf.to_string().contains("{\\rtf"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn text_and_formatting() {
|
||||
let rtf = br"{\rtf1\ansi{\fonttbl{\f0 Times;}}{\colortbl;\red0\green0\blue0;}{\*\generator Test;}
|
||||
\f0\fs24 Hello {\b bold} and {\i italic\i0 plain}\par
|
||||
Caf\'e9 \u8364? price\tab tab\line next\par
|
||||
}";
|
||||
let doc = read(rtf).unwrap();
|
||||
assert_eq!(text(rtf), "Hello bold and italic plain\nCafé € price\ttab\nnext");
|
||||
assert_eq!(doc.blocks.len(), 2, "a trailing \\par leaves no empty paragraph");
|
||||
assert!(fmt_of(&doc, "bold").bold && !fmt_of(&doc, "bold").italic);
|
||||
assert!(fmt_of(&doc, "italic").italic);
|
||||
let plain = fmt_of(&doc, " plain");
|
||||
assert!(!plain.italic && !plain.bold);
|
||||
assert_eq!(fmt_of(&doc, "Hello").half_points, Some(24));
|
||||
assert_eq!(fmt_of(&doc, "Hello").font.as_deref(), Some("Times"));
|
||||
assert!(!text(rtf).contains("Times") && !text(rtf).contains("Test"), "tables and starred destinations are not text");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn character_words() {
|
||||
let doc = read(br"{\rtf1{\colortbl;\red255\green0\blue0;\red0\green0\blue999;}\ul a\ulnone b\strike c\strike0 d\cf1 e\cf2 f\cf0 g\cf9 h\b\i\plain i\fs0 j\fs-8 k}").unwrap();
|
||||
assert!(fmt_of(&doc, "a").underline && !fmt_of(&doc, "b").underline);
|
||||
assert!(fmt_of(&doc, "c").strike && !fmt_of(&doc, "d").strike);
|
||||
assert_eq!(fmt_of(&doc, "e").color, Some([255, 0, 0]));
|
||||
assert_eq!(fmt_of(&doc, "f").color, Some([0, 0, 255]), "a component is clamped");
|
||||
assert_eq!(fmt_of(&doc, "g").color, None, "colour 0 is the default colour");
|
||||
assert_eq!(fmt_of(&doc, "h").color, None, "a colour the table does not have");
|
||||
assert_eq!(fmt_of(&doc, "i"), CharFmt::default(), "\\plain");
|
||||
assert_eq!((fmt_of(&doc, "j").half_points, fmt_of(&doc, "k").half_points), (None, None));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn special_characters_and_breaks() {
|
||||
assert_eq!(
|
||||
text(br"{\rtf1 a\~b\-c\_d\\e\{f\}g\emdash h\endash i\bullet j\lquote k\rquote l\ldblquote m\rdblquote}"),
|
||||
"a\u{A0}b\u{AD}c\u{2011}d\\e{f}g—h–i•j‘k’l“m”"
|
||||
);
|
||||
let breaks = format!("a{PAGE_BREAK}b{COLUMN_BREAK}c{LINE_BREAK}d\te");
|
||||
assert_eq!(text(br"{\rtf1 a\page b\column c\line d\tab e}"), breaks);
|
||||
// A backslash before a line end is a paragraph end; a bare line end is nothing.
|
||||
assert_eq!(text(b"{\\rtf1 a\\\nb\r\nc\\sect d}"), "a\nbc\nd");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn header_and_footer_variants_and_notes_are_skipped() {
|
||||
for w in ["header", "headerl", "headerr", "headerf", "footer", "footerl", "footerr", "footerf", "footnote", "annotation", "stylesheet", "info"] {
|
||||
let rtf = format!(r"{{\rtf1\ansi {{\{w} Page head\par}}Body text\par}}");
|
||||
assert_eq!(text(rtf.as_bytes()), "Body text", "{w}");
|
||||
}
|
||||
// Nothing inside a skipped destination comes back out of it, whatever it nests.
|
||||
assert_eq!(text(br"{\rtf1 {\header {\b {\fonttbl x} Head}}{\*\unknown {\field{\fldrslt hidden}}}Body}"), "Body");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_fields_result_is_its_text() {
|
||||
assert_eq!(text(br#"{\rtf1 See {\field{\*\fldinst HYPERLINK "https://example.com"}{\fldrslt {\ul the site}}}.}"#), "See the site.");
|
||||
assert_eq!(text(br"{\rtf1 {\field{\*\fldinst PAGE}{\*\fldrslt 3}} of 9}"), "3 of 9");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pictures_and_objects_are_counted_not_read() {
|
||||
let doc = read(
|
||||
br"{\rtf1 a{\pict\pngblip 89504e47}b{\object\objemb{\*\objdata 0105}}{\*\shppict{\pict\pngblip 00}}{\nonshppict{\pict\wmetafile8 00}}c}",
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(doc.pictures_left_out, 3, "a picture inside a skipped destination is not counted again");
|
||||
assert_eq!(doc.paras().map(Para::text).collect::<String>(), "abc");
|
||||
assert_eq!(text(b"{\\rtf1 a{\\pict\\bin4 }}{\\x}b}"), "ab", "binary data is not scanned for braces");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn out_of_range_unicode_wraps_and_hyphens_are_text() {
|
||||
assert_eq!(text(br"{\rtf1 a\u4294967361?b}"), "a\u{FFFF}b", "clamped to i32, then modulo 65536");
|
||||
assert_eq!(text(br"{\rtf1\ansi Rock\emdash-roll, x\tab-y, a\li-720 b}"), "Rock\u{2014}-roll, x\t-y, ab");
|
||||
assert_eq!(text(br"{\rtf1 {\b A\'4}B}"), "AB");
|
||||
assert_eq!(text(br"{\rtf1 a\'+fb}"), "a+fb");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unicode_fallbacks() {
|
||||
assert_eq!(text(br"{\rtf1 \u8220\ldblquote Hi\u8221\rdblquote x\u8212\emdash y}"), "\u{201C}Hi\u{201D} x\u{2014}y");
|
||||
assert_eq!(text(br"{\rtf1 \u8220\{Hi}"), "\u{201C}Hi", "a control symbol is one fallback character");
|
||||
assert_eq!(text(br"{\rtf1 \u8220\'93Hi}"), "\u{201C}Hi");
|
||||
assert_eq!(text(br"{\rtf1 \u8220{\b Hi}}"), "\u{201C}Hi", "a group is not a character");
|
||||
assert_eq!(text(br"{\rtf1 {\b \u8364} AB}"), "\u{20AC} AB", "the skip ends with the group");
|
||||
assert_eq!(text(br"{\rtf1 {\uc2\b \u8364} ABC}"), "\u{20AC} ABC");
|
||||
assert_eq!(text(br"{\rtf1\uc0 \u8364 5\uc2 \u8364xy5}"), "\u{20AC}5\u{20AC}5");
|
||||
assert_eq!(text(br"{\rtf1 \u8364\u8364?5}"), "\u{20AC}\u{20AC}5", "a \\u is never a fallback character");
|
||||
assert_eq!(text(br"{\rtf1 {\header \u8364}?x}"), "?x", "a \\u in a skipped destination leaves nothing to skip outside it");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nul_is_not_text() {
|
||||
assert_eq!(text(br"{\rtf1 \u0?abc\par a\'00b\par {\b\'00}}"), "abc\nab");
|
||||
assert_eq!(text(b"{\\rtf1 a\0b\\par}\0\0"), "ab");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn surrogate_pairs() {
|
||||
assert_eq!(text(br"{\rtf1\ansi\uc1 A\u-10179?\u-8704?B \u-10174?\u-8265?\u30000?}"), "A\u{1F600}B \u{20BB7}\u{7530}");
|
||||
assert_eq!(text(br"{\rtf1 A\u-10179?B\u-8704?C}"), "A\u{FFFD}B\u{FFFD}C", "half a pair is U+FFFD, as in Word");
|
||||
assert_eq!(text(br"{\rtf1 A\u-10179?\par\u-8704?C}"), "A\u{FFFD}\n\u{FFFD}C", "a pair does not span a paragraph end");
|
||||
assert_eq!(text(br"{\rtf1 A\u-10179?}"), "A\u{FFFD}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn code_pages() {
|
||||
assert_eq!(text(br"{\rtf1\ansi\ansicpg1251 \'cf\'f0\'e8\'e2\'e5\'f2}"), "Привет");
|
||||
assert_eq!(text(b"{\\rtf1\\ansi\\ansicpg1251 \xcf\xf0\xe8\xe2\xe5\xf2}"), "Привет", "raw bytes are text in the code page");
|
||||
assert_eq!(text(br"{\rtf1\mac Caf\'8e}"), "Café");
|
||||
assert_eq!(text(br"{\rtf1\pc Caf\'82}"), "Café");
|
||||
assert_eq!(text(br"{\rtf1\pca \'9d}"), "Ø");
|
||||
assert_eq!(text(br"{\rtf1\ansi Caf\'e9 \'80}"), "Café €", "Windows-1252 when the file names none");
|
||||
assert_eq!(text(br"{\rtf1\ansi\ansicpg9999 Caf\'e9}"), "Café", "an unknown code page is not taken");
|
||||
// The font's charset decides, then the default font's, then the document's.
|
||||
let rtf = br"{\rtf1\ansi\ansicpg1252\deff1{\fonttbl{\f0\fcharset204 Arial Cyr;}{\f1\fcharset161 Greek;}{\f2\cpg1250 CE;}{\f3 Plain;}}\'e1 \f0\'e1 \f2\'b3 \f3\'e1 \plain\'e1}";
|
||||
assert_eq!(text(rtf), "α б ł α α");
|
||||
}
|
||||
|
||||
/// A symbol font (`\fcharset2`: Symbol, Wingdings) has no letters: a byte from 0x80 is the font's
|
||||
/// own code, U+F000 + the byte, whatever the document's code page.
|
||||
#[test]
|
||||
fn a_symbol_fonts_bytes_are_its_own_codes() {
|
||||
for cpg in [1252, 1251] {
|
||||
let rtf = format!(r"{{\rtf1\ansi\ansicpg{cpg}{{\fonttbl{{\f0 Arial;}}{{\f1\fcharset2 Wingdings;}}}}\f1 J\'93\'d8\f0 \'93\f1\'fc\plain\'93}}");
|
||||
assert_eq!(text(rtf.as_bytes()), "J\u{F093}\u{F0D8}\u{201C}\u{F0FC}\u{201C}", "{cpg}");
|
||||
}
|
||||
// The symbol page is not for the font's own name in the table.
|
||||
let doc = read(br"{\rtf1{\fonttbl{\f0\fcharset2 Sym\'e9bol;}}\f0 x}").unwrap();
|
||||
assert_eq!(fmt_of(&doc, "x").font.as_deref(), Some("Symébol"));
|
||||
// A symbol default font's page is its own text's: not the text of a font chosen with \f,
|
||||
// and its again after \plain.
|
||||
let rtf = br"{\rtf1\ansi\deff0{\fonttbl{\f0\fcharset2 Symbol;}{\f1 Arial;}}\'b7 \f1\'b7 \plain\'b7}";
|
||||
assert_eq!(text(rtf), "\u{F0B7} \u{B7} \u{F0B7}");
|
||||
// A code page the entry names with \cpg is not replaced by the symbol charset.
|
||||
let rtf = br"{\rtf1{\fonttbl{\f0\cpg1251\fcharset2 Odd;}{\f1\fcharset2\cpg1251 Odd2;}}\f0\'c4\f1\'c4}";
|
||||
assert_eq!(text(rtf), "\u{414}\u{414}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn east_asian_text_decodes_across_tokens() {
|
||||
// 日本語 in Shift-JIS: 93 FA 96 7B 8C EA. 0x7B is `{` as a byte, so it must be `\'7b`.
|
||||
assert_eq!(text(br"{\rtf1\ansi\ansicpg932 \'93\'fa\'96\'7b\'8c\'ea}"), "日本語");
|
||||
assert_eq!(text(b"{\\rtf1\\ansi\\ansicpg932 \x93\xfa\\'96\\'7b\x8c\xea ok}"), "日本語 ok");
|
||||
assert_eq!(text(br"{\rtf1{\fonttbl{\f0\fcharset134 SimSun;}}\f0\'c4\'e3\'ba\'c3}"), "你好");
|
||||
assert_eq!(text(br"{\rtf1\ansicpg932 a\'93}b}"), "a\u{FFFD}b", "a lead byte left at a group end");
|
||||
assert_eq!(text(br"{\rtf1\ansicpg932 a\'93\par b}"), "a\u{FFFD}\nb");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn font_table_forms() {
|
||||
// Entries as groups or bare, `\'3b` for a `;` in a name, an alternative name in a starred
|
||||
// destination, an entry without its `;`.
|
||||
let rtf =
|
||||
br"{\rtf1{\fonttbl\f0\froman Times New Roman;\f1 A\'3bB;{\f2\fnil{\*\falt Other}DejaVu Sans;}{\f3 NoSemicolon}}\f0 a\f1 b\f2 c\f3 d\f9 e}";
|
||||
let doc = read(rtf).unwrap();
|
||||
let font = |s: &str| fmt_of(&doc, s).font;
|
||||
assert_eq!(font("a").as_deref(), Some("Times New Roman"));
|
||||
assert_eq!(font("b").as_deref(), Some("A;B"));
|
||||
assert_eq!(font("c").as_deref(), Some("DejaVu Sans"));
|
||||
assert_eq!(font("d").as_deref(), Some("NoSemicolon"));
|
||||
assert_eq!(font("e"), None, "a font the table does not have");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn outline_levels_and_indents() {
|
||||
let rtf = br"{\rtf1\ansi{\fonttbl{\f0 Arial;}}{\*\generator X;}\pard\outlinelevel0\b Title one\b0\par\pard\li720 Point \'93a\'94\par\pard\outlinelevel2\li1440 Sub\par\outlinelevel9 Deep\par\pard\li-720 Back\par}";
|
||||
let got: Vec<(String, Option<u8>, i32)> = paras(rtf).iter().map(|p| (p.text(), p.outline_level, p.left_indent)).collect();
|
||||
assert_eq!(
|
||||
got,
|
||||
[
|
||||
("Title one".to_string(), Some(0), 0),
|
||||
("Point “a”".to_string(), None, 720),
|
||||
("Sub".to_string(), Some(2), 1440),
|
||||
("Deep".to_string(), None, 1440),
|
||||
("Back".to_string(), None, -720)
|
||||
]
|
||||
);
|
||||
let huge = paras(br"{\rtf1\pard\li9223372036854775807 deep\par\pard\li-9223372036854775808 flat\par}");
|
||||
assert!(huge[0].left_indent > 1_000_000_000 && huge[1].left_indent <= 0, "{huge:?}");
|
||||
}
|
||||
|
||||
/// Word writes a list item's bullet as `{\listtext\pard\plain …\'b7\tab}` (older writers:
|
||||
/// `{\pntext …}`) after the paragraph's own `\li`. That group is the rendered bullet, not text.
|
||||
#[test]
|
||||
fn list_labels_are_not_text_and_keep_the_indent() {
|
||||
let rtf = br"{\rtf1\ansi\pard Title\par\pard\fi-360\li720{\listtext\pard\plain\f1\'b7\tab}Point\par\pard\li720{\pntext\f1\'b7\tab}Old\par}";
|
||||
let got: Vec<(String, i32)> = paras(rtf).iter().map(|p| (p.text(), p.left_indent)).collect();
|
||||
assert_eq!(got, [("Title".to_string(), 0), ("Point".to_string(), 720), ("Old".to_string(), 720)]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn alignment_indents_spacing_and_position() {
|
||||
let rtf = br"{\rtf1\pard\qc\li720\ri360\fi-360\sb120\sa240 x\super 2\sub i\nosupersub y\super z\super0 w\par\pard\qr r\par\pard\ql l\par\pard\qj j\par\pard n\par}";
|
||||
let doc = read(rtf).unwrap();
|
||||
let ps: Vec<&Para> = doc.paras().collect();
|
||||
assert_eq!((ps[0].left_indent, ps[0].right_indent, ps[0].first_line_indent, ps[0].space_before, ps[0].space_after), (720, 360, -360, 120, 240));
|
||||
assert_eq!(
|
||||
ps.iter().map(|p| p.align).collect::<Vec<_>>(),
|
||||
[Some(Align::Center), Some(Align::Right), Some(Align::Left), Some(Align::Justified), None]
|
||||
);
|
||||
assert_eq!((ps[4].left_indent, ps[4].space_after), (0, 0), "\\pard resets them");
|
||||
let va = |s: &str| fmt_of(&doc, s).vert_align;
|
||||
assert_eq!(
|
||||
(va("x"), va("2"), va("i"), va("y"), va("z"), va("w")),
|
||||
(VertAlign::Baseline, VertAlign::Superscript, VertAlign::Subscript, VertAlign::Baseline, VertAlign::Superscript, VertAlign::Baseline)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn paragraph_properties_end_with_their_group() {
|
||||
let got: Vec<i32> = paras(br"{\rtf1 {\li720 a\par}b\par{\pard\li360 c}\par}").iter().map(|p| p.left_indent).collect();
|
||||
assert_eq!(got, [720, 0, 0]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_paragraphs_are_kept_except_the_last() {
|
||||
let doc = read(br"{\rtf1 a\par\par b\par\par}").unwrap();
|
||||
assert_eq!(doc.blocks.len(), 4);
|
||||
assert_eq!(read(br"{\rtf1 }").unwrap().blocks.len(), 0);
|
||||
assert_eq!(read(br"{\rtf1 a}").unwrap().blocks.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tables() {
|
||||
let rtf = br"{\rtf1 Before\par\trowd\trgaph108\cellx1440\cellx4320{\intbl\fs20 A1}\cell{\intbl\b B1}\cell\row
|
||||
\trowd\cellx1440\cellx2880\cellx4320\pard\intbl A2\par second\cell\cell C2\cell\row\pard After\par\trowd\cellx100 X\cell\row}";
|
||||
let doc = read(rtf).unwrap();
|
||||
let [Block::Para(before), Block::Table(t), Block::Para(after), Block::Table(last)] = doc.blocks.as_slice() else { panic!("{doc:?}") };
|
||||
assert_eq!((before.text(), after.text()), ("Before".to_string(), "After".to_string()));
|
||||
let cells = |row: &[Cell]| row.iter().map(|c| (c.right, c.paras.iter().map(Para::text).collect::<Vec<_>>().join("|"))).collect::<Vec<_>>();
|
||||
assert_eq!(cells(&t.rows[0]), [(1440, "A1".to_string()), (4320, "B1".to_string())]);
|
||||
assert_eq!(cells(&t.rows[1]), [(1440, "A2|second".to_string()), (2880, String::new()), (4320, "C2".to_string())]);
|
||||
assert!(t.rows[0][1].paras[0].runs[0].fmt.bold);
|
||||
assert_eq!(cells(&last.rows[0]), [(100, "X".to_string())]);
|
||||
// Reading order.
|
||||
assert_eq!(doc.paras().map(Para::text).collect::<Vec<_>>().join(" "), "Before A1 B1 A2 second C2 After X");
|
||||
// A file cut inside a row keeps what the row had.
|
||||
let cut = read(br"{\rtf1\trowd\cellx100\intbl A\cell B").unwrap();
|
||||
let [Block::Table(t)] = cut.blocks.as_slice() else { panic!("{cut:?}") };
|
||||
assert_eq!(cells(&t.rows[0]), [(100, "A".to_string()), (0, "B".to_string())]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn groups_nested_past_the_limit_are_read_flat() {
|
||||
let deep = MAX_GROUPS * 4;
|
||||
let mut rtf = String::from("{\\rtf1 ");
|
||||
rtf.push_str(&"{\\b ".repeat(deep));
|
||||
rtf.push_str("deep");
|
||||
rtf.push_str(&"}".repeat(deep));
|
||||
rtf.push_str(" after}");
|
||||
let doc = read(rtf.as_bytes()).unwrap();
|
||||
assert_eq!(doc.paras().map(Para::text).collect::<String>(), "deep after");
|
||||
assert!(fmt_of(&doc, "deep").bold && !fmt_of(&doc, "after").bold, "the groups within the limit still close in step");
|
||||
// A skipped destination deeper than the limit is still skipped.
|
||||
let mut rtf = String::from("{\\rtf1 ");
|
||||
rtf.push_str(&"{".repeat(MAX_GROUPS - 3));
|
||||
rtf.push_str("{\\info {{{{secret}}}}}shown");
|
||||
assert_eq!(text(rtf.as_bytes()), "shown");
|
||||
// Unbalanced closes are ignored.
|
||||
assert_eq!(text(b"{\\rtf1 a}}}}b{c"), "abc");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn font_and_colour_tables_are_capped() {
|
||||
let mut rtf = String::from("{\\rtf1{\\fonttbl");
|
||||
for i in 0..MAX_FONTS + 10 {
|
||||
rtf.push_str(&format!("{{\\f{i} F{i};}}"));
|
||||
}
|
||||
rtf.push_str(&format!("{{\\f9000 {};}}{{\\f9001 {};}}", "n".repeat(MAX_FONT_NAME), "n".repeat(MAX_FONT_NAME + 1)));
|
||||
rtf.push_str("}{\\colortbl;");
|
||||
for i in 0..MAX_COLORS + 10 {
|
||||
rtf.push_str(&format!("\\red{}\\green1\\blue1;", i % 256));
|
||||
}
|
||||
rtf.push_str(&format!("}}\\f{} a\\f{} b\\f9000 x\\f9001 y\\cf{} c\\cf{} d}}", MAX_FONTS - 1, MAX_FONTS, MAX_COLORS - 1, MAX_COLORS));
|
||||
let doc = read(rtf.as_bytes()).unwrap();
|
||||
assert_eq!(fmt_of(&doc, "a").font, Some(format!("F{}", MAX_FONTS - 1)));
|
||||
assert_eq!(fmt_of(&doc, "b").font, None, "past the font limit");
|
||||
assert_eq!(fmt_of(&doc, "x").font, None, "the table was full before the 256-byte name");
|
||||
assert_eq!(fmt_of(&doc, "y").font, None);
|
||||
assert_eq!(fmt_of(&doc, "c").color, Some([((MAX_COLORS - 2) % 256) as u8, 1, 1]));
|
||||
assert_eq!(fmt_of(&doc, "d").color, None, "past the colour limit");
|
||||
// The name limit on its own.
|
||||
let rtf = format!("{{\\rtf1{{\\fonttbl{{\\f0 {};}}{{\\f1 {};}}}}\\f0 a\\f1 b}}", "n".repeat(MAX_FONT_NAME), "n".repeat(MAX_FONT_NAME + 1));
|
||||
let doc = read(rtf.as_bytes()).unwrap();
|
||||
assert_eq!(fmt_of(&doc, "a").font.map(|f| f.len()), Some(MAX_FONT_NAME));
|
||||
assert_eq!(fmt_of(&doc, "b").font, None);
|
||||
}
|
||||
|
||||
fn sample() -> Document {
|
||||
let fmt = |f: fn(&mut CharFmt)| {
|
||||
let mut c = CharFmt::default();
|
||||
f(&mut c);
|
||||
c
|
||||
};
|
||||
let run = |text: &str, fmt: CharFmt| Run { text: text.into(), fmt };
|
||||
let cell = |right, paras| Cell { right, paras };
|
||||
Document {
|
||||
blocks: vec![
|
||||
Block::Para(Para { runs: vec![run("Plan {Q3} \\ 100%", fmt(|c| c.bold = true))], outline_level: Some(0), ..Para::default() }),
|
||||
Block::Para(Para {
|
||||
runs: vec![
|
||||
run(" Grow — fast 😀 ", CharFmt::default()),
|
||||
run("Café\u{A0}résumé", fmt(|c| (c.italic, c.underline, c.half_points) = (true, true, Some(21)))),
|
||||
run("\ttab\nline", fmt(|c| (c.strike, c.color) = (true, Some([255, 0, 128])))),
|
||||
run("123 numbers first", fmt(|c| c.font = Some("Deja;Vu".into()))),
|
||||
run("2", fmt(|c| c.vert_align = VertAlign::Superscript)),
|
||||
run("i", fmt(|c| c.vert_align = VertAlign::Subscript)),
|
||||
],
|
||||
outline_level: Some(8),
|
||||
left_indent: 720,
|
||||
right_indent: 360,
|
||||
first_line_indent: -360,
|
||||
space_before: 120,
|
||||
space_after: 240,
|
||||
align: Some(Align::Justified),
|
||||
}),
|
||||
Block::Para(Para::default()),
|
||||
Block::Table(Table {
|
||||
rows: vec![
|
||||
vec![cell(1440, vec![para("A1")]), cell(4320, vec![para("B1"), Para::default(), para("B1 third")])],
|
||||
vec![
|
||||
cell(1440, vec![Para::default()]),
|
||||
cell(2880, vec![para("日本語")]),
|
||||
cell(4320, vec![Para { left_indent: -360, ..para("C2") }]),
|
||||
],
|
||||
],
|
||||
}),
|
||||
Block::Para(Para { left_indent: -100, ..para(&format!("after{PAGE_BREAK}page{COLUMN_BREAK}column")) }),
|
||||
Block::Table(Table { rows: vec![vec![cell(50, vec![para("last")])]] }),
|
||||
],
|
||||
pictures_left_out: 0,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn what_is_written_reads_back() {
|
||||
let doc = sample();
|
||||
let rtf = write(&doc, "Helvetica");
|
||||
assert!(rtf.is_ascii(), "7-bit");
|
||||
assert_eq!(read(rtf.as_bytes()).unwrap(), doc, "{rtf}");
|
||||
assert_eq!(write(&read(rtf.as_bytes()).unwrap(), "Helvetica"), rtf, "and writes the same file again");
|
||||
assert_eq!(read(write(&Document::default(), "x").as_bytes()).unwrap(), Document::default());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_written_file() {
|
||||
let doc = Document {
|
||||
blocks: vec![
|
||||
Block::Para(Para {
|
||||
runs: vec![Run { text: "Title {1}".into(), fmt: CharFmt { bold: true, half_points: Some(32), ..CharFmt::default() } }],
|
||||
outline_level: Some(0),
|
||||
..Para::default()
|
||||
}),
|
||||
Block::Para(Para { left_indent: 360, ..para("Body é") }),
|
||||
Block::Table(Table { rows: vec![vec![Cell { right: 1440, paras: vec![para("A")] }, Cell { right: 2880, paras: vec![] }], vec![]] }),
|
||||
],
|
||||
pictures_left_out: 7,
|
||||
};
|
||||
let red = CharFmt { color: Some([255, 0, 0]), font: Some("Courier New".into()), ..CharFmt::default() };
|
||||
let mut colored = doc.clone();
|
||||
colored.blocks.push(Block::Para(Para { runs: vec![Run { text: "red".into(), fmt: red }], ..Para::default() }));
|
||||
assert_eq!(
|
||||
write(&doc, "Liberation Sans"),
|
||||
"{\\rtf1\\ansi\\ansicpg1252\\deff0\\uc1{\\fonttbl{\\f0\\fnil Liberation Sans;}}\n\\pard\\outlinelevel0 {\\b\\fs32 Title \\{1\\}}\\par\n\\pard\\li360 Body \\u233?\\par\n\\trowd\\trgaph108\\cellx1440\\cellx2880\\pard\\intbl A\\cell\\cell\\row\n\\pard}"
|
||||
);
|
||||
let rtf = write(&colored, "Liberation Sans");
|
||||
assert!(rtf.contains("{\\fonttbl{\\f0\\fnil Liberation Sans;}{\\f1\\fnil Courier New;}}\n{\\colortbl;\\red255\\green0\\blue0;}\n"), "{rtf}");
|
||||
assert!(rtf.ends_with("\\pard\\pard {\\f1\\cf1 red}\\par\n}"), "{rtf}");
|
||||
// The default font named on a run needs no word.
|
||||
let named = Document {
|
||||
blocks: vec![Block::Para(Para {
|
||||
runs: vec![Run { text: "x".into(), fmt: CharFmt { font: Some("liberation sans".into()), ..CharFmt::default() } }],
|
||||
..Para::default()
|
||||
})],
|
||||
pictures_left_out: 0,
|
||||
};
|
||||
assert!(write(&named, "Liberation Sans").contains("\\pard x\\par"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn escaping() {
|
||||
assert_eq!(craft_rtf::esc("a\\b{c}d\te\nf\u{B}g\u{2028}h"), "a\\\\b\\{c\\}d\\tab e\\line f\\line g\\line h");
|
||||
assert_eq!(craft_rtf::esc("\u{A0}\u{AD}\u{2011}\u{C}\u{E}\u{1}\r"), "\\~\\-\\_\\page \\column ");
|
||||
assert_eq!(craft_rtf::esc("é€😀"), "\\u233?\\u8364?\\u-10179?\\u-8704?");
|
||||
assert_eq!(craft_rtf::esc_font("A;B\n{C}"), "A\\'3bB\\{C\\}");
|
||||
let mut s = String::from("x");
|
||||
craft_rtf::push_esc(&mut s, "{");
|
||||
assert_eq!(s, "x\\{");
|
||||
}
|
||||
|
||||
/// Every prefix of a file, and the file with each byte changed, reads without a panic.
|
||||
#[test]
|
||||
fn damaged_files_do_not_panic() {
|
||||
let rtf = write(&sample(), "Helvetica");
|
||||
let rtf = format!(
|
||||
"{rtf}{{\\rtf1\\ansicpg932{{\\fonttbl{{\\f0\\fcharset128 MS Mincho;}}}}{{\\info{{\\title x}}}}\\'93\\'fa\\u-10179?\\bin3 abc{{\\pict\\'00}}\\trowd\\cellx9 a\\cell\\row}}"
|
||||
);
|
||||
let bytes = rtf.as_bytes();
|
||||
for end in 0..=bytes.len() {
|
||||
let _ = read(&bytes[..end]);
|
||||
}
|
||||
for at in 0..bytes.len() {
|
||||
for b in [b'{', b'}', b'\\', b'\'', 0xFF, 0, b'9', b'-', b';'] {
|
||||
let mut damaged = bytes.to_vec();
|
||||
damaged[at] = b;
|
||||
let _ = read(&damaged);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user