diff --git a/crates/doc/src/encoding.rs b/crates/doc/src/encoding.rs index ee70d41512..1118391719 100644 --- a/crates/doc/src/encoding.rs +++ b/crates/doc/src/encoding.rs @@ -1,7 +1,8 @@ //! Windows-1252 (the byte encoding legacy Word text pieces use) and the other single-byte Windows //! code pages RTF files name with `\ansicpg` or a font's `\fcharset` (Cyrillic 1251, Central -//! European 1250, Greek, Turkish, Hebrew, Arabic, Baltic, Vietnamese, Thai), and Mac OS Roman -//! (10000, RTF `\mac`). +//! European 1250, Greek, Turkish, Hebrew, Arabic, Baltic, Vietnamese, Thai), Mac OS Roman +//! (10000, RTF `\mac`), and the East Asian multibyte code pages (932 Shift-JIS, 936 GBK, 949 UHC, +//! 950 Big5), whose bytes decode as runs rather than one at a time. /// Windows-1252 code points for 0x80..=0x9F. const CP1252: [u16; 32] = [ @@ -286,13 +287,47 @@ pub fn code_page_byte(code_page: u16, b: u8) -> char { if b >= 0x80 && OEM_CODE_PAGES.contains(&code_page) { oem_byte(code_page, b) } else { windows_byte(code_page, b) } } +/// The East Asian multibyte code pages a document format names with `\ansicpg` or a font's `\cpg`: +/// 932 Shift-JIS (Japanese), 936 GBK (Simplified Chinese), 949 UHC (Korean) and 950 Big5 +/// (Traditional Chinese). +pub const MULTIBYTE_CODE_PAGES: [u16; 4] = [932, 936, 949, 950]; + +/// Whether `code_page` is one of the East Asian multibyte code pages ([`MULTIBYTE_CODE_PAGES`]): +/// its bytes decode as a run, a lead byte together with its trail byte. +pub fn is_multibyte_code_page(code_page: u16) -> bool { + multibyte_encoding_name(code_page).is_some() +} + +/// The WHATWG encoding name of an East Asian multibyte code page (see [`MULTIBYTE_CODE_PAGES`]), +/// for `encoding_rs::Encoding::for_label`; `None` for the single-byte code pages, whose bytes +/// decode one at a time, and anything else. +pub fn multibyte_encoding_name(code_page: u16) -> Option<&'static str> { + Some(match code_page { + 932 => "shift_jis", + 936 => "gbk", + 949 => "euc-kr", + 950 => "big5", + _ => return None, + }) +} + +/// Whether a document format can name `code_page` for its text: a single-byte Windows, DOS or Mac +/// one ([`is_byte_code_page`]) or an East Asian multibyte one ([`is_multibyte_code_page`]). +pub fn is_text_code_page(code_page: u16) -> bool { + is_byte_code_page(code_page) || is_multibyte_code_page(code_page) +} + /// The code page of an RTF or Word font charset (`\fcharset`): 204 Russian → 1251, 238 Eastern -/// European → 1250, 77 Mac → 10000, 254 PC / 255 OEM → 437… `None` for the ANSI and default charsets (the -/// document's code page applies), symbol fonts and the double-byte East Asian charsets. +/// European → 1250, 77 Mac → 10000, 254 PC / 255 OEM → 437, 128 Japanese → 932, 129 Korean → 949, +/// 134 Simplified Chinese → 936, 136 Traditional Chinese → 950… `None` for the ANSI and default +/// charsets (the document's code page applies) and symbol fonts. pub fn charset_code_page(charset: i32) -> Option { Some(match charset { 77 => MAC_ROMAN_CODE_PAGE, - 254 | 255 => 437, + 128 => 932, + 129 => 949, + 134 => 936, + 136 => 950, 161 => 1253, 162 => 1254, 163 => 1258, @@ -302,6 +337,7 @@ pub fn charset_code_page(charset: i32) -> Option { 204 => 1251, 222 => 874, 238 => 1250, + 254 | 255 => 437, _ => return None, }) } @@ -347,6 +383,25 @@ mod tests { assert!(!is_byte_code_page(932) && !is_byte_code_page(0) && !SINGLE_BYTE_CODE_PAGES.contains(&10000)); } + /// #1653: the East Asian charsets and code pages RTF names are multibyte: their bytes decode + /// as runs (a lead byte and its trail byte together), not one at a time. + #[test] + fn east_asian_code_pages_are_multibyte() { + assert_eq!(charset_code_page(128), Some(932), "Japanese"); + assert_eq!(charset_code_page(129), Some(949), "Korean"); + assert_eq!(charset_code_page(134), Some(936), "Simplified Chinese"); + assert_eq!(charset_code_page(136), Some(950), "Traditional Chinese"); + assert_eq!(charset_code_page(255), Some(437), "OEM is single-byte"); + assert_eq!(MULTIBYTE_CODE_PAGES, [932, 936, 949, 950]); + for cp in MULTIBYTE_CODE_PAGES { + assert!(is_multibyte_code_page(cp) && is_text_code_page(cp), "{cp}"); + assert!(!is_byte_code_page(cp) && !SINGLE_BYTE_CODE_PAGES.contains(&cp), "{cp}"); + assert!(multibyte_encoding_name(cp).is_some(), "{cp}"); + } + assert!(is_text_code_page(1251) && is_text_code_page(437) && is_text_code_page(10000)); + assert!(!is_text_code_page(0) && !is_text_code_page(9999)); + } + #[test] fn oem_code_pages() { let decode = |cp: u16, bytes: &[u8]| bytes.iter().map(|&b| oem_byte(cp, b)).collect::(); diff --git a/crates/formats/src/rtf.rs b/crates/formats/src/rtf.rs index 95625360ef..7540134151 100644 --- a/crates/formats/src/rtf.rs +++ b/crates/formats/src/rtf.rs @@ -10,8 +10,11 @@ //! Import: a tolerant reader for groups and destinations (unknown `\*` destinations, headers, //! footers, notes and objects are skipped), `\par \line \tab \page`, `\uN` with `\ucN` //! fallback skipping, `\'hh` and 8-bit text in the document's code page (`\ansicpg`, `\mac`, `\pc`, `\pca`) or -//! the current font's (`\fcharset`, `\cpg`: Cyrillic, Central European, Greek, Mac OS Roman…; -//! Windows-1252 when none is named), the formatting words above, stylesheet-based +//! the current font's (`\fcharset`, `\cpg`: Cyrillic, Central European, Greek, Mac OS Roman, the +//! East Asian multibyte pages 932/936/949/950…; Windows-1252 when none is named). A multibyte +//! code page's bytes stream through one decoder, so a lead byte and its trail byte decode +//! together and a run that looks like a UTF-8 BOM is not mistaken for one; the formatting words +//! above, stylesheet-based //! headings and `\outlinelevel`, lists through `\ls`/`\ilvl`, fields, bookmarks, pictures and //! tables. Group nesting and picture sizes are capped. @@ -27,6 +30,12 @@ const MAX_GROUPS: usize = 512; const MAX_PICT: usize = 64 << 20; /// Most sections we read (`\sect`); later breaks end paragraphs only. const MAX_SECTIONS: usize = 10_000; +/// The encoding of an East Asian multibyte code page (932 Shift-JIS, 936 GBK, 949 UHC, 950 Big5), +/// which [`Reader::finish_multibyte`] decodes a run with; `None` for the single-byte code pages. +fn multibyte_encoding(code_page: u16) -> Option<&'static encoding_rs::Encoding> { + let label = wordcraft_doc::encoding::multibyte_encoding_name(code_page)?; + encoding_rs::Encoding::for_label(label.as_bytes()) +} // --------------------------------------------------------------------------------------------- // Export @@ -1144,6 +1153,11 @@ struct Reader { rev_name: String, skip_chars: usize, pending_high: Option, + /// A streaming decoder for the current multibyte code page, and the code page it was made for. + /// A lead byte and its trail byte decode together even when a text word, a group boundary or the + /// end of a chunk falls between them, so a run of any length never splits a character. + mb_decoder: Option, + mb_cp: u16, // Output. body: Vec, para: Para, @@ -1291,6 +1305,8 @@ impl Reader { } fn close(&mut self) { + // A group end decodes the multibyte bytes read in it, before the state is restored. + self.finish_multibyte(); self.skip_chars = 0; if self.overflow > 0 { self.overflow -= 1; @@ -1378,11 +1394,126 @@ impl Reader { } } - /// A byte of 8-bit text (`\'hh` or raw) as a char, in the current font's code page, else the - /// default font's, else the document's. - fn byte_char(&self, b: u8) -> char { - let cp = self.st.cp.or_else(|| self.font_cps.get(&self.deff).copied()).unwrap_or(if self.ansicpg == 0 { 1252 } else { self.ansicpg }); - wordcraft_doc::encoding::code_page_byte(cp, b) + /// The code page 8-bit text is read in: the current font's, else the default font's, else the + /// document's (Windows-1252 when none is named). + fn code_page(&self) -> u16 { + self.st.cp.or_else(|| self.font_cps.get(&self.deff).copied()).unwrap_or(if self.ansicpg == 0 { 1252 } else { self.ansicpg }) + } + + /// A byte of 8-bit text (`\'hh` or raw): a single-byte code page decodes it at once; a + /// multibyte one is fed to a streaming decoder ([`Reader::finish_multibyte`]), so a lead byte decodes + /// together with its trail byte. + fn byte(&mut self, b: u8) { + if self.skip_chars > 0 { + self.skip_chars -= 1; + return; + } + let cp = self.code_page(); + let Some(encoding) = multibyte_encoding(cp) else { + self.finish_multibyte(); + let c = wordcraft_doc::encoding::code_page_byte(cp, b); + self.ch(c); + return; + }; + // A code page switch ends the previous run (a lead byte left then becomes U+FFFD, as Word + // writes it) before the new decoder starts. + if self.mb_cp != cp || self.mb_decoder.is_none() { + self.finish_multibyte(); + self.mb_cp = cp; + self.mb_decoder = Some(encoding.new_decoder_without_bom_handling()); + } + self.decode_run(&[b]); + } + + /// Feed 8-bit text bytes to the open multibyte decoder and append what it emits. A single + /// `\'hh` byte decodes into a stack buffer (no heap allocation per byte of escaped text); a + /// raw run decodes in one call, so a long ASCII run in a Japanese, Chinese or Korean document + /// costs one append instead of one per byte. The destination's spare capacity is all + /// `decode_to_*` writes into, so a full buffer is drained and decoding continues. + fn decode_run(&mut self, bytes: &[u8]) { + // The decoder is taken out so the text it produced can be appended (which borrows the + // reader) before it goes back. + let Some(mut decoder) = self.mb_decoder.take() else { return }; + let mut read = 0; + if bytes.len() <= 16 { + // A character is at most 4 UTF-8 bytes, and a byte may also replace a pending lead + // byte (3 more), so one escaped byte always fits the stack buffer. + let mut buf = [0u8; 16]; + let (result, got, written, _) = decoder.decode_to_utf8(bytes, &mut buf, false); + read = got; + let full = matches!(result, encoding_rs::CoderResult::OutputFull); + if written > 0 + && let Ok(s) = std::str::from_utf8(&buf[..written]) + { + self.text(s); + } + if !full { + self.mb_decoder = Some(decoder); + return; + } + } + let mut out = String::with_capacity(bytes.len().saturating_mul(2)); + let mut rest = bytes.get(read..).unwrap_or_default(); + loop { + let (result, got, _) = decoder.decode_to_string(rest, &mut out, false); + rest = rest.get(got..).unwrap_or_default(); + if !matches!(result, encoding_rs::CoderResult::OutputFull) { + break; + } + out.reserve(16.max(rest.len().saturating_mul(2))); + } + self.mb_decoder = Some(decoder); + if !out.is_empty() { + self.text(&out); + } + } + + /// A run of raw 8-bit text from the file (not `\'hh` escapes): a single-byte code page maps + /// each byte to a char; a multibyte one decodes the whole run in one pass, so a long ASCII run + /// in a Japanese, Chinese or Korean document costs one decode and one append instead of one + /// per byte. + /// + /// The decoder is created without BOM handling: the code page comes from `\ansicpg`, `\cpg` or + /// `\fcharset` and must not change. With BOM sniffing, a run that starts `EF BB BF` in GBK, + /// Big5 or UHC (a valid character pair followed by a lead byte) would be dropped as a UTF-8 + /// BOM and the rest decoded as UTF-8. + fn run(&mut self, chunk: &[u8]) { + if self.skip_chars > 0 { + for b in chunk { + self.byte(*b); + } + return; + } + let cp = self.code_page(); + let Some(encoding) = multibyte_encoding(cp) else { + self.finish_multibyte(); + let mut out = String::with_capacity(chunk.len()); + for b in chunk { + out.push(wordcraft_doc::encoding::code_page_byte(cp, *b)); + } + if !out.is_empty() { + self.text(&out); + } + return; + }; + if self.mb_cp != cp || self.mb_decoder.is_none() { + self.finish_multibyte(); + self.mb_cp = cp; + self.mb_decoder = Some(encoding.new_decoder_without_bom_handling()); + } + self.decode_run(chunk); + } + + /// Ends the current multibyte run: the streaming decoder emits whatever it still holds (a lone + /// lead byte becomes U+FFFD, as Word writes it) and is dropped. A code page switch, a single-byte + /// run, a group end and the end of the document finish the run this way. + fn finish_multibyte(&mut self) { + let Some(mut decoder) = self.mb_decoder.take() else { return }; + let mut out = String::with_capacity(16); + let _ = decoder.decode_to_string(&[], &mut out, true); + if !out.is_empty() { + self.text(&out); + } } fn commit_style(&mut self) { @@ -1571,6 +1702,7 @@ impl Reader { self.skip_chars -= 1; return; } + self.finish_multibyte(); let mut buf = [0u8; 4]; self.text(c.encode_utf8(&mut buf)); } @@ -1728,6 +1860,9 @@ impl Reader { } fn word(&mut self, w: &str, p: Option, starred: bool, first_in_group: bool) { + // Every control word ends a multibyte run: the code page it is read in may change here + // (`\f`, `\fcharset`, `\cpg`, `\ansicpg`, `\pard`), and `\par` must see the text before it. + self.finish_multibyte(); let on = p != Some(0); let pv = p.unwrap_or(0); if self.st.dest == Dest::Skip && !(self.in_info() && matches!(w, "title" | "author" | "subject" | "keywords" | "doccomm" | "creatim")) { @@ -1850,9 +1985,9 @@ impl Reader { self.font_cp_cur = None; } "fcharset" => self.font_cp_cur = wordcraft_doc::encoding::charset_code_page(pv), - // A font's explicit code page (`\cpg1251`) wins over its charset. + // A font's explicit code page (`\cpg1251`, `\cpg932`) wins over its charset. "cpg" => { - if let Some(cp) = u16::try_from(pv).ok().filter(|cp| wordcraft_doc::encoding::is_byte_code_page(*cp)) { + if let Some(cp) = u16::try_from(pv).ok().filter(|cp| wordcraft_doc::encoding::is_text_code_page(*cp)) { self.font_cp_cur = Some(cp); } } @@ -2054,7 +2189,7 @@ impl Reader { } "fs" => self.st.fmt.size = if pv > 0 { Some((pv as f32 / 2.0).clamp(1.0, 1638.0)) } else { None }, "ansicpg" => { - if let Some(cp) = u16::try_from(pv).ok().filter(|cp| wordcraft_doc::encoding::is_byte_code_page(*cp)) { + if let Some(cp) = u16::try_from(pv).ok().filter(|cp| wordcraft_doc::encoding::is_text_code_page(*cp)) { self.ansicpg = cp; } } @@ -2555,10 +2690,14 @@ fn parse_full(bytes: &[u8]) -> Result { match h { Some(v) => { if r.st.dest == Dest::Pict { - r.text(&format!("{v:02x}")); + // Hex digits, two at a time, without a String per picture byte. + const HEX: &[u8; 16] = b"0123456789abcdef"; + let h = [HEX[usize::from(v >> 4)], HEX[usize::from(v & 0x0f)]]; + if let Ok(s) = std::str::from_utf8(&h) { + r.text(s); + } } else { - let c = r.byte_char(v); - r.ch(c); + r.byte(v); } i += 4; } @@ -2602,22 +2741,27 @@ fn parse_full(bytes: &[u8]) -> Result { j += 1; } let chunk = bytes.get(i..j).unwrap_or(b""); - if r.skip_chars == 0 && chunk.is_ascii() { + if r.st.dest == Dest::Pict && r.skip_chars == 0 && chunk.is_ascii() { + // Picture hex is not text: it needs no decoder, so neither a pending lead byte + // nor the code page's single-byte path applies to it. + if let Ok(s) = std::str::from_utf8(chunk) { + r.text(s); + } + } else if r.skip_chars == 0 && chunk.is_ascii() && r.mb_decoder.is_none() { + // Plain ASCII with no multibyte run open: append it as it is. if let Ok(s) = std::str::from_utf8(chunk) { r.text(s); } } else { - for b in chunk { - let c = r.byte_char(*b); - r.ch(c); - } + r.run(chunk); } i = j; first_in_group = false; } } } - // A header or footer left open by a truncated file ends here. + // A header or footer left open by a truncated file ends here, and so does a multibyte run. + r.finish_multibyte(); r.end_header_footer(); if r.para_has || !r.para.inlines.is_empty() { r.end_para(false); @@ -3024,6 +3168,126 @@ mod tests { // A code page number out of range is ignored, not truncated to another one. let f = parse(br"{\rtf1\ansi\ansicpg65973 caf\'e9\par}").unwrap(); assert_eq!(texts(&f), ["Normal:café"]); + // The OEM charset (255) is DOS code page 437, like 254. + let f = parse(br"{\rtf1\ansi\deff0{\fonttbl{\f0\fcharset255 Courier New;}}\f0 caf\'82\par}").unwrap(); + assert_eq!(texts(&f), ["Normal:café"]); + } + + /// #1653: the East Asian code pages (932 Shift-JIS, 936 GBK, 949 UHC, 950 Big5) decode their + /// bytes as a run, so a lead byte and its trail byte decode together, escaped or raw. + #[test] + fn east_asian_multibyte_code_pages() { + // Japanese `\ansicpg932` (CP932: 日 = 93 fa, 本 = 96 7b, 語 = 8c ea). + let f = parse(br"{\rtf1\ansi\ansicpg932 \'93\'fa\'96\'7b\'8c\'ea\par}").unwrap(); + assert_eq!(texts(&f), ["Normal:日本語"]); + // Raw 8-bit text (not `\'hh`), and raw bytes mixed with one escaped byte: RTF can't carry a + // raw `{`, `}` or `\`, so 本's trail byte 7b is written `\'7b` here. + let f = parse(b"{\\rtf1\\ansi\\ansicpg932 \x8a\xbf\x8e\x9a\\par}").unwrap(); + assert_eq!(texts(&f), ["Normal:漢字"]); + let f = parse(b"{\\rtf1\\ansi\\ansicpg932 \x93\xfa\x96\\'7b\x8c\xea\\par}").unwrap(); + assert_eq!(texts(&f), ["Normal:日本語"]); + // A font names the code page with `\fcharset128` or with `\cpg932`, and `\f0` selects it. + let f = parse(br"{\rtf1\ansi\ansicpg1252\deff0{\fonttbl{\f0\fcharset128 MS Mincho;}}\f0 \'93\'fa\'96\'7b\'8c\'ea\par}").unwrap(); + assert_eq!(texts(&f), ["Normal:日本語"]); + let f = parse(br"{\rtf1\ansi\ansicpg1252\deff0{\fonttbl{\f0\cpg932 MS Mincho;}}\f0 \'93\'fa\'96\'7b\'8c\'ea\par}").unwrap(); + assert_eq!(texts(&f), ["Normal:日本語"]); + // Raw 8-bit bytes in the font table (the font's name) decode in the document's code page. + let d = import(b"{\\rtf1\\ansi\\ansicpg932\\deff0{\\fonttbl{\\f0\\fcharset128 \x96\xbe\x92\xa9;}}\\f0 x\\par}").unwrap(); + assert_eq!(d.styles.default_chr.font.as_deref(), Some("明朝")); + // Korean `\ansicpg949` (UHC: 한국어 = c7 d1 b1 b9 be ee). + let f = parse(br"{\rtf1\ansi\ansicpg949 \'c7\'d1\'b1\'b9\'be\'ee\par}").unwrap(); + assert_eq!(texts(&f), ["Normal:한국어"]); + // Simplified Chinese `\ansicpg936` (GBK: 中文 = d6 d0 ce c4). + let f = parse(br"{\rtf1\ansi\ansicpg936 \'d6\'d0\'ce\'c4\par}").unwrap(); + assert_eq!(texts(&f), ["Normal:中文"]); + // Traditional Chinese `\ansicpg950` (Big5: 中文 = a4 a4 a4 e5). + let f = parse(br"{\rtf1\ansi\ansicpg950 \'a4\'a4\'a4\'e5\par}").unwrap(); + assert_eq!(texts(&f), ["Normal:中文"]); + // `\uN?` escapes still decode alongside multibyte text (`\u26412?` is 本, its `?` fallback + // skipped by `\uc1`). + let f = parse(br"{\rtf1\ansi\ansicpg932 \'93\'fa\u26412?\'8c\'ea x\par}").unwrap(); + assert_eq!(texts(&f), ["Normal:日本語 x"]); + // An invalid sequence decodes as U+FFFD, as Word writes it, and doesn't lose the rest. + let f = parse(b"{\\rtf1\\ansi\\ansicpg932 \x93\\par after\\par}").unwrap(); + assert_eq!(texts(&f), ["Normal:\u{FFFD}", "Normal:after"]); + let f = parse(b"{\\rtf1\\ansi\\ansicpg932 \x93\xfa\x93 x\x96\x8c\\par}").unwrap(); + assert_eq!(texts(&f), ["Normal:日\u{FFFD} x膜"]); + } + + /// The decoder never sniffs a BOM: the code page is named by `\ansicpg`, `\cpg` or `\fcharset`, + /// so a run that starts `EF BB BF` is a character pair and a lead byte, not a UTF-8 BOM (which + /// would switch the rest of the document to UTF-8 and garble it). + #[test] + fn a_bom_like_byte_triple_is_text_in_a_multibyte_code_page() { + // GBK EF BB = 锘, then BF B5 = 康; UHC EF BB = 癤, then BF C0 = 오; Big5 EF BB = 嚜, then + // BF 40 = 濃. + let f = parse(br"{\rtf1\ansi\ansicpg936 \'ef\'bb\'bf\'b5\par}").unwrap(); + assert_eq!(texts(&f), ["Normal:锘康"]); + let f = parse(br"{\rtf1\ansi\ansicpg949 \'ef\'bb\'bf\'c0\par}").unwrap(); + assert_eq!(texts(&f), ["Normal:癤오"]); + let f = parse(br"{\rtf1\ansi\ansicpg950 \'ef\'bb\'bf\'40\par}").unwrap(); + assert_eq!(texts(&f), ["Normal:嚜濃"]); + } + + /// A multibyte run longer than any fixed buffer keeps every character: the streaming decoder + /// holds a lead byte and its trail byte together across decode steps instead of cutting the + /// character at a buffer size (the review's finding on the byte-capped flush). + #[test] + fn a_long_multibyte_run_never_splits_a_character() { + let mut bytes = b"{\\rtf1\\ansi\\ansicpg932 ".to_vec(); + bytes.push(b'A'); + // 2048 × あ (CP932 82 a0) is 4096 bytes after the one ASCII byte: a fixed 4096-byte buffer + // would flush with the last あ's lead byte in one step and its trail byte in the next. + for _ in 0..2048 { + bytes.extend_from_slice(&[0x82, 0xA0]); + } + bytes.extend_from_slice(b"\\par}"); + let f = parse(&bytes).unwrap(); + let expected: String = std::iter::once('A').chain((0..2048).map(|_| 'あ')).collect(); + assert_eq!(texts(&f), [format!("Normal:{expected}")]); + } + + /// A lead byte with no trail byte at the end of the document becomes U+FFFD, as Word writes a + /// malformed sequence — the streaming decoder is finalised, not left holding it. + #[test] + fn an_incomplete_lead_byte_at_the_end_is_replacement() { + let f = parse(b"{\\rtf1\\ansi\\ansicpg932 \x93}").unwrap(); + assert_eq!(texts(&f), ["Normal:\u{FFFD}"]); + } + + /// #1653: a code page switch (`\f`, `\fcharset`, `\cpg`, `\ansicpg`), a group end and the end + /// of the document decode the multibyte bytes read so far, before the new code page applies. + #[test] + fn multibyte_code_page_switches_flush_pending_bytes() { + // Japanese (932) → ASCII → Cyrillic (`\fcharset204` with `\cpg1251`): each run decodes in + // its own code page, the last bytes before the switch included. + let f = parse( + br"{\rtf1\ansi\ansicpg932\deff0{\fonttbl{\f0 MS Mincho;}{\f1\fcharset204\cpg1251 Arial CYR;}}\f0 \'93\'fa\'96\'7b\'8c\'ea mixed \f1 \'cf\'f0\'e8\'e2\'e5\'f2\par}", + ) + .unwrap(); + assert_eq!(texts(&f), ["Normal:日本語 mixed Привет"]); + // A group end flushes: the group's 932 bytes decode before the outer 1252 text. + let f = + parse(br"{\rtf1\ansi\ansicpg1252\deff0{\fonttbl{\f0 Arial;}{\f1\fcharset128 MS Mincho;}}\f0 {\f1 \'93\'fa\'96\'7b\'8c\'ea}caf\'e9\par}") + .unwrap(); + assert_eq!(texts(&f), ["Normal:日本語café"]); + // An `\ansicpg` switch mid-document flushes too. + let f = parse(br"{\rtf1\ansi\ansicpg932 \'93\'fa\ansicpg1251 \'cf\'f0\'e8\par}").unwrap(); + assert_eq!(texts(&f), ["Normal:日При"]); + // A run left open at the end of the document isn't lost. + let f = parse(b"{\\rtf1\\ansi\\ansicpg932 \x93\xfa").unwrap(); + assert_eq!(texts(&f), ["Normal:日"]); + } + + /// A Windows-1252 document (the default) reads exactly as before: high bytes one at a time. + #[test] + fn windows_1252_text_is_unchanged() { + let f = parse(b"{\\rtf1\\ansi\\ansicpg1252 caf\xe9 na\xefve \x93quoted\x94 \\'80 5\\'b0\\par}").unwrap(); + assert_eq!(texts(&f), ["Normal:café naïve “quoted” € 5°"]); + // No `\ansicpg`: the default is 1252; the byte values of an East Asian code page keep + // their 1252 meaning. + let f = parse(b"{\\rtf1\\ansi \xe9\\'92\\'93 \\'fa\\par}").unwrap(); + assert_eq!(texts(&f), ["Normal:é’“ ú"]); } #[test] diff --git a/crates/formats/src/tests.rs b/crates/formats/src/tests.rs index 6b1a1a1844..8e49c3dce1 100644 --- a/crates/formats/src/tests.rs +++ b/crates/formats/src/tests.rs @@ -513,6 +513,12 @@ fn hostile_rtf_bits() { ] { let _ = import("rtf", s).unwrap(); } + // #1653: a flood of multibyte code-page bytes (leads and trails, raw and escaped) decodes as a + // stream and never panics or grows a buffer with the input. + let mut rtf = b"{\\rtf1\\ansi\\ansicpg932 ".to_vec(); + rtf.resize(rtf.len() + 200_000, 0x93); + rtf.extend_from_slice(b"\\'fa\\par\\'93"); + assert!(import("rtf", &rtf).unwrap().is_ok()); } proptest::proptest! { diff --git a/docs/file-format-parity.md b/docs/file-format-parity.md index c5be18734c..ac652c4244 100644 --- a/docs/file-format-parity.md +++ b/docs/file-format-parity.md @@ -1,6 +1,6 @@ # File-format parity with Microsoft Word -> **Last reviewed:** 2026-10-11 · **Last updated:** 2026-10-19 · **Change:** trivial (PDF export: lam-alef drawn as two glyphs at one x extracts in reading order, #2310; previously trivial: shape gradients and transparency, #1500; previously trivial: RTF import compares heading and code formatting with the file's stylesheet; previously trivial: text typed into a content control takes the control's formatting; localized placeholder styles recognised; previously trivial: forms protection keeps its content controls: no inserting, Properties or Remove Content Control; previously trivial: Word XML and web pages saved as `.doc` open by what they hold, #2853; previously trivial: a VML shape in a header cell is always placed on the page, user-checked in Word for Mac, #1775; previously trivial: `w:sym` characters keep their symbol font when the text's font changes and are written back as `w:sym`, #2036; previously trivial: RTF export writes sections (gap 19); previously trivial: ODT export writes sections (gap 19); previously trivial: two lines in one and horizontal in vertical (`w:eastAsianLayout`) kept on save; previously trivial: PDF outline targets follow leading page breaks; previously trivial: positional tabs `w:ptab` read and kept on save, #2367; previously trivial: DOCX: mixed right-to-left runs split and marked `w:rtl` with their language, #1543; previously trivial: `w:doNotTrackMoves` round-trips and turns move tracking off; previously trivial: legacy outline, shadow, emboss and imprint text effects drawn, #2534; previously trivial: DOCX glossary documents (building blocks) read, kept and written; previously trivial: cached field results keep their visible run formatting; previously trivial: cached REF results retain hyperlink destinations; previously trivial: CLI conversions can explicitly refresh fields; previously trivial: a copied equation also goes in the clipboard's MathML formats, #1297; previously trivial: a pasted HTML `
` is an indented paragraph, not the Quote style, #1270); previously trivial: every table style part round-trips, #714; previously trivial: VML freeforms read and kept on save; previously trivial: multi-paragraph field instructions no longer add blank lines; previously trivial: `flowChartOr` draws its circle and cross spokes, as in Word; previously trivial: RTF: Mac OS Roman and DOS text, #1653; previously trivial: VML text boxes keep their internal margins, fit to their text and vertical alignment; hidden drawings not shown; previously trivial: PDF export draws horizontal and vertical lines inside groups, #1474; previously trivial: symbol-font text exported as Unicode, #776; previously trivial: text-box margins, Resize shape to fit text and freeforms in groups, #743; previously trivial: formatted paste from the system clipboard's HTML and RTF, #687; previously trivial: independent complex-script formatting inheritance and round-trip regression tests; previously trivial: import/export fixes for TXT, RTF, ODT, HTML and LaTeX, #433–#440; previously major: first version; every format Word reads or writes, measured from the readers and writers on origin/main) · **Target:** Microsoft Word (Microsoft 365) for Mac 16.113.4, plus Word for Windows' extra formats +> **Last reviewed:** 2026-10-11 · **Last updated:** 2026-10-21 · **Change:** trivial (RTF import decodes the East Asian multibyte code pages, #1653; previously trivial: PDF export: lam-alef drawn as two glyphs at one x extracts in reading order, #2310; previously trivial: shape gradients and transparency, #1500; previously trivial: RTF import compares heading and code formatting with the file's stylesheet; previously trivial: text typed into a content control takes the control's formatting; localized placeholder styles recognised; previously trivial: forms protection keeps its content controls: no inserting, Properties or Remove Content Control; previously trivial: Word XML and web pages saved as `.doc` open by what they hold, #2853; previously trivial: a VML shape in a header cell is always placed on the page, user-checked in Word for Mac, #1775; previously trivial: `w:sym` characters keep their symbol font when the text's font changes and are written back as `w:sym`, #2036; previously trivial: RTF export writes sections (gap 19); previously trivial: ODT export writes sections (gap 19); previously trivial: two lines in one and horizontal in vertical (`w:eastAsianLayout`) kept on save; previously trivial: PDF outline targets follow leading page breaks; previously trivial: positional tabs `w:ptab` read and kept on save, #2367; previously trivial: DOCX: mixed right-to-left runs split and marked `w:rtl` with their language, #1543; previously trivial: `w:doNotTrackMoves` round-trips and turns move tracking off; previously trivial: legacy outline, shadow, emboss and imprint text effects drawn, #2534; previously trivial: DOCX glossary documents (building blocks) read, kept and written; previously trivial: cached field results keep their visible run formatting; previously trivial: cached REF results retain hyperlink destinations; previously trivial: CLI conversions can explicitly refresh fields; previously trivial: a copied equation also goes in the clipboard's MathML formats, #1297; previously trivial: a pasted HTML `
` is an indented paragraph, not the Quote style, #1270); previously trivial: every table style part round-trips, #714; previously trivial: VML freeforms read and kept on save; previously trivial: multi-paragraph field instructions no longer add blank lines; previously trivial: `flowChartOr` draws its circle and cross spokes, as in Word; previously trivial: RTF: Mac OS Roman and DOS text, #1653; previously trivial: VML text boxes keep their internal margins, fit to their text and vertical alignment; hidden drawings not shown; previously trivial: PDF export draws horizontal and vertical lines inside groups, #1474; previously trivial: symbol-font text exported as Unicode, #776; previously trivial: text-box margins, Resize shape to fit text and freeforms in groups, #743; previously trivial: formatted paste from the system clipboard's HTML and RTF, #687; previously trivial: independent complex-script formatting inheritance and round-trip regression tests; previously trivial: import/export fixes for TXT, RTF, ODT, HTML and LaTeX, #433–#440; previously major: first version; every format Word reads or writes, measured from the readers and writers on origin/main) · **Target:** Microsoft Word (Microsoft 365) for Mac 16.113.4, plus Word for Windows' extra formats Every format Word opens or saves, what WordCraft does with it, and how it's tested. Fidelity is the share of a typical real-world file's content and formatting that survives (read: shows correctly; @@ -41,9 +41,8 @@ Weights are the share of a Word user's file traffic (estimate). Read/write: ✅ | `.doc` Word 97-2003 (and `.dot`) | 8% | R/W | 🟡 | ❌ | ~60% | 25–40 | `crates/docbin/src/tests.rs`, `tests/drawing.rs` | Spec-based reader ([MS-DOC]): text, formatting, styles, sections, headers/footers, tables, lists, notes, fields, bookmarks; inline and floating pictures (PNG, JPEG, DIB, TIFF, EMF, WMF; Macintosh PICT as a sized placeholder) with crop, alt text, position and wrap; basic shapes, lines, groups and text boxes ([MS-ODRAW]). Table cell shading from every SHD record (`sprmTDefTableShd`, `sprmTDefTableShdRaw` and their 2nd/3rd ranges, with the Word 97 `Shd80` as fallback; a clear pattern shows its background colour); the row-end mark is not a cell, and each table has its own grid and properties (#1380). Horizontally merged rows stored as fewer, wider cells come with #1123. Freeform shapes, z-order and Word 6/95 files are dropped or refused. Password-protected files (XOR obfuscation, RC4, RC4 CryptoAPI) open after a password prompt, from the public specs; not yet tried on a file Word wrote (AES-encrypted ones are refused). No writer. DIB conversion caps BMP bytes (header included) and encoded PNG at the supplied BLIP byte limit, and uses it as the decoder's allocation limit; a BMP fallback is kept if it fits (related #1334 and #1554). | | PDF export | 8% | W | — | ✅ | ~80% | 6–10 | `crates/pdf/src/tests.rs` | krilla: embedded subsets, selectable text incl. field results (#133); Hebrew and Arabic copy and search as whole words (right-to-left glyphs written in visual order, #1383, also with fonts that draw lam-alef as two glyphs at one x, #2310); All Caps and Small Caps text extracts as the capitals shown, as from Word's PDFs, and never changes the case of other text (#1473), links, outline, tagging. Outline destinations follow leading page breaks to the heading text. Fonts it can't subset become outlines (#132). Print Markup (#589): comments in balloons beside each page (scaled to its paper) and tracked changes; HTML/Markdown/text copies end with a comment list. Horizontal and vertical lines inside groups are drawn (#1474). No PDF/A option, no "best for printing/online" choice | | PDF open (PDF Reflow, Word for Windows) | 1% | R | ❌ | — | 0% | 30–50 | — | Converting a PDF into an editable document. A sibling app (PdfCraft) may supply the parser | -| `.rtf` Rich Text Format | 3% | R/W | 🟡 | 🟡 | ~50% | 10–15 | `crates/formats/src/tests.rs` | ~1,500 lines: text, character/paragraph formatting, styles, lists, tables, pictures (`\shppict`, #154), fields, bookmarks. 8-bit text in the document's code page (`\ansicpg`) or the current font's (`\fcharset`, `\cpg`): Cyrillic, Central European, Greek, Turkish, Hebrew, Arabic, Baltic, Vietnamese, Thai (was Windows-1252 only), Mac OS Roman (`\mac`, `\fcharset77`) and DOS code pages (`\pc` 437, `\pca` 850, `\cpg` 437/737/850/852/857/866, `\fcharset254`) (#1653). Double-byte East Asian code pages (Japanese `\fcharset128`, Chinese, Korean) not decoded yet. Pasted from the system clipboard where the platform offers it (Windows, Wayland) when there is no HTML (#687). Tracked insertions and deletions read and written (`\revtbl`, `\revised`/`\deleted`, authors, `\revdttm` dates), so Accept/Reject All work after opening or a round trip; tracked paragraph marks (`\par` inside `\revised`/`\deleted` groups, including a `\sect` inside a deletion) and character and paragraph formatting revisions (`\crauth`/`\crdate` with `\oldcprops`, `\prauth`/`\prdate` with `\oldpprops`) read and written; section and table formatting revisions (`\oldsprops`, `\oldtprops`) not yet. Sections on import (#1378): each `\sect` ends a section with its own page size and orientation, margins, header/footer distance, columns, break type, page-number restart, start and format, title page and headers/footers (those it leaves out link to the previous); export still writes one section. Thin: comments, notes, section export, RTL | +| `.rtf` Rich Text Format | 3% | R/W | 🟡 | 🟡 | ~50% | 10–15 | `crates/formats/src/tests.rs` | ~1,500 lines: text, character/paragraph formatting, styles, lists, tables, pictures (`\shppict`, #154), fields, bookmarks. 8-bit text in the document's code page (`\ansicpg`) or the current font's (`\fcharset`, `\cpg`): Cyrillic, Central European, Greek, Turkish, Hebrew, Arabic, Baltic, Vietnamese, Thai (was Windows-1252 only), Mac OS Roman (`\mac`, `\fcharset77`) and DOS code pages (`\pc` 437, `\pca` 850, `\cpg` 437/737/850/852/857/866, `\fcharset254`) (#1653). East Asian multibyte code pages decoded as runs: 932 Shift-JIS (`\fcharset128`, Japanese), 936 GBK (134, Simplified Chinese), 949 UHC/windows-949 (129, Korean) and 950 Big5 (136, Traditional Chinese) through `\ansicpg`, `\cpg` or the font's charset, escaped or raw, a lead byte and its trail byte together; invalid sequences read as U+FFFD, and a run that starts `EF BB BF` is text, not a BOM (#1653). Pasted from the system clipboard where the platform offers it (Windows, Wayland) when there is no HTML (#687). Tracked insertions and deletions read and written (`\revtbl`, `\revised`/`\deleted`, authors, `\revdttm` dates), so Accept/Reject All work after opening or a round trip; tracked paragraph marks (`\par` inside `\revised`/`\deleted` groups, including a `\sect` inside a deletion) and character and paragraph formatting revisions (`\crauth`/`\crdate` with `\oldcprops`, `\prauth`/`\prdate` with `\oldpprops`) read and written; section and table formatting revisions (`\oldsprops`, `\oldtprops`) not yet. Sections on import (#1378): each `\sect` ends a section with its own page size and orientation, margins, header/footer distance, columns, break type, page-number restart, start and format, title page and headers/footers (those it leaves out link to the previous); export writes every section too (break type, page size and orientation, margins, header/footer distance, columns, page numbering, title page; not headers/footers). Thin: comments, notes, RTL | | `.odt` OpenDocument Text | 2% | R/W | 🟡 | 🟡 | ~50% | 8–12 | `crates/formats/src/tests.rs`, `crates/formats/src/odt.rs` unit tests, `crates/engine/tests/odt_sections.rs` | Styles, lists (#87), tables, pictures, headers/footers, page breaks (#86), keep with next (#2813), page size, orientation and margins (#810), sections: one per master-page change on a paragraph or table (next page, with its own page size, orientation, margins, columns and page-numbering restart) and per columned `text:section` (continuous, with its columns and separator line) (#1379); export writes them back: a master page and page layout per new-page section (size, orientation, margins, columns, page-numbering restart on the first paragraph or table), a `text:section` with its columns per continuous section whose columns differ from the page's (one column included), a list cut by a section break goes on numbering. Limits: a continuous section's page-numbering restart isn't written (ODF applies it only with a master page); a continuous section with the page's columns merges into the section before it on reopening; even and odd page starts are written as plain new pages; a tracked deletion of a section break is lost; an empty last section is dropped; tracked insertions and deletions with author and date, inserted and deleted paragraph marks and text inserted then deleted included (ODF change tracking; format changes read as none; pictures and tables inside a tracked deletion are dropped). No footnotes, comments, fields or RTL | -| `.rtf` Rich Text Format | 3% | R/W | 🟡 | 🟡 | ~50% | 10–15 | `crates/formats/src/tests.rs` | ~1,500 lines: text, character/paragraph formatting, styles, lists, tables, pictures (`\shppict`, #154), fields, bookmarks. 8-bit text in the document's code page (`\ansicpg`) or the current font's (`\fcharset`, `\cpg`): Cyrillic, Central European, Greek, Turkish, Hebrew, Arabic, Baltic, Vietnamese, Thai (was Windows-1252 only), Mac OS Roman (`\mac`, `\fcharset77`) and DOS code pages (`\pc` 437, `\pca` 850, `\cpg` 437/737/850/852/857/866, `\fcharset254`) (#1653). Double-byte East Asian code pages (Japanese `\fcharset128`, Chinese, Korean) not decoded yet. Pasted from the system clipboard where the platform offers it (Windows, Wayland) when there is no HTML (#687). Tracked insertions and deletions read and written (`\revtbl`, `\revised`/`\deleted`, authors, `\revdttm` dates), so Accept/Reject All work after opening or a round trip; paragraph-mark and formatting revisions not yet. Sections on import (#1378): each `\sect` ends a section with its own page size and orientation, margins, header/footer distance, columns, break type, page-number restart, start and format, title page and headers/footers (those it leaves out link to the previous); export writes every section too (break type, page size and orientation, margins, header/footer distance, columns, page numbering, title page; not headers/footers). Thin: comments, notes, RTL | | `.txt` Plain text | 2% | R/W | ✅ | ✅ | ~95% | 1–2 | unit tests | Encoding detection. Files with other extensions (`.m3u`/`.m3u8` playlists, logs, config files, no extension) open as text when they are text, Word and RTF files by their signature; binary files are refused (#569). A Word or RTF extension that contradicts the content also opens by signature, as in Word: RTF saved as `.doc` (hh.ru résumés, accounting exports), a `.docx` renamed `.doc`, a Word 97-2003 file renamed `.docx`; a Word XML Document (Word 2003 XML or Flat OPC) or a web page (``, ``) under a `.doc`, `.docx` or `.rtf` name opens as what it is (#2853, `tests::word_xml_and_web_pages_saved_as_doc_open_by_content`). Word's encoding choice dialog and line-ending options missing; such files save only through Save As | | `.htm/.html` Web Page and Web Page, Filtered | 1% | R/W | 🟡 | 🟡 | ~60% | 6–10 | `crates/formats/src/tests.rs` | Tables (#102), relative pictures embedded (#113), missing `` tolerated (#154). Header cells (``) bold unless their CSS says otherwise; `text-align` on a cell, or inherited from its row, row group or table, aligns the cell's paragraphs (#1381). HTML gives no page size, so it opens on the region's paper (A4 or Letter), as Markdown and plain text do; `@page` isn't read (#1381). No Word "Web Page" (with `_files` folder) round-trip. The same reader takes HTML from the system clipboard: Paste keeps another program's formatting (a Word document, a web page; Windows' "HTML Format" header understood), Merge Formatting keeps its emphasis (#687). A pasted `
` (LibreOffice's paragraph indented on both sides) becomes a Normal paragraph indented by the page's `blockquote` margins, else 0.5 in each side; opening an .html file keeps it as the Quote style (#1270). Equations are written as MathML (with the linear format as a UnicodeMath annotation) and read back from MathML: our annotation, else the presentation elements, else a TeX annotation (#816). A copy of one equation alone also goes in the clipboard's MathML formats (Windows "MathML" and "MathML Presentation", macOS `public.mathml.presentation` and `public.mathml`; W3C MathML 3 §6.2.4), which Microsoft documents Word reading first when it pastes math (#1297; the paste into Word is not yet verified) | | `.mht/.mhtml` Single File Web Page | <1% | R/W | ❌ | ❌ | 0% | 3–5 | — | | @@ -233,6 +232,7 @@ Imported text fields use the first nonempty cached result run's formatting on th | Date | Change | Summary | |---|---|---| +| 2026-10-21 | trivial | RTF import (#1653): the East Asian multibyte code pages are decoded — 932 Shift-JIS (Japanese), 936 GBK (Simplified Chinese), 949 UHC/windows-949 (Korean) and 950 Big5 (Traditional Chinese), named by `\ansicpg`, a font's `\cpg` or `\fcharset128`/`129`/`134`/`136` (and the OEM charset 255 as 437), escaped (`\'hh`) or raw. Bytes are decoded as a run, so a lead byte and its trail byte decode together; a code page switch, a group end and the end of the document decode what is pending; invalid sequences read as U+FFFD and never lose the rest of the text. The decoder does no BOM sniffing, so a run that starts `EF BB BF` in GBK, Big5 or UHC stays text, and ASCII runs and picture hex no longer decode byte by byte. Single-byte code pages, `\uN?` escapes and their `\ucN` skips are unchanged | | 2026-10-19 | trivial | Restrict Editing: with Filling in forms enforced, inserting content controls, their Properties and Remove Content Control are refused, as in Word (it greys them out), so a form's fields stay as they are and only their content is filled in | | 2026-10-19 | trivial | Content controls: text typed over a placeholder takes the control's own run properties (`w:sdtPr/w:rPr`, ECMA-376 §17.5.2.27), as in Word, and the placeholder style is recognised by its built-in name "Placeholder Text" whatever its localized id (German Word: `Platzhaltertext`), so entries in forms from localized Word no longer stay grey | | 2026-10-18 | trivial | A Word 2003 XML or Flat OPC document, or a web page, saved under a `.doc`, `.docx` or `.rtf` name opens by what it holds, as in Word, instead of going to the Word 97-2003 reader and showing a blank page (#2853) |