//! The text of an RTF file: `\uN` with its fallback characters, and 8-bit text in a code page. use crate::codepage::{code_page_byte, multibyte_encoding_name}; fn multibyte(code_page: u16) -> Option<&'static encoding_rs::Encoding> { encoding_rs::Encoding::for_label(multibyte_encoding_name(code_page)?.as_bytes()) } /// Decodes the text tokens of one RTF file into `String`s the caller supplies. /// /// The caller says which code page applies to each piece of 8-bit text (the current font's, /// else the default font's, else the document's) and how many fallback characters follow a /// `\uN` (`\ucN`, 1 when not set). /// /// - `\uN` is one UTF-16 code unit in the signed/unsigned 16-bit range. A high surrogate pairs /// only with a low surrogate in the very next `\u`; unmatched surrogates and out-of-range /// values are dropped, matching the historical DesignCraft reader. /// - After a `\uN`, the next `\ucN` characters are its fallback and are skipped: a text byte, /// a `\'hh`, a control word and a control symbol each count as one ([`TextDecoder::skip`]). /// The skipping ends with the group ([`TextDecoder::group`]). /// - A single-byte code page maps each byte. A multibyte one (932, 936, 949, 950) streams /// through one decoder, so a lead byte and its trail byte decode together even when a /// `\'hh`, a text run or a formatting word falls between them, and a run that happens to /// start `EF BB BF` is not taken for a UTF-8 byte order mark. A lead byte without its trail /// byte reads as U+FFFD. #[derive(Default)] pub struct TextDecoder { skip: usize, high: Option, mb: Option, mb_cp: u16, } impl TextDecoder { pub fn new() -> TextDecoder { TextDecoder::default() } /// A held high surrogate that nothing paired with. fn flush_high(&mut self) { self.high = None; } /// `\uN` with `uc` fallback characters to skip after it. pub fn unicode(&mut self, param: i32, uc: usize, out: &mut String) { self.finish_multibyte(out); if !(-32_768..=65_535).contains(¶m) { self.skip = uc; return; } let u = i64::from(param).rem_euclid(65_536) as u16; if (0xD800..0xDC00).contains(&u) { self.flush_high(); self.high = Some(u); } else { match self.high.take() { Some(h) if (0xDC00..0xE000).contains(&u) => { if let Some(c) = char::decode_utf16([h, u]).next().and_then(Result::ok) { out.push(c); } } Some(_) | None if (0xDC00..0xE000).contains(&u) => {} Some(_) | None => { if let Some(c) = char::from_u32(u32::from(u)) { out.push(c); } } } } self.skip = uc; } /// Is the control word or symbol just read a fallback character of the last `\uN`? Then it /// is used up and the caller ignores it. pub fn skip(&mut self) -> bool { let skipped = self.skip > 0; self.skip = self.skip.saturating_sub(1); skipped } /// A character a control word or symbol stands for (`\emdash`, `\~`). pub fn char(&mut self, c: char, out: &mut String) { self.finish_multibyte(out); self.flush_high(); out.push(c); } /// `\'hh`: one byte in `code_page`. pub fn byte(&mut self, b: u8, code_page: u16, out: &mut String) { if !self.skip() { self.decode(&[b], code_page, out); } } /// A run of text bytes in `code_page`. pub fn bytes(&mut self, chunk: &[u8], code_page: u16, out: &mut String) { let skip = self.skip.min(chunk.len()); self.skip -= skip; let chunk = chunk.get(skip..).unwrap_or_default(); if !chunk.is_empty() { self.decode(chunk, code_page, out); } } fn decode(&mut self, chunk: &[u8], cp: u16, out: &mut String) { self.flush_high(); if self.mb.is_none() && chunk.is_ascii() { // ASCII is itself in every code page here; with no lead byte pending, as it is. out.push_str(std::str::from_utf8(chunk).unwrap_or_default()); return; } let Some(encoding) = multibyte(cp) else { self.finish_multibyte(out); out.extend(chunk.iter().map(|b| code_page_byte(cp, *b))); return; }; // A code page switch ends the previous run before the new decoder starts. if self.mb_cp != cp { self.finish_multibyte(out); } self.mb_cp = cp; let decoder = self.mb.get_or_insert_with(|| encoding.new_decoder_without_bom_handling()); let mut rest = chunk; loop { out.reserve(rest.len().saturating_mul(3).saturating_add(16)); let (result, read, _) = decoder.decode_to_string(rest, out, false); rest = rest.get(read..).unwrap_or_default(); if !matches!(result, encoding_rs::CoderResult::OutputFull) { break; } } } /// Ends the current multibyte run: a lead byte still waiting for its trail byte becomes /// U+FFFD. fn finish_multibyte(&mut self, out: &mut String) { if let Some(mut decoder) = self.mb.take() { out.reserve(16); let _ = decoder.decode_to_string(&[], out, true); } } /// A group opened or closed: the fallback of a `\uN` ends with its group, and so does a /// multibyte run. pub fn group(&mut self, out: &mut String) { self.finish_multibyte(out); self.skip = 0; } /// The text ends here (a paragraph end, a cell end, the end of the file): nothing waits /// for a partner any longer. pub fn finish(&mut self, out: &mut String) { self.finish_multibyte(out); self.flush_high(); } } #[cfg(test)] mod tests { use super::*; fn units(ns: &[i32], uc: usize) -> String { let (mut d, mut out) = (TextDecoder::new(), String::new()); for n in ns { d.unicode(*n, uc, &mut out); } d.finish(&mut out); out } #[test] fn surrogates_pair_only_with_the_next_unit() { assert_eq!(units(&[-10179, -8704], 0), "\u{1F600}"); assert_eq!(units(&[0xD83D, 0xDE00], 0), "\u{1F600}", "written unsigned"); assert_eq!(units(&[8364], 0), "€"); assert_eq!(units(&[-10179], 0), "", "a lone high surrogate is dropped"); assert_eq!(units(&[-8704], 0), "", "a lone low surrogate is dropped"); assert_eq!(units(&[-8704, -10179], 0), "", "low then high are dropped"); assert_eq!(units(&[-10179, -10179, -8704], 0), "\u{1F600}", "an unpaired high is dropped before a pair"); assert_eq!(units(&[-10179, 65, -8704], 0), "A"); assert_eq!(units(&[65_536 + 65, -65_536 + 66], 0), "", "out-of-range units are dropped"); assert_eq!(units(&[i32::MAX, 65, i32::MIN], 0), "A", "extreme units are dropped"); } #[test] fn a_held_high_surrogate_does_not_pair_across_text() { let (mut d, mut out) = (TextDecoder::new(), String::new()); d.unicode(-10179, 0, &mut out); d.bytes(b"x", 1252, &mut out); d.unicode(-8704, 0, &mut out); d.unicode(-10179, 0, &mut out); d.char('\u{2014}', &mut out); d.unicode(-10179, 1, &mut out); // The fallback and a formatting group do not come between a pair. d.bytes(b"?", 1252, &mut out); d.group(&mut out); d.unicode(-8704, 1, &mut out); d.finish(&mut out); assert_eq!(out, "x\u{2014}\u{1F600}"); } #[test] fn fallback_characters_are_skipped() { let (mut d, mut out) = (TextDecoder::new(), String::new()); d.unicode(8364, 1, &mut out); d.bytes(b"? price", 1252, &mut out); d.unicode(8220, 2, &mut out); d.byte(0x93, 1252, &mut out); assert!(d.skip(), "a control word is one fallback character"); assert!(!d.skip()); d.unicode(233, 3, &mut out); d.bytes(b"e", 1252, &mut out); d.group(&mut out); d.bytes(b"AB", 1252, &mut out); assert_eq!(out, "\u{20AC} price\u{201C}\u{E9}AB", "the skip ends with the group"); } #[test] fn single_byte_code_pages() { let (mut d, mut out) = (TextDecoder::new(), String::new()); d.bytes(&[b'C', b'a', b'f', 0xE9, b' '], 1252, &mut out); d.byte(0xC4, 1251, &mut out); d.byte(0x8E, 10000, &mut out); d.byte(0x82, 437, &mut out); d.byte(0xE9, 9999, &mut out); assert_eq!(out, "Café Дééé"); } #[test] fn a_lead_byte_and_its_trail_byte_decode_together() { // 日本 in Shift-JIS is 93 FA 96 7B: the first character as two `\'hh`, the second as a // `\'hh` lead byte and a raw trail byte (`{`'s neighbour 0x7B is a text byte here). let (mut d, mut out) = (TextDecoder::new(), String::new()); d.byte(0x93, 932, &mut out); d.byte(0xFA, 932, &mut out); d.byte(0x96, 932, &mut out); d.bytes(b"\x7bA", 932, &mut out); assert_eq!(out, "日本A"); // A lead byte left at a group end, a code page switch or the end is U+FFFD. for end in 0..3 { let (mut d, mut out) = (TextDecoder::new(), String::new()); d.byte(0x93, 932, &mut out); match end { 0 => d.group(&mut out), 1 => d.bytes(&[0xE9], 1252, &mut out), _ => d.finish(&mut out), } assert!(out.starts_with('\u{FFFD}'), "{end}: {out:?}"); } // GBK text that starts EF BB BF is not a UTF-8 byte order mark. let (mut d, mut out) = (TextDecoder::new(), String::new()); d.bytes(&[0xEF, 0xBB, 0xBF, 0xC4, 0xE3], 936, &mut out); d.finish(&mut out); assert_eq!(out.chars().count(), 3, "{out:?}"); assert!(out.ends_with('\u{FFFD}') && !out.starts_with('\u{FEFF}'), "{out:?}"); // A long run decodes whole. let (mut d, mut out) = (TextDecoder::new(), String::new()); d.bytes(&[0x93, 0xFA].repeat(10_000), 932, &mut out); assert_eq!(out.chars().count(), 10_000); } }