From 4fa98367f8f7e177728f6948f6e71d6094957703 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sat, 14 Nov 2026 11:10:12 +0000 Subject: [PATCH] rtf: preserve legacy invalid Unicode handling --- crates/rtf/src/text.rs | 53 ++++++++++++++++++++-------------- crates/rtf/tests/read_write.rs | 16 ++++++---- 2 files changed, 42 insertions(+), 27 deletions(-) diff --git a/crates/rtf/src/text.rs b/crates/rtf/src/text.rs index b1411842e..7cca12d8e 100644 --- a/crates/rtf/src/text.rs +++ b/crates/rtf/src/text.rs @@ -12,8 +12,9 @@ fn multibyte(code_page: u16) -> Option<&'static encoding_rs::Encoding> { /// else the default font's, else the document's) and how many fallback characters follow a /// `\uN` (`\ucN`, 1 when not set). /// -/// - `\uN` is one UTF-16 code unit. A high surrogate pairs only with a low surrogate in the -/// very next `\u`; every surrogate that does not pair reads as U+FFFD, as in Word. +/// - `\uN` is one UTF-16 code unit in the signed/unsigned 16-bit range. A high surrogate pairs +/// only with a low surrogate in the very next `\u`; unmatched surrogates and out-of-range +/// values are dropped, matching the historical DesignCraft reader. /// - After a `\uN`, the next `\ucN` characters are its fallback and are skipped: a text byte, /// a `\'hh`, a control word and a control symbol each count as one ([`TextDecoder::skip`]). /// The skipping ends with the group ([`TextDecoder::group`]). @@ -36,27 +37,34 @@ impl TextDecoder { } /// A held high surrogate that nothing paired with. - fn flush_high(&mut self, out: &mut String) { - if self.high.take().is_some() { - out.push('\u{FFFD}'); - } + fn flush_high(&mut self) { + self.high = None; } /// `\uN` with `uc` fallback characters to skip after it. pub fn unicode(&mut self, param: i32, uc: usize, out: &mut String) { self.finish_multibyte(out); + if !(-32_768..=65_535).contains(¶m) { + self.skip = uc; + return; + } let u = i64::from(param).rem_euclid(65_536) as u16; if (0xD800..0xDC00).contains(&u) { - self.flush_high(out); + self.flush_high(); self.high = Some(u); } else { match self.high.take() { - Some(h) if (0xDC00..0xE000).contains(&u) => out.push(char::decode_utf16([h, u]).next().and_then(Result::ok).unwrap_or('\u{FFFD}')), - Some(_) => { - out.push('\u{FFFD}'); - out.push(char::from_u32(u32::from(u)).unwrap_or('\u{FFFD}')); + Some(h) if (0xDC00..0xE000).contains(&u) => { + if let Some(c) = char::decode_utf16([h, u]).next().and_then(Result::ok) { + out.push(c); + } + } + Some(_) | None if (0xDC00..0xE000).contains(&u) => {} + Some(_) | None => { + if let Some(c) = char::from_u32(u32::from(u)) { + out.push(c); + } } - None => out.push(char::from_u32(u32::from(u)).unwrap_or('\u{FFFD}')), } } self.skip = uc; @@ -73,7 +81,7 @@ impl TextDecoder { /// A character a control word or symbol stands for (`\emdash`, `\~`). pub fn char(&mut self, c: char, out: &mut String) { self.finish_multibyte(out); - self.flush_high(out); + self.flush_high(); out.push(c); } @@ -95,7 +103,7 @@ impl TextDecoder { } fn decode(&mut self, chunk: &[u8], cp: u16, out: &mut String) { - self.flush_high(out); + self.flush_high(); if self.mb.is_none() && chunk.is_ascii() { // ASCII is itself in every code page here; with no lead byte pending, as it is. out.push_str(std::str::from_utf8(chunk).unwrap_or_default()); @@ -143,7 +151,7 @@ impl TextDecoder { /// for a partner any longer. pub fn finish(&mut self, out: &mut String) { self.finish_multibyte(out); - self.flush_high(out); + self.flush_high(); } } @@ -165,12 +173,13 @@ mod tests { assert_eq!(units(&[-10179, -8704], 0), "\u{1F600}"); assert_eq!(units(&[0xD83D, 0xDE00], 0), "\u{1F600}", "written unsigned"); assert_eq!(units(&[8364], 0), "€"); - assert_eq!(units(&[-10179], 0), "\u{FFFD}", "a lone high surrogate at the end"); - assert_eq!(units(&[-8704], 0), "\u{FFFD}", "a lone low surrogate"); - assert_eq!(units(&[-8704, -10179], 0), "\u{FFFD}\u{FFFD}", "low then high"); - assert_eq!(units(&[-10179, -10179, -8704], 0), "\u{FFFD}\u{1F600}", "a high surrogate before a pair"); - assert_eq!(units(&[-10179, 65, -8704], 0), "\u{FFFD}A\u{FFFD}"); - assert_eq!(units(&[65536 + 65, -65536 + 66], 0), "AB", "taken modulo 65536"); + assert_eq!(units(&[-10179], 0), "", "a lone high surrogate is dropped"); + assert_eq!(units(&[-8704], 0), "", "a lone low surrogate is dropped"); + assert_eq!(units(&[-8704, -10179], 0), "", "low then high are dropped"); + assert_eq!(units(&[-10179, -10179, -8704], 0), "\u{1F600}", "an unpaired high is dropped before a pair"); + assert_eq!(units(&[-10179, 65, -8704], 0), "A"); + assert_eq!(units(&[65_536 + 65, -65_536 + 66], 0), "", "out-of-range units are dropped"); + assert_eq!(units(&[i32::MAX, 65, i32::MIN], 0), "A", "extreme units are dropped"); } #[test] @@ -187,7 +196,7 @@ mod tests { d.group(&mut out); d.unicode(-8704, 1, &mut out); d.finish(&mut out); - assert_eq!(out, "\u{FFFD}x\u{FFFD}\u{FFFD}\u{2014}\u{1F600}"); + assert_eq!(out, "x\u{2014}\u{1F600}"); } #[test] diff --git a/crates/rtf/tests/read_write.rs b/crates/rtf/tests/read_write.rs index 9ef8fbfb7..32209b992 100644 --- a/crates/rtf/tests/read_write.rs +++ b/crates/rtf/tests/read_write.rs @@ -103,8 +103,8 @@ fn pictures_and_objects_are_counted_not_read() { } #[test] -fn out_of_range_unicode_wraps_and_hyphens_are_text() { - assert_eq!(text(br"{\rtf1 a\u4294967361?b}"), "a\u{FFFF}b", "clamped to i32, then modulo 65536"); +fn invalid_unicode_and_hyphens_are_compatible() { + assert_eq!(text(br"{\rtf1 a\u4294967361?b}"), "ab", "out-of-range signed parameters are dropped"); assert_eq!(text(br"{\rtf1\ansi Rock\emdash-roll, x\tab-y, a\li-720 b}"), "Rock\u{2014}-roll, x\t-y, ab"); assert_eq!(text(br"{\rtf1 {\b A\'4}B}"), "AB"); assert_eq!(text(br"{\rtf1 a\'+fb}"), "a+fb"); @@ -132,9 +132,15 @@ fn nul_is_not_text() { #[test] fn surrogate_pairs() { assert_eq!(text(br"{\rtf1\ansi\uc1 A\u-10179?\u-8704?B \u-10174?\u-8265?\u30000?}"), "A\u{1F600}B \u{20BB7}\u{7530}"); - assert_eq!(text(br"{\rtf1 A\u-10179?B\u-8704?C}"), "A\u{FFFD}B\u{FFFD}C", "half a pair is U+FFFD, as in Word"); - assert_eq!(text(br"{\rtf1 A\u-10179?\par\u-8704?C}"), "A\u{FFFD}\n\u{FFFD}C", "a pair does not span a paragraph end"); - assert_eq!(text(br"{\rtf1 A\u-10179?}"), "A\u{FFFD}"); + assert_eq!(text(br"{\rtf1 A\u-10179?B\u-8704?C}"), "ABC", "unpaired surrogates are dropped, as in the previous DesignCraft reader"); + assert_eq!(text(br"{\rtf1 A\u-10179?\par\u-8704?C}"), "A\nC", "a pair does not span a paragraph end"); + assert_eq!(text(br"{\rtf1 A\u-10179?}"), "A"); +} + +#[test] +fn out_of_range_unicode_escapes_are_dropped() { + assert_eq!(text(br"{\rtf1 a\u4294967361?b}"), "ab"); + assert_eq!(text(br"{\rtf1 a\u-4294967231?b}"), "ab"); } #[test]