M8: an FF FE text string is searched as UTF-16LE, in /Info and the places applying leaves (#2448)

#2539 review: ISO 32000-2 §7.9.2.2.1 defines text strings in PDFDocEncoding,
UTF-16BE (FE FF) and UTF-8 (EF BB BF) only, so an FF FE string was read
through PDFDocEncoding and Cyrillic or CJK text in it could never match.
It is now read as UTF-16LE for the search (fail-closed), in the /Info scrub
and in the bookmark, attachment and comment counts.

Test: a_utf16le_string_is_searched_as_utf16le (failed before: 0 /Info
entries removed; with only the count reverted, bookmarks 0 instead of 1).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Pq7HyCfwzyRXj4VfQAVVdv

crates/redact/README.md

@@ -50,7 +50,9 @@ clear_marks(&mut doc, None)?;                        // remove marks without app
   "True" isn't part of "untrue". This over-matches only towards removal: "is a" also holds
   "Lisa". Both sides are compared in Unicode compatibility form (NFKC) and lower case, so a
   decomposed "Müller" (u and U+0308), full-width letters and a "fi" ligature from a
-  `ToUnicode` map match their plain forms. A value that shares only words of one or two
+  `ToUnicode` map match their plain forms. A string behind an `FF FE` mark, which ISO 32000-2
+  §7.9.2.2.1 doesn't define, is read as UTF-16LE, here and in the places reported below, rather
+  than through PDFDocEncoding, in which it could never match. A value that shares only words of one or two
   letters with the removed text, and isn't wholly part of it, stays:
   Sanitize's metadata category removes the rest. `Report::metadata` counts what went.
   - **Fail-closed.** When the check can't know what the removed text was, every `/Info`

crates/redact/src/elsewhere.rs

@@ -64,8 +64,9 @@ pub(crate) fn find(doc: &Document, removed: &[String], complete: bool) -> Elsewh
         return Elsewhere::default();
     }
     // A blank value holds nothing, even when everything counts.
-    let hit =
-        |o: Option<&Object>| o.and_then(|o| doc.resolve(o).as_string().map(|s| s.to_text())).is_some_and(|t| !t.trim().is_empty() && search.text(&t));
+    let hit = |o: Option<&Object>| {
+        o.and_then(|o| doc.resolve(o).as_string().map(crate::metadata::text_string)).is_some_and(|t| !t.trim().is_empty() && search.text(&t))
+    };
     Elsewhere {
         bookmarks: bookmarks(doc, &hit),
         attachments: attachments(doc, &hit),
@@ -182,7 +183,7 @@ fn comments(doc: &Document, search: &Search, hit: &dyn Fn(Option<&Object>) -> bo
 /// can't be read (strictly, as UTF-8, within the XMP limit).
 fn rich_text(doc: &Document, o: &Object) -> Option<String> {
     let xml = match &*doc.resolve(o) {
-        Object::String(s) => s.to_text(),
+        Object::String(s) => crate::metadata::text_string(s),
         Object::Stream(s) if s.filters().len() <= 1 => String::from_utf8(s.decoded_first_filter_strict(crate::metadata::MAX_XMP_BYTES).ok()?).ok()?,
         Object::Stream(_) => return None,
         _ => return Some(String::new()),

crates/redact/src/metadata.rs

@@ -358,6 +358,21 @@ pub(crate) fn decode_xmp(bytes: &[u8]) -> Option<String> {
     }
 }
 
+/// The text of a text string, as searched for the removed text. ISO 32000-2 §7.9.2.2.1 defines
+/// PDFDocEncoding, UTF-16BE (`FE FF`) and UTF-8 (`EF BB BF`) only, but a string behind an `FF FE`
+/// mark is UTF-16LE to any reader that takes it as such, and read through PDFDocEncoding it
+/// could never match: it is read as UTF-16LE (an odd last byte is left out, a lone surrogate
+/// becomes U+FFFD), so the search stays fail-closed (#2539 review).
+pub(crate) fn text_string(s: &pdfcraft_cos::PdfString) -> String {
+    match s.bytes.as_slice() {
+        [0xFF, 0xFE, rest @ ..] => {
+            let units = rest.as_chunks::<2>().0.iter().map(|&p| u16::from_le_bytes(p));
+            char::decode_utf16(units).map(|c| c.unwrap_or(char::REPLACEMENT_CHARACTER)).collect()
+        }
+        _ => s.to_text(),
+    }
+}
+
 /// The text values of an XMP stream, or `None` when it can't be read.
 fn xmp_values(doc: &Document, o: &Object) -> Option<Vec<String>> {
     let Object::Stream(s) = &*doc.resolve(o) else { return None };
@@ -466,7 +481,7 @@ pub(crate) fn scrub_within(doc: &mut Document, removed: &[String], complete: boo
     let info_obj = doc.trailer().get(b"Info").cloned();
     if let Some(info) = info_obj.as_ref().and_then(|i| doc.resolve(i).as_dict().cloned()) {
         let doomed: Vec<Vec<u8>> =
-            info.iter().filter(|(_, v)| doc.resolve(v).as_string().is_some_and(|s| search.text(&s.to_text()))).map(|(k, _)| k.to_vec()).collect();
+            info.iter().filter(|(_, v)| doc.resolve(v).as_string().is_some_and(|s| search.text(&text_string(s)))).map(|(k, _)| k.to_vec()).collect();
         if !doomed.is_empty() {
             n += doomed.len();
             let mut kept = info.clone();

crates/redact/src/tests/metadata_matching.rs

@@ -68,3 +68,31 @@ fn a_value_that_normalizes_to_far_more_text_counts_as_holding_it() {
     assert_eq!(metadata::scrub_within(&mut doc, &["Bluefin".into()], true, 1 << 28).unwrap(), 1);
     assert_eq!(info_keys(&doc), ["Producer"]);
 }
+
+/// `s` as UTF-16LE bytes in hex, behind an `FF FE` byte order mark.
+fn utf16le(s: &str) -> String {
+    format!("<FFFE{}>", s.encode_utf16().map(|u| format!("{:02X}{:02X}", u & 0xFF, u >> 8)).collect::<String>())
+}
+
+/// ISO 32000-2 §7.9.2.2.1 defines text strings in PDFDocEncoding, UTF-16BE (`FE FF`) and UTF-8
+/// (`EF BB BF`) only, but a string behind an `FF FE` mark is UTF-16LE to anyone who reads it as
+/// such; read through PDFDocEncoding, Cyrillic or CJK text in it could never match. It is
+/// searched as UTF-16LE in `/Info` and in the places applying leaves (here a bookmark title).
+#[test]
+fn a_utf16le_string_is_searched_as_utf16le() {
+    let info = format!("<< /Title {} /Subject {} /Producer (Test) >>", utf16le("Синий план"), utf16le("Годовой отчёт"));
+    let mut doc = with_metadata(b"BT /F1 10 Tf 10 100 Td (x) Tj ET", &info, "");
+    assert_eq!(metadata::scrub_within(&mut doc, &["СИНИЙ".into()], true, 1 << 28).unwrap(), 1);
+    assert_eq!(info_keys(&doc), ["Subject", "Producer"]);
+
+    // A bookmark titled the same way is counted as still holding it.
+    let objs: Vec<Vec<u8>> = vec![
+        b"<< /Type /Catalog /Pages 2 0 R /Outlines 4 0 R >>".to_vec(),
+        b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(),
+        b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] >>".to_vec(),
+        b"<< /Type /Outlines /First 5 0 R /Last 5 0 R /Count 1 >>".to_vec(),
+        format!("<< /Title {} /Parent 4 0 R /Dest [3 0 R /Fit] >>", utf16le("Синий план")).into_bytes(),
+    ];
+    let doc = Document::open(Arc::new(Pdf::new().objs(objs).build())).unwrap();
+    assert_eq!(crate::elsewhere::find(&doc, &["СИНИЙ".into()], true), Elsewhere { bookmarks: 1, ..Default::default() });
+}