From 17a934506452e3c2f88e72be8ddba309fbcd8eb9 Mon Sep 17 00:00:00 2001 From: mtlcl410 <2368091+mtlcl410@users.noreply.github.com> Date: Mon, 2 Nov 2026 00:56:20 +0800 Subject: [PATCH] M8: an FF FE text string is searched as UTF-16LE, in /Info and the places applying leaves (#2448) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #2539 review: ISO 32000-2 §7.9.2.2.1 defines text strings in PDFDocEncoding, UTF-16BE (FE FF) and UTF-8 (EF BB BF) only, so an FF FE string was read through PDFDocEncoding and Cyrillic or CJK text in it could never match. It is now read as UTF-16LE for the search (fail-closed), in the /Info scrub and in the bookmark, attachment and comment counts. Test: a_utf16le_string_is_searched_as_utf16le (failed before: 0 /Info entries removed; with only the count reverted, bookmarks 0 instead of 1). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Pq7HyCfwzyRXj4VfQAVVdv --- crates/redact/README.md | 4 ++- crates/redact/src/elsewhere.rs | 7 ++--- crates/redact/src/metadata.rs | 17 +++++++++++- crates/redact/src/tests/metadata_matching.rs | 28 ++++++++++++++++++++ 4 files changed, 51 insertions(+), 5 deletions(-) diff --git a/crates/redact/README.md b/crates/redact/README.md index 129602287c..2cba145390 100644 --- a/crates/redact/README.md +++ b/crates/redact/README.md @@ -50,7 +50,9 @@ clear_marks(&mut doc, None)?; // remove marks without app "True" isn't part of "untrue". This over-matches only towards removal: "is a" also holds "Lisa". Both sides are compared in Unicode compatibility form (NFKC) and lower case, so a decomposed "Müller" (u and U+0308), full-width letters and a "fi" ligature from a - `ToUnicode` map match their plain forms. A value that shares only words of one or two + `ToUnicode` map match their plain forms. A string behind an `FF FE` mark, which ISO 32000-2 + §7.9.2.2.1 doesn't define, is read as UTF-16LE, here and in the places reported below, rather + than through PDFDocEncoding, in which it could never match. A value that shares only words of one or two letters with the removed text, and isn't wholly part of it, stays: Sanitize's metadata category removes the rest. `Report::metadata` counts what went. - **Fail-closed.** When the check can't know what the removed text was, every `/Info` diff --git a/crates/redact/src/elsewhere.rs b/crates/redact/src/elsewhere.rs index 901775ac1b..4ebac7bad9 100644 --- a/crates/redact/src/elsewhere.rs +++ b/crates/redact/src/elsewhere.rs @@ -64,8 +64,9 @@ pub(crate) fn find(doc: &Document, removed: &[String], complete: bool) -> Elsewh return Elsewhere::default(); } // A blank value holds nothing, even when everything counts. - let hit = - |o: Option<&Object>| o.and_then(|o| doc.resolve(o).as_string().map(|s| s.to_text())).is_some_and(|t| !t.trim().is_empty() && search.text(&t)); + let hit = |o: Option<&Object>| { + o.and_then(|o| doc.resolve(o).as_string().map(crate::metadata::text_string)).is_some_and(|t| !t.trim().is_empty() && search.text(&t)) + }; Elsewhere { bookmarks: bookmarks(doc, &hit), attachments: attachments(doc, &hit), @@ -182,7 +183,7 @@ fn comments(doc: &Document, search: &Search, hit: &dyn Fn(Option<&Object>) -> bo /// can't be read (strictly, as UTF-8, within the XMP limit). fn rich_text(doc: &Document, o: &Object) -> Option { let xml = match &*doc.resolve(o) { - Object::String(s) => s.to_text(), + Object::String(s) => crate::metadata::text_string(s), Object::Stream(s) if s.filters().len() <= 1 => String::from_utf8(s.decoded_first_filter_strict(crate::metadata::MAX_XMP_BYTES).ok()?).ok()?, Object::Stream(_) => return None, _ => return Some(String::new()), diff --git a/crates/redact/src/metadata.rs b/crates/redact/src/metadata.rs index 9c6d435d27..18cff41907 100644 --- a/crates/redact/src/metadata.rs +++ b/crates/redact/src/metadata.rs @@ -358,6 +358,21 @@ pub(crate) fn decode_xmp(bytes: &[u8]) -> Option { } } +/// The text of a text string, as searched for the removed text. ISO 32000-2 §7.9.2.2.1 defines +/// PDFDocEncoding, UTF-16BE (`FE FF`) and UTF-8 (`EF BB BF`) only, but a string behind an `FF FE` +/// mark is UTF-16LE to any reader that takes it as such, and read through PDFDocEncoding it +/// could never match: it is read as UTF-16LE (an odd last byte is left out, a lone surrogate +/// becomes U+FFFD), so the search stays fail-closed (#2539 review). +pub(crate) fn text_string(s: &pdfcraft_cos::PdfString) -> String { + match s.bytes.as_slice() { + [0xFF, 0xFE, rest @ ..] => { + let units = rest.as_chunks::<2>().0.iter().map(|&p| u16::from_le_bytes(p)); + char::decode_utf16(units).map(|c| c.unwrap_or(char::REPLACEMENT_CHARACTER)).collect() + } + _ => s.to_text(), + } +} + /// The text values of an XMP stream, or `None` when it can't be read. fn xmp_values(doc: &Document, o: &Object) -> Option> { let Object::Stream(s) = &*doc.resolve(o) else { return None }; @@ -466,7 +481,7 @@ pub(crate) fn scrub_within(doc: &mut Document, removed: &[String], complete: boo let info_obj = doc.trailer().get(b"Info").cloned(); if let Some(info) = info_obj.as_ref().and_then(|i| doc.resolve(i).as_dict().cloned()) { let doomed: Vec> = - info.iter().filter(|(_, v)| doc.resolve(v).as_string().is_some_and(|s| search.text(&s.to_text()))).map(|(k, _)| k.to_vec()).collect(); + info.iter().filter(|(_, v)| doc.resolve(v).as_string().is_some_and(|s| search.text(&text_string(s)))).map(|(k, _)| k.to_vec()).collect(); if !doomed.is_empty() { n += doomed.len(); let mut kept = info.clone(); diff --git a/crates/redact/src/tests/metadata_matching.rs b/crates/redact/src/tests/metadata_matching.rs index d944dac766..441ad104a3 100644 --- a/crates/redact/src/tests/metadata_matching.rs +++ b/crates/redact/src/tests/metadata_matching.rs @@ -68,3 +68,31 @@ fn a_value_that_normalizes_to_far_more_text_counts_as_holding_it() { assert_eq!(metadata::scrub_within(&mut doc, &["Bluefin".into()], true, 1 << 28).unwrap(), 1); assert_eq!(info_keys(&doc), ["Producer"]); } + +/// `s` as UTF-16LE bytes in hex, behind an `FF FE` byte order mark. +fn utf16le(s: &str) -> String { + format!("", s.encode_utf16().map(|u| format!("{:02X}{:02X}", u & 0xFF, u >> 8)).collect::()) +} + +/// ISO 32000-2 §7.9.2.2.1 defines text strings in PDFDocEncoding, UTF-16BE (`FE FF`) and UTF-8 +/// (`EF BB BF`) only, but a string behind an `FF FE` mark is UTF-16LE to anyone who reads it as +/// such; read through PDFDocEncoding, Cyrillic or CJK text in it could never match. It is +/// searched as UTF-16LE in `/Info` and in the places applying leaves (here a bookmark title). +#[test] +fn a_utf16le_string_is_searched_as_utf16le() { + let info = format!("<< /Title {} /Subject {} /Producer (Test) >>", utf16le("Синий план"), utf16le("Годовой отчёт")); + let mut doc = with_metadata(b"BT /F1 10 Tf 10 100 Td (x) Tj ET", &info, ""); + assert_eq!(metadata::scrub_within(&mut doc, &["СИНИЙ".into()], true, 1 << 28).unwrap(), 1); + assert_eq!(info_keys(&doc), ["Subject", "Producer"]); + + // A bookmark titled the same way is counted as still holding it. + let objs: Vec> = vec![ + b"<< /Type /Catalog /Pages 2 0 R /Outlines 4 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] >>".to_vec(), + b"<< /Type /Outlines /First 5 0 R /Last 5 0 R /Count 1 >>".to_vec(), + format!("<< /Title {} /Parent 4 0 R /Dest [3 0 R /Fit] >>", utf16le("Синий план")).into_bytes(), + ]; + let doc = Document::open(Arc::new(Pdf::new().objs(objs).build())).unwrap(); + assert_eq!(crate::elsewhere::find(&doc, &["СИНИЙ".into()], true), Elsewhere { bookmarks: 1, ..Default::default() }); +}