M8: the metadata check compares NFKC, lower-case text, so Unicode forms can't hide the removed text (#2448)

#2539 review: a decomposed "Müller" in a UTF-16BE /Title, a U+FB01 ligature
from a ToUnicode map and full-width letters never matched their plain forms.
Both sides are now normalized to NFKC and lower case (unicode-normalization,
already in the build through pdfcraft-crypt's stringprep, so no new crate).
Normalized text more than four times its input's length (U+FDFA and the like)
isn't searched: it counts as holding the text.

Tests: unicode_forms_of_the_removed_text_match (failed before: Title, Subject
and Keywords kept), a_value_that_normalizes_to_far_more_text_counts_as_holding_it
(failed with the growth limit raised to 100: 0 removed).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Pq7HyCfwzyRXj4VfQAVVdv

Cargo.lock

@@ -4586,6 +4586,7 @@ dependencies = [
  "pdfcraft-testkit",
  "pdfcraft-xfa",
  "thiserror 2.0.21",
+ "unicode-normalization",
 ]
 
 [[package]]

crates/redact/Cargo.toml

@@ -18,6 +18,8 @@ pdfcraft-edit = { workspace = true }
 pdfcraft-annot = { workspace = true }
 pdfcraft-xfa = { workspace = true }
 thiserror = { workspace = true }
+# NFKC for the metadata check; already in the build through pdfcraft-crypt (stringprep).
+unicode-normalization = "0.1"
 
 [dev-dependencies]
 pdfcraft-filters = { workspace = true }

crates/redact/README.md

@@ -48,8 +48,10 @@ clear_marks(&mut doc, None)?;                        // remove marks without app
   "B L U E F I N", since a `TJ` gap can't be told from a word space). A value is wholly part of
   a run when its letters and digits are, starting and ending where the run's words do, so
   "True" isn't part of "untrue". This over-matches only towards removal: "is a" also holds
-  "Lisa". A value that shares
-  only words of one or two letters with the removed text, and isn't wholly part of it, stays:
+  "Lisa". Both sides are compared in Unicode compatibility form (NFKC) and lower case, so a
+  decomposed "Müller" (u and U+0308), full-width letters and a "fi" ligature from a
+  `ToUnicode` map match their plain forms. A value that shares only words of one or two
+  letters with the removed text, and isn't wholly part of it, stays:
   Sanitize's metadata category removes the rest. `Report::metadata` counts what went.
   - **Fail-closed.** When the check can't know what the removed text was, every `/Info`
     string and every catalog and page XMP stream goes: more than 100,000 runs or a run over
@@ -60,7 +62,8 @@ clear_marks(&mut doc, None)?;                        // remove marks without app
     search's work limit (2^28 steps) runs out, every entry and packet not yet checked goes
     too. An XMP packet that isn't valid UTF-8 or UTF-16, can't be decoded strictly (damaged or
     cut-short filter data, or more than one filter), or decodes to more than
-    16 MiB goes without being read.
+    16 MiB goes without being read, and a value whose normalized form would be more than four
+    times as long (U+FDFA alone is 18 letters) counts as holding the text.
   - **Not scrubbed, but reported** (Remove hidden information's categories remove them): bookmark
     titles, attachments (names in the document's list of embedded files, file names and
     descriptions, and the files of file attachment comments), the comments that remain (their

crates/redact/src/metadata.rs

@@ -23,6 +23,7 @@
 use std::collections::HashSet;
 
 use pdfcraft_cos::{Document, ObjRef, Object};
+use unicode_normalization::UnicodeNormalization;
 
 /// Terms longer than this are cut (matching still finds the prefix).
 const MAX_TERM_CHARS: usize = 4096;
@@ -33,10 +34,34 @@ pub(crate) const MAX_XMP_BYTES: usize = 16 * 1024 * 1024;
 /// do in all; past it every entry and packet counts as matching (fail-closed), so a page of
 /// removed text against a large packet, or against very many small values, stays quick.
 const MAX_WORK: usize = 1 << 28;
+/// Normalized text longer than this many times its input (plus 64 bytes) isn't searched: it
+/// counts as matching. Ordinary text barely changes length; a few compatibility characters
+/// (U+FDFA, the square katakana words) decompose to many.
+const MAX_GROWTH: usize = 4;
 
-/// Lower-case `s` with every run of whitespace made one space, trimmed.
-fn normalize(s: &str) -> String {
-    s.split_whitespace().flat_map(|w| w.chars().flat_map(char::to_lowercase).chain(std::iter::once(' '))).collect::<String>().trim_end().to_string()
+/// `s` in Unicode compatibility form (NFKC), lower case, every run of whitespace made one space,
+/// trimmed: "Mu\u{308}ller" (decomposed), "ＭÜＬＬＥＲ" (full width) and "Müller" are all
+/// "müller", and the ligature "fi" is "fi" (#2539 review). `None` when it would grow past
+/// [`MAX_GROWTH`] times `s`.
+fn normalize(s: &str) -> Option<String> {
+    let cap = s.len().saturating_mul(MAX_GROWTH).saturating_add(64);
+    let mut out = String::new();
+    let mut space = false;
+    for c in s.nfkc().flat_map(char::to_lowercase).nfkc() {
+        if c.is_whitespace() {
+            space = !out.is_empty();
+            continue;
+        }
+        if space {
+            out.push(' ');
+            space = false;
+        }
+        out.push(c);
+        if out.len() > cap {
+            return None;
+        }
+    }
+    Some(out)
 }
 
 fn significant(s: &str, min: usize) -> bool {
@@ -90,13 +115,14 @@ impl Run {
 /// The terms to look for: each removed run (at least two letters or digits), and each of its
 /// words with at least three, so "Smith, John" in `/Author` is caught when "John Smith" was
 /// redacted. Shorter pieces would match nearly any text. `None` when a run is longer than
-/// [`MAX_TERM_CHARS`] or there are more than [`MAX_TERMS`] terms: the caller then can't search
+/// [`MAX_TERM_CHARS`] (or grows past [`MAX_GROWTH`] when normalized) or there are more than
+/// [`MAX_TERMS`] terms: the caller then can't search
 /// for all of it and must clear the metadata instead.
 pub(crate) fn terms(removed: &[String]) -> Option<Vec<String>> {
     let mut seen = HashSet::new();
     let mut out = Vec::new();
     for run in removed {
-        let run = normalize(run);
+        let run = normalize(run)?;
         if run.chars().nth(MAX_TERM_CHARS).is_some() {
             return None;
         }
@@ -148,7 +174,7 @@ impl Work {
 
 /// Does `text` hold a term, as written or, for `squeezed` terms, by its letters and digits alone?
 fn mentions(text: &str, terms: &[String], squeezed: &[String], work: &Work) -> bool {
-    let text = normalize(text);
+    let Some(text) = normalize(text) else { return true };
     let letters = squeeze(&text);
     !work.spend(text.len().saturating_mul(terms.len().saturating_add(squeezed.len())))
         || terms.iter().any(|t| text.contains(t.as_str()))
@@ -161,7 +187,7 @@ fn mentions(text: &str, terms: &[String], squeezed: &[String], work: &Work) -> b
 /// edges, so an XMP value "True" doesn't go with "untrue". Too much text to search counts as
 /// part (fail-closed).
 fn within(value: &str, runs: &[Run], work: &Work) -> bool {
-    let value = normalize(value);
+    let Some(value) = normalize(value) else { return true };
     if !significant(&value, 3) {
         return false;
     }
@@ -368,16 +394,17 @@ impl Search {
     /// tests). `complete` is false when the collection of removed text was cut short; then, as
     /// when the terms exceed their limits, everything matches.
     pub(crate) fn new(removed: &[String], complete: bool, limit: usize) -> Self {
-        let lookup = match terms(removed).filter(|_| complete) {
-            Some(t) if t.is_empty() => Lookup::Nothing,
-            Some(t) => {
+        let runs: Option<Vec<String>> = removed.iter().map(|r| normalize(r)).collect();
+        let lookup = match (terms(removed).filter(|_| complete), runs) {
+            (Some(t), _) if t.is_empty() => Lookup::Nothing,
+            (Some(t), Some(runs)) => {
                 // The same run removed many times is searched for once.
                 let mut seen = HashSet::new();
-                let runs: Vec<Run> = removed.iter().map(|r| normalize(r)).filter(|r| seen.insert(r.clone())).map(Run::new).collect();
+                let runs: Vec<Run> = runs.into_iter().filter(|r| seen.insert(r.clone())).map(Run::new).collect();
                 let squeezed = squeezed_terms(&t);
                 Lookup::Terms(t, squeezed, runs)
             }
-            None => Lookup::Everything,
+            _ => Lookup::Everything,
         };
         Self { lookup, work: Work::new(limit) }
     }

crates/redact/src/tests/metadata_matching.rs

@@ -35,3 +35,36 @@ fn a_value_spaced_or_hyphenated_otherwise_matches() {
     assert!(!has_xmp(&doc), "the XMP title spelled it with a space");
     assert_eq!(r.metadata, 3, "{r:?}");
 }
+
+/// `s` as a UTF-16BE text string in hex, byte order mark first.
+fn utf16be(s: &str) -> String {
+    format!("<FEFF{}>", s.encode_utf16().map(|u| format!("{u:04X}")).collect::<String>())
+}
+
+/// The removed text and the metadata are compared after Unicode compatibility normalization
+/// (NFKC) and lower case on both sides: a decomposed "Müller" (u + U+0308) in a UTF-16BE `/Title`
+/// matches the composed one removed from the page, a "fi" ligature from a `ToUnicode` map matches
+/// "fi", and full-width letters match their ASCII forms.
+#[test]
+fn unicode_forms_of_the_removed_text_match() {
+    let info = format!(
+        "<< /Title {} /Subject (Finance plan) /Keywords {} /Author (Ada Lovelace) /Producer (Test) >>",
+        utf16be("Mu\u{308}ller"),
+        utf16be("\u{FF22}\u{FF2C}\u{FF35}\u{FF25}\u{FF26}\u{FF29}\u{FF2E}")
+    );
+    let mut doc = with_metadata(b"BT /F1 10 Tf 10 100 Td (x) Tj ET", &info, "");
+    let removed = ["M\u{FC}ller".to_string(), "\u{FB01}nance".to_string(), "Bluefin".to_string()];
+    let n = metadata::scrub_within(&mut doc, &removed, true, 1 << 28).unwrap();
+    assert_eq!(info_keys(&doc), ["Author", "Producer"]);
+    assert_eq!(n, 3);
+}
+
+/// A value whose normalized form would be far longer than itself (U+FDFA is 18 letters in NFKC)
+/// isn't normalized in full: it counts as holding the removed text (fail-closed).
+#[test]
+fn a_value_that_normalizes_to_far_more_text_counts_as_holding_it() {
+    let info = format!("<< /Subject {} /Producer (Test) >>", utf16be(&"\u{FDFA}".repeat(50)));
+    let mut doc = with_metadata(b"BT /F1 10 Tf 10 100 Td (x) Tj ET", &info, "");
+    assert_eq!(metadata::scrub_within(&mut doc, &["Bluefin".into()], true, 1 << 28).unwrap(), 1);
+    assert_eq!(info_keys(&doc), ["Producer"]);
+}