diff --git a/Cargo.lock b/Cargo.lock index fd186d995c..3cc808ab1c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4586,6 +4586,7 @@ dependencies = [ "pdfcraft-testkit", "pdfcraft-xfa", "thiserror 2.0.21", + "unicode-normalization", ] [[package]] diff --git a/crates/redact/Cargo.toml b/crates/redact/Cargo.toml index 8d1515d8a4..10cb97ccde 100644 --- a/crates/redact/Cargo.toml +++ b/crates/redact/Cargo.toml @@ -18,6 +18,8 @@ pdfcraft-edit = { workspace = true } pdfcraft-annot = { workspace = true } pdfcraft-xfa = { workspace = true } thiserror = { workspace = true } +# NFKC for the metadata check; already in the build through pdfcraft-crypt (stringprep). +unicode-normalization = "0.1" [dev-dependencies] pdfcraft-filters = { workspace = true } diff --git a/crates/redact/README.md b/crates/redact/README.md index fef032fa1b..129602287c 100644 --- a/crates/redact/README.md +++ b/crates/redact/README.md @@ -48,8 +48,10 @@ clear_marks(&mut doc, None)?; // remove marks without app "B L U E F I N", since a `TJ` gap can't be told from a word space). A value is wholly part of a run when its letters and digits are, starting and ending where the run's words do, so "True" isn't part of "untrue". This over-matches only towards removal: "is a" also holds - "Lisa". A value that shares - only words of one or two letters with the removed text, and isn't wholly part of it, stays: + "Lisa". Both sides are compared in Unicode compatibility form (NFKC) and lower case, so a + decomposed "Müller" (u and U+0308), full-width letters and a "fi" ligature from a + `ToUnicode` map match their plain forms. A value that shares only words of one or two + letters with the removed text, and isn't wholly part of it, stays: Sanitize's metadata category removes the rest. `Report::metadata` counts what went. - **Fail-closed.** When the check can't know what the removed text was, every `/Info` string and every catalog and page XMP stream goes: more than 100,000 runs or a run over @@ -60,7 +62,8 @@ clear_marks(&mut doc, None)?; // remove marks without app search's work limit (2^28 steps) runs out, every entry and packet not yet checked goes too. An XMP packet that isn't valid UTF-8 or UTF-16, can't be decoded strictly (damaged or cut-short filter data, or more than one filter), or decodes to more than - 16 MiB goes without being read. + 16 MiB goes without being read, and a value whose normalized form would be more than four + times as long (U+FDFA alone is 18 letters) counts as holding the text. - **Not scrubbed, but reported** (Remove hidden information's categories remove them): bookmark titles, attachments (names in the document's list of embedded files, file names and descriptions, and the files of file attachment comments), the comments that remain (their diff --git a/crates/redact/src/metadata.rs b/crates/redact/src/metadata.rs index 2eafdb63c9..9c6d435d27 100644 --- a/crates/redact/src/metadata.rs +++ b/crates/redact/src/metadata.rs @@ -23,6 +23,7 @@ use std::collections::HashSet; use pdfcraft_cos::{Document, ObjRef, Object}; +use unicode_normalization::UnicodeNormalization; /// Terms longer than this are cut (matching still finds the prefix). const MAX_TERM_CHARS: usize = 4096; @@ -33,10 +34,34 @@ pub(crate) const MAX_XMP_BYTES: usize = 16 * 1024 * 1024; /// do in all; past it every entry and packet counts as matching (fail-closed), so a page of /// removed text against a large packet, or against very many small values, stays quick. const MAX_WORK: usize = 1 << 28; +/// Normalized text longer than this many times its input (plus 64 bytes) isn't searched: it +/// counts as matching. Ordinary text barely changes length; a few compatibility characters +/// (U+FDFA, the square katakana words) decompose to many. +const MAX_GROWTH: usize = 4; -/// Lower-case `s` with every run of whitespace made one space, trimmed. -fn normalize(s: &str) -> String { - s.split_whitespace().flat_map(|w| w.chars().flat_map(char::to_lowercase).chain(std::iter::once(' '))).collect::().trim_end().to_string() +/// `s` in Unicode compatibility form (NFKC), lower case, every run of whitespace made one space, +/// trimmed: "Mu\u{308}ller" (decomposed), "ＭÜＬＬＥＲ" (full width) and "Müller" are all +/// "müller", and the ligature "fi" is "fi" (#2539 review). `None` when it would grow past +/// [`MAX_GROWTH`] times `s`. +fn normalize(s: &str) -> Option { + let cap = s.len().saturating_mul(MAX_GROWTH).saturating_add(64); + let mut out = String::new(); + let mut space = false; + for c in s.nfkc().flat_map(char::to_lowercase).nfkc() { + if c.is_whitespace() { + space = !out.is_empty(); + continue; + } + if space { + out.push(' '); + space = false; + } + out.push(c); + if out.len() > cap { + return None; + } + } + Some(out) } fn significant(s: &str, min: usize) -> bool { @@ -90,13 +115,14 @@ impl Run { /// The terms to look for: each removed run (at least two letters or digits), and each of its /// words with at least three, so "Smith, John" in `/Author` is caught when "John Smith" was /// redacted. Shorter pieces would match nearly any text. `None` when a run is longer than -/// [`MAX_TERM_CHARS`] or there are more than [`MAX_TERMS`] terms: the caller then can't search +/// [`MAX_TERM_CHARS`] (or grows past [`MAX_GROWTH`] when normalized) or there are more than +/// [`MAX_TERMS`] terms: the caller then can't search /// for all of it and must clear the metadata instead. pub(crate) fn terms(removed: &[String]) -> Option> { let mut seen = HashSet::new(); let mut out = Vec::new(); for run in removed { - let run = normalize(run); + let run = normalize(run)?; if run.chars().nth(MAX_TERM_CHARS).is_some() { return None; } @@ -148,7 +174,7 @@ impl Work { /// Does `text` hold a term, as written or, for `squeezed` terms, by its letters and digits alone? fn mentions(text: &str, terms: &[String], squeezed: &[String], work: &Work) -> bool { - let text = normalize(text); + let Some(text) = normalize(text) else { return true }; let letters = squeeze(&text); !work.spend(text.len().saturating_mul(terms.len().saturating_add(squeezed.len()))) || terms.iter().any(|t| text.contains(t.as_str())) @@ -161,7 +187,7 @@ fn mentions(text: &str, terms: &[String], squeezed: &[String], work: &Work) -> b /// edges, so an XMP value "True" doesn't go with "untrue". Too much text to search counts as /// part (fail-closed). fn within(value: &str, runs: &[Run], work: &Work) -> bool { - let value = normalize(value); + let Some(value) = normalize(value) else { return true }; if !significant(&value, 3) { return false; } @@ -368,16 +394,17 @@ impl Search { /// tests). `complete` is false when the collection of removed text was cut short; then, as /// when the terms exceed their limits, everything matches. pub(crate) fn new(removed: &[String], complete: bool, limit: usize) -> Self { - let lookup = match terms(removed).filter(|_| complete) { - Some(t) if t.is_empty() => Lookup::Nothing, - Some(t) => { + let runs: Option> = removed.iter().map(|r| normalize(r)).collect(); + let lookup = match (terms(removed).filter(|_| complete), runs) { + (Some(t), _) if t.is_empty() => Lookup::Nothing, + (Some(t), Some(runs)) => { // The same run removed many times is searched for once. let mut seen = HashSet::new(); - let runs: Vec = removed.iter().map(|r| normalize(r)).filter(|r| seen.insert(r.clone())).map(Run::new).collect(); + let runs: Vec = runs.into_iter().filter(|r| seen.insert(r.clone())).map(Run::new).collect(); let squeezed = squeezed_terms(&t); Lookup::Terms(t, squeezed, runs) } - None => Lookup::Everything, + _ => Lookup::Everything, }; Self { lookup, work: Work::new(limit) } } diff --git a/crates/redact/src/tests/metadata_matching.rs b/crates/redact/src/tests/metadata_matching.rs index b29ed51893..d944dac766 100644 --- a/crates/redact/src/tests/metadata_matching.rs +++ b/crates/redact/src/tests/metadata_matching.rs @@ -35,3 +35,36 @@ fn a_value_spaced_or_hyphenated_otherwise_matches() { assert!(!has_xmp(&doc), "the XMP title spelled it with a space"); assert_eq!(r.metadata, 3, "{r:?}"); } + +/// `s` as a UTF-16BE text string in hex, byte order mark first. +fn utf16be(s: &str) -> String { + format!("", s.encode_utf16().map(|u| format!("{u:04X}")).collect::()) +} + +/// The removed text and the metadata are compared after Unicode compatibility normalization +/// (NFKC) and lower case on both sides: a decomposed "Müller" (u + U+0308) in a UTF-16BE `/Title` +/// matches the composed one removed from the page, a "fi" ligature from a `ToUnicode` map matches +/// "fi", and full-width letters match their ASCII forms. +#[test] +fn unicode_forms_of_the_removed_text_match() { + let info = format!( + "<< /Title {} /Subject (Finance plan) /Keywords {} /Author (Ada Lovelace) /Producer (Test) >>", + utf16be("Mu\u{308}ller"), + utf16be("\u{FF22}\u{FF2C}\u{FF35}\u{FF25}\u{FF26}\u{FF29}\u{FF2E}") + ); + let mut doc = with_metadata(b"BT /F1 10 Tf 10 100 Td (x) Tj ET", &info, ""); + let removed = ["M\u{FC}ller".to_string(), "\u{FB01}nance".to_string(), "Bluefin".to_string()]; + let n = metadata::scrub_within(&mut doc, &removed, true, 1 << 28).unwrap(); + assert_eq!(info_keys(&doc), ["Author", "Producer"]); + assert_eq!(n, 3); +} + +/// A value whose normalized form would be far longer than itself (U+FDFA is 18 letters in NFKC) +/// isn't normalized in full: it counts as holding the removed text (fail-closed). +#[test] +fn a_value_that_normalizes_to_far_more_text_counts_as_holding_it() { + let info = format!("<< /Subject {} /Producer (Test) >>", utf16be(&"\u{FDFA}".repeat(50))); + let mut doc = with_metadata(b"BT /F1 10 Tf 10 100 Td (x) Tj ET", &info, ""); + assert_eq!(metadata::scrub_within(&mut doc, &["Bluefin".into()], true, 1 << 28).unwrap(), 1); + assert_eq!(info_keys(&doc), ["Producer"]); +}