Files
pdfcraft/crates/redact/src/metadata.rs
T
mtlcl410andClaude Opus 5.5 8c142bef2e Merge upstream/main into fix/redact-metadata-2448
Keeps main's recovery renewal after applying redactions alongside the
combined notification, and reads XMP and rich-text streams with their
/DecodeParms given by reference made direct, as main's stream readers do.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Pq7HyCHD1R3rywQBPCHXZA
2026-11-09 10:12:29 +08:00

548 lines
24 KiB
Rust

//! Metadata that repeats redacted text (#2448).
//!
//! [`crate::apply`] removes the glyphs under the marks, but the same words are often in the
//! document information dictionary (`/Title`, `/Subject`, `/Keywords`…) and the XMP metadata
//! that mirrors it. Once the page no longer shows them, the metadata must not keep them either.
//!
//! An `/Info` entry whose text contains a redacted term is removed, and so is one whose whole
//! value is part of a removed run (an author "J. Li" redacted from a sentence, whose words are too
//! short to be terms; the reverse check #2598 also proposed). An XMP stream whose text
//! (element content, attribute values other than namespace declarations, comments and
//! processing instructions, entities decoded) contains one, or has a value that is part of a
//! removed run, is removed whole: editing the XML in place could leave it malformed or keep the
//! term in another spelling. A PDF/A identification in that packet goes with it.
//!
//! Fail-closed: XMP that isn't UTF-8 or UTF-16, or whose filter data is damaged or behind more than
//! one filter (it is decoded strictly, so nothing after damage goes unsearched), goes; text too
//! large to search against every term counts as matching, and when the removed text itself was cut short (too many runs or terms,
//! or a run too long to keep) every `/Info` string and every XMP stream goes, since what was
//! never collected can't be searched for. Terms are matched without regard to case or spacing,
//! and by their letters and digits alone, so "Blue fin", "BLUE-FIN" and a tracked heading
//! collected as "B L U E F I N" all match "Bluefin" (#2539 review).
use std::collections::HashSet;
use pdfcraft_cos::{Document, ObjRef, Object};
use unicode_normalization::UnicodeNormalization;
/// Terms longer than this are cut (matching still finds the prefix).
const MAX_TERM_CHARS: usize = 4096;
const MAX_TERMS: usize = 20_000;
/// XMP packets larger than this, decoded, are removed without reading them.
pub(crate) const MAX_XMP_BYTES: usize = 16 * 1024 * 1024;
/// The searching (text length times terms, or value length times removed text) one scrub may
/// do in all; past it every entry and packet counts as matching (fail-closed), so a page of
/// removed text against a large packet, or against very many small values, stays quick.
const MAX_WORK: usize = 1 << 28;
/// Normalized text longer than this many times its input (plus 64 bytes) isn't searched: it
/// counts as matching. Ordinary text barely changes length; a few compatibility characters
/// (U+FDFA, the square katakana words) decompose to many.
const MAX_GROWTH: usize = 4;
/// `s` in Unicode compatibility form (NFKC), lower case, every run of whitespace made one space,
/// trimmed: "Mu\u{308}ller" (decomposed), "MÜLLER" (full width) and "Müller" are all
/// "müller", and the ligature "fi" is "fi" (#2539 review). `None` when it would grow past
/// [`MAX_GROWTH`] times `s`.
fn normalize(s: &str) -> Option<String> {
let cap = s.len().saturating_mul(MAX_GROWTH).saturating_add(64);
let mut out = String::new();
let mut space = false;
for c in s.nfkc().flat_map(char::to_lowercase).nfkc() {
if c.is_whitespace() {
space = !out.is_empty();
continue;
}
if space {
out.push(' ');
space = false;
}
out.push(c);
if out.len() > cap {
return None;
}
}
Some(out)
}
fn significant(s: &str, min: usize) -> bool {
s.chars().filter(|c| c.is_alphanumeric()).count() >= min
}
/// The letters and digits of `s` alone: "blue-fin" and "b l u e f i n" are both "bluefin".
fn squeeze(s: &str) -> String {
s.chars().filter(|c| c.is_alphanumeric()).collect()
}
/// A removed run (normalized), and its letters and digits alone with where its words begin and
/// end among them, so a value can be found in it whatever its spacing but still as whole words.
struct Run {
text: String,
letters: String,
/// `edges[i]`: byte `i` of `letters` begins a word (something other than a letter or digit
/// comes before it in the run, or nothing does), or `i` is the end. One entry per byte of
/// `letters`, and one for the end.
edges: Vec<bool>,
}
impl Run {
fn new(text: String) -> Self {
let mut letters = String::new();
let mut edges = Vec::new();
let mut gap = true;
for c in text.chars() {
if c.is_alphanumeric() {
edges.resize(letters.len(), false);
edges.push(gap);
letters.push(c);
gap = false;
} else {
gap = true;
}
}
edges.resize(letters.len(), false);
edges.push(true);
Self { text, letters, edges }
}
/// Are the letters and digits `value` (squeezed) among the run's, beginning and ending at
/// word edges? "bluefin" is in "b l u e f i n" and "bluef in"; "true" isn't in "untrue".
fn holds_letters(&self, value: &str) -> bool {
let edge = |i: usize| self.edges.get(i).copied().unwrap_or(false);
!value.is_empty() && self.letters.match_indices(value).any(|(at, m)| edge(at) && edge(at.saturating_add(m.len())))
}
}
/// The terms to look for: each removed run (at least two letters or digits), and each of its
/// words with at least three, so "Smith, John" in `/Author` is caught when "John Smith" was
/// redacted. Shorter pieces would match nearly any text. `None` when a run is longer than
/// [`MAX_TERM_CHARS`] (or grows past [`MAX_GROWTH`] when normalized) or there are more than
/// [`MAX_TERMS`] terms: the caller then can't search
/// for all of it and must clear the metadata instead.
pub(crate) fn terms(removed: &[String]) -> Option<Vec<String>> {
let mut seen = HashSet::new();
let mut out = Vec::new();
for run in removed {
let run = normalize(run)?;
if run.chars().nth(MAX_TERM_CHARS).is_some() {
return None;
}
let words = run.split(|c: char| !c.is_alphanumeric()).filter(|w| significant(w, 3)).map(str::to_string).collect::<Vec<_>>();
for t in std::iter::once(run.clone()).filter(|r| significant(r, 2)).chain(words) {
if seen.insert(t.clone()) {
if out.len() >= MAX_TERMS {
return None;
}
out.push(t);
}
}
}
Some(out)
}
/// The terms ([`terms`]) with three letters or digits or more, by their letters and digits
/// alone, each once: "blue fin" holds the term "bluefin" this way, and "bluefin" the run
/// "b l u e f i n".
fn squeezed_terms(terms: &[String]) -> Vec<String> {
let mut seen = HashSet::new();
terms.iter().filter(|t| significant(t, 3)).map(|t| squeeze(t)).filter(|t| seen.insert(t.clone())).collect()
}
/// The searching one scrub may do, across every `/Info` entry and XMP value: past it, each
/// further question is answered "yes" (fail-closed), so a packet of many small values can't add
/// up to more than one large one.
struct Work(std::cell::Cell<usize>);
impl Work {
fn new(limit: usize) -> Self {
Self(std::cell::Cell::new(limit))
}
/// Take `cost` from what is left; `false` (and nothing left) when it doesn't fit.
fn spend(&self, cost: usize) -> bool {
match self.0.get().checked_sub(cost) {
Some(left) => {
self.0.set(left);
true
}
None => {
self.0.set(0);
false
}
}
}
}
/// Does `text` hold a term, as written or, for `squeezed` terms, by its letters and digits alone?
fn mentions(text: &str, terms: &[String], squeezed: &[String], work: &Work) -> bool {
let Some(text) = normalize(text) else { return true };
let letters = squeeze(&text);
!work.spend(text.len().saturating_mul(terms.len().saturating_add(squeezed.len())))
|| terms.iter().any(|t| text.contains(t.as_str()))
|| squeezed.iter().any(|t| letters.contains(t.as_str()))
}
/// Is `value` itself part of one of the removed `runs`, as whole words, compared as written or by
/// letters and digits alone ("Bluefin" is part of "b l u e f i n")? It needs three letters or
/// digits, so an entry such as "by" doesn't go with every sentence that contains it, and word
/// edges, so an XMP value "True" doesn't go with "untrue". Too much text to search counts as
/// part (fail-closed).
fn within(value: &str, runs: &[Run], work: &Work) -> bool {
let Some(value) = normalize(value) else { return true };
if !significant(&value, 3) {
return false;
}
let letters = squeeze(&value);
let total = runs.iter().map(|r| r.text.len().saturating_add(r.letters.len())).fold(0usize, usize::saturating_add);
let word = |c: Option<char>| c.is_some_and(char::is_alphanumeric);
let whole = |r: &Run| {
r.holds_letters(&letters)
|| r.text.match_indices(value.as_str()).any(|(at, m)| {
let before = r.text.get(..at).and_then(|s| s.chars().next_back());
let after = r.text.get(at.saturating_add(m.len())..).and_then(|s| s.chars().next());
!(word(before) && value.starts_with(char::is_alphanumeric)) && !(word(after) && value.ends_with(char::is_alphanumeric))
})
};
!work.spend(value.len().saturating_mul(total)) || runs.iter().any(whole)
}
/// Decode the five predefined XML entities and character references; anything else stays.
fn unescape(s: &str) -> String {
let mut out = String::with_capacity(s.len());
let mut rest = s;
while let Some(at) = rest.find('&') {
out.push_str(rest.get(..at).unwrap_or(""));
let tail = rest.get(at..).unwrap_or("");
// A named entity ends within 12 bytes; looking further made a text of bare `&`s
// quadratic. A character reference may carry any number of leading zeros, so its digits
// are read to their end: each digit belongs to one `&` only, which keeps it linear.
let end = match tail.strip_prefix("&#") {
Some(digits) => {
let hex = digits.starts_with(['x', 'X']);
let from = 2 + usize::from(hex);
let run = tail.get(from..).unwrap_or("").bytes().take_while(|b| if hex { b.is_ascii_hexdigit() } else { b.is_ascii_digit() }).count();
let e = from + run;
(tail.as_bytes().get(e) == Some(&b';')).then_some(e)
}
None => tail.char_indices().take(13).find(|&(_, c)| c == ';').map(|(e, _)| e),
};
let entity = end.and_then(|e| tail.get(1..e));
let decoded = entity.and_then(|e| match e {
"amp" => Some('&'),
"lt" => Some('<'),
"gt" => Some('>'),
"quot" => Some('"'),
"apos" => Some('\''),
_ => {
let n = match e.strip_prefix("#x").or_else(|| e.strip_prefix("#X")) {
// Leading zeros dropped, so they can't overflow the number; `&#0;` isn't a
// character XML allows, and stays as written.
Some(h) => u32::from_str_radix(h.trim_start_matches('0'), 16).ok(),
None => e.strip_prefix('#').and_then(|d| d.trim_start_matches('0').parse().ok()),
};
n.and_then(char::from_u32)
}
});
match (decoded, end) {
(Some(c), Some(e)) => {
out.push(c);
rest = tail.get(e + 1..).unwrap_or("");
}
_ => {
out.push('&');
rest = tail.get(1..).unwrap_or("");
}
}
}
out.push_str(rest);
out
}
/// The text values an XML document carries, entities decoded: element content, attribute values
/// (XMP stores simple properties as attributes), CDATA, and the text of comments and processing
/// instructions other than the `xpacket` wrapper, which a reader can still see. Names and markup are left out, so a term can't
/// match a tag such as `dc:title`, and so are namespace declarations (`xmlns`), whose URIs
/// would match words such as "adobe" or "application" in every packet.
pub(crate) fn xml_values(xml: &str) -> Vec<String> {
let mut out = Vec::new();
let mut push = |s: String| {
if !s.trim().is_empty() {
out.push(s);
}
};
let mut rest = xml;
while !rest.is_empty() {
let Some(open) = rest.find('<') else {
push(unescape(rest));
break;
};
push(unescape(rest.get(..open).unwrap_or("")));
let tag = rest.get(open..).unwrap_or("");
let delimited = [("<![CDATA[", "]]>"), ("<!--", "-->"), ("<?", "?>")].into_iter().find(|(start, _)| tag.starts_with(start));
if let Some((start, stop)) = delimited {
let body = tag.get(start.len()..).unwrap_or("");
let end = body.find(stop).unwrap_or(body.len());
let text = body.get(..end).unwrap_or("");
// The packet wrapper's fixed `begin`/`id`/`end` would match words such as "end".
if !(start == "<?" && text.split_whitespace().next() == Some("xpacket")) {
push(text.to_string());
}
rest = body.get(end.saturating_add(stop.len())..).unwrap_or("");
continue;
}
// Attribute values inside the tag, up to its closing `>` (which can't be inside quotes).
let mut chars = tag.char_indices().skip(1);
let mut end = tag.len();
// The attribute name being read, and the last one read before whitespace: XML allows
// whitespace around `=` (`xmlns:x = "…"`).
let mut name = String::new();
let mut last = String::new();
while let Some((i, c)) = chars.next() {
match c {
'>' => {
end = i + 1;
break;
}
'"' | '\'' => {
let mut value = String::new();
for (_, d) in chars.by_ref() {
if d == c {
break;
}
value.push(d);
}
let attr = if name.is_empty() { &last } else { &name };
let declaration = attr == "xmlns" || attr.starts_with("xmlns:");
if !declaration {
push(unescape(&value));
}
name.clear();
last.clear();
}
'=' => {}
'/' => {
name.clear();
last.clear();
}
c if c.is_whitespace() => {
if !name.is_empty() {
last = std::mem::take(&mut name);
}
}
c => name.push(c),
}
}
rest = tag.get(end..).unwrap_or("");
}
out
}
/// Decode an XMP packet: UTF-8 (with or without a byte-order mark), or UTF-16 told by its mark
/// or, as a packet usually begins `<?xpacket` with the mark inside the attribute, by the zero
/// byte of its first character. `None` for anything else or any invalid sequence, so text that
/// can't be read faithfully is never searched lossily (the caller removes it).
pub(crate) fn decode_xmp(bytes: &[u8]) -> Option<String> {
let utf16 = |body: &[u8], be: bool| -> Option<String> {
let (units, rest) = body.as_chunks::<2>();
if !rest.is_empty() {
return None;
}
char::decode_utf16(units.iter().map(|&p| if be { u16::from_be_bytes(p) } else { u16::from_le_bytes(p) })).collect::<Result<String, _>>().ok()
};
match bytes {
[0xEF, 0xBB, 0xBF, rest @ ..] => std::str::from_utf8(rest).ok().map(str::to_string),
[0xFE, 0xFF, rest @ ..] => utf16(rest, true),
[0xFF, 0xFE, rest @ ..] => utf16(rest, false),
[0, a, ..] if *a != 0 => utf16(bytes, true),
[a, 0, ..] if *a != 0 => utf16(bytes, false),
_ => std::str::from_utf8(bytes).ok().map(str::to_string),
}
}
/// The text of a text string, as searched for the removed text. ISO 32000-2 §7.9.2.2.1 defines
/// PDFDocEncoding, UTF-16BE (`FE FF`) and UTF-8 (`EF BB BF`) only, but a string behind an `FF FE`
/// mark is UTF-16LE to any reader that takes it as such, and read through PDFDocEncoding it
/// could never match: it is read as UTF-16LE (an odd last byte is left out, a lone surrogate
/// becomes U+FFFD), so the search stays fail-closed (#2539 review).
pub(crate) fn text_string(s: &pdfcraft_cos::PdfString) -> String {
match s.bytes.as_slice() {
[0xFF, 0xFE, rest @ ..] => {
let units = rest.as_chunks::<2>().0.iter().map(|&p| u16::from_le_bytes(p));
char::decode_utf16(units).map(|c| c.unwrap_or(char::REPLACEMENT_CHARACTER)).collect()
}
_ => s.to_text(),
}
}
/// The text values of an XMP stream, or `None` when it can't be read.
fn xmp_values(doc: &Document, o: &Object) -> Option<Vec<String>> {
xmp_text(doc, o).map(|t| xml_values(&t))
}
/// The decoded text of an XMP stream, or `None` when it can't be read.
pub(crate) fn xmp_text(doc: &Document, o: &Object) -> Option<String> {
let Object::Stream(s) = &*doc.resolve(o) else { return None };
// Decoded strictly: a tolerant decode returns whatever came before the damage, and the part
// after it would go unsearched. A chain of more than one filter has no strict decoder here,
// so it isn't read either (fail-closed; XMP is rarely filtered at all). Decoding stops at the
// cap, so a small stream that inflates hugely costs no more memory.
if s.filters().len() > 1 {
return None;
}
// `/DecodeParms` given by reference are read too ([`Document::with_direct_parms`]).
let bytes = doc.with_direct_parms(s).decoded_first_filter_strict(MAX_XMP_BYTES).ok()?;
decode_xmp(&bytes)
}
/// What the removed text is looked for with, and the searching left to do it.
pub(crate) struct Search {
lookup: Lookup,
work: Work,
}
enum Lookup {
/// Nothing with text was removed: nothing can hold it.
Nothing,
/// The removed text couldn't be collected or searched for in full: everything counts as
/// holding it (fail-closed).
Everything,
/// The terms ([`terms`]), the same by their letters and digits alone ([`squeezed_terms`]),
/// and the distinct runs, normalized.
Terms(Vec<String>, Vec<String>, Vec<Run>),
}
impl Search {
/// The search for `removed` text, with `limit` steps of searching ([`MAX_WORK`] outside
/// tests). `complete` is false when the collection of removed text was cut short; then, as
/// when the terms exceed their limits, everything matches.
pub(crate) fn new(removed: &[String], complete: bool, limit: usize) -> Self {
let runs: Option<Vec<String>> = removed.iter().map(|r| normalize(r)).collect();
let lookup = match (terms(removed).filter(|_| complete), runs) {
(Some(t), _) if t.is_empty() => Lookup::Nothing,
(Some(t), Some(runs)) => {
// The same run removed many times is searched for once.
let mut seen = HashSet::new();
let runs: Vec<Run> = runs.into_iter().filter(|r| seen.insert(r.clone())).map(Run::new).collect();
let squeezed = squeezed_terms(&t);
Lookup::Terms(t, squeezed, runs)
}
_ => Lookup::Everything,
};
Self { lookup, work: Work::new(limit) }
}
/// [`Search::new`] with the whole work limit.
pub(crate) fn of(removed: &[String], complete: bool) -> Self {
Self::new(removed, complete, MAX_WORK)
}
/// Nothing with text was removed.
pub(crate) fn is_nothing(&self) -> bool {
matches!(self.lookup, Lookup::Nothing)
}
/// The removed text couldn't be searched for: everything matches.
pub(crate) fn is_everything(&self) -> bool {
matches!(self.lookup, Lookup::Everything)
}
/// Does `text` hold the removed text: a term, or wholly part of a removed run? Past the
/// work limit, yes (fail-closed).
pub(crate) fn text(&self, text: &str) -> bool {
match &self.lookup {
Lookup::Nothing => false,
Lookup::Everything => true,
Lookup::Terms(terms, squeezed, runs) => mentions(text, terms, squeezed, &self.work) || within(text, runs, &self.work),
}
}
/// Does the XMP stream `o` hold the removed text? A stream that can't be read does
/// (fail-closed), unless nothing with text was removed.
pub(crate) fn xmp(&self, doc: &Document, o: &Object) -> bool {
match &self.lookup {
Lookup::Nothing => false,
Lookup::Everything => true,
Lookup::Terms(terms, squeezed, runs) => xmp_values(doc, o)
.is_none_or(|values| mentions(&values.join(" "), terms, squeezed, &self.work) || values.iter().any(|v| within(v, runs, &self.work))),
}
}
}
/// What [`scrub`] did.
#[derive(Debug, Default)]
pub(crate) struct Scrubbed {
/// `/Info` entries and XMP streams removed.
pub(crate) removed: usize,
/// What became of the PDF/A and PDF/UA identification of a removed catalog packet.
pub(crate) identification: crate::Identification,
}
/// Remove the metadata that still holds `removed` text: matching `/Info` entries, and the
/// catalog's and pages' XMP streams that mention it. `complete` is false when the collection of
/// removed text was cut short; then, as when the terms themselves exceed their limits, every
/// `/Info` string and every XMP stream goes. A removed catalog packet's PDF/A and PDF/UA
/// identification is written back in a packet of its own ([`crate::identification`]).
pub(crate) fn scrub(doc: &mut Document, removed: &[String], complete: bool) -> Result<Scrubbed, pdfcraft_cos::CosError> {
scrub_within(doc, removed, complete, MAX_WORK)
}
/// [`scrub`] with the searching it may do given, so tests can run out of it.
pub(crate) fn scrub_within(doc: &mut Document, removed: &[String], complete: bool, limit: usize) -> Result<Scrubbed, pdfcraft_cos::CosError> {
let search = Search::new(removed, complete, limit);
let mut out = Scrubbed::default();
if search.is_nothing() {
return Ok(out);
}
let mut n = 0;
// The information dictionary, direct in the trailer or an object of its own.
let info_obj = doc.trailer().get(b"Info").cloned();
if let Some(info) = info_obj.as_ref().and_then(|i| doc.resolve(i).as_dict().cloned()) {
let doomed: Vec<Vec<u8>> =
info.iter().filter(|(_, v)| doc.resolve(v).as_string().is_some_and(|s| search.text(&text_string(s)))).map(|(k, _)| k.to_vec()).collect();
if !doomed.is_empty() {
n += doomed.len();
let mut kept = info.clone();
for k in &doomed {
kept.remove(k);
}
match info_obj.as_ref().and_then(Object::as_ref) {
Some(r) => doc.update_dict(r, |d| *d = kept)?,
None => doc.trailer_mut().set(b"Info".to_vec(), Object::Dict(kept)),
}
}
}
// XMP on the catalog and on every page.
let root = doc.root();
for holder in catalog_and_pages(doc) {
let Some(xmp) = doc.get(holder).as_dict().and_then(|d| d.get(b"Metadata").cloned()) else { continue };
if search.xmp(doc, &xmp) {
// The catalog's packet says whether the file conforms to PDF/A or PDF/UA: that much
// is written back, in a packet of its own.
let replacement = if Some(holder) == root {
let (id, packet) = crate::identification::keep(doc, &xmp, &search);
out.identification = id;
packet.map(|p| doc.add(Object::Stream(p)))
} else {
None
};
doc.update_dict(holder, |d| match replacement {
Some(r) => d.set(b"Metadata".to_vec(), Object::Ref(r)),
None => {
d.remove(b"Metadata");
}
})?;
n += 1;
}
}
out.removed = n;
Ok(out)
}
/// The catalog and the pages: the objects whose XMP [`scrub`] checks.
pub(crate) fn catalog_and_pages(doc: &Document) -> Vec<ObjRef> {
let mut holders: Vec<ObjRef> = doc.root().into_iter().collect();
holders.extend(pdfcraft_model::pages(doc).iter().map(|p| p.obj));
holders
}