From d7ce875dd692109789bcdad1b683ffc5a2d07dc7 Mon Sep 17 00:00:00 2001 From: eliets2 Date: Fri, 6 Nov 2026 10:15:07 +0200 Subject: [PATCH] redact: independent redaction proof machinery (snapshot, sweep, budget) (#5557) * redact: add the independent redaction proof machinery * redact: harden the proof's paint check and count removed codes The manifest counts character codes, not raw bytes: a composite font's codes are several bytes, so the byte length overstated what was cut. Annotation appearances are checked whatever their storage (a direct dictionary has no reference of its own to dedupe on), and a stream shared by several annotations is checked at each annotation's own placement. The graphics-state stack and the clip holes of one walk are bounded; deeper or more is unverifiable, never unbounded memory. Even-odd clips cut holes only where the rule can be told cheaply - all subpaths axis-aligned, disjoint, few - and anything else keeps its paint reported, failing closed. The /W widths of a descendant font cost bounded work, like a CMap. * redact: settle the survivors a page sweep counted before its work ran out A page that the search budget stops on left the counts made on earlier pages unsettled: the survivors were seen but never reported, and the regions that owned them were never marked. The sweep now stops and still settles, so a located survivor is reported where it sits (the page reads unswept either way, and the proof still fails). * redact: prove the redacted output of every supported encryption The authenticated-reopen regression now covers all four crypt algorithms the writer supports, not only AES-128: the caller's credentials prove the saved output, and empty or wrong credentials still fail closed. * redact: answer the review round on the proof A glyph lying under several regions is now accounted for by every one of them, so overlapping marks each carry their own share in a shared proof instead of the first one absorbing the rest and the others reading as if no text was there. Each finding is recorded once per region as it is found, so a page that paints over a region a thousand times neither grows the report nor the memory with repeats. The byte search walks the haystack once (the pattern's prefix function is built once), so a needle of one letter over a haystack full of it can no longer spend comparisons the search budget never charged for. A numeric XML character reference may carry any number of leading zeros; the padded form now decodes like the short one, so a survivor spelled that way in the XMP is still found. The /W widths of a descendant font are paid for from the shared work budget: a form invoked again rebuilds its fonts, and a per-font counter alone let the same array be paid for many times over. * redact: sweep the proof's survivors out of every place a reader could look Plant the removed text where the review listed it and expect the sweep to report it: a compressed object stream, a page thumbnail, a form field value, an XFA packet, and an object nothing refers to. A file whose page content is bigger than the proof may decode for one stream is now shown to come out unverified on its surfaces and its region, never clean. * redact: spare the names a redacted word legitimately sits in, and charge failed decodes A redacted word that is also a structural name (a /Subtype, a /Filter, a /BaseFont) is structure, not a leftover copy: the name surface is now swept against the matches the input itself had, and only a surplus of matches is a survivor. The input's names are counted when the snapshot is taken, for the region's needles and for strings added later alike. A decode that fails after doing part of its work paid nothing: the proof's cumulative decode allowance now charges every attempt, failures included, so the same bomb cannot be decoded (and fail) again and again for free. * redact: count the input's names once each, and say so when the count runs out The name allowance counted one entry per occurrence, so a document whose pages repeat the same structural name paid for the same scan again and again; names are now kept once, with their count. A name search that runs out of work read as no matches, which would have swept names with none allowed: that is reported on the text surface now, and the proof fails rather than pass on an uncounted allowance. * Bound redaction proof searches across many needles Assisted-by: OpenAI Codex * Redaction proof: drop find_all, which nothing calls Co-Authored-By: Claude Opus 5.5 * Cargo.lock: put aho-corasick in its sorted place Co-Authored-By: Claude Opus 5.5 * Redaction proof: review notes A word split across a TJ array is looked for whole. Overlapping marks count a removed copy once. A spent decode budget is a problem, not a raw-only stream. Short needles stay out of the attachment text search, and an indirect /Decode counts. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: eliets2 <287273517+eliets2@users.noreply.github.com> Co-authored-by: FredApps Co-authored-by: FredApps <153173854+FredApps@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 --- Cargo.lock | 11 + crates/redact/Cargo.toml | 2 + crates/redact/src/image.rs | 2 +- crates/redact/src/interp.rs | 100 +- crates/redact/src/lib.rs | 11 +- crates/redact/src/tests_verify.rs | 1380 ++++++++++++++++++++++ crates/redact/src/verify/extract.rs | 711 +++++++++++ crates/redact/src/verify/geometry.rs | 500 ++++++++ crates/redact/src/verify/limits.rs | 42 + crates/redact/src/verify/mod.rs | 389 ++++++ crates/redact/src/verify/search_tests.rs | 34 + crates/redact/src/verify/snapshot.rs | 532 +++++++++ crates/redact/src/verify/sweep.rs | 759 ++++++++++++ crates/redact/src/verify/util.rs | 301 +++++ 14 files changed, 4759 insertions(+), 15 deletions(-) create mode 100644 crates/redact/src/tests_verify.rs create mode 100644 crates/redact/src/verify/extract.rs create mode 100644 crates/redact/src/verify/geometry.rs create mode 100644 crates/redact/src/verify/limits.rs create mode 100644 crates/redact/src/verify/mod.rs create mode 100644 crates/redact/src/verify/search_tests.rs create mode 100644 crates/redact/src/verify/snapshot.rs create mode 100644 crates/redact/src/verify/sweep.rs create mode 100644 crates/redact/src/verify/util.rs diff --git a/Cargo.lock b/Cargo.lock index 4816a44ec0..e1f271ed12 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -143,6 +143,15 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + [[package]] name = "aligned-vec" version = "0.6.4" @@ -4610,6 +4619,7 @@ dependencies = [ name = "pdfcraft-redact" version = "0.18.1" dependencies = [ + "aho-corasick", "pdfcraft-annot", "pdfcraft-content", "pdfcraft-cos", @@ -4620,6 +4630,7 @@ dependencies = [ "pdfcraft-model", "pdfcraft-testkit", "pdfcraft-xfa", + "sha2 0.11.1", "thiserror 2.0.21", ] diff --git a/crates/redact/Cargo.toml b/crates/redact/Cargo.toml index fe21d13b32..c02335e3c4 100644 --- a/crates/redact/Cargo.toml +++ b/crates/redact/Cargo.toml @@ -9,6 +9,7 @@ license.workspace = true workspace = true [dependencies] +aho-corasick = "1" pdfcraft-cos = { workspace = true } pdfcraft-content = { workspace = true } pdfcraft-fonts = { workspace = true } @@ -20,6 +21,7 @@ pdfcraft-xfa = { workspace = true } # CCITT Group 4 and JBIG2 encoding for cleared one-bit images. pdfcraft-filters = { workspace = true } thiserror = { workspace = true } +sha2 = "0.11" [dev-dependencies] pdfcraft-testkit = { workspace = true } diff --git a/crates/redact/src/image.rs b/crates/redact/src/image.rs index 4b3c44b0fd..a99e179895 100644 --- a/crates/redact/src/image.rs +++ b/crates/redact/src/image.rs @@ -12,7 +12,7 @@ use pdfcraft_cos::{Dict, Document, ImageBudget, Object, Stream}; /// Most pixels an image may have to be cleared in place (larger ones are removed), or to be stored /// again without shared `/JBIG2Globals` (see [`without_globals`]). -const MAX_PIXELS: u64 = pdfcraft_cos::MAX_REDACTED_IMAGE_PIXELS; +pub(crate) const MAX_PIXELS: u64 = pdfcraft_cos::MAX_REDACTED_IMAGE_PIXELS; fn components(doc: &Document, s: &Stream) -> Option { let cs = doc.resolve(s.dict.get(b"ColorSpace")?); diff --git a/crates/redact/src/interp.rs b/crates/redact/src/interp.rs index 460085a320..7a2cd26eca 100644 --- a/crates/redact/src/interp.rs +++ b/crates/redact/src/interp.rs @@ -53,6 +53,23 @@ pub(crate) enum Mode { Verify, } +/// A run of glyphs cut out under a region, kept for the redaction proof +/// ([`Snapshot::capture`](crate::verify::Snapshot::capture)). +pub(crate) struct Removed { + /// Index of the region (in `Scope::rects`) the run lay under. + pub(crate) rect: usize, + /// The character codes as they stood in the string. + pub(crate) raw: Vec, + /// What the font says they showed (codes without a known meaning are left out). + pub(crate) text: String, + /// How many character codes the run holds (a composite font's codes are several bytes). + pub(crate) codes: usize, + /// Which text operator showed it, counted from the start of the page. + pub(crate) op: usize, + /// Where in that operator it starts: the operand's place in a `TJ` array, and the byte in it. + pub(crate) at: (usize, usize), +} + #[derive(Clone)] struct Gs { ctm: Matrix, @@ -124,6 +141,12 @@ pub(crate) struct Scope<'a> { xobject_oc_cache: HashMap, /// Content removed from hidden layers (blocks and XObjects). pub layer_blocks: usize, + /// Collects the glyph runs removed under a region in apply mode (`None` = not collected). + /// The redaction proof's snapshot is what sets it: it must know what applying is about to + /// remove, by the interpreter's own judgement, without trusting anything else to it. + pub removed: Option>, + /// How many text operators have been shown, for [`Removed::op`]. + shown: usize, fonts: HashMap, Rc>, used_names: Vec>, depth: usize, @@ -149,6 +172,8 @@ impl<'a> Scope<'a> { ocmd_cache: HashMap::new(), xobject_oc_cache: HashMap::new(), layer_blocks: 0, + removed: None, + shown: 0, fonts: HashMap::new(), used_names: Vec::new(), depth: 0, @@ -169,21 +194,33 @@ impl<'a> Scope<'a> { self.rects.iter().any(|r| contains(*r, b, 0.01)) } - /// Does a glyph box count as under a region? Any real overlap does (more than a fifth of + /// Which regions a glyph box counts as under? Any real overlap does (more than a fifth of /// the glyph or a point, whichever is less, in both directions), so neighbours that only /// touch an edge survive while partly covered glyphs go. Degenerate boxes (zero-size text) - /// count when their origin is inside. - fn glyph_hit(&self, b: [f64; 4], origin: (f64, f64)) -> bool { + /// count when their origin is inside. Empty = under no region (a glyph kept). A glyph may + /// lie under several regions at once (overlapping marks); each is reported. + fn glyph_rects(&self, b: [f64; 4], origin: (f64, f64)) -> Vec { let (gw, gh) = (b[2] - b[0], b[3] - b[1]); if gw < 0.01 || gh < 0.01 { - return self.rects.iter().any(|r| origin.0 >= r[0] && origin.0 <= r[2] && origin.1 >= r[1] && origin.1 <= r[3]); + return self + .rects + .iter() + .enumerate() + .filter(|(_, r)| origin.0 >= r[0] && origin.0 <= r[2] && origin.1 >= r[1] && origin.1 <= r[3]) + .map(|(i, _)| i) + .collect(); } let (mx, my) = ((gw * 0.2).min(1.0), (gh * 0.2).min(1.0)); - self.rects.iter().any(|r| { - let ox = b[2].min(r[2]) - b[0].max(r[0]); - let oy = b[3].min(r[3]) - b[1].max(r[1]); - ox > mx && oy > my - }) + self.rects + .iter() + .enumerate() + .filter(|(_, r)| { + let ox = b[2].min(r[2]) - b[0].max(r[0]); + let oy = b[3].min(r[3]) - b[1].max(r[1]); + ox > mx && oy > my + }) + .map(|(i, _)| i) + .collect() } } @@ -223,6 +260,8 @@ struct Glyph { /// Advance in unscaled text space (before `Th`). advance: f64, hit: bool, + /// The regions it lies under (empty for hidden text). + rects: Vec, } /// Process a scope's content streams (state flows from one stream to the next). @@ -644,11 +683,14 @@ fn clean_replacements(doc: &Document, resources: &Dict, original: &[Op], process /// Lay out a shown string (or `TJ` array) glyph by glyph, advancing `tm`. Returns the `TJ` /// items with the glyphs under a region replaced by displacements, and how many were removed. -fn show(scope: &Scope<'_>, gs: &Gs, tm: &mut Matrix, items: &[Object]) -> (Vec, usize) { +/// In apply mode the runs removed under each region are also handed to `scope.removed`. +fn show(scope: &mut Scope<'_>, gs: &Gs, tm: &mut Matrix, items: &[Object]) -> (Vec, usize) { let f = &gs.font; let size = gs.size; let mut out: Vec = Vec::new(); let mut removed = 0; + scope.shown += 1; + let op = scope.shown; let push_num = |out: &mut Vec, v: f64| { if v == 0.0 { return; @@ -661,7 +703,7 @@ fn show(scope: &Scope<'_>, gs: &Gs, tm: &mut Matrix, items: &[Object]) -> (Vec { let bytes = &s.bytes; @@ -674,18 +716,50 @@ fn show(scope: &Scope<'_>, gs: &Gs, tm: &mut Matrix, items: &[Object]) -> (Vec { let degenerate = b[2] - b[0] < 0.01 && b[3] - b[1] < 0.01; matches!(gs.render_mode, 3 | 7) || !(overlaps(page, b, 0.0) || degenerate) } - None => scope.glyph_hit(b, origin), + None => { + rects = scope.glyph_rects(b, origin); + !rects.is_empty() + } }; - glyphs.push(Glyph { bytes: pos..pos + len, advance, hit }); + glyphs.push(Glyph { bytes: pos..pos + len, advance, hit, rects }); *tm = Matrix::translate(advance * gs.scale, 0.0).then(tm); pos += len; } + if scope.mode == Mode::Apply + && let Some(sink) = scope.removed.as_mut() + { + // Consecutive removed glyphs under one region are one run, recorded for + // every region the glyphs lie under: overlapping marks each account for + // their own share of what was cut. + let mut under: Vec = glyphs.iter().flat_map(|g| g.rects.iter().copied()).collect(); + under.sort_unstable(); + under.dedup(); + for r in under { + let mut k = 0; + while let Some(g) = glyphs.get(k) { + if !g.rects.contains(&r) { + k += 1; + continue; + } + let mut end = k; + while glyphs.get(end + 1).is_some_and(|n| n.rects.contains(&r)) { + end += 1; + } + let span = g.bytes.start..glyphs.get(end).map_or(g.bytes.end, |l| l.bytes.end); + let at = (place, span.start); + let raw = bytes.get(span).unwrap_or_default().to_vec(); + sink.push(Removed { rect: r, text: f.decode(&raw), raw, codes: end - k + 1, op, at }); + k = end + 1; + } + } + } let mut run: Vec = Vec::new(); for g in &glyphs { if g.hit { diff --git a/crates/redact/src/lib.rs b/crates/redact/src/lib.rs index b6a8885c03..6bb8e5cc06 100644 --- a/crates/redact/src/lib.rs +++ b/crates/redact/src/lib.rs @@ -27,10 +27,14 @@ pub mod sanitize; mod tags; #[cfg(test)] mod tests; +#[cfg(test)] +mod tests_verify; pub(crate) mod type3; pub(crate) mod type3_tables; +pub mod verify; use interp::{Mode, Scope, process}; +pub use verify::{Proof, ProofOptions, Snapshot}; pub type Rgb = [f64; 3]; @@ -59,6 +63,11 @@ pub enum RedactError { "redaction appearance has invalid geometry or exceeds the 8 MiB budget; reduce the marked area, turn off repeated text or apply fewer pages at once (nothing was redacted)" )] OverlayLimit, + /// A stream the redaction created to draw the marks' boxes and labels can't be read back, so + /// the proof can't tell it apart from content that was there before and nothing can be + /// exempted from the sweep. + #[error("a stream the redaction created can't be read back, so the result can't be proven")] + OverlayUnreadable, /// The structure tree couldn't be cleaned completely (#3302): an element written in place that /// would need changing, an element whose inherited page can't be found, or a tree too large to /// walk. Tags that kept the removed text or annotation would undo the redaction. @@ -164,7 +173,7 @@ pub struct Report { pub overlays: Vec, } -fn rect_of(doc: &Document, o: Option<&Object>) -> Option<[f64; 4]> { +pub(crate) fn rect_of(doc: &Document, o: Option<&Object>) -> Option<[f64; 4]> { let v: Vec = doc.resolve(o?).as_array()?.iter().filter_map(|x| doc.resolve(x).as_f64()).collect(); (v.len() == 4 && v.iter().all(|x| x.is_finite())).then(|| [v[0].min(v[2]), v[1].min(v[3]), v[0].max(v[2]), v[1].max(v[3])]) } diff --git a/crates/redact/src/tests_verify.rs b/crates/redact/src/tests_verify.rs new file mode 100644 index 0000000000..ee443f3209 --- /dev/null +++ b/crates/redact/src/tests_verify.rs @@ -0,0 +1,1380 @@ +//! The independent redaction proof: what it catches, what it refuses to call clean. + +use std::sync::Arc; + +use pdfcraft_annot::{Meta, NewAnnotation, Shape, Style, add_annotation, rect_quad}; +use pdfcraft_cos::{Algorithm, Dict, Document, NewEncryption, Object, SaveOptions, Stream, write_full, write_incremental}; +use sha2::Digest; + +use super::*; +use crate::verify::tests_support::{self, count_text_operators, survivor_encodings}; +use crate::verify::{Status, Surface, SurfaceReport, Verdict}; + +const SECRET: &str = "SECRETVALUE"; + +pub(super) fn stream(dict: &str, data: &[u8]) -> Vec { + let mut v = format!("<< {dict} /Length {} >>\nstream\n", data.len()).into_bytes(); + v.extend_from_slice(data); + v.extend_from_slice(b"\nendstream"); + v +} + +pub(super) fn pdf_bytes(objs: Vec>, trailer: &str) -> Vec { + let mut out = b"%PDF-1.7\n".to_vec(); + let mut offs = Vec::new(); + for (i, o) in objs.iter().enumerate() { + offs.push(out.len()); + out.extend_from_slice(format!("{} 0 obj\n", i + 1).as_bytes()); + out.extend_from_slice(o); + out.extend_from_slice(b"\nendobj\n"); + } + let x = out.len(); + out.extend_from_slice(format!("xref\n0 {}\n0000000000 65535 f \n", objs.len() + 1).as_bytes()); + for o in offs { + out.extend_from_slice(format!("{o:010} 00000 n \n").as_bytes()); + } + out.extend_from_slice(format!("trailer\n<< /Size {} /Root 1 0 R {trailer} >>\nstartxref\n{x}\n%%EOF\n", objs.len() + 1).as_bytes()); + out +} + +pub(super) fn open(bytes: Vec) -> Document { + Document::open(Arc::new(bytes)).unwrap() +} + +/// Every glyph is 500 units wide (5 pt at 10 pt). +pub(super) const FONT: &str = + "<< /Type /Font /Subtype /TrueType /BaseFont /Arial /FirstChar 32 /LastChar 126 /Widths 6 0 R /FontDescriptor << /Ascent 800 /Descent -200 >> >>"; + +/// A composite font without a /ToUnicode map: its text can't be decoded. +const OPAQUE_FONT: &str = "<< /Type /Font /Subtype /Type0 /BaseFont /X /Encoding /Identity-H /DescendantFonts [<< /Type /Font /Subtype /CIDFontType2 /BaseFont /X /CIDSystemInfo << /Registry (Adobe) /Ordering (Identity) /Supplement 0 >> /DW 1000 >>] >>"; + +/// Options of a one-page file: catalog, page and trailer entries, page resources, and more +/// objects (numbered from 7). +#[derive(Default)] +struct Spec<'a> { + catalog: &'a str, + page: &'a str, + trailer: &'a str, + resources: &'a str, + extra: Vec>, +} + +/// "Public" at the top, the secret lower down at x 10..65, y 100..110. +const CONTENT: &[u8] = b"BT /F1 10 Tf 10 200 Td (Public) Tj ET BT /F1 10 Tf 10 100 Td (SECRETVALUE) Tj ET"; + +pub(super) const REGION: [f64; 4] = [8.0, 95.0, 70.0, 112.0]; + +fn build_with(content: &[u8], spec: Spec<'_>) -> Vec { + let mut objs: Vec> = vec![ + format!("<< /Type /Catalog /Pages 2 0 R {} >>", spec.catalog).into_bytes(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + format!( + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R {} /Resources << /Font << /F1 5 0 R >> {} >> >>", + spec.page, spec.resources + ) + .into_bytes(), + stream("", content), + FONT.as_bytes().to_vec(), + format!("[{}]", vec!["500"; 95].join(" ")).into_bytes(), + ]; + objs.extend(spec.extra); + pdf_bytes(objs, spec.trailer) +} + +pub(super) fn mark(doc: &mut Document, rects: &[[f64; 4]]) { + let shape = Shape::Redact { quads: rects.iter().map(|r| rect_quad(*r)).collect(), overlay: String::new(), look: Default::default() }; + let style = Style::default_for(&shape); + add_annotation(doc, &NewAnnotation { page: 0, shape, style, contents: String::new(), author: "Tester".into() }, &Meta::default()).unwrap(); +} + +/// The standard page with `spec`, the secret's region marked. +fn marked(spec: Spec<'_>) -> Document { + marked_content(CONTENT, spec) +} + +fn marked_content(content: &[u8], spec: Spec<'_>) -> Document { + let mut doc = open(build_with(content, spec)); + mark(&mut doc, &[REGION]); + doc +} + +/// Apply, and tell the snapshot which streams the apply created (its overlay boxes and labels). +fn applied(doc: &mut Document, snap: &mut Snapshot) -> Result { + let result = apply(doc, None); + if let Ok(report) = &result { + snap.set_overlay_streams(doc, &report.overlays)?; + } + result +} + +/// Capture, apply, prove. +pub(super) fn run(doc: &mut Document) -> (Result, Proof) { + let mut snap = Snapshot::capture(doc, None).unwrap(); + let result = applied(doc, &mut snap); + let proof = snap.prove(doc, None, &ProofOptions::default()); + (result, proof) +} + +/// Prove a document the redaction was never applied to (as a full save would write it). +pub(super) fn prove_unapplied(doc: &mut Document) -> Proof { + let snap = Snapshot::capture(doc, None).unwrap(); + doc.require_full_save(); + snap.prove(doc, None, &ProofOptions::default()) +} + +pub(super) fn surface(p: &Proof, s: Surface) -> &SurfaceReport { + p.surfaces.iter().find(|r| r.surface == s).unwrap() +} + +fn residue(r: &Result) -> bool { + matches!(r, Err(RedactError::Residue(n)) if *n > 0) +} + +#[test] +fn the_proof_reads_a_pages_streams_as_the_one_stream_they_are() { + // The TJ array of the secret ends the first piece and its operator starts the second + // (#153): the proof reads the pieces as the one stream they are, so the removed text is + // neither reported as a survivor nor does the page count as unreadable. + let objs: Vec> = vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents [4 0 R 7 0 R] /Resources << /Font << /F1 5 0 R >> >> >>".to_vec(), + stream("", b"BT /F1 10 Tf 10 100 Td [(SECRETVALUE)]"), + FONT.as_bytes().to_vec(), + format!("[{}]", vec!["500"; 95].join(" ")).into_bytes(), + stream("", b"TJ ET BT /F1 10 Tf 10 200 Td (Public) Tj ET"), + ]; + let mut doc = open(pdf_bytes(objs, "/Root 1 0 R")); + mark(&mut doc, &[REGION]); + let (r, proof) = run(&mut doc); + let r = r.unwrap(); + assert_eq!(r.glyphs, 11, "every glyph of the split string is removed: {r:?}"); + assert!(proof.passed(), "{}", proof.to_text()); + assert_eq!(proof.survivors(), 0); + let saved = write_incremental(&doc, &SaveOptions::default()).unwrap(); + assert!(!saved.windows(SECRET.len()).any(|w| w == SECRET.as_bytes()), "the secret is gone from the file"); +} + +#[test] +fn a_clean_redaction_passes_and_the_neighbour_is_untouched() { + let mut doc = marked(Spec::default()); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert!(proof.passed(), "{}", proof.to_text()); + assert_eq!(proof.entries.len(), 1); + let e = &proof.entries[0]; + assert_eq!((e.status, e.page, e.region), (Status::Verified, 0, REGION)); + assert_eq!((e.runs_removed, e.codes_removed), (1, SECRET.len())); + let content = page_content(&doc); + assert!(content.contains("(Public) Tj") && !content.contains(SECRET), "{content}"); + for s in &proof.surfaces { + assert!(matches!(s.verdict, Verdict::Clean | Verdict::Absent), "{s:?}"); + } + assert_eq!(surface(&proof, Surface::Revisions).verdict, Verdict::Clean); +} + +#[test] +fn the_manifest_has_digests_and_counts_but_no_plaintext() { + let mut doc = marked(Spec::default()); + // The digest of the original content: a shared proof must not carry it. + let original: String = sha2::Sha256::digest(page_content(&doc).as_bytes()).iter().map(|b| format!("{b:02x}")).collect(); + let (r, proof) = run(&mut doc); + r.unwrap(); + let e = &proof.entries[0]; + let after = e.page_sha256_after.as_deref().unwrap(); + assert!(after.len() == 64 && after.bytes().all(|c| c.is_ascii_hexdigit())); + assert_ne!(after, original); + for shown in [proof.to_text(), format!("{proof:?}")] { + assert!(!shown.contains(&original), "{shown}"); + } + // Two glyph-carrying operators before; the removed one left a number-only TJ, which carries + // no glyphs and is not counted. + assert_eq!((e.text_ops_before, e.text_ops_after), (Some(2), Some(1))); + assert!(e.removed_text.is_none()); + for shown in [proof.to_text(), format!("{proof:?}")] { + assert!(!shown.contains(SECRET), "{shown}"); + } + + // Plaintext only on request. + let mut doc = marked(Spec::default()); + let mut snap = Snapshot::capture(&doc, None).unwrap(); + assert!(!format!("{snap:?}").contains(SECRET)); + applied(&mut doc, &mut snap).unwrap(); + let with = snap.prove(&doc, None, &ProofOptions { include_plaintext: true }); + assert_eq!(with.entries[0].removed_text.as_deref(), Some(&[SECRET.to_string()][..])); +} + +#[test] +fn survivors_do_not_echo_the_text_either() { + let mut doc = marked(Spec { trailer: "/Info 7 0 R", extra: vec![b"<< /Title (SECRETVALUE) >>".to_vec()], ..Spec::default() }); + let (_, proof) = run(&mut doc); + assert!(!proof.passed()); + for shown in [proof.to_text(), format!("{proof:?}")] { + assert!(!shown.contains(SECRET), "{shown}"); + } +} + +#[test] +fn numeric_only_tj_is_not_glyph_carrying() { + assert_eq!(count_text_operators(b"BT (Tj) Tj [ -250 ] TJ [(x) -3 (y)] TJ () Tj [] TJ ET"), 2); + assert_eq!(count_text_operators(b"BT (a) ' 1 2 (b) \" [ 10 -10 ] TJ ET"), 2); +} + +#[test] +fn a_survivor_in_the_info_dictionary() { + let mut doc = + marked(Spec { trailer: "/Info 7 0 R", extra: vec![b"<< /Title (Report SECRETVALUE) /Author (Someone) >>".to_vec()], ..Spec::default() }); + let (_, proof) = run(&mut doc); + assert_eq!(surface(&proof, Surface::Info).verdict, Verdict::Survivor); + assert_eq!(proof.entries[0].status, Status::Failed); + assert!(!proof.passed()); +} + +#[test] +fn a_survivor_in_xmp() { + let xmp = stream("/Type /Metadata /Subtype /XML", b"Quarterly & SECRETVALUE"); + let mut doc = marked(Spec { catalog: "/Metadata 7 0 R", extra: vec![xmp], ..Spec::default() }); + let (_, proof) = run(&mut doc); + assert_eq!(surface(&proof, Surface::Xmp).verdict, Verdict::Survivor, "{}", proof.to_text()); + assert!(!proof.passed()); +} + +#[test] +fn a_long_numeric_reference_in_xmp_is_a_survivor() { + // A character reference may carry any number of leading zeros: the padded form reads the + // same, and is swept the same. + let xmp = stream("/Type /Metadata /Subtype /XML", b"SECRETVALUE"); + let mut doc = marked(Spec { catalog: "/Metadata 7 0 R", extra: vec![xmp], ..Spec::default() }); + let (_, proof) = run(&mut doc); + assert_eq!(surface(&proof, Surface::Xmp).verdict, Verdict::Survivor, "{}", proof.to_text()); + assert!(!proof.passed()); +} + +#[test] +fn a_survivor_in_an_annotation_string() { + let annot = b"<< /Type /Annot /Subtype /Text /Rect [200 200 220 220] /Contents (note: SECRETVALUE) >>".to_vec(); + let mut doc = marked(Spec { page: "/Annots [7 0 R]", extra: vec![annot], ..Spec::default() }); + let (_, proof) = run(&mut doc); + assert_eq!(surface(&proof, Surface::ObjectStrings).verdict, Verdict::Survivor); + assert_eq!(proof.entries[0].status, Status::Failed); + assert!(!proof.passed()); +} + +// ── Every place a reader could recover the removed text from ──────────────────────────────── +// Each test plants the removed text in one location and expects the sweep to report it there. + +#[test] +fn a_survivor_in_a_compressed_object_stream_is_found() { + // A full save packs non-stream objects into compressed object streams: a copy of the + // removed text left in one (an annotation away from the region) is neither in the stored + // bytes nor on any page, only inside the /ObjStm. The sweep reads the saved file like any + // reader, objects in object streams included, and reports the copy. + let annot = b"<< /Type /Annot /Subtype /Text /Rect [200 200 220 220] /Contents (note: SECRETVALUE) >>".to_vec(); + let mut doc = marked(Spec { page: "/Annots [7 0 R]", extra: vec![annot], ..Spec::default() }); + let mut snap = Snapshot::capture(&doc, None).unwrap(); + applied(&mut doc, &mut snap).unwrap(); + let saved = write_full(&doc, &SaveOptions::default()).unwrap(); + assert!(saved.windows(b"/ObjStm".len()).any(|w| w == b"/ObjStm"), "the output packs objects into object streams"); + assert!(!saved.windows(SECRET.len()).any(|w| w == SECRET.as_bytes()), "the plaintext is compressed away"); + let proof = snap.prove_saved(&saved, None, &ProofOptions::default()); + let s = surface(&proof, Surface::ObjectStrings); + assert_eq!(s.verdict, Verdict::Survivor, "{}", proof.to_text()); + assert_eq!(surface(&proof, Surface::RawBytes).verdict, Verdict::Clean, "the copy sits in the object stream, not the stored bytes"); + assert!(!proof.passed(), "{}", proof.to_text()); +} + +#[test] +fn a_survivor_in_a_page_thumbnail_is_found() { + // The page's /Thumb holds a picture of the page as it was, secret included (uncompressed + // pixels, so the stored bytes are the pixels). A thumbnail is not page content, so only + // the object sweep can find it. + let thumb = stream("/Type /XObject /Subtype /Image /Width 7 /Height 3 /ColorSpace /DeviceGray /BitsPerComponent 8", b"junk SECRETVALUE junk"); + let mut doc = marked(Spec { page: "/Thumb 7 0 R", extra: vec![thumb], ..Spec::default() }); + let (_, proof) = run(&mut doc); + let s = surface(&proof, Surface::DecodedStreams); + assert_eq!(s.verdict, Verdict::Survivor, "{}", proof.to_text()); + assert!(s.locations.iter().any(|l| l.contains("(raw)")), "{:?}", s.locations); + assert!(!proof.passed(), "{}", proof.to_text()); +} + +#[test] +fn a_survivor_in_a_form_field_value_is_found() { + // The redacted text also sits in the form: a text field whose /V holds it. The field + // dictionary is an object like any other, and its strings are swept wherever the field + // hangs (AcroForm and the page's annotations both point at it). + let field = b"<< /Type /Annot /Subtype /Widget /Rect [200 20 280 40] /FT /Tx /T (form) /V (note: SECRETVALUE) >>".to_vec(); + let spec = Spec { catalog: "/AcroForm << /Fields [7 0 R] >>", page: "/Annots [7 0 R]", extra: vec![field], ..Spec::default() }; + let mut doc = marked(spec); + let (_, proof) = run(&mut doc); + let s = surface(&proof, Surface::ObjectStrings); + assert_eq!(s.verdict, Verdict::Survivor, "{}", proof.to_text()); + // The field's copy is the only place left (the save renumbers objects, so the place is + // named, not numbered from the input file; both views of the string count as survivors). + assert!(s.survivors >= 1, "{}", proof.to_text()); + assert_eq!(s.locations.len(), 1, "{:?}", s.locations); + assert!(!proof.passed(), "{}", proof.to_text()); +} + +#[test] +fn a_survivor_in_an_xfa_packet_is_found() { + // An XFA form mirrors its field values in the /XFA packets, XML of their own. Redaction + // strips an XFA form outright (crates/xfa does that, not this machinery); the proof must + // not take the stripper on trust: whatever it left (a shape it refused, a leftover + // packet) is swept like any stream, and what the packet holds is reported. + let xfa = stream("", b"
SECRETVALUE
"); + let spec = Spec { catalog: "/AcroForm << /XFA 7 0 R /Fields [] >>", extra: vec![xfa], ..Spec::default() }; + let doc = marked(spec); + let snap = Snapshot::capture(&doc, None).unwrap(); + let proof = snap.prove_saved(doc.bytes(), None, &ProofOptions::default()); + let s = surface(&proof, Surface::DecodedStreams); + assert_eq!(s.verdict, Verdict::Survivor, "{}", proof.to_text()); + assert!(s.locations.iter().any(|l| l == "object 7 (decoded)"), "{:?}", s.locations); + assert!(!proof.passed(), "{}", proof.to_text()); +} + +#[test] +fn a_survivor_in_an_unused_object_is_found() { + // Object 7 is reachable from nothing: no catalog entry, no page resource. A reader that + // only walks the document tree misses it; the sweep walks every object the + // cross-reference table knows, orphans included, and names the object it sits in. + let orphan = format!("<< /Type /Annot /Subtype /Text /Contents (note: {SECRET}) >>").into_bytes(); + let doc = marked(Spec { extra: vec![orphan], ..Spec::default() }); + let snap = Snapshot::capture(&doc, None).unwrap(); + let proof = snap.prove_saved(doc.bytes(), None, &ProofOptions::default()); + let s = surface(&proof, Surface::ObjectStrings); + assert_eq!(s.verdict, Verdict::Survivor, "{}", proof.to_text()); + assert!(s.locations.iter().any(|l| l == "object 7"), "{:?}", s.locations); + assert!(!proof.passed(), "{}", proof.to_text()); +} + +/// The page draws "(Image)" under the marked region, and keeps an image XObject whose +/// `/Subtype /Image` is the same word. +fn image_word_doc() -> Document { + let image = stream("/Type /XObject /Subtype /Image /Width 20 /Height 20 /ColorSpace /DeviceGray /BitsPerComponent 8", &[0x80u8; 400]); + let content = b"BT /F1 10 Tf 10 100 Td (Image) Tj ET"; + let spec = Spec { resources: "/XObject << /Im1 7 0 R >>", extra: vec![image], ..Spec::default() }; + let mut doc = open(build_with(content, spec)); + mark(&mut doc, &[[0.0, 90.0, 40.0, 115.0]]); + doc +} + +#[test] +fn a_word_that_remains_in_a_structural_name_does_not_fail_the_proof() { + // "Image" is redacted from the page, and is also the /Subtype of the image the page keeps. + // The name is structure redaction never edits, not a leftover copy: the proof must pass, + // and the name surface must read clean, not unswept. + let mut doc = image_word_doc(); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert!(proof.passed(), "{}", proof.to_text()); + assert_eq!(surface(&proof, Surface::ObjectStrings).verdict, Verdict::Clean); + + // But a name the input did not have is a copy like any other: a surplus of name matches + // is a survivor, wherever it sits. + let snap = Snapshot::capture(&image_word_doc(), None).unwrap(); + let image = stream("/Type /XObject /Subtype /Image /Width 20 /Height 20 /ColorSpace /DeviceGray /BitsPerComponent 8", &[0x80u8; 400]); + let spec = Spec { resources: "/XObject << /Im1 7 0 R >>", catalog: "/Extra << /Stolen /Image >>", extra: vec![image], ..Spec::default() }; + let mut shown = open(build_with(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", spec)); + mark(&mut shown, &[[0.0, 90.0, 40.0, 115.0]]); + shown.require_full_save(); + let proof = snap.prove(&shown, None, &ProofOptions::default()); + assert_eq!(surface(&proof, Surface::ObjectStrings).verdict, Verdict::Survivor, "{}", proof.to_text()); + assert!(!proof.passed(), "{}", proof.to_text()); +} + +fn attachment_spec(payload: &[u8]) -> Spec<'static> { + Spec { + catalog: "/Names << /EmbeddedFiles << /Names [(a.bin) 7 0 R] >> >>", + extra: vec![b"<< /Type /Filespec /F (a.bin) /EF << /F 8 0 R >> >>".to_vec(), stream("/Type /EmbeddedFile", payload)], + ..Spec::default() + } +} + +#[test] +fn a_survivor_in_an_attachment() { + let mut doc = marked(attachment_spec(b"the SECRETVALUE is attached")); + let (_, proof) = run(&mut doc); + assert_eq!(surface(&proof, Surface::EmbeddedFiles).verdict, Verdict::Survivor); + assert!(!proof.passed()); + + // UTF-16LE with a byte order mark. + let mut le = vec![0xFF, 0xFE]; + le.extend(SECRET.encode_utf16().flat_map(u16::to_le_bytes)); + let mut doc = marked(attachment_spec(&le)); + let (_, proof) = run(&mut doc); + assert_eq!(surface(&proof, Surface::EmbeddedFiles).verdict, Verdict::Survivor); + assert!(!proof.passed()); +} + +#[test] +fn a_survivor_in_a_nested_attachment() { + let inner = build_with(b"BT /F1 10 Tf 10 100 Td (SECRETVALUE) Tj ET", Spec::default()); + let middle = build_with(b"", attachment_spec(&inner)); + let mut doc = marked(attachment_spec(&middle)); + let (_, proof) = run(&mut doc); + let s = surface(&proof, Surface::EmbeddedFiles); + assert_eq!(s.verdict, Verdict::Survivor); + assert!(s.locations.iter().any(|l| l.matches(" > ").count() >= 1), "{:?}", s.locations); + assert!(!proof.passed(), "{}", proof.to_text()); +} + +#[test] +fn a_clean_nested_attachment_is_swept_and_clean() { + let inner = build_with(b"BT /F1 10 Tf 10 100 Td (harmless) Tj ET", Spec::default()); + let mut doc = marked(attachment_spec(&inner)); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert!(proof.passed(), "{}", proof.to_text()); + assert_eq!(surface(&proof, Surface::EmbeddedFiles).verdict, Verdict::Clean); +} + +#[test] +fn unsweepable_attachments_fail_closed() { + for payload in [&b"PK\x03\x04 zipped"[..], b"%PDF-1.7 not really a pdf", b"\x1f\x8b gzip"] { + let mut doc = marked(attachment_spec(payload)); + let (r, proof) = run(&mut doc); + assert!(r.is_ok() || residue(&r), "only a residue refusal may stop the apply itself: {r:?}"); + assert_eq!(surface(&proof, Surface::EmbeddedFiles).verdict, Verdict::Unswept, "{}", String::from_utf8_lossy(payload)); + assert!(!proof.passed(), "a proof that could not sweep must not pass"); + } +} + +#[test] +fn an_undecodable_stream_fails_closed() { + let junk = stream("/Filter /NoSuchDecode", b"\x00\x01\x02 junk"); + let mut doc = marked(Spec { catalog: "/Extra 7 0 R", extra: vec![junk], ..Spec::default() }); + let (_, proof) = run(&mut doc); + assert_eq!(surface(&proof, Surface::DecodedStreams).verdict, Verdict::Unswept, "{}", proof.to_text()); + assert!(!proof.passed()); +} + +#[test] +fn an_incremental_remnant_fails_the_proof() { + let mut doc = marked(Spec::default()); + let mut snap = Snapshot::capture(&doc, None).unwrap(); + // A "redaction" saved as an incremental update: the page shows something else now, but the + // first revision still holds the original content. + let mut other = open(write_incremental(&doc, &SaveOptions::default()).unwrap()); + let new = other.add(Object::Stream(Stream::flate(Dict::new(), b"BT /F1 10 Tf 10 200 Td (Public) Tj ET"))); + let page = pdfcraft_model::pages(&other).swap_remove(0); + other.update_dict(page.obj, |d| d.set(b"Contents".to_vec(), Object::Ref(new))).unwrap(); + let saved = write_incremental(&other, &SaveOptions::default()).unwrap(); + let proof = snap.prove_saved(&saved, None, &ProofOptions::default()); + assert!(!proof.passed()); + assert_eq!(surface(&proof, Surface::Revisions).verdict, Verdict::Survivor); + assert_eq!(surface(&proof, Surface::RawBytes).verdict, Verdict::Survivor); + // The page text itself is clean: only the file structure gives it away. + assert_eq!(surface(&proof, Surface::ExtractedText).verdict, Verdict::Clean); + + // The real redaction, saved as apply() requires, is a single clean revision. + applied(&mut doc, &mut snap).unwrap(); + let full = write_incremental(&doc, &SaveOptions::default()).unwrap(); + let proof = snap.prove_saved(&full, None, &ProofOptions::default()); + assert!(proof.passed(), "{}", proof.to_text()); +} + +#[test] +fn the_secret_in_each_encoding_is_found() { + // Spelled as PDF syntax in a catalog entry. + let utf16: String = SECRET.encode_utf16().flat_map(u16::to_be_bytes).map(|b| format!("{b:02X}")).collect(); + let utf16le: String = SECRET.encode_utf16().flat_map(u16::to_le_bytes).map(|b| format!("{b:02X}")).collect(); + let hex8: String = SECRET.bytes().map(|b| format!("{b:02x}")).collect(); + let forms = [ + format!("({SECRET})"), + format!(""), + format!(""), + format!("", hex8.to_uppercase()), + format!("<{hex8}>"), + format!("<{utf16}>"), + "(\\123ECRETVALUE)".to_string(), + ]; + for f in forms { + let mut doc = marked(Spec { catalog: &format!("/Extra {f}"), ..Spec::default() }); + let (_, proof) = run(&mut doc); + assert!(!proof.passed(), "{f}: {}", proof.to_text()); + assert_eq!(surface(&proof, Surface::ObjectStrings).verdict, Verdict::Survivor, "{f}: {}", proof.to_text()); + } + + // Spelled as bytes in the file, including the backslash-escaped literal form. + let paren = "SECR(ET)X\\Y"; + let enc = survivor_encodings(paren); + assert!(enc.len() >= 10); + for spelling in enc { + let mut hay = b"junk ".to_vec(); + hay.extend(&spelling); + hay.extend(b" junk"); + assert!(tests_support::found(paren, &hay), "{}", String::from_utf8_lossy(&spelling)); + } + assert!(!tests_support::found(paren, b"nothing to see")); +} + +#[test] +fn a_rotated_page_is_proven_too() { + let mut doc = marked(Spec { page: "/Rotate 90", ..Spec::default() }); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert!(proof.passed(), "{}", proof.to_text()); + assert_eq!(proof.entries[0].status, Status::Verified); +} + +#[test] +fn a_region_without_text_is_recorded_as_such() { + let mut doc = open(build_with(CONTENT, Spec::default())); + mark(&mut doc, &[[150.0, 20.0, 200.0, 40.0]]); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert_eq!(proof.entries[0].status, Status::VerifiedNoTextInRegion); + assert!(proof.passed()); +} + +fn opaque_spec() -> Spec<'static> { + Spec { resources: "/Font << /F1 5 0 R /F2 7 0 R >>", extra: vec![OPAQUE_FONT.as_bytes().to_vec()], ..Spec::default() } +} + +#[test] +fn text_that_cannot_be_decoded_makes_a_region_unverifiable() { + let mut doc = open(build_with(b"BT /F2 10 Tf 10 200 Td <00410042> Tj ET", opaque_spec())); + mark(&mut doc, &[[150.0, 20.0, 200.0, 40.0]]); + let (r, proof) = run(&mut doc); + assert!(r.is_ok() || residue(&r), "only a residue refusal may stop the apply itself: {r:?}"); + assert_eq!(proof.entries[0].status, Status::Unverifiable); + assert!(!proof.passed(), "no PASS while a region is unverifiable"); +} + +#[test] +fn removed_text_of_an_undecodable_font_is_swept_by_its_codes() { + let content = b"BT /F2 10 Tf 10 100 Td <0041004200430044> Tj ET"; + let mut doc = open(build_with(content, opaque_spec())); + mark(&mut doc, &[[0.0, 90.0, 100.0, 120.0]]); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert_eq!(proof.entries[0].runs_removed, 1); + assert_eq!(proof.entries[0].status, Status::Verified, "{}", proof.to_text()); + + // Left in place, the codes are still on the page and the proof says so. + let mut doc = open(build_with(content, opaque_spec())); + mark(&mut doc, &[[0.0, 90.0, 100.0, 120.0]]); + let proof = prove_unapplied(&mut doc); + assert!(!proof.passed()); + assert_eq!(surface(&proof, Surface::ExtractedText).verdict, Verdict::Survivor); +} + +#[test] +fn codes_removed_counts_codes_not_bytes() { + // A composite font's codes are two bytes each: the manifest counts the codes, not the bytes. + let content = b"BT /F2 10 Tf 10 100 Td <0041004200430044> Tj ET"; + let mut doc = open(build_with(content, opaque_spec())); + mark(&mut doc, &[[0.0, 90.0, 100.0, 120.0]]); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert_eq!(proof.entries[0].runs_removed, 1); + assert_eq!(proof.entries[0].codes_removed, 4, "{}", proof.to_text()); + + // A simple font's codes are one byte each: the two counts agree. + let mut doc = marked(Spec::default()); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert_eq!((proof.entries[0].runs_removed, proof.entries[0].codes_removed), (1, SECRET.len())); +} + +#[test] +fn overlapping_marks_each_account_for_their_own_share() { + // Two regions over the same text: each gets the runs, codes and status it is owed — the + // second one is not waved through as "no text in the region". + let mut doc = open(build_with(CONTENT, Spec::default())); + mark(&mut doc, &[REGION, [7.0, 94.0, 71.0, 113.0]]); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert_eq!(proof.entries.len(), 2); + for e in &proof.entries { + assert_eq!((e.runs_removed, e.codes_removed), (1, SECRET.len()), "{}", proof.to_text()); + assert_eq!(e.status, Status::Verified, "{}", proof.to_text()); + } + + // Left in place, both regions name the survivor as theirs. + let mut doc = open(build_with(CONTENT, Spec::default())); + mark(&mut doc, &[REGION, [7.0, 94.0, 71.0, 113.0]]); + let proof = prove_unapplied(&mut doc); + assert!(!proof.passed()); + assert!(proof.entries.iter().all(|e| matches!(e.status, Status::Failed | Status::Unverifiable)), "{}", proof.to_text()); +} + +#[test] +fn nothing_applied_means_survivors_where_the_text_was() { + let mut doc = marked(Spec::default()); + let proof = prove_unapplied(&mut doc); + assert!(!proof.passed()); + assert_eq!(surface(&proof, Surface::ExtractedText).verdict, Verdict::Survivor); + assert_eq!(surface(&proof, Surface::DecodedStreams).verdict, Verdict::Survivor); + assert_eq!(proof.entries[0].status, Status::Failed); +} + +#[test] +fn a_document_that_would_save_incrementally_is_not_proven() { + let mut doc = marked(Spec::default()); + let mut snap = Snapshot::capture(&doc, None).unwrap(); + applied(&mut doc, &mut snap).unwrap(); + // A reopened copy no longer carries the full-save requirement. + let reopened = open(write_incremental(&doc, &SaveOptions::default()).unwrap()); + let proof = snap.prove(&reopened, None, &ProofOptions::default()); + assert_eq!(surface(&proof, Surface::Revisions).verdict, Verdict::Survivor); +} + +fn image_page() -> Document { + let image = stream("/Type /XObject /Subtype /Image /Width 20 /Height 20 /ColorSpace /DeviceGray /BitsPerComponent 8", &[0xFFu8; 400]); + let spec = Spec { resources: "/XObject << /Im1 7 0 R >>", extra: vec![image], ..Spec::default() }; + marked_region(b"q 100 0 0 100 100 100 cm /Im1 Do Q 0 0 1 rg 120 120 60 60 re f", spec) +} + +fn marked_region(content: &[u8], spec: Spec<'_>) -> Document { + let mut doc = open(build_with(content, spec)); + mark(&mut doc, &[[100.0, 100.0, 150.0, 150.0]]); + doc +} + +#[test] +fn images_and_paths_under_a_region_are_checked_independently() { + let mut doc = image_page(); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert!(proof.passed(), "{}", proof.to_text()); + assert_eq!(proof.entries[0].status, Status::VerifiedNoTextInRegion); + + // The same page unredacted: the image and the path are both still there. + let proof = prove_unapplied(&mut image_page()); + let e = &proof.entries[0]; + assert_eq!(e.status, Status::Failed); + assert!(e.detail.contains("image") && e.detail.contains("path"), "{}", e.detail); +} + +#[test] +fn a_shading_under_a_region_must_be_clipped() { + let sh = + b"<< /ShadingType 2 /ColorSpace /DeviceRGB /Coords [0 0 300 0] /Function << /FunctionType 2 /Domain [0 1] /C0 [1 0 0] /C1 [0 0 1] /N 1 >> >>" + .to_vec(); + let spec = || Spec { resources: "/Shading << /Sh1 7 0 R >>", extra: vec![sh.clone()], ..Spec::default() }; + let mut doc = marked_region(b"/Sh1 sh", spec()); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert!(proof.passed(), "{}", proof.to_text()); + + let proof = prove_unapplied(&mut marked_region(b"/Sh1 sh", spec())); + assert_eq!(proof.entries[0].status, Status::Failed); +} + +#[test] +fn an_even_odd_clip_around_the_region_hides_what_it_cuts_out() { + // How kept content is told to hide: an even-odd clip of the page and the region. A path + // painted inside the region under that clip is clipped away; the same path under a clip + // the rule cannot read (a triangle, overlapping inners), or with no clip, is reported. + let blank = marked_content(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", Spec::default()); + let snap = Snapshot::capture(&blank, None).unwrap(); + let prove = |tail: &str| -> Proof { + let content = format!("BT /F1 10 Tf 10 200 Td (Public) Tj ET {tail}"); + let mut shown = marked_content(content.as_bytes(), Spec::default()); + shown.require_full_save(); + snap.prove(&shown, None, &ProofOptions::default()) + }; + let proof = prove("q 0 0 300 300 re 8 95 62 17 re W* n 0 0 0 rg 10 100 20 10 re f Q"); + assert!(proof.passed(), "{}", proof.to_text()); + + // A triangle is not a rectangle: no hole is cut, and the paint is reported. + let proof = prove("q 0 0 300 300 re 8 95 m 70 95 l 8 112 l h W* n 0 0 0 rg 10 100 20 10 re f Q"); + assert_eq!(proof.entries[0].status, Status::Failed, "{}", proof.to_text()); + assert!(proof.entries[0].detail.contains("path"), "{}", proof.to_text()); + + // Overlapping inners change the parity: no hole is cut either. + let proof = prove("q 0 0 300 300 re 8 95 62 17 re 5 90 70 25 re W* n 0 0 0 rg 10 100 20 10 re f Q"); + assert_eq!(proof.entries[0].status, Status::Failed, "{}", proof.to_text()); + + // Unclipped, the same path is a plain finding. + let proof = prove("0 0 0 rg 10 100 20 10 re f"); + assert_eq!(proof.entries[0].status, Status::Failed, "{}", proof.to_text()); +} + +#[test] +fn a_graphics_state_stack_too_deep_to_follow_is_unverifiable() { + // Saved states carry their clip holes with them: a `q` stack deep enough to grow without + // bound is refused, not followed. + let blank = marked_content(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", Spec::default()); + let snap = Snapshot::capture(&blank, None).unwrap(); + let mut deep = Vec::new(); + for _ in 0..crate::verify::limits::MAX_GS_STACK + 8 { + deep.extend_from_slice(b"q "); + } + deep.extend_from_slice(b"0 0 0 rg 10 100 20 10 re f"); + let mut shown = marked_content(&deep, Spec::default()); + shown.require_full_save(); + let proof = snap.prove(&shown, None, &ProofOptions::default()); + assert_eq!(proof.entries[0].status, Status::Unverifiable, "{}", proof.to_text()); + assert!(proof.entries[0].detail.contains("too deeply"), "{}", proof.to_text()); + assert!(!proof.passed()); +} + +#[test] +fn a_page_full_of_paint_reports_its_finding_once() { + // Fifty paths over a region are one finding: the report names the problem once, not once + // per operator, and the report does not grow with the operator count. + let blank = marked_content(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", Spec::default()); + let snap = Snapshot::capture(&blank, None).unwrap(); + let mut content = b"BT /F1 10 Tf 10 200 Td (Public) Tj ET ".to_vec(); + for k in 0..50 { + content.extend_from_slice(&format!("0 0 1 rg {} 100 4 4 re f ", 10 + k).into_bytes()); + } + let mut shown = marked_content(&content, Spec::default()); + shown.require_full_save(); + let proof = snap.prove(&shown, None, &ProofOptions::default()); + assert_eq!(proof.entries[0].status, Status::Failed, "{}", proof.to_text()); + assert_eq!(proof.entries[0].detail.matches("a path still paints").count(), 1, "{}", proof.to_text()); +} + +#[test] +fn text_kept_elsewhere_does_not_fail_the_proof() { + // The same word is redacted in one place and legitimately remains in another. + let content = b"BT /F1 10 Tf 10 200 Td (SECRETVALUE) Tj ET BT /F1 10 Tf 10 100 Td (SECRETVALUE) Tj ET"; + let mut doc = marked_content(content, Spec::default()); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert!(proof.passed(), "{}", proof.to_text()); + assert!(proof.entries[0].detail.contains("remain in retained content")); + + // A copy that should have gone is still caught, by count. + let proof = prove_unapplied(&mut marked_content(content, Spec::default())); + assert_eq!(surface(&proof, Surface::ExtractedText).verdict, Verdict::Survivor); +} + +#[test] +fn the_proof_text_keeps_its_honesty_wording() { + // The closing caveat of a proof is a promise to whoever reads a shared proof: pin it so the + // wording cannot silently drop out of the text for either verdict. + let honesty = "Evidence, not a guarantee"; + let passed = Proof { entries: vec![], surfaces: vec![], failures: vec![] }; + assert!(passed.passed()); + assert!(passed.to_text().contains(honesty), "{}", passed.to_text()); + + let failed = Proof { entries: vec![], surfaces: vec![], failures: vec!["the marked text is still readable".into()] }; + assert!(!failed.passed()); + assert!(failed.to_text().contains(honesty), "{}", failed.to_text()); +} + +// ── Independence, scope and bounds of the sweep ───────────────────────────────────────────── + +/// Hang `obj` off the catalog (as `/Extra`) so the saved file keeps it. +pub(super) fn attach(doc: &mut Document, obj: Object) { + let r = doc.add(obj); + let root = doc.root().unwrap(); + doc.update_dict(root, |d| d.set(b"Extra".to_vec(), Object::Ref(r))).unwrap(); +} + +/// Add a content stream (with `dict`) after the page's own. +fn append_content(doc: &mut Document, dict: Dict, data: &[u8]) { + let new = doc.add(Object::Stream(Stream::from_raw(dict, data.to_vec()))); + let page = pdfcraft_model::pages(doc).swap_remove(0); + let first = page.dict.get(b"Contents").cloned().unwrap(); + doc.update_dict(page.obj, |d| d.set(b"Contents".to_vec(), Object::Array(vec![first, Object::Ref(new)]))).unwrap(); +} + +#[test] +fn a_glyph_the_interpreter_kept_is_found_by_placement_alone() { + // The snapshot is taken where the region holds no text, so it has nothing to search for: + // only the independent placement of glyphs can notice the text in the output. + let blank = marked_content(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", Spec::default()); + let snap = Snapshot::capture(&blank, None).unwrap(); + let mut shown = marked(Spec::default()); + shown.require_full_save(); + let proof = snap.prove(&shown, None, &ProofOptions::default()); + assert!(!proof.passed()); + let e = &proof.entries[0]; + assert_eq!(e.status, Status::Failed, "{}", proof.to_text()); + assert!(e.detail.contains("glyph(s) still show"), "{}", e.detail); + assert_eq!(proof.survivors(), 0, "no needle was involved"); + // The neighbour at y 200 is outside the region and is not reported. + assert!(!e.detail.contains("12 glyph"), "{}", e.detail); +} + +#[test] +fn glyphs_are_placed_with_the_text_matrix_and_the_form_matrix() { + // The secret drawn through a rotated/translated text matrix inside a scaled form: the + // placement has to follow `Tm` and the form's `/Matrix`, not the raw operands. + let form = stream( + "/Type /XObject /Subtype /Form /BBox [0 0 300 300] /Matrix [1 0 0 1 0 0] /Resources << /Font << /F1 5 0 R >> >>", + b"BT /F1 10 Tf 1 0 0 1 10 100 Tm (SECRETVALUE) Tj ET", + ); + let blank = marked_content(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", Spec::default()); + let snap = Snapshot::capture(&blank, None).unwrap(); + let mut doc = marked_content(b"/Fm1 Do", Spec { resources: "/XObject << /Fm1 7 0 R >>", extra: vec![form], ..Spec::default() }); + doc.require_full_save(); + let proof = snap.prove(&doc, None, &ProofOptions::default()); + assert_eq!(proof.entries[0].status, Status::Failed, "{}", proof.to_text()); + assert!(proof.entries[0].detail.contains("glyph(s) still show")); +} + +#[test] +fn an_ambiguous_needle_is_still_swept_outside_page_content() { + // The word also stays on the page legitimately (so page text, the raw file and decoded + // streams can only count it), but a copy in the Info dictionary is not page content. + let content = b"BT /F1 10 Tf 10 200 Td (SECRETVALUE) Tj ET BT /F1 10 Tf 10 100 Td (SECRETVALUE) Tj ET"; + let spec = Spec { trailer: "/Info 7 0 R", extra: vec![b"<< /Title (SECRETVALUE) >>".to_vec()], ..Spec::default() }; + let mut doc = marked_content(content, spec); + let (_, proof) = run(&mut doc); + assert_eq!(surface(&proof, Surface::Info).verdict, Verdict::Survivor, "{}", proof.to_text()); + assert!(!proof.passed()); + + // Likewise an XMP packet and an annotation string. + let xmp = stream("/Type /Metadata /Subtype /XML", b"SECRETVALUE"); + let mut doc = marked_content(content, Spec { catalog: "/Metadata 7 0 R", extra: vec![xmp], ..Spec::default() }); + assert_eq!(surface(&run(&mut doc).1, Surface::Xmp).verdict, Verdict::Survivor); + let annot = b"<< /Type /Annot /Subtype /Text /Rect [200 200 220 220] /Contents (SECRETVALUE) >>".to_vec(); + let mut doc = marked_content(content, Spec { page: "/Annots [7 0 R]", extra: vec![annot], ..Spec::default() }); + assert_eq!(surface(&run(&mut doc).1, Surface::ObjectStrings).verdict, Verdict::Survivor); +} + +#[test] +fn a_pcmark_key_in_the_input_does_not_exempt_a_stream() { + // A content stream that claims to be a redaction box paints inside the region, and was not + // created by this run: it counts as paint like any other. + let blank = b"BT /F1 10 Tf 10 200 Td (Public) Tj ET"; + let mut doc = marked_content(blank, Spec::default()); + let mut d = Dict::new(); + d.set(b"PCMark".to_vec(), Object::Name(b"Redaction".to_vec())); + append_content(&mut doc, d, b"0 0 0 rg 8 95 62 17 re f"); + let proof = prove_unapplied(&mut doc); + let e = &proof.entries[0]; + assert_eq!(e.status, Status::Failed, "{}", proof.to_text()); + assert!(e.detail.contains("path"), "{}", e.detail); +} + +#[test] +fn only_the_streams_this_run_created_are_exempt() { + // The same stream, declared as created by this run, is the box that was drawn on purpose. + let blank = b"BT /F1 10 Tf 10 200 Td (Public) Tj ET"; + let mut doc = marked_content(blank, Spec::default()); + let mut snap = Snapshot::capture(&doc, None).unwrap(); + let new = doc.add(Object::Stream(Stream::from_raw(Dict::new(), b"0 0 0 rg 8 95 62 17 re f".to_vec()))); + let page = pdfcraft_model::pages(&doc).swap_remove(0); + let first = page.dict.get(b"Contents").cloned().unwrap(); + doc.update_dict(page.obj, |d| d.set(b"Contents".to_vec(), Object::Array(vec![first, Object::Ref(new)]))).unwrap(); + doc.require_full_save(); + snap.set_overlay_streams(&doc, &[new]).unwrap(); + let proof = snap.prove(&doc, None, &ProofOptions::default()); + assert!(proof.passed(), "{}", proof.to_text()); +} + +#[test] +fn overlay_text_does_not_collide_with_what_was_removed() { + // "Account" is redacted and the label says "Account [redacted]": the label is not a survivor. + let content = b"BT /F1 10 Tf 10 100 Td (Account 1234) Tj ET"; + let mut doc = open(build_with(content, Spec::default())); + let shape = Shape::Redact { quads: vec![rect_quad([8.0, 95.0, 100.0, 112.0])], overlay: "Account [redacted]".into(), look: Default::default() }; + let style = Style::default_for(&shape); + add_annotation(&mut doc, &NewAnnotation { page: 0, shape, style, contents: String::new(), author: "T".into() }, &Meta::default()).unwrap(); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert!(proof.passed(), "{}", proof.to_text()); + assert_eq!(surface(&proof, Surface::ExtractedText).verdict, Verdict::Clean); +} + +#[test] +fn removed_annotation_and_field_text_is_searched_for_too() { + let spec = || Spec { trailer: "/Info 7 0 R", extra: vec![b"<< /Subject (call back ANNOTWORDS) >>".to_vec()], ..Spec::default() }; + // Not asked for: the proof does not know about it. + let mut doc = marked(spec()); + let (_, proof) = run(&mut doc); + assert!(proof.passed(), "{}", proof.to_text()); + // Asked for: it is found where it was left. + let mut doc = marked(spec()); + let mut snap = Snapshot::capture(&doc, None).unwrap(); + snap.add_removed_strings(&[" call back ANNOTWORDS ".to_string(), "x".to_string(), String::new()]); + let result = applied(&mut doc, &mut snap); + assert!(result.is_ok() || residue(&result)); + let proof = snap.prove(&doc, None, &ProofOptions::default()); + assert!(!proof.passed()); + assert_eq!(surface(&proof, Surface::Info).verdict, Verdict::Survivor, "{}", proof.to_text()); + // And an absent string is clean. + let mut doc = marked(Spec::default()); + let mut snap = Snapshot::capture(&doc, None).unwrap(); + snap.add_removed_strings(&["never present anywhere".to_string()]); + applied(&mut doc, &mut snap).unwrap(); + assert!(snap.prove(&doc, None, &ProofOptions::default()).passed()); +} + +#[test] +fn a_damaged_stream_tail_is_unswept_not_truncated() { + let text: Vec = (0..4000u32).map(|i| b'a' + (i * 7 % 23) as u8).collect(); + let mut s = Stream::flate(Dict::new(), &text); + let cut = s.raw.len() - 8; + s.raw = Arc::new(s.raw[..cut].to_vec()).into(); + // Nothing was removed (the snapshot's region is blank), so the only issue is the stream. + let snap = Snapshot::capture(&marked_content(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", Spec::default()), None).unwrap(); + let mut doc = marked_content(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", Spec::default()); + attach(&mut doc, Object::Stream(s)); + doc.require_full_save(); + let proof = snap.prove(&doc, None, &ProofOptions::default()); + assert_eq!(surface(&proof, Surface::DecodedStreams).verdict, Verdict::Unswept, "{}", proof.to_text()); + assert!(!proof.passed()); +} + +#[test] +fn a_damaged_page_content_tail_makes_the_page_unreadable() { + let text: Vec = b"BT /F1 10 Tf 10 100 Td (filler) Tj ET\n".repeat(200); + let mut s = Stream::flate(Dict::new(), &text); + let cut = s.raw.len() - 8; + s.raw = Arc::new(s.raw[..cut].to_vec()).into(); + let mut doc = marked(Spec::default()); + let page = pdfcraft_model::pages(&doc).swap_remove(0); + let new = doc.add(Object::Stream(s)); + doc.update_dict(page.obj, |d| d.set(b"Contents".to_vec(), Object::Ref(new))).unwrap(); + doc.require_full_save(); + let snap = Snapshot::capture(&marked(Spec::default()), None).unwrap(); + let proof = snap.prove(&doc, None, &ProofOptions::default()); + assert!(!proof.passed()); + assert_eq!(surface(&proof, Surface::ExtractedText).verdict, Verdict::Unswept, "{}", proof.to_text()); +} + +#[test] +fn a_malformed_form_matrix_or_bbox_is_unswept() { + for dict in ["/Matrix [1 0 0]", "/Matrix [1 0 0 1 0 (x)]", "/BBox [0 0 300]"] { + let form = stream(&format!("/Type /XObject /Subtype /Form /BBox [0 0 300 300] {dict}"), b"BT /F1 10 Tf 10 100 Td (x) Tj ET"); + let spec = Spec { resources: "/XObject << /Fm1 7 0 R >>", extra: vec![form], ..Spec::default() }; + let mut doc = marked_content(b"/Fm1 Do", spec); + let proof = prove_unapplied(&mut doc); + assert_eq!(surface(&proof, Surface::ExtractedText).verdict, Verdict::Unswept, "{dict}: {}", proof.to_text()); + assert!(!proof.passed()); + assert!(proof.entries[0].detail.contains("malformed"), "{dict}: {}", proof.to_text()); + } +} + +#[test] +fn images_hidden_in_patterns_soft_masks_and_type3_glyphs_are_not_skipped() { + let image = "BI /W 1 /H 1 /CS /G /BPC 8 ID \u{ff} EI"; + let tile = + |data: &str| stream("/PatternType 1 /PaintType 1 /TilingType 1 /BBox [0 0 10 10] /XStep 10 /YStep 10 /Resources << >>", data.as_bytes()); + let group = stream("/Type /XObject /Subtype /Form /BBox [0 0 300 300] /Group << /S /Transparency >> /Resources << >>", image.as_bytes()); + let proc = stream("", format!("1000 0 d0 {image}").as_bytes()); + let cases: Vec<(&str, Spec)> = vec![ + ("pattern", Spec { resources: "/Pattern << /P1 7 0 R >>", extra: vec![tile(image)], ..Spec::default() }), + ( + "soft mask", + Spec { resources: "/ExtGState << /G1 << /SMask << /Type /Mask /S /Luminosity /G 7 0 R >> >> >>", extra: vec![group], ..Spec::default() }, + ), + ( + "type3", + Spec { + resources: "/Font << /F3 << /Type /Font /Subtype /Type3 /FontBBox [0 0 1000 1000] /FontMatrix [0.001 0 0 0.001 0 0] /CharProcs << /a 7 0 R >> /Encoding << /Type /Encoding /Differences [97 /a] >> /FirstChar 97 /LastChar 97 /Widths [1000] >> >>", + extra: vec![proc], + ..Spec::default() + }, + ), + ]; + for (what, spec) in cases { + // The region holds no text and no visible image: only the hidden image is in question. + let mut doc = marked_content(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", spec); + let proof = prove_unapplied(&mut doc); + let e = &proof.entries[0]; + assert_eq!(e.status, Status::Unverifiable, "{what}: {}", proof.to_text()); + assert!(e.detail.contains("pattern, soft mask or Type3"), "{what}: {}", e.detail); + assert!(!proof.passed()); + } + // The same resources without an image are fine. + let spec = Spec { resources: "/Pattern << /P1 7 0 R >>", extra: vec![tile("0 0 5 5 re f")], ..Spec::default() }; + let mut doc = marked_content(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", spec); + let proof = prove_unapplied(&mut doc); + assert_eq!(proof.entries[0].status, Status::VerifiedNoTextInRegion, "{}", proof.to_text()); +} + +#[test] +fn the_sweep_is_bounded() { + // A CMap whose ranges rewrite the same 64Ki codes again and again costs work, whatever the map size. + let mut cmap = b"1 begincodespacerange <0000> endcodespacerange\n".to_vec(); + for _ in 0..200 { + cmap.extend_from_slice(b"100 beginbfrange\n"); + for _ in 0..100 { + cmap.extend_from_slice(b"<0000> <0041>\n"); + } + cmap.extend_from_slice(b"endbfrange\n"); + } + let start = std::time::Instant::now(); + assert!(!tests_support::cmap_is_complete(&cmap)); + assert!(start.elapsed().as_secs() < 20, "bounded work"); + // A small CMap is read in full. + assert!(tests_support::cmap_is_complete(b"1 beginbfchar <01> <0041> endbfchar")); + // Nested objects, cumulative decoding and searching all stop and say so. + assert!(tests_support::walks_in_full(60)); + assert!(!tests_support::walks_in_full(200)); + assert_eq!(tests_support::decode_budget_runs_out(), (true, true)); + assert_eq!(tests_support::search_budget_runs_out(), (true, true)); + // A decode that fails past the one-stream cap pays for the attempt: the same bomb cannot + // be decoded (and fail) again and again for free. + assert_eq!(tests_support::failed_decode_charges_the_budget(), (true, true, true)); +} + +#[test] +fn the_search_finds_what_is_there_whatever_the_prefix() { + assert!(tests_support::found("aaaab", b"xxaaaaaaab")); + assert!(tests_support::found("needle", b"needle")); + assert!(!tests_support::found("needle", b"needl")); + assert!(!tests_support::found("aaab", b"aaaaaaaaaa")); + assert!(tests_support::found("abc", b"zabc")); + // The worst shape for a skip-based search (a needle of one letter and a haystack full of + // it) is walked once per byte, and still finds what is there. + let needle = format!("{}b", "a".repeat(64)); + let mut hay = vec![b'a'; 20_000]; + assert!(!tests_support::found(&needle, &hay)); + hay.extend_from_slice(needle.as_bytes()); + assert!(tests_support::found(&needle, &hay)); +} + +#[test] +fn the_sweep_charge_stays_fair_for_a_bulk_of_needles() { + // 80 removed strings swept over one 5 MiB haystack (a stream decoded in full): the search + // really is one pass per needle, but the budget must not charge the haystack once per + // needle — that charge (haystack × needle × spelling: 80 × 20 × 5 MiB, about twice the 4 GiB + // work budget) would refuse an ordinary bulk redaction out of hand. The haystack is charged + // once per scan and a needle's own charge is capped, so the sweep stays under the budget. + let hay = vec![b'x'; 5 << 20]; + let word = "SECRETVALUE-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + assert_eq!(tests_support::bulk_counts(&hay, word, 80, 0), Some(0)); + // Genuinely unbounded work still fails closed: the region scanned is charged in full, so a + // haystack that no longer fits in the remaining budget is an unswept surface, never a pass. + assert_eq!(tests_support::bulk_counts(&hay, word, 80, crate::verify::tests_support::work_budget() - (1 << 20)), None); +} + +#[test] +fn survivors_seen_before_the_work_limit_are_still_reported() { + // The page sweep stops when the work budget runs out, but the survivors it has already + // counted are settled: a survivor on an earlier page is reported where it sits, not hidden + // behind the limit (the page still reads unswept, so the proof still fails). + let (reported, unswept) = tests_support::page_sweep_survivors_survive_the_limit(); + assert!(reported, "a survivor counted before the limit is reported"); + assert!(unswept, "the page the limit stopped on is unswept"); +} + +#[test] +fn a_file_past_the_proof_budget_is_unverified_never_clean() { + // A page whose content is bigger than the proof may decode for one stream: the budget + // stops the sweep, and the result must say so. The surfaces read unswept (a page content + // stream is never excused as raw-only), the region reads unverifiable, the proof fails. + // Work that stopped is "not verified", never a pass. + let big = vec![0u8; crate::verify::limits::MAX_STREAM + (6 << 20)]; + let content = Stream::flate(Dict::new(), &big); + drop(big); + let mut doc = open(build_with(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", Spec::default())); + let page = pdfcraft_model::pages(&doc).swap_remove(0); + let new = doc.add(Object::Stream(content)); + doc.update_dict(page.obj, |d| d.set(b"Contents".to_vec(), Object::Ref(new))).unwrap(); + doc.require_full_save(); + let snap = Snapshot::capture(&marked(Spec::default()), None).unwrap(); + let proof = snap.prove(&doc, None, &ProofOptions::default()); + assert!(!proof.passed(), "{}", proof.to_text()); + assert!(!proof.failures.is_empty(), "{}", proof.to_text()); + for s in [Surface::ExtractedText, Surface::DecodedStreams] { + let r = surface(&proof, s); + assert_eq!(r.verdict, Verdict::Unswept, "{s:?}: {}", proof.to_text()); + assert_eq!((r.survivors, r.raw_only), (0, 0), "{s:?}: {}", proof.to_text()); + } + assert!(matches!(proof.entries[0].status, Status::Unverifiable | Status::Failed), "the region is not verified: {}", proof.to_text()); +} + +#[test] +fn font_widths_parsing_shares_the_work_budget() { + // A form invoked again rebuilds its fonts, so the `/W` array is paid for from the shared + // work budget, not from a per-font counter that every rebuild resets. + let (full, refused) = tests_support::font_widths_charge_shares_the_budget(); + assert!(full, "a small /W array is read in full"); + assert!(refused, "a spent work budget refuses the /W parsing"); +} + +#[test] +fn an_unreadable_overlay_stream_is_its_own_error() { + let mut doc = marked(Spec::default()); + let mut snap = Snapshot::capture(&doc, None).unwrap(); + let mut dict = Dict::new(); + dict.set(b"Filter".to_vec(), Object::Name(b"FlateDecode".to_vec())); + let broken = doc.add(Object::Stream(Stream::from_raw(dict, b"not deflate data".to_vec()))); + let err = snap.set_overlay_streams(&doc, &[broken]).unwrap_err(); + assert_eq!(err, RedactError::OverlayUnreadable); + assert!(!err.to_string().contains("page 0"), "{err}"); +} + +// ── The caller's credentials ───────────────────────────────────────────────────────────────── + +#[test] +fn a_nonempty_user_password_document_is_proven_with_those_credentials() { + // The redacted document is protected with a user password and saved: the proof's subject is + // now an encrypted file, and only the caller's own credentials can read it back. Every + // supported encryption algorithm. + for alg in [Algorithm::Rc4_40, Algorithm::Rc4_128, Algorithm::Aes128, Algorithm::Aes256] { + let mut doc = marked(Spec::default()); + let mut snap = Snapshot::capture(&doc, None).unwrap(); + applied(&mut doc, &mut snap).unwrap(); + doc.set_encryption(&NewEncryption { + algorithm: alg, + user_password: "pw-secret", + owner_password: "owner-secret", + permissions: -1, + encrypt_metadata: true, + seed: [7; 32], + }) + .unwrap(); + + // The credentials the caller holds prove the saved output. + let proof = snap.prove(&doc, Some("pw-secret"), &ProofOptions::default()); + assert!(proof.passed(), "{alg:?}: {}", proof.to_text()); + + // An already saved file too. + let saved = write_full(&doc, &SaveOptions::default()).unwrap(); + let proof = snap.prove_saved(&saved, Some("pw-secret"), &ProofOptions::default()); + assert!(proof.passed(), "{alg:?}: {}", proof.to_text()); + + // Empty and wrong credentials fail closed: the output can't be read back, which is an + // unswept proof, never a pass. + for pw in [None, Some("wrong")] { + let proof = snap.prove_saved(&saved, pw, &ProofOptions::default()); + assert!(!proof.passed(), "{alg:?} {pw:?}: {}", proof.to_text()); + for s in &proof.surfaces { + assert_eq!(s.verdict, Verdict::Unswept, "{alg:?} {pw:?}: {}", proof.to_text()); + } + } + } +} + +// ── Compressed copies inside binary streams ────────────────────────────────────────────────── + +/// A stream object whose stored bytes are the Flate encoding of `data` (as `Stream::flate` +/// writes them), with the extra dictionary entries `dict`. +fn flate_object(dict: &str, data: &[u8]) -> Vec { + stream(&format!("{dict} /Filter /FlateDecode"), &Stream::flate(Dict::new(), data).raw) +} + +/// A one-page file whose resources point at a fake font: /F2 is a /TrueType font whose +/// /FontDescriptor carries `font_file` as its /FontFile2 (objects 7, 8 and 9). The font carries +/// `/Widths` (object 6) so the interpreter resolves it; nothing on the page draws with it. +fn font_file_spec(font_file: Vec) -> Spec<'static> { + Spec { + resources: "/Font << /F1 5 0 R /F2 7 0 R >>", + extra: vec![ + b"<< /Type /Font /Subtype /TrueType /BaseFont /X /FirstChar 32 /LastChar 126 /Widths 6 0 R /FontDescriptor 8 0 R >>".to_vec(), + b"<< /Type /FontDescriptor /FontName /X /Flags 4 /FontFile2 9 0 R >>".to_vec(), + font_file, + ], + ..Spec::default() + } +} + +#[test] +fn a_compressed_copy_in_a_fake_font_file_is_found() { + // The visible copy is redacted normally, but a Flate-compressed copy of the secret hides in + // a fake /FontFile2. /Length1 makes the sweep class the stream as a binary font program, so + // its stored bytes hold no plaintext: only the decoded data gives the copy away. + let spec = font_file_spec(flate_object("/Length1 900", SECRET.as_bytes())); + let mut doc = marked(spec); + let (_, proof) = run(&mut doc); + assert_eq!(surface(&proof, Surface::DecodedStreams).verdict, Verdict::Survivor, "{}", proof.to_text()); + let s = surface(&proof, Surface::DecodedStreams); + assert!(s.locations.iter().any(|l| l.contains("(decoded)")), "{:?}", s.locations); + assert!(!proof.passed()); +} + +#[test] +fn a_legitimate_font_file_is_swept_and_clean() { + // A font program that holds none of what was removed (as real ones do not) is searched and + // lets the redaction pass: no false survivor. + let spec = font_file_spec(flate_object("/Length1 900", b"\x00\x01\x00\x00\x00\x10glyph tables")); + let mut doc = marked(spec); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert!(proof.passed(), "{}", proof.to_text()); + assert_eq!(surface(&proof, Surface::DecodedStreams).verdict, Verdict::Clean, "{}", proof.to_text()); +} + +#[test] +fn a_font_program_that_will_not_decode_is_searched_as_stored_bytes() { + // An unrelated, damaged font program the page reaches is not a reason to refuse: nothing + // redacted touched it, so its stored bytes are searched and it is counted as raw only. + let raw = Stream::flate(Dict::new(), b"a font program without the secret").raw.as_ref().to_vec(); + let cut = raw.len() - 8; + let spec = font_file_spec(stream("/Length1 900 /Filter /FlateDecode", &raw[..cut])); + let mut doc = marked(spec); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert!(proof.passed(), "{}", proof.to_text()); + let s = surface(&proof, Surface::DecodedStreams); + assert_eq!((s.verdict, s.raw_only), (Verdict::Clean, 1), "{}", proof.to_text()); +} + +/// The standard page plus an image (object 7) drawn away from the secret. +fn image_doc(image: Vec) -> Document { + let mut content = CONTENT.to_vec(); + content.extend_from_slice(b" q 20 0 0 20 200 200 cm /Im1 Do Q"); + marked_content(&content, Spec { resources: "/XObject << /Im1 7 0 R >>", extra: vec![image], ..Spec::default() }) +} + +const IMG: &str = "/Type /XObject /Subtype /Image /Width 20 /Height 20 /ColorSpace /DeviceGray /BitsPerComponent 8 /Filter /FlateDecode"; + +fn assert_redacted_cleanly(mut doc: Document) -> Proof { + let (r, proof) = run(&mut doc); + r.unwrap(); + assert!(proof.passed(), "{}", proof.to_text()); + let saved = write_incremental(&doc, &SaveOptions::default()).unwrap(); + assert!(!saved.windows(SECRET.len()).any(|w| w == SECRET.as_bytes()), "the secret is gone from the file"); + proof +} + +#[test] +fn an_unrelated_truncated_flate_image_does_not_fail_the_proof() { + let raw = Stream::flate(Dict::new(), &[0x80u8; 400]).raw.as_ref().to_vec(); + let proof = assert_redacted_cleanly(image_doc(stream(IMG, &raw[..raw.len() - 8]))); + assert_eq!(surface(&proof, Surface::DecodedStreams).raw_only, 1, "{}", proof.to_text()); +} + +#[test] +fn an_unrelated_image_over_the_stream_cap_does_not_fail_the_proof() { + // Highly compressible: stored in a few dozen KiB, decoded past MAX_STREAM. + let big = vec![0u8; crate::verify::limits::MAX_STREAM + (6 << 20)]; + let raw = Stream::flate(Dict::new(), &big).raw.as_ref().to_vec(); + drop(big); + let proof = assert_redacted_cleanly(image_doc(stream(IMG, &raw))); + assert_eq!(surface(&proof, Surface::DecodedStreams).raw_only, 1, "{}", proof.to_text()); +} + +#[test] +fn an_unrelated_truncated_font_program_does_not_fail_the_redaction() { + let raw = Stream::flate(Dict::new(), b"glyph tables").raw.as_ref().to_vec(); + let doc = marked(font_file_spec(stream("/Length1 900 /Filter /FlateDecode", &raw[..raw.len() - 6]))); + assert_redacted_cleanly(doc); +} + +#[test] +fn a_secret_in_the_stored_bytes_of_a_font_program_is_still_found() { + // The font program cannot be decoded (it claims Flate but holds plain bytes), so only its + // stored bytes are searched: the copy of the secret in them is a survivor. + let spec = font_file_spec(stream("/Length1 900 /Filter /FlateDecode", format!("not deflate {SECRET} not deflate").as_bytes())); + let mut doc = marked(spec); + let (_, proof) = run(&mut doc); + let s = surface(&proof, Surface::DecodedStreams); + assert_eq!((s.verdict, s.raw_only), (Verdict::Survivor, 1), "{}", proof.to_text()); + assert!(s.locations.iter().any(|l| l.contains("(raw)")), "{:?}", s.locations); + assert!(!proof.to_text().contains(SECRET)); + assert!(!proof.passed()); +} + +/// The standard page plus an /Indexed colour space whose lookup table is object 7. +fn indexed_doc(lookup: Vec) -> Document { + marked_content(CONTENT, Spec { resources: "/ColorSpace << /CS1 [/Indexed /DeviceRGB 255 7 0 R] >>", extra: vec![lookup], ..Spec::default() }) +} + +#[test] +fn an_indexed_lookup_stream_is_searched_as_stored_bytes_only() { + // Unrelated and damaged: nothing redacted touched it, so it must not fail the proof. + let raw = Stream::flate(Dict::new(), &[0x40u8; 300]).raw.as_ref().to_vec(); + let proof = assert_redacted_cleanly(indexed_doc(stream("/Filter /FlateDecode", &raw[..raw.len() - 8]))); + assert_eq!(surface(&proof, Surface::DecodedStreams).raw_only, 1, "{}", proof.to_text()); + // A secret stored raw inside one that will not decode is still found. + let mut doc = indexed_doc(stream("/Filter /FlateDecode", format!("not deflate {SECRET} not deflate").as_bytes())); + let (_, proof) = run(&mut doc); + let s = surface(&proof, Surface::DecodedStreams); + assert_eq!((s.verdict, s.raw_only), (Verdict::Survivor, 1), "{}", proof.to_text()); + assert!(!proof.passed()); +} + +/// Capture a sound document, then damage the stream `num` (a Flate stream cut short, with the +/// extra dictionary entries `dict`) and prove it. +fn prove_damaged(mut doc: Document, num: u32, dict: &str, data: &[u8]) -> Proof { + let snap = Snapshot::capture(&doc, None).unwrap(); + let raw = Stream::flate(Dict::new(), data).raw.as_ref().to_vec(); + let cut = raw[..raw.len() - 8].to_vec(); + let broken = open(pdf_bytes(vec![b"<< /Type /Catalog >>".to_vec(), stream(&format!("{dict} /Filter /FlateDecode"), &cut)], "")); + let obj = broken.get(ObjRef::new(2, 0)).as_ref().clone(); + doc.set(ObjRef::new(num, 0), obj); + doc.require_full_save(); + snap.prove(&doc, None, &ProofOptions::default()) +} + +#[test] +fn damaged_content_appearance_and_metadata_still_fail_closed() { + // A truncated second content stream. + let objs: Vec> = vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents [4 0 R 7 0 R] /Resources << /Font << /F1 5 0 R >> >> >>".to_vec(), + stream("", CONTENT), + FONT.as_bytes().to_vec(), + format!("[{}]", vec!["500"; 95].join(" ")).into_bytes(), + stream("", b"BT /F1 10 Tf 10 200 Td (more text here) Tj ET"), + ]; + let mut doc = open(pdf_bytes(objs, "/Root 1 0 R")); + mark(&mut doc, &[REGION]); + let proof = prove_damaged(doc, 7, "", b"BT /F1 10 Tf 10 200 Td (more text here) Tj ET"); + assert!(!proof.passed()); + assert!(!surface(&proof, Surface::ExtractedText).problems.is_empty(), "{}", proof.to_text()); + assert!(!surface(&proof, Surface::DecodedStreams).problems.is_empty(), "{}", proof.to_text()); + + // A truncated appearance stream. + let annot = b"<< /Type /Annot /Subtype /Square /Rect [200 200 250 250] /AP << /N 8 0 R >> >>".to_vec(); + let ap = stream("/Type /XObject /Subtype /Form /BBox [0 0 50 50]", b"0 0 50 50 re f 0 0 1 rg 1 0 0 RG 2 w"); + let mut doc = marked(Spec { page: "/Annots [7 0 R]", extra: vec![annot, ap], ..Spec::default() }); + doc.require_full_save(); + let proof = prove_damaged(doc, 8, "/Type /XObject /Subtype /Form /BBox [0 0 50 50]", b"0 0 50 50 re f 0 0 1 rg 1 0 0 RG 2 w"); + assert!(!proof.passed()); + assert!(!surface(&proof, Surface::ExtractedText).problems.is_empty(), "{}", proof.to_text()); + + // A truncated XMP packet. + let xmp = stream("/Type /Metadata /Subtype /XML", b"some metadata of the file"); + let doc = marked(Spec { catalog: "/Metadata 7 0 R", extra: vec![xmp], ..Spec::default() }); + let proof = prove_damaged(doc, 7, "/Type /Metadata /Subtype /XML", b"some metadata of the file"); + assert!(!proof.passed()); + assert!(!surface(&proof, Surface::Xmp).problems.is_empty(), "{}", proof.to_text()); +} + +#[test] +fn an_annotation_appearance_is_checked_however_it_is_stored() { + // The region holds no text, so only the paint check can speak. An annotation written in + // place (a direct dictionary, no reference of its own) is checked like an indirect one, and + // a stream shared by two annotations is checked at each annotation's own placement, so the + // copy that lands over the region is seen even though the first placement does not reach it. + let blank = marked_content(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", Spec::default()); + let snap = Snapshot::capture(&blank, None).unwrap(); + let prove = |spec: Spec<'_>| -> Proof { + let mut shown = marked_content(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", spec); + shown.require_full_save(); + snap.prove(&shown, None, &ProofOptions::default()) + }; + let ap = stream("/Type /XObject /Subtype /Form /BBox [0 0 62 17] /Resources << >>", b"0 0 0 rg 10 5 20 10 re f"); + + let proof = prove(Spec { + page: "/Annots [<< /Type /Annot /Subtype /Square /Rect [8 95 70 112] /AP << /N 7 0 R >> >>]", + extra: vec![ap], + ..Spec::default() + }); + assert_eq!(proof.entries[0].status, Status::Failed, "{}", proof.to_text()); + assert!(proof.entries[0].detail.contains("path"), "{}", proof.to_text()); + + let ap = stream("/Type /XObject /Subtype /Form /BBox [0 0 62 17] /Resources << >>", b"0 0 0 rg 10 5 20 10 re f"); + let far = b"<< /Type /Annot /Subtype /Square /Rect [200 200 250 250] /AP << /N 7 0 R >> >>".to_vec(); + let near = b"<< /Type /Annot /Subtype /Square /Rect [8 95 70 112] /AP << /N 7 0 R >> >>".to_vec(); + let proof = prove(Spec { page: "/Annots [8 0 R 9 0 R]", extra: vec![ap, far, near], ..Spec::default() }); + assert_eq!(proof.entries[0].status, Status::Failed, "{}", proof.to_text()); + assert!(proof.entries[0].detail.contains("path"), "{}", proof.to_text()); +} + +#[test] +fn a_null_entry_in_the_contents_array_is_an_empty_piece() { + let objs: Vec> = vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents [4 0 R null] /Resources << /Font << /F1 5 0 R >> >> >>".to_vec(), + stream("", CONTENT), + FONT.as_bytes().to_vec(), + format!("[{}]", vec!["500"; 95].join(" ")).into_bytes(), + ]; + let mut doc = open(pdf_bytes(objs, "/Root 1 0 R")); + mark(&mut doc, &[REGION]); + assert_redacted_cleanly(doc); +} + +/// The page's decoded content streams, joined (what the redaction left on the page). +fn page_content(doc: &Document) -> String { + let p = &pdfcraft_model::pages(doc)[0]; + let (_, data) = page_streams(doc, &p.dict, 0).unwrap(); + data.iter().map(|d| String::from_utf8_lossy(d).into_owned()).collect::>().join("\n") +} diff --git a/crates/redact/src/verify/extract.rs b/crates/redact/src/verify/extract.rs new file mode 100644 index 0000000000..f6db96229b --- /dev/null +++ b/crates/redact/src/verify/extract.rs @@ -0,0 +1,711 @@ +//! A minimal text extractor, independent of the interpreter's font metrics and hit test. + +use std::collections::HashMap; + +use pdfcraft_content::{Matrix, parse}; +use pdfcraft_cos::{Dict, Document, ObjRef, Object, Stream}; + +use super::util::{Ctx, latin1, resolve_strict, squash}; +use crate::annots_of; +use crate::verify::limits::{MAX_CMAP, MAX_CMAP_ENTRIES, MAX_CMAP_WORK, MAX_DEPTH}; + +/// Is this one of the glyph-carrying text-showing operators (`Tj`, `'`, `"`, and `TJ` with at +/// least one non-empty string)? A `TJ` of numbers only (the gap redaction leaves for a removed +/// string) moves the cursor but shows nothing, so it does not count. +pub(super) fn carries_glyphs(op: &pdfcraft_content::Op) -> bool { + let nonempty = |o: &Object| o.as_string().is_some_and(|s| !s.bytes.is_empty()); + match op.op.as_slice() { + b"Tj" | b"'" | b"\"" => op.operands.last().is_some_and(nonempty), + b"TJ" => op.operands.first().and_then(Object::as_array).is_some_and(|a| a.iter().any(nonempty)), + _ => false, + } +} + +/// A page's text as two whitespace-free views: decoded text, and the character codes as Latin-1 +/// (what fonts without a usable mapping show). +#[derive(Default)] +pub(super) struct Extracted { + pub(super) text: String, + pub(super) raw: String, + pub(super) glyph_ops: usize, + /// Glyph-carrying operators whose text could not be decoded. + pub(super) undecodable_ops: usize, + /// Per region asked about: glyphs whose centre this extractor places inside it. + pub(super) shown_in: Vec, + /// Glyph-carrying operators drawn where this extractor can't place them (tiling patterns, + /// or an annotation appearance without a usable `/Rect`). + pub(super) unplaced: usize, +} + +/// How one font turns codes into text. +#[derive(Clone, Default)] +pub(super) struct Decoder { + pub(super) map: HashMap, + /// Bytes per code. + pub(super) width: usize, + /// No way to read these codes as text (composite font without a mapping). + pub(super) opaque: bool, +} + +impl Decoder { + fn latin1() -> Self { + Decoder { map: HashMap::new(), width: 1, opaque: false } + } + + /// `Err` when the font's own data can't be read in full (a damaged or oversized CMap). + fn new(doc: &Document, font: &Dict, ctx: &Ctx) -> Result { + let composite = font.name(b"Subtype") == Some(b"Type0"); + let mut d = Decoder { map: HashMap::new(), width: if composite { 2 } else { 1 }, opaque: false }; + let tu = font.get(b"ToUnicode").map(|t| doc.resolve(t)); + if let Some(Object::Stream(s)) = tu.as_deref() { + let data = ctx.decode(s)?; + if data.len() > MAX_CMAP { + return Err(()); + } + let (map, width) = parse_cmap(&data).ok_or(())?; + if !map.is_empty() { + d.map = map; + if let Some(w) = width { + d.width = w; + } + return Ok(d); + } + } + if composite { + d.opaque = true; + return Ok(d); + } + // Simple font: Latin-1, with the /Differences of the encoding. + if let Some(enc) = font.get(b"Encoding").map(|e| doc.resolve(e)) + && let Some(diff) = enc.as_dict().and_then(|e| e.get(b"Differences")).map(|x| doc.resolve(x)) + && let Some(items) = diff.as_array() + { + let mut code = 0u32; + for it in items { + match &*doc.resolve(it) { + Object::Int(i) => code = u32::try_from(*i).unwrap_or(0), + Object::Name(n) => { + if let Some(c) = std::str::from_utf8(n).ok().and_then(pdfcraft_fonts::pdf::glyph_unicode) { + d.map.insert(code, c.to_string()); + } + code = code.saturating_add(1); + } + _ => {} + } + } + } + Ok(d) + } + + /// Each code of a shown string, with the text it gives (`None`: no way to read it as text). + fn split(&self, s: &[u8]) -> Vec<(u32, Option)> { + let w = self.width.max(1); + s.chunks(w) + .map(|c| { + let code = code_of(c); + let text = if self.opaque { + None + } else { + match self.map.get(&code) { + Some(t) => Some(t.clone()), + None if self.map.is_empty() || w == 1 && code < 256 => Some(char::from(u8::try_from(code).unwrap_or(b'?')).to_string()), + None => None, + } + }; + (code, text) + }) + .collect() + } + + /// The text of a shown string. + fn decode(&self, s: &[u8]) -> String { + self.split(s).into_iter().filter_map(|(_, t)| t).collect() + } +} + +/// A font as the geometric check sees it: its decoder and its own simple width model. +#[derive(Clone)] +pub(super) struct Font { + pub(super) dec: Decoder, + /// Advance per code, in text space units per 1 em of font size (0.5 = half an em). + pub(super) widths: HashMap, + /// Advance of a code `widths` doesn't list. + pub(super) default_width: f64, + pub(super) vertical: bool, +} + +/// The width a font without usable `/Widths` is assumed to have (half an em). +pub(super) const FLAT_WIDTH: f64 = 0.5; + +impl Font { + fn flat() -> Self { + Font { dec: Decoder::latin1(), widths: HashMap::new(), default_width: FLAT_WIDTH, vertical: false } + } + + fn new(doc: &Document, font: &Dict, ctx: &Ctx) -> Result { + let dec = Decoder::new(doc, font, ctx)?; + let mut f = Font { dec, widths: HashMap::new(), default_width: FLAT_WIDTH, vertical: false }; + let num = |o: &Object| doc.resolve(o).as_f64(); + if font.name(b"Subtype") == Some(b"Type0") { + f.vertical = font.get(b"Encoding").map(|e| doc.resolve(e)).and_then(|e| e.as_name().map(|n| n.ends_with(b"-V"))).unwrap_or(false); + let desc = font.get(b"DescendantFonts").map(|d| doc.resolve(d)).and_then(|d| d.as_array().and_then(|a| a.first().cloned())); + let desc = desc.and_then(|d| doc.resolve(&d).as_dict().cloned()); + if let Some(d) = desc { + f.default_width = d.get(b"DW").and_then(num).map_or(1.0, |w| w / 1000.0); + // The array is charged to the shared work budget as it is taken (a form + // invoked again rebuilds its fonts, so a per-font counter alone would let the + // same work be paid for many times), and every entry is paid for there too, + // whether or not it grows the map (a range may rewrite the same codes again + // and again, and the array is bounded only by the file). + let items: Vec = match d.get(b"W").map(|w| doc.resolve(w)).and_then(|w| w.as_array().cloned()) { + Some(a) => { + if !ctx.charge(a.len()) { + return Err(()); + } + a + } + None => Vec::new(), + }; + let mut work = 0usize; + let mut step = |n: usize| -> Result<(), ()> { + work = work.saturating_add(n); + if work <= MAX_CMAP_WORK && ctx.charge(n) { Ok(()) } else { Err(()) } + }; + let mut i = 0; + while let Some(first) = items.get(i).and_then(num) { + let first = first.max(0.0) as u32; + match items.get(i + 1).map(|o| doc.resolve(o)) { + Some(a) if a.as_array().is_some() => { + step(a.as_array().map_or(0, Vec::len))?; + for (k, w) in a.as_array().into_iter().flatten().enumerate() { + if f.widths.len() >= MAX_CMAP_ENTRIES { + return Err(()); + } + if let (Some(w), Ok(k)) = (num(w), u32::try_from(k)) { + f.widths.insert(first.saturating_add(k), w / 1000.0); + } + } + i += 2; + } + Some(last) => { + let (Some(last), Some(w)) = (last.as_f64(), items.get(i + 2).and_then(num)) else { break }; + let last = (last.max(0.0) as u32).min(first.saturating_add(0xFFFF)); + // Each code the range covers is paid for, grown or not (a span is at + // most 0x10000 codes, so the count always fits). + step(last.saturating_sub(first) as usize + 1)?; + for c in first..=last { + if f.widths.len() >= MAX_CMAP_ENTRIES { + return Err(()); + } + f.widths.insert(c, w / 1000.0); + } + i += 3; + } + None => break, + } + } + } + } else { + // Glyph space units per em: 1000, or what a Type3 font's matrix says. + let scale = if font.name(b"Subtype") == Some(b"Type3") { + font.get(b"FontMatrix").map(|m| doc.resolve(m)).and_then(|m| m.as_array().and_then(|a| a.first().and_then(num))).unwrap_or(0.001) + } else { + 0.001 + }; + let first = font.get(b"FirstChar").and_then(num).map_or(0, |v| v.max(0.0) as u32); + let missing = font + .get(b"FontDescriptor") + .map(|d| doc.resolve(d)) + .and_then(|d| d.as_dict().and_then(|d| d.get(b"MissingWidth").and_then(num))) + .map(|w| w * scale); + if let Some(ws) = font.get(b"Widths").map(|w| doc.resolve(w)).and_then(|w| w.as_array().cloned()) { + if ws.len() > MAX_CMAP_ENTRIES { + return Err(()); + } + for (k, w) in ws.iter().enumerate() { + if let (Some(w), Ok(k)) = (num(w), u32::try_from(k)) { + f.widths.insert(first.saturating_add(k), w * scale); + } + } + f.default_width = missing.unwrap_or(0.0); + } else if let Some(std) = pdfcraft_fonts::Std14::from_base_font(&String::from_utf8_lossy(font.name(b"BaseFont").unwrap_or(b""))) { + // No /Widths: a standard font has exact published widths, by WinAnsi code (Symbol, + // ZapfDingbats and other encodings are refused before the proof is reached). + for code in 0..=255u8 { + f.widths.insert(u32::from(code), std.width(code) / 1000.0); + } + f.default_width = if std.is_fixed_pitch() { 0.6 } else { 0.0 }; + } + } + Ok(f) + } + + fn width(&self, code: u32) -> f64 { + self.widths.get(&code).copied().unwrap_or(self.default_width) + } +} + +pub(super) fn hex_token(tok: &[u8]) -> Option> { + let digits: Vec = tok.iter().filter_map(|c| (*c as char).to_digit(16).map(|d| d as u8)).collect(); + if digits.is_empty() || !digits.len().is_multiple_of(2) { + return None; + } + Some(digits.chunks(2).map(|p| p.iter().fold(0u8, |a, d| (a << 4) | d)).collect()) +} + +pub(super) fn utf16_text(b: &[u8]) -> String { + if let [c] = b { + return char::from(*c).to_string(); + } + let units: Vec = b.as_chunks::<2>().0.iter().map(|c| u16::from_be_bytes(*c)).collect(); + String::from_utf16_lossy(&units) +} + +pub(super) fn code_of(b: &[u8]) -> u32 { + b.iter().fold(0u32, |a, x| a.wrapping_shl(8) | u32::from(*x)) +} + +pub(super) enum Tok { + Hex(Vec), + Word(String), + Open, + Close, +} + +pub(super) fn cmap_tokens(data: &[u8]) -> Vec { + let mut out = Vec::new(); + let mut i = 0; + while let Some(&c) = data.get(i) { + match c { + b'<' if data.get(i + 1) != Some(&b'<') => { + let end = data.get(i..).and_then(|d| d.iter().position(|x| *x == b'>')).map_or(data.len(), |e| i + e); + if let Some(h) = data.get(i + 1..end).and_then(hex_token) { + out.push(Tok::Hex(h)); + } + i = end + 1; + } + b'[' => { + out.push(Tok::Open); + i += 1; + } + b']' => { + out.push(Tok::Close); + i += 1; + } + b'%' => { + while data.get(i).is_some_and(|x| !matches!(x, b'\n' | b'\r')) { + i += 1; + } + } + c if c.is_ascii_alphabetic() => { + let start = i; + while data.get(i).is_some_and(|x| x.is_ascii_alphanumeric()) { + i += 1; + } + out.push(Tok::Word(String::from_utf8_lossy(data.get(start..i).unwrap_or_default()).into_owned())); + } + _ => i += 1, + } + } + out +} + +/// `bfchar` / `bfrange` entries of a ToUnicode CMap, and the code length of its codespace. +/// `None` when the CMap is bigger than [`MAX_CMAP_ENTRIES`] entries or costs more than +/// [`MAX_CMAP_WORK`] steps: a map that is cut short would read as "no such code". +pub(super) fn parse_cmap(data: &[u8]) -> Option<(HashMap, Option)> { + let toks = cmap_tokens(data); + let mut map: HashMap = HashMap::new(); + let mut width = None; + // Every step is paid for, whether or not it grows the map (a range may rewrite the same + // codes again and again). + let mut work = 0usize; + let mut step = |n: usize| -> Option<()> { + work = work.saturating_add(n); + (work <= MAX_CMAP_WORK).then_some(()) + }; + let mut i = 0; + while let Some(t) = toks.get(i) { + i += 1; + let Tok::Word(w) = t else { continue }; + match w.as_str() { + "begincodespacerange" => { + if let Some(Tok::Hex(h)) = toks.get(i) { + width = Some(h.len()); + } + } + "beginbfchar" => { + while let (Some(Tok::Hex(src)), Some(Tok::Hex(dst))) = (toks.get(i), toks.get(i + 1)) { + step(1)?; + map.insert(code_of(src), utf16_text(dst)); + if map.len() > MAX_CMAP_ENTRIES { + return None; + } + i += 2; + } + } + "beginbfrange" => { + while let (Some(Tok::Hex(lo)), Some(Tok::Hex(hi))) = (toks.get(i), toks.get(i + 1)) { + let (lo, hi) = (code_of(lo), code_of(hi)); + let span = hi.saturating_sub(lo).min(0xFFFF); + match toks.get(i + 2) { + Some(Tok::Hex(dst)) => { + let base = utf16_text(dst); + step(usize::try_from(span).ok()?.saturating_add(1))?; + for k in 0..=span { + let mut chars: Vec = base.chars().collect(); + if let Some(last) = chars.last_mut() { + *last = char::from_u32(u32::from(*last).saturating_add(k)).unwrap_or('\u{fffd}'); + } + map.insert(lo.saturating_add(k), chars.into_iter().collect()); + } + if map.len() > MAX_CMAP_ENTRIES { + return None; + } + i += 3; + } + Some(Tok::Open) => { + i += 3; + let mut k = 0u32; + while let Some(Tok::Hex(d)) = toks.get(i) { + step(1)?; + map.insert(lo.saturating_add(k), utf16_text(d)); + if map.len() > MAX_CMAP_ENTRIES { + return None; + } + k = k.saturating_add(1); + i += 1; + } + i += 1; // the closing bracket + } + _ => break, + } + } + } + _ => {} + } + } + Some((map, width)) +} + +pub(super) fn res_dict(doc: &Document, res: &Dict, key: &[u8]) -> Dict { + res.get(key).map(|o| doc.resolve(o)).and_then(|o| o.as_dict().cloned()).unwrap_or_default() +} + +pub(super) fn push_squashed(into: &mut String, s: &str) { + into.extend(s.chars().filter(|c| !c.is_whitespace() && *c != '\u{feff}')); +} + +/// A form's or pattern's `/Matrix`: absent is the identity, present but malformed is an error +/// (guessing the identity would put its content in the wrong place). +pub(super) fn matrix_of(doc: &Document, dict: &Dict) -> Result { + match dict.get(b"Matrix") { + None => Ok(Matrix::IDENTITY), + Some(m) => { + let m = doc.resolve(m); + let items: Vec = m.as_array().ok_or(())?.iter().map(|x| (*doc.resolve(x)).clone()).collect(); + Matrix::from_operands(&items).filter(|m| m.0.iter().all(|v| v.is_finite())).ok_or(()) + } + } +} + +/// A form's `/BBox` as `[x0 y0 x1 y1]`: absent is `None`, present but malformed is an error. +pub(super) fn bbox_of(doc: &Document, dict: &Dict) -> Result, ()> { + let Some(b) = dict.get(b"BBox") else { return Ok(None) }; + let b = doc.resolve(b); + let v: Vec = + b.as_array().ok_or(())?.iter().map(|x| doc.resolve(x).as_f64().filter(|v| v.is_finite()).ok_or(())).collect::>()?; + match v[..] { + [a, b, c, d] => Ok(Some([a.min(c), b.min(d), a.max(c), b.max(d)])), + _ => Err(()), + } +} + +/// The content streams of a page as `(raw object, decoded)`, a missing or unreadable entry an +/// error. `skip_overlay` leaves out the streams this run drew the boxes and labels with. +pub(super) fn page_content(doc: &Document, page: &Dict, ctx: &Ctx, skip_overlay: bool) -> Result, ()> { + let list: Vec = match page.get(b"Contents") { + None => Vec::new(), + Some(c) => match &*resolve_strict(doc, c)? { + Object::Array(a) => a.clone(), + _ => vec![c.clone()], + }, + }; + let mut data: Vec = Vec::new(); + for o in &list { + match &*resolve_strict(doc, o)? { + Object::Stream(s) => { + let d = ctx.decode(s)?; + if skip_overlay && ctx.is_overlay(&d) { + continue; + } + data.extend(d); + data.push(b'\n'); + } + // A null entry is an empty piece, as for the redaction itself (ISO 32000-2 7.3.9). + Object::Null => {} + _ => return Err(()), + } + } + Ok(data) +} + +/// The text of a page: its content, the forms and patterns it reaches (visible or not), and the +/// appearance streams of its annotations; and which of `regions` still hold glyphs. +pub(super) fn extract_page(doc: &Document, page: &Dict, ctx: &Ctx, regions: &[[f64; 4]]) -> Result { + let data = page_content(doc, page, ctx, true)?; + let res = page.get(b"Resources").and_then(|r| doc.resolve(r).as_dict().cloned()).unwrap_or_default(); + let mut ex = Extractor { doc, ctx, regions, out: Extracted { shown_in: vec![0; regions.len()], ..Extracted::default() }, stack: Vec::new() }; + ex.scope(&data, &res, Matrix::IDENTITY, 0, true)?; + for a in annots_of(doc, page) { + let Some(d) = doc.resolve(&a).as_dict().cloned() else { continue }; + let Some(ap) = d.get(b"AP").and_then(|x| doc.resolve(x).as_dict().cloned()) else { continue }; + let rect = + d.get(b"Rect").and_then(|r| doc.resolve(r).as_array().map(|a| a.iter().filter_map(|x| doc.resolve(x).as_f64()).collect::>())); + for (_, state) in ap.iter() { + let s = doc.resolve(state); + let streams: Vec<(Option, Object)> = match &*s { + Object::Stream(_) => vec![(state.as_ref(), (*s).clone())], + Object::Dict(sub) => sub.iter().map(|(_, v)| (v.as_ref(), (*doc.resolve(v)).clone())).collect(), + _ => Vec::new(), + }; + for (r, o) in streams { + if let Object::Stream(st) = o { + // Where the appearance lands on the page (ISO 32000-2 §12.5.5), if it can be told. + let ctm = rect.as_deref().and_then(|r| appearance_matrix(doc, &st, r)); + // A font of the appearance is looked up in the appearance's own resources + // first (what the redaction itself does), then in the page's. + let own = st.dict.get(b"Resources").and_then(|x| doc.resolve(x).as_dict().cloned()); + let parent = own.as_ref().unwrap_or(&res); + ex.stream(&st, r, parent, ctm.unwrap_or_default(), 1, ctm.is_some())?; + } + } + } + } + Ok(ex.out) +} + +/// The matrix that maps an appearance stream's form space onto an annotation's `/Rect` +/// (ISO 32000-2 §12.5.5): the appearance's `/Matrix`, then the scale that fits the mapped +/// `/BBox` into the rectangle. `None` when that can't be told (no usable `/Rect`, `/BBox` or +/// `/Matrix`, or a mapped box that degenerates). +pub(crate) fn appearance_matrix(doc: &Document, ap: &Stream, rect: &[f64]) -> Option { + let [rx0, ry0, rx1, ry1] = rect[..] else { return None }; + let bbox = bbox_of(doc, &ap.dict).ok()??; + let m = matrix_of(doc, &ap.dict).ok()?; + let t = m.bbox(bbox); + let (tw, th) = (t[2] - t[0], t[3] - t[1]); + if tw <= 0.0 || th <= 0.0 { + return None; + } + let (rx0, rx1, ry0, ry1) = (rx0.min(rx1), rx0.max(rx1), ry0.min(ry1), ry0.max(ry1)); + let (sx, sy) = ((rx1 - rx0) / tw, (ry1 - ry0) / th); + let a = Matrix([sx, 0.0, 0.0, sy, rx0 - t[0] * sx, ry0 - t[1] * sy]); + Some(m.then(&a)) +} + +/// Text-state parameters that `q`/`Q` save and restore. +#[derive(Clone)] +pub(super) struct TextGs { + pub(super) ctm: Matrix, + pub(super) tc: f64, + pub(super) tw: f64, + pub(super) th: f64, + pub(super) tl: f64, + pub(super) rise: f64, + pub(super) size: f64, + pub(super) font: Vec, +} + +pub(super) struct Extractor<'a> { + pub(super) doc: &'a Document, + pub(super) ctx: &'a Ctx, + pub(super) regions: &'a [[f64; 4]], + pub(super) out: Extracted, + pub(super) stack: Vec, +} + +impl Extractor<'_> { + /// A form or pattern stream. `ctm` maps its space onto the page's; `placed` is false where + /// that mapping is not known. + fn stream(&mut self, s: &Stream, id: Option, parent: &Dict, ctm: Matrix, depth: usize, placed: bool) -> Result<(), ()> { + if let Some(r) = id { + if self.stack.contains(&r) { + return Ok(()); + } + self.stack.push(r); + } + let result = (|| { + let data = self.ctx.decode(s)?; + let own = s.dict.get(b"Resources").and_then(|r| self.doc.resolve(r).as_dict().cloned()); + self.scope(&data, own.as_ref().unwrap_or(parent), ctm, depth, placed) + })(); + if id.is_some() { + self.stack.pop(); + } + result + } + + /// Show the strings of one text-showing operator: extract their text, and place each glyph + /// to count the ones inside a region. `tm` moves along with the pen. + fn show_text(&mut self, font: &Font, op: &pdfcraft_content::Op, gs: &TextGs, tm: &mut Matrix, placed: bool) { + let o = op.op.as_slice(); + // Strings and TJ adjustments, in order. + let items: Vec<&Object> = match o { + b"TJ" => op.operands.first().and_then(Object::as_array).map(|a| a.iter().collect()).unwrap_or_default(), + _ => op.operands.last().into_iter().collect(), + }; + let mut any_text = false; + for it in items { + if let Some(n) = it.as_f64() { + let d = -n / 1000.0 * gs.size; + let m = if font.vertical { Matrix::translate(0.0, d) } else { Matrix::translate(d * gs.th, 0.0) }; + *tm = m.then(tm); + continue; + } + let Some(s) = it.as_string() else { continue }; + let t = font.dec.decode(&s.bytes); + any_text |= !squash(&t).is_empty(); + push_squashed(&mut self.out.text, &t); + push_squashed(&mut self.out.raw, &latin1(&s.bytes)); + for (code, text) in font.dec.split(&s.bytes) { + let w = font.width(code); + let space = text.as_deref().is_some_and(|t| t == " "); + // The glyph's centre, in text space (the box runs from 0.2 em below the + // baseline to 0.8 em above), and where the pen moves to. + let (centre, advance) = if font.vertical { + let adv = -gs.size + gs.tc + if space { gs.tw } else { 0.0 }; + ((0.0, gs.rise - 0.5 * gs.size), Matrix::translate(0.0, adv)) + } else { + let adv = (w * gs.size + gs.tc + if space { gs.tw } else { 0.0 }) * gs.th; + ((0.5 * w * gs.size * gs.th, gs.rise + 0.3 * gs.size), Matrix::translate(adv, 0.0)) + }; + let visible = text.as_deref().is_none_or(|t| !squash(t).is_empty()); + if placed && visible && !self.regions.is_empty() { + let (x, y) = tm.then(&gs.ctm).apply(centre.0, centre.1); + for (r, n) in self.regions.iter().zip(self.out.shown_in.iter_mut()) { + if x >= r[0] && x <= r[2] && y >= r[1] && y <= r[3] { + *n += 1; + } + } + } + *tm = advance.then(tm); + } + } + if carries_glyphs(op) { + self.out.glyph_ops += 1; + if !any_text { + self.out.undecodable_ops += 1; + } + if !placed { + self.out.unplaced += 1; + } + } + } + + fn scope(&mut self, data: &[u8], res: &Dict, ctm: Matrix, depth: usize, placed: bool) -> Result<(), ()> { + if depth > MAX_DEPTH { + return Err(()); + } + let doc = self.doc; + let fonts = res_dict(doc, res, b"Font"); + let xobjects = res_dict(doc, res, b"XObject"); + let mut cache: HashMap, Font> = HashMap::new(); + let mut gs = TextGs { ctm, tc: 0.0, tw: 0.0, th: 1.0, tl: 0.0, rise: 0.0, size: 0.0, font: Vec::new() }; + let mut saved: Vec = Vec::new(); + let (mut tm, mut tlm) = (Matrix::IDENTITY, Matrix::IDENTITY); + for op in parse(data).ops { + let o = op.op.as_slice(); + match o { + b"q" => { + if saved.len() >= 4096 { + return Err(()); + } + saved.push(gs.clone()); + } + b"Q" => { + if let Some(g) = saved.pop() { + gs = g; + } + } + b"cm" => { + if let Some(m) = op.nums::<6>() { + gs.ctm = Matrix(m).then(&gs.ctm); + } + } + b"BT" => { + tm = Matrix::IDENTITY; + tlm = Matrix::IDENTITY; + } + b"Tc" => gs.tc = op.num(0).unwrap_or(0.0), + b"Tw" => gs.tw = op.num(0).unwrap_or(0.0), + b"Tz" => gs.th = op.num(0).unwrap_or(100.0) / 100.0, + b"TL" => gs.tl = op.num(0).unwrap_or(0.0), + b"Ts" => gs.rise = op.num(0).unwrap_or(0.0), + b"Tf" => { + gs.font = op.name(0).map(<[u8]>::to_vec).unwrap_or_default(); + gs.size = op.num(1).unwrap_or(0.0); + } + b"Td" | b"TD" => { + if let Some([x, y]) = op.nums::<2>() { + if o == b"TD" { + gs.tl = -y; + } + tlm = Matrix::translate(x, y).then(&tlm); + tm = tlm; + } + } + b"Tm" => { + if let Some(m) = op.nums::<6>() { + tm = Matrix(m); + tlm = tm; + } + } + b"T*" => { + tlm = Matrix::translate(0.0, -gs.tl).then(&tlm); + tm = tlm; + } + b"Tj" | b"TJ" | b"'" | b"\"" => { + if matches!(o, b"'" | b"\"") { + tlm = Matrix::translate(0.0, -gs.tl).then(&tlm); + tm = tlm; + } + if o == b"\"" + && let (Some(aw), Some(ac)) = (op.num(0), op.num(1)) + { + (gs.tw, gs.tc) = (aw, ac); + } + if !cache.contains_key(&gs.font) { + let f = match fonts.get(&gs.font).map(|f| doc.resolve(f)) { + Some(f) => f.as_dict().map_or_else(|| Ok(Font::flat()), |d| Font::new(doc, d, self.ctx))?, + None => Font::flat(), + }; + cache.insert(gs.font.clone(), f); + } + let Some(font) = cache.get(&gs.font) else { continue }; + self.show_text(font, &op, &gs, &mut tm, placed); + } + b"Do" => { + let Some(name) = op.name(0) else { continue }; + let Some(entry) = xobjects.get(name) else { continue }; + if let Object::Stream(s) = &*doc.resolve(entry) + && s.dict.name(b"Subtype") == Some(b"Form") + { + let m = matrix_of(doc, &s.dict)?; + bbox_of(doc, &s.dict)?; + self.stream(s, entry.as_ref(), res, m.then(&gs.ctm), depth + 1, placed)?; + } + } + _ => {} + } + } + // Tiling patterns draw content of their own, repeated where the pattern is used: its + // glyphs can't be placed. + for (_, p) in res_dict(doc, res, b"Pattern").iter() { + if let Object::Stream(s) = &*doc.resolve(p) { + self.stream(s, p.as_ref(), res, Matrix::IDENTITY, depth + 1, false)?; + } + } + Ok(()) + } +} diff --git a/crates/redact/src/verify/geometry.rs b/crates/redact/src/verify/geometry.rs new file mode 100644 index 0000000000..fb69f91351 --- /dev/null +++ b/crates/redact/src/verify/geometry.rs @@ -0,0 +1,500 @@ +//! The non-text content under the regions: images, paths and shadings, and the placement of the +//! glyphs still shown, checked independently of the interpreter. + +use std::collections::HashSet; + +use pdfcraft_content::{Matrix, contains, overlaps, parse}; +use pdfcraft_cos::{Dict, Document, ObjRef, Object, Stream}; + +use super::extract::{appearance_matrix, bbox_of, matrix_of, page_content, res_dict}; +use super::util::{Ctx, image_codec}; +use crate::verify::limits::{MAX_CLIP_SUBPATHS, MAX_DEPTH, MAX_GS_STACK, MAX_HOLES, MAX_PIXELS}; + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub(super) enum Finding { + /// This many glyphs are placed inside the region. + Text(usize), + Image, + Path, + Shading, + Missing, + Unverifiable(&'static str), +} + +#[derive(Clone)] +pub(super) struct Gs { + pub(super) ctm: Matrix, + /// Device-space boxes cut out of the clip (even-odd clips with a hole). + pub(super) holes: Vec<[f64; 4]>, +} + +pub(super) struct Geo<'a> { + pub(super) doc: &'a Document, + pub(super) ctx: &'a Ctx, + pub(super) regions: &'a [[f64; 4]], + pub(super) found: Vec<(usize, Finding)>, + /// What `found` already holds: each finding is recorded once per region, so a page that + /// paints thousands of times cannot grow the report (or the memory) with repeats. + reported: HashSet<(usize, Finding)>, + /// Pattern, soft-mask and Type3 resources already searched for images. + pub(super) searched: HashSet, +} + +/// What still paints inside `regions` on a page of the output (the overlay boxes excepted): +/// images with pixels there, paths and shadings not clipped away from there. +pub(super) fn check_geometry(doc: &Document, page: &Dict, regions: &[[f64; 4]], ctx: &Ctx) -> Vec<(usize, Finding)> { + let unverifiable = |why: &'static str| (0..regions.len()).map(|i| (i, Finding::Unverifiable(why))).collect(); + // The redaction boxes are painted on purpose. + let Ok(data) = page_content(doc, page, ctx, true) else { return unverifiable("the page content can't be read in full") }; + let mut geo = Geo { doc, ctx, regions, found: Vec::new(), reported: HashSet::new(), searched: HashSet::new() }; + let res = page.get(b"Resources").and_then(|r| doc.resolve(r).as_dict().cloned()).unwrap_or_default(); + geo.run(&data, &res, Matrix::IDENTITY, Vec::new(), 0); + geo.appearances(page, &res); + geo.found +} + +impl Geo<'_> { + /// Record a finding for region `i`, once: a page that paints over a region a thousand + /// times is one finding, not a thousand. + fn report(&mut self, i: usize, f: Finding) { + if self.reported.insert((i, f.clone())) { + self.found.push((i, f)); + } + } + + /// The appearances of the annotations still on the page, checked like the page itself: their + /// content is placed with the appearance matrix (ISO 32000-2 §12.5.5) and swept for images, + /// paths and shadings under the regions. An appearance that has content but can't be placed + /// makes every region unverifiable: the redaction refuses such documents, so one in the + /// output means what it paints can't be vouched for. + fn appearances(&mut self, page: &Dict, res: &Dict) { + let doc = self.doc; + for a in crate::annots_of(doc, page) { + // A direct dictionary has no reference of its own; it is checked like an indirect + // one (what its appearance paints does not depend on how it is stored). + let Some(d) = doc.resolve(&a).as_dict().cloned() else { continue }; + // A redaction mark's appearance outlines the marked area on purpose (and applying + // removes the mark whole): it is exempt like the boxes drawn over the regions. + if d.name(b"Subtype") == Some(b"Redact") { + continue; + } + let Some(ap) = d.get(b"AP").and_then(|x| doc.resolve(x).as_dict().cloned()) else { continue }; + let rect = crate::rect_of(doc, d.get(b"Rect")); + for (_, state) in ap.iter() { + let s = doc.resolve(state); + let streams: Vec = match &*s { + Object::Stream(st) => vec![st.clone()], + Object::Dict(sub) => sub + .iter() + .filter_map(|(_, v)| match &*doc.resolve(v) { + Object::Stream(st) => Some(st.clone()), + _ => None, + }) + .collect(), + _ => Vec::new(), + }; + for st in streams { + // A stream shared by several annotations is checked at each annotation's + // own placement: where it lands comes from each `/Rect`, so the first + // placement tells nothing about the others. Repeated decodes spend the + // same bounded budget as everything else. + let Ok(data) = self.ctx.decode(&st) else { + self.unverifiable("an annotation appearance can't be read in full"); + continue; + }; + if data.is_empty() { + continue; + } + let Some(ctm) = rect.as_ref().and_then(|r| appearance_matrix(doc, &st, r)) else { + self.unverifiable("an annotation appearance can't be placed, so what it paints can't be checked"); + continue; + }; + let own = st.dict.get(b"Resources").and_then(|x| doc.resolve(x).as_dict().cloned()); + self.run(&data, own.as_ref().unwrap_or(res), ctm, Vec::new(), 1); + } + } + } + } + + fn touching(&self, b: [f64; 4]) -> Vec { + (0..self.regions.len()).filter(|&i| self.regions.get(i).is_some_and(|r| overlaps(*r, b, 0.0))).collect() + } + + /// Does something painting over `within` reach region `i` through the clip? Only the part of + /// the region the painting could touch has to be cut out. + fn unclipped(&self, i: usize, within: [f64; 4], gs: &Gs) -> bool { + let Some(r) = self.regions.get(i) else { return false }; + let reach = [r[0].max(within[0]), r[1].max(within[1]), r[2].min(within[2]), r[3].min(within[3])]; + !gs.holes.iter().any(|h| contains(*h, reach, 0.01)) + } + + fn run(&mut self, data: &[u8], res: &Dict, ctm: Matrix, holes: Vec<[f64; 4]>, depth: usize) { + if depth > MAX_DEPTH { + for i in 0..self.regions.len() { + self.report(i, Finding::Unverifiable("forms are nested too deep to check")); + } + return; + } + let xobjects = res_dict(self.doc, res, b"XObject"); + self.hidden_images(res, depth); + let mut gs = Gs { ctm, holes }; + let mut stack: Vec = Vec::new(); + // The current path in device space: its subpaths, and the clip operator it carries. + let mut sub: Vec> = Vec::new(); + let mut clip: Option = None; + for op in parse(data).ops { + let o = op.op.as_slice(); + if matches!(o, b"m" | b"l" | b"c" | b"v" | b"y" | b"h" | b"re" | b"W" | b"W*") { + extend_path(&op, &gs.ctm, &mut sub, &mut clip); + continue; + } + if matches!(o, b"S" | b"s" | b"f" | b"F" | b"f*" | b"B" | b"B*" | b"b" | b"b*" | b"n") { + self.paint_path(o, &mut gs, &sub, clip); + sub.clear(); + clip = None; + continue; + } + sub.clear(); + clip = None; + match o { + b"q" => { + // The saved state carries the holes with it, so an unbounded stack of them + // is unbounded memory: past the cap the walk says so instead of growing. + if stack.len() >= MAX_GS_STACK { + return self.unverifiable("the content stacks graphics states too deeply to check"); + } + stack.push(gs.clone()); + } + b"Q" => { + if let Some(g) = stack.pop() { + gs = g; + } + } + b"cm" => { + if let Some(m) = op.nums::<6>() { + gs.ctm = Matrix(m).then(&gs.ctm); + } + } + b"sh" => { + for i in 0..self.regions.len() { + if self.unclipped(i, [f64::MIN, f64::MIN, f64::MAX, f64::MAX], &gs) { + self.report(i, Finding::Shading); + } + } + } + b"BI" => { + for i in self.touching(gs.ctm.bbox([0.0, 0.0, 1.0, 1.0])) { + self.report(i, Finding::Image); + } + } + b"Do" => { + let Some(entry) = op.name(0).and_then(|n| xobjects.get(n)) else { continue }; + let obj = self.doc.resolve(entry); + let Object::Stream(s) = &*obj else { continue }; + match s.dict.name(b"Subtype") { + Some(b"Image") => { + for i in self.touching(gs.ctm.bbox([0.0, 0.0, 1.0, 1.0])) { + let Some(region) = self.regions.get(i) else { continue }; + match image_cleared(self.doc, self.ctx, s, &gs.ctm, *region) { + Ok(true) => {} + Ok(false) => self.report(i, Finding::Image), + Err(()) => self.report(i, Finding::Unverifiable("an image under the region can't be inspected")), + } + } + } + Some(b"Form") => self.form(s, &gs, res, depth), + _ => {} + } + } + _ => {} + } + } + } + + /// A path-painting operator `o` ends the current path: report what it paints over a region, + /// and let an even-odd clip add its holes to `gs`. + fn paint_path(&mut self, o: &[u8], gs: &mut Gs, sub: &[Vec<(f64, f64)>], clip: Option) { + let pts: Vec<(f64, f64)> = sub.iter().flatten().copied().collect(); + let bbox = pts.iter().fold(None, |b: Option<[f64; 4]>, &(u, v)| { + Some(match b { + None => [u, v, u, v], + Some(b) => [b[0].min(u), b[1].min(v), b[2].max(u), b[3].max(v)], + }) + }); + if o != b"n" + && let Some(b) = bbox + { + for i in self.touching(b) { + if self.unclipped(i, b, gs) { + self.report(i, Finding::Path); + } + } + } + // An even-odd clip whose subpaths are all axis-aligned rectangles lets through what an + // odd number of the inner rectangles covers. Where a region lies under the outer box + // and an odd number of inner ones, it is cut out of the clip; under an even number it + // is painted and must be reported. Anything that is not a stock of disjoint rectangles + // (a triangle, a rotated box, overlapping inners, too many of them) cannot be told + // cheaply, so no hole is cut: what the clip lets through is then reported as paint + // (fail closed). This is how redaction itself clips regions out of what it leaves. + if clip != Some(true) { + return; + } + let Some(rects) = sub.iter().map(|s| axis_aligned_rect(s)).collect::>>() else { return }; + if rects.len() < 2 || rects.len() > MAX_CLIP_SUBPATHS { + return; + } + let (outer, inners) = (rects[0], &rects[1..]); + // Inner rectangles must be disjoint, or their overlaps would change the parity the + // reader draws with. + let disjoint = inners.iter().enumerate().all(|(x, a)| inners.iter().skip(x + 1).all(|b| !overlaps(*a, *b, 0.0))); + let regions: Vec<[f64; 4]> = self.regions.to_vec(); + let mut cut = Vec::new(); + for r in regions { + if contains(outer, r, 0.0) && disjoint && inners.iter().filter(|b| contains(**b, r, 0.0)).count() % 2 == 1 { + cut.push(r); + } + } + if gs.holes.len() + cut.len() > MAX_HOLES { + return self.unverifiable("the content cuts clips with too many pieces to check"); + } + gs.holes.extend(cut); + } + + fn unverifiable(&mut self, why: &'static str) { + for i in 0..self.regions.len() { + self.report(i, Finding::Unverifiable(why)); + } + } + + fn form(&mut self, s: &Stream, gs: &Gs, parent: &Dict, depth: usize) { + let (Ok(m), Ok(bbox)) = (matrix_of(self.doc, &s.dict), bbox_of(self.doc, &s.dict)) else { + self.unverifiable("a form under the region has a malformed /Matrix or /BBox"); + return; + }; + let fctm = m.then(&gs.ctm); + if let Some(bb) = bbox + && self.touching(fctm.bbox(bb)).is_empty() + { + return; + } + let Ok(data) = self.ctx.decode(s) else { + self.unverifiable("a form under the region can't be decoded in full"); + return; + }; + let own = s.dict.get(b"Resources").and_then(|r| self.doc.resolve(r).as_dict().cloned()); + self.run(&data, own.as_ref().unwrap_or(parent), fctm, gs.holes.clone(), depth + 1); + } + + /// Images drawn from tiling patterns, soft-mask groups and Type3 glyph procedures can't be + /// placed or cleared by this check: if `res` reaches any, every region is unverifiable. + fn hidden_images(&mut self, res: &Dict, depth: usize) { + let doc = self.doc; + let mut streams: Vec<(Option, Stream)> = Vec::new(); + let take = |o: &Object, streams: &mut Vec<(Option, Stream)>| { + if let Object::Stream(s) = &*doc.resolve(o) { + streams.push((o.as_ref(), s.clone())); + } + }; + for (_, p) in res_dict(doc, res, b"Pattern").iter() { + take(p, &mut streams); + } + for (_, g) in res_dict(doc, res, b"ExtGState").iter() { + if let Some(Object::Dict(sm)) = doc.resolve(g).as_dict().and_then(|g| g.get(b"SMask")).map(|m| (*doc.resolve(m)).clone()).as_ref() + && let Some(group) = sm.get(b"G") + { + take(group, &mut streams); + } + } + for (_, f) in res_dict(doc, res, b"Font").iter() { + let Some(font) = doc.resolve(f).as_dict().cloned() else { continue }; + if font.name(b"Subtype") != Some(b"Type3") { + continue; + } + if let Some(procs) = font.get(b"CharProcs").and_then(|c| doc.resolve(c).as_dict().cloned()) { + for (_, p) in procs.iter() { + take(p, &mut streams); + } + } + } + for (id, s) in streams { + if let Some(r) = id + && !self.searched.insert(r) + { + continue; + } + match self.has_image(&s, res, depth + 1) { + Ok(false) => {} + Ok(true) => self.unverifiable("an image inside a pattern, soft mask or Type3 glyph can't be placed or cleared"), + Err(()) => self.unverifiable("a pattern, soft mask or Type3 glyph can't be read in full"), + } + } + } + + /// Does the content of `s` (forms it draws included) draw an image? + fn has_image(&mut self, s: &Stream, parent: &Dict, depth: usize) -> Result { + if depth > MAX_DEPTH { + return Err(()); + } + let data = self.ctx.decode(s)?; + let own = s.dict.get(b"Resources").and_then(|r| self.doc.resolve(r).as_dict().cloned()); + let res = own.as_ref().unwrap_or(parent); + let xobjects = res_dict(self.doc, res, b"XObject"); + for op in parse(&data).ops { + match op.op.as_slice() { + b"BI" => return Ok(true), + b"Do" => { + let Some(entry) = op.name(0).and_then(|n| xobjects.get(n)) else { continue }; + let Object::Stream(x) = &*self.doc.resolve(entry) else { continue }; + match x.dict.name(b"Subtype") { + Some(b"Image") => return Ok(true), + Some(b"Form") => { + if let Some(r) = entry.as_ref() + && !self.searched.insert(r) + { + continue; + } + if self.has_image(x, res, depth + 1)? { + return Ok(true); + } + } + _ => {} + } + } + _ => {} + } + } + Ok(false) + } +} + +/// Extend the current path (`sub`: its subpaths in device space; `clip`: the clip operator it +/// carries) by one construction operator. +fn extend_path(op: &pdfcraft_content::Op, ctm: &Matrix, sub: &mut Vec>, clip: &mut Option) { + let o = op.op.as_slice(); + let nums: Vec = op.operands.iter().filter_map(Object::as_f64).collect(); + let at = |x: f64, y: f64| ctm.apply(x, y); + match o { + b"m" => { + if let [x, y] = nums[..] { + sub.push(vec![at(x, y)]); + } + } + b"l" | b"c" | b"v" | b"y" => { + if let Some(cur) = sub.last_mut() { + cur.extend(nums.as_chunks::<2>().0.iter().map(|p| at(p[0], p[1]))); + } + } + b"re" => { + if let [x, y, w, h] = nums[..] { + sub.push(vec![at(x, y), at(x + w, y), at(x + w, y + h), at(x, y + h)]); + } + } + b"W" => *clip = Some(false), + b"W*" => *clip = Some(true), + _ => {} + } +} + +/// The axis-aligned rectangle a four-corner subpath draws, in device space: `None` for anything +/// else (open paths, more or fewer corners, skewed or rotated boxes, non-finite coordinates). +fn axis_aligned_rect(pts: &[(f64, f64)]) -> Option<[f64; 4]> { + // `re` leaves its rectangle open: the four corners are the whole subpath. A rectangle drawn + // by hand closes by repeating the first point at the end. Both spell the same four corners. + let pts = match pts { + [a, _, _, _, e] if a == e => &pts[..4], + _ => pts, + }; + let [p0, p1, p2, p3] = pts else { return None }; + let finite = |p: &(f64, f64)| p.0.is_finite() && p.1.is_finite(); + if !finite(p0) || !finite(p1) || !finite(p2) || !finite(p3) { + return None; + } + // The edges alternate between horizontal and vertical, whichever corner they start from and + // whichever way the rectangle is wound. + let flat = |a: &(f64, f64), b: &(f64, f64)| a.1 == b.1 && a.0 != b.0; + let steep = |a: &(f64, f64), b: &(f64, f64)| a.0 == b.0 && a.1 != b.1; + let even = flat(p0, p1) && steep(p1, p2) && flat(p2, p3) && steep(p3, p0); + let odd = steep(p0, p1) && flat(p1, p2) && steep(p2, p3) && flat(p3, p0); + if !even && !odd { + return None; + } + let (x0, x1) = (p0.0.min(p2.0), p0.0.max(p2.0)); + let (y0, y1) = (p0.1.min(p2.1), p0.1.max(p2.1)); + (x0 < x1 && y0 < y1).then_some([x0, y0, x1, y1]) +} + +/// Are the samples of image `s` whose centres fall in `region` all clear (zero; for image masks, +/// the sample that does not paint)? `Err` when the image can't be inspected. +#[allow(clippy::result_unit_err)] +pub(super) fn image_cleared(doc: &Document, ctx: &Ctx, s: &Stream, ctm: &Matrix, region: [f64; 4]) -> Result { + if image_codec(&s.dict) { + // Still encoded pixels: nothing was cleared. + return Ok(false); + } + let (w, h) = (s.dict.int(b"Width").ok_or(())?, s.dict.int(b"Height").ok_or(())?); + if w <= 0 || h <= 0 || (w as u64).saturating_mul(h as u64) > MAX_PIXELS { + return Err(()); + } + let (w, h) = (usize::try_from(w).map_err(|_| ())?, usize::try_from(h).map_err(|_| ())?); + let mask = matches!(s.dict.get(b"ImageMask"), Some(Object::Bool(true))); + let (ncomp, bpc) = if mask { + (1, 1) + } else { + let cs = doc.resolve(s.dict.get(b"ColorSpace").ok_or(())?); + let name = match &*cs { + Object::Name(n) => n.clone(), + Object::Array(a) => a.first().and_then(Object::as_name).ok_or(())?.to_vec(), + _ => return Err(()), + }; + let n = match name.as_slice() { + b"DeviceGray" | b"G" | b"CalGray" | b"Indexed" | b"I" | b"Separation" => 1, + b"DeviceRGB" | b"RGB" | b"CalRGB" | b"Lab" => 3, + b"DeviceCMYK" | b"CMYK" => 4, + // One component per colorant name. + b"DeviceN" => doc.resolve(cs.as_array().and_then(|a| a.get(1)).ok_or(())?).as_array().ok_or(())?.len(), + b"ICCBased" => match &*doc.resolve(cs.as_array().and_then(|a| a.get(1)).ok_or(())?) { + Object::Stream(icc) => usize::try_from(icc.dict.int(b"N").ok_or(())?).map_err(|_| ())?, + _ => return Err(()), + }, + _ => return Err(()), + }; + (n, usize::try_from(s.dict.int(b"BitsPerComponent").ok_or(())?).map_err(|_| ())?) + }; + if !matches!(bpc, 1 | 2 | 4 | 8 | 16) || ncomp == 0 || ncomp > 32 { + return Err(()); + } + let data = ctx.decode(s)?; + let bits = ncomp.checked_mul(bpc).ok_or(())?; + let row = w.checked_mul(bits).ok_or(())?.div_ceil(8); + if data.len() < row.checked_mul(h).ok_or(())? { + return Err(()); + } + let inverted = s.dict.get(b"Decode").is_some_and(|d| { + let d = doc.resolve(d); + d.as_array().and_then(|a| a.first()).is_some_and(|first| doc.resolve(first).as_f64() == Some(1.0)) + }); + let want = u8::from(mask && !inverted); + for j in 0..h { + let v = 1.0 - (j as f64 + 0.5) / h as f64; + // Rows that don't reach the region hold nothing to check. + let ((ax, ay), (bx, by)) = (ctm.apply(0.0, v), ctm.apply(1.0, v)); + if !overlaps(region, [ax.min(bx), ay.min(by) - 0.01, ax.max(bx), ay.max(by) + 0.01], 0.0) { + continue; + } + for i in 0..w { + let (x, y) = ctm.apply((i as f64 + 0.5) / w as f64, v); + if x < region[0] || x > region[2] || y < region[1] || y > region[3] { + continue; + } + let start = j.checked_mul(row).and_then(|v| v.checked_mul(8)).and_then(|v| v.checked_add(i.checked_mul(bits)?)).ok_or(())?; + for bit in start..start.checked_add(bits).ok_or(())? { + let byte = data.get(bit / 8).ok_or(())?; + if (byte >> (7 - bit % 8)) & 1 != want { + return Ok(false); + } + } + } + } + Ok(true) +} diff --git a/crates/redact/src/verify/limits.rs b/crates/redact/src/verify/limits.rs new file mode 100644 index 0000000000..e2972bb5f6 --- /dev/null +++ b/crates/redact/src/verify/limits.rs @@ -0,0 +1,42 @@ +//! Bounds of the redaction proof (`verify`). Hitting one is a failure the proof reports, never +//! a quiet skip: a proof that cannot sweep everything it must is not a passing proof. + +/// Most decoded bytes of one stream the proof reads. +pub(crate) const MAX_STREAM: usize = 64 << 20; +/// Most decoded bytes the whole proof reads, over every stream and every pass. +pub(crate) const MAX_TOTAL_DECODED: u64 = 2 << 30; +/// Most units of linear search work (input, patterns and reported matches) in one proof. +pub(crate) const MAX_WORK: u64 = 4 << 30; +/// Nested attachment levels that are swept (deeper ones are reported as unswept). +pub(crate) const MAX_NESTING: usize = 4; +/// Nesting bound of object walks (deeper is reported, never skipped silently). +pub(crate) const MAX_WALK: usize = 64; +/// Deepest graphics-state stack the paint check follows (saved states carry their clip holes +/// with them, so an unbounded stack is unbounded memory). Deeper is unverifiable. +pub(crate) const MAX_GS_STACK: usize = 4096; +/// Most clip holes one walk may cut (each is kept until the state that made it is restored). +/// More is unverifiable. +pub(crate) const MAX_HOLES: usize = 4096; +/// Most subpaths of one even-odd clip the hole rule examines (it compares them pairwise). +/// More, and no hole is cut: what the clip lets through is reported as paint. +pub(crate) const MAX_CLIP_SUBPATHS: usize = 64; +/// Most decoded bytes of an embedded CMap. +pub(crate) const MAX_CMAP: usize = 4 << 20; +/// Most CMap range steps one font may cost (a range that never grows the map still costs). +pub(crate) const MAX_CMAP_WORK: usize = 4 << 20; +/// Most entries one ToUnicode CMap may define. +pub(crate) const MAX_CMAP_ENTRIES: usize = 1 << 20; +/// Needles shorter than this (in characters) are too common to look for in raw bytes. +pub(crate) const MIN_NEEDLE: usize = 3; +/// Smallest needle looked for in the raw file and in decoded streams. Those hold object syntax, +/// numbers and binary data, where three characters match by chance (`/Length 612`, a +/// `/MediaBox`, pixels); shorter needles are still swept on page text, strings, names, Info, XMP +/// and attachment text. +pub(crate) const MIN_STRUCTURED_NEEDLE: usize = 6; +/// Deepest nesting of forms, patterns and appearance streams the proof's independent walks +/// follow: the bound the interpreter itself walks forms at, so the proof reaches everything a +/// redaction could have removed and the two cannot drift apart. +pub(crate) const MAX_DEPTH: usize = crate::interp::MAX_DEPTH; +/// Most pixels an image may have for the proof to inspect it (larger ones are removed by +/// [`apply`](crate::apply), and the proof's pixel look has the same ceiling: `crate::image`). +pub(crate) const MAX_PIXELS: u64 = crate::image::MAX_PIXELS; diff --git a/crates/redact/src/verify/mod.rs b/crates/redact/src/verify/mod.rs new file mode 100644 index 0000000000..b374f93fdc --- /dev/null +++ b/crates/redact/src/verify/mod.rs @@ -0,0 +1,389 @@ +//! An independent proof that redaction removed what it claimed to remove. +//! +//! [`apply`](crate::apply) cuts glyphs out with its own interpreter, so asking that interpreter +//! whether anything is left would only repeat its own mistakes. The proof therefore works from +//! the other end, and does not trust the interpreter's font metrics or hit test: +//! +//! 1. **Before** applying, [`Snapshot::capture`] records, per redaction region, the glyph runs the +//! interpreter is about to remove (their character codes and the text the font gives them), a +//! digest of each page's content, and the text of the whole document as a separate extractor +//! sees it. The strings the sweep looks for therefore *start* from what the interpreter +//! removed; a glyph it wrongly kept is caught by the geometric check below, not by a needle. +//! 2. **After** applying, [`Snapshot::prove`] serializes the document and sweeps the *output* +//! for those strings on every surface a reader could recover them from: the raw file, every +//! string object, every decoded stream (orphans included), the text of every page (read by a +//! minimal extractor of its own, not the interpreter's font metrics or hit test), the Info +//! dictionary, the XMP packet, embedded files (PDF attachments are swept recursively) and the +//! revision structure. Each string is looked for in every encoding a PDF can hold it in. +//! 3. It places every glyph still shown on the redacted pages with its own text-state tracking +//! and width model (the font's `/Widths`, else a flat guess) and fails a region when the +//! centre of a surviving glyph lies inside it, whatever the interpreter decided. +//! 4. It also re-checks the non-text content under each region — on the page and in the +//! appearance streams of the annotations left on it: no image still carries pixels there, and +//! no path or shading paints there unclipped. +//! +//! The proof fails closed: a survivor, but also anything it could not sweep (page content, a form +//! or an appearance stream, XMP, or an attachment that won't decode in full or can't be opened; a +//! nesting or work limit hit; an appearance with content that can't be placed on its page), makes +//! it fail. The one exception is binary data that redaction +//! never edits: a font program, colour profile, `/Indexed` lookup table, function sample table or +//! Flate/LZW image that is damaged or too long to decode is searched as stored bytes and reported +//! as `raw_only`, not as unswept. A proof that cannot fail is worthless. +//! +//! Work is bounded: every stream is decoded strictly under `limits::MAX_STREAM`, all decodes together +//! under `limits::MAX_TOTAL_DECODED`, and all searching under `limits::MAX_WORK`. A document that needs more +//! is reported as unswept, not skipped (binary data as above excepted). +//! +//! The result is a [`Proof`]: a per-region manifest (page, region, glyph counts, SHA-256 of +//! the page content after) and a verdict per surface. It holds counts and digests, +//! never the redacted text, unless [`ProofOptions::include_plaintext`] asks for it. +//! +//! Limits, stated plainly: text that exists only as pixels of an image is not text; a string +//! that also appears in content that was *not* redacted is counted, over the whole document, on +//! the page text, the raw file and the decoded streams (those hold the kept copy too) and fails +//! on a surplus; names are swept against the matches the input itself had, since a redacted word +//! that is also a `/Subtype` or a `/BaseFont` is structure, not a leftover copy, and a surplus of +//! matches still fails; every other surface is swept with no allowance. A word of six characters or more +//! that also spells a structural name (`/Filter`, `/MediaBox`) can still be reported by the raw-file and +//! decoded-stream surfaces, which search stored bytes and cannot tell a name from a copy: that is +//! over-reporting, never a pass. The raw file and the decoded +//! streams are searched for strings of at least six characters only (shorter ones match object +//! syntax and numbers by chance), skipping the trailer's file identifier and the +//! pixels of an image codec (DCT, JPX, JBIG2, CCITT), which are searched as stored bytes only; shorter strings are still found on the page text, +//! in strings, names, Info, XMP and attachment text. A region without extractable text is reported as such, and as +//! unverifiable when the page draws text this extractor cannot decode. The geometric check +//! places glyphs with the font's `/Widths` only: for a font without them the widths are a guess, +//! so a neighbouring glyph may be reported when the guess puts its centre inside a region. +//! `runs_removed` and `codes_removed` are exact counts of what was cut: a proof that is shared +//! discloses how much text sat under each region (never what it said). + +use std::fmt::Write as _; + +use pdfcraft_cos::ObjRef; + +mod extract; +mod geometry; +pub(crate) mod limits; +#[cfg(test)] +mod search_tests; +mod snapshot; +mod sweep; +mod util; + +pub use snapshot::Snapshot; +/// What the proof established for one region. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Status { + /// What was removed was swept for on every surface and is gone. + Verified, + /// The region held no extractable text: nothing was removed, and that is recorded as such. + VerifiedNoTextInRegion, + /// Neither "text was removed" nor "there was none" can be checked (the page draws text this + /// extractor can't decode, or an image under the region can't be inspected). The proof can't + /// pass while any region is unverifiable. + Unverifiable, + /// A survivor was found, or the mechanical evidence is missing. + Failed, +} + +/// The places a redacted string is looked for. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub enum Surface { + RawBytes, + ObjectStrings, + DecodedStreams, + ExtractedText, + Info, + Xmp, + EmbeddedFiles, + Revisions, +} + +const SURFACES: [Surface; 8] = [ + Surface::RawBytes, + Surface::ObjectStrings, + Surface::DecodedStreams, + Surface::ExtractedText, + Surface::Info, + Surface::Xmp, + Surface::EmbeddedFiles, + Surface::Revisions, +]; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Verdict { + /// Swept, nothing found. + Clean, + /// Swept, a redacted string (or an earlier revision) is still there. + Survivor, + /// Could not be swept; the proof fails. + Unswept, + /// There is nothing of this kind in the document. + Absent, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct SurfaceReport { + pub surface: Surface, + pub verdict: Verdict, + /// Streams, strings, pages or files looked at. + pub scanned: usize, + /// Streams scanned as stored bytes only: image-codec pixels (not text), and font, colour or + /// image streams that could not be decoded in full. + pub raw_only: usize, + pub survivors: usize, + /// Where each survivor sits ("object 12", "page 1", "attachment #1 > object 4"); never the text. + pub locations: Vec, + /// Why the surface could not be swept. + pub problems: Vec, +} + +/// The manifest entry of one redaction region. +#[derive(Clone, Debug, PartialEq)] +pub struct Excision { + /// 0-based page. + pub page: usize, + /// The region in user space. + pub region: [f64; 4], + /// The Redact annotation it came from. + pub mark: ObjRef, + pub status: Status, + pub detail: String, + /// Glyph runs (consecutive glyphs of one string) removed under the region, and their + /// character codes. + pub runs_removed: usize, + pub codes_removed: usize, + /// SHA-256 of the page's decoded content streams, joined, after redaction. There is no digest + /// of the content before: a proof is meant to be shared, and a hash of the original page + /// would let a reader confirm a guess at what it said. + pub page_sha256_after: Option, + /// Glyph-carrying text operators on the page (forms included) before and after. A `TJ` of + /// numbers only (the gap left for a removed string) carries no glyphs and does not count. + pub text_ops_before: Option, + pub text_ops_after: Option, + /// The removed text, only when [`ProofOptions::include_plaintext`] is set. + pub removed_text: Option>, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct Proof { + pub entries: Vec, + pub surfaces: Vec, + /// Located, plain reasons for every failure (never the redacted text). + pub failures: Vec, +} + +impl Proof { + /// Survivors found across all surfaces. + pub fn survivors(&self) -> usize { + self.surfaces.iter().map(|s| s.survivors).sum() + } + + /// Surfaces that could not be swept. + pub fn unswept(&self) -> usize { + self.surfaces.iter().filter(|s| s.verdict == Verdict::Unswept || !s.problems.is_empty()).count() + } + + pub fn unverifiable(&self) -> usize { + self.entries.iter().filter(|e| e.status == Status::Unverifiable).count() + } + + /// Pass only when nothing failed, nothing is unverifiable and no surface went unswept. + pub fn passed(&self) -> bool { + self.failures.is_empty() + && self.survivors() == 0 + && self.unswept() == 0 + && self.entries.iter().all(|e| matches!(e.status, Status::Verified | Status::VerifiedNoTextInRegion)) + } + + /// A human-readable report: the verdict, the manifest and the surfaces. It names places and + /// counts, and the removed text only if it was asked for. + pub fn to_text(&self) -> String { + let mut s = String::new(); + let _ = writeln!(s, "Redaction proof: {}", if self.passed() { "PASS" } else { "FAIL" }); + for f in &self.failures { + let _ = writeln!(s, " failure: {f}"); + } + for (i, e) in self.entries.iter().enumerate() { + let _ = writeln!( + s, + " region {} page {} {:?} {:?}: {} run(s), {} code(s) removed; text ops {:?} -> {:?}; page sha256 after {}{}", + i + 1, + e.page + 1, + e.region, + e.status, + e.runs_removed, + e.codes_removed, + e.text_ops_before, + e.text_ops_after, + e.page_sha256_after.as_deref().unwrap_or("-"), + if e.detail.is_empty() { String::new() } else { format!(" ({})", e.detail) }, + ); + } + for r in &self.surfaces { + let _ = writeln!( + s, + " {:?}: {:?} ({} scanned, {} raw only, {} survivor(s)){}", + r.surface, + r.verdict, + r.scanned, + r.raw_only, + r.survivors, + if r.locations.is_empty() { String::new() } else { format!(" at {}", r.locations.join(", ")) }, + ); + for p in &r.problems { + let _ = writeln!(s, " unswept: {p}"); + } + } + let _ = writeln!(s, " Evidence, not a guarantee: text that exists only as pixels of an image is not text, and is not searched."); + s + } +} + +/// What a proof may contain. +#[derive(Clone, Debug, Default)] +pub struct ProofOptions { + /// Include the removed text in the manifest. Off by default: a proof is meant to be shared, + /// and must not carry what it proves was removed. + pub include_plaintext: bool, +} + +/// Handles for the tests to reach the byte sweep without a whole document. +#[cfg(test)] +pub(crate) mod tests_support { + use pdfcraft_content::parse; + use pdfcraft_cos::{Dict, Object, Stream}; + + use super::extract::{carries_glyphs, parse_cmap}; + use super::limits::{MAX_STREAM, MAX_TOTAL_DECODED, MAX_WORK, MIN_STRUCTURED_NEEDLE}; + use super::sweep::{Needle, add_needle, bytes_counts, bytes_hits, walk_strings}; + use super::util::Ctx; + pub(crate) use super::util::survivor_encodings; + + /// The proof's whole search-work budget (for tests that pre-spend part of it). + pub(crate) fn work_budget() -> u64 { + MAX_WORK + } + + /// The byte sweep of `hay` for a bulk of `count` distinct needles, on a context already + /// `spent` units into its work budget: `Some` (with the needles that occur) while the sweep + /// stays under the budget, `None` once it ran out. + pub(crate) fn bulk_counts(hay: &[u8], text: &str, count: usize, spent: u64) -> Option { + let mut needles: Vec = Vec::new(); + for i in 0..count { + add_needle(&mut needles, false, format!("{text}{i:04}"), Some(i)); + } + let ctx = Ctx::new(Default::default()); + ctx.work.set(spent); + bytes_counts(&needles, hay, MIN_STRUCTURED_NEEDLE, &ctx).map(|hits| hits.len()) + } + + /// The glyph-carrying text-showing operators of a decoded content stream. + pub fn count_text_operators(decoded: &[u8]) -> usize { + parse(decoded).ops.iter().filter(|op| carries_glyphs(op)).count() + } + + /// Does the byte sweep for `text` find it in `hay`? + pub fn found(text: &str, hay: &[u8]) -> bool { + let mut needles: Vec = Vec::new(); + add_needle(&mut needles, false, text.to_string(), Some(0)); + bytes_hits(&needles, hay, 3, &Ctx::new(Default::default())).is_some_and(|h| !h.is_empty()) + } + + /// Does this CMap parse completely (not cut short by a bound)? + pub fn cmap_is_complete(data: &[u8]) -> bool { + parse_cmap(data).is_some() + } + + /// Is an object nested `depth` levels deep walked in full? + pub fn walks_in_full(depth: usize) -> bool { + let mut o = Object::Null; + for _ in 0..depth { + o = Object::Array(vec![o]); + } + walk_strings(&o, 0, &mut |_| {}) + } + + /// Does decoding fail once the cumulative decode budget is used up (and not before)? + pub fn decode_budget_runs_out() -> (bool, bool) { + let ctx = Ctx::new(Default::default()); + let s = Stream::from_raw(Dict::new(), vec![0u8; 100]); + let before = ctx.decode(&s).is_ok(); + let spent = Ctx::with_decode_left(50); + (before, spent.decode(&s).is_err()) + } + + /// Does a decode that fails past the one-stream cap pay for the attempt, so the same bomb + /// cannot be decoded (and fail) again and again for free? + /// `(refused, charged, then refused for want of allowance)`. + pub fn failed_decode_charges_the_budget() -> (bool, bool, bool) { + let big = vec![0u8; MAX_STREAM + (1 << 20)]; + let bomb = Stream::flate(Dict::new(), &big); + drop(big); + let small = Stream::from_raw(Dict::new(), vec![0u8; 100]); + let ctx = Ctx::with_decode_left(u64::try_from(MAX_STREAM + 50).unwrap_or(u64::MAX)); + let refused = ctx.decode(&bomb).is_err(); + let charged = ctx.decoded.get() == MAX_TOTAL_DECODED - 50; + let gone = ctx.decode(&small).is_err(); + (refused, charged, gone) + } + + /// Does the byte search report `None` once the work budget is used up (and not before)? + pub fn search_budget_runs_out() -> (bool, bool) { + let mut needles: Vec = Vec::new(); + add_needle(&mut needles, false, "secret".to_string(), Some(0)); + let ctx = Ctx::new(Default::default()); + let before = bytes_hits(&needles, b"some bytes", 3, &ctx).is_some(); + ctx.work.set(MAX_WORK); + (before, bytes_hits(&needles, b"some bytes", 3, &ctx).is_none()) + } + + /// Is a survivor counted on an earlier page still reported when the work budget runs out on + /// a later one? `(survivor reported, page left unswept)`. + pub fn page_sweep_survivors_survive_the_limit() -> (bool, bool) { + use super::Surface; + use super::sweep::{Acc, sweep_pages}; + let objs = vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R 4 0 R] /Count 2 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 5 0 R /Resources << /Font << /F1 6 0 R >> >> >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 7 0 R /Resources << /Font << /F1 6 0 R >> >> >>".to_vec(), + crate::tests_verify::stream("", b"BT /F1 10 Tf 10 100 Td (SECRETVALUE) Tj ET"), + b"<< /Type /Font /Subtype /TrueType /BaseFont /Arial >>".to_vec(), + crate::tests_verify::stream("", &format!("BT /F1 10 Tf 10 100 Td ({}) Tj ET", "x".repeat(700_000)).into_bytes()), + ]; + let doc = pdfcraft_cos::Document::open(std::sync::Arc::new(crate::tests_verify::pdf_bytes(objs, ""))).unwrap(); + let mut needles = Vec::new(); + add_needle(&mut needles, false, "SECRETVALUE".to_string(), Some(0)); + let mut acc = Acc::new(Ctx::new(Default::default())); + // Page 1 (the survivor's) fits in what is left; page 2 does not. + acc.ctx.work.set(MAX_WORK - 100_000); + sweep_pages(&needles, &doc, 0, "", &mut acc); + let text = acc.reps.get(&Surface::ExtractedText).unwrap(); + (text.survivors > 0, !text.problems.is_empty()) + } + + /// Does a font's `/W` parsing answer to the shared work budget, and not only to its own + /// counter (a form invoked again rebuilds its fonts, and would pay for the same array + /// again)? `(read in full, refused on a spent budget)`. + pub fn font_widths_charge_shares_the_budget() -> (bool, bool) { + use super::extract::extract_page; + let font = b"<< /Type /Font /Subtype /Type0 /BaseFont /X /Encoding /Identity-H /DescendantFonts [<< /Type /Font /Subtype /CIDFontType2 /BaseFont /X /DW 1000 /W [1 65535 250] >>] >>".to_vec(); + let objs = vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Resources << /Font << /F2 5 0 R >> >> >>".to_vec(), + crate::tests_verify::stream("", b"BT /F2 10 Tf 10 100 Td <00410042> Tj ET"), + font, + ]; + let bytes = std::sync::Arc::new(crate::tests_verify::pdf_bytes(objs, "")); + let probe = |spent: u64| -> bool { + let doc = pdfcraft_cos::Document::open(bytes.clone()).unwrap(); + let ctx = Ctx::new(Default::default()); + ctx.work.set(spent); + let page = &pdfcraft_model::pages(&doc)[0]; + extract_page(&doc, &page.dict, &ctx, &[]).is_ok() + }; + (probe(0), !probe(MAX_WORK - 10_000)) + } +} diff --git a/crates/redact/src/verify/search_tests.rs b/crates/redact/src/verify/search_tests.rs new file mode 100644 index 0000000000..4b8d84c243 --- /dev/null +++ b/crates/redact/src/verify/search_tests.rs @@ -0,0 +1,34 @@ +use super::{ + sweep::{add_needle, bytes_counts, bytes_hits}, + util::Ctx, +}; +use std::collections::HashSet; + +#[test] +fn a_thousand_needles_share_one_multimegabyte_scan() { + let mut needles = Vec::new(); + for i in 0..1000 { + add_needle(&mut needles, false, format!("SECRET_{i:04}"), None); + } + let hay = vec![b'z'; 8 << 20]; + assert_eq!(bytes_hits(&needles, &hay, 6, &Ctx::new(HashSet::new())), Some(Vec::new())); + assert_eq!(bytes_counts(&needles, &hay, 6, &Ctx::new(HashSet::new())), Some(Vec::new())); +} + +#[test] +fn overlapping_encodings_count_one_occurrence() { + let mut needles = Vec::new(); + add_needle(&mut needles, false, "SECRET42".into(), None); + let text: Vec = "SECRET42".encode_utf16().flat_map(u16::to_be_bytes).collect(); + let mut hay = vec![0xfe, 0xff]; + hay.extend(text); + assert_eq!(bytes_counts(&needles, &hay, 6, &Ctx::new(HashSet::new())), Some(vec![(0, 1)])); +} + +#[test] +fn match_count_exhaustion_is_incomplete_not_a_truncated_pass() { + let mut needles = Vec::new(); + add_needle(&mut needles, true, "SECRET42".into(), None); + let hay = b"SECRET42 ".repeat((1 << 16) + 1); + assert!(bytes_counts(&needles, &hay, 6, &Ctx::new(HashSet::new())).is_none()); +} diff --git a/crates/redact/src/verify/snapshot.rs b/crates/redact/src/verify/snapshot.rs new file mode 100644 index 0000000000..fe74571bb4 --- /dev/null +++ b/crates/redact/src/verify/snapshot.rs @@ -0,0 +1,532 @@ +//! The snapshot taken before applying (what is about to go) and the proof run on the output. + +use std::collections::{BTreeMap, HashMap, HashSet}; +use std::sync::Arc; + +use pdfcraft_content::Matrix; +use pdfcraft_cos::{Dict, Document, ObjRef, Object, SaveOptions, write_full}; + +use super::extract::extract_page; +use super::geometry::{Finding, check_geometry}; +use super::sweep::{Acc, Needle, add_needle, bytes_hits, sweep_doc, walk_strings}; +use super::util::{Ctx, count, latin1, sha256, squash, words}; +use super::{Excision, Proof, ProofOptions, SURFACES, Status, Surface}; +use crate::interp::{Mode, Removed, Scope, process}; +use crate::verify::limits::MIN_NEEDLE; +use crate::{Mark, RedactError, Report, page_streams}; + +pub(super) struct RegionSnap { + pub(super) page: usize, + pub(super) region: [f64; 4], + pub(super) mark: ObjRef, + pub(super) runs: Vec<(Vec, String)>, + /// The runs of one text operator joined, where it split them: `[(sec) (ret)] TJ` reads as + /// `secret`, which a copy elsewhere in the file holds whole. + pub(super) joined: Vec<(Vec, String)>, + /// The character codes the runs hold (a composite font's codes are several bytes). + pub(super) codes: usize, +} + +pub(super) struct PageSnap { + pub(super) sha: Option, + pub(super) text_ops: Option, + pub(super) undecodable_ops: usize, +} + +/// What redaction is about to remove, recorded before it does. +pub struct Snapshot { + regions: Vec, + pages: BTreeMap, + needles: Vec, + /// Things that made the "before" side unreadable (reported on the text surface). + problems: Vec, + /// The document's page text before redaction (decoded view, code view), for the allowance + /// of needles added later. + src: (String, String), + /// The names of the document (structure redaction never edits), for the name allowance of + /// needles added later. + names: BTreeMap, usize>, + /// Digests of the decoded content of the streams this run created to draw the redaction + /// boxes and labels (see [`Snapshot::set_overlay_streams`]). + overlay: HashSet, +} + +impl std::fmt::Debug for Snapshot { + // The snapshot holds what is about to be redacted: never print it. + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Snapshot").field("regions", &self.regions.len()).field("needles", &self.needles.len()).finish() + } +} + +impl Snapshot { + /// Record what redacting the marks on `pages` (0-based; `None` = all) will remove. Call it + /// before [`apply`](crate::apply); it changes nothing. + pub fn capture(doc: &Document, pages: Option<&[usize]>) -> Result { + let all = crate::marks(doc); + let chosen: Vec<&Mark> = all.iter().filter(|m| pages.is_none_or(|p| p.contains(&m.page))).collect(); + if chosen.is_empty() { + return Err(RedactError::NothingToApply); + } + Snapshot::capture_marks(doc, &chosen) + } + + pub(crate) fn capture_marks(doc: &Document, chosen: &[&Mark]) -> Result { + let mut by_page: Vec = chosen.iter().map(|m| m.page).collect(); + by_page.sort_unstable(); + by_page.dedup(); + let page_list = pdfcraft_model::pages(doc); + let ctx = Ctx::new(HashSet::new()); + let mut regions: Vec = Vec::new(); + let mut pages = BTreeMap::new(); + let mut images = pdfcraft_cos::ImageBudget::new(doc); + // What was cut, each copy once, however many overlapping marks it lay under: the allowance + // for what stays is what the source holds less this. + let (mut cut_text, mut cut_raw) = (String::new(), String::new()); + for &pi in &by_page { + let page = page_list.get(pi).ok_or(RedactError::Unreadable(pi + 1))?; + let page_marks: Vec<&&Mark> = chosen.iter().filter(|m| m.page == pi).collect(); + let rects: Vec<[f64; 4]> = page_marks.iter().flat_map(|m| m.rects.iter().copied()).collect(); + let (_, data) = page_streams(doc, &page.dict, pi)?; + let resources = page.dict.get(b"Resources").and_then(|r| doc.resolve(r).as_dict().cloned()).unwrap_or_default(); + // The interpreter's own removal path, on a scratch copy: what it takes out is what + // has to be gone. + let mut scratch = doc.clone(); + let mut sink = Report::default(); + let removed: Vec = { + let mut scope = Scope::new(&rects, Mode::Apply, &mut sink, pdfcraft_cos::DecodeBudget::page(), &mut images); + scope.removed = Some(Vec::new()); + process(&mut scratch, &mut scope, &data, &resources, Matrix::IDENTITY); + scope.removed.take().unwrap_or_default() + }; + let lines = joined(&removed); + let mut seen = HashSet::new(); + for line in lines.iter().filter(|l| seen.insert((l.op, l.at, l.raw.clone()))) { + cut_text.push('\u{1}'); + cut_text.push_str(&squash(&line.text)); + cut_raw.push('\u{1}'); + cut_raw.push_str(&squash(&latin1(&line.raw))); + } + let before = extract_page(doc, &page.dict, &ctx, &[]); + pages.insert( + pi, + PageSnap { + sha: Some(sha256(&data)), + text_ops: before.as_ref().ok().map(|e| e.glyph_ops), + undecodable_ops: before.as_ref().map_or(0, |e| e.undecodable_ops), + }, + ); + let mut k = 0; + for m in &page_marks { + for r in &m.rects { + let hits: Vec<&Removed> = removed.iter().filter(|x| x.rect == k).collect(); + let runs = hits.iter().map(|x| (x.raw.clone(), x.text.clone())).collect(); + let codes = hits.iter().map(|x| x.codes).sum(); + let joined = lines.iter().filter(|l| l.rect == k && l.pieces > 1).map(|l| (l.raw.clone(), l.text.clone())).collect(); + regions.push(RegionSnap { page: pi, region: *r, mark: m.obj, runs, joined, codes }); + k += 1; + } + } + } + // The document's text as the independent extractor reads it, for the allowance of + // strings that legitimately remain elsewhere. + let (mut src_text, mut src_raw) = (String::new(), String::new()); + let mut problems = Vec::new(); + for (i, p) in page_list.iter().enumerate() { + match extract_page(doc, &p.dict, &ctx, &[]) { + Ok(e) => { + src_text.push_str(&e.text); + src_raw.push_str(&e.raw); + } + Err(()) => { + problems.push(format!("page {} of the source can't be read, so strings left on it can't be told from removed ones", i + 1)) + } + } + } + let mut needles: Vec = Vec::new(); + for (ri, r) in regions.iter().enumerate() { + for (raw, text) in r.runs.iter().chain(&r.joined) { + let raw_s = latin1(raw); + for (raw_view, whole) in [(false, text.trim().to_string()), (true, raw_s)] { + let mut shown: Vec = Vec::new(); + if !squash(&whole).is_empty() { + shown.push(whole.clone()); + shown.extend(words(&whole).into_iter().filter(|w| *w != whole).map(str::to_string)); + } + for s in shown { + add_needle(&mut needles, raw_view, s, Some(ri)); + } + } + } + } + for (src, cut, raw) in [(&src_text, &cut_text, false), (&src_raw, &cut_raw, true)] { + match (super::sweep::key_counts(&needles, src, raw, &ctx), super::sweep::key_counts(&needles, cut, raw, &ctx)) { + (Some(source), Some(removed)) => { + let removed: BTreeMap = removed.into_iter().collect(); + for (i, total) in source { + if let Some(n) = needles.get_mut(i) { + n.allowance = total.saturating_sub(removed.get(&i).copied().unwrap_or(0)); + } + } + } + _ => problems.push("the source text allowances could not be counted in full".into()), + } + } + // The names the removed text legitimately sits in (a redacted word that is also a + // `/Subtype`, a `/Filter`, a `/BaseFont`): names are structure redaction never edits, + // so the output may hold the input's own matches, and only a surplus is a survivor. + let names = all_names(doc); + let name_allow: Vec> = needles.iter().map(|n| count_name_matches(&names, n, &ctx)).collect(); + if name_allow.iter().any(Option::is_none) { + problems.push("the name allowances could not be counted in full, so names are swept with none of them allowed".into()); + } + for (n, allow) in needles.iter_mut().zip(name_allow) { + n.name_allowance = allow.unwrap_or(0); + } + Ok(Snapshot { regions, pages, needles, problems, src: (src_text, src_raw), names, overlay: HashSet::new() }) + } + + /// Tell the proof which streams this run created to draw the redaction boxes and labels + /// (`created`: references into `doc`, the document after `apply`). Call it before + /// [`prove`](Snapshot::prove) / [`prove_saved`](Snapshot::prove_saved). + /// + /// Those streams paint over the regions on purpose and may show label text, so the proof + /// leaves them out of the text sweep and the paint check. They are recognised by the digest + /// of their decoded content (a save renumbers objects, so references would not survive it), + /// and *only* those: a stream that merely carries a `/PCMark /Redaction` key in the input is + /// not exempt. Streams that can't be read here are an error; nothing is then exempted. + pub fn set_overlay_streams(&mut self, doc: &Document, created: &[ObjRef]) -> Result<(), RedactError> { + let ctx = Ctx::new(HashSet::new()); + let mut digests = HashSet::new(); + for r in created { + let obj = doc.try_get(r.num).map_err(|_| RedactError::OverlayUnreadable)?; + let Object::Stream(s) = &*obj else { continue }; + let data = ctx.decode(s).map_err(|_| RedactError::OverlayUnreadable)?; + digests.insert(sha256(&[data])); + } + self.overlay = digests; + Ok(()) + } + + /// Also look for these strings in the output: text that left the document by another route + /// than a page's glyphs (a removed annotation's `/Contents` or `/RC`, a form field's value). + /// Each is searched whole, on every surface. A string that also legitimately remains as page + /// text is counted on the page-content surfaces only (as for the removed glyph runs). + /// Strings that are blank or shorter than the minimum needle are ignored. + pub fn add_removed_strings(&mut self, strings: &[String]) { + for s in strings { + let text = s.trim(); + let key = squash(text); + if key.chars().count() < MIN_NEEDLE { + continue; + } + let known = self.needles.iter().any(|n| !n.raw_view && n.key == key); + // No owner: a survivor fails the proof through its surface; no region carries it. + add_needle(&mut self.needles, false, text.to_string(), None); + if !known { + let ctx = Ctx::new(HashSet::new()); + let matches = self.needles.last().map(|n| count_name_matches(&self.names, n, &ctx)); + if let Some(n) = self.needles.last_mut() { + n.allowance = count(&self.src.0, &n.key); + n.name_allowance = matches.flatten().unwrap_or(0); + } + if matches == Some(None) { + self.problems.push("the name allowances could not be counted in full, so names are swept with none of them allowed".into()); + } + } + } + } + + /// Serialize `doc` the way a save would and sweep the result. The output is re-opened with + /// `password`, the credentials the caller opened the document with (`None` tries the empty + /// password): a save of an encrypted document is itself encrypted, and a proof that could + /// not read its own subject would prove nothing. + pub fn prove(&self, doc: &Document, password: Option<&str>, opts: &ProofOptions) -> Proof { + match write_full(doc, &SaveOptions::default()) { + Ok(bytes) => self.prove_inner(&bytes, Some(doc), password, opts), + Err(e) => { + let mut p = self.prove_inner(&[], None, password, opts); + p.failures.push(format!("the document could not be serialized for the sweep: {e}")); + p + } + } + } + + /// Sweep an already saved file (for example an incrementally saved one, whose earlier + /// revision still holds the original content). The file is opened with `password`, the + /// credentials it is read with. + pub fn prove_saved(&self, bytes: &[u8], password: Option<&str>, opts: &ProofOptions) -> Proof { + self.prove_inner(bytes, None, password, opts) + } + + fn prove_inner(&self, bytes: &[u8], live: Option<&Document>, password: Option<&str>, opts: &ProofOptions) -> Proof { + let mut acc = Acc::new(Ctx::new(self.overlay.clone())); + let mut failures: Vec = Vec::new(); + for p in &self.problems { + acc.problem(Surface::ExtractedText, p.clone()); + } + // Always the caller's credentials, never unconditionally the empty password: a + // document whose user password is not empty must be provable with that password, and + // must fail closed (an unswept output, never a false pass) without it. + let parsed = if bytes.is_empty() { None } else { Document::open_with_password(Arc::new(bytes.to_vec()), password).ok() }; + match &parsed { + Some(out) => sweep_doc(&self.needles, out, bytes, 0, "", &mut acc), + None => { + let why = "the output could not be read back".to_string(); + for s in SURFACES { + acc.problem(s, why.clone()); + } + failures.push(why); + } + } + if live.is_some_and(|d| !d.full_save_required()) { + acc.survivor(Surface::Revisions, "the document would be saved as an incremental update, keeping the original revision".into(), None); + } + + // The non-text content under each region, on the output. + let (findings, after) = match &parsed { + Some(out) => self.check_output(out, &acc.ctx), + None => Default::default(), + }; + let mut entries = Vec::with_capacity(self.regions.len()); + for (i, r) in self.regions.iter().enumerate() { + let (entry, failure) = self.entry(i, r, &findings, &after, &acc.failed_needles, opts); + failures.extend(failure); + entries.push(entry); + } + let surfaces = acc.finish(); + for s in &surfaces { + if s.survivors > 0 { + failures.push(format!("{:?}: {} survivor(s) at {}", s.surface, s.survivors, s.locations.join(", "))); + } + for p in &s.problems { + failures.push(format!("{:?} could not be swept: {p}", s.surface)); + } + } + Proof { entries, surfaces, failures } + } + + /// The non-text content under each region of the output (what still paints or shows there), + /// and per page the digest and glyph-operator count of what `apply` left. + fn check_output(&self, out: &Document, ctx: &Ctx) -> (HashMap>, PagesAfter) { + let mut findings: HashMap> = HashMap::new(); + let mut after: PagesAfter = BTreeMap::new(); + let out_pages = pdfcraft_model::pages(out); + for &pi in self.pages.keys() { + let idx: Vec = (0..self.regions.len()).filter(|&i| self.regions.get(i).is_some_and(|r| r.page == pi)).collect(); + let Some(page) = out_pages.get(pi) else { + for &i in &idx { + findings.entry(i).or_default().push(Finding::Missing); + } + continue; + }; + let rects: Vec<[f64; 4]> = idx.iter().filter_map(|&i| self.regions.get(i).map(|r| r.region)).collect(); + let (sha, ops, shown) = page_after(out, &page.dict, ctx, &rects); + after.insert(pi, (sha, ops)); + match shown { + Some(shown) => { + for (k, n) in shown.into_iter().enumerate() { + if let (Some(&i), true) = (idx.get(k), n > 0) { + findings.entry(i).or_default().push(Finding::Text(n)); + } + } + } + None => { + for &i in &idx { + findings + .entry(i) + .or_default() + .push(Finding::Unverifiable("the page text can't be placed, so what still shows under the region can't be checked")); + } + } + } + for (k, f) in check_geometry(out, &page.dict, &rects, ctx) { + if let Some(&i) = idx.get(k) { + findings.entry(i).or_default().push(f); + } + } + } + (findings, after) + } + + /// The manifest entry of region `i`, and why it failed when it did. + fn entry( + &self, + i: usize, + r: &RegionSnap, + findings: &HashMap>, + after: &PagesAfter, + failed_needles: &HashSet, + opts: &ProofOptions, + ) -> (Excision, Option) { + let mut failure = None; + let snap = self.pages.get(&r.page); + let (sha_after, ops_after) = after.get(&r.page).cloned().unwrap_or((None, None)); + let sha_before = snap.and_then(|s| s.sha.clone()); + let owned: Vec<&Needle> = self.needles.iter().filter(|n| n.owners.contains(&i)).collect(); + let survived = self.needles.iter().enumerate().filter(|(ni, n)| n.owners.contains(&i) && failed_needles.contains(ni)).count(); + let mut detail: Vec = Vec::new(); + let mut status = if r.runs.is_empty() { Status::VerifiedNoTextInRegion } else { Status::Verified }; + let fs = findings.get(&i).map(Vec::as_slice).unwrap_or_default(); + if survived > 0 { + status = Status::Failed; + detail.push(format!("{survived} removed string(s) still found")); + } + for f in fs { + match f { + Finding::Text(n) => detail.push(format!("{n} glyph(s) still show inside the region (placed independently of the redaction)")), + Finding::Image => detail.push("an image still carries pixels under the region".into()), + Finding::Path => detail.push("a path still paints inside the region".into()), + Finding::Shading => detail.push("a shading still paints inside the region".into()), + Finding::Missing => detail.push("the page is missing from the output".into()), + Finding::Unverifiable(why) => detail.push((*why).to_string()), + } + if !matches!(f, Finding::Unverifiable(_)) { + status = Status::Failed; + } else if status != Status::Failed { + status = Status::Unverifiable; + } + } + if sha_before.is_none() || sha_after.is_none() { + status = Status::Failed; + detail.push("the page content can't be read on one side, so it can't be compared".into()); + } else if !r.runs.is_empty() && sha_before == sha_after { + status = Status::Failed; + detail.push("glyphs were removed but the page content is unchanged".into()); + } + if r.runs.is_empty() && snap.is_some_and(|s| s.undecodable_ops > 0) && status != Status::Failed { + status = Status::Unverifiable; + detail.push("the page draws text that can't be decoded, so \"no text in the region\" can't be checked".into()); + } + let ambiguous = owned.iter().filter(|n| n.allowance > 0).count(); + if ambiguous > 0 { + detail.push(format!("{ambiguous} string(s) also remain in retained content; only a surplus over those copies counts")); + } + if matches!(status, Status::Failed | Status::Unverifiable) { + failure = Some(format!("page {} region {:?}: {}", r.page + 1, r.region, detail.join("; "))); + } + let entry = Excision { + page: r.page, + region: r.region, + mark: r.mark, + status, + detail: detail.join("; "), + runs_removed: r.runs.len(), + codes_removed: r.codes, + page_sha256_after: sha_after, + text_ops_before: snap.and_then(|s| s.text_ops), + text_ops_after: ops_after, + removed_text: opts.include_plaintext.then(|| r.runs.iter().map(|(_, t)| t.clone()).collect()), + }; + (entry, failure) + } +} + +/// Per page: the digest of its content in the output and its glyph-operator count. +type PagesAfter = BTreeMap, Option)>; + +/// Every name of the document (the values, wherever an object holds them), with how often +/// each occurs. +fn all_names(doc: &Document) -> BTreeMap, usize> { + let mut names: BTreeMap, usize> = BTreeMap::new(); + for num in doc.object_numbers() { + if let Ok(obj) = doc.try_get(num) { + walk_strings(&obj, 0, &mut |o| { + if let Object::Name(n) = o { + let seen = names.entry(n.clone()).or_default(); + *seen = seen.saturating_add(1); + } + }); + } + } + names +} + +/// How many of `names` carry `needle`, by the same search the sweep runs on a name. `None` +/// when the search work ran out: an unknown allowance must not read as "none allowed", so the +/// caller reports that (and sweeps names with no allowance). +fn count_name_matches(names: &BTreeMap, usize>, needle: &Needle, ctx: &Ctx) -> Option { + if needle.short() { + return Some(0); + } + let mut total = 0usize; + for (name, seen) in names { + if !bytes_hits(std::slice::from_ref(needle), name, MIN_NEEDLE, ctx)?.is_empty() { + total = total.saturating_add(*seen); + } + } + Some(total) +} + +/// The page's content digest and glyph-operator count in the output, and per region the glyphs +/// still placed inside it (`None` when the page's text can't be placed). The digest covers the +/// page's content as `apply` left it, the overlay streams included. +pub(super) fn page_after(doc: &Document, page: &Dict, ctx: &Ctx, regions: &[[f64; 4]]) -> (Option, Option, Option>) { + let sha = page_streams(doc, page, 0).ok().map(|(_, d)| sha256(&d)); + let e = extract_page(doc, page, ctx, regions).ok(); + (sha, e.as_ref().map(|e| e.glyph_ops), e.map(|e| e.shown_in)) +} + +/// One text operator's removed runs under one region, as a reader takes them: joined. +struct Line { + rect: usize, + op: usize, + at: (usize, usize), + raw: Vec, + text: String, + pieces: usize, +} + +/// The runs of `removed` joined by text operator and region, in the order they were shown. +fn joined(removed: &[Removed]) -> Vec { + let mut lines: Vec = Vec::new(); + let mut found: HashMap<(usize, usize), usize> = HashMap::new(); + for r in removed { + let line = found.get(&(r.op, r.rect)).copied().and_then(|i| lines.get_mut(i)); + match line { + Some(l) => { + l.raw.extend_from_slice(&r.raw); + l.text.push_str(&r.text); + l.pieces += 1; + } + None => { + found.insert((r.op, r.rect), lines.len()); + lines.push(Line { rect: r.rect, op: r.op, at: r.at, raw: r.raw.clone(), text: r.text.clone(), pieces: 1 }); + } + } + } + lines +} + +#[cfg(test)] +mod tests { + use super::*; + + fn run(rect: usize, op: usize, place: usize, text: &str) -> Removed { + Removed { rect, raw: text.as_bytes().to_vec(), text: text.to_string(), codes: text.len(), op, at: (place, 0) } + } + + #[test] + fn a_word_split_across_a_tj_array_is_one_line_for_each_region() { + let lines = joined(&[run(0, 1, 0, "sec"), run(1, 1, 0, "sec"), run(0, 1, 1, "ret"), run(1, 1, 1, "ret"), run(0, 2, 0, "other")]); + let shown: Vec<(usize, usize, &str, usize)> = lines.iter().map(|l| (l.rect, l.op, l.text.as_str(), l.pieces)).collect(); + assert_eq!(shown, [(0, 1, "secret", 2), (1, 1, "secret", 2), (0, 2, "other", 1)]); + } + + #[test] + fn overlapping_marks_are_one_copy_and_two_copies_stay_two() { + // Under two marks at once: the same operator, place and bytes, so one copy was cut. + let both = joined(&[run(0, 1, 0, "word"), run(1, 1, 0, "word")]); + let mut seen = HashSet::new(); + assert_eq!(both.iter().filter(|l| seen.insert((l.op, l.at, l.raw.clone()))).count(), 1); + // The same word twice in one array, one under each mark: two copies were cut. + let apart = joined(&[run(0, 1, 0, "word"), run(1, 1, 2, "word")]); + let mut seen = HashSet::new(); + assert_eq!(apart.iter().filter(|l| seen.insert((l.op, l.at, l.raw.clone()))).count(), 2); + } + + #[test] + fn a_spent_decode_budget_is_told_apart() { + assert!(!Ctx::new(HashSet::new()).decode_spent()); + assert!(Ctx::with_decode_left(0).decode_spent()); + } +} diff --git a/crates/redact/src/verify/sweep.rs b/crates/redact/src/verify/sweep.rs new file mode 100644 index 0000000000..73b44c005d --- /dev/null +++ b/crates/redact/src/verify/sweep.rs @@ -0,0 +1,759 @@ +//! Sweeping the serialized output for the removed strings, on every surface a reader could +//! recover them from. + +use std::collections::{BTreeMap, BTreeSet, HashSet}; +use std::sync::Arc; + +use pdfcraft_cos::{Dict, Document, ObjRef, Object, PdfString}; + +use super::extract::extract_page; +use super::util::{Ctx, decode_text, find, image_codec, spellings, squash, survivor_encodings, xml_unescape}; +use super::{SURFACES, Surface, SurfaceReport, Verdict}; +use crate::verify::limits::{MAX_NESTING, MAX_WALK, MIN_NEEDLE, MIN_STRUCTURED_NEEDLE}; + +/// A string the sweep looks for. +pub(super) struct Needle { + /// The code bytes as Latin-1 text (the "raw view" of a page) rather than decoded text. + pub(super) raw_view: bool, + /// As removed, whitespace included (for the byte encodings). + pub(super) shown: String, + /// `shown` without whitespace (for page text, which is compared whitespace-blind). + pub(super) key: String, + /// What the raw file may spell it as. + pub(super) bytes: Vec>, + pub(super) owners: BTreeSet, + /// Occurrences that legitimately remain in content that was not redacted. The surfaces that + /// hold page content (the page text, the raw file, decoded streams) count occurrences over + /// the whole document and fail on a surplus; every other surface (strings, Info, XMP, + /// attachments) is swept with no allowance. + pub(super) allowance: usize, + /// Names of the input that carry this needle (a redacted word that is also a `/Subtype`, a + /// `/Filter`, a `/BaseFont`). Names are structure redaction never edits, so the name surface + /// is swept against the input's own matches: the output may hold as many as the input did, + /// and only a surplus is a survivor. + pub(super) name_allowance: usize, +} + +impl Needle { + pub(super) fn short(&self) -> bool { + self.key.chars().count() < MIN_NEEDLE + } + + /// Long enough to be looked for in bytes (`min`: the shortest the haystack allows). + pub(super) fn at_least(&self, min: usize) -> bool { + self.key.chars().count() >= min + } +} + +pub(super) fn add_needle(set: &mut Vec, raw_view: bool, shown: String, owner: Option) { + let key = squash(&shown); + if key.is_empty() { + return; + } + if let Some(n) = set.iter_mut().find(|n| n.raw_view == raw_view && n.key == key) { + n.owners.extend(owner); + return; + } + let bytes = if raw_view { + // The code bytes themselves (Latin-1 round-trips them). + let raw: Vec = shown.chars().filter_map(|c| u8::try_from(u32::from(c)).ok()).collect(); + spellings([raw]) + } else { + survivor_encodings(&shown) + }; + set.push(Needle { raw_view, shown, key, bytes, owners: owner.into_iter().collect(), allowance: 0, name_allowance: 0 }); +} + +pub(super) struct Acc { + pub(super) reps: BTreeMap, + pub(super) failed_needles: HashSet, + pub(super) ctx: Ctx, +} + +impl Acc { + pub(super) fn new(ctx: Ctx) -> Self { + let mut a = Acc { reps: BTreeMap::new(), failed_needles: HashSet::new(), ctx }; + for s in SURFACES { + a.rep(s); + } + a + } + + pub(super) fn rep(&mut self, surface: Surface) -> &mut SurfaceReport { + self.reps.entry(surface).or_insert_with(|| SurfaceReport { + surface, + verdict: Verdict::Absent, + scanned: 0, + raw_only: 0, + survivors: 0, + locations: Vec::new(), + problems: Vec::new(), + }) + } + + pub(super) fn scanned(&mut self, s: Surface) { + self.rep(s).scanned += 1; + } + + pub(super) fn raw_only(&mut self, s: Surface) { + self.rep(s).raw_only += 1; + } + + pub(super) fn survivor(&mut self, s: Surface, loc: String, needle: Option) { + self.survivors(s, loc, needle, 1); + } + + /// `count` survivors at one place. + pub(super) fn survivors(&mut self, s: Surface, loc: String, needle: Option, count: usize) { + if let Some(n) = needle { + self.failed_needles.insert(n); + } + let r = self.rep(s); + r.survivors = r.survivors.saturating_add(count); + if !r.locations.contains(&loc) && r.locations.len() < 64 { + r.locations.push(loc); + } + } + + pub(super) fn problem(&mut self, s: Surface, msg: String) { + let r = self.rep(s); + if r.problems.len() < 64 { + r.problems.push(msg); + } + } + + /// The needles (at least `min` long) found in `hay`; running out of search work is a + /// problem of `surface`. + pub(super) fn bytes(&mut self, surface: Surface, needles: &[Needle], hay: &[u8], min: usize) -> Vec { + bytes_hits(needles, hay, min, &self.ctx).unwrap_or_else(|| { + self.problem(surface, "the search work limit was reached".into()); + Vec::new() + }) + } + + /// How often each needle (at least `min` long) occurs in `hay`, as `(needle, count)`. + pub(super) fn counts(&mut self, surface: Surface, needles: &[Needle], hay: &[u8], min: usize) -> Vec<(usize, usize)> { + bytes_counts(needles, hay, min, &self.ctx).unwrap_or_else(|| { + self.problem(surface, "the search work limit was reached".into()); + Vec::new() + }) + } + + /// Survivors and problems so far, over all surfaces. + pub(super) fn totals(&self) -> (usize, usize) { + (self.reps.values().map(|r| r.survivors).sum(), self.reps.values().map(|r| r.problems.len()).sum()) + } + + pub(super) fn finish(mut self) -> Vec { + for r in self.reps.values_mut() { + r.verdict = if r.survivors > 0 { + Verdict::Survivor + } else if !r.problems.is_empty() { + Verdict::Unswept + } else if r.scanned + r.raw_only == 0 { + Verdict::Absent + } else { + Verdict::Clean + }; + } + self.reps.into_values().collect() + } +} + +/// Occurrences of needles on one surface, per place, so that what legitimately remains +/// (a needle's allowance) is taken off the document-wide total once, not once per place. +#[derive(Default)] +pub(super) struct Tally(BTreeMap>); + +impl Tally { + pub(super) fn add(&mut self, needle: usize, loc: &str, count: usize) { + if count > 0 { + self.0.entry(needle).or_default().push((loc.to_string(), count)); + } + } + + /// Report what is left of each needle's occurrences after its allowance (none inside an + /// attachment, where nothing legitimately remains). + pub(super) fn settle(self, surface: Surface, needles: &[Needle], depth: usize, acc: &mut Acc) { + self.settle_with(surface, needles, depth, acc, |n| n.allowance) + } + + /// The name surface: the input's own name matches are the allowance (redaction never edits + /// names), so what is left is a surplus of matches over the input. + pub(super) fn settle_names(self, surface: Surface, needles: &[Needle], depth: usize, acc: &mut Acc) { + self.settle_with(surface, needles, depth, acc, |n| n.name_allowance) + } + + fn settle_with(self, surface: Surface, needles: &[Needle], depth: usize, acc: &mut Acc, allow_of: impl Fn(&Needle) -> usize) { + for (ni, places) in self.0 { + let mut allow = if depth == 0 { needles.get(ni).map_or(0, &allow_of) } else { 0 }; + for (loc, count) in places { + let used = count.min(allow); + allow -= used; + if count > used { + acc.survivors(surface, loc, Some(ni), count - used); + } + } + } + } +} + +/// Search all encodings in one bounded pass, including overlapping matches. +fn pattern_matches<'a>( + patterns: impl IntoIterator, + hay: &[u8], + ctx: &Ctx, + mut found: impl FnMut(usize, usize, usize) -> bool, +) -> Option<()> { + const MAX_PATTERNS: usize = 100_000; + const MAX_PATTERN_BYTES: usize = 16 << 20; + const MAX_MATCHES: usize = 1 << 20; + if !ctx.charge(hay.len()) { + return None; + } + let mut map: BTreeMap, Vec> = BTreeMap::new(); + let (mut bytes, mut links) = (0usize, 0usize); + for (owner, pattern) in patterns { + if pattern.is_empty() { + continue; + } + if !ctx.charge(pattern.len().saturating_add(1)) { + return None; + } + if !map.contains_key(pattern) { + bytes = bytes.checked_add(pattern.len())?; + if bytes > MAX_PATTERN_BYTES || map.len() >= MAX_PATTERNS { + return None; + } + } + let owners = map.entry(pattern.to_vec()).or_default(); + if owners.last() != Some(&owner) { + links = links.checked_add(1)?; + if links > MAX_PATTERNS { + return None; + } + owners.push(owner); + } + } + if map.is_empty() { + return Some(()); + } + let matcher = aho_corasick::AhoCorasick::new(map.keys()).ok()?; + let owners: Vec> = map.into_values().collect(); + let mut matches = 0usize; + for hit in matcher.find_overlapping_iter(hay) { + for &owner in owners.get(hit.pattern().as_usize())? { + matches = matches.checked_add(1)?; + if matches > MAX_MATCHES || !ctx.charge(1) || !found(owner, hit.start(), hit.end()) { + return None; + } + } + } + Some(()) +} + +/// Count normalized text keys in one pass, with the same non-overlap rule as `str::matches`. +pub(super) fn key_counts(needles: &[Needle], hay: &str, raw: bool, ctx: &Ctx) -> Option> { + key_counts_from(needles, hay, raw, ctx, 0) +} + +/// [`key_counts`] of the needles at least `min` long, so a one-letter needle can't use up the match +/// limits of a search it takes no part in. +fn key_counts_from(needles: &[Needle], hay: &str, raw: bool, ctx: &Ctx, min: usize) -> Option> { + let patterns = needles.iter().enumerate().filter(|(_, n)| n.raw_view == raw && n.at_least(min)).map(|(i, n)| (i, n.key.as_bytes())); + let mut spans: BTreeMap> = BTreeMap::new(); + pattern_matches(patterns, hay.as_bytes(), ctx, |i, a, b| { + let hits = spans.entry(i).or_default(); + if hits.len() >= MAX_COUNTED { + return false; + } + hits.push((a, b)); + true + })?; + Some( + spans + .into_iter() + .map(|(i, mut hits)| { + hits.sort_unstable(); + let (mut count, mut end) = (0, 0); + for (a, b) in hits { + if a >= end { + count += 1; + end = b; + } + } + (i, count) + }) + .collect(), + ) +} + +pub(super) fn bytes_hits(needles: &[Needle], hay: &[u8], min: usize, ctx: &Ctx) -> Option> { + let patterns = needles.iter().enumerate().filter(|(_, n)| n.at_least(min)).flat_map(|(i, n)| n.bytes.iter().map(move |b| (i, b.as_slice()))); + let mut found = BTreeSet::new(); + pattern_matches(patterns, hay, ctx, |i, _, _| { + found.insert(i); + true + })?; + Some(found.into_iter().collect()) +} + +/// A larger count is incomplete evidence, never a silently truncated count. +pub(super) const MAX_COUNTED: usize = 1 << 16; + +pub(super) fn bytes_counts(needles: &[Needle], hay: &[u8], min: usize, ctx: &Ctx) -> Option> { + let patterns = needles.iter().enumerate().filter(|(_, n)| n.at_least(min)).flat_map(|(i, n)| n.bytes.iter().map(move |b| (i, b.as_slice()))); + let mut spans: BTreeMap> = BTreeMap::new(); + pattern_matches(patterns, hay, ctx, |i, a, b| { + let hits = spans.entry(i).or_default(); + if hits.len() >= MAX_COUNTED { + return false; + } + hits.push((a, b)); + true + })?; + Some( + spans + .into_iter() + .map(|(i, mut spans)| { + spans.sort_unstable(); + spans.dedup(); + let (mut count, mut end) = (0usize, 0usize); + for (a, b) in spans { + if count == 0 || a >= end { + count += 1; + } + end = end.max(b); + } + (i, count) + }) + .collect(), + ) +} + +pub(super) fn string_hits(needles: &[Needle], value: &PdfString, ctx: &Ctx) -> Option> { + let text = value.to_text(); + let key = squash(&text); + if !ctx.charge(value.bytes.len().saturating_add(key.len())) { + return None; + } + let mut found: BTreeSet = bytes_hits(needles, &value.bytes, MIN_NEEDLE, ctx)?.into_iter().collect(); + let patterns = needles.iter().enumerate().filter(|(_, n)| !n.raw_view && !n.short()).map(|(i, n)| (i, n.key.as_bytes())); + pattern_matches(patterns, key.as_bytes(), ctx, |i, _, _| { + found.insert(i); + true + })?; + let tokens: BTreeSet<&str> = text.split(|c: char| !c.is_alphanumeric()).filter(|s| !s.is_empty()).collect(); + for (i, needle) in needles.iter().enumerate().filter(|(_, n)| n.short() && n.allowance == 0 && !n.raw_view) { + if !ctx.charge(needle.shown.len().saturating_add(1)) { + return None; + } + if tokens.contains(needle.shown.as_str()) { + found.insert(i); + } + } + Some(found.into_iter().collect()) +} + +/// Streams whose data is binary rather than text: images, embedded font programs, colour +/// profiles, cross-reference and object streams (the objects in the latter are swept as +/// objects). Their decoded data is swept like any other stream's; the pixels of an image codec +/// stay undecoded (text that exists only as pixels is not text), and one that won't decode (or is +/// past the cap) is searched as stored bytes only. A short needle "found" in stored bytes is +/// chance. +pub(super) fn binary_stream(d: &Dict) -> bool { + let subtype = d.name(b"Subtype"); + subtype == Some(b"Image".as_slice()) + || matches!(d.name(b"Type"), Some(b"XRef" | b"ObjStm")) + || matches!(subtype, Some(b"Type1C" | b"CIDFontType0C" | b"OpenType")) + || [b"Length1".as_slice(), b"Length2", b"Length3"].iter().any(|k| d.contains(k)) + || (d.contains(b"N") && !d.contains(b"Type") && subtype.is_none()) +} + +/// What a file identifier (`/ID` of the trailer) is spelled as in the file. +pub(super) fn id_spellings(doc: &Document) -> Vec> { + let Some(Object::Array(ids)) = doc.trailer().get(b"ID").map(|o| doc.resolve(o)).as_deref().cloned() else { return Vec::new() }; + ids.iter().filter_map(Object::as_string).flat_map(|s| spellings([s.bytes.to_vec()])).collect() +} + +/// Visit `o` and everything nested in it. False when something lay deeper than [`MAX_WALK`] +/// and was not visited: the caller reports that as unswept. +pub(super) fn walk_strings(o: &Object, depth: usize, f: &mut impl FnMut(&Object)) -> bool { + if depth > MAX_WALK { + return false; + } + f(o); + // Every child is visited even after one is too deep: nothing is skipped silently. + let mut ok = true; + match o { + Object::Array(a) => a.iter().for_each(|x| ok &= walk_strings(x, depth + 1, f)), + Object::Dict(d) => d.iter().for_each(|(_, v)| ok &= walk_strings(v, depth + 1, f)), + Object::Stream(s) => s.dict.iter().for_each(|(_, v)| ok &= walk_strings(v, depth + 1, f)), + _ => {} + } + ok +} + +/// Sweep a document (the output, or an attachment) on every surface. +pub(super) fn sweep_doc(needles: &[Needle], doc: &Document, bytes: &[u8], depth: usize, tag: &str, acc: &mut Acc) { + let (embedded, overlay_raw) = sweep_objects(needles, doc, depth, tag, acc); + sweep_file(needles, doc, bytes, overlay_raw, depth, tag, acc); + sweep_pages(needles, doc, depth, tag, acc); + sweep_info(needles, doc, tag, acc); + sweep_xmp(needles, doc, tag, acc); + + // Revision structure: one revision, no earlier content kept. + acc.scanned(Surface::Revisions); + let revs = doc.revisions().len(); + if revs > 1 || doc.trailer().contains(b"Prev") { + acc.survivor(Surface::Revisions, format!("{tag}{revs} revisions: earlier ones keep the original content"), None); + } + + sweep_attachments(needles, doc, &embedded, depth, tag, acc); +} + +/// Every object: strings, names and stream data (orphans included). Returns the embedded file +/// streams found, and the raw bytes of the overlay streams (to leave out of the raw sweep). +fn sweep_objects(needles: &[Needle], doc: &Document, depth: usize, tag: &str, acc: &mut Acc) -> (BTreeSet, Vec>) { + let mut embedded: BTreeSet = BTreeSet::new(); + let mut overlay_raw: Vec> = Vec::new(); + let mut in_streams = Tally::default(); + let mut in_names = Tally::default(); + let lookup = lookup_streams(doc); + for num in doc.object_numbers() { + let loc = format!("{tag}object {num}"); + let obj = match doc.try_get(num) { + Ok(o) => o, + Err(_) => { + acc.problem(Surface::ObjectStrings, format!("{loc} can't be read")); + continue; + } + }; + let mut hits: BTreeSet = BTreeSet::new(); + let mut spent = false; + let ctx = &acc.ctx; + // The file identifier is random bytes, never text: a short needle must not "read" it. + let without_id = match &*obj { + Object::Stream(s) if s.dict.name(b"Type") == Some(b"XRef") => { + let mut d = s.dict.clone(); + d.remove(b"ID"); + Some(Object::Dict(d)) + } + _ => None, + }; + let complete = walk_strings(without_id.as_ref().unwrap_or(&obj), 0, &mut |o| match o { + Object::String(s) => match string_hits(needles, s, ctx) { + Some(h) => hits.extend(h), + None => spent = true, + }, + Object::Name(n) => match bytes_hits(needles, n, MIN_NEEDLE, ctx) { + // A name the input had its own matches in is structure, not a leftover copy: + // the matches are tallied and settled against the input's count. + Some(h) => h.into_iter().for_each(|ni| in_names.add(ni, &loc, 1)), + None => spent = true, + }, + _ => {} + }); + if !complete { + acc.problem(Surface::ObjectStrings, format!("{loc} is nested too deeply to sweep in full")); + } + if spent { + acc.problem(Surface::ObjectStrings, "the search work limit was reached".into()); + } + acc.scanned(Surface::ObjectStrings); + for n in hits { + acc.survivor(Surface::ObjectStrings, loc.clone(), Some(n)); + } + // File specifications, wherever they sit (a name tree keeps them as separate objects, + // annotations and collections may inline them). + walk_strings(&obj, 0, &mut |o| { + if let Object::Dict(d) = o + && let Some(Object::Dict(ef)) = d.get(b"EF") + { + embedded.extend(ef.iter().filter_map(|(_, v)| v.as_ref()).map(|r| r.num)); + } + }); + if let Object::Stream(s) = &*obj { + if s.dict.name(b"Type") == Some(b"EmbeddedFile") { + embedded.insert(num); + } + acc.scanned(Surface::DecodedStreams); + if binary_stream(&s.dict) && image_codec(&s.dict) { + // The pixels of an image codec are still encoded and are not text: their stored + // bytes are searched for whole strings only. + acc.raw_only(Surface::DecodedStreams); + for n in acc.bytes(Surface::DecodedStreams, needles, &s.raw, MIN_STRUCTURED_NEEDLE) { + acc.survivor(Surface::DecodedStreams, format!("{loc} (raw)"), Some(n)); + } + } else if binary_stream(&s.dict) { + // A compressed copy of the removed text can hide in a font program or colour + // profile (with /Length1 in its dict, its stored bytes hold no plaintext): the + // stored bytes and the decoded data are both searched. Such a stream that won't + // decode (damaged, or past the cap) is searched as stored bytes only and counted + // as raw only: it holds no page content, and redaction never touches it. A + // cross-reference or object stream that won't decode is a problem of the proof. + for n in acc.bytes(Surface::DecodedStreams, needles, &s.raw, MIN_STRUCTURED_NEEDLE) { + acc.survivor(Surface::DecodedStreams, format!("{loc} (raw)"), Some(n)); + } + match acc.ctx.decode(s) { + Ok(d) => { + let loc = format!("{loc} (decoded)"); + for (n, c) in acc.counts(Surface::DecodedStreams, needles, &d, MIN_STRUCTURED_NEEDLE) { + in_streams.add(n, &loc, c); + } + } + Err(()) if acc.ctx.decode_spent() || matches!(s.dict.name(b"Type"), Some(b"XRef" | b"ObjStm")) => { + acc.problem(Surface::DecodedStreams, format!("{loc} can't be decoded in full")); + } + Err(()) => acc.raw_only(Surface::DecodedStreams), + } + } else { + match acc.ctx.decode(s) { + // The boxes and labels this run drew are not text to look for. + Ok(d) if acc.ctx.is_overlay(&d) => overlay_raw.push(s.raw.to_vec()), + Ok(d) => { + let loc = format!("{loc} (decoded)"); + for (n, c) in acc.counts(Surface::DecodedStreams, needles, &d, MIN_STRUCTURED_NEEDLE) { + in_streams.add(n, &loc, c); + } + } + // Colour-space lookup data and function samples are data tables, not content. + Err(()) if !acc.ctx.decode_spent() && (lookup.contains(&num) || s.dict.contains(b"FunctionType")) => { + acc.raw_only(Surface::DecodedStreams); + for n in acc.bytes(Surface::DecodedStreams, needles, &s.raw, MIN_STRUCTURED_NEEDLE) { + acc.survivor(Surface::DecodedStreams, format!("{loc} (raw)"), Some(n)); + } + } + Err(()) => acc.problem(Surface::DecodedStreams, format!("{loc} can't be decoded in full")), + } + } + } + } + in_names.settle_names(Surface::ObjectStrings, needles, depth, acc); + in_streams.settle(Surface::DecodedStreams, needles, depth, acc); + (embedded, overlay_raw) +} + +/// The streams that hold the lookup table of an `/Indexed` colour space (object numbers). +fn lookup_streams(doc: &Document) -> BTreeSet { + let mut out = BTreeSet::new(); + for num in doc.object_numbers() { + let Ok(obj) = doc.try_get(num) else { continue }; + walk_strings(&obj, 0, &mut |o| { + if let Object::Array(a) = o + && matches!(a.first(), Some(Object::Name(n)) if n == b"Indexed" || n == b"I") + && let Some(Object::Ref(r)) = a.get(3) + { + out.insert(r.num); + } + }); + } + out +} + +/// The raw bytes of the whole file, the overlay streams (stored as they are, or not at all) and +/// the file identifier (random bytes, regenerated on every write) aside. +fn sweep_file(needles: &[Needle], doc: &Document, bytes: &[u8], overlay_raw: Vec>, depth: usize, tag: &str, acc: &mut Acc) { + acc.scanned(Surface::RawBytes); + let mut mask = overlay_raw; + mask.extend(id_spellings(doc)); + let masked = mask_all(bytes, &mask, &acc.ctx); + let mut in_file = Tally::default(); + for (n, c) in acc.counts(Surface::RawBytes, needles, masked.as_deref().unwrap_or(bytes), MIN_STRUCTURED_NEEDLE) { + in_file.add(n, &format!("{tag}file"), c); + } + in_file.settle(Surface::RawBytes, needles, depth, acc); + if masked.is_none() && !mask.is_empty() { + acc.problem(Surface::RawBytes, "the search work limit was reached".into()); + } +} + +/// Embedded files, PDF attachments swept in full. +fn sweep_attachments(needles: &[Needle], doc: &Document, embedded: &BTreeSet, depth: usize, tag: &str, acc: &mut Acc) { + for &num in embedded { + let loc = format!("{tag}attachment object {num}"); + acc.scanned(Surface::EmbeddedFiles); + let obj = doc.get(ObjRef::new(num, doc.generation(num))); + let Object::Stream(s) = &*obj else { + acc.problem(Surface::EmbeddedFiles, format!("{loc} is not a stream")); + continue; + }; + let payload = match acc.ctx.decode(s) { + Ok(p) => p, + Err(()) => { + acc.problem(Surface::EmbeddedFiles, format!("{loc} can't be decoded in full")); + continue; + } + }; + sweep_payload(needles, &payload, depth, &loc, acc); + } +} + +/// `bytes` with every occurrence of `pats` zeroed (`None` when the work limit ran out, or +/// there is nothing to mask). +pub(super) fn mask_all(bytes: &[u8], pats: &[Vec], ctx: &Ctx) -> Option> { + if pats.iter().all(Vec::is_empty) { + return None; + } + let mut out = bytes.to_vec(); + for p in pats.iter().filter(|p| !p.is_empty()) { + if !ctx.charge(bytes.len()) { + return None; + } + let mut at = 0; + while let Some(w) = bytes.get(at..at + p.len()) { + if w == p.as_slice() { + if let Some(z) = out.get_mut(at..at + p.len()) { + z.fill(0); + } + at += p.len(); + } else { + at += 1; + } + } + } + Some(out) +} + +/// One attachment: a PDF is swept as a document, text is searched, containers are reported as +/// unswept (their content can't be seen), images are searched as raw bytes only. +pub(super) fn sweep_payload(needles: &[Needle], payload: &[u8], depth: usize, loc: &str, acc: &mut Acc) { + let head = payload.get(..1024).unwrap_or(payload); + let tag = format!("{loc} > "); + if find(head, b"%PDF-") { + if depth >= MAX_NESTING { + acc.problem(Surface::EmbeddedFiles, format!("{loc} is nested too deep to sweep")); + return; + } + match Document::open(Arc::new(payload.to_vec())) { + Ok(inner) => { + let before = acc.totals(); + sweep_doc(needles, &inner, payload, depth + 1, &tag, acc); + // What the attachment's own sweep found counts against the attachment too. + let after = acc.totals(); + if after.0 > before.0 { + acc.survivor(Surface::EmbeddedFiles, loc.to_string(), None); + } + if after.1 > before.1 { + acc.problem(Surface::EmbeddedFiles, format!("{loc} could not be swept in full")); + } + } + Err(_) => acc.problem(Surface::EmbeddedFiles, format!("{loc} is a PDF that can't be opened")), + } + return; + } + let starts = |sig: &[u8]| payload.starts_with(sig); + let container = starts(b"PK\x03\x04") + || starts(b"PK\x05\x06") + || starts(&[0x1F, 0x8B]) + || starts(b"BZh") + || starts(&[0xFD, b'7', b'z', b'X', b'Z', 0]) + || starts(&[b'7', b'z', 0xBC, 0xAF, 0x27, 0x1C]) + || starts(b"Rar!") + || starts(&[0xD0, 0xCF, 0x11, 0xE0]) + || payload.get(257..262) == Some(b"ustar"); + if container { + acc.problem(Surface::EmbeddedFiles, format!("{loc} is an archive or container that can't be opened")); + return; + } + let image = starts(&[0x89, b'P', b'N', b'G']) || starts(&[0xFF, 0xD8, 0xFF]) || starts(b"GIF8"); + if image { + acc.raw_only(Surface::EmbeddedFiles); + } + let min = if image { MIN_STRUCTURED_NEEDLE } else { MIN_NEEDLE }; + for n in acc.bytes(Surface::EmbeddedFiles, needles, payload, min) { + acc.survivor(Surface::EmbeddedFiles, loc.to_string(), Some(n)); + } + if !image { + let key = squash(&decode_text(payload)); + match key_counts_from(needles, &key, false, &acc.ctx, MIN_NEEDLE) { + Some(counts) => { + for (i, count) in counts { + if count > 0 { + acc.survivor(Surface::EmbeddedFiles, loc.to_string(), Some(i)); + } + } + } + None => acc.problem(Surface::EmbeddedFiles, format!("{loc} could not be searched in full")), + } + } +} + +pub(super) fn sweep_pages(needles: &[Needle], doc: &Document, depth: usize, tag: &str, acc: &mut Acc) { + let mut shown = Tally::default(); + 'pages: for (i, p) in pdfcraft_model::pages(doc).iter().enumerate() { + acc.scanned(Surface::ExtractedText); + let loc = format!("{tag}page {}", i + 1); + let Ok(e) = extract_page(doc, &p.dict, &acc.ctx, &[]) else { + acc.problem(Surface::ExtractedText, format!("{loc} can't be read")); + continue; + }; + for (hay, raw) in [(&e.text, false), (&e.raw, true)] { + let Some(counts) = key_counts(needles, hay, raw, &acc.ctx) else { + acc.problem(Surface::ExtractedText, "the search work limit was reached".into()); + break 'pages; + }; + for (ni, count) in counts { + shown.add(ni, &loc, count); + } + } + } + // A surplus over what legitimately remains anywhere in the document is a survivor. The + // counts made before the work ran out are settled too: a survivor already seen is + // reported where it sits, not hidden behind the limit. + shown.settle(Surface::ExtractedText, needles, depth, acc); +} + +pub(super) fn sweep_info(needles: &[Needle], doc: &Document, tag: &str, acc: &mut Acc) { + let Some(info) = doc.trailer().get(b"Info").map(|i| doc.resolve(i)) else { return }; + let Some(d) = info.as_dict() else { + acc.problem(Surface::Info, format!("{tag}Info is not a dictionary")); + return; + }; + for (k, v) in d.iter() { + acc.scanned(Surface::Info); + let v = doc.resolve(v); + let mut hits: BTreeSet = BTreeSet::new(); + let mut spent = false; + let ctx = &acc.ctx; + let complete = walk_strings(&v, 0, &mut |o| { + if let Object::String(s) = o { + match string_hits(needles, s, ctx) { + Some(h) => hits.extend(h), + None => spent = true, + } + } + }); + if !complete || spent { + acc.problem(Surface::Info, format!("{tag}Info /{} can't be swept in full", String::from_utf8_lossy(k))); + } + for n in hits { + acc.survivor(Surface::Info, format!("{tag}Info /{}", String::from_utf8_lossy(k)), Some(n)); + } + } +} + +pub(super) fn sweep_xmp(needles: &[Needle], doc: &Document, tag: &str, acc: &mut Acc) { + let Some(root) = doc.root() else { return }; + let Some(meta) = doc.get(root).as_dict().and_then(|c| c.get(b"Metadata")).map(|m| doc.resolve(m)) else { return }; + acc.scanned(Surface::Xmp); + let loc = format!("{tag}XMP metadata"); + let Object::Stream(s) = &*meta else { + acc.problem(Surface::Xmp, format!("{loc} is not a stream")); + return; + }; + let Ok(data) = acc.ctx.decode(s) else { + acc.problem(Surface::Xmp, format!("{loc} can't be decoded in full")); + return; + }; + let mut hits: BTreeSet = acc.bytes(Surface::Xmp, needles, &data, MIN_NEEDLE).into_iter().collect(); + let key = squash(&xml_unescape(&decode_text(&data))); + for (i, n) in needles.iter().enumerate() { + if n.at_least(MIN_NEEDLE) && !n.raw_view && key.contains(&n.key) { + hits.insert(i); + } + } + for n in hits { + acc.survivor(Surface::Xmp, loc.clone(), Some(n)); + } +} diff --git a/crates/redact/src/verify/util.rs b/crates/redact/src/verify/util.rs new file mode 100644 index 0000000000..77c4309cdd --- /dev/null +++ b/crates/redact/src/verify/util.rs @@ -0,0 +1,301 @@ +//! Shared helpers of the proof: what one proof may spend, text and byte search, and the +//! spellings a string can have in a PDF. + +use std::cell::Cell; +use std::collections::HashSet; +use std::fmt::Write as _; +use std::sync::Arc; + +use pdfcraft_cos::{DecodeBudget, Dict, Document, Object, Stream}; +use sha2::{Digest, Sha256}; + +use crate::verify::limits::{MAX_STREAM, MAX_TOTAL_DECODED, MAX_WORK, MIN_NEEDLE}; + +/// What one proof may spend, and what it knows about the document it checks. +pub(super) struct Ctx { + /// Digests of the decoded content of the overlay streams (their boxes and labels are drawn + /// on purpose and are neither text to look for nor paint to flag). + pub(super) overlay: HashSet, + /// Decoded bytes of the proof so far: one allowance for the whole proof, failures charged + /// too (see [`Ctx::decode`]), so a stream referenced many times, or one that fails after + /// decoding part of itself, cannot decode again for free. + pub(super) decoded: Cell, + pub(super) work: Cell, +} + +impl Ctx { + pub(super) fn new(overlay: HashSet) -> Self { + Ctx { overlay, decoded: Cell::new(0), work: Cell::new(0) } + } + + /// Decode `s` strictly: a stream that is damaged, longer than [`MAX_STREAM`], or one more + /// than [`MAX_TOTAL_DECODED`] allows is an error (never silently truncated data). Every + /// attempt is charged to the cumulative allowance, failures included: a decode that fails + /// past the one-stream cap pays the cap, and one that fails mid-way pays what it decoded + /// before the error — the work of finding out is not work a repeated reference gets for free. + pub(super) fn decode(&self, s: &Stream) -> Result, ()> { + let left = MAX_TOTAL_DECODED.saturating_sub(self.decoded.get()); + let cap = usize::try_from(left).unwrap_or(usize::MAX).min(MAX_STREAM); + let mut budget = DecodeBudget::new(cap, usize::MAX); + let result = budget.decode_strict(s); + // The attempt's own accounting covers a success (what it decoded), a stream cut short + // by corruption (what it decoded first) and one refused past the cap (the cap): charge + // it through, whichever way the attempt went. + let spent = cap.saturating_sub(budget.remaining()); + self.decoded.set(self.decoded.get().saturating_add(spent as u64)); + result.map_err(|_| ()) + } + + /// Is the cumulative decode allowance used up? A decode refused then says nothing about the + /// stream, so it must not pass for a stream that can't be decoded. + pub(super) fn decode_spent(&self) -> bool { + self.decoded.get() >= MAX_TOTAL_DECODED + } + + /// Spend `n` units of search work; false once [`MAX_WORK`] is used up. + pub(super) fn charge(&self, n: usize) -> bool { + let used = self.work.get().saturating_add(n as u64); + self.work.set(used); + used <= MAX_WORK + } + + /// Is this decoded content one of the overlay streams this run created? + pub(super) fn is_overlay(&self, data: &[u8]) -> bool { + !self.overlay.is_empty() && self.overlay.contains(&sha256(&[data])) + } + + /// (tests) A context whose cumulative decode allowance has `left` bytes left. + #[cfg(test)] + pub(super) fn with_decode_left(left: u64) -> Self { + Ctx { overlay: HashSet::new(), decoded: Cell::new(MAX_TOTAL_DECODED.saturating_sub(left)), work: Cell::new(0) } + } +} + +/// Follow `o` to its object; a reference that can't be loaded is an error (a missing object is +/// `Null` for [`Document::resolve`], which would read as "empty"). +pub(super) fn resolve_strict(doc: &Document, o: &Object) -> Result, ()> { + match o { + Object::Ref(r) => doc.try_get(r.num).map_err(|_| ()), + other => Ok(Arc::new(other.clone())), + } +} + +pub(super) fn hex(bytes: &[u8]) -> String { + bytes.iter().fold(String::with_capacity(bytes.len() * 2), |mut s, b| { + let _ = write!(s, "{b:02x}"); + s + }) +} + +pub(super) fn sha256>(parts: &[T]) -> String { + let mut h = Sha256::new(); + for p in parts { + let p = p.as_ref(); + h.update((p.len() as u64).to_be_bytes()); + h.update(p); + } + hex(h.finalize().as_slice()) +} + +/// The text of `s` without whitespace (page text is compared whitespace-blind: a gap in a `TJ` +/// is a space in one extraction and nothing in another). +pub(super) fn squash(s: &str) -> String { + s.chars().filter(|c| !c.is_whitespace() && *c != '\u{feff}').collect() +} + +pub(super) fn latin1(b: &[u8]) -> String { + b.iter().map(|&c| char::from(c)).collect() +} + +/// The words of a run worth looking for on their own (a run that survives in pieces). +pub(super) fn words(s: &str) -> Vec<&str> { + s.split_whitespace().filter(|w| w.chars().count() >= MIN_NEEDLE).collect() +} + +pub(super) fn count(hay: &str, needle: &str) -> usize { + if needle.is_empty() { 0 } else { hay.matches(needle).count() } +} + +/// Is `pat` in `hay`? +pub(super) fn find(hay: &[u8], pat: &[u8]) -> bool { + find_from(hay, pat, 0).is_some() +} + +/// Where `pat` first occurs in `hay` at or after `from`. The pattern's prefix function is +/// built once and the haystack is walked once: the search is worst-case linear, so +/// adversarial input (a needle of one letter and a haystack full of it) can neither make it +/// quadratic nor spend more comparisons than the caller charges for. +pub(super) fn find_from(hay: &[u8], pat: &[u8], from: usize) -> Option { + let m = pat.len(); + if m == 0 { + return None; + } + // fail[i]: the longest proper prefix of `pat` that ends at `i` and is also a suffix there. + let mut fail = vec![0usize; m]; + let mut k = 0; + for (i, &c) in pat.iter().enumerate().skip(1) { + while k > 0 && pat[k] != c { + k = fail[k - 1]; + } + if pat[k] == c { + k += 1; + } + fail[i] = k; + } + let mut k = 0; + for (at, &c) in hay.iter().enumerate().skip(from) { + while k > 0 && pat[k] != c { + k = fail[k - 1]; + } + if pat[k] == c { + k += 1; + if k == m { + return Some(at + 1 - m); + } + } + } + None +} + +pub(super) fn escape_literal(b: &[u8], octal: bool) -> Vec { + let mut out = Vec::with_capacity(b.len() * 2); + for &c in b { + if octal { + out.extend(format!("\\{c:03o}").bytes()); + continue; + } + match c { + b'(' | b')' | b'\\' => out.extend([b'\\', c]), + b'\n' => out.extend(*b"\\n"), + b'\r' => out.extend(*b"\\r"), + b'\t' => out.extend(*b"\\t"), + _ => out.push(c), + } + } + out +} + +/// Every way `text` can be spelled in a PDF: UTF-8, UTF-16BE (with and without a byte order +/// mark) and UTF-16LE, each as raw bytes, as a hex string (either case) and as a backslash- +/// escaped literal string (minimal escapes, and octal for every byte). +pub(crate) fn survivor_encodings(text: &str) -> Vec> { + let utf16 = |be: bool, bom: bool| -> Vec { + let mut v = Vec::new(); + if bom { + v.extend(if be { [0xFE, 0xFF] } else { [0xFF, 0xFE] }); + } + for u in text.encode_utf16() { + v.extend(if be { u.to_be_bytes() } else { u.to_le_bytes() }); + } + v + }; + let forms = [text.as_bytes().to_vec(), utf16(true, true), utf16(true, false), utf16(false, true), utf16(false, false)]; + spellings(forms) +} + +/// The spellings of a byte string (see [`survivor_encodings`]). +pub(super) fn spellings(forms: impl IntoIterator>) -> Vec> { + let mut out: Vec> = Vec::new(); + for f in forms { + if f.is_empty() { + continue; + } + let h = hex(&f); + out.push(h.to_uppercase().into_bytes()); + out.push(h.into_bytes()); + out.push(escape_literal(&f, false)); + out.push(escape_literal(&f, true)); + out.push(f); + } + let mut seen = HashSet::new(); + out.retain(|b| seen.insert(b.clone())); + out +} + +/// Text of a payload: UTF-16 when it has a byte order mark, else UTF-8 (lossy). +pub(super) fn decode_text(b: &[u8]) -> String { + let units = |be: bool, rest: &[u8]| -> String { + let u: Vec = rest.as_chunks::<2>().0.iter().map(|c| if be { u16::from_be_bytes(*c) } else { u16::from_le_bytes(*c) }).collect(); + String::from_utf16_lossy(&u) + }; + match b { + [0xFE, 0xFF, rest @ ..] => units(true, rest), + [0xFF, 0xFE, rest @ ..] => units(false, rest), + _ => String::from_utf8_lossy(b).into_owned(), + } +} + +pub(super) fn xml_unescape(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + let mut rest = s; + while let Some(at) = rest.find('&') { + out.push_str(rest.get(..at).unwrap_or_default()); + rest = rest.get(at..).unwrap_or_default(); + match xml_entity(rest) { + Some((c, len)) => { + out.push(c); + rest = rest.get(len..).unwrap_or_default(); + } + None => { + out.push('&'); + rest = rest.get(1..).unwrap_or_default(); + } + } + } + out.push_str(rest); + out +} + +/// The entity starting at `&`: the character it reads as, and how far it runs (through the +/// closing `;`). A numeric reference may carry any number of leading zeros, so its digits +/// are padded without a bound; more significant digits than a character value can hold, or +/// a missing `;`, is not an entity and the `&` stays as it is. Walking only the digits the +/// reference itself holds keeps the decode linear in the input. +fn xml_entity(s: &str) -> Option<(char, usize)> { + for (name, c) in [("&", '&'), ("<", '<'), (">", '>'), (""", '"'), ("'", '\'')] { + if s.starts_with(name) { + return Some((c, name.len())); + } + } + let body = s.strip_prefix("&#")?; + let (radix, digits) = match body.as_bytes().first() { + Some(b'x' | b'X') => (16u32, &body[1..]), + _ => (10, body), + }; + let mut value: u32 = 0; + let mut digits_held = false; + let mut end = None; + for (i, c) in digits.char_indices() { + match c.to_digit(radix) { + Some(d) => { + digits_held = true; + // Leading zeros only pad; past a character's worth of significant digits the + // multiplication overflows and this was never an entity. + value = value.checked_mul(radix)?.checked_add(d)?; + } + None => { + if c == ';' && digits_held { + end = Some(i); + } + break; + } + } + } + let end = end?; + let ch = char::from_u32(value)?; + Some((ch, 2 + usize::from(radix == 16) + end + 1)) +} + +/// Names of the filters of a stream dictionary. +pub(super) fn filter_names(d: &Dict) -> Vec> { + match d.get(b"Filter") { + Some(Object::Name(n)) => vec![n.clone()], + Some(Object::Array(a)) => a.iter().filter_map(|o| o.as_name().map(<[u8]>::to_vec)).collect(), + _ => Vec::new(), + } +} + +/// Image codecs: their data is pixels, still encoded. +pub(super) fn image_codec(d: &Dict) -> bool { + filter_names(d).iter().any(|n| matches!(n.as_slice(), b"DCTDecode" | b"DCT" | b"JPXDecode" | b"CCITTFaxDecode" | b"CCF" | b"JBIG2Decode")) +}