diff --git a/Cargo.lock b/Cargo.lock index 4816a44ec0..e1f271ed12 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -143,6 +143,15 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + [[package]] name = "aligned-vec" version = "0.6.4" @@ -4610,6 +4619,7 @@ dependencies = [ name = "pdfcraft-redact" version = "0.18.1" dependencies = [ + "aho-corasick", "pdfcraft-annot", "pdfcraft-content", "pdfcraft-cos", @@ -4620,6 +4630,7 @@ dependencies = [ "pdfcraft-model", "pdfcraft-testkit", "pdfcraft-xfa", + "sha2 0.11.1", "thiserror 2.0.21", ] diff --git a/crates/redact/Cargo.toml b/crates/redact/Cargo.toml index fe21d13b32..c02335e3c4 100644 --- a/crates/redact/Cargo.toml +++ b/crates/redact/Cargo.toml @@ -9,6 +9,7 @@ license.workspace = true workspace = true [dependencies] +aho-corasick = "1" pdfcraft-cos = { workspace = true } pdfcraft-content = { workspace = true } pdfcraft-fonts = { workspace = true } @@ -20,6 +21,7 @@ pdfcraft-xfa = { workspace = true } # CCITT Group 4 and JBIG2 encoding for cleared one-bit images. pdfcraft-filters = { workspace = true } thiserror = { workspace = true } +sha2 = "0.11" [dev-dependencies] pdfcraft-testkit = { workspace = true } diff --git a/crates/redact/src/image.rs b/crates/redact/src/image.rs index 4b3c44b0fd..a99e179895 100644 --- a/crates/redact/src/image.rs +++ b/crates/redact/src/image.rs @@ -12,7 +12,7 @@ use pdfcraft_cos::{Dict, Document, ImageBudget, Object, Stream}; /// Most pixels an image may have to be cleared in place (larger ones are removed), or to be stored /// again without shared `/JBIG2Globals` (see [`without_globals`]). -const MAX_PIXELS: u64 = pdfcraft_cos::MAX_REDACTED_IMAGE_PIXELS; +pub(crate) const MAX_PIXELS: u64 = pdfcraft_cos::MAX_REDACTED_IMAGE_PIXELS; fn components(doc: &Document, s: &Stream) -> Option { let cs = doc.resolve(s.dict.get(b"ColorSpace")?); diff --git a/crates/redact/src/interp.rs b/crates/redact/src/interp.rs index 460085a320..7a2cd26eca 100644 --- a/crates/redact/src/interp.rs +++ b/crates/redact/src/interp.rs @@ -53,6 +53,23 @@ pub(crate) enum Mode { Verify, } +/// A run of glyphs cut out under a region, kept for the redaction proof +/// ([`Snapshot::capture`](crate::verify::Snapshot::capture)). +pub(crate) struct Removed { + /// Index of the region (in `Scope::rects`) the run lay under. + pub(crate) rect: usize, + /// The character codes as they stood in the string. + pub(crate) raw: Vec, + /// What the font says they showed (codes without a known meaning are left out). + pub(crate) text: String, + /// How many character codes the run holds (a composite font's codes are several bytes). + pub(crate) codes: usize, + /// Which text operator showed it, counted from the start of the page. + pub(crate) op: usize, + /// Where in that operator it starts: the operand's place in a `TJ` array, and the byte in it. + pub(crate) at: (usize, usize), +} + #[derive(Clone)] struct Gs { ctm: Matrix, @@ -124,6 +141,12 @@ pub(crate) struct Scope<'a> { xobject_oc_cache: HashMap, /// Content removed from hidden layers (blocks and XObjects). pub layer_blocks: usize, + /// Collects the glyph runs removed under a region in apply mode (`None` = not collected). + /// The redaction proof's snapshot is what sets it: it must know what applying is about to + /// remove, by the interpreter's own judgement, without trusting anything else to it. + pub removed: Option>, + /// How many text operators have been shown, for [`Removed::op`]. + shown: usize, fonts: HashMap, Rc>, used_names: Vec>, depth: usize, @@ -149,6 +172,8 @@ impl<'a> Scope<'a> { ocmd_cache: HashMap::new(), xobject_oc_cache: HashMap::new(), layer_blocks: 0, + removed: None, + shown: 0, fonts: HashMap::new(), used_names: Vec::new(), depth: 0, @@ -169,21 +194,33 @@ impl<'a> Scope<'a> { self.rects.iter().any(|r| contains(*r, b, 0.01)) } - /// Does a glyph box count as under a region? Any real overlap does (more than a fifth of + /// Which regions a glyph box counts as under? Any real overlap does (more than a fifth of /// the glyph or a point, whichever is less, in both directions), so neighbours that only /// touch an edge survive while partly covered glyphs go. Degenerate boxes (zero-size text) - /// count when their origin is inside. - fn glyph_hit(&self, b: [f64; 4], origin: (f64, f64)) -> bool { + /// count when their origin is inside. Empty = under no region (a glyph kept). A glyph may + /// lie under several regions at once (overlapping marks); each is reported. + fn glyph_rects(&self, b: [f64; 4], origin: (f64, f64)) -> Vec { let (gw, gh) = (b[2] - b[0], b[3] - b[1]); if gw < 0.01 || gh < 0.01 { - return self.rects.iter().any(|r| origin.0 >= r[0] && origin.0 <= r[2] && origin.1 >= r[1] && origin.1 <= r[3]); + return self + .rects + .iter() + .enumerate() + .filter(|(_, r)| origin.0 >= r[0] && origin.0 <= r[2] && origin.1 >= r[1] && origin.1 <= r[3]) + .map(|(i, _)| i) + .collect(); } let (mx, my) = ((gw * 0.2).min(1.0), (gh * 0.2).min(1.0)); - self.rects.iter().any(|r| { - let ox = b[2].min(r[2]) - b[0].max(r[0]); - let oy = b[3].min(r[3]) - b[1].max(r[1]); - ox > mx && oy > my - }) + self.rects + .iter() + .enumerate() + .filter(|(_, r)| { + let ox = b[2].min(r[2]) - b[0].max(r[0]); + let oy = b[3].min(r[3]) - b[1].max(r[1]); + ox > mx && oy > my + }) + .map(|(i, _)| i) + .collect() } } @@ -223,6 +260,8 @@ struct Glyph { /// Advance in unscaled text space (before `Th`). advance: f64, hit: bool, + /// The regions it lies under (empty for hidden text). + rects: Vec, } /// Process a scope's content streams (state flows from one stream to the next). @@ -644,11 +683,14 @@ fn clean_replacements(doc: &Document, resources: &Dict, original: &[Op], process /// Lay out a shown string (or `TJ` array) glyph by glyph, advancing `tm`. Returns the `TJ` /// items with the glyphs under a region replaced by displacements, and how many were removed. -fn show(scope: &Scope<'_>, gs: &Gs, tm: &mut Matrix, items: &[Object]) -> (Vec, usize) { +/// In apply mode the runs removed under each region are also handed to `scope.removed`. +fn show(scope: &mut Scope<'_>, gs: &Gs, tm: &mut Matrix, items: &[Object]) -> (Vec, usize) { let f = &gs.font; let size = gs.size; let mut out: Vec = Vec::new(); let mut removed = 0; + scope.shown += 1; + let op = scope.shown; let push_num = |out: &mut Vec, v: f64| { if v == 0.0 { return; @@ -661,7 +703,7 @@ fn show(scope: &Scope<'_>, gs: &Gs, tm: &mut Matrix, items: &[Object]) -> (Vec { let bytes = &s.bytes; @@ -674,18 +716,50 @@ fn show(scope: &Scope<'_>, gs: &Gs, tm: &mut Matrix, items: &[Object]) -> (Vec { let degenerate = b[2] - b[0] < 0.01 && b[3] - b[1] < 0.01; matches!(gs.render_mode, 3 | 7) || !(overlaps(page, b, 0.0) || degenerate) } - None => scope.glyph_hit(b, origin), + None => { + rects = scope.glyph_rects(b, origin); + !rects.is_empty() + } }; - glyphs.push(Glyph { bytes: pos..pos + len, advance, hit }); + glyphs.push(Glyph { bytes: pos..pos + len, advance, hit, rects }); *tm = Matrix::translate(advance * gs.scale, 0.0).then(tm); pos += len; } + if scope.mode == Mode::Apply + && let Some(sink) = scope.removed.as_mut() + { + // Consecutive removed glyphs under one region are one run, recorded for + // every region the glyphs lie under: overlapping marks each account for + // their own share of what was cut. + let mut under: Vec = glyphs.iter().flat_map(|g| g.rects.iter().copied()).collect(); + under.sort_unstable(); + under.dedup(); + for r in under { + let mut k = 0; + while let Some(g) = glyphs.get(k) { + if !g.rects.contains(&r) { + k += 1; + continue; + } + let mut end = k; + while glyphs.get(end + 1).is_some_and(|n| n.rects.contains(&r)) { + end += 1; + } + let span = g.bytes.start..glyphs.get(end).map_or(g.bytes.end, |l| l.bytes.end); + let at = (place, span.start); + let raw = bytes.get(span).unwrap_or_default().to_vec(); + sink.push(Removed { rect: r, text: f.decode(&raw), raw, codes: end - k + 1, op, at }); + k = end + 1; + } + } + } let mut run: Vec = Vec::new(); for g in &glyphs { if g.hit { diff --git a/crates/redact/src/lib.rs b/crates/redact/src/lib.rs index b6a8885c03..6bb8e5cc06 100644 --- a/crates/redact/src/lib.rs +++ b/crates/redact/src/lib.rs @@ -27,10 +27,14 @@ pub mod sanitize; mod tags; #[cfg(test)] mod tests; +#[cfg(test)] +mod tests_verify; pub(crate) mod type3; pub(crate) mod type3_tables; +pub mod verify; use interp::{Mode, Scope, process}; +pub use verify::{Proof, ProofOptions, Snapshot}; pub type Rgb = [f64; 3]; @@ -59,6 +63,11 @@ pub enum RedactError { "redaction appearance has invalid geometry or exceeds the 8 MiB budget; reduce the marked area, turn off repeated text or apply fewer pages at once (nothing was redacted)" )] OverlayLimit, + /// A stream the redaction created to draw the marks' boxes and labels can't be read back, so + /// the proof can't tell it apart from content that was there before and nothing can be + /// exempted from the sweep. + #[error("a stream the redaction created can't be read back, so the result can't be proven")] + OverlayUnreadable, /// The structure tree couldn't be cleaned completely (#3302): an element written in place that /// would need changing, an element whose inherited page can't be found, or a tree too large to /// walk. Tags that kept the removed text or annotation would undo the redaction. @@ -164,7 +173,7 @@ pub struct Report { pub overlays: Vec, } -fn rect_of(doc: &Document, o: Option<&Object>) -> Option<[f64; 4]> { +pub(crate) fn rect_of(doc: &Document, o: Option<&Object>) -> Option<[f64; 4]> { let v: Vec = doc.resolve(o?).as_array()?.iter().filter_map(|x| doc.resolve(x).as_f64()).collect(); (v.len() == 4 && v.iter().all(|x| x.is_finite())).then(|| [v[0].min(v[2]), v[1].min(v[3]), v[0].max(v[2]), v[1].max(v[3])]) } diff --git a/crates/redact/src/tests_verify.rs b/crates/redact/src/tests_verify.rs new file mode 100644 index 0000000000..ee443f3209 --- /dev/null +++ b/crates/redact/src/tests_verify.rs @@ -0,0 +1,1380 @@ +//! The independent redaction proof: what it catches, what it refuses to call clean. + +use std::sync::Arc; + +use pdfcraft_annot::{Meta, NewAnnotation, Shape, Style, add_annotation, rect_quad}; +use pdfcraft_cos::{Algorithm, Dict, Document, NewEncryption, Object, SaveOptions, Stream, write_full, write_incremental}; +use sha2::Digest; + +use super::*; +use crate::verify::tests_support::{self, count_text_operators, survivor_encodings}; +use crate::verify::{Status, Surface, SurfaceReport, Verdict}; + +const SECRET: &str = "SECRETVALUE"; + +pub(super) fn stream(dict: &str, data: &[u8]) -> Vec { + let mut v = format!("<< {dict} /Length {} >>\nstream\n", data.len()).into_bytes(); + v.extend_from_slice(data); + v.extend_from_slice(b"\nendstream"); + v +} + +pub(super) fn pdf_bytes(objs: Vec>, trailer: &str) -> Vec { + let mut out = b"%PDF-1.7\n".to_vec(); + let mut offs = Vec::new(); + for (i, o) in objs.iter().enumerate() { + offs.push(out.len()); + out.extend_from_slice(format!("{} 0 obj\n", i + 1).as_bytes()); + out.extend_from_slice(o); + out.extend_from_slice(b"\nendobj\n"); + } + let x = out.len(); + out.extend_from_slice(format!("xref\n0 {}\n0000000000 65535 f \n", objs.len() + 1).as_bytes()); + for o in offs { + out.extend_from_slice(format!("{o:010} 00000 n \n").as_bytes()); + } + out.extend_from_slice(format!("trailer\n<< /Size {} /Root 1 0 R {trailer} >>\nstartxref\n{x}\n%%EOF\n", objs.len() + 1).as_bytes()); + out +} + +pub(super) fn open(bytes: Vec) -> Document { + Document::open(Arc::new(bytes)).unwrap() +} + +/// Every glyph is 500 units wide (5 pt at 10 pt). +pub(super) const FONT: &str = + "<< /Type /Font /Subtype /TrueType /BaseFont /Arial /FirstChar 32 /LastChar 126 /Widths 6 0 R /FontDescriptor << /Ascent 800 /Descent -200 >> >>"; + +/// A composite font without a /ToUnicode map: its text can't be decoded. +const OPAQUE_FONT: &str = "<< /Type /Font /Subtype /Type0 /BaseFont /X /Encoding /Identity-H /DescendantFonts [<< /Type /Font /Subtype /CIDFontType2 /BaseFont /X /CIDSystemInfo << /Registry (Adobe) /Ordering (Identity) /Supplement 0 >> /DW 1000 >>] >>"; + +/// Options of a one-page file: catalog, page and trailer entries, page resources, and more +/// objects (numbered from 7). +#[derive(Default)] +struct Spec<'a> { + catalog: &'a str, + page: &'a str, + trailer: &'a str, + resources: &'a str, + extra: Vec>, +} + +/// "Public" at the top, the secret lower down at x 10..65, y 100..110. +const CONTENT: &[u8] = b"BT /F1 10 Tf 10 200 Td (Public) Tj ET BT /F1 10 Tf 10 100 Td (SECRETVALUE) Tj ET"; + +pub(super) const REGION: [f64; 4] = [8.0, 95.0, 70.0, 112.0]; + +fn build_with(content: &[u8], spec: Spec<'_>) -> Vec { + let mut objs: Vec> = vec![ + format!("<< /Type /Catalog /Pages 2 0 R {} >>", spec.catalog).into_bytes(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + format!( + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R {} /Resources << /Font << /F1 5 0 R >> {} >> >>", + spec.page, spec.resources + ) + .into_bytes(), + stream("", content), + FONT.as_bytes().to_vec(), + format!("[{}]", vec!["500"; 95].join(" ")).into_bytes(), + ]; + objs.extend(spec.extra); + pdf_bytes(objs, spec.trailer) +} + +pub(super) fn mark(doc: &mut Document, rects: &[[f64; 4]]) { + let shape = Shape::Redact { quads: rects.iter().map(|r| rect_quad(*r)).collect(), overlay: String::new(), look: Default::default() }; + let style = Style::default_for(&shape); + add_annotation(doc, &NewAnnotation { page: 0, shape, style, contents: String::new(), author: "Tester".into() }, &Meta::default()).unwrap(); +} + +/// The standard page with `spec`, the secret's region marked. +fn marked(spec: Spec<'_>) -> Document { + marked_content(CONTENT, spec) +} + +fn marked_content(content: &[u8], spec: Spec<'_>) -> Document { + let mut doc = open(build_with(content, spec)); + mark(&mut doc, &[REGION]); + doc +} + +/// Apply, and tell the snapshot which streams the apply created (its overlay boxes and labels). +fn applied(doc: &mut Document, snap: &mut Snapshot) -> Result { + let result = apply(doc, None); + if let Ok(report) = &result { + snap.set_overlay_streams(doc, &report.overlays)?; + } + result +} + +/// Capture, apply, prove. +pub(super) fn run(doc: &mut Document) -> (Result, Proof) { + let mut snap = Snapshot::capture(doc, None).unwrap(); + let result = applied(doc, &mut snap); + let proof = snap.prove(doc, None, &ProofOptions::default()); + (result, proof) +} + +/// Prove a document the redaction was never applied to (as a full save would write it). +pub(super) fn prove_unapplied(doc: &mut Document) -> Proof { + let snap = Snapshot::capture(doc, None).unwrap(); + doc.require_full_save(); + snap.prove(doc, None, &ProofOptions::default()) +} + +pub(super) fn surface(p: &Proof, s: Surface) -> &SurfaceReport { + p.surfaces.iter().find(|r| r.surface == s).unwrap() +} + +fn residue(r: &Result) -> bool { + matches!(r, Err(RedactError::Residue(n)) if *n > 0) +} + +#[test] +fn the_proof_reads_a_pages_streams_as_the_one_stream_they_are() { + // The TJ array of the secret ends the first piece and its operator starts the second + // (#153): the proof reads the pieces as the one stream they are, so the removed text is + // neither reported as a survivor nor does the page count as unreadable. + let objs: Vec> = vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents [4 0 R 7 0 R] /Resources << /Font << /F1 5 0 R >> >> >>".to_vec(), + stream("", b"BT /F1 10 Tf 10 100 Td [(SECRETVALUE)]"), + FONT.as_bytes().to_vec(), + format!("[{}]", vec!["500"; 95].join(" ")).into_bytes(), + stream("", b"TJ ET BT /F1 10 Tf 10 200 Td (Public) Tj ET"), + ]; + let mut doc = open(pdf_bytes(objs, "/Root 1 0 R")); + mark(&mut doc, &[REGION]); + let (r, proof) = run(&mut doc); + let r = r.unwrap(); + assert_eq!(r.glyphs, 11, "every glyph of the split string is removed: {r:?}"); + assert!(proof.passed(), "{}", proof.to_text()); + assert_eq!(proof.survivors(), 0); + let saved = write_incremental(&doc, &SaveOptions::default()).unwrap(); + assert!(!saved.windows(SECRET.len()).any(|w| w == SECRET.as_bytes()), "the secret is gone from the file"); +} + +#[test] +fn a_clean_redaction_passes_and_the_neighbour_is_untouched() { + let mut doc = marked(Spec::default()); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert!(proof.passed(), "{}", proof.to_text()); + assert_eq!(proof.entries.len(), 1); + let e = &proof.entries[0]; + assert_eq!((e.status, e.page, e.region), (Status::Verified, 0, REGION)); + assert_eq!((e.runs_removed, e.codes_removed), (1, SECRET.len())); + let content = page_content(&doc); + assert!(content.contains("(Public) Tj") && !content.contains(SECRET), "{content}"); + for s in &proof.surfaces { + assert!(matches!(s.verdict, Verdict::Clean | Verdict::Absent), "{s:?}"); + } + assert_eq!(surface(&proof, Surface::Revisions).verdict, Verdict::Clean); +} + +#[test] +fn the_manifest_has_digests_and_counts_but_no_plaintext() { + let mut doc = marked(Spec::default()); + // The digest of the original content: a shared proof must not carry it. + let original: String = sha2::Sha256::digest(page_content(&doc).as_bytes()).iter().map(|b| format!("{b:02x}")).collect(); + let (r, proof) = run(&mut doc); + r.unwrap(); + let e = &proof.entries[0]; + let after = e.page_sha256_after.as_deref().unwrap(); + assert!(after.len() == 64 && after.bytes().all(|c| c.is_ascii_hexdigit())); + assert_ne!(after, original); + for shown in [proof.to_text(), format!("{proof:?}")] { + assert!(!shown.contains(&original), "{shown}"); + } + // Two glyph-carrying operators before; the removed one left a number-only TJ, which carries + // no glyphs and is not counted. + assert_eq!((e.text_ops_before, e.text_ops_after), (Some(2), Some(1))); + assert!(e.removed_text.is_none()); + for shown in [proof.to_text(), format!("{proof:?}")] { + assert!(!shown.contains(SECRET), "{shown}"); + } + + // Plaintext only on request. + let mut doc = marked(Spec::default()); + let mut snap = Snapshot::capture(&doc, None).unwrap(); + assert!(!format!("{snap:?}").contains(SECRET)); + applied(&mut doc, &mut snap).unwrap(); + let with = snap.prove(&doc, None, &ProofOptions { include_plaintext: true }); + assert_eq!(with.entries[0].removed_text.as_deref(), Some(&[SECRET.to_string()][..])); +} + +#[test] +fn survivors_do_not_echo_the_text_either() { + let mut doc = marked(Spec { trailer: "/Info 7 0 R", extra: vec![b"<< /Title (SECRETVALUE) >>".to_vec()], ..Spec::default() }); + let (_, proof) = run(&mut doc); + assert!(!proof.passed()); + for shown in [proof.to_text(), format!("{proof:?}")] { + assert!(!shown.contains(SECRET), "{shown}"); + } +} + +#[test] +fn numeric_only_tj_is_not_glyph_carrying() { + assert_eq!(count_text_operators(b"BT (Tj) Tj [ -250 ] TJ [(x) -3 (y)] TJ () Tj [] TJ ET"), 2); + assert_eq!(count_text_operators(b"BT (a) ' 1 2 (b) \" [ 10 -10 ] TJ ET"), 2); +} + +#[test] +fn a_survivor_in_the_info_dictionary() { + let mut doc = + marked(Spec { trailer: "/Info 7 0 R", extra: vec![b"<< /Title (Report SECRETVALUE) /Author (Someone) >>".to_vec()], ..Spec::default() }); + let (_, proof) = run(&mut doc); + assert_eq!(surface(&proof, Surface::Info).verdict, Verdict::Survivor); + assert_eq!(proof.entries[0].status, Status::Failed); + assert!(!proof.passed()); +} + +#[test] +fn a_survivor_in_xmp() { + let xmp = stream("/Type /Metadata /Subtype /XML", b"Quarterly & SECRETVALUE"); + let mut doc = marked(Spec { catalog: "/Metadata 7 0 R", extra: vec![xmp], ..Spec::default() }); + let (_, proof) = run(&mut doc); + assert_eq!(surface(&proof, Surface::Xmp).verdict, Verdict::Survivor, "{}", proof.to_text()); + assert!(!proof.passed()); +} + +#[test] +fn a_long_numeric_reference_in_xmp_is_a_survivor() { + // A character reference may carry any number of leading zeros: the padded form reads the + // same, and is swept the same. + let xmp = stream("/Type /Metadata /Subtype /XML", b"SECRETVALUE"); + let mut doc = marked(Spec { catalog: "/Metadata 7 0 R", extra: vec![xmp], ..Spec::default() }); + let (_, proof) = run(&mut doc); + assert_eq!(surface(&proof, Surface::Xmp).verdict, Verdict::Survivor, "{}", proof.to_text()); + assert!(!proof.passed()); +} + +#[test] +fn a_survivor_in_an_annotation_string() { + let annot = b"<< /Type /Annot /Subtype /Text /Rect [200 200 220 220] /Contents (note: SECRETVALUE) >>".to_vec(); + let mut doc = marked(Spec { page: "/Annots [7 0 R]", extra: vec![annot], ..Spec::default() }); + let (_, proof) = run(&mut doc); + assert_eq!(surface(&proof, Surface::ObjectStrings).verdict, Verdict::Survivor); + assert_eq!(proof.entries[0].status, Status::Failed); + assert!(!proof.passed()); +} + +// ── Every place a reader could recover the removed text from ──────────────────────────────── +// Each test plants the removed text in one location and expects the sweep to report it there. + +#[test] +fn a_survivor_in_a_compressed_object_stream_is_found() { + // A full save packs non-stream objects into compressed object streams: a copy of the + // removed text left in one (an annotation away from the region) is neither in the stored + // bytes nor on any page, only inside the /ObjStm. The sweep reads the saved file like any + // reader, objects in object streams included, and reports the copy. + let annot = b"<< /Type /Annot /Subtype /Text /Rect [200 200 220 220] /Contents (note: SECRETVALUE) >>".to_vec(); + let mut doc = marked(Spec { page: "/Annots [7 0 R]", extra: vec![annot], ..Spec::default() }); + let mut snap = Snapshot::capture(&doc, None).unwrap(); + applied(&mut doc, &mut snap).unwrap(); + let saved = write_full(&doc, &SaveOptions::default()).unwrap(); + assert!(saved.windows(b"/ObjStm".len()).any(|w| w == b"/ObjStm"), "the output packs objects into object streams"); + assert!(!saved.windows(SECRET.len()).any(|w| w == SECRET.as_bytes()), "the plaintext is compressed away"); + let proof = snap.prove_saved(&saved, None, &ProofOptions::default()); + let s = surface(&proof, Surface::ObjectStrings); + assert_eq!(s.verdict, Verdict::Survivor, "{}", proof.to_text()); + assert_eq!(surface(&proof, Surface::RawBytes).verdict, Verdict::Clean, "the copy sits in the object stream, not the stored bytes"); + assert!(!proof.passed(), "{}", proof.to_text()); +} + +#[test] +fn a_survivor_in_a_page_thumbnail_is_found() { + // The page's /Thumb holds a picture of the page as it was, secret included (uncompressed + // pixels, so the stored bytes are the pixels). A thumbnail is not page content, so only + // the object sweep can find it. + let thumb = stream("/Type /XObject /Subtype /Image /Width 7 /Height 3 /ColorSpace /DeviceGray /BitsPerComponent 8", b"junk SECRETVALUE junk"); + let mut doc = marked(Spec { page: "/Thumb 7 0 R", extra: vec![thumb], ..Spec::default() }); + let (_, proof) = run(&mut doc); + let s = surface(&proof, Surface::DecodedStreams); + assert_eq!(s.verdict, Verdict::Survivor, "{}", proof.to_text()); + assert!(s.locations.iter().any(|l| l.contains("(raw)")), "{:?}", s.locations); + assert!(!proof.passed(), "{}", proof.to_text()); +} + +#[test] +fn a_survivor_in_a_form_field_value_is_found() { + // The redacted text also sits in the form: a text field whose /V holds it. The field + // dictionary is an object like any other, and its strings are swept wherever the field + // hangs (AcroForm and the page's annotations both point at it). + let field = b"<< /Type /Annot /Subtype /Widget /Rect [200 20 280 40] /FT /Tx /T (form) /V (note: SECRETVALUE) >>".to_vec(); + let spec = Spec { catalog: "/AcroForm << /Fields [7 0 R] >>", page: "/Annots [7 0 R]", extra: vec![field], ..Spec::default() }; + let mut doc = marked(spec); + let (_, proof) = run(&mut doc); + let s = surface(&proof, Surface::ObjectStrings); + assert_eq!(s.verdict, Verdict::Survivor, "{}", proof.to_text()); + // The field's copy is the only place left (the save renumbers objects, so the place is + // named, not numbered from the input file; both views of the string count as survivors). + assert!(s.survivors >= 1, "{}", proof.to_text()); + assert_eq!(s.locations.len(), 1, "{:?}", s.locations); + assert!(!proof.passed(), "{}", proof.to_text()); +} + +#[test] +fn a_survivor_in_an_xfa_packet_is_found() { + // An XFA form mirrors its field values in the /XFA packets, XML of their own. Redaction + // strips an XFA form outright (crates/xfa does that, not this machinery); the proof must + // not take the stripper on trust: whatever it left (a shape it refused, a leftover + // packet) is swept like any stream, and what the packet holds is reported. + let xfa = stream("", b"
SECRETVALUE
"); + let spec = Spec { catalog: "/AcroForm << /XFA 7 0 R /Fields [] >>", extra: vec![xfa], ..Spec::default() }; + let doc = marked(spec); + let snap = Snapshot::capture(&doc, None).unwrap(); + let proof = snap.prove_saved(doc.bytes(), None, &ProofOptions::default()); + let s = surface(&proof, Surface::DecodedStreams); + assert_eq!(s.verdict, Verdict::Survivor, "{}", proof.to_text()); + assert!(s.locations.iter().any(|l| l == "object 7 (decoded)"), "{:?}", s.locations); + assert!(!proof.passed(), "{}", proof.to_text()); +} + +#[test] +fn a_survivor_in_an_unused_object_is_found() { + // Object 7 is reachable from nothing: no catalog entry, no page resource. A reader that + // only walks the document tree misses it; the sweep walks every object the + // cross-reference table knows, orphans included, and names the object it sits in. + let orphan = format!("<< /Type /Annot /Subtype /Text /Contents (note: {SECRET}) >>").into_bytes(); + let doc = marked(Spec { extra: vec![orphan], ..Spec::default() }); + let snap = Snapshot::capture(&doc, None).unwrap(); + let proof = snap.prove_saved(doc.bytes(), None, &ProofOptions::default()); + let s = surface(&proof, Surface::ObjectStrings); + assert_eq!(s.verdict, Verdict::Survivor, "{}", proof.to_text()); + assert!(s.locations.iter().any(|l| l == "object 7"), "{:?}", s.locations); + assert!(!proof.passed(), "{}", proof.to_text()); +} + +/// The page draws "(Image)" under the marked region, and keeps an image XObject whose +/// `/Subtype /Image` is the same word. +fn image_word_doc() -> Document { + let image = stream("/Type /XObject /Subtype /Image /Width 20 /Height 20 /ColorSpace /DeviceGray /BitsPerComponent 8", &[0x80u8; 400]); + let content = b"BT /F1 10 Tf 10 100 Td (Image) Tj ET"; + let spec = Spec { resources: "/XObject << /Im1 7 0 R >>", extra: vec![image], ..Spec::default() }; + let mut doc = open(build_with(content, spec)); + mark(&mut doc, &[[0.0, 90.0, 40.0, 115.0]]); + doc +} + +#[test] +fn a_word_that_remains_in_a_structural_name_does_not_fail_the_proof() { + // "Image" is redacted from the page, and is also the /Subtype of the image the page keeps. + // The name is structure redaction never edits, not a leftover copy: the proof must pass, + // and the name surface must read clean, not unswept. + let mut doc = image_word_doc(); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert!(proof.passed(), "{}", proof.to_text()); + assert_eq!(surface(&proof, Surface::ObjectStrings).verdict, Verdict::Clean); + + // But a name the input did not have is a copy like any other: a surplus of name matches + // is a survivor, wherever it sits. + let snap = Snapshot::capture(&image_word_doc(), None).unwrap(); + let image = stream("/Type /XObject /Subtype /Image /Width 20 /Height 20 /ColorSpace /DeviceGray /BitsPerComponent 8", &[0x80u8; 400]); + let spec = Spec { resources: "/XObject << /Im1 7 0 R >>", catalog: "/Extra << /Stolen /Image >>", extra: vec![image], ..Spec::default() }; + let mut shown = open(build_with(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", spec)); + mark(&mut shown, &[[0.0, 90.0, 40.0, 115.0]]); + shown.require_full_save(); + let proof = snap.prove(&shown, None, &ProofOptions::default()); + assert_eq!(surface(&proof, Surface::ObjectStrings).verdict, Verdict::Survivor, "{}", proof.to_text()); + assert!(!proof.passed(), "{}", proof.to_text()); +} + +fn attachment_spec(payload: &[u8]) -> Spec<'static> { + Spec { + catalog: "/Names << /EmbeddedFiles << /Names [(a.bin) 7 0 R] >> >>", + extra: vec![b"<< /Type /Filespec /F (a.bin) /EF << /F 8 0 R >> >>".to_vec(), stream("/Type /EmbeddedFile", payload)], + ..Spec::default() + } +} + +#[test] +fn a_survivor_in_an_attachment() { + let mut doc = marked(attachment_spec(b"the SECRETVALUE is attached")); + let (_, proof) = run(&mut doc); + assert_eq!(surface(&proof, Surface::EmbeddedFiles).verdict, Verdict::Survivor); + assert!(!proof.passed()); + + // UTF-16LE with a byte order mark. + let mut le = vec![0xFF, 0xFE]; + le.extend(SECRET.encode_utf16().flat_map(u16::to_le_bytes)); + let mut doc = marked(attachment_spec(&le)); + let (_, proof) = run(&mut doc); + assert_eq!(surface(&proof, Surface::EmbeddedFiles).verdict, Verdict::Survivor); + assert!(!proof.passed()); +} + +#[test] +fn a_survivor_in_a_nested_attachment() { + let inner = build_with(b"BT /F1 10 Tf 10 100 Td (SECRETVALUE) Tj ET", Spec::default()); + let middle = build_with(b"", attachment_spec(&inner)); + let mut doc = marked(attachment_spec(&middle)); + let (_, proof) = run(&mut doc); + let s = surface(&proof, Surface::EmbeddedFiles); + assert_eq!(s.verdict, Verdict::Survivor); + assert!(s.locations.iter().any(|l| l.matches(" > ").count() >= 1), "{:?}", s.locations); + assert!(!proof.passed(), "{}", proof.to_text()); +} + +#[test] +fn a_clean_nested_attachment_is_swept_and_clean() { + let inner = build_with(b"BT /F1 10 Tf 10 100 Td (harmless) Tj ET", Spec::default()); + let mut doc = marked(attachment_spec(&inner)); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert!(proof.passed(), "{}", proof.to_text()); + assert_eq!(surface(&proof, Surface::EmbeddedFiles).verdict, Verdict::Clean); +} + +#[test] +fn unsweepable_attachments_fail_closed() { + for payload in [&b"PK\x03\x04 zipped"[..], b"%PDF-1.7 not really a pdf", b"\x1f\x8b gzip"] { + let mut doc = marked(attachment_spec(payload)); + let (r, proof) = run(&mut doc); + assert!(r.is_ok() || residue(&r), "only a residue refusal may stop the apply itself: {r:?}"); + assert_eq!(surface(&proof, Surface::EmbeddedFiles).verdict, Verdict::Unswept, "{}", String::from_utf8_lossy(payload)); + assert!(!proof.passed(), "a proof that could not sweep must not pass"); + } +} + +#[test] +fn an_undecodable_stream_fails_closed() { + let junk = stream("/Filter /NoSuchDecode", b"\x00\x01\x02 junk"); + let mut doc = marked(Spec { catalog: "/Extra 7 0 R", extra: vec![junk], ..Spec::default() }); + let (_, proof) = run(&mut doc); + assert_eq!(surface(&proof, Surface::DecodedStreams).verdict, Verdict::Unswept, "{}", proof.to_text()); + assert!(!proof.passed()); +} + +#[test] +fn an_incremental_remnant_fails_the_proof() { + let mut doc = marked(Spec::default()); + let mut snap = Snapshot::capture(&doc, None).unwrap(); + // A "redaction" saved as an incremental update: the page shows something else now, but the + // first revision still holds the original content. + let mut other = open(write_incremental(&doc, &SaveOptions::default()).unwrap()); + let new = other.add(Object::Stream(Stream::flate(Dict::new(), b"BT /F1 10 Tf 10 200 Td (Public) Tj ET"))); + let page = pdfcraft_model::pages(&other).swap_remove(0); + other.update_dict(page.obj, |d| d.set(b"Contents".to_vec(), Object::Ref(new))).unwrap(); + let saved = write_incremental(&other, &SaveOptions::default()).unwrap(); + let proof = snap.prove_saved(&saved, None, &ProofOptions::default()); + assert!(!proof.passed()); + assert_eq!(surface(&proof, Surface::Revisions).verdict, Verdict::Survivor); + assert_eq!(surface(&proof, Surface::RawBytes).verdict, Verdict::Survivor); + // The page text itself is clean: only the file structure gives it away. + assert_eq!(surface(&proof, Surface::ExtractedText).verdict, Verdict::Clean); + + // The real redaction, saved as apply() requires, is a single clean revision. + applied(&mut doc, &mut snap).unwrap(); + let full = write_incremental(&doc, &SaveOptions::default()).unwrap(); + let proof = snap.prove_saved(&full, None, &ProofOptions::default()); + assert!(proof.passed(), "{}", proof.to_text()); +} + +#[test] +fn the_secret_in_each_encoding_is_found() { + // Spelled as PDF syntax in a catalog entry. + let utf16: String = SECRET.encode_utf16().flat_map(u16::to_be_bytes).map(|b| format!("{b:02X}")).collect(); + let utf16le: String = SECRET.encode_utf16().flat_map(u16::to_le_bytes).map(|b| format!("{b:02X}")).collect(); + let hex8: String = SECRET.bytes().map(|b| format!("{b:02x}")).collect(); + let forms = [ + format!("({SECRET})"), + format!(""), + format!(""), + format!("", hex8.to_uppercase()), + format!("<{hex8}>"), + format!("<{utf16}>"), + "(\\123ECRETVALUE)".to_string(), + ]; + for f in forms { + let mut doc = marked(Spec { catalog: &format!("/Extra {f}"), ..Spec::default() }); + let (_, proof) = run(&mut doc); + assert!(!proof.passed(), "{f}: {}", proof.to_text()); + assert_eq!(surface(&proof, Surface::ObjectStrings).verdict, Verdict::Survivor, "{f}: {}", proof.to_text()); + } + + // Spelled as bytes in the file, including the backslash-escaped literal form. + let paren = "SECR(ET)X\\Y"; + let enc = survivor_encodings(paren); + assert!(enc.len() >= 10); + for spelling in enc { + let mut hay = b"junk ".to_vec(); + hay.extend(&spelling); + hay.extend(b" junk"); + assert!(tests_support::found(paren, &hay), "{}", String::from_utf8_lossy(&spelling)); + } + assert!(!tests_support::found(paren, b"nothing to see")); +} + +#[test] +fn a_rotated_page_is_proven_too() { + let mut doc = marked(Spec { page: "/Rotate 90", ..Spec::default() }); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert!(proof.passed(), "{}", proof.to_text()); + assert_eq!(proof.entries[0].status, Status::Verified); +} + +#[test] +fn a_region_without_text_is_recorded_as_such() { + let mut doc = open(build_with(CONTENT, Spec::default())); + mark(&mut doc, &[[150.0, 20.0, 200.0, 40.0]]); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert_eq!(proof.entries[0].status, Status::VerifiedNoTextInRegion); + assert!(proof.passed()); +} + +fn opaque_spec() -> Spec<'static> { + Spec { resources: "/Font << /F1 5 0 R /F2 7 0 R >>", extra: vec![OPAQUE_FONT.as_bytes().to_vec()], ..Spec::default() } +} + +#[test] +fn text_that_cannot_be_decoded_makes_a_region_unverifiable() { + let mut doc = open(build_with(b"BT /F2 10 Tf 10 200 Td <00410042> Tj ET", opaque_spec())); + mark(&mut doc, &[[150.0, 20.0, 200.0, 40.0]]); + let (r, proof) = run(&mut doc); + assert!(r.is_ok() || residue(&r), "only a residue refusal may stop the apply itself: {r:?}"); + assert_eq!(proof.entries[0].status, Status::Unverifiable); + assert!(!proof.passed(), "no PASS while a region is unverifiable"); +} + +#[test] +fn removed_text_of_an_undecodable_font_is_swept_by_its_codes() { + let content = b"BT /F2 10 Tf 10 100 Td <0041004200430044> Tj ET"; + let mut doc = open(build_with(content, opaque_spec())); + mark(&mut doc, &[[0.0, 90.0, 100.0, 120.0]]); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert_eq!(proof.entries[0].runs_removed, 1); + assert_eq!(proof.entries[0].status, Status::Verified, "{}", proof.to_text()); + + // Left in place, the codes are still on the page and the proof says so. + let mut doc = open(build_with(content, opaque_spec())); + mark(&mut doc, &[[0.0, 90.0, 100.0, 120.0]]); + let proof = prove_unapplied(&mut doc); + assert!(!proof.passed()); + assert_eq!(surface(&proof, Surface::ExtractedText).verdict, Verdict::Survivor); +} + +#[test] +fn codes_removed_counts_codes_not_bytes() { + // A composite font's codes are two bytes each: the manifest counts the codes, not the bytes. + let content = b"BT /F2 10 Tf 10 100 Td <0041004200430044> Tj ET"; + let mut doc = open(build_with(content, opaque_spec())); + mark(&mut doc, &[[0.0, 90.0, 100.0, 120.0]]); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert_eq!(proof.entries[0].runs_removed, 1); + assert_eq!(proof.entries[0].codes_removed, 4, "{}", proof.to_text()); + + // A simple font's codes are one byte each: the two counts agree. + let mut doc = marked(Spec::default()); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert_eq!((proof.entries[0].runs_removed, proof.entries[0].codes_removed), (1, SECRET.len())); +} + +#[test] +fn overlapping_marks_each_account_for_their_own_share() { + // Two regions over the same text: each gets the runs, codes and status it is owed — the + // second one is not waved through as "no text in the region". + let mut doc = open(build_with(CONTENT, Spec::default())); + mark(&mut doc, &[REGION, [7.0, 94.0, 71.0, 113.0]]); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert_eq!(proof.entries.len(), 2); + for e in &proof.entries { + assert_eq!((e.runs_removed, e.codes_removed), (1, SECRET.len()), "{}", proof.to_text()); + assert_eq!(e.status, Status::Verified, "{}", proof.to_text()); + } + + // Left in place, both regions name the survivor as theirs. + let mut doc = open(build_with(CONTENT, Spec::default())); + mark(&mut doc, &[REGION, [7.0, 94.0, 71.0, 113.0]]); + let proof = prove_unapplied(&mut doc); + assert!(!proof.passed()); + assert!(proof.entries.iter().all(|e| matches!(e.status, Status::Failed | Status::Unverifiable)), "{}", proof.to_text()); +} + +#[test] +fn nothing_applied_means_survivors_where_the_text_was() { + let mut doc = marked(Spec::default()); + let proof = prove_unapplied(&mut doc); + assert!(!proof.passed()); + assert_eq!(surface(&proof, Surface::ExtractedText).verdict, Verdict::Survivor); + assert_eq!(surface(&proof, Surface::DecodedStreams).verdict, Verdict::Survivor); + assert_eq!(proof.entries[0].status, Status::Failed); +} + +#[test] +fn a_document_that_would_save_incrementally_is_not_proven() { + let mut doc = marked(Spec::default()); + let mut snap = Snapshot::capture(&doc, None).unwrap(); + applied(&mut doc, &mut snap).unwrap(); + // A reopened copy no longer carries the full-save requirement. + let reopened = open(write_incremental(&doc, &SaveOptions::default()).unwrap()); + let proof = snap.prove(&reopened, None, &ProofOptions::default()); + assert_eq!(surface(&proof, Surface::Revisions).verdict, Verdict::Survivor); +} + +fn image_page() -> Document { + let image = stream("/Type /XObject /Subtype /Image /Width 20 /Height 20 /ColorSpace /DeviceGray /BitsPerComponent 8", &[0xFFu8; 400]); + let spec = Spec { resources: "/XObject << /Im1 7 0 R >>", extra: vec![image], ..Spec::default() }; + marked_region(b"q 100 0 0 100 100 100 cm /Im1 Do Q 0 0 1 rg 120 120 60 60 re f", spec) +} + +fn marked_region(content: &[u8], spec: Spec<'_>) -> Document { + let mut doc = open(build_with(content, spec)); + mark(&mut doc, &[[100.0, 100.0, 150.0, 150.0]]); + doc +} + +#[test] +fn images_and_paths_under_a_region_are_checked_independently() { + let mut doc = image_page(); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert!(proof.passed(), "{}", proof.to_text()); + assert_eq!(proof.entries[0].status, Status::VerifiedNoTextInRegion); + + // The same page unredacted: the image and the path are both still there. + let proof = prove_unapplied(&mut image_page()); + let e = &proof.entries[0]; + assert_eq!(e.status, Status::Failed); + assert!(e.detail.contains("image") && e.detail.contains("path"), "{}", e.detail); +} + +#[test] +fn a_shading_under_a_region_must_be_clipped() { + let sh = + b"<< /ShadingType 2 /ColorSpace /DeviceRGB /Coords [0 0 300 0] /Function << /FunctionType 2 /Domain [0 1] /C0 [1 0 0] /C1 [0 0 1] /N 1 >> >>" + .to_vec(); + let spec = || Spec { resources: "/Shading << /Sh1 7 0 R >>", extra: vec![sh.clone()], ..Spec::default() }; + let mut doc = marked_region(b"/Sh1 sh", spec()); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert!(proof.passed(), "{}", proof.to_text()); + + let proof = prove_unapplied(&mut marked_region(b"/Sh1 sh", spec())); + assert_eq!(proof.entries[0].status, Status::Failed); +} + +#[test] +fn an_even_odd_clip_around_the_region_hides_what_it_cuts_out() { + // How kept content is told to hide: an even-odd clip of the page and the region. A path + // painted inside the region under that clip is clipped away; the same path under a clip + // the rule cannot read (a triangle, overlapping inners), or with no clip, is reported. + let blank = marked_content(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", Spec::default()); + let snap = Snapshot::capture(&blank, None).unwrap(); + let prove = |tail: &str| -> Proof { + let content = format!("BT /F1 10 Tf 10 200 Td (Public) Tj ET {tail}"); + let mut shown = marked_content(content.as_bytes(), Spec::default()); + shown.require_full_save(); + snap.prove(&shown, None, &ProofOptions::default()) + }; + let proof = prove("q 0 0 300 300 re 8 95 62 17 re W* n 0 0 0 rg 10 100 20 10 re f Q"); + assert!(proof.passed(), "{}", proof.to_text()); + + // A triangle is not a rectangle: no hole is cut, and the paint is reported. + let proof = prove("q 0 0 300 300 re 8 95 m 70 95 l 8 112 l h W* n 0 0 0 rg 10 100 20 10 re f Q"); + assert_eq!(proof.entries[0].status, Status::Failed, "{}", proof.to_text()); + assert!(proof.entries[0].detail.contains("path"), "{}", proof.to_text()); + + // Overlapping inners change the parity: no hole is cut either. + let proof = prove("q 0 0 300 300 re 8 95 62 17 re 5 90 70 25 re W* n 0 0 0 rg 10 100 20 10 re f Q"); + assert_eq!(proof.entries[0].status, Status::Failed, "{}", proof.to_text()); + + // Unclipped, the same path is a plain finding. + let proof = prove("0 0 0 rg 10 100 20 10 re f"); + assert_eq!(proof.entries[0].status, Status::Failed, "{}", proof.to_text()); +} + +#[test] +fn a_graphics_state_stack_too_deep_to_follow_is_unverifiable() { + // Saved states carry their clip holes with them: a `q` stack deep enough to grow without + // bound is refused, not followed. + let blank = marked_content(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", Spec::default()); + let snap = Snapshot::capture(&blank, None).unwrap(); + let mut deep = Vec::new(); + for _ in 0..crate::verify::limits::MAX_GS_STACK + 8 { + deep.extend_from_slice(b"q "); + } + deep.extend_from_slice(b"0 0 0 rg 10 100 20 10 re f"); + let mut shown = marked_content(&deep, Spec::default()); + shown.require_full_save(); + let proof = snap.prove(&shown, None, &ProofOptions::default()); + assert_eq!(proof.entries[0].status, Status::Unverifiable, "{}", proof.to_text()); + assert!(proof.entries[0].detail.contains("too deeply"), "{}", proof.to_text()); + assert!(!proof.passed()); +} + +#[test] +fn a_page_full_of_paint_reports_its_finding_once() { + // Fifty paths over a region are one finding: the report names the problem once, not once + // per operator, and the report does not grow with the operator count. + let blank = marked_content(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", Spec::default()); + let snap = Snapshot::capture(&blank, None).unwrap(); + let mut content = b"BT /F1 10 Tf 10 200 Td (Public) Tj ET ".to_vec(); + for k in 0..50 { + content.extend_from_slice(&format!("0 0 1 rg {} 100 4 4 re f ", 10 + k).into_bytes()); + } + let mut shown = marked_content(&content, Spec::default()); + shown.require_full_save(); + let proof = snap.prove(&shown, None, &ProofOptions::default()); + assert_eq!(proof.entries[0].status, Status::Failed, "{}", proof.to_text()); + assert_eq!(proof.entries[0].detail.matches("a path still paints").count(), 1, "{}", proof.to_text()); +} + +#[test] +fn text_kept_elsewhere_does_not_fail_the_proof() { + // The same word is redacted in one place and legitimately remains in another. + let content = b"BT /F1 10 Tf 10 200 Td (SECRETVALUE) Tj ET BT /F1 10 Tf 10 100 Td (SECRETVALUE) Tj ET"; + let mut doc = marked_content(content, Spec::default()); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert!(proof.passed(), "{}", proof.to_text()); + assert!(proof.entries[0].detail.contains("remain in retained content")); + + // A copy that should have gone is still caught, by count. + let proof = prove_unapplied(&mut marked_content(content, Spec::default())); + assert_eq!(surface(&proof, Surface::ExtractedText).verdict, Verdict::Survivor); +} + +#[test] +fn the_proof_text_keeps_its_honesty_wording() { + // The closing caveat of a proof is a promise to whoever reads a shared proof: pin it so the + // wording cannot silently drop out of the text for either verdict. + let honesty = "Evidence, not a guarantee"; + let passed = Proof { entries: vec![], surfaces: vec![], failures: vec![] }; + assert!(passed.passed()); + assert!(passed.to_text().contains(honesty), "{}", passed.to_text()); + + let failed = Proof { entries: vec![], surfaces: vec![], failures: vec!["the marked text is still readable".into()] }; + assert!(!failed.passed()); + assert!(failed.to_text().contains(honesty), "{}", failed.to_text()); +} + +// ── Independence, scope and bounds of the sweep ───────────────────────────────────────────── + +/// Hang `obj` off the catalog (as `/Extra`) so the saved file keeps it. +pub(super) fn attach(doc: &mut Document, obj: Object) { + let r = doc.add(obj); + let root = doc.root().unwrap(); + doc.update_dict(root, |d| d.set(b"Extra".to_vec(), Object::Ref(r))).unwrap(); +} + +/// Add a content stream (with `dict`) after the page's own. +fn append_content(doc: &mut Document, dict: Dict, data: &[u8]) { + let new = doc.add(Object::Stream(Stream::from_raw(dict, data.to_vec()))); + let page = pdfcraft_model::pages(doc).swap_remove(0); + let first = page.dict.get(b"Contents").cloned().unwrap(); + doc.update_dict(page.obj, |d| d.set(b"Contents".to_vec(), Object::Array(vec![first, Object::Ref(new)]))).unwrap(); +} + +#[test] +fn a_glyph_the_interpreter_kept_is_found_by_placement_alone() { + // The snapshot is taken where the region holds no text, so it has nothing to search for: + // only the independent placement of glyphs can notice the text in the output. + let blank = marked_content(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", Spec::default()); + let snap = Snapshot::capture(&blank, None).unwrap(); + let mut shown = marked(Spec::default()); + shown.require_full_save(); + let proof = snap.prove(&shown, None, &ProofOptions::default()); + assert!(!proof.passed()); + let e = &proof.entries[0]; + assert_eq!(e.status, Status::Failed, "{}", proof.to_text()); + assert!(e.detail.contains("glyph(s) still show"), "{}", e.detail); + assert_eq!(proof.survivors(), 0, "no needle was involved"); + // The neighbour at y 200 is outside the region and is not reported. + assert!(!e.detail.contains("12 glyph"), "{}", e.detail); +} + +#[test] +fn glyphs_are_placed_with_the_text_matrix_and_the_form_matrix() { + // The secret drawn through a rotated/translated text matrix inside a scaled form: the + // placement has to follow `Tm` and the form's `/Matrix`, not the raw operands. + let form = stream( + "/Type /XObject /Subtype /Form /BBox [0 0 300 300] /Matrix [1 0 0 1 0 0] /Resources << /Font << /F1 5 0 R >> >>", + b"BT /F1 10 Tf 1 0 0 1 10 100 Tm (SECRETVALUE) Tj ET", + ); + let blank = marked_content(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", Spec::default()); + let snap = Snapshot::capture(&blank, None).unwrap(); + let mut doc = marked_content(b"/Fm1 Do", Spec { resources: "/XObject << /Fm1 7 0 R >>", extra: vec![form], ..Spec::default() }); + doc.require_full_save(); + let proof = snap.prove(&doc, None, &ProofOptions::default()); + assert_eq!(proof.entries[0].status, Status::Failed, "{}", proof.to_text()); + assert!(proof.entries[0].detail.contains("glyph(s) still show")); +} + +#[test] +fn an_ambiguous_needle_is_still_swept_outside_page_content() { + // The word also stays on the page legitimately (so page text, the raw file and decoded + // streams can only count it), but a copy in the Info dictionary is not page content. + let content = b"BT /F1 10 Tf 10 200 Td (SECRETVALUE) Tj ET BT /F1 10 Tf 10 100 Td (SECRETVALUE) Tj ET"; + let spec = Spec { trailer: "/Info 7 0 R", extra: vec![b"<< /Title (SECRETVALUE) >>".to_vec()], ..Spec::default() }; + let mut doc = marked_content(content, spec); + let (_, proof) = run(&mut doc); + assert_eq!(surface(&proof, Surface::Info).verdict, Verdict::Survivor, "{}", proof.to_text()); + assert!(!proof.passed()); + + // Likewise an XMP packet and an annotation string. + let xmp = stream("/Type /Metadata /Subtype /XML", b"SECRETVALUE"); + let mut doc = marked_content(content, Spec { catalog: "/Metadata 7 0 R", extra: vec![xmp], ..Spec::default() }); + assert_eq!(surface(&run(&mut doc).1, Surface::Xmp).verdict, Verdict::Survivor); + let annot = b"<< /Type /Annot /Subtype /Text /Rect [200 200 220 220] /Contents (SECRETVALUE) >>".to_vec(); + let mut doc = marked_content(content, Spec { page: "/Annots [7 0 R]", extra: vec![annot], ..Spec::default() }); + assert_eq!(surface(&run(&mut doc).1, Surface::ObjectStrings).verdict, Verdict::Survivor); +} + +#[test] +fn a_pcmark_key_in_the_input_does_not_exempt_a_stream() { + // A content stream that claims to be a redaction box paints inside the region, and was not + // created by this run: it counts as paint like any other. + let blank = b"BT /F1 10 Tf 10 200 Td (Public) Tj ET"; + let mut doc = marked_content(blank, Spec::default()); + let mut d = Dict::new(); + d.set(b"PCMark".to_vec(), Object::Name(b"Redaction".to_vec())); + append_content(&mut doc, d, b"0 0 0 rg 8 95 62 17 re f"); + let proof = prove_unapplied(&mut doc); + let e = &proof.entries[0]; + assert_eq!(e.status, Status::Failed, "{}", proof.to_text()); + assert!(e.detail.contains("path"), "{}", e.detail); +} + +#[test] +fn only_the_streams_this_run_created_are_exempt() { + // The same stream, declared as created by this run, is the box that was drawn on purpose. + let blank = b"BT /F1 10 Tf 10 200 Td (Public) Tj ET"; + let mut doc = marked_content(blank, Spec::default()); + let mut snap = Snapshot::capture(&doc, None).unwrap(); + let new = doc.add(Object::Stream(Stream::from_raw(Dict::new(), b"0 0 0 rg 8 95 62 17 re f".to_vec()))); + let page = pdfcraft_model::pages(&doc).swap_remove(0); + let first = page.dict.get(b"Contents").cloned().unwrap(); + doc.update_dict(page.obj, |d| d.set(b"Contents".to_vec(), Object::Array(vec![first, Object::Ref(new)]))).unwrap(); + doc.require_full_save(); + snap.set_overlay_streams(&doc, &[new]).unwrap(); + let proof = snap.prove(&doc, None, &ProofOptions::default()); + assert!(proof.passed(), "{}", proof.to_text()); +} + +#[test] +fn overlay_text_does_not_collide_with_what_was_removed() { + // "Account" is redacted and the label says "Account [redacted]": the label is not a survivor. + let content = b"BT /F1 10 Tf 10 100 Td (Account 1234) Tj ET"; + let mut doc = open(build_with(content, Spec::default())); + let shape = Shape::Redact { quads: vec![rect_quad([8.0, 95.0, 100.0, 112.0])], overlay: "Account [redacted]".into(), look: Default::default() }; + let style = Style::default_for(&shape); + add_annotation(&mut doc, &NewAnnotation { page: 0, shape, style, contents: String::new(), author: "T".into() }, &Meta::default()).unwrap(); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert!(proof.passed(), "{}", proof.to_text()); + assert_eq!(surface(&proof, Surface::ExtractedText).verdict, Verdict::Clean); +} + +#[test] +fn removed_annotation_and_field_text_is_searched_for_too() { + let spec = || Spec { trailer: "/Info 7 0 R", extra: vec![b"<< /Subject (call back ANNOTWORDS) >>".to_vec()], ..Spec::default() }; + // Not asked for: the proof does not know about it. + let mut doc = marked(spec()); + let (_, proof) = run(&mut doc); + assert!(proof.passed(), "{}", proof.to_text()); + // Asked for: it is found where it was left. + let mut doc = marked(spec()); + let mut snap = Snapshot::capture(&doc, None).unwrap(); + snap.add_removed_strings(&[" call back ANNOTWORDS ".to_string(), "x".to_string(), String::new()]); + let result = applied(&mut doc, &mut snap); + assert!(result.is_ok() || residue(&result)); + let proof = snap.prove(&doc, None, &ProofOptions::default()); + assert!(!proof.passed()); + assert_eq!(surface(&proof, Surface::Info).verdict, Verdict::Survivor, "{}", proof.to_text()); + // And an absent string is clean. + let mut doc = marked(Spec::default()); + let mut snap = Snapshot::capture(&doc, None).unwrap(); + snap.add_removed_strings(&["never present anywhere".to_string()]); + applied(&mut doc, &mut snap).unwrap(); + assert!(snap.prove(&doc, None, &ProofOptions::default()).passed()); +} + +#[test] +fn a_damaged_stream_tail_is_unswept_not_truncated() { + let text: Vec = (0..4000u32).map(|i| b'a' + (i * 7 % 23) as u8).collect(); + let mut s = Stream::flate(Dict::new(), &text); + let cut = s.raw.len() - 8; + s.raw = Arc::new(s.raw[..cut].to_vec()).into(); + // Nothing was removed (the snapshot's region is blank), so the only issue is the stream. + let snap = Snapshot::capture(&marked_content(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", Spec::default()), None).unwrap(); + let mut doc = marked_content(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", Spec::default()); + attach(&mut doc, Object::Stream(s)); + doc.require_full_save(); + let proof = snap.prove(&doc, None, &ProofOptions::default()); + assert_eq!(surface(&proof, Surface::DecodedStreams).verdict, Verdict::Unswept, "{}", proof.to_text()); + assert!(!proof.passed()); +} + +#[test] +fn a_damaged_page_content_tail_makes_the_page_unreadable() { + let text: Vec = b"BT /F1 10 Tf 10 100 Td (filler) Tj ET\n".repeat(200); + let mut s = Stream::flate(Dict::new(), &text); + let cut = s.raw.len() - 8; + s.raw = Arc::new(s.raw[..cut].to_vec()).into(); + let mut doc = marked(Spec::default()); + let page = pdfcraft_model::pages(&doc).swap_remove(0); + let new = doc.add(Object::Stream(s)); + doc.update_dict(page.obj, |d| d.set(b"Contents".to_vec(), Object::Ref(new))).unwrap(); + doc.require_full_save(); + let snap = Snapshot::capture(&marked(Spec::default()), None).unwrap(); + let proof = snap.prove(&doc, None, &ProofOptions::default()); + assert!(!proof.passed()); + assert_eq!(surface(&proof, Surface::ExtractedText).verdict, Verdict::Unswept, "{}", proof.to_text()); +} + +#[test] +fn a_malformed_form_matrix_or_bbox_is_unswept() { + for dict in ["/Matrix [1 0 0]", "/Matrix [1 0 0 1 0 (x)]", "/BBox [0 0 300]"] { + let form = stream(&format!("/Type /XObject /Subtype /Form /BBox [0 0 300 300] {dict}"), b"BT /F1 10 Tf 10 100 Td (x) Tj ET"); + let spec = Spec { resources: "/XObject << /Fm1 7 0 R >>", extra: vec![form], ..Spec::default() }; + let mut doc = marked_content(b"/Fm1 Do", spec); + let proof = prove_unapplied(&mut doc); + assert_eq!(surface(&proof, Surface::ExtractedText).verdict, Verdict::Unswept, "{dict}: {}", proof.to_text()); + assert!(!proof.passed()); + assert!(proof.entries[0].detail.contains("malformed"), "{dict}: {}", proof.to_text()); + } +} + +#[test] +fn images_hidden_in_patterns_soft_masks_and_type3_glyphs_are_not_skipped() { + let image = "BI /W 1 /H 1 /CS /G /BPC 8 ID \u{ff} EI"; + let tile = + |data: &str| stream("/PatternType 1 /PaintType 1 /TilingType 1 /BBox [0 0 10 10] /XStep 10 /YStep 10 /Resources << >>", data.as_bytes()); + let group = stream("/Type /XObject /Subtype /Form /BBox [0 0 300 300] /Group << /S /Transparency >> /Resources << >>", image.as_bytes()); + let proc = stream("", format!("1000 0 d0 {image}").as_bytes()); + let cases: Vec<(&str, Spec)> = vec![ + ("pattern", Spec { resources: "/Pattern << /P1 7 0 R >>", extra: vec![tile(image)], ..Spec::default() }), + ( + "soft mask", + Spec { resources: "/ExtGState << /G1 << /SMask << /Type /Mask /S /Luminosity /G 7 0 R >> >> >>", extra: vec![group], ..Spec::default() }, + ), + ( + "type3", + Spec { + resources: "/Font << /F3 << /Type /Font /Subtype /Type3 /FontBBox [0 0 1000 1000] /FontMatrix [0.001 0 0 0.001 0 0] /CharProcs << /a 7 0 R >> /Encoding << /Type /Encoding /Differences [97 /a] >> /FirstChar 97 /LastChar 97 /Widths [1000] >> >>", + extra: vec![proc], + ..Spec::default() + }, + ), + ]; + for (what, spec) in cases { + // The region holds no text and no visible image: only the hidden image is in question. + let mut doc = marked_content(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", spec); + let proof = prove_unapplied(&mut doc); + let e = &proof.entries[0]; + assert_eq!(e.status, Status::Unverifiable, "{what}: {}", proof.to_text()); + assert!(e.detail.contains("pattern, soft mask or Type3"), "{what}: {}", e.detail); + assert!(!proof.passed()); + } + // The same resources without an image are fine. + let spec = Spec { resources: "/Pattern << /P1 7 0 R >>", extra: vec![tile("0 0 5 5 re f")], ..Spec::default() }; + let mut doc = marked_content(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", spec); + let proof = prove_unapplied(&mut doc); + assert_eq!(proof.entries[0].status, Status::VerifiedNoTextInRegion, "{}", proof.to_text()); +} + +#[test] +fn the_sweep_is_bounded() { + // A CMap whose ranges rewrite the same 64Ki codes again and again costs work, whatever the map size. + let mut cmap = b"1 begincodespacerange <0000> endcodespacerange\n".to_vec(); + for _ in 0..200 { + cmap.extend_from_slice(b"100 beginbfrange\n"); + for _ in 0..100 { + cmap.extend_from_slice(b"<0000> <0041>\n"); + } + cmap.extend_from_slice(b"endbfrange\n"); + } + let start = std::time::Instant::now(); + assert!(!tests_support::cmap_is_complete(&cmap)); + assert!(start.elapsed().as_secs() < 20, "bounded work"); + // A small CMap is read in full. + assert!(tests_support::cmap_is_complete(b"1 beginbfchar <01> <0041> endbfchar")); + // Nested objects, cumulative decoding and searching all stop and say so. + assert!(tests_support::walks_in_full(60)); + assert!(!tests_support::walks_in_full(200)); + assert_eq!(tests_support::decode_budget_runs_out(), (true, true)); + assert_eq!(tests_support::search_budget_runs_out(), (true, true)); + // A decode that fails past the one-stream cap pays for the attempt: the same bomb cannot + // be decoded (and fail) again and again for free. + assert_eq!(tests_support::failed_decode_charges_the_budget(), (true, true, true)); +} + +#[test] +fn the_search_finds_what_is_there_whatever_the_prefix() { + assert!(tests_support::found("aaaab", b"xxaaaaaaab")); + assert!(tests_support::found("needle", b"needle")); + assert!(!tests_support::found("needle", b"needl")); + assert!(!tests_support::found("aaab", b"aaaaaaaaaa")); + assert!(tests_support::found("abc", b"zabc")); + // The worst shape for a skip-based search (a needle of one letter and a haystack full of + // it) is walked once per byte, and still finds what is there. + let needle = format!("{}b", "a".repeat(64)); + let mut hay = vec![b'a'; 20_000]; + assert!(!tests_support::found(&needle, &hay)); + hay.extend_from_slice(needle.as_bytes()); + assert!(tests_support::found(&needle, &hay)); +} + +#[test] +fn the_sweep_charge_stays_fair_for_a_bulk_of_needles() { + // 80 removed strings swept over one 5 MiB haystack (a stream decoded in full): the search + // really is one pass per needle, but the budget must not charge the haystack once per + // needle — that charge (haystack × needle × spelling: 80 × 20 × 5 MiB, about twice the 4 GiB + // work budget) would refuse an ordinary bulk redaction out of hand. The haystack is charged + // once per scan and a needle's own charge is capped, so the sweep stays under the budget. + let hay = vec![b'x'; 5 << 20]; + let word = "SECRETVALUE-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + assert_eq!(tests_support::bulk_counts(&hay, word, 80, 0), Some(0)); + // Genuinely unbounded work still fails closed: the region scanned is charged in full, so a + // haystack that no longer fits in the remaining budget is an unswept surface, never a pass. + assert_eq!(tests_support::bulk_counts(&hay, word, 80, crate::verify::tests_support::work_budget() - (1 << 20)), None); +} + +#[test] +fn survivors_seen_before_the_work_limit_are_still_reported() { + // The page sweep stops when the work budget runs out, but the survivors it has already + // counted are settled: a survivor on an earlier page is reported where it sits, not hidden + // behind the limit (the page still reads unswept, so the proof still fails). + let (reported, unswept) = tests_support::page_sweep_survivors_survive_the_limit(); + assert!(reported, "a survivor counted before the limit is reported"); + assert!(unswept, "the page the limit stopped on is unswept"); +} + +#[test] +fn a_file_past_the_proof_budget_is_unverified_never_clean() { + // A page whose content is bigger than the proof may decode for one stream: the budget + // stops the sweep, and the result must say so. The surfaces read unswept (a page content + // stream is never excused as raw-only), the region reads unverifiable, the proof fails. + // Work that stopped is "not verified", never a pass. + let big = vec![0u8; crate::verify::limits::MAX_STREAM + (6 << 20)]; + let content = Stream::flate(Dict::new(), &big); + drop(big); + let mut doc = open(build_with(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", Spec::default())); + let page = pdfcraft_model::pages(&doc).swap_remove(0); + let new = doc.add(Object::Stream(content)); + doc.update_dict(page.obj, |d| d.set(b"Contents".to_vec(), Object::Ref(new))).unwrap(); + doc.require_full_save(); + let snap = Snapshot::capture(&marked(Spec::default()), None).unwrap(); + let proof = snap.prove(&doc, None, &ProofOptions::default()); + assert!(!proof.passed(), "{}", proof.to_text()); + assert!(!proof.failures.is_empty(), "{}", proof.to_text()); + for s in [Surface::ExtractedText, Surface::DecodedStreams] { + let r = surface(&proof, s); + assert_eq!(r.verdict, Verdict::Unswept, "{s:?}: {}", proof.to_text()); + assert_eq!((r.survivors, r.raw_only), (0, 0), "{s:?}: {}", proof.to_text()); + } + assert!(matches!(proof.entries[0].status, Status::Unverifiable | Status::Failed), "the region is not verified: {}", proof.to_text()); +} + +#[test] +fn font_widths_parsing_shares_the_work_budget() { + // A form invoked again rebuilds its fonts, so the `/W` array is paid for from the shared + // work budget, not from a per-font counter that every rebuild resets. + let (full, refused) = tests_support::font_widths_charge_shares_the_budget(); + assert!(full, "a small /W array is read in full"); + assert!(refused, "a spent work budget refuses the /W parsing"); +} + +#[test] +fn an_unreadable_overlay_stream_is_its_own_error() { + let mut doc = marked(Spec::default()); + let mut snap = Snapshot::capture(&doc, None).unwrap(); + let mut dict = Dict::new(); + dict.set(b"Filter".to_vec(), Object::Name(b"FlateDecode".to_vec())); + let broken = doc.add(Object::Stream(Stream::from_raw(dict, b"not deflate data".to_vec()))); + let err = snap.set_overlay_streams(&doc, &[broken]).unwrap_err(); + assert_eq!(err, RedactError::OverlayUnreadable); + assert!(!err.to_string().contains("page 0"), "{err}"); +} + +// ── The caller's credentials ───────────────────────────────────────────────────────────────── + +#[test] +fn a_nonempty_user_password_document_is_proven_with_those_credentials() { + // The redacted document is protected with a user password and saved: the proof's subject is + // now an encrypted file, and only the caller's own credentials can read it back. Every + // supported encryption algorithm. + for alg in [Algorithm::Rc4_40, Algorithm::Rc4_128, Algorithm::Aes128, Algorithm::Aes256] { + let mut doc = marked(Spec::default()); + let mut snap = Snapshot::capture(&doc, None).unwrap(); + applied(&mut doc, &mut snap).unwrap(); + doc.set_encryption(&NewEncryption { + algorithm: alg, + user_password: "pw-secret", + owner_password: "owner-secret", + permissions: -1, + encrypt_metadata: true, + seed: [7; 32], + }) + .unwrap(); + + // The credentials the caller holds prove the saved output. + let proof = snap.prove(&doc, Some("pw-secret"), &ProofOptions::default()); + assert!(proof.passed(), "{alg:?}: {}", proof.to_text()); + + // An already saved file too. + let saved = write_full(&doc, &SaveOptions::default()).unwrap(); + let proof = snap.prove_saved(&saved, Some("pw-secret"), &ProofOptions::default()); + assert!(proof.passed(), "{alg:?}: {}", proof.to_text()); + + // Empty and wrong credentials fail closed: the output can't be read back, which is an + // unswept proof, never a pass. + for pw in [None, Some("wrong")] { + let proof = snap.prove_saved(&saved, pw, &ProofOptions::default()); + assert!(!proof.passed(), "{alg:?} {pw:?}: {}", proof.to_text()); + for s in &proof.surfaces { + assert_eq!(s.verdict, Verdict::Unswept, "{alg:?} {pw:?}: {}", proof.to_text()); + } + } + } +} + +// ── Compressed copies inside binary streams ────────────────────────────────────────────────── + +/// A stream object whose stored bytes are the Flate encoding of `data` (as `Stream::flate` +/// writes them), with the extra dictionary entries `dict`. +fn flate_object(dict: &str, data: &[u8]) -> Vec { + stream(&format!("{dict} /Filter /FlateDecode"), &Stream::flate(Dict::new(), data).raw) +} + +/// A one-page file whose resources point at a fake font: /F2 is a /TrueType font whose +/// /FontDescriptor carries `font_file` as its /FontFile2 (objects 7, 8 and 9). The font carries +/// `/Widths` (object 6) so the interpreter resolves it; nothing on the page draws with it. +fn font_file_spec(font_file: Vec) -> Spec<'static> { + Spec { + resources: "/Font << /F1 5 0 R /F2 7 0 R >>", + extra: vec![ + b"<< /Type /Font /Subtype /TrueType /BaseFont /X /FirstChar 32 /LastChar 126 /Widths 6 0 R /FontDescriptor 8 0 R >>".to_vec(), + b"<< /Type /FontDescriptor /FontName /X /Flags 4 /FontFile2 9 0 R >>".to_vec(), + font_file, + ], + ..Spec::default() + } +} + +#[test] +fn a_compressed_copy_in_a_fake_font_file_is_found() { + // The visible copy is redacted normally, but a Flate-compressed copy of the secret hides in + // a fake /FontFile2. /Length1 makes the sweep class the stream as a binary font program, so + // its stored bytes hold no plaintext: only the decoded data gives the copy away. + let spec = font_file_spec(flate_object("/Length1 900", SECRET.as_bytes())); + let mut doc = marked(spec); + let (_, proof) = run(&mut doc); + assert_eq!(surface(&proof, Surface::DecodedStreams).verdict, Verdict::Survivor, "{}", proof.to_text()); + let s = surface(&proof, Surface::DecodedStreams); + assert!(s.locations.iter().any(|l| l.contains("(decoded)")), "{:?}", s.locations); + assert!(!proof.passed()); +} + +#[test] +fn a_legitimate_font_file_is_swept_and_clean() { + // A font program that holds none of what was removed (as real ones do not) is searched and + // lets the redaction pass: no false survivor. + let spec = font_file_spec(flate_object("/Length1 900", b"\x00\x01\x00\x00\x00\x10glyph tables")); + let mut doc = marked(spec); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert!(proof.passed(), "{}", proof.to_text()); + assert_eq!(surface(&proof, Surface::DecodedStreams).verdict, Verdict::Clean, "{}", proof.to_text()); +} + +#[test] +fn a_font_program_that_will_not_decode_is_searched_as_stored_bytes() { + // An unrelated, damaged font program the page reaches is not a reason to refuse: nothing + // redacted touched it, so its stored bytes are searched and it is counted as raw only. + let raw = Stream::flate(Dict::new(), b"a font program without the secret").raw.as_ref().to_vec(); + let cut = raw.len() - 8; + let spec = font_file_spec(stream("/Length1 900 /Filter /FlateDecode", &raw[..cut])); + let mut doc = marked(spec); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert!(proof.passed(), "{}", proof.to_text()); + let s = surface(&proof, Surface::DecodedStreams); + assert_eq!((s.verdict, s.raw_only), (Verdict::Clean, 1), "{}", proof.to_text()); +} + +/// The standard page plus an image (object 7) drawn away from the secret. +fn image_doc(image: Vec) -> Document { + let mut content = CONTENT.to_vec(); + content.extend_from_slice(b" q 20 0 0 20 200 200 cm /Im1 Do Q"); + marked_content(&content, Spec { resources: "/XObject << /Im1 7 0 R >>", extra: vec![image], ..Spec::default() }) +} + +const IMG: &str = "/Type /XObject /Subtype /Image /Width 20 /Height 20 /ColorSpace /DeviceGray /BitsPerComponent 8 /Filter /FlateDecode"; + +fn assert_redacted_cleanly(mut doc: Document) -> Proof { + let (r, proof) = run(&mut doc); + r.unwrap(); + assert!(proof.passed(), "{}", proof.to_text()); + let saved = write_incremental(&doc, &SaveOptions::default()).unwrap(); + assert!(!saved.windows(SECRET.len()).any(|w| w == SECRET.as_bytes()), "the secret is gone from the file"); + proof +} + +#[test] +fn an_unrelated_truncated_flate_image_does_not_fail_the_proof() { + let raw = Stream::flate(Dict::new(), &[0x80u8; 400]).raw.as_ref().to_vec(); + let proof = assert_redacted_cleanly(image_doc(stream(IMG, &raw[..raw.len() - 8]))); + assert_eq!(surface(&proof, Surface::DecodedStreams).raw_only, 1, "{}", proof.to_text()); +} + +#[test] +fn an_unrelated_image_over_the_stream_cap_does_not_fail_the_proof() { + // Highly compressible: stored in a few dozen KiB, decoded past MAX_STREAM. + let big = vec![0u8; crate::verify::limits::MAX_STREAM + (6 << 20)]; + let raw = Stream::flate(Dict::new(), &big).raw.as_ref().to_vec(); + drop(big); + let proof = assert_redacted_cleanly(image_doc(stream(IMG, &raw))); + assert_eq!(surface(&proof, Surface::DecodedStreams).raw_only, 1, "{}", proof.to_text()); +} + +#[test] +fn an_unrelated_truncated_font_program_does_not_fail_the_redaction() { + let raw = Stream::flate(Dict::new(), b"glyph tables").raw.as_ref().to_vec(); + let doc = marked(font_file_spec(stream("/Length1 900 /Filter /FlateDecode", &raw[..raw.len() - 6]))); + assert_redacted_cleanly(doc); +} + +#[test] +fn a_secret_in_the_stored_bytes_of_a_font_program_is_still_found() { + // The font program cannot be decoded (it claims Flate but holds plain bytes), so only its + // stored bytes are searched: the copy of the secret in them is a survivor. + let spec = font_file_spec(stream("/Length1 900 /Filter /FlateDecode", format!("not deflate {SECRET} not deflate").as_bytes())); + let mut doc = marked(spec); + let (_, proof) = run(&mut doc); + let s = surface(&proof, Surface::DecodedStreams); + assert_eq!((s.verdict, s.raw_only), (Verdict::Survivor, 1), "{}", proof.to_text()); + assert!(s.locations.iter().any(|l| l.contains("(raw)")), "{:?}", s.locations); + assert!(!proof.to_text().contains(SECRET)); + assert!(!proof.passed()); +} + +/// The standard page plus an /Indexed colour space whose lookup table is object 7. +fn indexed_doc(lookup: Vec) -> Document { + marked_content(CONTENT, Spec { resources: "/ColorSpace << /CS1 [/Indexed /DeviceRGB 255 7 0 R] >>", extra: vec![lookup], ..Spec::default() }) +} + +#[test] +fn an_indexed_lookup_stream_is_searched_as_stored_bytes_only() { + // Unrelated and damaged: nothing redacted touched it, so it must not fail the proof. + let raw = Stream::flate(Dict::new(), &[0x40u8; 300]).raw.as_ref().to_vec(); + let proof = assert_redacted_cleanly(indexed_doc(stream("/Filter /FlateDecode", &raw[..raw.len() - 8]))); + assert_eq!(surface(&proof, Surface::DecodedStreams).raw_only, 1, "{}", proof.to_text()); + // A secret stored raw inside one that will not decode is still found. + let mut doc = indexed_doc(stream("/Filter /FlateDecode", format!("not deflate {SECRET} not deflate").as_bytes())); + let (_, proof) = run(&mut doc); + let s = surface(&proof, Surface::DecodedStreams); + assert_eq!((s.verdict, s.raw_only), (Verdict::Survivor, 1), "{}", proof.to_text()); + assert!(!proof.passed()); +} + +/// Capture a sound document, then damage the stream `num` (a Flate stream cut short, with the +/// extra dictionary entries `dict`) and prove it. +fn prove_damaged(mut doc: Document, num: u32, dict: &str, data: &[u8]) -> Proof { + let snap = Snapshot::capture(&doc, None).unwrap(); + let raw = Stream::flate(Dict::new(), data).raw.as_ref().to_vec(); + let cut = raw[..raw.len() - 8].to_vec(); + let broken = open(pdf_bytes(vec![b"<< /Type /Catalog >>".to_vec(), stream(&format!("{dict} /Filter /FlateDecode"), &cut)], "")); + let obj = broken.get(ObjRef::new(2, 0)).as_ref().clone(); + doc.set(ObjRef::new(num, 0), obj); + doc.require_full_save(); + snap.prove(&doc, None, &ProofOptions::default()) +} + +#[test] +fn damaged_content_appearance_and_metadata_still_fail_closed() { + // A truncated second content stream. + let objs: Vec> = vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents [4 0 R 7 0 R] /Resources << /Font << /F1 5 0 R >> >> >>".to_vec(), + stream("", CONTENT), + FONT.as_bytes().to_vec(), + format!("[{}]", vec!["500"; 95].join(" ")).into_bytes(), + stream("", b"BT /F1 10 Tf 10 200 Td (more text here) Tj ET"), + ]; + let mut doc = open(pdf_bytes(objs, "/Root 1 0 R")); + mark(&mut doc, &[REGION]); + let proof = prove_damaged(doc, 7, "", b"BT /F1 10 Tf 10 200 Td (more text here) Tj ET"); + assert!(!proof.passed()); + assert!(!surface(&proof, Surface::ExtractedText).problems.is_empty(), "{}", proof.to_text()); + assert!(!surface(&proof, Surface::DecodedStreams).problems.is_empty(), "{}", proof.to_text()); + + // A truncated appearance stream. + let annot = b"<< /Type /Annot /Subtype /Square /Rect [200 200 250 250] /AP << /N 8 0 R >> >>".to_vec(); + let ap = stream("/Type /XObject /Subtype /Form /BBox [0 0 50 50]", b"0 0 50 50 re f 0 0 1 rg 1 0 0 RG 2 w"); + let mut doc = marked(Spec { page: "/Annots [7 0 R]", extra: vec![annot, ap], ..Spec::default() }); + doc.require_full_save(); + let proof = prove_damaged(doc, 8, "/Type /XObject /Subtype /Form /BBox [0 0 50 50]", b"0 0 50 50 re f 0 0 1 rg 1 0 0 RG 2 w"); + assert!(!proof.passed()); + assert!(!surface(&proof, Surface::ExtractedText).problems.is_empty(), "{}", proof.to_text()); + + // A truncated XMP packet. + let xmp = stream("/Type /Metadata /Subtype /XML", b"some metadata of the file"); + let doc = marked(Spec { catalog: "/Metadata 7 0 R", extra: vec![xmp], ..Spec::default() }); + let proof = prove_damaged(doc, 7, "/Type /Metadata /Subtype /XML", b"some metadata of the file"); + assert!(!proof.passed()); + assert!(!surface(&proof, Surface::Xmp).problems.is_empty(), "{}", proof.to_text()); +} + +#[test] +fn an_annotation_appearance_is_checked_however_it_is_stored() { + // The region holds no text, so only the paint check can speak. An annotation written in + // place (a direct dictionary, no reference of its own) is checked like an indirect one, and + // a stream shared by two annotations is checked at each annotation's own placement, so the + // copy that lands over the region is seen even though the first placement does not reach it. + let blank = marked_content(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", Spec::default()); + let snap = Snapshot::capture(&blank, None).unwrap(); + let prove = |spec: Spec<'_>| -> Proof { + let mut shown = marked_content(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", spec); + shown.require_full_save(); + snap.prove(&shown, None, &ProofOptions::default()) + }; + let ap = stream("/Type /XObject /Subtype /Form /BBox [0 0 62 17] /Resources << >>", b"0 0 0 rg 10 5 20 10 re f"); + + let proof = prove(Spec { + page: "/Annots [<< /Type /Annot /Subtype /Square /Rect [8 95 70 112] /AP << /N 7 0 R >> >>]", + extra: vec![ap], + ..Spec::default() + }); + assert_eq!(proof.entries[0].status, Status::Failed, "{}", proof.to_text()); + assert!(proof.entries[0].detail.contains("path"), "{}", proof.to_text()); + + let ap = stream("/Type /XObject /Subtype /Form /BBox [0 0 62 17] /Resources << >>", b"0 0 0 rg 10 5 20 10 re f"); + let far = b"<< /Type /Annot /Subtype /Square /Rect [200 200 250 250] /AP << /N 7 0 R >> >>".to_vec(); + let near = b"<< /Type /Annot /Subtype /Square /Rect [8 95 70 112] /AP << /N 7 0 R >> >>".to_vec(); + let proof = prove(Spec { page: "/Annots [8 0 R 9 0 R]", extra: vec![ap, far, near], ..Spec::default() }); + assert_eq!(proof.entries[0].status, Status::Failed, "{}", proof.to_text()); + assert!(proof.entries[0].detail.contains("path"), "{}", proof.to_text()); +} + +#[test] +fn a_null_entry_in_the_contents_array_is_an_empty_piece() { + let objs: Vec> = vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents [4 0 R null] /Resources << /Font << /F1 5 0 R >> >> >>".to_vec(), + stream("", CONTENT), + FONT.as_bytes().to_vec(), + format!("[{}]", vec!["500"; 95].join(" ")).into_bytes(), + ]; + let mut doc = open(pdf_bytes(objs, "/Root 1 0 R")); + mark(&mut doc, &[REGION]); + assert_redacted_cleanly(doc); +} + +/// The page's decoded content streams, joined (what the redaction left on the page). +fn page_content(doc: &Document) -> String { + let p = &pdfcraft_model::pages(doc)[0]; + let (_, data) = page_streams(doc, &p.dict, 0).unwrap(); + data.iter().map(|d| String::from_utf8_lossy(d).into_owned()).collect::>().join("\n") +} diff --git a/crates/redact/src/verify/extract.rs b/crates/redact/src/verify/extract.rs new file mode 100644 index 0000000000..f6db96229b --- /dev/null +++ b/crates/redact/src/verify/extract.rs @@ -0,0 +1,711 @@ +//! A minimal text extractor, independent of the interpreter's font metrics and hit test. + +use std::collections::HashMap; + +use pdfcraft_content::{Matrix, parse}; +use pdfcraft_cos::{Dict, Document, ObjRef, Object, Stream}; + +use super::util::{Ctx, latin1, resolve_strict, squash}; +use crate::annots_of; +use crate::verify::limits::{MAX_CMAP, MAX_CMAP_ENTRIES, MAX_CMAP_WORK, MAX_DEPTH}; + +/// Is this one of the glyph-carrying text-showing operators (`Tj`, `'`, `"`, and `TJ` with at +/// least one non-empty string)? A `TJ` of numbers only (the gap redaction leaves for a removed +/// string) moves the cursor but shows nothing, so it does not count. +pub(super) fn carries_glyphs(op: &pdfcraft_content::Op) -> bool { + let nonempty = |o: &Object| o.as_string().is_some_and(|s| !s.bytes.is_empty()); + match op.op.as_slice() { + b"Tj" | b"'" | b"\"" => op.operands.last().is_some_and(nonempty), + b"TJ" => op.operands.first().and_then(Object::as_array).is_some_and(|a| a.iter().any(nonempty)), + _ => false, + } +} + +/// A page's text as two whitespace-free views: decoded text, and the character codes as Latin-1 +/// (what fonts without a usable mapping show). +#[derive(Default)] +pub(super) struct Extracted { + pub(super) text: String, + pub(super) raw: String, + pub(super) glyph_ops: usize, + /// Glyph-carrying operators whose text could not be decoded. + pub(super) undecodable_ops: usize, + /// Per region asked about: glyphs whose centre this extractor places inside it. + pub(super) shown_in: Vec, + /// Glyph-carrying operators drawn where this extractor can't place them (tiling patterns, + /// or an annotation appearance without a usable `/Rect`). + pub(super) unplaced: usize, +} + +/// How one font turns codes into text. +#[derive(Clone, Default)] +pub(super) struct Decoder { + pub(super) map: HashMap, + /// Bytes per code. + pub(super) width: usize, + /// No way to read these codes as text (composite font without a mapping). + pub(super) opaque: bool, +} + +impl Decoder { + fn latin1() -> Self { + Decoder { map: HashMap::new(), width: 1, opaque: false } + } + + /// `Err` when the font's own data can't be read in full (a damaged or oversized CMap). + fn new(doc: &Document, font: &Dict, ctx: &Ctx) -> Result { + let composite = font.name(b"Subtype") == Some(b"Type0"); + let mut d = Decoder { map: HashMap::new(), width: if composite { 2 } else { 1 }, opaque: false }; + let tu = font.get(b"ToUnicode").map(|t| doc.resolve(t)); + if let Some(Object::Stream(s)) = tu.as_deref() { + let data = ctx.decode(s)?; + if data.len() > MAX_CMAP { + return Err(()); + } + let (map, width) = parse_cmap(&data).ok_or(())?; + if !map.is_empty() { + d.map = map; + if let Some(w) = width { + d.width = w; + } + return Ok(d); + } + } + if composite { + d.opaque = true; + return Ok(d); + } + // Simple font: Latin-1, with the /Differences of the encoding. + if let Some(enc) = font.get(b"Encoding").map(|e| doc.resolve(e)) + && let Some(diff) = enc.as_dict().and_then(|e| e.get(b"Differences")).map(|x| doc.resolve(x)) + && let Some(items) = diff.as_array() + { + let mut code = 0u32; + for it in items { + match &*doc.resolve(it) { + Object::Int(i) => code = u32::try_from(*i).unwrap_or(0), + Object::Name(n) => { + if let Some(c) = std::str::from_utf8(n).ok().and_then(pdfcraft_fonts::pdf::glyph_unicode) { + d.map.insert(code, c.to_string()); + } + code = code.saturating_add(1); + } + _ => {} + } + } + } + Ok(d) + } + + /// Each code of a shown string, with the text it gives (`None`: no way to read it as text). + fn split(&self, s: &[u8]) -> Vec<(u32, Option)> { + let w = self.width.max(1); + s.chunks(w) + .map(|c| { + let code = code_of(c); + let text = if self.opaque { + None + } else { + match self.map.get(&code) { + Some(t) => Some(t.clone()), + None if self.map.is_empty() || w == 1 && code < 256 => Some(char::from(u8::try_from(code).unwrap_or(b'?')).to_string()), + None => None, + } + }; + (code, text) + }) + .collect() + } + + /// The text of a shown string. + fn decode(&self, s: &[u8]) -> String { + self.split(s).into_iter().filter_map(|(_, t)| t).collect() + } +} + +/// A font as the geometric check sees it: its decoder and its own simple width model. +#[derive(Clone)] +pub(super) struct Font { + pub(super) dec: Decoder, + /// Advance per code, in text space units per 1 em of font size (0.5 = half an em). + pub(super) widths: HashMap, + /// Advance of a code `widths` doesn't list. + pub(super) default_width: f64, + pub(super) vertical: bool, +} + +/// The width a font without usable `/Widths` is assumed to have (half an em). +pub(super) const FLAT_WIDTH: f64 = 0.5; + +impl Font { + fn flat() -> Self { + Font { dec: Decoder::latin1(), widths: HashMap::new(), default_width: FLAT_WIDTH, vertical: false } + } + + fn new(doc: &Document, font: &Dict, ctx: &Ctx) -> Result { + let dec = Decoder::new(doc, font, ctx)?; + let mut f = Font { dec, widths: HashMap::new(), default_width: FLAT_WIDTH, vertical: false }; + let num = |o: &Object| doc.resolve(o).as_f64(); + if font.name(b"Subtype") == Some(b"Type0") { + f.vertical = font.get(b"Encoding").map(|e| doc.resolve(e)).and_then(|e| e.as_name().map(|n| n.ends_with(b"-V"))).unwrap_or(false); + let desc = font.get(b"DescendantFonts").map(|d| doc.resolve(d)).and_then(|d| d.as_array().and_then(|a| a.first().cloned())); + let desc = desc.and_then(|d| doc.resolve(&d).as_dict().cloned()); + if let Some(d) = desc { + f.default_width = d.get(b"DW").and_then(num).map_or(1.0, |w| w / 1000.0); + // The array is charged to the shared work budget as it is taken (a form + // invoked again rebuilds its fonts, so a per-font counter alone would let the + // same work be paid for many times), and every entry is paid for there too, + // whether or not it grows the map (a range may rewrite the same codes again + // and again, and the array is bounded only by the file). + let items: Vec = match d.get(b"W").map(|w| doc.resolve(w)).and_then(|w| w.as_array().cloned()) { + Some(a) => { + if !ctx.charge(a.len()) { + return Err(()); + } + a + } + None => Vec::new(), + }; + let mut work = 0usize; + let mut step = |n: usize| -> Result<(), ()> { + work = work.saturating_add(n); + if work <= MAX_CMAP_WORK && ctx.charge(n) { Ok(()) } else { Err(()) } + }; + let mut i = 0; + while let Some(first) = items.get(i).and_then(num) { + let first = first.max(0.0) as u32; + match items.get(i + 1).map(|o| doc.resolve(o)) { + Some(a) if a.as_array().is_some() => { + step(a.as_array().map_or(0, Vec::len))?; + for (k, w) in a.as_array().into_iter().flatten().enumerate() { + if f.widths.len() >= MAX_CMAP_ENTRIES { + return Err(()); + } + if let (Some(w), Ok(k)) = (num(w), u32::try_from(k)) { + f.widths.insert(first.saturating_add(k), w / 1000.0); + } + } + i += 2; + } + Some(last) => { + let (Some(last), Some(w)) = (last.as_f64(), items.get(i + 2).and_then(num)) else { break }; + let last = (last.max(0.0) as u32).min(first.saturating_add(0xFFFF)); + // Each code the range covers is paid for, grown or not (a span is at + // most 0x10000 codes, so the count always fits). + step(last.saturating_sub(first) as usize + 1)?; + for c in first..=last { + if f.widths.len() >= MAX_CMAP_ENTRIES { + return Err(()); + } + f.widths.insert(c, w / 1000.0); + } + i += 3; + } + None => break, + } + } + } + } else { + // Glyph space units per em: 1000, or what a Type3 font's matrix says. + let scale = if font.name(b"Subtype") == Some(b"Type3") { + font.get(b"FontMatrix").map(|m| doc.resolve(m)).and_then(|m| m.as_array().and_then(|a| a.first().and_then(num))).unwrap_or(0.001) + } else { + 0.001 + }; + let first = font.get(b"FirstChar").and_then(num).map_or(0, |v| v.max(0.0) as u32); + let missing = font + .get(b"FontDescriptor") + .map(|d| doc.resolve(d)) + .and_then(|d| d.as_dict().and_then(|d| d.get(b"MissingWidth").and_then(num))) + .map(|w| w * scale); + if let Some(ws) = font.get(b"Widths").map(|w| doc.resolve(w)).and_then(|w| w.as_array().cloned()) { + if ws.len() > MAX_CMAP_ENTRIES { + return Err(()); + } + for (k, w) in ws.iter().enumerate() { + if let (Some(w), Ok(k)) = (num(w), u32::try_from(k)) { + f.widths.insert(first.saturating_add(k), w * scale); + } + } + f.default_width = missing.unwrap_or(0.0); + } else if let Some(std) = pdfcraft_fonts::Std14::from_base_font(&String::from_utf8_lossy(font.name(b"BaseFont").unwrap_or(b""))) { + // No /Widths: a standard font has exact published widths, by WinAnsi code (Symbol, + // ZapfDingbats and other encodings are refused before the proof is reached). + for code in 0..=255u8 { + f.widths.insert(u32::from(code), std.width(code) / 1000.0); + } + f.default_width = if std.is_fixed_pitch() { 0.6 } else { 0.0 }; + } + } + Ok(f) + } + + fn width(&self, code: u32) -> f64 { + self.widths.get(&code).copied().unwrap_or(self.default_width) + } +} + +pub(super) fn hex_token(tok: &[u8]) -> Option> { + let digits: Vec = tok.iter().filter_map(|c| (*c as char).to_digit(16).map(|d| d as u8)).collect(); + if digits.is_empty() || !digits.len().is_multiple_of(2) { + return None; + } + Some(digits.chunks(2).map(|p| p.iter().fold(0u8, |a, d| (a << 4) | d)).collect()) +} + +pub(super) fn utf16_text(b: &[u8]) -> String { + if let [c] = b { + return char::from(*c).to_string(); + } + let units: Vec = b.as_chunks::<2>().0.iter().map(|c| u16::from_be_bytes(*c)).collect(); + String::from_utf16_lossy(&units) +} + +pub(super) fn code_of(b: &[u8]) -> u32 { + b.iter().fold(0u32, |a, x| a.wrapping_shl(8) | u32::from(*x)) +} + +pub(super) enum Tok { + Hex(Vec), + Word(String), + Open, + Close, +} + +pub(super) fn cmap_tokens(data: &[u8]) -> Vec { + let mut out = Vec::new(); + let mut i = 0; + while let Some(&c) = data.get(i) { + match c { + b'<' if data.get(i + 1) != Some(&b'<') => { + let end = data.get(i..).and_then(|d| d.iter().position(|x| *x == b'>')).map_or(data.len(), |e| i + e); + if let Some(h) = data.get(i + 1..end).and_then(hex_token) { + out.push(Tok::Hex(h)); + } + i = end + 1; + } + b'[' => { + out.push(Tok::Open); + i += 1; + } + b']' => { + out.push(Tok::Close); + i += 1; + } + b'%' => { + while data.get(i).is_some_and(|x| !matches!(x, b'\n' | b'\r')) { + i += 1; + } + } + c if c.is_ascii_alphabetic() => { + let start = i; + while data.get(i).is_some_and(|x| x.is_ascii_alphanumeric()) { + i += 1; + } + out.push(Tok::Word(String::from_utf8_lossy(data.get(start..i).unwrap_or_default()).into_owned())); + } + _ => i += 1, + } + } + out +} + +/// `bfchar` / `bfrange` entries of a ToUnicode CMap, and the code length of its codespace. +/// `None` when the CMap is bigger than [`MAX_CMAP_ENTRIES`] entries or costs more than +/// [`MAX_CMAP_WORK`] steps: a map that is cut short would read as "no such code". +pub(super) fn parse_cmap(data: &[u8]) -> Option<(HashMap, Option)> { + let toks = cmap_tokens(data); + let mut map: HashMap = HashMap::new(); + let mut width = None; + // Every step is paid for, whether or not it grows the map (a range may rewrite the same + // codes again and again). + let mut work = 0usize; + let mut step = |n: usize| -> Option<()> { + work = work.saturating_add(n); + (work <= MAX_CMAP_WORK).then_some(()) + }; + let mut i = 0; + while let Some(t) = toks.get(i) { + i += 1; + let Tok::Word(w) = t else { continue }; + match w.as_str() { + "begincodespacerange" => { + if let Some(Tok::Hex(h)) = toks.get(i) { + width = Some(h.len()); + } + } + "beginbfchar" => { + while let (Some(Tok::Hex(src)), Some(Tok::Hex(dst))) = (toks.get(i), toks.get(i + 1)) { + step(1)?; + map.insert(code_of(src), utf16_text(dst)); + if map.len() > MAX_CMAP_ENTRIES { + return None; + } + i += 2; + } + } + "beginbfrange" => { + while let (Some(Tok::Hex(lo)), Some(Tok::Hex(hi))) = (toks.get(i), toks.get(i + 1)) { + let (lo, hi) = (code_of(lo), code_of(hi)); + let span = hi.saturating_sub(lo).min(0xFFFF); + match toks.get(i + 2) { + Some(Tok::Hex(dst)) => { + let base = utf16_text(dst); + step(usize::try_from(span).ok()?.saturating_add(1))?; + for k in 0..=span { + let mut chars: Vec = base.chars().collect(); + if let Some(last) = chars.last_mut() { + *last = char::from_u32(u32::from(*last).saturating_add(k)).unwrap_or('\u{fffd}'); + } + map.insert(lo.saturating_add(k), chars.into_iter().collect()); + } + if map.len() > MAX_CMAP_ENTRIES { + return None; + } + i += 3; + } + Some(Tok::Open) => { + i += 3; + let mut k = 0u32; + while let Some(Tok::Hex(d)) = toks.get(i) { + step(1)?; + map.insert(lo.saturating_add(k), utf16_text(d)); + if map.len() > MAX_CMAP_ENTRIES { + return None; + } + k = k.saturating_add(1); + i += 1; + } + i += 1; // the closing bracket + } + _ => break, + } + } + } + _ => {} + } + } + Some((map, width)) +} + +pub(super) fn res_dict(doc: &Document, res: &Dict, key: &[u8]) -> Dict { + res.get(key).map(|o| doc.resolve(o)).and_then(|o| o.as_dict().cloned()).unwrap_or_default() +} + +pub(super) fn push_squashed(into: &mut String, s: &str) { + into.extend(s.chars().filter(|c| !c.is_whitespace() && *c != '\u{feff}')); +} + +/// A form's or pattern's `/Matrix`: absent is the identity, present but malformed is an error +/// (guessing the identity would put its content in the wrong place). +pub(super) fn matrix_of(doc: &Document, dict: &Dict) -> Result { + match dict.get(b"Matrix") { + None => Ok(Matrix::IDENTITY), + Some(m) => { + let m = doc.resolve(m); + let items: Vec = m.as_array().ok_or(())?.iter().map(|x| (*doc.resolve(x)).clone()).collect(); + Matrix::from_operands(&items).filter(|m| m.0.iter().all(|v| v.is_finite())).ok_or(()) + } + } +} + +/// A form's `/BBox` as `[x0 y0 x1 y1]`: absent is `None`, present but malformed is an error. +pub(super) fn bbox_of(doc: &Document, dict: &Dict) -> Result, ()> { + let Some(b) = dict.get(b"BBox") else { return Ok(None) }; + let b = doc.resolve(b); + let v: Vec = + b.as_array().ok_or(())?.iter().map(|x| doc.resolve(x).as_f64().filter(|v| v.is_finite()).ok_or(())).collect::>()?; + match v[..] { + [a, b, c, d] => Ok(Some([a.min(c), b.min(d), a.max(c), b.max(d)])), + _ => Err(()), + } +} + +/// The content streams of a page as `(raw object, decoded)`, a missing or unreadable entry an +/// error. `skip_overlay` leaves out the streams this run drew the boxes and labels with. +pub(super) fn page_content(doc: &Document, page: &Dict, ctx: &Ctx, skip_overlay: bool) -> Result, ()> { + let list: Vec = match page.get(b"Contents") { + None => Vec::new(), + Some(c) => match &*resolve_strict(doc, c)? { + Object::Array(a) => a.clone(), + _ => vec![c.clone()], + }, + }; + let mut data: Vec = Vec::new(); + for o in &list { + match &*resolve_strict(doc, o)? { + Object::Stream(s) => { + let d = ctx.decode(s)?; + if skip_overlay && ctx.is_overlay(&d) { + continue; + } + data.extend(d); + data.push(b'\n'); + } + // A null entry is an empty piece, as for the redaction itself (ISO 32000-2 7.3.9). + Object::Null => {} + _ => return Err(()), + } + } + Ok(data) +} + +/// The text of a page: its content, the forms and patterns it reaches (visible or not), and the +/// appearance streams of its annotations; and which of `regions` still hold glyphs. +pub(super) fn extract_page(doc: &Document, page: &Dict, ctx: &Ctx, regions: &[[f64; 4]]) -> Result { + let data = page_content(doc, page, ctx, true)?; + let res = page.get(b"Resources").and_then(|r| doc.resolve(r).as_dict().cloned()).unwrap_or_default(); + let mut ex = Extractor { doc, ctx, regions, out: Extracted { shown_in: vec![0; regions.len()], ..Extracted::default() }, stack: Vec::new() }; + ex.scope(&data, &res, Matrix::IDENTITY, 0, true)?; + for a in annots_of(doc, page) { + let Some(d) = doc.resolve(&a).as_dict().cloned() else { continue }; + let Some(ap) = d.get(b"AP").and_then(|x| doc.resolve(x).as_dict().cloned()) else { continue }; + let rect = + d.get(b"Rect").and_then(|r| doc.resolve(r).as_array().map(|a| a.iter().filter_map(|x| doc.resolve(x).as_f64()).collect::>())); + for (_, state) in ap.iter() { + let s = doc.resolve(state); + let streams: Vec<(Option, Object)> = match &*s { + Object::Stream(_) => vec![(state.as_ref(), (*s).clone())], + Object::Dict(sub) => sub.iter().map(|(_, v)| (v.as_ref(), (*doc.resolve(v)).clone())).collect(), + _ => Vec::new(), + }; + for (r, o) in streams { + if let Object::Stream(st) = o { + // Where the appearance lands on the page (ISO 32000-2 §12.5.5), if it can be told. + let ctm = rect.as_deref().and_then(|r| appearance_matrix(doc, &st, r)); + // A font of the appearance is looked up in the appearance's own resources + // first (what the redaction itself does), then in the page's. + let own = st.dict.get(b"Resources").and_then(|x| doc.resolve(x).as_dict().cloned()); + let parent = own.as_ref().unwrap_or(&res); + ex.stream(&st, r, parent, ctm.unwrap_or_default(), 1, ctm.is_some())?; + } + } + } + } + Ok(ex.out) +} + +/// The matrix that maps an appearance stream's form space onto an annotation's `/Rect` +/// (ISO 32000-2 §12.5.5): the appearance's `/Matrix`, then the scale that fits the mapped +/// `/BBox` into the rectangle. `None` when that can't be told (no usable `/Rect`, `/BBox` or +/// `/Matrix`, or a mapped box that degenerates). +pub(crate) fn appearance_matrix(doc: &Document, ap: &Stream, rect: &[f64]) -> Option { + let [rx0, ry0, rx1, ry1] = rect[..] else { return None }; + let bbox = bbox_of(doc, &ap.dict).ok()??; + let m = matrix_of(doc, &ap.dict).ok()?; + let t = m.bbox(bbox); + let (tw, th) = (t[2] - t[0], t[3] - t[1]); + if tw <= 0.0 || th <= 0.0 { + return None; + } + let (rx0, rx1, ry0, ry1) = (rx0.min(rx1), rx0.max(rx1), ry0.min(ry1), ry0.max(ry1)); + let (sx, sy) = ((rx1 - rx0) / tw, (ry1 - ry0) / th); + let a = Matrix([sx, 0.0, 0.0, sy, rx0 - t[0] * sx, ry0 - t[1] * sy]); + Some(m.then(&a)) +} + +/// Text-state parameters that `q`/`Q` save and restore. +#[derive(Clone)] +pub(super) struct TextGs { + pub(super) ctm: Matrix, + pub(super) tc: f64, + pub(super) tw: f64, + pub(super) th: f64, + pub(super) tl: f64, + pub(super) rise: f64, + pub(super) size: f64, + pub(super) font: Vec, +} + +pub(super) struct Extractor<'a> { + pub(super) doc: &'a Document, + pub(super) ctx: &'a Ctx, + pub(super) regions: &'a [[f64; 4]], + pub(super) out: Extracted, + pub(super) stack: Vec, +} + +impl Extractor<'_> { + /// A form or pattern stream. `ctm` maps its space onto the page's; `placed` is false where + /// that mapping is not known. + fn stream(&mut self, s: &Stream, id: Option, parent: &Dict, ctm: Matrix, depth: usize, placed: bool) -> Result<(), ()> { + if let Some(r) = id { + if self.stack.contains(&r) { + return Ok(()); + } + self.stack.push(r); + } + let result = (|| { + let data = self.ctx.decode(s)?; + let own = s.dict.get(b"Resources").and_then(|r| self.doc.resolve(r).as_dict().cloned()); + self.scope(&data, own.as_ref().unwrap_or(parent), ctm, depth, placed) + })(); + if id.is_some() { + self.stack.pop(); + } + result + } + + /// Show the strings of one text-showing operator: extract their text, and place each glyph + /// to count the ones inside a region. `tm` moves along with the pen. + fn show_text(&mut self, font: &Font, op: &pdfcraft_content::Op, gs: &TextGs, tm: &mut Matrix, placed: bool) { + let o = op.op.as_slice(); + // Strings and TJ adjustments, in order. + let items: Vec<&Object> = match o { + b"TJ" => op.operands.first().and_then(Object::as_array).map(|a| a.iter().collect()).unwrap_or_default(), + _ => op.operands.last().into_iter().collect(), + }; + let mut any_text = false; + for it in items { + if let Some(n) = it.as_f64() { + let d = -n / 1000.0 * gs.size; + let m = if font.vertical { Matrix::translate(0.0, d) } else { Matrix::translate(d * gs.th, 0.0) }; + *tm = m.then(tm); + continue; + } + let Some(s) = it.as_string() else { continue }; + let t = font.dec.decode(&s.bytes); + any_text |= !squash(&t).is_empty(); + push_squashed(&mut self.out.text, &t); + push_squashed(&mut self.out.raw, &latin1(&s.bytes)); + for (code, text) in font.dec.split(&s.bytes) { + let w = font.width(code); + let space = text.as_deref().is_some_and(|t| t == " "); + // The glyph's centre, in text space (the box runs from 0.2 em below the + // baseline to 0.8 em above), and where the pen moves to. + let (centre, advance) = if font.vertical { + let adv = -gs.size + gs.tc + if space { gs.tw } else { 0.0 }; + ((0.0, gs.rise - 0.5 * gs.size), Matrix::translate(0.0, adv)) + } else { + let adv = (w * gs.size + gs.tc + if space { gs.tw } else { 0.0 }) * gs.th; + ((0.5 * w * gs.size * gs.th, gs.rise + 0.3 * gs.size), Matrix::translate(adv, 0.0)) + }; + let visible = text.as_deref().is_none_or(|t| !squash(t).is_empty()); + if placed && visible && !self.regions.is_empty() { + let (x, y) = tm.then(&gs.ctm).apply(centre.0, centre.1); + for (r, n) in self.regions.iter().zip(self.out.shown_in.iter_mut()) { + if x >= r[0] && x <= r[2] && y >= r[1] && y <= r[3] { + *n += 1; + } + } + } + *tm = advance.then(tm); + } + } + if carries_glyphs(op) { + self.out.glyph_ops += 1; + if !any_text { + self.out.undecodable_ops += 1; + } + if !placed { + self.out.unplaced += 1; + } + } + } + + fn scope(&mut self, data: &[u8], res: &Dict, ctm: Matrix, depth: usize, placed: bool) -> Result<(), ()> { + if depth > MAX_DEPTH { + return Err(()); + } + let doc = self.doc; + let fonts = res_dict(doc, res, b"Font"); + let xobjects = res_dict(doc, res, b"XObject"); + let mut cache: HashMap, Font> = HashMap::new(); + let mut gs = TextGs { ctm, tc: 0.0, tw: 0.0, th: 1.0, tl: 0.0, rise: 0.0, size: 0.0, font: Vec::new() }; + let mut saved: Vec = Vec::new(); + let (mut tm, mut tlm) = (Matrix::IDENTITY, Matrix::IDENTITY); + for op in parse(data).ops { + let o = op.op.as_slice(); + match o { + b"q" => { + if saved.len() >= 4096 { + return Err(()); + } + saved.push(gs.clone()); + } + b"Q" => { + if let Some(g) = saved.pop() { + gs = g; + } + } + b"cm" => { + if let Some(m) = op.nums::<6>() { + gs.ctm = Matrix(m).then(&gs.ctm); + } + } + b"BT" => { + tm = Matrix::IDENTITY; + tlm = Matrix::IDENTITY; + } + b"Tc" => gs.tc = op.num(0).unwrap_or(0.0), + b"Tw" => gs.tw = op.num(0).unwrap_or(0.0), + b"Tz" => gs.th = op.num(0).unwrap_or(100.0) / 100.0, + b"TL" => gs.tl = op.num(0).unwrap_or(0.0), + b"Ts" => gs.rise = op.num(0).unwrap_or(0.0), + b"Tf" => { + gs.font = op.name(0).map(<[u8]>::to_vec).unwrap_or_default(); + gs.size = op.num(1).unwrap_or(0.0); + } + b"Td" | b"TD" => { + if let Some([x, y]) = op.nums::<2>() { + if o == b"TD" { + gs.tl = -y; + } + tlm = Matrix::translate(x, y).then(&tlm); + tm = tlm; + } + } + b"Tm" => { + if let Some(m) = op.nums::<6>() { + tm = Matrix(m); + tlm = tm; + } + } + b"T*" => { + tlm = Matrix::translate(0.0, -gs.tl).then(&tlm); + tm = tlm; + } + b"Tj" | b"TJ" | b"'" | b"\"" => { + if matches!(o, b"'" | b"\"") { + tlm = Matrix::translate(0.0, -gs.tl).then(&tlm); + tm = tlm; + } + if o == b"\"" + && let (Some(aw), Some(ac)) = (op.num(0), op.num(1)) + { + (gs.tw, gs.tc) = (aw, ac); + } + if !cache.contains_key(&gs.font) { + let f = match fonts.get(&gs.font).map(|f| doc.resolve(f)) { + Some(f) => f.as_dict().map_or_else(|| Ok(Font::flat()), |d| Font::new(doc, d, self.ctx))?, + None => Font::flat(), + }; + cache.insert(gs.font.clone(), f); + } + let Some(font) = cache.get(&gs.font) else { continue }; + self.show_text(font, &op, &gs, &mut tm, placed); + } + b"Do" => { + let Some(name) = op.name(0) else { continue }; + let Some(entry) = xobjects.get(name) else { continue }; + if let Object::Stream(s) = &*doc.resolve(entry) + && s.dict.name(b"Subtype") == Some(b"Form") + { + let m = matrix_of(doc, &s.dict)?; + bbox_of(doc, &s.dict)?; + self.stream(s, entry.as_ref(), res, m.then(&gs.ctm), depth + 1, placed)?; + } + } + _ => {} + } + } + // Tiling patterns draw content of their own, repeated where the pattern is used: its + // glyphs can't be placed. + for (_, p) in res_dict(doc, res, b"Pattern").iter() { + if let Object::Stream(s) = &*doc.resolve(p) { + self.stream(s, p.as_ref(), res, Matrix::IDENTITY, depth + 1, false)?; + } + } + Ok(()) + } +} diff --git a/crates/redact/src/verify/geometry.rs b/crates/redact/src/verify/geometry.rs new file mode 100644 index 0000000000..fb69f91351 --- /dev/null +++ b/crates/redact/src/verify/geometry.rs @@ -0,0 +1,500 @@ +//! The non-text content under the regions: images, paths and shadings, and the placement of the +//! glyphs still shown, checked independently of the interpreter. + +use std::collections::HashSet; + +use pdfcraft_content::{Matrix, contains, overlaps, parse}; +use pdfcraft_cos::{Dict, Document, ObjRef, Object, Stream}; + +use super::extract::{appearance_matrix, bbox_of, matrix_of, page_content, res_dict}; +use super::util::{Ctx, image_codec}; +use crate::verify::limits::{MAX_CLIP_SUBPATHS, MAX_DEPTH, MAX_GS_STACK, MAX_HOLES, MAX_PIXELS}; + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub(super) enum Finding { + /// This many glyphs are placed inside the region. + Text(usize), + Image, + Path, + Shading, + Missing, + Unverifiable(&'static str), +} + +#[derive(Clone)] +pub(super) struct Gs { + pub(super) ctm: Matrix, + /// Device-space boxes cut out of the clip (even-odd clips with a hole). + pub(super) holes: Vec<[f64; 4]>, +} + +pub(super) struct Geo<'a> { + pub(super) doc: &'a Document, + pub(super) ctx: &'a Ctx, + pub(super) regions: &'a [[f64; 4]], + pub(super) found: Vec<(usize, Finding)>, + /// What `found` already holds: each finding is recorded once per region, so a page that + /// paints thousands of times cannot grow the report (or the memory) with repeats. + reported: HashSet<(usize, Finding)>, + /// Pattern, soft-mask and Type3 resources already searched for images. + pub(super) searched: HashSet, +} + +/// What still paints inside `regions` on a page of the output (the overlay boxes excepted): +/// images with pixels there, paths and shadings not clipped away from there. +pub(super) fn check_geometry(doc: &Document, page: &Dict, regions: &[[f64; 4]], ctx: &Ctx) -> Vec<(usize, Finding)> { + let unverifiable = |why: &'static str| (0..regions.len()).map(|i| (i, Finding::Unverifiable(why))).collect(); + // The redaction boxes are painted on purpose. + let Ok(data) = page_content(doc, page, ctx, true) else { return unverifiable("the page content can't be read in full") }; + let mut geo = Geo { doc, ctx, regions, found: Vec::new(), reported: HashSet::new(), searched: HashSet::new() }; + let res = page.get(b"Resources").and_then(|r| doc.resolve(r).as_dict().cloned()).unwrap_or_default(); + geo.run(&data, &res, Matrix::IDENTITY, Vec::new(), 0); + geo.appearances(page, &res); + geo.found +} + +impl Geo<'_> { + /// Record a finding for region `i`, once: a page that paints over a region a thousand + /// times is one finding, not a thousand. + fn report(&mut self, i: usize, f: Finding) { + if self.reported.insert((i, f.clone())) { + self.found.push((i, f)); + } + } + + /// The appearances of the annotations still on the page, checked like the page itself: their + /// content is placed with the appearance matrix (ISO 32000-2 §12.5.5) and swept for images, + /// paths and shadings under the regions. An appearance that has content but can't be placed + /// makes every region unverifiable: the redaction refuses such documents, so one in the + /// output means what it paints can't be vouched for. + fn appearances(&mut self, page: &Dict, res: &Dict) { + let doc = self.doc; + for a in crate::annots_of(doc, page) { + // A direct dictionary has no reference of its own; it is checked like an indirect + // one (what its appearance paints does not depend on how it is stored). + let Some(d) = doc.resolve(&a).as_dict().cloned() else { continue }; + // A redaction mark's appearance outlines the marked area on purpose (and applying + // removes the mark whole): it is exempt like the boxes drawn over the regions. + if d.name(b"Subtype") == Some(b"Redact") { + continue; + } + let Some(ap) = d.get(b"AP").and_then(|x| doc.resolve(x).as_dict().cloned()) else { continue }; + let rect = crate::rect_of(doc, d.get(b"Rect")); + for (_, state) in ap.iter() { + let s = doc.resolve(state); + let streams: Vec = match &*s { + Object::Stream(st) => vec![st.clone()], + Object::Dict(sub) => sub + .iter() + .filter_map(|(_, v)| match &*doc.resolve(v) { + Object::Stream(st) => Some(st.clone()), + _ => None, + }) + .collect(), + _ => Vec::new(), + }; + for st in streams { + // A stream shared by several annotations is checked at each annotation's + // own placement: where it lands comes from each `/Rect`, so the first + // placement tells nothing about the others. Repeated decodes spend the + // same bounded budget as everything else. + let Ok(data) = self.ctx.decode(&st) else { + self.unverifiable("an annotation appearance can't be read in full"); + continue; + }; + if data.is_empty() { + continue; + } + let Some(ctm) = rect.as_ref().and_then(|r| appearance_matrix(doc, &st, r)) else { + self.unverifiable("an annotation appearance can't be placed, so what it paints can't be checked"); + continue; + }; + let own = st.dict.get(b"Resources").and_then(|x| doc.resolve(x).as_dict().cloned()); + self.run(&data, own.as_ref().unwrap_or(res), ctm, Vec::new(), 1); + } + } + } + } + + fn touching(&self, b: [f64; 4]) -> Vec { + (0..self.regions.len()).filter(|&i| self.regions.get(i).is_some_and(|r| overlaps(*r, b, 0.0))).collect() + } + + /// Does something painting over `within` reach region `i` through the clip? Only the part of + /// the region the painting could touch has to be cut out. + fn unclipped(&self, i: usize, within: [f64; 4], gs: &Gs) -> bool { + let Some(r) = self.regions.get(i) else { return false }; + let reach = [r[0].max(within[0]), r[1].max(within[1]), r[2].min(within[2]), r[3].min(within[3])]; + !gs.holes.iter().any(|h| contains(*h, reach, 0.01)) + } + + fn run(&mut self, data: &[u8], res: &Dict, ctm: Matrix, holes: Vec<[f64; 4]>, depth: usize) { + if depth > MAX_DEPTH { + for i in 0..self.regions.len() { + self.report(i, Finding::Unverifiable("forms are nested too deep to check")); + } + return; + } + let xobjects = res_dict(self.doc, res, b"XObject"); + self.hidden_images(res, depth); + let mut gs = Gs { ctm, holes }; + let mut stack: Vec = Vec::new(); + // The current path in device space: its subpaths, and the clip operator it carries. + let mut sub: Vec> = Vec::new(); + let mut clip: Option = None; + for op in parse(data).ops { + let o = op.op.as_slice(); + if matches!(o, b"m" | b"l" | b"c" | b"v" | b"y" | b"h" | b"re" | b"W" | b"W*") { + extend_path(&op, &gs.ctm, &mut sub, &mut clip); + continue; + } + if matches!(o, b"S" | b"s" | b"f" | b"F" | b"f*" | b"B" | b"B*" | b"b" | b"b*" | b"n") { + self.paint_path(o, &mut gs, &sub, clip); + sub.clear(); + clip = None; + continue; + } + sub.clear(); + clip = None; + match o { + b"q" => { + // The saved state carries the holes with it, so an unbounded stack of them + // is unbounded memory: past the cap the walk says so instead of growing. + if stack.len() >= MAX_GS_STACK { + return self.unverifiable("the content stacks graphics states too deeply to check"); + } + stack.push(gs.clone()); + } + b"Q" => { + if let Some(g) = stack.pop() { + gs = g; + } + } + b"cm" => { + if let Some(m) = op.nums::<6>() { + gs.ctm = Matrix(m).then(&gs.ctm); + } + } + b"sh" => { + for i in 0..self.regions.len() { + if self.unclipped(i, [f64::MIN, f64::MIN, f64::MAX, f64::MAX], &gs) { + self.report(i, Finding::Shading); + } + } + } + b"BI" => { + for i in self.touching(gs.ctm.bbox([0.0, 0.0, 1.0, 1.0])) { + self.report(i, Finding::Image); + } + } + b"Do" => { + let Some(entry) = op.name(0).and_then(|n| xobjects.get(n)) else { continue }; + let obj = self.doc.resolve(entry); + let Object::Stream(s) = &*obj else { continue }; + match s.dict.name(b"Subtype") { + Some(b"Image") => { + for i in self.touching(gs.ctm.bbox([0.0, 0.0, 1.0, 1.0])) { + let Some(region) = self.regions.get(i) else { continue }; + match image_cleared(self.doc, self.ctx, s, &gs.ctm, *region) { + Ok(true) => {} + Ok(false) => self.report(i, Finding::Image), + Err(()) => self.report(i, Finding::Unverifiable("an image under the region can't be inspected")), + } + } + } + Some(b"Form") => self.form(s, &gs, res, depth), + _ => {} + } + } + _ => {} + } + } + } + + /// A path-painting operator `o` ends the current path: report what it paints over a region, + /// and let an even-odd clip add its holes to `gs`. + fn paint_path(&mut self, o: &[u8], gs: &mut Gs, sub: &[Vec<(f64, f64)>], clip: Option) { + let pts: Vec<(f64, f64)> = sub.iter().flatten().copied().collect(); + let bbox = pts.iter().fold(None, |b: Option<[f64; 4]>, &(u, v)| { + Some(match b { + None => [u, v, u, v], + Some(b) => [b[0].min(u), b[1].min(v), b[2].max(u), b[3].max(v)], + }) + }); + if o != b"n" + && let Some(b) = bbox + { + for i in self.touching(b) { + if self.unclipped(i, b, gs) { + self.report(i, Finding::Path); + } + } + } + // An even-odd clip whose subpaths are all axis-aligned rectangles lets through what an + // odd number of the inner rectangles covers. Where a region lies under the outer box + // and an odd number of inner ones, it is cut out of the clip; under an even number it + // is painted and must be reported. Anything that is not a stock of disjoint rectangles + // (a triangle, a rotated box, overlapping inners, too many of them) cannot be told + // cheaply, so no hole is cut: what the clip lets through is then reported as paint + // (fail closed). This is how redaction itself clips regions out of what it leaves. + if clip != Some(true) { + return; + } + let Some(rects) = sub.iter().map(|s| axis_aligned_rect(s)).collect::>>() else { return }; + if rects.len() < 2 || rects.len() > MAX_CLIP_SUBPATHS { + return; + } + let (outer, inners) = (rects[0], &rects[1..]); + // Inner rectangles must be disjoint, or their overlaps would change the parity the + // reader draws with. + let disjoint = inners.iter().enumerate().all(|(x, a)| inners.iter().skip(x + 1).all(|b| !overlaps(*a, *b, 0.0))); + let regions: Vec<[f64; 4]> = self.regions.to_vec(); + let mut cut = Vec::new(); + for r in regions { + if contains(outer, r, 0.0) && disjoint && inners.iter().filter(|b| contains(**b, r, 0.0)).count() % 2 == 1 { + cut.push(r); + } + } + if gs.holes.len() + cut.len() > MAX_HOLES { + return self.unverifiable("the content cuts clips with too many pieces to check"); + } + gs.holes.extend(cut); + } + + fn unverifiable(&mut self, why: &'static str) { + for i in 0..self.regions.len() { + self.report(i, Finding::Unverifiable(why)); + } + } + + fn form(&mut self, s: &Stream, gs: &Gs, parent: &Dict, depth: usize) { + let (Ok(m), Ok(bbox)) = (matrix_of(self.doc, &s.dict), bbox_of(self.doc, &s.dict)) else { + self.unverifiable("a form under the region has a malformed /Matrix or /BBox"); + return; + }; + let fctm = m.then(&gs.ctm); + if let Some(bb) = bbox + && self.touching(fctm.bbox(bb)).is_empty() + { + return; + } + let Ok(data) = self.ctx.decode(s) else { + self.unverifiable("a form under the region can't be decoded in full"); + return; + }; + let own = s.dict.get(b"Resources").and_then(|r| self.doc.resolve(r).as_dict().cloned()); + self.run(&data, own.as_ref().unwrap_or(parent), fctm, gs.holes.clone(), depth + 1); + } + + /// Images drawn from tiling patterns, soft-mask groups and Type3 glyph procedures can't be + /// placed or cleared by this check: if `res` reaches any, every region is unverifiable. + fn hidden_images(&mut self, res: &Dict, depth: usize) { + let doc = self.doc; + let mut streams: Vec<(Option, Stream)> = Vec::new(); + let take = |o: &Object, streams: &mut Vec<(Option, Stream)>| { + if let Object::Stream(s) = &*doc.resolve(o) { + streams.push((o.as_ref(), s.clone())); + } + }; + for (_, p) in res_dict(doc, res, b"Pattern").iter() { + take(p, &mut streams); + } + for (_, g) in res_dict(doc, res, b"ExtGState").iter() { + if let Some(Object::Dict(sm)) = doc.resolve(g).as_dict().and_then(|g| g.get(b"SMask")).map(|m| (*doc.resolve(m)).clone()).as_ref() + && let Some(group) = sm.get(b"G") + { + take(group, &mut streams); + } + } + for (_, f) in res_dict(doc, res, b"Font").iter() { + let Some(font) = doc.resolve(f).as_dict().cloned() else { continue }; + if font.name(b"Subtype") != Some(b"Type3") { + continue; + } + if let Some(procs) = font.get(b"CharProcs").and_then(|c| doc.resolve(c).as_dict().cloned()) { + for (_, p) in procs.iter() { + take(p, &mut streams); + } + } + } + for (id, s) in streams { + if let Some(r) = id + && !self.searched.insert(r) + { + continue; + } + match self.has_image(&s, res, depth + 1) { + Ok(false) => {} + Ok(true) => self.unverifiable("an image inside a pattern, soft mask or Type3 glyph can't be placed or cleared"), + Err(()) => self.unverifiable("a pattern, soft mask or Type3 glyph can't be read in full"), + } + } + } + + /// Does the content of `s` (forms it draws included) draw an image? + fn has_image(&mut self, s: &Stream, parent: &Dict, depth: usize) -> Result { + if depth > MAX_DEPTH { + return Err(()); + } + let data = self.ctx.decode(s)?; + let own = s.dict.get(b"Resources").and_then(|r| self.doc.resolve(r).as_dict().cloned()); + let res = own.as_ref().unwrap_or(parent); + let xobjects = res_dict(self.doc, res, b"XObject"); + for op in parse(&data).ops { + match op.op.as_slice() { + b"BI" => return Ok(true), + b"Do" => { + let Some(entry) = op.name(0).and_then(|n| xobjects.get(n)) else { continue }; + let Object::Stream(x) = &*self.doc.resolve(entry) else { continue }; + match x.dict.name(b"Subtype") { + Some(b"Image") => return Ok(true), + Some(b"Form") => { + if let Some(r) = entry.as_ref() + && !self.searched.insert(r) + { + continue; + } + if self.has_image(x, res, depth + 1)? { + return Ok(true); + } + } + _ => {} + } + } + _ => {} + } + } + Ok(false) + } +} + +/// Extend the current path (`sub`: its subpaths in device space; `clip`: the clip operator it +/// carries) by one construction operator. +fn extend_path(op: &pdfcraft_content::Op, ctm: &Matrix, sub: &mut Vec>, clip: &mut Option) { + let o = op.op.as_slice(); + let nums: Vec = op.operands.iter().filter_map(Object::as_f64).collect(); + let at = |x: f64, y: f64| ctm.apply(x, y); + match o { + b"m" => { + if let [x, y] = nums[..] { + sub.push(vec![at(x, y)]); + } + } + b"l" | b"c" | b"v" | b"y" => { + if let Some(cur) = sub.last_mut() { + cur.extend(nums.as_chunks::<2>().0.iter().map(|p| at(p[0], p[1]))); + } + } + b"re" => { + if let [x, y, w, h] = nums[..] { + sub.push(vec![at(x, y), at(x + w, y), at(x + w, y + h), at(x, y + h)]); + } + } + b"W" => *clip = Some(false), + b"W*" => *clip = Some(true), + _ => {} + } +} + +/// The axis-aligned rectangle a four-corner subpath draws, in device space: `None` for anything +/// else (open paths, more or fewer corners, skewed or rotated boxes, non-finite coordinates). +fn axis_aligned_rect(pts: &[(f64, f64)]) -> Option<[f64; 4]> { + // `re` leaves its rectangle open: the four corners are the whole subpath. A rectangle drawn + // by hand closes by repeating the first point at the end. Both spell the same four corners. + let pts = match pts { + [a, _, _, _, e] if a == e => &pts[..4], + _ => pts, + }; + let [p0, p1, p2, p3] = pts else { return None }; + let finite = |p: &(f64, f64)| p.0.is_finite() && p.1.is_finite(); + if !finite(p0) || !finite(p1) || !finite(p2) || !finite(p3) { + return None; + } + // The edges alternate between horizontal and vertical, whichever corner they start from and + // whichever way the rectangle is wound. + let flat = |a: &(f64, f64), b: &(f64, f64)| a.1 == b.1 && a.0 != b.0; + let steep = |a: &(f64, f64), b: &(f64, f64)| a.0 == b.0 && a.1 != b.1; + let even = flat(p0, p1) && steep(p1, p2) && flat(p2, p3) && steep(p3, p0); + let odd = steep(p0, p1) && flat(p1, p2) && steep(p2, p3) && flat(p3, p0); + if !even && !odd { + return None; + } + let (x0, x1) = (p0.0.min(p2.0), p0.0.max(p2.0)); + let (y0, y1) = (p0.1.min(p2.1), p0.1.max(p2.1)); + (x0 < x1 && y0 < y1).then_some([x0, y0, x1, y1]) +} + +/// Are the samples of image `s` whose centres fall in `region` all clear (zero; for image masks, +/// the sample that does not paint)? `Err` when the image can't be inspected. +#[allow(clippy::result_unit_err)] +pub(super) fn image_cleared(doc: &Document, ctx: &Ctx, s: &Stream, ctm: &Matrix, region: [f64; 4]) -> Result { + if image_codec(&s.dict) { + // Still encoded pixels: nothing was cleared. + return Ok(false); + } + let (w, h) = (s.dict.int(b"Width").ok_or(())?, s.dict.int(b"Height").ok_or(())?); + if w <= 0 || h <= 0 || (w as u64).saturating_mul(h as u64) > MAX_PIXELS { + return Err(()); + } + let (w, h) = (usize::try_from(w).map_err(|_| ())?, usize::try_from(h).map_err(|_| ())?); + let mask = matches!(s.dict.get(b"ImageMask"), Some(Object::Bool(true))); + let (ncomp, bpc) = if mask { + (1, 1) + } else { + let cs = doc.resolve(s.dict.get(b"ColorSpace").ok_or(())?); + let name = match &*cs { + Object::Name(n) => n.clone(), + Object::Array(a) => a.first().and_then(Object::as_name).ok_or(())?.to_vec(), + _ => return Err(()), + }; + let n = match name.as_slice() { + b"DeviceGray" | b"G" | b"CalGray" | b"Indexed" | b"I" | b"Separation" => 1, + b"DeviceRGB" | b"RGB" | b"CalRGB" | b"Lab" => 3, + b"DeviceCMYK" | b"CMYK" => 4, + // One component per colorant name. + b"DeviceN" => doc.resolve(cs.as_array().and_then(|a| a.get(1)).ok_or(())?).as_array().ok_or(())?.len(), + b"ICCBased" => match &*doc.resolve(cs.as_array().and_then(|a| a.get(1)).ok_or(())?) { + Object::Stream(icc) => usize::try_from(icc.dict.int(b"N").ok_or(())?).map_err(|_| ())?, + _ => return Err(()), + }, + _ => return Err(()), + }; + (n, usize::try_from(s.dict.int(b"BitsPerComponent").ok_or(())?).map_err(|_| ())?) + }; + if !matches!(bpc, 1 | 2 | 4 | 8 | 16) || ncomp == 0 || ncomp > 32 { + return Err(()); + } + let data = ctx.decode(s)?; + let bits = ncomp.checked_mul(bpc).ok_or(())?; + let row = w.checked_mul(bits).ok_or(())?.div_ceil(8); + if data.len() < row.checked_mul(h).ok_or(())? { + return Err(()); + } + let inverted = s.dict.get(b"Decode").is_some_and(|d| { + let d = doc.resolve(d); + d.as_array().and_then(|a| a.first()).is_some_and(|first| doc.resolve(first).as_f64() == Some(1.0)) + }); + let want = u8::from(mask && !inverted); + for j in 0..h { + let v = 1.0 - (j as f64 + 0.5) / h as f64; + // Rows that don't reach the region hold nothing to check. + let ((ax, ay), (bx, by)) = (ctm.apply(0.0, v), ctm.apply(1.0, v)); + if !overlaps(region, [ax.min(bx), ay.min(by) - 0.01, ax.max(bx), ay.max(by) + 0.01], 0.0) { + continue; + } + for i in 0..w { + let (x, y) = ctm.apply((i as f64 + 0.5) / w as f64, v); + if x < region[0] || x > region[2] || y < region[1] || y > region[3] { + continue; + } + let start = j.checked_mul(row).and_then(|v| v.checked_mul(8)).and_then(|v| v.checked_add(i.checked_mul(bits)?)).ok_or(())?; + for bit in start..start.checked_add(bits).ok_or(())? { + let byte = data.get(bit / 8).ok_or(())?; + if (byte >> (7 - bit % 8)) & 1 != want { + return Ok(false); + } + } + } + } + Ok(true) +} diff --git a/crates/redact/src/verify/limits.rs b/crates/redact/src/verify/limits.rs new file mode 100644 index 0000000000..e2972bb5f6 --- /dev/null +++ b/crates/redact/src/verify/limits.rs @@ -0,0 +1,42 @@ +//! Bounds of the redaction proof (`verify`). Hitting one is a failure the proof reports, never +//! a quiet skip: a proof that cannot sweep everything it must is not a passing proof. + +/// Most decoded bytes of one stream the proof reads. +pub(crate) const MAX_STREAM: usize = 64 << 20; +/// Most decoded bytes the whole proof reads, over every stream and every pass. +pub(crate) const MAX_TOTAL_DECODED: u64 = 2 << 30; +/// Most units of linear search work (input, patterns and reported matches) in one proof. +pub(crate) const MAX_WORK: u64 = 4 << 30; +/// Nested attachment levels that are swept (deeper ones are reported as unswept). +pub(crate) const MAX_NESTING: usize = 4; +/// Nesting bound of object walks (deeper is reported, never skipped silently). +pub(crate) const MAX_WALK: usize = 64; +/// Deepest graphics-state stack the paint check follows (saved states carry their clip holes +/// with them, so an unbounded stack is unbounded memory). Deeper is unverifiable. +pub(crate) const MAX_GS_STACK: usize = 4096; +/// Most clip holes one walk may cut (each is kept until the state that made it is restored). +/// More is unverifiable. +pub(crate) const MAX_HOLES: usize = 4096; +/// Most subpaths of one even-odd clip the hole rule examines (it compares them pairwise). +/// More, and no hole is cut: what the clip lets through is reported as paint. +pub(crate) const MAX_CLIP_SUBPATHS: usize = 64; +/// Most decoded bytes of an embedded CMap. +pub(crate) const MAX_CMAP: usize = 4 << 20; +/// Most CMap range steps one font may cost (a range that never grows the map still costs). +pub(crate) const MAX_CMAP_WORK: usize = 4 << 20; +/// Most entries one ToUnicode CMap may define. +pub(crate) const MAX_CMAP_ENTRIES: usize = 1 << 20; +/// Needles shorter than this (in characters) are too common to look for in raw bytes. +pub(crate) const MIN_NEEDLE: usize = 3; +/// Smallest needle looked for in the raw file and in decoded streams. Those hold object syntax, +/// numbers and binary data, where three characters match by chance (`/Length 612`, a +/// `/MediaBox`, pixels); shorter needles are still swept on page text, strings, names, Info, XMP +/// and attachment text. +pub(crate) const MIN_STRUCTURED_NEEDLE: usize = 6; +/// Deepest nesting of forms, patterns and appearance streams the proof's independent walks +/// follow: the bound the interpreter itself walks forms at, so the proof reaches everything a +/// redaction could have removed and the two cannot drift apart. +pub(crate) const MAX_DEPTH: usize = crate::interp::MAX_DEPTH; +/// Most pixels an image may have for the proof to inspect it (larger ones are removed by +/// [`apply`](crate::apply), and the proof's pixel look has the same ceiling: `crate::image`). +pub(crate) const MAX_PIXELS: u64 = crate::image::MAX_PIXELS; diff --git a/crates/redact/src/verify/mod.rs b/crates/redact/src/verify/mod.rs new file mode 100644 index 0000000000..b374f93fdc --- /dev/null +++ b/crates/redact/src/verify/mod.rs @@ -0,0 +1,389 @@ +//! An independent proof that redaction removed what it claimed to remove. +//! +//! [`apply`](crate::apply) cuts glyphs out with its own interpreter, so asking that interpreter +//! whether anything is left would only repeat its own mistakes. The proof therefore works from +//! the other end, and does not trust the interpreter's font metrics or hit test: +//! +//! 1. **Before** applying, [`Snapshot::capture`] records, per redaction region, the glyph runs the +//! interpreter is about to remove (their character codes and the text the font gives them), a +//! digest of each page's content, and the text of the whole document as a separate extractor +//! sees it. The strings the sweep looks for therefore *start* from what the interpreter +//! removed; a glyph it wrongly kept is caught by the geometric check below, not by a needle. +//! 2. **After** applying, [`Snapshot::prove`] serializes the document and sweeps the *output* +//! for those strings on every surface a reader could recover them from: the raw file, every +//! string object, every decoded stream (orphans included), the text of every page (read by a +//! minimal extractor of its own, not the interpreter's font metrics or hit test), the Info +//! dictionary, the XMP packet, embedded files (PDF attachments are swept recursively) and the +//! revision structure. Each string is looked for in every encoding a PDF can hold it in. +//! 3. It places every glyph still shown on the redacted pages with its own text-state tracking +//! and width model (the font's `/Widths`, else a flat guess) and fails a region when the +//! centre of a surviving glyph lies inside it, whatever the interpreter decided. +//! 4. It also re-checks the non-text content under each region — on the page and in the +//! appearance streams of the annotations left on it: no image still carries pixels there, and +//! no path or shading paints there unclipped. +//! +//! The proof fails closed: a survivor, but also anything it could not sweep (page content, a form +//! or an appearance stream, XMP, or an attachment that won't decode in full or can't be opened; a +//! nesting or work limit hit; an appearance with content that can't be placed on its page), makes +//! it fail. The one exception is binary data that redaction +//! never edits: a font program, colour profile, `/Indexed` lookup table, function sample table or +//! Flate/LZW image that is damaged or too long to decode is searched as stored bytes and reported +//! as `raw_only`, not as unswept. A proof that cannot fail is worthless. +//! +//! Work is bounded: every stream is decoded strictly under `limits::MAX_STREAM`, all decodes together +//! under `limits::MAX_TOTAL_DECODED`, and all searching under `limits::MAX_WORK`. A document that needs more +//! is reported as unswept, not skipped (binary data as above excepted). +//! +//! The result is a [`Proof`]: a per-region manifest (page, region, glyph counts, SHA-256 of +//! the page content after) and a verdict per surface. It holds counts and digests, +//! never the redacted text, unless [`ProofOptions::include_plaintext`] asks for it. +//! +//! Limits, stated plainly: text that exists only as pixels of an image is not text; a string +//! that also appears in content that was *not* redacted is counted, over the whole document, on +//! the page text, the raw file and the decoded streams (those hold the kept copy too) and fails +//! on a surplus; names are swept against the matches the input itself had, since a redacted word +//! that is also a `/Subtype` or a `/BaseFont` is structure, not a leftover copy, and a surplus of +//! matches still fails; every other surface is swept with no allowance. A word of six characters or more +//! that also spells a structural name (`/Filter`, `/MediaBox`) can still be reported by the raw-file and +//! decoded-stream surfaces, which search stored bytes and cannot tell a name from a copy: that is +//! over-reporting, never a pass. The raw file and the decoded +//! streams are searched for strings of at least six characters only (shorter ones match object +//! syntax and numbers by chance), skipping the trailer's file identifier and the +//! pixels of an image codec (DCT, JPX, JBIG2, CCITT), which are searched as stored bytes only; shorter strings are still found on the page text, +//! in strings, names, Info, XMP and attachment text. A region without extractable text is reported as such, and as +//! unverifiable when the page draws text this extractor cannot decode. The geometric check +//! places glyphs with the font's `/Widths` only: for a font without them the widths are a guess, +//! so a neighbouring glyph may be reported when the guess puts its centre inside a region. +//! `runs_removed` and `codes_removed` are exact counts of what was cut: a proof that is shared +//! discloses how much text sat under each region (never what it said). + +use std::fmt::Write as _; + +use pdfcraft_cos::ObjRef; + +mod extract; +mod geometry; +pub(crate) mod limits; +#[cfg(test)] +mod search_tests; +mod snapshot; +mod sweep; +mod util; + +pub use snapshot::Snapshot; +/// What the proof established for one region. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Status { + /// What was removed was swept for on every surface and is gone. + Verified, + /// The region held no extractable text: nothing was removed, and that is recorded as such. + VerifiedNoTextInRegion, + /// Neither "text was removed" nor "there was none" can be checked (the page draws text this + /// extractor can't decode, or an image under the region can't be inspected). The proof can't + /// pass while any region is unverifiable. + Unverifiable, + /// A survivor was found, or the mechanical evidence is missing. + Failed, +} + +/// The places a redacted string is looked for. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub enum Surface { + RawBytes, + ObjectStrings, + DecodedStreams, + ExtractedText, + Info, + Xmp, + EmbeddedFiles, + Revisions, +} + +const SURFACES: [Surface; 8] = [ + Surface::RawBytes, + Surface::ObjectStrings, + Surface::DecodedStreams, + Surface::ExtractedText, + Surface::Info, + Surface::Xmp, + Surface::EmbeddedFiles, + Surface::Revisions, +]; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Verdict { + /// Swept, nothing found. + Clean, + /// Swept, a redacted string (or an earlier revision) is still there. + Survivor, + /// Could not be swept; the proof fails. + Unswept, + /// There is nothing of this kind in the document. + Absent, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct SurfaceReport { + pub surface: Surface, + pub verdict: Verdict, + /// Streams, strings, pages or files looked at. + pub scanned: usize, + /// Streams scanned as stored bytes only: image-codec pixels (not text), and font, colour or + /// image streams that could not be decoded in full. + pub raw_only: usize, + pub survivors: usize, + /// Where each survivor sits ("object 12", "page 1", "attachment #1 > object 4"); never the text. + pub locations: Vec, + /// Why the surface could not be swept. + pub problems: Vec, +} + +/// The manifest entry of one redaction region. +#[derive(Clone, Debug, PartialEq)] +pub struct Excision { + /// 0-based page. + pub page: usize, + /// The region in user space. + pub region: [f64; 4], + /// The Redact annotation it came from. + pub mark: ObjRef, + pub status: Status, + pub detail: String, + /// Glyph runs (consecutive glyphs of one string) removed under the region, and their + /// character codes. + pub runs_removed: usize, + pub codes_removed: usize, + /// SHA-256 of the page's decoded content streams, joined, after redaction. There is no digest + /// of the content before: a proof is meant to be shared, and a hash of the original page + /// would let a reader confirm a guess at what it said. + pub page_sha256_after: Option, + /// Glyph-carrying text operators on the page (forms included) before and after. A `TJ` of + /// numbers only (the gap left for a removed string) carries no glyphs and does not count. + pub text_ops_before: Option, + pub text_ops_after: Option, + /// The removed text, only when [`ProofOptions::include_plaintext`] is set. + pub removed_text: Option>, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct Proof { + pub entries: Vec, + pub surfaces: Vec, + /// Located, plain reasons for every failure (never the redacted text). + pub failures: Vec, +} + +impl Proof { + /// Survivors found across all surfaces. + pub fn survivors(&self) -> usize { + self.surfaces.iter().map(|s| s.survivors).sum() + } + + /// Surfaces that could not be swept. + pub fn unswept(&self) -> usize { + self.surfaces.iter().filter(|s| s.verdict == Verdict::Unswept || !s.problems.is_empty()).count() + } + + pub fn unverifiable(&self) -> usize { + self.entries.iter().filter(|e| e.status == Status::Unverifiable).count() + } + + /// Pass only when nothing failed, nothing is unverifiable and no surface went unswept. + pub fn passed(&self) -> bool { + self.failures.is_empty() + && self.survivors() == 0 + && self.unswept() == 0 + && self.entries.iter().all(|e| matches!(e.status, Status::Verified | Status::VerifiedNoTextInRegion)) + } + + /// A human-readable report: the verdict, the manifest and the surfaces. It names places and + /// counts, and the removed text only if it was asked for. + pub fn to_text(&self) -> String { + let mut s = String::new(); + let _ = writeln!(s, "Redaction proof: {}", if self.passed() { "PASS" } else { "FAIL" }); + for f in &self.failures { + let _ = writeln!(s, " failure: {f}"); + } + for (i, e) in self.entries.iter().enumerate() { + let _ = writeln!( + s, + " region {} page {} {:?} {:?}: {} run(s), {} code(s) removed; text ops {:?} -> {:?}; page sha256 after {}{}", + i + 1, + e.page + 1, + e.region, + e.status, + e.runs_removed, + e.codes_removed, + e.text_ops_before, + e.text_ops_after, + e.page_sha256_after.as_deref().unwrap_or("-"), + if e.detail.is_empty() { String::new() } else { format!(" ({})", e.detail) }, + ); + } + for r in &self.surfaces { + let _ = writeln!( + s, + " {:?}: {:?} ({} scanned, {} raw only, {} survivor(s)){}", + r.surface, + r.verdict, + r.scanned, + r.raw_only, + r.survivors, + if r.locations.is_empty() { String::new() } else { format!(" at {}", r.locations.join(", ")) }, + ); + for p in &r.problems { + let _ = writeln!(s, " unswept: {p}"); + } + } + let _ = writeln!(s, " Evidence, not a guarantee: text that exists only as pixels of an image is not text, and is not searched."); + s + } +} + +/// What a proof may contain. +#[derive(Clone, Debug, Default)] +pub struct ProofOptions { + /// Include the removed text in the manifest. Off by default: a proof is meant to be shared, + /// and must not carry what it proves was removed. + pub include_plaintext: bool, +} + +/// Handles for the tests to reach the byte sweep without a whole document. +#[cfg(test)] +pub(crate) mod tests_support { + use pdfcraft_content::parse; + use pdfcraft_cos::{Dict, Object, Stream}; + + use super::extract::{carries_glyphs, parse_cmap}; + use super::limits::{MAX_STREAM, MAX_TOTAL_DECODED, MAX_WORK, MIN_STRUCTURED_NEEDLE}; + use super::sweep::{Needle, add_needle, bytes_counts, bytes_hits, walk_strings}; + use super::util::Ctx; + pub(crate) use super::util::survivor_encodings; + + /// The proof's whole search-work budget (for tests that pre-spend part of it). + pub(crate) fn work_budget() -> u64 { + MAX_WORK + } + + /// The byte sweep of `hay` for a bulk of `count` distinct needles, on a context already + /// `spent` units into its work budget: `Some` (with the needles that occur) while the sweep + /// stays under the budget, `None` once it ran out. + pub(crate) fn bulk_counts(hay: &[u8], text: &str, count: usize, spent: u64) -> Option { + let mut needles: Vec = Vec::new(); + for i in 0..count { + add_needle(&mut needles, false, format!("{text}{i:04}"), Some(i)); + } + let ctx = Ctx::new(Default::default()); + ctx.work.set(spent); + bytes_counts(&needles, hay, MIN_STRUCTURED_NEEDLE, &ctx).map(|hits| hits.len()) + } + + /// The glyph-carrying text-showing operators of a decoded content stream. + pub fn count_text_operators(decoded: &[u8]) -> usize { + parse(decoded).ops.iter().filter(|op| carries_glyphs(op)).count() + } + + /// Does the byte sweep for `text` find it in `hay`? + pub fn found(text: &str, hay: &[u8]) -> bool { + let mut needles: Vec = Vec::new(); + add_needle(&mut needles, false, text.to_string(), Some(0)); + bytes_hits(&needles, hay, 3, &Ctx::new(Default::default())).is_some_and(|h| !h.is_empty()) + } + + /// Does this CMap parse completely (not cut short by a bound)? + pub fn cmap_is_complete(data: &[u8]) -> bool { + parse_cmap(data).is_some() + } + + /// Is an object nested `depth` levels deep walked in full? + pub fn walks_in_full(depth: usize) -> bool { + let mut o = Object::Null; + for _ in 0..depth { + o = Object::Array(vec![o]); + } + walk_strings(&o, 0, &mut |_| {}) + } + + /// Does decoding fail once the cumulative decode budget is used up (and not before)? + pub fn decode_budget_runs_out() -> (bool, bool) { + let ctx = Ctx::new(Default::default()); + let s = Stream::from_raw(Dict::new(), vec![0u8; 100]); + let before = ctx.decode(&s).is_ok(); + let spent = Ctx::with_decode_left(50); + (before, spent.decode(&s).is_err()) + } + + /// Does a decode that fails past the one-stream cap pay for the attempt, so the same bomb + /// cannot be decoded (and fail) again and again for free? + /// `(refused, charged, then refused for want of allowance)`. + pub fn failed_decode_charges_the_budget() -> (bool, bool, bool) { + let big = vec![0u8; MAX_STREAM + (1 << 20)]; + let bomb = Stream::flate(Dict::new(), &big); + drop(big); + let small = Stream::from_raw(Dict::new(), vec![0u8; 100]); + let ctx = Ctx::with_decode_left(u64::try_from(MAX_STREAM + 50).unwrap_or(u64::MAX)); + let refused = ctx.decode(&bomb).is_err(); + let charged = ctx.decoded.get() == MAX_TOTAL_DECODED - 50; + let gone = ctx.decode(&small).is_err(); + (refused, charged, gone) + } + + /// Does the byte search report `None` once the work budget is used up (and not before)? + pub fn search_budget_runs_out() -> (bool, bool) { + let mut needles: Vec = Vec::new(); + add_needle(&mut needles, false, "secret".to_string(), Some(0)); + let ctx = Ctx::new(Default::default()); + let before = bytes_hits(&needles, b"some bytes", 3, &ctx).is_some(); + ctx.work.set(MAX_WORK); + (before, bytes_hits(&needles, b"some bytes", 3, &ctx).is_none()) + } + + /// Is a survivor counted on an earlier page still reported when the work budget runs out on + /// a later one? `(survivor reported, page left unswept)`. + pub fn page_sweep_survivors_survive_the_limit() -> (bool, bool) { + use super::Surface; + use super::sweep::{Acc, sweep_pages}; + let objs = vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R 4 0 R] /Count 2 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 5 0 R /Resources << /Font << /F1 6 0 R >> >> >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 7 0 R /Resources << /Font << /F1 6 0 R >> >> >>".to_vec(), + crate::tests_verify::stream("", b"BT /F1 10 Tf 10 100 Td (SECRETVALUE) Tj ET"), + b"<< /Type /Font /Subtype /TrueType /BaseFont /Arial >>".to_vec(), + crate::tests_verify::stream("", &format!("BT /F1 10 Tf 10 100 Td ({}) Tj ET", "x".repeat(700_000)).into_bytes()), + ]; + let doc = pdfcraft_cos::Document::open(std::sync::Arc::new(crate::tests_verify::pdf_bytes(objs, ""))).unwrap(); + let mut needles = Vec::new(); + add_needle(&mut needles, false, "SECRETVALUE".to_string(), Some(0)); + let mut acc = Acc::new(Ctx::new(Default::default())); + // Page 1 (the survivor's) fits in what is left; page 2 does not. + acc.ctx.work.set(MAX_WORK - 100_000); + sweep_pages(&needles, &doc, 0, "", &mut acc); + let text = acc.reps.get(&Surface::ExtractedText).unwrap(); + (text.survivors > 0, !text.problems.is_empty()) + } + + /// Does a font's `/W` parsing answer to the shared work budget, and not only to its own + /// counter (a form invoked again rebuilds its fonts, and would pay for the same array + /// again)? `(read in full, refused on a spent budget)`. + pub fn font_widths_charge_shares_the_budget() -> (bool, bool) { + use super::extract::extract_page; + let font = b"<< /Type /Font /Subtype /Type0 /BaseFont /X /Encoding /Identity-H /DescendantFonts [<< /Type /Font /Subtype /CIDFontType2 /BaseFont /X /DW 1000 /W [1 65535 250] >>] >>".to_vec(); + let objs = vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Resources << /Font << /F2 5 0 R >> >> >>".to_vec(), + crate::tests_verify::stream("", b"BT /F2 10 Tf 10 100 Td <00410042> Tj ET"), + font, + ]; + let bytes = std::sync::Arc::new(crate::tests_verify::pdf_bytes(objs, "")); + let probe = |spent: u64| -> bool { + let doc = pdfcraft_cos::Document::open(bytes.clone()).unwrap(); + let ctx = Ctx::new(Default::default()); + ctx.work.set(spent); + let page = &pdfcraft_model::pages(&doc)[0]; + extract_page(&doc, &page.dict, &ctx, &[]).is_ok() + }; + (probe(0), !probe(MAX_WORK - 10_000)) + } +} diff --git a/crates/redact/src/verify/search_tests.rs b/crates/redact/src/verify/search_tests.rs new file mode 100644 index 0000000000..4b8d84c243 --- /dev/null +++ b/crates/redact/src/verify/search_tests.rs @@ -0,0 +1,34 @@ +use super::{ + sweep::{add_needle, bytes_counts, bytes_hits}, + util::Ctx, +}; +use std::collections::HashSet; + +#[test] +fn a_thousand_needles_share_one_multimegabyte_scan() { + let mut needles = Vec::new(); + for i in 0..1000 { + add_needle(&mut needles, false, format!("SECRET_{i:04}"), None); + } + let hay = vec![b'z'; 8 << 20]; + assert_eq!(bytes_hits(&needles, &hay, 6, &Ctx::new(HashSet::new())), Some(Vec::new())); + assert_eq!(bytes_counts(&needles, &hay, 6, &Ctx::new(HashSet::new())), Some(Vec::new())); +} + +#[test] +fn overlapping_encodings_count_one_occurrence() { + let mut needles = Vec::new(); + add_needle(&mut needles, false, "SECRET42".into(), None); + let text: Vec = "SECRET42".encode_utf16().flat_map(u16::to_be_bytes).collect(); + let mut hay = vec![0xfe, 0xff]; + hay.extend(text); + assert_eq!(bytes_counts(&needles, &hay, 6, &Ctx::new(HashSet::new())), Some(vec![(0, 1)])); +} + +#[test] +fn match_count_exhaustion_is_incomplete_not_a_truncated_pass() { + let mut needles = Vec::new(); + add_needle(&mut needles, true, "SECRET42".into(), None); + let hay = b"SECRET42 ".repeat((1 << 16) + 1); + assert!(bytes_counts(&needles, &hay, 6, &Ctx::new(HashSet::new())).is_none()); +} diff --git a/crates/redact/src/verify/snapshot.rs b/crates/redact/src/verify/snapshot.rs new file mode 100644 index 0000000000..fe74571bb4 --- /dev/null +++ b/crates/redact/src/verify/snapshot.rs @@ -0,0 +1,532 @@ +//! The snapshot taken before applying (what is about to go) and the proof run on the output. + +use std::collections::{BTreeMap, HashMap, HashSet}; +use std::sync::Arc; + +use pdfcraft_content::Matrix; +use pdfcraft_cos::{Dict, Document, ObjRef, Object, SaveOptions, write_full}; + +use super::extract::extract_page; +use super::geometry::{Finding, check_geometry}; +use super::sweep::{Acc, Needle, add_needle, bytes_hits, sweep_doc, walk_strings}; +use super::util::{Ctx, count, latin1, sha256, squash, words}; +use super::{Excision, Proof, ProofOptions, SURFACES, Status, Surface}; +use crate::interp::{Mode, Removed, Scope, process}; +use crate::verify::limits::MIN_NEEDLE; +use crate::{Mark, RedactError, Report, page_streams}; + +pub(super) struct RegionSnap { + pub(super) page: usize, + pub(super) region: [f64; 4], + pub(super) mark: ObjRef, + pub(super) runs: Vec<(Vec, String)>, + /// The runs of one text operator joined, where it split them: `[(sec) (ret)] TJ` reads as + /// `secret`, which a copy elsewhere in the file holds whole. + pub(super) joined: Vec<(Vec, String)>, + /// The character codes the runs hold (a composite font's codes are several bytes). + pub(super) codes: usize, +} + +pub(super) struct PageSnap { + pub(super) sha: Option, + pub(super) text_ops: Option, + pub(super) undecodable_ops: usize, +} + +/// What redaction is about to remove, recorded before it does. +pub struct Snapshot { + regions: Vec, + pages: BTreeMap, + needles: Vec, + /// Things that made the "before" side unreadable (reported on the text surface). + problems: Vec, + /// The document's page text before redaction (decoded view, code view), for the allowance + /// of needles added later. + src: (String, String), + /// The names of the document (structure redaction never edits), for the name allowance of + /// needles added later. + names: BTreeMap, usize>, + /// Digests of the decoded content of the streams this run created to draw the redaction + /// boxes and labels (see [`Snapshot::set_overlay_streams`]). + overlay: HashSet, +} + +impl std::fmt::Debug for Snapshot { + // The snapshot holds what is about to be redacted: never print it. + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Snapshot").field("regions", &self.regions.len()).field("needles", &self.needles.len()).finish() + } +} + +impl Snapshot { + /// Record what redacting the marks on `pages` (0-based; `None` = all) will remove. Call it + /// before [`apply`](crate::apply); it changes nothing. + pub fn capture(doc: &Document, pages: Option<&[usize]>) -> Result { + let all = crate::marks(doc); + let chosen: Vec<&Mark> = all.iter().filter(|m| pages.is_none_or(|p| p.contains(&m.page))).collect(); + if chosen.is_empty() { + return Err(RedactError::NothingToApply); + } + Snapshot::capture_marks(doc, &chosen) + } + + pub(crate) fn capture_marks(doc: &Document, chosen: &[&Mark]) -> Result { + let mut by_page: Vec = chosen.iter().map(|m| m.page).collect(); + by_page.sort_unstable(); + by_page.dedup(); + let page_list = pdfcraft_model::pages(doc); + let ctx = Ctx::new(HashSet::new()); + let mut regions: Vec = Vec::new(); + let mut pages = BTreeMap::new(); + let mut images = pdfcraft_cos::ImageBudget::new(doc); + // What was cut, each copy once, however many overlapping marks it lay under: the allowance + // for what stays is what the source holds less this. + let (mut cut_text, mut cut_raw) = (String::new(), String::new()); + for &pi in &by_page { + let page = page_list.get(pi).ok_or(RedactError::Unreadable(pi + 1))?; + let page_marks: Vec<&&Mark> = chosen.iter().filter(|m| m.page == pi).collect(); + let rects: Vec<[f64; 4]> = page_marks.iter().flat_map(|m| m.rects.iter().copied()).collect(); + let (_, data) = page_streams(doc, &page.dict, pi)?; + let resources = page.dict.get(b"Resources").and_then(|r| doc.resolve(r).as_dict().cloned()).unwrap_or_default(); + // The interpreter's own removal path, on a scratch copy: what it takes out is what + // has to be gone. + let mut scratch = doc.clone(); + let mut sink = Report::default(); + let removed: Vec = { + let mut scope = Scope::new(&rects, Mode::Apply, &mut sink, pdfcraft_cos::DecodeBudget::page(), &mut images); + scope.removed = Some(Vec::new()); + process(&mut scratch, &mut scope, &data, &resources, Matrix::IDENTITY); + scope.removed.take().unwrap_or_default() + }; + let lines = joined(&removed); + let mut seen = HashSet::new(); + for line in lines.iter().filter(|l| seen.insert((l.op, l.at, l.raw.clone()))) { + cut_text.push('\u{1}'); + cut_text.push_str(&squash(&line.text)); + cut_raw.push('\u{1}'); + cut_raw.push_str(&squash(&latin1(&line.raw))); + } + let before = extract_page(doc, &page.dict, &ctx, &[]); + pages.insert( + pi, + PageSnap { + sha: Some(sha256(&data)), + text_ops: before.as_ref().ok().map(|e| e.glyph_ops), + undecodable_ops: before.as_ref().map_or(0, |e| e.undecodable_ops), + }, + ); + let mut k = 0; + for m in &page_marks { + for r in &m.rects { + let hits: Vec<&Removed> = removed.iter().filter(|x| x.rect == k).collect(); + let runs = hits.iter().map(|x| (x.raw.clone(), x.text.clone())).collect(); + let codes = hits.iter().map(|x| x.codes).sum(); + let joined = lines.iter().filter(|l| l.rect == k && l.pieces > 1).map(|l| (l.raw.clone(), l.text.clone())).collect(); + regions.push(RegionSnap { page: pi, region: *r, mark: m.obj, runs, joined, codes }); + k += 1; + } + } + } + // The document's text as the independent extractor reads it, for the allowance of + // strings that legitimately remain elsewhere. + let (mut src_text, mut src_raw) = (String::new(), String::new()); + let mut problems = Vec::new(); + for (i, p) in page_list.iter().enumerate() { + match extract_page(doc, &p.dict, &ctx, &[]) { + Ok(e) => { + src_text.push_str(&e.text); + src_raw.push_str(&e.raw); + } + Err(()) => { + problems.push(format!("page {} of the source can't be read, so strings left on it can't be told from removed ones", i + 1)) + } + } + } + let mut needles: Vec = Vec::new(); + for (ri, r) in regions.iter().enumerate() { + for (raw, text) in r.runs.iter().chain(&r.joined) { + let raw_s = latin1(raw); + for (raw_view, whole) in [(false, text.trim().to_string()), (true, raw_s)] { + let mut shown: Vec = Vec::new(); + if !squash(&whole).is_empty() { + shown.push(whole.clone()); + shown.extend(words(&whole).into_iter().filter(|w| *w != whole).map(str::to_string)); + } + for s in shown { + add_needle(&mut needles, raw_view, s, Some(ri)); + } + } + } + } + for (src, cut, raw) in [(&src_text, &cut_text, false), (&src_raw, &cut_raw, true)] { + match (super::sweep::key_counts(&needles, src, raw, &ctx), super::sweep::key_counts(&needles, cut, raw, &ctx)) { + (Some(source), Some(removed)) => { + let removed: BTreeMap = removed.into_iter().collect(); + for (i, total) in source { + if let Some(n) = needles.get_mut(i) { + n.allowance = total.saturating_sub(removed.get(&i).copied().unwrap_or(0)); + } + } + } + _ => problems.push("the source text allowances could not be counted in full".into()), + } + } + // The names the removed text legitimately sits in (a redacted word that is also a + // `/Subtype`, a `/Filter`, a `/BaseFont`): names are structure redaction never edits, + // so the output may hold the input's own matches, and only a surplus is a survivor. + let names = all_names(doc); + let name_allow: Vec> = needles.iter().map(|n| count_name_matches(&names, n, &ctx)).collect(); + if name_allow.iter().any(Option::is_none) { + problems.push("the name allowances could not be counted in full, so names are swept with none of them allowed".into()); + } + for (n, allow) in needles.iter_mut().zip(name_allow) { + n.name_allowance = allow.unwrap_or(0); + } + Ok(Snapshot { regions, pages, needles, problems, src: (src_text, src_raw), names, overlay: HashSet::new() }) + } + + /// Tell the proof which streams this run created to draw the redaction boxes and labels + /// (`created`: references into `doc`, the document after `apply`). Call it before + /// [`prove`](Snapshot::prove) / [`prove_saved`](Snapshot::prove_saved). + /// + /// Those streams paint over the regions on purpose and may show label text, so the proof + /// leaves them out of the text sweep and the paint check. They are recognised by the digest + /// of their decoded content (a save renumbers objects, so references would not survive it), + /// and *only* those: a stream that merely carries a `/PCMark /Redaction` key in the input is + /// not exempt. Streams that can't be read here are an error; nothing is then exempted. + pub fn set_overlay_streams(&mut self, doc: &Document, created: &[ObjRef]) -> Result<(), RedactError> { + let ctx = Ctx::new(HashSet::new()); + let mut digests = HashSet::new(); + for r in created { + let obj = doc.try_get(r.num).map_err(|_| RedactError::OverlayUnreadable)?; + let Object::Stream(s) = &*obj else { continue }; + let data = ctx.decode(s).map_err(|_| RedactError::OverlayUnreadable)?; + digests.insert(sha256(&[data])); + } + self.overlay = digests; + Ok(()) + } + + /// Also look for these strings in the output: text that left the document by another route + /// than a page's glyphs (a removed annotation's `/Contents` or `/RC`, a form field's value). + /// Each is searched whole, on every surface. A string that also legitimately remains as page + /// text is counted on the page-content surfaces only (as for the removed glyph runs). + /// Strings that are blank or shorter than the minimum needle are ignored. + pub fn add_removed_strings(&mut self, strings: &[String]) { + for s in strings { + let text = s.trim(); + let key = squash(text); + if key.chars().count() < MIN_NEEDLE { + continue; + } + let known = self.needles.iter().any(|n| !n.raw_view && n.key == key); + // No owner: a survivor fails the proof through its surface; no region carries it. + add_needle(&mut self.needles, false, text.to_string(), None); + if !known { + let ctx = Ctx::new(HashSet::new()); + let matches = self.needles.last().map(|n| count_name_matches(&self.names, n, &ctx)); + if let Some(n) = self.needles.last_mut() { + n.allowance = count(&self.src.0, &n.key); + n.name_allowance = matches.flatten().unwrap_or(0); + } + if matches == Some(None) { + self.problems.push("the name allowances could not be counted in full, so names are swept with none of them allowed".into()); + } + } + } + } + + /// Serialize `doc` the way a save would and sweep the result. The output is re-opened with + /// `password`, the credentials the caller opened the document with (`None` tries the empty + /// password): a save of an encrypted document is itself encrypted, and a proof that could + /// not read its own subject would prove nothing. + pub fn prove(&self, doc: &Document, password: Option<&str>, opts: &ProofOptions) -> Proof { + match write_full(doc, &SaveOptions::default()) { + Ok(bytes) => self.prove_inner(&bytes, Some(doc), password, opts), + Err(e) => { + let mut p = self.prove_inner(&[], None, password, opts); + p.failures.push(format!("the document could not be serialized for the sweep: {e}")); + p + } + } + } + + /// Sweep an already saved file (for example an incrementally saved one, whose earlier + /// revision still holds the original content). The file is opened with `password`, the + /// credentials it is read with. + pub fn prove_saved(&self, bytes: &[u8], password: Option<&str>, opts: &ProofOptions) -> Proof { + self.prove_inner(bytes, None, password, opts) + } + + fn prove_inner(&self, bytes: &[u8], live: Option<&Document>, password: Option<&str>, opts: &ProofOptions) -> Proof { + let mut acc = Acc::new(Ctx::new(self.overlay.clone())); + let mut failures: Vec = Vec::new(); + for p in &self.problems { + acc.problem(Surface::ExtractedText, p.clone()); + } + // Always the caller's credentials, never unconditionally the empty password: a + // document whose user password is not empty must be provable with that password, and + // must fail closed (an unswept output, never a false pass) without it. + let parsed = if bytes.is_empty() { None } else { Document::open_with_password(Arc::new(bytes.to_vec()), password).ok() }; + match &parsed { + Some(out) => sweep_doc(&self.needles, out, bytes, 0, "", &mut acc), + None => { + let why = "the output could not be read back".to_string(); + for s in SURFACES { + acc.problem(s, why.clone()); + } + failures.push(why); + } + } + if live.is_some_and(|d| !d.full_save_required()) { + acc.survivor(Surface::Revisions, "the document would be saved as an incremental update, keeping the original revision".into(), None); + } + + // The non-text content under each region, on the output. + let (findings, after) = match &parsed { + Some(out) => self.check_output(out, &acc.ctx), + None => Default::default(), + }; + let mut entries = Vec::with_capacity(self.regions.len()); + for (i, r) in self.regions.iter().enumerate() { + let (entry, failure) = self.entry(i, r, &findings, &after, &acc.failed_needles, opts); + failures.extend(failure); + entries.push(entry); + } + let surfaces = acc.finish(); + for s in &surfaces { + if s.survivors > 0 { + failures.push(format!("{:?}: {} survivor(s) at {}", s.surface, s.survivors, s.locations.join(", "))); + } + for p in &s.problems { + failures.push(format!("{:?} could not be swept: {p}", s.surface)); + } + } + Proof { entries, surfaces, failures } + } + + /// The non-text content under each region of the output (what still paints or shows there), + /// and per page the digest and glyph-operator count of what `apply` left. + fn check_output(&self, out: &Document, ctx: &Ctx) -> (HashMap>, PagesAfter) { + let mut findings: HashMap> = HashMap::new(); + let mut after: PagesAfter = BTreeMap::new(); + let out_pages = pdfcraft_model::pages(out); + for &pi in self.pages.keys() { + let idx: Vec = (0..self.regions.len()).filter(|&i| self.regions.get(i).is_some_and(|r| r.page == pi)).collect(); + let Some(page) = out_pages.get(pi) else { + for &i in &idx { + findings.entry(i).or_default().push(Finding::Missing); + } + continue; + }; + let rects: Vec<[f64; 4]> = idx.iter().filter_map(|&i| self.regions.get(i).map(|r| r.region)).collect(); + let (sha, ops, shown) = page_after(out, &page.dict, ctx, &rects); + after.insert(pi, (sha, ops)); + match shown { + Some(shown) => { + for (k, n) in shown.into_iter().enumerate() { + if let (Some(&i), true) = (idx.get(k), n > 0) { + findings.entry(i).or_default().push(Finding::Text(n)); + } + } + } + None => { + for &i in &idx { + findings + .entry(i) + .or_default() + .push(Finding::Unverifiable("the page text can't be placed, so what still shows under the region can't be checked")); + } + } + } + for (k, f) in check_geometry(out, &page.dict, &rects, ctx) { + if let Some(&i) = idx.get(k) { + findings.entry(i).or_default().push(f); + } + } + } + (findings, after) + } + + /// The manifest entry of region `i`, and why it failed when it did. + fn entry( + &self, + i: usize, + r: &RegionSnap, + findings: &HashMap>, + after: &PagesAfter, + failed_needles: &HashSet, + opts: &ProofOptions, + ) -> (Excision, Option) { + let mut failure = None; + let snap = self.pages.get(&r.page); + let (sha_after, ops_after) = after.get(&r.page).cloned().unwrap_or((None, None)); + let sha_before = snap.and_then(|s| s.sha.clone()); + let owned: Vec<&Needle> = self.needles.iter().filter(|n| n.owners.contains(&i)).collect(); + let survived = self.needles.iter().enumerate().filter(|(ni, n)| n.owners.contains(&i) && failed_needles.contains(ni)).count(); + let mut detail: Vec = Vec::new(); + let mut status = if r.runs.is_empty() { Status::VerifiedNoTextInRegion } else { Status::Verified }; + let fs = findings.get(&i).map(Vec::as_slice).unwrap_or_default(); + if survived > 0 { + status = Status::Failed; + detail.push(format!("{survived} removed string(s) still found")); + } + for f in fs { + match f { + Finding::Text(n) => detail.push(format!("{n} glyph(s) still show inside the region (placed independently of the redaction)")), + Finding::Image => detail.push("an image still carries pixels under the region".into()), + Finding::Path => detail.push("a path still paints inside the region".into()), + Finding::Shading => detail.push("a shading still paints inside the region".into()), + Finding::Missing => detail.push("the page is missing from the output".into()), + Finding::Unverifiable(why) => detail.push((*why).to_string()), + } + if !matches!(f, Finding::Unverifiable(_)) { + status = Status::Failed; + } else if status != Status::Failed { + status = Status::Unverifiable; + } + } + if sha_before.is_none() || sha_after.is_none() { + status = Status::Failed; + detail.push("the page content can't be read on one side, so it can't be compared".into()); + } else if !r.runs.is_empty() && sha_before == sha_after { + status = Status::Failed; + detail.push("glyphs were removed but the page content is unchanged".into()); + } + if r.runs.is_empty() && snap.is_some_and(|s| s.undecodable_ops > 0) && status != Status::Failed { + status = Status::Unverifiable; + detail.push("the page draws text that can't be decoded, so \"no text in the region\" can't be checked".into()); + } + let ambiguous = owned.iter().filter(|n| n.allowance > 0).count(); + if ambiguous > 0 { + detail.push(format!("{ambiguous} string(s) also remain in retained content; only a surplus over those copies counts")); + } + if matches!(status, Status::Failed | Status::Unverifiable) { + failure = Some(format!("page {} region {:?}: {}", r.page + 1, r.region, detail.join("; "))); + } + let entry = Excision { + page: r.page, + region: r.region, + mark: r.mark, + status, + detail: detail.join("; "), + runs_removed: r.runs.len(), + codes_removed: r.codes, + page_sha256_after: sha_after, + text_ops_before: snap.and_then(|s| s.text_ops), + text_ops_after: ops_after, + removed_text: opts.include_plaintext.then(|| r.runs.iter().map(|(_, t)| t.clone()).collect()), + }; + (entry, failure) + } +} + +/// Per page: the digest of its content in the output and its glyph-operator count. +type PagesAfter = BTreeMap, Option)>; + +/// Every name of the document (the values, wherever an object holds them), with how often +/// each occurs. +fn all_names(doc: &Document) -> BTreeMap, usize> { + let mut names: BTreeMap, usize> = BTreeMap::new(); + for num in doc.object_numbers() { + if let Ok(obj) = doc.try_get(num) { + walk_strings(&obj, 0, &mut |o| { + if let Object::Name(n) = o { + let seen = names.entry(n.clone()).or_default(); + *seen = seen.saturating_add(1); + } + }); + } + } + names +} + +/// How many of `names` carry `needle`, by the same search the sweep runs on a name. `None` +/// when the search work ran out: an unknown allowance must not read as "none allowed", so the +/// caller reports that (and sweeps names with no allowance). +fn count_name_matches(names: &BTreeMap, usize>, needle: &Needle, ctx: &Ctx) -> Option { + if needle.short() { + return Some(0); + } + let mut total = 0usize; + for (name, seen) in names { + if !bytes_hits(std::slice::from_ref(needle), name, MIN_NEEDLE, ctx)?.is_empty() { + total = total.saturating_add(*seen); + } + } + Some(total) +} + +/// The page's content digest and glyph-operator count in the output, and per region the glyphs +/// still placed inside it (`None` when the page's text can't be placed). The digest covers the +/// page's content as `apply` left it, the overlay streams included. +pub(super) fn page_after(doc: &Document, page: &Dict, ctx: &Ctx, regions: &[[f64; 4]]) -> (Option, Option, Option>) { + let sha = page_streams(doc, page, 0).ok().map(|(_, d)| sha256(&d)); + let e = extract_page(doc, page, ctx, regions).ok(); + (sha, e.as_ref().map(|e| e.glyph_ops), e.map(|e| e.shown_in)) +} + +/// One text operator's removed runs under one region, as a reader takes them: joined. +struct Line { + rect: usize, + op: usize, + at: (usize, usize), + raw: Vec, + text: String, + pieces: usize, +} + +/// The runs of `removed` joined by text operator and region, in the order they were shown. +fn joined(removed: &[Removed]) -> Vec { + let mut lines: Vec = Vec::new(); + let mut found: HashMap<(usize, usize), usize> = HashMap::new(); + for r in removed { + let line = found.get(&(r.op, r.rect)).copied().and_then(|i| lines.get_mut(i)); + match line { + Some(l) => { + l.raw.extend_from_slice(&r.raw); + l.text.push_str(&r.text); + l.pieces += 1; + } + None => { + found.insert((r.op, r.rect), lines.len()); + lines.push(Line { rect: r.rect, op: r.op, at: r.at, raw: r.raw.clone(), text: r.text.clone(), pieces: 1 }); + } + } + } + lines +} + +#[cfg(test)] +mod tests { + use super::*; + + fn run(rect: usize, op: usize, place: usize, text: &str) -> Removed { + Removed { rect, raw: text.as_bytes().to_vec(), text: text.to_string(), codes: text.len(), op, at: (place, 0) } + } + + #[test] + fn a_word_split_across_a_tj_array_is_one_line_for_each_region() { + let lines = joined(&[run(0, 1, 0, "sec"), run(1, 1, 0, "sec"), run(0, 1, 1, "ret"), run(1, 1, 1, "ret"), run(0, 2, 0, "other")]); + let shown: Vec<(usize, usize, &str, usize)> = lines.iter().map(|l| (l.rect, l.op, l.text.as_str(), l.pieces)).collect(); + assert_eq!(shown, [(0, 1, "secret", 2), (1, 1, "secret", 2), (0, 2, "other", 1)]); + } + + #[test] + fn overlapping_marks_are_one_copy_and_two_copies_stay_two() { + // Under two marks at once: the same operator, place and bytes, so one copy was cut. + let both = joined(&[run(0, 1, 0, "word"), run(1, 1, 0, "word")]); + let mut seen = HashSet::new(); + assert_eq!(both.iter().filter(|l| seen.insert((l.op, l.at, l.raw.clone()))).count(), 1); + // The same word twice in one array, one under each mark: two copies were cut. + let apart = joined(&[run(0, 1, 0, "word"), run(1, 1, 2, "word")]); + let mut seen = HashSet::new(); + assert_eq!(apart.iter().filter(|l| seen.insert((l.op, l.at, l.raw.clone()))).count(), 2); + } + + #[test] + fn a_spent_decode_budget_is_told_apart() { + assert!(!Ctx::new(HashSet::new()).decode_spent()); + assert!(Ctx::with_decode_left(0).decode_spent()); + } +} diff --git a/crates/redact/src/verify/sweep.rs b/crates/redact/src/verify/sweep.rs new file mode 100644 index 0000000000..73b44c005d --- /dev/null +++ b/crates/redact/src/verify/sweep.rs @@ -0,0 +1,759 @@ +//! Sweeping the serialized output for the removed strings, on every surface a reader could +//! recover them from. + +use std::collections::{BTreeMap, BTreeSet, HashSet}; +use std::sync::Arc; + +use pdfcraft_cos::{Dict, Document, ObjRef, Object, PdfString}; + +use super::extract::extract_page; +use super::util::{Ctx, decode_text, find, image_codec, spellings, squash, survivor_encodings, xml_unescape}; +use super::{SURFACES, Surface, SurfaceReport, Verdict}; +use crate::verify::limits::{MAX_NESTING, MAX_WALK, MIN_NEEDLE, MIN_STRUCTURED_NEEDLE}; + +/// A string the sweep looks for. +pub(super) struct Needle { + /// The code bytes as Latin-1 text (the "raw view" of a page) rather than decoded text. + pub(super) raw_view: bool, + /// As removed, whitespace included (for the byte encodings). + pub(super) shown: String, + /// `shown` without whitespace (for page text, which is compared whitespace-blind). + pub(super) key: String, + /// What the raw file may spell it as. + pub(super) bytes: Vec>, + pub(super) owners: BTreeSet, + /// Occurrences that legitimately remain in content that was not redacted. The surfaces that + /// hold page content (the page text, the raw file, decoded streams) count occurrences over + /// the whole document and fail on a surplus; every other surface (strings, Info, XMP, + /// attachments) is swept with no allowance. + pub(super) allowance: usize, + /// Names of the input that carry this needle (a redacted word that is also a `/Subtype`, a + /// `/Filter`, a `/BaseFont`). Names are structure redaction never edits, so the name surface + /// is swept against the input's own matches: the output may hold as many as the input did, + /// and only a surplus is a survivor. + pub(super) name_allowance: usize, +} + +impl Needle { + pub(super) fn short(&self) -> bool { + self.key.chars().count() < MIN_NEEDLE + } + + /// Long enough to be looked for in bytes (`min`: the shortest the haystack allows). + pub(super) fn at_least(&self, min: usize) -> bool { + self.key.chars().count() >= min + } +} + +pub(super) fn add_needle(set: &mut Vec, raw_view: bool, shown: String, owner: Option) { + let key = squash(&shown); + if key.is_empty() { + return; + } + if let Some(n) = set.iter_mut().find(|n| n.raw_view == raw_view && n.key == key) { + n.owners.extend(owner); + return; + } + let bytes = if raw_view { + // The code bytes themselves (Latin-1 round-trips them). + let raw: Vec = shown.chars().filter_map(|c| u8::try_from(u32::from(c)).ok()).collect(); + spellings([raw]) + } else { + survivor_encodings(&shown) + }; + set.push(Needle { raw_view, shown, key, bytes, owners: owner.into_iter().collect(), allowance: 0, name_allowance: 0 }); +} + +pub(super) struct Acc { + pub(super) reps: BTreeMap, + pub(super) failed_needles: HashSet, + pub(super) ctx: Ctx, +} + +impl Acc { + pub(super) fn new(ctx: Ctx) -> Self { + let mut a = Acc { reps: BTreeMap::new(), failed_needles: HashSet::new(), ctx }; + for s in SURFACES { + a.rep(s); + } + a + } + + pub(super) fn rep(&mut self, surface: Surface) -> &mut SurfaceReport { + self.reps.entry(surface).or_insert_with(|| SurfaceReport { + surface, + verdict: Verdict::Absent, + scanned: 0, + raw_only: 0, + survivors: 0, + locations: Vec::new(), + problems: Vec::new(), + }) + } + + pub(super) fn scanned(&mut self, s: Surface) { + self.rep(s).scanned += 1; + } + + pub(super) fn raw_only(&mut self, s: Surface) { + self.rep(s).raw_only += 1; + } + + pub(super) fn survivor(&mut self, s: Surface, loc: String, needle: Option) { + self.survivors(s, loc, needle, 1); + } + + /// `count` survivors at one place. + pub(super) fn survivors(&mut self, s: Surface, loc: String, needle: Option, count: usize) { + if let Some(n) = needle { + self.failed_needles.insert(n); + } + let r = self.rep(s); + r.survivors = r.survivors.saturating_add(count); + if !r.locations.contains(&loc) && r.locations.len() < 64 { + r.locations.push(loc); + } + } + + pub(super) fn problem(&mut self, s: Surface, msg: String) { + let r = self.rep(s); + if r.problems.len() < 64 { + r.problems.push(msg); + } + } + + /// The needles (at least `min` long) found in `hay`; running out of search work is a + /// problem of `surface`. + pub(super) fn bytes(&mut self, surface: Surface, needles: &[Needle], hay: &[u8], min: usize) -> Vec { + bytes_hits(needles, hay, min, &self.ctx).unwrap_or_else(|| { + self.problem(surface, "the search work limit was reached".into()); + Vec::new() + }) + } + + /// How often each needle (at least `min` long) occurs in `hay`, as `(needle, count)`. + pub(super) fn counts(&mut self, surface: Surface, needles: &[Needle], hay: &[u8], min: usize) -> Vec<(usize, usize)> { + bytes_counts(needles, hay, min, &self.ctx).unwrap_or_else(|| { + self.problem(surface, "the search work limit was reached".into()); + Vec::new() + }) + } + + /// Survivors and problems so far, over all surfaces. + pub(super) fn totals(&self) -> (usize, usize) { + (self.reps.values().map(|r| r.survivors).sum(), self.reps.values().map(|r| r.problems.len()).sum()) + } + + pub(super) fn finish(mut self) -> Vec { + for r in self.reps.values_mut() { + r.verdict = if r.survivors > 0 { + Verdict::Survivor + } else if !r.problems.is_empty() { + Verdict::Unswept + } else if r.scanned + r.raw_only == 0 { + Verdict::Absent + } else { + Verdict::Clean + }; + } + self.reps.into_values().collect() + } +} + +/// Occurrences of needles on one surface, per place, so that what legitimately remains +/// (a needle's allowance) is taken off the document-wide total once, not once per place. +#[derive(Default)] +pub(super) struct Tally(BTreeMap>); + +impl Tally { + pub(super) fn add(&mut self, needle: usize, loc: &str, count: usize) { + if count > 0 { + self.0.entry(needle).or_default().push((loc.to_string(), count)); + } + } + + /// Report what is left of each needle's occurrences after its allowance (none inside an + /// attachment, where nothing legitimately remains). + pub(super) fn settle(self, surface: Surface, needles: &[Needle], depth: usize, acc: &mut Acc) { + self.settle_with(surface, needles, depth, acc, |n| n.allowance) + } + + /// The name surface: the input's own name matches are the allowance (redaction never edits + /// names), so what is left is a surplus of matches over the input. + pub(super) fn settle_names(self, surface: Surface, needles: &[Needle], depth: usize, acc: &mut Acc) { + self.settle_with(surface, needles, depth, acc, |n| n.name_allowance) + } + + fn settle_with(self, surface: Surface, needles: &[Needle], depth: usize, acc: &mut Acc, allow_of: impl Fn(&Needle) -> usize) { + for (ni, places) in self.0 { + let mut allow = if depth == 0 { needles.get(ni).map_or(0, &allow_of) } else { 0 }; + for (loc, count) in places { + let used = count.min(allow); + allow -= used; + if count > used { + acc.survivors(surface, loc, Some(ni), count - used); + } + } + } + } +} + +/// Search all encodings in one bounded pass, including overlapping matches. +fn pattern_matches<'a>( + patterns: impl IntoIterator, + hay: &[u8], + ctx: &Ctx, + mut found: impl FnMut(usize, usize, usize) -> bool, +) -> Option<()> { + const MAX_PATTERNS: usize = 100_000; + const MAX_PATTERN_BYTES: usize = 16 << 20; + const MAX_MATCHES: usize = 1 << 20; + if !ctx.charge(hay.len()) { + return None; + } + let mut map: BTreeMap, Vec> = BTreeMap::new(); + let (mut bytes, mut links) = (0usize, 0usize); + for (owner, pattern) in patterns { + if pattern.is_empty() { + continue; + } + if !ctx.charge(pattern.len().saturating_add(1)) { + return None; + } + if !map.contains_key(pattern) { + bytes = bytes.checked_add(pattern.len())?; + if bytes > MAX_PATTERN_BYTES || map.len() >= MAX_PATTERNS { + return None; + } + } + let owners = map.entry(pattern.to_vec()).or_default(); + if owners.last() != Some(&owner) { + links = links.checked_add(1)?; + if links > MAX_PATTERNS { + return None; + } + owners.push(owner); + } + } + if map.is_empty() { + return Some(()); + } + let matcher = aho_corasick::AhoCorasick::new(map.keys()).ok()?; + let owners: Vec> = map.into_values().collect(); + let mut matches = 0usize; + for hit in matcher.find_overlapping_iter(hay) { + for &owner in owners.get(hit.pattern().as_usize())? { + matches = matches.checked_add(1)?; + if matches > MAX_MATCHES || !ctx.charge(1) || !found(owner, hit.start(), hit.end()) { + return None; + } + } + } + Some(()) +} + +/// Count normalized text keys in one pass, with the same non-overlap rule as `str::matches`. +pub(super) fn key_counts(needles: &[Needle], hay: &str, raw: bool, ctx: &Ctx) -> Option> { + key_counts_from(needles, hay, raw, ctx, 0) +} + +/// [`key_counts`] of the needles at least `min` long, so a one-letter needle can't use up the match +/// limits of a search it takes no part in. +fn key_counts_from(needles: &[Needle], hay: &str, raw: bool, ctx: &Ctx, min: usize) -> Option> { + let patterns = needles.iter().enumerate().filter(|(_, n)| n.raw_view == raw && n.at_least(min)).map(|(i, n)| (i, n.key.as_bytes())); + let mut spans: BTreeMap> = BTreeMap::new(); + pattern_matches(patterns, hay.as_bytes(), ctx, |i, a, b| { + let hits = spans.entry(i).or_default(); + if hits.len() >= MAX_COUNTED { + return false; + } + hits.push((a, b)); + true + })?; + Some( + spans + .into_iter() + .map(|(i, mut hits)| { + hits.sort_unstable(); + let (mut count, mut end) = (0, 0); + for (a, b) in hits { + if a >= end { + count += 1; + end = b; + } + } + (i, count) + }) + .collect(), + ) +} + +pub(super) fn bytes_hits(needles: &[Needle], hay: &[u8], min: usize, ctx: &Ctx) -> Option> { + let patterns = needles.iter().enumerate().filter(|(_, n)| n.at_least(min)).flat_map(|(i, n)| n.bytes.iter().map(move |b| (i, b.as_slice()))); + let mut found = BTreeSet::new(); + pattern_matches(patterns, hay, ctx, |i, _, _| { + found.insert(i); + true + })?; + Some(found.into_iter().collect()) +} + +/// A larger count is incomplete evidence, never a silently truncated count. +pub(super) const MAX_COUNTED: usize = 1 << 16; + +pub(super) fn bytes_counts(needles: &[Needle], hay: &[u8], min: usize, ctx: &Ctx) -> Option> { + let patterns = needles.iter().enumerate().filter(|(_, n)| n.at_least(min)).flat_map(|(i, n)| n.bytes.iter().map(move |b| (i, b.as_slice()))); + let mut spans: BTreeMap> = BTreeMap::new(); + pattern_matches(patterns, hay, ctx, |i, a, b| { + let hits = spans.entry(i).or_default(); + if hits.len() >= MAX_COUNTED { + return false; + } + hits.push((a, b)); + true + })?; + Some( + spans + .into_iter() + .map(|(i, mut spans)| { + spans.sort_unstable(); + spans.dedup(); + let (mut count, mut end) = (0usize, 0usize); + for (a, b) in spans { + if count == 0 || a >= end { + count += 1; + } + end = end.max(b); + } + (i, count) + }) + .collect(), + ) +} + +pub(super) fn string_hits(needles: &[Needle], value: &PdfString, ctx: &Ctx) -> Option> { + let text = value.to_text(); + let key = squash(&text); + if !ctx.charge(value.bytes.len().saturating_add(key.len())) { + return None; + } + let mut found: BTreeSet = bytes_hits(needles, &value.bytes, MIN_NEEDLE, ctx)?.into_iter().collect(); + let patterns = needles.iter().enumerate().filter(|(_, n)| !n.raw_view && !n.short()).map(|(i, n)| (i, n.key.as_bytes())); + pattern_matches(patterns, key.as_bytes(), ctx, |i, _, _| { + found.insert(i); + true + })?; + let tokens: BTreeSet<&str> = text.split(|c: char| !c.is_alphanumeric()).filter(|s| !s.is_empty()).collect(); + for (i, needle) in needles.iter().enumerate().filter(|(_, n)| n.short() && n.allowance == 0 && !n.raw_view) { + if !ctx.charge(needle.shown.len().saturating_add(1)) { + return None; + } + if tokens.contains(needle.shown.as_str()) { + found.insert(i); + } + } + Some(found.into_iter().collect()) +} + +/// Streams whose data is binary rather than text: images, embedded font programs, colour +/// profiles, cross-reference and object streams (the objects in the latter are swept as +/// objects). Their decoded data is swept like any other stream's; the pixels of an image codec +/// stay undecoded (text that exists only as pixels is not text), and one that won't decode (or is +/// past the cap) is searched as stored bytes only. A short needle "found" in stored bytes is +/// chance. +pub(super) fn binary_stream(d: &Dict) -> bool { + let subtype = d.name(b"Subtype"); + subtype == Some(b"Image".as_slice()) + || matches!(d.name(b"Type"), Some(b"XRef" | b"ObjStm")) + || matches!(subtype, Some(b"Type1C" | b"CIDFontType0C" | b"OpenType")) + || [b"Length1".as_slice(), b"Length2", b"Length3"].iter().any(|k| d.contains(k)) + || (d.contains(b"N") && !d.contains(b"Type") && subtype.is_none()) +} + +/// What a file identifier (`/ID` of the trailer) is spelled as in the file. +pub(super) fn id_spellings(doc: &Document) -> Vec> { + let Some(Object::Array(ids)) = doc.trailer().get(b"ID").map(|o| doc.resolve(o)).as_deref().cloned() else { return Vec::new() }; + ids.iter().filter_map(Object::as_string).flat_map(|s| spellings([s.bytes.to_vec()])).collect() +} + +/// Visit `o` and everything nested in it. False when something lay deeper than [`MAX_WALK`] +/// and was not visited: the caller reports that as unswept. +pub(super) fn walk_strings(o: &Object, depth: usize, f: &mut impl FnMut(&Object)) -> bool { + if depth > MAX_WALK { + return false; + } + f(o); + // Every child is visited even after one is too deep: nothing is skipped silently. + let mut ok = true; + match o { + Object::Array(a) => a.iter().for_each(|x| ok &= walk_strings(x, depth + 1, f)), + Object::Dict(d) => d.iter().for_each(|(_, v)| ok &= walk_strings(v, depth + 1, f)), + Object::Stream(s) => s.dict.iter().for_each(|(_, v)| ok &= walk_strings(v, depth + 1, f)), + _ => {} + } + ok +} + +/// Sweep a document (the output, or an attachment) on every surface. +pub(super) fn sweep_doc(needles: &[Needle], doc: &Document, bytes: &[u8], depth: usize, tag: &str, acc: &mut Acc) { + let (embedded, overlay_raw) = sweep_objects(needles, doc, depth, tag, acc); + sweep_file(needles, doc, bytes, overlay_raw, depth, tag, acc); + sweep_pages(needles, doc, depth, tag, acc); + sweep_info(needles, doc, tag, acc); + sweep_xmp(needles, doc, tag, acc); + + // Revision structure: one revision, no earlier content kept. + acc.scanned(Surface::Revisions); + let revs = doc.revisions().len(); + if revs > 1 || doc.trailer().contains(b"Prev") { + acc.survivor(Surface::Revisions, format!("{tag}{revs} revisions: earlier ones keep the original content"), None); + } + + sweep_attachments(needles, doc, &embedded, depth, tag, acc); +} + +/// Every object: strings, names and stream data (orphans included). Returns the embedded file +/// streams found, and the raw bytes of the overlay streams (to leave out of the raw sweep). +fn sweep_objects(needles: &[Needle], doc: &Document, depth: usize, tag: &str, acc: &mut Acc) -> (BTreeSet, Vec>) { + let mut embedded: BTreeSet = BTreeSet::new(); + let mut overlay_raw: Vec> = Vec::new(); + let mut in_streams = Tally::default(); + let mut in_names = Tally::default(); + let lookup = lookup_streams(doc); + for num in doc.object_numbers() { + let loc = format!("{tag}object {num}"); + let obj = match doc.try_get(num) { + Ok(o) => o, + Err(_) => { + acc.problem(Surface::ObjectStrings, format!("{loc} can't be read")); + continue; + } + }; + let mut hits: BTreeSet = BTreeSet::new(); + let mut spent = false; + let ctx = &acc.ctx; + // The file identifier is random bytes, never text: a short needle must not "read" it. + let without_id = match &*obj { + Object::Stream(s) if s.dict.name(b"Type") == Some(b"XRef") => { + let mut d = s.dict.clone(); + d.remove(b"ID"); + Some(Object::Dict(d)) + } + _ => None, + }; + let complete = walk_strings(without_id.as_ref().unwrap_or(&obj), 0, &mut |o| match o { + Object::String(s) => match string_hits(needles, s, ctx) { + Some(h) => hits.extend(h), + None => spent = true, + }, + Object::Name(n) => match bytes_hits(needles, n, MIN_NEEDLE, ctx) { + // A name the input had its own matches in is structure, not a leftover copy: + // the matches are tallied and settled against the input's count. + Some(h) => h.into_iter().for_each(|ni| in_names.add(ni, &loc, 1)), + None => spent = true, + }, + _ => {} + }); + if !complete { + acc.problem(Surface::ObjectStrings, format!("{loc} is nested too deeply to sweep in full")); + } + if spent { + acc.problem(Surface::ObjectStrings, "the search work limit was reached".into()); + } + acc.scanned(Surface::ObjectStrings); + for n in hits { + acc.survivor(Surface::ObjectStrings, loc.clone(), Some(n)); + } + // File specifications, wherever they sit (a name tree keeps them as separate objects, + // annotations and collections may inline them). + walk_strings(&obj, 0, &mut |o| { + if let Object::Dict(d) = o + && let Some(Object::Dict(ef)) = d.get(b"EF") + { + embedded.extend(ef.iter().filter_map(|(_, v)| v.as_ref()).map(|r| r.num)); + } + }); + if let Object::Stream(s) = &*obj { + if s.dict.name(b"Type") == Some(b"EmbeddedFile") { + embedded.insert(num); + } + acc.scanned(Surface::DecodedStreams); + if binary_stream(&s.dict) && image_codec(&s.dict) { + // The pixels of an image codec are still encoded and are not text: their stored + // bytes are searched for whole strings only. + acc.raw_only(Surface::DecodedStreams); + for n in acc.bytes(Surface::DecodedStreams, needles, &s.raw, MIN_STRUCTURED_NEEDLE) { + acc.survivor(Surface::DecodedStreams, format!("{loc} (raw)"), Some(n)); + } + } else if binary_stream(&s.dict) { + // A compressed copy of the removed text can hide in a font program or colour + // profile (with /Length1 in its dict, its stored bytes hold no plaintext): the + // stored bytes and the decoded data are both searched. Such a stream that won't + // decode (damaged, or past the cap) is searched as stored bytes only and counted + // as raw only: it holds no page content, and redaction never touches it. A + // cross-reference or object stream that won't decode is a problem of the proof. + for n in acc.bytes(Surface::DecodedStreams, needles, &s.raw, MIN_STRUCTURED_NEEDLE) { + acc.survivor(Surface::DecodedStreams, format!("{loc} (raw)"), Some(n)); + } + match acc.ctx.decode(s) { + Ok(d) => { + let loc = format!("{loc} (decoded)"); + for (n, c) in acc.counts(Surface::DecodedStreams, needles, &d, MIN_STRUCTURED_NEEDLE) { + in_streams.add(n, &loc, c); + } + } + Err(()) if acc.ctx.decode_spent() || matches!(s.dict.name(b"Type"), Some(b"XRef" | b"ObjStm")) => { + acc.problem(Surface::DecodedStreams, format!("{loc} can't be decoded in full")); + } + Err(()) => acc.raw_only(Surface::DecodedStreams), + } + } else { + match acc.ctx.decode(s) { + // The boxes and labels this run drew are not text to look for. + Ok(d) if acc.ctx.is_overlay(&d) => overlay_raw.push(s.raw.to_vec()), + Ok(d) => { + let loc = format!("{loc} (decoded)"); + for (n, c) in acc.counts(Surface::DecodedStreams, needles, &d, MIN_STRUCTURED_NEEDLE) { + in_streams.add(n, &loc, c); + } + } + // Colour-space lookup data and function samples are data tables, not content. + Err(()) if !acc.ctx.decode_spent() && (lookup.contains(&num) || s.dict.contains(b"FunctionType")) => { + acc.raw_only(Surface::DecodedStreams); + for n in acc.bytes(Surface::DecodedStreams, needles, &s.raw, MIN_STRUCTURED_NEEDLE) { + acc.survivor(Surface::DecodedStreams, format!("{loc} (raw)"), Some(n)); + } + } + Err(()) => acc.problem(Surface::DecodedStreams, format!("{loc} can't be decoded in full")), + } + } + } + } + in_names.settle_names(Surface::ObjectStrings, needles, depth, acc); + in_streams.settle(Surface::DecodedStreams, needles, depth, acc); + (embedded, overlay_raw) +} + +/// The streams that hold the lookup table of an `/Indexed` colour space (object numbers). +fn lookup_streams(doc: &Document) -> BTreeSet { + let mut out = BTreeSet::new(); + for num in doc.object_numbers() { + let Ok(obj) = doc.try_get(num) else { continue }; + walk_strings(&obj, 0, &mut |o| { + if let Object::Array(a) = o + && matches!(a.first(), Some(Object::Name(n)) if n == b"Indexed" || n == b"I") + && let Some(Object::Ref(r)) = a.get(3) + { + out.insert(r.num); + } + }); + } + out +} + +/// The raw bytes of the whole file, the overlay streams (stored as they are, or not at all) and +/// the file identifier (random bytes, regenerated on every write) aside. +fn sweep_file(needles: &[Needle], doc: &Document, bytes: &[u8], overlay_raw: Vec>, depth: usize, tag: &str, acc: &mut Acc) { + acc.scanned(Surface::RawBytes); + let mut mask = overlay_raw; + mask.extend(id_spellings(doc)); + let masked = mask_all(bytes, &mask, &acc.ctx); + let mut in_file = Tally::default(); + for (n, c) in acc.counts(Surface::RawBytes, needles, masked.as_deref().unwrap_or(bytes), MIN_STRUCTURED_NEEDLE) { + in_file.add(n, &format!("{tag}file"), c); + } + in_file.settle(Surface::RawBytes, needles, depth, acc); + if masked.is_none() && !mask.is_empty() { + acc.problem(Surface::RawBytes, "the search work limit was reached".into()); + } +} + +/// Embedded files, PDF attachments swept in full. +fn sweep_attachments(needles: &[Needle], doc: &Document, embedded: &BTreeSet, depth: usize, tag: &str, acc: &mut Acc) { + for &num in embedded { + let loc = format!("{tag}attachment object {num}"); + acc.scanned(Surface::EmbeddedFiles); + let obj = doc.get(ObjRef::new(num, doc.generation(num))); + let Object::Stream(s) = &*obj else { + acc.problem(Surface::EmbeddedFiles, format!("{loc} is not a stream")); + continue; + }; + let payload = match acc.ctx.decode(s) { + Ok(p) => p, + Err(()) => { + acc.problem(Surface::EmbeddedFiles, format!("{loc} can't be decoded in full")); + continue; + } + }; + sweep_payload(needles, &payload, depth, &loc, acc); + } +} + +/// `bytes` with every occurrence of `pats` zeroed (`None` when the work limit ran out, or +/// there is nothing to mask). +pub(super) fn mask_all(bytes: &[u8], pats: &[Vec], ctx: &Ctx) -> Option> { + if pats.iter().all(Vec::is_empty) { + return None; + } + let mut out = bytes.to_vec(); + for p in pats.iter().filter(|p| !p.is_empty()) { + if !ctx.charge(bytes.len()) { + return None; + } + let mut at = 0; + while let Some(w) = bytes.get(at..at + p.len()) { + if w == p.as_slice() { + if let Some(z) = out.get_mut(at..at + p.len()) { + z.fill(0); + } + at += p.len(); + } else { + at += 1; + } + } + } + Some(out) +} + +/// One attachment: a PDF is swept as a document, text is searched, containers are reported as +/// unswept (their content can't be seen), images are searched as raw bytes only. +pub(super) fn sweep_payload(needles: &[Needle], payload: &[u8], depth: usize, loc: &str, acc: &mut Acc) { + let head = payload.get(..1024).unwrap_or(payload); + let tag = format!("{loc} > "); + if find(head, b"%PDF-") { + if depth >= MAX_NESTING { + acc.problem(Surface::EmbeddedFiles, format!("{loc} is nested too deep to sweep")); + return; + } + match Document::open(Arc::new(payload.to_vec())) { + Ok(inner) => { + let before = acc.totals(); + sweep_doc(needles, &inner, payload, depth + 1, &tag, acc); + // What the attachment's own sweep found counts against the attachment too. + let after = acc.totals(); + if after.0 > before.0 { + acc.survivor(Surface::EmbeddedFiles, loc.to_string(), None); + } + if after.1 > before.1 { + acc.problem(Surface::EmbeddedFiles, format!("{loc} could not be swept in full")); + } + } + Err(_) => acc.problem(Surface::EmbeddedFiles, format!("{loc} is a PDF that can't be opened")), + } + return; + } + let starts = |sig: &[u8]| payload.starts_with(sig); + let container = starts(b"PK\x03\x04") + || starts(b"PK\x05\x06") + || starts(&[0x1F, 0x8B]) + || starts(b"BZh") + || starts(&[0xFD, b'7', b'z', b'X', b'Z', 0]) + || starts(&[b'7', b'z', 0xBC, 0xAF, 0x27, 0x1C]) + || starts(b"Rar!") + || starts(&[0xD0, 0xCF, 0x11, 0xE0]) + || payload.get(257..262) == Some(b"ustar"); + if container { + acc.problem(Surface::EmbeddedFiles, format!("{loc} is an archive or container that can't be opened")); + return; + } + let image = starts(&[0x89, b'P', b'N', b'G']) || starts(&[0xFF, 0xD8, 0xFF]) || starts(b"GIF8"); + if image { + acc.raw_only(Surface::EmbeddedFiles); + } + let min = if image { MIN_STRUCTURED_NEEDLE } else { MIN_NEEDLE }; + for n in acc.bytes(Surface::EmbeddedFiles, needles, payload, min) { + acc.survivor(Surface::EmbeddedFiles, loc.to_string(), Some(n)); + } + if !image { + let key = squash(&decode_text(payload)); + match key_counts_from(needles, &key, false, &acc.ctx, MIN_NEEDLE) { + Some(counts) => { + for (i, count) in counts { + if count > 0 { + acc.survivor(Surface::EmbeddedFiles, loc.to_string(), Some(i)); + } + } + } + None => acc.problem(Surface::EmbeddedFiles, format!("{loc} could not be searched in full")), + } + } +} + +pub(super) fn sweep_pages(needles: &[Needle], doc: &Document, depth: usize, tag: &str, acc: &mut Acc) { + let mut shown = Tally::default(); + 'pages: for (i, p) in pdfcraft_model::pages(doc).iter().enumerate() { + acc.scanned(Surface::ExtractedText); + let loc = format!("{tag}page {}", i + 1); + let Ok(e) = extract_page(doc, &p.dict, &acc.ctx, &[]) else { + acc.problem(Surface::ExtractedText, format!("{loc} can't be read")); + continue; + }; + for (hay, raw) in [(&e.text, false), (&e.raw, true)] { + let Some(counts) = key_counts(needles, hay, raw, &acc.ctx) else { + acc.problem(Surface::ExtractedText, "the search work limit was reached".into()); + break 'pages; + }; + for (ni, count) in counts { + shown.add(ni, &loc, count); + } + } + } + // A surplus over what legitimately remains anywhere in the document is a survivor. The + // counts made before the work ran out are settled too: a survivor already seen is + // reported where it sits, not hidden behind the limit. + shown.settle(Surface::ExtractedText, needles, depth, acc); +} + +pub(super) fn sweep_info(needles: &[Needle], doc: &Document, tag: &str, acc: &mut Acc) { + let Some(info) = doc.trailer().get(b"Info").map(|i| doc.resolve(i)) else { return }; + let Some(d) = info.as_dict() else { + acc.problem(Surface::Info, format!("{tag}Info is not a dictionary")); + return; + }; + for (k, v) in d.iter() { + acc.scanned(Surface::Info); + let v = doc.resolve(v); + let mut hits: BTreeSet = BTreeSet::new(); + let mut spent = false; + let ctx = &acc.ctx; + let complete = walk_strings(&v, 0, &mut |o| { + if let Object::String(s) = o { + match string_hits(needles, s, ctx) { + Some(h) => hits.extend(h), + None => spent = true, + } + } + }); + if !complete || spent { + acc.problem(Surface::Info, format!("{tag}Info /{} can't be swept in full", String::from_utf8_lossy(k))); + } + for n in hits { + acc.survivor(Surface::Info, format!("{tag}Info /{}", String::from_utf8_lossy(k)), Some(n)); + } + } +} + +pub(super) fn sweep_xmp(needles: &[Needle], doc: &Document, tag: &str, acc: &mut Acc) { + let Some(root) = doc.root() else { return }; + let Some(meta) = doc.get(root).as_dict().and_then(|c| c.get(b"Metadata")).map(|m| doc.resolve(m)) else { return }; + acc.scanned(Surface::Xmp); + let loc = format!("{tag}XMP metadata"); + let Object::Stream(s) = &*meta else { + acc.problem(Surface::Xmp, format!("{loc} is not a stream")); + return; + }; + let Ok(data) = acc.ctx.decode(s) else { + acc.problem(Surface::Xmp, format!("{loc} can't be decoded in full")); + return; + }; + let mut hits: BTreeSet = acc.bytes(Surface::Xmp, needles, &data, MIN_NEEDLE).into_iter().collect(); + let key = squash(&xml_unescape(&decode_text(&data))); + for (i, n) in needles.iter().enumerate() { + if n.at_least(MIN_NEEDLE) && !n.raw_view && key.contains(&n.key) { + hits.insert(i); + } + } + for n in hits { + acc.survivor(Surface::Xmp, loc.clone(), Some(n)); + } +} diff --git a/crates/redact/src/verify/util.rs b/crates/redact/src/verify/util.rs new file mode 100644 index 0000000000..77c4309cdd --- /dev/null +++ b/crates/redact/src/verify/util.rs @@ -0,0 +1,301 @@ +//! Shared helpers of the proof: what one proof may spend, text and byte search, and the +//! spellings a string can have in a PDF. + +use std::cell::Cell; +use std::collections::HashSet; +use std::fmt::Write as _; +use std::sync::Arc; + +use pdfcraft_cos::{DecodeBudget, Dict, Document, Object, Stream}; +use sha2::{Digest, Sha256}; + +use crate::verify::limits::{MAX_STREAM, MAX_TOTAL_DECODED, MAX_WORK, MIN_NEEDLE}; + +/// What one proof may spend, and what it knows about the document it checks. +pub(super) struct Ctx { + /// Digests of the decoded content of the overlay streams (their boxes and labels are drawn + /// on purpose and are neither text to look for nor paint to flag). + pub(super) overlay: HashSet, + /// Decoded bytes of the proof so far: one allowance for the whole proof, failures charged + /// too (see [`Ctx::decode`]), so a stream referenced many times, or one that fails after + /// decoding part of itself, cannot decode again for free. + pub(super) decoded: Cell, + pub(super) work: Cell, +} + +impl Ctx { + pub(super) fn new(overlay: HashSet) -> Self { + Ctx { overlay, decoded: Cell::new(0), work: Cell::new(0) } + } + + /// Decode `s` strictly: a stream that is damaged, longer than [`MAX_STREAM`], or one more + /// than [`MAX_TOTAL_DECODED`] allows is an error (never silently truncated data). Every + /// attempt is charged to the cumulative allowance, failures included: a decode that fails + /// past the one-stream cap pays the cap, and one that fails mid-way pays what it decoded + /// before the error — the work of finding out is not work a repeated reference gets for free. + pub(super) fn decode(&self, s: &Stream) -> Result, ()> { + let left = MAX_TOTAL_DECODED.saturating_sub(self.decoded.get()); + let cap = usize::try_from(left).unwrap_or(usize::MAX).min(MAX_STREAM); + let mut budget = DecodeBudget::new(cap, usize::MAX); + let result = budget.decode_strict(s); + // The attempt's own accounting covers a success (what it decoded), a stream cut short + // by corruption (what it decoded first) and one refused past the cap (the cap): charge + // it through, whichever way the attempt went. + let spent = cap.saturating_sub(budget.remaining()); + self.decoded.set(self.decoded.get().saturating_add(spent as u64)); + result.map_err(|_| ()) + } + + /// Is the cumulative decode allowance used up? A decode refused then says nothing about the + /// stream, so it must not pass for a stream that can't be decoded. + pub(super) fn decode_spent(&self) -> bool { + self.decoded.get() >= MAX_TOTAL_DECODED + } + + /// Spend `n` units of search work; false once [`MAX_WORK`] is used up. + pub(super) fn charge(&self, n: usize) -> bool { + let used = self.work.get().saturating_add(n as u64); + self.work.set(used); + used <= MAX_WORK + } + + /// Is this decoded content one of the overlay streams this run created? + pub(super) fn is_overlay(&self, data: &[u8]) -> bool { + !self.overlay.is_empty() && self.overlay.contains(&sha256(&[data])) + } + + /// (tests) A context whose cumulative decode allowance has `left` bytes left. + #[cfg(test)] + pub(super) fn with_decode_left(left: u64) -> Self { + Ctx { overlay: HashSet::new(), decoded: Cell::new(MAX_TOTAL_DECODED.saturating_sub(left)), work: Cell::new(0) } + } +} + +/// Follow `o` to its object; a reference that can't be loaded is an error (a missing object is +/// `Null` for [`Document::resolve`], which would read as "empty"). +pub(super) fn resolve_strict(doc: &Document, o: &Object) -> Result, ()> { + match o { + Object::Ref(r) => doc.try_get(r.num).map_err(|_| ()), + other => Ok(Arc::new(other.clone())), + } +} + +pub(super) fn hex(bytes: &[u8]) -> String { + bytes.iter().fold(String::with_capacity(bytes.len() * 2), |mut s, b| { + let _ = write!(s, "{b:02x}"); + s + }) +} + +pub(super) fn sha256>(parts: &[T]) -> String { + let mut h = Sha256::new(); + for p in parts { + let p = p.as_ref(); + h.update((p.len() as u64).to_be_bytes()); + h.update(p); + } + hex(h.finalize().as_slice()) +} + +/// The text of `s` without whitespace (page text is compared whitespace-blind: a gap in a `TJ` +/// is a space in one extraction and nothing in another). +pub(super) fn squash(s: &str) -> String { + s.chars().filter(|c| !c.is_whitespace() && *c != '\u{feff}').collect() +} + +pub(super) fn latin1(b: &[u8]) -> String { + b.iter().map(|&c| char::from(c)).collect() +} + +/// The words of a run worth looking for on their own (a run that survives in pieces). +pub(super) fn words(s: &str) -> Vec<&str> { + s.split_whitespace().filter(|w| w.chars().count() >= MIN_NEEDLE).collect() +} + +pub(super) fn count(hay: &str, needle: &str) -> usize { + if needle.is_empty() { 0 } else { hay.matches(needle).count() } +} + +/// Is `pat` in `hay`? +pub(super) fn find(hay: &[u8], pat: &[u8]) -> bool { + find_from(hay, pat, 0).is_some() +} + +/// Where `pat` first occurs in `hay` at or after `from`. The pattern's prefix function is +/// built once and the haystack is walked once: the search is worst-case linear, so +/// adversarial input (a needle of one letter and a haystack full of it) can neither make it +/// quadratic nor spend more comparisons than the caller charges for. +pub(super) fn find_from(hay: &[u8], pat: &[u8], from: usize) -> Option { + let m = pat.len(); + if m == 0 { + return None; + } + // fail[i]: the longest proper prefix of `pat` that ends at `i` and is also a suffix there. + let mut fail = vec![0usize; m]; + let mut k = 0; + for (i, &c) in pat.iter().enumerate().skip(1) { + while k > 0 && pat[k] != c { + k = fail[k - 1]; + } + if pat[k] == c { + k += 1; + } + fail[i] = k; + } + let mut k = 0; + for (at, &c) in hay.iter().enumerate().skip(from) { + while k > 0 && pat[k] != c { + k = fail[k - 1]; + } + if pat[k] == c { + k += 1; + if k == m { + return Some(at + 1 - m); + } + } + } + None +} + +pub(super) fn escape_literal(b: &[u8], octal: bool) -> Vec { + let mut out = Vec::with_capacity(b.len() * 2); + for &c in b { + if octal { + out.extend(format!("\\{c:03o}").bytes()); + continue; + } + match c { + b'(' | b')' | b'\\' => out.extend([b'\\', c]), + b'\n' => out.extend(*b"\\n"), + b'\r' => out.extend(*b"\\r"), + b'\t' => out.extend(*b"\\t"), + _ => out.push(c), + } + } + out +} + +/// Every way `text` can be spelled in a PDF: UTF-8, UTF-16BE (with and without a byte order +/// mark) and UTF-16LE, each as raw bytes, as a hex string (either case) and as a backslash- +/// escaped literal string (minimal escapes, and octal for every byte). +pub(crate) fn survivor_encodings(text: &str) -> Vec> { + let utf16 = |be: bool, bom: bool| -> Vec { + let mut v = Vec::new(); + if bom { + v.extend(if be { [0xFE, 0xFF] } else { [0xFF, 0xFE] }); + } + for u in text.encode_utf16() { + v.extend(if be { u.to_be_bytes() } else { u.to_le_bytes() }); + } + v + }; + let forms = [text.as_bytes().to_vec(), utf16(true, true), utf16(true, false), utf16(false, true), utf16(false, false)]; + spellings(forms) +} + +/// The spellings of a byte string (see [`survivor_encodings`]). +pub(super) fn spellings(forms: impl IntoIterator>) -> Vec> { + let mut out: Vec> = Vec::new(); + for f in forms { + if f.is_empty() { + continue; + } + let h = hex(&f); + out.push(h.to_uppercase().into_bytes()); + out.push(h.into_bytes()); + out.push(escape_literal(&f, false)); + out.push(escape_literal(&f, true)); + out.push(f); + } + let mut seen = HashSet::new(); + out.retain(|b| seen.insert(b.clone())); + out +} + +/// Text of a payload: UTF-16 when it has a byte order mark, else UTF-8 (lossy). +pub(super) fn decode_text(b: &[u8]) -> String { + let units = |be: bool, rest: &[u8]| -> String { + let u: Vec = rest.as_chunks::<2>().0.iter().map(|c| if be { u16::from_be_bytes(*c) } else { u16::from_le_bytes(*c) }).collect(); + String::from_utf16_lossy(&u) + }; + match b { + [0xFE, 0xFF, rest @ ..] => units(true, rest), + [0xFF, 0xFE, rest @ ..] => units(false, rest), + _ => String::from_utf8_lossy(b).into_owned(), + } +} + +pub(super) fn xml_unescape(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + let mut rest = s; + while let Some(at) = rest.find('&') { + out.push_str(rest.get(..at).unwrap_or_default()); + rest = rest.get(at..).unwrap_or_default(); + match xml_entity(rest) { + Some((c, len)) => { + out.push(c); + rest = rest.get(len..).unwrap_or_default(); + } + None => { + out.push('&'); + rest = rest.get(1..).unwrap_or_default(); + } + } + } + out.push_str(rest); + out +} + +/// The entity starting at `&`: the character it reads as, and how far it runs (through the +/// closing `;`). A numeric reference may carry any number of leading zeros, so its digits +/// are padded without a bound; more significant digits than a character value can hold, or +/// a missing `;`, is not an entity and the `&` stays as it is. Walking only the digits the +/// reference itself holds keeps the decode linear in the input. +fn xml_entity(s: &str) -> Option<(char, usize)> { + for (name, c) in [("&", '&'), ("<", '<'), (">", '>'), (""", '"'), ("'", '\'')] { + if s.starts_with(name) { + return Some((c, name.len())); + } + } + let body = s.strip_prefix("&#")?; + let (radix, digits) = match body.as_bytes().first() { + Some(b'x' | b'X') => (16u32, &body[1..]), + _ => (10, body), + }; + let mut value: u32 = 0; + let mut digits_held = false; + let mut end = None; + for (i, c) in digits.char_indices() { + match c.to_digit(radix) { + Some(d) => { + digits_held = true; + // Leading zeros only pad; past a character's worth of significant digits the + // multiplication overflows and this was never an entity. + value = value.checked_mul(radix)?.checked_add(d)?; + } + None => { + if c == ';' && digits_held { + end = Some(i); + } + break; + } + } + } + let end = end?; + let ch = char::from_u32(value)?; + Some((ch, 2 + usize::from(radix == 16) + end + 1)) +} + +/// Names of the filters of a stream dictionary. +pub(super) fn filter_names(d: &Dict) -> Vec> { + match d.get(b"Filter") { + Some(Object::Name(n)) => vec![n.clone()], + Some(Object::Array(a)) => a.iter().filter_map(|o| o.as_name().map(<[u8]>::to_vec)).collect(), + _ => Vec::new(), + } +} + +/// Image codecs: their data is pixels, still encoded. +pub(super) fn image_codec(d: &Dict) -> bool { + filter_names(d).iter().any(|n| matches!(n.as_slice(), b"DCTDecode" | b"DCT" | b"JPXDecode" | b"CCITTFaxDecode" | b"CCF" | b"JBIG2Decode")) +}