From 2976caeb32ade27c728457e32fae793835076913 Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Fri, 6 Nov 2026 16:54:55 +0100 Subject: [PATCH] M8: the forms redaction follows through resources are read one operator at a time resource_ops parsed the content and each form it follows whole before checking its 2,000,000-operator limit, and content_draws parsed whole streams with no limit at all. A parsed operator takes about 120 bytes, and a form of `n` operators outside the mark cost 78 to 138 bytes of memory per byte of content. Measured as the growth of the process's peak: 629 MiB at 8 MiB, 1,253 MiB at 16 MiB and 4,415 MiB at 32 MiB, so tens of GiB at the 256 MiB the page's allowance admits, from a file of a few hundred KiB. The page's check of what it still draws (prune), added in this branch, reads such forms on pages with no fonts or properties too. On main the font and property cleanups read them, and the unlinking check (content_draws) on any page that shares a dictionary with a removed image, so the content_draws half fixes what main does too. Both now read with pdfcraft_content::ops, one operator at a time. resource_ops holds at most one past its limit, and content_draws holds no operators, as it only looks for Do. Measured as the peak resident memory of the whole process (dev profile): the page's check over a form of 8, 16 and 64 MiB of `n` takes 311, 327 and 422 MiB, and the unlinking check over one of 16 MiB takes 69 MiB, against 1,269 before (2,527 MiB at 32 MiB). This bounds the operators held, not their operands: the parser still holds the operands that no operator ends, so content of operands alone costs about 30 bytes per byte (974 MiB for 32 MiB of `0`). Bounding those is pdfcraft-content's, for every reader of content. It also drops resource_ops' check that it followed no more than 2,000,000 forms. Each form it follows is a Do among the operators it holds, which the operator limit already bounds, so that check could never fail; cargo-mutants found it untested. Test the_forms_followed_through_resources_are_read_one_operator_at_a_time (tests/operator_memory.rs, a binary of its own) measures how much the process's peak grows over 32 MiB of `n` operators through each check, on Linux, and fails with either read reverted. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/redact/src/interp/prune.rs | 25 ++++--- crates/redact/tests/operator_memory.rs | 99 ++++++++++++++++++++++++++ 2 files changed, 115 insertions(+), 9 deletions(-) create mode 100644 crates/redact/tests/operator_memory.rs diff --git a/crates/redact/src/interp/prune.rs b/crates/redact/src/interp/prune.rs index 97701a26c6..8d48a0f54c 100644 --- a/crates/redact/src/interp/prune.rs +++ b/crates/redact/src/interp/prune.rs @@ -32,8 +32,9 @@ pub(crate) fn prune(doc: &Document, budget: &mut DecodeBudget, xo: &mut Dict, re /// `None` when that cannot be told (a form that will not decode, or nesting past the limit). fn content_draws(doc: &Document, content: &[Vec], xo: &Dict, name: &[u8], depth: usize, seen: &mut HashSet) -> Option { let joined = Pieces::join(content); - let ops = joined.parse().ops; - for op in ops.iter().filter(|op| op.is("Do")) { + // One operator at a time, since only the `Do`s matter: a stream of millions of operators is + // never held whole. + for op in pdfcraft_content::ops(joined.data()).filter(|op| op.is("Do")) { let Some(n) = op.last_name() else { continue }; if n == name { return Some(true); @@ -280,13 +281,17 @@ pub(super) fn clip_out(rects: &[[f64; 4]], ctm: &Matrix, around: [f64; 4]) -> Op /// with its `/DecodeParms` given by reference, its decoded bytes charged to `budget`, so the /// cleanups of one page that walk the same forms spend one allowance between them. An error /// when that can't all be read: the parser can't read all of `content` or of such a form, the walk -/// passes 2,000,000 operators, or such a form doesn't decode or runs past `budget`. +/// passes 2,000,000 operators, or such a form doesn't decode or runs past `budget`. Operators are +/// read one at a time, so the walk stops at one past the limit rather than holding them all. pub(crate) fn resource_ops(doc: &Document, budget: &mut DecodeBudget, res: &Dict, content: &[Vec]) -> Result, CosError> { - let parsed = Pieces::join(content).parse(); - if parsed.skipped > 0 || parsed.ops.len() > 2_000_000 { + const MAX_OPS: usize = 2_000_000; + let joined = Pieces::join(content); + let mut read = pdfcraft_content::ops(joined.data()); + let mut ops: Vec = read.by_ref().take(MAX_OPS + 1).collect(); + // Under the limit the reader ran to the end, so `skipped` counts all of it. + if read.skipped() > 0 || ops.len() > MAX_OPS { return Err(CosError::Syntax { offset: 0, detail: "redaction could not fully read the content that uses these resources".into() }); } - let mut ops = parsed.ops; let xobjects = res_dict(doc, res, b"XObject"); let mut seen = HashSet::new(); let mut cursor = 0; @@ -299,11 +304,13 @@ pub(crate) fn resource_ops(doc: &Document, budget: &mut DecodeBudget, res: &Dict && s.dict.name(b"Subtype") == Some(b"Form") && s.dict.get(b"Resources").is_none_or(|r| doc.resolve(r).as_dict().is_none()) { - let parsed = pdfcraft_content::parse(&budget.decode(&doc.with_direct_parms(s))?); - if seen.len() > 2_000_000 || ops.len().saturating_add(parsed.ops.len()) > 2_000_000 || parsed.skipped > 0 { + let data = budget.decode(&doc.with_direct_parms(s))?; + let mut read = pdfcraft_content::ops(&data); + ops.extend(read.by_ref().take((MAX_OPS + 1).saturating_sub(ops.len()))); + // `seen` gains at most one form for each `Do` in `ops`, so this bounds it too. + if ops.len() > MAX_OPS || read.skipped() > 0 { return Err(CosError::Syntax { offset: 0, detail: "redaction could not fully inspect inherited form resources".into() }); } - ops.extend(parsed.ops); } cursor += 1; } diff --git a/crates/redact/tests/operator_memory.rs b/crates/redact/tests/operator_memory.rs new file mode 100644 index 0000000000..925a091a0d --- /dev/null +++ b/crates/redact/tests/operator_memory.rs @@ -0,0 +1,99 @@ +//! Memory regressions for reading the forms a redaction follows through resources: their +//! operators are read one at a time, so a form of millions of them is never held whole. A test +//! binary of its own, so that the one test is all the process runs while its peak is measured. + +use std::sync::Arc; + +use pdfcraft_annot::{Meta, NewAnnotation, Shape, Style, add_annotation, rect_quad}; +use pdfcraft_cos::{Document, ObjRef, Object, Stream}; +use pdfcraft_redact::apply; +use pdfcraft_testkit::{Pdf, stream_obj}; + +const IMAGE: &str = "/Type /XObject /Subtype /Image /Width 1 /Height 1 /ColorSpace /DeviceGray /BitsPerComponent 8"; + +/// 32 MiB of `n` operators: 16 million of them, eight times the walk's limit. Held as parsed +/// operators, at about 120 bytes each, nearly 2 GiB. +fn operators() -> Vec { + b"n\n".repeat(16 << 20) +} + +/// `form` with `content`, compressed. +fn compress(doc: &mut Document, form: ObjRef, content: &[u8]) { + let Object::Stream(s) = &*doc.get(form) else { panic!("{form:?} is a stream") }; + let dict = s.dict.clone(); + doc.set(form, Object::Stream(Stream::flate(dict, content))); +} + +fn mark(doc: &mut Document) { + let shape = Shape::Redact { quads: vec![rect_quad([0.0, 200.0, 300.0, 300.0])], overlay: String::new(), look: Default::default() }; + let style = Style::default_for(&shape); + add_annotation(doc, &NewAnnotation { page: 0, shape, style, contents: String::new(), author: "Synthetic test".into() }, &Meta::default()) + .unwrap(); +} + +/// How much the process's peak resident memory grows, in MiB, while `f` runs. `None` where the +/// peak can't be reset (`/proc/self/clear_refs` is Linux's). +fn peak_growth(f: impl FnOnce() -> T) -> Option<(T, u64)> { + fn peak_kib() -> Option { + let status = std::fs::read_to_string("/proc/self/status").ok()?; + status.lines().find_map(|l| l.strip_prefix("VmHWM:"))?.split_whitespace().next()?.parse().ok() + } + std::fs::write("/proc/self/clear_refs", "5").ok()?; + let before = peak_kib()?; + let out = f(); + Some((out, peak_kib()?.saturating_sub(before) >> 10)) +} + +/// Two reads of a form of 16 million operators: +/// - the page's check of what it still draws walks a form without resources of its own outside +/// the mark, and refuses at 2,000,001 operators; +/// - the unlinking check of a page nobody marked, which shares the marked page's resources, +/// looks through such a form for the image the marked page removed, and finds no `Do`. +/// +/// Each parsed every operator first: the peak grew by 4,415 and 2,527 MiB. Now the first holds at +/// most one operator past the walk's limit, and the second none. +#[test] +fn the_forms_followed_through_resources_are_read_one_operator_at_a_time() { + // The page's check: an image under the mark, and the form outside it. + let mut doc = Document::open(Arc::new( + Pdf::new() + .objs(vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Resources << /XObject << /Im1 5 0 R /Fm 6 0 R >> >> >>" + .to_vec(), + stream_obj("", b"q 50 0 0 50 20 230 cm /Im1 Do Q /Fm Do"), + stream_obj(IMAGE, b"o"), + stream_obj("/Type /XObject /Subtype /Form /BBox [0 0 100 100]", b""), + ]) + .build(), + )) + .unwrap(); + compress(&mut doc, ObjRef::new(6, 0), &operators()); + mark(&mut doc); + let Some((result, grew)) = peak_growth(|| apply(&mut doc, None)) else { return }; + assert!(result.is_err(), "the walk refuses past 2,000,000 operators: {result:?}"); + assert!(grew < 1024, "the page's check grew the peak by {grew} MiB"); + + // The unlinking check: two pages share the /Pages node's resources; the second draws the form. + let mut doc = Document::open(Arc::new( + Pdf::new() + .objs(vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [6 0 R 7 0 R] /Count 2 /Resources << /XObject << /Im1 5 0 R /Fm 3 0 R >> >> >>".to_vec(), + stream_obj("/Type /XObject /Subtype /Form /BBox [0 0 100 100]", b""), + stream_obj("", b"q 50 0 0 50 20 230 cm /Im1 Do Q"), + stream_obj(IMAGE, b"o"), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 8 0 R >>".to_vec(), + stream_obj("", b"/Fm Do"), + ]) + .build(), + )) + .unwrap(); + compress(&mut doc, ObjRef::new(3, 0), &operators()); + mark(&mut doc); + let Some((result, grew)) = peak_growth(|| apply(&mut doc, Some(&[0]))) else { return }; + assert_eq!(result.map(|r| r.images_removed).ok(), Some(1), "the marked page's image goes"); + assert!(grew < 1024, "the unlinking check grew the peak by {grew} MiB"); +}