diff --git a/crates/redact/README.md b/crates/redact/README.md index 51ac11f086..3537bfc547 100644 --- a/crates/redact/README.md +++ b/crates/redact/README.md @@ -148,8 +148,10 @@ used by surviving pages or forms are kept. A redacted page that inherits `/Resources` from the page tree gets them pinned (by reference; a direct inherited dictionary is made one shared object first) before it is rewritten. The parent's `/Resources` is then retired unless another page still inherits it, so it cannot keep a removed -XObject reachable. Surviving form XObjects lose the resource entries for removed XObjects their own -content doesn't draw, for the same reason. Pages that aren't redacted are left untouched. +XObject reachable. Surviving form XObjects lose the resource entries for removed XObjects that +neither their own content nor a form without resources of its own drawn through them still draws, +for the same reason, and a redacted page keeps a name under that rule too. Pages that aren't +redacted are left untouched. ## Remove hidden information / Sanitize diff --git a/crates/redact/src/interp.rs b/crates/redact/src/interp.rs index 40f07f1f42..1e69cb32e2 100644 --- a/crates/redact/src/interp.rs +++ b/crates/redact/src/interp.rs @@ -228,6 +228,12 @@ impl<'a> Scope<'a> { } } + /// What is left of the page's allowance once the content is walked, for the cleanups after it + /// that read the forms drawing through the page's resources ([`prune`]) to go on spending. + pub(crate) fn into_budget(self) -> DecodeBudget { + self.budget + } + /// Keep at most `room` bytes of removed text: what earlier pages left of the document's /// [`MAX_REMOVED_BYTES`]. pub fn with_removed_room(mut self, room: usize) -> Self { @@ -1096,7 +1102,9 @@ fn xobject( out.doomed.push(r); return Some(None); } - // Strict, like page content: a Form read only in part counts as undecodable. + // Strict, like page content: a Form read only in part counts as undecodable. With its + // `/DecodeParms` given by reference too, as the cleanups after the walk read it: read + // without them, it looks empty and is left as it was, drawing whatever lies under a mark. let data = if scope.budget.enter_form() { scope.budget.decode_strict(doc, s) } else { @@ -1173,11 +1181,11 @@ fn xobject( let mut cleaned_res = res.clone(); // Install rewritten nested forms before finding live property names: a new form may // still inherit its caller's resources, and its new Do name must resolve here. - if !inner.xobjects.is_empty() || !inner.replaced.is_empty() { + // `prune_form_resources` below then drops the XObjects this form no longer draws, the + // ones it replaced among them, keeping those a form drawing through these resources + // still draws. + if !inner.xobjects.is_empty() { let mut xo = res_dict(doc, &cleaned_res, b"XObject"); - if let Some(data) = &new_data { - prune(&mut xo, &inner.replaced, std::slice::from_ref(data)); - } for (name, reference) in &inner.xobjects { xo.set(name.clone(), Object::Ref(*reference)); } @@ -1185,20 +1193,20 @@ fn xobject( } if let Some(data) = &new_data { let content = std::slice::from_ref(data); - match prune_properties(doc, &mut cleaned_res, content) { + match prune_properties(doc, &mut scope.budget, &mut cleaned_res, content) { Ok(retired) => out.retired_properties.extend(retired), Err(error) => { out.error = Some(error); return None; } } - if let Err(error) = prune_form_resources(doc, &mut cleaned_res, content) { + if let Err(error) = prune_form_resources(doc, &mut scope.budget, &mut cleaned_res, content) { out.error = Some(error); return None; } // No page: annotation appearances draw through a page's resources, never through a // form's. The form's own soft masks are still counted. - if let Err(error) = crate::type3::prune_type3_from_resources(doc, &mut cleaned_res, content, None) { + if let Err(error) = crate::type3::prune_type3_from_resources(doc, &mut scope.budget, &mut cleaned_res, content, None) { out.error = Some(error); return None; } diff --git a/crates/redact/src/interp/prune.rs b/crates/redact/src/interp/prune.rs index 2bfae6391f..39089601ec 100644 --- a/crates/redact/src/interp/prune.rs +++ b/crates/redact/src/interp/prune.rs @@ -3,49 +3,81 @@ use super::{ is_vertical, num, overlaps, page_streams, res_dict, }; -/// Drop every `replaced` resource name whose `Do` no longer appears in the rewritten `content`, -/// so a full-rewrite save stops reaching the original image or form it named. A name that some -/// surviving `Do` still draws (the same image shown again outside a mark) is kept. -pub(crate) fn prune(xo: &mut Dict, replaced: &[Vec], content: &[Vec]) { +/// Drop every `replaced` name of the XObject resources `xo` that the rewritten `content` no +/// longer draws, so a full-rewrite save stops reaching the original image or form it named. A +/// name still drawn is kept: by a surviving `Do` (the same image shown again outside a mark), or +/// by a form without /Resources of its own, which draws through `xo` too. Such forms are read +/// within `budget`, what is left of the page's allowance ([`resource_ops`]). Fails, as the cleanup +/// of the fonts and properties these forms draw with does, when that can't be told: such a form +/// doesn't decode (an image codec or a /Filter it can't run, say) or runs past `budget`, the walk +/// passes 2,000,000 operators, or the parser can't read all of such a form or of `content`. +/// Corrupt data in such a form is read, tolerantly, up to the damage, which may be nothing. +pub(crate) fn prune(doc: &Document, budget: &mut DecodeBudget, xo: &mut Dict, replaced: &[Vec], content: &[Vec]) -> Result<(), CosError> { if replaced.is_empty() { - return; + return Ok(()); } - let joined = Pieces::join(content); - let ops = joined.parse().ops; - let still_drawn = |n: &[u8]| ops.iter().any(|op| op.is("Do") && op.last_name() == Some(n)); + let res = Dict::from_iter([(b"XObject".to_vec(), Object::Dict(xo.clone()))]); + let ops = resource_ops(doc, budget, &res, content)?; + let drawn: HashSet<&[u8]> = ops.iter().filter(|op| op.is("Do")).filter_map(Op::last_name).collect(); for n in replaced { - if !still_drawn(n) { + if !drawn.contains(n.as_slice()) { xo.remove(n); } } + Ok(()) } /// The XObject names of the resource dictionary `xo` that `content` draws: directly, or through /// a form that has no /Resources of its own (such a form draws through its caller's). Each such -/// form is read once, however many names are asked about. The flag says whether some of what -/// `content` draws could not be read (a form that will not decode, or nesting past the limit), so -/// a name missing from the set may still be drawn. +/// form is read once, however many names are asked about, with its `/DecodeParms` given by +/// reference. The flag says whether some of what `content` draws could not be read (a form that +/// will not decode, or nesting past the limit), so a name missing from the set may still be drawn. +/// Operators are read one at a time, since only the `Do`s matter: no stream's are held whole, and +/// each form is decoded only in its turn. fn names_content_draws(doc: &Document, content: &[Vec], xo: &Dict) -> (HashSet>, bool) { let mut drawn = HashSet::new(); let mut seen = HashSet::new(); - let mut pending = vec![(Pieces::join(content).parse().ops, 0)]; + let joined = Pieces::join(content); + // What is still to read, each with its depth: `content` itself (`None`), then the forms it draws. + let mut pending: Vec<(Option, usize)> = vec![(None, 0)]; // A form that will not decode makes the answer uncertain, but it does not stop the forms // already queued from being read: a name drawn only inside one of those need not be unknown. let mut unknown = false; - while let Some((ops, depth)) = pending.pop() { - for n in ops.iter().filter(|op| op.is("Do")).filter_map(Op::last_name) { + while let Some((form, depth)) = pending.pop() { + let data = match form { + None => std::borrow::Cow::Borrowed(joined.data()), + Some(r) => { + let obj = doc.get(r); + let Object::Stream(s) = &*obj else { continue }; + match doc.with_direct_parms(s).decoded() { + Ok(data) => std::borrow::Cow::Owned(data), + Err(_) => { + unknown = true; + continue; + } + } + } + }; + for op in pdfcraft_content::ops(&data).filter(|op| op.is("Do")) { + let Some(n) = op.last_name() else { continue }; if !drawn.insert(n.to_vec()) { continue; } let Some(r) = xo.get(n).and_then(Object::as_ref) else { continue }; let obj = doc.get(r); let Object::Stream(s) = &*obj else { continue }; - if s.dict.name(b"Subtype") != Some(b"Form") || s.dict.contains(b"Resources") || !seen.insert(r) { + // A /Resources that isn't a dictionary (null, or a missing object) is no resources of + // its own, as for `resource_ops` and the interpreter. + if s.dict.name(b"Subtype") != Some(b"Form") + || s.dict.get(b"Resources").is_some_and(|r| doc.resolve(r).as_dict().is_some()) + || !seen.insert(r) + { continue; } - match s.decoded() { - Ok(data) if depth < MAX_DEPTH => pending.push((pdfcraft_content::parse(&data).ops, depth + 1)), - _ => unknown = true, + if depth < MAX_DEPTH { + pending.push((Some(r), depth + 1)); + } else { + unknown = true; } } } @@ -322,7 +354,8 @@ fn prune_retired_properties(doc: &Document, res: &mut Dict, names: &[Vec], c } let mut used = HashSet::new(); for content in contents { - for op in resource_ops(doc, res, content)? { + // Each is one page's content or one form's, with the allowance of a page. + for op in resource_ops(doc, &mut DecodeBudget::page(), res, content)? { if matches!(op.op.as_slice(), b"BDC" | b"DP") && let Some(name) = op.name(1) { @@ -383,7 +416,7 @@ fn unlink_from_forms(doc: &mut Document, holders: &[ObjRef], doomed: &HashSet Op Some(ops) } -pub(crate) fn resource_ops(doc: &Document, res: &Dict, content: &[Vec]) -> Result, CosError> { - let parsed = Pieces::join(content).parse(); - if parsed.skipped > 0 || parsed.ops.len() > 2_000_000 { - return Err(CosError::Syntax { offset: 0, detail: "redaction could not fully inspect property resources".into() }); +/// The operators of `content`, and of every form without /Resources of its own that it draws +/// through `res`, nested ones too: what draws with these resources. Each such form is read once, +/// with its `/DecodeParms` given by reference, its decoded bytes charged to `budget`, so the +/// cleanups of one page that walk the same forms spend one allowance between them. An error +/// when that can't all be read: the parser can't read all of `content` or of such a form, the walk +/// passes 2,000,000 operators, or such a form doesn't decode or runs past `budget`. Operators are +/// read one at a time, so the walk stops at one past the limit rather than holding them all. +pub(crate) fn resource_ops(doc: &Document, budget: &mut DecodeBudget, res: &Dict, content: &[Vec]) -> Result, CosError> { + const MAX_OPS: usize = 2_000_000; + let joined = Pieces::join(content); + let mut read = pdfcraft_content::ops(joined.data()); + let mut ops: Vec = read.by_ref().take(MAX_OPS + 1).collect(); + // Under the limit the reader ran to the end, so `skipped` counts all of it. + if read.skipped() > 0 || ops.len() > MAX_OPS { + return Err(CosError::Syntax { offset: 0, detail: "redaction could not fully read the content that uses these resources".into() }); } - let mut ops = parsed.ops; let xobjects = res_dict(doc, res, b"XObject"); let mut seen = HashSet::new(); let mut cursor = 0; @@ -455,24 +498,26 @@ pub(crate) fn resource_ops(doc: &Document, res: &Dict, content: &[Vec]) -> R && s.dict.name(b"Subtype") == Some(b"Form") && s.dict.get(b"Resources").is_none_or(|r| doc.resolve(r).as_dict().is_none()) { - let parsed = pdfcraft_content::parse(&s.decoded()?); - if seen.len() > 2_000_000 || ops.len().saturating_add(parsed.ops.len()) > 2_000_000 || parsed.skipped > 0 { + let data = budget.decode(&doc.with_direct_parms(s))?; + let mut read = pdfcraft_content::ops(&data); + ops.extend(read.by_ref().take((MAX_OPS + 1).saturating_sub(ops.len()))); + // `seen` gains at most one form for each `Do` in `ops`, so this bounds it too. + if ops.len() > MAX_OPS || read.skipped() > 0 { return Err(CosError::Syntax { offset: 0, detail: "redaction could not fully inspect inherited form resources".into() }); } - ops.extend(parsed.ops); } cursor += 1; } Ok(ops) } -pub(crate) fn prune_properties(doc: &Document, res: &mut Dict, content: &[Vec]) -> Result, CosError> { +pub(crate) fn prune_properties(doc: &Document, budget: &mut DecodeBudget, res: &mut Dict, content: &[Vec]) -> Result, CosError> { let mut properties = res_dict(doc, res, b"Properties"); if properties.is_empty() { return Ok(Vec::new()); } let mut retired = Vec::new(); - let ops = resource_ops(doc, res, content)?; + let ops = resource_ops(doc, budget, res, content)?; let names: Vec> = properties.iter().map(|(n, _)| n.to_vec()).collect(); for name in names { if !ops.iter().any(|op| matches!(op.op.as_slice(), b"BDC" | b"DP") && op.name(1) == Some(&name)) @@ -489,8 +534,8 @@ pub(crate) fn prune_properties(doc: &Document, res: &mut Dict, content: &[Vec]) -> Result<(), CosError> { - let ops = resource_ops(doc, res, content)?; +pub(super) fn prune_form_resources(doc: &Document, budget: &mut DecodeBudget, res: &mut Dict, content: &[Vec]) -> Result<(), CosError> { + let ops = resource_ops(doc, budget, res, content)?; let mut xo = res_dict(doc, res, b"XObject"); let names: Vec> = xo.iter().map(|(n, _)| n.to_vec()).collect(); for name in names { diff --git a/crates/redact/src/lib.rs b/crates/redact/src/lib.rs index 8846b7a501..c0d9ff967b 100644 --- a/crates/redact/src/lib.rs +++ b/crates/redact/src/lib.rs @@ -812,14 +812,15 @@ fn redact_page_content( let mut budget = pdfcraft_cos::DecodeBudget::page(); let (list, data) = page_streams_within(doc, &page.dict, pi, &mut budget)?; let resources = page.dict.get(b"Resources").and_then(|r| doc.resolve(r).as_dict().cloned()).unwrap_or_default(); - let out = { + // What the walk leaves of the page's allowance, the cleanups below spend. + let (out, mut budget) = { let mut scope = Scope::new(rects, Mode::Apply, report, budget, images).with_removed_room(*removed_room); let out = process(doc, &mut scope, &data, &resources, pdfcraft_content::Matrix::IDENTITY); scope.flush(); removed_text.append(&mut scope.removed_text); *removed_cut |= scope.removed_cut; *removed_room = scope.removed_room; - out + (out, scope.into_budget()) }; // Fail closed: the interpreter lays vertical-writing text out horizontally, so it cannot // tell whether those glyphs lie under a mark. Rather than report success and possibly @@ -861,6 +862,11 @@ fn redact_page_content( let mut res = resources.clone(); if !out.xobjects.is_empty() || !out.replaced.is_empty() { let mut xo = res.get(b"XObject").and_then(|x| doc.resolve(x).as_dict().cloned()).unwrap_or_default(); + // The rewritten XObjects first, as for a rewritten form: one left without resources of + // its own draws through these, and `prune` follows it only under a name it can find. + for (n, r) in &out.xobjects { + xo.set(n.clone(), Object::Ref(*r)); + } // A full save keeps whatever is still referenced: unlink the originals this page no // longer draws so the removed image or form bytes cannot survive in the saved file. let page_content: Vec> = new_list @@ -875,10 +881,7 @@ fn redact_page_content( _ => Vec::new(), }) .collect(); - interp::prune(&mut xo, &out.replaced, &page_content); - for (n, r) in &out.xobjects { - xo.set(n.clone(), Object::Ref(*r)); - } + interp::prune(doc, &mut budget, &mut xo, &out.replaced, &page_content)?; res.set(b"XObject".to_vec(), Object::Dict(xo)); } let (_, cleaned_data) = page_streams( @@ -895,8 +898,8 @@ fn redact_page_content( // A Type 3 glyph without resources paints with patterns nobody tracked: keep them all. let painted = if out.keeps_all_patterns { &out.patterns_seen } else { &out.patterns_painted }; report.patterns_removed += interp::prune_patterns(doc, &mut res, &out.patterns_seen, painted, pruned_patterns); - retired_properties.extend(interp::prune_properties(doc, &mut res, &cleaned_data)?); - type3::prune_type3_from_resources(doc, &mut res, &cleaned_data, Some(&page.dict))?; + retired_properties.extend(interp::prune_properties(doc, &mut budget, &mut res, &cleaned_data)?); + type3::prune_type3_from_resources(doc, &mut budget, &mut res, &cleaned_data, Some(&page.dict))?; doc.update_dict(page.obj, |d| { d.set(b"Contents".to_vec(), Object::Array(new_list)); d.set(b"Resources".to_vec(), Object::Dict(res)); diff --git a/crates/redact/src/sanitize.rs b/crates/redact/src/sanitize.rs index f51197d16d..0cc457c492 100644 --- a/crates/redact/src/sanitize.rs +++ b/crates/redact/src/sanitize.rs @@ -662,7 +662,8 @@ fn content_pass(doc: &mut Document, text: bool, layers: bool, write: bool) -> Re let blocks = scope.layer_blocks; let glyphs = scope.report.glyphs; let retained_for_page = std::mem::take(&mut scope.retained_off_layers); - drop(scope); + // What the walk leaves of the page's allowance, the cleanups below spend. + let mut budget = scope.into_budget(); total += out.residue + blocks + glyphs; retained_off_layers.extend(retained_for_page); if !write { @@ -694,16 +695,18 @@ fn content_pass(doc: &mut Document, text: bool, layers: bool, write: bool) -> Re let mut res = resources.clone(); if !out.xobjects.is_empty() || !out.replaced.is_empty() { let mut xo = res.get(b"XObject").and_then(|x| doc.resolve(x).as_dict().cloned()).unwrap_or_default(); - // A full save keeps whatever is still referenced: unlink the originals this page no - // longer draws so the removed form or image bytes cannot survive in the saved file. - crate::interp::prune(&mut xo, &out.replaced, &page_content); + // The rewritten XObjects first, as on Apply Redactions' pages: `prune` follows a + // rewritten form left without resources of its own only under a name it can find. for (n, r) in &out.xobjects { xo.set(n.clone(), Object::Ref(*r)); } + // A full save keeps whatever is still referenced: unlink the originals this page no + // longer draws so the removed form or image bytes cannot survive in the saved file. + crate::interp::prune(doc, &mut budget, &mut xo, &out.replaced, &page_content)?; res.set(b"XObject".to_vec(), Object::Dict(xo)); } - retired_properties.extend(crate::interp::prune_properties(doc, &mut res, &page_content)?); - crate::type3::prune_type3_from_resources(doc, &mut res, &page_content, Some(&page.dict))?; + retired_properties.extend(crate::interp::prune_properties(doc, &mut budget, &mut res, &page_content)?); + crate::type3::prune_type3_from_resources(doc, &mut budget, &mut res, &page_content, Some(&page.dict))?; doc.update_dict(page.obj, |d| { d.set(b"Contents".to_vec(), Object::Array(new_list)); d.set(b"Resources".to_vec(), Object::Dict(res)); diff --git a/crates/redact/src/tests.rs b/crates/redact/src/tests.rs index 031296a01e..c50f44d048 100644 --- a/crates/redact/src/tests.rs +++ b/crates/redact/src/tests.rs @@ -2117,6 +2117,479 @@ fn an_image_a_form_drew_through_the_pages_resources_does_not_survive() { assert!(!bytes.windows(16).any(|w| w == b"REDIMGONECOVERED"), "the image the form drew survived the save"); } +/// How a form can have no resources of its own: no /Resources, a null one, or a reference to an +/// object that isn't there (which is null too). +const NO_RESOURCES: [&str; 3] = ["", "/Resources null", "/Resources 99 0 R"]; + +/// The page draws an image under the mark and, outside it, a form without /Resources of its own +/// that draws the same image through the page's: the page keeps the name, whether the form lies +/// wholly outside the mark or is entered and left as it was. +#[test] +fn an_image_a_form_draws_again_through_the_pages_resources_is_kept() { + for bbox in ["[0 0 100 100]", "[0 0 300 300]"] { + for none in NO_RESOURCES { + let mut doc = one_page( + b"q 50 0 0 50 20 230 cm /Im1 Do Q /FmBare Do", + "/XObject << /FmBare 7 0 R /Im1 8 0 R >>", + vec![ + stream(&format!("/Type /XObject /Subtype /Form /BBox {bbox} {none}"), b"q 50 0 0 50 20 20 cm /Im1 Do Q"), + stream("/Type /XObject /Subtype /Image /Width 16 /Height 1 /ColorSpace /DeviceGray /BitsPerComponent 8", b"PUBIMGSHOWNAGAIN"), + ], + ); + mark(&mut doc, 0, &[[0.0, 200.0, 300.0, 300.0]], ""); + let r = apply(&mut doc, None).unwrap(); + assert_eq!((r.images_removed, r.forms_rewritten), (1, 0), "{bbox} {none}: {r:?}"); + let saved = Document::open(Arc::new(write_full(&doc, &SaveOptions::default()).unwrap())).unwrap(); + assert!(content(&saved, 0).contains("/FmBare Do"), "{bbox} {none}"); + assert_eq!(named_image(&saved, 0, b"Im1").as_deref(), Some(&b"PUBIMGSHOWNAGAIN"[..]), "{bbox} {none}: the form still draws it"); + } + } +} + +/// The same one level down: a form with resources of its own draws the image under the mark and, +/// outside it, a form without any that draws it again through the first form's. +#[test] +fn an_image_a_nested_form_draws_again_through_its_callers_resources_is_kept() { + for none in NO_RESOURCES { + let mut doc = one_page( + b"/FmOwn Do", + "/XObject << /FmOwn 7 0 R >>", + vec![ + stream( + "/Type /XObject /Subtype /Form /BBox [0 0 300 300] /Resources << /XObject << /FmBare 8 0 R /Im1 9 0 R >> >>", + b"q 50 0 0 50 20 230 cm /Im1 Do Q /FmBare Do", + ), + stream(&format!("/Type /XObject /Subtype /Form /BBox [0 0 100 100] {none}"), b"q 50 0 0 50 20 20 cm /Im1 Do Q"), + stream("/Type /XObject /Subtype /Image /Width 16 /Height 1 /ColorSpace /DeviceGray /BitsPerComponent 8", b"PUBIMGSHOWNAGAIN"), + ], + ); + mark(&mut doc, 0, &[[0.0, 200.0, 300.0, 300.0]], ""); + let r = apply(&mut doc, None).unwrap(); + assert_eq!((r.images_removed, r.forms_rewritten), (1, 1), "{none}: {r:?}"); + let saved = Document::open(Arc::new(write_full(&doc, &SaveOptions::default()).unwrap())).unwrap(); + assert!(content(&saved, 0).contains("/PCRedacted1 Do"), "{none}"); + let form = named_stream(&saved, 0, b"PCRedacted1").expect("the rewritten form"); + assert!(form.decoded().unwrap().windows(10).any(|w| w == b"/FmBare Do"), "{none}"); + let res = saved.resolve(form.dict.get(b"Resources").unwrap()).as_dict().cloned().unwrap(); + let xo = saved.resolve(res.get(b"XObject").unwrap()).as_dict().cloned().unwrap(); + let image = xo.get(b"Im1").and_then(Object::as_ref).unwrap_or_else(|| panic!("{none}: the rewritten form still names /Im1")); + let Object::Stream(image) = &*saved.get(image) else { panic!("/Im1 is a stream") }; + assert_eq!(decoded_image(&saved, image).unwrap(), b"PUBIMGSHOWNAGAIN", "{none}"); + } +} + +/// A form without /Resources that won't decode may draw the image under the mark through the +/// page's: the redaction is refused. A page with fonts or properties to clean up refused it +/// already; this one has neither. +#[test] +fn a_form_that_cannot_be_read_refuses_the_redaction_of_an_image_it_may_draw() { + let mut doc = pdf(vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Resources << /XObject << /FmBroken 5 0 R /Im1 6 0 R >> >> >>" + .to_vec(), + stream("", b"q 50 0 0 50 20 230 cm /Im1 Do Q /FmBroken Do"), + stream("/Type /XObject /Subtype /Form /BBox [0 0 100 100] /Filter /DCTDecode", b"q 50 0 0 50 20 20 cm /Im1 Do Q"), + stream("/Type /XObject /Subtype /Image /Width 16 /Height 1 /ColorSpace /DeviceGray /BitsPerComponent 8", b"PUBIMGSHOWNAGAIN"), + ]); + let Object::Stream(broken) = &*doc.get(ObjRef::new(5, 0)) else { panic!("the form is a stream") }; + assert!(broken.decoded().is_err(), "the fixture's form must not decode"); + mark(&mut doc, 0, &[[0.0, 200.0, 300.0, 300.0]], ""); + assert!(matches!(apply(&mut doc, None), Err(RedactError::Cos(pdfcraft_cos::CosError::Filter(_))))); +} + +/// A page of two content streams, with no fonts or properties: the first draws an image at +/// (20, 230), 50 pt square, named `image` in the page's resources; the second is `tail`, which +/// draws nothing. `catalog` goes in the catalog, and `image_dict` in the image's dictionary. +fn image_and_a_tail(catalog: &str, image: &str, image_dict: &str, tail: &[u8]) -> Document { + pdf(vec![ + format!("<< /Type /Catalog /Pages 2 0 R {catalog} >>").into_bytes(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + format!("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents [4 0 R 5 0 R] /Resources << /XObject << /{image} 6 0 R >> >> >>") + .into_bytes(), + stream("", format!("q 50 0 0 50 20 230 cm /{image} Do Q").as_bytes()), + stream("", tail), + stream( + &format!("/Type /XObject /Subtype /Image /Width 16 /Height 1 /ColorSpace /DeviceGray /BitsPerComponent 8 {image_dict}"), + b"REDIMGONECOVERED", + ), + b"<< /Type /OCG /Name (Secret layer) >>".to_vec(), + ]) +} + +/// The refusal of the check of what content still draws through its resources (`resource_ops`). +fn unread_content_refusal(result: &Result) -> bool { + matches!(result, Err(RedactError::Cos(pdfcraft_cos::CosError::Syntax { detail, .. })) + if detail == "redaction could not fully read the content that uses these resources") +} + +/// What the page draws can't be told when a content stream the redaction leaves as it was holds +/// bytes the parser can't read: the redaction of an image is refused, as it was already on a page +/// with fonts or properties to clean up. +#[test] +fn unreadable_bytes_in_a_stream_left_as_it_was_refuse_the_redaction_of_an_image() { + for (tail, refused) in [(&b"q Q )"[..], true), (b"q Q", false)] { + let mut doc = image_and_a_tail("", "Im1", "", tail); + mark(&mut doc, 0, &[[0.0, 200.0, 300.0, 300.0]], ""); + let result = apply(&mut doc, None); + if refused { + assert!(unread_content_refusal(&result), "{result:?}"); + } else { + assert_eq!(result.map(|r| r.images_removed).ok(), Some(1)); + } + } +} + +/// Remove Hidden Information makes the same check: removing an off layer's image is refused on a +/// page whose drawing can't be told. +#[test] +fn unreadable_bytes_in_a_stream_left_as_it_was_refuse_removing_a_hidden_image() { + use crate::sanitize::{Hidden, remove_hidden}; + for (tail, refused) in [(&b"q Q )"[..], true), (b"q Q", false)] { + let mut doc = image_and_a_tail("/OCProperties << /OCGs [7 0 R] /D << /OFF [7 0 R] >> >>", "ImOC", "/OC 7 0 R", tail); + let result = remove_hidden(&mut doc, &[Hidden::HiddenLayers]); + if refused { + assert!(unread_content_refusal(&result), "{result:?}"); + } else { + assert!(result.unwrap().iter().any(|(h, n)| *h == Hidden::HiddenLayers && *n > 0)); + } + } +} + +/// Remove Hidden Information keeps a name that a form without resources still draws, too. Hidden +/// text includes text wholly off the page, so what it removes from a form depends on where it is +/// drawn: the page's /Fm, drawn off the page, is rewritten, while /G draws /Fm again on the page, +/// where its text shows, and stays as it was. The page still names /Fm, with its text. +#[test] +fn removing_hidden_text_keeps_a_form_a_form_without_resources_still_draws() { + use crate::sanitize::{Hidden, remove_hidden}; + for none in NO_RESOURCES { + let mut doc = pdf(vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Resources << /XObject << /Fm 5 0 R /G 6 0 R >> >> >>".to_vec(), + stream("", b"q 1 0 0 1 1000 0 cm /Fm Do Q /G Do"), + stream( + "/Type /XObject /Subtype /Form /BBox [0 0 300 300] /Resources << /Font << /F1 7 0 R >> >>", + b"BT /F1 10 Tf 20 20 Td (KEEPTEXT) Tj ET", + ), + stream(&format!("/Type /XObject /Subtype /Form /BBox [0 0 300 300] {none}"), b"/Fm Do"), + FONT.replace("95 0 R", "8 0 R").into_bytes(), + widths(), + ]); + let removed = remove_hidden(&mut doc, &[Hidden::HiddenText]).unwrap(); + assert!(removed.iter().any(|(h, n)| *h == Hidden::HiddenText && *n > 0), "{none:?}: nothing was removed"); + let saved = Document::open(Arc::new(write_full(&doc, &SaveOptions::default()).unwrap())).unwrap(); + assert!(content(&saved, 0).contains("/G Do"), "{none:?}: /G was rewritten"); + let fm = named_stream(&saved, 0, b"Fm").and_then(|s| s.decoded().ok()); + assert!(fm.is_some_and(|d| d.windows(8).any(|w| w == b"KEEPTEXT")), "{none:?}: the page lost /Fm, which /G still draws"); + } +} + +/// The page's own check follows only forms without resources of their own: a form that names a +/// different image /Im1 in resources of its own doesn't keep the page's redacted /Im1. +#[test] +fn the_pages_check_does_not_follow_a_form_with_resources_of_its_own() { + let image = "/Type /XObject /Subtype /Image /Width 1 /Height 1 /ColorSpace /DeviceGray /BitsPerComponent 8"; + let doc = pdf(vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [] /Count 0 >>".to_vec(), + stream("/Type /XObject /Subtype /Form /BBox [0 0 1 1] /Resources << /XObject << /Im1 4 0 R >> >>", b"/Im1 Do"), + stream(image, b"o"), + stream(image, b"p"), + ]); + let mut xo = Dict::from_iter([(b"FmOwn".to_vec(), Object::Ref(ObjRef::new(3, 0))), (b"Im1".to_vec(), Object::Ref(ObjRef::new(5, 0)))]); + crate::interp::prune(&doc, &mut pdfcraft_cos::DecodeBudget::page(), &mut xo, &[b"Im1".to_vec()], &[b"/FmOwn Do".to_vec()]).unwrap(); + assert!(!xo.contains(b"Im1") && xo.contains(b"FmOwn"), "{xo:?}"); +} + +/// What content draws through its resources is read up to 2,000,000 operators, those of the forms +/// it draws without resources of their own included; past that it is refused. +#[test] +fn what_content_draws_through_its_resources_is_read_up_to_two_million_operators() { + let doc = Document::new_empty(); + for (ops, read) in [(2_000_000, true), (2_000_001, false)] { + let content = b"n ".repeat(ops); + assert_eq!( + crate::interp::resource_ops(&doc, &mut pdfcraft_cos::DecodeBudget::page(), &Dict::new(), &[content]).is_ok(), + read, + "{ops} operators" + ); + } + // The content's own `/Fm Do` is one of them. + for (ops, read) in [(1_999_999, true), (2_000_000, false)] { + let doc = pdf(vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [] /Count 0 >>".to_vec(), + stream("/Type /XObject /Subtype /Form /BBox [0 0 1 1]", b"n ".repeat(ops)), + ]); + let res = Dict::from_iter([(b"XObject".to_vec(), Object::Dict(Dict::from_iter([(b"Fm".to_vec(), Object::Ref(ObjRef::new(3, 0)))])))]); + assert_eq!( + crate::interp::resource_ops(&doc, &mut pdfcraft_cos::DecodeBudget::page(), &res, &[b"/Fm Do".to_vec()]).is_ok(), + read, + "a form of {ops} operators" + ); + } +} + +/// `content` behind a PNG predictor (Up, one column) whose `/DecodeParms` are given by reference, +/// in a stream with `entries` too: read without the parameters, its bytes are the predictor's tags +/// and differences, not the operators. +fn behind_parms_by_reference(doc: &mut Document, entries: &[(&str, Object)], content: &[u8]) -> Stream { + let mut rows = Vec::new(); + let mut above = 0u8; + for &b in content { + rows.extend([2, b.wrapping_sub(above)]); + above = b; + } + let parms = doc.add(Object::Dict(Dict::from_iter([(b"Predictor".to_vec(), Object::Int(12))]))); + let mut s = Stream::flate(Dict::new(), &rows); + for (key, value) in entries { + s.dict.set(key.as_bytes().to_vec(), value.clone()); + } + s.dict.set(b"DecodeParms".to_vec(), Object::Ref(parms)); + s +} + +/// A form's entries: `/Subtype /Form` and `bbox`. +fn form_entries(bbox: [i64; 4]) -> Vec<(&'static str, Object)> { + vec![("Subtype", Object::name("Form")), ("BBox", Object::Array(bbox.into_iter().map(Object::Int).collect()))] +} + +/// A form outside the mark draws the image again through the page's resources, its content behind +/// a predictor whose `/DecodeParms` are given by reference. Both checks of whether the page still +/// draws it read the form through them, the page's (`resource_ops`) and the unlinking one at the +/// end (`names_content_draws`): the page keeps the image for the form. +#[test] +fn an_image_a_form_with_parameters_by_reference_draws_again_is_kept() { + let mut doc = one_page( + b"q 50 0 0 50 20 230 cm /Im1 Do Q /FmBare Do", + "/XObject << /FmBare 7 0 R /Im1 8 0 R >>", + vec![ + stream("/Type /XObject /Subtype /Form /BBox [0 0 100 100]", b""), + stream("/Type /XObject /Subtype /Image /Width 16 /Height 1 /ColorSpace /DeviceGray /BitsPerComponent 8", b"PUBIMGSHOWNAGAIN"), + ], + ); + let form = behind_parms_by_reference(&mut doc, &form_entries([0, 0, 100, 100]), b"q 50 0 0 50 20 20 cm /Im1 Do Q"); + doc.set(ObjRef::new(7, 0), form); + mark(&mut doc, 0, &[[0.0, 200.0, 300.0, 300.0]], ""); + let r = apply(&mut doc, None).unwrap(); + assert_eq!(r.images_removed, 1, "{r:?}"); + let saved = Document::open(Arc::new(write_full(&doc, &SaveOptions::default()).unwrap())).unwrap(); + assert_eq!(named_image(&saved, 0, b"Im1").as_deref(), Some(&b"PUBIMGSHOWNAGAIN"[..]), "the form still draws it"); +} + +/// A surviving form outside the mark shares the page's resources and draws the image itself, its +/// content behind a predictor whose `/DecodeParms` are given by reference: unlinking the redacted +/// image from the forms' resources reads it through them, and keeps the image the form draws. +#[test] +fn an_image_a_surviving_form_with_parameters_by_reference_draws_is_kept() { + let mut doc = pdf(vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Resources 5 0 R >>".to_vec(), + stream("", b"q 100 0 0 100 0 0 cm /Im Do Q /Fm Do"), + b"<< /XObject << /Im 6 0 R /Fm 7 0 R >> >>".to_vec(), + stream("/Type /XObject /Subtype /Image /Width 11 /Height 1 /ColorSpace /DeviceGray /BitsPerComponent 8", b"IMAGESECRET"), + stream("/Type /XObject /Subtype /Form /BBox [150 150 300 300]", b""), + ]); + let mut entries = form_entries([150, 150, 300, 300]); + entries.push(("Resources", Object::Ref(ObjRef::new(5, 0)))); + let form = behind_parms_by_reference(&mut doc, &entries, b"q 50 0 0 50 200 200 cm /Im Do Q"); + doc.set(ObjRef::new(7, 0), form); + mark(&mut doc, 0, &[[0.0, 0.0, 120.0, 120.0]], ""); + assert_eq!(apply(&mut doc, None).unwrap().images_removed, 1); + assert!(surviving_references::full_save_holds(&doc, b"IMAGESECRET"), "the image the form still draws outside the mark"); +} + +/// A form without resources of its own crosses the mark and draws an image, or text, under it, +/// its content behind a predictor whose `/DecodeParms` are given by reference. The walk reads it +/// through them and removes what lies under the mark: the image, whether the page draws it under +/// the mark too or only the form does, and the text's glyphs. Read without them, the form looked +/// empty and was left as it was, still drawing them, and the page's check, which reads it with +/// them, kept the image's name for it. +#[test] +fn what_a_form_with_parameters_by_reference_draws_under_the_mark_is_removed() { + let mut doc = one_page(b"/FmBare Do", "/XObject << /FmBare 7 0 R >>", vec![stream("/Type /XObject /Subtype /Form /BBox [0 0 100 100]", b"")]); + let mut entries = form_entries([0, 0, 100, 100]); + // Its drawing moved up by 210: the text lands at y 250, under the mark. + entries.push(("Matrix", Object::Array([1, 0, 0, 1, 0, 210].into_iter().map(Object::Int).collect()))); + let form = behind_parms_by_reference(&mut doc, &entries, b"BT /F1 10 Tf 20 40 Td (SECRETVALUE) Tj ET"); + doc.set(ObjRef::new(7, 0), form); + mark(&mut doc, 0, &[[0.0, 200.0, 300.0, 300.0]], ""); + let r = apply(&mut doc, None).unwrap(); + assert_eq!((r.glyphs, r.forms_rewritten), (11, 1), "the text the form draws under the mark: {r:?}"); + for page in [&b"q 50 0 0 50 20 230 cm /Im1 Do Q /FmBare Do"[..], b"/FmBare Do"] { + let mut doc = one_page( + page, + "/XObject << /FmBare 7 0 R /Im1 8 0 R >>", + vec![ + stream("/Type /XObject /Subtype /Form /BBox [0 0 100 100]", b""), + stream("/Type /XObject /Subtype /Image /Width 16 /Height 1 /ColorSpace /DeviceGray /BitsPerComponent 8", b"REDIMGONECOVERED"), + ], + ); + let mut entries = form_entries([0, 0, 100, 100]); + // Its drawing moved up by 210: the image lands at y 230 to 280, under the mark. + entries.push(("Matrix", Object::Array([1, 0, 0, 1, 0, 210].into_iter().map(Object::Int).collect()))); + let form = behind_parms_by_reference(&mut doc, &entries, b"q 50 0 0 50 20 20 cm /Im1 Do Q"); + doc.set(ObjRef::new(7, 0), form); + mark(&mut doc, 0, &[[0.0, 200.0, 300.0, 300.0]], ""); + let r = apply(&mut doc, None).unwrap(); + assert_eq!(r.forms_rewritten, 1, "{}: {r:?}", String::from_utf8_lossy(page)); + let bytes = write_full(&doc, &SaveOptions::default()).unwrap(); + assert!(!bytes.windows(16).any(|w| w == b"REDIMGONECOVERED"), "{}: the image the form drew under the mark", String::from_utf8_lossy(page)); + } +} + +/// A form without resources of its own that the redaction rewrites gets a new name. It keeps none +/// of its own either when its copy of the page's loses nothing, which takes a form drawing itself +/// (here off the page, so outside the mark), and then goes on drawing through the page's: itself +/// and, through a form it draws unchanged, the image under the mark again outside it. The page's +/// check of what it still draws follows the rewritten form under its new name, so the page keeps +/// both names for it. +#[test] +fn an_image_a_rewritten_form_draws_again_through_the_pages_resources_is_kept() { + let mut doc = one_page( + b"q 50 0 0 50 20 230 cm /Im1 Do Q /FmBare Do", + "/XObject << /FmBare 7 0 R /Inner 8 0 R /Im1 9 0 R >>", + vec![ + stream( + "/Type /XObject /Subtype /Form /BBox [0 0 300 300]", + b"BT /F1 10 Tf 20 250 Td (SECRET) Tj ET /Inner Do q 1 0 0 1 0 -1000 cm /FmBare Do Q", + ), + stream("/Type /XObject /Subtype /Form /BBox [0 0 100 100]", b"q 50 0 0 50 20 20 cm /Im1 Do Q"), + stream("/Type /XObject /Subtype /Image /Width 16 /Height 1 /ColorSpace /DeviceGray /BitsPerComponent 8", b"PUBIMGSHOWNAGAIN"), + ], + ); + mark(&mut doc, 0, &[[0.0, 200.0, 300.0, 300.0]], ""); + let r = apply(&mut doc, None).unwrap(); + assert_eq!((r.images_removed, r.forms_rewritten), (1, 1), "{r:?}"); + let saved = Document::open(Arc::new(write_full(&doc, &SaveOptions::default()).unwrap())).unwrap(); + let page = content(&saved, 0); + let drawn: Vec> = + pdfcraft_content::parse(page.as_bytes()).ops.iter().filter(|op| op.is("Do")).filter_map(|op| op.last_name().map(<[u8]>::to_vec)).collect(); + assert!(drawn.len() == 1 && drawn[0] != b"FmBare", "the form is drawn under its new name: {page}"); + let rewritten = named_stream(&saved, 0, &drawn[0]).unwrap(); + assert!(!rewritten.dict.contains(b"Resources"), "the rewritten form keeps no resources of its own"); + assert!(named_stream(&saved, 0, b"FmBare").is_some(), "the rewritten form still draws the original"); + assert_eq!(named_image(&saved, 0, b"Im1").as_deref(), Some(&b"PUBIMGSHOWNAGAIN"[..]), "the unchanged form still draws it"); +} + +/// The walk of what content draws through its resources charges the forms it reads to the +/// allowance it is given: a form decoding to more than is left is refused, one within it read. +#[test] +fn what_content_draws_through_its_resources_is_read_within_its_allowance() { + let doc = pdf(vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [] /Count 0 >>".to_vec(), + stream("/Type /XObject /Subtype /Form /BBox [0 0 1 1]", b"n ".repeat(1000)), + ]); + let res = Dict::from_iter([(b"XObject".to_vec(), Object::Dict(Dict::from_iter([(b"Fm".to_vec(), Object::Ref(ObjRef::new(3, 0)))])))]); + for (allowance, read) in [(2000, true), (1999, false)] { + let mut budget = pdfcraft_cos::DecodeBudget::new(allowance, 0); + assert_eq!(crate::interp::resource_ops(&doc, &mut budget, &res, &[b"/Fm Do".to_vec()]).is_ok(), read, "{allowance} bytes"); + } +} + +/// The cleanups after the content walk spend what the walk left of the page's allowance, one +/// allowance between them. A form without resources of its own that crosses the mark is read by +/// the walk, then again by the check of what the page still draws and by the font cleanup: at +/// 100 MiB, three reads are more than the page's 256 MiB, and the redaction is refused; at +/// 50 MiB it goes on. +#[test] +fn the_cleanups_after_the_content_walk_spend_what_it_left_of_the_pages_allowance() { + for (mib, refused) in [(100, true), (50, false)] { + let blank = pdfcraft_filters::encode_flate(&vec![b' '; mib << 20]); + let mut doc = one_page( + b"q 50 0 0 50 20 230 cm /Im1 Do Q /FmBare Do", + "/XObject << /FmBare 7 0 R /Im1 8 0 R >>", + vec![ + stream("/Type /XObject /Subtype /Form /BBox [0 0 300 300] /Filter /FlateDecode", &blank), + stream("/Type /XObject /Subtype /Image /Width 1 /Height 1 /ColorSpace /DeviceGray /BitsPerComponent 8", b"o"), + ], + ); + mark(&mut doc, 0, &[[0.0, 200.0, 300.0, 300.0]], ""); + assert_eq!(apply(&mut doc, None).is_err(), refused, "{mib} MiB"); + } +} + +/// The cleanups of a form the redaction rewrites spend the page's allowance too. The rewritten +/// form has resources of its own, so the page's cleanups don't follow it, and draws through them a +/// form without any that crosses the mark: the walk reads that form, then the check of what the +/// rewritten form still draws and its font cleanup read it again. At 100 MiB, three reads are more +/// than the page's 256 MiB, and the redaction is refused; at 50 MiB it goes on. +#[test] +fn the_cleanups_of_a_rewritten_form_spend_the_pages_allowance() { + for (mib, refused) in [(100, true), (50, false)] { + let blank = pdfcraft_filters::encode_flate(&vec![b' '; mib << 20]); + let mut doc = one_page( + b"/FmText Do", + "/XObject << /FmText 7 0 R >>", + vec![ + stream( + "/Type /XObject /Subtype /Form /BBox [0 0 300 300] /Resources << /Font << /F1 5 0 R >> /XObject << /FmBare 8 0 R >> >>", + b"BT /F1 10 Tf 20 250 Td (SECRET) Tj ET /FmBare Do", + ), + stream("/Type /XObject /Subtype /Form /BBox [0 0 300 300] /Filter /FlateDecode", &blank), + ], + ); + mark(&mut doc, 0, &[[0.0, 200.0, 300.0, 300.0]], ""); + assert_eq!(apply(&mut doc, None).is_err(), refused, "{mib} MiB"); + } +} + +/// Removing hidden text spends what its content walk left of the page's allowance too. The page +/// draws a form holding hidden text, which is rewritten, and a form without resources of its own: +/// the walk reads the second, then the check of what the page still draws and the font cleanup +/// read it again. At 100 MiB, three reads are more than the page's 256 MiB, and the removal is +/// refused; at 50 MiB it goes on. +#[test] +fn removing_hidden_text_spends_what_the_content_walk_left_of_the_pages_allowance() { + use crate::sanitize::{Hidden, remove_hidden}; + for (mib, refused) in [(100, true), (50, false)] { + let blank = pdfcraft_filters::encode_flate(&vec![b' '; mib << 20]); + let mut doc = one_page( + b"/FmHidden Do /FmBare Do", + "/XObject << /FmHidden 7 0 R /FmBare 8 0 R >>", + vec![ + stream("/Type /XObject /Subtype /Form /BBox [0 0 300 300]", b"BT /F1 10 Tf 3 Tr 20 250 Td (HIDDEN) Tj ET"), + stream("/Type /XObject /Subtype /Form /BBox [0 0 300 300] /Filter /FlateDecode", &blank), + ], + ); + assert_eq!(remove_hidden(&mut doc, &[Hidden::HiddenText]).is_err(), refused, "{mib} MiB"); + } +} + +/// A form with resources of its own draws through those, not its caller's: one on another page +/// that names a different image /Im1 doesn't keep the redacted /Im1 in the dictionary both pages +/// inherit. +#[test] +fn a_form_with_resources_of_its_own_does_not_keep_its_callers_image() { + let image = "/Type /XObject /Subtype /Image /Width 16 /Height 1 /ColorSpace /DeviceGray /BitsPerComponent 8"; + let mut doc = pdf(vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R 5 0 R] /Count 2 /Resources << /XObject << /Im1 7 0 R /FmOwn 8 0 R >> >> >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R >>".to_vec(), + stream("", b"q 50 0 0 50 20 230 cm /Im1 Do Q"), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 6 0 R >>".to_vec(), + stream("", b"/FmOwn Do"), + stream(image, b"REDIMGONECOVERED"), + stream("/Type /XObject /Subtype /Form /BBox [0 0 300 300] /Resources << /XObject << /Im1 9 0 R >> >>", b"q 50 0 0 50 20 20 cm /Im1 Do Q"), + stream(image, b"PUBIMGSHOWNAGAIN"), + ]); + mark(&mut doc, 0, &[[0.0, 200.0, 300.0, 300.0]], ""); + let r = apply(&mut doc, None).unwrap(); + assert_eq!(r.images_removed, 1, "{r:?}"); + let bytes = write_full(&doc, &SaveOptions::default()).unwrap(); + assert!(!bytes.windows(16).any(|w| w == b"REDIMGONECOVERED"), "the redacted image survived the save"); + let saved = Document::open(Arc::new(bytes)).unwrap(); + let form = named_stream(&saved, 1, b"FmOwn").expect("page 2 still draws its form"); + let res = saved.resolve(form.dict.get(b"Resources").unwrap()).as_dict().cloned().unwrap(); + let xo = saved.resolve(res.get(b"XObject").unwrap()).as_dict().cloned().unwrap(); + let Object::Stream(own) = &*saved.get(xo.get(b"Im1").and_then(Object::as_ref).unwrap()) else { panic!("/Im1 is a stream") }; + assert_eq!(decoded_image(&saved, own).unwrap(), b"PUBIMGSHOWNAGAIN", "the form's own image stays"); +} + /// Remove Hidden Information has the same rewrite: an off layer's image named by an inherited /// /Resources must leave the saved file too. #[test] diff --git a/crates/redact/src/tests/surviving_references.rs b/crates/redact/src/tests/surviving_references.rs index d0be2a194a..0d171b4008 100644 --- a/crates/redact/src/tests/surviving_references.rs +++ b/crates/redact/src/tests/surviving_references.rs @@ -185,6 +185,73 @@ fn a_redacted_type3_glyphs_image_does_not_survive_in_a_surviving_glyphs_form() { assert!(full_save_holds(&doc, b"PUBLIC_IMAGE_B"), "the image the surviving glyph's form draws was lost"); } +/// The same Form, its content behind a predictor whose `/DecodeParms` are given by reference: the +/// cleanup of its resources reads it through them, and keeps the image it draws. Read without +/// them, this content's bytes are all operators that use no resource, so nothing refuses and +/// the image would go. +#[test] +fn an_image_a_type3_glyphs_form_with_parameters_by_reference_draws_is_kept() { + let mut doc = pdf(vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Resources 5 0 R >>".to_vec(), + stream("", b"BT /T3 10 Tf 10 100 Td (AB) Tj ET"), + b"<< /Font << /T3 6 0 R >> >>".to_vec(), + b"<< /Type /Font /Subtype /Type3 /FontBBox [0 -200 1000 800] /FontMatrix [0.001 0 0 0.001 0 0] /FirstChar 65 /LastChar 66 /Widths [500 500] /Encoding << /Differences [65 /A /B] >> /CharProcs << /A 7 0 R /B 8 0 R >> /Resources << /XObject << /ImA 9 0 R /Fm 10 0 R >> >> >>".to_vec(), + stream("", b"/ImA Do"), + stream("", b"/Fm Do"), + stream("/Type /XObject /Subtype /Image /Width 1 /Height 1 /ColorSpace /DeviceGray /BitsPerComponent 8", b"CONFIDENTIAL_IMAGE_A"), + stream("/Type /XObject /Subtype /Form /BBox [0 0 1000 1000]", b""), + stream("/Type /XObject /Subtype /Image /Width 1 /Height 1 /ColorSpace /DeviceGray /BitsPerComponent 8", b"PUBLIC_IMAGE_B"), + ]); + let mut entries = super::form_entries([0, 0, 1000, 1000]); + entries.push(( + "Resources", + Object::Dict(Dict::from_iter([( + b"XObject".to_vec(), + Object::Dict(Dict::from_iter([(b"ImA".to_vec(), Object::Ref(ObjRef::new(9, 0))), (b"ImB".to_vec(), Object::Ref(ObjRef::new(11, 0)))])), + )])), + )); + let form = super::behind_parms_by_reference(&mut doc, &entries, b"q 1000 0 0 1000 0 0 cm /ImB Do Q"); + doc.set(ObjRef::new(10, 0), form); + mark(&mut doc, 0, &[[9.0, 95.0, 14.9, 110.0]], ""); + assert_eq!(apply(&mut doc, None).unwrap().glyphs, 1); + assert!(!full_save_holds(&doc, b"CONFIDENTIAL_IMAGE_A"), "the removed glyph's image survives through the surviving glyph's form"); + assert!(full_save_holds(&doc, b"PUBLIC_IMAGE_B"), "the image the surviving glyph's form draws was lost"); +} + +/// The Form a kept glyph draws is walked, for the names its content uses, within the allowance of +/// a page: two forms without resources of their own that it draws, each past half of it, refuse +/// the redaction rather than being decoded in full. Two small ones are the control. +#[test] +fn the_form_a_kept_type3_glyph_draws_is_walked_within_the_allowance_of_a_page() { + for (size, refused) in [(pdfcraft_cos::MAX_PAGE_DECODED / 2 + 1, true), (1, false)] { + let packed = pdfcraft_filters::encode_flate(&vec![b' '; size]); + let blank = || stream("/Type /XObject /Subtype /Form /BBox [0 0 1000 1000] /Filter /FlateDecode", packed.as_slice()); + let mut doc = pdf(vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Resources 5 0 R >>".to_vec(), + stream("", b"BT /T3 10 Tf 10 100 Td (AB) Tj ET"), + b"<< /Font << /T3 6 0 R >> >>".to_vec(), + b"<< /Type /Font /Subtype /Type3 /FontBBox [0 -200 1000 800] /FontMatrix [0.001 0 0 0.001 0 0] /FirstChar 65 /LastChar 66 /Widths [500 500] /Encoding << /Differences [65 /A /B] >> /CharProcs << /A 7 0 R /B 8 0 R >> /Resources << /XObject << /ImA 9 0 R /Fm 10 0 R >> >> >>".to_vec(), + stream("", b"/ImA Do"), + stream("", b"/Fm Do"), + stream("/Type /XObject /Subtype /Image /Width 1 /Height 1 /ColorSpace /DeviceGray /BitsPerComponent 8", b"CONFIDENTIAL_IMAGE_A"), + stream("/Type /XObject /Subtype /Form /BBox [0 0 1000 1000] /Resources << /XObject << /Sub1 11 0 R /Sub2 12 0 R >> >>", b"/Sub1 Do /Sub2 Do"), + blank(), + blank(), + ]); + mark(&mut doc, 0, &[[9.0, 95.0, 14.9, 110.0]], ""); + let out = apply(&mut doc, None); + if refused { + assert!(matches!(out, Err(RedactError::Cos(pdfcraft_cos::CosError::Filter(_)))), "forms of {size} bytes each: {out:?}"); + } else { + assert_eq!(out.unwrap().glyphs, 1, "forms of {size} bytes each"); + } + } +} + /// When all glyphs of a Type 3 font are redacted, the entire font is removed from Resources (F-011). #[test] fn all_redacted_type3_glyphs_cause_font_removal_from_resources() { @@ -323,6 +390,139 @@ fn type3_font_used_in_a_form_xobject_has_its_redacted_glyph_removed() { assert!(!full_save_holds(&doc, b"SECRET_FORM_GLYPH_PAYLOAD"), "redacted glyph program survived"); } +/// A Type 3 font with glyphs A (its program object 7) and B (object 8), at object 6. +const TYPE3: &str = "<< /Type /Font /Subtype /Type3 /FontBBox [0 -200 1000 800] /FontMatrix [0.001 0 0 0.001 0 0] /FirstChar 65 /LastChar 66 /Widths [500 500] /Encoding << /Differences [65 /A /B] >> /CharProcs << /A 7 0 R /B 8 0 R >> >>"; + +/// A form outside the mark shows glyph B of a Type 3 font, its content behind a predictor whose +/// `/DecodeParms` are given by reference: the font's cleanup reads it through them, and keeps B. +#[test] +fn a_type3_glyph_a_form_with_parameters_by_reference_shows_is_kept() { + let mut doc = pdf(vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Resources 5 0 R >>".to_vec(), + stream("", b"BT /T3 10 Tf 10 100 Td (A) Tj ET /Fm Do"), + b"<< /Font << /T3 6 0 R >> /XObject << /Fm 9 0 R >> >>".to_vec(), + TYPE3.as_bytes().to_vec(), + stream("", b"% SECRET_TYPE3_GLYPH_A"), + stream("", b"% PUBLIC_TYPE3_GLYPH_B"), + stream("/Type /XObject /Subtype /Form /BBox [0 0 300 50]", b""), + ]); + let mut entries = super::form_entries([0, 0, 300, 50]); + entries.push(("Resources", Object::Ref(ObjRef::new(5, 0)))); + let form = super::behind_parms_by_reference(&mut doc, &entries, b"BT /T3 10 Tf 10 20 Td (B) Tj ET"); + doc.set(ObjRef::new(9, 0), form); + mark(&mut doc, 0, &[[9.0, 95.0, 16.0, 110.0]], ""); + assert_eq!(apply(&mut doc, None).unwrap().glyphs, 1); + assert!(!full_save_holds(&doc, b"SECRET_TYPE3_GLYPH_A"), "the redacted glyph program survived"); + assert!(full_save_holds(&doc, b"PUBLIC_TYPE3_GLYPH_B"), "the form still shows B"); +} + +/// A form the page lists but doesn't draw shows glyph B, its content behind a predictor whose +/// `/DecodeParms` are given by reference: the font's cleanup reads every surviving form, this one +/// through them, and keeps B. +#[test] +fn a_type3_glyph_an_undrawn_form_with_parameters_by_reference_shows_is_kept() { + let mut doc = pdf(vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Resources 5 0 R >>".to_vec(), + stream("", b"BT /T3 10 Tf 10 100 Td (A) Tj ET"), + b"<< /Font << /T3 6 0 R >> /XObject << /Fm 9 0 R >> >>".to_vec(), + TYPE3.as_bytes().to_vec(), + stream("", b"% SECRET_TYPE3_GLYPH_A"), + stream("", b"% PUBLIC_TYPE3_GLYPH_B"), + stream("/Type /XObject /Subtype /Form /BBox [0 0 300 50]", b""), + ]); + let mut entries = super::form_entries([0, 0, 300, 50]); + entries.push(( + "Resources", + Object::Dict(Dict::from_iter([(b"Font".to_vec(), Object::Dict(Dict::from_iter([(b"T3".to_vec(), Object::Ref(ObjRef::new(6, 0)))])))])), + )); + let form = super::behind_parms_by_reference(&mut doc, &entries, b"BT /T3 10 Tf 10 20 Td (B) Tj ET"); + doc.set(ObjRef::new(9, 0), form); + mark(&mut doc, 0, &[[9.0, 95.0, 16.0, 110.0]], ""); + assert_eq!(apply(&mut doc, None).unwrap().glyphs, 1); + assert!(!full_save_holds(&doc, b"SECRET_TYPE3_GLYPH_A"), "the redacted glyph program survived"); + assert!(full_save_holds(&doc, b"PUBLIC_TYPE3_GLYPH_B"), "the undrawn form still shows B"); +} + +/// Glyph B's program draws an image from the font's resources, behind a predictor whose +/// `/DecodeParms` are given by reference: the font's cleanup reads it through them, and keeps the +/// image B still draws while the one only the redacted A drew goes. +#[test] +fn an_image_a_type3_glyph_with_parameters_by_reference_draws_is_kept() { + let image = "/Type /XObject /Subtype /Image /Width 12 /Height 1 /ColorSpace /DeviceGray /BitsPerComponent 8"; + let font = TYPE3.replace("/CharProcs", "/Resources << /XObject << /ImA 9 0 R /ImB 10 0 R >> >> /CharProcs"); + let mut doc = pdf(vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Resources 5 0 R >>".to_vec(), + stream("", b"BT /T3 10 Tf 10 100 Td (A) Tj 0 100 Td (B) Tj ET"), + b"<< /Font << /T3 6 0 R >> >>".to_vec(), + font.into_bytes(), + stream("", b"/ImA Do"), + stream("", b""), + stream(image, b"SECRET_IMG_A"), + stream(image, b"PUBLIC_IMG_B"), + ]); + let glyph = super::behind_parms_by_reference(&mut doc, &[], b"/ImB Do"); + doc.set(ObjRef::new(8, 0), glyph); + mark(&mut doc, 0, &[[9.0, 95.0, 16.0, 110.0]], ""); + assert_eq!(apply(&mut doc, None).unwrap().glyphs, 1); + assert!(!full_save_holds(&doc, b"SECRET_IMG_A"), "the image only the redacted glyph drew survived"); + assert!(full_save_holds(&doc, b"PUBLIC_IMG_B"), "glyph B still draws its image"); +} + +/// Whether `out` is the redaction refused with a syntax error whose detail holds `detail`. +fn refused_for(out: &Result, detail: &str) -> bool { + matches!(out, Err(RedactError::Cos(pdfcraft_cos::CosError::Syntax { detail: d, .. })) if d.contains(detail)) +} + +/// The font's cleanup scans every page and form for the glyphs they show. A string left open in a +/// form nothing else reads (resources of its own, outside the mark) runs to the end of the +/// stream, hiding the glyph the form shows after it, so the redaction is refused: with the check +/// off, that glyph loses its program. +#[test] +fn unreadable_bytes_in_a_form_the_type3_cleanup_scans_refuse_the_redaction() { + let mut doc = pdf(vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Resources 5 0 R >>".to_vec(), + stream("", b"BT /T3 10 Tf 10 100 Td (A) Tj ET /Fm Do"), + b"<< /Font << /T3 6 0 R >> /XObject << /Fm 9 0 R >> >>".to_vec(), + TYPE3.as_bytes().to_vec(), + stream("", b"% SECRET_TYPE3_GLYPH_A"), + stream("", b"% PUBLIC_TYPE3_GLYPH_B"), + stream("/Type /XObject /Subtype /Form /BBox [0 0 300 50] /Resources << /Font << /T3 6 0 R >> >>", b"BT /T3 10 Tf 10 20 Td ( (B) Tj ET"), + ]); + mark(&mut doc, 0, &[[9.0, 95.0, 16.0, 110.0]], ""); + let out = apply(&mut doc, None); + assert!(refused_for(&out, "content stream syntax"), "{out:?}"); +} + +/// The font's cleanup reads the program of each glyph it keeps, for the resources it uses. A +/// string left open there runs to the end of the program, hiding the image it draws after it, so +/// the redaction is refused: with the check off, the font's resources lose that image. +#[test] +fn unreadable_bytes_in_a_kept_type3_glyph_program_refuse_the_redaction() { + let font = TYPE3.replace("/CharProcs", "/Resources << /XObject << /ImB 9 0 R >> >> /CharProcs"); + let mut doc = pdf(vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Resources 5 0 R >>".to_vec(), + stream("", b"BT /T3 10 Tf 10 100 Td (A) Tj 0 100 Td (B) Tj ET"), + b"<< /Font << /T3 6 0 R >> >>".to_vec(), + font.into_bytes(), + stream("", b"% SECRET_TYPE3_GLYPH_A"), + stream("", b"500 0 d0 ( /ImB Do"), + stream("/Type /XObject /Subtype /Image /Width 1 /Height 1 /ColorSpace /DeviceGray /BitsPerComponent 8", b"o"), + ]); + mark(&mut doc, 0, &[[9.0, 95.0, 16.0, 110.0]], ""); + let out = apply(&mut doc, None); + assert!(refused_for(&out, "glyph stream syntax"), "{out:?}"); +} + /// A Form the surviving glyph draws holds `/Resources` as an array that references the removed /// glyph's image: it can't be pruned, so the redaction refuses rather than keep the image. #[test] @@ -430,6 +630,36 @@ fn a_type3_font_only_an_appearance_without_resources_selects_stays_on_the_page() assert!(saved_first_page_lists_font(&doc, b"T4"), "the page no longer lists the font the appearance selects"); } +/// The same appearance, its content behind a predictor whose `/DecodeParms` are given by +/// reference: the page's cleanup reads it through them, and keeps the font it selects. +#[test] +fn a_type3_font_only_an_appearance_with_parameters_by_reference_selects_stays_on_the_page() { + let mut doc = pdf(vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Resources 5 0 R /Annots [8 0 R] >>".to_vec(), + stream("", b"BT /T3 10 Tf 10 100 Td (AC) Tj ET"), + b"<< /Font << /T3 6 0 R /T4 7 0 R >> >>".to_vec(), + type3_abc(10), + type3_abc(13), + b"<< /Type /Annot /Subtype /FreeText /Rect [150 200 250 220] /P 3 0 R /DA (/T4 10 Tf 0 g) /AP << /N 9 0 R >> >>".to_vec(), + stream("/Type /XObject /Subtype /Form /BBox [0 0 100 20]", b""), + stream("", b"% REDACTED_GLYPH_A"), + stream("", b"% UNUSED_GLYPH_B"), + stream("", b"% PAGE_GLYPH_C"), + stream("", b"% UNUSED_T4_GLYPH_A"), + stream("", b"% APPEARANCE_GLYPH_B"), + stream("", b"% UNUSED_T4_GLYPH_C"), + ]); + let appearance = super::behind_parms_by_reference(&mut doc, &super::form_entries([0, 0, 100, 20]), b"BT /T4 10 Tf 2 5 Td (B) Tj ET"); + doc.set(ObjRef::new(9, 0), appearance); + mark(&mut doc, 0, &[[9.0, 95.0, 14.9, 110.0]], ""); + assert_eq!(apply(&mut doc, None).unwrap().glyphs, 1); + assert!(!full_save_holds(&doc, b"REDACTED_GLYPH_A"), "the redacted glyph program survived"); + assert!(full_save_holds(&doc, b"APPEARANCE_GLYPH_B"), "the glyph the appearance shows was unlinked"); + assert!(saved_first_page_lists_font(&doc, b"T4"), "the page no longer lists the font the appearance selects"); +} + /// The operators of a page's appearances count toward one total, as the content's do. Two /// appearances under the cap alone pass it together, and the redaction is refused rather than /// judged on part of what draws through the page's fonts. @@ -483,6 +713,35 @@ fn a_page_s_appearances_share_one_decoded_byte_allowance() { assert!(refused.is_err(), "two appearances past the page's decoded-byte allowance together were read in full: {refused:?}"); } +/// The appearances are read within what the page's content left of the page's allowance, not +/// within one of their own: 1 MiB of content and an appearance within half a MiB of the allowance +/// together pass it, though the appearance alone fits. A small appearance is the control. +#[test] +fn a_page_s_appearances_spend_what_its_content_left_of_the_allowance() { + let mut content = b"BT /T3 10 Tf 10 100 Td (AC) Tj ET".to_vec(); + content.resize(1 << 20, b' '); + let content = pdfcraft_filters::encode_flate(&content); + for (size, refused) in [(pdfcraft_cos::MAX_PAGE_DECODED - (1 << 19), true), (1, false)] { + let appearance = pdfcraft_filters::encode_flate(&vec![b' '; size]); + let mut doc = pdf(vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Resources 5 0 R /Annots [7 0 R] >>".to_vec(), + stream("/Filter /FlateDecode", content.as_slice()), + b"<< /Font << /T3 6 0 R >> >>".to_vec(), + type3_abc(9), + b"<< /Type /Annot /Subtype /FreeText /Rect [150 200 250 220] /P 3 0 R /AP << /N 8 0 R >> >>".to_vec(), + stream("/Type /XObject /Subtype /Form /BBox [0 0 100 20] /Filter /FlateDecode", appearance.as_slice()), + stream("", b"% GLYPH_A"), + stream("", b"% GLYPH_B"), + stream("", b"% GLYPH_C"), + ]); + mark(&mut doc, 0, &[[9.0, 95.0, 14.9, 110.0]], ""); + let out = apply(&mut doc, None); + assert_eq!(out.is_err(), refused, "an appearance of {size} bytes: {out:?}"); + } +} + /// Whether the first page of a full save of `doc` lists the font `name` in its resources. fn saved_first_page_lists_font(doc: &Document, name: &[u8]) -> bool { let bytes = write_full(doc, &SaveOptions::default()).unwrap(); @@ -524,6 +783,41 @@ fn a_type3_font_only_a_soft_mask_without_resources_selects_stays_on_the_page() { assert!(saved_first_page_lists_font(&doc, b"T4"), "the page no longer lists the font the soft mask selects"); } +/// The same group, its content behind a predictor whose `/DecodeParms` are given by reference: +/// the page's cleanup reads it through them, and keeps the font it selects. +#[test] +fn a_type3_font_only_a_soft_mask_with_parameters_by_reference_selects_stays_on_the_page() { + let mut doc = pdf(vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Resources 5 0 R >>".to_vec(), + stream("", b"BT /T3 10 Tf 10 100 Td (AC) Tj ET /GS1 gs 150 150 100 100 re f"), + b"<< /Font << /T3 6 0 R /T4 7 0 R >> /ExtGState << /GS1 << /Type /ExtGState /SMask << /Type /Mask /S /Luminosity /G 8 0 R >> >> >> >>" + .to_vec(), + type3_abc(9), + type3_abc(12), + stream("/Type /XObject /Subtype /Form /BBox [0 0 300 300]", b""), + stream("", b"% REDACTED_GLYPH_A"), + stream("", b"% UNUSED_GLYPH_B"), + stream("", b"% PAGE_GLYPH_C"), + stream("", b"% UNUSED_T4_GLYPH_A"), + stream("", b"% MASK_GLYPH_B"), + stream("", b"% UNUSED_T4_GLYPH_C"), + ]); + let mut entries = super::form_entries([0, 0, 300, 300]); + entries.push(( + "Group", + Object::Dict(Dict::from_iter([(b"S".to_vec(), Object::name("Transparency")), (b"CS".to_vec(), Object::name("DeviceGray"))])), + )); + let group = super::behind_parms_by_reference(&mut doc, &entries, b"1 g BT /T4 40 Tf 160 180 Td (B) Tj ET"); + doc.set(ObjRef::new(8, 0), group); + mark(&mut doc, 0, &[[9.0, 95.0, 14.9, 110.0]], ""); + assert_eq!(apply(&mut doc, None).unwrap().glyphs, 1); + assert!(!full_save_holds(&doc, b"REDACTED_GLYPH_A"), "the redacted glyph program survived"); + assert!(full_save_holds(&doc, b"MASK_GLYPH_B"), "the glyph the soft mask shows was unlinked"); + assert!(saved_first_page_lists_font(&doc, b"T4"), "the page no longer lists the font the soft mask selects"); +} + /// The cell of a tiling pattern another page fills with shows glyph B: its program is kept. (On /// the redacted page itself, a pattern that shows text refuses the redaction: `pattern_guard`.) #[test] diff --git a/crates/redact/src/type3.rs b/crates/redact/src/type3.rs index 7347bfdb82..4c3f2211e8 100644 --- a/crates/redact/src/type3.rs +++ b/crates/redact/src/type3.rs @@ -131,11 +131,13 @@ struct UsedInGlyphs { } fn collect_used_resources_in_stream(data: &[u8], used: &mut UsedInGlyphs) -> Result<(), CosError> { - let parsed = pdfcraft_content::parse(data); - if parsed.skipped > 0 { + // One operator at a time, so a glyph program of millions of them is never held whole. A byte + // that can't be read anywhere in it still refuses, once the loop has read it all. + let mut read = pdfcraft_content::ops(data); + read.by_ref().for_each(|op| record_used(&op, used)); + if read.skipped() > 0 { return Err(CosError::Syntax { offset: 0, detail: "redaction could not fully inspect glyph stream syntax".into() }); } - parsed.ops.iter().for_each(|op| record_used(op, used)); Ok(()) } @@ -218,9 +220,13 @@ fn prune_font_forms(doc: &mut Document, font_res: &Dict) -> Result<(), CosError> } }, }; - let data = form.decoded()?; + let data = doc.with_direct_parms(form).decoded()?; let mut used = UsedInGlyphs::default(); - crate::interp::resource_ops(doc, &res, std::slice::from_ref(&data))?.iter().for_each(|op| record_used(op, &mut used)); + // Each Form's walk has the allowance of a page, as `prune_retired_properties` gives each + // content it reads. + crate::interp::resource_ops(doc, &mut pdfcraft_cos::DecodeBudget::page(), &res, std::slice::from_ref(&data))? + .iter() + .for_each(|op| record_used(op, &mut used)); // `rg`, `g` and `k` draw through these entries when the Form keeps them as its defaults, // and nothing cites them with `cs`/`CS`: pruning them would change the Form's colours. used.colorspaces.extend([b"DefaultRGB".to_vec(), b"DefaultGray".to_vec(), b"DefaultCMYK".to_vec()]); @@ -287,15 +293,14 @@ fn scan_content_stream( return Err(CosError::Syntax { offset: 0, detail: "redaction exceeded max depth inspecting Type 3 font content".into() }); } let joined = Pieces::join(content); - let parsed = joined.parse(); - if parsed.skipped > 0 { - return Err(CosError::Syntax { offset: 0, detail: "redaction could not fully inspect content stream syntax".into() }); - } + // One operator at a time, so content of millions of them is never held whole. The scan only + // reads, and a byte that can't be read anywhere still refuses, once the loop has read it all. + let mut read = pdfcraft_content::ops(joined.data()); let mut stack: Vec> = Vec::new(); let mut current_font = initial_font; - for op in parsed.ops { + for op in read.by_ref() { match op.op.as_slice() { b"q" => stack.push(current_font), b"Q" => { @@ -322,6 +327,9 @@ fn scan_content_stream( _ => apply_text_op(ctx, res, &op, &mut current_font), } } + if read.skipped() > 0 { + return Err(CosError::Syntax { offset: 0, detail: "redaction could not fully inspect content stream syntax".into() }); + } Ok(()) } @@ -490,7 +498,7 @@ fn add_soft_mask_ops(doc: &Document, res: &Dict, ops: &mut Vec 0 || ops.len().saturating_add(parsed.ops.len()) > 2_000_000 { return Err(CosError::Syntax { offset: 0, detail: "redaction could not fully inspect soft mask content".into() }); } @@ -517,7 +525,7 @@ fn scan_stream_object( if !ctx.seen_forms.insert((r, current_font, through)) { return Ok(()); } - let data = s.decoded()?; + let data = ctx.doc.with_direct_parms(s).decoded()?; match own_res { Some(own) => scan_content_stream(ctx, std::slice::from_ref(&data), r, &own, current_font, depth + 1), None => scan_content_stream(ctx, std::slice::from_ref(&data), holder_ref, res, current_font, depth + 1), @@ -533,8 +541,15 @@ fn gs_font_object(doc: &Document, state_obj: &Object) -> Option { /// Prune Type 3 fonts from a rewritten page or form's `/Resources` dictionary if the rewritten /// content no longer draws any text using that font name. `page` is the page's dictionary when -/// `res` are a page's resources. -pub(crate) fn prune_type3_from_resources(doc: &Document, res: &mut Dict, content: &[Vec], page: Option<&Dict>) -> Result<(), CosError> { +/// `res` are a page's resources. What it reads is charged to `budget`, what is left of the page's +/// allowance. +pub(crate) fn prune_type3_from_resources( + doc: &Document, + budget: &mut pdfcraft_cos::DecodeBudget, + res: &mut Dict, + content: &[Vec], + page: Option<&Dict>, +) -> Result<(), CosError> { let mut fonts = res_dict(doc, res, b"Font"); if fonts.is_empty() { return Ok(()); @@ -542,20 +557,20 @@ pub(crate) fn prune_type3_from_resources(doc: &Document, res: &mut Dict, content // The operators of forms that draw through these resources count too: dropping a font a child // form still shows would leave its text without one. - let mut ops = crate::interp::resource_ops(doc, res, content)?; + let mut ops = crate::interp::resource_ops(doc, budget, res, content)?; // So do those of a page's annotation appearances, and of soft mask groups, without resources of // their own. They are read only when there is a Type 3 font to keep, so other files are not // refused over them. if fonts.iter().any(|(_, f)| doc.resolve(f).as_dict().is_some_and(|d| d.name(b"Subtype") == Some(b"Type3"))) { if let Some(page) = page { // One total for all of them, as for the content: each appearance under the cap could - // still add up to far more than it. Their decoded bytes share one page's allowance - // too, and each is decoded only in its turn, so they are never all held at once. - let mut budget = pdfcraft_cos::DecodeBudget::page(); + // still add up to far more than it. Their decoded bytes, and their forms', are charged to + // the page's allowance, which the content and its forms spend too, and each is decoded + // only in its turn, so they are never all held at once. for r in resourceless_appearances(doc, page) { let Object::Stream(s) = &*doc.get(r) else { continue }; - let data = budget.decode(s)?; - let more = crate::interp::resource_ops(doc, res, std::slice::from_ref(&data))?; + let data = budget.decode(&doc.with_direct_parms(s))?; + let more = crate::interp::resource_ops(doc, budget, res, std::slice::from_ref(&data))?; if ops.len().saturating_add(more.len()) > 2_000_000 { return Err(CosError::Syntax { offset: 0, detail: "redaction could not fully inspect annotation appearances".into() }); } @@ -696,7 +711,7 @@ pub(crate) fn unlink_type3(doc: &mut Document) -> Result<(), RedactError> { for (form_ref, form_res) in &form_holders { if !ctx.seen_forms.iter().any(|(form, _, _)| form == form_ref) { let Object::Stream(s) = &*ctx.doc.get(*form_ref) else { continue }; - let data = s.decoded()?; + let data = ctx.doc.with_direct_parms(s).decoded()?; scan_content_stream(&mut ctx, std::slice::from_ref(&data), *form_ref, form_res, None, 0)?; } } @@ -830,7 +845,7 @@ fn clean_type3_font( } .into()); }; - let data = s.decoded()?; + let data = doc.with_direct_parms(s).decoded()?; collect_used_resources_in_stream(&data, &mut used_in_glyphs)?; } diff --git a/crates/redact/tests/operator_memory.rs b/crates/redact/tests/operator_memory.rs new file mode 100644 index 0000000000..f360f83cdd --- /dev/null +++ b/crates/redact/tests/operator_memory.rs @@ -0,0 +1,142 @@ +//! Memory regressions for the streams a redaction reads besides its walk: the forms it follows +//! through resources, and what the Type 3 cleanup scans and reads. Their operators are read one +//! at a time, so a stream of millions of them is never held whole. A test binary of its own, so +//! that the one test is all the process runs while its peak is measured. + +use std::sync::Arc; + +use pdfcraft_annot::{Meta, NewAnnotation, Shape, Style, add_annotation, rect_quad}; +use pdfcraft_cos::{Document, ObjRef, Object, Stream}; +use pdfcraft_redact::apply; +use pdfcraft_testkit::{Pdf, stream_obj}; + +const IMAGE: &str = "/Type /XObject /Subtype /Image /Width 1 /Height 1 /ColorSpace /DeviceGray /BitsPerComponent 8"; + +/// 32 MiB of `n` operators: 16 million of them, eight times the walk's limit. Held as parsed +/// operators, at about 120 bytes each, nearly 2 GiB. +fn operators() -> Vec { + b"n\n".repeat(16 << 20) +} + +/// `form` with `content`, compressed. +fn compress(doc: &mut Document, form: ObjRef, content: &[u8]) { + let Object::Stream(s) = &*doc.get(form) else { panic!("{form:?} is a stream") }; + let dict = s.dict.clone(); + doc.set(form, Object::Stream(Stream::flate(dict, content))); +} + +fn mark(doc: &mut Document) { + let shape = Shape::Redact { quads: vec![rect_quad([0.0, 200.0, 300.0, 300.0])], overlay: String::new(), look: Default::default() }; + let style = Style::default_for(&shape); + add_annotation(doc, &NewAnnotation { page: 0, shape, style, contents: String::new(), author: "Synthetic test".into() }, &Meta::default()) + .unwrap(); +} + +/// How much the process's peak resident memory grows, in MiB, while `f` runs. `None` where the +/// peak can't be reset (`/proc/self/clear_refs` is Linux's). +fn peak_growth(f: impl FnOnce() -> T) -> Option<(T, u64)> { + fn peak_kib() -> Option { + let status = std::fs::read_to_string("/proc/self/status").ok()?; + status.lines().find_map(|l| l.strip_prefix("VmHWM:"))?.split_whitespace().next()?.parse().ok() + } + std::fs::write("/proc/self/clear_refs", "5").ok()?; + let before = peak_kib()?; + let out = f(); + Some((out, peak_kib()?.saturating_sub(before) >> 10)) +} + +/// A page that draws an image under the mark, a form `form` with resources of its own outside it, +/// and a glyph of a Type 3 font outside it, whose program is `glyph`: the Type 3 cleanup reads +/// every page and form, and the programs of the glyphs it keeps. +fn type3_page(form: &[u8], glyph: &[u8]) -> Document { + let mut doc = Document::open(Arc::new( + Pdf::new() + .objs(vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Resources << /Font << /T3 7 0 R >> /XObject << /Im1 5 0 R /Fm 6 0 R >> >> >>" + .to_vec(), + stream_obj("", b"q 50 0 0 50 20 230 cm /Im1 Do Q /Fm Do BT /T3 10 Tf 10 10 Td (a) Tj ET"), + stream_obj(IMAGE, b"o"), + stream_obj("/Type /XObject /Subtype /Form /BBox [0 0 100 100] /Resources << >>", b""), + b"<< /Type /Font /Subtype /Type3 /FontBBox [0 0 1000 1000] /FontMatrix [0.001 0 0 0.001 0 0] /CharProcs << /a 8 0 R >> /Encoding << /Differences [97 /a] >> /FirstChar 97 /LastChar 97 /Widths [1000] /Resources << >> >>" + .to_vec(), + stream_obj("", b""), + ]) + .build(), + )) + .unwrap(); + compress(&mut doc, ObjRef::new(6, 0), form); + compress(&mut doc, ObjRef::new(8, 0), glyph); + mark(&mut doc); + doc +} + +/// Four reads of a stream of 16 million operators: +/// - the page's check of what it still draws walks a form without resources of its own outside +/// the mark, and refuses at 2,000,001 operators; +/// - the unlinking check of a page nobody marked, which shares the marked page's resources, +/// looks through such a form for the image the marked page removed, and finds no `Do`; +/// - the Type 3 cleanup scans a form with resources of its own that nothing else reads, for the +/// glyphs it shows; +/// - the Type 3 cleanup reads the program of a glyph it keeps, for the resources it uses. +/// +/// Each parsed every operator first: the peak grew by 4,415, 2,526, 2,526 and 2,494 MiB. Now +/// the first holds at most one operator past the walk's limit, and the others none. +#[test] +fn the_forms_followed_through_resources_are_read_one_operator_at_a_time() { + // The page's check: an image under the mark, and the form outside it. + let mut doc = Document::open(Arc::new( + Pdf::new() + .objs(vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Resources << /XObject << /Im1 5 0 R /Fm 6 0 R >> >> >>" + .to_vec(), + stream_obj("", b"q 50 0 0 50 20 230 cm /Im1 Do Q /Fm Do"), + stream_obj(IMAGE, b"o"), + stream_obj("/Type /XObject /Subtype /Form /BBox [0 0 100 100]", b""), + ]) + .build(), + )) + .unwrap(); + compress(&mut doc, ObjRef::new(6, 0), &operators()); + mark(&mut doc); + let Some((result, grew)) = peak_growth(|| apply(&mut doc, None)) else { return }; + assert!(result.is_err(), "the walk refuses past 2,000,000 operators: {result:?}"); + assert!(grew < 1024, "the page's check grew the peak by {grew} MiB"); + + // The unlinking check: two pages share the /Pages node's resources; the second draws the form. + let mut doc = Document::open(Arc::new( + Pdf::new() + .objs(vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [6 0 R 7 0 R] /Count 2 /Resources << /XObject << /Im1 5 0 R /Fm 3 0 R >> >> >>".to_vec(), + stream_obj("/Type /XObject /Subtype /Form /BBox [0 0 100 100]", b""), + stream_obj("", b"q 50 0 0 50 20 230 cm /Im1 Do Q"), + stream_obj(IMAGE, b"o"), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 8 0 R >>".to_vec(), + stream_obj("", b"/Fm Do"), + ]) + .build(), + )) + .unwrap(); + compress(&mut doc, ObjRef::new(3, 0), &operators()); + mark(&mut doc); + let Some((result, grew)) = peak_growth(|| apply(&mut doc, Some(&[0]))) else { return }; + assert_eq!(result.map(|r| r.images_removed).ok(), Some(1), "the marked page's image goes"); + assert!(grew < 1024, "the unlinking check grew the peak by {grew} MiB"); + + let mut doc = type3_page(&operators(), b"1000 0 d0"); + let Some((result, grew)) = peak_growth(|| apply(&mut doc, None)) else { return }; + assert_eq!(result.map(|r| r.images_removed).ok(), Some(1), "the Type 3 scan: the image goes"); + assert!(grew < 1024, "the Type 3 cleanup's scan grew the peak by {grew} MiB"); + + let mut glyph = b"1000 0 d0\n".to_vec(); + glyph.extend(operators()); + let mut doc = type3_page(b"", &glyph); + let Some((result, grew)) = peak_growth(|| apply(&mut doc, None)) else { return }; + assert_eq!(result.map(|r| r.images_removed).ok(), Some(1), "the glyph program: the image goes"); + assert!(grew < 1024, "the Type 3 cleanup's read of a glyph program grew the peak by {grew} MiB"); +}