Remove Hidden Information unlinks images nothing draws

Delete Image takes only the Do out of the page, and Replace Image adds a
new name and leaves the old image listed. Remove Hidden Information only
pruned the names its own content pass replaced, so a deleted image stayed
in the page's resources and in the full-rewrite save.

Every run now unlinks the image and form XObjects that nothing draws from
the page-tree resource dictionaries that name them. A name stays when
anything drawing through the dictionary uses it with Do: the pages, their
annotations' appearances, and every form, tiling pattern, Type 3 glyph and
soft mask they reach, with /Resources of their own or not, since a name
those don't define is looked up in the page's. Going by name keeps more
than it must, never less. Each stream is read once against one decode
budget; a page or stream that won't decode, or a spent budget, leaves the
dictionary as it is. The page-tree walk is shared with unlink_doomed.

crates/redact/src/interp.rs

@@ -31,7 +31,7 @@ mod prune;
 
 use layers::strip_hidden_layers;
 use prune::{clip_out, extgstate_font, prune_form_resources};
-pub(crate) use prune::{prune, prune_patterns, prune_properties, resource_ops, unlink_doomed, unlink_pruned_patterns};
+pub(crate) use prune::{prune, prune_patterns, prune_properties, resource_ops, unlink_doomed, unlink_pruned_patterns, unlink_undrawn};
 
 /// How deep this pass follows form XObjects. A form past it is removed rather than walked, which
 /// is the fail-safe choice for a redaction -- marked content inside a form we cannot read would

crates/redact/src/interp/prune.rs

@@ -1,6 +1,6 @@
 use super::{
-    CosError, Dict, Document, HashMap, HashSet, MAX_DEPTH, Matrix, Metrics, ObjRef, Object, Op, Pieces, Rc, RedactError, Scope, is_vertical, num,
-    overlaps, page_streams, res_dict,
+    CosError, DecodeBudget, Dict, Document, HashMap, HashSet, MAX_DEPTH, Matrix, Metrics, ObjRef, Object, Op, Pieces, Rc, RedactError, Scope,
+    is_vertical, num, overlaps, page_streams, res_dict,
 };
 
 /// Drop every `replaced` resource name whose `Do` no longer appears in the rewritten `content`,
@@ -49,29 +49,12 @@ fn content_draws(doc: &Document, content: &[Vec<u8>], xo: &Dict, name: &[u8], de
     Some(false)
 }
 
-/// Unlink the XObjects a rewrite dropped or replaced (`doomed`) from every page-tree resource
-/// dictionary that still names them. The rewritten page has its own pruned /Resources, but the
-/// originals stay reachable, and so stay in a full-rewrite save, when:
-/// - the page inherited its /Resources from a /Pages node, which still lists them;
-/// - several pages share one resource dictionary, and another page still refers to it;
-/// - a form without /Resources drew them through the page's dictionary.
-///
-/// An entry goes unless a page that draws through that dictionary still draws the name (the same
-/// image shown on another page, or again outside a mark). A dictionary that loses an entry is
-/// replaced by a private copy on its node, so a dictionary object shared with other nodes is
-/// left as it is for them. Retired properties are likewise unlinked only when none of the
-/// holder's pages or inheriting forms names them. Fails closed when use cannot be determined.
-pub(crate) fn unlink_doomed(doc: &mut Document, doomed: &[ObjRef], retired_properties: &[Object]) -> Result<(), RedactError> {
-    crate::type3::unlink_type3(doc)?;
-    if doomed.is_empty() && retired_properties.is_empty() {
-        return Ok(());
-    }
-    let doomed: HashSet<ObjRef> = doomed.iter().copied().collect();
-    let Some(top) = doc.root().and_then(|root| doc.get(root).as_dict().and_then(|d| d.reference(b"Pages"))) else { return Ok(()) };
-    // Every page-tree node with a /Resources of its own, and the pages that draw through it
-    // (the nearest such node above a page, or the page itself). Cycles are cut as in
-    // `pdfcraft_model::pages`.
+/// Every page-tree node with a /Resources of its own, and the pages that draw through it (the
+/// nearest such node above a page, or the page itself). Cycles are cut as in
+/// `pdfcraft_model::pages`.
+fn resource_holders(doc: &Document) -> Vec<(ObjRef, Vec<ObjRef>)> {
     let mut holders: Vec<(ObjRef, Vec<ObjRef>)> = Vec::new();
+    let Some(top) = doc.root().and_then(|root| doc.get(root).as_dict().and_then(|d| d.reference(b"Pages"))) else { return holders };
     let mut seen = HashSet::new();
     let mut stack: Vec<(ObjRef, Option<usize>)> = vec![(top, None)];
     while let Some((node, mut holder)) = stack.pop() {
@@ -92,6 +75,152 @@ pub(crate) fn unlink_doomed(doc: &mut Document, doomed: &[ObjRef], retired_prope
             h.1.push(node);
         }
     }
+    holders
+}
+
+/// Unlink the image and form XObjects nothing draws from the page-tree resource dictionaries
+/// that still name them: Delete Image and Replace Image take an image off the page but leave it
+/// listed there, and so in a full-rewrite save. A name stays when anything that draws through the
+/// dictionary uses it ([`names_drawn`]), and a dictionary whose use can't be told is left as it
+/// is. A dictionary that loses an entry is replaced by a private copy on its node, as in
+/// [`unlink_doomed`]. Returns how many names went.
+pub(crate) fn unlink_undrawn(doc: &mut Document) -> Result<usize, RedactError> {
+    let index: HashMap<ObjRef, usize> = pdfcraft_model::pages(doc).iter().enumerate().map(|(i, p)| (p.obj, i)).collect();
+    let mut n = 0;
+    for (node, users) in resource_holders(doc) {
+        let Some(mut res) = doc.get(node).as_dict().and_then(|d| d.get(b"Resources")).and_then(|r| doc.resolve(r).as_dict().cloned()) else {
+            continue;
+        };
+        let mut xo = res_dict(doc, &res, b"XObject");
+        let listed: Vec<Vec<u8>> = xo
+            .iter()
+            .filter(|(_, v)| {
+                v.as_ref().is_some_and(|r| matches!(&*doc.get(r), Object::Stream(s) if matches!(s.dict.name(b"Subtype"), Some(b"Image" | b"Form"))))
+            })
+            .map(|(name, _)| name.to_vec())
+            .collect();
+        if listed.is_empty() {
+            continue;
+        }
+        let Some(drawn) = names_drawn(doc, &res, &users, &index) else { continue };
+        let mut pruned = false;
+        for name in listed.iter().filter(|name| !drawn.contains(*name)) {
+            xo.remove(name);
+            pruned = true;
+            n += 1;
+        }
+        if pruned {
+            res.set(b"XObject".to_vec(), Object::Dict(xo));
+            doc.update_dict(node, |d| d.set(b"Resources".to_vec(), Object::Dict(res)))?;
+        }
+    }
+    Ok(n)
+}
+
+/// Every name used with `Do` by what draws through `res`, the resource dictionary of `pages`:
+/// their content, their annotations' appearances, and every form, tiling pattern, Type 3 glyph
+/// and soft mask those can reach, with /Resources of their own or not. A name such a stream
+/// doesn't define is looked up in the resources of whatever draws it, up to the page's, so a `Do`
+/// anywhere in there may draw from `res`. Going by name keeps more than it must, never less.
+/// Each stream is read once, against one budget. `None` when that can't be told: a page or
+/// stream that won't decode, or the budget spent.
+fn names_drawn(doc: &Document, res: &Dict, pages: &[ObjRef], index: &HashMap<ObjRef, usize>) -> Option<HashSet<Vec<u8>>> {
+    let mut drawn = HashSet::new();
+    let mut note = |data: &[u8]| {
+        for op in pdfcraft_content::parse(data).ops.iter().filter(|op| op.is("Do")) {
+            if let Some(name) = op.last_name() {
+                drawn.insert(name.to_vec());
+            }
+        }
+    };
+    // Streams still to read, and resource dictionaries still to look through.
+    let mut streams: Vec<ObjRef> = Vec::new();
+    let mut dicts: Vec<Dict> = vec![res.clone()];
+    for page in pages {
+        let dict = doc.get(*page).as_dict().cloned().unwrap_or_default();
+        let (_, content) = page_streams(doc, &dict, index.get(page).copied().unwrap_or(0)).ok()?;
+        content.iter().for_each(|c| note(c));
+        let annots = dict.get(b"Annots").map(|a| doc.resolve(a)).and_then(|a| a.as_array().cloned()).unwrap_or_default();
+        for annot in annots {
+            let ap = doc.resolve(&annot).as_dict().and_then(|a| a.get(b"AP")).map(|ap| doc.resolve(ap)).and_then(|ap| ap.as_dict().cloned());
+            // Each of /N, /R and /D is an appearance, or a dictionary of them by state.
+            for (_, look) in ap.unwrap_or_default().iter() {
+                match look.as_ref() {
+                    Some(r) if matches!(&*doc.get(r), Object::Stream(_)) => streams.push(r),
+                    _ => streams.extend(doc.resolve(look).as_dict().into_iter().flat_map(|states| states.iter().filter_map(|(_, v)| v.as_ref()))),
+                }
+            }
+        }
+    }
+    let mut budget = DecodeBudget::page();
+    let mut read = HashSet::new();
+    let mut looked = 0;
+    loop {
+        while let Some(dict) = dicts.get(looked) {
+            streams.extend(drawn_through(doc, dict));
+            looked += 1;
+        }
+        let Some(r) = streams.pop() else { break };
+        if !read.insert(r) {
+            continue;
+        }
+        let obj = doc.get(r);
+        let Object::Stream(s) = &*obj else { continue };
+        if !budget.enter_form() {
+            return None;
+        }
+        note(&budget.decode_strict(s).ok()?);
+        if let Some(own) = s.dict.get(b"Resources").map(|o| doc.resolve(o)).and_then(|o| o.as_dict().cloned()) {
+            dicts.push(own);
+        }
+    }
+    Some(drawn)
+}
+
+/// The content streams the resource dictionary `res` lists: its form XObjects, its tiling
+/// patterns, its Type 3 fonts' glyph procedures and its soft masks' groups.
+fn drawn_through(doc: &Document, res: &Dict) -> Vec<ObjRef> {
+    let is_stream = |r: &ObjRef| matches!(&*doc.get(*r), Object::Stream(_));
+    let mut found: Vec<ObjRef> = Vec::new();
+    for (_, x) in res_dict(doc, res, b"XObject").iter() {
+        found.extend(x.as_ref().filter(|r| matches!(&*doc.get(*r), Object::Stream(s) if s.dict.name(b"Subtype") == Some(b"Form"))));
+    }
+    for (_, p) in res_dict(doc, res, b"Pattern").iter() {
+        found.extend(p.as_ref().filter(|r| matches!(&*doc.get(*r), Object::Stream(s) if s.dict.int(b"PatternType") == Some(1))));
+    }
+    for (_, f) in res_dict(doc, res, b"Font").iter() {
+        let font = doc.resolve(f);
+        let Some(font) = font.as_dict().filter(|f| f.name(b"Subtype") == Some(b"Type3")) else { continue };
+        let procs = font.get(b"CharProcs").map(|c| doc.resolve(c)).and_then(|c| c.as_dict().cloned()).unwrap_or_default();
+        found.extend(procs.iter().filter_map(|(_, v)| v.as_ref()).filter(is_stream));
+    }
+    for (_, g) in res_dict(doc, res, b"ExtGState").iter() {
+        let gs = doc.resolve(g);
+        let mask = gs.as_dict().and_then(|g| g.get(b"SMask")).map(|m| doc.resolve(m));
+        found.extend(mask.as_ref().and_then(|m| m.as_dict()).and_then(|m| m.reference(b"G")).filter(is_stream));
+    }
+    found
+}
+
+/// Unlink the XObjects a rewrite dropped or replaced (`doomed`) from every page-tree resource
+/// dictionary that still names them. The rewritten page has its own pruned /Resources, but the
+/// originals stay reachable, and so stay in a full-rewrite save, when:
+/// - the page inherited its /Resources from a /Pages node, which still lists them;
+/// - several pages share one resource dictionary, and another page still refers to it;
+/// - a form without /Resources drew them through the page's dictionary.
+///
+/// An entry goes unless a page that draws through that dictionary still draws the name (the same
+/// image shown on another page, or again outside a mark). A dictionary that loses an entry is
+/// replaced by a private copy on its node, so a dictionary object shared with other nodes is
+/// left as it is for them. Retired properties are likewise unlinked only when none of the
+/// holder's pages or inheriting forms names them. Fails closed when use cannot be determined.
+pub(crate) fn unlink_doomed(doc: &mut Document, doomed: &[ObjRef], retired_properties: &[Object]) -> Result<(), RedactError> {
+    crate::type3::unlink_type3(doc)?;
+    if doomed.is_empty() && retired_properties.is_empty() {
+        return Ok(());
+    }
+    let doomed: HashSet<ObjRef> = doomed.iter().copied().collect();
+    let holders = resource_holders(doc);
     let index: HashMap<ObjRef, usize> = pdfcraft_model::pages(doc).iter().enumerate().map(|(i, p)| (p.obj, i)).collect();
     let holder_nodes: Vec<ObjRef> = holders.iter().map(|(node, _)| *node).collect();
     for (node, users) in holders {

crates/redact/src/sanitize.rs

@@ -671,6 +671,9 @@ pub fn remove_hidden(doc: &mut Document, which: &[Hidden]) -> Result<Vec<(Hidden
             doc.set(r, o);
         }
     }
+    // Images taken off their page (Delete Image, Replace Image) are still listed in its
+    // resources: the full rewrite would keep them.
+    crate::interp::unlink_undrawn(doc)?;
     if done.is_empty() {
         return Err(RedactError::NothingToApply);
     }

crates/redact/src/tests.rs

@@ -4207,3 +4207,103 @@ fn a_redacted_form_xobject_drops_the_source_text_of_added_content() {
     let bytes = write_full(&doc, &SaveOptions { object_streams: false, ..SaveOptions::default() }).unwrap();
     assert!(!bytes.windows(4).any(|w| w == b"1234"), "redacted text left in the saved file");
 }
+
+/// The full rewrite of the document made of `objs` after Remove Hidden Information, run for its
+/// metadata so it has something to do.
+fn hidden_information_removed(objs: Vec<Vec<u8>>) -> Vec<u8> {
+    use crate::sanitize::{Hidden, remove_hidden};
+    let mut doc = pdf(objs);
+    let info = doc.add(Object::Dict({
+        let mut d = Dict::new();
+        d.set(b"Title".to_vec(), pdfcraft_cos::PdfString::text("Plan"));
+        d
+    }));
+    doc.trailer_mut().set(b"Info".to_vec(), Object::Ref(info));
+    remove_hidden(&mut doc, &[Hidden::Metadata]).unwrap();
+    write_full(&doc, &SaveOptions { object_streams: false, ..SaveOptions::default() }).unwrap()
+}
+
+fn gray_image(data: &[u8]) -> Vec<u8> {
+    stream(&format!("/Type /XObject /Subtype /Image /Width {} /Height 1 /ColorSpace /DeviceGray /BitsPerComponent 8", data.len()), data)
+}
+
+/// An image Delete Image took off the page is still listed in its resources: Remove Hidden
+/// Information unlinks it, so the full rewrite leaves it out. Images a page still draws stay,
+/// directly, through a form without /Resources, or on another page sharing the dictionary.
+#[test]
+fn images_no_page_draws_leave_with_hidden_information() {
+    let bytes = hidden_information_removed(vec![
+        b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(),
+        b"<< /Type /Pages /Kids [3 0 R 4 0 R] /Count 2 >>".to_vec(),
+        b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 5 0 R /Resources 7 0 R >>".to_vec(),
+        b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 6 0 R /Resources 7 0 R >>".to_vec(),
+        stream("", b"q 50 0 0 50 20 20 cm /Shown Do Q /Fm Do"),
+        stream("", b"q 50 0 0 50 20 20 cm /Other Do Q"),
+        b"<< /XObject << /Shown 8 0 R /Deleted 9 0 R /Nested 10 0 R /Other 11 0 R /Fm 12 0 R >> >>".to_vec(),
+        gray_image(b"SHOWNIMAGEBYTES"),
+        gray_image(b"DELETEDIMAGEBYTES"),
+        gray_image(b"NESTEDIMAGEBYTES"),
+        gray_image(b"OTHERPAGEIMAGE"),
+        stream("/Type /XObject /Subtype /Form /BBox [0 0 300 300]", b"q 10 0 0 10 200 200 cm /Nested Do Q"),
+    ]);
+    let has = |needle: &[u8]| bytes.windows(needle.len()).any(|w| w == needle);
+    assert!(!has(b"DELETEDIMAGEBYTES"), "the deleted image survived the save");
+    for kept in [&b"SHOWNIMAGEBYTES"[..], b"NESTEDIMAGEBYTES", b"OTHERPAGEIMAGE"] {
+        assert!(has(kept), "{} is drawn and must stay", String::from_utf8_lossy(kept));
+    }
+}
+
+/// A name a pattern, a form, an appearance or a soft mask doesn't define is looked up in the
+/// page's resources, /Resources of its own or not: an image drawn only that way is still shown,
+/// so it stays. One no content names goes.
+#[test]
+fn images_drawn_through_the_page_resources_stay() {
+    let bytes = hidden_information_removed(vec![
+        b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(),
+        b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(),
+        b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Resources 5 0 R /Annots [6 0 R] >>".to_vec(),
+        stream("", b"q /Pattern cs /Pt scn 0 0 10 10 re f Q /Own Do /Loop Do q /Gs gs 0 0 5 5 re f Q"),
+        b"<< /XObject << /ByPattern 7 0 R /ByOwnForm 8 0 R /ByAppearance 9 0 R /ByMask 10 0 R /Deleted 11 0 R /Own 12 0 R /Loop 16 0 R >> \
+           /Pattern << /Pt 13 0 R >> /ExtGState << /Gs << /SMask << /S /Luminosity /G 14 0 R >> >> >> >>"
+            .to_vec(),
+        b"<< /Type /Annot /Subtype /Square /Rect [0 0 50 50] /AP << /N 15 0 R >> >>".to_vec(),
+        gray_image(b"BYPATTERNIMAGE"),
+        gray_image(b"BYOWNFORMIMAGE"),
+        gray_image(b"BYAPPEARANCEIMAGE"),
+        gray_image(b"BYMASKIMAGE"),
+        gray_image(b"DELETEDIMAGEBYTES"),
+        stream("/Type /XObject /Subtype /Form /BBox [0 0 10 10] /Resources << /ProcSet [/PDF] >>", b"/ByOwnForm Do"),
+        stream("/Type /Pattern /PatternType 1 /PaintType 1 /TilingType 1 /BBox [0 0 10 10] /XStep 10 /YStep 10", b"/ByPattern Do"),
+        stream("/Type /XObject /Subtype /Form /BBox [0 0 10 10] /Group << /S /Transparency /CS /DeviceGray >>", b"/ByMask Do"),
+        stream("/Type /XObject /Subtype /Form /BBox [0 0 50 50]", b"/ByAppearance Do"),
+        // Draws itself: read once.
+        stream("/Type /XObject /Subtype /Form /BBox [0 0 10 10] /Resources << /XObject << /Loop 16 0 R >> >>", b"/Loop Do"),
+    ]);
+    let has = |needle: &[u8]| bytes.windows(needle.len()).any(|w| w == needle);
+    assert!(!has(b"DELETEDIMAGEBYTES"), "the deleted image survived the save");
+    for kept in [&b"BYPATTERNIMAGE"[..], b"BYOWNFORMIMAGE", b"BYAPPEARANCEIMAGE", b"BYMASKIMAGE"] {
+        assert!(has(kept), "{} is drawn and must stay", String::from_utf8_lossy(kept));
+    }
+}
+
+/// A Type 3 glyph draws through the page's resources too: its image stays, and the dictionary
+/// is still cleaned of what nothing draws.
+#[test]
+fn images_drawn_by_type3_glyphs_stay() {
+    let bytes = hidden_information_removed(vec![
+        b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(),
+        b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(),
+        b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Resources 5 0 R >>".to_vec(),
+        stream("", b"BT /T3 12 Tf (a) Tj ET"),
+        b"<< /XObject << /ByGlyph 7 0 R /Deleted 8 0 R >> /Font << /T3 6 0 R >> >>".to_vec(),
+        b"<< /Type /Font /Subtype /Type3 /FontBBox [0 0 10 10] /FontMatrix [0.1 0 0 0.1 0 0] /CharProcs << /a 9 0 R >> \
+           /Encoding << /Differences [97 /a] >> /FirstChar 97 /LastChar 97 /Widths [10] >>"
+            .to_vec(),
+        gray_image(b"BYGLYPHIMAGE"),
+        gray_image(b"DELETEDIMAGEBYTES"),
+        stream("", b"10 0 d0 /ByGlyph Do"),
+    ]);
+    let has = |needle: &[u8]| bytes.windows(needle.len()).any(|w| w == needle);
+    assert!(!has(b"DELETEDIMAGEBYTES"), "the deleted image survived the save");
+    assert!(has(b"BYGLYPHIMAGE"), "the glyph's image is drawn and must stay");
+}