Merge #6458: Remove Hidden Information unlinks images nothing draws

# Conflicts:
#	crates/redact/src/tests.rs
This commit is contained in:
bflatastic committed 2026-11-09 08:02:23 +07:00
commit 2e380a42b5
4 files changed
+339 -29

No files matched your search

+1 -1
View File
@@ -31,7 +31,7 @@ mod prune;
use layers::strip_hidden_layers;
use prune::{clip_out, extgstate_font, prune_form_resources};
pub(crate) use prune::{prune, prune_patterns, prune_properties, resource_ops, unlink_doomed, unlink_pruned_patterns};
pub(crate) use prune::{prune, prune_patterns, prune_properties, resource_ops, unlink_doomed, unlink_pruned_patterns, unlink_undrawn};
/// How deep this pass follows form XObjects. A form past it is removed rather than walked, which
/// is the fail-safe choice for a redaction -- marked content inside a form we cannot read would
+186 -27
View File
@@ -1,6 +1,6 @@
use super::{
CosError, Dict, Document, HashMap, HashSet, MAX_DEPTH, Matrix, Metrics, ObjRef, Object, Op, Pieces, Rc, RedactError, Scope, is_vertical, num,
overlaps, page_streams, res_dict,
CosError, DecodeBudget, Dict, Document, HashMap, HashSet, MAX_DEPTH, Matrix, Metrics, ObjRef, Object, Op, Pieces, Rc, RedactError, Scope,
is_vertical, num, overlaps, page_streams, res_dict,
};
/// Drop every `replaced` resource name whose `Do` no longer appears in the rewritten `content`,
@@ -49,6 +49,189 @@ fn content_draws(doc: &Document, content: &[Vec<u8>], xo: &Dict, name: &[u8], de
Some(false)
}
/// Every page-tree node with a /Resources of its own, and the pages that draw through it (the
/// nearest such node above a page, or the page itself). Cycles are cut as in
/// `pdfcraft_model::pages`. The flag is false when a node was reached twice (listed under two
/// parents, or in a cycle) or the walk stopped early: a page may then draw through a holder it
/// isn't listed under.
fn resource_holders(doc: &Document) -> (Vec<(ObjRef, Vec<ObjRef>)>, bool) {
let mut holders: Vec<(ObjRef, Vec<ObjRef>)> = Vec::new();
let mut whole = true;
let Some(top) = doc.root().and_then(|root| doc.get(root).as_dict().and_then(|d| d.reference(b"Pages"))) else { return (holders, whole) };
let mut seen = HashSet::new();
let mut stack: Vec<(ObjRef, Option<usize>)> = vec![(top, None)];
while let Some((node, mut holder)) = stack.pop() {
if !seen.insert(node) || seen.len() > 1_000_000 {
whole = false;
continue;
}
let obj = doc.get(node);
let Some(d) = obj.as_dict() else { continue };
if d.contains(b"Resources") {
holders.push((node, Vec::new()));
holder = Some(holders.len() - 1);
}
let is_pages = d.name(b"Type") == Some(b"Pages") || (d.contains(b"Kids") && d.name(b"Type") != Some(b"Page"));
if is_pages {
let kids = d.get(b"Kids").map(|k| doc.resolve(k)).and_then(|k| k.as_array().cloned()).unwrap_or_default();
stack.extend(kids.iter().filter_map(Object::as_ref).map(|k| (k, holder)));
} else if let Some(h) = holder.and_then(|i| holders.get_mut(i)) {
h.1.push(node);
}
}
(holders, whole)
}
/// Unlink the image and form XObjects nothing draws from the page-tree resource dictionaries
/// that still name them: Delete Image and Replace Image take an image off the page but leave it
/// listed there, and so in a full-rewrite save. A name stays when anything that draws through the
/// dictionary uses it ([`names_drawn`]), and a dictionary whose use can't be told is left as it
/// is, as is every one when a page is listed twice in the page tree. A dictionary that loses an
/// entry is replaced by a private copy on its node, as in [`unlink_doomed`]. Returns how many
/// names went.
pub(crate) fn unlink_undrawn(doc: &mut Document) -> Result<usize, RedactError> {
let index: HashMap<ObjRef, usize> = pdfcraft_model::pages(doc).iter().enumerate().map(|(i, p)| (p.obj, i)).collect();
let (holders, whole) = resource_holders(doc);
if !whole {
return Ok(0);
}
let mut n = 0;
for (node, users) in holders {
let Some(mut res) = doc.get(node).as_dict().and_then(|d| d.get(b"Resources")).and_then(|r| doc.resolve(r).as_dict().cloned()) else {
continue;
};
let mut xo = res_dict(doc, &res, b"XObject");
let listed: Vec<Vec<u8>> = xo
.iter()
.filter(|(_, v)| {
v.as_ref().is_some_and(|r| matches!(&*doc.get(r), Object::Stream(s) if matches!(s.dict.name(b"Subtype"), Some(b"Image" | b"Form"))))
})
.map(|(name, _)| name.to_vec())
.collect();
if listed.is_empty() {
continue;
}
let mut pruned = false;
// A form that goes may have been all that drew another name: look again while names go.
for _ in 0..MAX_DEPTH {
let Some(drawn) = names_drawn(doc, &res, &users, &index) else { break };
let mut gone = 0;
for name in listed.iter().filter(|name| !drawn.contains(*name)) {
gone += usize::from(xo.remove(name).is_some());
}
if gone == 0 {
break;
}
n += gone;
pruned = true;
res.set(b"XObject".to_vec(), Object::Dict(xo.clone()));
}
if pruned {
doc.update_dict(node, |d| d.set(b"Resources".to_vec(), Object::Dict(res)))?;
}
}
Ok(n)
}
/// Every name used with `Do` by what draws through `res`, the resource dictionary of `pages`:
/// their content, their annotations' appearances, and every form, tiling pattern, Type 3 glyph
/// and soft mask those can reach, with /Resources of their own or not. A name such a stream
/// doesn't define is looked up in the resources of whatever draws it, up to the page's, so a `Do`
/// anywhere in there may draw from `res`. Going by name keeps more than it must, never less.
/// Each stream is read once, against one budget. `None` when that can't be told: a page or
/// stream that won't decode, or the budget spent.
fn names_drawn(doc: &Document, res: &Dict, pages: &[ObjRef], index: &HashMap<ObjRef, usize>) -> Option<HashSet<Vec<u8>>> {
let mut drawn = HashSet::new();
let mut note = |data: &[u8]| {
for op in pdfcraft_content::parse(data).ops.iter().filter(|op| op.is("Do")) {
if let Some(name) = op.last_name() {
drawn.insert(name.to_vec());
}
}
};
// Streams still to read, and resource dictionaries still to look through.
let mut streams: Vec<ObjRef> = Vec::new();
let mut dicts: Vec<Dict> = vec![res.clone()];
for page in pages {
let dict = doc.get(*page).as_dict().cloned().unwrap_or_default();
let (_, content) = page_streams(doc, &dict, index.get(page).copied().unwrap_or(0)).ok()?;
// One stream may end with a name its `Do` begins the next with: read them as one.
note(Pieces::join(&content).data());
let annots = dict.get(b"Annots").map(|a| doc.resolve(a)).and_then(|a| a.as_array().cloned()).unwrap_or_default();
for annot in annots {
let ap = doc.resolve(&annot).as_dict().and_then(|a| a.get(b"AP")).map(|ap| doc.resolve(ap)).and_then(|ap| ap.as_dict().cloned());
// Each of /N, /R and /D is an appearance, or a dictionary of them by state.
for (_, look) in ap.unwrap_or_default().iter() {
match look.as_ref() {
Some(r) if matches!(&*doc.resolve(look), Object::Stream(_)) => streams.push(r),
_ => streams.extend(doc.resolve(look).as_dict().into_iter().flat_map(|states| states.iter().filter_map(|(_, v)| v.as_ref()))),
}
}
}
}
let mut budget = DecodeBudget::page();
let mut read = HashSet::new();
let mut looked = 0;
loop {
while let Some(dict) = dicts.get(looked) {
let (found, fonts) = drawn_through(doc, dict);
looked += 1;
streams.extend(found);
// Type 3 fonts' own resources, each look charged like a form so fonts naming each
// other can't go round for ever.
for font_res in fonts {
if !budget.enter_form() {
return None;
}
dicts.push(font_res);
}
}
let Some(r) = streams.pop() else { break };
if !read.insert(r) {
continue;
}
let obj = doc.resolve(&Object::Ref(r));
let Object::Stream(s) = &*obj else { continue };
if !budget.enter_form() {
return None;
}
note(&budget.decode_strict(s).ok()?);
if let Some(own) = s.dict.get(b"Resources").map(|o| doc.resolve(o)).and_then(|o| o.as_dict().cloned()) {
dicts.push(own);
}
}
Some(drawn)
}
/// The content streams the resource dictionary `res` lists: its form XObjects, its tiling
/// patterns, its Type 3 fonts' glyph procedures and its soft masks' groups. Then the /Resources
/// of those Type 3 fonts, which their glyphs look names up in first. An object that holds only a
/// reference stands for what it names, as the renderer reads it.
fn drawn_through(doc: &Document, res: &Dict) -> (Vec<ObjRef>, Vec<Dict>) {
let is_stream = |r: &ObjRef| matches!(&*doc.resolve(&Object::Ref(*r)), Object::Stream(_));
let mut found: Vec<ObjRef> = Vec::new();
let mut fonts: Vec<Dict> = Vec::new();
for (_, x) in res_dict(doc, res, b"XObject").iter() {
found.extend(x.as_ref().filter(|_| matches!(&*doc.resolve(x), Object::Stream(s) if s.dict.name(b"Subtype") == Some(b"Form"))));
}
for (_, p) in res_dict(doc, res, b"Pattern").iter() {
found.extend(p.as_ref().filter(|_| matches!(&*doc.resolve(p), Object::Stream(s) if s.dict.int(b"PatternType") == Some(1))));
}
for (_, f) in res_dict(doc, res, b"Font").iter() {
let font = doc.resolve(f);
let Some(font) = font.as_dict().filter(|f| f.name(b"Subtype") == Some(b"Type3")) else { continue };
let procs = font.get(b"CharProcs").map(|c| doc.resolve(c)).and_then(|c| c.as_dict().cloned()).unwrap_or_default();
found.extend(procs.iter().filter_map(|(_, v)| v.as_ref()).filter(is_stream));
fonts.extend(font.get(b"Resources").map(|r| doc.resolve(r)).and_then(|r| r.as_dict().cloned()));
}
for (_, g) in res_dict(doc, res, b"ExtGState").iter() {
let gs = doc.resolve(g);
let mask = gs.as_dict().and_then(|g| g.get(b"SMask")).map(|m| doc.resolve(m));
found.extend(mask.as_ref().and_then(|m| m.as_dict()).and_then(|m| m.reference(b"G")).filter(is_stream));
}
(found, fonts)
}
/// Unlink the XObjects a rewrite dropped or replaced (`doomed`) from every page-tree resource
/// dictionary that still names them. The rewritten page has its own pruned /Resources, but the
/// originals stay reachable, and so stay in a full-rewrite save, when:
@@ -67,31 +250,7 @@ pub(crate) fn unlink_doomed(doc: &mut Document, doomed: &[ObjRef], retired_prope
return Ok(());
}
let doomed: HashSet<ObjRef> = doomed.iter().copied().collect();
let Some(top) = doc.root().and_then(|root| doc.get(root).as_dict().and_then(|d| d.reference(b"Pages"))) else { return Ok(()) };
// Every page-tree node with a /Resources of its own, and the pages that draw through it
// (the nearest such node above a page, or the page itself). Cycles are cut as in
// `pdfcraft_model::pages`.
let mut holders: Vec<(ObjRef, Vec<ObjRef>)> = Vec::new();
let mut seen = HashSet::new();
let mut stack: Vec<(ObjRef, Option<usize>)> = vec![(top, None)];
while let Some((node, mut holder)) = stack.pop() {
if !seen.insert(node) || seen.len() > 1_000_000 {
continue;
}
let obj = doc.get(node);
let Some(d) = obj.as_dict() else { continue };
if d.contains(b"Resources") {
holders.push((node, Vec::new()));
holder = Some(holders.len() - 1);
}
let is_pages = d.name(b"Type") == Some(b"Pages") || (d.contains(b"Kids") && d.name(b"Type") != Some(b"Page"));
if is_pages {
let kids = d.get(b"Kids").map(|k| doc.resolve(k)).and_then(|k| k.as_array().cloned()).unwrap_or_default();
stack.extend(kids.iter().filter_map(Object::as_ref).map(|k| (k, holder)));
} else if let Some(h) = holder.and_then(|i| holders.get_mut(i)) {
h.1.push(node);
}
}
let (holders, _) = resource_holders(doc);
let index: HashMap<ObjRef, usize> = pdfcraft_model::pages(doc).iter().enumerate().map(|(i, p)| (p.obj, i)).collect();
let holder_nodes: Vec<ObjRef> = holders.iter().map(|(node, _)| *node).collect();
for (node, users) in holders {
+4 -1
View File
@@ -790,7 +790,10 @@ pub fn remove_hidden(doc: &mut Document, which: &[Hidden]) -> Result<Vec<(Hidden
doc.set(r, o);
}
}
if done.is_empty() {
// Images taken off their page (Delete Image, Replace Image) are still listed in its
// resources: the full rewrite would keep them. Unlinking them is a removal of its own.
let unlinked = crate::interp::unlink_undrawn(doc)?;
if done.is_empty() && unlinked == 0 {
return Err(RedactError::NothingToApply);
}
// The rewrite drops the revisions that held the removed content, so it is a different file from
+148
View File
@@ -4606,3 +4606,151 @@ fn private_data_takes_opi_links_and_alternate_images() {
assert!(!has(b"ALTERNATEORIGINAL"), "the alternate image survived the save");
assert!(!has(b"OPISERVERPATH"), "the OPI link survived the save");
}
/// The full rewrite of the document made of `objs` after Remove Hidden Information, run for its
/// metadata so it has something to do.
fn hidden_information_removed(objs: Vec<Vec<u8>>) -> Vec<u8> {
use crate::sanitize::{Hidden, remove_hidden};
let mut doc = pdf(objs);
let info = doc.add(Object::Dict({
let mut d = Dict::new();
d.set(b"Title".to_vec(), pdfcraft_cos::PdfString::text("Plan"));
d
}));
doc.trailer_mut().set(b"Info".to_vec(), Object::Ref(info));
remove_hidden(&mut doc, &[Hidden::Metadata]).unwrap();
write_full(&doc, &SaveOptions { object_streams: false, ..SaveOptions::default() }).unwrap()
}
fn gray_image(data: &[u8]) -> Vec<u8> {
stream(&format!("/Type /XObject /Subtype /Image /Width {} /Height 1 /ColorSpace /DeviceGray /BitsPerComponent 8", data.len()), data)
}
/// An image Delete Image took off the page is still listed in its resources: Remove Hidden
/// Information unlinks it, so the full rewrite leaves it out, as it does one only an unused form
/// draws. Images a page still draws stay, directly, through a form without /Resources, or on
/// another page sharing the dictionary, its name and `Do` in two content streams.
#[test]
fn images_no_page_draws_leave_with_hidden_information() {
let bytes = hidden_information_removed(vec![
b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(),
b"<< /Type /Pages /Kids [3 0 R 4 0 R] /Count 2 >>".to_vec(),
b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 5 0 R /Resources 7 0 R >>".to_vec(),
b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents [6 0 R 15 0 R] /Resources 7 0 R >>".to_vec(),
stream("", b"q 50 0 0 50 20 20 cm /Shown Do Q /Fm Do"),
stream("", b"q 50 0 0 50 20 20 cm /Other"),
b"<< /XObject << /Shown 8 0 R /Deleted 9 0 R /Nested 10 0 R /Other 11 0 R /Fm 12 0 R /Unused 13 0 R /Orphan 14 0 R >> >>".to_vec(),
gray_image(b"SHOWNIMAGEBYTES"),
gray_image(b"DELETEDIMAGEBYTES"),
gray_image(b"NESTEDIMAGEBYTES"),
gray_image(b"OTHERPAGEIMAGE"),
stream("/Type /XObject /Subtype /Form /BBox [0 0 300 300]", b"q 10 0 0 10 200 200 cm /Nested Do Q"),
stream("/Type /XObject /Subtype /Form /BBox [0 0 300 300]", b"/Orphan Do"),
gray_image(b"ORPHANIMAGEBYTES"),
stream("", b"Do Q"),
]);
let has = |needle: &[u8]| bytes.windows(needle.len()).any(|w| w == needle);
assert!(!has(b"DELETEDIMAGEBYTES"), "the deleted image survived the save");
assert!(!has(b"ORPHANIMAGEBYTES"), "an image only an unused form draws survived the save");
for kept in [&b"SHOWNIMAGEBYTES"[..], b"NESTEDIMAGEBYTES", b"OTHERPAGEIMAGE"] {
assert!(has(kept), "{} is drawn and must stay", String::from_utf8_lossy(kept));
}
}
/// A page listed under two /Pages nodes draws through either one's resources, whichever a reader
/// comes to it from: neither loses the image it draws.
#[test]
fn a_page_listed_twice_keeps_both_parents_images() {
let bytes = hidden_information_removed(vec![
b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(),
b"<< /Type /Pages /Kids [3 0 R 4 0 R] /Count 2 >>".to_vec(),
b"<< /Type /Pages /Parent 2 0 R /Kids [5 0 R] /Count 1 /Resources << /XObject << /Im 7 0 R >> >> >>".to_vec(),
b"<< /Type /Pages /Parent 2 0 R /Kids [5 0 R] /Count 1 /Resources << /XObject << /Im 8 0 R >> >> >>".to_vec(),
b"<< /Type /Page /Parent 3 0 R /MediaBox [0 0 300 300] /Contents 6 0 R >>".to_vec(),
stream("", b"q 50 0 0 50 20 20 cm /Im Do Q"),
gray_image(b"FIRSTPARENTIMAGE"),
gray_image(b"SECONDPARENTIMAGE"),
]);
let has = |needle: &[u8]| bytes.windows(needle.len()).any(|w| w == needle);
for kept in [&b"FIRSTPARENTIMAGE"[..], b"SECONDPARENTIMAGE"] {
assert!(has(kept), "{} is drawn and must stay", String::from_utf8_lossy(kept));
}
}
/// Unlinking an image nothing draws is a removal of its own: with nothing else to remove, it is
/// still applied, not reported as nothing to do.
#[test]
fn unlinking_alone_still_applies() {
use crate::sanitize::{Hidden, remove_hidden};
let mut doc = pdf(vec![
b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(),
b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(),
b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Resources << /XObject << /Deleted 4 0 R >> >> >>".to_vec(),
gray_image(b"DELETEDIMAGEBYTES"),
]);
assert_eq!(remove_hidden(&mut doc, &[Hidden::Metadata]), Ok(Vec::new()));
let bytes = write_full(&doc, &SaveOptions { object_streams: false, ..SaveOptions::default() }).unwrap();
assert!(!bytes.windows(17).any(|w| w == b"DELETEDIMAGEBYTES"), "the deleted image survived the save");
}
/// A name a pattern, a form, an appearance or a soft mask doesn't define is looked up in the
/// page's resources, /Resources of its own or not: an image drawn only that way is still shown,
/// so it stays, as one drawn by a form named through an object that holds only a reference. One
/// no content names goes.
#[test]
fn images_drawn_through_the_page_resources_stay() {
let bytes = hidden_information_removed(vec![
b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(),
b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(),
b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Resources 5 0 R /Annots [6 0 R] >>".to_vec(),
stream("", b"q /Pattern cs /Pt scn 0 0 10 10 re f Q /Own Do /Loop Do q /Gs gs 0 0 5 5 re f Q /Chained Do"),
b"<< /XObject << /ByPattern 7 0 R /ByOwnForm 8 0 R /ByAppearance 9 0 R /ByMask 10 0 R /Deleted 11 0 R /Own 12 0 R /Loop 16 0 R \
/Chained 17 0 R /ByChain 19 0 R >> /Pattern << /Pt 13 0 R >> /ExtGState << /Gs << /SMask << /S /Luminosity /G 14 0 R >> >> >> >>"
.to_vec(),
b"<< /Type /Annot /Subtype /Square /Rect [0 0 50 50] /AP << /N 15 0 R >> >>".to_vec(),
gray_image(b"BYPATTERNIMAGE"),
gray_image(b"BYOWNFORMIMAGE"),
gray_image(b"BYAPPEARANCEIMAGE"),
gray_image(b"BYMASKIMAGE"),
gray_image(b"DELETEDIMAGEBYTES"),
stream("/Type /XObject /Subtype /Form /BBox [0 0 10 10] /Resources << /ProcSet [/PDF] >>", b"/ByOwnForm Do"),
stream("/Type /Pattern /PatternType 1 /PaintType 1 /TilingType 1 /BBox [0 0 10 10] /XStep 10 /YStep 10", b"/ByPattern Do"),
stream("/Type /XObject /Subtype /Form /BBox [0 0 10 10] /Group << /S /Transparency /CS /DeviceGray >>", b"/ByMask Do"),
stream("/Type /XObject /Subtype /Form /BBox [0 0 50 50]", b"/ByAppearance Do"),
// Draws itself: read once.
stream("/Type /XObject /Subtype /Form /BBox [0 0 10 10] /Resources << /XObject << /Loop 16 0 R >> >>", b"/Loop Do"),
b"18 0 R".to_vec(),
stream("/Type /XObject /Subtype /Form /BBox [0 0 10 10]", b"/ByChain Do"),
gray_image(b"BYCHAINIMAGE"),
]);
let has = |needle: &[u8]| bytes.windows(needle.len()).any(|w| w == needle);
assert!(!has(b"DELETEDIMAGEBYTES"), "the deleted image survived the save");
for kept in [&b"BYPATTERNIMAGE"[..], b"BYOWNFORMIMAGE", b"BYAPPEARANCEIMAGE", b"BYMASKIMAGE", b"BYCHAINIMAGE"] {
assert!(has(kept), "{} is drawn and must stay", String::from_utf8_lossy(kept));
}
}
/// A Type 3 glyph draws through the page's resources too, as does a form in the font's own
/// resources: their images stay, and the dictionary is still cleaned of what nothing draws.
#[test]
fn images_drawn_by_type3_glyphs_stay() {
let bytes = hidden_information_removed(vec![
b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(),
b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(),
b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Resources 5 0 R >>".to_vec(),
stream("", b"BT /T3 12 Tf (a) Tj ET"),
b"<< /XObject << /ByGlyph 7 0 R /Deleted 8 0 R /ByFontForm 11 0 R >> /Font << /T3 6 0 R >> >>".to_vec(),
b"<< /Type /Font /Subtype /Type3 /FontBBox [0 0 10 10] /FontMatrix [0.1 0 0 0.1 0 0] /CharProcs << /a 9 0 R >> \
/Encoding << /Differences [97 /a] >> /FirstChar 97 /LastChar 97 /Widths [10] /Resources << /XObject << /Fx 10 0 R >> >> >>"
.to_vec(),
gray_image(b"BYGLYPHIMAGE"),
gray_image(b"DELETEDIMAGEBYTES"),
stream("", b"10 0 d0 /ByGlyph Do /Fx Do"),
stream("/Type /XObject /Subtype /Form /BBox [0 0 10 10]", b"/ByFontForm Do"),
gray_image(b"BYFONTFORMIMAGE"),
]);
let has = |needle: &[u8]| bytes.windows(needle.len()).any(|w| w == needle);
assert!(!has(b"DELETEDIMAGEBYTES"), "the deleted image survived the save");
assert!(has(b"BYGLYPHIMAGE"), "the glyph's image is drawn and must stay");
assert!(has(b"BYFONTFORMIMAGE"), "the image a form in the font's resources draws must stay");
}