From 7f9554f29b704a4f011673504e069c715d65fb60 Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Fri, 30 Oct 2026 18:36:52 +0100 Subject: [PATCH 1/5] M8: a cleared black-and-white scan is stored as Group 4, or JBIG2 again, when smaller Redaction wrote every cleared copy as Flate, so a redacted Group 4 or JBIG2 scan page grew several times over until the optimizer ran. A cleared one-bit image of one component is now stored as the smallest of Flate, CCITT Group 4 and, when it was JBIG2, JBIG2 (PDF 1.4, which such a file already is), as stored, dictionary included: a tiny image can lose by Group 4's parameters. Both are coded from the cleared samples, so nothing of the original's code is carried over; rows Group 4 or JBIG2 can't code (too wide) leave Flate. Images of more bits stay Flate. Tests: a cleared one-bit Flate scan of ragged word blocks (where Group 4 is smaller than Flate) comes out Group 4 and an eight-bit image Flate, each decoding to the cleared samples (the whole buffer); a tiny image whose Group 4 code is shorter but whose stream isn't, and rows wider than Group 4 codes, stay Flate and clear; the redacted JBIG2 scan of #5461's test comes out JBIG2, smaller than Flate, with no globals and no stream holding the original code. The tests' image helpers decode through Document::decoded_image, as image codecs need. Co-Authored-By: Claude Opus 5.5 --- crates/redact/Cargo.toml | 3 +- crates/redact/README.md | 6 +- crates/redact/src/image.rs | 41 +++++++++++- crates/redact/src/tests.rs | 117 +++++++++++++++++++++++++++++++++-- parity/acrobat-features.toml | 4 +- 5 files changed, 157 insertions(+), 14 deletions(-) diff --git a/crates/redact/Cargo.toml b/crates/redact/Cargo.toml index 8d1515d8a4..fe21d13b32 100644 --- a/crates/redact/Cargo.toml +++ b/crates/redact/Cargo.toml @@ -17,8 +17,9 @@ pdfcraft-forms = { workspace = true } pdfcraft-edit = { workspace = true } pdfcraft-annot = { workspace = true } pdfcraft-xfa = { workspace = true } +# CCITT Group 4 and JBIG2 encoding for cleared one-bit images. +pdfcraft-filters = { workspace = true } thiserror = { workspace = true } [dev-dependencies] -pdfcraft-filters = { workspace = true } pdfcraft-testkit = { workspace = true } diff --git a/crates/redact/README.md b/crates/redact/README.md index af43e958d8..e52c1d8b98 100644 --- a/crates/redact/README.md +++ b/crates/redact/README.md @@ -21,7 +21,8 @@ clear_marks(&mut doc, None)?; // remove marks without app (8/16-bit and 1/2/4-bit images, image masks; Flate/LZW/RL/A85/AHx, and CCITT fax or JBIG2 as the last filter, decoded through `Document::decoded_image_within`, a JBIG2 image with its `/JBIG2Globals`; `/DecodeParms` given by reference, numbers inside included, count). The copy - is Flate, and refers to no globals: when no other image uses them, they leave the file with the + is Flate, or for a one-bit image of one component (a scan as a rule) CCITT Group 4, or JBIG2 + when it was JBIG2, whichever is smallest as stored, coded from the cleared samples; it refers to no globals: when no other image uses them, they leave the file with the original on the full rewrite; when one does, they stay as they are, and may hold the shapes of marks under the area (not where they were). Codecs redaction doesn't decode (DCT, JPX) are removed whole (fail-closed), and so is a JBIG2 image whose page isn't its size (the renderer @@ -109,5 +110,4 @@ Patterns are heuristic: dates, protocol identifiers and longer number runs are r Custom code sets aren't supported yet. Not yet: pattern locales, struct-tree/alt-text cleanup, metadata and hidden-information -sanitising (M8.4), re-encoding of DCT images (they are removed instead), and storing a cleared black-and-white -image as CCITT Group 4 or JBIG2 again (it is Flate, larger; the optimizer makes it smaller). +sanitising (M8.4), re-encoding of DCT images (they are removed instead). diff --git a/crates/redact/src/image.rs b/crates/redact/src/image.rs index 6c7a6bf93b..e9606c5e30 100644 --- a/crates/redact/src/image.rs +++ b/crates/redact/src/image.rs @@ -1,13 +1,14 @@ //! Clearing the pixels of an image under redaction regions. The image is decoded, every pixel //! whose footprint intersects a region is set to zero (no ink for image masks), and the result is -//! written as a new Flate image; the original object is untouched (other pages may use it). +//! written as a new image, Flate, or for a one-bit image of one component Group 4, or JBIG2 when it +//! was JBIG2, whichever is smallest; the original object is untouched (other pages may use it). //! CCITT fax and JBIG2 images decode through `Document::decoded_image_within`, but for Group 3 on //! bytes, which PdfCraft's renderer may draw as other rows, and a JBIG2 page of other rows than the //! image's. Images whose codec PdfCraft can't decode (DCT, JPX), and those, return `None`, and the //! caller removes the whole image instead; that is fail-closed. use pdfcraft_content::{Matrix, overlaps}; -use pdfcraft_cos::{Document, Object, Stream}; +use pdfcraft_cos::{Dict, Document, Object, Stream}; /// Most pixels an image may have to be cleared in place (larger ones are removed). const MAX_PIXELS: u64 = 64 << 20; @@ -165,5 +166,39 @@ fn decode_and_clear(doc: &Document, s: &Stream, ctm: &Matrix, rects: &[[f64; 4]] dict.remove(b"Length"); dict.remove(b"DecodeParms"); dict.remove(b"Alternates"); - Some(Some(Stream::flate(dict, &data))) + Some(Some(smallest(dict, &data, w, h, ncomp == 1 && bpc == 1, is_jbig2))) +} + +/// The cleared samples `data` of a `w` × `h` image as a stream with `dict`: Flate, or, for one bit +/// of one component (`bilevel`, a scan as a rule, which Flate stores several times larger), CCITT +/// Group 4, or JBIG2 when the image was JBIG2 (`jbig2`; JBIG2 is PDF 1.4, which such a file already +/// is), whichever is smallest as stored, dictionary included (a tiny image can lose by Group 4's +/// parameters). Both decode to `data`, cleared before it is coded, so nothing of the original's +/// code is carried over; Group 4 or JBIG2 that can't code the rows (too wide) is left out. +fn smallest(dict: Dict, data: &[u8], w: usize, h: usize, bilevel: bool, jbig2: bool) -> Stream { + let flate = Stream::flate(dict.clone(), data); + let Some((columns, rows)) = u32::try_from(w).ok().zip(u32::try_from(h).ok()).filter(|_| bilevel) else { return flate }; + let g4 = pdfcraft_filters::encode_g4(data, columns, rows).ok().map(|coded| { + let mut dict = dict.clone(); + let mut parms = Dict::new(); + parms.set(b"K".to_vec(), Object::Int(-1)); + parms.set(b"Columns".to_vec(), Object::Int(i64::from(columns))); + parms.set(b"Rows".to_vec(), Object::Int(i64::from(rows))); + dict.set(b"Filter".to_vec(), Object::name("CCITTFaxDecode")); + dict.set(b"DecodeParms".to_vec(), Object::Dict(parms)); + Stream::from_raw(dict, coded) + }); + let jbig = jbig2.then(|| pdfcraft_filters::encode_jbig2(data, columns, rows).ok()).flatten().map(|coded| { + let mut dict = dict.clone(); + dict.set(b"Filter".to_vec(), Object::name("JBIG2Decode")); + Stream::from_raw(dict, coded) + }); + [g4, jbig].into_iter().flatten().fold(flate, |best, s| if stored_len(&s) < stored_len(&best) { s } else { best }) +} + +/// The bytes stream `s` takes in a file: its data and its dictionary as written. +fn stored_len(s: &Stream) -> usize { + let mut dict = Vec::new(); + pdfcraft_cos::serialize(&Object::Dict(s.dict.clone()), &mut dict); + dict.len().saturating_add(s.raw.len()) } diff --git a/crates/redact/src/tests.rs b/crates/redact/src/tests.rs index 08faef69a7..2e76bb0bac 100644 --- a/crates/redact/src/tests.rs +++ b/crates/redact/src/tests.rs @@ -519,13 +519,19 @@ fn two_pages_one_resource_dictionary(inherited: bool, again: bool) -> Document { ]) } -/// The samples of the image a page's (possibly inherited) resources name `name`. +/// The samples of the image a page's (possibly inherited) resources name `name`, decoded through +/// its image codec too (`Document::decoded_image`). fn named_image(doc: &Document, page: usize, name: &[u8]) -> Option> { + named_stream(doc, page, name).and_then(|s| doc.decoded_image(&s).ok()) +} + +/// The image a page's (possibly inherited) resources name `name`. +fn named_stream(doc: &Document, page: usize, name: &[u8]) -> Option { let p = pdfcraft_model::pages(doc).swap_remove(page); let res = doc.resolve(p.dict.get(b"Resources")?).as_dict().cloned()?; let xo = doc.resolve(res.get(b"XObject")?).as_dict().cloned()?; match &*doc.get(xo.get(name)?.as_ref()?) { - Object::Stream(s) => s.decoded().ok(), + Object::Stream(s) => Some(s.clone()), _ => None, } } @@ -833,7 +839,8 @@ fn a_jbig2_image_that_cannot_be_cleared_for_certain_is_removed_whole() { /// A scan stored as JBIG2 symbols, with globals, keeps every pixel outside the area once it is /// redacted and saved, and the pixels under it are cleared; neither its JBIG2 code nor its globals, -/// which no other image uses, stay in the file. +/// which no other image uses, stay in the file. The cleared copy is coded afresh as JBIG2, smaller +/// than Flate, and refers to no globals. #[test] fn a_redacted_jbig2_scan_keeps_neither_its_code_nor_its_globals() { const LETTERS: [[&str; 7]; 3] = [ @@ -854,6 +861,7 @@ fn a_redacted_jbig2_scan_keeps_neither_its_code_nor_its_globals() { } } let coded = pdfcraft_filters::encode_jbig2(&bits, w as u32, h as u32).unwrap(); + let original = coded.clone(); assert!(coded.len() < pdfcraft_filters::encode_jbig2_generic(&bits, w as u32, h as u32).unwrap().len(), "symbol coded"); let dict = format!( "/Type /XObject /Subtype /Image /Width {w} /Height {h} /BitsPerComponent 1 /ColorSpace /DeviceGray /Filter /JBIG2Decode /DecodeParms << /JBIG2Globals 8 0 R >>" @@ -877,13 +885,111 @@ fn a_redacted_jbig2_scan_keeps_neither_its_code_nor_its_globals() { let saved = Document::open(Arc::new(write_full(&doc, &SaveOptions::default()).unwrap())).unwrap(); assert_eq!(named_image(&saved, 0, b"PCRedacted1"), Some(cleared)); assert_no_secret(&doc, b"LEAK_GLOBALS"); + let copy = named_stream(&saved, 0, b"PCRedacted1").unwrap(); + assert_eq!(copy.dict.name(b"Filter"), Some(&b"JBIG2Decode"[..])); + assert!(!copy.dict.contains(b"DecodeParms"), "the copy refers to no globals"); + let flate = pdfcraft_filters::encode_flate(&named_image(&saved, 0, b"PCRedacted1").unwrap()); + assert!(copy.raw.len() < flate.len(), "JBIG2 {} bytes, Flate {}", copy.raw.len(), flate.len()); for (r, object) in saved.scan_objects() { if let Object::Stream(s) = &*object { - assert_ne!(s.dict.name(b"Filter"), Some(&b"JBIG2Decode"[..]), "the JBIG2 image stays as {r:?}"); + assert_ne!(&s.raw[..], &original[..], "the original JBIG2 code stays as {r:?}"); } } } +/// A cleared one-bit image of one component, a scan as a rule, is stored as CCITT Group 4 when +/// that is smaller than Flate, whatever it was stored as (Flate here), and never as JBIG2 unless it +/// was JBIG2; an eight-bit image stays Flate. The copy decodes to the cleared samples. +#[test] +fn a_cleared_black_and_white_scan_is_stored_as_group_4() { + // 800 × 200: lines of word-sized dark blocks 14 rows tall, their edges ragged by a sample from + // row to row, as a scan's are; Group 4 codes such rows from the row above. + let (w, h) = (800usize, 200usize); + let stride = w / 8; + let mut seed = 0x9E37_79B9_7F4A_7C15u64; + let mut next = move || { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + seed + }; + let mut bits = vec![0xFFu8; stride * h]; + for top in (3..h - 14).step_by(20) { + let mut x = 4; + while x + 70 < w { + let len = 20 + (next() % 40) as usize; + for y in top..top + 14 { + let (left, right) = (x + (next() % 2) as usize, x + len - (next() % 2) as usize); + for xx in left..right { + bits[y * stride + xx / 8] &= !(0x80 >> (xx % 8)); + } + } + x += len + 6 + (next() % 6) as usize; + } + } + let g4 = pdfcraft_filters::encode_g4(&bits, w as u32, h as u32).unwrap(); + assert!(g4.len() < pdfcraft_filters::encode_flate(&bits).len(), "Group 4 is smaller than Flate here"); + let grey: Vec = (0..w * h).map(|i| (i % 251) as u8).collect(); + for (dict, data, filter) in [ + ( + format!("/Type /XObject /Subtype /Image /Width {w} /Height {h} /BitsPerComponent 1 /ColorSpace /DeviceGray /Filter /FlateDecode"), + pdfcraft_filters::encode_flate(&bits), + &b"CCITTFaxDecode"[..], + ), + ( + format!("/Type /XObject /Subtype /Image /Width {w} /Height {h} /BitsPerComponent 8 /ColorSpace /DeviceGray /Filter /FlateDecode"), + pdfcraft_filters::encode_flate(&grey), + &b"FlateDecode"[..], + ), + ] { + let mut doc = one_page(b"q 200 0 0 50 0 100 cm /Im Do Q", "/XObject << /Im 7 0 R >>", vec![stream(&dict, data)]); + // The left quarter of the top half. + mark(&mut doc, 0, &[[0.0, 125.0, 50.0, 150.0]], ""); + assert_eq!(apply(&mut doc, None).unwrap().images_cleared, 1, "{dict}"); + let saved = Document::open(Arc::new(write_full(&doc, &SaveOptions::default()).unwrap())).unwrap(); + let copy = named_stream(&saved, 0, b"PCRedacted1").unwrap(); + assert_eq!(copy.dict.name(b"Filter"), Some(filter), "{dict}"); + // Covered: rows 0 to 99 (y 150 down to 125, a quarter point a row), samples 0 to 199. + let mut expected = if filter == b"FlateDecode" { grey.clone() } else { bits.clone() }; + for y in 0..100 { + for x in 0..200 { + if filter == b"FlateDecode" { + expected[y * w + x] = 0; + } else { + expected[y * stride + x / 8] &= !(0x80 >> (x % 8)); + } + } + } + assert_eq!(named_image(&saved, 0, b"PCRedacted1").unwrap(), expected, "{dict}"); + } +} + +/// A cleared one-bit image is stored as the smallest stream, its dictionary counted: a tiny one, +/// whose Group 4 code is shorter than Flate's but whose `/DecodeParms` aren't, stays Flate. Rows +/// wider than Group 4 codes (`MAX_G4_COLUMNS`) stay Flate too, and still clear. +#[test] +fn a_cleared_image_group_4_wouldnt_shrink_stays_flate() { + let wide = pdfcraft_filters::MAX_G4_COLUMNS as usize + 1; + for (w, h) in [(3usize, 2usize), (wide, 2)] { + let stride = w.div_ceil(8); + let dict = format!("/Type /XObject /Subtype /Image /Width {w} /Height {h} /BitsPerComponent 1 /ColorSpace /DeviceGray /Filter /FlateDecode"); + let mut doc = one_page( + b"q 200 0 0 20 0 100 cm /Im Do Q", + "/XObject << /Im 7 0 R >>", + vec![stream(&dict, pdfcraft_filters::encode_flate(&vec![0xFF; stride * h]))], + ); + // The top row's left half. + mark(&mut doc, 0, &[[0.0, 112.0, 99.0, 118.0]], ""); + assert_eq!(apply(&mut doc, None).unwrap().images_cleared, 1, "{w} × {h}"); + let saved = Document::open(Arc::new(write_full(&doc, &SaveOptions::default()).unwrap())).unwrap(); + assert_eq!(named_stream(&saved, 0, b"PCRedacted1").unwrap().dict.name(b"Filter"), Some(&b"FlateDecode"[..]), "{w} × {h}"); + let samples = named_image(&saved, 0, b"PCRedacted1").unwrap(); + let sample = |x: usize| samples.get(x / 8).is_some_and(|b| b & (0x80 >> (x % 8)) != 0); + assert!(!sample(0) && sample(w - 1), "{w} × {h}: the top row's first sample is cleared and its last kept"); + assert!(samples.get(stride..).is_some_and(|row| row.iter().all(|b| *b == 0xFF)), "{w} × {h}: the bottom row is kept"); + } +} + /// Apply Redactions gives the page its own pruned /Resources, but the dictionary it drew through /// before (inherited from the /Pages node, or shared with another page) still named the removed /// image and form, so a full-rewrite save kept them. @@ -2525,7 +2631,8 @@ fn assert_no_secret(doc: &Document, secret: &[u8]) { pdfcraft_cos::serialize(&object, &mut bytes); assert!(!bytes.windows(secret.len()).any(|w| w == secret), "secret remains in object {r:?}"); if let Object::Stream(s) = &*object { - let data = s.decoded().unwrap(); + // Image codecs, a cleared scan coded as Group 4 or JBIG2, decode through the document. + let data = s.decoded().or_else(|_| saved.decoded_image(s)).unwrap(); assert!(!data.windows(secret.len()).any(|w| w == secret), "secret remains in stream {r:?}"); } } diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index 4878eac627..de1d9d9e38 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -5804,8 +5804,8 @@ milestone = "M8" status = "shipped" commands = ["redact.apply"] tools = ["redact_apply"] -evidence = ["crates/redact/src/tests.rs::images_are_removed_or_have_their_pixels_cleared", "crates/redact/src/tests.rs::redaction_clears_pixels_intersected_without_covering_their_centres", "crates/redact/src/tests.rs::a_fax_image_has_its_covered_pixels_cleared_instead_of_being_removed", "crates/redact/src/tests.rs::group_3_on_bytes_is_removed_whole_and_other_fax_images_cleared", "crates/redact/src/tests.rs::an_image_whose_predictor_is_given_by_reference_has_the_covered_pixel_cleared", "crates/redact/src/tests.rs::a_jbig2_image_has_its_covered_pixels_cleared_instead_of_being_removed", "crates/redact/src/tests.rs::a_jbig2_image_that_cannot_be_cleared_for_certain_is_removed_whole", "crates/redact/src/tests.rs::a_redacted_jbig2_scan_keeps_neither_its_code_nor_its_globals"] -notes = "intersected pixel footprints are cleared conservatively in a copy, CCITT fax and JBIG2 images (scans stored as Group 4 or JBIG2, with their /JBIG2Globals) included; DCT/JPX images, Group 3 on bytes (which the bootstrap renderer may draw as other rows than the decoder reads), and a JBIG2 image whose page isn't its size, removed whole. The copy is Flate (larger than a Group 4 or JBIG2 scan) and refers to no globals; globals another image still uses stay, and may hold the shapes of marks under the area (not where they were)" +evidence = ["crates/redact/src/tests.rs::images_are_removed_or_have_their_pixels_cleared", "crates/redact/src/tests.rs::redaction_clears_pixels_intersected_without_covering_their_centres", "crates/redact/src/tests.rs::a_fax_image_has_its_covered_pixels_cleared_instead_of_being_removed", "crates/redact/src/tests.rs::group_3_on_bytes_is_removed_whole_and_other_fax_images_cleared", "crates/redact/src/tests.rs::an_image_whose_predictor_is_given_by_reference_has_the_covered_pixel_cleared", "crates/redact/src/tests.rs::a_jbig2_image_has_its_covered_pixels_cleared_instead_of_being_removed", "crates/redact/src/tests.rs::a_jbig2_image_that_cannot_be_cleared_for_certain_is_removed_whole", "crates/redact/src/tests.rs::a_redacted_jbig2_scan_keeps_neither_its_code_nor_its_globals", "crates/redact/src/tests.rs::a_cleared_black_and_white_scan_is_stored_as_group_4"] +notes = "intersected pixel footprints are cleared conservatively in a copy, CCITT fax and JBIG2 images (scans stored as Group 4 or JBIG2, with their /JBIG2Globals) included; DCT/JPX images, Group 3 on bytes (which the bootstrap renderer may draw as other rows than the decoder reads), and a JBIG2 image whose page isn't its size, removed whole. The copy is Flate, or for a one-bit image of one component Group 4, or JBIG2 when it was JBIG2, whichever is smallest as stored, and refers to no globals; globals another image still uses stay, and may hold the shapes of marks under the area (not where they were)" [[feature]] id = "protect.redact-vectors" From 689a1e34cfc3883fd821fad8dfad5e64ae2cfaf6 Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Sat, 31 Oct 2026 00:41:23 +0100 Subject: [PATCH 2/5] M8: test: a cleared image of more than one bit stays Flate Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/redact/src/tests.rs | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/crates/redact/src/tests.rs b/crates/redact/src/tests.rs index 2e76bb0bac..4a0a40fd25 100644 --- a/crates/redact/src/tests.rs +++ b/crates/redact/src/tests.rs @@ -990,6 +990,29 @@ fn a_cleared_image_group_4_wouldnt_shrink_stays_flate() { } } +/// Only one bit of one component goes to Group 4 or JBIG2. An 8-bit grey image here starts as white +/// as a bilevel scan and goes on as noise Flate can't shrink, so Group 4, which would read only the +/// first rows' worth of bytes, would look far smaller while losing the picture. +#[test] +fn a_cleared_image_of_more_than_one_bit_stays_flate() { + let (w, h) = (256usize, 256usize); + let mut data = vec![0xFF; w * h]; + let mut state = 0x1234_5678u32; + for b in data.iter_mut().skip(w * h / 8) { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + *b = state.to_le_bytes()[0]; + } + let dict = format!("/Type /XObject /Subtype /Image /Width {w} /Height {h} /BitsPerComponent 8 /ColorSpace /DeviceGray /Filter /FlateDecode"); + let mut doc = one_page(b"q 200 0 0 20 0 100 cm /Im Do Q", "/XObject << /Im 7 0 R >>", vec![stream(&dict, pdfcraft_filters::encode_flate(&data))]); + mark(&mut doc, 0, &[[0.0, 112.0, 99.0, 118.0]], ""); + assert_eq!(apply(&mut doc, None).unwrap().images_cleared, 1); + let saved = Document::open(Arc::new(write_full(&doc, &SaveOptions::default()).unwrap())).unwrap(); + assert_eq!(named_stream(&saved, 0, b"PCRedacted1").unwrap().dict.name(b"Filter"), Some(&b"FlateDecode"[..])); + assert_eq!(named_image(&saved, 0, b"PCRedacted1").unwrap().get(h * w / 2..), data.get(h * w / 2..), "the rows below the mark are kept"); +} + /// Apply Redactions gives the page its own pruned /Resources, but the dictionary it drew through /// before (inherited from the /Pages node, or shared with another page) still named the removed /// image and form, so a full-rewrite save kept them. From 53874375cbf7afe77be0df35bbeb32a0d5110c4a Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Sat, 31 Oct 2026 02:53:16 +0100 Subject: [PATCH 3/5] M8: test: a cleared copy stored as Group 4 or JBIG2 can be redacted again; the README's not-yet list The README still listed metadata and hidden-information sanitising as not done, though sanitize does it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/redact/README.md | 4 +-- crates/redact/src/tests.rs | 60 +++++++++++++++++++++++++++++++++----- 2 files changed, 54 insertions(+), 10 deletions(-) diff --git a/crates/redact/README.md b/crates/redact/README.md index e52c1d8b98..0a33c4e20b 100644 --- a/crates/redact/README.md +++ b/crates/redact/README.md @@ -109,5 +109,5 @@ Patterns are heuristic: dates, protocol identifiers and longer number runs are r `CodeSet::from_id("foia")?.overlay(&["(b)(6)"])` gives the overlay text for the picked codes. Custom code sets aren't supported yet. -Not yet: pattern locales, struct-tree/alt-text cleanup, metadata and hidden-information -sanitising (M8.4), re-encoding of DCT images (they are removed instead). +Not yet: pattern locales, struct-tree/alt-text cleanup, re-encoding of DCT images (they are removed +instead). diff --git a/crates/redact/src/tests.rs b/crates/redact/src/tests.rs index 4a0a40fd25..9db24fff4e 100644 --- a/crates/redact/src/tests.rs +++ b/crates/redact/src/tests.rs @@ -897,14 +897,9 @@ fn a_redacted_jbig2_scan_keeps_neither_its_code_nor_its_globals() { } } -/// A cleared one-bit image of one component, a scan as a rule, is stored as CCITT Group 4 when -/// that is smaller than Flate, whatever it was stored as (Flate here), and never as JBIG2 unless it -/// was JBIG2; an eight-bit image stays Flate. The copy decodes to the cleared samples. -#[test] -fn a_cleared_black_and_white_scan_is_stored_as_group_4() { - // 800 × 200: lines of word-sized dark blocks 14 rows tall, their edges ragged by a sample from - // row to row, as a scan's are; Group 4 codes such rows from the row above. - let (w, h) = (800usize, 200usize); +/// `w` × `h` rows, `w` a multiple of 8: lines of word-sized dark blocks 14 rows tall, their edges +/// ragged by a sample from row to row, as a scan's are; Group 4 codes such rows from the row above. +fn ragged_scan(w: usize, h: usize) -> Vec { let stride = w / 8; let mut seed = 0x9E37_79B9_7F4A_7C15u64; let mut next = move || { @@ -927,6 +922,17 @@ fn a_cleared_black_and_white_scan_is_stored_as_group_4() { x += len + 6 + (next() % 6) as usize; } } + bits +} + +/// A cleared one-bit image of one component, a scan as a rule, is stored as CCITT Group 4 when +/// that is smaller than Flate, whatever it was stored as (Flate here), and never as JBIG2 unless it +/// was JBIG2; an eight-bit image stays Flate. The copy decodes to the cleared samples. +#[test] +fn a_cleared_black_and_white_scan_is_stored_as_group_4() { + let (w, h) = (800usize, 200usize); + let stride = w / 8; + let bits = ragged_scan(w, h); let g4 = pdfcraft_filters::encode_g4(&bits, w as u32, h as u32).unwrap(); assert!(g4.len() < pdfcraft_filters::encode_flate(&bits).len(), "Group 4 is smaller than Flate here"); let grey: Vec = (0..w * h).map(|i| (i % 251) as u8).collect(); @@ -1013,6 +1019,44 @@ fn a_cleared_image_of_more_than_one_bit_stays_flate() { assert_eq!(named_image(&saved, 0, b"PCRedacted1").unwrap().get(h * w / 2..), data.get(h * w / 2..), "the rows below the mark are kept"); } +/// A copy stored as Group 4 or JBIG2 is read like any image: a second redaction, across a save, +/// clears it again rather than removing it whole, keeps what the first cleared, and stores it +/// smaller than Flate again. +#[test] +fn a_cleared_copy_stored_as_group_4_or_jbig2_can_be_redacted_again() { + let (w, h) = (800usize, 200usize); + let stride = w / 8; + let bits = ragged_scan(w, h); + let image = format!("/Type /XObject /Subtype /Image /Width {w} /Height {h} /BitsPerComponent 1 /ColorSpace /DeviceGray"); + for (filter, data, wanted) in [ + ("/FlateDecode", pdfcraft_filters::encode_flate(&bits), &b"CCITTFaxDecode"[..]), + ("/JBIG2Decode", pdfcraft_filters::encode_jbig2(&bits, w as u32, h as u32).unwrap(), &b"JBIG2Decode"[..]), + ] { + let dict = format!("{image} /Filter {filter}"); + let mut doc = one_page(b"q 200 0 0 50 0 100 cm /Im Do Q", "/XObject << /Im 7 0 R >>", vec![stream(&dict, data)]); + mark(&mut doc, 0, &[[0.0, 125.0, 50.0, 150.0]], ""); + assert_eq!(apply(&mut doc, None).unwrap().images_cleared, 1, "{filter}"); + let mut saved = Document::open(Arc::new(write_full(&doc, &SaveOptions::default()).unwrap())).unwrap(); + assert_eq!(named_stream(&saved, 0, b"PCRedacted1").unwrap().dict.name(b"Filter"), Some(wanted), "{filter}"); + mark(&mut saved, 0, &[[150.0, 100.0, 200.0, 125.0]], ""); + let report = apply(&mut saved, None).unwrap(); + assert_eq!((report.images_cleared, report.images_removed), (1, 0), "{filter}: cleared again, not removed"); + let again = Document::open(Arc::new(write_full(&saved, &SaveOptions::default()).unwrap())).unwrap(); + // Rows 0 to 99 of the left quarter, then 100 to 199 of the right. + let mut expected = bits.clone(); + for (rows, columns) in [(0..100, 0..200), (100..200, 600..800)] { + for y in rows { + for x in columns.clone() { + expected[y * stride + x / 8] &= !(0x80 >> (x % 8)); + } + } + } + let names: Vec> = (1..4).map(|n| format!("PCRedacted{n}").into_bytes()).collect(); + let stored = names.iter().find_map(|n| named_image(&again, 0, n)); + assert_eq!(stored, Some(expected), "{filter}"); + } +} + /// Apply Redactions gives the page its own pruned /Resources, but the dictionary it drew through /// before (inherited from the /Pages node, or shared with another page) still named the removed /// image and form, so a full-rewrite save kept them. From 566007529516dddab991f4dfc54da664189dfd6d Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Sun, 1 Nov 2026 12:15:58 +0100 Subject: [PATCH 4/5] M8: the test of rows too wide for Group 4 says Group 4 refused them Flate coming out didn't show that: Group 4 could have been coded and lost on size. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/redact/src/tests.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/redact/src/tests.rs b/crates/redact/src/tests.rs index 9db24fff4e..c61e7e60f3 100644 --- a/crates/redact/src/tests.rs +++ b/crates/redact/src/tests.rs @@ -978,6 +978,7 @@ fn a_cleared_image_group_4_wouldnt_shrink_stays_flate() { let wide = pdfcraft_filters::MAX_G4_COLUMNS as usize + 1; for (w, h) in [(3usize, 2usize), (wide, 2)] { let stride = w.div_ceil(8); + assert_eq!(pdfcraft_filters::encode_g4(&vec![0xFF; stride * h], w as u32, h as u32).is_ok(), w != wide, "{w} × {h}"); let dict = format!("/Type /XObject /Subtype /Image /Width {w} /Height {h} /BitsPerComponent 1 /ColorSpace /DeviceGray /Filter /FlateDecode"); let mut doc = one_page( b"q 200 0 0 20 0 100 cm /Im Do Q", From 616ddca371a49cee3a3f187b0d086e5058e7662d Mon Sep 17 00:00:00 2001 From: FredApps <153173854+FredApps@users.noreply.github.com> Date: Sun, 1 Nov 2026 22:48:29 +0100 Subject: [PATCH 5/5] M8: images stored again without shared JBIG2 globals are stored as a cleared copy is An image that shared a redacted image's /JBIG2Globals is now stored again as the smallest of Flate, Group 4 or JBIG2 coded alone (smallest), like a cleared copy, rather than always Flate, so the other pages of a scan don't balloon either. The shared-globals test's second page is a ragged scan and checks it is no longer Flate and is smaller. --- crates/redact/README.md | 4 ++-- crates/redact/src/image.rs | 8 ++++++-- crates/redact/src/lib.rs | 12 ++++++------ crates/redact/src/tests.rs | 13 ++++++++----- parity/acrobat-features.toml | 2 +- 5 files changed, 23 insertions(+), 16 deletions(-) diff --git a/crates/redact/README.md b/crates/redact/README.md index e590d0a03d..9c024d1b55 100644 --- a/crates/redact/README.md +++ b/crates/redact/README.md @@ -25,8 +25,8 @@ clear_marks(&mut doc, None)?; // remove marks without app when it was JBIG2, whichever is smallest as stored, coded from the cleared samples; it refers to no globals. Globals hold the shapes of every image that shares them (a scan's pages, often), those of marks under the area included, so every other image that uses the globals of a removed - or replaced image is decoded and stored again in place as Flate without them - (`Report::images_restored`), and they leave the file on the full rewrite; one that can't be + or replaced image is decoded and stored again in place without them, as a copy is (Flate, + Group 4 or JBIG2, whichever is smallest; `Report::images_restored`), and they leave the file on the full rewrite; one that can't be decoded for certain refuses the redaction (`RedactError::SharedJbig2Globals`). Codecs redaction doesn't decode (DCT, JPX) are removed whole (fail-closed), and so is a JBIG2 image whose page isn't its size (the renderer draws the whole page into the image, so the rows cleared wouldn't be the ones shown) or of more diff --git a/crates/redact/src/image.rs b/crates/redact/src/image.rs index 802c7a47ec..a96f4715e6 100644 --- a/crates/redact/src/image.rs +++ b/crates/redact/src/image.rs @@ -204,7 +204,8 @@ fn stored_len(s: &Stream) -> usize { } /// JBIG2 image `s` stored again without its `/JBIG2Globals`: decoded through them, as it is drawn, -/// and written as a Flate image of the same samples, its other entries kept. `None` when it can't +/// and written as the same samples, Flate, Group 4 or JBIG2 coded alone, whichever is smallest +/// (`smallest`), its other entries kept. `None` when it can't /// be decoded for certain (too large, data that doesn't decode, a JBIG2 page of another size than /// the image, or not a one-bit image of one component). Redaction does this to every image that /// shares a redacted image's globals, since their symbol dictionary holds the shapes of marks drawn @@ -231,5 +232,8 @@ pub(crate) fn without_globals(doc: &Document, s: &Stream) -> Option { data.truncate(bytes); let mut dict = s.dict.clone(); dict.remove(b"Length"); - Some(Stream::flate(dict, &data)) + // The globals are named in the parameters; the copy is coded alone, as `clear`'s are. + dict.remove(b"DecodeParms"); + dict.remove(b"Alternates"); + Some(smallest(dict, &data, w, h, true, true)) } diff --git a/crates/redact/src/lib.rs b/crates/redact/src/lib.rs index e1891ecfef..fad55c431f 100644 --- a/crates/redact/src/lib.rs +++ b/crates/redact/src/lib.rs @@ -138,8 +138,8 @@ pub struct Report { pub glyphs: usize, pub images_removed: usize, pub images_cleared: usize, - /// JBIG2 images, anywhere in the document, stored again as Flate without the `/JBIG2Globals` - /// they shared with a redacted image, so those globals leave the file. + /// JBIG2 images, anywhere in the document, stored again (as a cleared copy is) without the + /// `/JBIG2Globals` they shared with a redacted image, so those globals leave the file. pub images_restored: usize, pub paths_removed: usize, pub paths_clipped: usize, @@ -551,10 +551,10 @@ const MAX_JBIG2_WALK: usize = 100_000; /// of marks drawn only under the area too. The image itself is replaced or removed, but the globals /// would stay for as long as another image uses them. So every image that uses the globals of a /// removed or replaced image (`doomed`), or of one drawn in a removed or replaced form, is decoded -/// and stored again in place as Flate, without them, and a full save then leaves them out. An -/// image that can't be stored again refuses the redaction (nothing changes, as `apply` works on a -/// copy), but for a removed or replaced one, which is either unlinked or still drawn whole on -/// another page. Returns how many images that weren't removed or replaced were stored again. +/// and stored again in place without them, as a cleared copy is, and a full save then leaves them +/// out. An image that can't be stored again refuses the redaction (nothing changes, as `apply` +/// works on a copy), but for a removed or replaced one, which is either unlinked or still drawn +/// whole on another page. Returns how many images that weren't removed or replaced were stored again. fn restore_shared_jbig2(doc: &mut Document, doomed: &[ObjRef]) -> Result { let mut globals = HashSet::new(); let mut seen = HashSet::new(); diff --git a/crates/redact/src/tests.rs b/crates/redact/src/tests.rs index 22eac1296b..74b8407056 100644 --- a/crates/redact/src/tests.rs +++ b/crates/redact/src/tests.rs @@ -1154,10 +1154,8 @@ fn globals_a_redacted_jbig2_image_shares_with_another_page_leave_the_file() { } } draw(&mut first, &SIGNATURE, 2, 10); - let mut second = vec![0xFFu8; stride * h]; - for x in (3..w - 6).step_by(7) { - draw(&mut second, &LETTER, 20, x); - } + // The second page is a scan's ragged text, which Group 4 or JBIG2 stores smaller than Flate. + let second = ragged_scan(w, h); // The first page's symbol dictionary (its second segment) goes to the globals, on no page; the // second page's segments are numbered past it. let mut segments = jbig2_segments(&pdfcraft_filters::encode_jbig2(&first, w as u32, h as u32).unwrap()); @@ -1219,7 +1217,12 @@ fn globals_a_redacted_jbig2_image_shares_with_another_page_leave_the_file() { } let saved = reopen(&doc); assert_eq!(picture(&saved, 0, b"PCRedacted1"), Some(cleared)); - assert_eq!(picture(&saved, 1, b"Im"), Some(second), "the second page keeps its picture"); + assert_eq!(picture(&saved, 1, b"Im"), Some(second.clone()), "the second page keeps its picture"); + // Stored again as a scan is: Group 4 or JBIG2 when smaller than Flate. + let restored = named_stream(&saved, 1, b"Im").unwrap(); + let flate = pdfcraft_filters::encode_flate(&second); + assert_ne!(restored.dict.name(b"Filter"), Some(&b"FlateDecode"[..])); + assert!(restored.raw.len() < flate.len(), "{:?} {} bytes, Flate {}", restored.dict.name(b"Filter"), restored.raw.len(), flate.len()); // The dictionary's data (after its 11-byte header) codes the signature's bitmap. let coded = &dictionary[11..]; for (r, object) in saved.scan_objects() { diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index ebafe530be..8b923d6ce7 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -5825,7 +5825,7 @@ status = "shipped" commands = ["redact.apply"] tools = ["redact_apply"] evidence = ["crates/redact/src/tests.rs::images_are_removed_or_have_their_pixels_cleared", "crates/redact/src/tests.rs::redaction_clears_pixels_intersected_without_covering_their_centres", "crates/redact/src/tests.rs::a_fax_image_has_its_covered_pixels_cleared_instead_of_being_removed", "crates/redact/src/tests.rs::group_3_on_bytes_is_removed_whole_and_other_fax_images_cleared", "crates/redact/src/tests.rs::an_image_whose_predictor_is_given_by_reference_has_the_covered_pixel_cleared", "crates/redact/src/tests.rs::a_jbig2_image_has_its_covered_pixels_cleared_instead_of_being_removed", "crates/redact/src/tests.rs::a_jbig2_image_that_cannot_be_cleared_for_certain_is_removed_whole", "crates/redact/src/tests.rs::a_redacted_jbig2_scan_keeps_neither_its_code_nor_its_globals", "crates/redact/src/tests.rs::a_cleared_black_and_white_scan_is_stored_as_group_4", "crates/redact/src/tests.rs::globals_a_redacted_jbig2_image_shares_with_another_page_leave_the_file"] -notes = "intersected pixel footprints are cleared conservatively in a copy, CCITT fax and JBIG2 images (scans stored as Group 4 or JBIG2, with their /JBIG2Globals) included; DCT/JPX images, Group 3 on bytes (which the bootstrap renderer may draw as other rows than the decoder reads), and a JBIG2 image whose page isn't its size, removed whole. The copy is Flate, or for a one-bit image of one component Group 4, or JBIG2 when it was JBIG2, whichever is smallest as stored, and refers to no globals; every other image using a redacted image's globals is stored again as Flate without them, so they leave the file (one that can't be decoded refuses the redaction)" +notes = "intersected pixel footprints are cleared conservatively in a copy, CCITT fax and JBIG2 images (scans stored as Group 4 or JBIG2, with their /JBIG2Globals) included; DCT/JPX images, Group 3 on bytes (which the bootstrap renderer may draw as other rows than the decoder reads), and a JBIG2 image whose page isn't its size, removed whole. The copy is Flate, or for a one-bit image of one component Group 4, or JBIG2 when it was JBIG2, whichever is smallest as stored, and refers to no globals; every other image using a redacted image's globals is stored again without them (Flate, Group 4 or JBIG2, whichever is smallest), so they leave the file (one that can't be decoded refuses the redaction)" [[feature]] id = "protect.redact-vectors"