Merge pull request #5474 from Piohrun/feat/redact-bilevel-codec

M8: a redacted black-and-white scan is stored as Group 4, or JBIG2 again, when smaller (protect.redact-image-pixels)

crates/redact/Cargo.toml

@@ -17,8 +17,9 @@ pdfcraft-forms = { workspace = true }
 pdfcraft-edit = { workspace = true }
 pdfcraft-annot = { workspace = true }
 pdfcraft-xfa = { workspace = true }
+# CCITT Group 4 and JBIG2 encoding for cleared one-bit images.
+pdfcraft-filters = { workspace = true }
 thiserror = { workspace = true }
 
 [dev-dependencies]
-pdfcraft-filters = { workspace = true }
 pdfcraft-testkit = { workspace = true }

crates/redact/README.md

@@ -21,10 +21,12 @@ clear_marks(&mut doc, None)?;                        // remove marks without app
   (8/16-bit and 1/2/4-bit images, image masks; Flate/LZW/RL/A85/AHx, and CCITT fax or JBIG2 as
   the last filter, decoded through `Document::decoded_image_within`, a JBIG2 image with its
   `/JBIG2Globals`; `/DecodeParms` given by reference, numbers inside included, count). The copy
-  is Flate, and refers to no globals. Globals hold the shapes of every image that shares them (a
-  scan's pages, often), those of marks under the area included, so every other image that uses
-  the globals of a removed or replaced image is decoded and stored again in place as Flate without
-  them (`Report::images_restored`), and they leave the file on the full rewrite; one that can't be
+  is Flate, or for a one-bit image of one component (a scan as a rule) CCITT Group 4, or JBIG2
+  when it was JBIG2, whichever is smallest as stored, coded from the cleared samples; it refers to
+  no globals. Globals hold the shapes of every image that shares them (a scan's pages, often),
+  those of marks under the area included, so every other image that uses the globals of a removed
+  or replaced image is decoded and stored again in place without them, as a copy is (Flate,
+  Group 4 or JBIG2, whichever is smallest; `Report::images_restored`), and they leave the file on the full rewrite; one that can't be
   decoded for certain refuses the redaction (`RedactError::SharedJbig2Globals`). Codecs redaction doesn't decode (DCT, JPX) are
   removed whole (fail-closed), and so is a JBIG2 image whose page isn't its size (the renderer
   draws the whole page into the image, so the rows cleared wouldn't be the ones shown) or of more
@@ -110,6 +112,5 @@ Patterns are heuristic: dates, protocol identifiers and longer number runs are r
 `CodeSet::from_id("foia")?.overlay(&["(b)(6)"])` gives the overlay text for the picked codes.
 Custom code sets aren't supported yet.
 
-Not yet: pattern locales, struct-tree/alt-text cleanup, metadata and hidden-information
-sanitising (M8.4), re-encoding of DCT images (they are removed instead), and storing a cleared black-and-white
-image as CCITT Group 4 or JBIG2 again (it is Flate, larger; the optimizer makes it smaller).
+Not yet: pattern locales, struct-tree/alt-text cleanup, re-encoding of DCT images (they are removed
+instead).

crates/redact/src/image.rs

@@ -1,13 +1,14 @@
 //! Clearing the pixels of an image under redaction regions. The image is decoded, every pixel
 //! whose footprint intersects a region is set to zero (no ink for image masks), and the result is
-//! written as a new Flate image; the original object is untouched (other pages may use it).
+//! written as a new image, Flate, or for a one-bit image of one component Group 4, or JBIG2 when it
+//! was JBIG2, whichever is smallest; the original object is untouched (other pages may use it).
 //! CCITT fax and JBIG2 images decode through `Document::decoded_image_within`, but for Group 3 on
 //! bytes, which PdfCraft's renderer may draw as other rows, and a JBIG2 page of other rows than the
 //! image's. Images whose codec PdfCraft can't decode (DCT, JPX), and those, return `None`, and the
 //! caller removes the whole image instead; that is fail-closed.
 
 use pdfcraft_content::{Matrix, overlaps};
-use pdfcraft_cos::{Document, Object, Stream};
+use pdfcraft_cos::{Dict, Document, Object, Stream};
 
 /// Most pixels an image may have to be cleared in place (larger ones are removed).
 const MAX_PIXELS: u64 = 64 << 20;
@@ -165,11 +166,46 @@ fn decode_and_clear(doc: &Document, s: &Stream, ctm: &Matrix, rects: &[[f64; 4]]
     dict.remove(b"Length");
     dict.remove(b"DecodeParms");
     dict.remove(b"Alternates");
-    Some(Some(Stream::flate(dict, &data)))
+    Some(Some(smallest(dict, &data, w, h, ncomp == 1 && bpc == 1, is_jbig2)))
+}
+
+/// The cleared samples `data` of a `w` × `h` image as a stream with `dict`: Flate, or, for one bit
+/// of one component (`bilevel`, a scan as a rule, which Flate stores several times larger), CCITT
+/// Group 4, or JBIG2 when the image was JBIG2 (`jbig2`; JBIG2 is PDF 1.4, which such a file already
+/// is), whichever is smallest as stored, dictionary included (a tiny image can lose by Group 4's
+/// parameters). Both decode to `data`, cleared before it is coded, so nothing of the original's
+/// code is carried over; Group 4 or JBIG2 that can't code the rows (too wide) is left out.
+fn smallest(dict: Dict, data: &[u8], w: usize, h: usize, bilevel: bool, jbig2: bool) -> Stream {
+    let flate = Stream::flate(dict.clone(), data);
+    let Some((columns, rows)) = u32::try_from(w).ok().zip(u32::try_from(h).ok()).filter(|_| bilevel) else { return flate };
+    let g4 = pdfcraft_filters::encode_g4(data, columns, rows).ok().map(|coded| {
+        let mut dict = dict.clone();
+        let mut parms = Dict::new();
+        parms.set(b"K".to_vec(), Object::Int(-1));
+        parms.set(b"Columns".to_vec(), Object::Int(i64::from(columns)));
+        parms.set(b"Rows".to_vec(), Object::Int(i64::from(rows)));
+        dict.set(b"Filter".to_vec(), Object::name("CCITTFaxDecode"));
+        dict.set(b"DecodeParms".to_vec(), Object::Dict(parms));
+        Stream::from_raw(dict, coded)
+    });
+    let jbig = jbig2.then(|| pdfcraft_filters::encode_jbig2(data, columns, rows).ok()).flatten().map(|coded| {
+        let mut dict = dict.clone();
+        dict.set(b"Filter".to_vec(), Object::name("JBIG2Decode"));
+        Stream::from_raw(dict, coded)
+    });
+    [g4, jbig].into_iter().flatten().fold(flate, |best, s| if stored_len(&s) < stored_len(&best) { s } else { best })
+}
+
+/// The bytes stream `s` takes in a file: its data and its dictionary as written.
+fn stored_len(s: &Stream) -> usize {
+    let mut dict = Vec::new();
+    pdfcraft_cos::serialize(&Object::Dict(s.dict.clone()), &mut dict);
+    dict.len().saturating_add(s.raw.len())
 }
 
 /// JBIG2 image `s` stored again without its `/JBIG2Globals`: decoded through them, as it is drawn,
-/// and written as a Flate image of the same samples, its other entries kept. `None` when it can't
+/// and written as the same samples, Flate, Group 4 or JBIG2 coded alone, whichever is smallest
+/// (`smallest`), its other entries kept. `None` when it can't
 /// be decoded for certain (too large, data that doesn't decode, a JBIG2 page of another size than
 /// the image, or not a one-bit image of one component). Redaction does this to every image that
 /// shares a redacted image's globals, since their symbol dictionary holds the shapes of marks drawn
@@ -196,5 +232,8 @@ pub(crate) fn without_globals(doc: &Document, s: &Stream) -> Option<Stream> {
     data.truncate(bytes);
     let mut dict = s.dict.clone();
     dict.remove(b"Length");
-    Some(Stream::flate(dict, &data))
+    // The globals are named in the parameters; the copy is coded alone, as `clear`'s are.
+    dict.remove(b"DecodeParms");
+    dict.remove(b"Alternates");
+    Some(smallest(dict, &data, w, h, true, true))
 }

crates/redact/src/lib.rs

@@ -138,8 +138,8 @@ pub struct Report {
     pub glyphs: usize,
     pub images_removed: usize,
     pub images_cleared: usize,
-    /// JBIG2 images, anywhere in the document, stored again as Flate without the `/JBIG2Globals`
-    /// they shared with a redacted image, so those globals leave the file.
+    /// JBIG2 images, anywhere in the document, stored again (as a cleared copy is) without the
+    /// `/JBIG2Globals` they shared with a redacted image, so those globals leave the file.
     pub images_restored: usize,
     pub paths_removed: usize,
     pub paths_clipped: usize,
@@ -551,10 +551,10 @@ const MAX_JBIG2_WALK: usize = 100_000;
 /// of marks drawn only under the area too. The image itself is replaced or removed, but the globals
 /// would stay for as long as another image uses them. So every image that uses the globals of a
 /// removed or replaced image (`doomed`), or of one drawn in a removed or replaced form, is decoded
-/// and stored again in place as Flate, without them, and a full save then leaves them out. An
-/// image that can't be stored again refuses the redaction (nothing changes, as `apply` works on a
-/// copy), but for a removed or replaced one, which is either unlinked or still drawn whole on
-/// another page. Returns how many images that weren't removed or replaced were stored again.
+/// and stored again in place without them, as a cleared copy is, and a full save then leaves them
+/// out. An image that can't be stored again refuses the redaction (nothing changes, as `apply`
+/// works on a copy), but for a removed or replaced one, which is either unlinked or still drawn
+/// whole on another page. Returns how many images that weren't removed or replaced were stored again.
 fn restore_shared_jbig2(doc: &mut Document, doomed: &[ObjRef]) -> Result<usize, RedactError> {
     let mut globals = HashSet::new();
     let mut seen = HashSet::new();

crates/redact/src/tests.rs

@@ -519,13 +519,19 @@ fn two_pages_one_resource_dictionary(inherited: bool, again: bool) -> Document {
     ])
 }
 
-/// The samples of the image a page's (possibly inherited) resources name `name`.
+/// The samples of the image a page's (possibly inherited) resources name `name`, decoded through
+/// its image codec too (`Document::decoded_image`).
 fn named_image(doc: &Document, page: usize, name: &[u8]) -> Option<Vec<u8>> {
+    named_stream(doc, page, name).and_then(|s| doc.decoded_image(&s).ok())
+}
+
+/// The image a page's (possibly inherited) resources name `name`.
+fn named_stream(doc: &Document, page: usize, name: &[u8]) -> Option<Stream> {
     let p = pdfcraft_model::pages(doc).swap_remove(page);
     let res = doc.resolve(p.dict.get(b"Resources")?).as_dict().cloned()?;
     let xo = doc.resolve(res.get(b"XObject")?).as_dict().cloned()?;
     match &*doc.get(xo.get(name)?.as_ref()?) {
-        Object::Stream(s) => s.decoded().ok(),
+        Object::Stream(s) => Some(s.clone()),
         _ => None,
     }
 }
@@ -887,7 +893,8 @@ fn a_jbig2_image_that_cannot_be_cleared_for_certain_is_removed_whole() {
 
 /// A scan stored as JBIG2 symbols, with globals, keeps every pixel outside the area once it is
 /// redacted and saved, and the pixels under it are cleared; neither its JBIG2 code nor its globals,
-/// which no other image uses, stay in the file.
+/// which no other image uses, stay in the file. The cleared copy is coded afresh as JBIG2, smaller
+/// than Flate, and refers to no globals.
 #[test]
 fn a_redacted_jbig2_scan_keeps_neither_its_code_nor_its_globals() {
     const LETTERS: [[&str; 7]; 3] = [
@@ -908,6 +915,7 @@ fn a_redacted_jbig2_scan_keeps_neither_its_code_nor_its_globals() {
         }
     }
     let coded = pdfcraft_filters::encode_jbig2(&bits, w as u32, h as u32).unwrap();
+    let original = coded.clone();
     assert!(coded.len() < pdfcraft_filters::encode_jbig2_generic(&bits, w as u32, h as u32).unwrap().len(), "symbol coded");
     let dict = format!(
         "/Type /XObject /Subtype /Image /Width {w} /Height {h} /BitsPerComponent 1 /ColorSpace /DeviceGray /Filter /JBIG2Decode /DecodeParms << /JBIG2Globals 8 0 R >>"
@@ -931,13 +939,179 @@ fn a_redacted_jbig2_scan_keeps_neither_its_code_nor_its_globals() {
     let saved = Document::open(Arc::new(write_full(&doc, &SaveOptions::default()).unwrap())).unwrap();
     assert_eq!(named_image(&saved, 0, b"PCRedacted1"), Some(cleared));
     assert_no_secret(&doc, b"LEAK_GLOBALS");
+    let copy = named_stream(&saved, 0, b"PCRedacted1").unwrap();
+    assert_eq!(copy.dict.name(b"Filter"), Some(&b"JBIG2Decode"[..]));
+    assert!(!copy.dict.contains(b"DecodeParms"), "the copy refers to no globals");
+    let flate = pdfcraft_filters::encode_flate(&named_image(&saved, 0, b"PCRedacted1").unwrap());
+    assert!(copy.raw.len() < flate.len(), "JBIG2 {} bytes, Flate {}", copy.raw.len(), flate.len());
     for (r, object) in saved.scan_objects() {
         if let Object::Stream(s) = &*object {
-            assert_ne!(s.dict.name(b"Filter"), Some(&b"JBIG2Decode"[..]), "the JBIG2 image stays as {r:?}");
+            assert_ne!(&s.raw[..], &original[..], "the original JBIG2 code stays as {r:?}");
         }
     }
 }
 
+/// `w` × `h` rows, `w` a multiple of 8: lines of word-sized dark blocks 14 rows tall, their edges
+/// ragged by a sample from row to row, as a scan's are; Group 4 codes such rows from the row above.
+fn ragged_scan(w: usize, h: usize) -> Vec<u8> {
+    let stride = w / 8;
+    let mut seed = 0x9E37_79B9_7F4A_7C15u64;
+    let mut next = move || {
+        seed ^= seed << 13;
+        seed ^= seed >> 7;
+        seed ^= seed << 17;
+        seed
+    };
+    let mut bits = vec![0xFFu8; stride * h];
+    for top in (3..h - 14).step_by(20) {
+        let mut x = 4;
+        while x + 70 < w {
+            let len = 20 + (next() % 40) as usize;
+            for y in top..top + 14 {
+                let (left, right) = (x + (next() % 2) as usize, x + len - (next() % 2) as usize);
+                for xx in left..right {
+                    bits[y * stride + xx / 8] &= !(0x80 >> (xx % 8));
+                }
+            }
+            x += len + 6 + (next() % 6) as usize;
+        }
+    }
+    bits
+}
+
+/// A cleared one-bit image of one component, a scan as a rule, is stored as CCITT Group 4 when
+/// that is smaller than Flate, whatever it was stored as (Flate here), and never as JBIG2 unless it
+/// was JBIG2; an eight-bit image stays Flate. The copy decodes to the cleared samples.
+#[test]
+fn a_cleared_black_and_white_scan_is_stored_as_group_4() {
+    let (w, h) = (800usize, 200usize);
+    let stride = w / 8;
+    let bits = ragged_scan(w, h);
+    let g4 = pdfcraft_filters::encode_g4(&bits, w as u32, h as u32).unwrap();
+    assert!(g4.len() < pdfcraft_filters::encode_flate(&bits).len(), "Group 4 is smaller than Flate here");
+    let grey: Vec<u8> = (0..w * h).map(|i| (i % 251) as u8).collect();
+    for (dict, data, filter) in [
+        (
+            format!("/Type /XObject /Subtype /Image /Width {w} /Height {h} /BitsPerComponent 1 /ColorSpace /DeviceGray /Filter /FlateDecode"),
+            pdfcraft_filters::encode_flate(&bits),
+            &b"CCITTFaxDecode"[..],
+        ),
+        (
+            format!("/Type /XObject /Subtype /Image /Width {w} /Height {h} /BitsPerComponent 8 /ColorSpace /DeviceGray /Filter /FlateDecode"),
+            pdfcraft_filters::encode_flate(&grey),
+            &b"FlateDecode"[..],
+        ),
+    ] {
+        let mut doc = one_page(b"q 200 0 0 50 0 100 cm /Im Do Q", "/XObject << /Im 7 0 R >>", vec![stream(&dict, data)]);
+        // The left quarter of the top half.
+        mark(&mut doc, 0, &[[0.0, 125.0, 50.0, 150.0]], "");
+        assert_eq!(apply(&mut doc, None).unwrap().images_cleared, 1, "{dict}");
+        let saved = Document::open(Arc::new(write_full(&doc, &SaveOptions::default()).unwrap())).unwrap();
+        let copy = named_stream(&saved, 0, b"PCRedacted1").unwrap();
+        assert_eq!(copy.dict.name(b"Filter"), Some(filter), "{dict}");
+        // Covered: rows 0 to 99 (y 150 down to 125, a quarter point a row), samples 0 to 199.
+        let mut expected = if filter == b"FlateDecode" { grey.clone() } else { bits.clone() };
+        for y in 0..100 {
+            for x in 0..200 {
+                if filter == b"FlateDecode" {
+                    expected[y * w + x] = 0;
+                } else {
+                    expected[y * stride + x / 8] &= !(0x80 >> (x % 8));
+                }
+            }
+        }
+        assert_eq!(named_image(&saved, 0, b"PCRedacted1").unwrap(), expected, "{dict}");
+    }
+}
+
+/// A cleared one-bit image is stored as the smallest stream, its dictionary counted: a tiny one,
+/// whose Group 4 code is shorter than Flate's but whose `/DecodeParms` aren't, stays Flate. Rows
+/// wider than Group 4 codes (`MAX_G4_COLUMNS`) stay Flate too, and still clear.
+#[test]
+fn a_cleared_image_group_4_wouldnt_shrink_stays_flate() {
+    let wide = pdfcraft_filters::MAX_G4_COLUMNS as usize + 1;
+    for (w, h) in [(3usize, 2usize), (wide, 2)] {
+        let stride = w.div_ceil(8);
+        assert_eq!(pdfcraft_filters::encode_g4(&vec![0xFF; stride * h], w as u32, h as u32).is_ok(), w != wide, "{w} × {h}");
+        let dict = format!("/Type /XObject /Subtype /Image /Width {w} /Height {h} /BitsPerComponent 1 /ColorSpace /DeviceGray /Filter /FlateDecode");
+        let mut doc = one_page(
+            b"q 200 0 0 20 0 100 cm /Im Do Q",
+            "/XObject << /Im 7 0 R >>",
+            vec![stream(&dict, pdfcraft_filters::encode_flate(&vec![0xFF; stride * h]))],
+        );
+        // The top row's left half.
+        mark(&mut doc, 0, &[[0.0, 112.0, 99.0, 118.0]], "");
+        assert_eq!(apply(&mut doc, None).unwrap().images_cleared, 1, "{w} × {h}");
+        let saved = Document::open(Arc::new(write_full(&doc, &SaveOptions::default()).unwrap())).unwrap();
+        assert_eq!(named_stream(&saved, 0, b"PCRedacted1").unwrap().dict.name(b"Filter"), Some(&b"FlateDecode"[..]), "{w} × {h}");
+        let samples = named_image(&saved, 0, b"PCRedacted1").unwrap();
+        let sample = |x: usize| samples.get(x / 8).is_some_and(|b| b & (0x80 >> (x % 8)) != 0);
+        assert!(!sample(0) && sample(w - 1), "{w} × {h}: the top row's first sample is cleared and its last kept");
+        assert!(samples.get(stride..).is_some_and(|row| row.iter().all(|b| *b == 0xFF)), "{w} × {h}: the bottom row is kept");
+    }
+}
+
+/// Only one bit of one component goes to Group 4 or JBIG2. An 8-bit grey image here starts as white
+/// as a bilevel scan and goes on as noise Flate can't shrink, so Group 4, which would read only the
+/// first rows' worth of bytes, would look far smaller while losing the picture.
+#[test]
+fn a_cleared_image_of_more_than_one_bit_stays_flate() {
+    let (w, h) = (256usize, 256usize);
+    let mut data = vec![0xFF; w * h];
+    let mut state = 0x1234_5678u32;
+    for b in data.iter_mut().skip(w * h / 8) {
+        state ^= state << 13;
+        state ^= state >> 17;
+        state ^= state << 5;
+        *b = state.to_le_bytes()[0];
+    }
+    let dict = format!("/Type /XObject /Subtype /Image /Width {w} /Height {h} /BitsPerComponent 8 /ColorSpace /DeviceGray /Filter /FlateDecode");
+    let mut doc = one_page(b"q 200 0 0 20 0 100 cm /Im Do Q", "/XObject << /Im 7 0 R >>", vec![stream(&dict, pdfcraft_filters::encode_flate(&data))]);
+    mark(&mut doc, 0, &[[0.0, 112.0, 99.0, 118.0]], "");
+    assert_eq!(apply(&mut doc, None).unwrap().images_cleared, 1);
+    let saved = Document::open(Arc::new(write_full(&doc, &SaveOptions::default()).unwrap())).unwrap();
+    assert_eq!(named_stream(&saved, 0, b"PCRedacted1").unwrap().dict.name(b"Filter"), Some(&b"FlateDecode"[..]));
+    assert_eq!(named_image(&saved, 0, b"PCRedacted1").unwrap().get(h * w / 2..), data.get(h * w / 2..), "the rows below the mark are kept");
+}
+
+/// A copy stored as Group 4 or JBIG2 is read like any image: a second redaction, across a save,
+/// clears it again rather than removing it whole, keeps what the first cleared, and stores it
+/// smaller than Flate again.
+#[test]
+fn a_cleared_copy_stored_as_group_4_or_jbig2_can_be_redacted_again() {
+    let (w, h) = (800usize, 200usize);
+    let stride = w / 8;
+    let bits = ragged_scan(w, h);
+    let image = format!("/Type /XObject /Subtype /Image /Width {w} /Height {h} /BitsPerComponent 1 /ColorSpace /DeviceGray");
+    for (filter, data, wanted) in [
+        ("/FlateDecode", pdfcraft_filters::encode_flate(&bits), &b"CCITTFaxDecode"[..]),
+        ("/JBIG2Decode", pdfcraft_filters::encode_jbig2(&bits, w as u32, h as u32).unwrap(), &b"JBIG2Decode"[..]),
+    ] {
+        let dict = format!("{image} /Filter {filter}");
+        let mut doc = one_page(b"q 200 0 0 50 0 100 cm /Im Do Q", "/XObject << /Im 7 0 R >>", vec![stream(&dict, data)]);
+        mark(&mut doc, 0, &[[0.0, 125.0, 50.0, 150.0]], "");
+        assert_eq!(apply(&mut doc, None).unwrap().images_cleared, 1, "{filter}");
+        let mut saved = Document::open(Arc::new(write_full(&doc, &SaveOptions::default()).unwrap())).unwrap();
+        assert_eq!(named_stream(&saved, 0, b"PCRedacted1").unwrap().dict.name(b"Filter"), Some(wanted), "{filter}");
+        mark(&mut saved, 0, &[[150.0, 100.0, 200.0, 125.0]], "");
+        let report = apply(&mut saved, None).unwrap();
+        assert_eq!((report.images_cleared, report.images_removed), (1, 0), "{filter}: cleared again, not removed");
+        let again = Document::open(Arc::new(write_full(&saved, &SaveOptions::default()).unwrap())).unwrap();
+        // Rows 0 to 99 of the left quarter, then 100 to 199 of the right.
+        let mut expected = bits.clone();
+        for (rows, columns) in [(0..100, 0..200), (100..200, 600..800)] {
+            for y in rows {
+                for x in columns.clone() {
+                    expected[y * stride + x / 8] &= !(0x80 >> (x % 8));
+                }
+            }
+        }
+        let names: Vec<Vec<u8>> = (1..4).map(|n| format!("PCRedacted{n}").into_bytes()).collect();
+        let stored = names.iter().find_map(|n| named_image(&again, 0, n));
+        assert_eq!(stored, Some(expected), "{filter}");
+    }
+}
+
 /// The segments of a JBIG2 stream `encode_jbig2` wrote, each with its header: a header is the
 /// segment's number (4 bytes), type, flags whose top three bits count the segments it refers to,
 /// those (a byte each), its page (a byte) and its data's length (4 bytes; ISO/IEC 14492 7.2).
@@ -980,10 +1154,8 @@ fn globals_a_redacted_jbig2_image_shares_with_another_page_leave_the_file() {
         }
     }
     draw(&mut first, &SIGNATURE, 2, 10);
-    let mut second = vec![0xFFu8; stride * h];
-    for x in (3..w - 6).step_by(7) {
-        draw(&mut second, &LETTER, 20, x);
-    }
+    // The second page is a scan's ragged text, which Group 4 or JBIG2 stores smaller than Flate.
+    let second = ragged_scan(w, h);
     // The first page's symbol dictionary (its second segment) goes to the globals, on no page; the
     // second page's segments are numbered past it.
     let mut segments = jbig2_segments(&pdfcraft_filters::encode_jbig2(&first, w as u32, h as u32).unwrap());
@@ -1045,7 +1217,12 @@ fn globals_a_redacted_jbig2_image_shares_with_another_page_leave_the_file() {
     }
     let saved = reopen(&doc);
     assert_eq!(picture(&saved, 0, b"PCRedacted1"), Some(cleared));
-    assert_eq!(picture(&saved, 1, b"Im"), Some(second), "the second page keeps its picture");
+    assert_eq!(picture(&saved, 1, b"Im"), Some(second.clone()), "the second page keeps its picture");
+    // Stored again as a scan is: Group 4 or JBIG2 when smaller than Flate.
+    let restored = named_stream(&saved, 1, b"Im").unwrap();
+    let flate = pdfcraft_filters::encode_flate(&second);
+    assert_ne!(restored.dict.name(b"Filter"), Some(&b"FlateDecode"[..]));
+    assert!(restored.raw.len() < flate.len(), "{:?} {} bytes, Flate {}", restored.dict.name(b"Filter"), restored.raw.len(), flate.len());
     // The dictionary's data (after its 11-byte header) codes the signature's bitmap.
     let coded = &dictionary[11..];
     for (r, object) in saved.scan_objects() {
@@ -2706,7 +2883,8 @@ fn assert_no_secret(doc: &Document, secret: &[u8]) {
         pdfcraft_cos::serialize(&object, &mut bytes);
         assert!(!bytes.windows(secret.len()).any(|w| w == secret), "secret remains in object {r:?}");
         if let Object::Stream(s) = &*object {
-            let data = s.decoded().unwrap();
+            // Image codecs, a cleared scan coded as Group 4 or JBIG2, decode through the document.
+            let data = s.decoded().or_else(|_| saved.decoded_image(s)).unwrap();
             assert!(!data.windows(secret.len()).any(|w| w == secret), "secret remains in stream {r:?}");
         }
     }

parity/acrobat-features.toml

@@ -5824,8 +5824,8 @@ milestone = "M8"
 status = "shipped"
 commands = ["redact.apply"]
 tools = ["redact_apply"]
-evidence = ["crates/redact/src/tests.rs::images_are_removed_or_have_their_pixels_cleared", "crates/redact/src/tests.rs::redaction_clears_pixels_intersected_without_covering_their_centres", "crates/redact/src/tests.rs::a_fax_image_has_its_covered_pixels_cleared_instead_of_being_removed", "crates/redact/src/tests.rs::group_3_on_bytes_is_removed_whole_and_other_fax_images_cleared", "crates/redact/src/tests.rs::an_image_whose_predictor_is_given_by_reference_has_the_covered_pixel_cleared", "crates/redact/src/tests.rs::a_jbig2_image_has_its_covered_pixels_cleared_instead_of_being_removed", "crates/redact/src/tests.rs::a_jbig2_image_that_cannot_be_cleared_for_certain_is_removed_whole", "crates/redact/src/tests.rs::a_redacted_jbig2_scan_keeps_neither_its_code_nor_its_globals", "crates/redact/src/tests.rs::globals_a_redacted_jbig2_image_shares_with_another_page_leave_the_file"]
-notes = "intersected pixel footprints are cleared conservatively in a copy, CCITT fax and JBIG2 images (scans stored as Group 4 or JBIG2, with their /JBIG2Globals) included; DCT/JPX images, Group 3 on bytes (which the bootstrap renderer may draw as other rows than the decoder reads), and a JBIG2 image whose page isn't its size, removed whole. The copy is Flate (larger than a Group 4 or JBIG2 scan) and refers to no globals; every other image using a redacted image's globals is stored again as Flate without them, so they leave the file (one that can't be decoded refuses the redaction)"
+evidence = ["crates/redact/src/tests.rs::images_are_removed_or_have_their_pixels_cleared", "crates/redact/src/tests.rs::redaction_clears_pixels_intersected_without_covering_their_centres", "crates/redact/src/tests.rs::a_fax_image_has_its_covered_pixels_cleared_instead_of_being_removed", "crates/redact/src/tests.rs::group_3_on_bytes_is_removed_whole_and_other_fax_images_cleared", "crates/redact/src/tests.rs::an_image_whose_predictor_is_given_by_reference_has_the_covered_pixel_cleared", "crates/redact/src/tests.rs::a_jbig2_image_has_its_covered_pixels_cleared_instead_of_being_removed", "crates/redact/src/tests.rs::a_jbig2_image_that_cannot_be_cleared_for_certain_is_removed_whole", "crates/redact/src/tests.rs::a_redacted_jbig2_scan_keeps_neither_its_code_nor_its_globals", "crates/redact/src/tests.rs::a_cleared_black_and_white_scan_is_stored_as_group_4", "crates/redact/src/tests.rs::globals_a_redacted_jbig2_image_shares_with_another_page_leave_the_file"]
+notes = "intersected pixel footprints are cleared conservatively in a copy, CCITT fax and JBIG2 images (scans stored as Group 4 or JBIG2, with their /JBIG2Globals) included; DCT/JPX images, Group 3 on bytes (which the bootstrap renderer may draw as other rows than the decoder reads), and a JBIG2 image whose page isn't its size, removed whole. The copy is Flate, or for a one-bit image of one component Group 4, or JBIG2 when it was JBIG2, whichever is smallest as stored, and refers to no globals; every other image using a redacted image's globals is stored again without them (Flate, Group 4 or JBIG2, whichever is smallest), so they leave the file (one that can't be decoded refuses the redaction)"
 
 [[feature]]
 id = "protect.redact-vectors"