M11: Create PDF stores black-and-white images at one bit, as Group 4 when smaller

A scan saved as a 1-bit PNG decodes to eight-bit grey and was stored
that way, larger than the PNG it came from; a bilevel TIFF was stored at
one bit but with Flate. Now any image of only pure black and white
(1-bit, eight-bit grey, a two-colour palette, RGB without a profile) is
stored at one bit per pixel, as CCITT Group 4 or Flate, whichever is
smaller as stored. A grey profile stays; RGB with a profile keeps its
colours.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BfMGHQNgdq5yf5tKZHGR3z

Cargo.lock

@@ -4333,9 +4333,11 @@ name = "pdfcraft-create"
 version = "0.15.0"
 dependencies = [
  "encoding_rs",
+ "hayro-ccitt",
  "hayro-syntax",
  "image",
  "pdfcraft-cos",
+ "pdfcraft-filters",
  "pdfcraft-fonts",
  "pdfcraft-model",
  "pdfcraft-xfa",

crates/create/Cargo.toml

@@ -10,6 +10,7 @@ workspace = true
 
 [dependencies]
 pdfcraft-cos = { workspace = true }
+pdfcraft-filters = { workspace = true }
 pdfcraft-model = { workspace = true }
 pdfcraft-fonts = { workspace = true }
 # For `image::bmp_with_pixel_offset`, shared with XFA pictures.
@@ -24,5 +25,6 @@ thiserror = { workspace = true }
 encoding_rs = "0.8"
 
 [dev-dependencies]
+hayro-ccitt.workspace = true
 image = { version = "0.25", default-features = false, features = ["bmp", "gif", "jpeg"] }
 hayro-syntax = { workspace = true }

crates/create/src/lib.rs

@@ -2,7 +2,7 @@
 
 #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
 
-use pdfcraft_cos::{Dict, Document, ObjRef, Object, PdfString, Stream};
+use pdfcraft_cos::{Dict, Document, ObjRef, Object, PdfString, Stream, serialize};
 
 mod extract;
 mod upright_image;
@@ -108,9 +108,42 @@ fn jpeg_icc(mut chunks: Vec<(u8, u8, &[u8])>) -> Option<Vec<u8>> {
     complete.then(|| chunks.iter().flat_map(|c| c.2.iter().copied()).collect())
 }
 
+/// The samples of a black-and-white image packed one bit per pixel (1 for white, each row
+/// starting on a byte): eight-bit grey (`n` 1) or RGB (`n` 3) whose every pixel is pure black or
+/// pure white, or one-bit grey, already packed (a bilevel TIFF). `None` for anything else, found
+/// at its first other pixel.
+fn bilevel(data: &[u8], (w, h): (u32, u32), n: usize, bits: Option<i64>) -> Option<Vec<u8>> {
+    let (w, h) = (usize::try_from(w).ok()?, usize::try_from(h).ok()?);
+    let stride = w.div_ceil(8);
+    let size = stride.checked_mul(h)?;
+    match (bits, n) {
+        (Some(1), 1) => data.get(..size).map(<[u8]>::to_vec),
+        (Some(8), 1 | 3) if w > 0 => {
+            let samples = data.get(..w.checked_mul(h)?.checked_mul(n)?)?;
+            let mut packed = vec![0u8; size];
+            for (i, px) in samples.chunks_exact(n).enumerate() {
+                let white = match px {
+                    [0] | [0, 0, 0] => false,
+                    [255] | [255, 255, 255] => true,
+                    _ => return None,
+                };
+                if white && let Some(b) = packed.get_mut((i / w) * stride + (i % w) / 8) {
+                    *b |= 0x80 >> (i % w % 8);
+                }
+            }
+            Some(packed)
+        }
+        _ => None,
+    }
+}
+
 /// Add `img` to `doc` as an image XObject. An ICC profile becomes an `/ICCBased` colour space
 /// (with the device space as `/Alternate`); `profiles` shares one object between images that
 /// carry the same profile.
+///
+/// A black-and-white image is stored at one bit per pixel, as CCITT Group 4 when that is smaller
+/// than Flate: losslessly, and several times smaller than the eight bits a scan saved as a 1-bit
+/// PNG decodes to. RGB keeps its colour space when it has a colour profile.
 fn add_image(doc: &mut Document, img: Embedded, profiles: &mut Vec<(Vec<u8>, ObjRef)>) -> ObjRef {
     let mut d = img.dict;
     d.set(b"Type".to_vec(), Object::name("XObject"));
@@ -118,12 +151,48 @@ fn add_image(doc: &mut Document, img: Embedded, profiles: &mut Vec<(Vec<u8>, Obj
     d.set(b"Width".to_vec(), Object::Int(img.px.0 as i64));
     d.set(b"Height".to_vec(), Object::Int(img.px.1 as i64));
     let device = d.name(b"ColorSpace").map(<[u8]>::to_vec);
-    let n = match device.as_deref() {
+    let mut n = match device.as_deref() {
         Some(b"DeviceGray") => 1,
         Some(b"DeviceRGB") => 3,
         Some(b"DeviceCMYK") => 4,
         _ => 0,
     };
+    let (mut data, mut filtered) = (img.data, img.filtered);
+    let profiled_rgb = n == 3 && img.icc.is_some();
+    if !filtered
+        && !profiled_rgb
+        && let Some(packed) = bilevel(&data, img.px, n, d.int(b"BitsPerComponent"))
+    {
+        if n == 3 {
+            d.set(b"ColorSpace".to_vec(), Object::name("DeviceGray"));
+            n = 1;
+        }
+        d.set(b"BitsPerComponent".to_vec(), Object::Int(1));
+        // The smaller as stored, dictionary included: Group 4's parameters outweigh its saving on
+        // a tiny image.
+        let stored = |d: &Dict, data: &[u8]| {
+            let mut out = Vec::new();
+            serialize(&Object::Dict(d.clone()), &mut out);
+            out.len().saturating_add(data.len())
+        };
+        let mut flate_dict = d.clone();
+        flate_dict.set(b"Filter".to_vec(), Object::name("FlateDecode"));
+        let flate = pdfcraft_filters::encode_flate(&packed);
+        (d, data) = match pdfcraft_filters::encode_g4(&packed, img.px.0, img.px.1) {
+            Ok(g4) => {
+                let mut parms = Dict::new();
+                parms.set(b"K".to_vec(), Object::Int(-1));
+                parms.set(b"Columns".to_vec(), Object::Int(i64::from(img.px.0)));
+                parms.set(b"Rows".to_vec(), Object::Int(i64::from(img.px.1)));
+                let mut g4_dict = d.clone();
+                g4_dict.set(b"Filter".to_vec(), Object::name("CCITTFaxDecode"));
+                g4_dict.set(b"DecodeParms".to_vec(), Object::Dict(parms));
+                if stored(&g4_dict, &g4) < stored(&flate_dict, &flate) { (g4_dict, g4) } else { (flate_dict, flate) }
+            }
+            Err(_) => (flate_dict, flate),
+        };
+        filtered = true;
+    }
     if let (Some(icc), Some(device)) = (img.icc.and_then(|p| icc_for(p, n)), device) {
         let r = match profiles.iter().find(|(p, _)| *p == icc) {
             Some((_, r)) => *r,
@@ -142,7 +211,7 @@ fn add_image(doc: &mut Document, img: Embedded, profiles: &mut Vec<(Vec<u8>, Obj
         let mr = doc.add(Object::Stream(Stream::flate(m, &alpha)));
         d.set(b"SMask".to_vec(), Object::Ref(mr));
     }
-    let stream = if img.filtered { Stream::from_raw(d, img.data) } else { Stream::flate(d, &img.data) };
+    let stream = if filtered { Stream::from_raw(d, data) } else { Stream::flate(d, &data) };
     doc.add(Object::Stream(stream))
 }
 

crates/create/src/tests.rs

@@ -1338,3 +1338,171 @@ fn a_tiny_gif_on_a_huge_screen_is_refused() {
     let doc = reopen(&from_images(&[("small.gif".into(), gif_on_screen(40, 30))]).unwrap());
     assert_eq!(media(&pages(&doc)[0]), [0.0, 0.0, 40.0, 30.0]);
 }
+
+/// A PNG of `w` × `h` samples `data` in `color` at `depth`, with a palette or an ICC profile.
+fn png_of(w: u32, h: u32, color: png::ColorType, depth: png::BitDepth, data: &[u8], palette: Option<Vec<u8>>, icc: Option<Vec<u8>>) -> Vec<u8> {
+    let mut out = Vec::new();
+    {
+        let mut info = png::Info::with_size(w, h);
+        info.color_type = color;
+        info.bit_depth = depth;
+        info.palette = palette.map(Into::into);
+        info.icc_profile = icc.map(Into::into);
+        let mut writer = png::Encoder::with_info(&mut out, info).unwrap().write_header().unwrap();
+        writer.write_image_data(data).unwrap();
+    }
+    out
+}
+
+/// Rows of `w` one-bit samples (1 white), each on a new byte: lines of glyphs drawn from a few
+/// shapes at uneven spacing, whose edges move a pixel now and then, as a scan of text has them.
+/// (Clean, evenly spaced strokes favour Flate over Group 4.)
+fn strokes(w: u32, h: u32) -> Vec<u8> {
+    let stride = (w as usize).div_ceil(8);
+    let mut data = vec![0xFF; stride * h as usize];
+    let mut seed = 0x9E37_79B9_7F4A_7C15u64;
+    let mut next = move || {
+        seed ^= seed << 13;
+        seed ^= seed >> 7;
+        seed ^= seed << 17;
+        seed
+    };
+    let (w, h) = (w as usize, h as usize);
+    for line in (4..h.saturating_sub(18)).step_by(24) {
+        let mut x = 8;
+        while x + 16 < w {
+            let wide = 5 + (next() % 4) as usize;
+            let shape = next() % 4;
+            let mut fill = |x0: usize, x1: usize, y0: usize, y1: usize| {
+                for y in line + y0..line + y1 {
+                    let ragged = |r: u64| match r % 8 {
+                        0 => 0,
+                        1 => 2,
+                        _ => 1,
+                    };
+                    let (left, right) = ((x + x0 + 1).saturating_sub(ragged(next())), x + x1 + ragged(next()));
+                    for xx in left..right.saturating_sub(1).min(w) {
+                        data[y * stride + xx / 8] &= !(0x80 >> (xx % 8));
+                    }
+                }
+            };
+            match shape {
+                0 => fill(0, 2, 0, 14),
+                1 => {
+                    fill(0, wide, 4, 6);
+                    fill(0, wide, 12, 14);
+                    fill(0, 2, 4, 14);
+                    fill(wide - 2, wide, 4, 14);
+                }
+                2 => {
+                    fill(0, 2, 4, 14);
+                    fill(wide - 2, wide, 6, 14);
+                    fill(0, wide - 1, 4, 6);
+                }
+                _ => {
+                    fill(0, 2, 7, 14);
+                    fill(0, 2, 2, 4);
+                }
+            }
+            x += wide + 2 + (next() % 2) as usize + if next() % 6 == 0 { 7 } else { 0 };
+        }
+    }
+    data
+}
+
+/// The one-bit samples (true for white) a Group 4 image stream decodes to, through an
+/// independent decoder.
+fn g4_rows(s: &Stream) -> Vec<Vec<bool>> {
+    struct Rows(Vec<Vec<bool>>, Vec<bool>, usize);
+    impl hayro_ccitt::Decoder for Rows {
+        fn push_pixel(&mut self, white: bool) {
+            self.1.push(white);
+        }
+        fn push_pixel_chunk(&mut self, white: bool, chunks: u32) {
+            self.1.extend(std::iter::repeat_n(white, chunks as usize * 8));
+        }
+        fn next_line(&mut self) {
+            let mut row = std::mem::take(&mut self.1);
+            row.truncate(self.2);
+            self.0.push(row);
+        }
+    }
+    let (w, h) = (s.dict.int(b"Width").unwrap() as u32, s.dict.int(b"Height").unwrap() as u32);
+    let settings = hayro_ccitt::DecodeSettings {
+        columns: w,
+        rows: h,
+        end_of_block: true,
+        end_of_line: false,
+        rows_are_byte_aligned: false,
+        encoding: hayro_ccitt::EncodingMode::Group4,
+        invert_black: false,
+    };
+    let mut out = Rows(Vec::new(), Vec::new(), w as usize);
+    hayro_ccitt::decode(&s.raw, &mut out, &mut hayro_ccitt::DecoderContext::new(settings)).unwrap();
+    out.0
+}
+
+fn image_stream(doc: &Document, page: usize) -> Stream {
+    let p = &pages(doc)[page];
+    let res = doc.resolve(p.get(b"Resources").unwrap()).as_dict().cloned().unwrap();
+    let xo = doc.resolve(res.get(b"XObject").unwrap()).as_dict().cloned().unwrap();
+    match &*doc.resolve(xo.get(b"Im0").unwrap()) {
+        Object::Stream(s) => s.clone(),
+        _ => panic!("not an image"),
+    }
+}
+
+/// Black-and-white pictures are stored at one bit per pixel, as CCITT Group 4 when that is
+/// smaller, whichever way the file holds them: a 1-bit PNG (which decodes to eight bits), eight-bit
+/// grey of only black and white, a two-colour palette, black-and-white RGB. The bits are the
+/// picture's. A grey profile stays; a picture with any other shade, colour, or an RGB profile is
+/// stored as before.
+#[test]
+fn black_and_white_images_are_stored_at_one_bit() {
+    use png::{BitDepth, ColorType};
+    let (w, h) = (1200u32, 240u32);
+    let bits = strokes(w, h);
+    let stride = (w as usize).div_ceil(8);
+    let white = |x: usize, y: usize| bits[y * stride + x / 8] & (0x80 >> (x % 8)) != 0;
+    let expected: Vec<Vec<bool>> = (0..h as usize).map(|y| (0..w as usize).map(|x| white(x, y)).collect()).collect();
+    let grey: Vec<u8> = expected.iter().flatten().map(|&wh| if wh { 255 } else { 0 }).collect();
+    let rgb: Vec<u8> = grey.iter().flat_map(|&v| [v, v, v]).collect();
+    let pictures = [
+        png_of(w, h, ColorType::Grayscale, BitDepth::One, &bits, None, None),
+        png_of(w, h, ColorType::Grayscale, BitDepth::Eight, &grey, None, None),
+        png_of(w, h, ColorType::Indexed, BitDepth::One, &bits.iter().map(|b| !b).collect::<Vec<_>>(), Some(vec![255, 255, 255, 0, 0, 0]), None),
+        png_of(w, h, ColorType::Rgb, BitDepth::Eight, &rgb, None, None),
+        png_of(w, h, ColorType::Grayscale, BitDepth::One, &bits, None, Some(icc_profile(b"GRAY"))),
+    ];
+    let files: Vec<(String, Vec<u8>)> = pictures.into_iter().enumerate().map(|(i, p)| (format!("{i}.png"), p)).collect();
+    let doc = reopen(&from_images(&files).unwrap());
+    for page in 0..5 {
+        let s = image_stream(&doc, page);
+        assert_eq!((s.dict.int(b"BitsPerComponent"), s.dict.name(b"Filter")), (Some(1), Some(&b"CCITTFaxDecode"[..])), "page {page}");
+        assert_eq!(g4_rows(&s), expected, "page {page}");
+        assert!(s.raw.len() < bits.len(), "page {page}: {} bytes, smaller than its one-bit samples", s.raw.len());
+    }
+    assert_eq!(image_of(&doc, 3).name(b"ColorSpace"), Some(&b"DeviceGray"[..]), "black-and-white RGB becomes grey");
+    assert_eq!(icc_of(&doc, &image_of(&doc, 4)).map(|(n, ..)| n), Some(1), "the grey profile stays");
+
+    // Another shade, colour, an RGB profile: as before. A tiny picture: one bit, but Flate.
+    let mut shaded = grey.clone();
+    shaded[5] = 128;
+    let mut coloured = rgb.clone();
+    coloured[0] = 200;
+    let others = [
+        png_of(w, h, ColorType::Grayscale, BitDepth::Eight, &shaded, None, None),
+        png_of(w, h, ColorType::Rgb, BitDepth::Eight, &coloured, None, None),
+        png_of(w, h, ColorType::Rgb, BitDepth::Eight, &rgb, None, Some(icc_profile(b"RGB "))),
+        png_of(2, 2, ColorType::Grayscale, BitDepth::Eight, &[0, 255, 255, 0], None, None),
+    ];
+    let files: Vec<(String, Vec<u8>)> = others.into_iter().enumerate().map(|(i, p)| (format!("o{i}.png"), p)).collect();
+    let doc = reopen(&from_images(&files).unwrap());
+    let kept: Vec<(Option<i64>, Option<Vec<u8>>)> =
+        (0..4).map(|p| (image_of(&doc, p).int(b"BitsPerComponent"), image_of(&doc, p).name(b"Filter").map(<[u8]>::to_vec))).collect();
+    assert_eq!(
+        kept[..3],
+        [(Some(8), Some(b"FlateDecode".to_vec())), (Some(8), Some(b"FlateDecode".to_vec())), (Some(8), Some(b"FlateDecode".to_vec()))]
+    );
+    assert_eq!(kept[3], (Some(1), Some(b"FlateDecode".to_vec())), "Group 4 isn't smaller for 2 × 2");
+}