From 9883da3bc33397d522b086fa0035ecbdaa7a4def Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Sat, 24 Oct 2026 00:56:00 +0200 Subject: [PATCH] M11: Create PDF stores black-and-white images at one bit, as Group 4 when smaller A scan saved as a 1-bit PNG decodes to eight-bit grey and was stored that way, larger than the PNG it came from; a bilevel TIFF was stored at one bit but with Flate. Now any image of only pure black and white (1-bit, eight-bit grey, a two-colour palette, RGB without a profile) is stored at one bit per pixel, as CCITT Group 4 or Flate, whichever is smaller as stored. A grey profile stays; RGB with a profile keeps its colours. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BfMGHQNgdq5yf5tKZHGR3z --- Cargo.lock | 2 + crates/create/Cargo.toml | 2 + crates/create/src/lib.rs | 75 ++++++++++++++++- crates/create/src/tests.rs | 168 +++++++++++++++++++++++++++++++++++++ 4 files changed, 244 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 42426bc688..e2427a1b0a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4333,9 +4333,11 @@ name = "pdfcraft-create" version = "0.15.0" dependencies = [ "encoding_rs", + "hayro-ccitt", "hayro-syntax", "image", "pdfcraft-cos", + "pdfcraft-filters", "pdfcraft-fonts", "pdfcraft-model", "pdfcraft-xfa", diff --git a/crates/create/Cargo.toml b/crates/create/Cargo.toml index 12daad621c..56cebc2af6 100644 --- a/crates/create/Cargo.toml +++ b/crates/create/Cargo.toml @@ -10,6 +10,7 @@ workspace = true [dependencies] pdfcraft-cos = { workspace = true } +pdfcraft-filters = { workspace = true } pdfcraft-model = { workspace = true } pdfcraft-fonts = { workspace = true } # For `image::bmp_with_pixel_offset`, shared with XFA pictures. @@ -24,5 +25,6 @@ thiserror = { workspace = true } encoding_rs = "0.8" [dev-dependencies] +hayro-ccitt.workspace = true image = { version = "0.25", default-features = false, features = ["bmp", "gif", "jpeg"] } hayro-syntax = { workspace = true } diff --git a/crates/create/src/lib.rs b/crates/create/src/lib.rs index a6fb28b3d6..36b52d4211 100644 --- a/crates/create/src/lib.rs +++ b/crates/create/src/lib.rs @@ -2,7 +2,7 @@ #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] -use pdfcraft_cos::{Dict, Document, ObjRef, Object, PdfString, Stream}; +use pdfcraft_cos::{Dict, Document, ObjRef, Object, PdfString, Stream, serialize}; mod extract; mod upright_image; @@ -108,9 +108,42 @@ fn jpeg_icc(mut chunks: Vec<(u8, u8, &[u8])>) -> Option> { complete.then(|| chunks.iter().flat_map(|c| c.2.iter().copied()).collect()) } +/// The samples of a black-and-white image packed one bit per pixel (1 for white, each row +/// starting on a byte): eight-bit grey (`n` 1) or RGB (`n` 3) whose every pixel is pure black or +/// pure white, or one-bit grey, already packed (a bilevel TIFF). `None` for anything else, found +/// at its first other pixel. +fn bilevel(data: &[u8], (w, h): (u32, u32), n: usize, bits: Option) -> Option> { + let (w, h) = (usize::try_from(w).ok()?, usize::try_from(h).ok()?); + let stride = w.div_ceil(8); + let size = stride.checked_mul(h)?; + match (bits, n) { + (Some(1), 1) => data.get(..size).map(<[u8]>::to_vec), + (Some(8), 1 | 3) if w > 0 => { + let samples = data.get(..w.checked_mul(h)?.checked_mul(n)?)?; + let mut packed = vec![0u8; size]; + for (i, px) in samples.chunks_exact(n).enumerate() { + let white = match px { + [0] | [0, 0, 0] => false, + [255] | [255, 255, 255] => true, + _ => return None, + }; + if white && let Some(b) = packed.get_mut((i / w) * stride + (i % w) / 8) { + *b |= 0x80 >> (i % w % 8); + } + } + Some(packed) + } + _ => None, + } +} + /// Add `img` to `doc` as an image XObject. An ICC profile becomes an `/ICCBased` colour space /// (with the device space as `/Alternate`); `profiles` shares one object between images that /// carry the same profile. +/// +/// A black-and-white image is stored at one bit per pixel, as CCITT Group 4 when that is smaller +/// than Flate: losslessly, and several times smaller than the eight bits a scan saved as a 1-bit +/// PNG decodes to. RGB keeps its colour space when it has a colour profile. fn add_image(doc: &mut Document, img: Embedded, profiles: &mut Vec<(Vec, ObjRef)>) -> ObjRef { let mut d = img.dict; d.set(b"Type".to_vec(), Object::name("XObject")); @@ -118,12 +151,48 @@ fn add_image(doc: &mut Document, img: Embedded, profiles: &mut Vec<(Vec, Obj d.set(b"Width".to_vec(), Object::Int(img.px.0 as i64)); d.set(b"Height".to_vec(), Object::Int(img.px.1 as i64)); let device = d.name(b"ColorSpace").map(<[u8]>::to_vec); - let n = match device.as_deref() { + let mut n = match device.as_deref() { Some(b"DeviceGray") => 1, Some(b"DeviceRGB") => 3, Some(b"DeviceCMYK") => 4, _ => 0, }; + let (mut data, mut filtered) = (img.data, img.filtered); + let profiled_rgb = n == 3 && img.icc.is_some(); + if !filtered + && !profiled_rgb + && let Some(packed) = bilevel(&data, img.px, n, d.int(b"BitsPerComponent")) + { + if n == 3 { + d.set(b"ColorSpace".to_vec(), Object::name("DeviceGray")); + n = 1; + } + d.set(b"BitsPerComponent".to_vec(), Object::Int(1)); + // The smaller as stored, dictionary included: Group 4's parameters outweigh its saving on + // a tiny image. + let stored = |d: &Dict, data: &[u8]| { + let mut out = Vec::new(); + serialize(&Object::Dict(d.clone()), &mut out); + out.len().saturating_add(data.len()) + }; + let mut flate_dict = d.clone(); + flate_dict.set(b"Filter".to_vec(), Object::name("FlateDecode")); + let flate = pdfcraft_filters::encode_flate(&packed); + (d, data) = match pdfcraft_filters::encode_g4(&packed, img.px.0, img.px.1) { + Ok(g4) => { + let mut parms = Dict::new(); + parms.set(b"K".to_vec(), Object::Int(-1)); + parms.set(b"Columns".to_vec(), Object::Int(i64::from(img.px.0))); + parms.set(b"Rows".to_vec(), Object::Int(i64::from(img.px.1))); + let mut g4_dict = d.clone(); + g4_dict.set(b"Filter".to_vec(), Object::name("CCITTFaxDecode")); + g4_dict.set(b"DecodeParms".to_vec(), Object::Dict(parms)); + if stored(&g4_dict, &g4) < stored(&flate_dict, &flate) { (g4_dict, g4) } else { (flate_dict, flate) } + } + Err(_) => (flate_dict, flate), + }; + filtered = true; + } if let (Some(icc), Some(device)) = (img.icc.and_then(|p| icc_for(p, n)), device) { let r = match profiles.iter().find(|(p, _)| *p == icc) { Some((_, r)) => *r, @@ -142,7 +211,7 @@ fn add_image(doc: &mut Document, img: Embedded, profiles: &mut Vec<(Vec, Obj let mr = doc.add(Object::Stream(Stream::flate(m, &alpha))); d.set(b"SMask".to_vec(), Object::Ref(mr)); } - let stream = if img.filtered { Stream::from_raw(d, img.data) } else { Stream::flate(d, &img.data) }; + let stream = if filtered { Stream::from_raw(d, data) } else { Stream::flate(d, &data) }; doc.add(Object::Stream(stream)) } diff --git a/crates/create/src/tests.rs b/crates/create/src/tests.rs index 5cd34a0f1a..f94fc020a1 100644 --- a/crates/create/src/tests.rs +++ b/crates/create/src/tests.rs @@ -1338,3 +1338,171 @@ fn a_tiny_gif_on_a_huge_screen_is_refused() { let doc = reopen(&from_images(&[("small.gif".into(), gif_on_screen(40, 30))]).unwrap()); assert_eq!(media(&pages(&doc)[0]), [0.0, 0.0, 40.0, 30.0]); } + +/// A PNG of `w` × `h` samples `data` in `color` at `depth`, with a palette or an ICC profile. +fn png_of(w: u32, h: u32, color: png::ColorType, depth: png::BitDepth, data: &[u8], palette: Option>, icc: Option>) -> Vec { + let mut out = Vec::new(); + { + let mut info = png::Info::with_size(w, h); + info.color_type = color; + info.bit_depth = depth; + info.palette = palette.map(Into::into); + info.icc_profile = icc.map(Into::into); + let mut writer = png::Encoder::with_info(&mut out, info).unwrap().write_header().unwrap(); + writer.write_image_data(data).unwrap(); + } + out +} + +/// Rows of `w` one-bit samples (1 white), each on a new byte: lines of glyphs drawn from a few +/// shapes at uneven spacing, whose edges move a pixel now and then, as a scan of text has them. +/// (Clean, evenly spaced strokes favour Flate over Group 4.) +fn strokes(w: u32, h: u32) -> Vec { + let stride = (w as usize).div_ceil(8); + let mut data = vec![0xFF; stride * h as usize]; + let mut seed = 0x9E37_79B9_7F4A_7C15u64; + let mut next = move || { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + seed + }; + let (w, h) = (w as usize, h as usize); + for line in (4..h.saturating_sub(18)).step_by(24) { + let mut x = 8; + while x + 16 < w { + let wide = 5 + (next() % 4) as usize; + let shape = next() % 4; + let mut fill = |x0: usize, x1: usize, y0: usize, y1: usize| { + for y in line + y0..line + y1 { + let ragged = |r: u64| match r % 8 { + 0 => 0, + 1 => 2, + _ => 1, + }; + let (left, right) = ((x + x0 + 1).saturating_sub(ragged(next())), x + x1 + ragged(next())); + for xx in left..right.saturating_sub(1).min(w) { + data[y * stride + xx / 8] &= !(0x80 >> (xx % 8)); + } + } + }; + match shape { + 0 => fill(0, 2, 0, 14), + 1 => { + fill(0, wide, 4, 6); + fill(0, wide, 12, 14); + fill(0, 2, 4, 14); + fill(wide - 2, wide, 4, 14); + } + 2 => { + fill(0, 2, 4, 14); + fill(wide - 2, wide, 6, 14); + fill(0, wide - 1, 4, 6); + } + _ => { + fill(0, 2, 7, 14); + fill(0, 2, 2, 4); + } + } + x += wide + 2 + (next() % 2) as usize + if next() % 6 == 0 { 7 } else { 0 }; + } + } + data +} + +/// The one-bit samples (true for white) a Group 4 image stream decodes to, through an +/// independent decoder. +fn g4_rows(s: &Stream) -> Vec> { + struct Rows(Vec>, Vec, usize); + impl hayro_ccitt::Decoder for Rows { + fn push_pixel(&mut self, white: bool) { + self.1.push(white); + } + fn push_pixel_chunk(&mut self, white: bool, chunks: u32) { + self.1.extend(std::iter::repeat_n(white, chunks as usize * 8)); + } + fn next_line(&mut self) { + let mut row = std::mem::take(&mut self.1); + row.truncate(self.2); + self.0.push(row); + } + } + let (w, h) = (s.dict.int(b"Width").unwrap() as u32, s.dict.int(b"Height").unwrap() as u32); + let settings = hayro_ccitt::DecodeSettings { + columns: w, + rows: h, + end_of_block: true, + end_of_line: false, + rows_are_byte_aligned: false, + encoding: hayro_ccitt::EncodingMode::Group4, + invert_black: false, + }; + let mut out = Rows(Vec::new(), Vec::new(), w as usize); + hayro_ccitt::decode(&s.raw, &mut out, &mut hayro_ccitt::DecoderContext::new(settings)).unwrap(); + out.0 +} + +fn image_stream(doc: &Document, page: usize) -> Stream { + let p = &pages(doc)[page]; + let res = doc.resolve(p.get(b"Resources").unwrap()).as_dict().cloned().unwrap(); + let xo = doc.resolve(res.get(b"XObject").unwrap()).as_dict().cloned().unwrap(); + match &*doc.resolve(xo.get(b"Im0").unwrap()) { + Object::Stream(s) => s.clone(), + _ => panic!("not an image"), + } +} + +/// Black-and-white pictures are stored at one bit per pixel, as CCITT Group 4 when that is +/// smaller, whichever way the file holds them: a 1-bit PNG (which decodes to eight bits), eight-bit +/// grey of only black and white, a two-colour palette, black-and-white RGB. The bits are the +/// picture's. A grey profile stays; a picture with any other shade, colour, or an RGB profile is +/// stored as before. +#[test] +fn black_and_white_images_are_stored_at_one_bit() { + use png::{BitDepth, ColorType}; + let (w, h) = (1200u32, 240u32); + let bits = strokes(w, h); + let stride = (w as usize).div_ceil(8); + let white = |x: usize, y: usize| bits[y * stride + x / 8] & (0x80 >> (x % 8)) != 0; + let expected: Vec> = (0..h as usize).map(|y| (0..w as usize).map(|x| white(x, y)).collect()).collect(); + let grey: Vec = expected.iter().flatten().map(|&wh| if wh { 255 } else { 0 }).collect(); + let rgb: Vec = grey.iter().flat_map(|&v| [v, v, v]).collect(); + let pictures = [ + png_of(w, h, ColorType::Grayscale, BitDepth::One, &bits, None, None), + png_of(w, h, ColorType::Grayscale, BitDepth::Eight, &grey, None, None), + png_of(w, h, ColorType::Indexed, BitDepth::One, &bits.iter().map(|b| !b).collect::>(), Some(vec![255, 255, 255, 0, 0, 0]), None), + png_of(w, h, ColorType::Rgb, BitDepth::Eight, &rgb, None, None), + png_of(w, h, ColorType::Grayscale, BitDepth::One, &bits, None, Some(icc_profile(b"GRAY"))), + ]; + let files: Vec<(String, Vec)> = pictures.into_iter().enumerate().map(|(i, p)| (format!("{i}.png"), p)).collect(); + let doc = reopen(&from_images(&files).unwrap()); + for page in 0..5 { + let s = image_stream(&doc, page); + assert_eq!((s.dict.int(b"BitsPerComponent"), s.dict.name(b"Filter")), (Some(1), Some(&b"CCITTFaxDecode"[..])), "page {page}"); + assert_eq!(g4_rows(&s), expected, "page {page}"); + assert!(s.raw.len() < bits.len(), "page {page}: {} bytes, smaller than its one-bit samples", s.raw.len()); + } + assert_eq!(image_of(&doc, 3).name(b"ColorSpace"), Some(&b"DeviceGray"[..]), "black-and-white RGB becomes grey"); + assert_eq!(icc_of(&doc, &image_of(&doc, 4)).map(|(n, ..)| n), Some(1), "the grey profile stays"); + + // Another shade, colour, an RGB profile: as before. A tiny picture: one bit, but Flate. + let mut shaded = grey.clone(); + shaded[5] = 128; + let mut coloured = rgb.clone(); + coloured[0] = 200; + let others = [ + png_of(w, h, ColorType::Grayscale, BitDepth::Eight, &shaded, None, None), + png_of(w, h, ColorType::Rgb, BitDepth::Eight, &coloured, None, None), + png_of(w, h, ColorType::Rgb, BitDepth::Eight, &rgb, None, Some(icc_profile(b"RGB "))), + png_of(2, 2, ColorType::Grayscale, BitDepth::Eight, &[0, 255, 255, 0], None, None), + ]; + let files: Vec<(String, Vec)> = others.into_iter().enumerate().map(|(i, p)| (format!("o{i}.png"), p)).collect(); + let doc = reopen(&from_images(&files).unwrap()); + let kept: Vec<(Option, Option>)> = + (0..4).map(|p| (image_of(&doc, p).int(b"BitsPerComponent"), image_of(&doc, p).name(b"Filter").map(<[u8]>::to_vec))).collect(); + assert_eq!( + kept[..3], + [(Some(8), Some(b"FlateDecode".to_vec())), (Some(8), Some(b"FlateDecode".to_vec())), (Some(8), Some(b"FlateDecode".to_vec()))] + ); + assert_eq!(kept[3], (Some(1), Some(b"FlateDecode".to_vec())), "Group 4 isn't smaller for 2 × 2"); +}