From d9d02c42be093bf4765eb55367ca1394038669ad Mon Sep 17 00:00:00 2001 From: aiongg Date: Sat, 24 Oct 2026 07:26:55 +0000 Subject: [PATCH 1/3] hayro-syntax: port PdfCraft's trailer /Prev reference (ledger 24) A trailer /Prev written as N G R is followed: it is the offset the referenced object holds when the section's own entries place it, and otherwise offset N. Each section's entries are read before its Prev is followed. From PdfCraft #2400, with its two xref unit tests; the local xref-stream bounds are kept. --- vendor/README.craft-reader.md | 10 ++ vendor/hayro-syntax/SOURCE.json | 2 +- vendor/hayro-syntax/src/xref.rs | 166 ++++++++++++++++++++++++++------ 3 files changed, 150 insertions(+), 28 deletions(-) diff --git a/vendor/README.craft-reader.md b/vendor/README.craft-reader.md index 1a560e73..51c7195b 100644 --- a/vendor/README.craft-reader.md +++ b/vendor/README.craft-reader.md @@ -176,6 +176,16 @@ raster tests are adapted in `tests/upstream_raster.rs` and the text contracts in `tests/upstream_text.rs`. (P8)'s PdfCraft test needs a Japanese substitute font and is not ported. +Fixes from the [PdfCraft revision 0b670caf4a090e9d83a7f41a84b4cb2144a2d33e ledger](https://github.com/storytold/pdfcraft/blob/0b670caf4a090e9d83a7f41a84b4cb2144a2d33e/vendor/README.md), +by its item ids, each marked `PdfCraft patch` in the source. hayro-syntax (24), PdfCraft +#2400: `xref::prev_offset` reads a trailer `/Prev` written as a reference `N G R` as the +offset that object holds when the section's own entries place it (iText's +`indirectPrev.pdf`), and otherwise as offset N (tiff2pdf), so the earlier sections' pages +are seen. Each section's entries are read before its `Prev` is followed; the local +xref-stream bounds above apply to them unchanged. Tests: +`xref::tests::prev_as_reference_reaches_the_earlier_section` and +`xref::tests::prev_as_reference_to_an_object_holding_the_offset_reads_that_offset`. + This remains the patched 0.7 backend. It retains the existing Bradford Lab conversion, floating component precision and codec versions, rather than claiming full 0.8 equivalence. The independent color oracles, backend tests and facade contracts cover diff --git a/vendor/hayro-syntax/SOURCE.json b/vendor/hayro-syntax/SOURCE.json index d2234ad3..c3838bc9 100644 --- a/vendor/hayro-syntax/SOURCE.json +++ b/vendor/hayro-syntax/SOURCE.json @@ -338,7 +338,7 @@ "path": "src/xref.rs", "source_sha256": "51ffeeb7d4ac035bae2cd5ba9288c180088ae06cdcefe2b29dfeda560a793aa0", "source_git_blob": "20e33a410489d42e438a3f2b87d70c7bb8d66eae", - "sha256": "fa8d24280df15ffe5628057fdcfa2a3d2a9f42539b6f9096b2feb836ac54ac30" + "sha256": "98fb56eb5c2da8b0c40202ed30b2f464874c1143436e72eda4e49f2333dd7c7f" }, { "path": "src/decode_limits_tests.rs", diff --git a/vendor/hayro-syntax/src/xref.rs b/vendor/hayro-syntax/src/xref.rs index 15858ac4..4f9de535 100644 --- a/vendor/hayro-syntax/src/xref.rs +++ b/vendor/hayro-syntax/src/xref.rs @@ -759,6 +759,34 @@ fn populate_xref_impl<'a>(data: &'a [u8], pos: usize, xref_map: &mut XrefMap, li populate_xref_impl_inner(data, pos, xref_map, &mut visited, limits) } +/// The offset of the previous xref section (`Prev`). +// PdfCraft patch: (PdfCraft #2400) some writers give `Prev` as a reference `N G R`. When this +// section's own entries (`own`) place that object and it holds an offset (iText's +// `indirectPrev.pdf`), that is the offset; otherwise its object number is (tiff2pdf, in JHOVE's +// PDF-HUL-157 files), as other readers take it. +fn prev_offset(data: &[u8], trailer: &Dict<'_>, own: &XrefMap) -> Option { + if let Some(prev) = trailer.get::(PREV) { + return usize::try_from(prev).ok(); + } + + let r = trailer.get_ref(PREV)?; + let id = ObjectIdentifier::new(r.obj_number, r.gen_number); + let held = match own.get(&id) { + Some(EntryType::Normal(at)) => { + let mut reader = Reader::new(data); + reader.jump(*at); + reader + .read_with_context::>(&ReaderContext::dummy()) + .filter(|o| o.id() == &id) + .map(|o| o.get()) + .filter(|p| *p >= 0) + } + _ => None, + }; + + usize::try_from(held.unwrap_or(r.obj_number)).ok() +} + /// Maximum number of allowed xref `Prev` pointers before we abort. const MAX_XREF_CHAIN_DEPTH: usize = 256; @@ -832,17 +860,8 @@ fn populate_from_xref_table<'a>( reader.skip_white_spaces(); let mut max_obj = 0; - - if let Some(prev) = trailer.get::(PREV) { - // First insert the entries from any previous xref tables. - populate_xref_impl_inner(data, prev as usize, insert_map, visited, limits)?; - } - - // In hybrid files, entries in `XRefStm` should have higher priority, therefore we insert them - // after looking at `PREV`. - if let Some(xref_stm) = trailer.get::(XREF_STM) { - populate_xref_impl_inner(data, xref_stm as usize, insert_map, visited, limits)?; - } + // PdfCraft patch: (PdfCraft #2400) this section's entries are read first, for `prev_offset`. + let mut own = XrefMap::default(); while let Some(header) = reader.read_without_context::() { reader.skip_white_spaces(); @@ -858,7 +877,7 @@ fn populate_from_xref_table<'a>( // Specification says we should ignore any object number > SIZE, but probably // not important? if entry.used { - insert_map.insert( + own.insert( ObjectIdentifier::new(obj_number as i32, entry.gen_number), EntryType::Normal(entry.offset), ); @@ -866,6 +885,19 @@ fn populate_from_xref_table<'a>( } } + if let Some(prev) = prev_offset(data, &trailer, &own) { + // First insert the entries from any previous xref tables. + populate_xref_impl_inner(data, prev, insert_map, visited, limits)?; + } + + // In hybrid files, entries in `XRefStm` should have higher priority, therefore we insert them + // after looking at `PREV`. + if let Some(xref_stm) = trailer.get::(XREF_STM) { + populate_xref_impl_inner(data, xref_stm as usize, insert_map, visited, limits)?; + } + + insert_map.extend(own); + Some(trailer.data()) } @@ -878,11 +910,6 @@ fn populate_from_xref_stream<'a>( ) -> Option<&'a [u8]> { let stream = reader.read_with_context::>>(&ReaderContext::dummy_with_limits(limits))?.get(); - if let Some(prev) = stream.dict().get::(PREV) { - // First insert the entries from any previous xref tables. - let _ = populate_xref_impl_inner(data, prev as usize, insert_map, visited, limits)?; - } - let size = stream.dict().get::(SIZE)?; let [f1_len, f2_len, f3_len] = stream.dict().get::<[u8; 3]>(W)?; @@ -895,6 +922,8 @@ fn populate_from_xref_stream<'a>( } let xref_data = stream.decoded().ok()?; + // PdfCraft patch: (PdfCraft #2400) this section's entries are read first, for `prev_offset`. + let mut own = XrefMap::default(); let mut xref_reader = Reader::new(xref_data.as_ref()); if let Some(arr) = stream.dict().get::>(INDEX) { @@ -908,21 +937,20 @@ fn populate_from_xref_stream<'a>( f1_len, f2_len, f3_len, - insert_map, + &mut own, )?; } } else { - xref_stream_subsection( - &mut xref_reader, - 0, - size, - f1_len, - f2_len, - f3_len, - insert_map, - )?; + xref_stream_subsection(&mut xref_reader, 0, size, f1_len, f2_len, f3_len, &mut own)?; } + if let Some(prev) = prev_offset(data, stream.dict(), &own) { + // First insert the entries from any previous xref tables. + let _ = populate_xref_impl_inner(data, prev, insert_map, visited, limits)?; + } + + insert_map.extend(own); + Some(stream.dict().data()) } @@ -1450,6 +1478,90 @@ mod tests { let _result = populate_xref_impl(pdf.as_ref(), xref_pos, &mut xref_map, DecodeLimits::default()); } + #[test] + fn prev_as_reference_reaches_the_earlier_section() { + let mut pdf = b"%PDF-1.0\n1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n".to_vec(); + let first_xref_pos = pdf.len(); + pdf.extend_from_slice( + format!( + "xref\n\ + 0 2\n\ + 0000000000 65535 f \n\ + 0000000009 00000 n \n\ + trailer\n<< /Size 2 /Root 1 0 R >>\n\ + startxref\n{first_xref_pos}\n%%EOF\n" + ) + .as_bytes(), + ); + let second_xref_pos = pdf.len(); + pdf.extend_from_slice( + format!( + "xref\n\ + 0 1\n\ + 0000000000 65535 f \n\ + trailer\n<< /Size 2 /Root 1 0 R /Prev {first_xref_pos} 0 R >>\n\ + startxref\n{second_xref_pos}\n%%EOF" + ) + .as_bytes(), + ); + + let mut xref_map = FxHashMap::default(); + let xref_pos = find_last_xref_pos(pdf.as_ref()).unwrap(); + assert_eq!(xref_pos, second_xref_pos); + populate_xref_impl( + pdf.as_ref(), + xref_pos, + &mut xref_map, + DecodeLimits::default(), + ) + .unwrap(); + assert!(xref_map.contains_key(&ObjectIdentifier::new(1, 0))); + } + + #[test] + fn prev_as_reference_to_an_object_holding_the_offset_reads_that_offset() { + let mut pdf = b"%PDF-1.0\n1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n".to_vec(); + let first_xref_pos = pdf.len(); + pdf.extend_from_slice( + format!( + "xref\n\ + 0 2\n\ + 0000000000 65535 f \n\ + 0000000009 00000 n \n\ + trailer\n<< /Size 2 /Root 1 0 R >>\n\ + startxref\n{first_xref_pos}\n%%EOF\n" + ) + .as_bytes(), + ); + // Object 3 holds the offset; read as offset 3, `Prev` would point inside the header. + let held_pos = pdf.len(); + pdf.extend_from_slice(format!("3 0 obj\n{first_xref_pos}\nendobj\n").as_bytes()); + let second_xref_pos = pdf.len(); + pdf.extend_from_slice( + format!( + "xref\n\ + 3 1\n\ + {held_pos:010} 00000 n \n\ + trailer\n<< /Size 4 /Root 1 0 R /Prev 3 0 R >>\n\ + startxref\n{second_xref_pos}\n%%EOF" + ) + .as_bytes(), + ); + + let mut xref_map = FxHashMap::default(); + let xref_pos = find_last_xref_pos(pdf.as_ref()).unwrap(); + assert_eq!(xref_pos, second_xref_pos); + populate_xref_impl( + pdf.as_ref(), + xref_pos, + &mut xref_map, + DecodeLimits::default(), + ) + .unwrap(); + assert!(xref_map.contains_key(&ObjectIdentifier::new(1, 0))); + assert!(xref_map.contains_key(&ObjectIdentifier::new(3, 0))); + } + #[test] fn find_last_xref_uses_last_startxref() { let pdf = b"%PDF-1.0\nstartxref\n5\n%%EOF\nstartxref\n42\n%%EOF"; From d2ffefcddf5148ad50fcc49b910021423a519873 Mon Sep 17 00:00:00 2001 From: aiongg Date: Sat, 24 Oct 2026 07:32:06 +0000 Subject: [PATCH 2/3] hayro-syntax: port PdfCraft's JP2 palette without /ColorSpace (ledger P5) A JPX image whose dictionary has no /ColorSpace, or one that does not resolve, takes its colors from the JP2 palette instead of drawing its raw indices as gray. Resolved palette colors are 8-bit samples whatever the index precision. From PdfCraft #3184, with its three jpx unit tests. --- vendor/README.craft-reader.md | 10 ++ vendor/hayro-syntax/SOURCE.json | 2 +- vendor/hayro-syntax/src/filter/jpx.rs | 144 +++++++++++++++++++++++++- 3 files changed, 152 insertions(+), 4 deletions(-) diff --git a/vendor/README.craft-reader.md b/vendor/README.craft-reader.md index 51c7195b..cd135c5a 100644 --- a/vendor/README.craft-reader.md +++ b/vendor/README.craft-reader.md @@ -185,6 +185,16 @@ are seen. Each section's entries are read before its `Prev` is followed; the loc xref-stream bounds above apply to them unchanged. Tests: `xref::tests::prev_as_reference_reaches_the_earlier_section` and `xref::tests::prev_as_reference_to_an_object_holding_the_offset_reads_that_offset`. +hayro-syntax (P5), PdfCraft #3184: `filter::jpx::decode` applies a JP2 palette (`pclr`) when +the image dictionary has no `/ColorSpace`, or one that does not resolve, since the colors then +come from the JP2 header (ISO 32000-2 §7.4.9). With a `/ColorSpace` the raw indices are kept, +as an `/Indexed` space needs. Resolved palette colors are 8-bit samples whatever the +precision of the indices; an image without a palette keeps its own. Tests: +`filter::jpx::tests::jpx_palette_is_applied_without_a_color_space`, +`filter::jpx::tests::jpx_palette_indices_stay_raw_for_an_indexed_color_space` and +`filter::jpx::tests::palette_colours_are_eight_bit_and_other_samples_keep_their_precision`. +PdfCraft's raster test `a_jp2_palette_keeps_its_colours_whatever_the_index_precision` is not +ported: the unit tests show the same behavior. This remains the patched 0.7 backend. It retains the existing Bradford Lab conversion, floating component precision and codec versions, rather than claiming full 0.8 diff --git a/vendor/hayro-syntax/SOURCE.json b/vendor/hayro-syntax/SOURCE.json index c3838bc9..e02661ba 100644 --- a/vendor/hayro-syntax/SOURCE.json +++ b/vendor/hayro-syntax/SOURCE.json @@ -164,7 +164,7 @@ "path": "src/filter/jpx.rs", "source_sha256": "f0a2b01c692c086d30a26562d7548f9e837af166be737d2c543de7c886a45f82", "source_git_blob": "74efd8b31e2b2935ff2bbceda2c1ffd36ea9c20d", - "sha256": "f0a2b01c692c086d30a26562d7548f9e837af166be737d2c543de7c886a45f82" + "sha256": "b0c295f815547377409a8810e9cbef35e4d6b506a3c3507669da71fcfe9d4701" }, { "path": "src/filter/lzw_flate.rs", diff --git a/vendor/hayro-syntax/src/filter/jpx.rs b/vendor/hayro-syntax/src/filter/jpx.rs index 74efd8b3..7268bb04 100644 --- a/vendor/hayro-syntax/src/filter/jpx.rs +++ b/vendor/hayro-syntax/src/filter/jpx.rs @@ -21,7 +21,13 @@ pub(crate) fn decode(data: &[u8], params: &ImageDecodeParams) -> Option Option ImageColorSpace::Gray, ColorSpace::RGB => ImageColorSpace::Rgb, @@ -98,6 +112,15 @@ pub(crate) fn decode(data: &[u8], params: &ImageDecodeParams) -> Option bool { + boxes(data) + .find(|&(kind, _, _)| kind == *b"jp2h") + .and_then(|(_, start, end)| data.get(start..end)) + .is_some_and(|header| boxes(header).any(|(kind, _, _)| kind == *b"pclr")) +} + /// The enumerated colour spaces (`EnumCS`, ISO 15444-1 I.5.3.3 and 15444-2 M.11.7.2) the decoder /// handles: CMYK, CIELab, sRGB, greyscale, sYCC, e-sRGB and ROMM-RGB. const DECODED_ENUMERATED: [u32; 7] = [12, 14, 16, 17, 18, 20, 21]; @@ -221,8 +244,9 @@ fn scale( #[cfg(test)] mod tests { - use super::{scale, without_refused_enumerated_colour}; + use super::{decode, scale, without_refused_enumerated_colour}; use crate::bit_reader::{BitReader, bit_mask}; + use crate::object::stream::{ImageColorSpace, ImageDecodeParams}; use alloc::vec::Vec; #[test] @@ -358,4 +382,118 @@ mod tests { assert_eq!(without_refused_enumerated_colour(&data), None, "{bad:#x}"); } } + + /// A 4x2 JP2 written for this test: one 8-bit component whose samples (0, 1) index a + /// two-entry sRGB palette (`pclr` + `cmap`): red, then blue. Rows: 0 1 0 1 / 1 0 1 0. + const PALETTE_JP2: &[u8] = &[ + 0x00, 0x00, 0x00, 0x0c, 0x6a, 0x50, 0x20, 0x20, 0x0d, 0x0a, 0x87, 0x0a, 0x00, 0x00, 0x00, + 0x14, 0x66, 0x74, 0x79, 0x70, 0x6a, 0x70, 0x32, 0x20, 0x00, 0x00, 0x00, 0x00, 0x6a, 0x70, + 0x32, 0x20, 0x00, 0x00, 0x00, 0x55, 0x6a, 0x70, 0x32, 0x68, 0x00, 0x00, 0x00, 0x16, 0x69, + 0x68, 0x64, 0x72, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x04, 0x00, 0x01, 0x07, 0x07, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x0f, 0x63, 0x6f, 0x6c, 0x72, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x10, 0x00, 0x00, 0x00, 0x14, 0x70, 0x63, 0x6c, 0x72, 0x00, 0x02, 0x03, 0x07, 0x07, + 0x07, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0x00, 0x00, 0x00, 0x14, 0x63, 0x6d, 0x61, 0x70, + 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x02, 0x00, 0x00, 0x00, + 0x8c, 0x6a, 0x70, 0x32, 0x63, 0xff, 0x4f, 0xff, 0x51, 0x00, 0x29, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x04, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x01, 0x07, 0x01, 0x01, 0xff, 0x52, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x01, 0x00, 0x01, + 0x04, 0x04, 0x00, 0x01, 0xff, 0x5c, 0x00, 0x07, 0x40, 0x40, 0x48, 0x48, 0x50, 0xff, 0x64, + 0x00, 0x25, 0x00, 0x01, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x20, 0x62, 0x79, 0x20, + 0x4f, 0x70, 0x65, 0x6e, 0x4a, 0x50, 0x45, 0x47, 0x20, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, + 0x6e, 0x20, 0x32, 0x2e, 0x35, 0x2e, 0x34, 0xff, 0x90, 0x00, 0x0a, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x17, 0x00, 0x01, 0xff, 0x93, 0xcf, 0xb4, 0x08, 0x08, 0x67, 0x90, 0x07, 0x41, 0x08, + 0xff, 0xd9, + ]; + + /// PdfCraft #3179: with no `/ColorSpace` in the image dictionary, the JP2 palette gives the + /// colors. Raw indices (0 and 1) would come back as a gray image, drawn as noise. + #[test] + fn jpx_palette_is_applied_without_a_color_space() { + let params = ImageDecodeParams { + width: 4, + height: 2, + ..ImageDecodeParams::default() + }; + let result = decode(PALETTE_JP2, ¶ms).unwrap(); + let image = result.image_data.unwrap(); + assert!(matches!(image.color_space, Some(ImageColorSpace::Rgb))); + let red = [255, 0, 0]; + let blue = [0, 0, 255]; + let expected: Vec = [red, blue, red, blue, blue, red, blue, red].concat(); + assert_eq!(&*result.data, &expected[..]); + } + + /// With an `/Indexed` color space in the dictionary, the PDF supplies the palette, so the raw + /// indices are what the image holds. + #[test] + fn jpx_palette_indices_stay_raw_for_an_indexed_color_space() { + let params = ImageDecodeParams { + is_indexed: true, + num_components: Some(1), + width: 4, + height: 2, + ..ImageDecodeParams::default() + }; + let result = decode(PALETTE_JP2, ¶ms).unwrap(); + assert_eq!(&*result.data, &[0, 1, 0, 1, 1, 0, 1, 0][..]); + } + + /// A 1x1, 1-bit codestream (one empty packet, so its sample is the midpoint 1) as a JP2 file + /// whose header holds `extra` boxes after `ihdr` and `colr`. + fn one_bit_jp2(extra: &[u8]) -> Vec { + let jp2_box = |kind: &[u8], body: &[u8]| { + let mut b = ((8 + body.len()) as u32).to_be_bytes().to_vec(); + b.extend_from_slice(kind); + b.extend_from_slice(body); + b + }; + let mut codestream = alloc::vec![0xff, 0x4f, 0xff, 0x51, 0, 41, 0, 0]; + for value in [1_u32, 1, 0, 0, 1, 1, 0, 0] { + codestream.extend_from_slice(&value.to_be_bytes()); + } + codestream.extend_from_slice(&[0, 1, 0, 1, 1]); + codestream.extend_from_slice(&[0xff, 0x52, 0, 12, 0, 0, 0, 1, 0, 0, 4, 4, 0, 1]); + codestream.extend_from_slice(&[0xff, 0x5c, 0, 4, 0x40, 0x40]); + codestream.extend_from_slice(&[0xff, 0x90, 0, 10, 0, 0, 0, 0, 0, 15, 0, 1, 0xff, 0x93, 0]); + codestream.extend_from_slice(&[0xff, 0xd9]); + let mut header = jp2_box(b"ihdr", &[0, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0, 7, 0, 0]); + header.extend(jp2_box(b"colr", &[1, 0, 0, 0, 0, 0, 16])); + header.extend_from_slice(extra); + let mut out = alloc::vec![0, 0, 0, 12, b'j', b'P', b' ', b' ', 0x0d, 0x0a, 0x87, 0x0a]; + out.extend(jp2_box(b"ftyp", b"jp2 \0\0\0\0jp2 ")); + out.extend(jp2_box(b"jp2h", &header)); + out.extend(jp2_box(b"jp2c", &codestream)); + out + } + + /// PdfCraft #3184: resolved palette colors are 8-bit samples whatever the indices' precision; + /// an image without a palette keeps its own (1 bit here). + #[test] + fn palette_colours_are_eight_bit_and_other_samples_keep_their_precision() { + let params = ImageDecodeParams { + width: 1, + height: 1, + ..ImageDecodeParams::default() + }; + let bpc = |data: &[u8]| { + decode(data, ¶ms) + .and_then(|r| r.image_data) + .map(|d| d.bits_per_component) + }; + let mut palette = Vec::new(); + for (kind, body) in [ + ( + &b"pclr"[..], + &[0, 2, 3, 7, 7, 7, 255, 0, 0, 64, 128, 192][..], + ), + (b"cmap", &[0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 1, 2]), + ] { + palette.extend_from_slice(&((8 + body.len()) as u32).to_be_bytes()); + palette.extend_from_slice(kind); + palette.extend_from_slice(body); + } + assert_eq!(bpc(&one_bit_jp2(&palette)), Some(8)); + assert_eq!(bpc(&one_bit_jp2(&[])), Some(1)); + } } From d118f471b87aa2ae4099d959db711b2505df4cd4 Mon Sep 17 00:00:00 2001 From: aiongg Date: Sat, 24 Oct 2026 07:32:18 +0000 Subject: [PATCH 3/3] hayro-syntax: port PdfCraft's unfiltered inline-image length (#3107) From PdfCraft #3438 with its look-ahead bound (01af6cba3): an inline image without a filter ends at the byte length its /W, /H, /CS, /BPC and /IM give when white space and EI follow there, so an EI inside its samples does not end it. Anything else keeps the EI look-ahead. A computed end past the content window waits for more data only while it fits within MAX_DECODED_STREAM. Tests: PdfCraft's six content::tests unit tests. --- vendor/README.craft-reader.md | 11 + vendor/hayro-syntax/SOURCE.json | 2 +- vendor/hayro-syntax/src/content/mod.rs | 385 ++++++++++++++++++++++++- 3 files changed, 395 insertions(+), 3 deletions(-) diff --git a/vendor/README.craft-reader.md b/vendor/README.craft-reader.md index cd135c5a..419e9ad9 100644 --- a/vendor/README.craft-reader.md +++ b/vendor/README.craft-reader.md @@ -196,6 +196,17 @@ precision of the indices; an image without a palette keeps its own. Tests: PdfCraft's raster test `a_jp2_palette_keeps_its_colours_whatever_the_index_precision` is not ported: the unit tests show the same behavior. +From [PdfCraft revision 0b670caf4a090e9d83a7f41a84b4cb2144a2d33e](https://github.com/storytold/pdfcraft/tree/0b670caf4a090e9d83a7f41a84b4cb2144a2d33e/vendor), +unledgered and marked `PdfCraft patch` in the source: hayro-syntax #3438 (issue #3107), with +its look-ahead bound from 01af6cba3. An inline image without a filter ends at the byte +length its `/W`, `/H`, `/CS`, `/BPC` and `/IM` give (rows padded to whole bytes) when white +space and `EI` follow there, so an `EI` inside its samples does not end it. Conflicting +long and short keys, filters and other color spaces keep the `EI` look-ahead. When the +computed end lies past the content window, the tokenizer asks for more data only while the +image, its header and the terminator fit within `MAX_DECODED_STREAM`, and otherwise falls +back to the look-ahead. Its unit tests in `content::tests` come with it. PdfCraft's raster +test is not ported: the windowed tokenizer tests cover the same content streams. + This remains the patched 0.7 backend. It retains the existing Bradford Lab conversion, floating component precision and codec versions, rather than claiming full 0.8 equivalence. The independent color oracles, backend tests and facade contracts cover diff --git a/vendor/hayro-syntax/SOURCE.json b/vendor/hayro-syntax/SOURCE.json index e02661ba..20f4c121 100644 --- a/vendor/hayro-syntax/SOURCE.json +++ b/vendor/hayro-syntax/SOURCE.json @@ -68,7 +68,7 @@ "path": "src/content/mod.rs", "source_sha256": "b0f36badb2688c90691c27a4e5dcdfec7639858081553242744dd7bc39ffc0d0", "source_git_blob": "8e47324d0b0bd90ed15efe5ff1e9de0cb46e75d9", - "sha256": "aa0883ba92dd05655bfd8d54195584bab2c0b28111d76ec73c5381a0f23b6181" + "sha256": "ae0b0596d2304adc4109294cc7af681698071060d9e8cd6d44f0bb46baffec6b" }, { "path": "src/content/ops.rs", diff --git a/vendor/hayro-syntax/src/content/mod.rs b/vendor/hayro-syntax/src/content/mod.rs index ee749cd9..5b19d509 100644 --- a/vendor/hayro-syntax/src/content/mod.rs +++ b/vendor/hayro-syntax/src/content/mod.rs @@ -35,7 +35,7 @@ pub mod ops; use crate::content::ops::TypedInstruction; use crate::object; -use crate::object::dict::InlineImageDict; +use crate::object::dict::{Dict, InlineImageDict}; use crate::object::name::{Name, skip_name_like}; use crate::object::{Array, Null, Number, Object, Stream}; use crate::reader::Reader; @@ -242,7 +242,33 @@ impl<'a> UntypedIter<'a> { let mut lookahead_touched = false; let mut accepted = false; - 'outer: while let Some(pos) = find_needle(self.reader.tail()?, b"EI") { + // PdfCraft patch: an unfiltered inline image's byte length is determined by + // its parameters (and `/L` or `/Length` when present and consistent), so + // `EI` inside its samples does not cut the image short (#3107, PR #3438). + match unfiltered_image_end( + stream_data, + &dict, + self.complete, + start_offset.saturating_sub(self.committed), + ) { + UnfilteredEnd::Found(end_data, end_after_ei) => { + let image_data = stream_data.get(..end_data)?; + let stream = Stream::new(image_data, dict.clone()); + self.stack.push(Object::Stream(stream))?; + self.reader.read_bytes(end_after_ei)?; + self.reader.skip_white_spaces(); + accepted = true; + } + UnfilteredEnd::NeedsMore => { + self.needs_more = true; + return None; + } + UnfilteredEnd::Fallback => {} + } + + 'outer: while !accepted + && let Some(pos) = find_needle(self.reader.tail()?, b"EI") + { self.reader.read_bytes(pos)?; if self.reader.peek_bytes(2) == Some(b"EI") { @@ -858,9 +884,175 @@ mod macros { pub(crate) use op6; } +/// PdfCraft patch: outcome of checking an unfiltered inline image's deterministic byte length. +enum UnfilteredEnd { + /// Deterministic end found: `(end_data, end_after_ei)`. + Found(usize, usize), + /// Window ran out before the terminator after the expected sample bytes could be decided. + NeedsMore, + /// Not an unfiltered image with a valid terminator at its computed length; use the heuristic. + Fallback, +} + +/// PdfCraft patch: an unfiltered inline image's sample byte count, including byte padding per row, +/// and `/Length` or `/L` when present and consistent (#3107). +fn unfiltered_image_len(dict: &Dict<'_>) -> Option { + for key in [b"Filter".as_slice(), b"F"] { + if let Some(filter) = dict.get::>(key) + && !matches!(filter, Object::Null(_)) + && !matches!(&filter, Object::Array(filters) if filters.iter::>().next().is_none()) + { + return None; + } + } + for (full, short) in [ + (b"Width".as_slice(), b"W".as_slice()), + (b"Height", b"H"), + (b"ColorSpace", b"CS"), + (b"BitsPerComponent", b"BPC"), + (b"ImageMask", b"IM"), + ] { + if let (Some(full), Some(short)) = + (dict.get::>(full), dict.get::>(short)) + && full != short + { + return None; + } + } + let get = |full: &[u8], short: &[u8]| { + dict.get::>(full) + .or_else(|| dict.get::>(short)) + }; + let int = |obj: &Object<'_>| match obj { + Object::Number(n) => i32::try_from(n.as_i64()) + .ok() + .filter(|&i| Number::from_i32(i) == *n) + .map(i64::from), + _ => None, + }; + let width = usize::try_from(int(&get(b"Width", b"W")?)?) + .ok() + .filter(|n| *n > 0)?; + let height = usize::try_from(int(&get(b"Height", b"H")?)?) + .ok() + .filter(|n| *n > 0)?; + let mask = match get(b"ImageMask", b"IM") { + None | Some(Object::Null(_) | Object::Boolean(false)) => false, + Some(Object::Boolean(true)) => true, + _ => return None, + }; + let bits = match get(b"BitsPerComponent", b"BPC") { + None | Some(Object::Null(_)) => None, + Some(value) => Some(int(&value)?), + }; + let (components, bits) = if mask { + if bits.is_some_and(|bits| bits != 1) { + return None; + } + (1usize, 1usize) + } else { + let (components, max_bits) = match get(b"ColorSpace", b"CS")? { + Object::Name(name) => match name.as_ref() { + b"DeviceGray" | b"G" => (1, 16), + b"DeviceRGB" | b"RGB" => (3, 16), + b"DeviceCMYK" | b"CMYK" => (4, 16), + _ => return None, + }, + Object::Array(space) => { + let mut items = space.iter::>(); + if !matches!(items.next(), Some(Object::Name(n)) if matches!(n.as_ref(), b"Indexed" | b"I")) + || items.count() != 3 + { + return None; + } + (1, 8) + } + _ => return None, + }; + let bits = match bits? { + bits @ (1 | 2 | 4 | 8 | 16) => bits as usize, + _ => return None, + }; + if bits > max_bits { + return None; + } + (components, bits) + }; + let row = width + .checked_mul(components)? + .checked_mul(bits)? + .div_ceil(8); + row.checked_mul(height) +} + +/// PdfCraft patch: locate the deterministic `EI` terminator of an unfiltered inline image (#3107). +fn unfiltered_image_end( + data: &[u8], + dict: &Dict<'_>, + complete: bool, + header_bytes: usize, +) -> UnfilteredEnd { + let Some(end_data) = unfiltered_image_len(dict) else { + return UnfilteredEnd::Fallback; + }; + // An inline image shares the content window's budget, not the larger decoded-image + // budget. Include its dictionary/header and the shortest whitespace + EI + delimiter + // look-ahead, or one more byte beyond any separator whitespace already present. + // Missing bytes that cannot fit must use the existing heuristic, not pin the window + // and discard following content. Already-available exact ends still win. + let missing_end = if complete + || end_data + .checked_add(4) + .and_then(|minimum| data.len().checked_add(1).map(|next| minimum.max(next))) + .and_then(|needed| needed.checked_add(header_bytes)) + .is_none_or(|needed| needed > crate::MAX_DECODED_STREAM) + { + UnfilteredEnd::Fallback + } else { + UnfilteredEnd::NeedsMore + }; + let Some(after) = data.get(end_data..) else { + return missing_end; + }; + let Some(&first) = after.first() else { + return missing_end; + }; + if !is_white_space_character(first) { + return UnfilteredEnd::Fallback; + } + let Some(separator) = after.iter().position(|&b| !is_white_space_character(b)) else { + return missing_end; + }; + let Some(rest) = after.get(separator..) else { + return UnfilteredEnd::Fallback; + }; + if rest.first() != Some(&b'E') { + return UnfilteredEnd::Fallback; + } + let Some(&second) = rest.get(1) else { + return missing_end; + }; + if second != b'I' { + return UnfilteredEnd::Fallback; + } + let Some(end_after_ei) = end_data + .checked_add(separator) + .and_then(|ei| ei.checked_add(2)) + else { + return UnfilteredEnd::Fallback; + }; + match rest.get(2) { + Some(&b) if !is_regular_character(b) => UnfilteredEnd::Found(end_data, end_after_ei), + Some(_) => UnfilteredEnd::Fallback, + None if complete => UnfilteredEnd::Found(end_data, end_after_ei), + None => missing_end, + } +} + #[cfg(test)] mod tests { use super::{ContentTokenizer, UntypedIter}; + use crate::reader::ReaderExt; fn operators(data: &[u8]) -> Vec> { let mut tokenizer = UntypedIter::new(data); @@ -1081,4 +1273,193 @@ mod tests { assert_eq!(windowed, expected, "{:?}", String::from_utf8_lossy(&data)); } } + + #[test] + fn unfiltered_inline_image_with_ei_in_samples_is_not_cut_early() { + // 48-byte unfiltered DeviceGray image whose first four bytes are `0EI ` + // followed by valid page content operators (`Q BT ... Tj ET q`). + let data = b"q 100 0 0 100 72 650 cm \ +BI /W 48 /H 1 /CS /G /BPC 8 ID 0EI Q BT /F1 18 Tf 72 600 Td (Injected) Tj ET q! EI Q \ +BT /F1 14 Tf 72 550 Td (After) Tj ET"; + assert_eq!( + operators(data), + vec![ + b"q".to_vec(), + b"cm".to_vec(), + b"BI".to_vec(), + b"Q".to_vec(), + b"BT".to_vec(), + b"Tf".to_vec(), + b"Td".to_vec(), + b"Tj".to_vec(), + b"ET".to_vec(), + ] + ); + let expected = slice_instructions(data); + for piece in 1..=data.len() { + assert_eq!( + windowed_instructions(data, || piece), + expected, + "piece size {piece}" + ); + } + } + + #[test] + fn oversized_unfiltered_inline_image_does_not_hold_following_content() { + for length in [ + crate::MAX_DECODED_STREAM - 1, + crate::MAX_DECODED_STREAM, + crate::MAX_DECODED_STREAM + 1, + ] { + // The next BI supplies the existing EI heuristic's bounded look-ahead. The + // stream is not exhausted: an impossible declaration must not pin this window. + let data = format!( + "BI /W {length} /H 1 /CS /G /BPC 8 ID x EI BT (After) Tj ET \ + BI /W 1 /H 1 /CS /G /BPC 8 ID y EI Q" + ); + let mut tokenizer = UntypedIter::new_window(data.as_bytes(), false); + for expected in [b"BI".as_slice(), b"BT", b"Tj", b"ET"] { + let instruction = tokenizer + .next_instruction() + .expect("following content survives"); + assert_eq!( + instruction.operator.as_ref(), + expected, + "declared length {length}" + ); + } + } + } + + #[test] + fn unfiltered_inline_image_budget_includes_header_and_terminator() { + let mut reader = crate::reader::Reader::new(b"/W 1 /H 1 /CS /G /BPC 8 ID "); + let inline = reader + .read_without_context::>() + .expect("synthetic inline dictionary"); + let dict = inline.get_dict(); + let fits = crate::MAX_DECODED_STREAM - 5; // One sample + whitespace/EI/delimiter. + assert!(matches!( + super::unfiltered_image_end(b"", dict, false, fits), + super::UnfilteredEnd::NeedsMore + )); + assert!(matches!( + super::unfiltered_image_end(b"x ", dict, false, fits), + super::UnfilteredEnd::Fallback + )); + assert!(matches!( + super::unfiltered_image_end(b"x EI", dict, false, fits + 1), + super::UnfilteredEnd::Fallback + )); + assert!(matches!( + super::unfiltered_image_end(b"x EI", dict, true, usize::MAX), + super::UnfilteredEnd::Found(1, 4) + )); + for header in [fits + 1, usize::MAX] { + assert!(matches!( + super::unfiltered_image_end(b"", dict, false, header), + super::UnfilteredEnd::Fallback + )); + // The guard is only for missing bytes, never a reason to reinterpret available samples. + assert!(matches!( + super::unfiltered_image_end(b"x EI Q", dict, false, header), + super::UnfilteredEnd::Found(1, 4) + )); + } + } + + #[test] + fn large_unfiltered_inline_image_still_waits_for_its_samples() { + // A valid image larger than the interpreter's initial 64 KiB window must + // still use its exact length rather than execute operators in its samples. + let length = (64 << 10) + 1; + let mut data = format!("BI /W {length} /H 1 /CS /G /BPC 8 ID ").into_bytes(); + let sample_start = data.len(); + data.extend_from_slice(b"0EI Q BT (Injected) Tj ET q"); + data.resize(sample_start + length, b'0'); + data.extend_from_slice(b" EI BT (After) Tj ET"); + let mut partial = UntypedIter::new_window(&data[..64 << 10], false); + assert!(partial.next_instruction().is_none()); + assert!(partial.needs_more); + assert_eq!(partial.committed, 0); + assert_eq!( + operators(&data), + [ + b"BI".to_vec(), + b"BT".to_vec(), + b"Tj".to_vec(), + b"ET".to_vec() + ] + ); + let expected = slice_instructions(&data); + for piece in [4096, 64 << 10, data.len()] { + assert_eq!( + windowed_instructions(&data, || piece), + expected, + "piece size {piece}" + ); + } + } + + #[test] + fn unfiltered_inline_image_variants_and_fallbacks() { + // ImageMask with 1-bit row padding (width 10 -> 2 bytes per row, height 2 -> 4 bytes) + let mask_stream = b"BI /IM true /W 10 /H 2 ID 0EI\n EI Q"; + assert_eq!(operators(mask_stream), vec![b"BI".to_vec(), b"Q".to_vec()]); + let expected_mask = slice_instructions(mask_stream); + for piece in 1..=mask_stream.len() { + assert_eq!(windowed_instructions(mask_stream, || piece), expected_mask); + } + + // Indexed color space with 4-bit samples (width 5 -> 3 bytes per row, height 2 -> 6 bytes) + // and explicit /L matching computed length. + let indexed_stream = + b"BI /Width 5 /Height 2 /CS [/I /RGB 1 <000000ffffff>] /BPC 4 /L 6 ID 0EI Q! EI Q"; + assert_eq!( + operators(indexed_stream), + vec![b"BI".to_vec(), b"Q".to_vec()] + ); + let expected_indexed = slice_instructions(indexed_stream); + for piece in 1..=indexed_stream.len() { + assert_eq!( + windowed_instructions(indexed_stream, || piece), + expected_indexed + ); + } + + // Conflicting /W and /Width falls back to the heuristic scan; mismatched /L does not disrupt the verified layout. + let mismatch_l = b"BI /W 4 /H 1 /CS /G /BPC 8 /L 99 ID abcd EI Q"; + assert_eq!(operators(mismatch_l), vec![b"BI".to_vec(), b"Q".to_vec()]); + let conflict_w = b"BI /W 4 /Width 8 /H 1 /CS /G /BPC 8 ID abcd EI Q"; + assert_eq!(operators(conflict_w), vec![b"BI".to_vec(), b"Q".to_vec()]); + } + + #[test] + fn each_layout_keeps_its_computed_length() { + let inject = b" EI Q BT /F1 9 Tf (X) Tj ET q"; + let case = |params: &str, len: usize| { + let mut samples = inject.to_vec(); + samples.resize(len, b'0'); + let mut data = format!("BI {params} ID ").into_bytes(); + data.extend_from_slice(&samples); + data.extend_from_slice(b" EI Q"); + data + }; + for (params, len) in [ + ("/W 16 /H 2 /CS /RGB /BPC 8", 96), + ("/W 12 /H 2 /CS /CMYK /BPC 8", 96), + ("/W 70 /H 4 /CS /G /BPC 1", 36), + ("/IM true /W 70 /H 4", 36), + ("/W 30 /H 2 /CS [/I /RGB 1 <000000ffffff>] /BPC 4 /L 30", 30), + ("/W 15 /H 2 /CS /G /BPC 16 /L 60", 60), + ("/W 47 /H 1 /CS /G /BPC 8 /L 5", 47), + ] { + assert_eq!( + operators(&case(params, len)), + vec![b"BI".to_vec(), b"Q".to_vec()], + "{params}" + ); + } + } }