Merge part 4 (syntax)

# Conflicts:
#	vendor/README.craft-reader.md

vendor/README.craft-reader.md

@@ -205,6 +205,37 @@ OCR text (mode 3) keeps a heading drawn through its clip. The unit tests
 glyph_names_resolve_to_the_first_glyph_with_that_name}` come with (49) and (50); the #3116
 contract is in `tests/upstream_text.rs`.
 
+Fixes from the [PdfCraft revision 0b670caf4a090e9d83a7f41a84b4cb2144a2d33e ledger](https://github.com/storytold/pdfcraft/blob/0b670caf4a090e9d83a7f41a84b4cb2144a2d33e/vendor/README.md),
+by its item ids, each marked `PdfCraft patch` in the source. hayro-syntax (24), PdfCraft
+#2400: `xref::prev_offset` reads a trailer `/Prev` written as a reference `N G R` as the
+offset that object holds when the section's own entries place it (iText's
+`indirectPrev.pdf`), and otherwise as offset N (tiff2pdf), so the earlier sections' pages
+are seen. Each section's entries are read before its `Prev` is followed; the local
+xref-stream bounds above apply to them unchanged. Tests:
+`xref::tests::prev_as_reference_reaches_the_earlier_section` and
+`xref::tests::prev_as_reference_to_an_object_holding_the_offset_reads_that_offset`.
+hayro-syntax (P5), PdfCraft #3184: `filter::jpx::decode` applies a JP2 palette (`pclr`) when
+the image dictionary has no `/ColorSpace`, or one that does not resolve, since the colors then
+come from the JP2 header (ISO 32000-2 §7.4.9). With a `/ColorSpace` the raw indices are kept,
+as an `/Indexed` space needs. Resolved palette colors are 8-bit samples whatever the
+precision of the indices; an image without a palette keeps its own. Tests:
+`filter::jpx::tests::jpx_palette_is_applied_without_a_color_space`,
+`filter::jpx::tests::jpx_palette_indices_stay_raw_for_an_indexed_color_space` and
+`filter::jpx::tests::palette_colours_are_eight_bit_and_other_samples_keep_their_precision`.
+PdfCraft's raster test `a_jp2_palette_keeps_its_colours_whatever_the_index_precision` is not
+ported: the unit tests show the same behavior.
+
+From [PdfCraft revision 0b670caf4a090e9d83a7f41a84b4cb2144a2d33e](https://github.com/storytold/pdfcraft/tree/0b670caf4a090e9d83a7f41a84b4cb2144a2d33e/vendor),
+unledgered and marked `PdfCraft patch` in the source: hayro-syntax #3438 (issue #3107), with
+its look-ahead bound from 01af6cba3. An inline image without a filter ends at the byte
+length its `/W`, `/H`, `/CS`, `/BPC` and `/IM` give (rows padded to whole bytes) when white
+space and `EI` follow there, so an `EI` inside its samples does not end it. Conflicting
+long and short keys, filters and other color spaces keep the `EI` look-ahead. When the
+computed end lies past the content window, the tokenizer asks for more data only while the
+image, its header and the terminator fit within `MAX_DECODED_STREAM`, and otherwise falls
+back to the look-ahead. Its unit tests in `content::tests` come with it. PdfCraft's raster
+test is not ported: the windowed tokenizer tests cover the same content streams.
+
 This remains the patched 0.7 backend. It retains the existing Bradford Lab conversion,
 floating component precision and codec versions, rather than claiming full 0.8
 equivalence. The independent color oracles, backend tests and facade contracts cover

vendor/hayro-syntax/SOURCE.json

@@ -68,7 +68,7 @@
       "path": "src/content/mod.rs",
       "source_sha256": "b0f36badb2688c90691c27a4e5dcdfec7639858081553242744dd7bc39ffc0d0",
       "source_git_blob": "8e47324d0b0bd90ed15efe5ff1e9de0cb46e75d9",
-      "sha256": "aa0883ba92dd05655bfd8d54195584bab2c0b28111d76ec73c5381a0f23b6181"
+      "sha256": "ae0b0596d2304adc4109294cc7af681698071060d9e8cd6d44f0bb46baffec6b"
     },
     {
       "path": "src/content/ops.rs",
@@ -164,7 +164,7 @@
       "path": "src/filter/jpx.rs",
       "source_sha256": "f0a2b01c692c086d30a26562d7548f9e837af166be737d2c543de7c886a45f82",
       "source_git_blob": "74efd8b31e2b2935ff2bbceda2c1ffd36ea9c20d",
-      "sha256": "f0a2b01c692c086d30a26562d7548f9e837af166be737d2c543de7c886a45f82"
+      "sha256": "b0c295f815547377409a8810e9cbef35e4d6b506a3c3507669da71fcfe9d4701"
     },
     {
       "path": "src/filter/lzw_flate.rs",
@@ -338,7 +338,7 @@
       "path": "src/xref.rs",
       "source_sha256": "51ffeeb7d4ac035bae2cd5ba9288c180088ae06cdcefe2b29dfeda560a793aa0",
       "source_git_blob": "20e33a410489d42e438a3f2b87d70c7bb8d66eae",
-      "sha256": "fa8d24280df15ffe5628057fdcfa2a3d2a9f42539b6f9096b2feb836ac54ac30"
+      "sha256": "98fb56eb5c2da8b0c40202ed30b2f464874c1143436e72eda4e49f2333dd7c7f"
     },
     {
       "path": "src/decode_limits_tests.rs",

vendor/hayro-syntax/src/content/mod.rs

@@ -35,7 +35,7 @@ pub mod ops;
 
 use crate::content::ops::TypedInstruction;
 use crate::object;
-use crate::object::dict::InlineImageDict;
+use crate::object::dict::{Dict, InlineImageDict};
 use crate::object::name::{Name, skip_name_like};
 use crate::object::{Array, Null, Number, Object, Stream};
 use crate::reader::Reader;
@@ -242,7 +242,33 @@ impl<'a> UntypedIter<'a> {
                     let mut lookahead_touched = false;
                     let mut accepted = false;
 
-                    'outer: while let Some(pos) = find_needle(self.reader.tail()?, b"EI") {
+                    // PdfCraft patch: an unfiltered inline image's byte length is determined by
+                    // its parameters (and `/L` or `/Length` when present and consistent), so
+                    // `EI` inside its samples does not cut the image short (#3107, PR #3438).
+                    match unfiltered_image_end(
+                        stream_data,
+                        &dict,
+                        self.complete,
+                        start_offset.saturating_sub(self.committed),
+                    ) {
+                        UnfilteredEnd::Found(end_data, end_after_ei) => {
+                            let image_data = stream_data.get(..end_data)?;
+                            let stream = Stream::new(image_data, dict.clone());
+                            self.stack.push(Object::Stream(stream))?;
+                            self.reader.read_bytes(end_after_ei)?;
+                            self.reader.skip_white_spaces();
+                            accepted = true;
+                        }
+                        UnfilteredEnd::NeedsMore => {
+                            self.needs_more = true;
+                            return None;
+                        }
+                        UnfilteredEnd::Fallback => {}
+                    }
+
+                    'outer: while !accepted
+                        && let Some(pos) = find_needle(self.reader.tail()?, b"EI")
+                    {
                         self.reader.read_bytes(pos)?;
 
                         if self.reader.peek_bytes(2) == Some(b"EI") {
@@ -858,9 +884,175 @@ mod macros {
     pub(crate) use op6;
 }
 
+/// PdfCraft patch: outcome of checking an unfiltered inline image's deterministic byte length.
+enum UnfilteredEnd {
+    /// Deterministic end found: `(end_data, end_after_ei)`.
+    Found(usize, usize),
+    /// Window ran out before the terminator after the expected sample bytes could be decided.
+    NeedsMore,
+    /// Not an unfiltered image with a valid terminator at its computed length; use the heuristic.
+    Fallback,
+}
+
+/// PdfCraft patch: an unfiltered inline image's sample byte count, including byte padding per row,
+/// and `/Length` or `/L` when present and consistent (#3107).
+fn unfiltered_image_len(dict: &Dict<'_>) -> Option<usize> {
+    for key in [b"Filter".as_slice(), b"F"] {
+        if let Some(filter) = dict.get::<Object<'_>>(key)
+            && !matches!(filter, Object::Null(_))
+            && !matches!(&filter, Object::Array(filters) if filters.iter::<Object<'_>>().next().is_none())
+        {
+            return None;
+        }
+    }
+    for (full, short) in [
+        (b"Width".as_slice(), b"W".as_slice()),
+        (b"Height", b"H"),
+        (b"ColorSpace", b"CS"),
+        (b"BitsPerComponent", b"BPC"),
+        (b"ImageMask", b"IM"),
+    ] {
+        if let (Some(full), Some(short)) =
+            (dict.get::<Object<'_>>(full), dict.get::<Object<'_>>(short))
+            && full != short
+        {
+            return None;
+        }
+    }
+    let get = |full: &[u8], short: &[u8]| {
+        dict.get::<Object<'_>>(full)
+            .or_else(|| dict.get::<Object<'_>>(short))
+    };
+    let int = |obj: &Object<'_>| match obj {
+        Object::Number(n) => i32::try_from(n.as_i64())
+            .ok()
+            .filter(|&i| Number::from_i32(i) == *n)
+            .map(i64::from),
+        _ => None,
+    };
+    let width = usize::try_from(int(&get(b"Width", b"W")?)?)
+        .ok()
+        .filter(|n| *n > 0)?;
+    let height = usize::try_from(int(&get(b"Height", b"H")?)?)
+        .ok()
+        .filter(|n| *n > 0)?;
+    let mask = match get(b"ImageMask", b"IM") {
+        None | Some(Object::Null(_) | Object::Boolean(false)) => false,
+        Some(Object::Boolean(true)) => true,
+        _ => return None,
+    };
+    let bits = match get(b"BitsPerComponent", b"BPC") {
+        None | Some(Object::Null(_)) => None,
+        Some(value) => Some(int(&value)?),
+    };
+    let (components, bits) = if mask {
+        if bits.is_some_and(|bits| bits != 1) {
+            return None;
+        }
+        (1usize, 1usize)
+    } else {
+        let (components, max_bits) = match get(b"ColorSpace", b"CS")? {
+            Object::Name(name) => match name.as_ref() {
+                b"DeviceGray" | b"G" => (1, 16),
+                b"DeviceRGB" | b"RGB" => (3, 16),
+                b"DeviceCMYK" | b"CMYK" => (4, 16),
+                _ => return None,
+            },
+            Object::Array(space) => {
+                let mut items = space.iter::<Object<'_>>();
+                if !matches!(items.next(), Some(Object::Name(n)) if matches!(n.as_ref(), b"Indexed" | b"I"))
+                    || items.count() != 3
+                {
+                    return None;
+                }
+                (1, 8)
+            }
+            _ => return None,
+        };
+        let bits = match bits? {
+            bits @ (1 | 2 | 4 | 8 | 16) => bits as usize,
+            _ => return None,
+        };
+        if bits > max_bits {
+            return None;
+        }
+        (components, bits)
+    };
+    let row = width
+        .checked_mul(components)?
+        .checked_mul(bits)?
+        .div_ceil(8);
+    row.checked_mul(height)
+}
+
+/// PdfCraft patch: locate the deterministic `EI` terminator of an unfiltered inline image (#3107).
+fn unfiltered_image_end(
+    data: &[u8],
+    dict: &Dict<'_>,
+    complete: bool,
+    header_bytes: usize,
+) -> UnfilteredEnd {
+    let Some(end_data) = unfiltered_image_len(dict) else {
+        return UnfilteredEnd::Fallback;
+    };
+    // An inline image shares the content window's budget, not the larger decoded-image
+    // budget. Include its dictionary/header and the shortest whitespace + EI + delimiter
+    // look-ahead, or one more byte beyond any separator whitespace already present.
+    // Missing bytes that cannot fit must use the existing heuristic, not pin the window
+    // and discard following content. Already-available exact ends still win.
+    let missing_end = if complete
+        || end_data
+            .checked_add(4)
+            .and_then(|minimum| data.len().checked_add(1).map(|next| minimum.max(next)))
+            .and_then(|needed| needed.checked_add(header_bytes))
+            .is_none_or(|needed| needed > crate::MAX_DECODED_STREAM)
+    {
+        UnfilteredEnd::Fallback
+    } else {
+        UnfilteredEnd::NeedsMore
+    };
+    let Some(after) = data.get(end_data..) else {
+        return missing_end;
+    };
+    let Some(&first) = after.first() else {
+        return missing_end;
+    };
+    if !is_white_space_character(first) {
+        return UnfilteredEnd::Fallback;
+    }
+    let Some(separator) = after.iter().position(|&b| !is_white_space_character(b)) else {
+        return missing_end;
+    };
+    let Some(rest) = after.get(separator..) else {
+        return UnfilteredEnd::Fallback;
+    };
+    if rest.first() != Some(&b'E') {
+        return UnfilteredEnd::Fallback;
+    }
+    let Some(&second) = rest.get(1) else {
+        return missing_end;
+    };
+    if second != b'I' {
+        return UnfilteredEnd::Fallback;
+    }
+    let Some(end_after_ei) = end_data
+        .checked_add(separator)
+        .and_then(|ei| ei.checked_add(2))
+    else {
+        return UnfilteredEnd::Fallback;
+    };
+    match rest.get(2) {
+        Some(&b) if !is_regular_character(b) => UnfilteredEnd::Found(end_data, end_after_ei),
+        Some(_) => UnfilteredEnd::Fallback,
+        None if complete => UnfilteredEnd::Found(end_data, end_after_ei),
+        None => missing_end,
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::{ContentTokenizer, UntypedIter};
+    use crate::reader::ReaderExt;
 
     fn operators(data: &[u8]) -> Vec<Vec<u8>> {
         let mut tokenizer = UntypedIter::new(data);
@@ -1081,4 +1273,193 @@ mod tests {
             assert_eq!(windowed, expected, "{:?}", String::from_utf8_lossy(&data));
         }
     }
+
+    #[test]
+    fn unfiltered_inline_image_with_ei_in_samples_is_not_cut_early() {
+        // 48-byte unfiltered DeviceGray image whose first four bytes are `0EI `
+        // followed by valid page content operators (`Q BT ... Tj ET q`).
+        let data = b"q 100 0 0 100 72 650 cm \
+BI /W 48 /H 1 /CS /G /BPC 8 ID 0EI Q BT /F1 18 Tf 72 600 Td (Injected) Tj ET q! EI Q \
+BT /F1 14 Tf 72 550 Td (After) Tj ET";
+        assert_eq!(
+            operators(data),
+            vec![
+                b"q".to_vec(),
+                b"cm".to_vec(),
+                b"BI".to_vec(),
+                b"Q".to_vec(),
+                b"BT".to_vec(),
+                b"Tf".to_vec(),
+                b"Td".to_vec(),
+                b"Tj".to_vec(),
+                b"ET".to_vec(),
+            ]
+        );
+        let expected = slice_instructions(data);
+        for piece in 1..=data.len() {
+            assert_eq!(
+                windowed_instructions(data, || piece),
+                expected,
+                "piece size {piece}"
+            );
+        }
+    }
+
+    #[test]
+    fn oversized_unfiltered_inline_image_does_not_hold_following_content() {
+        for length in [
+            crate::MAX_DECODED_STREAM - 1,
+            crate::MAX_DECODED_STREAM,
+            crate::MAX_DECODED_STREAM + 1,
+        ] {
+            // The next BI supplies the existing EI heuristic's bounded look-ahead. The
+            // stream is not exhausted: an impossible declaration must not pin this window.
+            let data = format!(
+                "BI /W {length} /H 1 /CS /G /BPC 8 ID x EI BT (After) Tj ET \
+                 BI /W 1 /H 1 /CS /G /BPC 8 ID y EI Q"
+            );
+            let mut tokenizer = UntypedIter::new_window(data.as_bytes(), false);
+            for expected in [b"BI".as_slice(), b"BT", b"Tj", b"ET"] {
+                let instruction = tokenizer
+                    .next_instruction()
+                    .expect("following content survives");
+                assert_eq!(
+                    instruction.operator.as_ref(),
+                    expected,
+                    "declared length {length}"
+                );
+            }
+        }
+    }
+
+    #[test]
+    fn unfiltered_inline_image_budget_includes_header_and_terminator() {
+        let mut reader = crate::reader::Reader::new(b"/W 1 /H 1 /CS /G /BPC 8 ID ");
+        let inline = reader
+            .read_without_context::<crate::object::dict::InlineImageDict<'_>>()
+            .expect("synthetic inline dictionary");
+        let dict = inline.get_dict();
+        let fits = crate::MAX_DECODED_STREAM - 5; // One sample + whitespace/EI/delimiter.
+        assert!(matches!(
+            super::unfiltered_image_end(b"", dict, false, fits),
+            super::UnfilteredEnd::NeedsMore
+        ));
+        assert!(matches!(
+            super::unfiltered_image_end(b"x    ", dict, false, fits),
+            super::UnfilteredEnd::Fallback
+        ));
+        assert!(matches!(
+            super::unfiltered_image_end(b"x EI", dict, false, fits + 1),
+            super::UnfilteredEnd::Fallback
+        ));
+        assert!(matches!(
+            super::unfiltered_image_end(b"x EI", dict, true, usize::MAX),
+            super::UnfilteredEnd::Found(1, 4)
+        ));
+        for header in [fits + 1, usize::MAX] {
+            assert!(matches!(
+                super::unfiltered_image_end(b"", dict, false, header),
+                super::UnfilteredEnd::Fallback
+            ));
+            // The guard is only for missing bytes, never a reason to reinterpret available samples.
+            assert!(matches!(
+                super::unfiltered_image_end(b"x EI Q", dict, false, header),
+                super::UnfilteredEnd::Found(1, 4)
+            ));
+        }
+    }
+
+    #[test]
+    fn large_unfiltered_inline_image_still_waits_for_its_samples() {
+        // A valid image larger than the interpreter's initial 64 KiB window must
+        // still use its exact length rather than execute operators in its samples.
+        let length = (64 << 10) + 1;
+        let mut data = format!("BI /W {length} /H 1 /CS /G /BPC 8 ID ").into_bytes();
+        let sample_start = data.len();
+        data.extend_from_slice(b"0EI Q BT (Injected) Tj ET q");
+        data.resize(sample_start + length, b'0');
+        data.extend_from_slice(b" EI BT (After) Tj ET");
+        let mut partial = UntypedIter::new_window(&data[..64 << 10], false);
+        assert!(partial.next_instruction().is_none());
+        assert!(partial.needs_more);
+        assert_eq!(partial.committed, 0);
+        assert_eq!(
+            operators(&data),
+            [
+                b"BI".to_vec(),
+                b"BT".to_vec(),
+                b"Tj".to_vec(),
+                b"ET".to_vec()
+            ]
+        );
+        let expected = slice_instructions(&data);
+        for piece in [4096, 64 << 10, data.len()] {
+            assert_eq!(
+                windowed_instructions(&data, || piece),
+                expected,
+                "piece size {piece}"
+            );
+        }
+    }
+
+    #[test]
+    fn unfiltered_inline_image_variants_and_fallbacks() {
+        // ImageMask with 1-bit row padding (width 10 -> 2 bytes per row, height 2 -> 4 bytes)
+        let mask_stream = b"BI /IM true /W 10 /H 2 ID 0EI\n EI Q";
+        assert_eq!(operators(mask_stream), vec![b"BI".to_vec(), b"Q".to_vec()]);
+        let expected_mask = slice_instructions(mask_stream);
+        for piece in 1..=mask_stream.len() {
+            assert_eq!(windowed_instructions(mask_stream, || piece), expected_mask);
+        }
+
+        // Indexed color space with 4-bit samples (width 5 -> 3 bytes per row, height 2 -> 6 bytes)
+        // and explicit /L matching computed length.
+        let indexed_stream =
+            b"BI /Width 5 /Height 2 /CS [/I /RGB 1 <000000ffffff>] /BPC 4 /L 6 ID 0EI Q! EI Q";
+        assert_eq!(
+            operators(indexed_stream),
+            vec![b"BI".to_vec(), b"Q".to_vec()]
+        );
+        let expected_indexed = slice_instructions(indexed_stream);
+        for piece in 1..=indexed_stream.len() {
+            assert_eq!(
+                windowed_instructions(indexed_stream, || piece),
+                expected_indexed
+            );
+        }
+
+        // Conflicting /W and /Width falls back to the heuristic scan; mismatched /L does not disrupt the verified layout.
+        let mismatch_l = b"BI /W 4 /H 1 /CS /G /BPC 8 /L 99 ID abcd EI Q";
+        assert_eq!(operators(mismatch_l), vec![b"BI".to_vec(), b"Q".to_vec()]);
+        let conflict_w = b"BI /W 4 /Width 8 /H 1 /CS /G /BPC 8 ID abcd EI Q";
+        assert_eq!(operators(conflict_w), vec![b"BI".to_vec(), b"Q".to_vec()]);
+    }
+
+    #[test]
+    fn each_layout_keeps_its_computed_length() {
+        let inject = b" EI Q BT /F1 9 Tf (X) Tj ET q";
+        let case = |params: &str, len: usize| {
+            let mut samples = inject.to_vec();
+            samples.resize(len, b'0');
+            let mut data = format!("BI {params} ID ").into_bytes();
+            data.extend_from_slice(&samples);
+            data.extend_from_slice(b" EI Q");
+            data
+        };
+        for (params, len) in [
+            ("/W 16 /H 2 /CS /RGB /BPC 8", 96),
+            ("/W 12 /H 2 /CS /CMYK /BPC 8", 96),
+            ("/W 70 /H 4 /CS /G /BPC 1", 36),
+            ("/IM true /W 70 /H 4", 36),
+            ("/W 30 /H 2 /CS [/I /RGB 1 <000000ffffff>] /BPC 4 /L 30", 30),
+            ("/W 15 /H 2 /CS /G /BPC 16 /L 60", 60),
+            ("/W 47 /H 1 /CS /G /BPC 8 /L 5", 47),
+        ] {
+            assert_eq!(
+                operators(&case(params, len)),
+                vec![b"BI".to_vec(), b"Q".to_vec()],
+                "{params}"
+            );
+        }
+    }
 }

vendor/hayro-syntax/src/filter/jpx.rs

@@ -21,7 +21,13 @@ pub(crate) fn decode(data: &[u8], params: &ImageDecodeParams) -> Option<FilterRe
     use crate::object::stream::ImageColorSpace;
 
     let settings = DecodeSettings {
-        resolve_palette_indices: false,
+        // PdfCraft patch: (PdfCraft #3184) with no /ColorSpace in the image dictionary the colors
+        // come from the JP2 header (ISO 32000-2 §7.4.9), so its palette (`pclr`) is applied here;
+        // with a /ColorSpace the header color data is ignored, and an /Indexed space takes the raw
+        // indices. A /ColorSpace that does not resolve also leaves `num_components` unset: the
+        // header's colors are then the only ones there are, so the palette is applied too (raw
+        // indices would be drawn as gray).
+        resolve_palette_indices: params.num_components.is_none(),
         strict: false,
         target_resolution: params.target_dimension,
     };
@@ -41,7 +47,15 @@ pub(crate) fn decode(data: &[u8], params: &ImageDecodeParams) -> Option<FilterRe
 
     let width = image.width();
     let height = image.height();
-    let bpc = params.bpc.unwrap_or(image.original_bit_depth());
+    // PdfCraft patch: (PdfCraft #3184) a resolved palette gives colors, decoded to 8 bits like
+    // every sample; the codestream's precision is that of the indices (1 bit, say), and rescaling
+    // the colors to it kept only 0 or 255 per channel.
+    let palette_resolved = settings.resolve_palette_indices && has_palette(data);
+    let bpc = params.bpc.unwrap_or(if palette_resolved {
+        8
+    } else {
+        image.original_bit_depth()
+    });
     let cs = match image.color_space() {
         ColorSpace::Gray => ImageColorSpace::Gray,
         ColorSpace::RGB => ImageColorSpace::Rgb,
@@ -98,6 +112,15 @@ pub(crate) fn decode(data: &[u8], params: &ImageDecodeParams) -> Option<FilterRe
     })
 }
 
+/// PdfCraft patch: (PdfCraft #3184) whether JP2 file `data` has a palette box (`pclr`) in its
+/// header box (`jp2h`, ISO 15444-1 I.5.3.4). A raw codestream has none.
+fn has_palette(data: &[u8]) -> bool {
+    boxes(data)
+        .find(|&(kind, _, _)| kind == *b"jp2h")
+        .and_then(|(_, start, end)| data.get(start..end))
+        .is_some_and(|header| boxes(header).any(|(kind, _, _)| kind == *b"pclr"))
+}
+
 /// The enumerated colour spaces (`EnumCS`, ISO 15444-1 I.5.3.3 and 15444-2 M.11.7.2) the decoder
 /// handles: CMYK, CIELab, sRGB, greyscale, sYCC, e-sRGB and ROMM-RGB.
 const DECODED_ENUMERATED: [u32; 7] = [12, 14, 16, 17, 18, 20, 21];
@@ -221,8 +244,9 @@ fn scale(
 
 #[cfg(test)]
 mod tests {
-    use super::{scale, without_refused_enumerated_colour};
+    use super::{decode, scale, without_refused_enumerated_colour};
     use crate::bit_reader::{BitReader, bit_mask};
+    use crate::object::stream::{ImageColorSpace, ImageDecodeParams};
     use alloc::vec::Vec;
 
     #[test]
@@ -358,4 +382,118 @@ mod tests {
             assert_eq!(without_refused_enumerated_colour(&data), None, "{bad:#x}");
         }
     }
+
+    /// A 4x2 JP2 written for this test: one 8-bit component whose samples (0, 1) index a
+    /// two-entry sRGB palette (`pclr` + `cmap`): red, then blue. Rows: 0 1 0 1 / 1 0 1 0.
+    const PALETTE_JP2: &[u8] = &[
+        0x00, 0x00, 0x00, 0x0c, 0x6a, 0x50, 0x20, 0x20, 0x0d, 0x0a, 0x87, 0x0a, 0x00, 0x00, 0x00,
+        0x14, 0x66, 0x74, 0x79, 0x70, 0x6a, 0x70, 0x32, 0x20, 0x00, 0x00, 0x00, 0x00, 0x6a, 0x70,
+        0x32, 0x20, 0x00, 0x00, 0x00, 0x55, 0x6a, 0x70, 0x32, 0x68, 0x00, 0x00, 0x00, 0x16, 0x69,
+        0x68, 0x64, 0x72, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x04, 0x00, 0x01, 0x07, 0x07,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x0f, 0x63, 0x6f, 0x6c, 0x72, 0x01, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x10, 0x00, 0x00, 0x00, 0x14, 0x70, 0x63, 0x6c, 0x72, 0x00, 0x02, 0x03, 0x07, 0x07,
+        0x07, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0x00, 0x00, 0x00, 0x14, 0x63, 0x6d, 0x61, 0x70,
+        0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x02, 0x00, 0x00, 0x00,
+        0x8c, 0x6a, 0x70, 0x32, 0x63, 0xff, 0x4f, 0xff, 0x51, 0x00, 0x29, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x04, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x01, 0x07, 0x01, 0x01, 0xff, 0x52, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x01, 0x00, 0x01,
+        0x04, 0x04, 0x00, 0x01, 0xff, 0x5c, 0x00, 0x07, 0x40, 0x40, 0x48, 0x48, 0x50, 0xff, 0x64,
+        0x00, 0x25, 0x00, 0x01, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x20, 0x62, 0x79, 0x20,
+        0x4f, 0x70, 0x65, 0x6e, 0x4a, 0x50, 0x45, 0x47, 0x20, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f,
+        0x6e, 0x20, 0x32, 0x2e, 0x35, 0x2e, 0x34, 0xff, 0x90, 0x00, 0x0a, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x17, 0x00, 0x01, 0xff, 0x93, 0xcf, 0xb4, 0x08, 0x08, 0x67, 0x90, 0x07, 0x41, 0x08,
+        0xff, 0xd9,
+    ];
+
+    /// PdfCraft #3179: with no `/ColorSpace` in the image dictionary, the JP2 palette gives the
+    /// colors. Raw indices (0 and 1) would come back as a gray image, drawn as noise.
+    #[test]
+    fn jpx_palette_is_applied_without_a_color_space() {
+        let params = ImageDecodeParams {
+            width: 4,
+            height: 2,
+            ..ImageDecodeParams::default()
+        };
+        let result = decode(PALETTE_JP2, &params).unwrap();
+        let image = result.image_data.unwrap();
+        assert!(matches!(image.color_space, Some(ImageColorSpace::Rgb)));
+        let red = [255, 0, 0];
+        let blue = [0, 0, 255];
+        let expected: Vec<u8> = [red, blue, red, blue, blue, red, blue, red].concat();
+        assert_eq!(&*result.data, &expected[..]);
+    }
+
+    /// With an `/Indexed` color space in the dictionary, the PDF supplies the palette, so the raw
+    /// indices are what the image holds.
+    #[test]
+    fn jpx_palette_indices_stay_raw_for_an_indexed_color_space() {
+        let params = ImageDecodeParams {
+            is_indexed: true,
+            num_components: Some(1),
+            width: 4,
+            height: 2,
+            ..ImageDecodeParams::default()
+        };
+        let result = decode(PALETTE_JP2, &params).unwrap();
+        assert_eq!(&*result.data, &[0, 1, 0, 1, 1, 0, 1, 0][..]);
+    }
+
+    /// A 1x1, 1-bit codestream (one empty packet, so its sample is the midpoint 1) as a JP2 file
+    /// whose header holds `extra` boxes after `ihdr` and `colr`.
+    fn one_bit_jp2(extra: &[u8]) -> Vec<u8> {
+        let jp2_box = |kind: &[u8], body: &[u8]| {
+            let mut b = ((8 + body.len()) as u32).to_be_bytes().to_vec();
+            b.extend_from_slice(kind);
+            b.extend_from_slice(body);
+            b
+        };
+        let mut codestream = alloc::vec![0xff, 0x4f, 0xff, 0x51, 0, 41, 0, 0];
+        for value in [1_u32, 1, 0, 0, 1, 1, 0, 0] {
+            codestream.extend_from_slice(&value.to_be_bytes());
+        }
+        codestream.extend_from_slice(&[0, 1, 0, 1, 1]);
+        codestream.extend_from_slice(&[0xff, 0x52, 0, 12, 0, 0, 0, 1, 0, 0, 4, 4, 0, 1]);
+        codestream.extend_from_slice(&[0xff, 0x5c, 0, 4, 0x40, 0x40]);
+        codestream.extend_from_slice(&[0xff, 0x90, 0, 10, 0, 0, 0, 0, 0, 15, 0, 1, 0xff, 0x93, 0]);
+        codestream.extend_from_slice(&[0xff, 0xd9]);
+        let mut header = jp2_box(b"ihdr", &[0, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0, 7, 0, 0]);
+        header.extend(jp2_box(b"colr", &[1, 0, 0, 0, 0, 0, 16]));
+        header.extend_from_slice(extra);
+        let mut out = alloc::vec![0, 0, 0, 12, b'j', b'P', b' ', b' ', 0x0d, 0x0a, 0x87, 0x0a];
+        out.extend(jp2_box(b"ftyp", b"jp2 \0\0\0\0jp2 "));
+        out.extend(jp2_box(b"jp2h", &header));
+        out.extend(jp2_box(b"jp2c", &codestream));
+        out
+    }
+
+    /// PdfCraft #3184: resolved palette colors are 8-bit samples whatever the indices' precision;
+    /// an image without a palette keeps its own (1 bit here).
+    #[test]
+    fn palette_colours_are_eight_bit_and_other_samples_keep_their_precision() {
+        let params = ImageDecodeParams {
+            width: 1,
+            height: 1,
+            ..ImageDecodeParams::default()
+        };
+        let bpc = |data: &[u8]| {
+            decode(data, &params)
+                .and_then(|r| r.image_data)
+                .map(|d| d.bits_per_component)
+        };
+        let mut palette = Vec::new();
+        for (kind, body) in [
+            (
+                &b"pclr"[..],
+                &[0, 2, 3, 7, 7, 7, 255, 0, 0, 64, 128, 192][..],
+            ),
+            (b"cmap", &[0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 1, 2]),
+        ] {
+            palette.extend_from_slice(&((8 + body.len()) as u32).to_be_bytes());
+            palette.extend_from_slice(kind);
+            palette.extend_from_slice(body);
+        }
+        assert_eq!(bpc(&one_bit_jp2(&palette)), Some(8));
+        assert_eq!(bpc(&one_bit_jp2(&[])), Some(1));
+    }
 }

vendor/hayro-syntax/src/xref.rs

@@ -759,6 +759,34 @@ fn populate_xref_impl<'a>(data: &'a [u8], pos: usize, xref_map: &mut XrefMap, li
     populate_xref_impl_inner(data, pos, xref_map, &mut visited, limits)
 }
 
+/// The offset of the previous xref section (`Prev`).
+// PdfCraft patch: (PdfCraft #2400) some writers give `Prev` as a reference `N G R`. When this
+// section's own entries (`own`) place that object and it holds an offset (iText's
+// `indirectPrev.pdf`), that is the offset; otherwise its object number is (tiff2pdf, in JHOVE's
+// PDF-HUL-157 files), as other readers take it.
+fn prev_offset(data: &[u8], trailer: &Dict<'_>, own: &XrefMap) -> Option<usize> {
+    if let Some(prev) = trailer.get::<i32>(PREV) {
+        return usize::try_from(prev).ok();
+    }
+
+    let r = trailer.get_ref(PREV)?;
+    let id = ObjectIdentifier::new(r.obj_number, r.gen_number);
+    let held = match own.get(&id) {
+        Some(EntryType::Normal(at)) => {
+            let mut reader = Reader::new(data);
+            reader.jump(*at);
+            reader
+                .read_with_context::<IndirectObject<i32>>(&ReaderContext::dummy())
+                .filter(|o| o.id() == &id)
+                .map(|o| o.get())
+                .filter(|p| *p >= 0)
+        }
+        _ => None,
+    };
+
+    usize::try_from(held.unwrap_or(r.obj_number)).ok()
+}
+
 /// Maximum number of allowed xref `Prev` pointers before we abort.
 const MAX_XREF_CHAIN_DEPTH: usize = 256;
 
@@ -832,17 +860,8 @@ fn populate_from_xref_table<'a>(
     reader.skip_white_spaces();
 
     let mut max_obj = 0;
-
-    if let Some(prev) = trailer.get::<i32>(PREV) {
-        // First insert the entries from any previous xref tables.
-        populate_xref_impl_inner(data, prev as usize, insert_map, visited, limits)?;
-    }
-
-    // In hybrid files, entries in `XRefStm` should have higher priority, therefore we insert them
-    // after looking at `PREV`.
-    if let Some(xref_stm) = trailer.get::<i32>(XREF_STM) {
-        populate_xref_impl_inner(data, xref_stm as usize, insert_map, visited, limits)?;
-    }
+    // PdfCraft patch: (PdfCraft #2400) this section's entries are read first, for `prev_offset`.
+    let mut own = XrefMap::default();
 
     while let Some(header) = reader.read_without_context::<SubsectionHeader>() {
         reader.skip_white_spaces();
@@ -858,7 +877,7 @@ fn populate_from_xref_table<'a>(
             // Specification says we should ignore any object number > SIZE, but probably
             // not important?
             if entry.used {
-                insert_map.insert(
+                own.insert(
                     ObjectIdentifier::new(obj_number as i32, entry.gen_number),
                     EntryType::Normal(entry.offset),
                 );
@@ -866,6 +885,19 @@ fn populate_from_xref_table<'a>(
         }
     }
 
+    if let Some(prev) = prev_offset(data, &trailer, &own) {
+        // First insert the entries from any previous xref tables.
+        populate_xref_impl_inner(data, prev, insert_map, visited, limits)?;
+    }
+
+    // In hybrid files, entries in `XRefStm` should have higher priority, therefore we insert them
+    // after looking at `PREV`.
+    if let Some(xref_stm) = trailer.get::<i32>(XREF_STM) {
+        populate_xref_impl_inner(data, xref_stm as usize, insert_map, visited, limits)?;
+    }
+
+    insert_map.extend(own);
+
     Some(trailer.data())
 }
 
@@ -878,11 +910,6 @@ fn populate_from_xref_stream<'a>(
 ) -> Option<&'a [u8]> {
     let stream = reader.read_with_context::<IndirectObject<Stream<'_>>>(&ReaderContext::dummy_with_limits(limits))?.get();
 
-    if let Some(prev) = stream.dict().get::<i32>(PREV) {
-        // First insert the entries from any previous xref tables.
-        let _ = populate_xref_impl_inner(data, prev as usize, insert_map, visited, limits)?;
-    }
-
     let size = stream.dict().get::<u32>(SIZE)?;
 
     let [f1_len, f2_len, f3_len] = stream.dict().get::<[u8; 3]>(W)?;
@@ -895,6 +922,8 @@ fn populate_from_xref_stream<'a>(
     }
 
     let xref_data = stream.decoded().ok()?;
+    // PdfCraft patch: (PdfCraft #2400) this section's entries are read first, for `prev_offset`.
+    let mut own = XrefMap::default();
     let mut xref_reader = Reader::new(xref_data.as_ref());
 
     if let Some(arr) = stream.dict().get::<Array<'_>>(INDEX) {
@@ -908,21 +937,20 @@ fn populate_from_xref_stream<'a>(
                 f1_len,
                 f2_len,
                 f3_len,
-                insert_map,
+                &mut own,
             )?;
         }
     } else {
-        xref_stream_subsection(
-            &mut xref_reader,
-            0,
-            size,
-            f1_len,
-            f2_len,
-            f3_len,
-            insert_map,
-        )?;
+        xref_stream_subsection(&mut xref_reader, 0, size, f1_len, f2_len, f3_len, &mut own)?;
     }
 
+    if let Some(prev) = prev_offset(data, stream.dict(), &own) {
+        // First insert the entries from any previous xref tables.
+        let _ = populate_xref_impl_inner(data, prev, insert_map, visited, limits)?;
+    }
+
+    insert_map.extend(own);
+
     Some(stream.dict().data())
 }
 
@@ -1450,6 +1478,90 @@ mod tests {
         let _result = populate_xref_impl(pdf.as_ref(), xref_pos, &mut xref_map, DecodeLimits::default());
     }
 
+    #[test]
+    fn prev_as_reference_reaches_the_earlier_section() {
+        let mut pdf = b"%PDF-1.0\n1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n".to_vec();
+        let first_xref_pos = pdf.len();
+        pdf.extend_from_slice(
+            format!(
+                "xref\n\
+                 0 2\n\
+                 0000000000 65535 f \n\
+                 0000000009 00000 n \n\
+                 trailer\n<< /Size 2 /Root 1 0 R >>\n\
+                 startxref\n{first_xref_pos}\n%%EOF\n"
+            )
+            .as_bytes(),
+        );
+        let second_xref_pos = pdf.len();
+        pdf.extend_from_slice(
+            format!(
+                "xref\n\
+                 0 1\n\
+                 0000000000 65535 f \n\
+                 trailer\n<< /Size 2 /Root 1 0 R /Prev {first_xref_pos} 0 R >>\n\
+                 startxref\n{second_xref_pos}\n%%EOF"
+            )
+            .as_bytes(),
+        );
+
+        let mut xref_map = FxHashMap::default();
+        let xref_pos = find_last_xref_pos(pdf.as_ref()).unwrap();
+        assert_eq!(xref_pos, second_xref_pos);
+        populate_xref_impl(
+            pdf.as_ref(),
+            xref_pos,
+            &mut xref_map,
+            DecodeLimits::default(),
+        )
+        .unwrap();
+        assert!(xref_map.contains_key(&ObjectIdentifier::new(1, 0)));
+    }
+
+    #[test]
+    fn prev_as_reference_to_an_object_holding_the_offset_reads_that_offset() {
+        let mut pdf = b"%PDF-1.0\n1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n".to_vec();
+        let first_xref_pos = pdf.len();
+        pdf.extend_from_slice(
+            format!(
+                "xref\n\
+                 0 2\n\
+                 0000000000 65535 f \n\
+                 0000000009 00000 n \n\
+                 trailer\n<< /Size 2 /Root 1 0 R >>\n\
+                 startxref\n{first_xref_pos}\n%%EOF\n"
+            )
+            .as_bytes(),
+        );
+        // Object 3 holds the offset; read as offset 3, `Prev` would point inside the header.
+        let held_pos = pdf.len();
+        pdf.extend_from_slice(format!("3 0 obj\n{first_xref_pos}\nendobj\n").as_bytes());
+        let second_xref_pos = pdf.len();
+        pdf.extend_from_slice(
+            format!(
+                "xref\n\
+                 3 1\n\
+                 {held_pos:010} 00000 n \n\
+                 trailer\n<< /Size 4 /Root 1 0 R /Prev 3 0 R >>\n\
+                 startxref\n{second_xref_pos}\n%%EOF"
+            )
+            .as_bytes(),
+        );
+
+        let mut xref_map = FxHashMap::default();
+        let xref_pos = find_last_xref_pos(pdf.as_ref()).unwrap();
+        assert_eq!(xref_pos, second_xref_pos);
+        populate_xref_impl(
+            pdf.as_ref(),
+            xref_pos,
+            &mut xref_map,
+            DecodeLimits::default(),
+        )
+        .unwrap();
+        assert!(xref_map.contains_key(&ObjectIdentifier::new(1, 0)));
+        assert!(xref_map.contains_key(&ObjectIdentifier::new(3, 0)));
+    }
+
     #[test]
     fn find_last_xref_uses_last_startxref() {
         let pdf = b"%PDF-1.0\nstartxref\n5\n%%EOF\nstartxref\n42\n%%EOF";