From 4b57bd1eb6a635008169440593156d810c930e6d Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Mon, 2 Nov 2026 13:03:25 +0100 Subject: [PATCH] M8: what is made direct to read a stream is small, however often it is named direct_parms copied every value it followed, and direct() every /Filter entry, so each reference to one large array copied it again: about 5.5 MB a reference, 11 GB from a file of about 225 KB. Now, as Document::with_direct_parms does, a size is a number, /Filter a name or an array of names, and direct_parms() keeps of each entry the numbers and flags decoders read (DECODE_PARMS_KEYS, now public in cos) and /JBIG2Globals as given, for as many entries as there are filters. Anything else is null. A kept image whose own /DecodeParms come through a chain is stored again with them, which 2eddb1bc9 made so: what_stays tests it now. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/cos/src/document.rs | 1 + crates/cos/src/document/objects.rs | 2 +- crates/cos/src/lib.rs | 2 +- crates/redact/src/globals.rs | 62 ++++++++++++++++++++++-------- crates/redact/src/tests.rs | 22 +++++++++++ 5 files changed, 70 insertions(+), 19 deletions(-) diff --git a/crates/cos/src/document.rs b/crates/cos/src/document.rs index 082d1910d1..0508c4024c 100644 --- a/crates/cos/src/document.rs +++ b/crates/cos/src/document.rs @@ -19,6 +19,7 @@ mod reader; mod state; mod xref; +pub use objects::DECODE_PARMS_KEYS; #[cfg(test)] pub(crate) use reader::find; #[cfg(test)] diff --git a/crates/cos/src/document/objects.rs b/crates/cos/src/document/objects.rs index 39a4d9a6da..8f2e7a973f 100644 --- a/crates/cos/src/document/objects.rs +++ b/crates/cos/src/document/objects.rs @@ -781,7 +781,7 @@ impl Document { /// The `/DecodeParms` entries the decoders read (ISO 32000-2 Tables 8, 11 and 13: the Flate and /// LZW predictors, CCITT, DCT's colour transform): numbers and flags a file may give by /// reference, which [`Dict::int`] doesn't follow. -const DECODE_PARMS_KEYS: [&[u8]; 13] = [ +pub const DECODE_PARMS_KEYS: [&[u8]; 13] = [ b"Predictor", b"Colors", b"BitsPerComponent", diff --git a/crates/cos/src/lib.rs b/crates/cos/src/lib.rs index 76dc0b6fe7..9d45d8f982 100644 --- a/crates/cos/src/lib.rs +++ b/crates/cos/src/lib.rs @@ -24,7 +24,7 @@ mod writer; pub use budget::{DecodeBudget, MAX_PAGE_DECODED, MAX_PAGE_FORMS}; pub use bytes::Bytes; -pub use document::{Document, ReadState, Revision, SourceIdentity, XrefEntry}; +pub use document::{DECODE_PARMS_KEYS, Document, ReadState, Revision, SourceIdentity, XrefEntry}; pub use object::{Dict, MAX_DECODED, MAX_JBIG2_GLOBALS, Name, ObjRef, Object, PdfString, Stream}; pub use parser::{Lexer, is_whitespace, parse_indirect}; pub use pdfcraft_crypt::{ diff --git a/crates/redact/src/globals.rs b/crates/redact/src/globals.rs index b095059a7e..476b3d955b 100644 --- a/crates/redact/src/globals.rs +++ b/crates/redact/src/globals.rs @@ -9,7 +9,7 @@ use std::collections::{HashMap, HashSet}; use std::hash::{Hash, Hasher}; use std::sync::Arc; -use pdfcraft_cos::{Document, MAX_JBIG2_GLOBALS, Object, Stream, reachable}; +use pdfcraft_cos::{DECODE_PARMS_KEYS, Dict, Document, MAX_JBIG2_GLOBALS, Object, Stream, reachable}; /// The references followed from `/JBIG2Globals` towards a stream, so that a ring of them ends. const MAX_HOPS: usize = 32; @@ -60,44 +60,72 @@ fn named(doc: &Document, s: &Stream) -> Vec { found } -/// `v`, or what references lead to from it, inside an array too. -fn direct(doc: &Document, v: &Object) -> Object { +/// A `/Filter` through any references, as a decoder reads it: a name, or an array of names, each +/// entry through references too. Anything else is null, so nothing large is copied however often +/// the file refers to it. +fn direct_filter(doc: &Document, v: &Object) -> Object { + let name = |o: &Object| if matches!(o, Object::Name(_)) { o.clone() } else { Object::Null }; match &*followed(doc, v) { - Object::Array(a) => Object::Array(a.iter().map(|e| (*followed(doc, e)).clone()).collect()), - other => other.clone(), + Object::Array(a) => Object::Array(a.iter().map(|e| name(&followed(doc, e))).collect()), + other => name(other), } } -/// `/DecodeParms` as a reader takes them, through any references: the array or dictionary, each -/// entry, and each value made direct, but `/JBIG2Globals`, which stays the reference it is. -fn direct_parms(doc: &Document, v: &Object) -> Object { +/// A size through any references: a number, or null. +fn direct_number(doc: &Document, v: &Object) -> Object { + match &*followed(doc, v) { + number @ (Object::Int(_) | Object::Real(_)) => number.clone(), + _ => Object::Null, + } +} + +/// `/DecodeParms` as a reader takes them, through any references, for `filters` filters: of each +/// entry, the numbers and flags the decoders read ([`DECODE_PARMS_KEYS`]), and `/JBIG2Globals` as +/// given. An entry that isn't a dictionary is null. As `Document::with_direct_parms`, nothing large +/// is copied however often the file refers to it. +fn direct_parms(doc: &Document, v: &Object, filters: usize) -> Object { let entry = |e: &Object| match &*followed(doc, e) { Object::Dict(d) => { - let mut made = d.clone(); - for (key, value) in d.iter().filter(|(key, _)| key.as_slice() != b"JBIG2Globals") { - made.set(key.to_vec(), (*followed(doc, value)).clone()); + let mut made = Dict::new(); + for key in DECODE_PARMS_KEYS { + let Some(value) = d.get(key) else { continue }; + let value = followed(doc, value); + if matches!(*value, Object::Int(_) | Object::Real(_) | Object::Bool(_)) { + made.set(key.to_vec(), (*value).clone()); + } + } + if let Some(globals) = d.get(b"JBIG2Globals") { + made.set(b"JBIG2Globals".to_vec(), globals.clone()); } Object::Dict(made) } - other => other.clone(), + _ => Object::Null, }; match &*followed(doc, v) { - Object::Array(a) => Object::Array(a.iter().map(entry).collect()), + Object::Array(a) => Object::Array(a.iter().take(filters).map(entry).collect()), other => entry(other), } } /// `s` with what `decoded_image` and `decoded_within` read as given made direct: its size, filters /// and their parameters. -fn readable(doc: &Document, s: &Stream) -> Stream { +pub(crate) fn readable(doc: &Document, s: &Stream) -> Stream { let mut readable = s.clone(); - for key in [b"Width".as_slice(), b"Height", b"Filter"] { + for key in [b"Width".as_slice(), b"Height"] { if let Some(v) = s.dict.get(key) { - readable.dict.set(key.to_vec(), direct(doc, v)); + readable.dict.set(key.to_vec(), direct_number(doc, v)); } } + if let Some(v) = s.dict.get(b"Filter") { + readable.dict.set(b"Filter".to_vec(), direct_filter(doc, v)); + } if let Some(v) = s.dict.get(b"DecodeParms") { - readable.dict.set(b"DecodeParms".to_vec(), direct_parms(doc, v)); + let filters = match readable.dict.get(b"Filter") { + Some(Object::Array(a)) => a.len(), + Some(_) => 1, + None => 0, + }; + readable.dict.set(b"DecodeParms".to_vec(), direct_parms(doc, v, filters)); } readable } diff --git a/crates/redact/src/tests.rs b/crates/redact/src/tests.rs index 48d00da332..eb1d1d96dd 100644 --- a/crates/redact/src/tests.rs +++ b/crates/redact/src/tests.rs @@ -1565,6 +1565,7 @@ fn what_stays_is_stored_without_the_dictionary_however_it_names_it() { (format!("{image} /F 99 0 R"), scan.page.clone(), vec![]), // Through a reference to a reference: /F to null, the size, the filter, the colour space. (format!("{image} /F 10 0 R"), scan.page.clone(), vec![b"11 0 R".to_vec(), b"null".to_vec()]), + (shared_image("/DecodeParms 10 0 R"), scan.page.clone(), vec![b"11 0 R".to_vec(), b"<< /JBIG2Globals 9 0 R >>".to_vec()]), (image.replace("/Width 400", "/Width 10 0 R"), scan.page.clone(), vec![b"11 0 R".to_vec(), b"400".to_vec()]), (image.replace("/Height 40", "/Height 10 0 R"), scan.page.clone(), vec![b"11 0 R".to_vec(), b"40".to_vec()]), (image.replace("/Filter /JBIG2Decode", "/Filter 10 0 R"), scan.page.clone(), vec![b"11 0 R".to_vec(), b"/JBIG2Decode".to_vec()]), @@ -1791,6 +1792,27 @@ fn a_dictionary_that_names_itself_isnt_counted_as_stored() { assert!(matches!(apply(&mut doc, None), Err(RedactError::SharedJbig2Globals(1)))); } +/// What a stream's size, filters and parameters are made direct to is small, however often they +/// refer to a large object: only numbers, names and the parameters decoders read are copied. +#[test] +fn what_is_made_direct_is_small_however_often_a_large_object_is_named() { + let big = format!("[{}]", "1 ".repeat(100_000)); + let keys: String = (0..50).map(|k| format!(" /K{k} 2 0 R")).collect(); + // Three filters, and parameters for 2,001: the entries past the filters are dropped. + let (filters, entries) = (" 2 0 R".repeat(2), " 2 0 R".repeat(2000)); + let dict = + format!("/Width 2 0 R /Height 2 0 R /Filter [/FlateDecode{filters}] /DecodeParms [<< /Predictor 3 0 R /Columns 2 0 R{keys} >>{entries}]"); + let doc = pdf(vec![b"<< /Type /Catalog >>".to_vec(), big.into_bytes(), b"4 0 R".to_vec(), b"12".to_vec(), stream(&dict, b"")]); + let Object::Stream(s) = &*doc.get(ObjRef::new(5, 0)) else { panic!("object 5 is the stream") }; + let made = crate::globals::readable(&doc, s); + let mut bytes = Vec::new(); + pdfcraft_cos::serialize(&Object::Dict(made.dict.clone()), &mut bytes); + assert!(bytes.len() < 4096, "{} bytes made direct", bytes.len()); + let parms = made.dict.get(b"DecodeParms").and_then(Object::as_array).unwrap(); + assert_eq!(parms.first().and_then(Object::as_dict).and_then(|d| d.get(b"Predictor")), Some(&Object::Int(12))); + assert_eq!(parms.len(), 3, "an entry for each filter"); +} + /// A copy of the dictionary stored with a predictor is found whatever way its parameters are given, /// through a chain of references among them, as a reader that follows them decodes it. Through a /// chain, PdfCraft can't decode the image with the copy's symbols to store it again, so the run is