diff --git a/crates/automation/src/tools.rs b/crates/automation/src/tools.rs index 91abb54061..b78acbe0f9 100644 --- a/crates/automation/src/tools.rs +++ b/crates/automation/src/tools.rs @@ -1442,7 +1442,7 @@ pub fn tools() -> Vec { .with(schema(json!({ "doc": doc() }), &["doc"])), t("doc_open_revision", "Open a revision", "Open saved revision `revision` (1 = the oldest) of a document as a new, unsaved document, to see the file as it was then.") .with(schema(json!({ "doc": doc(), "revision": { "type": "integer", "minimum": 1 } }), &["doc", "revision"])), - t("doc_optimize", "Optimize PDF", "Write an optimized copy to `path` (Acrobat's PDF Optimizer). preset: standard (default; Reduce File Size's choices) or mobile (images to 96 ppi above 144, JPEG high quality); the options below change it. compatibility: retain (default; keeps the document's version, or 1.5 when object streams are written), 1.4, 1.5, 1.6, 1.7 or 2.0. merge_fragments (default true) first puts a page picture drawn as many stacked image strips (as some PDF printers write it) back together into one image, so it can be resampled (merged_images, fragments_merged). color / gray: { downsample, ppi, above_ppi, compression: jpeg|zip|retain, quality 1–100 } (defaults: downsample to 150 ppi above 225, JPEG 60). mono: { downsample, ppi, above_ppi, compression: ccitt_g4|jbig2|retain } for black-and-white images and image masks (default: averaged down to 300 ppi above 450 ppi, counted in images_resampled; stored losslessly as CCITT Group 4 or JBIG2 (or 1-bit Flate, once resampled or packed from more bits) when that is smaller, Group 3 fax images, JBIG2 images, black-and-white RGB and indexed images using only palette entries 0 and 1 among them, counted in images_recompressed; retain resamples only, a CCITT image staying CCITT, a JBIG2 image JBIG2 and any other becoming Flate). JBIG2 images past what one run may decode stay as they are, counted in images_refused. Images are measured where pages draw them and replaced only when smaller. discard_*: thumbnails (default true), alternate_images (true), tags, print_settings; flate_unencoded (true); object_streams (default true; always off for 1.4); lzw_to_flate (true) decodes LZW streams and stores them with Flate instead, leaving any it cannot decode unchanged (lzw_streams counts the streams converted); remove_invalid_links and remove_unreferenced_dests (true); images and forms the pages list but never draw are always dropped (unused_xobjects). discard: Remove Hidden Information categories (metadata, attachments, comments, form-fields, hidden-text, hidden-layers, bookmarks, links-actions-scripts, private-data). Signed documents are refused. The open document is unchanged.") + t("doc_optimize", "Optimize PDF", "Write an optimized copy to `path` (Acrobat's PDF Optimizer). preset: standard (default; Reduce File Size's choices) or mobile (images to 96 ppi above 144, JPEG high quality); the options below change it. compatibility: retain (default; keeps the document's version, or 1.5 when object streams are written), 1.4, 1.5, 1.6, 1.7 or 2.0. merge_fragments (default true) first puts a page picture drawn as many stacked image strips (as some PDF printers write it) back together into one image, so it can be resampled (merged_images, fragments_merged). color / gray: { downsample, ppi, above_ppi, compression: jpeg|zip|retain, quality 1–100 } (defaults: downsample to 150 ppi above 225, JPEG 60). mono: { downsample, ppi, above_ppi, compression: ccitt_g4|jbig2|retain } for black-and-white images and image masks (default: averaged down to 300 ppi above 450 ppi, counted in images_resampled; stored losslessly as CCITT Group 4 or JBIG2 (or 1-bit Flate, once resampled or packed from more bits) when that is smaller, Group 3 fax images, JBIG2 images, black-and-white RGB and indexed images using only palette entries 0 and 1 among them, counted in images_recompressed; retain resamples only, a CCITT image staying CCITT, a JBIG2 image JBIG2 and any other becoming Flate). Images past what one run may decode stay as they are, counted in images_refused. Images are measured where pages draw them and replaced only when smaller. discard_*: thumbnails (default true), alternate_images (true), tags, print_settings; flate_unencoded (true); object_streams (default true; always off for 1.4); lzw_to_flate (true) decodes LZW streams and stores them with Flate instead, leaving any it cannot decode unchanged (lzw_streams counts the streams converted); remove_invalid_links and remove_unreferenced_dests (true); images and forms the pages list but never draw are always dropped (unused_xobjects). discard: Remove Hidden Information categories (metadata, attachments, comments, form-fields, hidden-text, hidden-layers, bookmarks, links-actions-scripts, private-data). Signed documents are refused. The open document is unchanged.") .cmd("optimize.advanced") .with(schema( json!({ diff --git a/crates/cos/README.md b/crates/cos/README.md index f3195a46dd..ff192d2e6e 100644 --- a/crates/cos/README.md +++ b/crates/cos/README.md @@ -94,5 +94,6 @@ recognisable in a diff and byte-stable when re-serialized. (`Document::decoded_image`, a JBIG2 image's `/JBIG2Globals` within `MAX_JBIG2_GLOBALS`, 32 MiB, their filters undone once a document for the images that share them, up to 64 MiB kept, while their segments are decoded for every image, and `ImageBudget` for an operation over many images, - which share one allowance of JBIG2 decoding and are refused past it); + which share one allowance of decoding, filters and fax and JBIG2 decoders alike, and are refused + past it); DCT and JPX are still decoded through the bootstrap renderer (`core.image-filters`, M2.6). diff --git a/crates/cos/src/budget.rs b/crates/cos/src/budget.rs index f982a39b45..b2ea9a82fd 100644 --- a/crates/cos/src/budget.rs +++ b/crates/cos/src/budget.rs @@ -6,15 +6,17 @@ use pdfcraft_filters::FilterError; use crate::{CosError, Document, ObjRef, Object, Stream}; -/// The steps of JBIG2 decoding ([`hayro_jbig2::Image::decode_within`]) an operation over a -/// document's images may take whatever the file's size. A step is about a pixel, and takes 6 to 16 -/// ns to decode, so this is 2 to 5 minutes, and about as long again for what an operation does with -/// the pixels. A page scanned at 300 dpi takes about 17 million, 8.4 for its pixels and that again -/// to decode them, so this alone reads about 1,000 of them. +/// The steps of decoding an operation over a document's images may take whatever the file's size. +/// A step of JBIG2 decoding ([`hayro_jbig2::Image::decode_within`]) is about a pixel, and takes 6 to +/// 16 ns, so this is 2 to 5 minutes, and about as long again for what an operation does with the +/// pixels. A page scanned at 300 dpi takes about 17 million, 8.4 for its pixels and that again to +/// decode them, so this alone reads about 1,000 of them. A byte into or out of a filter is a step +/// too, and takes less: 3 ns at most, or up to 19 ns through a predictor on samples of a bit. const IMAGE_WORK: u64 = 1 << 34; /// The steps each byte of the file adds to [`IMAGE_WORK`], so that a long scan is read whole: such a /// page pays for itself with about 2 KB of the file, about the smallest a page of text is stored -/// in. So what a hostile file can ask for grows with its length, by about 14 minutes for 10 MB. +/// in, and a stream of any filter this reads can be decoded about 8 times at Flate's best, 1,032 to +/// one. So what a hostile file can ask for grows with its length, by about 14 minutes for 10 MB. const IMAGE_WORK_PER_BYTE: u64 = 1 << 13; /// Decoded bytes one page's content walk may produce in all: its `/Contents` streams and every @@ -30,14 +32,18 @@ type Decoder = fn(&Stream, usize) -> Result<(Vec, usize), (FilterError, usiz /// What one operation over a document's images may still spend decoding them: exporting them, /// optimizing them, clearing them for redaction. /// -/// An image has its own limit, which doesn't bound a file of thousands of images made to reach it. -/// JBIG2 is what such a file would use: a few hundred bytes can ask for seconds of decoding, and an -/// image that shares `/JBIG2Globals` decodes them again. So those images share this allowance, and -/// past it they aren't decoded. A caller then treats them as it does an image it can't read, and +/// An image has its own limit, which doesn't bound a file of thousands of images made to reach it: +/// a few hundred bytes of JBIG2 can ask for seconds of decoding, 4 KB of Flate inside Flate for a +/// gigabyte, and an image that shares `/JBIG2Globals` decodes them again. So images share this +/// allowance, whatever they do with what they decode and whether or not it works, and past it they +/// aren't decoded. A caller then treats them as it does an image it can't read, and /// [`ImageBudget::refused`] says how many there were. +/// +/// JPEG and JPEG 2000 data aren't charged: their decoders don't run here, and what one asks for +/// grows with its length, as the allowance does. #[derive(Debug)] pub struct ImageBudget { - jbig2_work: u64, + work: u64, refused: usize, } @@ -48,28 +54,46 @@ impl ImageBudget { Self::with_work(IMAGE_WORK.saturating_add(bytes.saturating_mul(IMAGE_WORK_PER_BYTE))) } - /// An allowance of `jbig2_work` steps of JBIG2 decoding. - pub fn with_work(jbig2_work: u64) -> Self { - Self { jbig2_work, refused: 0 } + /// An allowance of `work` steps of decoding. + pub fn with_work(work: u64) -> Self { + Self { work, refused: 0 } } /// Image `s` decoded like [`Document::decoded_image_within`], charged to the allowance. pub fn decode(&mut self, doc: &Document, s: &Stream, max: usize) -> Result, CosError> { - let decoded = doc.decoded_image_spending(s, max, &mut self.jbig2_work); - if decoded.is_err() && self.jbig2_work == 0 && matches!(s.filters().last(), Some((pdfcraft_filters::Filter::Jbig2, _))) { - self.refused += 1; - } + let decoded = doc.decoded_image_spending(s, max, &mut self.work); + self.count_refused(s, &decoded); decoded } - /// The JBIG2 images refused for lack of allowance. + /// Image `s` through its filters alone, charged to the allowance: [`Stream::decoded_within`] + /// with its `/DecodeParms` given by reference read too ([`Document::with_direct_parms`]), for an + /// image that has no fax or JBIG2 decoder to go through. + pub fn decode_filters(&mut self, doc: &Document, s: &Stream, max: usize) -> Result, CosError> { + let decoded = doc.with_direct_parms(s).decoded_spending(max, &mut self.work); + self.count_refused(s, &decoded); + decoded + } + + /// An image that failed with nothing left was refused, unless it has a filter that costs + /// nothing here, which fails whatever is left. + fn count_refused(&mut self, s: &Stream, decoded: &Result, CosError>) { + use pdfcraft_filters::Filter; + let charged = + s.readable_filters().is_ok_and(|chain| chain.iter().all(|(f, _)| f.is_decodable() || matches!(f, Filter::Jbig2 | Filter::CcittFax))); + if decoded.is_err() && self.work == 0 && charged { + self.refused += 1; + } + } + + /// The decodes refused for lack of allowance: an image read twice over is counted twice. pub fn refused(&self) -> usize { self.refused } - /// The steps of JBIG2 decoding still allowed. + /// The steps of decoding still allowed. pub fn work_left(&self) -> u64 { - self.jbig2_work + self.work } } @@ -216,8 +240,9 @@ mod image_tests { Stream::from_raw(dict, pdfcraft_filters::encode_jbig2_generic(&[0xFF; 4], 16, 2).unwrap()) } - /// Each JBIG2 image spends what it took from one allowance, and those past it are refused and - /// counted. Other images aren't charged, and a failure of theirs isn't counted as one of these. + /// Each image spends what it took from one allowance, and those past it are refused and + /// counted, whatever their filters. What can't be decoded here costs nothing, and fails + /// without being one of these. #[test] fn the_images_of_an_operation_share_its_allowance() { let doc = Document::new_empty(); @@ -236,12 +261,180 @@ mod image_tests { assert!(budget.decode(&doc, &image, 1 << 20).is_err()); assert_eq!(budget.refused(), refused); } - let flate = Stream::flate(Dict::new(), b"samples"); - assert_eq!(budget.decode(&doc, &flate, 1 << 20).unwrap(), b"samples", "not JBIG2: no charge"); + assert!(budget.decode(&doc, &Stream::flate(Dict::new(), b"samples"), 1 << 20).is_err()); + assert!(budget.decode(&doc, &Stream::from_raw(Dict::new(), b"samples".to_vec()), 1 << 20).is_err()); + assert_eq!(budget.refused(), 4, "other images share it"); + for name in ["NoSuchFilter", "DCTDecode", "JPXDecode"] { + let mut dict = Dict::new(); + dict.set(b"Filter".to_vec(), Object::name(name)); + assert!(budget.decode(&doc, &Stream::from_raw(dict, b"data".to_vec()), 1 << 20).is_err()); + assert_eq!(budget.refused(), 4, "{name} isn't one of these"); + } let mut dict = Dict::new(); - dict.set(b"Filter".to_vec(), Object::name("NoSuchFilter")); + dict.set(b"Filter".to_vec(), Object::Int(5)); assert!(budget.decode(&doc, &Stream::from_raw(dict, b"data".to_vec()), 1 << 20).is_err()); - assert_eq!(budget.refused(), 2, "not JBIG2: not one of these"); + assert_eq!(budget.refused(), 4, "a /Filter that can't be read isn't either"); + } + + /// What a file of images each inflating a megabyte asks for: 4 KB of Flate inside Flate can ask + /// for a gigabyte, and these aren't JBIG2, so without the allowance nothing would stop them. + #[test] + fn filter_bombs_stop_at_the_allowance() { + let doc = Document::new_empty(); + let samples = vec![0; 1 << 20]; + let inner = pdfcraft_filters::encode_flate(&samples); + let mut dict = Dict::new(); + dict.set(b"Filter".to_vec(), Object::Array(vec![Object::name("FlateDecode"), Object::name("FlateDecode")])); + let bomb = Stream::from_raw(dict, pdfcraft_filters::encode_flate(&inner)); + let cost = (bomb.raw.len() + inner.len() + samples.len()) as u64; + let mut budget = ImageBudget::with_work(2 * cost + 1); + for left in [cost + 1, 1] { + assert_eq!(budget.decode(&doc, &bomb, 1 << 20).unwrap(), samples); + assert_eq!(budget.work_left(), left); + } + assert!(budget.decode(&doc, &bomb, 1 << 20).is_err()); + assert_eq!((budget.work_left(), budget.refused()), (0, 1), "the first it can't pay for ends it"); + } + + /// Filters in front of a JBIG2 decoder pay as they run, on top of what the decoder takes, and + /// one that fails has still paid: nothing about the JBIG2 data is known by then, and each image + /// of a file can do this. + #[test] + fn filters_in_front_of_jbig2_pay_whether_or_not_they_work() { + let doc = Document::new_empty(); + let alone = jbig2(None); + let mut budget = ImageBudget::with_work(u64::MAX); + let samples = budget.decode(&doc, &alone, 1 << 20).unwrap(); + let decoding = u64::MAX - budget.work_left(); + let behind = |raw: Vec| { + let mut dict = alone.dict.clone(); + dict.set(b"Filter".to_vec(), Object::Array(vec![Object::name("FlateDecode"), Object::name("JBIG2Decode")])); + Stream::from_raw(dict, raw) + }; + + let image = behind(pdfcraft_filters::encode_flate(&alone.raw)); + let mut budget = ImageBudget::with_work(u64::MAX); + assert_eq!(budget.decode(&doc, &image, 1 << 20).unwrap(), samples); + assert_eq!(u64::MAX - budget.work_left(), decoding + (image.raw.len() + alone.raw.len()) as u64); + + // Past this image's limit: an error, with what finding out took paid and the rest left. + let bomb = behind(pdfcraft_filters::encode_flate(&vec![0; 1 << 16])); + let mut budget = ImageBudget::with_work(u64::MAX); + assert!(budget.decode(&doc, &bomb, 1000).is_err()); + assert_eq!(u64::MAX - budget.work_left(), bomb.raw.len() as u64 + 1000); + assert_eq!(budget.refused(), 0, "that image's limit isn't the run's"); + + // A small allowance is a limit too: it stops there, and the image is refused. + let mut budget = ImageBudget::with_work(bomb.raw.len() as u64 + 10); + assert!(budget.decode(&doc, &bomb, 1 << 20).is_err()); + assert_eq!((budget.work_left(), budget.refused()), (0, 1)); + } + + /// Globals that are read again for every image pay every time, and fail having paid. Kept + /// ones have no filters left to undo, so they pay for being read and nothing more. + #[test] + fn globals_pay_for_their_filters_each_time_they_are_decoded() { + let skipped = [0, 0, 0, 0, 0x3E, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1]; + let raw = pdfcraft_filters::encode_flate(&skipped); + let stream = |doc: &Document, r: ObjRef| match &*doc.get(r) { + Object::Stream(s) => s.clone(), + _ => panic!("not a stream"), + }; + let spent = |doc: &Document, s: &Stream| { + let mut budget = ImageBudget::with_work(u64::MAX); + budget.decode(doc, s, 1 << 20).unwrap(); + u64::MAX - budget.work_left() + }; + let mut doc = Document::new_empty(); + let mut dict = Dict::new(); + dict.set(b"Filter".to_vec(), Object::name("FlateDecode")); + let globals = doc.add(Object::Stream(Stream::from_raw(dict, raw.clone()))); + let mut parms = Dict::new(); + parms.set(b"JBIG2Globals".to_vec(), Object::Ref(globals)); + let image = doc.add(Object::Stream(jbig2(Some(Object::Dict(parms))))); + let root = doc.root().unwrap(); + doc.update_dict(root, |c| c.set(b"Image".to_vec(), Object::Ref(image))).unwrap(); + + // Added in memory, so not kept: the filters are undone for each read. + let s = stream(&doc, image); + let first = spent(&doc, &s); + assert_eq!(spent(&doc, &s), first, "not kept, so paid for again"); + + // Read from a file, so kept: the second read has no filters to undo. + let file = crate::write_full(&doc, &crate::SaveOptions::default()).unwrap(); + let doc = Document::open(std::sync::Arc::new(file)).unwrap(); + let image = doc.get(doc.root().unwrap()).as_dict().unwrap().reference(b"Image").unwrap(); + let s = stream(&doc, image); + let filters = (raw.len() + skipped.len()) as u64; + assert_eq!(spent(&doc, &s), first, "the first read undoes them"); + assert_eq!(spent(&doc, &s), first - filters, "kept"); + + // Globals that fail pay too, with the image not refused: these go past their own limit. + let over = pdfcraft_filters::encode_flate(&vec![0; crate::MAX_JBIG2_GLOBALS + 1]); + let mut doc = Document::new_empty(); + let mut dict = Dict::new(); + dict.set(b"Filter".to_vec(), Object::name("FlateDecode")); + let globals = doc.add(Object::Stream(Stream::from_raw(dict, over.clone()))); + let mut parms = Dict::new(); + parms.set(b"JBIG2Globals".to_vec(), Object::Ref(globals)); + let image = jbig2(Some(Object::Dict(parms))); + let mut budget = ImageBudget::with_work(u64::MAX); + assert!(budget.decode(&doc, &image, 1 << 20).unwrap_err().to_string().contains("globals")); + assert_eq!(u64::MAX - budget.work_left(), (over.len() + crate::MAX_JBIG2_GLOBALS) as u64); + assert_eq!(budget.refused(), 0); + } + + /// A fax image pays for what comes before its decoder and for the rows it asks for, and past + /// the allowance it is refused like any other. + #[test] + fn fax_images_share_the_allowance() { + let doc = Document::new_empty(); + let (w, h) = (4096u32, 64u32); + let coded = pdfcraft_filters::encode_g4(&vec![0xFF; 512 * h as usize], w, h).unwrap(); + let mut parms = Dict::new(); + parms.set(b"K".to_vec(), Object::Int(-1)); + parms.set(b"Columns".to_vec(), Object::Int(i64::from(w))); + let mut dict = Dict::new(); + dict.set(b"Width".to_vec(), Object::Int(i64::from(w))); + dict.set(b"Height".to_vec(), Object::Int(i64::from(h))); + dict.set(b"Filter".to_vec(), Object::Array(vec![Object::name("FlateDecode"), Object::name("CCITTFaxDecode")])); + dict.set(b"DecodeParms".to_vec(), Object::Array(vec![Object::Null, Object::Dict(parms)])); + let image = Stream::from_raw(dict, pdfcraft_filters::encode_flate(&coded)); + let cost = (image.raw.len() + coded.len() + 4 * coded.len() + 512 * h as usize) as u64; + let mut budget = ImageBudget::with_work(cost); + assert_eq!(budget.decode(&doc, &image, 1 << 20).unwrap().len(), 512 * h as usize); + assert_eq!(budget.work_left(), 0, "exactly that"); + assert!(budget.decode(&doc, &image, 1 << 20).is_err()); + assert_eq!(budget.refused(), 1); + } + + /// An image read through its filters alone reads parameters given by reference, is charged, and + /// is refused like the others. It isn't given to a decoder. + #[test] + fn filters_alone_are_charged_too() { + let mut doc = Document::new_empty(); + let p = pdfcraft_filters::Params { predictor: 12, columns: 4, ..pdfcraft_filters::Params::default() }; + let rows = vec![7u8; 400]; + let mut parms = Dict::new(); + parms.set(b"Predictor".to_vec(), Object::Int(12)); + parms.set(b"Columns".to_vec(), Object::Int(4)); + let parms = doc.add(Object::Dict(parms)); + let mut dict = Dict::new(); + dict.set(b"Filter".to_vec(), Object::name("FlateDecode")); + dict.set(b"DecodeParms".to_vec(), Object::Ref(parms)); + let raw = pdfcraft_filters::encode(&pdfcraft_filters::Filter::Flate, &p, &rows).unwrap(); + let image = Stream::from_raw(dict, raw); + let mut budget = ImageBudget::with_work(u64::MAX); + assert_eq!(budget.decode_filters(&doc, &image, 1 << 20).unwrap(), rows); + let cost = u64::MAX - budget.work_left(); + // The tag byte of each of its 100 rows counts, which the predictor takes off. + assert_eq!(cost, (image.raw.len() + rows.len() + 100) as u64); + let mut budget = ImageBudget::with_work(cost - 1); + assert!(budget.decode_filters(&doc, &image, 1 << 20).is_err()); + assert_eq!((budget.work_left(), budget.refused()), (0, 1)); + let mut budget = ImageBudget::with_work(u64::MAX); + assert!(budget.decode_filters(&doc, &jbig2(None), 1 << 20).is_err()); + assert_eq!(budget.work_left(), u64::MAX, "no decoder here, so nothing was read"); } /// The images of a file share its globals, so an image refused doesn't read them: this one's @@ -258,7 +451,7 @@ mod image_tests { let read = ImageBudget::with_work(u64::MAX).decode(&doc, &image, 1 << 20).unwrap_err().to_string(); assert!(read.contains("globals"), "{read}"); let refused = ImageBudget::with_work(0).decode(&doc, &image, 1 << 20).unwrap_err().to_string(); - assert!(refused.contains("more JBIG2 decoding than"), "{refused}"); + assert!(refused.contains("more image decoding than"), "{refused}"); } /// A longer file gets more, by its length, and an empty one the fixed part. @@ -276,6 +469,8 @@ mod image_tests { const PAGE: u64 = 17_000_000; const _: () = assert!(IMAGE_WORK >= 1_000 * PAGE); const _: () = assert!(IMAGE_WORK_PER_BYTE * 2_100 >= PAGE); + // And that a stream can be decoded about 8 times at Flate's best, 1,032 to one, and the byte in. + const _: () = assert!(IMAGE_WORK_PER_BYTE >= 7 * 1_033); } #[cfg(test)] diff --git a/crates/cos/src/document/objects.rs b/crates/cos/src/document/objects.rs index d9f8026a61..9d47fd9bd3 100644 --- a/crates/cos/src/document/objects.rs +++ b/crates/cos/src/document/objects.rs @@ -298,18 +298,19 @@ impl Document { self.decoded_image_spending(s, max, &mut unbounded) } - /// [`Document::decoded_image_within`], a JBIG2 image spending from `work`, what an operation - /// may still do across its images ([`ImageBudget`](crate::ImageBudget)). With nothing left a - /// JBIG2 image is refused before its globals are read. + /// [`Document::decoded_image_within`], spending from `work`, what an operation may still do + /// across its images ([`ImageBudget`](crate::ImageBudget)): each filter as it runs, a JBIG2 + /// image's globals when they have to be read, and the fax or JBIG2 decoder. With nothing left an + /// image is refused before anything is read, its globals included. pub(crate) fn decoded_image_spending(&self, s: &Stream, max: usize, work: &mut u64) -> Result, CosError> { let max = max.min(MAX_DECODED); let s = &*self.with_direct_parms(s); let filter = |e: pdfcraft_filters::FilterError| CosError::Filter(e.to_string()); let chain = s.readable_filters().map_err(filter)?; // The filters before the image codec, such as ASCII85Decode, are undone first. - let undo = |before: &[(pdfcraft_filters::Filter, pdfcraft_filters::Params)]| match before { + let undo = |before: &[(pdfcraft_filters::Filter, pdfcraft_filters::Params)], work: &mut u64| match before { [] => Ok(Cow::Borrowed(&s.raw[..])), - _ => pdfcraft_filters::decode_tolerant(before, &s.raw, max).map(|(data, _)| Cow::Owned(data)).map_err(filter), + _ => pdfcraft_filters::decode_tolerant_within(before, &s.raw, max, work).map(Cow::Owned).map_err(filter), }; // `/Width` and `/Height` are read as given directly, as every caller reads them first: one // given by reference has the image refused there, before it gets here. @@ -319,11 +320,11 @@ impl Document { return Err(CosError::Filter("a JBIG2 image's /Width or /Height isn't a size".into())); }; // Nothing to read for what can't be decoded: the images of a file share its globals. - let globals = if *work == 0 { None } else { self.jbig2_globals(s)? }; - let data = if *work == 0 { Cow::Borrowed(&[][..]) } else { undo(before)? }; + let globals = if *work == 0 { None } else { self.jbig2_globals(s, work)? }; + let data = if *work == 0 { Cow::Borrowed(&[][..]) } else { undo(before, work)? }; return pdfcraft_filters::decode_jbig2_within(&data, globals.as_deref().map(Vec::as_slice), columns, height, max, work).map_err(filter); } - let Some(p) = self.fax_params(s) else { return s.decoded_within(max) }; + let Some(p) = self.fax_params(s) else { return s.decoded_spending(max, work) }; // As many rows as the image has: a /Rows below /Height would cut the picture short, and // rows past /Height aren't drawn. let p = pdfcraft_filters::CcittParams { rows: 0, ..p }; @@ -332,8 +333,8 @@ impl Document { } let stride = usize::try_from(p.columns).unwrap_or(usize::MAX).div_ceil(8).max(1); let rows = u32::try_from(max / stride).unwrap_or(u32::MAX).min(height.unwrap_or(0)); - let data = undo(chain.split_last().map_or(&[][..], |(_, before)| before))?; - pdfcraft_filters::decode_ccitt(&data, &p, rows).map_err(filter) + let data = undo(chain.split_last().map_or(&[][..], |(_, before)| before), work)?; + pdfcraft_filters::decode_ccitt_within(&data, &p, rows, work).map_err(filter) } /// What the `/JBIG2Globals` stream of JBIG2 image `s` (in its last filter's `/DecodeParms`) @@ -347,8 +348,9 @@ impl Document { /// it was read (the same in every clone) and with its `/DecodeParms` direct (by reference they /// are another object, which can change while this one doesn't), has its filters undone once a /// document and is kept, up to [`JBIG2_GLOBALS_HELD`] in all; others are read each time. That - /// saves the filters only: the segments it holds are decoded again for every image. - fn jbig2_globals(&self, s: &Stream) -> Result>>, CosError> { + /// saves the filters only: the segments it holds are decoded again for every image. Undoing + /// the filters spends from `work`, so globals read for every image pay every time. + fn jbig2_globals(&self, s: &Stream, work: &mut u64) -> Result>>, CosError> { let Some(entry) = self.last_filter_parms(s).get(b"JBIG2Globals").cloned() else { return Ok(None) }; let held = match entry { Object::Ref(r) if !self.overlay.contains_key(&r.num) => Some(r.num), @@ -360,7 +362,7 @@ impl Document { let Object::Stream(g) = &*self.resolve(&entry) else { return Ok(None) }; let readable = self.with_direct_parms(g); let held = held.filter(|_| matches!(readable, Cow::Borrowed(_))); - let mut data = readable.decoded_within(MAX_JBIG2_GLOBALS).map_err(|e| { + let mut data = readable.decoded_spending(MAX_JBIG2_GLOBALS, work).map_err(|e| { let detail = match e { CosError::Filter(detail) => detail, e => e.to_string(), diff --git a/crates/cos/src/object.rs b/crates/cos/src/object.rs index 400fc29962..5d437faa82 100644 --- a/crates/cos/src/object.rs +++ b/crates/cos/src/object.rs @@ -306,6 +306,14 @@ impl Stream { self.decode_chain(max, decode) } + /// [`Stream::decoded_within`] spending from `work`, what an operation may still do across a + /// document's images ([`pdfcraft_filters::decode_tolerant_within`]). + pub(crate) fn decoded_spending(&self, max: usize, work: &mut u64) -> Result, CosError> { + let filter = |e: pdfcraft_filters::FilterError| CosError::Filter(e.to_string()); + let chain = self.readable_filters().map_err(filter)?; + pdfcraft_filters::decode_tolerant_within(&chain, &self.raw, max.min(MAX_DECODED), work).map_err(filter) + } + /// The stream decoded through its filters by `decode`, or copied when it has none, within /// `max` and [`MAX_DECODED`]. fn decode_chain( diff --git a/crates/create/README.md b/crates/create/README.md index 7b629a56e4..46281772e4 100644 --- a/crates/create/README.md +++ b/crates/create/README.md @@ -32,8 +32,8 @@ other filters too, with the `/JBIG2Globals` of their own `/DecodeParms` entry, a whose JBIG2 page isn't its size (PdfCraft's renderer draws the whole page into the image; the page is refused before it is decoded), or whose globals can't be read or decode to more than 32 MiB (they are decoded before the page's size is known, so they have a limit of their own), is skipped, and such a mask is left out and the image exported opaque, as JPX and CCITT masks are. -Past what one export may decode of JBIG2, which its images share, an image is skipped, and so is one whose mask is past it, which -would otherwise come out showing what the mask hides. An export also stops at 2 GiB of files held, or twice the document if that +Past what one export may decode, which its images share, filters and fax and JBIG2 decoders alike (`ImageBudget`; JPEG data isn't +counted), an image is skipped, and so is one whose mask is past it, which would otherwise come out showing what the mask hides. An export also stops at 2 GiB of files held, or twice the document if that is more, since they stay in memory until it ends. Indexed Decode values select palette entries after rounding and clipping. Color-key masks compare the original samples. Missing, null or malformed Decode arrays keep the default mapping. diff --git a/crates/create/src/extract.rs b/crates/create/src/extract.rs index 9631aa0c91..6019a0d093 100644 --- a/crates/create/src/extract.rs +++ b/crates/create/src/extract.rs @@ -165,7 +165,7 @@ fn image(doc: &Document, s: &Stream, budget: &mut ImageBudget) -> Result<(&'stat let mut alpha = if mask { None } else { soft_mask(doc, s, w, h, budget).or_else(|| stencil_mask(doc, s, w, h, budget)) }; // Without its mask the image would come out opaque, showing what the mask hides. if budget.refused() > refused { - return Err("more JBIG2 decoding than one operation may do".into()); + return Err("more image decoding than one operation may do".into()); } let key = if mask { None } else { colour_key(doc, s) }; // Keep the existing JPEG and stencil-mask Decode handling separate from lossless colour images. diff --git a/crates/create/src/tests.rs b/crates/create/src/tests.rs index 55d0612637..17197b91e9 100644 --- a/crates/create/src/tests.rs +++ b/crates/create/src/tests.rs @@ -2264,7 +2264,7 @@ fn jbig2_images_past_the_exports_allowance_are_left_out_with_the_reason() { let export = crate::extract::extract_images_within(&doc, &[0], 1, &mut ImageBudget::with_work(cost), usize::MAX); assert_eq!(export.images.len(), 1); assert_eq!(export.skipped.len(), 1); - assert!(export.skipped[0].2.contains("more JBIG2 decoding than"), "{:?}", export.skipped); + assert!(export.skipped[0].2.contains("more image decoding than"), "{:?}", export.skipped); // What an export holds is bounded too: the files are all in memory until it ends. let export = crate::extract::extract_images_within(&doc, &[0], 1, &mut ImageBudget::with_work(u64::MAX), 1); assert_eq!(export.images.len(), 1, "the first goes past it"); @@ -2276,6 +2276,42 @@ fn jbig2_images_past_the_exports_allowance_are_left_out_with_the_reason() { assert!(crate::extract::image_file_within(&doc, b, &mut budget).is_err()); } +/// An image that isn't JBIG2 shares the allowance too: a file of filter bombs would hold an export +/// as long as one of JBIG2. What isn't decoded here, a JPEG, is still exported. +#[test] +fn flate_images_past_the_exports_allowance_are_left_out_with_the_reason() { + use pdfcraft_cos::{Dict, Document, ImageBudget, Object, Stream}; + let image = |filter: &str, raw: Vec| { + let mut dict = Dict::new(); + dict.set(b"Subtype".to_vec(), Object::name("Image")); + dict.set(b"Width".to_vec(), Object::Int(2)); + dict.set(b"Height".to_vec(), Object::Int(2)); + dict.set(b"BitsPerComponent".to_vec(), Object::Int(8)); + dict.set(b"ColorSpace".to_vec(), Object::name("DeviceGray")); + dict.set(b"Filter".to_vec(), Object::name(filter)); + Stream::from_raw(dict, raw) + }; + let flate = image("FlateDecode", pdfcraft_filters::encode_flate(&[0x80; 4])); + let mut cost = ImageBudget::with_work(u64::MAX); + cost.decode(&Document::new_empty(), &flate, 1 << 20).unwrap(); + let cost = u64::MAX - cost.work_left(); + + let mut doc = Document::new_empty(); + let (a, b) = (doc.add(Object::Stream(flate.clone())), doc.add(Object::Stream(flate))); + let mut budget = ImageBudget::with_work(cost); + assert!(crate::extract::image_file_within(&doc, a, &mut budget).is_ok()); + let refused = crate::extract::image_file_within(&doc, b, &mut budget).unwrap_err(); + assert!(refused.contains("more image decoding than"), "{refused}"); + assert_eq!(budget.refused(), 1); + + let jpeg = image::RgbImage::from_pixel(2, 2, image::Rgb([9, 9, 9])); + let mut bytes = Vec::new(); + jpeg.write_to(&mut std::io::Cursor::new(&mut bytes), image::ImageFormat::Jpeg).unwrap(); + let jpeg = doc.add(Object::Stream(image("DCTDecode", bytes))); + assert!(crate::extract::image_file_within(&doc, jpeg, &mut budget).is_ok(), "nothing charged"); + assert_eq!(budget.refused(), 1); +} + /// A mask past the allowance can't be left out as an unreadable one is, or the image comes out /// opaque, showing what the mask hides: the image goes, with the reason. #[test] @@ -2290,17 +2326,21 @@ fn an_image_whose_jbig2_mask_is_past_the_allowance_is_left_out() { mask.set(b"Filter".to_vec(), Object::name("JBIG2Decode")); let mut doc = Document::new_empty(); let mask = doc.add(Object::Stream(Stream::from_raw(mask, pdfcraft_filters::encode_jbig2_generic(&[0xFF; 4], 16, 2).unwrap()))); + // A JPEG, which nothing charges, so only its mask can be what stops it. + let mut jpeg = Vec::new(); + image::RgbImage::from_pixel(1, 1, image::Rgb([9, 9, 9])).write_to(&mut std::io::Cursor::new(&mut jpeg), image::ImageFormat::Jpeg).unwrap(); let mut image = Dict::new(); image.set(b"Subtype".to_vec(), Object::name("Image")); image.set(b"Width".to_vec(), Object::Int(1)); image.set(b"Height".to_vec(), Object::Int(1)); image.set(b"BitsPerComponent".to_vec(), Object::Int(8)); - image.set(b"ColorSpace".to_vec(), Object::name("DeviceGray")); + image.set(b"ColorSpace".to_vec(), Object::name("DeviceRGB")); + image.set(b"Filter".to_vec(), Object::name("DCTDecode")); image.set(b"SMask".to_vec(), Object::Ref(mask)); - let image = doc.add(Object::Stream(Stream::from_raw(image, vec![0x80]))); + let image = doc.add(Object::Stream(Stream::from_raw(image, jpeg))); assert!(crate::extract::image_file_within(&doc, image, &mut ImageBudget::with_work(u64::MAX)).is_ok()); let refused = crate::extract::image_file_within(&doc, image, &mut ImageBudget::with_work(0)).unwrap_err(); - assert!(refused.contains("more JBIG2 decoding than"), "{refused}"); + assert!(refused.contains("more image decoding than"), "{refused}"); } /// A large honest file keeps its images: what an export may hold grows with the file past the fixed diff --git a/crates/filters/README.md b/crates/filters/README.md index 59403e4a2b..dcf070c4c1 100644 --- a/crates/filters/README.md +++ b/crates/filters/README.md @@ -21,6 +21,7 @@ decode(chain, data, max_output) -> Result, FilterError> decode_counted(chain, data, max_output) -> Result<(Vec, usize /*decoded*/), (FilterError, usize /*decoded*/)> decode_tolerant(chain, data, max_output) -> Result<(Vec, bool /*partial*/), FilterError> decode_tolerant_counted(chain, data, max_output) -> Result<(Vec, bool /*partial*/, usize /*decoded*/), (FilterError, usize /*decoded*/)> +decode_tolerant_within(chain, data, max_output, work: &mut u64) -> Result, FilterError> encode(filter, params, data) -> Result, FilterError> encode_flate(data) -> Vec // zlib, level 6 encode_g4(data, columns, rows) -> Result, FilterError> // 1-bit rows as CCITT Group 4; columns <= MAX_G4_COLUMNS @@ -35,6 +36,7 @@ encode_jbig2(data, columns, rows) -> Result, FilterError> // 1-bit row - Image codecs (DCT, JPX, JBIG2, CCITT), `Crypt` and unknown filters anywhere in a chain give `Unsupported`: callers keep such data encoded. So does a chain longer than `MAX_CHAIN` (8) filters, before any stage runs. Decryption belongs to `crypt`. - `max_output` bounds every intermediate buffer (including the pre-predictor buffer) and gives `LimitExceeded(max_output)`. This also holds when Flate retries raw deflate after a misleading zlib header, in both strict and tolerant decoding. - `decode_counted` and `decode_tolerant_counted` are for a caller charging a budget (`pdfcraft_cos::DecodeBudget`). `max_output` bounds all stages together (each stage gets what the earlier ones left), and they give the bytes every stage that ran decoded, on success and on failure. A stage's output counts as its decoder produced it, before a predictor or a later stage shrinks it, and a failed decode pays for what it decoded first. +- `decode_tolerant_within` is for a run of images (`pdfcraft_cos::ImageBudget`), as `decode_jbig2_within` and `decode_ccitt_within` are, all three spending from one `work`, in the JBIG2 decoder's steps. `max_output` bounds each stage, as in `decode_tolerant`, and no stage decodes past what `work` has left. It pays a step a byte of `data` first and a step a byte each stage decodes, so a chain that fails has paid for what it did: 0.2 to 3 ns a byte in or out, up to 19 ns through a predictor on samples of a bit, against the 6 to 16 ns of a JBIG2 step. What `work` can't pay for spends all of it, and gives `Unsupported`; with nothing left nothing is read. A chain that can't be decoded at all costs nothing and gives its usual error. - `decode_tolerant` turns `Corrupt` into "what was decoded so far" plus `partial = true`, then runs the rest of the chain on it (predictors are applied to partial output too). `LimitExceeded` and `Unsupported` are still errors. ## Behaviour diff --git a/crates/filters/src/ccitt.rs b/crates/filters/src/ccitt.rs index 7c188d701c..a839c24c02 100644 --- a/crates/filters/src/ccitt.rs +++ b/crates/filters/src/ccitt.rs @@ -123,6 +123,22 @@ pub fn decode_ccitt(data: &[u8], p: &CcittParams, max_rows: u32) -> Result Result, FilterError> { + let rows = u32::try_from(p.rows).ok().filter(|r| *r > 0).map_or(max_rows, |r| r.min(max_rows)); + let stride = u64::try_from(p.columns).unwrap_or(0).div_ceil(8); + crate::pay(work, stride.saturating_mul(u64::from(rows)).saturating_add((data.len() as u64).saturating_mul(CODE_WORK)))?; + decode_ccitt(data, p, max_rows) +} + /// Whether CCITT fax `data` starts with an end-of-line code: a 1 after 11 zeros or more, fill /// included. T.4 sends one before the first row when it sends them at all, and no other code starts /// with more than eight zeros. @@ -189,6 +205,42 @@ impl hayro_ccitt::Decoder for Rows { mod tests { use super::*; + /// A run pays for the rows an image asks for and the code it reads, before any of it is + /// decoded: rows the data doesn't hold are paid for too, as a few bytes can ask for any number. + #[test] + fn a_run_of_images_pays_for_the_rows_it_asks_for() { + let (w, h) = (4096u32, 64u32); + let stride = (w as usize).div_ceil(8); + let data = encode_g4(&vec![0xFF; stride * h as usize], w, h).unwrap(); + let p = CcittParams { k: -1, columns: i64::from(w), ..CcittParams::default() }; + let whole = decode_ccitt(&data, &p, h).unwrap(); + let cost = (whole.len() + 4 * data.len()) as u64; + let mut work = cost; + assert_eq!(decode_ccitt_within(&data, &p, h, &mut work).unwrap(), whole, "exactly enough"); + assert_eq!(work, 0); + let mut work = cost + 7; + decode_ccitt_within(&data, &p, h, &mut work).unwrap(); + assert_eq!(work, 7, "only what it took"); + let mut work = cost - 1; + let refused = decode_ccitt_within(&data, &p, h, &mut work).unwrap_err(); + assert_eq!(refused.to_string(), crate::work_spent().to_string(), "a step short"); + assert_eq!(work, 0, "spends all that was left"); + // Nothing left: not read at all, so bad columns aren't reported as bad columns. + let bad = CcittParams { columns: 0, ..p }; + assert!(decode_ccitt_within(&data, &bad, h, &mut 0).is_err_and(|e| e.to_string() == crate::work_spent().to_string())); + // No code at all, for the same rows. + let mut work = u64::MAX; + decode_ccitt_within(&[], &p, h, &mut work).unwrap(); + assert_eq!(u64::MAX - work, whole.len() as u64); + // `/Rows` below what is asked for, and no rows. + let mut work = u64::MAX; + decode_ccitt_within(&data, &CcittParams { rows: 3, ..p }, h, &mut work).unwrap(); + assert_eq!(u64::MAX - work, (3 * stride + 4 * data.len()) as u64); + let mut work = u64::MAX; + decode_ccitt_within(&data, &p, 0, &mut work).unwrap(); + assert_eq!(u64::MAX - work, 4 * data.len() as u64); + } + /// The rows decoded by an independent decoder (`hayro-ccitt`), as 1 for white and 0 for black. pub(crate) fn decode(data: &[u8], columns: u32, rows: u32) -> Vec> { struct Rows(Vec>, Vec, usize); diff --git a/crates/filters/src/jbig2/decode.rs b/crates/filters/src/jbig2/decode.rs index fe3148ff10..9a3ed9fbde 100644 --- a/crates/filters/src/jbig2/decode.rs +++ b/crates/filters/src/jbig2/decode.rs @@ -38,11 +38,10 @@ pub fn decode_jbig2_within(data: &[u8], globals: Option<&[u8]>, columns: u32, ro if rows == 0 { return Ok(Vec::new()); } - let spent = || FilterError::Unsupported("more JBIG2 decoding than one operation may do".into()); if *work == 0 { - return Err(spent()); + return Err(crate::work_spent()); } - hayro_jbig2::charge_reading(work, data.len().saturating_add(globals.map_or(0, <[u8]>::len))).map_err(|_| spent())?; + hayro_jbig2::charge_reading(work, data.len().saturating_add(globals.map_or(0, <[u8]>::len))).map_err(|_| crate::work_spent())?; let corrupt = |detail: String| FilterError::Corrupt { filter: "JBIG2Decode", detail }; let image = hayro_jbig2::Image::new_embedded(data, globals).map_err(|e| corrupt(e.to_string()))?; let (width, height) = (image.width(), image.height()); @@ -59,7 +58,7 @@ pub fn decode_jbig2_within(data: &[u8], globals: Option<&[u8]>, columns: u32, ro } let mut out = Rows::new(width, height, false); // Stopped by what was left, it isn't a damaged image, whatever the decoder calls it. - image.decode_within(&mut out, work).map_err(|e| if *work == 0 { spent() } else { corrupt(e.to_string()) })?; + image.decode_within(&mut out, work).map_err(|e| if *work == 0 { crate::work_spent() } else { corrupt(e.to_string()) })?; Ok(out.out) } diff --git a/crates/filters/src/lib.rs b/crates/filters/src/lib.rs index 6dc4813f78..bcafc132bd 100644 --- a/crates/filters/src/lib.rs +++ b/crates/filters/src/lib.rs @@ -28,7 +28,7 @@ mod runlength; use std::borrow::Cow; -pub use ccitt::{CcittParams, MAX_CCITT_COLUMNS, MAX_G4_COLUMNS, decode_ccitt, encode_g4}; +pub use ccitt::{CcittParams, MAX_CCITT_COLUMNS, MAX_G4_COLUMNS, decode_ccitt, decode_ccitt_within, encode_g4}; pub use jbig2::{MAX_JBIG2_COLUMNS, MAX_JBIG2_PIXELS, decode_jbig2, decode_jbig2_within, encode_jbig2, encode_jbig2_generic}; /// A PDF stream filter, identified by its `/Filter` name. @@ -91,7 +91,8 @@ impl Filter { matches!(self, Filter::Dct | Filter::Jpx | Filter::Jbig2 | Filter::CcittFax) } - fn is_decodable(&self) -> bool { + /// True for the general-purpose filters, which a chain can hold ([`decode`]). + pub fn is_decodable(&self) -> bool { matches!(self, Filter::Flate | Filter::Lzw | Filter::AsciiHex | Filter::Ascii85 | Filter::RunLength) } } @@ -143,13 +144,33 @@ impl Failure { pub(crate) type Step = Result, Failure>; +/// What decoding says when a run of images has nothing left to spend on it. +pub(crate) fn work_spent() -> FilterError { + FilterError::Unsupported("more image decoding than one operation may do".into()) +} + +/// Takes `steps` from `work`, what a run of images may still cost. What it can't pay for spends +/// all of it. +pub(crate) fn pay(work: &mut u64, steps: u64) -> Result<(), FilterError> { + match work.checked_sub(steps) { + Some(left) => { + *work = left; + Ok(()) + } + None => { + *work = 0; + Err(work_spent()) + } + } +} + /// Decodes `data` through `chain` (first element applied first). /// /// Image codecs, `Crypt` and unknown filters anywhere in the chain give /// [`FilterError::Unsupported`] (callers keep such data encoded), as does a chain longer than /// [`MAX_CHAIN`]. `max_output` bounds every intermediate buffer. pub fn decode(chain: &[(Filter, Params)], data: &[u8], max_output: usize) -> Result, FilterError> { - run(chain, data, max_output, true, false).map(|(v, _, _)| v).map_err(|(e, _)| e) + run(chain, data, max_output, None, true).map(|(v, _, _)| v).map_err(|(e, _)| e) } /// Like [`decode`] for a caller charging an allowance: `max_output` bounds the bytes all stages @@ -157,42 +178,83 @@ pub fn decode(chain: &[(Filter, Params)], data: &[u8], max_output: usize) -> Res /// bytes, on success and on failure. A stage's output counts even when a later stage or its /// predictor shrinks it. pub fn decode_counted(chain: &[(Filter, Params)], data: &[u8], max_output: usize) -> Result<(Vec, usize), (FilterError, usize)> { - run(chain, data, max_output, true, true).map(|(v, _, spent)| (v, spent)) + run(chain, data, max_output, Some(max_output), true).map(|(v, _, spent)| (v, spent)) } /// Like [`decode`], but on corrupt data returns what was decoded before the error and /// reports `partial = true` (the behaviour viewers expect). The remaining filters of the /// chain still run on the partial data. Limit and unsupported errors are still errors. pub fn decode_tolerant(chain: &[(Filter, Params)], data: &[u8], max_output: usize) -> Result<(Vec, bool), FilterError> { - run(chain, data, max_output, false, false).map(|(v, partial, _)| (v, partial)).map_err(|(e, _)| e) + run(chain, data, max_output, None, false).map(|(v, partial, _)| (v, partial)).map_err(|(e, _)| e) } /// [`decode_tolerant`] bounded and counted as [`decode_counted`] is. pub fn decode_tolerant_counted(chain: &[(Filter, Params)], data: &[u8], max_output: usize) -> Result<(Vec, bool, usize), (FilterError, usize)> { - run(chain, data, max_output, false, true) + run(chain, data, max_output, Some(max_output), false) +} + +/// [`decode_tolerant`], spending from `work`, what decoding may still cost across a run of images: +/// a step for each byte of `data`, paid before anything is decoded, and one for each byte a stage +/// decodes, so a chain that fails, or a stage that goes past `max_output`, has paid for what it did. +/// No stage decodes past what is left: a chain that `work` can't pay for spends all of it and stops +/// there, so with nothing left nothing is read. A chain that can't be decoded at all costs nothing, +/// whatever is left, so what an image's filters are is known from the error. +/// +/// Unlike [`decode_tolerant`], no filters at all gives `data` itself, within `max_output`. +pub fn decode_tolerant_within(chain: &[(Filter, Params)], data: &[u8], max_output: usize, work: &mut u64) -> Result, FilterError> { + decodable(chain)?; + pay(work, data.len() as u64)?; + if chain.is_empty() { + return if data.len() > max_output { Err(FilterError::LimitExceeded(max_output)) } else { Ok(data.to_vec()) }; + } + let left = usize::try_from(*work).unwrap_or(usize::MAX); + match run(chain, data, max_output, Some(left), false) { + Ok((out, _, decoded)) => pay(work, decoded as u64).map(|()| out), + // The stage stopped at the lower of the two, which took as much to find out. + Err((FilterError::LimitExceeded(_), decoded)) if left.saturating_sub(decoded) > max_output => { + pay(work, (decoded as u64).saturating_add(max_output as u64))?; + Err(FilterError::LimitExceeded(max_output)) + } + Err((FilterError::LimitExceeded(_), _)) => { + *work = 0; + Err(work_spent()) + } + Err((e, decoded)) => pay(work, decoded as u64).and(Err(e)), + } } /// Longest filter chain decoded. Real streams use one to three filters; every stage may produce /// up to the output limit, so the chain length multiplies the work. pub const MAX_CHAIN: usize = 8; -/// The decoded data, whether it is partial and the bytes all stages decoded, or the error and -/// the bytes decoded before it. `max` bounds each stage, or with `cumulative` all of them -/// together. -fn run(chain: &[(Filter, Params)], data: &[u8], max: usize, strict: bool, cumulative: bool) -> Result<(Vec, bool, usize), (FilterError, usize)> { +/// Whether a chain is one [`run`] takes, and if not, the error. +fn decodable(chain: &[(Filter, Params)]) -> Result<(), FilterError> { if chain.len() > MAX_CHAIN { - return Err((FilterError::Unsupported(format!("a chain of {} filters", chain.len())), 0)); + return Err(FilterError::Unsupported(format!("a chain of {} filters", chain.len()))); } - if let Some((f, _)) = chain.iter().find(|(f, _)| !f.is_decodable()) { - return Err((FilterError::Unsupported(f.name().to_owned()), 0)); + match chain.iter().find(|(f, _)| !f.is_decodable()) { + Some((f, _)) => Err(FilterError::Unsupported(f.name().to_owned())), + None => Ok(()), } +} + +/// The decoded data, whether it is partial and the bytes all stages decoded, or the error and +/// the bytes decoded before it. `max` bounds each stage, and `total` all of them together. +fn run( + chain: &[(Filter, Params)], + data: &[u8], + max: usize, + total: Option, + strict: bool, +) -> Result<(Vec, bool, usize), (FilterError, usize)> { + decodable(chain).map_err(|e| (e, 0))?; let mut cur: Cow<[u8]> = Cow::Borrowed(data); let mut partial = false; let mut spent = 0usize; for (filter, params) in chain { #[cfg(test)] STAGE_BYTES.with(|c| c.set(c.get() + cur.len())); - let limit = if cumulative { max.saturating_sub(spent) } else { max }; + let limit = total.map_or(max, |total| max.min(total.saturating_sub(spent))); let (step, work) = decode_one(filter, params, &cur, limit); spent = spent.saturating_add(work); cur = Cow::Owned(match step { @@ -428,6 +490,9 @@ mod tests { assert_eq!(decode(&chain, &data, max).map(|d| d.len()).ok(), Some(1 << 19)); assert!(matches!(decode_counted(&chain, &data, max), Err((FilterError::LimitExceeded(m), _)) if m == max)); assert_eq!(decode_counted(&chain, &data, 2 << 20).map(|(_, spent)| spent).ok(), Some((1 << 20) + (1 << 19))); + let tolerant = decode_tolerant_counted(&chain, &data, 2 << 20).map(|(out, partial, spent)| (out.len(), partial, spent)); + assert_eq!(tolerant.ok(), Some((1 << 19, false, (1 << 20) + (1 << 19)))); + assert!(matches!(decode_tolerant_counted(&chain, &data, max), Err((FilterError::LimitExceeded(m), _)) if m == max)); } /// A PNG predictor drops a tag byte per row, but Flate produced it: the count includes it, @@ -446,4 +511,123 @@ mod tests { assert!(matches!(e, FilterError::Corrupt { .. }), "{e}"); assert_eq!(spent, bad.len()); } + + fn refused(e: &FilterError) -> bool { + e.to_string() == work_spent().to_string() + } + + /// A chain pays for what it reads and what each stage decodes, down to the byte: Flate inflates + /// hex that ASCIIHex halves, so a stage counts when a later one shrinks it. + #[test] + fn a_chain_spends_down_to_the_byte() { + let hex = b"ab".repeat(1 << 10); + let data = encode_flate(&hex); + let chain = [(Filter::Flate, Params::default()), (Filter::AsciiHex, Params::default())]; + let whole = decode(&chain, &data, 1 << 20).unwrap(); + let cost = (data.len() + hex.len() + whole.len()) as u64; + let mut work = u64::MAX; + assert_eq!(decode_tolerant_within(&chain, &data, 1 << 20, &mut work).unwrap(), whole); + assert_eq!(u64::MAX - work, cost); + let mut work = cost; + assert_eq!(decode_tolerant_within(&chain, &data, 1 << 20, &mut work).unwrap(), whole, "exactly enough"); + assert_eq!(work, 0); + let mut work = cost + 7; + decode_tolerant_within(&chain, &data, 1 << 20, &mut work).unwrap(); + assert_eq!(work, 7, "only what it took"); + let mut work = cost - 1; + assert!(refused(&decode_tolerant_within(&chain, &data, 1 << 20, &mut work).unwrap_err()), "a step short"); + assert_eq!(work, 0, "spends all that was left"); + // Each stage has `max_output` to itself, as in a plain decode: together they are past it. + let mut work = u64::MAX; + assert_eq!(decode_tolerant_within(&chain, &data, hex.len(), &mut work).unwrap(), whole); + } + + /// No stage runs past what is left: the first stage of this bomb stops there, so the second + /// isn't given anything to read, and with too little to read it nothing runs at all. + #[test] + fn a_chain_stops_at_what_is_left() { + let inner = encode_flate(&vec![0; 1 << 20]); + let bomb = encode_flate(&inner); + let chain = [(Filter::Flate, Params::default()), (Filter::Flate, Params::default())]; + let read = |work: u64| { + super::STAGE_BYTES.with(|c| c.set(0)); + let mut work = work; + assert!(refused(&decode_tolerant_within(&chain, &bomb, usize::MAX, &mut work).unwrap_err())); + assert_eq!(work, 0); + super::STAGE_BYTES.with(|c| c.get()) + }; + assert_eq!(read((bomb.len() + inner.len()) as u64 + 10), bomb.len() + inner.len(), "the second stage stops"); + assert_eq!(read(bomb.len() as u64 + 10), bomb.len(), "the first stops, and the second never runs"); + assert_eq!(read(bomb.len() as u64 - 1), 0, "it can't pay to be read"); + } + + /// Past `max_output` the chain has failed, and hasn't run out of anything: it pays what it took + /// to find out, and what is left stays. Whichever limit a stage stops at is the one reported. + #[test] + fn a_stage_past_its_limit_pays_for_finding_out() { + let data = encode_flate(&vec![0; 1 << 16]); + let chain = [(Filter::Flate, Params::default())]; + let mut work = u64::MAX; + assert!(matches!(decode_tolerant_within(&chain, &data, 1000, &mut work), Err(FilterError::LimitExceeded(1000)))); + assert_eq!(u64::MAX - work, data.len() as u64 + 1000); + // The later stage of two, which pays for the first as well. + let bomb = encode_flate(&data); + let two = [(Filter::Flate, Params::default()), (Filter::Flate, Params::default())]; + let mut work = u64::MAX; + assert!(matches!(decode_tolerant_within(&two, &bomb, data.len(), &mut work), Err(FilterError::LimitExceeded(_)))); + assert_eq!(u64::MAX - work, (bomb.len() + 2 * data.len()) as u64); + // Both limits at once: a run that can't go on is what a caller needs to hear. + let mut work = data.len() as u64 + 1000; + assert!(refused(&decode_tolerant_within(&chain, &data, 1000, &mut work).unwrap_err())); + assert_eq!(work, 0); + let mut work = data.len() as u64 + 1001; + assert!(matches!(decode_tolerant_within(&chain, &data, 1000, &mut work), Err(FilterError::LimitExceeded(1000)))); + assert_eq!(work, 1, "the image's limit, with a step to spare"); + } + + /// What a damaged stream decoded is paid for, however the decode ends. + #[test] + fn a_chain_that_fails_has_paid_for_what_it_did() { + let data: Vec = (0..20_000u32).map(|i| (i % 97) as u8 ^ (i / 300) as u8).collect(); + let mut cut = encode_flate(&data); + cut.truncate(cut.len() / 2); + let chain = [(Filter::Flate, Params::default())]; + let (partial, _) = decode_tolerant(&chain, &cut, 1 << 20).unwrap(); + assert!(!partial.is_empty()); + let mut work = u64::MAX; + assert_eq!(decode_tolerant_within(&chain, &cut, 1 << 20, &mut work).unwrap(), partial); + assert_eq!(u64::MAX - work, (cut.len() + partial.len()) as u64); + // A predictor that comes to nothing, after all of this was inflated for it. + let p = Params { predictor: 99, ..Params::default() }; + let data = encode_flate(&vec![0; 1 << 16]); + let mut work = u64::MAX; + assert!(decode_tolerant_within(&[(Filter::Flate, p)], &data, 1 << 20, &mut work).unwrap().is_empty()); + assert_eq!(u64::MAX - work, data.len() as u64 + (1 << 16)); + } + + /// With nothing left nothing is read. What can't be decoded at all never costs anything, and + /// says so whatever is left. Without filters, what is read is the data. + #[test] + fn what_a_chain_is_shows_whatever_is_left() { + let flate = [(Filter::Flate, Params::default())]; + let mut work = 0; + assert!(refused(&decode_tolerant_within(&flate, b"garbage", 1 << 20, &mut work).unwrap_err())); + let long = vec![(Filter::Flate, Params::default()); MAX_CHAIN + 1]; + for chain in [vec![(Filter::Dct, Params::default())], vec![(Filter::Jbig2, Params::default())], long] { + for left in [0, 100] { + let mut work = left; + let e = decode_tolerant_within(&chain, b"data", 1 << 20, &mut work).unwrap_err(); + assert!(matches!(e, FilterError::Unsupported(_)) && !refused(&e), "{e}"); + assert_eq!(work, left); + } + } + let mut work = 100; + assert_eq!(decode_tolerant_within(&[], b"data", 4, &mut work).unwrap(), b"data"); + assert_eq!(work, 96); + assert!(matches!(decode_tolerant_within(&[], b"data", 3, &mut work), Err(FilterError::LimitExceeded(3)))); + assert_eq!(work, 92); + let mut work = 3; + assert!(refused(&decode_tolerant_within(&[], b"data", 4, &mut work).unwrap_err())); + assert_eq!(work, 0); + } } diff --git a/crates/optimize/README.md b/crates/optimize/README.md index d6083733da..a30e3b197b 100644 --- a/crates/optimize/README.md +++ b/crates/optimize/README.md @@ -100,9 +100,11 @@ let report = optimize_with_progress(&mut doc, &settings, &mut |stage| keep_going - Lossless JBIG2 uses bounded generic-region encoding or exact-copy symbol dictionaries when smaller. No lossy symbol substitution is used. Export, redaction and this step read JBIG2 - images, with bounded shared globals. All of a run's JBIG2 images share one allowance of - decoding (`pdfcraft_cos::ImageBudget`): about 1,000 scanned pages' worth, plus more for each - byte of the file. Images past it stay as they are, and `Report::images_refused` counts them. + images, with bounded shared globals. All of a run's images share one allowance of decoding + (`pdfcraft_cos::ImageBudget`), filters and fax and JBIG2 decoders alike, JPEG data aside: about + 1,000 scanned pages' worth, plus more for each byte of the file, enough to decode a stream + about 8 times at Flate's best. Images past it stay as they are, and `Report::images_refused` + counts them. - **Discard objects:** thumbnails, alternate images, document tags, print settings. - **Clean up:** Flate for streams with no filter; for streams with an outer ASCII85 or ASCIIHex layer, that layer is removed when the result is smaller (the inner filters keep their encoded diff --git a/crates/optimize/src/fragments.rs b/crates/optimize/src/fragments.rs index c069de2f95..55aca519f0 100644 --- a/crates/optimize/src/fragments.rs +++ b/crates/optimize/src/fragments.rs @@ -18,7 +18,7 @@ //! drops them. use pdfcraft_content::{Matrix, Op, parse, serialize_ops}; -use pdfcraft_cos::{DecodeBudget, Dict, Document, ObjRef, Object, Stream}; +use pdfcraft_cos::{DecodeBudget, Dict, Document, ImageBudget, ObjRef, Object, Stream}; use crate::images::{components, content_bytes, decodable, page_resources, pixels, resample}; use crate::{OptimizeError, Report, Stage}; @@ -152,7 +152,7 @@ fn run_end(ops: &[Op], i: usize, count: usize) -> usize { } /// The run's pixels as one image, top row first: `(pixels, width, height)`. -fn merge(doc: &Document, run: &[Fragment], n: usize) -> Option<(Vec, u32, u32)> { +fn merge(doc: &Document, run: &[Fragment], n: usize, budget: &mut ImageBudget) -> Option<(Vec, u32, u32)> { let (top, bottom) = (run.first()?.rect[3], run.last()?.rect[1]); let width = run.iter().map(|f| f.width).max()?; // Pixel rows per unit of height: the finest fragment's. @@ -176,7 +176,7 @@ fn merge(doc: &Document, run: &[Fragment], n: usize) -> Option<(Vec, u32, u3 } let obj = doc.get(f.image); let Object::Stream(s) = &*obj else { return None }; - let px = pixels(doc, s, n)?; + let px = pixels(doc, s, n, budget)?; let px = if (f.width, f.height) == (width, r1 - r0) { px } else { resample(&px, n, f.width, f.height, width, r1 - r0)? }; out.get_mut(r0 as usize * row..r1 as usize * row)?.copy_from_slice(px.get(..(r1 - r0) as usize * row)?); } @@ -230,7 +230,13 @@ fn add_xobject(doc: &mut Document, page: ObjRef, name: &[u8], image: ObjRef) -> } /// Merge the fragment runs of one page; returns how many fragments were merged. -fn merge_page(doc: &mut Document, page: ObjRef, report: &mut Report, progress: &mut dyn FnMut(Stage) -> bool) -> Result { +fn merge_page( + doc: &mut Document, + page: ObjRef, + report: &mut Report, + progress: &mut dyn FnMut(Stage) -> bool, + images: &mut ImageBudget, +) -> Result { let Some(contents) = doc.get(page).as_dict().and_then(|d| d.get(b"Contents").cloned()) else { return Ok(0) }; // A page whose content isn't all known is left alone: rewriting it would drop the rest. let Some(data) = content_bytes(doc, &contents, &mut DecodeBudget::page()) else { return Ok(0) }; @@ -286,7 +292,7 @@ fn merge_page(doc: &mut Document, page: ObjRef, report: &mut Report, progress: & if !progress(Stage::Fragments) { return Err(OptimizeError::Cancelled); } - let Some((px, w, h)) = merge(doc, &run, n) else { + let Some((px, w, h)) = merge(doc, &run, n, images) else { out.extend(ops.get(i..j).unwrap_or_default().iter().cloned()); i = j; continue; @@ -320,9 +326,15 @@ fn merge_page(doc: &mut Document, page: ObjRef, report: &mut Report, progress: & Ok(merged) } -pub(crate) fn run(doc: &mut Document, pages: &[ObjRef], report: &mut Report, progress: &mut dyn FnMut(Stage) -> bool) -> Result<(), OptimizeError> { +pub(crate) fn run( + doc: &mut Document, + pages: &[ObjRef], + report: &mut Report, + progress: &mut dyn FnMut(Stage) -> bool, + budget: &mut ImageBudget, +) -> Result<(), OptimizeError> { for p in pages { - report.fragments_merged += merge_page(doc, *p, report, progress)?; + report.fragments_merged += merge_page(doc, *p, report, progress, budget)?; } Ok(()) } @@ -343,7 +355,42 @@ mod tests { Fragment { image, rect: [0.0, edge, 1.0, 1.0], width: 1 << 16, height: 1 }, Fragment { image, rect: [0.0, 0.0, 1.0, edge], width: 1 << 16, height: 1 }, ]; - assert!(merge(&Document::new_empty(), &run, 3).is_none()); + assert!(merge(&Document::new_empty(), &run, 3, &mut ImageBudget::with_work(u64::MAX)).is_none()); + } + + /// Merging reads the images the pass after it reads, from the same allowance: with nothing + /// left the strips stay as they are, and the image that couldn't be read is counted. + #[test] + fn merging_spends_the_runs_allowance() { + let mut doc = Document::new_empty(); + let mut image = Dict::new(); + for (k, v) in + [("Subtype", Object::name("Image")), ("Width", Object::Int(8)), ("Height", Object::Int(1)), ("BitsPerComponent", Object::Int(8))] + { + image.set(k.as_bytes().to_vec(), v); + } + image.set(b"ColorSpace".to_vec(), Object::name("DeviceGray")); + let mut xobjects = Dict::new(); + let mut drawn = String::new(); + for row in 0..8 { + let r = doc.add(Object::Stream(Stream::flate(image.clone(), &[row * 30; 8]))); + xobjects.set(format!("I{row}").into_bytes(), Object::Ref(r)); + drawn.push_str(&format!("q 80 0 0 10 0 {} cm /I{row} Do Q\n", 70 - 10 * i32::from(row))); + } + let content = doc.add(Object::Stream(Stream::flate(Dict::new(), drawn.as_bytes()))); + let mut resources = Dict::new(); + resources.set(b"XObject".to_vec(), Object::Dict(xobjects)); + let mut page = Dict::new(); + page.set(b"Type".to_vec(), Object::name("Page")); + page.set(b"Resources".to_vec(), Object::Dict(resources)); + page.set(b"Contents".to_vec(), Object::Ref(content)); + let page = doc.add(Object::Dict(page)); + let mut budget = ImageBudget::with_work(0); + assert_eq!(merge_page(&mut doc.clone(), page, &mut Report::default(), &mut |_| true, &mut budget).unwrap(), 0); + assert_eq!(budget.refused(), 1); + let mut budget = ImageBudget::with_work(u64::MAX); + assert_eq!(merge_page(&mut doc, page, &mut Report::default(), &mut |_| true, &mut budget).unwrap(), 8); + assert!(budget.work_left() < u64::MAX); } /// Many images drawn on top of each other are not one picture; looking for runs among them reads @@ -370,7 +417,7 @@ mod tests { page.set(b"Resources".to_vec(), Object::Dict(resources)); let page = doc.add(Object::Dict(page)); GROUP_READS.with(|c| c.set(0)); - let merged = merge_page(&mut doc, page, &mut Report::default(), &mut |_| true); + let merged = merge_page(&mut doc, page, &mut Report::default(), &mut |_| true, &mut ImageBudget::with_work(u64::MAX)); assert_eq!(merged.ok(), Some(0)); let reads = GROUP_READS.with(|c| c.get()); assert!(reads <= 40 * 4 * groups, "{reads} group reads"); diff --git a/crates/optimize/src/images.rs b/crates/optimize/src/images.rs index cd2317590d..5f2aba8e5a 100644 --- a/crates/optimize/src/images.rs +++ b/crates/optimize/src/images.rs @@ -479,7 +479,7 @@ fn count_decoded(_n: u64) { /// every row, and the image would be written back wrong. `None` for a JPEG whose colour transform /// readers may disagree about ([`ycc_encoded`]), and for data that decodes past what the samples /// can need ([`allowance`]). -pub(crate) fn pixels(doc: &Document, s: &Stream, n: usize) -> Option> { +pub(crate) fn pixels(doc: &Document, s: &Stream, n: usize, budget: &mut ImageBudget) -> Option> { let (w, h) = (u32::try_from(s.dict.int(b"Width")?).ok()?, u32::try_from(s.dict.int(b"Height")?).ok()?); if w == 0 || h == 0 || u64::from(w) * u64::from(h) > MAX_PIXELS { return None; @@ -494,7 +494,7 @@ pub(crate) fn pixels(doc: &Document, s: &Stream, n: usize) -> Option> { count_decoded(u64::from(w) * u64::from(h)); (v.len() == bytes).then_some(v) } else { - let v = doc.with_direct_parms(s).decoded_within(allowance(bytes, h)?).ok()?; + let v = budget.decode_filters(doc, s, allowance(bytes, h)?).ok()?; Some(v.get(..bytes)?.to_vec()) } } @@ -700,7 +700,14 @@ fn image_stream(old: &Dict, px: &[u8], n: usize, w: u32, h: u32, compression: Co /// One image: resample and/or recompress per `settings`; `ppi` is its effective resolution. /// Returns the new image and its new soft mask, if the result is smaller. -fn process(doc: &Document, s: &Stream, ppi: f64, settings: &ImageSettings, n: usize) -> Option<(Stream, Option<(ObjRef, Stream)>)> { +fn process( + doc: &Document, + s: &Stream, + ppi: f64, + settings: &ImageSettings, + n: usize, + budget: &mut ImageBudget, +) -> Option<(Stream, Option<(ObjRef, Stream)>)> { let d = &s.dict; if d.get(b"Decode").is_some_and(|a| !is_identity_decode(doc, a, n)) || d.contains(b"Mask") @@ -732,7 +739,7 @@ fn process(doc: &Document, s: &Stream, ppi: f64, settings: &ImageSettings, n: us } else { (w, h) }; - let px = pixels(doc, s, n)?; + let px = pixels(doc, s, n, budget)?; let px = if (nw, nh) != (w, h) { resample(&px, n, w, h, nw, nh)? } else { px }; let new = image_stream(d, &px, n, nw, nh, settings.compression, was_dct)?; // A soft mask must keep the image's dimensions: resample it with the image (lossless). @@ -745,7 +752,7 @@ fn process(doc: &Document, s: &Stream, ppi: f64, settings: &ImageSettings, n: us return None; } let (mw, mh) = (u32::try_from(m.dict.int(b"Width")?).ok()?, u32::try_from(m.dict.int(b"Height")?).ok()?); - let mpx = pixels(doc, m, 1)?; + let mpx = pixels(doc, m, 1, budget)?; let mpx = resample(&mpx, 1, mw, mh, nw, nh)?; smask = Some((mr, image_stream(&m.dict, &mpx, 1, nw, nh, Compression::Flate, false)?)); } @@ -888,7 +895,7 @@ fn one_bit(doc: &Document, s: &Stream, pack: bool, defaults: bool, budget: &mut let row_len = wu.checked_mul(n)?.checked_mul(bpc)?.div_ceil(8); // Through its parameters given by reference too: without a PNG predictor's tag bytes taken // off, rows of 0 and 255 (or palette entries 0 and 1) would still pass below, each shifted. - let samples = doc.with_direct_parms(s).decoded_within(allowance(row_len.checked_mul(hu)?, h)?).ok()?; + let samples = budget.decode_filters(doc, s, allowance(row_len.checked_mul(hu)?, h)?).ok()?; Some((packed_rows(&samples, row_len, wu, hu, (n, bpc, white))?, w, h, Some(packed))) } @@ -1175,24 +1182,14 @@ fn mono(doc: &Document, s: &Stream, ppi: f64, defaults: bool, settings: &MonoSet (best.0 < crate::stored_len(s)?).then_some((best.1, resampled)) } +/// One pass over the document's images, within `budget`. An image past it stays as it is, and is +/// counted in the report. pub(crate) fn run( doc: &mut Document, pages: &[ObjRef], settings: &Settings, report: &mut Report, progress: &mut dyn FnMut(Stage) -> bool, -) -> Result<(), OptimizeError> { - let mut budget = ImageBudget::new(doc); - run_within(doc, pages, settings, report, progress, &mut budget) -} - -/// [`run`] within `budget`. An image past it stays as it is, and is counted in the report. -fn run_within( - doc: &mut Document, - pages: &[ObjRef], - settings: &Settings, - report: &mut Report, - progress: &mut dyn FnMut(Stage) -> bool, budget: &mut ImageBudget, ) -> Result<(), OptimizeError> { let (ppi, mut defaults, read) = walk_pages(doc, pages); @@ -1222,6 +1219,7 @@ fn run_within( let obj = doc.get(r); let Object::Stream(s) = &*obj else { continue }; report.images += 1; + let refused = budget.refused(); // Packed from more bits a sample, it is lossless at one bit, but the grey or colour // settings can make it smaller still (below). let packed_from_more = s.dict.int(b"BitsPerComponent").is_some_and(|b| b > 1); @@ -1242,7 +1240,9 @@ fn run_within( continue; } let n = s.dict.get(b"ColorSpace").and_then(|cs| components(doc, cs)); - let processed = n.and_then(|n| process(doc, s, ppi, if n == 1 { &settings.gray } else { &settings.color }, n)); + let processed = n.and_then(|n| process(doc, s, ppi, if n == 1 { &settings.gray } else { &settings.color }, n, budget)); + // Read to be packed and again to be resampled, it is still one image. + report.images_refused += usize::from(budget.refused() > refused); // A dithered scan, say, downsampled and JPEG-compressed: the smaller wins, counting a soft // mask resampled with it. let smask_len = |smask: Option<&(ObjRef, Stream)>| match smask { @@ -1282,7 +1282,6 @@ fn run_within( doc.set(mr, Object::Stream(m)); } } - report.images_refused = budget.refused(); if !progress(Stage::Images { done: total, total }) { return Err(OptimizeError::Cancelled); } @@ -1317,13 +1316,16 @@ mod dimension_tests { dict.set(b"Width".to_vec(), Object::Int(width)); dict.set(b"Height".to_vec(), Object::Int(height)); let stream = Stream::from_raw(dict, vec![0; 4]); - assert!(pixels(&Document::new_empty(), &stream, channels).is_none(), "{width} x {height} x {channels}"); + assert!( + pixels(&Document::new_empty(), &stream, channels, &mut ImageBudget::with_work(u64::MAX)).is_none(), + "{width} x {height} x {channels}" + ); } let mut dict = Dict::new(); dict.set(b"Width".to_vec(), Object::Int(2)); dict.set(b"Height".to_vec(), Object::Int(2)); let stream = Stream::from_raw(dict, vec![1, 2, 3, 4, 5]); - assert_eq!(pixels(&Document::new_empty(), &stream, 1), Some(vec![1, 2, 3, 4])); + assert_eq!(pixels(&Document::new_empty(), &stream, 1, &mut ImageBudget::with_work(u64::MAX)), Some(vec![1, 2, 3, 4])); } /// JBIG2 alone, with Group 4 asked for, is decoded only to be resampled. A cost guard: dropped, @@ -1393,7 +1395,7 @@ mod dimension_tests { let settings = Settings { mono: MonoSettings { compression: MonoCompression::CcittG4, ..MonoSettings::REDUCE }, ..Settings::default() }; let mut report = Report::default(); let mut budget = ImageBudget::with_work(2 * cost); - run_within(&mut doc, &[page], &settings, &mut report, &mut |_| true, &mut budget).unwrap(); + run(&mut doc, &[page], &settings, &mut report, &mut |_| true, &mut budget).unwrap(); assert_eq!(report.images_refused, 1); assert_eq!(budget.work_left(), 0, "two decoded and the third refused"); assert_eq!(report.images_resampled, 2); @@ -1401,6 +1403,111 @@ mod dimension_tests { assert_eq!(changed, 2, "the third is as it was"); } + /// Images that aren't JBIG2 share the run's allowance too, both on the way to being resampled + /// and on the way to being packed to a bit a sample: a file of filter bombs would hold a run + /// as long as one of JBIG2, and those past it stay as they are. + #[test] + fn flate_images_past_the_runs_allowance_stay_as_they_are() { + // Noise, resampled, or black and white, packed. + let mut seed = 0x2545_F491_4F6C_DD1Du64; + let noise: Vec = (0..256 * 256) + .map(|_| { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + seed as u8 + }) + .collect(); + let black_and_white: Vec = noise.iter().map(|v| if v & 1 == 0 { 0 } else { 255 }).collect(); + for (name, samples, settings) in [ + ("resampled", &noise, Settings::default()), + ( + "packed", + &black_and_white, + Settings { mono: MonoSettings { compression: MonoCompression::CcittG4, ..MonoSettings::REDUCE }, ..Settings::default() }, + ), + ] { + let page_of = |images: usize| { + let mut doc = Document::new_empty(); + let mut dict = Dict::new(); + dict.set(b"Subtype".to_vec(), Object::name("Image")); + dict.set(b"Width".to_vec(), Object::Int(256)); + dict.set(b"Height".to_vec(), Object::Int(256)); + dict.set(b"BitsPerComponent".to_vec(), Object::Int(8)); + dict.set(b"ColorSpace".to_vec(), Object::name("DeviceGray")); + let image = Stream::flate(dict, samples); + let refs: Vec = (0..images).map(|_| doc.add(Object::Stream(image.clone()))).collect(); + let mut xobjects = Dict::new(); + let mut drawn = String::from("q 10 0 0 10 0 0 cm "); + for (i, r) in refs.iter().enumerate() { + xobjects.set(format!("I{i}").into_bytes(), Object::Ref(*r)); + drawn.push_str(&format!("/I{i} Do ")); + } + drawn.push('Q'); + let mut resources = Dict::new(); + resources.set(b"XObject".to_vec(), Object::Dict(xobjects)); + let content = doc.add(Object::Stream(Stream::flate(Dict::new(), drawn.as_bytes()))); + let mut page = Dict::new(); + page.set(b"Type".to_vec(), Object::name("Page")); + page.set(b"Resources".to_vec(), Object::Dict(resources)); + page.set(b"Contents".to_vec(), Object::Ref(content)); + let page = doc.add(Object::Dict(page)); + (doc, page, refs, image) + }; + let (mut doc, page, ..) = page_of(1); + let mut budget = ImageBudget::with_work(u64::MAX); + let mut report = Report::default(); + run(&mut doc, &[page], &settings, &mut report, &mut |_| true, &mut budget).unwrap(); + let cost = u64::MAX - budget.work_left(); + assert!(cost > 0, "{name}"); + assert_eq!(report.images_refused, 0, "{name}"); + + let (mut doc, page, refs, image) = page_of(3); + let mut budget = ImageBudget::with_work(2 * cost); + let mut report = Report::default(); + run(&mut doc, &[page], &settings, &mut report, &mut |_| true, &mut budget).unwrap(); + assert_eq!(report.images_refused, 1, "{name}: {report:?}"); + assert_eq!(budget.work_left(), 0, "{name}"); + let changed = refs.iter().filter(|r| matches!(&*doc.get(**r), Object::Stream(s) if *s != image)).count(); + assert_eq!(changed, 2, "{name}: the third is as it was"); + } + } + + /// A soft mask resampled with its image is read from the same allowance as the image. + #[test] + fn a_soft_mask_resampled_with_its_image_spends_the_allowance() { + let mut seed = 0x2545_F491_4F6C_DD1Du64; + let noise: Vec = (0..256 * 256) + .map(|_| { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + seed as u8 + }) + .collect(); + let image = |space: &str| { + let mut dict = Dict::new(); + dict.set(b"Subtype".to_vec(), Object::name("Image")); + dict.set(b"Width".to_vec(), Object::Int(256)); + dict.set(b"Height".to_vec(), Object::Int(256)); + dict.set(b"BitsPerComponent".to_vec(), Object::Int(8)); + dict.set(b"ColorSpace".to_vec(), Object::name(space)); + Stream::flate(dict, &noise) + }; + let mut doc = Document::new_empty(); + let mask = image("DeviceGray"); + let cost = (mask.raw.len() + noise.len()) as u64; + let masked = doc.add(Object::Stream(mask)); + let mut with_mask = image("DeviceGray"); + with_mask.dict.set(b"SMask".to_vec(), Object::Ref(masked)); + let spent = |s: &Stream| { + let mut budget = ImageBudget::with_work(u64::MAX); + process(&doc, s, 720.0, &Settings::default().gray, 1, &mut budget).expect("resampled, and smaller"); + u64::MAX - budget.work_left() + }; + assert_eq!(spent(&with_mask), spent(&image("DeviceGray")) + cost); + } + /// Decoded only to be resampled, what can't be averaged whatever it holds isn't decoded, so it /// takes nothing from the images that can be, and isn't counted as refused past the allowance. #[test] @@ -1448,9 +1555,9 @@ mod dimension_tests { Stream::from_raw(dict, data.clone()) }; DECODED_PIXELS.with(|c| c.set(0)); - assert!(pixels(&Document::new_empty(), &stream(10, 10), 1).is_none()); + assert!(pixels(&Document::new_empty(), &stream(10, 10), 1, &mut ImageBudget::with_work(u64::MAX)).is_none()); assert_eq!(DECODED_PIXELS.with(|c| c.get()), 0, "pixels decoded"); - assert_eq!(pixels(&Document::new_empty(), &stream(1000, 1000), 1).map(|p| p.len()), Some(1000 * 1000)); + assert_eq!(pixels(&Document::new_empty(), &stream(1000, 1000), 1, &mut ImageBudget::with_work(u64::MAX)).map(|p| p.len()), Some(1000 * 1000)); } } diff --git a/crates/optimize/src/lib.rs b/crates/optimize/src/lib.rs index 92671d2381..90d109a809 100644 --- a/crates/optimize/src/lib.rs +++ b/crates/optimize/src/lib.rs @@ -279,7 +279,7 @@ pub struct Report { pub images: usize, pub images_resampled: usize, pub images_recompressed: usize, - /// JBIG2 images left as they are: decoding them all would take more than one run may spend. + /// Images left as they are: decoding them all would take more than one run may spend. pub images_refused: usize, /// Encoded image bytes before and after (images that changed). pub image_bytes_before: usize, @@ -331,10 +331,12 @@ pub fn optimize_with_progress(doc: &mut Document, settings: &Settings, progress: } let mut report = Report::default(); let pages = pdfcraft_annot::page_refs(doc).map_err(|_| OptimizeError::NoPages)?; + // Merging fragments and the pass after it read the same images, so they share the allowance. + let mut budget = pdfcraft_cos::ImageBudget::new(doc); if settings.merge_fragments { - fragments::run(doc, &pages, &mut report, progress)?; + fragments::run(doc, &pages, &mut report, progress, &mut budget)?; } - images::run(doc, &pages, settings, &mut report, progress)?; + images::run(doc, &pages, settings, &mut report, progress, &mut budget)?; if !progress(Stage::CleanUp) { return Err(OptimizeError::Cancelled); } diff --git a/crates/redact/README.md b/crates/redact/README.md index 726f387376..04fcd788fb 100644 --- a/crates/redact/README.md +++ b/crates/redact/README.md @@ -33,7 +33,8 @@ clear_marks(&mut doc, None)?; // remove marks without app and a byte per row, plus 4 KiB (ASCIIHex text broken into lines under Flate is a little over twice; for a fax image's code, about a byte per pixel), are removed whole; cleared copies drop `/Alternates` so alternate pixels cannot survive; - A run that would decode more JBIG2 than one redaction may (`pdfcraft_cos::ImageBudget`) is refused + A run that would decode more of its images than one redaction may (`pdfcraft_cos::ImageBudget`, + filters and fax and JBIG2 decoders alike) is refused and changes nothing: such an image can't be cleared, and removing it whole would take what no mark covers with it. - **vectors**: covered paths are removed; partly covered paths and shadings are clipped so diff --git a/crates/redact/src/tests.rs b/crates/redact/src/tests.rs index a547193dce..e5ff4621f7 100644 --- a/crates/redact/src/tests.rs +++ b/crates/redact/src/tests.rs @@ -780,6 +780,28 @@ fn a_jbig2_image_past_the_runs_allowance_stops_the_run() { assert!(crate::apply_within(&mut doc, None, pdfcraft_cos::ImageBudget::with_work(u64::MAX)).is_ok()); } +/// A Flate image past the run's allowance stops the run as a JBIG2 image does: removed whole, it +/// would take what no mark covers with it, unreported. A JPEG, which clearing doesn't read, is +/// removed whole as it always was, and spends nothing. +#[test] +fn a_flate_image_past_the_runs_allowance_stops_the_run() { + let dict = "/Type /XObject /Subtype /Image /Width 2 /Height 2 /BitsPerComponent 8 /ColorSpace /DeviceGray /Filter /FlateDecode"; + let page = || one_page(b"q 160 0 0 20 0 0 cm /Im Do Q", "/XObject << /Im 7 0 R >>", vec![stream(dict, pdfcraft_filters::encode_flate(&[9; 4]))]); + let mut doc = page(); + mark(&mut doc, 0, &[[0.0, 12.0, 75.0, 18.0]], ""); + let refused = crate::apply_within(&mut doc, None, pdfcraft_cos::ImageBudget::with_work(0)); + assert!(matches!(refused, Err(crate::RedactError::ImagesPastAllowance)), "{refused:?}"); + let mut doc = page(); + mark(&mut doc, 0, &[[0.0, 12.0, 75.0, 18.0]], ""); + assert!(crate::apply_within(&mut doc, None, pdfcraft_cos::ImageBudget::with_work(u64::MAX)).is_ok()); + + let dict = "/Type /XObject /Subtype /Image /Width 2 /Height 2 /BitsPerComponent 8 /ColorSpace /DeviceGray /Filter /DCTDecode"; + let mut doc = one_page(b"q 160 0 0 20 0 0 cm /Im Do Q", "/XObject << /Im 7 0 R >>", vec![stream(dict, b"not read")]); + mark(&mut doc, 0, &[[0.0, 12.0, 75.0, 18.0]], ""); + let report = crate::apply_within(&mut doc, None, pdfcraft_cos::ImageBudget::with_work(0)).unwrap(); + assert_eq!(report.images_removed, 1); +} + /// A JBIG2 image (a scan the optimizer stored as JBIG2) has its covered pixels cleared in a copy, /// like a fax image, instead of being removed whole: with `/JBIG2Globals`, its `/DecodeParms` by /// reference, behind another filter with its globals at its own place, or as a mask, its decode diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index def0b97e37..f125e26506 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -7409,7 +7409,7 @@ status = "partial" commands = ["optimize.advanced", "optimize.reduce"] tools = ["doc_optimize", "doc_reduce"] evidence = ["crates/optimize/tests/optimize.rs::eight_bit_black_and_white_images_become_one_bit", "crates/filters/src/ccitt.rs::bitmaps_round_trip", "crates/filters/src/ccitt.rs::a_page_of_text_like_marks_round_trips_and_shrinks", "crates/filters/src/ccitt.rs::what_it_cannot_encode_is_an_error", "crates/optimize/tests/optimize.rs::black_and_white_images_become_ccitt_group_4", "crates/optimize/tests/optimize.rs::group_3_fax_images_become_group_4", "crates/optimize/tests/optimize.rs::rgb_and_two_colour_indexed_black_and_white_images_become_one_bit", "crates/optimize/tests/optimize.rs::black_and_white_rgb_used_beyond_the_page_content_read_stays_rgb", "crates/optimize/tests/optimize.rs::black_and_white_rgb_drawn_only_where_the_optimizer_reads_becomes_grey", "crates/optimize/tests/optimize.rs::black_and_white_rgb_in_resources_its_page_tree_and_a_form_share_becomes_grey", "crates/automation/tests/it/automation.rs::doc_optimize_stores_black_and_white_images_as_ccitt_group_4", "crates/ui-egui/tests/it/create.rs::the_pdf_optimizer_dialog_saves_an_optimized_copy", "crates/filters/src/mq.rs::the_annex_h_test_sequence_encodes_to_its_bytes", "crates/filters/src/jbig2.rs::bitmaps_round_trip", "crates/filters/src/jbig2.rs::a_page_of_text_like_marks_round_trips_and_shrinks", "crates/optimize/tests/optimize.rs::black_and_white_images_become_jbig2", "crates/automation/tests/it/automation.rs::doc_optimize_stores_black_and_white_images_as_jbig2", "crates/filters/src/jbig2/symbols.rs::repeated_marks_are_stored_once", "crates/filters/src/jbig2/symbols.rs::scattered_marks_round_trip", "crates/optimize/tests/optimize.rs::rendered_text_becomes_jbig2_symbols", "crates/optimize/tests/optimize.rs::jbig2_scans_are_averaged_and_stored_again", "crates/optimize/src/images.rs::jbig2_images_past_the_runs_allowance_stay_as_they_are"] -notes = "All of a run's JBIG2 images share one allowance of decoding (about 1,000 scanned pages' worth, more for a longer file); those past it stay as they are and are counted in the report, which doc_optimize returns and the panel doesn't show yet. CCITT Group 4, lossless: PDF Optimizer ▸ Images ▸ Monochrome Images (on by default, so Reduce File Size does it too; doc_optimize mono) stores a 1-bit image of one colour component (grey, indexed, a separation) or an image mask, kept with no filter or general-purpose ones, as Group 4 when that is smaller as stored; the samples decode to the same bits, so colour space, decode array and masks stay. Packed to one bit first, then stored the same way (colour-key masks aside): eight-bit grey of only black and white, eight-bit DeviceRGB of only black and white (as DeviceGray; calibrated RGB, RGB with a profile, RGB drawn or listed where /DefaultGray or /DefaultRGB is in force or referred to from anywhere but the page and form resources read (an annotation's appearance or a pattern drawing it, directly or through resources or a form shared with a page; another image's /Alternates), and RGB whose soft mask has a /Matte aside), and an indexed image of 2, 4 or 8 bits that uses only palette entries 0 and 1 (keeping its palette), unless the grey or colour settings make it smaller still, as they can a dithered scan downsampled to JPEG, its soft mask counted. Group 4 wins on scans; on clean computer-drawn 1-bit art Flate can be smaller, and the image then stays. Group 3 fax images are stored again as Group 4 when that is smaller (damaged Group 3 with end-of-line codes as the decoder recovers it, as it reports no damage there; byte-aligned Group 3 with end-of-line codes but none before the first row, not as T.4 sends them, as PdfCraft draws it, which at /K 0 may be other rows than were coded). CCITT fax images are decoded to be downsampled (optimize.downsample-images), and a resampled image is stored as Group 4 or 1-bit Flate, whichever is smaller. Lossless JBIG2 with bounded exact-copy symbol coding is available too, with image export; lossy JBIG2 remains unsupported. JBIG2 images are read too: averaged like any (under Retain stored as JBIG2 again), and, not resampled, stored again as the compression asks when that is smaller (as JBIG2 symbols where its own coding is a generic region, say); a symbol-coded scan, averaged, loses its exact repeats and is often larger, and then stays as it is; one whose JBIG2 page isn't its size is left alone. Not yet: inline images, soft masks and /Mask images, Group 4 for rows wider than 32,767 samples." +notes = "All of a run's images share one allowance of decoding, filters included and JPEG data aside (about 1,000 scanned pages' worth, more for a longer file); those past it stay as they are and are counted in the report, which doc_optimize returns and the panel doesn't show yet. CCITT Group 4, lossless: PDF Optimizer ▸ Images ▸ Monochrome Images (on by default, so Reduce File Size does it too; doc_optimize mono) stores a 1-bit image of one colour component (grey, indexed, a separation) or an image mask, kept with no filter or general-purpose ones, as Group 4 when that is smaller as stored; the samples decode to the same bits, so colour space, decode array and masks stay. Packed to one bit first, then stored the same way (colour-key masks aside): eight-bit grey of only black and white, eight-bit DeviceRGB of only black and white (as DeviceGray; calibrated RGB, RGB with a profile, RGB drawn or listed where /DefaultGray or /DefaultRGB is in force or referred to from anywhere but the page and form resources read (an annotation's appearance or a pattern drawing it, directly or through resources or a form shared with a page; another image's /Alternates), and RGB whose soft mask has a /Matte aside), and an indexed image of 2, 4 or 8 bits that uses only palette entries 0 and 1 (keeping its palette), unless the grey or colour settings make it smaller still, as they can a dithered scan downsampled to JPEG, its soft mask counted. Group 4 wins on scans; on clean computer-drawn 1-bit art Flate can be smaller, and the image then stays. Group 3 fax images are stored again as Group 4 when that is smaller (damaged Group 3 with end-of-line codes as the decoder recovers it, as it reports no damage there; byte-aligned Group 3 with end-of-line codes but none before the first row, not as T.4 sends them, as PdfCraft draws it, which at /K 0 may be other rows than were coded). CCITT fax images are decoded to be downsampled (optimize.downsample-images), and a resampled image is stored as Group 4 or 1-bit Flate, whichever is smaller. Lossless JBIG2 with bounded exact-copy symbol coding is available too, with image export; lossy JBIG2 remains unsupported. JBIG2 images are read too: averaged like any (under Retain stored as JBIG2 again), and, not resampled, stored again as the compression asks when that is smaller (as JBIG2 symbols where its own coding is a generic region, say); a symbol-coded scan, averaged, loses its exact repeats and is often larger, and then stays as it is; one whose JBIG2 page isn't its size is left alone. Not yet: inline images, soft masks and /Mask images, Group 4 for rows wider than 32,767 samples." [[feature]] id = "optimize.only-if-smaller"