diff --git a/Cargo.toml b/Cargo.toml index edd9e682b7..19714f41a6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -64,7 +64,9 @@ pdfcraft-automation = { path = "crates/automation", version = "0.17.0", default- hayro = "0.7.1" hayro-syntax = "0.7.2" lopdf = "0.45" -flate2 = { version = "1.1", default-features = false, features = ["rust_backend"] } +# zlib-rs, which the Windows build already got through craft-update: miniz_oxide spends about +# 1.7 µs on every deflate block, empty ones included (`pdfcraft_filters::FLATE_READ_WORK`). +flate2 = { version = "1.1", default-features = false, features = ["zlib-rs"] } weezl = "0.2" # CCITT Group 4 encoding (pdfcraft-filters); already in the tree through tiff. fax = "0.2.7" diff --git a/crates/cos/src/budget.rs b/crates/cos/src/budget.rs index d3e4a71974..ef68bfffdf 100644 --- a/crates/cos/src/budget.rs +++ b/crates/cos/src/budget.rs @@ -13,12 +13,12 @@ use crate::{CosError, Document, ObjRef, Object, Stream}; /// decode them, so this alone reads about 1,000 of them. A byte into or out of a filter is a step /// too, and takes less: 3 ns at most, or up to 19 ns through a predictor on samples of a bit. A /// byte Flate reads is [`pdfcraft_filters::FLATE_READ_WORK`] steps besides, since inflating blocks -/// that make nothing takes up to 1.4 µs a byte: about 11 ns a step. +/// that make nothing takes up to 21 ns a byte: about 7 ns a step. const IMAGE_WORK: u64 = 1 << 34; /// The steps each byte of the file adds to [`IMAGE_WORK`], so that a long scan is read whole: such a /// page pays for itself with about 2 KB of the file, about the smallest a page of text is stored -/// in, and a stream of any filter this reads can be decoded about 7 times at Flate's best, 1,032 to -/// one, its reading included. So what a hostile file can ask for grows with its length, by about 14 minutes for 10 MB. +/// in, and a stream of any filter this reads can be decoded about 8 times at Flate's best, 1,032 to +/// one. So what a hostile file can ask for grows with its length, by about 14 minutes for 10 MB. const IMAGE_WORK_PER_BYTE: u64 = 1 << 13; /// Decoded bytes one page's content walk may produce in all: its `/Contents` streams and every @@ -591,6 +591,35 @@ mod image_tests { assert_eq!((budget.work_left(), budget.refused()), (0, 1)); } + /// What Flate reads is priced for blocks that make nothing, and an image an encoder made pays + /// little for it: a scan-like picture stored in under half its size costs no more than half + /// again its bytes, samples and pixels. Redaction decodes a picture once for each page that draws it, so + /// a letterhead on hundreds of pages has to fit as it did unpriced. + #[test] + fn an_image_an_encoder_made_pays_little_for_its_reading() { + let doc = Document::new_empty(); + let mut seed = 0x2545_F491_4F6C_DD1Du64; + let samples: Vec = (0..256 * 256usize) + .map(|i| { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + ((i % 256) as u8 / 16) ^ (seed as u8 & 3) + }) + .collect(); + let mut dict = Dict::new(); + dict.set(b"Width".to_vec(), Object::Int(256)); + dict.set(b"Height".to_vec(), Object::Int(256)); + let image = Stream::flate(dict, &samples); + assert!(image.raw.len() * 2 < samples.len(), "stored smaller: {}", image.raw.len()); + let unpriced = (image.raw.len() + samples.len()) as u64 + 256 * 256; + let mut budget = ImageBudget::with_work(u64::MAX); + budget.decode(&doc, &image, 1 << 20).unwrap(); + let paid = u64::MAX - budget.work_left(); + assert_eq!(paid, unpriced + read(image.raw.len())); + assert!(2 * paid <= 3 * unpriced, "{paid} steps against {unpriced}"); + } + /// An image pays for the pixels its data holds, not those its size declares: one claiming /// 131072 by 131072 pixels with a few bytes of Flate costs what those bytes give, and the /// images after it are decoded as they would be without it. diff --git a/crates/create/src/extract.rs b/crates/create/src/extract.rs index d5103071e6..8fc7fea984 100644 --- a/crates/create/src/extract.rs +++ b/crates/create/src/extract.rs @@ -184,11 +184,11 @@ fn image(doc: &Document, s: &Stream, budget: &mut ImageBudget) -> Result<(&'stat // In a PDF a CMYK JPEG's samples are ink amounts unless /Decode inverts them, while image // viewers read a CMYK JPEG file as inverted (Adobe's convention): as is, it would show as // a negative. - let as_is_negative = jpeg && !inverted && matches!(space(doc, s.dict.get(b"ColorSpace")), Ok(Space::Cmyk)); + let as_is_negative = jpeg && !inverted && matches!(space(doc, s.dict.get(b"ColorSpace"), budget), Ok(Space::Cmyk)); if jpeg && alpha.is_none() && key.is_none() && !as_is_negative { return as_is(); } - let mut space = if mask { Space::Gray } else { space(doc, s.dict.get(b"ColorSpace"))? }; + let mut space = if mask { Space::Gray } else { space(doc, s.dict.get(b"ColorSpace"), budget)? }; let (samples, bpc) = if jpeg { if !budget.pay_jpeg((w as u64).saturating_mul(h as u64).saturating_mul(space.components() as u64), &s.raw) { return Err(FilterError::Spent.to_string()); @@ -436,7 +436,10 @@ impl Space { /// A lookup table holds at most 256 entries of 4 bytes; decoding stops well past that. const MAX_LOOKUP: usize = 64 << 10; -fn space(doc: &Document, cs: Option<&Object>) -> Result { +/// The colour space `cs` names; an indexed one's lookup table is read within `budget`, as the +/// image's samples are, since each image reads it again and its filters can do work that makes +/// nothing. +fn space(doc: &Document, cs: Option<&Object>, budget: &mut ImageBudget) -> Result { let Some(cs) = cs else { return Err("the image has no colour space".into()) }; let cs = doc.resolve(cs); let unsupported = |n: &[u8]| Err(format!("{} images can't be exported yet", String::from_utf8_lossy(n))); @@ -470,7 +473,7 @@ fn space(doc: &Document, cs: Option<&Object>) -> Result { if nested { return Err("nested indexed colour spaces aren't valid".into()); } - let base = space(doc, a.get(1))?; + let base = space(doc, a.get(1), budget)?; let hival = match a.get(2).map(|value| doc.resolve(value)).as_deref() { Some(Object::Int(value)) if (0..=255).contains(value) => *value as u8, _ => return Err("an indexed colour space with an invalid hival".into()), @@ -478,10 +481,14 @@ fn space(doc: &Document, cs: Option<&Object>) -> Result { let table = match a.get(3).map(|t| doc.resolve(t)).as_deref() { Some(Object::String(s)) => s.bytes.clone(), // Through its parameters given by reference too, as the image's samples. - Some(Object::Stream(t)) => doc - .with_direct_parms(t) - .decoded_within(MAX_LOOKUP) - .map_err(|_| "an indexed colour space with an oversized or damaged lookup table".to_string())?, + Some(Object::Stream(t)) => { + let refused = budget.refused(); + match budget.decode_filters(doc, t, MAX_LOOKUP) { + Ok(table) => table, + Err(_) if budget.refused() > refused => return Err(FilterError::Spent.to_string()), + Err(_) => return Err("an indexed colour space with an oversized or damaged lookup table".into()), + } + } _ => return Err("an indexed colour space without a lookup table".into()), }; Ok(Space::Indexed(Box::new(base), hival, table)) diff --git a/crates/create/src/tests.rs b/crates/create/src/tests.rs index a7a5611ffc..644fed4bdb 100644 --- a/crates/create/src/tests.rs +++ b/crates/create/src/tests.rs @@ -2312,6 +2312,38 @@ fn flate_images_past_the_exports_allowance_are_left_out_with_the_reason() { assert_eq!(budget.refused(), 1); } +/// An indexed image's lookup table, given as a stream, is read within the export's allowance as +/// the image's samples are: each image reads it again, and its filters can do work that makes +/// nothing (Flate over empty blocks). Past the allowance the image is left out with the reason. +#[test] +fn a_lookup_table_is_read_within_the_exports_allowance() { + use pdfcraft_cos::{Dict, Document, ImageBudget, Object, PdfString, Stream}; + let table = [255, 0, 0, 0, 0, 255]; + let lookup = Stream::flate(Dict::new(), &table); + let exported = |as_stream: bool, work: u64| { + let mut doc = Document::new_empty(); + let table = if as_stream { Object::Ref(doc.add(Object::Stream(lookup.clone()))) } else { Object::String(PdfString::literal(table.to_vec())) }; + let mut dict = Dict::new(); + dict.set(b"Subtype".to_vec(), Object::name("Image")); + dict.set(b"Width".to_vec(), Object::Int(2)); + dict.set(b"Height".to_vec(), Object::Int(1)); + dict.set(b"BitsPerComponent".to_vec(), Object::Int(8)); + dict.set(b"ColorSpace".to_vec(), Object::Array(vec![Object::name("Indexed"), Object::name("DeviceRGB"), Object::Int(1), table])); + let image = doc.add(Object::Stream(Stream::flate(dict, &[0, 1]))); + let mut budget = ImageBudget::with_work(work); + let file = crate::extract::image_file_within(&doc, image, &mut budget); + (file, work - budget.work_left(), budget.refused()) + }; + let (direct, image_alone, _) = exported(false, u64::MAX); + let (streamed, both, _) = exported(true, u64::MAX); + assert_eq!(streamed.unwrap(), direct.unwrap(), "the same picture"); + let read = (lookup.raw.len() + table.len()) as u64 + pdfcraft_filters::FLATE_READ_WORK * lookup.raw.len() as u64; + assert_eq!(both, image_alone + read, "the table is paid for as it is read"); + let (refused, _, count) = exported(true, image_alone); + assert!(refused.unwrap_err().contains("more image decoding than")); + assert_eq!(count, 1); +} + /// A JPEG is written as it is, at no cost, unless something means decoding it, here a mask: that /// pays for its samples, and past the allowance the image goes, with the reason, as it would come out without /// its transparency otherwise. diff --git a/crates/filters/README.md b/crates/filters/README.md index 38666e6369..6c831ebe54 100644 --- a/crates/filters/README.md +++ b/crates/filters/README.md @@ -37,7 +37,7 @@ encode_jbig2(data, columns, rows) -> Result, FilterError> // 1-bit row - Image codecs (DCT, JPX, JBIG2, CCITT), `Crypt` and unknown filters anywhere in a chain give `Unsupported`: callers keep such data encoded. So does a chain longer than `MAX_CHAIN` (8) filters, before any stage runs. Decryption belongs to `crypt`. - `max_output` bounds every intermediate buffer (including the pre-predictor buffer) and gives `LimitExceeded(max_output)`. This also holds when Flate retries raw deflate after a misleading zlib header, in both strict and tolerant decoding. - `decode_counted` and `decode_tolerant_counted` are for a caller charging a budget (`pdfcraft_cos::DecodeBudget`). `max_output` bounds all stages together (each stage gets what the earlier ones left), and they give the bytes every stage that ran decoded, on success and on failure. A stage's output counts as its decoder produced it, before a predictor or a later stage shrinks it, and a failed decode pays for what it decoded first. -- `decode_tolerant_within` is for a run of images (`pdfcraft_cos::ImageBudget`), as `decode_within` (its strict form), `decode_jbig2_within` and `decode_ccitt_within` are, all spending from one `work`, in the JBIG2 decoder's steps. `max_output` bounds each stage, as in `decode_tolerant`, and no stage decodes past what `work` has left. It pays a step a byte of `data` first and a step a byte each stage decodes, so a chain that fails has paid for what it did: 0.2 to 3 ns a byte in or out, up to 19 ns through a predictor on samples of a bit, against the 6 to 16 ns of a JBIG2 step. A Flate stage also pays `FLATE_READ_WORK` (128) steps a byte it reads, before it runs, and again if it reads them a second time as raw deflate: inflating does work for every block it reads, and a stream of empty blocks took 1.4 µs a byte to make nothing, about 11 ns a step at that price. What `work` can't pay for spends all of it, and gives `Spent`; with nothing left nothing is read. A chain that can't be decoded at all costs nothing and gives its usual error. +- `decode_tolerant_within` is for a run of images (`pdfcraft_cos::ImageBudget`), as `decode_within` (its strict form), `decode_jbig2_within` and `decode_ccitt_within` are, all spending from one `work`, in the JBIG2 decoder's steps. `max_output` bounds each stage, as in `decode_tolerant`, and no stage decodes past what `work` has left. It pays a step a byte of `data` first and a step a byte each stage decodes, so a chain that fails has paid for what it did: 0.2 to 3 ns a byte in or out, up to 19 ns through a predictor on samples of a bit, against the 6 to 16 ns of a JBIG2 step. A Flate stage also pays `FLATE_READ_WORK` (2) steps a byte it reads, before it runs, and again if it reads them a second time as raw deflate: inflating does work for every block it reads, and a stream of empty dynamic blocks took 21 ns a byte to make nothing, about 7 ns a step at that price. Inflating is flate2's zlib-rs backend on every platform; miniz_oxide, before, took 1.4 µs a byte on empty fixed blocks. What `work` can't pay for spends all of it, and gives `Spent`; with nothing left nothing is read. A chain that can't be decoded at all costs nothing and gives its usual error. - `decode_tolerant` turns `Corrupt` into "what was decoded so far" plus `partial = true`, then runs the rest of the chain on it (predictors are applied to partial output too). `LimitExceeded` and `Unsupported` are still errors. ## Behaviour diff --git a/crates/filters/src/lib.rs b/crates/filters/src/lib.rs index cf10d0586b..5a679d5270 100644 --- a/crates/filters/src/lib.rs +++ b/crates/filters/src/lib.rs @@ -258,15 +258,16 @@ fn spending(chain: &[(Filter, Params)], data: &[u8], max_output: usize, work: &m pub const MAX_CHAIN: usize = 8; /// The steps each byte Flate reads costs where decoding is paid for ([`decode_tolerant_within`], -/// [`decode_within`]), besides a step for each byte it makes. Inflating does work for every block -/// it reads, whether or not the block makes anything: an empty fixed-Huffman block is ten bits -/// (RFC 1951 §3.2.6), and a stream of them took 1,383 ns a byte to make nothing (miniz_oxide -/// 0.8.9, release build), one of a literal each 768 ns a byte, and empty dynamic blocks 106. At -/// this price the worst of these comes to about 11 ns a step, inside what a step of JBIG2 -/// decoding takes. Data a real encoder makes took 0.4 to 3.2 ns for each byte it made, its reading -/// included, so it pays far more than reading takes; each byte of a file still adds 64 times this -/// price to an operation's allowance (8,192 steps, `IMAGE_WORK_PER_BYTE` in `pdfcraft_cos`). -pub const FLATE_READ_WORK: u64 = 1 << 7; +/// [`decode_within`]), besides the step it was paid as input and a step for each byte it makes. +/// Inflating does work for every block it reads, whether or not the block makes anything: an empty +/// fixed-Huffman block is ten bits (RFC 1951 §3.2.6). With flate2's zlib-rs backend (zlib-rs +/// 0.6.8, release build) the slowest blocks measured, empty dynamic ones, took 21 ns a byte read +/// to make nothing, empty fixed ones 9 and fixed ones of a literal each 6, so at this price the +/// worst comes to about 7 ns a step, inside what a step of JBIG2 decoding takes. (miniz_oxide +/// 0.8.9, flate2's backend here before, builds a block's tables for every block, and took 1,383 ns +/// a byte read on empty fixed blocks.) Data a real encoder makes took 0.2 to 2.3 ns a byte made, its +/// reading included, so an image pays twice its stored length more than it would unpriced. +pub const FLATE_READ_WORK: u64 = 2; /// Whether a chain is one [`run`] takes, and if not, the error. fn decodable(chain: &[(Filter, Params)]) -> Result<(), FilterError> {