From 10172d7a94d93f9c5030478c81bf6fdf6760e54b Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Mon, 2 Nov 2026 20:36:50 +0100 Subject: [PATCH] M2.6: a Flate stage pays for what it reads The image allowance charged a Flate stage the bytes it made, and its input only as what the stage before it made. Inflating does work for every block it reads, whether or not the block makes anything: a stream of empty fixed-Huffman blocks, ten bits each, took 1,383 ns a byte to make nothing (miniz_oxide 0.9.1, release), one of a literal each 768 ns a byte, empty dynamic blocks 106, against the 3 ns a step budget.rs states. A 250-byte stream of four Flates over empty blocks took 47 s and was charged about 34 million steps, so a review of #5698 worked out about 7 hours before a 240 KB file of them was refused, through clearing, Optimize, export, compare and redaction's images and dictionaries alike. Where decoding is paid for (decode_tolerant_within, decode_within), a Flate stage now pays FLATE_READ_WORK, 128 steps, for each byte it reads, before it runs: a stage the allowance can't pay for doesn't start. Per byte read, the worst of the shapes measured comes to about 11 ns a step at that price, inside what a JBIG2 step takes. The raw retry of a zlib header that only looked valid reads the bytes again, so it pays again, makes no more than is left once it has, and is refused when it can't be paid for. Byte counts for a byte allowance (decode_counted, decode_tolerant_counted, as a page's content uses) are unchanged. With a tiny file's allowance (2^34 steps), the 250-byte stream is decoded three times in 141 s, and the fourth stops before its last stage reads the empty blocks (8.2 ns a step, release). Data an encoder makes took 0.4 to 3.2 ns a byte it made, reading included, and each byte of a file adds 64 times the price to an operation's allowance. Tests: a_flate_stage_pays_for_what_it_reads (empty blocks: 129 steps a byte, refused a step short with nothing inflated) and a_second_reading_is_paid_for; the exact costs in filters, cos and optimize tests add what Flate reads. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/cos/src/budget.rs | 37 ++++--- crates/filters/README.md | 2 +- crates/filters/src/flate.rs | 51 +++++++--- crates/filters/src/lib.rs | 159 ++++++++++++++++++++++++------- crates/optimize/src/fragments.rs | 4 +- crates/optimize/src/images.rs | 4 +- 6 files changed, 191 insertions(+), 66 deletions(-) diff --git a/crates/cos/src/budget.rs b/crates/cos/src/budget.rs index 376b108be3..d3e4a71974 100644 --- a/crates/cos/src/budget.rs +++ b/crates/cos/src/budget.rs @@ -11,12 +11,14 @@ use crate::{CosError, Document, ObjRef, Object, Stream}; /// 16 ns, so this is 2 to 5 minutes, and about as long again for what an operation does with the /// pixels. A page scanned at 300 dpi takes about 17 million, 8.4 for its pixels and that again to /// decode them, so this alone reads about 1,000 of them. A byte into or out of a filter is a step -/// too, and takes less: 3 ns at most, or up to 19 ns through a predictor on samples of a bit. +/// too, and takes less: 3 ns at most, or up to 19 ns through a predictor on samples of a bit. A +/// byte Flate reads is [`pdfcraft_filters::FLATE_READ_WORK`] steps besides, since inflating blocks +/// that make nothing takes up to 1.4 µs a byte: about 11 ns a step. const IMAGE_WORK: u64 = 1 << 34; /// The steps each byte of the file adds to [`IMAGE_WORK`], so that a long scan is read whole: such a /// page pays for itself with about 2 KB of the file, about the smallest a page of text is stored -/// in, and a stream of any filter this reads can be decoded about 8 times at Flate's best, 1,032 to -/// one. So what a hostile file can ask for grows with its length, by about 14 minutes for 10 MB. +/// in, and a stream of any filter this reads can be decoded about 7 times at Flate's best, 1,032 to +/// one, its reading included. So what a hostile file can ask for grows with its length, by about 14 minutes for 10 MB. const IMAGE_WORK_PER_BYTE: u64 = 1 << 13; /// Decoded bytes one page's content walk may produce in all: its `/Contents` streams and every @@ -361,6 +363,11 @@ mod image_tests { Stream::from_raw(dict, pdfcraft_filters::encode_jbig2_generic(&[0xFF; 4], 16, 2).unwrap()) } + /// What a Flate stage pays to read `n` bytes, besides what it makes. + fn read(n: usize) -> u64 { + pdfcraft_filters::FLATE_READ_WORK * n as u64 + } + /// Each image spends what it took from one allowance, and those past it are refused and /// counted, whatever their filters. What can't be decoded here costs nothing, and fails /// without being one of these. @@ -407,7 +414,7 @@ mod image_tests { let mut dict = Dict::new(); dict.set(b"Filter".to_vec(), Object::Array(vec![Object::name("FlateDecode"), Object::name("FlateDecode")])); let bomb = Stream::from_raw(dict, pdfcraft_filters::encode_flate(&inner)); - let cost = (bomb.raw.len() + inner.len() + samples.len()) as u64; + let cost = (bomb.raw.len() + inner.len() + samples.len()) as u64 + read(bomb.raw.len()) + read(inner.len()); let mut budget = ImageBudget::with_work(2 * cost + 1); for left in [cost + 1, 1] { assert_eq!(budget.decode(&doc, &bomb, 1 << 20).unwrap(), samples); @@ -436,17 +443,17 @@ mod image_tests { let image = behind(pdfcraft_filters::encode_flate(&alone.raw)); let mut budget = ImageBudget::with_work(u64::MAX); assert_eq!(budget.decode(&doc, &image, 1 << 20).unwrap(), samples); - assert_eq!(u64::MAX - budget.work_left(), decoding + (image.raw.len() + alone.raw.len()) as u64); + assert_eq!(u64::MAX - budget.work_left(), decoding + (image.raw.len() + alone.raw.len()) as u64 + read(image.raw.len())); // Past this image's limit: an error, with what finding out took paid and the rest left. let bomb = behind(pdfcraft_filters::encode_flate(&vec![0; 1 << 16])); let mut budget = ImageBudget::with_work(u64::MAX); assert!(budget.decode(&doc, &bomb, 1000).is_err()); - assert_eq!(u64::MAX - budget.work_left(), bomb.raw.len() as u64 + 1000); + assert_eq!(u64::MAX - budget.work_left(), bomb.raw.len() as u64 + read(bomb.raw.len()) + 1000); assert_eq!(budget.refused(), 0, "that image's limit isn't the run's"); // A small allowance is a limit too: it stops there, and the image is refused. - let mut budget = ImageBudget::with_work(bomb.raw.len() as u64 + 10); + let mut budget = ImageBudget::with_work(bomb.raw.len() as u64 + read(bomb.raw.len()) + 10); assert!(budget.decode(&doc, &bomb, 1 << 20).is_err()); assert_eq!((budget.work_left(), budget.refused()), (0, 1)); } @@ -486,7 +493,7 @@ mod image_tests { let doc = Document::open(std::sync::Arc::new(file)).unwrap(); let image = doc.get(doc.root().unwrap()).as_dict().unwrap().reference(b"Image").unwrap(); let s = stream(&doc, image); - let filters = (raw.len() + skipped.len()) as u64; + let filters = (raw.len() + skipped.len()) as u64 + read(raw.len()); assert_eq!(spent(&doc, &s), first, "the first read undoes them"); assert_eq!(spent(&doc, &s), first - filters, "kept"); @@ -501,7 +508,7 @@ mod image_tests { let image = jbig2(Some(Object::Dict(parms))); let mut budget = ImageBudget::with_work(u64::MAX); assert!(budget.decode(&doc, &image, 1 << 20).unwrap_err().to_string().contains("globals")); - assert_eq!(u64::MAX - budget.work_left(), (over.len() + crate::MAX_JBIG2_GLOBALS) as u64); + assert_eq!(u64::MAX - budget.work_left(), (over.len() + crate::MAX_JBIG2_GLOBALS) as u64 + read(over.len())); assert_eq!(budget.refused(), 0); } @@ -522,7 +529,7 @@ mod image_tests { dict.set(b"DecodeParms".to_vec(), Object::Array(vec![Object::Null, Object::Dict(parms)])); let image = Stream::from_raw(dict, pdfcraft_filters::encode_flate(&coded)); let pixels = u64::from(w) * u64::from(h); - let cost = (image.raw.len() + coded.len() + 4 * coded.len() + 512 * h as usize) as u64 + pixels; + let cost = (image.raw.len() + coded.len() + 4 * coded.len() + 512 * h as usize) as u64 + read(image.raw.len()) + pixels; let mut budget = ImageBudget::with_work(cost); assert_eq!(budget.decode(&doc, &image, 1 << 20).unwrap().len(), 512 * h as usize); assert_eq!(budget.work_left(), 0, "exactly that"); @@ -547,7 +554,7 @@ mod image_tests { }; let rows = vec![0xFFu8; 512 * 1024]; let flate = blank(Some("FlateDecode"), pdfcraft_filters::encode_flate(&rows)); - let cost = (flate.raw.len() + rows.len()) as u64 + 4096 * 1024; + let cost = (flate.raw.len() + rows.len()) as u64 + read(flate.raw.len()) + 4096 * 1024; let mut budget = ImageBudget::with_work(cost); assert_eq!(budget.decode(&doc, &flate, 1 << 24).unwrap(), rows); assert_eq!((budget.work_left(), budget.refused()), (0, 0), "exactly that"); @@ -603,8 +610,8 @@ mod image_tests { budget.decode(&doc, s, 1 << 24).unwrap(); u64::MAX - budget.work_left() }; - assert_eq!(cost(&claims), (claims.raw.len() + 16 + 16 * 8) as u64, "its bytes, and a pixel a bit of them"); - assert_eq!(cost(&image), (image.raw.len() + 64 * 64) as u64 + 64 * 64, "its bytes, and its pixels"); + assert_eq!(cost(&claims), (claims.raw.len() + 16 + 16 * 8) as u64 + read(claims.raw.len()), "its bytes, and a pixel a bit of them"); + assert_eq!(cost(&image), (image.raw.len() + 64 * 64) as u64 + read(image.raw.len()) + 64 * 64, "its bytes, and its pixels"); for decode in [ImageBudget::decode, ImageBudget::decode_filters] { let mut budget = ImageBudget::with_work(cost(&claims) + cost(&image)); assert_eq!(decode(&mut budget, &doc, &claims, 1 << 24).unwrap(), [0; 16]); @@ -621,7 +628,7 @@ mod image_tests { let samples = vec![7u8; 1000]; let raw = pdfcraft_filters::encode_flate(&samples); let chain = [(Filter::Flate, Params::default())]; - let cost = (raw.len() + samples.len()) as u64; + let cost = (raw.len() + samples.len()) as u64 + read(raw.len()); let mut budget = ImageBudget::with_work(cost + 10); assert_eq!(budget.decode_chain(&chain, &raw, 1 << 20).unwrap(), samples); assert_eq!(budget.work_left(), 10); @@ -658,7 +665,7 @@ mod image_tests { assert_eq!(budget.decode_filters(&doc, &image, 1 << 20).unwrap(), rows); let cost = u64::MAX - budget.work_left(); // The tag byte of each of its 100 rows counts, which the predictor takes off. - assert_eq!(cost, (image.raw.len() + rows.len() + 100) as u64); + assert_eq!(cost, (image.raw.len() + rows.len() + 100) as u64 + read(image.raw.len())); let mut budget = ImageBudget::with_work(cost - 1); assert!(budget.decode_filters(&doc, &image, 1 << 20).is_err()); assert_eq!((budget.work_left(), budget.refused()), (0, 1)); diff --git a/crates/filters/README.md b/crates/filters/README.md index 90e1485177..38666e6369 100644 --- a/crates/filters/README.md +++ b/crates/filters/README.md @@ -37,7 +37,7 @@ encode_jbig2(data, columns, rows) -> Result, FilterError> // 1-bit row - Image codecs (DCT, JPX, JBIG2, CCITT), `Crypt` and unknown filters anywhere in a chain give `Unsupported`: callers keep such data encoded. So does a chain longer than `MAX_CHAIN` (8) filters, before any stage runs. Decryption belongs to `crypt`. - `max_output` bounds every intermediate buffer (including the pre-predictor buffer) and gives `LimitExceeded(max_output)`. This also holds when Flate retries raw deflate after a misleading zlib header, in both strict and tolerant decoding. - `decode_counted` and `decode_tolerant_counted` are for a caller charging a budget (`pdfcraft_cos::DecodeBudget`). `max_output` bounds all stages together (each stage gets what the earlier ones left), and they give the bytes every stage that ran decoded, on success and on failure. A stage's output counts as its decoder produced it, before a predictor or a later stage shrinks it, and a failed decode pays for what it decoded first. -- `decode_tolerant_within` is for a run of images (`pdfcraft_cos::ImageBudget`), as `decode_within` (its strict form), `decode_jbig2_within` and `decode_ccitt_within` are, all spending from one `work`, in the JBIG2 decoder's steps. `max_output` bounds each stage, as in `decode_tolerant`, and no stage decodes past what `work` has left. It pays a step a byte of `data` first and a step a byte each stage decodes, so a chain that fails has paid for what it did: 0.2 to 3 ns a byte in or out, up to 19 ns through a predictor on samples of a bit, against the 6 to 16 ns of a JBIG2 step. What `work` can't pay for spends all of it, and gives `Spent`; with nothing left nothing is read. A chain that can't be decoded at all costs nothing and gives its usual error. +- `decode_tolerant_within` is for a run of images (`pdfcraft_cos::ImageBudget`), as `decode_within` (its strict form), `decode_jbig2_within` and `decode_ccitt_within` are, all spending from one `work`, in the JBIG2 decoder's steps. `max_output` bounds each stage, as in `decode_tolerant`, and no stage decodes past what `work` has left. It pays a step a byte of `data` first and a step a byte each stage decodes, so a chain that fails has paid for what it did: 0.2 to 3 ns a byte in or out, up to 19 ns through a predictor on samples of a bit, against the 6 to 16 ns of a JBIG2 step. A Flate stage also pays `FLATE_READ_WORK` (128) steps a byte it reads, before it runs, and again if it reads them a second time as raw deflate: inflating does work for every block it reads, and a stream of empty blocks took 1.4 µs a byte to make nothing, about 11 ns a step at that price. What `work` can't pay for spends all of it, and gives `Spent`; with nothing left nothing is read. A chain that can't be decoded at all costs nothing and gives its usual error. - `decode_tolerant` turns `Corrupt` into "what was decoded so far" plus `partial = true`, then runs the rest of the chain on it (predictors are applied to partial output too). `LimitExceeded` and `Unsupported` are still errors. ## Behaviour diff --git a/crates/filters/src/flate.rs b/crates/filters/src/flate.rs index 1c38f72a53..ffe46207b7 100644 --- a/crates/filters/src/flate.rs +++ b/crates/filters/src/flate.rs @@ -21,25 +21,48 @@ fn zlib_header(data: &[u8]) -> Option<(u8, u8)> { ok.then_some((cmf, flg)) } -pub(crate) fn decode(data: &[u8], max: usize) -> Step { +/// `data` inflated within `max`, and whether it was read twice: a zlib header that only looked +/// valid is read again as raw deflate, within `again`, what a caller paying for what is read has +/// left for it once it has paid for the second reading. `None`, that reading can't be paid for, +/// and is [`FilterError::Spent`](crate::FilterError::Spent). +pub(crate) fn decode(data: &[u8], max: usize, again: Option) -> (Step, bool) { match zlib_header(data) { - Some((_, flg)) if flg & 0x20 != 0 => Err(Failure::corrupt(NAME, "zlib preset dictionary is not allowed in PDF", Vec::new())), + Some((_, flg)) if flg & 0x20 != 0 => (Err(Failure::corrupt(NAME, "zlib preset dictionary is not allowed in PDF", Vec::new())), false), Some(_) => match inflate_raw(&data[2..], max) { // A header that only looked valid: retry the whole input as raw deflate. - Err(f) if f.partial.is_empty() && matches!(f.error, crate::FilterError::Corrupt { .. }) => match inflate_raw(data, max) { - // Keep the original corruption diagnostic only if the retry is corrupt - // too. A limit error must remain fatal even for tolerant decoding. What the - // retry decoded isn't kept, but a caller charging for work has to hear of it. - Err(retry) if matches!(retry.error, crate::FilterError::Corrupt { .. }) => Err(Failure { discarded: retry.partial.len(), ..f }), - retry => retry, - }, - r => r, + Err(f) if f.partial.is_empty() && matches!(f.error, crate::FilterError::Corrupt { .. }) => { + let Some(again) = again else { + return (Err(Failure { error: crate::FilterError::Spent, partial: Vec::new(), discarded: 0 }), false); + }; + let step = match inflate_raw(data, again) { + // Keep the original corruption diagnostic only if the retry is corrupt + // too. A limit error must remain fatal even for tolerant decoding. What the + // retry decoded isn't kept, but a caller charging for work has to hear of it. + Err(retry) if matches!(retry.error, crate::FilterError::Corrupt { .. }) => Err(Failure { discarded: retry.partial.len(), ..f }), + retry => retry, + }; + (step, true) + } + r => (r, false), }, - None => inflate_raw(data, max), + None => (inflate_raw(data, max), false), } } +#[cfg(test)] +thread_local! { + /// Bytes inflating gave back on this thread, its partial output included. + pub(crate) static INFLATED: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + fn inflate_raw(input: &[u8], max: usize) -> Step { + let step = inflate(input, max); + #[cfg(test)] + INFLATED.with(|c| c.set(c.get() + step.as_ref().map_or_else(|f| f.partial.len(), Vec::len))); + step +} + +fn inflate(input: &[u8], max: usize) -> Step { let mut d = Decompress::new(false); let mut out: Vec = Vec::new(); // One byte beyond the limit lets us tell "exactly max" from "more than max". @@ -87,7 +110,7 @@ mod tests { use crate::FilterError; fn dec(d: &[u8]) -> Result, FilterError> { - decode(d, 1 << 24).map_err(|f| f.error) + decode(d, 1 << 24, Some(1 << 24)).0.map_err(|f| f.error) } #[test] @@ -144,7 +167,7 @@ mod tests { fn corrupt_and_truncated() { let data: Vec = (0..10_000u32).map(|i| (i % 13) as u8).collect(); let z = encode(&data); - match decode(&z[..z.len() / 2], 1 << 24) { + match decode(&z[..z.len() / 2], 1 << 24, Some(1 << 24)).0 { Err(f) => { assert!(matches!(f.error, FilterError::Corrupt { .. })); assert_eq!(&data[..f.partial.len()], &f.partial[..]); @@ -168,6 +191,6 @@ mod tests { } fn dec_limit(d: &[u8], max: usize) -> Result, FilterError> { - decode(d, max).map_err(|f| f.error) + decode(d, max, Some(max)).0.map_err(|f| f.error) } } diff --git a/crates/filters/src/lib.rs b/crates/filters/src/lib.rs index 181f96ab66..cf10d0586b 100644 --- a/crates/filters/src/lib.rs +++ b/crates/filters/src/lib.rs @@ -188,7 +188,7 @@ pub(crate) fn pay(work: &mut u64, steps: u64) -> Result<(), FilterError> { /// [`FilterError::Unsupported`] (callers keep such data encoded), as does a chain longer than /// [`MAX_CHAIN`]. `max_output` bounds every intermediate buffer. pub fn decode(chain: &[(Filter, Params)], data: &[u8], max_output: usize) -> Result, FilterError> { - run(chain, data, max_output, None, true).map(|(v, _, _)| v).map_err(|(e, _)| e) + run(chain, data, max_output, None, true, false).map(|(v, _, _)| v).map_err(|(e, _)| e) } /// Like [`decode`] for a caller charging an allowance: `max_output` bounds the bytes all stages @@ -196,27 +196,28 @@ pub fn decode(chain: &[(Filter, Params)], data: &[u8], max_output: usize) -> Res /// bytes, on success and on failure. A stage's output counts even when a later stage or its /// predictor shrinks it. pub fn decode_counted(chain: &[(Filter, Params)], data: &[u8], max_output: usize) -> Result<(Vec, usize), (FilterError, usize)> { - run(chain, data, max_output, Some(max_output), true).map(|(v, _, spent)| (v, spent)) + run(chain, data, max_output, Some(max_output), true, false).map(|(v, _, spent)| (v, spent)) } /// Like [`decode`], but on corrupt data returns what was decoded before the error and /// reports `partial = true` (the behaviour viewers expect). The remaining filters of the /// chain still run on the partial data. Limit and unsupported errors are still errors. pub fn decode_tolerant(chain: &[(Filter, Params)], data: &[u8], max_output: usize) -> Result<(Vec, bool), FilterError> { - run(chain, data, max_output, None, false).map(|(v, partial, _)| (v, partial)).map_err(|(e, _)| e) + run(chain, data, max_output, None, false, false).map(|(v, partial, _)| (v, partial)).map_err(|(e, _)| e) } /// [`decode_tolerant`] bounded and counted as [`decode_counted`] is. pub fn decode_tolerant_counted(chain: &[(Filter, Params)], data: &[u8], max_output: usize) -> Result<(Vec, bool, usize), (FilterError, usize)> { - run(chain, data, max_output, Some(max_output), false) + run(chain, data, max_output, Some(max_output), false, false) } /// [`decode_tolerant`], spending from `work`, what decoding may still cost across a run of images: -/// a step for each byte of `data`, paid before anything is decoded, and one for each byte a stage -/// decodes, so a chain that fails, or a stage that goes past `max_output`, has paid for what it did. -/// No stage decodes past what is left: a chain that `work` can't pay for spends all of it and stops -/// there, so with nothing left nothing is read. A chain that can't be decoded at all costs nothing, -/// whatever is left, so what an image's filters are is known from the error. +/// a step for each byte of `data`, paid before anything is decoded, [`FLATE_READ_WORK`] for each +/// byte a Flate stage reads, paid before it runs (again if it reads them twice), and one for each +/// byte a stage decodes, so a chain that fails, or a stage that goes past `max_output`, has paid for +/// what it did. No stage decodes past what is left: a chain that `work` can't pay for spends all of +/// it and stops there, so with nothing left nothing is read. A chain that can't be decoded at all +/// costs nothing, whatever is left, so what an image's filters are is known from the error. /// /// Unlike [`decode_tolerant`], no filters at all gives `data` itself, within `max_output`. pub fn decode_tolerant_within(chain: &[(Filter, Params)], data: &[u8], max_output: usize, work: &mut u64) -> Result, FilterError> { @@ -236,14 +237,15 @@ fn spending(chain: &[(Filter, Params)], data: &[u8], max_output: usize, work: &m return if data.len() > max_output { Err(FilterError::LimitExceeded(max_output)) } else { Ok(data.to_vec()) }; } let left = usize::try_from(*work).unwrap_or(usize::MAX); - match run(chain, data, max_output, Some(left), strict) { + match run(chain, data, max_output, Some(left), strict, true) { Ok((out, _, decoded)) => pay(work, decoded as u64).map(|()| out), // The stage stopped at the lower of the two, which took as much to find out. Err((FilterError::LimitExceeded(_), decoded)) if left.saturating_sub(decoded) > max_output => { pay(work, (decoded as u64).saturating_add(max_output as u64))?; Err(FilterError::LimitExceeded(max_output)) } - Err((FilterError::LimitExceeded(_), _)) => { + // Past what is left, a stage stops, or a Flate stage doesn't start. + Err((FilterError::LimitExceeded(_) | FilterError::Spent, _)) => { *work = 0; Err(FilterError::Spent) } @@ -255,6 +257,17 @@ fn spending(chain: &[(Filter, Params)], data: &[u8], max_output: usize, work: &m /// up to the output limit, so the chain length multiplies the work. pub const MAX_CHAIN: usize = 8; +/// The steps each byte Flate reads costs where decoding is paid for ([`decode_tolerant_within`], +/// [`decode_within`]), besides a step for each byte it makes. Inflating does work for every block +/// it reads, whether or not the block makes anything: an empty fixed-Huffman block is ten bits +/// (RFC 1951 §3.2.6), and a stream of them took 1,383 ns a byte to make nothing (miniz_oxide +/// 0.8.9, release build), one of a literal each 768 ns a byte, and empty dynamic blocks 106. At +/// this price the worst of these comes to about 11 ns a step, inside what a step of JBIG2 +/// decoding takes. Data a real encoder makes took 0.4 to 3.2 ns for each byte it made, its reading +/// included, so it pays far more than reading takes; each byte of a file still adds 64 times this +/// price to an operation's allowance (8,192 steps, `IMAGE_WORK_PER_BYTE` in `pdfcraft_cos`). +pub const FLATE_READ_WORK: u64 = 1 << 7; + /// Whether a chain is one [`run`] takes, and if not, the error. fn decodable(chain: &[(Filter, Params)]) -> Result<(), FilterError> { if chain.len() > MAX_CHAIN { @@ -268,22 +281,39 @@ fn decodable(chain: &[(Filter, Params)]) -> Result<(), FilterError> { /// The decoded data, whether it is partial and the bytes all stages decoded, or the error and /// the bytes decoded before it. `max` bounds each stage, and `total` all of them together. +/// `priced`, the count is steps: each Flate stage also pays [`FLATE_READ_WORK`] for each byte it +/// reads, before it runs, so a stage `total` can't pay for doesn't run +/// ([`FilterError::Spent`]). fn run( chain: &[(Filter, Params)], data: &[u8], max: usize, total: Option, strict: bool, + priced: bool, ) -> Result<(Vec, bool, usize), (FilterError, usize)> { decodable(chain).map_err(|e| (e, 0))?; let mut cur: Cow<[u8]> = Cow::Borrowed(data); let mut partial = false; let mut spent = 0usize; + let price = usize::try_from(FLATE_READ_WORK).unwrap_or(usize::MAX); for (filter, params) in chain { + let read = if priced && matches!(filter, Filter::Flate) { price.saturating_mul(cur.len()) } else { 0 }; + let room = total.map_or(usize::MAX, |total| total.saturating_sub(spent)); + if read > room { + return Err((FilterError::Spent, spent)); + } + spent = spent.saturating_add(read); #[cfg(test)] STAGE_BYTES.with(|c| c.set(c.get() + cur.len())); let limit = total.map_or(max, |total| max.min(total.saturating_sub(spent))); - let (step, work) = decode_one(filter, params, &cur, limit); + // A second reading of the same bytes (Flate's raw retry) is paid for as the first was, and + // makes no more than is left once it has been. + let again = room.checked_sub(read.saturating_mul(2)).map(|left| max.min(left)); + let (step, work, again) = decode_one(filter, params, &cur, limit, again); + if again { + spent = spent.saturating_add(read); + } spent = spent.saturating_add(work); cur = Cow::Owned(match step { Ok(v) => v, @@ -298,15 +328,21 @@ fn run( Ok((cur.into_owned(), partial, spent)) } -/// One stage's result and the bytes its decoder produced (before any predictor). -fn decode_one(filter: &Filter, params: &Params, data: &[u8], max: usize) -> (Step, usize) { +/// One stage's result, the bytes its decoder produced (before any predictor), and whether it +/// read its input twice, which it does only within `again` ([`flate::decode`]). +fn decode_one(filter: &Filter, params: &Params, data: &[u8], max: usize, again: Option) -> (Step, usize, bool) { + let once = |(step, n): (Step, usize)| (step, n, false); match filter { - Filter::Flate => with_predictor(flate::decode(data, max), params, "FlateDecode"), - Filter::Lzw => with_predictor(lzw::decode(data, params.early_change != 0, max), params, "LZWDecode"), - Filter::AsciiHex => counted(asciihex::decode(data, max)), - Filter::Ascii85 => counted(ascii85::decode(data, max)), - Filter::RunLength => counted(runlength::decode(data, max)), - other => (Err(Failure { error: FilterError::Unsupported(other.name().to_owned()), partial: Vec::new(), discarded: 0 }), 0), + Filter::Flate => { + let (step, twice) = flate::decode(data, max, again); + let (step, n) = with_predictor(step, params, "FlateDecode"); + (step, n, twice) + } + Filter::Lzw => once(with_predictor(lzw::decode(data, params.early_change != 0, max), params, "LZWDecode")), + Filter::AsciiHex => once(counted(asciihex::decode(data, max))), + Filter::Ascii85 => once(counted(ascii85::decode(data, max))), + Filter::RunLength => once(counted(runlength::decode(data, max))), + other => (Err(Failure { error: FilterError::Unsupported(other.name().to_owned()), partial: Vec::new(), discarded: 0 }), 0, false), } } @@ -552,7 +588,7 @@ mod tests { let data = encode_flate(&hex); let chain = [(Filter::Flate, Params::default()), (Filter::AsciiHex, Params::default())]; let whole = decode(&chain, &data, 1 << 20).unwrap(); - let cost = (data.len() + hex.len() + whole.len()) as u64; + let cost = (data.len() + hex.len() + whole.len()) as u64 + FLATE_READ_WORK * data.len() as u64; let mut work = u64::MAX; assert_eq!(decode_tolerant_within(&chain, &data, 1 << 20, &mut work).unwrap(), whole); assert_eq!(u64::MAX - work, cost); @@ -584,8 +620,11 @@ mod tests { assert_eq!(work, 0); super::STAGE_BYTES.with(|c| c.get()) }; - assert_eq!(read((bomb.len() + inner.len()) as u64 + 10), bomb.len() + inner.len(), "the second stage stops"); - assert_eq!(read(bomb.len() as u64 + 10), bomb.len(), "the first stops, and the second never runs"); + let paid = |n: usize| (1 + FLATE_READ_WORK) * n as u64; + assert_eq!(read(paid(bomb.len()) + paid(inner.len()) + 10), bomb.len() + inner.len(), "the second stage stops"); + assert_eq!(read(paid(bomb.len()) + FLATE_READ_WORK * inner.len() as u64 - 1), bomb.len(), "the second can't pay to read"); + assert_eq!(read(paid(bomb.len()) + 10), bomb.len(), "the first stops, and the second never runs"); + assert_eq!(read(paid(bomb.len()) - 1), 0, "the first can't pay to read"); assert_eq!(read(bomb.len() as u64 - 1), 0, "it can't pay to be read"); } @@ -597,18 +636,19 @@ mod tests { let chain = [(Filter::Flate, Params::default())]; let mut work = u64::MAX; assert!(matches!(decode_tolerant_within(&chain, &data, 1000, &mut work), Err(FilterError::LimitExceeded(1000)))); - assert_eq!(u64::MAX - work, data.len() as u64 + 1000); + let read = |n: usize| FLATE_READ_WORK * n as u64; + assert_eq!(u64::MAX - work, data.len() as u64 + read(data.len()) + 1000); // The later stage of two, which pays for the first as well. let bomb = encode_flate(&data); let two = [(Filter::Flate, Params::default()), (Filter::Flate, Params::default())]; let mut work = u64::MAX; assert!(matches!(decode_tolerant_within(&two, &bomb, data.len(), &mut work), Err(FilterError::LimitExceeded(_)))); - assert_eq!(u64::MAX - work, (bomb.len() + 2 * data.len()) as u64); + assert_eq!(u64::MAX - work, (bomb.len() + 2 * data.len()) as u64 + read(bomb.len()) + read(data.len())); // Both limits at once: a run that can't go on is what a caller needs to hear. - let mut work = data.len() as u64 + 1000; + let mut work = data.len() as u64 + read(data.len()) + 1000; assert!(refused(&decode_tolerant_within(&chain, &data, 1000, &mut work).unwrap_err())); assert_eq!(work, 0); - let mut work = data.len() as u64 + 1001; + let mut work = data.len() as u64 + read(data.len()) + 1001; assert!(matches!(decode_tolerant_within(&chain, &data, 1000, &mut work), Err(FilterError::LimitExceeded(1000)))); assert_eq!(work, 1, "the image's limit, with a step to spare"); } @@ -624,21 +664,22 @@ mod tests { assert!(!partial.is_empty()); let mut work = u64::MAX; assert_eq!(decode_tolerant_within(&chain, &cut, 1 << 20, &mut work).unwrap(), partial); - assert_eq!(u64::MAX - work, (cut.len() + partial.len()) as u64); + let read = |n: usize| FLATE_READ_WORK * n as u64; + assert_eq!(u64::MAX - work, (cut.len() + partial.len()) as u64 + read(cut.len())); // Read strictly it is an error, for the same work. let mut work = u64::MAX; assert!(matches!(decode_within(&chain, &cut, 1 << 20, &mut work), Err(FilterError::Corrupt { .. }))); - assert_eq!(u64::MAX - work, (cut.len() + partial.len()) as u64); + assert_eq!(u64::MAX - work, (cut.len() + partial.len()) as u64 + read(cut.len())); let whole = encode_flate(&data); let mut work = u64::MAX; assert_eq!(decode_within(&chain, &whole, 1 << 20, &mut work).unwrap(), data); - assert_eq!(u64::MAX - work, (whole.len() + data.len()) as u64); + assert_eq!(u64::MAX - work, (whole.len() + data.len()) as u64 + read(whole.len())); // A predictor that comes to nothing, after all of this was inflated for it. let p = Params { predictor: 99, ..Params::default() }; let data = encode_flate(&vec![0; 1 << 16]); let mut work = u64::MAX; assert!(decode_tolerant_within(&[(Filter::Flate, p)], &data, 1 << 20, &mut work).unwrap().is_empty()); - assert_eq!(u64::MAX - work, data.len() as u64 + (1 << 16)); + assert_eq!(u64::MAX - work, data.len() as u64 + read(data.len()) + (1 << 16)); } /// A stream that reads as zlib only until it fails, and as raw deflate until that fails too, @@ -660,13 +701,67 @@ mod tests { assert_eq!(spent, 0x0601); let mut work = u64::MAX; assert!(decode_tolerant_within(&chain, &data, 1 << 20, &mut work).unwrap().is_empty()); - assert_eq!(u64::MAX - work, (data.len() + 0x0601) as u64); + // Read twice: as zlib, then as raw deflate. + assert_eq!(u64::MAX - work, (data.len() + 0x0601) as u64 + 2 * FLATE_READ_WORK * data.len() as u64); // Through a predictor, which would otherwise lose the count. let p = Params { predictor: 12, columns: 4, ..Params::default() }; let (_, _, spent) = decode_tolerant_counted(&[(Filter::Flate, p)], &data, 1 << 20).unwrap(); assert_eq!(spent, 0x0601); } + /// Inflating does work for every block it reads, whether or not the block makes anything: empty + /// fixed-Huffman blocks, ten bits each, make nothing however many there are. Each byte of them + /// is paid for at Flate's price before any is read. + #[test] + fn a_flate_stage_pays_for_what_it_reads() { + // Four empty blocks in five bytes, as raw deflate, then an empty final block. + let mut hollow = [0x02, 0x08, 0x20, 0x80, 0x00].repeat(1 << 12); + hollow.extend([0x03, 0x00]); + let chain = [(Filter::Flate, Params::default())]; + assert_eq!(decode(&chain, &hollow, 1 << 20).unwrap(), b""); + let cost = (1 + FLATE_READ_WORK) * hollow.len() as u64; + let mut work = u64::MAX; + assert_eq!(decode_tolerant_within(&chain, &hollow, 1 << 20, &mut work).unwrap(), b""); + assert_eq!(u64::MAX - work, cost); + let mut work = cost; + assert!(decode_tolerant_within(&chain, &hollow, 1 << 20, &mut work).is_ok(), "exactly enough"); + assert_eq!(work, 0); + super::STAGE_BYTES.with(|c| c.set(0)); + let mut work = cost - 1; + assert!(refused(&decode_tolerant_within(&chain, &hollow, 1 << 20, &mut work).unwrap_err())); + assert_eq!((work, super::STAGE_BYTES.with(|c| c.get())), (0, 0), "nothing was inflated"); + // Counted for a byte allowance, as a page's content is, reading costs nothing. + assert_eq!(decode_counted(&chain, &hollow, 1 << 20).unwrap(), (Vec::new(), 0)); + } + + /// A zlib header that only looked valid is read again as raw deflate, and the second reading + /// is paid for as the first was. One that can't be paid for is refused, not read as damaged. + #[test] + fn a_second_reading_is_paid_for() { + // As zlib: a block of a reserved type at once. As raw deflate: a stored block of this many + // zero bytes, then an empty final block. + let mut data = vec![0x78, 0x01, 0x06, 0xFE, 0xF9]; + data.extend(vec![0u8; 0x0601]); + data.extend([0x03, 0x00]); + let chain = [(Filter::Flate, Params::default())]; + assert_eq!(decode(&chain, &data, 1 << 20).unwrap(), vec![0u8; 0x0601]); + let read = FLATE_READ_WORK * data.len() as u64; + let mut work = (data.len() + 0x0601) as u64 + 2 * read; + assert_eq!(decode_tolerant_within(&chain, &data, 1 << 20, &mut work).unwrap(), vec![0u8; 0x0601]); + assert_eq!(work, 0, "exactly enough"); + // Enough to read it once and not twice: it isn't read again. + super::flate::INFLATED.with(|c| c.set(0)); + let mut work = data.len() as u64 + 2 * read - 1; + assert!(refused(&decode_tolerant_within(&chain, &data, 1 << 20, &mut work).unwrap_err())); + assert_eq!((work, super::flate::INFLATED.with(|c| c.get())), (0, 0)); + // Enough to read it twice, and a byte short of what it makes: the second reading makes no + // more than is left once it has been paid for. + super::flate::INFLATED.with(|c| c.set(0)); + let mut work = (data.len() + 0x0601) as u64 + 2 * read - 1; + assert!(refused(&decode_tolerant_within(&chain, &data, 1 << 20, &mut work).unwrap_err())); + assert!(super::flate::INFLATED.with(|c| c.get()) < 0x0601, "it stopped short"); + } + /// With nothing left nothing is read. What can't be decoded at all never costs anything, and /// says so whatever is left. Without filters, what is read is the data. #[test] diff --git a/crates/optimize/src/fragments.rs b/crates/optimize/src/fragments.rs index f74b091984..b5662a2bb6 100644 --- a/crates/optimize/src/fragments.rs +++ b/crates/optimize/src/fragments.rs @@ -394,8 +394,8 @@ mod tests { let mut budget = ImageBudget::with_work(0); assert_eq!(merge_page(&mut doc.clone(), page, &mut Report::default(), &mut |_| true, &mut budget).unwrap(), 0); assert_eq!(budget.refused(), 1); - // Reading each strip, and building the picture of 8 rows of 8 they make. - let cost = raw + 8 * (8 + 8) + 8 * 8; + // Reading each strip, inflating it, and building the picture of 8 rows of 8 they make. + let cost = (1 + pdfcraft_filters::FLATE_READ_WORK) * raw + 8 * (8 + 8) + 8 * 8; let mut budget = ImageBudget::with_work(u64::MAX); assert_eq!(merge_page(&mut doc.clone(), page, &mut Report::default(), &mut |_| true, &mut budget).unwrap(), 8); assert_eq!(u64::MAX - budget.work_left(), cost); diff --git a/crates/optimize/src/images.rs b/crates/optimize/src/images.rs index 92e68582c2..3ad9409f7d 100644 --- a/crates/optimize/src/images.rs +++ b/crates/optimize/src/images.rs @@ -1515,7 +1515,7 @@ mod dimension_tests { dict.set(b"BitsPerComponent".to_vec(), Object::Int(8)); dict.set(b"ColorSpace".to_vec(), Object::name("DeviceGray")); let image = Stream::flate(dict, &samples); - let read = (image.raw.len() + samples.len()) as u64 + 256 * 256; + let read = (image.raw.len() + samples.len()) as u64 + pdfcraft_filters::FLATE_READ_WORK * image.raw.len() as u64 + 256 * 256; let original = doc.add(Object::Stream(image.clone())); let mut xobjects = Dict::new(); xobjects.set(b"I0".to_vec(), Object::Ref(original)); @@ -1607,7 +1607,7 @@ mod dimension_tests { }; let mut doc = Document::new_empty(); let mask = image("DeviceGray"); - let cost = (mask.raw.len() + noise.len()) as u64 + 256 * 256; + let cost = (mask.raw.len() + noise.len()) as u64 + pdfcraft_filters::FLATE_READ_WORK * mask.raw.len() as u64 + 256 * 256; let masked = doc.add(Object::Stream(mask)); let mut with_mask = image("DeviceGray"); with_mask.dict.set(b"SMask".to_vec(), Object::Ref(masked));