mirror of
https://github.com/storytold/craft-libs.git
synced 2026-11-14 06:22:24 +00:00
craft-pdf: the PDF filter chain from PdfCraft (filters), on craft-stream-codecs (#1033)
* Add craft-stream-codecs: the codec modules of PdfCraft's stream filters LZW, ASCII85, ASCIIHex, RunLength, the TIFF and PNG predictors, bounded CCITT fax decoding (through hayro-ccitt) with Group 4 encoding (through fax), and the lossless JBIG2 encoder with its MQ coder, from pdfcraft-filters at PdfCraft revision 80e26e674d1aa33078e83c8139875047f3289eb6. The crate sits below craft-pdf and craft-tiff: it knows no container. The PDF filter chain, Flate and the bounded JBIG2 decoding entry are not part of it. * craft-pdf: the PDF filter chain from PdfCraft (`filters`) Adds the `filters` feature: the /Filter chain of pdfcraft-filters (PdfCraft revision 80e26e674d1aa33078e83c8139875047f3289eb6) as craft_pdf::filters, over the codecs in craft-stream-codecs. It adds Flate, strict and tolerant chain decoding, byte accounting and a shared work allowance, and encoding, with the chain's unit tests, its property tests and its fuzz target. The code is unchanged apart from the imports of the moved codecs; FilterError is craft-stream-codecs' CodecError under its old name.
This commit is contained in:
1 parent
a7a00c7b0a
commit
97f4971666
12 files changed
+1427
No files matched your search
Generated
+2
@@ -1561,6 +1561,8 @@ version = "0.1.0"
|
||||
dependencies = [
|
||||
"aes",
|
||||
"bytemuck",
|
||||
"craft-stream-codecs",
|
||||
"flate2",
|
||||
"hayro",
|
||||
"hayro-cmap",
|
||||
"hayro-interpret",
|
||||
|
||||
@@ -18,6 +18,7 @@ raster = ["interpret", "dep:hayro", "dep:bytemuck"]
|
||||
write = ["dep:krilla", "dep:thiserror"]
|
||||
write-pdf = ["write", "krilla/pdf"]
|
||||
write-simple-text = ["write", "krilla/simple-text"]
|
||||
filters = ["dep:craft-stream-codecs", "dep:flate2", "dep:thiserror"]
|
||||
crypt = ["dep:aes", "dep:md-5", "dep:sha2", "dep:stringprep", "dep:zeroize", "dep:thiserror"]
|
||||
|
||||
[dependencies]
|
||||
@@ -27,6 +28,9 @@ hayro = { workspace = true, optional = true }
|
||||
bytemuck = { workspace = true, optional = true }
|
||||
krilla = { workspace = true, optional = true }
|
||||
thiserror = { workspace = true, optional = true }
|
||||
# The filter chain (`filters`). flate2 with zlib-rs, the backend the chain's Flate prices were measured on.
|
||||
craft-stream-codecs = { workspace = true, optional = true }
|
||||
flate2 = { version = "1.1", default-features = false, features = ["zlib-rs"], optional = true }
|
||||
# The standard security handler (`crypt`): pure Rust, MIT OR Apache-2.0, builds for wasm.
|
||||
aes = { version = "0.9", optional = true }
|
||||
md-5 = { version = "0.11", optional = true }
|
||||
|
||||
@@ -12,6 +12,7 @@ No feature is enabled by default.
|
||||
| `write` | Guarded Krilla authoring; no shared reader dependency |
|
||||
| `write-pdf` | `write` plus Krilla's embedded-PDF support |
|
||||
| `write-simple-text` | `write` plus Krilla's optional text shaper and `draw_text` |
|
||||
| `filters` | The `/Filter` chain for editing and writing PDF files: decode and encode, with limits and accounting. No reader or writer dependency |
|
||||
| `crypt` | The standard security handler: decrypt, encrypt and create encryption. No reader or writer dependency |
|
||||
|
||||
## Reading and rendering
|
||||
@@ -110,6 +111,33 @@ read-only consumer, then repeat with `--target wasm32-unknown-unknown` and the t
|
||||
These backend checks supplement the root workspace gates; excluded vendor crates are not
|
||||
covered by `cargo test --workspace` alone.
|
||||
|
||||
## Stream filters
|
||||
|
||||
`filters` is the `/Filter` chain of a PDF stream (ISO 32000-2 §7.4) for code that edits or
|
||||
writes PDF files. The reader decodes streams on its own and does not use it. The codecs are
|
||||
[`craft-stream-codecs`](../stream-codecs/README.md); this module adds Flate (flate2 with its
|
||||
zlib-rs backend), the chain and its accounting.
|
||||
|
||||
- `decode(chain, data, max_output)` runs the general-purpose filters first to last (Flate,
|
||||
LZW, ASCIIHex, ASCII85, RunLength, with the TIFF and PNG predictors). Image codecs, `Crypt`,
|
||||
unknown filters and a chain longer than `MAX_CHAIN` (8) give `Unsupported`, so the caller
|
||||
keeps such data encoded. `max_output` bounds every stage and gives `LimitExceeded`.
|
||||
- `decode_tolerant` returns what was decoded before damage, with `partial = true`, and runs the
|
||||
rest of the chain on it.
|
||||
- `decode_counted`, `decode_tolerant_counted` and `decode_tolerant_counted_each` also return the
|
||||
bytes every stage decoded, on success and on failure, for a caller charging a budget.
|
||||
- `decode_within`, `decode_tolerant_within` and `decode_image_tolerant_within` spend from a
|
||||
caller-owned `work` allowance shared by a run of images, as `decode_ccitt_within` does; what
|
||||
`work` cannot pay for gives `Spent`. Flate pays `FLATE_READ_WORK` steps a byte it reads.
|
||||
- `encode(filter, params, data)` writes Flate (zlib level 6), LZW, ASCIIHex, ASCII85 and
|
||||
RunLength, with a predictor first for Flate and LZW. `encode_g4`, `encode_jbig2`,
|
||||
`encode_jbig2_generic` and `encode_jbig2_shared` code 1-bit images.
|
||||
|
||||
The code comes from PdfCraft's `pdfcraft-filters` crate (revision
|
||||
`80e26e674d1aa33078e83c8139875047f3289eb6`): its `lib.rs` and `flate.rs` here, its codecs in
|
||||
`craft-stream-codecs`. Tests: `cargo test -p craft-pdf --no-default-features --features filters`.
|
||||
A fuzz target is in `crates/pdf/fuzz` (`cargo +nightly fuzz run filters`, from `crates/pdf`).
|
||||
|
||||
## Encryption
|
||||
|
||||
`crypt` is the PDF standard security handler (ISO 32000-2 §7.6) for code that edits or writes
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
target/
|
||||
corpus/
|
||||
artifacts/
|
||||
coverage/
|
||||
Cargo.lock
|
||||
@@ -0,0 +1,23 @@
|
||||
[package]
|
||||
name = "craft-pdf-fuzz"
|
||||
version = "0.0.0"
|
||||
publish = false
|
||||
edition = "2024"
|
||||
|
||||
[package.metadata]
|
||||
cargo-fuzz = true
|
||||
|
||||
[dependencies]
|
||||
libfuzzer-sys = "0.4"
|
||||
craft-pdf = { path = "..", default-features = false, features = ["filters"] }
|
||||
|
||||
# Its own workspace: cargo-fuzz needs nightly and sanitizer flags, so this crate stays out of the
|
||||
# craft-libs workspace and its lockfile.
|
||||
[workspace]
|
||||
|
||||
[[bin]]
|
||||
name = "filters"
|
||||
path = "fuzz_targets/filters.rs"
|
||||
test = false
|
||||
doc = false
|
||||
bench = false
|
||||
@@ -0,0 +1,32 @@
|
||||
//! Feeds arbitrary bytes to every decoder, with parameters taken from the first bytes.
|
||||
#![no_main]
|
||||
|
||||
use libfuzzer_sys::fuzz_target;
|
||||
use craft_pdf::filters::{Filter, Params, decode, decode_tolerant};
|
||||
|
||||
const FILTERS: [Filter; 5] = [Filter::Flate, Filter::Lzw, Filter::AsciiHex, Filter::Ascii85, Filter::RunLength];
|
||||
const PREDICTORS: [i64; 8] = [1, 2, 10, 11, 12, 13, 14, 15];
|
||||
const BPC: [i64; 5] = [1, 2, 4, 8, 16];
|
||||
|
||||
fuzz_target!(|data: &[u8]| {
|
||||
let Some((head, body)) = data.split_first_chunk::<4>() else { return };
|
||||
// Up to two chained filters; the second only when head[0] says so.
|
||||
let params = Params {
|
||||
predictor: PREDICTORS[usize::from(head[1] & 7)],
|
||||
colors: i64::from(head[1] >> 3 & 3) + 1,
|
||||
bits_per_component: BPC[usize::from(head[2]) % BPC.len()],
|
||||
columns: i64::from(head[3]) + 1,
|
||||
early_change: i64::from(head[2] >> 7),
|
||||
};
|
||||
let mut chain = vec![(FILTERS[usize::from(head[0] & 7) % FILTERS.len()].clone(), params.clone())];
|
||||
if head[0] & 0x80 != 0 {
|
||||
chain.insert(0, (FILTERS[usize::from(head[0] >> 3 & 7) % FILTERS.len()].clone(), Params::default()));
|
||||
}
|
||||
let max = 1 << 22;
|
||||
if let Ok(out) = decode(&chain, body, max) {
|
||||
assert!(out.len() <= max);
|
||||
}
|
||||
if let Ok((out, _)) = decode_tolerant(&chain, body, max) {
|
||||
assert!(out.len() <= max);
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,865 @@
|
||||
//! PDF stream filters (ISO 32000-2 §7.4): the `/Filter` chain of a stream, for code that edits
|
||||
//! or writes PDF files. The reader (`read`) decodes streams on its own.
|
||||
//!
|
||||
//! Decoding covers the general-purpose filters (`FlateDecode`, `LZWDecode` with
|
||||
//! TIFF/PNG predictors, `ASCIIHexDecode`, `ASCII85Decode`, `RunLengthDecode`).
|
||||
//! Image codecs (`DCTDecode`, `JPXDecode`, `JBIG2Decode`, `CCITTFaxDecode`),
|
||||
//! `Crypt` and unknown filters are reported as [`FilterError::Unsupported`] so
|
||||
//! callers keep such data encoded. CCITT fax images decode on their own
|
||||
//! ([`decode_ccitt_within`]), spending from an allowance the caller shares across a run of
|
||||
//! images.
|
||||
//!
|
||||
//! The codecs themselves are `craft-stream-codecs`, re-exported here; this module adds Flate,
|
||||
//! the chain (stage order, limits across stages, strict and tolerant decoding) and its
|
||||
//! accounting. 1-bit samples can be encoded as CCITT Group 4 ([`encode_g4`]) or lossless JBIG2
|
||||
//! ([`encode_jbig2`]), a group of images sharing their marks through `/JBIG2Globals`
|
||||
//! ([`encode_jbig2_shared`]).
|
||||
//!
|
||||
//! Every decoder is bounded by a caller-supplied `max_output` (decompression-bomb
|
||||
//! defence) and never panics on malformed input.
|
||||
|
||||
mod flate;
|
||||
|
||||
use std::borrow::Cow;
|
||||
|
||||
pub use craft_stream_codecs::{
|
||||
CcittParams, CodecError as FilterError, MAX_CCITT_COLUMNS, MAX_G4_COLUMNS, MAX_JBIG2_COLUMNS, MAX_JBIG2_PIXELS, Params, SharedJbig2,
|
||||
decode_ccitt_within, encode_g4, encode_jbig2, encode_jbig2_generic, encode_jbig2_shared,
|
||||
};
|
||||
use craft_stream_codecs::{Failure, Step, pay};
|
||||
use craft_stream_codecs::{ascii85, asciihex, lzw, predictor, runlength};
|
||||
|
||||
/// A PDF stream filter, identified by its `/Filter` name.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Hash)]
|
||||
pub enum Filter {
|
||||
Flate,
|
||||
Lzw,
|
||||
AsciiHex,
|
||||
Ascii85,
|
||||
RunLength,
|
||||
Dct,
|
||||
Jpx,
|
||||
Jbig2,
|
||||
CcittFax,
|
||||
Crypt,
|
||||
/// Any other name (kept verbatim, lossily converted to UTF-8).
|
||||
Unknown(String),
|
||||
}
|
||||
|
||||
impl Filter {
|
||||
/// Parses a filter name. Accepts the full names (ISO 32000-2 Table 6) and the
|
||||
/// inline-image abbreviations (Table 92): `Fl`, `LZW`, `AHx`, `A85`, `RL`, `DCT`, `CCF`.
|
||||
/// A leading `/` is ignored.
|
||||
pub fn from_name(name: &[u8]) -> Filter {
|
||||
let n = name.strip_prefix(b"/").unwrap_or(name);
|
||||
match n {
|
||||
b"FlateDecode" | b"Fl" => Filter::Flate,
|
||||
b"LZWDecode" | b"LZW" => Filter::Lzw,
|
||||
b"ASCIIHexDecode" | b"AHx" => Filter::AsciiHex,
|
||||
b"ASCII85Decode" | b"A85" => Filter::Ascii85,
|
||||
b"RunLengthDecode" | b"RL" => Filter::RunLength,
|
||||
b"DCTDecode" | b"DCT" => Filter::Dct,
|
||||
b"JPXDecode" => Filter::Jpx,
|
||||
b"JBIG2Decode" => Filter::Jbig2,
|
||||
b"CCITTFaxDecode" | b"CCF" => Filter::CcittFax,
|
||||
b"Crypt" => Filter::Crypt,
|
||||
other => Filter::Unknown(String::from_utf8_lossy(other).into_owned()),
|
||||
}
|
||||
}
|
||||
|
||||
/// The full PDF name (without the leading `/`).
|
||||
pub fn name(&self) -> &str {
|
||||
match self {
|
||||
Filter::Flate => "FlateDecode",
|
||||
Filter::Lzw => "LZWDecode",
|
||||
Filter::AsciiHex => "ASCIIHexDecode",
|
||||
Filter::Ascii85 => "ASCII85Decode",
|
||||
Filter::RunLength => "RunLengthDecode",
|
||||
Filter::Dct => "DCTDecode",
|
||||
Filter::Jpx => "JPXDecode",
|
||||
Filter::Jbig2 => "JBIG2Decode",
|
||||
Filter::CcittFax => "CCITTFaxDecode",
|
||||
Filter::Crypt => "Crypt",
|
||||
Filter::Unknown(s) => s,
|
||||
}
|
||||
}
|
||||
|
||||
/// True for the image codecs (DCT, JPX, JBIG2, CCITT fax).
|
||||
pub fn is_image_codec(&self) -> bool {
|
||||
matches!(self, Filter::Dct | Filter::Jpx | Filter::Jbig2 | Filter::CcittFax)
|
||||
}
|
||||
|
||||
/// True for the general-purpose filters, which a chain can hold ([`decode`]).
|
||||
pub fn is_decodable(&self) -> bool {
|
||||
matches!(self, Filter::Flate | Filter::Lzw | Filter::AsciiHex | Filter::Ascii85 | Filter::RunLength)
|
||||
}
|
||||
}
|
||||
|
||||
/// Decodes `data` through `chain` (first element applied first).
|
||||
///
|
||||
/// Image codecs, `Crypt` and unknown filters anywhere in the chain give
|
||||
/// [`FilterError::Unsupported`] (callers keep such data encoded), as does a chain longer than
|
||||
/// [`MAX_CHAIN`]. `max_output` bounds every intermediate buffer.
|
||||
pub fn decode(chain: &[(Filter, Params)], data: &[u8], max_output: usize) -> Result<Vec<u8>, FilterError> {
|
||||
run(chain, data, max_output, None, true, false, None).map(|(v, _, _)| v).map_err(|(e, _)| e)
|
||||
}
|
||||
|
||||
/// Like [`decode`] for a caller charging an allowance: `max_output` bounds the bytes all stages
|
||||
/// decode together, each stage getting what the earlier ones left, and the result gives those
|
||||
/// bytes, on success and on failure. A stage's output counts even when a later stage or its
|
||||
/// predictor shrinks it.
|
||||
pub fn decode_counted(chain: &[(Filter, Params)], data: &[u8], max_output: usize) -> Result<(Vec<u8>, usize), (FilterError, usize)> {
|
||||
run(chain, data, max_output, Some(max_output), true, false, None).map(|(v, _, spent)| (v, spent))
|
||||
}
|
||||
|
||||
/// Like [`decode`], but on corrupt data returns what was decoded before the error and
|
||||
/// reports `partial = true` (the behaviour viewers expect). The remaining filters of the
|
||||
/// chain still run on the partial data. Limit and unsupported errors are still errors.
|
||||
pub fn decode_tolerant(chain: &[(Filter, Params)], data: &[u8], max_output: usize) -> Result<(Vec<u8>, bool), FilterError> {
|
||||
run(chain, data, max_output, None, false, false, None).map(|(v, partial, _)| (v, partial)).map_err(|(e, _)| e)
|
||||
}
|
||||
|
||||
/// [`decode_tolerant`] bounded and counted as [`decode_counted`] is.
|
||||
pub fn decode_tolerant_counted(chain: &[(Filter, Params)], data: &[u8], max_output: usize) -> Result<(Vec<u8>, bool, usize), (FilterError, usize)> {
|
||||
run(chain, data, max_output, Some(max_output), false, false, None)
|
||||
}
|
||||
|
||||
/// [`decode_tolerant_counted`] with the two bounds apart: `max_stage` bounds every stage, as
|
||||
/// [`decode_tolerant`]'s bound does, and `max_output` the bytes all stages decode together. A
|
||||
/// stage stopped by either gives [`FilterError::LimitExceeded`] and the bytes decoded before it.
|
||||
pub fn decode_tolerant_counted_each(
|
||||
chain: &[(Filter, Params)],
|
||||
data: &[u8],
|
||||
max_stage: usize,
|
||||
max_output: usize,
|
||||
) -> Result<(Vec<u8>, bool, usize), (FilterError, usize)> {
|
||||
run(chain, data, max_stage, Some(max_output), false, false, None)
|
||||
}
|
||||
|
||||
/// [`decode_tolerant`], spending from `work`, what decoding may still cost across a run of images:
|
||||
/// a step for each byte of `data`, paid before anything is decoded, [`FLATE_READ_WORK`] for each
|
||||
/// byte a Flate stage reads, paid before it runs (again if it reads them twice), and one for each
|
||||
/// byte a stage decodes, so a chain that fails, or a stage that goes past `max_output`, has paid for
|
||||
/// what it did. No stage decodes past what is left: a chain that `work` can't pay for spends all of
|
||||
/// it and stops there, so with nothing left nothing is read. A chain that can't be decoded at all
|
||||
/// costs nothing, whatever is left, so what an image's filters are is known from the error.
|
||||
///
|
||||
/// Unlike [`decode_tolerant`], no filters at all gives `data` itself, within `max_output`.
|
||||
pub fn decode_tolerant_within(chain: &[(Filter, Params)], data: &[u8], max_output: usize, work: &mut u64) -> Result<Vec<u8>, FilterError> {
|
||||
spending(chain, data, max_output, work, false, None)
|
||||
}
|
||||
|
||||
/// [`decode_tolerant_within`] for the samples of an image `rows` high (its `/Height`), which the
|
||||
/// last stage gives. When that stage is Flate with a PNG predictor and gives exactly `rows` rows
|
||||
/// without the filter-type byte each predicted row starts with (ISO 32000-2 §7.4.4.4), its writer
|
||||
/// declared the predictor without applying it: the output is taken as the samples, as other
|
||||
/// readers take it, and reported as the damage it is, like any other corruption tolerated here.
|
||||
/// Any other output has its predictor undone as [`decode_tolerant_within`] does.
|
||||
pub fn decode_image_tolerant_within(
|
||||
chain: &[(Filter, Params)],
|
||||
data: &[u8],
|
||||
max_output: usize,
|
||||
rows: u32,
|
||||
work: &mut u64,
|
||||
) -> Result<Vec<u8>, FilterError> {
|
||||
spending(chain, data, max_output, work, false, usize::try_from(rows).ok())
|
||||
}
|
||||
|
||||
/// [`decode`], spending from `work` as [`decode_tolerant_within`] does: corrupt data is an error,
|
||||
/// having paid for what was decoded before it.
|
||||
pub fn decode_within(chain: &[(Filter, Params)], data: &[u8], max_output: usize, work: &mut u64) -> Result<Vec<u8>, FilterError> {
|
||||
spending(chain, data, max_output, work, true, None)
|
||||
}
|
||||
|
||||
fn spending(
|
||||
chain: &[(Filter, Params)],
|
||||
data: &[u8],
|
||||
max_output: usize,
|
||||
work: &mut u64,
|
||||
strict: bool,
|
||||
rows: Option<usize>,
|
||||
) -> Result<Vec<u8>, FilterError> {
|
||||
decodable(chain)?;
|
||||
pay(work, data.len() as u64)?;
|
||||
if chain.is_empty() {
|
||||
return if data.len() > max_output { Err(FilterError::LimitExceeded(max_output)) } else { Ok(data.to_vec()) };
|
||||
}
|
||||
let left = usize::try_from(*work).unwrap_or(usize::MAX);
|
||||
match run(chain, data, max_output, Some(left), strict, true, rows) {
|
||||
Ok((out, _, decoded)) => pay(work, decoded as u64).map(|()| out),
|
||||
// The stage stopped at the lower of the two, which took as much to find out.
|
||||
Err((FilterError::LimitExceeded(_), decoded)) if left.saturating_sub(decoded) > max_output => {
|
||||
pay(work, (decoded as u64).saturating_add(max_output as u64))?;
|
||||
Err(FilterError::LimitExceeded(max_output))
|
||||
}
|
||||
// Past what is left, a stage stops, or a Flate stage doesn't start.
|
||||
Err((FilterError::LimitExceeded(_) | FilterError::Spent, _)) => {
|
||||
*work = 0;
|
||||
Err(FilterError::Spent)
|
||||
}
|
||||
Err((e, decoded)) => pay(work, decoded as u64).and(Err(e)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Longest filter chain decoded. Real streams use one to three filters; every stage may produce
|
||||
/// up to the output limit, so the chain length multiplies the work.
|
||||
pub const MAX_CHAIN: usize = 8;
|
||||
|
||||
/// The steps each byte Flate reads costs where decoding is paid for ([`decode_tolerant_within`],
|
||||
/// [`decode_within`]), besides the step it was paid as input and a step for each byte it makes.
|
||||
/// Inflating does work for every block it reads, whether or not the block makes anything: an empty
|
||||
/// fixed-Huffman block is ten bits (RFC 1951 §3.2.6). With flate2's zlib-rs backend (zlib-rs
|
||||
/// 0.6.8, release build) the slowest blocks measured, empty dynamic ones, took 21 ns a byte read
|
||||
/// to make nothing, empty fixed ones 9 and fixed ones of a literal each 6, so at this price the
|
||||
/// worst per byte read comes to about 7 ns a step, inside what a step of JBIG2 decoding takes.
|
||||
/// (miniz_oxide 0.9.1, flate2's backend here before, took about 1.7 µs for every fixed-Huffman
|
||||
/// block, empty or not: 1,383 ns a byte read on empty ones.) A stage's start isn't priced, about
|
||||
/// 0.25 µs: every way to repeat one takes bytes of the file, so what starts cost is bounded by the
|
||||
/// file's length. Data a real encoder makes took 0.2 to 2.3 ns a byte made, its reading included, so
|
||||
/// an image pays twice its stored length more than it would unpriced.
|
||||
pub const FLATE_READ_WORK: u64 = 2;
|
||||
|
||||
/// Whether a chain is one [`run`] takes, and if not, the error.
|
||||
fn decodable(chain: &[(Filter, Params)]) -> Result<(), FilterError> {
|
||||
if chain.len() > MAX_CHAIN {
|
||||
return Err(FilterError::Unsupported(format!("a chain of {} filters", chain.len())));
|
||||
}
|
||||
match chain.iter().find(|(f, _)| !f.is_decodable()) {
|
||||
Some((f, _)) => Err(FilterError::Unsupported(f.name().to_owned())),
|
||||
None => Ok(()),
|
||||
}
|
||||
}
|
||||
|
||||
/// The decoded data, whether it is partial and the bytes all stages decoded, or the error and
|
||||
/// the bytes decoded before it. `max` bounds each stage, and `total` all of them together. With
|
||||
/// `priced`, the count is steps: each Flate stage also pays [`FLATE_READ_WORK`] for each byte it
|
||||
/// reads, before it runs, so a stage `total` can't pay for doesn't run
|
||||
/// ([`FilterError::Spent`]). `rows` is the height of the image whose samples the last stage
|
||||
/// produces, if it does ([`decode_image_tolerant_within`]).
|
||||
fn run(
|
||||
chain: &[(Filter, Params)],
|
||||
data: &[u8],
|
||||
max: usize,
|
||||
total: Option<usize>,
|
||||
strict: bool,
|
||||
priced: bool,
|
||||
rows: Option<usize>,
|
||||
) -> Result<(Vec<u8>, bool, usize), (FilterError, usize)> {
|
||||
decodable(chain).map_err(|e| (e, 0))?;
|
||||
let mut cur: Cow<[u8]> = Cow::Borrowed(data);
|
||||
let mut partial = false;
|
||||
let mut spent = 0usize;
|
||||
let price = usize::try_from(FLATE_READ_WORK).unwrap_or(usize::MAX);
|
||||
for (i, (filter, params)) in chain.iter().enumerate() {
|
||||
let read = if priced && matches!(filter, Filter::Flate) { price.saturating_mul(cur.len()) } else { 0 };
|
||||
let room = total.map_or(usize::MAX, |total| total.saturating_sub(spent));
|
||||
if read > room {
|
||||
return Err((FilterError::Spent, spent));
|
||||
}
|
||||
spent = spent.saturating_add(read);
|
||||
#[cfg(test)]
|
||||
STAGE_BYTES.with(|c| c.set(c.get() + cur.len()));
|
||||
let limit = total.map_or(max, |total| max.min(total.saturating_sub(spent)));
|
||||
// A second reading of the same bytes (Flate's raw retry) is paid for as the first was, and
|
||||
// makes no more than is left once it has been.
|
||||
let rows = rows.filter(|_| i + 1 == chain.len());
|
||||
let again = room.checked_sub(read.saturating_mul(2)).map(|left| max.min(left));
|
||||
let (step, work, again) = decode_one(filter, params, &cur, limit, again, rows);
|
||||
if again {
|
||||
spent = spent.saturating_add(read);
|
||||
}
|
||||
spent = spent.saturating_add(work);
|
||||
cur = Cow::Owned(match step {
|
||||
Ok(v) => v,
|
||||
Err(Failure { error: FilterError::Corrupt { .. }, partial: out, .. }) if !strict => {
|
||||
partial = true;
|
||||
out
|
||||
}
|
||||
Err(Failure { error: FilterError::LimitExceeded(_), .. }) => return Err((FilterError::LimitExceeded(max), spent)),
|
||||
Err(f) => return Err((f.error, spent)),
|
||||
});
|
||||
}
|
||||
Ok((cur.into_owned(), partial, spent))
|
||||
}
|
||||
|
||||
/// One stage's result, the bytes its decoder produced (before any predictor), and whether it
|
||||
/// read its input twice, which it does only within `again` ([`flate::decode`]); `rows` as for
|
||||
/// [`run`].
|
||||
fn decode_one(filter: &Filter, params: &Params, data: &[u8], max: usize, again: Option<usize>, rows: Option<usize>) -> (Step, usize, bool) {
|
||||
let once = |(step, n): (Step, usize)| (step, n, false);
|
||||
match filter {
|
||||
Filter::Flate => {
|
||||
let (step, twice) = flate::decode(data, max, again);
|
||||
let (step, n) = with_predictor(step, params, "FlateDecode", rows);
|
||||
(step, n, twice)
|
||||
}
|
||||
// LZW rows are predicted whatever their length, as the renderer reads them.
|
||||
Filter::Lzw => once(with_predictor(lzw::decode(data, params.early_change != 0, max), params, "LZWDecode", None)),
|
||||
Filter::AsciiHex => once(counted(asciihex::decode(data, max))),
|
||||
Filter::Ascii85 => once(counted(ascii85::decode(data, max))),
|
||||
Filter::RunLength => once(counted(runlength::decode(data, max))),
|
||||
other => (Err(Failure { error: FilterError::Unsupported(other.name().to_owned()), partial: Vec::new(), discarded: 0 }), 0, false),
|
||||
}
|
||||
}
|
||||
|
||||
/// A step with the bytes it produced: its output, or its partial output and what it threw away.
|
||||
fn counted(step: Step) -> (Step, usize) {
|
||||
let n = match &step {
|
||||
Ok(v) => v.len(),
|
||||
Err(f) => f.partial.len().saturating_add(f.discarded),
|
||||
};
|
||||
(step, n)
|
||||
}
|
||||
|
||||
/// Applies the predictor to a decoder's result, including to partial output on failure. The
|
||||
/// count is the decoder's output: a PNG predictor drops a tag byte per row, but the decoder
|
||||
/// produced it. `rows` (an image's height, for its last stage) applies only to data that decoded
|
||||
/// whole: data cut short by damage has its predictor undone as it always had.
|
||||
fn with_predictor(step: Step, params: &Params, name: &'static str, rows: Option<usize>) -> (Step, usize) {
|
||||
let (step, n) = counted(step);
|
||||
let step = match step {
|
||||
Ok(v) => predictor::decode(params, v, name, rows),
|
||||
Err(Failure { error, partial, discarded }) => {
|
||||
let partial = match predictor::decode(params, partial, name, None) {
|
||||
Ok(v) => v,
|
||||
Err(f) => f.partial,
|
||||
};
|
||||
Err(Failure { error, partial, discarded })
|
||||
}
|
||||
};
|
||||
(step, n)
|
||||
}
|
||||
|
||||
/// Encodes `data` with `filter`. Supported: Flate (zlib, level 6), LZW, ASCIIHex,
|
||||
/// ASCII85, RunLength. For Flate and LZW, `params.predictor >= 2` applies the
|
||||
/// predictor first (TIFF 2, PNG 10–15; 15 chooses the per-row optimum).
|
||||
pub fn encode(filter: &Filter, params: &Params, data: &[u8]) -> Result<Vec<u8>, FilterError> {
|
||||
match filter {
|
||||
Filter::Flate => Ok(flate::encode(&predictor::encode(params, data)?)),
|
||||
Filter::Lzw => Ok(lzw::encode(&predictor::encode(params, data)?, params.early_change != 0)),
|
||||
Filter::AsciiHex => Ok(asciihex::encode(data)),
|
||||
Filter::Ascii85 => Ok(ascii85::encode(data)),
|
||||
Filter::RunLength => Ok(runlength::encode(data)),
|
||||
other => Err(FilterError::Unsupported(format!("encoding with {}", other.name()))),
|
||||
}
|
||||
}
|
||||
|
||||
/// zlib (RFC 1950) at compression level 6.
|
||||
pub fn encode_flate(data: &[u8]) -> Vec<u8> {
|
||||
flate::encode(data)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
thread_local! {
|
||||
/// Bytes handed to decoding stages on this thread.
|
||||
static STAGE_BYTES: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#[test]
|
||||
fn a_long_filter_chain_is_refused_before_decoding() {
|
||||
// `FF FF` run-length decodes to itself, so every stage of the chain repeats the work.
|
||||
let data = vec![0xFF; 4096];
|
||||
let chain = |n: usize| vec![(super::Filter::RunLength, super::Params::default()); n];
|
||||
super::STAGE_BYTES.with(|c| c.set(0));
|
||||
let long = super::decode(&chain(1000), &data, 1 << 20);
|
||||
let read = super::STAGE_BYTES.with(|c| c.get());
|
||||
assert!(read <= 16 * data.len(), "{read} bytes decoded through the chain");
|
||||
assert!(matches!(long, Err(super::FilterError::Unsupported(_))), "{long:?}");
|
||||
assert_eq!(super::decode(&chain(super::MAX_CHAIN), &data, 1 << 20).ok(), Some(data));
|
||||
}
|
||||
|
||||
/// [`decode_tolerant_counted_each`] holds every stage to its own bound as well as the stages
|
||||
/// to their total: a Flate stage that would make more than that bound stops there, though
|
||||
/// the total has room, and the stage after it gets nothing; within both, the chain reads
|
||||
/// whole, one stage's output charged on top of another's.
|
||||
#[test]
|
||||
fn each_stage_is_held_to_its_own_bound_within_the_total() {
|
||||
let hex = "00".repeat(1 << 16).into_bytes();
|
||||
let data = super::flate::encode(&hex);
|
||||
let chain = [(Filter::Flate, Params::default()), (Filter::AsciiHex, Params::default())];
|
||||
// The total alone lets the Flate stage make all of the hex text, and the hex stage read it.
|
||||
STAGE_BYTES.with(|c| c.set(0));
|
||||
let whole = decode_tolerant_counted(&chain, &data, 3 << 16).map(|(v, _, _)| v);
|
||||
assert_eq!(whole.ok(), Some(vec![0u8; 1 << 16]));
|
||||
assert_eq!(STAGE_BYTES.with(|c| c.get()), data.len() + hex.len());
|
||||
// Each stage held to 64 KiB, the Flate stage stops there and the hex stage gets nothing.
|
||||
STAGE_BYTES.with(|c| c.set(0));
|
||||
let held = decode_tolerant_counted_each(&chain, &data, 1 << 16, 3 << 16);
|
||||
assert!(matches!(held, Err((FilterError::LimitExceeded(m), 0)) if m == 1 << 16), "{held:?}");
|
||||
assert_eq!(STAGE_BYTES.with(|c| c.get()), data.len());
|
||||
// Within both, it reads whole.
|
||||
let (out, _, spent) = decode_tolerant_counted_each(&chain, &data, 2 << 16, 3 << 16).unwrap();
|
||||
assert_eq!((out.len(), spent), (1 << 16, hex.len() + (1 << 16)));
|
||||
}
|
||||
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn names_and_abbreviations() {
|
||||
for (full, abbr, f) in [
|
||||
(&b"FlateDecode"[..], &b"Fl"[..], Filter::Flate),
|
||||
(b"LZWDecode", b"LZW", Filter::Lzw),
|
||||
(b"ASCIIHexDecode", b"AHx", Filter::AsciiHex),
|
||||
(b"ASCII85Decode", b"A85", Filter::Ascii85),
|
||||
(b"RunLengthDecode", b"RL", Filter::RunLength),
|
||||
(b"DCTDecode", b"DCT", Filter::Dct),
|
||||
(b"CCITTFaxDecode", b"CCF", Filter::CcittFax),
|
||||
] {
|
||||
assert_eq!(Filter::from_name(full), f);
|
||||
assert_eq!(Filter::from_name(abbr), f);
|
||||
assert_eq!(f.name().as_bytes(), full);
|
||||
}
|
||||
assert_eq!(Filter::from_name(b"/JPXDecode"), Filter::Jpx);
|
||||
assert_eq!(Filter::from_name(b"JBIG2Decode"), Filter::Jbig2);
|
||||
assert_eq!(Filter::from_name(b"Crypt"), Filter::Crypt);
|
||||
assert_eq!(Filter::from_name(b"Foo"), Filter::Unknown("Foo".into()));
|
||||
assert_eq!(Filter::Unknown("Foo".into()).name(), "Foo");
|
||||
assert!(Filter::Dct.is_image_codec() && Filter::CcittFax.is_image_codec());
|
||||
assert!(!Filter::Flate.is_image_codec() && !Filter::Crypt.is_image_codec());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn default_params() {
|
||||
let p = Params::default();
|
||||
assert_eq!((p.predictor, p.colors, p.bits_per_component, p.columns, p.early_change), (1, 1, 8, 1, 1));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unsupported_filters() {
|
||||
for f in [Filter::Dct, Filter::Jpx, Filter::Jbig2, Filter::CcittFax, Filter::Crypt, Filter::Unknown("X".into())] {
|
||||
let chain = [(Filter::AsciiHex, Params::default()), (f.clone(), Params::default())];
|
||||
assert!(matches!(decode(&chain, b"00>", 100), Err(FilterError::Unsupported(_))));
|
||||
assert!(matches!(decode_tolerant(&chain, b"00>", 100), Err(FilterError::Unsupported(_))));
|
||||
assert!(matches!(encode(&f, &Params::default(), b"x"), Err(FilterError::Unsupported(_))));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_chain_is_identity() {
|
||||
assert_eq!(decode(&[], b"abc", 1).unwrap(), b"abc");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chain_ascii85_flate() {
|
||||
let data = b"Hello, chained filters! Hello, chained filters!".repeat(10);
|
||||
let z = encode_flate(&data);
|
||||
let a = encode(&Filter::Ascii85, &Params::default(), &z).unwrap();
|
||||
let chain = [(Filter::Ascii85, Params::default()), (Filter::Flate, Params::default())];
|
||||
assert_eq!(decode(&chain, &a, 1 << 20).unwrap(), data);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chain_three_filters_with_predictor() {
|
||||
let p = Params { predictor: 12, colors: 3, columns: 5, ..Params::default() };
|
||||
let data: Vec<u8> = (0..300u32).map(|i| (i * 7 % 251) as u8).collect();
|
||||
let lzw = encode(&Filter::Lzw, &p, &data).unwrap();
|
||||
let rl = encode(&Filter::RunLength, &Params::default(), &lzw).unwrap();
|
||||
let hex = encode(&Filter::AsciiHex, &Params::default(), &rl).unwrap();
|
||||
let chain = [(Filter::AsciiHex, Params::default()), (Filter::RunLength, Params::default()), (Filter::Lzw, p)];
|
||||
assert_eq!(decode(&chain, &hex, 1 << 20).unwrap(), data);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn limit_applies_to_intermediate_buffers() {
|
||||
let hex = encode(&Filter::AsciiHex, &Params::default(), &[7u8; 100]).unwrap();
|
||||
let chain = [(Filter::AsciiHex, Params::default())];
|
||||
assert!(matches!(decode(&chain, &hex, 99), Err(FilterError::LimitExceeded(99))));
|
||||
assert_eq!(decode(&chain, &hex, 100).unwrap().len(), 100);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn raw_flate_fallback_preserves_output_limits() {
|
||||
// RFC 1951 stored blocks: the first is nonfinal with LEN=257 and
|
||||
// NLEN=!257, followed by a final empty block. Its ignored alignment
|
||||
// bits make the first two bytes also look like a valid zlib header.
|
||||
let data = vec![b'A'; 257];
|
||||
let mut raw = vec![0x78, 0x01, 0x01, 0xfe, 0xfe];
|
||||
raw.extend_from_slice(&data);
|
||||
raw.extend_from_slice(&[0x01, 0x00, 0x00, 0xff, 0xff]);
|
||||
let chain = [(Filter::Flate, Params::default())];
|
||||
for max in [0, 1, 128, 256] {
|
||||
assert!(matches!(decode(&chain, &raw, max), Err(FilterError::LimitExceeded(m)) if m == max));
|
||||
assert!(matches!(decode_tolerant(&chain, &raw, max), Err(FilterError::LimitExceeded(m)) if m == max));
|
||||
}
|
||||
for max in [257, 258] {
|
||||
assert_eq!(decode(&chain, &raw, max).unwrap(), data);
|
||||
assert_eq!(decode_tolerant(&chain, &raw, max).unwrap(), (data.clone(), false));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tolerant_flate_returns_partial() {
|
||||
let data: Vec<u8> = (0..20_000u32).map(|i| (i % 97) as u8 ^ (i / 300) as u8).collect();
|
||||
let mut z = encode_flate(&data);
|
||||
let cut = z.len() / 2;
|
||||
z.truncate(cut);
|
||||
let chain = [(Filter::Flate, Params::default())];
|
||||
assert!(matches!(decode(&chain, &z, 1 << 20), Err(FilterError::Corrupt { .. })));
|
||||
let (out, partial) = decode_tolerant(&chain, &z, 1 << 20).unwrap();
|
||||
assert!(partial);
|
||||
assert!(!out.is_empty() && out.len() < data.len());
|
||||
assert_eq!(&data[..out.len()], &out[..]);
|
||||
// Complete data is not partial.
|
||||
let (out, partial) = decode_tolerant(&chain, &encode_flate(&data), 1 << 20).unwrap();
|
||||
assert!(!partial);
|
||||
assert_eq!(out, data);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tolerant_partial_flows_through_predictor() {
|
||||
let p = Params { predictor: 12, columns: 4, ..Params::default() };
|
||||
let data: Vec<u8> = (0..4000u32).map(|i| (i * 31 % 256) as u8).collect();
|
||||
let mut z = encode(&Filter::Flate, &p, &data).unwrap();
|
||||
z.truncate(z.len() - 20);
|
||||
let (out, partial) = decode_tolerant(&[(Filter::Flate, p)], &z, 1 << 20).unwrap();
|
||||
assert!(partial);
|
||||
assert_eq!(&data[..out.len()], &out[..]);
|
||||
}
|
||||
|
||||
/// A counted decode bounds what all stages decode together: Flate inflating 1 MiB of hex
|
||||
/// that ASCIIHex halves is 1.5 MiB of work, past a 1.25 MiB allowance. A plain decode keeps
|
||||
/// the limit per stage.
|
||||
#[test]
|
||||
fn a_counted_decode_bounds_all_stages_together() {
|
||||
let hex: Vec<u8> = std::iter::repeat_n(b"ab".as_slice(), 1 << 19).flatten().copied().collect();
|
||||
let data = encode_flate(&hex);
|
||||
let chain = [(Filter::Flate, Params::default()), (Filter::AsciiHex, Params::default())];
|
||||
let max = (1 << 20) + (1 << 18);
|
||||
assert_eq!(decode(&chain, &data, max).map(|d| d.len()).ok(), Some(1 << 19));
|
||||
assert!(matches!(decode_counted(&chain, &data, max), Err((FilterError::LimitExceeded(m), _)) if m == max));
|
||||
assert_eq!(decode_counted(&chain, &data, 2 << 20).map(|(_, spent)| spent).ok(), Some((1 << 20) + (1 << 19)));
|
||||
let tolerant = decode_tolerant_counted(&chain, &data, 2 << 20).map(|(out, partial, spent)| (out.len(), partial, spent));
|
||||
assert_eq!(tolerant.ok(), Some((1 << 19, false, (1 << 20) + (1 << 19))));
|
||||
assert!(matches!(decode_tolerant_counted(&chain, &data, max), Err((FilterError::LimitExceeded(m), _)) if m == max));
|
||||
}
|
||||
|
||||
/// A PNG predictor drops a tag byte per row, but Flate produced it: the count includes it,
|
||||
/// also when the predictor fails on a bad tag.
|
||||
#[test]
|
||||
fn a_counted_decode_counts_output_before_the_predictor() {
|
||||
let params = Params { predictor: 12, ..Params::default() };
|
||||
let rows = vec![7u8; 1 << 16];
|
||||
let data = encode(&Filter::Flate, ¶ms, &rows).unwrap();
|
||||
let chain = [(Filter::Flate, params.clone())];
|
||||
let (out, spent) = decode_counted(&chain, &data, usize::MAX).unwrap();
|
||||
assert!(out == rows);
|
||||
assert_eq!(spent, 2 << 16);
|
||||
let bad: Vec<u8> = std::iter::repeat_n([0u8, 7], 1 << 15).flatten().chain([9, 7]).collect();
|
||||
let (e, spent) = decode_counted(&chain, &encode_flate(&bad), usize::MAX).unwrap_err();
|
||||
assert!(matches!(e, FilterError::Corrupt { .. }), "{e}");
|
||||
assert_eq!(spent, bad.len());
|
||||
}
|
||||
|
||||
fn refused(e: &FilterError) -> bool {
|
||||
matches!(e, FilterError::Spent)
|
||||
}
|
||||
|
||||
/// A chain pays for what it reads and what each stage decodes, down to the byte: Flate inflates
|
||||
/// hex that ASCIIHex halves, so a stage counts when a later one shrinks it.
|
||||
#[test]
|
||||
fn a_chain_spends_down_to_the_byte() {
|
||||
let hex = b"ab".repeat(1 << 10);
|
||||
let data = encode_flate(&hex);
|
||||
let chain = [(Filter::Flate, Params::default()), (Filter::AsciiHex, Params::default())];
|
||||
let whole = decode(&chain, &data, 1 << 20).unwrap();
|
||||
let cost = (data.len() + hex.len() + whole.len()) as u64 + FLATE_READ_WORK * data.len() as u64;
|
||||
let mut work = u64::MAX;
|
||||
assert_eq!(decode_tolerant_within(&chain, &data, 1 << 20, &mut work).unwrap(), whole);
|
||||
assert_eq!(u64::MAX - work, cost);
|
||||
let mut work = cost;
|
||||
assert_eq!(decode_tolerant_within(&chain, &data, 1 << 20, &mut work).unwrap(), whole, "exactly enough");
|
||||
assert_eq!(work, 0);
|
||||
let mut work = cost + 7;
|
||||
decode_tolerant_within(&chain, &data, 1 << 20, &mut work).unwrap();
|
||||
assert_eq!(work, 7, "only what it took");
|
||||
let mut work = cost - 1;
|
||||
assert!(refused(&decode_tolerant_within(&chain, &data, 1 << 20, &mut work).unwrap_err()), "a step short");
|
||||
assert_eq!(work, 0, "spends all that was left");
|
||||
// Each stage has `max_output` to itself, as in a plain decode: together they are past it.
|
||||
let mut work = u64::MAX;
|
||||
assert_eq!(decode_tolerant_within(&chain, &data, hex.len(), &mut work).unwrap(), whole);
|
||||
}
|
||||
|
||||
/// No stage runs past what is left: the first stage of this bomb stops there, so the second
|
||||
/// isn't given anything to read, and with too little to read it nothing runs at all.
|
||||
#[test]
|
||||
fn a_chain_stops_at_what_is_left() {
|
||||
let inner = encode_flate(&vec![0; 1 << 20]);
|
||||
let bomb = encode_flate(&inner);
|
||||
let chain = [(Filter::Flate, Params::default()), (Filter::Flate, Params::default())];
|
||||
let read = |work: u64| {
|
||||
super::STAGE_BYTES.with(|c| c.set(0));
|
||||
let mut work = work;
|
||||
assert!(refused(&decode_tolerant_within(&chain, &bomb, usize::MAX, &mut work).unwrap_err()));
|
||||
assert_eq!(work, 0);
|
||||
super::STAGE_BYTES.with(|c| c.get())
|
||||
};
|
||||
let paid = |n: usize| (1 + FLATE_READ_WORK) * n as u64;
|
||||
assert_eq!(read(paid(bomb.len()) + paid(inner.len()) + 10), bomb.len() + inner.len(), "the second stage stops");
|
||||
assert_eq!(read(paid(bomb.len()) + FLATE_READ_WORK * inner.len() as u64 - 1), bomb.len(), "the second can't pay to read");
|
||||
assert_eq!(read(paid(bomb.len()) + 10), bomb.len(), "the first stops, and the second never runs");
|
||||
assert_eq!(read(paid(bomb.len()) - 1), 0, "the first can't pay to read");
|
||||
assert_eq!(read(bomb.len() as u64 - 1), 0, "it can't pay to be read");
|
||||
}
|
||||
|
||||
/// Past `max_output` the chain has failed, and hasn't run out of anything: it pays what it took
|
||||
/// to find out, and what is left stays. Whichever limit a stage stops at is the one reported.
|
||||
#[test]
|
||||
fn a_stage_past_its_limit_pays_for_finding_out() {
|
||||
let data = encode_flate(&vec![0; 1 << 16]);
|
||||
let chain = [(Filter::Flate, Params::default())];
|
||||
let mut work = u64::MAX;
|
||||
assert!(matches!(decode_tolerant_within(&chain, &data, 1000, &mut work), Err(FilterError::LimitExceeded(1000))));
|
||||
let read = |n: usize| FLATE_READ_WORK * n as u64;
|
||||
assert_eq!(u64::MAX - work, data.len() as u64 + read(data.len()) + 1000);
|
||||
// The later stage of two, which pays for the first as well.
|
||||
let bomb = encode_flate(&data);
|
||||
let two = [(Filter::Flate, Params::default()), (Filter::Flate, Params::default())];
|
||||
let mut work = u64::MAX;
|
||||
assert!(matches!(decode_tolerant_within(&two, &bomb, data.len(), &mut work), Err(FilterError::LimitExceeded(_))));
|
||||
assert_eq!(u64::MAX - work, (bomb.len() + 2 * data.len()) as u64 + read(bomb.len()) + read(data.len()));
|
||||
// Both limits at once: a run that can't go on is what a caller needs to hear.
|
||||
let mut work = data.len() as u64 + read(data.len()) + 1000;
|
||||
assert!(refused(&decode_tolerant_within(&chain, &data, 1000, &mut work).unwrap_err()));
|
||||
assert_eq!(work, 0);
|
||||
let mut work = data.len() as u64 + read(data.len()) + 1001;
|
||||
assert!(matches!(decode_tolerant_within(&chain, &data, 1000, &mut work), Err(FilterError::LimitExceeded(1000))));
|
||||
assert_eq!(work, 1, "the image's limit, with a step to spare");
|
||||
}
|
||||
|
||||
/// What a damaged stream decoded is paid for, however the decode ends.
|
||||
#[test]
|
||||
fn a_chain_that_fails_has_paid_for_what_it_did() {
|
||||
let data: Vec<u8> = (0..20_000u32).map(|i| (i % 97) as u8 ^ (i / 300) as u8).collect();
|
||||
let mut cut = encode_flate(&data);
|
||||
cut.truncate(cut.len() / 2);
|
||||
let chain = [(Filter::Flate, Params::default())];
|
||||
let (partial, _) = decode_tolerant(&chain, &cut, 1 << 20).unwrap();
|
||||
assert!(!partial.is_empty());
|
||||
let mut work = u64::MAX;
|
||||
assert_eq!(decode_tolerant_within(&chain, &cut, 1 << 20, &mut work).unwrap(), partial);
|
||||
let read = |n: usize| FLATE_READ_WORK * n as u64;
|
||||
assert_eq!(u64::MAX - work, (cut.len() + partial.len()) as u64 + read(cut.len()));
|
||||
// Read strictly it is an error, for the same work.
|
||||
let mut work = u64::MAX;
|
||||
assert!(matches!(decode_within(&chain, &cut, 1 << 20, &mut work), Err(FilterError::Corrupt { .. })));
|
||||
assert_eq!(u64::MAX - work, (cut.len() + partial.len()) as u64 + read(cut.len()));
|
||||
let whole = encode_flate(&data);
|
||||
let mut work = u64::MAX;
|
||||
assert_eq!(decode_within(&chain, &whole, 1 << 20, &mut work).unwrap(), data);
|
||||
assert_eq!(u64::MAX - work, (whole.len() + data.len()) as u64 + read(whole.len()));
|
||||
// A predictor that comes to nothing, after all of this was inflated for it.
|
||||
let p = Params { predictor: 99, ..Params::default() };
|
||||
let data = encode_flate(&vec![0; 1 << 16]);
|
||||
let mut work = u64::MAX;
|
||||
assert!(decode_tolerant_within(&[(Filter::Flate, p)], &data, 1 << 20, &mut work).unwrap().is_empty());
|
||||
assert_eq!(u64::MAX - work, data.len() as u64 + read(data.len()) + (1 << 16));
|
||||
}
|
||||
|
||||
/// A stream that reads as zlib only until it fails, and as raw deflate until that fails too,
|
||||
/// gives back no data, and has still decoded everything the retry did.
|
||||
#[test]
|
||||
fn a_retry_that_fails_pays_for_what_it_decoded() {
|
||||
// Stored as raw deflate: this many bytes, then a block of a reserved type. As zlib: the
|
||||
// same reserved type at once, so the first reading fails with nothing.
|
||||
let mut data = vec![0x78, 0x01, 0x06, 0xFE, 0xF9];
|
||||
data.extend(vec![0u8; 0x0601]);
|
||||
data.push(0x07);
|
||||
let chain = [(Filter::Flate, Params::default())];
|
||||
let (out, partial) = decode_tolerant(&chain, &data, 1 << 20).unwrap();
|
||||
assert!(out.is_empty() && partial, "what is read stays as it was");
|
||||
let (_, _, spent) = decode_tolerant_counted(&chain, &data, 1 << 20).unwrap();
|
||||
assert_eq!(spent, 0x0601);
|
||||
let (e, spent) = decode_counted(&chain, &data, 1 << 20).unwrap_err();
|
||||
assert!(matches!(e, FilterError::Corrupt { .. }), "{e}");
|
||||
assert_eq!(spent, 0x0601);
|
||||
let mut work = u64::MAX;
|
||||
assert!(decode_tolerant_within(&chain, &data, 1 << 20, &mut work).unwrap().is_empty());
|
||||
// Read twice: as zlib, then as raw deflate.
|
||||
assert_eq!(u64::MAX - work, (data.len() + 0x0601) as u64 + 2 * FLATE_READ_WORK * data.len() as u64);
|
||||
// Through a predictor, which would otherwise lose the count.
|
||||
let p = Params { predictor: 12, columns: 4, ..Params::default() };
|
||||
let (_, _, spent) = decode_tolerant_counted(&[(Filter::Flate, p)], &data, 1 << 20).unwrap();
|
||||
assert_eq!(spent, 0x0601);
|
||||
}
|
||||
|
||||
/// Inflating does work for every block it reads, whether or not the block makes anything: empty
|
||||
/// fixed-Huffman blocks, ten bits each, make nothing however many there are. Each byte of them
|
||||
/// is paid for at Flate's price before any is read.
|
||||
#[test]
|
||||
fn a_flate_stage_pays_for_what_it_reads() {
|
||||
// Four empty blocks in five bytes, as raw deflate, then an empty final block.
|
||||
let mut hollow = [0x02, 0x08, 0x20, 0x80, 0x00].repeat(1 << 12);
|
||||
hollow.extend([0x03, 0x00]);
|
||||
let chain = [(Filter::Flate, Params::default())];
|
||||
assert_eq!(decode(&chain, &hollow, 1 << 20).unwrap(), b"");
|
||||
let cost = (1 + FLATE_READ_WORK) * hollow.len() as u64;
|
||||
let mut work = u64::MAX;
|
||||
assert_eq!(decode_tolerant_within(&chain, &hollow, 1 << 20, &mut work).unwrap(), b"");
|
||||
assert_eq!(u64::MAX - work, cost);
|
||||
let mut work = cost;
|
||||
assert!(decode_tolerant_within(&chain, &hollow, 1 << 20, &mut work).is_ok(), "exactly enough");
|
||||
assert_eq!(work, 0);
|
||||
super::STAGE_BYTES.with(|c| c.set(0));
|
||||
let mut work = cost - 1;
|
||||
assert!(refused(&decode_tolerant_within(&chain, &hollow, 1 << 20, &mut work).unwrap_err()));
|
||||
assert_eq!((work, super::STAGE_BYTES.with(|c| c.get())), (0, 0), "nothing was inflated");
|
||||
// Counted for a byte allowance, as a page's content is, reading costs nothing.
|
||||
assert_eq!(decode_counted(&chain, &hollow, 1 << 20).unwrap(), (Vec::new(), 0));
|
||||
}
|
||||
|
||||
/// A zlib header that only looked valid is read again as raw deflate, and the second reading
|
||||
/// is paid for as the first was. One that can't be paid for is refused, not read as damaged.
|
||||
#[test]
|
||||
fn a_second_reading_is_paid_for() {
|
||||
// As zlib: a block of a reserved type at once. As raw deflate: a stored block of this many
|
||||
// zero bytes, then an empty final block.
|
||||
let mut data = vec![0x78, 0x01, 0x06, 0xFE, 0xF9];
|
||||
data.extend(vec![0u8; 0x0601]);
|
||||
data.extend([0x03, 0x00]);
|
||||
let chain = [(Filter::Flate, Params::default())];
|
||||
assert_eq!(decode(&chain, &data, 1 << 20).unwrap(), vec![0u8; 0x0601]);
|
||||
let read = FLATE_READ_WORK * data.len() as u64;
|
||||
let mut work = (data.len() + 0x0601) as u64 + 2 * read;
|
||||
assert_eq!(decode_tolerant_within(&chain, &data, 1 << 20, &mut work).unwrap(), vec![0u8; 0x0601]);
|
||||
assert_eq!(work, 0, "exactly enough");
|
||||
// Enough to read it once and not twice: it isn't read again.
|
||||
super::flate::INFLATED.with(|c| c.set(0));
|
||||
let mut work = data.len() as u64 + 2 * read - 1;
|
||||
assert!(refused(&decode_tolerant_within(&chain, &data, 1 << 20, &mut work).unwrap_err()));
|
||||
assert_eq!((work, super::flate::INFLATED.with(|c| c.get())), (0, 0));
|
||||
// Enough to read it twice, and a byte short of what it makes: the second reading makes no
|
||||
// more than is left once it has been paid for.
|
||||
super::flate::INFLATED.with(|c| c.set(0));
|
||||
let mut work = (data.len() + 0x0601) as u64 + 2 * read - 1;
|
||||
assert!(refused(&decode_tolerant_within(&chain, &data, 1 << 20, &mut work).unwrap_err()));
|
||||
assert!(super::flate::INFLATED.with(|c| c.get()) < 0x0601, "it stopped short");
|
||||
}
|
||||
|
||||
/// With nothing left nothing is read. What can't be decoded at all never costs anything, and
|
||||
/// says so whatever is left. Without filters, what is read is the data.
|
||||
#[test]
|
||||
fn what_a_chain_is_shows_whatever_is_left() {
|
||||
let flate = [(Filter::Flate, Params::default())];
|
||||
let mut work = 0;
|
||||
assert!(refused(&decode_tolerant_within(&flate, b"garbage", 1 << 20, &mut work).unwrap_err()));
|
||||
let long = vec![(Filter::Flate, Params::default()); MAX_CHAIN + 1];
|
||||
for chain in [vec![(Filter::Dct, Params::default())], vec![(Filter::Jbig2, Params::default())], long] {
|
||||
for left in [0, 100] {
|
||||
let mut work = left;
|
||||
let e = decode_tolerant_within(&chain, b"data", 1 << 20, &mut work).unwrap_err();
|
||||
assert!(matches!(e, FilterError::Unsupported(_)) && !refused(&e), "{e}");
|
||||
assert_eq!(work, left);
|
||||
}
|
||||
}
|
||||
let mut work = 100;
|
||||
assert_eq!(decode_tolerant_within(&[], b"data", 4, &mut work).unwrap(), b"data");
|
||||
assert_eq!(work, 96);
|
||||
assert!(matches!(decode_tolerant_within(&[], b"data", 3, &mut work), Err(FilterError::LimitExceeded(3))));
|
||||
assert_eq!(work, 92);
|
||||
let mut work = 3;
|
||||
assert!(refused(&decode_tolerant_within(&[], b"data", 4, &mut work).unwrap_err()));
|
||||
assert_eq!(work, 0);
|
||||
}
|
||||
|
||||
/// Four rows of three RGB pixels, every sample above 4 (none reads as a PNG filter type).
|
||||
fn rgb_rows() -> (Params, Vec<u8>) {
|
||||
let p = Params { predictor: 15, colors: 3, columns: 3, ..Params::default() };
|
||||
(p, (0..36u8).map(|i| 200 - i).collect())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_image_whose_rows_carry_no_png_tags_is_read_as_its_samples_and_reported() {
|
||||
let (p, samples) = rgb_rows();
|
||||
let chain = [(Filter::Flate, p.clone())];
|
||||
let data = encode_flate(&samples);
|
||||
let mut work = u64::MAX;
|
||||
assert_eq!(decode_image_tolerant_within(&chain, &data, 1 << 20, 4, &mut work).unwrap(), samples);
|
||||
// The repair is reported as the damage it is: tolerant decoding says so, and strict refuses.
|
||||
let (out, partial, _) = run(&chain, &data, 1 << 20, None, false, false, Some(4)).map_err(|(e, _)| e).unwrap();
|
||||
assert_eq!((out, partial), (samples.clone(), true));
|
||||
let strict = run(&chain, &data, 1 << 20, None, true, false, Some(4)).map(|_| ()).map_err(|(e, _)| e.to_string());
|
||||
assert!(strict.as_ref().is_err_and(|e| e.contains("carry no filter-type bytes")), "{strict:?}");
|
||||
// The same data that isn't known to be an image's rows still goes through the predictor.
|
||||
let mut work = u64::MAX;
|
||||
assert_ne!(decode_tolerant_within(&chain, &data, 1 << 20, &mut work).unwrap(), samples);
|
||||
// LZW is read as before, as the renderer reads it.
|
||||
let lzw = [(Filter::Lzw, p)];
|
||||
let data = encode(&Filter::Lzw, &Params::default(), &samples).unwrap();
|
||||
let mut plain = u64::MAX;
|
||||
assert_eq!(
|
||||
decode_image_tolerant_within(&lzw, &data, 1 << 20, 4, &mut work).ok(),
|
||||
decode_tolerant_within(&lzw, &data, 1 << 20, &mut plain).ok()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_image_whose_rows_carry_png_tags_has_its_predictor_undone() {
|
||||
let (p, samples) = rgb_rows();
|
||||
let chain = [(Filter::Flate, p.clone())];
|
||||
let data = encode(&Filter::Flate, &p, &samples).unwrap();
|
||||
let (mut a, mut b) = (u64::MAX, u64::MAX);
|
||||
assert_eq!(decode_image_tolerant_within(&chain, &data, 1 << 20, 4, &mut a).unwrap(), samples);
|
||||
assert_eq!(decode_tolerant_within(&chain, &data, 1 << 20, &mut b).unwrap(), samples);
|
||||
assert_eq!(a, b, "the same work");
|
||||
let (_, partial, _) = run(&chain, &data, 1 << 20, None, false, false, Some(4)).map_err(|(e, _)| e).unwrap();
|
||||
assert!(!partial);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn image_data_of_neither_length_decodes_as_it_did() {
|
||||
let (p, _) = rgb_rows();
|
||||
let chain = [(Filter::Flate, p.clone())];
|
||||
for len in [0, 1, 9, 27, 35, 37, 39, 40, 41, 72] {
|
||||
let data = encode_flate(&(0..len).map(|i| (i * 7 % 5) as u8).collect::<Vec<_>>());
|
||||
let (mut a, mut b) = (u64::MAX, u64::MAX);
|
||||
let image = decode_image_tolerant_within(&chain, &data, 1 << 20, 4, &mut a).map_err(|e| e.to_string());
|
||||
let plain = decode_tolerant_within(&chain, &data, 1 << 20, &mut b).map_err(|e| e.to_string());
|
||||
assert_eq!(image, plain, "{len} bytes");
|
||||
}
|
||||
// Rows of the untagged length for another height, or no height.
|
||||
let samples = encode_flate(&[9; 36]);
|
||||
for rows in [0, 3, 5, u32::MAX] {
|
||||
let (mut a, mut b) = (u64::MAX, u64::MAX);
|
||||
assert_eq!(
|
||||
decode_image_tolerant_within(&chain, &samples, 1 << 20, rows, &mut a).ok(),
|
||||
decode_tolerant_within(&chain, &samples, 1 << 20, &mut b).ok(),
|
||||
"{rows} rows"
|
||||
);
|
||||
}
|
||||
// Only the last stage produces the rows: an earlier one is predicted as before.
|
||||
let hex: Vec<u8> = [9u8; 36].iter().flat_map(|b| format!("{b:02x}").into_bytes()).collect();
|
||||
let earlier = [(Filter::Flate, p.clone()), (Filter::AsciiHex, Params::default())];
|
||||
let data = encode_flate(&hex[..36]);
|
||||
let (mut a, mut b) = (u64::MAX, u64::MAX);
|
||||
assert_eq!(
|
||||
decode_image_tolerant_within(&earlier, &data, 1 << 20, 4, &mut a).ok(),
|
||||
decode_tolerant_within(&earlier, &data, 1 << 20, &mut b).ok()
|
||||
);
|
||||
// TIFF predictor 2 and no predictor are left alone.
|
||||
for predictor in [1, 2] {
|
||||
let chain = [(Filter::Flate, Params { predictor, ..p.clone() })];
|
||||
let (mut a, mut b) = (u64::MAX, u64::MAX);
|
||||
assert_eq!(
|
||||
decode_image_tolerant_within(&chain, &samples, 1 << 20, 4, &mut a).ok(),
|
||||
decode_tolerant_within(&chain, &samples, 1 << 20, &mut b).ok()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn image_rows_of_no_bytes_or_past_memory_do_not_panic() {
|
||||
for (colors, bits_per_component, columns) in [(0, 8, 3), (3, 8, 0), (3, 8, -1), (255, 16, i64::MAX), (1, 1, i64::MAX), (3, 3, 3)] {
|
||||
let chain = [(Filter::Flate, Params { predictor: 15, colors, bits_per_component, columns, early_change: 1 })];
|
||||
for len in [0, 1, 36] {
|
||||
let data = encode_flate(&vec![7; len]);
|
||||
for rows in [0, 1, 4, u32::MAX] {
|
||||
let mut work = u64::MAX;
|
||||
let _ = decode_image_tolerant_within(&chain, &data, 1 << 20, rows, &mut work);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,305 @@
|
||||
//! FlateDecode (ISO 32000-2 §7.4.4): zlib (RFC 1950) wrapping deflate (RFC 1951).
|
||||
//!
|
||||
//! Real-world quirks handled:
|
||||
//! - the Adler-32 trailer is not required (some producers omit or corrupt it), because the
|
||||
//! deflate stream is inflated raw after the 2-byte zlib header;
|
||||
//! - streams without a valid zlib header are inflated as raw deflate;
|
||||
//! - data after the end of the deflate stream is ignored.
|
||||
//!
|
||||
//! A back-reference to before the start of the output points at nothing, and is damage, as zlib
|
||||
//! has it: read strictly the stream is refused, and read tolerantly what came before it is kept.
|
||||
//! (miniz_oxide, flate2's backend here before zlib-rs, copied zeros and went on.) So is a dynamic
|
||||
//! block whose code has no end-of-block symbol, and nothing of that block is kept.
|
||||
|
||||
use std::io::Write;
|
||||
|
||||
use flate2::{Compression, Decompress, FlushDecompress, Status};
|
||||
|
||||
use craft_stream_codecs::{Failure, Step};
|
||||
|
||||
use super::FilterError;
|
||||
|
||||
const NAME: &str = "FlateDecode";
|
||||
|
||||
/// Checks the RFC 1950 header: CM = 8, CINFO ≤ 7 and the FCHECK multiple-of-31 rule.
|
||||
fn zlib_header(data: &[u8]) -> Option<(u8, u8)> {
|
||||
let (&cmf, &flg) = (data.first()?, data.get(1)?);
|
||||
let ok = cmf & 0x0f == 8 && cmf >> 4 <= 7 && (u16::from(cmf) << 8 | u16::from(flg)) % 31 == 0;
|
||||
ok.then_some((cmf, flg))
|
||||
}
|
||||
|
||||
/// `data` inflated within `max`, and whether it was read twice: a zlib header that only looked
|
||||
/// valid is read again as raw deflate, within `again`, what a caller paying for what is read has
|
||||
/// left for it once it has paid for the second reading. `None`, that reading can't be paid for,
|
||||
/// and is [`FilterError::Spent`].
|
||||
pub(crate) fn decode(data: &[u8], max: usize, again: Option<usize>) -> (Step, bool) {
|
||||
match zlib_header(data) {
|
||||
Some((_, flg)) if flg & 0x20 != 0 => (Err(Failure::corrupt(NAME, "zlib preset dictionary is not allowed in PDF", Vec::new())), false),
|
||||
Some(_) => match inflate_raw(&data[2..], max) {
|
||||
// A header that only looked valid: retry the whole input as raw deflate.
|
||||
Err(f) if f.partial.is_empty() && matches!(f.error, FilterError::Corrupt { .. }) => {
|
||||
let Some(again) = again else {
|
||||
return (Err(Failure { error: FilterError::Spent, partial: Vec::new(), discarded: 0 }), false);
|
||||
};
|
||||
let step = match inflate_raw(data, again) {
|
||||
// Keep the original corruption diagnostic only if the retry is corrupt
|
||||
// too. A limit error must remain fatal even for tolerant decoding. What the
|
||||
// retry decoded isn't kept, but a caller charging for work has to hear of it.
|
||||
Err(retry) if matches!(retry.error, FilterError::Corrupt { .. }) => Err(Failure { discarded: retry.partial.len(), ..f }),
|
||||
retry => retry,
|
||||
};
|
||||
(step, true)
|
||||
}
|
||||
r => (r, false),
|
||||
},
|
||||
None => (inflate_raw(data, max), false),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
thread_local! {
|
||||
/// Bytes inflating gave back on this thread, its partial output included.
|
||||
pub(crate) static INFLATED: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
|
||||
}
|
||||
|
||||
fn inflate_raw(input: &[u8], max: usize) -> Step {
|
||||
let step = inflate(input, max);
|
||||
#[cfg(test)]
|
||||
INFLATED.with(|c| c.set(c.get() + step.as_ref().map_or_else(|f| f.partial.len(), Vec::len)));
|
||||
step
|
||||
}
|
||||
|
||||
fn inflate(input: &[u8], max: usize) -> Step {
|
||||
let mut d = Decompress::new(false);
|
||||
let mut out: Vec<u8> = Vec::new();
|
||||
// One byte beyond the limit lets us tell "exactly max" from "more than max".
|
||||
let cap = max.saturating_add(1);
|
||||
loop {
|
||||
if out.len() == out.capacity() {
|
||||
if out.len() >= cap {
|
||||
return Err(Failure::limit(max));
|
||||
}
|
||||
let grow = out.capacity().max(input.len().saturating_mul(2)).max(4096).min(cap - out.len());
|
||||
out.reserve_exact(grow);
|
||||
}
|
||||
let consumed = d.total_in() as usize;
|
||||
let produced = out.len();
|
||||
let res = d.decompress_vec(&input[consumed.min(input.len())..], &mut out, FlushDecompress::None);
|
||||
if out.len() > max {
|
||||
return Err(Failure::limit(max));
|
||||
}
|
||||
match res {
|
||||
Err(e) => return Err(Failure::corrupt(NAME, e.to_string(), out)),
|
||||
Ok(Status::StreamEnd) => return Ok(out),
|
||||
Ok(Status::Ok | Status::BufError) => {
|
||||
let progressed = d.total_in() as usize != consumed || out.len() != produced;
|
||||
let room_left = out.len() < out.capacity();
|
||||
if room_left && (d.total_in() as usize >= input.len() || !progressed) {
|
||||
return Err(Failure::corrupt(NAME, "unexpected end of deflate data", out));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// zlib at level 6.
|
||||
// The documented never-crash exception: writing into a `Vec` cannot fail.
|
||||
#[allow(clippy::expect_used)]
|
||||
pub(crate) fn encode(data: &[u8]) -> Vec<u8> {
|
||||
let mut e = flate2::write::ZlibEncoder::new(Vec::with_capacity(data.len() / 2 + 64), Compression::new(6));
|
||||
e.write_all(data).expect("in-memory write");
|
||||
e.finish().expect("in-memory write")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn dec(d: &[u8]) -> Result<Vec<u8>, FilterError> {
|
||||
decode(d, 1 << 24, Some(1 << 24)).0.map_err(|f| f.error)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn known_zlib_vector() {
|
||||
// zlib("a"): header 78 9C, fixed-Huffman block, Adler-32 0x00620062.
|
||||
assert_eq!(dec(&[0x78, 0x9c, 0x4b, 0x04, 0x00, 0x00, 0x62, 0x00, 0x62]).unwrap(), b"a");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hand_built_stored_block() {
|
||||
// RFC 1951 §3.2.4 stored block: BFINAL=1 BTYPE=00, LEN=5, NLEN=!5, "hello",
|
||||
// then Adler-32("hello") = 0x062C0215.
|
||||
let v = [0x78, 0x01, 0x01, 0x05, 0x00, 0xfa, 0xff, b'h', b'e', b'l', b'l', b'o', 0x06, 0x2c, 0x02, 0x15];
|
||||
assert_eq!(dec(&v).unwrap(), b"hello");
|
||||
// Missing Adler-32 is tolerated.
|
||||
assert_eq!(dec(&v[..12]).unwrap(), b"hello");
|
||||
// Wrong Adler-32 is tolerated (as in other readers).
|
||||
let mut bad = v;
|
||||
bad[15] ^= 0xff;
|
||||
assert_eq!(dec(&bad).unwrap(), b"hello");
|
||||
// Trailing garbage after the stream is ignored.
|
||||
let mut trailing = v.to_vec();
|
||||
trailing.extend_from_slice(b"\r\nendstream");
|
||||
assert_eq!(dec(&trailing).unwrap(), b"hello");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn raw_deflate_fallback() {
|
||||
let data = b"raw deflate without any zlib header, repeated repeated repeated".repeat(4);
|
||||
let mut e = flate2::write::DeflateEncoder::new(Vec::new(), Compression::new(6));
|
||||
e.write_all(&data).unwrap();
|
||||
let raw = e.finish().unwrap();
|
||||
assert!(zlib_header(&raw).is_none());
|
||||
assert_eq!(dec(&raw).unwrap(), data);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn round_trip_and_missing_checksum() {
|
||||
let data: Vec<u8> = (0..50_000u32).map(|i| (i * i % 251) as u8).collect();
|
||||
let z = encode(&data);
|
||||
assert_eq!(&z[..2], &[0x78, 0x9c]);
|
||||
assert_eq!(dec(&z).unwrap(), data);
|
||||
assert_eq!(dec(&z[..z.len() - 4]).unwrap(), data);
|
||||
assert_eq!(dec(&z[..z.len() - 2]).unwrap(), data);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_input() {
|
||||
assert_eq!(dec(&encode(b"")).unwrap(), b"");
|
||||
assert!(matches!(dec(b""), Err(FilterError::Corrupt { .. })));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn corrupt_and_truncated() {
|
||||
let data: Vec<u8> = (0..10_000u32).map(|i| (i % 13) as u8).collect();
|
||||
let z = encode(&data);
|
||||
match decode(&z[..z.len() / 2], 1 << 24, Some(1 << 24)).0 {
|
||||
Err(f) => {
|
||||
assert!(matches!(f.error, FilterError::Corrupt { .. }));
|
||||
assert_eq!(&data[..f.partial.len()], &f.partial[..]);
|
||||
}
|
||||
Ok(_) => panic!("truncated stream decoded"),
|
||||
}
|
||||
// BTYPE = 11 is reserved (RFC 1951 §3.2.3).
|
||||
assert!(matches!(dec(&[0x78, 0x9c, 0x07, 0, 0, 0]), Err(FilterError::Corrupt { .. })));
|
||||
// Preset dictionary.
|
||||
assert!(matches!(dec(&[0x78, 0x20, 0, 0, 0, 0]), Err(FilterError::Corrupt { .. })));
|
||||
// One whose header is followed by data that inflates: refused all the same.
|
||||
assert!(matches!(dec(&[0x78, 0xBB, 0x4b, 0x04, 0x00]), Err(FilterError::Corrupt { .. })));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bomb_is_bounded() {
|
||||
let z = encode(&vec![0u8; 16 << 20]);
|
||||
assert!(z.len() < 32 * 1024);
|
||||
assert!(matches!(dec_limit(&z, 1 << 20), Err(FilterError::LimitExceeded(n)) if n == 1 << 20));
|
||||
assert_eq!(dec_limit(&z, 16 << 20).unwrap().len(), 16 << 20);
|
||||
assert!(matches!(dec_limit(&z, (16 << 20) - 1), Err(FilterError::LimitExceeded(_))));
|
||||
assert!(matches!(dec_limit(&z, 0), Err(FilterError::LimitExceeded(0))));
|
||||
}
|
||||
|
||||
fn dec_limit(d: &[u8], max: usize) -> Result<Vec<u8>, FilterError> {
|
||||
decode(d, max, Some(max)).0.map_err(|f| f.error)
|
||||
}
|
||||
|
||||
/// Deflate's bits: header fields and extra bits least significant first, Huffman codes most
|
||||
/// significant first (RFC 1951 §3.1.1).
|
||||
#[derive(Default)]
|
||||
struct Bits {
|
||||
out: Vec<u8>,
|
||||
acc: u32,
|
||||
n: u32,
|
||||
}
|
||||
|
||||
impl Bits {
|
||||
fn put(&mut self, value: u32, len: u32) {
|
||||
for i in 0..len {
|
||||
self.acc |= ((value >> i) & 1) << self.n;
|
||||
self.n += 1;
|
||||
if self.n == 8 {
|
||||
self.out.push(self.acc as u8);
|
||||
(self.acc, self.n) = (0, 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn code(&mut self, code: u32, len: u32) {
|
||||
for i in (0..len).rev() {
|
||||
self.put((code >> i) & 1, 1);
|
||||
}
|
||||
}
|
||||
|
||||
fn finish(mut self) -> Vec<u8> {
|
||||
if self.n > 0 {
|
||||
self.out.push(self.acc as u8);
|
||||
}
|
||||
self.out
|
||||
}
|
||||
}
|
||||
|
||||
/// A dynamic block whose literal and length code has no end-of-block symbol can't end (RFC 1951
|
||||
/// §3.2.5): it is refused, and nothing of it is kept.
|
||||
#[test]
|
||||
fn a_dynamic_code_without_an_end_is_damage() {
|
||||
let mut bits = Bits::default();
|
||||
bits.put(1, 1); // the final block,
|
||||
bits.put(2, 2); // dynamic Huffman codes (§3.2.7)
|
||||
bits.put(0, 5); // 257 literal and length codes
|
||||
bits.put(0, 5); // 1 distance code
|
||||
bits.put(14, 4); // 18 code length codes, in their order up to the one for 1
|
||||
for symbol in [16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1] {
|
||||
bits.put(u32::from(symbol == 18 || symbol == 1), 3);
|
||||
}
|
||||
// Code length codes: 1 is 0, 18 is 1. 'A' and 'B' get a bit each, 256 none.
|
||||
let zeros = |bits: &mut Bits, n: u32| {
|
||||
bits.code(1, 1);
|
||||
bits.put(n - 11, 7);
|
||||
};
|
||||
zeros(&mut bits, 65);
|
||||
bits.code(0, 1);
|
||||
bits.code(0, 1);
|
||||
zeros(&mut bits, 138);
|
||||
zeros(&mut bits, 52);
|
||||
bits.code(0, 1); // the distance code: a bit
|
||||
for _ in 0..4 {
|
||||
bits.code(0, 1); // 'A'
|
||||
}
|
||||
let data = bits.finish();
|
||||
assert!(zlib_header(&data).is_none(), "read as raw deflate");
|
||||
match decode(&data, 1 << 20, Some(1 << 20)).0 {
|
||||
Err(f) => {
|
||||
assert!(matches!(f.error, FilterError::Corrupt { .. }), "{}", f.error);
|
||||
assert_eq!(f.partial, b"");
|
||||
}
|
||||
Ok(out) => panic!("read as {out:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
/// A back-reference to before the start of the output points at nothing: read strictly the
|
||||
/// stream is refused, and what came before it is what a tolerant reading keeps. miniz_oxide,
|
||||
/// flate2's backend here before zlib-rs, copied zeros and went on, so this also shows which
|
||||
/// backend inflates (the price of reading Flate, `FLATE_READ_WORK`, is set for zlib-rs).
|
||||
#[test]
|
||||
fn a_copy_from_before_the_start_is_damage() {
|
||||
let literal = |bits: &mut Bits, b: u8| bits.code(0x30 + u32::from(b), 8);
|
||||
let mut bits = Bits::default();
|
||||
bits.put(1, 1); // the final block,
|
||||
bits.put(1, 2); // fixed Huffman codes (§3.2.6)
|
||||
literal(&mut bits, b'A');
|
||||
literal(&mut bits, b'B');
|
||||
bits.code(1, 7); // length 3
|
||||
bits.code(4, 5); // distance 5 (code 4 and an extra bit of 0), three bytes before the start
|
||||
bits.put(0, 1);
|
||||
literal(&mut bits, b'C');
|
||||
bits.code(0, 7); // end of block
|
||||
let data = bits.finish();
|
||||
assert!(zlib_header(&data).is_none(), "read as raw deflate");
|
||||
match decode(&data, 1 << 20, Some(1 << 20)).0 {
|
||||
Err(f) => {
|
||||
assert!(matches!(f.error, FilterError::Corrupt { .. }), "{}", f.error);
|
||||
assert_eq!(f.partial, b"AB");
|
||||
}
|
||||
Ok(out) => panic!("read as {out:?}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -6,6 +6,8 @@
|
||||
|
||||
#[cfg(feature = "crypt")]
|
||||
pub mod crypt;
|
||||
#[cfg(feature = "filters")]
|
||||
pub mod filters;
|
||||
#[cfg(feature = "interpret")]
|
||||
pub use hayro_interpret as interpret;
|
||||
mod link;
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
//! Tests of the `filters` feature. Not built for the web: proptest does not build there.
|
||||
#![cfg(all(feature = "filters", not(target_arch = "wasm32")))]
|
||||
|
||||
mod proptests;
|
||||
@@ -0,0 +1,7 @@
|
||||
# Seeds for failure cases proptest has generated in the past. It is
|
||||
# automatically read and these particular cases re-run before any
|
||||
# novel cases are generated.
|
||||
#
|
||||
# It is recommended to check this file in to source control so that
|
||||
# everyone who runs the test benefits from these saved cases.
|
||||
cc 690714485bf1c84d6f7788b0205b5d62942b9774699a364161224381ebb08f22 # shrinks to filter = Flate, params = Params { predictor: 10, colors: 1, bits_per_component: 1, columns: 1, early_change: 0 }, data = [74, 4, 40, 11, 72, 5, 123, 154, 191, 76, 212, 154, 73, 3, 113, 92, 51, 190, 160, 141, 12, 106, 83, 206, 91, 214, 202, 23, 39, 151, 117, 114, 220, 76, 204, 235, 186, 176, 80, 229, 63, 79, 67, 63, 177, 94, 142, 223, 140, 248, 1, 203, 157, 88, 175, 77, 69, 155, 159, 152, 47, 216, 81, 234, 169, 144, 235, 202, 75, 191, 233, 128, 225, 162, 96, 126, 245, 57, 135, 167, 184, 53, 138, 215, 153, 248, 165, 18, 81, 128, 44, 139, 223, 76, 191, 58, 137, 48, 93, 14, 80, 58, 30, 63, 40, 235, 68, 159, 137, 113, 123, 24, 150, 84, 78, 190, 207, 36, 189, 220, 192, 161, 121, 179, 1, 242, 152, 205, 64, 67, 50, 245, 94, 185, 81, 132, 27, 25, 122, 7, 223, 178, 147, 70, 27, 130, 226, 217, 54, 87, 112, 14, 196, 133, 193, 27, 147, 30, 38, 182, 108, 72, 75, 107, 113, 244, 65, 9, 62, 11, 66, 234, 146, 180, 56, 85, 170, 255, 38, 37, 159, 48, 3, 159, 125, 230, 34, 110, 7, 58, 92, 66, 14, 29, 50, 98, 56, 228, 118, 243, 32, 99, 158, 252, 246, 249, 185, 106, 11, 127, 218, 104, 216, 204, 23, 59, 85, 34, 123, 28, 253, 14, 147, 201, 205, 176, 60, 39, 249, 143, 205, 190, 183, 83, 27, 213, 23, 225, 172, 43, 225, 31, 62, 205, 53, 88, 202, 200, 75, 152, 252, 105, 20, 95, 20, 0, 229, 170, 23, 222, 252, 208, 41, 206, 133, 167, 41, 89, 208, 136, 98, 203, 93, 112, 18, 75, 54, 217, 191, 163, 165, 88, 137, 203, 7, 66, 218, 184, 251, 48, 25, 144, 29, 183, 119, 101, 192, 234, 116, 81, 211, 255, 22, 16, 20, 36, 190, 253, 73, 246, 9, 141, 99, 90, 178, 252, 93, 112, 233, 213, 42, 127, 227, 208, 125, 129, 41, 99, 111, 228, 123, 106, 131, 39, 204, 158, 18, 6, 221, 168, 135, 60, 107, 156, 254, 93, 255, 163, 34, 142, 110, 16, 207, 84, 243, 180, 145, 42, 240, 252, 64, 53, 225, 211, 231, 255, 215, 221, 181, 69, 8, 59, 134, 254, 98, 16, 157, 100, 115, 145, 190, 115, 49, 153, 255, 242, 159, 248, 37, 192, 48, 133, 42, 158, 35, 111, 7, 101, 1, 12, 131, 117, 149, 23, 224, 80, 101, 146, 235, 24, 214, 166, 254, 39, 121, 65, 109, 7, 115, 4, 126, 238, 207, 122, 72, 78, 170, 80, 239, 13, 233, 101, 110, 101, 121, 219, 159, 194, 78, 33, 198, 239, 149, 64, 6, 105, 138, 106, 34, 82, 253, 66, 74, 219, 101, 247, 133, 23, 140, 254, 221, 164, 77, 66, 28, 202, 253, 230, 152, 200, 164, 143, 126, 92, 231, 72, 120, 249, 230, 119, 24, 109, 185, 137, 26, 206, 91, 90, 76, 140, 134, 137, 67, 83, 52, 105, 135, 184, 180, 231, 213, 190, 252, 144, 129, 85, 96, 4, 62, 70, 197, 73, 164, 61, 209, 150, 227, 138, 92, 25, 117, 84, 161, 81, 247, 94, 53, 207, 93, 59, 223, 222, 87, 255, 216, 219, 143, 36, 255, 131, 117, 9, 30, 144, 196, 209, 139, 253, 187, 193, 126, 243, 21, 66, 19, 226, 95, 166, 245, 11, 63, 220, 30, 55, 255, 172, 108, 87, 187, 2, 3, 242, 243, 175, 134, 88, 128, 97, 68, 102, 241, 94, 109, 219, 219, 6, 38, 155, 249, 194, 33, 175, 48, 141, 41, 82, 186, 85, 232, 157, 160, 145, 233, 103, 190, 239, 106, 247, 241, 11, 216, 90, 56, 154, 18, 119, 33, 181, 109, 145, 80, 158, 51, 129, 150, 147, 255, 14, 135, 86, 62, 250, 84, 8, 235, 237, 246, 227, 137, 112, 175, 239, 226, 182, 92, 188, 236, 254, 87, 49, 18, 208, 139, 48, 200, 151, 201, 196, 182, 182], max = 1319
|
||||
@@ -0,0 +1,150 @@
|
||||
//! Round-trip property tests (encode → decode) for every encodable filter and predictor,
|
||||
//! and fuzz-style tests feeding arbitrary bytes and parameters to every decoder.
|
||||
|
||||
use craft_pdf::filters::{Filter, FilterError, Params, decode, decode_tolerant, encode};
|
||||
use proptest::prelude::*;
|
||||
|
||||
const MAX: usize = 1 << 24;
|
||||
|
||||
fn encodable() -> impl Strategy<Value = Filter> {
|
||||
prop_oneof![Just(Filter::Flate), Just(Filter::Lzw), Just(Filter::AsciiHex), Just(Filter::Ascii85), Just(Filter::RunLength),]
|
||||
}
|
||||
|
||||
fn any_filter() -> impl Strategy<Value = Filter> {
|
||||
prop_oneof![
|
||||
encodable(),
|
||||
Just(Filter::Dct),
|
||||
Just(Filter::Jpx),
|
||||
Just(Filter::Jbig2),
|
||||
Just(Filter::CcittFax),
|
||||
Just(Filter::Crypt),
|
||||
Just(Filter::Unknown("Foo".into())),
|
||||
]
|
||||
}
|
||||
|
||||
fn valid_params() -> impl Strategy<Value = Params> {
|
||||
(prop::sample::select(vec![1i64, 2, 10, 11, 12, 13, 14, 15]), 1i64..=5, prop::sample::select(vec![1i64, 2, 4, 8, 16]), 1i64..=40, 0i64..=1)
|
||||
.prop_map(|(predictor, colors, bits_per_component, columns, early_change)| Params {
|
||||
predictor,
|
||||
colors,
|
||||
bits_per_component,
|
||||
columns,
|
||||
early_change,
|
||||
})
|
||||
}
|
||||
|
||||
/// Anything, including nonsense values.
|
||||
fn wild_params() -> impl Strategy<Value = Params> {
|
||||
let wild = prop_oneof![Just(i64::MIN), Just(i64::MAX), Just(-1i64), Just(0i64), 0i64..=40, any::<i64>()];
|
||||
(
|
||||
prop_oneof![valid_params().prop_map(|p| p.predictor), wild.clone()],
|
||||
prop_oneof![1i64..=5, wild.clone()],
|
||||
prop_oneof![Just(8i64), Just(16), Just(1), wild.clone()],
|
||||
prop_oneof![1i64..=40, wild.clone()],
|
||||
wild,
|
||||
)
|
||||
.prop_map(|(predictor, colors, bits_per_component, columns, early_change)| Params {
|
||||
predictor,
|
||||
colors,
|
||||
bits_per_component,
|
||||
columns,
|
||||
early_change,
|
||||
})
|
||||
}
|
||||
|
||||
/// Low-entropy data (so LZW/Flate/RunLength exercise their repeat paths) mixed with noise.
|
||||
fn data() -> impl Strategy<Value = Vec<u8>> {
|
||||
prop_oneof![prop::collection::vec(any::<u8>(), 0..2000), prop::collection::vec(prop::sample::select(vec![0u8, 1, 7, 255]), 0..6000),]
|
||||
}
|
||||
|
||||
proptest! {
|
||||
#[test]
|
||||
fn round_trip(filter in encodable(), params in valid_params(), data in data()) {
|
||||
let enc = encode(&filter, ¶ms, &data).unwrap();
|
||||
let chain = [(filter, params)];
|
||||
prop_assert_eq!(&decode(&chain, &enc, MAX).unwrap(), &data);
|
||||
let (out, partial) = decode_tolerant(&chain, &enc, MAX).unwrap();
|
||||
prop_assert!(!partial);
|
||||
prop_assert_eq!(&out, &data);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn round_trip_chain(a in encodable(), b in encodable(), params in valid_params(), data in data()) {
|
||||
let inner = encode(&b, ¶ms, &data).unwrap();
|
||||
let outer = encode(&a, &Params::default(), &inner).unwrap();
|
||||
let chain = [(a, Params::default()), (b, params)];
|
||||
prop_assert_eq!(decode(&chain, &outer, MAX).unwrap(), data);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lzw_long_round_trip(early in 0i64..=1, data in prop::collection::vec(0u8..4, 20_000..60_000)) {
|
||||
let p = Params { early_change: early, ..Params::default() };
|
||||
let enc = encode(&Filter::Lzw, &p, &data).unwrap();
|
||||
prop_assert_eq!(decode(&[(Filter::Lzw, p)], &enc, MAX).unwrap(), data);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn output_limit_is_respected(filter in encodable(), params in valid_params(), data in data(), max in 0usize..3000) {
|
||||
let enc = encode(&filter, ¶ms, &data).unwrap();
|
||||
// PNG predictors add a tag byte per row to the intermediate (pre-predictor) buffer.
|
||||
let png = matches!(filter, Filter::Flate | Filter::Lzw) && params.predictor >= 10;
|
||||
let row = (params.colors * params.bits_per_component * params.columns + 7) as usize / 8;
|
||||
let intermediate = if png { data.len() + data.len().div_ceil(row) } else { data.len() };
|
||||
match decode(&[(filter, params)], &enc, max) {
|
||||
Ok(out) => prop_assert!(out.len() <= max && out == data),
|
||||
Err(FilterError::LimitExceeded(m)) => prop_assert!(m == max && intermediate > max),
|
||||
Err(e) => prop_assert!(false, "unexpected {e}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoders_never_panic(
|
||||
chain in prop::collection::vec((any_filter(), wild_params()), 0..4),
|
||||
data in prop::collection::vec(any::<u8>(), 0..3000),
|
||||
max in prop_oneof![Just(0usize), 1usize..100_000],
|
||||
) {
|
||||
if let Ok(out) = decode(&chain, &data, max) {
|
||||
prop_assert!(out.len() <= max || chain.is_empty());
|
||||
}
|
||||
if let Ok((out, _)) = decode_tolerant(&chain, &data, max) {
|
||||
prop_assert!(out.len() <= max || chain.is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn single_decoders_never_panic(filter in encodable(), params in wild_params(), data in prop::collection::vec(any::<u8>(), 0..3000)) {
|
||||
let chain = [(filter, params)];
|
||||
let _ = decode(&chain, &data, 1 << 20);
|
||||
let _ = decode_tolerant(&chain, &data, 1 << 20);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn encoders_never_panic(filter in any_filter(), params in wild_params(), data in prop::collection::vec(any::<u8>(), 0..2000)) {
|
||||
if let Ok(enc) = encode(&filter, ¶ms, &data) {
|
||||
prop_assert_eq!(decode(&[(filter, params)], &enc, MAX).unwrap(), data);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mutated_streams_never_panic(filter in encodable(), params in valid_params(), data in data(), flips in prop::collection::vec((any::<prop::sample::Index>(), any::<u8>()), 1..8)) {
|
||||
let mut enc = encode(&filter, ¶ms, &data).unwrap();
|
||||
for (i, x) in flips {
|
||||
if !enc.is_empty() {
|
||||
let k = i.index(enc.len());
|
||||
enc[k] ^= x;
|
||||
}
|
||||
}
|
||||
let chain = [(filter, params)];
|
||||
let _ = decode(&chain, &enc, 1 << 20);
|
||||
let _ = decode_tolerant(&chain, &enc, 1 << 20);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn flate_bomb_in_chain() {
|
||||
let bomb = craft_pdf::filters::encode_flate(&vec![0u8; 32 << 20]);
|
||||
let a85 = encode(&Filter::Ascii85, &Params::default(), &bomb).unwrap();
|
||||
let chain = [(Filter::Ascii85, Params::default()), (Filter::Flate, Params::default())];
|
||||
assert!(matches!(decode(&chain, &a85, 1 << 20), Err(FilterError::LimitExceeded(_))));
|
||||
assert!(matches!(decode_tolerant(&chain, &a85, 1 << 20), Err(FilterError::LimitExceeded(_))));
|
||||
}
|
||||
Reference in new issue
Block a user