diff --git a/crates/exr/src/decode.rs b/crates/exr/src/decode.rs index 2797e3641..04d135e27 100644 --- a/crates/exr/src/decode.rs +++ b/crates/exr/src/decode.rs @@ -281,8 +281,9 @@ pub(super) fn read_meta(bytes: &[u8], limits: &Limits) -> Result { return Err(Error::LimitExceeded(format!("{total} bytes across {} parts exceed {}", meta.headers.len(), limits.max_bytes))); } // A flat part's samples are allocated before its chunks are read, so the file must be able - // to hold the flat parts' samples. DWA's run-length channels are zlib-packed on top and reach - // about 132000:1, so DWA parts are counted at a quarter. Deep parts size their samples from + // to hold the flat parts' samples. DWA's run-length channels are zlib-packed on top (RLE up to + // 64:1, zlib up to about 1032:1, about 66000:1 together), so DWA parts are counted at a quarter + // (with room to spare). Deep parts size their samples from // the file's own counts and are not counted here. let declared = flat.saturating_sub(dwa).saturating_add(dwa / 4); let capacity = (bytes.len() as u64).saturating_mul(limits.max_ratio).saturating_add(1 << 20); @@ -308,7 +309,7 @@ fn text_attributes(header: &Header) -> Vec<(String, String)> { out } -fn part_info(index: usize, header: &Header) -> PartInfo { +pub(crate) fn part_info(index: usize, header: &Header) -> PartInfo { let (w, h) = window_size(header.layer_size); let display = &header.shared_attributes.display_window; let (dw, dh) = window_size(display.size); @@ -449,6 +450,7 @@ pub fn decode_display_part( header_limits.max_pixels = u64::MAX; header_limits.max_bytes = u64::MAX; let meta = read_meta(bytes, &header_limits)?; + check_dwa_chunks(bytes, &meta)?; let info = meta.headers.get(index).map(|header| part_info(index, header)).ok_or_else(|| malformed(format!("part {index}")))?; if info.deep { return Err(Error::Unsupported("deep data (use the deep reader)".into())); @@ -756,6 +758,16 @@ fn tables_start(bytes: &[u8]) -> Result { Ok(cursor.position()) } +/// Checks the chunks of the file's DWAA/DWAB parts before the `exr` crate decompresses them (see +/// [`crate::dwa_guard`]: a few bytes of counters can make it allocate terabytes and abort). +fn check_dwa_chunks(bytes: &[u8], meta: &MetaData) -> Result<()> { + if !meta.headers.iter().any(|h| crate::dwa_guard::is_dwa(h.compression)) { + return Ok(()); + } + let tables = usize::try_from(tables_start(bytes)?).map_err(|_| malformed("header size"))?; + crate::dwa_guard::check_file(bytes, meta, tables).map_err(malformed) +} + /// Stable dense ordinal without materializing every tile. The library's block iterator /// internally allocates a Vec of all coordinates, which is unsuitable for cropped reads. pub(super) fn flat_block_ordinal(header: &Header, tile: TileCoordinates) -> Result { @@ -944,6 +956,7 @@ fn read_flat_part_chunks(bytes: &[u8], meta: &MetaData, index: usize, reader: &m /// [`Error::Unsupported`]; the file's other parts may be deep. pub fn decode_part(bytes: &[u8], index: usize, keep: &dyn Fn(&str) -> bool, limits: &Limits) -> Result { let meta = read_meta(bytes, limits)?; + check_dwa_chunks(bytes, &meta)?; let header = meta.headers.get(index).ok_or_else(|| malformed(format!("part {index} of {}", meta.headers.len())))?; if header.deep { return Err(Error::Unsupported("deep data (use the deep reader)".into())); diff --git a/crates/exr/src/decode_tests.rs b/crates/exr/src/decode_tests.rs index 89f44d506..0f63d8108 100644 --- a/crates/exr/src/decode_tests.rs +++ b/crates/exr/src/decode_tests.rs @@ -354,13 +354,13 @@ fn mixed_headers_with_deep_size( } /// Sample `i` of a flat channel: `R` is 1, 2, 3, …, `G` 10 more and `B` 100 more, top row first. -fn mixed_flat(channel: f32, i: usize) -> f32 { +pub(super) fn mixed_flat(channel: f32, i: usize) -> f32 { channel + 1.0 + i as f32 } /// A mixed file with a 3 × 2 flat part, and its chunk offsets as `offsets[part][chunk]`. The /// tiled flat part's chunks hold the tiles at (0, 0) and (1, 0). -fn mixed_file(deep_first: bool, tiled: bool) -> (Vec, [[usize; 2]; 2]) { +pub(super) fn mixed_file(deep_first: bool, tiled: bool) -> (Vec, [[usize; 2]; 2]) { let (w, h) = (3usize, 2usize); let mut out = mixed_headers(deep_first, tiled, (w, h)); let chunks: Vec>> = (0..2i32) diff --git a/crates/exr/src/dwa_guard.rs b/crates/exr/src/dwa_guard.rs new file mode 100644 index 000000000..a0b9c7546 --- /dev/null +++ b/crates/exr/src/dwa_guard.rs @@ -0,0 +1,161 @@ +//! A check of DWAA/DWAB chunks before the `exr` crate decompresses them. +//! +//! A DWA payload starts with eleven little-endian `u64` counters, which the `exr` crate (1.74.3, +//! `src/compression/dwa/chunk_header.rs`, `DwaHeader::parse`) reads in this order: version, +//! unknown uncompressed size, unknown compressed size, AC compressed size, DC compressed size, +//! RLE compressed size, RLE uncompressed size, RLE raw size, AC count, DC count, AC compression. +//! The crate checks them only against `i64::MAX`, then inflates sections with these sizes as the +//! size hint (`src/compression/dwa/section_stream.rs`: `inflate(section, expected)` with the +//! unknown uncompressed size, `ac_count * 2` for deflated AC, `dc_count * 2`, and the RLE +//! uncompressed size), and its inflater allocates the hint up front. A 100-byte chunk declaring +//! 2^44 DC values makes it allocate 32 TiB: the process aborts, which `catch_unwind` can't stop. +//! +//! What a valid chunk declares is bounded by its block (`width × height` pixels of all the +//! part's channels, `raw` bytes uncompressed): +//! - the unknown (losslessly zipped) channels and the RLE channels are some of the block's +//! channels stored planar, so their uncompressed sizes are at most `raw`; +//! - the RLE token stream adds at most one count byte per 127 literal bytes (runs of three or +//! more shrink), so it stays within `2 × raw`, a bound that leaves room for any encoder; +//! - the lossy channels are cut into 8 × 8 DCT blocks over the block padded to multiples of 8: +//! one DC value and at most 63 AC values per DCT block and channel. + +use exr::block::chunk::{CompressedBlock, CompressedScanLineBlock, CompressedTileBlock, TileCoordinates}; +use exr::math::Vec2; +use exr::meta::header::Header; +use exr::prelude::Compression; + +/// The counters' bytes at the start of a DWA payload. +const COUNTERS: usize = 11 * 8; + +/// Whether chunks of `compression` go through this check. +pub(crate) fn is_dwa(compression: Compression) -> bool { + matches!(compression, Compression::DWAA(_) | Compression::DWAB(_)) +} + +/// What the check needs of a DWA part: the bytes of one pixel of all its channels, and how many +/// channels it has. `None` when the pixel's bytes overflow. +fn shape(header: &Header) -> Option<(usize, usize)> { + let pixel_bytes = header.channels.list.iter().map(|c| c.sample_type.bytes_per_sample()).try_fold(0usize, usize::checked_add)?; + Some((pixel_bytes, header.channels.list.len())) +} + +/// Checks the DWA payload `payload` of a block of `width × height` pixels of the part `header` +/// describes: an error message when a counter declares more than a valid chunk of that block +/// can hold. A payload as long as the block's raw bytes is stored uncompressed (the crate doesn't +/// read counters then); one shorter than the counters is left to the crate, which refuses it. +pub(crate) fn check_payload(header: &Header, size: (usize, usize), payload: &[u8]) -> Result<(), String> { + if !is_dwa(header.compression) { + return Ok(()); + } + check_shaped(shape(header).ok_or("a DWA pixel size overflows")?, size, payload) +} + +/// [`check_payload`] with the part's [`shape`] worked out already. +fn check_shaped((pixel_bytes, channels): (usize, usize), (width, height): (usize, usize), payload: &[u8]) -> Result<(), String> { + let raw = pixel_bytes.checked_mul(width).and_then(|b| b.checked_mul(height)).ok_or("a DWA block size overflows")?; + if payload.len() == raw { + return Ok(()); + } + let Some(counters) = payload.get(..COUNTERS) else { return Ok(()) }; + let counter = |i: usize| counters.get(i * 8..i * 8 + 8).and_then(|b| <[u8; 8]>::try_from(b).ok()).map_or(u64::MAX, u64::from_le_bytes); + let dct_blocks = width.div_ceil(8).checked_mul(height.div_ceil(8)).and_then(|n| n.checked_mul(channels)).ok_or("a DWA block size overflows")?; + let raw = raw as u64; + let bounds: [(usize, &str, u64); 5] = [ + (1, "unknown uncompressed size", raw), + (6, "RLE uncompressed size", raw.saturating_mul(2)), + (7, "RLE raw size", raw), + (8, "AC count", (dct_blocks as u64).saturating_mul(63)), + (9, "DC count", dct_blocks as u64), + ]; + for (i, name, most) in bounds { + let value = counter(i); + // (also keeps `count * 2` within a 32-bit usize: the bounds are block sizes, which fit) + if value > most || usize::try_from(value.saturating_mul(2)).is_err() { + return Err(format!("a DWA chunk declares a {name} of {value}, more than its {width} × {height} block can hold ({most})")); + } + } + Ok(()) +} + +/// The pixel size of the block the `exr` crate decompresses a chunk of the part `header` +/// describes into, from the chunk's coordinates: the crate's own placement +/// (`Header::get_block_data_indices`, then `get_absolute_block_pixel_coordinates`), so a scan-line +/// `y` the crate rounds into a block (it divides toward zero: rows just above the data window land +/// in block 0) is sized as that block. An error where the crate can't place the chunk. +fn block_size(header: &Header, block: &CompressedBlock) -> Result<(usize, usize), String> { + if let CompressedBlock::Tile(tile) = block { + // (the crate's level size asserts on levels of 64 and more; no valid file has 32) + if tile.coordinates.level_index.0 >= 32 || tile.coordinates.level_index.1 >= 32 { + return Err("a DWA tile of an impossible level".into()); + } + } + let tile = header.get_block_data_indices(block).map_err(|e| format!("a DWA chunk the decoder can't place: {e}"))?; + let bounds = header.get_absolute_block_pixel_coordinates(tile).map_err(|e| format!("a DWA chunk the decoder can't place: {e}"))?; + Ok((bounds.size.0, bounds.size.1)) +} + +/// Checks every chunk the offset tables point at that is, or names, a flat DWA part (`tables` is +/// where the offset tables start), before the `exr` crate's whole-image reader, which offers no +/// hook between reading a chunk and decompressing it, gets the bytes. It fails closed: a chunk of +/// a DWA part whose part number, coordinates, size or payload can't be read, or whose block can't +/// be placed, is an error, not left to the crate. +pub(crate) fn check_file(bytes: &[u8], meta: &exr::meta::MetaData, tables: usize) -> Result<(), String> { + if !meta.headers.iter().any(|h| !h.deep && is_dwa(h.compression)) { + return Ok(()); + } + let multipart = meta.requirements.is_multilayer(); + let word = |at: usize| bytes.get(at..at.checked_add(4)?).and_then(|b| <[u8; 4]>::try_from(b).ok()).map(i32::from_le_bytes); + let cut = || "a DWA chunk cut off by the end of the file".to_string(); + // Per part: its shape when it is a flat DWA part (worked out once, not per chunk). + let shapes: Vec> = meta + .headers + .iter() + .map(|h| if !h.deep && is_dwa(h.compression) { shape(h).ok_or("a DWA pixel size overflows").map(Some) } else { Ok(None) }) + .collect::>()?; + let mut table = tables; + for (table_part, header) in meta.headers.iter().enumerate() { + let entries = + bytes.get(table..table.saturating_add(header.chunk_count.saturating_mul(8))).ok_or("the offset tables run past the end of the file")?; + table = table.saturating_add(header.chunk_count.saturating_mul(8)); + for entry in entries.as_chunks::<8>().0 { + let offset = usize::try_from(u64::from_le_bytes(*entry)).ok(); + // The part the crate decodes the chunk as: the one it names (multi-part files). + let (part, mut at) = match (multipart, offset) { + (false, Some(at)) => (Some(table_part), at), + (true, Some(at)) => (word(at).and_then(|p| usize::try_from(p).ok()), at.saturating_add(4)), + (_, None) => (None, 0), + }; + let named = part.and_then(|p| shapes.get(p).copied().flatten().map(|s| (p, s))); + let Some((part, shape)) = named else { + // (not a DWA chunk, unless it sits in a DWA part's table and can't be read) + if shapes.get(table_part).copied().flatten().is_some() && part.is_none_or(|p| p >= meta.headers.len()) { + return Err(cut()); + } + continue; + }; + let header = meta.headers.get(part).ok_or_else(cut)?; + let block = match header.blocks { + exr::meta::BlockDescription::ScanLines => { + let y = word(at).ok_or_else(cut)?; + at = at.saturating_add(4); + CompressedBlock::ScanLine(CompressedScanLineBlock { y_coordinate: y, compressed_pixels_le: Vec::new() }) + } + exr::meta::BlockDescription::Tiles(_) => { + let tile = [word(at), word(at.saturating_add(4)), word(at.saturating_add(8)), word(at.saturating_add(12))]; + at = at.saturating_add(16); + let [Some(x), Some(y), Some(lx), Some(ly)] = tile else { return Err(cut()) }; + let index = |v: i32| usize::try_from(v).map_err(|_| "a DWA tile at a negative position".to_string()); + let coordinates = TileCoordinates { tile_index: Vec2(index(x)?, index(y)?), level_index: Vec2(index(lx)?, index(ly)?) }; + CompressedBlock::Tile(CompressedTileBlock { coordinates, compressed_pixels_le: Vec::new() }) + } + }; + let size = block_size(header, &block)?; + let len = word(at).ok_or_else(cut)?; + let len = usize::try_from(len).map_err(|_| "a DWA chunk declaring a negative size".to_string())?; + let start = at.saturating_add(4); + let payload = bytes.get(start..start.saturating_add(len)).ok_or_else(cut)?; + check_shaped(shape, size, payload)?; + } + } + Ok(()) +} diff --git a/crates/exr/src/lib.rs b/crates/exr/src/lib.rs index 78afd427e..9ff66f95d 100644 --- a/crates/exr/src/lib.rs +++ b/crates/exr/src/lib.rs @@ -15,6 +15,8 @@ //! - [`decode`]: bounded part listing, flat decoding (scan lines and tiles, every compression the //! `exr` crate reads) and writing, on top of the `exr` crate. //! - [`deep`]: deep scan lines and tiles (NONE, RLE, ZIPS) and their flattening. +//! - [`seek`]: any channels, display rows and reduced resolution of a file's flat parts, read from +//! a seekable source and delivered band by band (or collected into planes). //! - [`stream`]: one flat scan-line part read block by block, for pictures larger than memory. //! - [`cryptomatte_samples`]: a file's Cryptomatte layers and their per-pixel (ID, coverage) ranks. @@ -36,4 +38,10 @@ pub mod deep; #[cfg(all(test, feature = "decode"))] mod deep_tests; #[cfg(feature = "decode")] +mod dwa_guard; +#[cfg(feature = "decode")] +pub mod seek; +#[cfg(all(test, feature = "decode"))] +mod seek_tests; +#[cfg(feature = "decode")] pub mod stream; diff --git a/crates/exr/src/seek.rs b/crates/exr/src/seek.rs new file mode 100644 index 000000000..03f0fa506 --- /dev/null +++ b/crates/exr/src/seek.rs @@ -0,0 +1,1551 @@ +//! Selective reading of flat OpenEXR parts from a seekable source (feature `decode`): the headers +//! are indexed once, then any channels, any display rows and a reduced resolution are read from +//! the file without loading the rest of it. +//! +//! - [`Index::build`] reads the headers and the offset tables only, through `Read + Seek`, after +//! checking what they declare against [`Limits`] and the source's length. It allocates nothing +//! sized by pixels, and the index holds no pixel data, so it can be kept per file. +//! - [`read_bands`] reads what a [`Request`] asks for and delivers it incrementally: each +//! compression block becomes [`Band`]s (one per requested channel: a rectangle of rows) handed +//! to a callback on the calling thread, in file order. Blocks are decoded on worker threads; the +//! calling thread hands over what has been decoded each time it has read a piece of the file +//! and while it waits, so the first bands arrive long before the request has been read. A +//! caller that writes into tiles or shows lines while the rest is still being read never holds +//! a whole plane. +//! - [`read_planes`] is a collector over [`read_bands`]: the same request delivered as one +//! display-window plane per channel once everything is read. +//! +//! A request names channels as `(part, channel)` pairs (positions in [`Index::parts`]), a range of +//! display-window rows, and a level: level `n` keeps every `2^n`-th row and column of the display +//! window, so a reduced picture is made without allocating the full-resolution one. Rows and band +//! positions count from the display window's top-left corner (0, 0), not in the file's absolute +//! coordinates (which [`PartInfo`]'s windows use). Only the chunks that hold a requested channel +//! and a requested row are fetched, chunks that follow each other in the file in one read, and +//! they are decompressed on several threads while the next ones are being read. Samples are +//! converted to `f32` (integer samples above 2^24 lose precision); outside a part's data window +//! (and in rows no chunk holds) a plane has the channel's fill value. +//! +//! A tile writer (an image editor storing a picture in tiles at its own depth) uses it like this: +//! [`Index::build`], then [`Index::estimate`] to reserve the working memory and size a progress +//! bar (one step per chunk), then [`read_bands_try`] with the colour channels as one group +//! ([`Request::groups`]), so each block arrives as one band of interleaved pixels to write into +//! the tiles it covers. Its sink checks the tile budget or scratch disk before each band and +//! returns its own error to stop the read; the read winds down and hands that error back. +//! +//! Deep parts are not read here (see [`crate::deep`]); a flat part beside them is. Tiled parts +//! are read from their largest resolution level. Every size a header declares is checked before +//! it is used and every allocation is bounded by a limit or reserved fallibly. +//! +//! Panics: the `exr` crate's header parser and decompressors can panic on some damaged input. +//! Where panics unwind (native targets), those calls are wrapped in `catch_unwind` and a panic +//! becomes [`Error::Malformed`]. Where panics abort (`wasm32-unknown-unknown`), `catch_unwind` +//! does nothing, so this module checks beforehand what it knows the crate would panic on: the +//! chunk layout is computed here with checked arithmetic instead of the crate's (which panics on +//! some sizes), the crate's block-size formula is checked for overflow, and a chunk's part, +//! coordinates, level and declared size are checked before the crate reads it. A panic inside +//! the crate's header +//! parsing or a decompressor on damaged data is not ruled out there. + +use std::io::{Cursor, Read, Seek, SeekFrom}; +use std::ops::Range; +use std::panic::{AssertUnwindSafe, catch_unwind}; +use std::sync::mpsc::{self, Receiver, RecvTimeoutError, Sender}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use exr::block::UncompressedBlock; +use exr::block::chunk::Chunk; +use exr::math::Vec2; +use exr::meta::MetaData; +use exr::meta::attribute::SampleType as ExrSample; +use exr::meta::header::Header; + +use crate::decode::{Limits, PartInfo, Window, check_previews, part_info}; + +/// Why a read failed. More kinds may be added: a `match` needs a catch-all arm. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +#[non_exhaustive] +pub enum Error { + /// The file is damaged, or a chunk can't be decompressed. + #[error("OpenEXR: {0}")] + Malformed(String), + /// A valid file this reader does not cover (subsampled channels, channels of deep parts). + #[error("unsupported OpenEXR: {0}")] + Unsupported(String), + /// A valid file whose compression this reader does not decode (HTJ2K): the name of the + /// compression. An app may hand such a file to another decoder. + #[error("unsupported OpenEXR compression: {0}")] + UnsupportedCompression(String), + /// The file declares more than [`Limits`] (or this module's own bounds) allow, or the request + /// needs more than [`Request::max_bytes`]: a lasting refusal of this file or request. + #[error("OpenEXR exceeds a limit: {0}")] + LimitExceeded(String), + /// The system could not give memory the read needed (within its limits) just now: another + /// attempt, with more memory free, may succeed. + #[error("OpenEXR: out of memory: {0}")] + OutOfMemory(String), + /// The source failed just now (a seek or read error other than the file ending early): + /// another attempt may succeed. + #[error("OpenEXR: cannot read: {0}")] + Io(String), + /// The `cancel` callback asked to stop. + #[error("OpenEXR: cancelled")] + Cancelled, +} + +/// Result of this module's functions. +pub type Result = std::result::Result; + +fn malformed(e: impl std::fmt::Display) -> Error { + Error::Malformed(e.to_string()) +} + +fn io(e: impl std::fmt::Display) -> Error { + Error::Io(e.to_string()) +} + +/// A failed `read_exact` of the headers or tables: the file ending early is a damaged file, any +/// other error is the source failing. +fn read_failed(e: std::io::Error) -> Error { + if e.kind() == std::io::ErrorKind::UnexpectedEof { malformed(e) } else { io(e) } +} + +fn exr_error(e: exr::error::Error) -> Error { + match e { + exr::error::Error::NotSupported(message) => Error::Unsupported(message.to_string()), + exr::error::Error::Io(e) => io(e), + e => malformed(e), + } +} + +const MAGIC: [u8; 4] = [0x76, 0x2f, 0x31, 0x01]; +/// Headers of all parts together: at most this many bytes, parts, and attributes per part. +const META_BYTES: usize = 8 << 20; +const MAX_PARTS: usize = 1024; +const MAX_ATTRIBUTES: usize = 4096; +/// The most chunks all parts of a file may have together. The offset tables and the index hold +/// a few bytes per chunk, so a header declaring billions of tiny tiles (whose tables a short file +/// can't hold anyway) is refused before anything is sized by it. 16 million chunks is a 64K × +/// 64K picture in 16 × 16 tiles. +const MAX_CHUNKS: usize = 1 << 24; +/// The most bytes one decompressed block may take in a read that sets no +/// [`Request::max_block_bytes`] and whose [`Request::max_bytes`] is smaller: the decompressor +/// allocates a block's bytes before it looks at the data, so a hostile header (a data window a +/// billion pixels wide over one tiny chunk) would otherwise ask for gigabytes. Such a read allows +/// blocks up to `max(this, max_bytes)`: a caller that budgets more memory can read files with +/// larger blocks (wide DWAB parts with many channels hold 512 MiB a block). +const MAX_BLOCK_BYTES: usize = 256 << 20; +/// Chunks are read in pieces: a run of chunks that follow each other in the file, read with one +/// request (one round trip to a network share, not one per chunk). The first piece is small so +/// the first bands arrive early; each later one may be twice as large, up to [`PIECE_BYTES`], so +/// a long read makes few requests. A piece holds at most [`PIECE_CHUNKS`] chunks, and a single +/// chunk larger than the limit is read alone. +const FIRST_PIECE_BYTES: u64 = 256 << 10; +const PIECE_BYTES: u64 = 4 << 20; +const PIECE_CHUNKS: usize = 256; +/// The most decoded bytes a read keeps in flight: the decompressed blocks and the bands of the +/// chunks read but not yet handed to the sink. Compressed bytes say little about this (a flat +/// colour compresses a thousandfold), so the reader estimates each chunk's decoded size from its +/// header and waits for the sink to catch up before reading more. +const IN_FLIGHT_BYTES: u64 = 64 << 20; +/// How often a wait for the workers polls `cancel`. +const CANCEL_POLL: Duration = Duration::from_millis(50); + +/// One chunk of a part's full-resolution level, as [`Index::chunks`] lists it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ChunkInfo { + /// Where the chunk starts in the file. + pub offset: u64, + /// The bytes read for it: up to the next chunk in the file (or the file's end), at most a + /// block's bytes plus its header; 0 when its offset lies outside the chunk area (past the + /// file's end, or in the headers or offset tables): a read that needs it fails. + pub len: u64, + /// The column of its top-left pixel in the part's data window, counted from the data + /// window's left edge (0). + pub x: usize, + /// The row of its top-left pixel, counted from the data window's top edge (0). + pub y: usize, + /// Its width in pixels (a tile at the right edge, or a scan-line block, may be narrower than + /// the tile size; a scan-line block is as wide as the data window). + pub width: usize, + /// Its height in pixels (the last scan-line block or the bottom tiles may hold fewer rows). + pub height: usize, +} + +/// One chunk of the full-resolution level of a part, in offset-table order. +#[derive(Debug, Clone, Copy)] +struct ChunkRef { + /// Where it starts in the file, and how many bytes to read for it (0: its offset is not valid, + /// it is never read). + offset: u64, + len: u64, + /// Its top-left pixel in the part's data window (0-based) and its size. + x: usize, + y: usize, + cols: usize, + rows: usize, + /// Its tile column and row (scan-line blocks: column 0, row = block number). + tile: (usize, usize), +} + +/// Where one part's samples go in the display window, and its chunks. +#[derive(Debug, Clone)] +struct PartLayout { + /// The part's data window position in the display window. + offset: (i64, i64), + /// The data window's size. + size: (usize, usize), + /// The bytes of one pixel of all its channels, uncompressed. + pixel_bytes: usize, + /// The chunks of the full-resolution level (smaller levels of tiled parts are never read). + chunks: Vec, +} + +/// What an OpenEXR file holds, from its headers and offset tables: the facts [`read_bands`] needs +/// to fetch only the chunks a request touches. Holds no pixel data. +#[derive(Debug, Clone)] +pub struct Index { + meta: MetaData, + parts: Vec, + layout: Vec, + display: Window, + len: u64, + /// [`Limits::max_ratio`] at build: how many bytes one byte of a chunk may decompress to. + max_ratio: u64, +} + +impl Index { + /// Reads the headers and offset tables of the file in `source` and checks them against + /// `limits`: the window and pixel limits. `max_bytes` bounds what the whole file may decode + /// to in [`crate::decode`] and plays no part here (a read is bounded by + /// [`Request::max_bytes`]); `max_ratio` is kept in the index, and a read refuses a chunk whose + /// block is more than its bytes can decompress to (see [`Index::check_blocks`]). Nothing is + /// sized by pixels. + /// `cancel` is polled while reading. + /// The display window and every data window are held to `limits.max_pixels`; see + /// [`Index::build_with_display_limit`] to hold the display window to its own bound. + pub fn build(source: &mut R, limits: &Limits, cancel: &dyn Fn() -> bool) -> Result { + Index::build_with_display_limit(source, limits, limits.max_pixels, cancel) + } + + /// [`Index::build`] with the display window held to `max_display_pixels` instead of + /// `limits.max_pixels`, which then bounds the data windows only: an app may allow parts larger + /// than the picture it shows (overscan) while keeping the planes it allocates smaller. + pub fn build_with_display_limit( + source: &mut R, + limits: &Limits, + max_display_pixels: u64, + cancel: &dyn Fn() -> bool, + ) -> Result { + let len = source.seek(SeekFrom::End(0)).map_err(io)?; + source.seek(SeekFrom::Start(0)).map_err(io)?; + let prefix = read_headers(source, cancel)?; + check_previews(&prefix).map_err(malformed)?; + // (the crate's own parsing panics on some malformed headers: reported, never raised, + // where panics unwind) + let meta = catch_unwind(AssertUnwindSafe(|| MetaData::read_from_buffered(Cursor::new(&prefix), false))) + .map_err(|_| malformed("invalid OpenEXR headers"))? + .map_err(exr_error)?; + if meta.headers.is_empty() || meta.headers.len() > MAX_PARTS { + return Err(malformed("part count")); + } + for (i, h) in meta.headers.iter().enumerate() { + if matches!(h.compression, exr::prelude::Compression::HTJ2K32 | exr::prelude::Compression::HTJ2K256) { + return Err(Error::UnsupportedCompression("HTJ2K".into())); + } + check_part(i, h, limits)?; + } + let first = meta.headers.first().ok_or_else(|| malformed("missing header"))?; + let display = first.shared_attributes.display_window; + let (dw, dh) = (display.size.0, display.size.1); + let display_pixels = (dw as u64).saturating_mul(dh as u64); + if dw == 0 || dh == 0 || dw > limits.max_width as usize || dh > limits.max_height as usize || display_pixels > max_display_pixels { + return Err(Error::LimitExceeded(format!("display window of {dw}x{dh}"))); + } + let tables = read_tables(source, &meta, prefix.len() as u64, len, cancel)?; + let chunks_start = prefix.len() as u64 + tables.iter().map(|t| t.len() as u64 * 8).sum::(); + let layout = layout_of(&meta, &tables, chunks_start, len)?; + let mut parts: Vec = meta.headers.iter().enumerate().map(|(i, h)| part_info(i, h)).collect(); + // (an index stays small however much a renderer writes into the headers: the text + // attributes are kept in the part infos, the crate's own copies are dropped) + let mut meta = meta; + for h in &mut meta.headers { + h.shared_attributes.other.clear(); + h.own_attributes.other.clear(); + h.own_attributes.preview = None; + } + for (info, h) in parts.iter_mut().zip(&meta.headers) { + info.deep = h.deep; + } + let display = parts.first().map_or(Window { x: 0, y: 0, width: 0, height: 0 }, |p| p.display_window); + Ok(Index { meta, parts, layout, display, len, max_ratio: limits.max_ratio }) + } + + /// Every part's header facts: channels (in the order a request's `(part, channel)` pairs + /// count them), windows (in the file's absolute coordinates), compression and text attributes. + pub fn parts(&self) -> &[PartInfo] { + &self.parts + } + + /// The display window of the file (the first part's, in absolute coordinates): every plane + /// covers it, and a request's rows and a band's position count from its top-left corner. + pub fn display(&self) -> Window { + self.display + } + + /// The chunks of part `part`'s full-resolution level, in offset-table order (none for a deep + /// part or a part that doesn't exist): where each lies in the file and which pixels of the + /// part's data window it holds. For diagnostics and tests; [`read_bands`] picks the chunks a + /// request needs by itself. + pub fn chunks(&self, part: usize) -> impl Iterator + '_ { + self.layout.get(part).map_or(&[][..], |p| p.chunks.as_slice()).iter().map(|c| ChunkInfo { + offset: c.offset, + len: c.len, + x: c.x, + y: c.y, + width: c.cols, + height: c.rows, + }) + } + + /// Checks every chunk of the full-resolution level of every flat part against what its bytes + /// can decompress to (see [`Index::build`]'s `max_ratio`), from the index alone: an app can + /// refuse a file before another decoder allocates by its headers. A chunk's bytes are counted + /// here as the gap to the next chunk in the file (the index holds no payload sizes); a read + /// makes the same check for the chunks it needs, and again, once a chunk is read, against the + /// payload size the chunk declares, and also holds each block to its own byte limit. + pub fn check_blocks(&self) -> Result<()> { + for (pi, part) in self.layout.iter().enumerate() { + let Some(header) = self.meta.headers.get(pi).filter(|h| !h.deep) else { continue }; + for c in &part.chunks { + self.check_ratio(pi, header, part, c, c.len)?; + } + } + Ok(()) + } + + /// Refuses chunk `c` of part `pi` when its decompressed block is more than `len` bytes can + /// hold: at most `max_ratio` bytes per byte (four times that for DWA, whose RLE stage, up to + /// 64 : 1, and zlib, up to about 1032 : 1, together reach about 66 000 : 1), plus 1 MiB. A + /// hostile header can declare a block of gigabytes over a chunk of a few bytes, and the + /// decompressors allocate the whole block first. + fn check_ratio(&self, pi: usize, header: &Header, part: &PartLayout, c: &ChunkRef, len: u64) -> Result<()> { + let block = (c.cols as u64).saturating_mul(c.rows as u64).saturating_mul(part.pixel_bytes as u64); + let ratio = if crate::dwa_guard::is_dwa(header.compression) { self.max_ratio.saturating_mul(4) } else { self.max_ratio }; + let most = len.checked_mul(ratio).and_then(|b| b.checked_add(1 << 20)).unwrap_or(u64::MAX); + if block > most { + return Err(Error::LimitExceeded(format!( + "a block of part {pi} decompresses to {block} bytes, more than its {len}-byte chunk can hold ({most})" + ))); + } + Ok(()) + } + + /// The file's length when it was indexed. + pub fn file_len(&self) -> u64 { + self.len + } + + /// About the bytes the index holds in memory. + pub fn bytes(&self) -> usize { + let chunks: usize = self.layout.iter().map(|p| p.chunks.len()).sum(); + let infos: usize = self + .parts + .iter() + .map(|p| { + let channels: usize = p.channels.iter().map(|c| c.name.len() + 48).sum(); + let text: usize = p.text_attributes.iter().map(|(k, v)| k.len() + v.len() + 48).sum(); + channels + text + p.compression.len() + 256 + }) + .sum(); + chunks.saturating_mul(std::mem::size_of::()).saturating_add(infos).saturating_add(self.meta.headers.len().saturating_mul(1024)) + } +} + +/// The headers of the file in `r` (up to and including the empty name that ends them), as raw +/// bytes: bounded in size, parts and attributes before anything parses them. +fn read_headers(r: &mut R, cancel: &dyn Fn() -> bool) -> Result> { + let mut start = [0u8; 8]; + r.read_exact(&mut start).map_err(read_failed)?; + if start.get(..4) != Some(&MAGIC[..]) { + return Err(malformed("OpenEXR signature")); + } + let version = start.get(4..8).and_then(|v| <[u8; 4]>::try_from(v).ok()).map(u32::from_le_bytes).ok_or_else(|| malformed("version"))?; + let multipart = version & 0x1000 != 0; + let mut prefix = start.to_vec(); + let mut headers = 0usize; + let mut pending: Option = None; + loop { + let mut attributes = 0usize; + // One header: attributes (name, type, size, value) up to an empty name. + loop { + if cancel() { + return Err(Error::Cancelled); + } + if !read_name(r, &mut prefix, pending.take())? { + break; + } + attributes += 1; + if attributes > MAX_ATTRIBUTES { + return Err(Error::LimitExceeded("more than 4096 attributes in a header".into())); + } + read_name(r, &mut prefix, None)?; + let mut size = [0u8; 4]; + r.read_exact(&mut size).map_err(read_failed)?; + prefix.extend_from_slice(&size); + let n = usize::try_from(i32::from_le_bytes(size)).map_err(malformed)?; + let end = + prefix.len().checked_add(n).filter(|end| *end <= META_BYTES).ok_or_else(|| Error::LimitExceeded("headers over 8 MiB".into()))?; + let begin = prefix.len(); + prefix.try_reserve_exact(n).map_err(|_| Error::OutOfMemory("header allocation".into()))?; + prefix.resize(end, 0); + for bytes in prefix.get_mut(begin..).ok_or_else(|| malformed("attribute size"))?.chunks_mut(64 << 10) { + if cancel() { + return Err(Error::Cancelled); + } + r.read_exact(bytes).map_err(read_failed)?; + } + } + headers += 1; + if headers > MAX_PARTS { + return Err(Error::LimitExceeded("more than 1024 parts".into())); + } + if !multipart { + return Ok(prefix); + } + // A multi-part header list ends with one more empty name. + let mut next = [0u8; 1]; + r.read_exact(&mut next).map_err(read_failed)?; + match next.first().copied() { + Some(0) | None => { + prefix.push(0); + return Ok(prefix); + } + Some(b) => pending = Some(b), + } + } +} + +/// One NUL-terminated name (at most 255 bytes) appended to `prefix`, `first` already read; `false` +/// for the empty name that ends a header. +fn read_name(r: &mut R, prefix: &mut Vec, first: Option) -> Result { + let mut i = 0usize; + let mut next = first; + loop { + let b = match next.take() { + Some(b) => b, + None => { + let mut one = [0u8; 1]; + r.read_exact(&mut one).map_err(read_failed)?; + one.first().copied().unwrap_or(0) + } + }; + prefix.push(b); + if b == 0 { + return Ok(i != 0); + } + i += 1; + if i > 255 { + return Err(Error::LimitExceeded("an attribute name over 255 bytes".into())); + } + } +} + +/// What a header may declare before any chunk of the part is sized by it. +fn check_part(index: usize, h: &Header, limits: &Limits) -> Result<()> { + if h.deep { + // (flat parts beside it are read; a request for a channel of this one is refused) + return Ok(()); + } + let (w, ht) = (h.layer_size.0, h.layer_size.1); + if w == 0 || ht == 0 { + return Err(malformed(format!("empty data window of part {index}"))); + } + if w > limits.max_width as usize || ht > limits.max_height as usize || (w as u64).saturating_mul(ht as u64) > limits.max_pixels { + return Err(Error::LimitExceeded(format!("part {index} of {w}x{ht}"))); + } + if h.channels.list.is_empty() || h.channels.list.len() > 4096 { + return Err(malformed(format!("channel count of part {index}"))); + } + // (the decompressor's actual block is no larger than the crate's bound, so it fits too; how + // large a block may be depends on the read: see `read_bands`) + if !crate_bound_fits(h) { + return Err(malformed(format!("the block size of part {index} overflows"))); + } + Ok(()) +} + +/// The bytes of all channels of one pixel of `h`, checked. +fn pixel_bytes(h: &Header) -> Option { + h.channels.list.iter().map(|c| c.sample_type.bytes_per_sample()).try_fold(0usize, usize::checked_add) +} + +/// The bytes of the largest block the decompressor allocates for a chunk of `h`: a pixel's bytes +/// times a tile's area, or times the width and the rows of a scan-line block (no more than the +/// part has). `None` when it overflows. +fn block_bytes(h: &Header) -> Option { + let pixels = match h.blocks { + exr::meta::BlockDescription::Tiles(t) => t.tile_size.0.checked_mul(t.tile_size.1)?, + exr::meta::BlockDescription::ScanLines => h.compression.scan_lines_per_block().min(h.layer_size.1).checked_mul(h.layer_size.0)?, + }; + pixel_bytes(h)?.checked_mul(pixels) +} + +/// Whether the `exr` crate's own bound on a chunk of `h` (`Header::max_block_byte_size`: like +/// [`block_bytes`] but with every scan line a block may hold, even past the part's last row) can +/// be computed: the crate computes it without overflow checks. It is not a limit here: before +/// the crate reads a chunk, [`check_chunk_header`] has made sure the size the chunk declares is +/// no more than the bytes it holds, so the crate never allocates by that bound. +fn crate_bound_fits(h: &Header) -> bool { + let pixels = match h.blocks { + exr::meta::BlockDescription::Tiles(t) => t.tile_size.0.checked_mul(t.tile_size.1), + exr::meta::BlockDescription::ScanLines => h.compression.scan_lines_per_block().checked_mul(h.layer_size.0), + }; + pixels.zip(pixel_bytes(h)).and_then(|(pixels, bytes)| pixels.checked_mul(bytes)).is_some() +} + +/// The offset tables, one per part, read from where the headers end. +fn read_tables(r: &mut R, meta: &MetaData, headers_end: u64, len: u64, cancel: &dyn Fn() -> bool) -> Result>> { + let chunks: usize = meta.headers.iter().map(|h| h.chunk_count).fold(0, usize::saturating_add); + if chunks > MAX_CHUNKS { + return Err(Error::LimitExceeded(format!("{chunks} chunks, more than {MAX_CHUNKS}"))); + } + // Eight bytes of table per chunk: a header can't claim more chunks than the file holds. + if (chunks as u64).saturating_mul(8) > len.saturating_sub(headers_end) { + return Err(malformed("offset tables longer than the file")); + } + r.seek(SeekFrom::Start(headers_end)).map_err(io)?; + let mut tables = Vec::new(); + tables.try_reserve_exact(meta.headers.len()).map_err(|_| Error::OutOfMemory("offset table allocation".into()))?; + let mut buf = vec![0u8; 8 * 8192]; + for h in &meta.headers { + let mut table: Vec = Vec::new(); + table.try_reserve_exact(h.chunk_count).map_err(|_| Error::OutOfMemory("offset table allocation".into()))?; + let mut left = h.chunk_count; + while left > 0 { + if cancel() { + return Err(Error::Cancelled); + } + let take = left.min(8192); + let bytes = buf.get_mut(..take * 8).ok_or_else(|| malformed("table buffer"))?; + r.read_exact(bytes).map_err(read_failed)?; + table.extend(bytes.as_chunks::<8>().0.iter().map(|c| u64::from_le_bytes(*c))); + left -= take; + } + tables.push(table); + } + Ok(tables) +} + +/// Each part's chunks of the full-resolution level: where they are and which pixels they hold. A +/// chunk ends where the next one in the file starts (or at the end of the file); an offset +/// outside the chunk area is kept as a chunk that is never read. +/// +/// The layout is computed here rather than with the `exr` crate's block iterator, which allocates +/// for every level and panics on some sizes. The full-resolution level comes first in the offset +/// table of every level mode (scan lines, single-level tiles, mip and rip maps), its blocks row by +/// row and left to right. +fn layout_of(meta: &MetaData, tables: &[Vec], chunks_start: u64, len: u64) -> Result> { + let allocation = |_| Error::OutOfMemory("chunk layout allocation".into()); + let first = meta.headers.first().ok_or_else(|| malformed("missing header"))?; + let display = first.shared_attributes.display_window; + let mut starts: Vec = Vec::new(); + starts.try_reserve_exact(tables.iter().map(Vec::len).fold(0, usize::saturating_add)).map_err(allocation)?; + starts.extend(tables.iter().flatten().copied().filter(|o| *o >= chunks_start && *o < len)); + starts.sort_unstable(); + starts.dedup(); + let end_of = |offset: u64| match starts.binary_search(&offset) { + Ok(i) => starts.get(i + 1).copied().unwrap_or(len), + Err(_) => offset, + }; + let mut out = Vec::new(); + out.try_reserve_exact(meta.headers.len()).map_err(allocation)?; + for (pi, h) in meta.headers.iter().enumerate() { + let pos = h.own_attributes.layer_position; + let offset = (i64::from(pos.0) - i64::from(display.position.0), i64::from(pos.1) - i64::from(display.position.1)); + if h.deep { + out.push(PartLayout { offset, size: (0, 0), pixel_bytes: 0, chunks: Vec::new() }); + continue; + } + let table = tables.get(pi).ok_or_else(|| malformed("offset table"))?; + // A chunk is at most its block's bytes plus its own small header (part number, position + // and sizes): never read more for one. + let most = (block_bytes(h).ok_or_else(|| malformed("block size"))? as u64).saturating_add(64); + let (w, ht) = (h.layer_size.0, h.layer_size.1); + let (bw, bh) = match h.blocks { + exr::meta::BlockDescription::Tiles(t) => (t.tile_size.0, t.tile_size.1), + exr::meta::BlockDescription::ScanLines => (w, h.compression.scan_lines_per_block()), + }; + if bw == 0 || bh == 0 { + return Err(malformed(format!("block size of part {pi}"))); + } + let across = w.div_ceil(bw); + let count = + across.checked_mul(ht.div_ceil(bh)).filter(|n| *n <= table.len()).ok_or_else(|| malformed(format!("chunk count of part {pi}")))?; + let mut chunks = Vec::new(); + chunks.try_reserve_exact(count).map_err(allocation)?; + for (i, at) in table.iter().copied().enumerate().take(count) { + let tile = (i % across, i / across); + // (inside the data window: a tile index is below the count of blocks across or down) + let (x, y) = (tile.0.saturating_mul(bw), tile.1.saturating_mul(bh)); + let valid = at >= chunks_start && at < len; + let bytes = if valid { end_of(at).saturating_sub(at).min(most) } else { 0 }; + chunks.push(ChunkRef { offset: at, len: bytes, x, y, cols: bw.min(w.saturating_sub(x)), rows: bh.min(ht.saturating_sub(y)), tile }); + } + let pixel_bytes = pixel_bytes(h).ok_or_else(|| malformed("pixel size"))?; + out.push(PartLayout { offset, size: (w, ht), pixel_bytes, chunks }); + } + Ok(out) +} + +/// What to read. +#[derive(Debug, Clone)] +pub struct Request<'a> { + /// The channels wanted as `(part, channel)` (positions in [`Index::parts`] and in the part's + /// [`PartInfo::channels`]), in the order [`Band::slot`] and the planes of [`read_planes`] count + /// them. A channel may be named more than once (each slot gets its own bands). A channel of a + /// deep part is refused. + pub channels: &'a [(usize, usize)], + /// The value of each requested channel (same order) where the file has no samples; missing + /// entries count as 0. Only [`read_planes`] uses it ([`read_bands`] delivers samples only). + pub fill: &'a [f32], + /// The rows to read, counted from the display window's top row (0, whatever the display + /// window's absolute position), at full resolution; `None`: all. + pub rows: Option>, + /// Every `2^level`-th row and column of the display window (0: full resolution, at most 16). + pub level: u32, + /// The most bytes of samples the read may produce: [`read_bands`] checks the requested rows + /// at the level, [`read_planes`] the whole display-window planes it allocates. Without + /// [`Request::max_block_bytes`], it also raises the most one decompressed block may take + /// above the 256 MiB default. + pub max_bytes: usize, + /// The most threads that decompress (`None`: as many as the machine runs at once; `Some(0)`: + /// none, everything is decoded on the calling thread). Never more than the chunks need. + pub workers: Option, + /// The most bytes one decompressed block may take (`None`: `max(256 MiB, max_bytes)`). A + /// needed chunk whose block is larger is refused before anything is read. + /// + /// It is a limit of its own because the `exr` crate's decompressors allocate a whole block at + /// once and can't report that the memory isn't there: the process aborts. A block must stay + /// below what the caller is sure the system can give, even when the read's total budget + /// (`max_bytes`, which allocations this module makes fallibly count against) is far larger. A + /// block is one strip of the file (at most 256 scan lines, for DWAB): 8 GiB is a 16K-wide + /// strip of 256 lines of 512 f32 channels. + pub max_block_bytes: Option, + /// Runs of request slots delivered together, as one band per block whose pixels hold the + /// slots' values in slot order (`0..4` over R, G, B, A of a part: one RGBA band): for a caller + /// that stores pixels interleaved. Every slot of a group must name a channel of the same part + /// (they then share every block), and no slot may be in two groups; slots in no group come one + /// band each, as without groups. Empty: no groups. [`read_planes`] ignores it. + pub groups: &'a [Range], +} + +impl<'a> Request<'a> { + /// Every row of `channels` at full resolution, fill 0, up to `max_bytes` of samples, with as + /// many decoding threads as the machine runs at once. + pub fn new(channels: &'a [(usize, usize)], max_bytes: usize) -> Request<'a> { + Request { channels, fill: &[], rows: None, level: 0, max_bytes, workers: None, max_block_bytes: None, groups: &[] } + } +} + +/// The size of the planes of a read at `level` of a `width` × `height` display window. +pub fn level_size((width, height): (usize, usize), level: u32) -> (usize, usize) { + let stride = 1usize << level.min(16); + (width.div_ceil(stride), height.div_ceil(stride)) +} + +/// A rectangle of rows of one requested channel, or of a group of them ([`Request::groups`]), +/// decoded: `samples` holds `height` rows of `width` pixels, row by row, each pixel +/// `values_per_pixel` values in slot order. `x` and `y` are in the output grid, counted from the +/// display window's top-left corner (0, 0, not its absolute position): display-window pixels at +/// level 0, every `2^level`-th pixel above. +#[derive(Debug, Clone, PartialEq)] +pub struct Band { + /// The position of the channel in [`Request::channels`]; for a group, its first slot. + pub slot: usize, + pub x: usize, + pub y: usize, + pub width: usize, + pub height: usize, + /// The values per pixel: 1, or the size of the group the band is of. + pub values_per_pixel: usize, + /// The chunk the band was decoded from: its number in the read, from 0 in file order, counting + /// the chunks [`Index::estimate`] counts. A chunk may give no band (none of its rows on the + /// level's grid, say) or several; the bands of a chunk come together, so progress is the + /// highest `chunk` seen plus one, out of [`Estimate::chunks`]. + pub chunk: usize, + pub samples: Vec, +} + +/// What a read did, for measurements and tests. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct Stats { + /// Chunks fetched, per part. + pub chunks: Vec, + /// Bytes of chunks read from the source. + pub bytes: u64, + /// Scan lines (or tile rows) decompressed. + pub lines: usize, + /// The most decoded bytes the read had in flight at once, as estimated from the headers: the + /// decompressed blocks and the bands of the chunks read but not yet handed to the sink. + pub in_flight: u64, +} + +/// What a read will take, from [`Index::estimate`], before anything is read. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct Estimate { + /// The chunks it fetches: one unit of progress each (a read hands every chunk's bands over + /// together). + pub chunks: usize, + /// The bytes of those chunks it reads from the source. + pub compressed_bytes: u64, + /// An upper estimate of the most memory the read holds at once besides what the sink keeps: + /// the decoded bytes in flight (as [`Stats::in_flight`] counts them: each chunk's + /// decompressed block and its bands, exactly as large as they are made, grouped or not), as + /// much again for the decompressors' scratch, two pieces of compressed bytes, and each + /// worker's copy of the chunk it decodes. + pub working_bytes: u64, + /// The bytes of the planes [`read_planes`] would allocate for the request (whole display + /// window planes, one per slot: it ignores [`Request::groups`]). Not part of `working_bytes`, + /// and not checked here: [`read_planes`] checks it against [`Request::max_bytes`] when it + /// runs, a caller that collects bands itself checks what it allocates. + pub planes_bytes: u64, +} + +/// The planes of [`read_planes`]: one per requested channel, `width` × `height` (the display +/// window at the request's level). +#[derive(Debug, Clone, PartialEq)] +pub struct Planes { + pub width: usize, + pub height: usize, + pub planes: Vec>, + pub stats: Stats, +} + +/// How a read with a fallible sink ([`read_bands_try`]) ended early. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Stop { + /// The read failed. + Read(Error), + /// The sink returned this error; nothing was handed to it after. + Sink(E), +} + +impl From for Stop { + fn from(e: Error) -> Self { + Stop::Read(e) + } +} + +impl std::fmt::Display for Stop { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Stop::Read(e) => e.fmt(f), + Stop::Sink(e) => e.fmt(f), + } + } +} + +impl std::error::Error for Stop { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + Stop::Read(e) => Some(e), + Stop::Sink(_) => None, + } + } +} + +/// The planes' geometry for [`chunk_bands`]. +struct Geometry<'a> { + /// The display window's size and the output's (at the level). + size: (usize, usize), + out: (usize, usize), + stride: i64, + /// The display rows read. + rows: Range, + /// Each part's channel → the request slots that name it. + slots: &'a [Vec>], + /// Per request slot: the band it goes into (the first slot of its group, or itself), that + /// band's values per pixel, and the slot's place among them. + units: &'a [(usize, usize, usize)], +} + +impl Geometry<'_> { + /// A display row is read: in the rows, on the level's grid. + fn wanted(&self, row: i64) -> bool { + row >= self.rows.start && row < self.rows.end && row.rem_euclid(self.stride) == 0 + } + + /// A chunk holding display rows `a..b` has a row read. + fn any_wanted(&self, a: i64, b: i64) -> bool { + let first = a.max(self.rows.start); + // (saturating: the rows are clamped to the display window, but the chunk's top may not be) + let first = first.saturating_add((self.stride - first.rem_euclid(self.stride)) % self.stride); + first < b.min(self.rows.end) + } + + /// How many rows are read (in the rows, on the level's grid). + fn rows_read(&self) -> usize { + let (a, b) = (self.rows.start.max(0), self.rows.end.max(0)); + let count = |n: i64| n.saturating_add(self.stride - 1) / self.stride; + usize::try_from(count(b).saturating_sub(count(a))).unwrap_or(0) + } +} + +/// A chunk the request needs, in the order the jobs are numbered (file order). +struct Planned { + part: usize, + offset: u64, + len: u64, + chunk: ChunkRef, + /// Its estimated decoded bytes: the decompressed block and the bands of the requested slots. + cost: u64, +} + +/// What a request needs, worked out from the index alone: the shared first step of a read and of +/// [`Index::estimate`]. +struct Plan { + /// Per part and channel: the request slots that name it. + slots: Vec>>, + size: (usize, usize), + out: (usize, usize), + stride: i64, + rows: Range, + /// The chunks to read, in file order. + jobs: Vec, + /// The decoding threads to start. + workers: usize, + /// Per request slot: see [`Geometry::units`]. + units: Vec<(usize, usize, usize)>, +} + +impl Plan { + fn geometry(&self) -> Geometry<'_> { + Geometry { size: self.size, out: self.out, stride: self.stride, rows: self.rows.clone(), slots: &self.slots, units: &self.units } + } +} + +/// Checks `request` against `index` and lists the chunks it needs (see [`read_bands`] for the +/// checks): nothing is read. +fn plan(index: &Index, request: &Request) -> Result { + let level = request.level.min(16); + let stride = 1i64 << level; + let (w, h) = (index.display.width as usize, index.display.height as usize); + let out = level_size((w, h), level); + // Per part and channel: the request slots that name it. + let mut slots: Vec>> = index.parts.iter().map(|p| vec![Vec::new(); p.channels.len()]).collect(); + for (slot, (part, channel)) in request.channels.iter().enumerate() { + let info = index.parts.get(*part).ok_or_else(|| malformed("no such part"))?; + if info.deep { + return Err(Error::Unsupported("a channel of a deep part".into())); + } + let c = info.channels.get(*channel).ok_or_else(|| malformed("no such channel"))?; + if c.sampling != (1, 1) { + return Err(Error::Unsupported("a subsampled channel".into())); + } + if let Some(s) = slots.get_mut(*part).and_then(|p| p.get_mut(*channel)) { + s.push(slot); + } + } + // Groups: slots of one part (so they share every block), none in two groups. + let mut units: Vec<(usize, usize, usize)> = (0..request.channels.len()).map(|slot| (slot, 1, 0)).collect(); + let mut grouped = vec![false; request.channels.len()]; + for group in request.groups { + let members = request + .channels + .get(group.clone()) + .filter(|m| !m.is_empty()) + .ok_or_else(|| malformed(format!("channel group {group:?} is empty or past the requested channels")))?; + if members.iter().any(|(part, _)| Some(*part) != members.first().map(|m| m.0)) { + return Err(Error::Unsupported(format!("channel group {group:?} spans parts"))); + } + for taken in grouped.get_mut(group.clone()).unwrap_or_default() { + if std::mem::replace(taken, true) { + return Err(malformed(format!("channel group {group:?} overlaps another"))); + } + } + for (k, unit) in units.get_mut(group.clone()).unwrap_or_default().iter_mut().enumerate() { + *unit = (group.start, group.len(), k); + } + } + let rows = request.rows.clone().unwrap_or(0..h as i64); + // (both ends inside the display window, so an empty or reversed range at either end of i64 + // stays empty instead of overflowing when aligned to the level's grid) + let rows = rows.start.clamp(0, h as i64)..rows.end.clamp(0, h as i64); + let geometry = Geometry { size: (w, h), out, stride, rows: rows.clone(), slots: &slots, units: &units }; + // What the request delivers: the rows read, at the level, of every requested channel. + let need = plane_bytes((out.0, geometry.rows_read()), request.channels.len()); + if need > request.max_bytes { + return Err(Error::LimitExceeded(format!("the requested rows need {need} bytes, more than the {} the read may use", request.max_bytes))); + } + // The chunks to read, in file order. The decompressor allocates each one's whole block, so a + // block may take no more than the read allows (see `Request::max_block_bytes`). + let most_block = request.max_block_bytes.unwrap_or(MAX_BLOCK_BYTES.max(request.max_bytes)) as u64; + let mut jobs: Vec = Vec::new(); + for (pi, part) in index.layout.iter().enumerate() { + let requested = slots.get(pi).map_or(0, |s| s.iter().map(Vec::len).sum::()); + if requested == 0 { + continue; + } + for c in &part.chunks { + let top = (c.y as i64).saturating_add(part.offset.1); + if !geometry.any_wanted(top, top.saturating_add(c.rows as i64)) { + continue; + } + if c.len == 0 { + return Err(malformed("a chunk offset outside the chunk area of the file")); + } + if let Some(header) = index.meta.headers.get(pi) { + index.check_ratio(pi, header, part, c, c.len)?; + } + let pixels = (c.cols as u64).saturating_mul(c.rows as u64); + let block = pixels.saturating_mul(part.pixel_bytes as u64); + if block > most_block { + return Err(Error::LimitExceeded(format!( + "a block of part {pi} decompresses to {block} bytes, more than the {most_block} one block of this read may use" + ))); + } + let cost = block.saturating_add(pixels.saturating_mul(requested as u64).saturating_mul(4)); + jobs.try_reserve(1).map_err(|_| Error::OutOfMemory("chunk list allocation".into()))?; + jobs.push(Planned { part: pi, offset: c.offset, len: c.len, chunk: *c, cost }); + } + } + jobs.sort_by_key(|j| j.offset); + // (no more threads than chunks can be in flight: two pieces) + let workers = + request.workers.unwrap_or_else(|| std::thread::available_parallelism().map_or(1, usize::from)).min(jobs.len()).min(2 * PIECE_CHUNKS); + Ok(Plan { slots, size: (w, h), out, stride, rows, jobs, workers, units }) +} + +impl Index { + /// What reading `request` will take, before anything is read: the chunks (one unit of + /// progress each), their bytes, the most memory the read holds at once, and the planes + /// [`read_planes`] allocates. Fails as the read would on the same request (limits, unknown or + /// deep channels), so an app can reserve memory and size a progress bar first. + pub fn estimate(&self, request: &Request) -> Result { + estimate_within(self, request, IN_FLIGHT_BYTES) + } +} + +/// [`Index::estimate`] for a read keeping at most `budget` decoded bytes in flight. +pub(crate) fn estimate_within(index: &Index, request: &Request, budget: u64) -> Result { + let plan = plan(index, request)?; + let sum = |f: fn(&Planned) -> u64| plan.jobs.iter().map(f).fold(0u64, u64::saturating_add); + let most = |f: fn(&Planned) -> u64| plan.jobs.iter().map(f).max().unwrap_or(0); + let (compressed, longest) = (sum(|j| j.len), most(|j| j.len)); + // As the read bounds them: decoded bytes within the budget or one chunk, two pieces. + let decoded = sum(|j| j.cost).min(budget.max(most(|j| j.cost))); + let pieces = compressed.min(PIECE_BYTES.max(longest).saturating_mul(2)); + let copies = (plan.workers.max(1) as u64).saturating_mul(longest).min(compressed); + let (w, h) = level_size(plan.size, request.level.min(16)); + Ok(Estimate { + chunks: plan.jobs.len(), + compressed_bytes: compressed, + working_bytes: decoded.saturating_mul(2).saturating_add(pieces).saturating_add(copies), + planes_bytes: plane_bytes((w, h), request.channels.len()) as u64, + }) +} + +/// The bands of what `request` asks for, handed to `sink` one decoded block at a time, in file +/// order: by offset in the file, so a part written bottom-up (`LineOrder::Decreasing`) arrives +/// bottom-up, and the chunks of several parts arrive interleaved as the file stores them. Within +/// a block, the bands come by request slot (a group at its first slot). +/// +/// The chunks are read on the calling thread, in pieces of chunks that follow each other in the +/// file, and decompressed on worker threads ([`Request::workers`]) while the next piece is read. +/// Each time a piece has been read, and while waiting, the calling thread hands every band that +/// is next in file order to `sink`, so the first bands arrive after the first few hundred +/// kilobytes have been read, not after the whole request, and `sink` needs to be neither `Send` +/// nor `Sync`. +/// +/// Memory is bounded. At most one piece is read ahead of what has been delivered, so at most two +/// pieces of compressed bytes (8 MiB, or one larger chunk) are held. Each chunk's decoded bytes +/// are estimated from the headers as its decompressed block plus 4 bytes per pixel for each +/// requested slot of its part (see [`Stats::in_flight`]); the chunks in flight stay within +/// 64 MiB of that, and a chunk whose estimate alone is over 64 MiB is decoded alone, once +/// everything before it has been delivered, so however many workers run, at most one such chunk +/// is held at a time. A block may take no more than [`Request::max_block_bytes`] (by default +/// 256 MiB, or [`Request::max_bytes`] if that is more); a needed chunk over it is refused +/// ([`Error::LimitExceeded`]) before anything is read. +/// For one big chunk the real peak is several times its block: its compressed bytes in the +/// piece, the `exr` crate's copy of its payload (a chunk's declared size is checked against the +/// bytes it holds first, so that copy is no larger than the chunk), the decompressed block, the +/// decompressor's scratch buffers, and the `f32` bands; roughly 3 to 4 times +/// `max(256 MiB, max_bytes)` at worst ([`Index::estimate`] gives the bound for a request). Where +/// threads can't be started (WebAssembly) the chunks are decoded on the calling thread instead. +/// `cancel` is polled before each piece, before each chunk is handed to `sink`, and at least +/// every 50 ms while waiting for the workers. +/// +/// A band lies inside the planes the request describes ([`level_size`] of the display window); +/// rows or columns no chunk or data window covers are not delivered. A chunk the request needs +/// whose offset lies outside the file fails the read before anything is read or delivered +/// ([`Error::Malformed`]). A chunk that can't be decoded or isn't the block its offset-table +/// entry stands for fails the read when its turn comes: the bands before it have been delivered +/// by then. +pub fn read_bands( + source: &mut R, + index: &Index, + request: &Request, + cancel: &dyn Fn() -> bool, + sink: &mut dyn FnMut(Band), +) -> Result { + read_bands_within(source, index, request, cancel, sink, IN_FLIGHT_BYTES) +} + +/// [`read_bands`] that keeps what it did in `stats` as it goes, so the counts are there also when +/// the read fails or is cancelled (they then say how far it got). `stats` is reset first. +pub fn read_bands_with_stats( + source: &mut R, + index: &Index, + request: &Request, + cancel: &dyn Fn() -> bool, + sink: &mut dyn FnMut(Band), + stats: &mut Stats, +) -> Result<()> { + read_infallible(source, index, request, cancel, sink, IN_FLIGHT_BYTES, stats) +} + +/// [`read_bands_with_stats`] with a sink that can stop the read with an error of its own (a tile +/// store out of room, a full scratch disk): nothing more is read or handed to the sink after it +/// fails, the decoding threads wind down as on a cancel, and the sink's error comes back as +/// [`Stop::Sink`], unchanged. `stats` says how far the read got either way; after a sink error +/// its `lines` include the chunk whose band the sink refused (the chunk was decoded whole). +/// +/// This is also the way to pull: a caller that wants to decide band by band whether to go on (and +/// check a budget before each one) does that in the sink. A pulling iterator would have to run the +/// read on a thread of its own, because the read hands bands over from the middle of its loop. +pub fn read_bands_try( + source: &mut R, + index: &Index, + request: &Request, + cancel: &dyn Fn() -> bool, + sink: &mut dyn FnMut(Band) -> std::result::Result<(), E>, + stats: &mut Stats, +) -> std::result::Result<(), Stop> { + read_into(source, index, request, cancel, sink, IN_FLIGHT_BYTES, stats) +} + +/// [`read_bands`] keeping at most `budget` decoded bytes in flight (the tests use a small one). +pub(crate) fn read_bands_within( + source: &mut R, + index: &Index, + request: &Request, + cancel: &dyn Fn() -> bool, + sink: &mut dyn FnMut(Band), + budget: u64, +) -> Result { + let mut stats = Stats::default(); + read_infallible(source, index, request, cancel, sink, budget, &mut stats)?; + Ok(stats) +} + +/// [`read_into`] with a sink that can't fail. +fn read_infallible( + source: &mut R, + index: &Index, + request: &Request, + cancel: &dyn Fn() -> bool, + sink: &mut dyn FnMut(Band), + budget: u64, + stats: &mut Stats, +) -> Result<()> { + let mut sink = |band: Band| -> std::result::Result<(), std::convert::Infallible> { + sink(band); + Ok(()) + }; + read_into(source, index, request, cancel, &mut sink, budget, stats).map_err(|stop| match stop { + Stop::Read(e) => e, + Stop::Sink(never) => match never {}, + }) +} + +/// The read behind [`read_bands`], counting into `stats`. +fn read_into( + source: &mut R, + index: &Index, + request: &Request, + cancel: &dyn Fn() -> bool, + sink: &mut dyn FnMut(Band) -> std::result::Result<(), E>, + budget: u64, + stats: &mut Stats, +) -> std::result::Result<(), Stop> { + *stats = Stats { chunks: vec![0; index.layout.len()], ..Default::default() }; + let plan = plan(index, request)?; + let geometry = plan.geometry(); + let jobs = &plan.jobs; + if jobs.is_empty() { + return Ok(()); + } + // (declared outside the scope because the workers borrow the receiving end) + let (to_workers, queue) = mpsc::channel::(); + let queue = Mutex::new(queue); + std::thread::scope(|scope| { + // Owned by this closure so that every way out of it (an error, a cancel, the sink's + // error) drops the sending end, which ends the workers' loops before the scope waits. + let to_workers = to_workers; + let (done, results) = mpsc::channel(); + let mut started = 0usize; + for _ in 0..plan.workers { + let (queue, done, geometry) = (&queue, done.clone(), &geometry); + let worker = + std::thread::Builder::new().name("exr-decode".into()).spawn_scoped(scope, move || decode_worker(index, geometry, queue, &done)); + started += usize::from(worker.is_ok()); + } + // (only the workers hold senders now: if they all stop, waiting for a result fails + // instead of blocking forever) + drop(done); + let mut order = InOrder::default(); + let mut pos: Option = None; + let mut most = FIRST_PIECE_BYTES; + let mut submitted = 0usize; + // The decoded bytes of the chunks submitted and not yet delivered. + let mut in_flight = 0u64; + // The sequence number of the first chunk of the piece read last. + let mut last_piece = 0usize; + // Hands over what is next in order and takes it off the bytes in flight. + let mut deliver = |order: &mut InOrder, in_flight: &mut u64, stats: &mut Stats| -> std::result::Result<(), Stop> { + let before = order.next; + let delivered = order.deliver(sink, stats, cancel); + let done: u64 = jobs.get(before..order.next).unwrap_or_default().iter().map(|j| j.cost).fold(0, u64::saturating_add); + *in_flight = in_flight.saturating_sub(done); + delivered + }; + while submitted < jobs.len() { + if cancel() { + return Err(Error::Cancelled.into()); + } + let rest = jobs.get(submitted..).unwrap_or_default(); + let take = piece_len(rest, most, budget); + let piece = rest.get(..take).unwrap_or_default(); + let cost = piece.iter().map(|j| j.cost).fold(0, u64::saturating_add); + // Read ahead by one piece at most, and keep the decoded bytes in flight within the + // budget (a piece over it alone is read once everything before it is delivered). + // (`order.next` is a chunk not received yet whenever this waits, so a result is on + // its way: everything decoded on this thread is delivered at once.) + while order.next < last_piece || (order.next < submitted && in_flight.saturating_add(cost) > budget) { + let (seq, result) = next_result(&results, cancel)?; + order.accept(seq, result); + deliver(&mut order, &mut in_flight, stats)?; + if cancel() { + return Err(Error::Cancelled.into()); + } + } + most = most.saturating_mul(2).min(PIECE_BYTES); + let Piece { data, chunks } = read_piece(source, piece, &mut pos, stats)?; + let data = Arc::new(data); + last_piece = submitted; + in_flight = in_flight.saturating_add(cost); + stats.in_flight = stats.in_flight.max(in_flight); + for (planned, range) in piece.iter().zip(chunks) { + let job = Job { seq: submitted, part: planned.part, chunk: planned.chunk, data: Arc::clone(&data), range }; + submitted += 1; + if started == 0 { + order.accept(job.seq, decode_job(index, &geometry, &job)); + } else if let Err(mpsc::SendError(job)) = to_workers.send(job) { + // (the queue outlives this loop, so a send doesn't fail; if it ever did, the + // chunk is decoded here rather than lost) + order.accept(job.seq, decode_job(index, &geometry, &job)); + } + } + while let Ok((seq, result)) = results.try_recv() { + order.accept(seq, result); + } + deliver(&mut order, &mut in_flight, stats)?; + } + // Everything is read: the rest as it is decoded. + while order.next < submitted { + if cancel() { + return Err(Error::Cancelled.into()); + } + let (seq, result) = next_result(&results, cancel)?; + order.accept(seq, result); + deliver(&mut order, &mut in_flight, stats)?; + } + Ok(()) + }) +} + +/// The next decoded chunk from the workers, polling `cancel` while waiting. (Only called while a +/// worker holds a chunk: on the calling thread everything is decoded before it would wait.) +fn next_result(results: &Receiver<(usize, Decoded)>, cancel: &dyn Fn() -> bool) -> Result<(usize, Decoded)> { + loop { + match results.recv_timeout(CANCEL_POLL) { + Ok(result) => return Ok(result), + Err(RecvTimeoutError::Timeout) => { + if cancel() { + return Err(Error::Cancelled); + } + } + Err(RecvTimeoutError::Disconnected) => return Err(malformed("a decoding thread stopped")), + } + } +} + +/// [`read_bands`] collected: one plane per requested channel covering the whole display window +/// at the request's level (also when [`Request::rows`] selects only some rows: the other rows +/// keep the fill value), filled with [`Request::fill`] where nothing was read. The planes are +/// reserved up front and count against [`Request::max_bytes`] (an allocation the system refuses +/// is [`Error::OutOfMemory`], not an abort), so the cost is known before the first chunk is +/// read. +pub fn read_planes(source: &mut R, index: &Index, request: &Request, cancel: &dyn Fn() -> bool) -> Result { + let (w, h) = (index.display.width as usize, index.display.height as usize); + let (ow, oh) = level_size((w, h), request.level.min(16)); + let n = ow.checked_mul(oh).ok_or_else(|| Error::LimitExceeded("display window".into()))?; + let need = plane_bytes((ow, oh), request.channels.len()); + if need > request.max_bytes { + return Err(Error::LimitExceeded(format!("{w}×{h} channels need {need} bytes, more than the {} the read may use", request.max_bytes))); + } + let mut planes: Vec> = Vec::new(); + planes.try_reserve_exact(request.channels.len()).map_err(|_| Error::OutOfMemory("plane list allocation".into()))?; + for slot in 0..request.channels.len() { + let mut plane: Vec = Vec::new(); + plane.try_reserve_exact(n).map_err(|_| Error::OutOfMemory(format!("{need} bytes of planes: the system can't give that memory now")))?; + plane.resize(n, request.fill.get(slot).copied().unwrap_or(0.0)); + planes.push(plane); + } + // (one plane per slot: groups would interleave) + let request = Request { groups: &[], ..request.clone() }; + let stats = read_bands(source, index, &request, cancel, &mut |band: Band| { + let Some(plane) = planes.get_mut(band.slot) else { return }; + for r in 0..band.height { + let start = (band.y + r).checked_mul(ow).and_then(|row| row.checked_add(band.x)); + let (Some(start), Some(from)) = (start, r.checked_mul(band.width)) else { continue }; + if let (Some(dst), Some(src)) = + (plane.get_mut(start..start.saturating_add(band.width)), band.samples.get(from..from.saturating_add(band.width))) + { + dst.copy_from_slice(src); + } + } + })?; + Ok(Planes { width: ow, height: oh, planes, stats }) +} + +/// The bytes of `count` planes of `size`, saturating. +fn plane_bytes((w, h): (usize, usize), count: usize) -> usize { + w.saturating_mul(h).saturating_mul(count).saturating_mul(std::mem::size_of::()) +} + +/// One chunk to decode: its sequence number in file order, its part, where it belongs, and where +/// its bytes lie in the piece it was read with (shared by all the chunks of that piece). +struct Job { + seq: usize, + part: usize, + chunk: ChunkRef, + data: Arc>, + range: Range, +} + +/// What decoding one chunk gives: its bands and the scan lines decompressed. +type Decoded = Result<(Vec, usize)>; + +/// A decoding thread: takes chunks from `queue` until it is closed, and sends each one's bands +/// back through `done` (until nobody listens any more: the read has ended early). +fn decode_worker(index: &Index, g: &Geometry, queue: &Mutex>, done: &Sender<(usize, Decoded)>) { + loop { + // (the lock is held while waiting for the next chunk, so the workers take turns waiting; + // a poisoned lock can't happen, no worker panics while holding it, and would end this one) + let next = match queue.lock() { + Ok(queue) => queue.recv(), + Err(_) => return, + }; + let Ok(job) = next else { return }; + let seq = job.seq; + let mut report = Report { seq, done, sent: false }; + let decoded = decode_job(index, g, &job); + // (its share of the piece's bytes goes now, not after the result is taken: the two-pieces + // bound counts on it) + drop(job); + report.sent = true; + if done.send((seq, decoded)).is_err() { + return; + } + } +} + +/// Reports a chunk as failed if the worker decoding it unwinds before sending its result, so the +/// calling thread, which waits for every chunk in order, never waits for one that won't come. +struct Report<'a> { + seq: usize, + done: &'a Sender<(usize, Decoded)>, + sent: bool, +} + +impl Drop for Report<'_> { + fn drop(&mut self) { + if !self.sent { + let _ = self.done.send((self.seq, Err(malformed("a decoding thread stopped")))); + } + } +} + +/// The bands of one chunk. A panic in the `exr` crate's decompressors is reported as a damaged +/// chunk, never raised (where panics unwind). +fn decode_job(index: &Index, g: &Geometry, job: &Job) -> Decoded { + let buf = job.data.get(job.range.clone()).unwrap_or_default(); + let decoded = catch_unwind(AssertUnwindSafe(|| chunk_bands(index, job.part, &job.chunk, buf, g))) + .unwrap_or_else(|_| Err(malformed("a chunk could not be decompressed"))); + decoded.map(|(mut bands, lines)| { + for band in &mut bands { + band.chunk = job.seq; + } + (bands, lines) + }) +} + +/// Puts decoded chunks back into file order: the workers finish them in any order, the sink gets +/// them by sequence number. +#[derive(Default)] +struct InOrder { + /// The sequence number of the next chunk to deliver. + next: usize, + /// Chunks decoded ahead of it. + waiting: std::collections::BTreeMap, +} + +impl InOrder { + fn accept(&mut self, seq: usize, decoded: Decoded) { + self.waiting.insert(seq, decoded); + } + + /// Hands every chunk that is next in order to `sink`, polling `cancel` before each one; the + /// first chunk that failed to decode ends the read with its error (after the chunks before it + /// were delivered). + /// The sink's error stops the delivery at once (no band after it). + fn deliver( + &mut self, + sink: &mut dyn FnMut(Band) -> std::result::Result<(), E>, + stats: &mut Stats, + cancel: &dyn Fn() -> bool, + ) -> std::result::Result<(), Stop> { + while self.waiting.contains_key(&self.next) { + if cancel() { + return Err(Error::Cancelled.into()); + } + let Some(decoded) = self.waiting.remove(&self.next) else { break }; + self.next += 1; + let (bands, lines) = decoded?; + stats.lines = stats.lines.saturating_add(lines); + for band in bands { + sink(band).map_err(Stop::Sink)?; + } + } + Ok(()) + } +} + +/// The bytes of a piece, and each of its chunks' range in them (in the piece's order). +struct Piece { + data: Vec, + chunks: Vec>, +} + +/// How many of `jobs` (chunks in file order) the next piece takes: chunks that each start where +/// the one before ends, up to [`PIECE_CHUNKS`] of them, `most` bytes or `budget` decoded bytes, +/// at least one. +fn piece_len(jobs: &[Planned], most: u64, budget: u64) -> usize { + let mut take = 0; + let (mut bytes, mut cost) = (0u64, 0u64); + let mut end = None; + while let Some(job) = jobs.get(take) { + if take > 0 + && (take >= PIECE_CHUNKS || end != Some(job.offset) || bytes.saturating_add(job.len) > most || cost.saturating_add(job.cost) > budget) + { + break; + } + bytes = bytes.saturating_add(job.len); + cost = cost.saturating_add(job.cost); + end = Some(job.offset.saturating_add(job.len)); + take += 1; + } + take +} + +/// The raw bytes of the chunks `piece` (in file order, each starting where the one before ends), +/// read with one request after a seek when the source stands elsewhere (`pos`), and each chunk's +/// range in them. A chunk the file ends in keeps the bytes there are. +fn read_piece(source: &mut R, piece: &[Planned], pos: &mut Option, stats: &mut Stats) -> Result { + let Some(start) = piece.first().map(|j| j.offset) else { return Ok(Piece { data: Vec::new(), chunks: Vec::new() }) }; + let total = piece.iter().map(|j| j.len).fold(0u64, u64::saturating_add); + let mut data = Vec::new(); + data.try_reserve_exact(usize::try_from(total).map_err(|_| malformed("chunk size"))?) + .map_err(|_| Error::OutOfMemory("chunk allocation".into()))?; + if *pos != Some(start) { + source.seek(SeekFrom::Start(start)).map_err(io)?; + } + (&mut *source).take(total).read_to_end(&mut data).map_err(io)?; + *pos = Some(start.saturating_add(data.len() as u64)); + stats.bytes = stats.bytes.saturating_add(data.len() as u64); + let mut chunks = Vec::new(); + chunks.try_reserve_exact(piece.len()).map_err(|_| Error::OutOfMemory("chunk allocation".into()))?; + for job in piece { + let a = usize::try_from(job.offset.saturating_sub(start)).unwrap_or(usize::MAX).min(data.len()); + let b = usize::try_from(job.len).map_or(data.len(), |l| a.saturating_add(l)).min(data.len()); + chunks.push(a..b); + if let Some(c) = stats.chunks.get_mut(job.part) { + *c += 1; + } + } + Ok(Piece { data, chunks }) +} + +/// Checks the header of the chunk `buf` of flat part `pi` (`header`), which its offset table +/// says is `expected`, before the `exr` crate reads it: the crate allocates and zero-fills the +/// payload size a chunk declares (up to its own bound per block, which can be hundreds of +/// megabytes) before reading it, and computes a tile's position from the level it declares +/// (panicking on some). The header is the part number (in multi-part files), then the first +/// scan line's y or the tile's column, row and level (x, y), then the payload size, all `i32`. +fn check_chunk_header(meta: &MetaData, header: &Header, pi: usize, expected: &ChunkRef, buf: &[u8]) -> Result> { + let word = |i: usize| buf.get(i * 4..i * 4 + 4).and_then(|b| <[u8; 4]>::try_from(b).ok()).map(i32::from_le_bytes); + let short = || malformed("a chunk shorter than its header"); + let mut at = 0usize; + if meta.requirements.is_multilayer() { + if usize::try_from(word(0).ok_or_else(short)?).ok() != Some(pi) { + return Err(malformed("a chunk of another part where its offset table points")); + } + at = 1; + } + let in_place = match header.blocks { + exr::meta::BlockDescription::ScanLines => { + let y = word(at).ok_or_else(short)?; + at += 1; + i64::from(y) == i64::from(header.own_attributes.layer_position.1).saturating_add(expected.y as i64) + } + exr::meta::BlockDescription::Tiles(_) => { + let tile = [word(at), word(at + 1), word(at + 2), word(at + 3)]; + at += 4; + let [Some(x), Some(y), Some(lx), Some(ly)] = tile else { return Err(short()) }; + (usize::try_from(x).ok(), usize::try_from(y).ok()) == (Some(expected.tile.0), Some(expected.tile.1)) && (lx, ly) == (0, 0) + } + }; + if !in_place { + return Err(malformed("a chunk is not the block its offset table entry stands for")); + } + let size = usize::try_from(word(at).ok_or_else(short)?).map_err(|_| malformed("a chunk declaring a negative size"))?; + if size > buf.len().saturating_sub((at + 1) * 4) { + return Err(malformed("a chunk declaring more bytes than it holds")); + } + let start = (at + 1) * 4; + Ok(start..start + size) +} + +/// Decompress the chunk `buf` of part `pi`, which its offset table says is `expected`, and convert +/// the lines of the requested channels to bands (`g.slots`: each channel's request slots): the +/// bands, and the scan lines decompressed. A chunk that names another part, level or position +/// than its offset-table entry, or declares more bytes than it holds, is refused before the +/// `exr` crate reads it. +fn chunk_bands(index: &Index, pi: usize, expected: &ChunkRef, buf: &[u8], g: &Geometry) -> Result<(Vec, usize)> { + let meta = &index.meta; + let (Some(header), Some(part)) = (meta.headers.get(pi), index.layout.get(pi)) else { return Err(malformed("no such part")) }; + let payload = check_chunk_header(meta, header, pi, expected, buf)?; + // (a DWA chunk's counters can make the crate allocate terabytes: see `crate::dwa_guard`) + let payload = buf.get(payload).unwrap_or_default(); + // (planned against the gap to the next chunk; the payload the chunk declares may be smaller) + index.check_ratio(pi, header, part, expected, payload.len() as u64)?; + crate::dwa_guard::check_payload(header, (expected.cols, expected.rows), payload).map_err(malformed)?; + let mut read = buf; + let chunk = Chunk::read(&mut read, meta).map_err(exr_error)?; + let block = UncompressedBlock::decompress_chunk(chunk, meta, false).map_err(exr_error)?; + if block.index.level != Vec2(0, 0) || block.index.pixel_position != Vec2(expected.x, expected.y) { + return Err(malformed("a chunk is not the block its offset table entry stands for")); + } + let slots = g.slots.get(pi).map_or(&[][..], Vec::as_slice); + let lines = block.index.pixel_size.1; + let ((lw, lh), (w, h)) = (part.size, g.size); + let (ow, _) = g.out; + // The output rows of the block: its data-window rows that land in the display window on a + // wanted row. They are consecutive in the output grid (wanted rows are every stride-th). + let (by, bh) = (block.index.pixel_position.1, block.index.pixel_size.1); + let out_rows: Vec = (by..by.saturating_add(bh)) + .filter_map(|y| usize::try_from(y as i64 + part.offset.1).ok()) + .filter(|ty| *ty < h && g.wanted(*ty as i64)) + .map(|ty| ty / g.stride as usize) + .collect(); + let (Some(&top), Some(&bottom)) = (out_rows.first(), out_rows.last()) else { return Ok((Vec::new(), lines)) }; + let height = bottom.saturating_sub(top).saturating_add(1); + // One band per output unit (a group, or a slot of its own), each row written straight into it + // at its place: no band is copied or regrown. + let mut bands: std::collections::BTreeMap = std::collections::BTreeMap::new(); + // (typed: the line reader is generic over the sample type, and an untyped `0` would make the + // integer buffer `i32`, turning ids from 2^31 up negative) + let (mut f16s, mut f32s, mut u32s): (Vec, Vec, Vec) = (Vec::new(), Vec::new(), Vec::new()); + for line in block.lines(&header.channels) { + let Some(wanted) = slots.get(line.location.channel).filter(|s| !s.is_empty()) else { continue }; + let Some(kind) = header.channels.list.get(line.location.channel).map(|c| c.sample_type) else { continue }; + let (x0, y, count) = (line.location.position.0, line.location.position.1, line.location.sample_count); + // The samples of data-window row `y`, columns `x0..x0 + count`, at display row `ty` from + // display column `tx0`; clipped to the display window. + if y >= lh || x0.checked_add(count).is_none_or(|end| end > lw) { + continue; + } + let Some(ty) = usize::try_from(y as i64 + part.offset.1).ok().filter(|ty| *ty < h) else { continue }; + if !g.wanted(ty as i64) { + continue; + } + let tx0 = x0 as i64 + part.offset.0; + let skip = usize::try_from(-tx0).unwrap_or(0).min(count); + let tx = usize::try_from(tx0.max(0)).unwrap_or(usize::MAX); + let take = (count - skip).min(w.saturating_sub(tx)); + if take == 0 { + continue; + } + let samples: Vec = match kind { + ExrSample::F16 => { + f16s.resize(count, exr::prelude::f16::ZERO); + line.read_samples_into_slice(&mut f16s).map_err(exr_error)?; + let mut out = vec![0.0; take]; + if let Some(src) = f16s.get(skip..skip + take) { + half::slice::HalfFloatSliceExt::convert_to_f32_slice(src, &mut out); + } + out + } + ExrSample::F32 => { + f32s.resize(count, 0.0); + line.read_samples_into_slice(&mut f32s).map_err(exr_error)?; + f32s.iter().skip(skip).take(take).copied().collect() + } + ExrSample::U32 => { + u32s.resize(count, 0); + line.read_samples_into_slice(&mut u32s).map_err(exr_error)?; + u32s.iter().skip(skip).take(take).map(|s| *s as f32).collect() + } + }; + let (row, col, values) = if g.stride == 1 { + (ty, tx, samples) + } else { + // A reduced level: every stride-th column of the row, on the level's grid. + let stride = g.stride as usize; + let first = tx.div_ceil(stride) * stride; + let values: Vec = samples.iter().enumerate().filter(|(j, _)| (tx + j) % stride == 0).map(|(_, v)| *v).collect(); + (ty / stride, first / stride, values) + }; + if values.is_empty() || col.saturating_add(values.len()) > ow { + continue; + } + // (a channel requested more than once goes into each of its slots) + for slot in wanted { + let Some(&(unit, n, k)) = g.units.get(*slot) else { continue }; + let band = match bands.entry(unit) { + std::collections::btree_map::Entry::Occupied(band) => band.into_mut(), + std::collections::btree_map::Entry::Vacant(place) => { + let len = height.saturating_mul(values.len()).saturating_mul(n); + let mut samples = Vec::new(); + samples.try_reserve_exact(len).map_err(|_| Error::OutOfMemory("band allocation".into()))?; + samples.resize(len, 0.0); + place.insert(Band { slot: unit, x: col, y: top, width: values.len(), height, values_per_pixel: n, chunk: 0, samples }) + } + }; + // (every line of a block covers the same columns; a block whose lines don't is damaged) + if band.x != col || band.width != values.len() || row < top || row > bottom { + return Err(malformed("the lines of a block disagree on where they lie")); + } + let base = (row - top).saturating_mul(band.width).saturating_mul(n); + for (i, v) in values.iter().enumerate() { + if let Some(slot) = band.samples.get_mut(base + i * n + k) { + *slot = *v; + } + } + } + } + drop(block); + Ok((bands.into_values().collect(), lines)) +} diff --git a/crates/exr/src/seek_tests.rs b/crates/exr/src/seek_tests.rs new file mode 100644 index 000000000..010492235 --- /dev/null +++ b/crates/exr/src/seek_tests.rs @@ -0,0 +1,1525 @@ +//! Tests of [`crate::seek`]: files are built with the `exr` crate and read back through the +//! seekable reader; the whole-part decoder ([`crate::decode`]) is the oracle for what a channel +//! should hold. +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] +// (lists of channel groups with one group are arrays of one range on purpose) +#![allow(clippy::single_range_in_vec_init)] + +use std::cell::Cell; +use std::io::{Cursor, Read, Seek, SeekFrom}; + +use exr::math::Vec2; +use exr::prelude::{ + AnyChannel, AnyChannels, Blocks, Compression, Encoding, FlatSamples, ImageAttributes, IntegerBounds, Layer, LayerAttributes, LineOrder, + WritableImage, +}; + +use super::decode::{Limits, decode_display_part}; +use super::seek::*; + +fn value(i: usize, k: usize) -> f32 { + (i * 7 + k * 1000) as f32 / 4.0 +} + +/// A layer of `w × h` at `at` in the display window, with these channels (f32 ramps, each its own). +fn layer( + name: Option<&str>, + at: (i32, i32), + (w, h): (usize, usize), + channels: &[&str], + tiles: bool, + compression: Compression, +) -> Layer> { + let mut attrs = name.map_or_else(LayerAttributes::default, LayerAttributes::named); + attrs.layer_position = Vec2(at.0, at.1); + let list = channels.iter().enumerate().map(|(k, n)| AnyChannel::new(*n, FlatSamples::F32((0..w * h).map(|i| value(i, k)).collect()))).collect(); + let blocks = if tiles { Blocks::Tiles(Vec2(4, 4)) } else { Blocks::ScanLines }; + Layer::new((w, h), attrs, Encoding { compression, blocks, line_order: LineOrder::Increasing }, AnyChannels::sort(list)) +} + +fn file(layers: Vec>>, display: (usize, usize)) -> Vec { + let image = exr::image::Image::from_layers(ImageAttributes::new(IntegerBounds::new((0, 0), display)), layers); + let mut out = Cursor::new(Vec::new()); + image.write().to_buffered(&mut out).unwrap(); + out.into_inner() +} + +fn never() -> bool { + false +} + +fn indexed(bytes: &[u8]) -> Index { + Index::build(&mut Cursor::new(bytes), &Limits::default(), &never).unwrap() +} + +/// What the whole-part decoder places in the display window for each channel of part `part`. +fn oracle(bytes: &[u8], part: usize, fill: f32) -> Vec> { + decode_display_part(bytes, part, &|_| true, &|_| fill, &Limits::default()).unwrap().channels.into_iter().map(|c| c.samples).collect() +} + +fn channels_of(index: &Index, part: usize) -> Vec<(usize, usize)> { + (0..index.parts()[part].channels.len()).map(|c| (part, c)).collect() +} + +#[test] +fn every_compression_scan_lines_and_tiles_read_like_the_whole_part_decoder() { + for compression in + [Compression::Uncompressed, Compression::RLE, Compression::ZIP1, Compression::ZIP16, Compression::PIZ, Compression::PXR24, Compression::B44] + { + for tiles in [false, true] { + let bytes = file(vec![layer(None, (0, 0), (9, 7), &["R", "G", "B", "A"], tiles, compression)], (9, 7)); + let index = indexed(&bytes); + let channels = channels_of(&index, 0); + let read = read_planes(&mut Cursor::new(&bytes[..]), &index, &Request::new(&channels, usize::MAX), &never).unwrap(); + assert_eq!((read.width, read.height), (9, 7)); + assert_eq!(read.planes, oracle(&bytes, 0, 0.0), "{compression:?} tiles {tiles}"); + } + } +} + +#[test] +fn only_the_requested_channels_are_returned_in_the_requested_order() { + let bytes = file(vec![layer(None, (0, 0), (8, 4), &["R", "G", "B", "A"], false, Compression::ZIP1)], (8, 4)); + let index = indexed(&bytes); + let names: Vec<&str> = index.parts()[0].channels.iter().map(|c| c.name.as_str()).collect(); + assert_eq!(names, ["A", "B", "G", "R"]); + let whole = oracle(&bytes, 0, 0.0); + // R, then A (stored first), the others not read. + let read = read_planes(&mut Cursor::new(&bytes[..]), &index, &Request::new(&[(0, 3), (0, 0)], usize::MAX), &never).unwrap(); + assert_eq!(read.planes, [whole[3].clone(), whole[0].clone()]); +} + +#[test] +fn a_part_with_no_requested_channel_is_not_read() { + let bytes = file( + vec![ + layer(Some("depth"), (0, 0), (16, 16), &["Z"], false, Compression::ZIP1), + layer(Some("beauty"), (0, 0), (16, 16), &["R", "G", "B", "A"], true, Compression::PIZ), + ], + (16, 16), + ); + let index = indexed(&bytes); + assert_eq!(index.parts().len(), 2); + let depth = read_planes(&mut Cursor::new(&bytes[..]), &index, &Request::new(&[(0, 0)], usize::MAX), &never).unwrap(); + assert_eq!(depth.planes, oracle(&bytes, 0, 0.0)); + assert!(depth.stats.chunks[0] > 0); + assert_eq!(depth.stats.chunks[1], 0, "the beauty part was never touched"); + // Channels of both parts in one read, in request order. + let both = read_planes(&mut Cursor::new(&bytes[..]), &index, &Request::new(&[(1, 3), (0, 0)], usize::MAX), &never).unwrap(); + assert_eq!(both.planes, [oracle(&bytes, 1, 0.0)[3].clone(), oracle(&bytes, 0, 0.0)[0].clone()]); +} + +#[test] +fn only_the_chunks_holding_the_requested_rows_are_read() { + // One line per chunk (ZIPS-like layout through ZIP1): rows 4..8 are four chunks. + let bytes = file(vec![layer(None, (0, 0), (8, 16), &["Y"], false, Compression::ZIP1)], (8, 16)); + let index = indexed(&bytes); + let whole = oracle(&bytes, 0, 0.0)[0].clone(); + let request = Request { rows: Some(4..8), fill: &[-1.0], ..Request::new(&[(0, 0)], usize::MAX) }; + let read = read_planes(&mut Cursor::new(&bytes[..]), &index, &request, &never).unwrap(); + assert_eq!(read.stats.chunks[0], 4); + for row in 0..16 { + let got = &read.planes[0][row * 8..row * 8 + 8]; + if (4..8).contains(&row) { + assert_eq!(got, &whole[row * 8..row * 8 + 8], "row {row}"); + } else { + assert_eq!(got, [-1.0; 8], "row {row} keeps the fill value"); + } + } + // Tiles: rows 5..6 touch the one tile row (4..8) only. + let tiled = file(vec![layer(None, (0, 0), (8, 16), &["Y"], true, Compression::ZIP1)], (8, 16)); + let tiled_index = indexed(&tiled); + let request = Request { rows: Some(5..6), ..Request::new(&[(0, 0)], usize::MAX) }; + let read = read_planes(&mut Cursor::new(&tiled[..]), &tiled_index, &request, &never).unwrap(); + assert_eq!(read.stats.chunks[0], 2, "two 4-wide tiles of the one tile row"); + assert_eq!(&read.planes[0][5 * 8..6 * 8], &oracle(&tiled, 0, 0.0)[0][5 * 8..6 * 8]); +} + +#[test] +fn a_reduced_level_keeps_every_nth_row_and_column() { + // 9 × 7: level 1 is 5 × 4, level 2 is 3 × 2. + for tiles in [false, true] { + let bytes = file(vec![layer(None, (0, 0), (9, 7), &["Y"], tiles, Compression::ZIP1)], (9, 7)); + let index = indexed(&bytes); + let whole = oracle(&bytes, 0, 0.0)[0].clone(); + for level in 0..=3u32 { + let stride = 1usize << level; + let request = Request { level, ..Request::new(&[(0, 0)], usize::MAX) }; + let read = read_planes(&mut Cursor::new(&bytes[..]), &index, &request, &never).unwrap(); + assert_eq!((read.width, read.height), level_size((9, 7), level)); + let expected: Vec = + (0..7).step_by(stride).flat_map(|y| (0..9).step_by(stride).map(move |x| (y, x))).map(|(y, x)| whole[y * 9 + x]).collect(); + assert_eq!(read.planes[0], expected, "level {level} tiles {tiles}"); + } + } +} + +#[test] +fn a_reduced_level_reads_only_the_chunks_with_a_row_on_its_grid() { + // One line per chunk: level 2 needs every 4th row of 16. + let bytes = file(vec![layer(None, (0, 0), (8, 16), &["Y"], false, Compression::ZIP1)], (8, 16)); + let index = indexed(&bytes); + let request = Request { level: 2, ..Request::new(&[(0, 0)], usize::MAX) }; + let read = read_planes(&mut Cursor::new(&bytes[..]), &index, &request, &never).unwrap(); + assert_eq!(read.stats.chunks[0], 4); +} + +#[test] +fn a_part_inside_a_larger_display_window_is_placed_and_clipped_like_the_whole_part_decoder() { + // Data window 3 × 2 at (2, 1) in a 6 × 4 display window; another at (-1, -1) cut by the edge. + for (at, size) in [((2, 1), (3, 2)), ((-1, -1), (4, 3)), ((5, 3), (4, 4)), ((-9, 0), (2, 2))] { + let bytes = file(vec![layer(None, at, size, &["R", "G"], false, Compression::ZIP1)], (6, 4)); + let index = indexed(&bytes); + let channels = channels_of(&index, 0); + let request = Request { fill: &[9.0, 9.0], ..Request::new(&channels, usize::MAX) }; + let read = read_planes(&mut Cursor::new(&bytes[..]), &index, &request, &never).unwrap(); + assert_eq!(read.planes, oracle(&bytes, 0, 9.0), "data window {size:?} at {at:?}"); + } +} + +#[test] +fn bands_cover_the_planes_exactly_and_arrive_in_row_order() { + for tiles in [false, true] { + let bytes = file(vec![layer(None, (0, 0), (12, 10), &["R", "G"], tiles, Compression::ZIP16)], (12, 10)); + let index = indexed(&bytes); + let channels = channels_of(&index, 0); + let mut planes = [vec![f32::NAN; 120], vec![f32::NAN; 120]]; + let mut seen = 0usize; + // Per slot, the (row, column) of the last band: file order is block row by block row, + // and left to right within one (tiles), so it only grows. + let mut last: [Option<(usize, usize)>; 2] = [None, None]; + let stats = read_bands(&mut Cursor::new(&bytes[..]), &index, &Request::new(&channels, usize::MAX), &never, &mut |band: Band| { + assert!(last[band.slot] < Some((band.y, band.x)), "slot {}: {:?} after {:?}", band.slot, (band.y, band.x), last[band.slot]); + last[band.slot] = Some((band.y, band.x)); + assert_eq!(band.samples.len(), band.width * band.height); + assert!(band.x + band.width <= 12 && band.y + band.height <= 10); + for r in 0..band.height { + for c in 0..band.width { + planes[band.slot][(band.y + r) * 12 + band.x + c] = band.samples[r * band.width + c]; + seen += 1; + } + } + }) + .unwrap(); + assert_eq!(seen, 240, "every sample once"); + assert!(stats.chunks[0] > 0 && stats.lines > 0); + assert_eq!(planes.to_vec(), oracle(&bytes, 0, 0.0), "tiles {tiles}"); + } +} + +/// A source that counts the bytes read from it. +struct Counting<'a> { + inner: Cursor<&'a [u8]>, + read: &'a Cell, +} + +impl Read for Counting<'_> { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + let n = self.inner.read(buf)?; + self.read.set(self.read.get() + n as u64); + Ok(n) + } +} + +impl Seek for Counting<'_> { + fn seek(&mut self, pos: SeekFrom) -> std::io::Result { + self.inner.seek(pos) + } +} + +#[test] +fn the_first_bands_arrive_before_the_whole_request_has_been_read() { + // Uncompressed lines of 1024 f32 (one chunk each): about 5 MiB in all, more than the first + // two batches. + let bytes = file(vec![layer(None, (0, 0), (1024, 600), &["R", "G"], false, Compression::Uncompressed)], (1024, 600)); + let index = indexed(&bytes); + let channels = channels_of(&index, 0); + let read = Cell::new(0u64); + let mut at_first_band = None; + let mut source = Counting { inner: Cursor::new(&bytes[..]), read: &read }; + let before = read.get(); + // The rows each channel's bands have reached: they must keep growing (file order). + let mut reached = [0usize; 2]; + let mut bands = 0usize; + read_bands(&mut source, &index, &Request::new(&channels, usize::MAX), &never, &mut |band: Band| { + at_first_band.get_or_insert(read.get() - before); + assert_eq!(band.y, reached[band.slot], "slot {} out of order", band.slot); + reached[band.slot] = band.y + band.height; + bands += 1; + }) + .unwrap(); + let total = read.get() - before; + let first = at_first_band.unwrap(); + assert!(first < total / 2, "first band after {first} of {total} bytes"); + assert_eq!((reached, bands), ([600, 600], 1200), "every row of both channels, one band per chunk and channel"); +} + +#[test] +fn without_threads_the_chunks_are_decoded_on_the_calling_thread_with_the_same_result() { + // (where threads can't be started, as in WebAssembly, no worker runs) + let bytes = file(vec![layer(None, (0, 0), (1024, 300), &["R", "G"], false, Compression::Uncompressed)], (1024, 300)); + let index = indexed(&bytes); + let channels = channels_of(&index, 0); + let whole = oracle(&bytes, 0, 0.0); + let read = Cell::new(0u64); + let mut source = Counting { inner: Cursor::new(&bytes[..]), read: &read }; + let mut planes = vec![vec![f32::NAN; 1024 * 300]; 2]; + let mut reached = [0usize; 2]; + let mut at_first_band = None; + let request = Request { workers: Some(0), ..Request::new(&channels, usize::MAX) }; + let stats = read_bands(&mut source, &index, &request, &never, &mut |band: Band| { + at_first_band.get_or_insert(read.get()); + assert_eq!(band.y, reached[band.slot], "file order"); + reached[band.slot] = band.y + band.height; + let plane = &mut planes[band.slot]; + plane[band.y * 1024..(band.y + band.height) * 1024].copy_from_slice(&band.samples); + }) + .unwrap(); + assert_eq!(planes, whole); + assert_eq!(stats.chunks[0], 300); + assert!(at_first_band.unwrap() < stats.bytes / 2, "delivered piece by piece"); +} + +#[test] +fn a_read_can_be_called_off() { + let bytes = file(vec![layer(None, (0, 0), (16, 16), &["R"], false, Compression::ZIP1)], (16, 16)); + let index = indexed(&bytes); + let count = Cell::new(0usize); + let cancel = || { + count.set(count.get() + 1); + count.get() > 1 + }; + let got = read_planes(&mut Cursor::new(&bytes[..]), &index, &Request::new(&[(0, 0)], usize::MAX), &cancel); + assert_eq!(got.err(), Some(Error::Cancelled)); + let building = Index::build(&mut Cursor::new(&bytes[..]), &Limits::default(), &|| true); + assert_eq!(building.err(), Some(Error::Cancelled)); +} + +#[test] +fn a_request_over_the_byte_limit_is_refused_before_anything_is_read() { + let bytes = file(vec![layer(None, (0, 0), (16, 16), &["R", "G"], false, Compression::ZIP1)], (16, 16)); + let index = indexed(&bytes); + let read = Cell::new(0u64); + let mut source = Counting { inner: Cursor::new(&bytes[..]), read: &read }; + // Two planes of 16 × 16 f32 are 2048 bytes. + let got = read_planes(&mut source, &index, &Request::new(&[(0, 0), (0, 1)], 2047), &never); + assert!(matches!(got, Err(Error::LimitExceeded(_))), "{got:?}"); + assert_eq!(read.get(), 0, "no chunk was fetched"); + let got = read_planes(&mut source, &index, &Request::new(&[(0, 0), (0, 1)], 2048), &never).unwrap(); + assert_eq!(got.planes.len(), 2); + // A reduced level counts its smaller planes. + let request = Request { level: 1, ..Request::new(&[(0, 0), (0, 1)], 512) }; + assert_eq!(read_planes(&mut source, &index, &request, &never).unwrap().width, 8); +} + +#[test] +fn requests_for_channels_that_do_not_exist_are_refused() { + let bytes = file(vec![layer(None, (0, 0), (4, 4), &["R"], false, Compression::ZIP1)], (4, 4)); + let index = indexed(&bytes); + for channels in [[(1, 0)], [(0, 1)]] { + let got = read_planes(&mut Cursor::new(&bytes[..]), &index, &Request::new(&channels, usize::MAX), &never); + assert!(matches!(got, Err(Error::Malformed(_))), "{channels:?}: {got:?}"); + } + // Nothing requested is an empty read. + let none = read_planes(&mut Cursor::new(&bytes[..]), &index, &Request::new(&[], usize::MAX), &never).unwrap(); + assert!(none.planes.is_empty() && none.stats.bytes == 0); +} + +#[test] +fn headers_declaring_more_than_the_limits_are_refused_before_any_allocation() { + // An 8 × 8 data window, but the limits allow 4 × 4 only. + let bytes = file(vec![layer(None, (0, 0), (8, 8), &["R"], false, Compression::ZIP1)], (8, 8)); + let limits = Limits { max_width: 4, max_height: 4, ..Limits::default() }; + let got = Index::build(&mut Cursor::new(&bytes[..]), &limits, &never); + assert!(matches!(&got, Err(Error::LimitExceeded(m)) if m.contains("part 0")), "{got:?}"); + // A 2 × 2 data window (within the limit) in an 8 × 8 display window over it. + let small = file(vec![layer(None, (0, 0), (2, 2), &["R"], false, Compression::ZIP1)], (8, 8)); + let limits = Limits { max_pixels: 16, ..Limits::default() }; + let got = Index::build(&mut Cursor::new(&small[..]), &limits, &never); + assert!(matches!(&got, Err(Error::LimitExceeded(m)) if m.contains("display window")), "{got:?}"); + assert!(Index::build(&mut Cursor::new(&small[..]), &Limits { max_pixels: 64, ..Limits::default() }, &never).is_ok()); +} + +#[test] +fn hostile_files_are_refused_and_never_panic() { + let bytes = file( + vec![ + layer(Some("a"), (0, 0), (9, 7), &["R", "G"], false, Compression::ZIP1), + layer(Some("b"), (1, 1), (5, 5), &["Z"], true, Compression::PIZ), + ], + (9, 7), + ); + let try_read = |data: &[u8]| { + let Ok(index) = Index::build(&mut Cursor::new(data), &Limits::default(), &never) else { return }; + let channels: Vec<(usize, usize)> = (0..index.parts().len()).flat_map(|p| channels_of(&index, p)).collect(); + let _ = read_planes(&mut Cursor::new(data), &index, &Request::new(&channels, 1 << 24), &never); + }; + // Every truncation. + for n in 0..bytes.len() { + try_read(&bytes[..n]); + } + // A flipped byte at every position (headers, tables and chunks). + for at in 0..bytes.len() { + let mut broken = bytes.clone(); + broken[at] ^= 0xFF; + try_read(&broken); + } + // Garbage and an empty source. + try_read(&[]); + try_read(&[0x76, 0x2f, 0x31, 0x01, 2, 0, 0, 0]); + try_read(&vec![0xAB; 4096]); +} + +#[test] +fn headers_over_the_attribute_and_size_bounds_are_refused() { + let mut bytes = vec![0x76, 0x2f, 0x31, 0x01, 2, 0, 0, 0]; + // An attribute that claims 100 MB. + bytes.extend_from_slice(b"comments\0string\0"); + bytes.extend_from_slice(&100_000_000i32.to_le_bytes()); + bytes.extend_from_slice(&[0; 64]); + assert!(matches!(Index::build(&mut Cursor::new(&bytes[..]), &Limits::default(), &never), Err(Error::LimitExceeded(_)))); + // A name that never ends. + let mut bytes = vec![0x76, 0x2f, 0x31, 0x01, 2, 0, 0, 0]; + bytes.extend_from_slice(&[b'a'; 1000]); + assert!(matches!(Index::build(&mut Cursor::new(&bytes[..]), &Limits::default(), &never), Err(Error::LimitExceeded(_)))); +} + +#[test] +fn a_tiled_file_with_mip_levels_is_read_from_its_largest_level_only() { + use exr::math::RoundingMode; + use exr::prelude::Levels; + // 9 × 7 in 4 × 4 tiles: level 0 is 3 × 2 tiles, then 4 × 3, 2 × 1 and 1 × 1 pixels in one + // tile each. The smaller levels hold a value no full-resolution pixel has, so a wrong level + // would show in the planes. + let (w, h) = (9usize, 7usize); + let sizes: Vec = exr::meta::mip_map_levels(RoundingMode::Down, Vec2(w, h)).map(|(_, size)| size.area()).collect(); + assert!(sizes.len() > 1); + let levels = |k: usize| Levels::Mip { + rounding_mode: RoundingMode::Down, + level_data: sizes + .iter() + .enumerate() + .map(|(level, n)| FlatSamples::F32(if level == 0 { (0..*n).map(|i| value(i, k)).collect() } else { vec![1e9; *n] })) + .collect(), + }; + let channels = AnyChannels::sort(vec![AnyChannel::new("R", levels(0)), AnyChannel::new("G", levels(1))].into()); + let encoding = Encoding { compression: Compression::ZIP1, blocks: Blocks::Tiles(Vec2(4, 4)), line_order: LineOrder::Increasing }; + let image = exr::image::Image::from_layers( + ImageAttributes::new(IntegerBounds::new((0, 0), (w, h))), + vec![Layer::new((w, h), LayerAttributes::default(), encoding, channels)], + ); + let mut out = Cursor::new(Vec::new()); + image.write().to_buffered(&mut out).unwrap(); + let bytes = out.into_inner(); + let index = indexed(&bytes); + let read = read_planes(&mut Cursor::new(&bytes[..]), &index, &Request::new(&[(0, 0), (0, 1)], usize::MAX), &never).unwrap(); + // (the part stores G before R) + let expected = |k: usize| (0..w * h).map(|i| value(i, k)).collect::>(); + assert_eq!(read.planes, [expected(1), expected(0)]); + assert_eq!(read.stats.chunks[0], 6, "the six tiles of level 0, none of the smaller levels"); + // A reduced read takes its rows from the largest level too. + let half = read_planes(&mut Cursor::new(&bytes[..]), &index, &Request { level: 1, ..Request::new(&[(0, 1)], usize::MAX) }, &never).unwrap(); + let every_other: Vec = (0..h).step_by(2).flat_map(|y| (0..w).step_by(2).map(move |x| value(y * w + x, 0))).collect(); + assert_eq!(half.planes[0], every_other); +} + +#[test] +fn a_flat_part_beside_a_deep_part_is_read_and_the_deep_part_is_refused() { + use super::decode_tests::{mixed_file, mixed_flat}; + for (deep_first, tiled) in [(false, false), (true, false), (false, true), (true, true)] { + let (bytes, _) = mixed_file(deep_first, tiled); + let (flat, deep) = if deep_first { (1, 0) } else { (0, 1) }; + let index = indexed(&bytes); + assert_eq!((index.parts()[flat].deep, index.parts()[deep].deep), (false, true)); + let names: Vec<&str> = index.parts()[flat].channels.iter().map(|c| c.name.as_str()).collect(); + assert_eq!(names, ["B", "G", "R"]); + let read = read_planes(&mut Cursor::new(&bytes[..]), &index, &Request::new(&[(flat, 2), (flat, 0)], usize::MAX), &never).unwrap(); + let expected = |channel: f32| (0..6).map(|i| mixed_flat(channel, i)).collect::>(); + assert_eq!(read.planes, [expected(0.0), expected(100.0)], "deep first {deep_first} tiled {tiled}"); + assert_eq!((read.stats.chunks[flat], read.stats.chunks[deep]), (2, 0), "the deep part's chunks are never read"); + for channels in [vec![(deep, 0)], vec![(flat, 0), (deep, 0)]] { + let got = read_planes(&mut Cursor::new(&bytes[..]), &index, &Request::new(&channels, usize::MAX), &never); + assert!(matches!(got, Err(Error::Unsupported(_))), "{channels:?}: {got:?}"); + } + } +} + +#[test] +fn a_subsampled_channel_is_refused() { + let mut bytes = file(vec![layer(None, (0, 0), (4, 2), &["R"], false, Compression::Uncompressed)], (4, 2)); + // The `exr` crate's writer emits no subsampling: the x sampling of the one channel in the + // channel list is patched to 2 (the 4 × 2 window is a multiple of it). + let tag = b"channels\0chlist\0"; + let attr = bytes.windows(tag.len()).position(|window| window == tag).unwrap(); + let payload = attr + tag.len() + 4; + let name_end = payload + bytes[payload..].iter().position(|byte| *byte == 0).unwrap(); + let x_sampling = name_end + 1 + 4 + 1 + 3; + bytes[x_sampling..x_sampling + 4].copy_from_slice(&2u32.to_le_bytes()); + let index = indexed(&bytes); + assert_eq!(index.parts()[0].channels[0].sampling, (2, 1)); + let got = read_planes(&mut Cursor::new(&bytes[..]), &index, &Request::new(&[(0, 0)], usize::MAX), &never); + assert!(matches!(got, Err(Error::Unsupported(_))), "{got:?}"); +} + +#[test] +fn half_float_and_integer_channels_convert_like_the_whole_part_decoder() { + use exr::prelude::f16; + let (w, h) = (7usize, 5usize); + let halves: Vec = (0..w * h).map(|i| f16::from_f32(i as f32 / 8.0 - 2.0)).collect(); + // Integers beyond f32's exact range too (object ids): they round like the oracle's. + let ids: Vec = (0..w * h).map(|i| (i as u32).wrapping_mul(2_654_435_761)).collect(); + for tiles in [false, true] { + let list = vec![ + AnyChannel::new("Y", FlatSamples::F16(halves.clone())), + AnyChannel::new("id", FlatSamples::U32(ids.clone())), + AnyChannel::new("Z", FlatSamples::F32(vec![0.5; w * h])), + ]; + let blocks = if tiles { Blocks::Tiles(Vec2(4, 4)) } else { Blocks::ScanLines }; + let layer = Layer::new( + (w, h), + LayerAttributes::default(), + Encoding { compression: Compression::ZIP16, blocks, line_order: LineOrder::Increasing }, + AnyChannels::sort(list.into()), + ); + // Placed partly outside the display window, so the conversion and the clipping meet. + let mut layer = layer; + layer.attributes.layer_position = Vec2(-2, 1); + let bytes = file(vec![layer], (6, 5)); + let index = indexed(&bytes); + let channels = channels_of(&index, 0); + let request = Request { fill: &[7.0, 7.0, 7.0], ..Request::new(&channels, usize::MAX) }; + let read = read_planes(&mut Cursor::new(&bytes[..]), &index, &request, &never).unwrap(); + assert_eq!(read.planes, oracle(&bytes, 0, 7.0), "tiles {tiles}"); + assert!(read.planes.iter().all(|p| p.iter().any(|v| *v != 7.0))); + } +} + +#[test] +fn the_index_reports_its_parts_and_a_size() { + let bytes = file(vec![layer(Some("beauty"), (0, 0), (8, 8), &["R", "G", "B"], false, Compression::ZIP16)], (8, 8)); + let index = indexed(&bytes); + assert_eq!(index.parts()[0].name.as_deref(), Some("beauty")); + assert_eq!((index.display().width, index.display().height), (8, 8)); + assert_eq!(index.file_len(), bytes.len() as u64); + assert!(index.bytes() > 0 && index.bytes() < 64 << 10, "{}", index.bytes()); +} + +/// Writes `layers` into a file whose display window is `display` (absolute position and size). +fn file_at(layers: Vec>>, display: ((i32, i32), (usize, usize))) -> Vec { + let image = exr::image::Image::from_layers(ImageAttributes::new(IntegerBounds::new(display.0, display.1)), layers); + let mut out = Cursor::new(Vec::new()); + image.write().to_buffered(&mut out).unwrap(); + out.into_inner() +} + +/// Where the offset table of a single-part file with `count` chunks starts: the position whose +/// smallest entry is where the table ends. +fn table_at(bytes: &[u8], count: usize) -> usize { + let entry = |p: usize| u64::from_le_bytes(bytes[p..p + 8].try_into().unwrap()) as usize; + (8..bytes.len() - count * 8).find(|p| (0..count).map(|i| entry(p + i * 8)).min() == Some(p + count * 8)).unwrap() +} + +/// Sets the box2i attribute `name` (`dataWindow`, `displayWindow`) of a file to `bounds`. +fn patch_box(bytes: &mut [u8], name: &str, bounds: [i32; 4]) { + let tag = format!("{name}\0box2i\0"); + let at = bytes.windows(tag.len()).position(|w| w == tag.as_bytes()).unwrap() + tag.len() + 4; + for (i, v) in bounds.iter().enumerate() { + bytes[at + i * 4..at + i * 4 + 4].copy_from_slice(&v.to_le_bytes()); + } +} + +/// A tiled 9 × 7 file in 4 × 4 tiles with these levels of one channel `R`. +fn tiled_levels(levels: exr::prelude::Levels) -> Vec { + let encoding = Encoding { compression: Compression::ZIP1, blocks: Blocks::Tiles(Vec2(4, 4)), line_order: LineOrder::Increasing }; + let channels = AnyChannels::sort(vec![AnyChannel::new("R", levels)].into()); + let image = exr::image::Image::from_layers( + ImageAttributes::new(IntegerBounds::new((0, 0), (9, 7))), + vec![Layer::new((9, 7), LayerAttributes::default(), encoding, channels)], + ); + let mut out = Cursor::new(Vec::new()); + image.write().to_buffered(&mut out).unwrap(); + out.into_inner() +} + +#[test] +fn a_channel_requested_twice_is_delivered_in_both_slots() { + for tiles in [false, true] { + let bytes = file(vec![layer(None, (0, 0), (9, 7), &["R", "G"], tiles, Compression::ZIP1)], (9, 7)); + let index = indexed(&bytes); + let whole = oracle(&bytes, 0, 0.0); + let read = read_planes(&mut Cursor::new(&bytes[..]), &index, &Request::new(&[(0, 1), (0, 0), (0, 1)], usize::MAX), &never).unwrap(); + assert_eq!(read.planes, [whole[1].clone(), whole[0].clone(), whole[1].clone()], "tiles {tiles}"); + } +} + +#[test] +fn a_block_is_bounded_by_the_bytes_the_decompressor_allocates_for_it() { + // One row 4 Mi pixels wide of one f32 channel: 16 MiB. PIZ holds 32 rows a block, but a + // one-row part's block is that row; 16 such rows of four channels in one ZIP16 block are 1 GiB. + // Both index; a read refuses the 1 GiB block unless its `max_bytes` allows that much. + // (`cancel` stops each read that passes the check before a chunk is fetched.) + let stop = || true; + for (compression, (w, h), channels, big) in + [(Compression::PIZ, (4 << 20, 1), &["R"][..], false), (Compression::ZIP16, (4 << 20, 16), &["R", "G", "B", "A"][..], true)] + { + let mut bytes = file(vec![layer(None, (0, 0), (1, h), channels, false, compression)], (1, h)); + patch_box(&mut bytes, "dataWindow", [0, 0, w - 1, h as i32 - 1]); + let index = Index::build(&mut Cursor::new(&bytes[..]), &Limits::none(), &never).unwrap(); + let read = |max_bytes| read_bands(&mut Cursor::new(&bytes[..]), &index, &Request::new(&[(0, 0)], max_bytes), &stop, &mut |_| {}); + let got = read(256 << 20); + if big { + assert!(matches!(&got, Err(Error::LimitExceeded(m)) if m.contains("block")), "{compression:?}: {got:?}"); + assert_eq!(read(1 << 30).err(), Some(Error::Cancelled), "{compression:?}: allowed with max_bytes 1 GiB"); + } else { + assert_eq!(got.err(), Some(Error::Cancelled), "{compression:?}"); + } + } +} + +#[test] +fn a_wide_dwab_part_with_half_a_gigabyte_blocks_reads_only_with_a_budget_that_allows_them() { + // 16K wide, 256 lines (one DWAB block), 64 half channels: 128 bytes a pixel, 512 MiB a block. + let mut bytes = many_channels(1, 1, 64, true, Compression::DWAB(None)); + patch_box(&mut bytes, "dataWindow", [0, 0, 16383, 255]); + patch_box(&mut bytes, "displayWindow", [0, 0, 16383, 255]); + let index = Index::build(&mut Cursor::new(&bytes[..]), &Limits::none(), &never).unwrap(); + assert_eq!(index.chunks(0).map(|c| (c.width, c.height)).collect::>(), [(16384, 256)]); + let stop = || true; + let read = |max_bytes| read_bands(&mut Cursor::new(&bytes[..]), &index, &Request::new(&[(0, 0)], max_bytes), &stop, &mut |_| {}); + let got = read(256 << 20); + assert!(matches!(&got, Err(Error::LimitExceeded(m)) if m.contains("block")), "{got:?}"); + assert_eq!(read((512 << 20) - 1).err().map(|e| matches!(e, Error::LimitExceeded(_))), Some(true)); + assert_eq!(read(512 << 20).err(), Some(Error::Cancelled), "a 512 MiB block within max_bytes passes the check"); + assert_eq!(read(1 << 30).err(), Some(Error::Cancelled)); +} + +// (budgets over 4 GiB: a 64-bit usize) +#[cfg(target_pointer_width = "64")] +#[test] +fn a_block_is_held_to_max_block_bytes_whatever_the_total_budget() { + // The same 512 MiB DWAB block, read with a 64 GiB budget: `max_block_bytes` decides. The + // source panics if a chunk is fetched, so a missing check fails at once. + let mut bytes = many_channels(1, 1, 64, true, Compression::DWAB(None)); + patch_box(&mut bytes, "dataWindow", [0, 0, 16383, 255]); + patch_box(&mut bytes, "displayWindow", [0, 0, 16383, 255]); + let index = Index::build(&mut Cursor::new(&bytes[..]), &Limits::none(), &never).unwrap(); + let chunks = index.chunks(0).next().unwrap().offset; + let read = |max_block_bytes, cancel: &dyn Fn() -> bool| { + let request = Request { max_block_bytes, ..Request::new(&[(0, 0)], 64 << 30) }; + read_bands(&mut NoChunks { inner: Cursor::new(&bytes[..]), chunks }, &index, &request, cancel, &mut |_| {}) + }; + let got = read(Some(256 << 20), &never); + assert!(matches!(&got, Err(Error::LimitExceeded(m)) if m.contains("one block")), "{got:?}"); + // (cancelled right after the check, before a chunk is fetched) + assert_eq!(read(Some(1 << 30), &|| true).err(), Some(Error::Cancelled), "a 1 GiB block limit passes the check"); + // `None`: max(256 MiB, max_bytes), here 64 GiB. + assert_eq!(read(None, &|| true).err(), Some(Error::Cancelled)); +} + +#[test] +fn a_chunk_over_the_in_flight_budget_is_decoded_alone() { + // 64 rows of one f32 channel, 16 wide, one row a chunk: each chunk holds a 64-byte block and + // 64 bytes of bands, 128 bytes in flight; with a budget of 100 no two may be in flight at + // once, however many workers there are. + let bytes = file(vec![layer(None, (0, 0), (16, 64), &["R"], false, Compression::ZIP1)], (16, 64)); + let index = indexed(&bytes); + let whole = oracle(&bytes, 0, 0.0); + for _ in 0..20 { + let mut plane = vec![f32::NAN; 16 * 64]; + let request = Request { workers: Some(8), ..Request::new(&[(0, 0)], usize::MAX) }; + let mut sink = |b: Band| plane[b.y * 16..(b.y + b.height) * 16].copy_from_slice(&b.samples); + let stats = super::seek::read_bands_within(&mut Cursor::new(&bytes[..]), &index, &request, &never, &mut sink, 100).unwrap(); + assert_eq!(stats.in_flight, 128, "one chunk at a time"); + assert_eq!(vec![plane], whole); + } +} + +/// A file of `channels` channels named `c0`, `c1`, … of `w × h` pixels, f32 or half. +fn many_channels(w: usize, h: usize, channels: usize, half: bool, compression: Compression) -> Vec { + use exr::prelude::f16; + let list: Vec> = (0..channels) + .map(|k| { + let samples = (0..w * h).map(|i| ((i * 3 + k * 7) % 50) as f32 / 8.0); + let samples = if half { FlatSamples::F16(samples.map(f16::from_f32).collect()) } else { FlatSamples::F32(samples.collect()) }; + AnyChannel::new(format!("c{k:03}").as_str(), samples) + }) + .collect(); + let encoding = Encoding { compression, blocks: Blocks::ScanLines, line_order: LineOrder::Increasing }; + file(vec![Layer::new((w, h), LayerAttributes::default(), encoding, AnyChannels::sort(list.into()))], (w, h)) +} + +#[test] +fn wide_dwab_parts_with_many_channels_are_accepted() { + // DWAB holds 256 scan lines a block, and the `exr` crate's bound on a chunk counts all 256 + // even in a part with fewer rows: over 256 MiB for these, though each block is a few MiB. + let dwab = Compression::DWAB(None); + for (w, h, channels, half) in [(65536, 1, 16, false), (4096, 8, 70, false), (8192, 4, 65, true)] { + let mut bytes = many_channels(1, 1, channels, half, dwab); + patch_box(&mut bytes, "dataWindow", [0, 0, w - 1, h - 1]); + let got = Index::build(&mut Cursor::new(&bytes[..]), &Limits::none(), &never); + assert!(got.is_ok(), "{w} × {h}, {channels} channels: {got:?}"); + } + // A real one: one row 4096 wide of 65 f32 channels (the crate's bound: 256 × 1 MiB). + let bytes = many_channels(4096, 1, 65, false, dwab); + let index = indexed(&bytes); + let channels = channels_of(&index, 0); + let read = read_planes(&mut Cursor::new(&bytes[..]), &index, &Request::new(&channels, usize::MAX), &never).unwrap(); + assert_eq!(read.planes, oracle(&bytes, 0, 0.0)); +} + +#[test] +fn a_chunk_declaring_more_bytes_than_it_holds_is_refused_before_the_crate_reads_it() { + // A one-pixel-wide part in 4096 × 16 tiles: each chunk holds 16 pixels, but the `exr` crate + // would allocate whatever size up to a whole tile (256 KiB) a chunk declares. + let part = layer(None, (0, 0), (1, 32), &["R"], false, Compression::ZIP1); + let part = + Layer::new((1, 32), LayerAttributes::default(), Encoding { blocks: Blocks::Tiles(Vec2(4096, 16)), ..part.encoding }, part.channel_data); + let good = file(vec![part], (1, 32)); + let table = table_at(&good, 2); + let chunk = u64::from_le_bytes(good[table..table + 8].try_into().unwrap()) as usize; + assert!(indexed(&good).chunks(0).all(|c| (c.width, c.height) == (1, 16))); + for size in [4096 * 16 * 4 - 1, -1i32] { + let mut bytes = good.clone(); + // (tile column, row, level x and y, then the size) + bytes[chunk + 16..chunk + 20].copy_from_slice(&size.to_le_bytes()); + let index = indexed(&bytes); + let got = read_planes(&mut Cursor::new(&bytes[..]), &index, &Request::new(&[(0, 0)], usize::MAX), &never); + assert!(matches!(&got, Err(Error::Malformed(m)) if m.starts_with("a chunk declaring")), "size {size}: {got:?}"); + } +} + +#[test] +fn a_chunk_naming_another_part_is_refused() { + use super::decode_tests::mixed_file; + // The flat part's first chunk names the deep part. + let (mut bytes, offsets) = mixed_file(false, false); + bytes[offsets[0][0]..offsets[0][0] + 4].copy_from_slice(&1i32.to_le_bytes()); + let index = indexed(&bytes); + let got = read_planes(&mut Cursor::new(&bytes[..]), &index, &Request::new(&[(0, 0)], usize::MAX), &never); + assert!(matches!(&got, Err(Error::Malformed(m)) if m.contains("another part")), "{got:?}"); +} + +#[test] +fn a_failed_or_cancelled_read_still_reports_how_far_it_got() { + // Row 400's chunk of 600 claims another row (see the damaged-chunk test). + let mut bytes = file(vec![layer(None, (0, 0), (1024, 600), &["R", "G"], false, Compression::Uncompressed)], (1024, 600)); + let at = bytes.len() - (600 - 400) * 8200; + bytes[at..at + 4].copy_from_slice(&7i32.to_le_bytes()); + let index = indexed(&bytes); + let channels = channels_of(&index, 0); + let request = Request { workers: Some(4), ..Request::new(&channels, usize::MAX) }; + let mut stats = Stats::default(); + let got = read_bands_with_stats(&mut Cursor::new(&bytes[..]), &index, &request, &never, &mut |_| {}, &mut stats); + assert!(matches!(got, Err(Error::Malformed(_))), "{got:?}"); + assert_eq!(stats.lines, 400, "the lines delivered before the damaged chunk"); + assert!(stats.chunks[0] > 400 && stats.bytes >= 400 * 8200, "{stats:?}"); + // Called off after the first band. + let stop = Cell::new(false); + let got = read_bands_with_stats(&mut Cursor::new(&bytes[..]), &index, &request, &|| stop.get(), &mut |_| stop.set(true), &mut stats); + assert_eq!(got.err(), Some(Error::Cancelled)); + assert!(stats.lines > 0 && stats.lines < 600 && stats.chunks[0] > 0, "{stats:?}"); +} + +#[test] +fn the_index_lists_the_chunks_of_the_full_resolution_level() { + use super::decode_tests::mixed_file; + use exr::math::RoundingMode; + use exr::prelude::Levels; + let sizes: Vec = exr::meta::mip_map_levels(RoundingMode::Down, Vec2(9, 7)).map(|(_, size)| size.area()).collect(); + let bytes = + tiled_levels(Levels::Mip { rounding_mode: RoundingMode::Down, level_data: sizes.iter().map(|n| FlatSamples::F32(vec![1.0; *n])).collect() }); + let index = indexed(&bytes); + let chunks: Vec = index.chunks(0).collect(); + let tiles: Vec<(usize, usize, usize, usize)> = chunks.iter().map(|c| (c.x, c.y, c.width, c.height)).collect(); + assert_eq!(tiles, [(0, 0, 4, 4), (4, 0, 4, 4), (8, 0, 1, 4), (0, 4, 4, 3), (4, 4, 4, 3), (8, 4, 1, 3)]); + assert!(chunks.iter().all(|c| c.len > 0 && c.offset > 0 && c.offset + c.len <= bytes.len() as u64)); + assert_eq!(index.chunks(1).count(), 0, "no such part"); + let (mixed, offsets) = mixed_file(true, false); + let index = indexed(&mixed); + assert_eq!(index.chunks(0).count(), 0, "a deep part"); + assert_eq!(index.chunks(1).map(|c| (c.offset as usize, c.y, c.height)).collect::>(), [(offsets[1][0], 0, 1), (offsets[1][1], 1, 1)]); +} + +#[test] +fn htj2k_compression_is_its_own_error() { + let mut bytes = file(vec![layer(None, (0, 0), (4, 4), &["R"], false, Compression::ZIP1)], (4, 4)); + let tag = b"compression\0compression\0"; + let at = bytes.windows(tag.len()).position(|w| w == tag).unwrap() + tag.len() + 4; + for code in [10u8, 11] { + bytes[at] = code; + let got = Index::build(&mut Cursor::new(&bytes[..]), &Limits::default(), &never); + assert_eq!(got.err(), Some(Error::UnsupportedCompression("HTJ2K".into())), "code {code}"); + } +} + +#[test] +fn the_display_window_may_have_its_own_pixel_limit() { + // A 16 × 16 data window (overscan) over an 8 × 8 display window. + let bytes = file(vec![layer(None, (-4, -4), (16, 16), &["R"], false, Compression::ZIP1)], (8, 8)); + let limits = Limits { max_pixels: 256, ..Limits::default() }; + let build = |display| Index::build_with_display_limit(&mut Cursor::new(&bytes[..]), &limits, display, &never); + assert!(build(64).is_ok()); + assert!(matches!(&build(63), Err(Error::LimitExceeded(m)) if m.contains("display window")), "{:?}", build(63)); + // The data windows stay bound by `max_pixels`. + let tight = Limits { max_pixels: 255, ..Limits::default() }; + let got = Index::build_with_display_limit(&mut Cursor::new(&bytes[..]), &tight, 1 << 20, &never); + assert!(matches!(&got, Err(Error::LimitExceeded(m)) if m.contains("part 0")), "{got:?}"); + // `build` holds both to `max_pixels`. + assert!(Index::build(&mut Cursor::new(&bytes[..]), &Limits { max_pixels: 63, ..Limits::default() }, &never).is_err()); + assert!(Index::build(&mut Cursor::new(&bytes[..]), &limits, &never).is_ok()); +} + +#[test] +fn rows_and_band_positions_count_from_the_display_window_s_corner() { + // Display window 12 × 9 at (10, 20); the data window 8 × 5 at (11, 22), i.e. (1, 2) inside it. + let bytes = file_at(vec![layer(None, (11, 22), (8, 5), &["Y"], false, Compression::ZIP1)], ((10, 20), (12, 9))); + let index = indexed(&bytes); + assert_eq!((index.display().x, index.display().y), (10, 20)); + let whole = oracle(&bytes, 0, -1.0)[0].clone(); + let request = Request { rows: Some(3..6), fill: &[-1.0], ..Request::new(&[(0, 0)], usize::MAX) }; + let read = read_planes(&mut Cursor::new(&bytes[..]), &index, &request, &never).unwrap(); + for row in 0..9 { + let got = &read.planes[0][row * 12..row * 12 + 12]; + if (3..6).contains(&row) { + assert_eq!(got, &whole[row * 12..row * 12 + 12], "row {row}"); + assert!(got.iter().any(|v| *v != -1.0), "row {row} has samples"); + } else { + assert_eq!(got, [-1.0; 12], "row {row}"); + } + } + let mut placed = Vec::new(); + read_bands(&mut Cursor::new(&bytes[..]), &index, &request, &never, &mut |b: Band| placed.push((b.x, b.y, b.width, b.height))).unwrap(); + assert_eq!(placed, [(1, 3, 8, 1), (1, 4, 8, 1), (1, 5, 8, 1)]); +} + +#[test] +fn bands_are_bounded_by_the_rows_they_deliver_and_planes_by_the_display_window() { + let bytes = file(vec![layer(None, (0, 0), (16, 16), &["R", "G"], false, Compression::ZIP1)], (16, 16)); + let index = indexed(&bytes); + // Rows 4..8 of two channels: 4 × 16 × 2 f32, 512 bytes; the whole planes are 2048. + let rows = |max_bytes| Request { rows: Some(4..8), ..Request::new(&[(0, 0), (0, 1)], max_bytes) }; + let mut count = 0; + read_bands(&mut Cursor::new(&bytes[..]), &index, &rows(512), &never, &mut |_| count += 1).unwrap(); + assert_eq!(count, 8); + let got = read_bands(&mut Cursor::new(&bytes[..]), &index, &rows(511), &never, &mut |_| {}); + assert!(matches!(got, Err(Error::LimitExceeded(_))), "{got:?}"); + let got = read_planes(&mut Cursor::new(&bytes[..]), &index, &rows(512), &never); + assert!(matches!(got, Err(Error::LimitExceeded(_))), "read_planes allocates the whole planes: {got:?}"); + // At level 2 rows 4..8 hold one row on the grid (row 4): 4 × 2 f32. + let reduced = |max_bytes| Request { level: 2, ..rows(max_bytes) }; + read_bands(&mut Cursor::new(&bytes[..]), &index, &reduced(32), &never, &mut |_| {}).unwrap(); + assert!(read_bands(&mut Cursor::new(&bytes[..]), &index, &reduced(31), &never, &mut |_| {}).is_err()); +} + +#[test] +fn a_header_declaring_more_chunks_than_the_bound_is_refused_before_the_tables_are_read() { + // One column of 2^24 + 1 rows, one row a chunk: the file can't hold its table, but the count + // alone is refused first. + let mut bytes = file(vec![layer(None, (0, 0), (1, 1), &["R"], false, Compression::Uncompressed)], (1, 1)); + patch_box(&mut bytes, "dataWindow", [0, 0, 0, 1 << 24]); + patch_box(&mut bytes, "displayWindow", [0, 0, 0, 1 << 24]); + let got = Index::build(&mut Cursor::new(&bytes[..]), &Limits::none(), &never); + assert!(matches!(&got, Err(Error::LimitExceeded(m)) if m.contains("chunks")), "{got:?}"); +} + +/// A source that fails with `kind` once `after` bytes have been read. +struct Failing<'a> { + inner: Cursor<&'a [u8]>, + after: u64, + kind: std::io::ErrorKind, +} + +impl Read for Failing<'_> { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + if self.inner.position() >= self.after { + return Err(std::io::Error::new(self.kind, "the share went away")); + } + let room = (self.after - self.inner.position()) as usize; + let n = buf.len().min(room); + self.inner.read(&mut buf[..n]) + } +} + +impl Seek for Failing<'_> { + fn seek(&mut self, pos: SeekFrom) -> std::io::Result { + self.inner.seek(pos) + } +} + +#[test] +fn a_failing_source_is_an_io_error_and_a_short_file_is_malformed() { + let bytes = file(vec![layer(None, (0, 0), (8, 8), &["R"], false, Compression::ZIP1)], (8, 8)); + for after in [4u64, 40, 200] { + let mut source = Failing { inner: Cursor::new(&bytes[..]), after, kind: std::io::ErrorKind::Other }; + let got = Index::build(&mut source, &Limits::default(), &never); + assert!(matches!(got, Err(Error::Io(_))), "after {after}: {got:?}"); + let got = Index::build(&mut Cursor::new(&bytes[..after as usize]), &Limits::default(), &never); + assert!(matches!(got, Err(Error::Malformed(_))), "cut at {after}: {got:?}"); + } +} + +#[test] +fn a_read_called_off_while_workers_hold_chunks_stops() { + let bytes = file(vec![layer(None, (0, 0), (1024, 600), &["R", "G"], false, Compression::Uncompressed)], (1024, 600)); + let index = indexed(&bytes); + let channels = channels_of(&index, 0); + let stop = Cell::new(false); + let mut bands = 0usize; + let request = Request { workers: Some(2), ..Request::new(&channels, usize::MAX) }; + let got = read_bands(&mut Cursor::new(&bytes[..]), &index, &request, &|| stop.get(), &mut |_| { + bands += 1; + stop.set(true); + }); + assert_eq!(got.err(), Some(Error::Cancelled)); + assert!(bands > 0 && bands < 1200, "{bands} bands"); +} + +#[test] +fn a_part_written_bottom_up_arrives_bottom_up() { + let mut part = layer(None, (0, 0), (8, 12), &["Y"], false, Compression::ZIP1); + part.encoding.line_order = LineOrder::Decreasing; + let bytes = file(vec![part], (8, 12)); + let index = indexed(&bytes); + let mut rows = Vec::new(); + let mut plane = vec![f32::NAN; 96]; + read_bands(&mut Cursor::new(&bytes[..]), &index, &Request::new(&[(0, 0)], usize::MAX), &never, &mut |b: Band| { + rows.push(b.y); + plane[b.y * 8..(b.y + b.height) * 8].copy_from_slice(&b.samples); + }) + .unwrap(); + assert_eq!(rows, (0..12).rev().collect::>(), "file order"); + assert_eq!(vec![plane], oracle(&bytes, 0, 0.0)); +} + +#[test] +fn a_tiled_file_with_rip_levels_is_read_from_its_largest_level_only() { + use exr::image::RipMaps; + use exr::math::RoundingMode; + use exr::prelude::Levels; + let (w, h) = (9usize, 7usize); + let sizes: Vec = exr::meta::rip_map_levels(RoundingMode::Down, Vec2(w, h)).map(|(_, size)| size.area()).collect(); + let level_count = Vec2(exr::meta::compute_level_count(RoundingMode::Down, w), exr::meta::compute_level_count(RoundingMode::Down, h)); + assert!(sizes.len() > 1); + let map_data = sizes + .iter() + .enumerate() + .map(|(level, n)| FlatSamples::F32(if level == 0 { (0..*n).map(|i| value(i, 0)).collect() } else { vec![1e9; *n] })) + .collect(); + let bytes = tiled_levels(Levels::Rip { rounding_mode: RoundingMode::Down, level_data: RipMaps { map_data, level_count } }); + let index = indexed(&bytes); + let read = read_planes(&mut Cursor::new(&bytes[..]), &index, &Request::new(&[(0, 0)], usize::MAX), &never).unwrap(); + assert_eq!(read.planes[0], (0..w * h).map(|i| value(i, 0)).collect::>()); + assert_eq!(read.stats.chunks[0], 6); +} + +#[test] +fn rows_an_odd_data_window_and_a_level_together_on_several_threads() { + // A 37 × 29 data window at (3, 5) in a 50 × 40 display window, one row a chunk. + let bytes = file(vec![layer(None, (3, 5), (37, 29), &["R", "G"], false, Compression::ZIP1)], (50, 40)); + let index = indexed(&bytes); + let whole = oracle(&bytes, 0, 9.0); + for level in [0u32, 1, 2] { + let stride = 1usize << level; + let request = Request { rows: Some(7..33), level, fill: &[9.0, 9.0], workers: Some(4), ..Request::new(&[(0, 1), (0, 0)], usize::MAX) }; + let read = read_planes(&mut Cursor::new(&bytes[..]), &index, &request, &never).unwrap(); + for (slot, channel) in [(0, 1), (1, 0)] { + let expected: Vec = (0..40) + .step_by(stride) + .flat_map(|y| (0..50).step_by(stride).map(move |x| (y, x))) + .map(|(y, x)| if (7..33).contains(&y) { whole[channel][y * 50 + x] } else { 9.0 }) + .collect(); + assert_eq!(read.planes[slot], expected, "level {level} slot {slot}"); + } + } +} + +#[test] +fn a_damaged_chunk_in_the_middle_ends_the_read_after_the_bands_before_it() { + // 600 uncompressed chunks of 8200 bytes (y, size, 2 × 1024 f32), the last at the file's end: + // row 400's chunk claims another row. The read spans several pieces and threads. + let mut bytes = file(vec![layer(None, (0, 0), (1024, 600), &["R", "G"], false, Compression::Uncompressed)], (1024, 600)); + let at = bytes.len() - (600 - 400) * 8200; + assert_eq!(&bytes[at..at + 4], &400i32.to_le_bytes()); + bytes[at..at + 4].copy_from_slice(&7i32.to_le_bytes()); + let index = indexed(&bytes); + let channels = channels_of(&index, 0); + let mut reached = [0usize; 2]; + let request = Request { workers: Some(4), ..Request::new(&channels, usize::MAX) }; + let got = read_bands(&mut Cursor::new(&bytes[..]), &index, &request, &never, &mut |b: Band| { + assert_eq!(b.y, reached[b.slot]); + reached[b.slot] = b.y + b.height; + }); + assert!(matches!(got, Err(Error::Malformed(_))), "{got:?}"); + assert_eq!(reached, [400, 400], "every row before the damaged chunk, nothing after it"); +} + +#[test] +fn a_chunk_that_is_not_the_block_its_table_entry_stands_for_is_refused() { + // Scan lines: the entries of rows 0 and 1 swapped; reading row 0 fetches row 1's chunk. + let mut bytes = file(vec![layer(None, (0, 0), (8, 16), &["Y"], false, Compression::ZIP1)], (8, 16)); + let table = table_at(&bytes, 16); + let (a, b) = (bytes[table..table + 8].to_vec(), bytes[table + 8..table + 16].to_vec()); + bytes[table..table + 8].copy_from_slice(&b); + bytes[table + 8..table + 16].copy_from_slice(&a); + let index = indexed(&bytes); + let request = Request { rows: Some(0..1), ..Request::new(&[(0, 0)], usize::MAX) }; + let got = read_planes(&mut Cursor::new(&bytes[..]), &index, &request, &never); + assert!(matches!(got, Err(Error::Malformed(_))), "{got:?}"); + // Mip-mapped tiles: the first full-resolution tile's entry points at a smaller level's tile. + use exr::math::RoundingMode; + use exr::prelude::Levels; + let sizes: Vec = exr::meta::mip_map_levels(RoundingMode::Down, Vec2(9, 7)).map(|(_, size)| size.area()).collect(); + let mut mip = + tiled_levels(Levels::Mip { rounding_mode: RoundingMode::Down, level_data: sizes.iter().map(|n| FlatSamples::F32(vec![1.0; *n])).collect() }); + let table = table_at(&mip, 6 + sizes.len() - 1); + let smaller = mip[table + 6 * 8..table + 7 * 8].to_vec(); + mip[table..table + 8].copy_from_slice(&smaller); + let index = indexed(&mip); + let got = read_planes(&mut Cursor::new(&mip[..]), &index, &Request::new(&[(0, 0)], usize::MAX), &never); + assert!(matches!(got, Err(Error::Malformed(_))), "{got:?}"); +} + +#[test] +fn the_decoded_bytes_in_flight_stay_within_the_budget_however_well_the_file_compresses() { + // A flat colour in ZIP1 lines: each 1024-pixel line of two f32 channels compresses to a few + // dozen bytes but decodes to 8 KiB (and 8 KiB of bands), so a 256 KiB piece would hold + // thousands of them. + let (w, h) = (1024usize, 600usize); + let list = vec![AnyChannel::new("R", FlatSamples::F32(vec![0.25; w * h])), AnyChannel::new("G", FlatSamples::F32(vec![0.5; w * h]))]; + let encoding = Encoding { compression: Compression::ZIP1, blocks: Blocks::ScanLines, line_order: LineOrder::Increasing }; + let bytes = file(vec![Layer::new((w, h), LayerAttributes::default(), encoding, AnyChannels::sort(list.into()))], (w, h)); + let index = indexed(&bytes); + let channels = channels_of(&index, 0); + let budget = 64 << 10; + let mut rows = 0usize; + let request = Request { workers: Some(4), ..Request::new(&channels, usize::MAX) }; + let mut sink = |b: Band| { + assert!(b.samples.iter().all(|v| *v == 0.25 || *v == 0.5)); + rows += b.height; + }; + let stats = super::seek::read_bands_within(&mut Cursor::new(&bytes[..]), &index, &request, &never, &mut sink, budget).unwrap(); + assert_eq!(rows, 2 * h); + assert!(stats.bytes < 64 << 10, "{} bytes: well compressed", stats.bytes); + assert!(stats.in_flight > 0 && stats.in_flight <= budget, "{} decoded bytes in flight", stats.in_flight); +} + +#[test] +fn cancel_is_polled_before_each_chunk_reaches_the_sink() { + // Decoded on the calling thread, the first piece's ~30 chunks are ready at once; a cancel + // the sink raises after the first chunk's two bands stops the rest. + let bytes = file(vec![layer(None, (0, 0), (1024, 600), &["R", "G"], false, Compression::Uncompressed)], (1024, 600)); + let index = indexed(&bytes); + let channels = channels_of(&index, 0); + let stop = Cell::new(false); + let mut bands = 0usize; + let request = Request { workers: Some(0), ..Request::new(&channels, usize::MAX) }; + let got = read_bands(&mut Cursor::new(&bytes[..]), &index, &request, &|| stop.get(), &mut |_| { + bands += 1; + stop.set(true); + }); + assert_eq!(got.err(), Some(Error::Cancelled)); + assert_eq!(bands, 2, "one chunk (both channels), nothing after the cancel"); +} + +#[test] +fn a_chunk_offset_into_the_headers_is_outside_the_chunk_area() { + let mut bytes = file(vec![layer(None, (0, 0), (8, 16), &["Y"], false, Compression::ZIP1)], (8, 16)); + let table = table_at(&bytes, 16); + bytes[table..table + 8].copy_from_slice(&8u64.to_le_bytes()); + let index = indexed(&bytes); + assert_eq!(index.chunks(0).next().map(|c| c.len), Some(0)); + let got = read_planes(&mut Cursor::new(&bytes[..]), &index, &Request::new(&[(0, 0)], usize::MAX), &never); + assert!(matches!(&got, Err(Error::Malformed(m)) if m.contains("outside the chunk area")), "{got:?}"); +} + +/// A single-part scan-line DWA file of `w × h` with these half channels (a ramp each). +fn dwa_file(compression: Compression, (w, h): (usize, usize), names: &[&str], tiles: bool) -> Vec { + use exr::prelude::f16; + let list: Vec> = names + .iter() + .enumerate() + .map(|(k, n)| AnyChannel::new(*n, FlatSamples::F16((0..w * h).map(|i| f16::from_f32(((i % w) + k) as f32 / 64.0)).collect()))) + .collect(); + let blocks = if tiles { Blocks::Tiles(Vec2(16, 16)) } else { Blocks::ScanLines }; + let encoding = Encoding { compression, blocks, line_order: LineOrder::Increasing }; + file(vec![Layer::new((w, h), LayerAttributes::default(), encoding, AnyChannels::sort(list.into()))], (w, h)) +} + +/// `bytes` with DWA counter `field` (0: version … 9: DC count) of the first chunk set to `value`. +fn with_dwa_counter(bytes: &[u8], field: usize, value: u64) -> Vec { + let index = indexed(bytes); + let chunk = index.chunks(0).next().unwrap(); + let at = chunk.offset as usize; + // (y, then the payload size: compressed, so it's no raw copy of the 64 × 32 half block) + assert_ne!(i32::from_le_bytes(bytes[at + 4..at + 8].try_into().unwrap()), 64 * 32 * 2); + let mut out = bytes.to_vec(); + out[at + 8 + 8 * field..at + 16 + 8 * field].copy_from_slice(&value.to_le_bytes()); + out +} + +/// Every reader of the crate refuses `bytes` as damaged, and none aborts. +fn refused_by_every_reader(bytes: &[u8], what: &str) { + use crate::decode::{Error as DecodeError, decode_display_part, decode_part}; + let index = indexed(bytes); + let got = read_planes(&mut Cursor::new(bytes), &index, &Request::new(&[(0, 0)], usize::MAX), &never); + assert!(matches!(&got, Err(Error::Malformed(m)) if m.contains("DWA")), "seek, {what}: {got:?}"); + let got = decode_display_part(bytes, 0, &|_| true, &|_| 0.0, &Limits::default()).map(|_| ()); + assert!(matches!(&got, Err(DecodeError::Malformed(m)) if m.contains("DWA")), "decode_display_part, {what}: {got:?}"); + let got = decode_part(bytes, 0, &|_| true, &Limits::default()).map(|_| ()); + assert!(matches!(&got, Err(DecodeError::Malformed(m)) if m.contains("DWA")), "decode_part, {what}: {got:?}"); + let got = crate::stream::Decoder::new(Cursor::new(bytes), &never).and_then(|mut d| { + while d.next_band(&never)?.is_some() {} + Ok(()) + }); + assert!(matches!(&got, Err(DecodeError::Malformed(m)) if m.contains("DWA")), "stream, {what}: {got:?}"); +} + +#[test] +fn a_dwa_chunk_declaring_2_to_the_44_dc_values_is_refused_not_an_abort() { + // The case that aborted the process: DWAA 64 × 32, one half channel R, DC count 2^44. + let bytes = with_dwa_counter(&dwa_file(Compression::DWAA(None), (64, 32), &["R"], false), 9, 1 << 44); + refused_by_every_reader(&bytes, "DC count 2^44"); +} + +#[test] +fn a_dwa_chunk_just_above_the_data_window_is_checked_as_the_block_the_decoder_rounds_it_into() { + // The `exr` crate divides a scan line's offset from the data window toward zero, so a chunk + // at y = -1 (DWAA, 32 lines) or -255 (DWAB, 256 lines) becomes block 0 and is decompressed. + use crate::decode::{Error as DecodeError, decode_display_part, decode_part}; + for (compression, y) in [(Compression::DWAA(None), -1i32), (Compression::DWAA(None), -31), (Compression::DWAB(None), -255)] { + let mut bytes = with_dwa_counter(&dwa_file(compression, (64, 32), &["R"], false), 9, 1 << 40); + let at = indexed(&bytes).chunks(0).next().unwrap().offset as usize; + bytes[at..at + 4].copy_from_slice(&y.to_le_bytes()); + let got = decode_part(&bytes, 0, &|_| true, &Limits::default()).map(|_| ()); + assert!(matches!(&got, Err(DecodeError::Malformed(m)) if m.contains("DWA")), "decode_part, {compression:?} y {y}: {got:?}"); + let got = decode_display_part(&bytes, 0, &|_| true, &|_| 0.0, &Limits::default()).map(|_| ()); + assert!(matches!(&got, Err(DecodeError::Malformed(m)) if m.contains("DWA")), "decode_display_part, {compression:?} y {y}: {got:?}"); + } +} + +#[test] +fn a_dwa_chunk_the_scan_cannot_read_is_refused_not_left_to_the_decoder() { + use crate::decode::{Error as DecodeError, decode_part}; + let good = dwa_file(Compression::DWAA(None), (64, 32), &["R"], false); + let table = table_at(&good, 1); + // The entry points past the end of the file, and into the last bytes of it. + for offset in [good.len() as u64 + 100, good.len() as u64 - 6] { + let mut bytes = good.clone(); + bytes[table..table + 8].copy_from_slice(&offset.to_le_bytes()); + let got = decode_part(&bytes, 0, &|_| true, &Limits::default()).map(|_| ()); + assert!(matches!(&got, Err(DecodeError::Malformed(m)) if m.contains("DWA")), "offset {offset}: {got:?}"); + } +} + +#[test] +fn a_chunk_declaring_a_small_payload_before_padding_is_held_to_what_the_payload_can_hold() { + // A 1 × 1 PIZ file patched to 16 Mi pixels wide (a 64 MiB block) with 100 KiB of padding + // after its one chunk: the gap to the file's end could hold the block at 1024 : 1, the few + // dozen bytes the chunk declares can't. + let mut bytes = file(vec![layer(None, (0, 0), (1, 1), &["R"], false, Compression::PIZ)], (1, 1)); + patch_box(&mut bytes, "dataWindow", [0, 0, (16 << 20) - 1, 0]); + bytes.extend(std::iter::repeat_n(0u8, 100 << 10)); + let limits = Limits { max_ratio: 1024, ..Limits::none() }; + let index = Index::build(&mut Cursor::new(&bytes[..]), &limits, &never).unwrap(); + assert_eq!(index.check_blocks(), Ok(()), "the gap is enough"); + let got = read_planes(&mut Cursor::new(&bytes[..]), &index, &Request::new(&[(0, 0)], 1 << 30), &never); + assert!(matches!(&got, Err(Error::LimitExceeded(m)) if m.contains("can hold")), "{got:?}"); +} + +#[test] +fn every_dwa_counter_the_inflater_sizes_by_is_bounded_by_the_block() { + for compression in [Compression::DWAA(None), Compression::DWAB(None)] { + // (fields: 1 unknown uncompressed size, 6 RLE uncompressed size, 7 RLE raw size, 8 AC + // count, 9 DC count; a value just over each bound, and a huge one) + let good = dwa_file(compression, (64, 32), &["R"], false); + // The block: 64 × 32 half, 4096 bytes; 8 × 4 DCT blocks of one channel. + for (field, over) in [(1, 4097u64), (6, 8193), (7, 4097), (8, 32 * 63 + 1), (9, 33)] { + for value in [over, 1 << 44, i64::MAX as u64] { + refused_by_every_reader(&with_dwa_counter(&good, field, value), &format!("{compression:?} counter {field} = {value}")); + } + } + } +} + +#[test] +fn valid_dwa_files_pass_the_guard() { + // Odd sizes (blocks padded to 8 × 8 for the DCT), lossy and lossless channels, one-row and + // one-column parts, scan lines and tiles. + for compression in [Compression::DWAA(None), Compression::DWAB(None)] { + for (size, names, tiles) in [ + ((1, 1), &["R"][..], false), + ((9, 7), &["B", "G", "R"][..], false), + ((1, 300), &["Y", "A"][..], false), + ((300, 1), &["R", "G", "B", "A", "Z"][..], false), + ((64, 32), &["R"][..], false), + ((37, 41), &["R", "G", "B", "A"][..], true), + ] { + let bytes = dwa_file(compression, size, names, tiles); + let index = indexed(&bytes); + assert_eq!(index.check_blocks(), Ok(())); + let channels = channels_of(&index, 0); + let read = read_planes(&mut Cursor::new(&bytes[..]), &index, &Request::new(&channels, usize::MAX), &never); + assert_eq!(read.map(|r| r.planes).ok(), Some(oracle(&bytes, 0, 0.0)), "{compression:?} {size:?} {names:?} tiles {tiles}"); + assert!(crate::decode::decode_part(&bytes, 0, &|_| true, &Limits::default()).is_ok()); + if !tiles { + let mut stream = crate::stream::Decoder::new(Cursor::new(&bytes[..]), &never); + if let Ok(d) = stream.as_mut() { + while d.next_band(&never).unwrap().is_some() {} + } + } + } + } +} + +/// A source that panics when anything at or past `chunks` (where the chunks start) is read: a +/// read that a check should have refused fails fast instead of allocating. +#[cfg(target_pointer_width = "64")] +struct NoChunks<'a> { + inner: Cursor<&'a [u8]>, + chunks: u64, +} + +#[cfg(target_pointer_width = "64")] +impl Read for NoChunks<'_> { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + assert!(self.inner.position() < self.chunks, "a chunk was read"); + self.inner.read(buf) + } +} + +#[cfg(target_pointer_width = "64")] +impl Seek for NoChunks<'_> { + fn seek(&mut self, pos: SeekFrom) -> std::io::Result { + self.inner.seek(pos) + } +} + +// (budgets over 4 GiB: a 64-bit usize) +#[cfg(target_pointer_width = "64")] +#[test] +fn a_tiny_chunk_declaring_a_gigabyte_block_is_refused_before_it_is_read() { + // A 1 × 1 file whose data window is patched to 2^30 - 2 pixels wide: one row of one f32 + // channel is a 4 GiB block over a chunk of a few dozen bytes. Within an app's 8 GiB budget + // the block cap allows it; the bytes can't hold it. + let limits = Limits { max_ratio: Limits::default().max_ratio, ..Limits::none() }; + for compression in [Compression::PIZ, Compression::DWAB(None), Compression::ZIP16] { + let mut bytes = file(vec![layer(None, (0, 0), (1, 1), &["R"], false, compression)], (1, 1)); + patch_box(&mut bytes, "dataWindow", [0, 0, (1 << 30) - 3, 0]); + let index = Index::build(&mut Cursor::new(&bytes[..]), &limits, &never).unwrap(); + let chunk = index.chunks(0).next().unwrap(); + assert!(chunk.len < 200 && chunk.width == (1 << 30) - 2, "{chunk:?}"); + let got = index.check_blocks(); + assert!(matches!(&got, Err(Error::LimitExceeded(m)) if m.contains("can hold")), "{compression:?}: {got:?}"); + let mut source = NoChunks { inner: Cursor::new(&bytes[..]), chunks: chunk.offset }; + let got = read_planes(&mut source, &index, &Request::new(&[(0, 0)], 8 << 30), &never); + assert!(matches!(&got, Err(Error::LimitExceeded(m)) if m.contains("can hold")), "{compression:?}: {got:?}"); + } +} + +#[test] +fn black_frames_that_compress_extremely_well_still_read() { + // 4096 × 256 × 4 half channels of zero in DWAB (one 8 MiB block) and of f32 in ZIP16. + use exr::prelude::f16; + let (w, h) = (4096usize, 256usize); + for (compression, half) in [(Compression::DWAB(None), true), (Compression::ZIP16, false)] { + let list: Vec> = ["R", "G", "B", "A"] + .iter() + .map(|n| AnyChannel::new(*n, if half { FlatSamples::F16(vec![f16::ZERO; w * h]) } else { FlatSamples::F32(vec![0.0; w * h]) })) + .collect(); + let encoding = Encoding { compression, blocks: Blocks::ScanLines, line_order: LineOrder::Increasing }; + let bytes = file(vec![Layer::new((w, h), LayerAttributes::default(), encoding, AnyChannels::sort(list.into()))], (w, h)); + let index = indexed(&bytes); + let smallest = index.chunks(0).map(|c| c.len).min().unwrap(); + assert_eq!(index.check_blocks(), Ok(()), "{compression:?}, chunks from {smallest} bytes"); + let channels = channels_of(&index, 0); + let read = read_planes(&mut Cursor::new(&bytes[..]), &index, &Request::new(&channels, usize::MAX), &never).unwrap(); + assert!(read.planes.iter().all(|p| p.len() == w * h && p.iter().all(|v| *v == 0.0)), "{compression:?}"); + } +} + +#[test] +fn the_ratio_bound_allows_a_mebibyte_more_and_four_times_as_much_for_dwa() { + use exr::prelude::f16; + let build = |bytes: &[u8], max_ratio| Index::build(&mut Cursor::new(bytes), &Limits { max_ratio, ..Limits::default() }, &never).unwrap(); + // A black ZIP16 block of 512 × 16 f32 RGBA is 128 KiB over a few hundred bytes: refused at + // one byte per byte but for the 1 MiB every chunk is allowed besides. + let zip = file(vec![layer(None, (0, 0), (512, 16), &["R", "G", "B", "A"], false, Compression::ZIP16)], (512, 16)); + let list: Vec> = + ["R", "G", "B", "A"].iter().map(|n| AnyChannel::new(*n, FlatSamples::F32(vec![0.0; 512 * 16]))).collect(); + let encoding = Encoding { compression: Compression::ZIP16, blocks: Blocks::ScanLines, line_order: LineOrder::Increasing }; + let black = file(vec![Layer::new((512, 16), LayerAttributes::default(), encoding, AnyChannels::sort(list.into()))], (512, 16)); + for bytes in [&zip, &black] { + let index = build(bytes, 1); + assert!(index.chunks(0).all(|c| c.len < 128 << 10)); + assert_eq!(index.check_blocks(), Ok(())); + } + // A black DWAB block of 4096 × 256 half RGBA (8 MiB): with a ratio that lets its bytes hold + // between a quarter of the block and the block (less the 1 MiB), only the fourfold DWA + // allowance admits it. + let list: Vec> = + ["R", "G", "B", "A"].iter().map(|n| AnyChannel::new(*n, FlatSamples::F16(vec![f16::ZERO; 4096 * 256]))).collect(); + let encoding = Encoding { compression: Compression::DWAB(None), blocks: Blocks::ScanLines, line_order: LineOrder::Increasing }; + let dwab = file(vec![Layer::new((4096, 256), LayerAttributes::default(), encoding, AnyChannels::sort(list.into()))], (4096, 256)); + let len = indexed(&dwab).chunks(0).next().unwrap().len; + let ratio = (5u64 << 20) / len; // len × ratio ≈ 5 MiB: 8 MiB > that + 1 MiB, but ≤ 4 × that + 1 MiB + let index = build(&dwab, ratio); + assert_eq!(index.check_blocks(), Ok(()), "{len}-byte chunk, ratio {ratio}"); + assert!(matches!(build(&dwab, ratio / 8).check_blocks(), Err(Error::LimitExceeded(_)))); +} + +#[cfg(target_pointer_width = "64")] +#[test] +fn memory_the_system_cannot_give_is_out_of_memory_not_a_limit() { + // A display window of (2^30 - 2)^2 pixels: one f32 plane is 4 EiB, which no system maps, so + // the reservation fails at once (nothing is touched). + let mut bytes = file(vec![layer(None, (0, 0), (1, 1), &["R"], false, Compression::ZIP1)], (1, 1)); + patch_box(&mut bytes, "displayWindow", [0, 0, (1 << 30) - 3, (1 << 30) - 3]); + let index = Index::build(&mut Cursor::new(&bytes[..]), &Limits::none(), &never).unwrap(); + let got = read_planes(&mut Cursor::new(&bytes[..]), &index, &Request::new(&[(0, 0)], usize::MAX), &never); + assert!(matches!(got, Err(Error::OutOfMemory(_))), "{got:?}"); +} + +#[test] +#[allow(clippy::reversed_empty_ranges)] // (reversed ranges are what is tested) +fn empty_reversed_and_extreme_row_ranges_read_without_overflow() { + // (alignment to a level's grid near the ends of i64 overflowed before the rows were clamped) + let bytes = file(vec![layer(None, (0, 0), (9, 7), &["R"], false, Compression::ZIP1)], (9, 7)); + let index = indexed(&bytes); + let whole = oracle(&bytes, 0, 0.0)[0].clone(); + for level in 0..=16u32 { + for rows in [i64::MAX..i64::MAX, i64::MIN..i64::MIN, i64::MAX - 1..i64::MAX, 5..2, i64::MAX..i64::MIN, -3..-1, 7..9] { + let request = Request { rows: Some(rows.clone()), level, ..Request::new(&[(0, 0)], usize::MAX) }; + let mut bands = 0; + read_bands(&mut Cursor::new(&bytes[..]), &index, &request, &never, &mut |_| bands += 1).unwrap(); + assert_eq!(bands, 0, "{rows:?} at level {level} holds no row"); + assert_eq!(index.estimate(&request).unwrap().chunks, 0); + } + // Everything, from one end of i64 to the other. + let request = Request { rows: Some(i64::MIN..i64::MAX), level, ..Request::new(&[(0, 0)], usize::MAX) }; + let read = read_planes(&mut Cursor::new(&bytes[..]), &index, &request, &never).unwrap(); + let stride = 1usize << level; + let expected: Vec = + (0..7).step_by(stride).flat_map(|y| (0..9).step_by(stride).map(move |x| (y, x))).map(|(y, x)| whole[y * 9 + x]).collect(); + assert_eq!(read.planes[0], expected, "level {level}"); + } +} + +#[test] +fn a_sink_can_stop_the_read_with_its_own_error() { + // 600 chunks on 4 threads: the sink refuses its third band. Nothing reaches it after that, the + // read returns (no hang) with the sink's error unchanged, and the counts say how far it got. + let bytes = file(vec![layer(None, (0, 0), (1024, 600), &["R", "G"], false, Compression::Uncompressed)], (1024, 600)); + let index = indexed(&bytes); + let channels = channels_of(&index, 0); + let request = Request { workers: Some(4), ..Request::new(&channels, usize::MAX) }; + let mut calls = 0usize; + let mut stats = Stats::default(); + let got = read_bands_try( + &mut Cursor::new(&bytes[..]), + &index, + &request, + &never, + &mut |_: Band| { + calls += 1; + if calls >= 3 { Err("the tile store is full") } else { Ok(()) } + }, + &mut stats, + ); + assert_eq!(got, Err(Stop::Sink("the tile store is full"))); + assert_eq!(calls, 3, "no band after the sink's error"); + assert!(stats.lines >= 1 && stats.lines < 600 && stats.bytes > 0, "{stats:?}"); + // A sink that never fails reads everything, as `read_bands` does. + let mut bands = 0usize; + let mut sink = |_: Band| -> std::result::Result<(), ()> { + bands += 1; + Ok(()) + }; + read_bands_try(&mut Cursor::new(&bytes[..]), &index, &request, &never, &mut sink, &mut stats).unwrap(); + assert_eq!((bands, stats.lines), (1200, 600)); + // The read's own errors come back as such. + let mut damaged = bytes.clone(); + let at = damaged.len() - 200 * 8200; + damaged[at..at + 4].copy_from_slice(&7i32.to_le_bytes()); + let index = indexed(&damaged); + let got = read_bands_try(&mut Cursor::new(&damaged[..]), &index, &request, &never, &mut |_| Ok::<(), ()>(()), &mut stats); + assert!(matches!(got, Err(Stop::Read(Error::Malformed(_)))), "{got:?}"); +} + +/// The planes of `bands` (grouped or not) for `slots` slots of `w × h`, de-interleaved. +fn planes_of(bands: &[Band], slots: usize, (w, h): (usize, usize)) -> Vec> { + let mut planes = vec![vec![-1e30; w * h]; slots]; + for b in bands { + assert_eq!(b.samples.len(), b.width * b.height * b.values_per_pixel); + for r in 0..b.height { + for c in 0..b.width { + for k in 0..b.values_per_pixel { + planes[b.slot + k][(b.y + r) * w + b.x + c] = b.samples[(r * b.width + c) * b.values_per_pixel + k]; + } + } + } + } + planes +} + +#[test] +fn grouped_slots_arrive_interleaved_as_one_band_per_block() { + for tiles in [false, true] { + let bytes = file(vec![layer(None, (0, 0), (13, 11), &["R", "G", "B", "A", "Z"], tiles, Compression::ZIP1)], (13, 11)); + let index = indexed(&bytes); + // Stored A B G R Z: ask for R G B A as a group, Z alone, and G again in a group of one. + let names: Vec<&str> = index.parts()[0].channels.iter().map(|c| c.name.as_str()).collect(); + let at = |n: &str| (0, names.iter().position(|m| *m == n).unwrap()); + let channels = [at("R"), at("G"), at("B"), at("A"), at("Z"), at("G")]; + for (rows, level) in [(None, 0u32), (Some(3..9), 0), (None, 1)] { + let base = Request { rows: rows.clone(), level, ..Request::new(&channels, usize::MAX) }; + let mut single = Vec::new(); + read_bands(&mut Cursor::new(&bytes[..]), &index, &base, &never, &mut |b| single.push(b)).unwrap(); + let groups = [0..4, 5..6]; + let mut grouped = Vec::new(); + read_bands(&mut Cursor::new(&bytes[..]), &index, &Request { groups: &groups, ..base.clone() }, &never, &mut |b| grouped.push(b)).unwrap(); + assert!(single.iter().all(|b| b.values_per_pixel == 1)); + assert!(grouped.iter().all(|b| b.values_per_pixel == if b.slot == 0 { 4 } else { 1 }), "{tiles} {rows:?} {level}"); + assert_eq!(grouped.iter().filter(|b| b.slot == 0).count(), single.iter().filter(|b| b.slot == 0).count(), "one band per block"); + let size = level_size((13, 11), level); + assert_eq!(planes_of(&grouped, 6, size), planes_of(&single, 6, size), "tiles {tiles} rows {rows:?} level {level}"); + // `read_planes` ignores groups. + let with = read_planes(&mut Cursor::new(&bytes[..]), &index, &Request { groups: &groups, ..base.clone() }, &never).unwrap(); + assert_eq!(with, read_planes(&mut Cursor::new(&bytes[..]), &index, &base, &never).unwrap()); + } + } +} + +#[test] +fn groups_must_be_slots_of_one_part_each_in_one_group() { + let bytes = file( + vec![ + layer(Some("a"), (0, 0), (8, 8), &["R", "G"], false, Compression::ZIP1), + layer(Some("b"), (0, 0), (8, 8), &["Z"], false, Compression::ZIP1), + ], + (8, 8), + ); + let index = indexed(&bytes); + let channels = [(0, 0), (0, 1), (1, 0)]; + let read = |groups: &[std::ops::Range]| { + read_bands(&mut Cursor::new(&bytes[..]), &index, &Request { groups, ..Request::new(&channels, usize::MAX) }, &never, &mut |_| {}) + }; + assert!(matches!(read(&[1..3]), Err(Error::Unsupported(m)) if m.contains("spans parts"))); + for bad in [&[0..2, 1..2][..], &[2..4], &[1..1]] { + assert!(matches!(read(bad), Err(Error::Malformed(_))), "{bad:?}"); + } + assert!(read(&[0..2]).is_ok()); +} + +#[test] +fn the_estimate_covers_what_the_read_does() { + // A flat colour in ZIP16 (decodes to far more than it reads), lines of a ramp, and tiles. + let (w, h) = (1024usize, 600usize); + let list = vec![AnyChannel::new("R", FlatSamples::F32(vec![0.25; w * h])), AnyChannel::new("G", FlatSamples::F32(vec![0.5; w * h]))]; + let encoding = Encoding { compression: Compression::ZIP16, blocks: Blocks::ScanLines, line_order: LineOrder::Increasing }; + let flat = file(vec![Layer::new((w, h), LayerAttributes::default(), encoding, AnyChannels::sort(list.into()))], (w, h)); + let ramp = file(vec![layer(None, (0, 0), (1024, 600), &["R", "G"], false, Compression::Uncompressed)], (1024, 600)); + let tiled = file(vec![layer(None, (0, 0), (300, 200), &["R", "G", "B"], true, Compression::PIZ)], (300, 200)); + for bytes in [&flat, &ramp, &tiled] { + let index = indexed(bytes); + let channels = channels_of(&index, 0); + for (rows, level, workers) in [(None, 0u32, None), (Some(50..120), 0, Some(4)), (None, 1, Some(0))] { + let request = Request { rows: rows.clone(), level, workers, ..Request::new(&channels, usize::MAX) }; + let estimate = index.estimate(&request).unwrap(); + let read = read_planes(&mut Cursor::new(&bytes[..]), &index, &request, &never).unwrap(); + let what = format!("{rows:?} level {level}: {estimate:?} {:?}", read.stats); + assert_eq!(estimate.chunks, read.stats.chunks.iter().sum::(), "{what}"); + assert_eq!(estimate.compressed_bytes, read.stats.bytes, "{what}"); + assert!(estimate.working_bytes >= read.stats.in_flight, "{what}"); + let (ow, oh) = level_size((index.display().width as usize, index.display().height as usize), level); + assert_eq!(estimate.planes_bytes, (ow * oh * 4 * channels.len()) as u64, "{what}"); + // Measured from what is decoded: the largest chunk's block and bands (decoded, and + // as much again for scratch) fit in the working bytes. + let largest = decoded_per_chunk(bytes, &index, &request).into_iter().max().unwrap_or(0); + assert!(estimate.working_bytes >= 2 * largest, "{what}, largest chunk {largest}"); + } + } + // It refuses what the read refuses. + let index = indexed(&ramp); + assert!(matches!(index.estimate(&Request::new(&[(0, 0)], 1024)), Err(Error::LimitExceeded(_)))); + assert!(matches!(index.estimate(&Request::new(&[(0, 9)], usize::MAX)), Err(Error::Malformed(_)))); +} + +/// Per chunk of a read of part 0 of `bytes`: its decompressed block's bytes (all of the part's +/// channels over the chunk's pixels) plus the bytes of the bands it gave, as decoded. +fn decoded_per_chunk(bytes: &[u8], index: &Index, request: &Request) -> Vec { + let pixel_bytes: u64 = index.parts()[0].channels.iter().map(|c| if c.sample == crate::decode::SampleType::F16 { 2 } else { 4 }).sum(); + let mut bands_of: std::collections::BTreeMap = std::collections::BTreeMap::new(); + read_bands(&mut Cursor::new(bytes), index, request, &never, &mut |b: Band| { + assert_eq!(b.samples.len(), b.width * b.height * b.values_per_pixel, "exactly sized"); + assert_eq!(b.samples.capacity(), b.samples.len(), "no slack"); + *bands_of.entry(b.chunk).or_default() += 4 * b.samples.len() as u64; + }) + .unwrap(); + // The chunks a read fetches, in file order, as the bands number them. + let mut chunks: Vec = index.chunks(0).collect(); + chunks.sort_by_key(|c| c.offset); + let estimate = index.estimate(request).unwrap(); + let needed: Vec = chunks + .into_iter() + .filter(|c| { + let rows = request.rows.clone().unwrap_or(0..i64::MAX); + let stride = 1i64 << request.level; + (c.y as i64..(c.y + c.height) as i64).any(|r| rows.contains(&r) && r % stride == 0) + }) + .collect(); + assert_eq!(needed.len(), estimate.chunks); + needed.iter().enumerate().map(|(i, c)| (c.width * c.height) as u64 * pixel_bytes + bands_of.get(&i).copied().unwrap_or(0)).collect() +} + +#[test] +fn a_grouped_read_with_duplicates_keeps_one_chunk_in_flight_at_its_decoded_size() { + // 1024 × 64 RGBA f32 in ZIP16: 16-row blocks of 256 KiB. R, G, B, A and R again as one group: + // each block's band is 16 × 1024 × 5 values (320 KiB), so a chunk is 576 KiB decoded, over + // a 64 KiB budget: one at a time. + let bytes = file(vec![layer(None, (0, 0), (1024, 64), &["R", "G", "B", "A"], false, Compression::ZIP16)], (1024, 64)); + let index = indexed(&bytes); + let names: Vec<&str> = index.parts()[0].channels.iter().map(|c| c.name.as_str()).collect(); + let at = |n: &str| (0, names.iter().position(|m| *m == n).unwrap()); + let channels = [at("R"), at("G"), at("B"), at("A"), at("R")]; + let groups = [0..5]; + let request = Request { groups: &groups, workers: Some(4), ..Request::new(&channels, usize::MAX) }; + let decoded = decoded_per_chunk(&bytes, &index, &request); + assert_eq!(decoded, vec![(256 + 320) << 10; 4], "block and band of each chunk, as decoded"); + let budget = 64 << 10; + let mut bands = Vec::new(); + let stats = super::seek::read_bands_within(&mut Cursor::new(&bytes[..]), &index, &request, &never, &mut |b| bands.push(b), budget).unwrap(); + assert_eq!(stats.in_flight, (256 + 320) << 10, "one chunk at a time, counted at its decoded size"); + let estimate = super::seek::estimate_within(&index, &request, budget).unwrap(); + assert!(estimate.working_bytes >= 2 * ((256 + 320) << 10), "{estimate:?}"); + // The interleaved values, duplicates included, are those of per-channel reads. + let mut single = Vec::new(); + read_bands(&mut Cursor::new(&bytes[..]), &index, &Request::new(&channels, usize::MAX), &never, &mut |b| single.push(b)).unwrap(); + assert_eq!(planes_of(&bands, 5, (1024, 64)), planes_of(&single, 5, (1024, 64))); +} + +#[test] +fn bands_say_which_chunk_they_came_from_for_progress() { + // Tiles of 4 × 4 over 9 × 7 (six chunks), and one-row chunks read at level 2 over some rows. + let tiled = file(vec![layer(None, (0, 0), (9, 7), &["R", "G"], true, Compression::ZIP1)], (9, 7)); + let lines = file(vec![layer(None, (0, 0), (8, 16), &["Y"], false, Compression::ZIP1)], (8, 16)); + for (bytes, request) in [ + (&tiled, Request::new(&[(0, 0), (0, 1)], usize::MAX)), + (&lines, Request { rows: Some(3..14), level: 2, workers: Some(2), ..Request::new(&[(0, 0)], usize::MAX) }), + ] { + let index = indexed(bytes); + let estimate = index.estimate(&request).unwrap(); + let mut seen = Vec::new(); + read_bands(&mut Cursor::new(&bytes[..]), &index, &request, &never, &mut |b| seen.push(b.chunk)).unwrap(); + assert!(seen.windows(2).all(|w| w[0] <= w[1]), "in order: {seen:?}"); + seen.dedup(); + assert_eq!(seen, (0..estimate.chunks).collect::>(), "every chunk, numbered as the estimate counts"); + } +} + +#[test] +fn a_stop_s_source_is_the_read_error() { + use std::error::Error as _; + let read: Stop = Stop::Read(Error::Cancelled); + assert_eq!(read.source().map(|e| e.to_string()), Some(Error::Cancelled.to_string())); + assert!(Stop::Sink("full".to_string()).source().is_none()); +} diff --git a/crates/exr/src/stream.rs b/crates/exr/src/stream.rs index 8e69e534e..c3d942e94 100644 --- a/crates/exr/src/stream.rs +++ b/crates/exr/src/stream.rs @@ -262,6 +262,9 @@ impl Decoder { } self.source.read_exact(bytes).map_err(invalid)?; } + // (a DWA chunk's counters can make the crate allocate terabytes: see `crate::dwa_guard`) + let lines = self.rows.min((self.info.height as usize).saturating_sub(row as usize)); + crate::dwa_guard::check_payload(header, (self.info.width as usize, lines), &packed).map_err(invalid)?; let chunk = Chunk { layer_index: 0, compressed_block: CompressedBlock::ScanLine(CompressedScanLineBlock { y_coordinate: y, compressed_pixels_le: packed }),