From ea4ac2e5edb4199d007873781a4c25c757132d69 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Mon, 16 Nov 2026 23:07:23 +0100 Subject: [PATCH] Refuse OpenEXR blocks larger than their chunk can hold craft-exr now holds every block to what its chunk's bytes can decompress to, using the ratio the index was built with: the index is built with craft-exr's default ratio (no longer unbounded), a file with such a block is refused when indexed and by safe_to_decode before the image crate sees it, and each read checks it again. Memory the system refuses during a read is a read that failed just now, not remembered, instead of a file too large. --- crates/media/src/exr_index.rs | 37 +++++++++++++++++++--------- crates/media/src/exr_planes/tests.rs | 19 ++++++++++++++ crates/media/src/exr_read.rs | 21 ++++++++++------ 3 files changed, 59 insertions(+), 18 deletions(-) diff --git a/crates/media/src/exr_index.rs b/crates/media/src/exr_index.rs index cd9a7ef2d..988c3b370 100644 --- a/crates/media/src/exr_index.rs +++ b/crates/media/src/exr_index.rs @@ -10,9 +10,11 @@ //! it and never decode pixels. The reader ([`crate::exr_read`]) fetches the chunks it lists. //! //! Everything is validated before anything is sized by it: the display window -//! ([`crate::exr_channels::MAX_DISPLAY_PIXELS`]), the data windows (four times that), the blocks, -//! the chunk count (eight bytes of offset table each, so never more than the file can hold), and -//! every offset (inside the chunk area; a chunk with a bad offset is never read). +//! ([`crate::exr_channels::MAX_DISPLAY_PIXELS`]), the data windows (four times that), the chunk +//! count (eight bytes of offset table each, so never more than the file can hold), and every +//! offset (inside the chunk area; a chunk with a bad offset is never read). Every block is held +//! to what its chunk's bytes can decompress to ([`limits`]) when the file is indexed and before +//! it is handed to another decoder ([`safe_to_decode`]); each read checks it again. use std::collections::{HashMap, VecDeque}; use std::io::SeekFrom; @@ -144,11 +146,15 @@ pub(crate) const HTJ2K: &str = "OpenEXR HTJ2K compression isn't supported yet: r const MAX_DATA_PIXELS: usize = 4 * MAX_DISPLAY_PIXELS; /// What `craft-exr` may index: data windows of at most [`MAX_DATA_PIXELS`], of any width or -/// height (the display window is checked against [`MAX_DISPLAY_PIXELS`] in [`build`]). -/// `craft-exr` bounds every block itself (256 MiB uncompressed). +/// height (the display window is checked against [`MAX_DISPLAY_PIXELS`] in [`build`]), and +/// blocks of at most `craft-exr`'s default ratio of bytes per byte of their chunk (four times +/// that for DWA, plus 1 MiB): a hostile header declaring a block of gigabytes over a chunk of a +/// few bytes is refused, by every read and by [`safe_to_decode`]. A read's planes are bounded by +/// its own cap ([`crate::exr_read::MAX_DECODE_BYTES`]). fn limits() -> Limits { let mut limits = Limits::none(); limits.max_pixels = MAX_DATA_PIXELS as u64; + limits.max_ratio = Limits::default().max_ratio; limits } @@ -159,12 +165,17 @@ fn index_file(src: &mut dyn ReadSeek) -> Result { seek::Index::build(src, &limits(), &|| false) } -/// The headers of the OpenEXR `bytes` are safe to hand to a decoder: they can be read and no -/// part's data window or block is larger than this reader takes (#3271 review). A file this -/// reader doesn't cover (deep parts, HTJ2K) is left to the decoder; damaged headers, a part too -/// large and a failed read are refused. +/// The headers of the OpenEXR `bytes` are safe to hand to a decoder (the `image` crate, which +/// allocates every block by its header): they can be read, no part's data window is larger than +/// this reader takes, and no block is larger than its chunk can decompress to ([`limits`]) +/// (#3271 review). A file this reader doesn't cover (deep parts, HTJ2K) is left to the decoder; +/// damaged headers, a part or a block too large, and a failed read are refused. pub(crate) fn safe_to_decode(bytes: &[u8]) -> bool { - matches!(index_file(&mut std::io::Cursor::new(bytes)), Ok(_) | Err(seek::Error::Unsupported(_) | seek::Error::UnsupportedCompression(_))) + match index_file(&mut std::io::Cursor::new(bytes)) { + Ok(index) => index.check_blocks().is_ok(), + Err(seek::Error::Unsupported(_) | seek::Error::UnsupportedCompression(_)) => true, + Err(_) => false, + } } /// The channel list of `part` as the `exr` crate's validation took it (#3271): sorted by name, @@ -183,9 +194,13 @@ fn channels_valid(part: &PartInfo) -> bool { pub(crate) fn build(src: &mut dyn ReadSeek, stamp: FileStamp) -> Result { let file = index_file(src).map_err(|e| match e { seek::Error::UnsupportedCompression(name) if name.contains("HTJ2K") => IndexError::Unsupported(HTJ2K), - seek::Error::Io(_) => IndexError::Io, + // (memory refused just now: another attempt may succeed, nothing is remembered) + seek::Error::Io(_) | seek::Error::OutOfMemory(_) => IndexError::Io, _ => IndexError::Unreadable, })?; + // A block larger than its chunk can decompress to (a hostile header): the file is refused + // as a whole, as every read of it would refuse that chunk. + file.check_blocks().map_err(|_| IndexError::Unreadable)?; let parts = file.parts(); // Deep data isn't read (the whole file was refused before). if parts.iter().any(|p| p.deep || !channels_valid(p)) { diff --git a/crates/media/src/exr_planes/tests.rs b/crates/media/src/exr_planes/tests.rs index d4fd2d824..02f99a2c1 100644 --- a/crates/media/src/exr_planes/tests.rs +++ b/crates/media/src/exr_planes/tests.rs @@ -1544,6 +1544,25 @@ fn a_failure_is_forgotten_once_the_file_changes() { /// #3271 review: a failed picture request (too large here) doesn't refuse a later request for /// the other channels: "all channels" is every channel but the picture's, so it doesn't ask for /// what failed. +/// Memory the system refuses during a read is a read that failed just now, not a file too large: +/// it isn't remembered, and the next request reads again (a file too large is remembered). +#[test] +fn an_allocation_failure_is_not_remembered() { + use crate::exr_read::{ReadError, read_error}; + assert_eq!(read_error(craft_exr::seek::Error::OutOfMemory("planes".into())), ReadError::Io("planes".into())); + assert!(matches!(read_error(craft_exr::seek::Error::LimitExceeded("planes".into())), ReadError::TooLarge(_))); + // A read that failed just now (here: the file can't be opened) is a stop, and the next + // request for the same channels reads them. + let bytes: Arc<[u8]> = single_part_zips().into(); + let path = "/oom-not-remembered.exr"; + let store = PlaneStore::new(CACHE_BUDGET); + let index = crate::exr_index::get(path, &ExrSource::Bytes(bytes.clone())).unwrap(); + let want = Want { all: false, ids: index.picture_ids().into_iter().collect() }; + let gone = ExrSource::File(std::env::temp_dir().join("ec-oom-not-remembered-missing.exr")); + assert_eq!(store.get(path, &gone, &index, &want, Use::Other).err(), Some(Refused::Stopped)); + assert!(store.get(path, &ExrSource::Bytes(bytes), &index, &want, Use::Other).is_ok(), "read again, not refused from memory"); +} + #[test] fn a_failed_picture_leaves_the_other_channels_readable() { let bytes: Arc<[u8]> = diff --git a/crates/media/src/exr_read.rs b/crates/media/src/exr_read.rs index 611e9e5fb..ca30f6994 100644 --- a/crates/media/src/exr_read.rs +++ b/crates/media/src/exr_read.rs @@ -221,6 +221,19 @@ pub(crate) const MAX_DECODE_BYTES: usize = 8 << 30; #[cfg(not(target_pointer_width = "64"))] pub(crate) const MAX_DECODE_BYTES: usize = 512 << 20; +/// What a failed `craft-exr` read means here. Memory the system refused just now is a read that +/// failed ([`ReadError::Io`]: the plane store doesn't remember it, another read may succeed), +/// not a file too large (remembered until the file changes). +pub(crate) fn read_error(e: seek::Error) -> ReadError { + match e { + seek::Error::Cancelled => ReadError::Cancelled, + seek::Error::Io(why) | seek::Error::OutOfMemory(why) => ReadError::Io(why), + seek::Error::LimitExceeded(why) => ReadError::TooLarge(format!("its OpenEXR channels need more memory than one file may use ({why})")), + seek::Error::Malformed(why) | seek::Error::Unsupported(why) | seek::Error::UnsupportedCompression(why) => ReadError::Unreadable(why), + other => ReadError::Unreadable(other.to_string()), + } +} + /// The most threads one read decompresses on (beside the caller's), and the size of a file /// whose chunks are decompressed on the calling thread alone. const READ_WORKERS: usize = 4; @@ -274,13 +287,7 @@ fn read(src: &mut dyn ReadSeek, index: &ExrIndex, req: &ReadRequest) -> Result<( // pool the caller runs on: a few at most, and none for a small file (its chunks are // decompressed on the calling thread). request.workers = Some(if index.file.file_len() < SMALL_FILE { 0 } else { rayon::current_num_threads().min(READ_WORKERS) }); - let planes = seek::read_planes(src, &index.file, &request, &filmcraft_media::cancel::cancelled).map_err(|e| match e { - seek::Error::Cancelled => ReadError::Cancelled, - seek::Error::Io(why) => ReadError::Io(why), - seek::Error::LimitExceeded(why) => ReadError::TooLarge(format!("its OpenEXR channels need more memory than one file may use ({why})")), - seek::Error::Malformed(why) | seek::Error::Unsupported(why) | seek::Error::UnsupportedCompression(why) => ReadError::Unreadable(why), - other => ReadError::Unreadable(other.to_string()), - })?; + let planes = seek::read_planes(src, &index.file, &request, &filmcraft_media::cancel::cancelled).map_err(read_error)?; let stats = ReadStats { chunks: planes.stats.chunks, bytes: planes.stats.bytes, lines: planes.stats.lines }; Ok((planes.planes, stats)) }