mirror of
https://github.com/storytold/effectcraft.git
synced 2026-11-19 16:45:28 +00:00
Refuse OpenEXR blocks larger than their chunk can hold
craft-exr now holds every block to what its chunk's bytes can decompress to, using the ratio the index was built with: the index is built with craft-exr's default ratio (no longer unbounded), a file with such a block is refused when indexed and by safe_to_decode before the image crate sees it, and each read checks it again. Memory the system refuses during a read is a read that failed just now, not remembered, instead of a file too large.
This commit is contained in:
1 parent
c1aea6d65a
commit
ea4ac2e5ed
3 files changed
+59
-18
No files matched your search
@@ -10,9 +10,11 @@
|
||||
//! it and never decode pixels. The reader ([`crate::exr_read`]) fetches the chunks it lists.
|
||||
//!
|
||||
//! Everything is validated before anything is sized by it: the display window
|
||||
//! ([`crate::exr_channels::MAX_DISPLAY_PIXELS`]), the data windows (four times that), the blocks,
|
||||
//! the chunk count (eight bytes of offset table each, so never more than the file can hold), and
|
||||
//! every offset (inside the chunk area; a chunk with a bad offset is never read).
|
||||
//! ([`crate::exr_channels::MAX_DISPLAY_PIXELS`]), the data windows (four times that), the chunk
|
||||
//! count (eight bytes of offset table each, so never more than the file can hold), and every
|
||||
//! offset (inside the chunk area; a chunk with a bad offset is never read). Every block is held
|
||||
//! to what its chunk's bytes can decompress to ([`limits`]) when the file is indexed and before
|
||||
//! it is handed to another decoder ([`safe_to_decode`]); each read checks it again.
|
||||
|
||||
use std::collections::{HashMap, VecDeque};
|
||||
use std::io::SeekFrom;
|
||||
@@ -144,11 +146,15 @@ pub(crate) const HTJ2K: &str = "OpenEXR HTJ2K compression isn't supported yet: r
|
||||
const MAX_DATA_PIXELS: usize = 4 * MAX_DISPLAY_PIXELS;
|
||||
|
||||
/// What `craft-exr` may index: data windows of at most [`MAX_DATA_PIXELS`], of any width or
|
||||
/// height (the display window is checked against [`MAX_DISPLAY_PIXELS`] in [`build`]).
|
||||
/// `craft-exr` bounds every block itself (256 MiB uncompressed).
|
||||
/// height (the display window is checked against [`MAX_DISPLAY_PIXELS`] in [`build`]), and
|
||||
/// blocks of at most `craft-exr`'s default ratio of bytes per byte of their chunk (four times
|
||||
/// that for DWA, plus 1 MiB): a hostile header declaring a block of gigabytes over a chunk of a
|
||||
/// few bytes is refused, by every read and by [`safe_to_decode`]. A read's planes are bounded by
|
||||
/// its own cap ([`crate::exr_read::MAX_DECODE_BYTES`]).
|
||||
fn limits() -> Limits {
|
||||
let mut limits = Limits::none();
|
||||
limits.max_pixels = MAX_DATA_PIXELS as u64;
|
||||
limits.max_ratio = Limits::default().max_ratio;
|
||||
limits
|
||||
}
|
||||
|
||||
@@ -159,12 +165,17 @@ fn index_file(src: &mut dyn ReadSeek) -> Result<seek::Index, seek::Error> {
|
||||
seek::Index::build(src, &limits(), &|| false)
|
||||
}
|
||||
|
||||
/// The headers of the OpenEXR `bytes` are safe to hand to a decoder: they can be read and no
|
||||
/// part's data window or block is larger than this reader takes (#3271 review). A file this
|
||||
/// reader doesn't cover (deep parts, HTJ2K) is left to the decoder; damaged headers, a part too
|
||||
/// large and a failed read are refused.
|
||||
/// The headers of the OpenEXR `bytes` are safe to hand to a decoder (the `image` crate, which
|
||||
/// allocates every block by its header): they can be read, no part's data window is larger than
|
||||
/// this reader takes, and no block is larger than its chunk can decompress to ([`limits`])
|
||||
/// (#3271 review). A file this reader doesn't cover (deep parts, HTJ2K) is left to the decoder;
|
||||
/// damaged headers, a part or a block too large, and a failed read are refused.
|
||||
pub(crate) fn safe_to_decode(bytes: &[u8]) -> bool {
|
||||
matches!(index_file(&mut std::io::Cursor::new(bytes)), Ok(_) | Err(seek::Error::Unsupported(_) | seek::Error::UnsupportedCompression(_)))
|
||||
match index_file(&mut std::io::Cursor::new(bytes)) {
|
||||
Ok(index) => index.check_blocks().is_ok(),
|
||||
Err(seek::Error::Unsupported(_) | seek::Error::UnsupportedCompression(_)) => true,
|
||||
Err(_) => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// The channel list of `part` as the `exr` crate's validation took it (#3271): sorted by name,
|
||||
@@ -183,9 +194,13 @@ fn channels_valid(part: &PartInfo) -> bool {
|
||||
pub(crate) fn build(src: &mut dyn ReadSeek, stamp: FileStamp) -> Result<ExrIndex, IndexError> {
|
||||
let file = index_file(src).map_err(|e| match e {
|
||||
seek::Error::UnsupportedCompression(name) if name.contains("HTJ2K") => IndexError::Unsupported(HTJ2K),
|
||||
seek::Error::Io(_) => IndexError::Io,
|
||||
// (memory refused just now: another attempt may succeed, nothing is remembered)
|
||||
seek::Error::Io(_) | seek::Error::OutOfMemory(_) => IndexError::Io,
|
||||
_ => IndexError::Unreadable,
|
||||
})?;
|
||||
// A block larger than its chunk can decompress to (a hostile header): the file is refused
|
||||
// as a whole, as every read of it would refuse that chunk.
|
||||
file.check_blocks().map_err(|_| IndexError::Unreadable)?;
|
||||
let parts = file.parts();
|
||||
// Deep data isn't read (the whole file was refused before).
|
||||
if parts.iter().any(|p| p.deep || !channels_valid(p)) {
|
||||
|
||||
@@ -1544,6 +1544,25 @@ fn a_failure_is_forgotten_once_the_file_changes() {
|
||||
/// #3271 review: a failed picture request (too large here) doesn't refuse a later request for
|
||||
/// the other channels: "all channels" is every channel but the picture's, so it doesn't ask for
|
||||
/// what failed.
|
||||
/// Memory the system refuses during a read is a read that failed just now, not a file too large:
|
||||
/// it isn't remembered, and the next request reads again (a file too large is remembered).
|
||||
#[test]
|
||||
fn an_allocation_failure_is_not_remembered() {
|
||||
use crate::exr_read::{ReadError, read_error};
|
||||
assert_eq!(read_error(craft_exr::seek::Error::OutOfMemory("planes".into())), ReadError::Io("planes".into()));
|
||||
assert!(matches!(read_error(craft_exr::seek::Error::LimitExceeded("planes".into())), ReadError::TooLarge(_)));
|
||||
// A read that failed just now (here: the file can't be opened) is a stop, and the next
|
||||
// request for the same channels reads them.
|
||||
let bytes: Arc<[u8]> = single_part_zips().into();
|
||||
let path = "/oom-not-remembered.exr";
|
||||
let store = PlaneStore::new(CACHE_BUDGET);
|
||||
let index = crate::exr_index::get(path, &ExrSource::Bytes(bytes.clone())).unwrap();
|
||||
let want = Want { all: false, ids: index.picture_ids().into_iter().collect() };
|
||||
let gone = ExrSource::File(std::env::temp_dir().join("ec-oom-not-remembered-missing.exr"));
|
||||
assert_eq!(store.get(path, &gone, &index, &want, Use::Other).err(), Some(Refused::Stopped));
|
||||
assert!(store.get(path, &ExrSource::Bytes(bytes), &index, &want, Use::Other).is_ok(), "read again, not refused from memory");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_failed_picture_leaves_the_other_channels_readable() {
|
||||
let bytes: Arc<[u8]> =
|
||||
|
||||
@@ -221,6 +221,19 @@ pub(crate) const MAX_DECODE_BYTES: usize = 8 << 30;
|
||||
#[cfg(not(target_pointer_width = "64"))]
|
||||
pub(crate) const MAX_DECODE_BYTES: usize = 512 << 20;
|
||||
|
||||
/// What a failed `craft-exr` read means here. Memory the system refused just now is a read that
|
||||
/// failed ([`ReadError::Io`]: the plane store doesn't remember it, another read may succeed),
|
||||
/// not a file too large (remembered until the file changes).
|
||||
pub(crate) fn read_error(e: seek::Error) -> ReadError {
|
||||
match e {
|
||||
seek::Error::Cancelled => ReadError::Cancelled,
|
||||
seek::Error::Io(why) | seek::Error::OutOfMemory(why) => ReadError::Io(why),
|
||||
seek::Error::LimitExceeded(why) => ReadError::TooLarge(format!("its OpenEXR channels need more memory than one file may use ({why})")),
|
||||
seek::Error::Malformed(why) | seek::Error::Unsupported(why) | seek::Error::UnsupportedCompression(why) => ReadError::Unreadable(why),
|
||||
other => ReadError::Unreadable(other.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
/// The most threads one read decompresses on (beside the caller's), and the size of a file
|
||||
/// whose chunks are decompressed on the calling thread alone.
|
||||
const READ_WORKERS: usize = 4;
|
||||
@@ -274,13 +287,7 @@ fn read(src: &mut dyn ReadSeek, index: &ExrIndex, req: &ReadRequest) -> Result<(
|
||||
// pool the caller runs on: a few at most, and none for a small file (its chunks are
|
||||
// decompressed on the calling thread).
|
||||
request.workers = Some(if index.file.file_len() < SMALL_FILE { 0 } else { rayon::current_num_threads().min(READ_WORKERS) });
|
||||
let planes = seek::read_planes(src, &index.file, &request, &filmcraft_media::cancel::cancelled).map_err(|e| match e {
|
||||
seek::Error::Cancelled => ReadError::Cancelled,
|
||||
seek::Error::Io(why) => ReadError::Io(why),
|
||||
seek::Error::LimitExceeded(why) => ReadError::TooLarge(format!("its OpenEXR channels need more memory than one file may use ({why})")),
|
||||
seek::Error::Malformed(why) | seek::Error::Unsupported(why) | seek::Error::UnsupportedCompression(why) => ReadError::Unreadable(why),
|
||||
other => ReadError::Unreadable(other.to_string()),
|
||||
})?;
|
||||
let planes = seek::read_planes(src, &index.file, &request, &filmcraft_media::cancel::cancelled).map_err(read_error)?;
|
||||
let stats = ReadStats { chunks: planes.stats.chunks, bytes: planes.stats.bytes, lines: planes.stats.lines };
|
||||
Ok((planes.planes, stats))
|
||||
}
|
||||
Reference in new issue
Block a user