mirror of
https://github.com/storytold/effectcraft.git
synced 2026-11-19 16:45:28 +00:00
OpenEXR: a file's channels may take what the machine's RAM allows, not a fixed 8 GiB (#3749)
The limit of one read is decided here (the 8 GiB floor, or the memory budget when that is more) and handed to craft-exr as the read's max_bytes, so the planes it reserves and the largest block it decompresses follow it. craft-exr reserves the planes fallibly, so the reader's own fallible plane allocation is not needed: memory the system can't give is a read that failed just now (read_error), a plane over the limit a refusal for size.
This commit is contained in:
1 parent
ea4ac2e5ed
commit
b591f95ce4
7 files changed
+153
-9
No files matched your search
@@ -122,6 +122,11 @@ impl Session {
|
||||
let layers: std::sync::Arc<dyn effectcraft_render::memory::MemoryHolder> = self.layer_cache.clone();
|
||||
accountant.register(&layers);
|
||||
accountant.set_budget(b.bytes);
|
||||
// (what one file's decoded planes may take follows it once the RAM is read, at least the
|
||||
// fixed limit; until then the host's reader asks when a request is over that, #3749)
|
||||
if b.installed.is_some() {
|
||||
effectcraft_render::memory::set_decode_limit(Some(b.bytes));
|
||||
}
|
||||
// The process's measured memory as well (#3301: memory nobody accounts for, such as GPU
|
||||
// staging, effect buffers and the allocator's fragmentation): over the purge limit the
|
||||
// caches give way by what it is over, over the hard limit also the frames a playing
|
||||
|
||||
@@ -117,9 +117,19 @@ pub fn session() -> Session {
|
||||
// the settings, and the caches have no limits of their own below it. Without a memory
|
||||
// reading yet, the budget for unknown RAM.
|
||||
s.apply_cache_budgets();
|
||||
// The CLI and the MCP server read no memory: a request for a file's planes over the fixed
|
||||
// limit asks for it once (#3749).
|
||||
effectcraft_engine::render::memory::set_decode_limit_source(Some(installed_memory_budget));
|
||||
s
|
||||
}
|
||||
|
||||
/// The memory budget of the installed RAM with the default settings: what one file's decoded
|
||||
/// planes may take when a request is over the fixed limit and the session read no memory.
|
||||
fn installed_memory_budget() -> Option<usize> {
|
||||
let memory = effectcraft_engine::sysinfo::memory()?;
|
||||
Some(effectcraft_engine::prefs::Prefs::default().memory_budget(Some(memory), false).bytes)
|
||||
}
|
||||
|
||||
/// External effect helpers run as child processes: native builds only.
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
fn helper_hooks() -> Option<effectcraft_engine::HelperHooks> {
|
||||
|
||||
@@ -150,7 +150,8 @@ const MAX_DATA_PIXELS: usize = 4 * MAX_DISPLAY_PIXELS;
|
||||
/// blocks of at most `craft-exr`'s default ratio of bytes per byte of their chunk (four times
|
||||
/// that for DWA, plus 1 MiB): a hostile header declaring a block of gigabytes over a chunk of a
|
||||
/// few bytes is refused, by every read and by [`safe_to_decode`]. A read's planes are bounded by
|
||||
/// its own cap ([`crate::exr_read::MAX_DECODE_BYTES`]).
|
||||
/// its own cap ([`crate::exr_read::MAX_DECODE_BYTES`], or more with more memory:
|
||||
/// [`effectcraft_render::memory::decode_limit`]).
|
||||
fn limits() -> Limits {
|
||||
let mut limits = Limits::none();
|
||||
limits.max_pixels = MAX_DATA_PIXELS as u64;
|
||||
|
||||
@@ -434,14 +434,15 @@ pub(crate) fn set_cap(path: &str, bytes: usize) {
|
||||
CAPS.lock().unwrap_or_else(PoisonError::into_inner).push((path.to_string(), bytes));
|
||||
}
|
||||
|
||||
/// The most bytes of planes one read of `path` may allocate.
|
||||
/// The most bytes of planes one read of `path` may allocate by the caller's own limit (none: the
|
||||
/// system's, [`effectcraft_render::memory::decode_limit`]).
|
||||
fn cap_of(path: &str) -> usize {
|
||||
#[cfg(test)]
|
||||
if let Some((_, c)) = CAPS.lock().unwrap_or_else(PoisonError::into_inner).iter().find(|(p, _)| p == path) {
|
||||
return *c;
|
||||
}
|
||||
let _ = path;
|
||||
crate::exr_read::MAX_DECODE_BYTES
|
||||
usize::MAX
|
||||
}
|
||||
|
||||
impl PlaneStore {
|
||||
|
||||
@@ -1580,6 +1580,41 @@ fn a_failed_picture_leaves_the_other_channels_readable() {
|
||||
assert!(rest.aux.get("Z").is_some());
|
||||
}
|
||||
|
||||
/// #3749: a read over the fixed floor is allowed up to what the system's memory gives (the
|
||||
/// session's figure), never under the floor, and refused with a reason beyond it. The floor is
|
||||
/// small here: the real one is 8 GiB.
|
||||
#[test]
|
||||
fn a_read_over_the_floor_is_allowed_up_to_the_systems_memory() {
|
||||
let bytes = write(vec![part(None, &RGBA, (8, 8), Compression::ZIP1)], (8, 8));
|
||||
let mut src = std::io::Cursor::new(bytes.as_slice());
|
||||
let index = crate::exr_index::build(&mut src, FileStamp::Virtual).unwrap();
|
||||
// Four channels of 8×8 f32 are 1024 bytes; the floor is 512.
|
||||
let request = ReadRequest { floor: 512, ..ReadRequest::full(vec![0, 1, 2, 3]) };
|
||||
let read = |src: &mut std::io::Cursor<&[u8]>| crate::exr_read::read_planes(src, &index, &request).0;
|
||||
effectcraft_render::memory::set_decode_limit(None);
|
||||
effectcraft_render::memory::set_decode_limit_source(None);
|
||||
assert!(matches!(read(&mut src), Err(crate::exr_read::ReadError::TooLarge(_))), "over the floor, memory unknown");
|
||||
effectcraft_render::memory::set_decode_limit(Some(2048));
|
||||
assert_eq!(read(&mut src).map(|p| p.len()).ok(), Some(4), "the system's memory allows it");
|
||||
effectcraft_render::memory::set_decode_limit(Some(800));
|
||||
assert!(matches!(read(&mut src), Err(crate::exr_read::ReadError::TooLarge(_))), "still more than the system's memory");
|
||||
effectcraft_render::memory::set_decode_limit(None);
|
||||
}
|
||||
|
||||
/// #3749: a plane the system cannot give is not an aborted process: `craft-exr` asks for the
|
||||
/// planes' memory first and reports what it can't get, a read that failed just now (asked again,
|
||||
/// not remembered); a plane over the read's limit (the decode limit handed to it) is a refusal
|
||||
/// for size with `craft-exr`'s reason.
|
||||
#[test]
|
||||
fn a_plane_the_system_cannot_give_is_refused_not_allocated() {
|
||||
use crate::exr_read::{ReadError, read_error};
|
||||
use craft_exr::seek::Error;
|
||||
let gone = read_error(Error::OutOfMemory("4096 bytes of planes".into()));
|
||||
assert!(matches!(&gone, ReadError::Io(why) if why.contains("4096 bytes of planes")), "{gone:?}");
|
||||
let over = read_error(Error::LimitExceeded("the requested rows need 4096 bytes".into()));
|
||||
assert!(matches!(&over, ReadError::TooLarge(why) if why.contains("4096 bytes")), "{over:?}");
|
||||
}
|
||||
|
||||
/// A two-rank Cryptomatte file of `size` with a picture: "cube" and "sphere" meet along a
|
||||
/// diagonal whose pixels hold both IDs (rank 00 the nearer, 55-75 %, rank 01 the other one with
|
||||
/// the rest), IDs and coverage stored as 32-bit floats.
|
||||
|
||||
@@ -178,14 +178,17 @@ pub(crate) struct ReadRequest {
|
||||
pub(crate) rows: Option<Range<i64>>,
|
||||
/// Every 2^level-th row and column of the display window (0: full resolution).
|
||||
pub(crate) level: u32,
|
||||
/// The most bytes of planes the read may allocate.
|
||||
/// The most bytes of planes the read may allocate (the caller's own limit; `usize::MAX`: only
|
||||
/// the system's, [`effectcraft_render::memory::decode_limit`]).
|
||||
pub(crate) max_bytes: usize,
|
||||
/// The fixed limit the system's memory may raise ([`MAX_DECODE_BYTES`]).
|
||||
pub(crate) floor: usize,
|
||||
}
|
||||
|
||||
impl ReadRequest {
|
||||
/// Every row of `channels` at full resolution.
|
||||
pub(crate) fn full(channels: Vec<usize>) -> ReadRequest {
|
||||
ReadRequest { channels, rows: None, level: 0, max_bytes: MAX_DECODE_BYTES }
|
||||
ReadRequest { channels, rows: None, level: 0, max_bytes: usize::MAX, floor: MAX_DECODE_BYTES }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -214,8 +217,10 @@ pub(crate) enum ReadError {
|
||||
Cancelled,
|
||||
}
|
||||
|
||||
/// The most bytes of planes one read allocates (every requested channel is a display-window
|
||||
/// plane of f32): a header can ask for far more than the file holds.
|
||||
/// The most bytes of planes one read allocates at least (every requested channel is a
|
||||
/// display-window plane of f32): a header can ask for far more than the file holds, and a file's
|
||||
/// size on disk says nothing about it. With more memory in the machine the limit is higher
|
||||
/// ([`effectcraft_render::memory::decode_limit`], #3749).
|
||||
#[cfg(target_pointer_width = "64")]
|
||||
pub(crate) const MAX_DECODE_BYTES: usize = 8 << 30;
|
||||
#[cfg(not(target_pointer_width = "64"))]
|
||||
@@ -256,14 +261,19 @@ pub(crate) fn read_planes(src: &mut dyn ReadSeek, index: &ExrIndex, req: &ReadRe
|
||||
}
|
||||
}
|
||||
|
||||
fn gb(bytes: usize) -> f64 {
|
||||
bytes as f64 / f64::from(1u32 << 30)
|
||||
}
|
||||
|
||||
fn read(src: &mut dyn ReadSeek, index: &ExrIndex, req: &ReadRequest) -> Result<(Vec<Vec<f32>>, ReadStats), ReadError> {
|
||||
let (w, h) = index.size();
|
||||
let (ow, oh) = seek::level_size((w, h), req.level.min(16));
|
||||
let n = ow.checked_mul(oh).ok_or_else(|| ReadError::Unreadable("display window".into()))?;
|
||||
let need = n.saturating_mul(req.channels.len()).saturating_mul(std::mem::size_of::<f32>());
|
||||
let most = req.max_bytes.min(MAX_DECODE_BYTES);
|
||||
// (the limit is decided here and handed to `craft-exr` as the read's `max_bytes`: the planes
|
||||
// it reserves, fallibly, and the largest block it decompresses follow it)
|
||||
let most = req.max_bytes.min(effectcraft_render::memory::decode_limit(req.floor, need));
|
||||
if need > most {
|
||||
let gb = |b: usize| b as f64 / f64::from(1u32 << 30);
|
||||
return Err(ReadError::TooLarge(format!(
|
||||
"its {}×{} OpenEXR channels need {:.1} GB of memory, more than the {:.1} GB one file may use",
|
||||
w,
|
||||
|
||||
@@ -419,6 +419,44 @@ impl Accountant {
|
||||
}
|
||||
}
|
||||
|
||||
/// What the system's memory allows one request for a file's decoded planes (an OpenEXR's
|
||||
/// channels) to take; 0: not known yet ([`decode_limit`]).
|
||||
static DECODE_LIMIT: AtomicUsize = AtomicUsize::new(0);
|
||||
|
||||
/// Reads that limit from the system when nothing set it ([`set_decode_limit_source`]).
|
||||
static DECODE_SOURCE: Mutex<Option<fn() -> Option<usize>>> = Mutex::new(None);
|
||||
|
||||
/// Set what the system's memory allows one request for a file's decoded planes to take
|
||||
/// (`None`: unknown again). The session sets it with the memory budget once the installed RAM
|
||||
/// is read.
|
||||
pub fn set_decode_limit(bytes: Option<usize>) {
|
||||
DECODE_LIMIT.store(bytes.unwrap_or(0), Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Where [`decode_limit`] reads the limit from while nothing set it: the CLI and the MCP server
|
||||
/// read no memory otherwise. The reading may take a second, so it is only asked for when a
|
||||
/// request is over the fixed floor.
|
||||
pub fn set_decode_limit_source(source: Option<fn() -> Option<usize>>) {
|
||||
*DECODE_SOURCE.lock().unwrap_or_else(PoisonError::into_inner) = source;
|
||||
}
|
||||
|
||||
/// The most bytes one request of `need` bytes for a file's decoded planes may take: `floor`
|
||||
/// (the fixed limit), or what the system's memory allows when that is more. The system is
|
||||
/// asked only when `need` is over `floor` (and once: the answer is kept, also "unknown", which
|
||||
/// leaves `floor`). The browser keeps `floor`.
|
||||
pub fn decode_limit(floor: usize, need: usize) -> usize {
|
||||
if need <= floor || cfg!(target_arch = "wasm32") {
|
||||
return floor;
|
||||
}
|
||||
let mut known = DECODE_LIMIT.load(Ordering::Relaxed);
|
||||
if known == 0 {
|
||||
let source = *DECODE_SOURCE.lock().unwrap_or_else(PoisonError::into_inner);
|
||||
known = source.and_then(|read| read()).unwrap_or(floor);
|
||||
DECODE_LIMIT.store(known, Ordering::Relaxed);
|
||||
}
|
||||
floor.max(known)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -831,4 +869,48 @@ mod tests {
|
||||
acc.set_process_memory(process_now, 1_000, 2_000);
|
||||
assert!(preview.held() <= 200, "{}", preview.held());
|
||||
}
|
||||
|
||||
static ASKED: AtomicUsize = AtomicUsize::new(0);
|
||||
|
||||
fn ram_says() -> Option<usize> {
|
||||
ASKED.fetch_add(1, Ordering::SeqCst);
|
||||
Some(300)
|
||||
}
|
||||
|
||||
fn ram_unknown() -> Option<usize> {
|
||||
ASKED.fetch_add(1, Ordering::SeqCst);
|
||||
None
|
||||
}
|
||||
|
||||
/// #3749: a request for a file's planes may take the fixed floor, or the system's memory
|
||||
/// when that is more: the system is asked only for a request over the floor, once.
|
||||
#[test]
|
||||
fn a_request_over_the_floor_is_limited_by_the_systems_memory() {
|
||||
let _serial = PROCESS_SERIAL.lock().unwrap_or_else(PoisonError::into_inner);
|
||||
set_decode_limit(None);
|
||||
set_decode_limit_source(Some(ram_says));
|
||||
ASKED.store(0, Ordering::SeqCst);
|
||||
// Under the floor: nobody is asked.
|
||||
assert_eq!(decode_limit(100, 50), 100);
|
||||
assert_eq!(ASKED.load(Ordering::SeqCst), 0, "a request under the floor reads no memory");
|
||||
// Over it: the system's 300, asked once.
|
||||
assert_eq!(decode_limit(100, 200), 300);
|
||||
assert_eq!(decode_limit(100, 250), 300);
|
||||
assert_eq!(ASKED.load(Ordering::SeqCst), 1, "asked once, then kept");
|
||||
// A smaller figure than the floor never lowers it.
|
||||
set_decode_limit(Some(40));
|
||||
assert_eq!(decode_limit(100, 200), 100);
|
||||
// The session's figure replaces the asked one.
|
||||
set_decode_limit(Some(900));
|
||||
assert_eq!(decode_limit(100, 200), 900);
|
||||
// Unknown memory leaves the floor, and is not asked again.
|
||||
set_decode_limit(None);
|
||||
set_decode_limit_source(Some(ram_unknown));
|
||||
ASKED.store(0, Ordering::SeqCst);
|
||||
assert_eq!(decode_limit(100, 200), 100);
|
||||
assert_eq!(decode_limit(100, 200), 100);
|
||||
assert_eq!(ASKED.load(Ordering::SeqCst), 1, "an unknown reading is kept too");
|
||||
set_decode_limit(None);
|
||||
set_decode_limit_source(None);
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user