From 45811427c1f519acabbec53b4314ca8f20382502 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sun, 8 Nov 2026 02:56:07 +0100 Subject: [PATCH] Footage cache (#3271): a new OpenEXR frame opens its file once exr_index::get_opened keeps the file open when indexing opened it; the plane store's decode reads from that handle (get_opened) instead of opening the file again. Served requests drop it. --- crates/media/src/exr_index.rs | 12 ++++-- crates/media/src/exr_planes.rs | 59 ++++++++++++++++++---------- crates/media/src/exr_planes/tests.rs | 17 ++++++++ 3 files changed, 65 insertions(+), 23 deletions(-) diff --git a/crates/media/src/exr_index.rs b/crates/media/src/exr_index.rs index 53922dc07..49ba5cb4b 100644 --- a/crates/media/src/exr_index.rs +++ b/crates/media/src/exr_index.rs @@ -22,7 +22,7 @@ use exr::meta::MetaData; use exr::meta::attribute::{AttributeValue, IntegerBounds, Text}; use crate::exr_channels::{MAX_DISPLAY_PIXELS, Rgb, channel_name, is_depth, parse_manifest, part_name, picture_names}; -use crate::exr_read::{ExrSource, ReadSeek}; +use crate::exr_read::{ExrSource, Opened, ReadSeek}; /// One channel of the file, in file order (parts in order, each part's channels as its header /// lists them): the order of [`AuxChannels::channels`] when every channel is read. @@ -417,10 +417,16 @@ pub(crate) fn keep(path: &str, index: Arc) { /// [`forget`]). A file whose identity changed since (size, modification time) is indexed again; /// a read checks it again on the file it opens. pub(crate) fn get(path: &str, source: &ExrSource) -> Result, IndexError> { + get_opened(path, source).map(|(ix, _)| ix) +} + +/// [`get`], with the file left open when indexing it opened it: a decode that follows reads from +/// it instead of opening the file again (one open per new frame, #3271). +pub(crate) fn get_opened(path: &str, source: &ExrSource) -> Result<(Arc, Option), IndexError> { let kept = lock().map.get(path).cloned(); if let Some(ix) = kept { match source.stamp() { - Ok(stamp) if stamp == ix.stamp => return Ok(ix), + Ok(stamp) if stamp == ix.stamp => return Ok((ix, None)), Ok(_) => {} Err(e) => { log::debug!("media: {path}: {e}"); @@ -434,7 +440,7 @@ pub(crate) fn get(path: &str, source: &ExrSource) -> Result, Index })?; let ix = Arc::new(build(&mut *opened.reader, opened.len, opened.stamp)?); keep(path, ix.clone()); - Ok(ix) + Ok((ix, Some(opened))) } /// The index of `path` if it is kept. diff --git a/crates/media/src/exr_planes.rs b/crates/media/src/exr_planes.rs index dff10e48f..0a66ac4ce 100644 --- a/crates/media/src/exr_planes.rs +++ b/crates/media/src/exr_planes.rs @@ -30,7 +30,7 @@ use crate::convert::AlphaOp; use crate::exr_channels::CACHE_BUDGET; use crate::exr_channels::{Rgb, planes_picture}; use crate::exr_index::{self, ExrIndex, IndexError}; -use crate::exr_read::{ExrSource, ReadError, ReadRequest, read_planes}; +use crate::exr_read::{ExrSource, Opened, ReadError, ReadRequest, read_planes}; /// At most this many entries are kept (unreadable files hold no bytes). const CACHE_FILES: usize = 64; @@ -68,7 +68,7 @@ fn asks_at_least(want: &Want, failed: &Want) -> bool { /// Decode `want` for this request alone (kept nowhere), noting the file it read. fn copy(path: &str, source: &ExrSource, index: &Arc, want: &Want) -> Result, Refused> { - let b = run(path, source, &Flight::new(index.clone(), want.clone()), None); + let b = run(path, source, &Flight::new(index.clone(), want.clone()), None, None); if let Ok(b) = &b { crate::reads::note(b.file.clone()); } @@ -337,9 +337,24 @@ impl PlaneStore { /// file it was read from is noted ([`crate::reads`]). A request that keeps finding the /// decodes it waits for called off decodes a copy itself. pub(crate) fn get(&self, path: &str, source: &ExrSource, index: &Arc, want: &Want, by: Use) -> Result, Refused> { + self.get_opened(path, source, index, want, by, None) + } + + /// [`PlaneStore::get`] with the file already open (`opened`, as indexing it opened it): a + /// decode reads from it instead of opening the file again, so a new frame opens its file + /// once (#3271). Dropped unused when the store serves the request. + pub(crate) fn get_opened( + &self, + path: &str, + source: &ExrSource, + index: &Arc, + want: &Want, + by: Use, + mut opened: Option, + ) -> Result, Refused> { let mut retries = 0; for _ in 0..ROUNDS { - match self.step(path, source, index, want, by) { + match self.step(path, source, index, want, by, &mut opened) { Step::Done(r) => return r, Step::Again { retry: true } if retries >= RETRIES => break, Step::Again { retry } => retries += u32::from(retry), @@ -349,7 +364,7 @@ impl PlaneStore { } /// One round of [`PlaneStore::get`]. - fn step(&self, path: &str, source: &ExrSource, index: &Arc, want: &Want, by: Use) -> Step { + fn step(&self, path: &str, source: &ExrSource, index: &Arc, want: &Want, by: Use, opened: &mut Option) -> Step { let level = 0; let mut st = self.lock(); st.clock += 1; @@ -403,7 +418,7 @@ impl PlaneStore { #[cfg(test)] crate::exr_channels::count_decode(true); let mut owning = Owning { store: self, path, level, flight: &flight, ended: false }; - let outcome = run(path, source, &flight, base); + let outcome = run(path, source, &flight, base, opened.take()); match owning.end(outcome) { Ok(b) if b.covers(index, want) => { let mut st = self.lock(); @@ -564,7 +579,7 @@ impl PlaneStore { /// Run `flight`'s decode on the footage decode threads (here when this is one of them, or /// there are none) and block until it is done. -fn run(path: &str, source: &ExrSource, flight: &Arc, base: Option>) -> Result, Refused> { +fn run(path: &str, source: &ExrSource, flight: &Arc, base: Option>, opened: Option) -> Result, Refused> { #[cfg(not(target_arch = "wasm32"))] if let Some(threads) = crate::pool::decode_threads() && threads.current_thread_index().is_none() @@ -574,20 +589,25 @@ fn run(path: &str, source: &ExrSource, flight: &Arc, base: Option, base: Option>) -> Result, Refused> { +fn decode(path: &str, source: &ExrSource, flight: &Arc, base: Option>, opened: Option) -> Result, Refused> { let _decoding = Decoding::new(); - // (a share that doesn't answer just now isn't remembered as unreadable) - let mut opened = source.open().map_err(|e| { + // The file as indexing opened it, else opened now (a share that doesn't answer just now + // isn't remembered as unreadable). + let opened = match opened { + Some(o) => Ok(o), + None => source.open(), + }; + let mut opened = opened.map_err(|e| { log::warn!("media: {path}: {e}"); Refused::Stopped })?; @@ -676,8 +696,8 @@ pub(crate) fn is_exr(path: &str) -> bool { } /// The index of `path`, and the store's verdict on an unreadable one. -fn index_of(path: &str, source: &ExrSource) -> Result, IndexError> { - exr_index::get(path, source) +fn index_of(path: &str, source: &ExrSource) -> Result<(Arc, Option), IndexError> { + exr_index::get_opened(path, source) } /// The ids `need` reads of the file `index` describes (`None`: all). @@ -702,14 +722,13 @@ pub(crate) enum Picture { pub(crate) fn picture(path: &str, source: &ExrSource, op: AlphaOp, extra: Option<&AuxNeed>) -> Picture { let mut extra = extra; for _ in 0..3 { - let index = match index_of(path, source) { + let (index, opened) = match index_of(path, source) { Ok(ix) => ix, Err(IndexError::Unsupported(why)) => return Picture::Done(Err(why.to_string())), Err(_) => return Picture::Legacy, }; let Some(ids) = index.picture_ids() else { return Picture::Legacy }; - let alone = Want { all: false, ids: ids.into_iter().collect() }; - let mut want = alone.clone(); + let mut want = Want { all: false, ids: ids.into_iter().collect() }; if let Some(need) = extra { match need_ids(&index, need) { Some(more) => want.ids.extend(more), @@ -717,7 +736,7 @@ pub(crate) fn picture(path: &str, source: &ExrSource, op: AlphaOp, extra: Option } } let by = if extra.is_some() { Use::Picture } else { Use::Other }; - let batch = match store().get(path, source, &index, &want, by) { + let batch = match store().get_opened(path, source, &index, &want, by, opened) { Ok(b) => b, Err(Refused::Changed) => continue, // The channels with it are too many: the picture alone (the effects ask for theirs). @@ -770,12 +789,12 @@ fn picture_of(index: &ExrIndex, batch: &Batch, op: AlphaOp) -> Option { /// store. `None` when it isn't a readable OpenEXR (or the request was called off). pub(crate) fn aux(path: &str, source: &ExrSource, need: &AuxNeed) -> Option> { for _ in 0..2 { - let index = index_of(path, source).ok()?; + let (index, opened) = index_of(path, source).ok()?; let want = match need_ids(&index, need) { Some(ids) => Want { all: false, ids }, None => Want { all: true, ids: BTreeSet::new() }, }; - match store().get(path, source, &index, &want, Use::Other) { + match store().get_opened(path, source, &index, &want, Use::Other, opened) { Ok(b) => return Some(b.aux.clone()), Err(Refused::Changed) => continue, Err(_) => return None, @@ -787,7 +806,7 @@ pub(crate) fn aux(path: &str, source: &ExrSource, need: &AuxNeed) -> Option Option> { - index_of(path, source).ok().map(|ix| ix.names.clone()) + exr_index::get(path, source).ok().map(|ix| ix.names.clone()) } /// Drop what is kept of `path`: its index and planes. diff --git a/crates/media/src/exr_planes/tests.rs b/crates/media/src/exr_planes/tests.rs index f28cfb90d..90f33ea39 100644 --- a/crates/media/src/exr_planes/tests.rs +++ b/crates/media/src/exr_planes/tests.rs @@ -804,3 +804,20 @@ fn rows_and_levels_on_every_compression() { assert_eq!(stats.chunks, [4], "two rows of two tiles"); assert_eq!(band.unwrap()[0][10 * 32..20 * 32], full[0][10 * 32..20 * 32]); } + +/// #3271: a new frame of an OpenEXR on disk opens its file once: indexing it and decoding the +/// picture share the open file. +#[test] +fn a_new_frame_opens_its_file_once() { + let dir = std::env::temp_dir().join(format!("ec-3271-open-once-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let file = dir.join("plate_0001.exr"); + std::fs::write(&file, multipart()).unwrap(); + let f = crate::probe(&file).unwrap(); + let before = crate::opens_under(&dir); + let pool = crate::MediaPool::new(); + pool.frame_at(&f, Tick::ZERO).unwrap(); + assert_eq!(crate::opens_under(&dir) - before, 1, "one open for the index and the decode"); + assert_eq!(decodes_of(&f.path).len(), 1); + let _ = std::fs::remove_dir_all(&dir); +}