Footage cache (#3271): a stopped decode isn't remembered; port the #474 store tests

A decode that panicked, or a file that couldn't be opened just then, ends Stopped: not kept as
unreadable, the next request decodes again. Decode threads never wait for planes. Headers and
offset tables are read through one buffer. The #474 cache tests are ported to the plane store.
This commit is contained in:
vangel76 committed 2026-11-08 02:02:37 +01:00
1 parent 804067b9da
commit e7271f367d
3 files changed
+205 -47

No files matched your search

+43 -32
View File
@@ -116,7 +116,7 @@ pub(crate) enum IndexError {
/// Bytes counted as they are read (the offset tables start where the headers end).
struct Counted<'r> {
inner: &'r mut dyn ReadSeek,
inner: &'r mut dyn Read,
read: u64,
}
@@ -129,19 +129,30 @@ impl Read for Counted<'_> {
}
/// The headers of the file in `src` as the high-level reader takes them (headers read strictly,
/// then validated leniently: the files that opened before open now).
fn headers(src: &mut dyn ReadSeek) -> Result<(MetaData, u64), IndexError> {
let (strict, read) = {
let mut counted = Counted { inner: &mut *src, read: 0 };
let strict = MetaData::read_from_buffered(&mut counted, true).and_then(|meta| MetaData::validate(&meta.headers, false).map(|_| meta));
(strict, counted.read)
/// then validated leniently: the files that opened before open now), checked before anything is
/// sized by them, and the offset tables: the headers, the tables and where the chunks start.
/// Read through one buffer (attribute names are read a byte at a time); the bytes taken from it
/// are counted, so the chunk area's start is exact.
fn headers_and_tables(src: &mut dyn ReadSeek, len: u64) -> Result<(MetaData, exr::meta::OffsetTables, u64), IndexError> {
let strict = {
let mut buffered = std::io::BufReader::with_capacity(64 << 10, &mut *src);
let mut counted = Counted { inner: &mut buffered, read: 0 };
match MetaData::read_from_buffered(&mut counted, true).and_then(|meta| MetaData::validate(&meta.headers, false).map(|_| meta)) {
Ok(meta) => {
let header_end = counted.read;
check(&meta, len, header_end)?;
let tables = MetaData::read_offset_tables(&mut exr::io::PeekRead::new(&mut counted), &meta.headers).map_err(|_| IndexError::Unreadable)?;
Some((meta, tables, counted.read))
}
Err(_) => None,
}
};
match strict {
Ok(meta) => Ok((meta, read)),
Err(_) => {
Some(parsed) => Ok(parsed),
None => {
// (an HTJ2K file fails here: say why)
src.seek(SeekFrom::Start(0)).map_err(|_| IndexError::Io)?;
match MetaData::read_from_buffered(&mut *src, false).ok().as_ref().and_then(unsupported) {
match MetaData::read_from_buffered(std::io::BufReader::new(&mut *src), false).ok().as_ref().and_then(unsupported) {
Some(why) => Err(IndexError::Unsupported(why)),
None => Err(IndexError::Unreadable),
}
@@ -149,21 +160,9 @@ fn headers(src: &mut dyn ReadSeek) -> Result<(MetaData, u64), IndexError> {
}
}
/// [`crate::exr_channels::unsupported_compression`] of parsed headers.
pub(crate) fn unsupported(meta: &MetaData) -> Option<&'static str> {
use exr::compression::Compression;
meta.headers
.iter()
.any(|h| matches!(h.compression, Compression::HTJ2K32 | Compression::HTJ2K256))
.then_some("OpenEXR HTJ2K compression isn't supported yet: re-save the file with ZIP, PIZ or DWAA compression")
}
/// Index the OpenEXR file in `src` (`len` bytes, identity `stamp`), reading its headers and
/// offset tables only.
pub(crate) fn build(src: &mut dyn ReadSeek, len: u64, stamp: FileStamp) -> Result<ExrIndex, IndexError> {
src.seek(SeekFrom::Start(0)).map_err(|_| IndexError::Io)?;
let (meta, header_end) = headers(&mut *src)?;
if let Some(why) = unsupported(&meta) {
/// Refuse headers this reader doesn't take or that ask for more than the file can hold.
fn check(meta: &MetaData, len: u64, header_end: u64) -> Result<(), IndexError> {
if let Some(why) = unsupported(meta) {
return Err(IndexError::Unsupported(why));
}
let first = meta.headers.first().ok_or(IndexError::Unreadable)?;
@@ -180,13 +179,25 @@ pub(crate) fn build(src: &mut dyn ReadSeek, len: u64, stamp: FileStamp) -> Resul
if (chunks as u64).saturating_mul(8) > len.saturating_sub(header_end) {
return Err(IndexError::Unreadable);
}
let tables = {
let mut counted = Counted { inner: &mut *src, read: 0 };
let tables = MetaData::read_offset_tables(&mut exr::io::PeekRead::new(&mut counted), &meta.headers).map_err(|_| IndexError::Unreadable)?;
(tables, counted.read)
};
let (tables, table_bytes) = tables;
let chunks_start = header_end.saturating_add(table_bytes);
Ok(())
}
/// [`crate::exr_channels::unsupported_compression`] of parsed headers.
pub(crate) fn unsupported(meta: &MetaData) -> Option<&'static str> {
use exr::compression::Compression;
meta.headers
.iter()
.any(|h| matches!(h.compression, Compression::HTJ2K32 | Compression::HTJ2K256))
.then_some("OpenEXR HTJ2K compression isn't supported yet: re-save the file with ZIP, PIZ or DWAA compression")
}
/// Index the OpenEXR file in `src` (`len` bytes, identity `stamp`), reading its headers and
/// offset tables only.
pub(crate) fn build(src: &mut dyn ReadSeek, len: u64, stamp: FileStamp) -> Result<ExrIndex, IndexError> {
src.seek(SeekFrom::Start(0)).map_err(|_| IndexError::Io)?;
let (meta, tables, chunks_start) = headers_and_tables(&mut *src, len)?;
let first = meta.headers.first().ok_or(IndexError::Unreadable)?;
let display = first.shared_attributes.display_window;
// Where each chunk ends: at the next chunk's start, or at the end of the file.
let mut starts: Vec<u64> = tables.iter().flatten().copied().filter(|o| *o >= chunks_start && *o < len).collect();
starts.sort_unstable();
+18 -8
View File
@@ -115,6 +115,9 @@ pub(crate) enum Refused {
Changed,
/// The file was forgotten while it decoded: ask again.
Forgotten,
/// The decode stopped (it panicked, or the file couldn't be opened just then): not
/// remembered, the next request decodes again.
Stopped,
}
/// A decode in progress: what it reads (until it starts reading) and its outcome.
@@ -194,7 +197,7 @@ impl Owning<'_> {
impl Drop for Owning<'_> {
fn drop(&mut self) {
if !self.ended {
let _ = self.end(Err(Refused::Unreadable));
let _ = self.end(Err(Refused::Stopped));
}
}
}
@@ -316,7 +319,7 @@ impl PlaneStore {
}
// Not all it asks for: ask again (the next decode reads what is missing).
Waited::Got(Ok(_)) => {}
Waited::Got(Err(Refused::Cancelled | Refused::Forgotten)) if retries < RETRIES => retries += 1,
Waited::Got(Err(Refused::Cancelled | Refused::Forgotten | Refused::Stopped)) if retries < RETRIES => retries += 1,
Waited::Got(Err(e)) => return Err(e),
Waited::Cancelled => return Err(Refused::Cancelled),
// A copy, kept nowhere.
@@ -342,7 +345,7 @@ impl PlaneStore {
crate::reads::note(b.file.clone());
return Ok(b);
}
Ok(_) | Err(Refused::Forgotten) if retries < RETRIES => retries += 1,
Ok(_) | Err(Refused::Forgotten | Refused::Stopped) if retries < RETRIES => retries += 1,
Ok(_) | Err(Refused::Forgotten) => return Err(Refused::Cancelled),
Err(e) => return Err(e),
}
@@ -373,7 +376,7 @@ impl PlaneStore {
let clock = st.clock;
let Some(e) = st.entries.iter_mut().find(|e| e.path == path && e.level == level && e.flight.as_ref().is_some_and(|f| Arc::ptr_eq(f, flight))) else {
return match outcome {
Err(Refused::Cancelled) => Err(Refused::Cancelled),
Err(e @ (Refused::Cancelled | Refused::Stopped)) => Err(e),
_ => Err(Refused::Forgotten),
};
};
@@ -391,7 +394,7 @@ impl PlaneStore {
e.batch = None;
e.bytes = 0;
}
Err(Refused::Cancelled | Refused::Forgotten) => {}
Err(Refused::Cancelled | Refused::Forgotten | Refused::Stopped) => {}
}
Self::evict(&mut st, self.budget, path, level);
outcome
@@ -422,7 +425,10 @@ impl PlaneStore {
/// decode doesn't wait (see the module notes); the time limit is a safety net.
#[cfg(not(target_arch = "wasm32"))]
fn wait(&self, flight: &Flight) -> Waited {
if DECODING.with(std::cell::Cell::get) > 0 {
// A decode thread never waits for planes (#3270: a decode doesn't depend on the requests
// waiting for it; an owner on a frame worker queues its decode on these threads).
let on_decode_thread = crate::pool::decode_threads().is_some_and(|t| t.current_thread_index().is_some());
if on_decode_thread || DECODING.with(std::cell::Cell::get) > 0 {
return Waited::No;
}
let until = std::time::Instant::now() + WAIT_LIMIT;
@@ -491,7 +497,8 @@ fn run(path: &str, source: &ExrSource, flight: &Arc<Flight>, base: Option<Arc<Ba
let _ = tx.send(hints.run(|| decode(&path, &source, &flight, base)));
});
// A plain block: this thread runs nothing else meanwhile.
return rx.recv().unwrap_or(Err(Refused::Unreadable));
// (a decode that panicked sent nothing)
return rx.recv().unwrap_or(Err(Refused::Stopped));
}
decode(path, source, flight, base)
}
@@ -499,9 +506,10 @@ fn run(path: &str, source: &ExrSource, flight: &Arc<Flight>, base: Option<Arc<Ba
/// Decode what `flight` asks for and `base` doesn't hold, and make the union of both.
fn decode(path: &str, source: &ExrSource, flight: &Arc<Flight>, base: Option<Arc<Batch>>) -> Result<Arc<Batch>, Refused> {
let _decoding = Decoding::new();
// (a share that doesn't answer just now isn't remembered as unreadable)
let mut opened = source.open().map_err(|e| {
log::warn!("media: {path}: {e}");
Refused::Unreadable
Refused::Stopped
})?;
let index = flight.state().index.clone();
if opened.stamp != index.stamp {
@@ -624,6 +632,8 @@ pub(crate) fn picture(path: &str, source: &ExrSource, op: AlphaOp, extra: Option
Ok(b) => b,
Err(Refused::Changed) => continue,
Err(Refused::Cancelled | Refused::Forgotten) => return Picture::Cancelled,
// (the whole file's decode says why)
Err(Refused::Stopped) => return Picture::Legacy,
Err(Refused::TooLarge(why)) => return Picture::Done(Err(why)),
Err(Refused::Unreadable) => return Picture::Legacy,
};
+144 -7
View File
@@ -91,9 +91,6 @@ impl Temp {
std::fs::write(&path, bytes).unwrap();
Temp(path)
}
fn path(&self) -> String {
self.0.to_string_lossy().to_string()
}
}
impl Drop for Temp {
@@ -123,8 +120,8 @@ fn chunks(r: &DecodeRecord) -> usize {
fn a_picture_reads_only_its_part() {
let bytes = multipart();
let file = Temp::new("picture-part", &bytes);
let path = file.path();
let f = crate::probe(&file.0).unwrap();
let path = f.path.clone();
let pool = crate::MediaPool::new();
let img = pool.frame_at(&f, Tick::ZERO).unwrap();
assert_eq!(bits(&img), bits(&image_crates(&bytes, &f)), "the image crate's picture");
@@ -132,7 +129,9 @@ fn a_picture_reads_only_its_part() {
assert_eq!(d.len(), 1, "{d:?}");
assert_eq!(d[0].channels, ["beauty.A", "beauty.B", "beauty.G", "beauty.R"]);
assert_eq!(d[0].stats.chunks, [SIZE.1, 0, 0], "only the beauty part's chunks");
assert!(d[0].stats.bytes * 2 < bytes.len() as u64, "{} of {} bytes", d[0].stats.bytes, bytes.len());
let index = crate::exr_index::cached(&path).unwrap();
let beauty: u64 = index.parts[0].chunks.iter().map(|c| c.len).sum();
assert_eq!(d[0].stats.bytes, beauty, "the beauty part's bytes and nothing else");
}
/// #3271: EXtractoR reading 3 channels of a multi-part file reads only their part and converts
@@ -142,8 +141,8 @@ fn a_picture_reads_only_its_part() {
fn three_aux_channels_read_only_their_part() {
let bytes = multipart();
let file = Temp::new("aux-part", &bytes);
let path = file.path();
let f = crate::probe(&file.0).unwrap();
let path = f.path.clone();
let pool = crate::MediaPool::new();
let need = AuxNeed::Lookups(["Crypto_Material00.R", "Crypto_Material00.G", "Crypto_Material00.B"].map(|n| AuxLookup::Name(n.into())).to_vec());
let aux = pool.aux_for(ItemId(1), &f, Tick::ZERO, &need).unwrap();
@@ -187,8 +186,8 @@ fn multipart_channels_are_found_by_their_stored_names() {
fn picture_and_aux_share_one_read_and_one_decode() {
let bytes = single_part_zips();
let file = Temp::new("one-decode", &bytes);
let path = file.path();
let f = crate::probe(&file.0).unwrap();
let path = f.path.clone();
let pool = crate::MediaPool::new();
let need = AuxNeed::Lookups(vec![AuxLookup::Name("normal.R".into()), AuxLookup::Crypto { name: String::new(), index: 0 }]);
pool.set_aux_footage(std::slice::from_ref(&f.path));
@@ -437,3 +436,141 @@ fn picture_from_planes_is_the_image_crates() {
}
}
}
/// #474 / #3271: a decode that stops (a panic, on the decode threads or on the thread asking)
/// leaves nothing behind: the entry isn't "decoding" or "unreadable", the next request decodes.
#[test]
fn a_decode_that_panics_is_not_left_in_the_store() {
let bytes: Arc<[u8]> = write(vec![part(None, &RGBA, (8, 8), Compression::ZIP1)], (8, 8)).into();
let all = Want { all: true, ids: Default::default() };
for (k, inline) in [false, true].into_iter().enumerate() {
let path = format!("/3271-panics-{k}.exr");
let source = ExrSource::Bytes(bytes.clone());
let store = PlaneStore::new(CACHE_BUDGET);
let index = crate::exr_index::get(&path, &source).unwrap();
let panicked = Arc::new(AtomicU64::new(0));
{
let (watch, panicked) = (path.clone(), panicked.clone());
set_before_read(Some(Arc::new(move |p: &str| {
if p == watch && panicked.fetch_add(1, Ordering::SeqCst) == 0 {
panic!("a decoder bug");
}
})));
}
let ask = || store.get(&path, &source, &index, &all, Use::Other);
if inline {
// On a decode thread the decode runs right there, and the panic comes through.
let threads = crate::pool::decode_threads().unwrap();
let r = threads.install(|| std::panic::catch_unwind(std::panic::AssertUnwindSafe(ask)));
assert!(r.is_err(), "the panic came through");
assert!(store.cached().is_empty(), "nothing left");
assert!(ask().is_ok(), "decodes again");
} else {
// Handed to the decode threads: the request asks again by itself.
assert!(ask().is_ok());
}
set_before_read(None);
assert_eq!(panicked.load(Ordering::SeqCst), 2, "inline: {inline}");
assert_eq!(store.cached(), [path.clone()]);
}
}
/// #474: a file counts as used when its decode finishes: one asked for first but decoded after
/// another is the newer of the two, so concurrent frames don't evict each other's files before
/// their effects read them.
#[test]
fn a_file_is_used_as_of_its_decode() {
let bytes: Arc<[u8]> = write(vec![part(None, &RGBA, (32, 32), Compression::ZIP1)], (32, 32)).into();
let one = 4 * (32 * 32 * 4 + 1);
let store = Arc::new(PlaneStore::new(2 * one));
let all = Want { all: true, ids: Default::default() };
let get = {
let (store, bytes, all) = (store.clone(), bytes.clone(), all.clone());
move |p: &str| {
let source = ExrSource::Bytes(bytes.clone());
let index = crate::exr_index::get(p, &source).unwrap();
store.get(p, &source, &index, &all, Use::Other).map(|_| ())
}
};
{
let get = get.clone();
let nested = Arc::new(AtomicU64::new(0));
set_before_read(Some(Arc::new(move |p: &str| {
// Another frame's file is asked for and decoded while this one decodes.
if p == "/3271-used-first.exr" && nested.fetch_add(1, Ordering::SeqCst) == 0 {
get("/3271-used-second.exr").unwrap();
}
})));
}
get("/3271-used-first.exr").unwrap();
set_before_read(None);
get("/3271-used-third.exr").unwrap();
assert_eq!(store.cached(), ["/3271-used-first.exr", "/3271-used-third.exr"], "the file decoded first went first");
}
/// #474: frames rendered at once (each asks for its picture, then its effects for the channels
/// while other frames decode) decode each file once, though the budget holds only one: a
/// picture's planes are kept for its effects, and planes in use are never dropped.
#[test]
fn concurrent_frames_decode_each_file_once() {
let rgbz = [("R", false), ("G", false), ("B", false), ("Z", false)];
let bytes: Arc<[u8]> = write(vec![part(None, &rgbz, (64, 64), Compression::ZIP1)], (64, 64)).into();
let store = PlaneStore::new(4 * (64 * 64 * 4 + 1));
let all = Want { all: true, ids: Default::default() };
let (next, frames) = (std::sync::atomic::AtomicUsize::new(0), 24);
std::thread::scope(|s| {
for _ in 0..4 {
s.spawn(|| {
loop {
let i = next.fetch_add(1, Ordering::Relaxed);
if i >= frames {
break;
}
let path = format!("/3271-frames/{i:04}.exr");
let source = ExrSource::Bytes(bytes.clone());
let index = crate::exr_index::get(&path, &source).unwrap();
drop(store.get(&path, &source, &index, &all, Use::Picture).unwrap());
std::thread::sleep(std::time::Duration::from_millis(3));
let b = store.get(&path, &source, &index, &all, Use::Other).unwrap();
std::thread::sleep(std::time::Duration::from_millis(2));
assert_eq!(b.aux.get("Z").and_then(|z| z.first()).copied(), Some(value(0, 3, false)));
}
});
}
});
for i in 0..frames {
assert_eq!(decodes_of(&format!("/3271-frames/{i:04}.exr")).len(), 1, "frame {i}: one decode");
}
}
/// #474: once the project no longer names footage (its effect removed), later frames decode
/// only their picture's channels again; what `aux()` learned is dropped with each new set too.
#[test]
fn a_new_aux_footage_set_returns_to_picture_only() {
let bytes: Arc<[u8]> = single_part_zips().into();
let paths: Vec<String> = (1..=4).map(|i| format!("/3271-unset_{i:04}.exr")).collect();
let pool = crate::MediaPool::new();
for p in &paths {
pool.add_bytes(p, bytes.clone());
}
let mut f = crate::probe_bytes(&paths[0], bytes.clone()).unwrap();
f.kind = effectcraft_project::FootageKind::Sequence;
f.sequence = paths.clone();
let at = |i: i64| f.frame_rate.tick_of(i);
let all = crate::exr_index::get(&paths[0], &ExrSource::Bytes(bytes.clone())).unwrap().channels.len();
// The channels the frame's decode read.
let frame = |i: usize| {
pool.frame_at(&f, at(i as i64)).unwrap();
decodes_of(&paths[i]).first().map(|d| d.channels.len())
};
pool.set_aux_footage(std::slice::from_ref(&f.path));
assert_eq!(frame(0), Some(all), "given: the picture with the channels");
pool.set_aux_footage(&[]);
assert!(!shared().is_held(&paths[0]), "no longer kept once no effect reads it");
assert_eq!(frame(1), Some(4), "no longer given: the picture alone");
// Learned by `aux()`, then a new set without it: picture only again.
pool.aux(ItemId(1), &f, at(1)).unwrap();
assert_eq!(frame(2), Some(all), "learned");
pool.set_aux_footage(&[]);
assert_eq!(frame(3), Some(4), "a new set drops what was learned");
}