From 88fa8ccf155afb55b294ed046562a068c00541fe Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sun, 8 Nov 2026 06:10:35 +0100 Subject: [PATCH] Footage cache (#3271): the memory accountant frees queued plane sets A queued plane write shares the plane store's set, which keeps the store from letting go of it; trimming the disk queue now drops such writes too, so the store can free the set in the same balance. The reader uses saturating offsets, and the batch test fails a later batch by position. --- crates/media/src/exr_planes/tests.rs | 20 +++++++------ crates/media/src/exr_read.rs | 6 ++-- crates/render/src/disk_cache.rs | 44 ++++++++++++++++++++++------ 3 files changed, 49 insertions(+), 21 deletions(-) diff --git a/crates/media/src/exr_planes/tests.rs b/crates/media/src/exr_planes/tests.rs index 89d2cecb9..56443b4f0 100644 --- a/crates/media/src/exr_planes/tests.rs +++ b/crates/media/src/exr_planes/tests.rs @@ -1039,13 +1039,13 @@ fn a_region_of_interest_reads_only_its_rows() { } } -/// A source that calls the request off after `cancel_after` reads, or fails every read after -/// `fail_after` (tests of a read in several batches). +/// A source that calls the request off after `cancel_after` reads, or fails every read from +/// byte `fail_from` on (tests of a read in several batches). struct Flaky { inner: std::io::Cursor>, reads: usize, cancel_after: Option<(usize, Arc)>, - fail_after: Option, + fail_from: Option, } impl std::io::Read for Flaky { @@ -1056,7 +1056,7 @@ impl std::io::Read for Flaky { { flag.store(true, Ordering::SeqCst); } - if self.fail_after.is_some_and(|n| self.reads > n) { + if self.fail_from.is_some_and(|at| self.inner.position() >= at) { return Err(std::io::Error::other("the share went away")); } self.inner.read(buf) @@ -1103,18 +1103,20 @@ fn a_read_in_several_batches_is_whole_and_stops_between_them() { } // Called off while the first batch is read: it stops before the next. let flag = Arc::new(std::sync::atomic::AtomicBool::new(false)); - let mut flaky = Flaky { inner: std::io::Cursor::new(bytes.clone()), reads: 0, cancel_after: Some((1, flag.clone())), fail_after: None }; + let mut flaky = Flaky { inner: std::io::Cursor::new(bytes.clone()), reads: 0, cancel_after: Some((1, flag.clone())), fail_from: None }; let (got, stats) = filmcraft_media::cancel::with_cancel(&flag, || crate::exr_read::read_planes(&mut flaky, &ix, &all)); assert_eq!(got.err(), Some(ReadError::Cancelled)); assert!(stats.lines < h, "stopped early: {} lines", stats.lines); // Chunks that follow each other in the file are read in one go, not one read per chunk. - let mut counted = Flaky { inner: std::io::Cursor::new(bytes.clone()), reads: 0, cancel_after: None, fail_after: None }; + let mut counted = Flaky { inner: std::io::Cursor::new(bytes.clone()), reads: 0, cancel_after: None, fail_from: None }; assert!(crate::exr_read::read_planes(&mut counted, &ix, &all).0.is_ok()); assert!(counted.reads <= 12, "{} reads for {h} chunks", counted.reads); - // A read failing in the second batch. - let mut flaky = Flaky { inner: std::io::Cursor::new(bytes), reads: 0, cancel_after: None, fail_after: Some(1) }; - let (got, _) = crate::exr_read::read_planes(&mut flaky, &ix, &all); + // A read failing in a later batch (at chunk 300): an I/O error after the first was decoded. + let at = ix.parts[0].chunks[300].offset; + let mut flaky = Flaky { inner: std::io::Cursor::new(bytes), reads: 0, cancel_after: None, fail_from: Some(at) }; + let (got, stats) = crate::exr_read::read_planes(&mut flaky, &ix, &all); assert!(matches!(got, Err(ReadError::Io(_))), "{got:?}"); + assert!(stats.lines > 0, "the batches before it were decoded"); } /// #3271: dropping entries for bytes (a decode over the budget, the memory accountant) drops diff --git a/crates/media/src/exr_read.rs b/crates/media/src/exr_read.rs index f28fe27f7..d37da11ef 100644 --- a/crates/media/src/exr_read.rs +++ b/crates/media/src/exr_read.rs @@ -328,11 +328,11 @@ fn read_batch(src: &mut dyn ReadSeek, batch: &[(usize, u64, u64)], pos: &mut Opt src.seek(SeekFrom::Start(start)).map_err(|e| ReadError::Io(e.to_string()))?; } let at = raw.data.len(); - (&mut *src).take(end - start).read_to_end(&mut raw.data).map_err(|e| ReadError::Io(e.to_string()))?; - let got = raw.data.len() - at; + (&mut *src).take(end.saturating_sub(start)).read_to_end(&mut raw.data).map_err(|e| ReadError::Io(e.to_string()))?; + let got = raw.data.len().saturating_sub(at); *pos = Some(start.saturating_add(got as u64)); for (pi, offset, len) in batch.get(i..j).unwrap_or_default() { - let a = usize::try_from(offset - start).map_or(raw.data.len(), |o| at.saturating_add(o)).min(raw.data.len()); + let a = usize::try_from(offset.saturating_sub(start)).map_or(raw.data.len(), |o| at.saturating_add(o)).min(raw.data.len()); let b = usize::try_from(*len).map_or(raw.data.len(), |l| a.saturating_add(l)).min(raw.data.len()); raw.chunks.push((*pi, a..b)); if let Some(c) = stats.chunks.get_mut(*pi) { diff --git a/crates/render/src/disk_cache.rs b/crates/render/src/disk_cache.rs index 5554765c6..a48fa7d0b 100644 --- a/crates/render/src/disk_cache.rs +++ b/crates/render/src/disk_cache.rs @@ -454,6 +454,28 @@ impl Queue { self.frames.is_empty() && self.jobs.is_empty() } + /// Drop queued layer and plane writes, newest first, until the ones left hold at most `bytes` + /// of their own and of plane sets the plane store also holds (a queued plane set keeps the + /// store from letting go of it, so it is dropped too); frame writes are never dropped (a + /// fully cached comp would come back with gaps). The writes dropped. + #[cfg_attr(target_arch = "wasm32", allow(dead_code))] + fn trim(&mut self, bytes: usize) -> u64 { + let pins = |p: &Payload| match p { + Payload::Planes(..) => p.held(), + _ => p.unique(), + }; + let held = |q: &Queue| q.jobs.iter().map(|j| if let Job::Write(_, _, _, p) = j { pins(p) } else { 0 }).fold(0, usize::saturating_add); + let mut dropped = 0; + while held(self) > bytes { + let Some(i) = self.jobs.iter().rposition(|j| matches!(j, Job::Write(_, _, _, p) if pins(p) > 0)) else { break }; + if let Some(Job::Write(_, _, _, p)) = self.jobs.remove(i) { + self.job_bytes = self.job_bytes.saturating_sub(p.held()); + } + dropped += 1; + } + dropped + } + /// The bytes only the queued layer and plane writes hold (no frames: see the memory holder). #[cfg_attr(target_arch = "wasm32", allow(dead_code))] fn unique_job_bytes(&self) -> usize { @@ -1489,17 +1511,11 @@ impl crate::memory::MemoryHolder for DiskCache { fn priority(&self) -> u8 { 0 } - /// Drops queued layer and plane writes (newest first) that alone hold their bytes; frame - /// writes are never dropped (a fully cached comp would come back with gaps). + /// Drops queued layer and plane writes (newest first), see [`Queue::trim`]. fn trim_to(&self, bytes: usize) { let Ok(mut q) = self.queue.0.lock() else { return }; - while q.unique_job_bytes() > bytes { - let Some(i) = q.jobs.iter().rposition(|j| matches!(j, Job::Write(_, _, _, p) if p.unique() > 0)) else { break }; - if let Some(Job::Write(_, _, _, p)) = q.jobs.remove(i) { - q.job_bytes = q.job_bytes.saturating_sub(p.held()); - } - self.dropped.fetch_add(1, Ordering::Relaxed); - } + let dropped = q.trim(bytes); + self.dropped.fetch_add(dropped, Ordering::Relaxed); } } @@ -2103,6 +2119,16 @@ mod tests { assert_eq!(job.unique(), 69 * 8 * 4, "the queue alone holds them"); let aux = kept.upgrade().unwrap(); drop(job); + // Trimmed by the memory accountant, a queued plane set is dropped also while the store + // holds it (it keeps the store from letting go); frames stay. + { + let mut q = dc.queue.0.lock().unwrap(); + q.jobs.push_back(Job::Write(Kind::Plane, 1, FORMAT_PLANES_FILES, Payload::Planes(aux.clone(), planes.clone(), files.clone()))); + q.frames.push_back(Job::Write(Kind::Frame, 2, FORMAT_RGBA8_FILES, Payload::Frame8(1, 1, vec![0; 4], files.clone()))); + assert_eq!(q.trim(0), 1); + assert!(q.jobs.is_empty() && q.frames.len() == 1); + q.frames.clear(); + } assert!(dc.put_planes(&aux, &planes, &files, 0)); dc.flush(); assert_eq!(dc.keys(Kind::Plane).len(), 69);