Footage cache (#3271): the memory accountant frees queued plane sets

A queued plane write shares the plane store's set, which keeps the store
from letting go of it; trimming the disk queue now drops such writes
too, so the store can free the set in the same balance. The reader uses
saturating offsets, and the batch test fails a later batch by position.
This commit is contained in:
vangel76 committed 2026-11-08 06:10:35 +01:00
1 parent dfe9aedfe4
commit 88fa8ccf15
3 files changed
+49 -21

No files matched your search

+11 -9
View File
@@ -1039,13 +1039,13 @@ fn a_region_of_interest_reads_only_its_rows() {
}
}
/// A source that calls the request off after `cancel_after` reads, or fails every read after
/// `fail_after` (tests of a read in several batches).
/// A source that calls the request off after `cancel_after` reads, or fails every read from
/// byte `fail_from` on (tests of a read in several batches).
struct Flaky {
inner: std::io::Cursor<Vec<u8>>,
reads: usize,
cancel_after: Option<(usize, Arc<std::sync::atomic::AtomicBool>)>,
fail_after: Option<usize>,
fail_from: Option<u64>,
}
impl std::io::Read for Flaky {
@@ -1056,7 +1056,7 @@ impl std::io::Read for Flaky {
{
flag.store(true, Ordering::SeqCst);
}
if self.fail_after.is_some_and(|n| self.reads > n) {
if self.fail_from.is_some_and(|at| self.inner.position() >= at) {
return Err(std::io::Error::other("the share went away"));
}
self.inner.read(buf)
@@ -1103,18 +1103,20 @@ fn a_read_in_several_batches_is_whole_and_stops_between_them() {
}
// Called off while the first batch is read: it stops before the next.
let flag = Arc::new(std::sync::atomic::AtomicBool::new(false));
let mut flaky = Flaky { inner: std::io::Cursor::new(bytes.clone()), reads: 0, cancel_after: Some((1, flag.clone())), fail_after: None };
let mut flaky = Flaky { inner: std::io::Cursor::new(bytes.clone()), reads: 0, cancel_after: Some((1, flag.clone())), fail_from: None };
let (got, stats) = filmcraft_media::cancel::with_cancel(&flag, || crate::exr_read::read_planes(&mut flaky, &ix, &all));
assert_eq!(got.err(), Some(ReadError::Cancelled));
assert!(stats.lines < h, "stopped early: {} lines", stats.lines);
// Chunks that follow each other in the file are read in one go, not one read per chunk.
let mut counted = Flaky { inner: std::io::Cursor::new(bytes.clone()), reads: 0, cancel_after: None, fail_after: None };
let mut counted = Flaky { inner: std::io::Cursor::new(bytes.clone()), reads: 0, cancel_after: None, fail_from: None };
assert!(crate::exr_read::read_planes(&mut counted, &ix, &all).0.is_ok());
assert!(counted.reads <= 12, "{} reads for {h} chunks", counted.reads);
// A read failing in the second batch.
let mut flaky = Flaky { inner: std::io::Cursor::new(bytes), reads: 0, cancel_after: None, fail_after: Some(1) };
let (got, _) = crate::exr_read::read_planes(&mut flaky, &ix, &all);
// A read failing in a later batch (at chunk 300): an I/O error after the first was decoded.
let at = ix.parts[0].chunks[300].offset;
let mut flaky = Flaky { inner: std::io::Cursor::new(bytes), reads: 0, cancel_after: None, fail_from: Some(at) };
let (got, stats) = crate::exr_read::read_planes(&mut flaky, &ix, &all);
assert!(matches!(got, Err(ReadError::Io(_))), "{got:?}");
assert!(stats.lines > 0, "the batches before it were decoded");
}
/// #3271: dropping entries for bytes (a decode over the budget, the memory accountant) drops
+3 -3
View File
@@ -328,11 +328,11 @@ fn read_batch(src: &mut dyn ReadSeek, batch: &[(usize, u64, u64)], pos: &mut Opt
src.seek(SeekFrom::Start(start)).map_err(|e| ReadError::Io(e.to_string()))?;
}
let at = raw.data.len();
(&mut *src).take(end - start).read_to_end(&mut raw.data).map_err(|e| ReadError::Io(e.to_string()))?;
let got = raw.data.len() - at;
(&mut *src).take(end.saturating_sub(start)).read_to_end(&mut raw.data).map_err(|e| ReadError::Io(e.to_string()))?;
let got = raw.data.len().saturating_sub(at);
*pos = Some(start.saturating_add(got as u64));
for (pi, offset, len) in batch.get(i..j).unwrap_or_default() {
let a = usize::try_from(offset - start).map_or(raw.data.len(), |o| at.saturating_add(o)).min(raw.data.len());
let a = usize::try_from(offset.saturating_sub(start)).map_or(raw.data.len(), |o| at.saturating_add(o)).min(raw.data.len());
let b = usize::try_from(*len).map_or(raw.data.len(), |l| a.saturating_add(l)).min(raw.data.len());
raw.chunks.push((*pi, a..b));
if let Some(c) = stats.chunks.get_mut(*pi) {
+35 -9
View File
@@ -454,6 +454,28 @@ impl Queue {
self.frames.is_empty() && self.jobs.is_empty()
}
/// Drop queued layer and plane writes, newest first, until the ones left hold at most `bytes`
/// of their own and of plane sets the plane store also holds (a queued plane set keeps the
/// store from letting go of it, so it is dropped too); frame writes are never dropped (a
/// fully cached comp would come back with gaps). The writes dropped.
#[cfg_attr(target_arch = "wasm32", allow(dead_code))]
fn trim(&mut self, bytes: usize) -> u64 {
let pins = |p: &Payload| match p {
Payload::Planes(..) => p.held(),
_ => p.unique(),
};
let held = |q: &Queue| q.jobs.iter().map(|j| if let Job::Write(_, _, _, p) = j { pins(p) } else { 0 }).fold(0, usize::saturating_add);
let mut dropped = 0;
while held(self) > bytes {
let Some(i) = self.jobs.iter().rposition(|j| matches!(j, Job::Write(_, _, _, p) if pins(p) > 0)) else { break };
if let Some(Job::Write(_, _, _, p)) = self.jobs.remove(i) {
self.job_bytes = self.job_bytes.saturating_sub(p.held());
}
dropped += 1;
}
dropped
}
/// The bytes only the queued layer and plane writes hold (no frames: see the memory holder).
#[cfg_attr(target_arch = "wasm32", allow(dead_code))]
fn unique_job_bytes(&self) -> usize {
@@ -1489,17 +1511,11 @@ impl crate::memory::MemoryHolder for DiskCache {
fn priority(&self) -> u8 {
0
}
/// Drops queued layer and plane writes (newest first) that alone hold their bytes; frame
/// writes are never dropped (a fully cached comp would come back with gaps).
/// Drops queued layer and plane writes (newest first), see [`Queue::trim`].
fn trim_to(&self, bytes: usize) {
let Ok(mut q) = self.queue.0.lock() else { return };
while q.unique_job_bytes() > bytes {
let Some(i) = q.jobs.iter().rposition(|j| matches!(j, Job::Write(_, _, _, p) if p.unique() > 0)) else { break };
if let Some(Job::Write(_, _, _, p)) = q.jobs.remove(i) {
q.job_bytes = q.job_bytes.saturating_sub(p.held());
}
self.dropped.fetch_add(1, Ordering::Relaxed);
}
let dropped = q.trim(bytes);
self.dropped.fetch_add(dropped, Ordering::Relaxed);
}
}
@@ -2103,6 +2119,16 @@ mod tests {
assert_eq!(job.unique(), 69 * 8 * 4, "the queue alone holds them");
let aux = kept.upgrade().unwrap();
drop(job);
// Trimmed by the memory accountant, a queued plane set is dropped also while the store
// holds it (it keeps the store from letting go); frames stay.
{
let mut q = dc.queue.0.lock().unwrap();
q.jobs.push_back(Job::Write(Kind::Plane, 1, FORMAT_PLANES_FILES, Payload::Planes(aux.clone(), planes.clone(), files.clone())));
q.frames.push_back(Job::Write(Kind::Frame, 2, FORMAT_RGBA8_FILES, Payload::Frame8(1, 1, vec![0; 4], files.clone())));
assert_eq!(q.trim(0), 1);
assert!(q.jobs.is_empty() && q.frames.len() == 1);
q.frames.clear();
}
assert!(dc.put_planes(&aux, &planes, &files, 0));
dc.flush();
assert_eq!(dc.keys(Kind::Plane).len(), 69);