mirror of
https://github.com/storytold/effectcraft.git
synced 2026-11-20 00:55:43 +00:00
Footage cache (#3271): the memory accountant frees queued plane sets
A queued plane write shares the plane store's set, which keeps the store from letting go of it; trimming the disk queue now drops such writes too, so the store can free the set in the same balance. The reader uses saturating offsets, and the batch test fails a later batch by position.
This commit is contained in:
1 parent
dfe9aedfe4
commit
88fa8ccf15
3 files changed
+49
-21
No files matched your search
@@ -1039,13 +1039,13 @@ fn a_region_of_interest_reads_only_its_rows() {
|
||||
}
|
||||
}
|
||||
|
||||
/// A source that calls the request off after `cancel_after` reads, or fails every read after
|
||||
/// `fail_after` (tests of a read in several batches).
|
||||
/// A source that calls the request off after `cancel_after` reads, or fails every read from
|
||||
/// byte `fail_from` on (tests of a read in several batches).
|
||||
struct Flaky {
|
||||
inner: std::io::Cursor<Vec<u8>>,
|
||||
reads: usize,
|
||||
cancel_after: Option<(usize, Arc<std::sync::atomic::AtomicBool>)>,
|
||||
fail_after: Option<usize>,
|
||||
fail_from: Option<u64>,
|
||||
}
|
||||
|
||||
impl std::io::Read for Flaky {
|
||||
@@ -1056,7 +1056,7 @@ impl std::io::Read for Flaky {
|
||||
{
|
||||
flag.store(true, Ordering::SeqCst);
|
||||
}
|
||||
if self.fail_after.is_some_and(|n| self.reads > n) {
|
||||
if self.fail_from.is_some_and(|at| self.inner.position() >= at) {
|
||||
return Err(std::io::Error::other("the share went away"));
|
||||
}
|
||||
self.inner.read(buf)
|
||||
@@ -1103,18 +1103,20 @@ fn a_read_in_several_batches_is_whole_and_stops_between_them() {
|
||||
}
|
||||
// Called off while the first batch is read: it stops before the next.
|
||||
let flag = Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||
let mut flaky = Flaky { inner: std::io::Cursor::new(bytes.clone()), reads: 0, cancel_after: Some((1, flag.clone())), fail_after: None };
|
||||
let mut flaky = Flaky { inner: std::io::Cursor::new(bytes.clone()), reads: 0, cancel_after: Some((1, flag.clone())), fail_from: None };
|
||||
let (got, stats) = filmcraft_media::cancel::with_cancel(&flag, || crate::exr_read::read_planes(&mut flaky, &ix, &all));
|
||||
assert_eq!(got.err(), Some(ReadError::Cancelled));
|
||||
assert!(stats.lines < h, "stopped early: {} lines", stats.lines);
|
||||
// Chunks that follow each other in the file are read in one go, not one read per chunk.
|
||||
let mut counted = Flaky { inner: std::io::Cursor::new(bytes.clone()), reads: 0, cancel_after: None, fail_after: None };
|
||||
let mut counted = Flaky { inner: std::io::Cursor::new(bytes.clone()), reads: 0, cancel_after: None, fail_from: None };
|
||||
assert!(crate::exr_read::read_planes(&mut counted, &ix, &all).0.is_ok());
|
||||
assert!(counted.reads <= 12, "{} reads for {h} chunks", counted.reads);
|
||||
// A read failing in the second batch.
|
||||
let mut flaky = Flaky { inner: std::io::Cursor::new(bytes), reads: 0, cancel_after: None, fail_after: Some(1) };
|
||||
let (got, _) = crate::exr_read::read_planes(&mut flaky, &ix, &all);
|
||||
// A read failing in a later batch (at chunk 300): an I/O error after the first was decoded.
|
||||
let at = ix.parts[0].chunks[300].offset;
|
||||
let mut flaky = Flaky { inner: std::io::Cursor::new(bytes), reads: 0, cancel_after: None, fail_from: Some(at) };
|
||||
let (got, stats) = crate::exr_read::read_planes(&mut flaky, &ix, &all);
|
||||
assert!(matches!(got, Err(ReadError::Io(_))), "{got:?}");
|
||||
assert!(stats.lines > 0, "the batches before it were decoded");
|
||||
}
|
||||
|
||||
/// #3271: dropping entries for bytes (a decode over the budget, the memory accountant) drops
|
||||
|
||||
@@ -328,11 +328,11 @@ fn read_batch(src: &mut dyn ReadSeek, batch: &[(usize, u64, u64)], pos: &mut Opt
|
||||
src.seek(SeekFrom::Start(start)).map_err(|e| ReadError::Io(e.to_string()))?;
|
||||
}
|
||||
let at = raw.data.len();
|
||||
(&mut *src).take(end - start).read_to_end(&mut raw.data).map_err(|e| ReadError::Io(e.to_string()))?;
|
||||
let got = raw.data.len() - at;
|
||||
(&mut *src).take(end.saturating_sub(start)).read_to_end(&mut raw.data).map_err(|e| ReadError::Io(e.to_string()))?;
|
||||
let got = raw.data.len().saturating_sub(at);
|
||||
*pos = Some(start.saturating_add(got as u64));
|
||||
for (pi, offset, len) in batch.get(i..j).unwrap_or_default() {
|
||||
let a = usize::try_from(offset - start).map_or(raw.data.len(), |o| at.saturating_add(o)).min(raw.data.len());
|
||||
let a = usize::try_from(offset.saturating_sub(start)).map_or(raw.data.len(), |o| at.saturating_add(o)).min(raw.data.len());
|
||||
let b = usize::try_from(*len).map_or(raw.data.len(), |l| a.saturating_add(l)).min(raw.data.len());
|
||||
raw.chunks.push((*pi, a..b));
|
||||
if let Some(c) = stats.chunks.get_mut(*pi) {
|
||||
|
||||
@@ -454,6 +454,28 @@ impl Queue {
|
||||
self.frames.is_empty() && self.jobs.is_empty()
|
||||
}
|
||||
|
||||
/// Drop queued layer and plane writes, newest first, until the ones left hold at most `bytes`
|
||||
/// of their own and of plane sets the plane store also holds (a queued plane set keeps the
|
||||
/// store from letting go of it, so it is dropped too); frame writes are never dropped (a
|
||||
/// fully cached comp would come back with gaps). The writes dropped.
|
||||
#[cfg_attr(target_arch = "wasm32", allow(dead_code))]
|
||||
fn trim(&mut self, bytes: usize) -> u64 {
|
||||
let pins = |p: &Payload| match p {
|
||||
Payload::Planes(..) => p.held(),
|
||||
_ => p.unique(),
|
||||
};
|
||||
let held = |q: &Queue| q.jobs.iter().map(|j| if let Job::Write(_, _, _, p) = j { pins(p) } else { 0 }).fold(0, usize::saturating_add);
|
||||
let mut dropped = 0;
|
||||
while held(self) > bytes {
|
||||
let Some(i) = self.jobs.iter().rposition(|j| matches!(j, Job::Write(_, _, _, p) if pins(p) > 0)) else { break };
|
||||
if let Some(Job::Write(_, _, _, p)) = self.jobs.remove(i) {
|
||||
self.job_bytes = self.job_bytes.saturating_sub(p.held());
|
||||
}
|
||||
dropped += 1;
|
||||
}
|
||||
dropped
|
||||
}
|
||||
|
||||
/// The bytes only the queued layer and plane writes hold (no frames: see the memory holder).
|
||||
#[cfg_attr(target_arch = "wasm32", allow(dead_code))]
|
||||
fn unique_job_bytes(&self) -> usize {
|
||||
@@ -1489,17 +1511,11 @@ impl crate::memory::MemoryHolder for DiskCache {
|
||||
fn priority(&self) -> u8 {
|
||||
0
|
||||
}
|
||||
/// Drops queued layer and plane writes (newest first) that alone hold their bytes; frame
|
||||
/// writes are never dropped (a fully cached comp would come back with gaps).
|
||||
/// Drops queued layer and plane writes (newest first), see [`Queue::trim`].
|
||||
fn trim_to(&self, bytes: usize) {
|
||||
let Ok(mut q) = self.queue.0.lock() else { return };
|
||||
while q.unique_job_bytes() > bytes {
|
||||
let Some(i) = q.jobs.iter().rposition(|j| matches!(j, Job::Write(_, _, _, p) if p.unique() > 0)) else { break };
|
||||
if let Some(Job::Write(_, _, _, p)) = q.jobs.remove(i) {
|
||||
q.job_bytes = q.job_bytes.saturating_sub(p.held());
|
||||
}
|
||||
self.dropped.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
let dropped = q.trim(bytes);
|
||||
self.dropped.fetch_add(dropped, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2103,6 +2119,16 @@ mod tests {
|
||||
assert_eq!(job.unique(), 69 * 8 * 4, "the queue alone holds them");
|
||||
let aux = kept.upgrade().unwrap();
|
||||
drop(job);
|
||||
// Trimmed by the memory accountant, a queued plane set is dropped also while the store
|
||||
// holds it (it keeps the store from letting go); frames stay.
|
||||
{
|
||||
let mut q = dc.queue.0.lock().unwrap();
|
||||
q.jobs.push_back(Job::Write(Kind::Plane, 1, FORMAT_PLANES_FILES, Payload::Planes(aux.clone(), planes.clone(), files.clone())));
|
||||
q.frames.push_back(Job::Write(Kind::Frame, 2, FORMAT_RGBA8_FILES, Payload::Frame8(1, 1, vec![0; 4], files.clone())));
|
||||
assert_eq!(q.trim(0), 1);
|
||||
assert!(q.jobs.is_empty() && q.frames.len() == 1);
|
||||
q.frames.clear();
|
||||
}
|
||||
assert!(dc.put_planes(&aux, &planes, &files, 0));
|
||||
dc.flush();
|
||||
assert_eq!(dc.keys(Kind::Plane).len(), 69);
|
||||
|
||||
Reference in new issue
Block a user