mirror of
https://github.com/storytold/effectcraft.git
synced 2026-11-19 16:45:28 +00:00
Disk cache: never serve a RAM layer buffer made before a drop of its files
A render that passed the drop check could sit in put_layer's encode while Reload Footage cleared the layer cache and dropped the file, then publish its old pixels to memory after the purge; get, and a fresh render's insert, then served them. The disk-hit promotion went through the same insert_mem. Every way the cache hands out a RAM buffer (get, the kept-first path of insert, an existing entry in insert_mem) now checks the entry's files against the store's drops from their own generation (ReadFiles::since): an entry made before a drop of one of its files is forgotten and the lookup misses, so a fresh render's buffer replaces it. insert_mem also re-checks a buffer just before publishing it.
This commit is contained in:
1 parent
9d3152f4d2
commit
4c29667bf0
1 file changed
+133
-14
+133
-14
@@ -154,19 +154,13 @@ impl LayerCache {
|
||||
|
||||
/// The buffer kept under `key` and the source files it was made of (see [`crate::reads`]).
|
||||
/// A buffer from the disk cache or another store comes with the files its entry lists, after
|
||||
/// they were found unchanged.
|
||||
/// they were found unchanged. A buffer made of a file dropped since its reads began (File ▸
|
||||
/// Reload Footage) is never served: it is forgotten, and the lookup is a miss.
|
||||
pub fn get(&self, key: u64) -> Option<(Arc<Buf>, Arc<ReadFiles>)> {
|
||||
let mut g = self.inner.lock().ok()?;
|
||||
g.clock += 1;
|
||||
let now = g.clock;
|
||||
match g.map.get_mut(&key) {
|
||||
Some(e) => {
|
||||
e.last_use = now;
|
||||
let found = (e.buf.clone(), e.files.clone());
|
||||
g.hits += 1;
|
||||
Some(found)
|
||||
}
|
||||
match self.kept(key, true) {
|
||||
Some(found) => Some(found),
|
||||
None => {
|
||||
let mut g = self.inner.lock().ok()?;
|
||||
g.misses += 1;
|
||||
drop(g);
|
||||
let store = self.persistent();
|
||||
@@ -207,7 +201,7 @@ impl LayerCache {
|
||||
if self.gated() {
|
||||
return (buf, files);
|
||||
}
|
||||
if let Some(kept) = self.inner.lock().ok().and_then(|g| g.map.get(&key).map(|e| (e.buf.clone(), e.files.clone()))) {
|
||||
if let Some(kept) = self.kept(key, false) {
|
||||
return kept;
|
||||
}
|
||||
let store = self.persistent();
|
||||
@@ -231,6 +225,44 @@ impl LayerCache {
|
||||
self.insert_mem(key, buf, files)
|
||||
}
|
||||
|
||||
/// The buffer kept in memory under `key` with its files, if it is not made of a file dropped
|
||||
/// since its reads began (one that is, is forgotten). `hit`: count it as a hit and as just
|
||||
/// used. (A render that started before a drop may publish its buffer after the drop's
|
||||
/// purge, Reload Footage clearing the cache while the buffer was being written: the check
|
||||
/// here, not the order of the two, keeps it from being served.)
|
||||
fn kept(&self, key: u64, hit: bool) -> Option<(Arc<Buf>, Arc<ReadFiles>)> {
|
||||
let found = {
|
||||
let mut g = self.inner.lock().ok()?;
|
||||
g.clock += 1;
|
||||
let now = g.clock;
|
||||
let e = g.map.get_mut(&key)?;
|
||||
if hit {
|
||||
e.last_use = now;
|
||||
}
|
||||
(e.buf.clone(), e.files.clone())
|
||||
};
|
||||
if self.made_before_a_drop(&found.1) {
|
||||
if let Ok(mut g) = self.inner.lock()
|
||||
&& g.map.get(&key).is_some_and(|e| Arc::ptr_eq(&e.files, &found.1))
|
||||
&& let Some(e) = g.map.remove(&key)
|
||||
{
|
||||
g.bytes = g.bytes.saturating_sub(e.bytes);
|
||||
}
|
||||
return None;
|
||||
}
|
||||
if hit && let Ok(mut g) = self.inner.lock() {
|
||||
g.hits += 1;
|
||||
}
|
||||
Some(found)
|
||||
}
|
||||
|
||||
/// Whether a buffer made of `files` holds a file as it was before a drop that came after its
|
||||
/// reads began ([`ReadFiles::since`]; a list that doesn't say counts as made before every
|
||||
/// drop).
|
||||
fn made_before_a_drop(&self, files: &ReadFiles) -> bool {
|
||||
self.persistent().is_some_and(|s| s.dropped_since(files, files.since().unwrap_or(0)))
|
||||
}
|
||||
|
||||
/// The persistent store's drop generation now ([`LayerStore::drop_generation`]), taken by a
|
||||
/// render before it starts reading (`None` without a store).
|
||||
pub fn drop_generation(&self) -> Option<u64> {
|
||||
@@ -248,8 +280,19 @@ impl LayerCache {
|
||||
g.clock += 1;
|
||||
let now = g.clock;
|
||||
if let Some(e) = g.map.get_mut(&key) {
|
||||
e.last_use = now;
|
||||
return (e.buf.clone(), e.files.clone());
|
||||
// (one made before a drop gives way: see [`LayerCache::kept`])
|
||||
if !self.made_before_a_drop(&e.files) {
|
||||
e.last_use = now;
|
||||
return (e.buf.clone(), e.files.clone());
|
||||
}
|
||||
if let Some(old) = g.map.remove(&key) {
|
||||
g.bytes = g.bytes.saturating_sub(old.bytes);
|
||||
}
|
||||
}
|
||||
// (checked again just before it is published: a drop may have come since `insert`
|
||||
// checked; one that comes later is caught when the buffer is looked up)
|
||||
if self.made_before_a_drop(&files) {
|
||||
return (buf, files);
|
||||
}
|
||||
g.map.insert(key, Entry { buf: buf.clone(), files: files.clone(), bytes, last_use: now });
|
||||
g.bytes += bytes;
|
||||
@@ -1004,6 +1047,82 @@ mod shared_insert_tests {
|
||||
drop((cache, dc));
|
||||
}
|
||||
|
||||
/// A disk cache whose layer writes stop at a barrier until the test lets them go on.
|
||||
struct PausedWrites {
|
||||
dc: Arc<crate::disk_cache::DiskCache>,
|
||||
arrived: std::sync::Barrier,
|
||||
go_on: std::sync::Barrier,
|
||||
}
|
||||
|
||||
impl super::LayerStore for PausedWrites {
|
||||
fn get_layer(&self, key: u128, virtual_id: Option<crate::disk_cache::VirtualFileId<'_>>) -> Option<(Arc<Buf>, crate::reads::ReadFiles)> {
|
||||
super::LayerStore::get_layer(&*self.dc, key, virtual_id)
|
||||
}
|
||||
fn put_layer(&self, key: u128, buf: &Buf, files: &crate::disk_cache::EntryFiles, since: u64) {
|
||||
self.arrived.wait();
|
||||
self.go_on.wait();
|
||||
super::LayerStore::put_layer(&*self.dc, key, buf, files, since);
|
||||
}
|
||||
fn drop_generation(&self) -> u64 {
|
||||
self.dc.drop_generation()
|
||||
}
|
||||
fn dropped_since(&self, files: &crate::reads::ReadFiles, since: u64) -> bool {
|
||||
self.dc.dropped_since(files, since)
|
||||
}
|
||||
}
|
||||
|
||||
/// Reload Footage while a render from before it is writing its buffer: the cache is cleared
|
||||
/// and the file dropped after the buffer passed its check, before it is kept. The buffer
|
||||
/// then reaches memory after the purge, but it is never served, and a render after the drop
|
||||
/// keeps its own buffer (#3191 review).
|
||||
#[test]
|
||||
fn a_buffer_kept_after_the_purge_of_a_reload_is_not_served() {
|
||||
use crate::reads::{FileStamp, ReadFile, ReadLog};
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let dc = crate::disk_cache::DiskCache::open(dir.path(), 1 << 30).unwrap();
|
||||
let store = Arc::new(PausedWrites { dc: dc.clone(), arrived: std::sync::Barrier::new(2), go_on: std::sync::Barrier::new(2) });
|
||||
let cache = Arc::new(LayerCache::default());
|
||||
cache.set_store(Some(store.clone()));
|
||||
let buf = || Arc::new(Buf { img: Image::new(2, 2), offset: [0.0; 2], scale: 1.0 });
|
||||
let files = |cache: &LayerCache| {
|
||||
let log = ReadLog::starting_at(cache.drop_generation().unwrap());
|
||||
log.note(ReadFile::new("/a.png", FileStamp::Native { len: 3, modified: Some(1) }));
|
||||
Arc::new(log.into_files())
|
||||
};
|
||||
// The old render: a miss, its reads, then its insert, which stops in the disk write.
|
||||
assert!(cache.get(1).is_none());
|
||||
let old_files = files(&cache);
|
||||
let old = buf();
|
||||
let writer = {
|
||||
let (cache, old) = (cache.clone(), old.clone());
|
||||
std::thread::spawn(move || cache.insert(1, old, old_files))
|
||||
};
|
||||
store.arrived.wait();
|
||||
// Reload Footage meanwhile: the layer cache is purged and the file dropped.
|
||||
cache.clear();
|
||||
dc.drop_files(["/a.png"]);
|
||||
dc.flush();
|
||||
store.go_on.wait();
|
||||
let _ = writer.join().unwrap();
|
||||
dc.flush();
|
||||
assert!(!dc.contains(crate::disk_cache::Kind::Layer, crate::disk_cache::layer_disk_key(1)), "not on disk");
|
||||
assert!(cache.get(1).is_none(), "the old pixels are not served from memory");
|
||||
// A render after the drop keeps its own buffer.
|
||||
let fresh = buf();
|
||||
let new_files = files(&cache);
|
||||
let mut kept = None;
|
||||
std::thread::scope(|s| {
|
||||
s.spawn(|| {
|
||||
store.arrived.wait();
|
||||
store.go_on.wait();
|
||||
});
|
||||
kept = Some(cache.insert(1, fresh.clone(), new_files).0);
|
||||
});
|
||||
let kept = kept.unwrap();
|
||||
assert!(Arc::ptr_eq(&kept, &fresh) && !Arc::ptr_eq(&kept, &old));
|
||||
assert!(Arc::ptr_eq(&cache.get(1).unwrap().0, &fresh));
|
||||
}
|
||||
|
||||
/// A render that gets another render's buffer gets that buffer's files too (#3184): when a
|
||||
/// file changed between the two reads, its frame lists the file with both identities, so the
|
||||
/// list is incomplete and the frame is not kept on disk with the new identity and the old
|
||||
|
||||
Reference in new issue
Block a user