Footage cache (#3271): the picture's planes are never kept beside the converted picture

The plane store kept the R, G, B, A planes a picture was made of whenever the picture was read with channels for its effects (or with all of them), beside the converted picture in the media pool's frame cache: the same pixels twice. Once the picture is converted, its planes now leave the stored set (in place when nobody else holds it), but for a picture channel an effect names itself (EXtractoR reading R). 'All channels' means every channel but the picture's. A test reads a picture with one channel, with a named picture channel and with all channels, and checks exactly what stays and its bytes; it fails on the old code.
This commit is contained in:
vangel76 committed 2026-11-08 13:11:16 +01:00
1 parent d4cb440527
commit 89705dc2c2
3 files changed
+131 -28

No files matched your search

+7
View File
@@ -96,6 +96,13 @@ impl ExrIndex {
}
}
/// Every channel but the picture's (what [`crate::exr_planes`] means by all channels: the
/// picture is kept converted, by the media pool, not as planes too).
pub(crate) fn aux_ids(&self) -> Vec<usize> {
let picture = self.picture_ids().unwrap_or_default();
(0..self.channels.len()).filter(|i| !picture.contains(i)).collect()
}
/// The channels named `names` (exact plane names, each once, in file order).
pub(crate) fn ids_named<'a>(&self, names: impl IntoIterator<Item = &'a str>) -> Vec<usize> {
let names: std::collections::HashSet<&str> = names.into_iter().collect();
+78 -10
View File
@@ -83,7 +83,8 @@ pub(crate) struct Batch {
pub(crate) index: Arc<ExrIndex>,
/// The channels held, in file order (parallel to `aux.channels`).
pub(crate) ids: Vec<usize>,
/// Every channel of the file.
/// Every channel of the file but the picture's ([`ExrIndex::aux_ids`]; the picture's may be
/// there too).
pub(crate) all: bool,
pub(crate) aux: Arc<AuxChannels>,
/// The file as read.
@@ -104,14 +105,15 @@ impl Batch {
/// It holds what `want` asks of the file `index` describes.
fn covers(&self, index: &Arc<ExrIndex>, want: &Want) -> bool {
Arc::ptr_eq(&self.index, index) && (self.all || (!want.all && want.ids.iter().all(|id| self.ids.binary_search(id).is_ok())))
Arc::ptr_eq(&self.index, index) && (self.all || !want.all) && want.ids.iter().all(|id| self.ids.binary_search(id).is_ok())
}
}
/// The channels a request asks for.
#[derive(Clone, Debug, Default, PartialEq)]
pub(crate) struct Want {
/// Every channel.
/// Every channel but the picture's ([`ExrIndex::aux_ids`]): the picture's planes are asked
/// for by name (`ids`).
pub(crate) all: bool,
pub(crate) ids: BTreeSet<usize>,
}
@@ -554,6 +556,55 @@ impl PlaneStore {
}
}
/// Keep only the channels `keep` of the planes `b` of `path` (a picture converted: the pool
/// keeps the picture, its planes would be a second copy of it); nothing when none is left.
/// The set is trimmed in place when nobody else holds it, else the kept planes are copied.
pub(crate) fn keep_only(&self, path: &str, b: Arc<Batch>, keep: &BTreeSet<usize>) {
if b.ids.iter().all(|id| keep.contains(id)) {
return;
}
let mut st = self.lock();
let Some(at) = st.entries.iter().position(|e| e.path == path && e.batch.as_ref().is_some_and(|k| Arc::ptr_eq(k, &b))) else { return };
let held = st.held.iter().any(|(p, h)| p == path && Arc::ptr_eq(h, &b));
// (taken out of the entry and the held list: then this is likely the only holder)
if let Some(e) = st.entries.get_mut(at) {
e.batch = None;
e.bytes = 0;
}
st.held.retain(|(p, h)| !(p == path && Arc::ptr_eq(h, &b)));
let slim = match Arc::try_unwrap(b) {
Ok(mut owned) => {
let mut aux = Arc::try_unwrap(owned.aux).unwrap_or_else(|shared| AuxChannels::clone(&shared));
let channels = std::mem::take(&mut aux.channels);
aux.channels = owned.ids.iter().zip(channels).filter(|(id, _)| keep.contains(id)).map(|(_, c)| c).collect();
owned.ids.retain(|id| keep.contains(id));
aux.listed = owned.index.channels.iter().map(|c| c.name.clone()).collect();
owned.aux = Arc::new(aux);
owned
}
Err(shared) => {
let mut aux = AuxChannels::new(shared.aux.width, shared.aux.height, shared.aux.scale);
aux.manifests = shared.aux.manifests.clone();
aux.aliases = shared.aux.aliases.clone();
aux.listed = shared.index.channels.iter().map(|c| c.name.clone()).collect();
aux.channels = shared.ids.iter().zip(&shared.aux.channels).filter(|(id, _)| keep.contains(id)).map(|(_, c)| c.clone()).collect();
let ids = shared.ids.iter().copied().filter(|id| keep.contains(id)).collect();
Batch { index: shared.index.clone(), ids, all: shared.all, aux: Arc::new(aux), file: shared.file.clone() }
}
};
if slim.ids.is_empty() {
return;
}
let slim = Arc::new(slim);
if let Some(e) = st.entries.get_mut(at) {
e.bytes = slim.bytes();
e.batch = Some(slim.clone());
}
if held {
st.held.push_back((path.to_string(), slim));
}
}
/// Stop keeping the planes `b` of `path` (a picture alone, converted: the pool keeps the
/// picture, the planes would be a second copy of it).
pub(crate) fn release(&self, path: &str, b: &Arc<Batch>) {
@@ -720,6 +771,15 @@ impl PlaneStore {
self.lock().held.clear();
}
/// The channels kept of `path` and their bytes (tests).
#[cfg(test)]
pub(crate) fn kept(&self, path: &str) -> Option<(Vec<String>, usize)> {
let st = self.lock();
let e = st.entries.iter().find(|e| e.path == path)?;
let b = e.batch.as_ref()?;
Some((b.aux.channels.iter().map(|(n, _)| n.clone()).collect(), e.bytes))
}
#[cfg(test)]
pub(crate) fn cached(&self) -> Vec<String> {
self.lock().entries.iter().filter(|e| e.batch.is_some() || !e.failed.is_empty()).map(|e| e.path.clone()).collect()
@@ -858,8 +918,12 @@ fn decode(path: &str, source: &ExrSource, flight: &Arc<Flight>, base: Option<Arc
fs.sealed = true;
fs.want.clone()
};
let wanted: Vec<usize> =
if want.all { (0..index.channels.len()).collect() } else { want.ids.iter().copied().filter(|i| *i < index.channels.len()).collect() };
let wanted: Vec<usize> = if want.all {
index.aux_ids().into_iter().chain(want.ids.iter().copied()).collect::<BTreeSet<usize>>().into_iter().collect()
} else {
want.ids.iter().copied().collect()
};
let wanted: Vec<usize> = wanted.into_iter().filter(|i| *i < index.channels.len()).collect();
let base = base.filter(|b| Arc::ptr_eq(&b.index, &index));
let missing: Vec<usize> = wanted.iter().copied().filter(|id| !base.as_ref().is_some_and(|b| b.ids.binary_search(id).is_ok())).collect();
let names: Vec<String> = missing.iter().filter_map(|i| index.channels.get(*i)).map(|c| c.name.clone()).collect();
@@ -923,11 +987,11 @@ fn decode(path: &str, source: &ExrSource, flight: &Arc<Flight>, base: Option<Arc
}
new.sort_by_key(|(id, _)| *id);
new.dedup_by_key(|(id, _)| *id);
let all = new.len() == index.channels.len();
let all = index.aux_ids().iter().all(|id| new.binary_search_by_key(id, |(i, _)| *i).is_ok());
let mut aux = AuxChannels::new(w as u32, h as u32, 1.0);
aux.manifests = index.names.manifests.clone();
aux.aliases = index.names.aliases.clone();
if !all {
if new.len() != index.channels.len() {
aux.listed = index.channels.iter().map(|c| c.name.clone()).collect();
}
let ids: Vec<usize> = new.iter().map(|(id, _)| *id).collect();
@@ -1103,9 +1167,13 @@ pub(crate) fn picture(path: &str, source: &ExrSource, op: AlphaOp, extra: Option
Err(Refused::Stopped | Refused::Unreadable) => return Picture::Legacy,
};
let img = picture_of(&index, &batch, op);
// A picture read alone is kept by the pool once converted, not twice.
if extra.is_none() && !batch.all && batch.ids.len() == want.ids.len() {
store().release(path, &batch);
// The picture is kept by the pool once converted, not twice: its planes go, but for those
// an effect names (EXtractoR reading "R").
if img.is_some() {
let named = extra.and_then(|n| need_ids(&index, n)).unwrap_or_default();
let picture: BTreeSet<usize> = index.picture_ids().unwrap_or_default().into_iter().filter(|id| !named.contains(id)).collect();
let keep: BTreeSet<usize> = batch.ids.iter().copied().filter(|id| !picture.contains(id)).collect();
store().keep_only(path, batch, &keep);
}
return match img {
Some(img) => Picture::Done(Ok(img)),
+46 -18
View File
@@ -181,7 +181,7 @@ fn multipart_channels_are_found_by_their_stored_names() {
assert_eq!(aux.manifest(&layer).map(<[_]>::len), Some(1));
// The names alone, from the headers.
let names = pool.aux_names(ItemId(1), &f, Tick::ZERO).unwrap();
assert_eq!(names.names(), aux.names());
assert_eq!(names.names(), aux.listed.iter().map(String::as_str).collect::<Vec<_>>());
assert!(names.channels.iter().all(|(_, p)| p.is_empty()), "no pixels decoded");
}
@@ -271,7 +271,7 @@ fn concurrent_requests_decode_a_file_once() {
.map(|_| {
s.spawn(|| {
barrier.wait();
store.get(path, &source, &index, &Want { all: true, ids: Default::default() }, Use::Other).unwrap()
store.get(path, &source, &index, &Want { all: true, ids: (0..4).collect() }, Use::Other).unwrap()
})
})
.collect();
@@ -311,7 +311,7 @@ fn the_store_evicts_least_recently_used_by_bytes() {
let bytes: Arc<[u8]> = write(vec![part(None, &RGBA, (32, 32), Compression::ZIP1)], (32, 32)).into();
let one = 4 * (32 * 32 * 4 + 1);
let store = PlaneStore::new(2 * one);
let all = Want { all: true, ids: Default::default() };
let all = Want { all: true, ids: (0..4).collect() };
let get = |p: &str| {
let source = ExrSource::Bytes(bytes.clone());
let index = crate::exr_index::get(p, &source).unwrap();
@@ -350,7 +350,7 @@ fn a_decode_thread_finishing_late_does_not_pin_the_planes() {
let bytes: Arc<[u8]> = write(vec![part(None, &RGBA, (32, 32), Compression::ZIP1)], (32, 32)).into();
let one = 4 * (32 * 32 * 4 + 1);
let store = PlaneStore::new(2 * one);
let all = Want { all: true, ids: Default::default() };
let all = Want { all: true, ids: (0..4).collect() };
let get = |p: &str| {
let source = ExrSource::Bytes(bytes.clone());
let index = crate::exr_index::get(p, &source).unwrap();
@@ -397,7 +397,7 @@ fn a_file_forgotten_while_it_decodes_is_not_kept() {
})),
);
}
let got = store.get(path, &source, &index, &Want { all: true, ids: Default::default() }, Use::Other);
let got = store.get(path, &source, &index, &Want { all: true, ids: (0..4).collect() }, Use::Other);
set_before_read(path, None);
assert!(got.is_ok());
assert_eq!(decodes_of(path).len(), 2, "decoded again after the reload");
@@ -414,7 +414,7 @@ fn a_request_after_forget_does_not_join_the_old_decode() {
let source = ExrSource::Bytes(bytes);
let store = Arc::new(PlaneStore::new(CACHE_BUDGET));
let index = crate::exr_index::get(path, &source).unwrap();
let all = Want { all: true, ids: Default::default() };
let all = Want { all: true, ids: (0..4).collect() };
let calls = Arc::new(AtomicU64::new(0));
let (tx, rx) = std::sync::mpsc::channel();
{
@@ -460,7 +460,7 @@ fn forgetting_another_file_keeps_a_decode_in_flight() {
let bytes: Arc<[u8]> = write(vec![part(None, &RGBA, (8, 8), Compression::ZIP1)], (8, 8)).into();
let (a, b) = ("/3271-forget-other-a.exr", "/3271-forget-other-b.exr");
let store = Arc::new(PlaneStore::new(CACHE_BUDGET));
let all = Want { all: true, ids: Default::default() };
let all = Want { all: true, ids: (0..4).collect() };
let source = ExrSource::Bytes(bytes);
let index = crate::exr_index::get(a, &source).unwrap();
// B is in the store first.
@@ -502,7 +502,7 @@ fn a_pictures_planes_are_held_for_its_effects() {
let source = ExrSource::Bytes(bytes);
let store = PlaneStore::new(1);
let index = crate::exr_index::get(path, &source).unwrap();
let all = Want { all: true, ids: Default::default() };
let all = Want { all: true, ids: (0..4).collect() };
drop(store.get(path, &source, &index, &all, Use::Picture).unwrap());
assert!(store.is_held(path));
drop(store.get(path, &source, &index, &all, Use::Other).unwrap());
@@ -534,7 +534,7 @@ fn given_aux_footage_decodes_once_from_the_first_frame() {
assert_eq!(take_decodes(), [0, 2]);
let d = decodes_of(learned);
assert_eq!(d[0].channels.len(), 4, "the picture alone");
assert_eq!(d[1].channels.len(), crate::exr_index::cached(learned).unwrap().channels.len(), "all: a picture read alone isn't kept twice");
assert_eq!(d[1].channels.len(), crate::exr_index::cached(learned).unwrap().aux_ids().len(), "all but the picture: a picture read alone isn't kept twice");
}
/// #474: the picture made of the planes of a file with exact R, G, B channels is the `image`
@@ -576,7 +576,7 @@ fn picture_from_planes_is_the_image_crates() {
#[test]
fn a_decode_that_panics_is_not_left_in_the_store() {
let bytes: Arc<[u8]> = write(vec![part(None, &RGBA, (8, 8), Compression::ZIP1)], (8, 8)).into();
let all = Want { all: true, ids: Default::default() };
let all = Want { all: true, ids: (0..4).collect() };
for (k, inline) in [false, true].into_iter().enumerate() {
let path = format!("/3271-panics-{k}.exr");
let source = ExrSource::Bytes(bytes.clone());
@@ -620,7 +620,7 @@ fn a_file_is_used_as_of_its_decode() {
let bytes: Arc<[u8]> = write(vec![part(None, &RGBA, (32, 32), Compression::ZIP1)], (32, 32)).into();
let one = 4 * (32 * 32 * 4 + 1);
let store = Arc::new(PlaneStore::new(2 * one));
let all = Want { all: true, ids: Default::default() };
let all = Want { all: true, ids: (0..4).collect() };
let get = {
let (store, bytes, all) = (store.clone(), bytes.clone(), all.clone());
move |p: &str| {
@@ -656,7 +656,7 @@ fn concurrent_frames_decode_each_file_once() {
let rgbz = [("R", false), ("G", false), ("B", false), ("Z", false)];
let bytes: Arc<[u8]> = write(vec![part(None, &rgbz, (64, 64), Compression::ZIP1)], (64, 64)).into();
let store = PlaneStore::new(4 * (64 * 64 * 4 + 1));
let all = Want { all: true, ids: Default::default() };
let all = Want { all: true, ids: (0..4).collect() };
let (next, frames) = (std::sync::atomic::AtomicUsize::new(0), 24);
std::thread::scope(|s| {
for _ in 0..4 {
@@ -769,7 +769,7 @@ fn a_failed_request_fails_only_requests_as_large() {
let store = PlaneStore::new(CACHE_BUDGET);
let index = crate::exr_index::get(path, &source).unwrap();
let picture = Want { all: false, ids: index.picture_ids().unwrap().into_iter().collect() };
let all = Want { all: true, ids: Default::default() };
let all = Want { all: true, ids: (0..4).collect() };
let held = store.get(path, &source, &index, &picture, Use::Picture).unwrap();
assert_eq!(store.get(path, &source, &index, &all, Use::Other).err(), Some(Refused::Unreadable));
assert!(Arc::ptr_eq(&store.get(path, &source, &index, &picture, Use::Other).unwrap(), &held), "served from the store");
@@ -804,7 +804,7 @@ fn a_picture_too_large_with_its_channels_is_read_alone() {
fn the_store_keeps_a_bounded_number_of_files() {
let bytes: Arc<[u8]> = write(vec![part(None, &[("R", true)], (2, 2), Compression::ZIP1)], (2, 2)).into();
let store = PlaneStore::new(CACHE_BUDGET);
let all = Want { all: true, ids: Default::default() };
let all = Want { all: true, ids: (0..4).collect() };
for i in 0..CACHE_FILES + 10 {
let path = format!("/3271-many/{i}.exr");
let source = ExrSource::Bytes(bytes.clone());
@@ -994,7 +994,7 @@ fn a_decode_thread_takes_over_a_queued_decode() {
let source = ExrSource::Bytes(bytes);
let store = Arc::new(PlaneStore::new(CACHE_BUDGET));
let index = crate::exr_index::get(path, &source).unwrap();
let all = Want { all: true, ids: Default::default() };
let all = Want { all: true, ids: (0..4).collect() };
let threads = crate::pool::decode_threads().unwrap();
let (tx, rx) = std::sync::mpsc::channel();
// When the asker queues its decode: a decode thread asks for the same planes.
@@ -1045,7 +1045,7 @@ fn the_plane_store_is_trimmed_by_the_accountant() {
}
let bytes: Arc<[u8]> = write(vec![part(None, &RGBA, (32, 32), Compression::ZIP1)], (32, 32)).into();
let store = Arc::new(PlaneStore::new(CACHE_BUDGET));
let all = Want { all: true, ids: Default::default() };
let all = Want { all: true, ids: (0..4).collect() };
let get = |p: &str, by: Use| {
let source = ExrSource::Bytes(bytes.clone());
let index = crate::exr_index::get(p, &source).unwrap();
@@ -1250,7 +1250,7 @@ fn trimming_for_bytes_keeps_remembered_failures() {
let bytes: Arc<[u8]> = write(vec![part(None, &RGBA, (32, 32), Compression::ZIP1)], (32, 32)).into();
let one = 4 * (32 * 32 * 4 + 1);
let store = PlaneStore::new(one);
let all = Want { all: true, ids: Default::default() };
let all = Want { all: true, ids: (0..4).collect() };
let mut broken = bytes.to_vec();
broken.truncate(broken.len() - 64);
let broken = ExrSource::Bytes(broken.into());
@@ -1327,7 +1327,7 @@ fn frames_and_planes_make_room_for_each_other() {
acc.register(h);
}
acc.set_group_budget(FOOTAGE, budget);
let all = Want { all: true, ids: Default::default() };
let all = Want { all: true, ids: (0..4).collect() };
let planes = |p: &str| {
let source = ExrSource::Bytes(bytes.clone());
let index = crate::exr_index::get(p, &source).unwrap();
@@ -1354,3 +1354,31 @@ fn frames_and_planes_make_room_for_each_other() {
assert!(store.cached().contains(&"/3271-share-p6.exr".to_string()), "the newest plane set stays");
assert!(acc.total() <= budget, "{} of {budget}", acc.total());
}
/// #3271 (rule: picture channels are never kept twice): once the picture is converted (the
/// media pool keeps it), its R, G, B, A planes leave the plane store; the channels the effects
/// read stay, and a picture channel an effect names itself (EXtractoR reading "R") stays too.
#[test]
fn the_picture_planes_are_not_kept_beside_the_picture() {
let bytes: Arc<[u8]> =
write(vec![part(None, &[("R", true), ("G", true), ("B", true), ("A", true), ("Z", false), ("N", false)], (16, 8), Compression::ZIP1)], (16, 8)).into();
let source = ExrSource::Bytes(bytes);
let op = crate::convert::AlphaOp::of(&Footage::default(), FloatCurve::Linear);
let plane = 16 * 8 * 4;
for (path, need, kept) in [
("/3271-slim-aux.exr", AuxNeed::Lookups(vec![AuxLookup::Name("Z".into())]), vec!["Z"]),
("/3271-slim-named.exr", AuxNeed::Lookups(vec![AuxLookup::Name("Z".into()), AuxLookup::Name("R".into())]), vec!["R", "Z"]),
("/3271-slim-all.exr", AuxNeed::All, vec!["N", "Z"]),
] {
let Picture::Done(Ok(_)) = picture(path, &source, op, Some(&need)) else { panic!("{path}: a picture") };
let (names, bytes) = shared().kept(path).expect("the effects' channels are kept");
assert_eq!(names, kept, "{path}");
assert_eq!(bytes, kept.iter().map(|n| n.len() + plane).sum::<usize>(), "{path}");
forget(path);
}
// The picture alone keeps nothing.
let path = "/3271-slim-none.exr";
let Picture::Done(Ok(_)) = picture(path, &source, op, None) else { panic!("a picture") };
assert!(shared().kept(path).is_none());
forget(path);
}