mirror of
https://github.com/storytold/effectcraft.git
synced 2026-11-19 16:45:28 +00:00
OpenEXR limit: the decode that let planes go no longer keeps them for its waiters, and an entry with planes let go but still in use is kept (#3750 review)
This commit is contained in:
1 parent
51bb54efde
commit
cc639b9cd6
2 files changed
+35
-24
No files matched your search
@@ -80,7 +80,7 @@ fn asks_at_least(index: &ExrIndex, want: &Want, failed: &Want) -> bool {
|
||||
/// Decode `want` for this request alone (kept nowhere), noting the file it read. `live`: planes
|
||||
/// of the file let go but still in memory ([`PlaneStore::live`]).
|
||||
fn copy(path: &str, source: &ExrSource, index: &Arc<ExrIndex>, want: &Want, live: usize) -> Result<Arc<Batch>, Refused> {
|
||||
let b = run(path, source, &Flight::new(index.clone(), want.clone(), live), None, None);
|
||||
let b = run(path, source, &Flight::new(index.clone(), want.clone(), live), None);
|
||||
if let Ok(b) = &b {
|
||||
crate::reads::note(b.file.clone());
|
||||
}
|
||||
@@ -179,8 +179,10 @@ struct Flight {
|
||||
phase: std::sync::atomic::AtomicU8,
|
||||
handed: Mutex<Option<Result<Arc<Batch>, Refused>>>,
|
||||
handed_ready: Condvar,
|
||||
/// What the store held when the decode was asked for (the decode adds to it).
|
||||
base: Option<Arc<Batch>>,
|
||||
/// What the store held when the decode was asked for (the decode adds to it). Taken when the
|
||||
/// store lets it go ([`PlaneStore::let_go`]): the requests still holding the flight (its
|
||||
/// waiters, until they wake) don't keep those planes in memory.
|
||||
base: Mutex<Option<Arc<Batch>>>,
|
||||
/// Planes of the file the store let go ([`Refused::LetGo`]) that are still in memory when the
|
||||
/// decode was asked for: a caller holds them, or a decode that let them go hasn't ended yet
|
||||
/// ([`PlaneStore::live`]). They count toward the limit with the planes read.
|
||||
@@ -231,7 +233,7 @@ impl Flight {
|
||||
phase: std::sync::atomic::AtomicU8::new(QUEUED),
|
||||
handed: Mutex::new(None),
|
||||
handed_ready: Condvar::new(),
|
||||
base,
|
||||
base: Mutex::new(base),
|
||||
live,
|
||||
since,
|
||||
hints: filmcraft_media::cancel::Context::current(),
|
||||
@@ -284,6 +286,11 @@ impl Flight {
|
||||
}
|
||||
}
|
||||
|
||||
/// [`Flight::base`] (`None` once let go).
|
||||
fn base(&self) -> Option<Arc<Batch>> {
|
||||
self.base.lock().unwrap_or_else(PoisonError::into_inner).clone()
|
||||
}
|
||||
|
||||
fn state(&self) -> MutexGuard<'_, FlightState> {
|
||||
self.state.lock().unwrap_or_else(PoisonError::into_inner)
|
||||
}
|
||||
@@ -320,9 +327,12 @@ impl Entry {
|
||||
self.let_go.iter().map(|(_, n)| *n).fold(0, usize::saturating_add)
|
||||
}
|
||||
|
||||
/// Dropping it would free nothing or lose work in progress: a decode runs, or planes are held.
|
||||
/// Dropping it would free nothing or lose work in progress: a decode runs, or planes are held
|
||||
/// (also planes it let go: dropping the entry would forget that they are still in memory).
|
||||
fn in_use(&self) -> bool {
|
||||
self.flight.is_some() || self.batch.as_ref().is_some_and(|b| Arc::strong_count(b) > 1 || Arc::strong_count(&b.aux) > 1)
|
||||
self.flight.is_some()
|
||||
|| self.batch.as_ref().is_some_and(|b| Arc::strong_count(b) > 1 || Arc::strong_count(&b.aux) > 1)
|
||||
|| self.let_go.iter().any(|(w, _)| w.strong_count() > 0)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -608,20 +618,19 @@ impl PlaneStore {
|
||||
Waited::Got(Err(Refused::TooLarge(_))) if flight.asks_beyond(index, want) => Step::Again { retry: true },
|
||||
Waited::Got(Err(e)) => Step::Done(Err(e)),
|
||||
Waited::Cancelled => Step::Done(Err(Refused::Cancelled)),
|
||||
// A copy, kept nowhere (with the planes let go still in memory now: a decode
|
||||
// that let them go may not have ended yet).
|
||||
// A copy, kept nowhere (with the planes let go that something still holds now).
|
||||
Waited::No => Step::Done(copy(path, source, index, want, self.live(path))),
|
||||
};
|
||||
}
|
||||
// This request decodes: what it asks for, minus what the store holds.
|
||||
let base = e.batch.clone().filter(|b| Arc::ptr_eq(&b.index, index));
|
||||
let flight = Flight::with_base(index.clone(), want.clone(), base.clone(), live, since);
|
||||
let flight = Flight::with_base(index.clone(), want.clone(), base, live, since);
|
||||
e.flight = Some(flight.clone());
|
||||
drop(st);
|
||||
#[cfg(test)]
|
||||
crate::exr_channels::count_decode(true);
|
||||
let mut owning = Owning { store: self, path, flight: &flight, ended: false };
|
||||
let outcome = run(path, source, &flight, base, opened.take());
|
||||
let outcome = run(path, source, &flight, opened.take());
|
||||
match owning.end(outcome) {
|
||||
Ok(b) if b.covers(index, want) => {
|
||||
let mut st = self.lock();
|
||||
@@ -741,7 +750,7 @@ impl PlaneStore {
|
||||
let took = clock.saturating_sub(flight.asked).max(1);
|
||||
// A decode adding channels to what was held: both decodes made the planes; one
|
||||
// that read them all anew replaces what was held, and its cost.
|
||||
e.cost_us = if flight.base.is_some() { e.cost_us.saturating_add(took) } else { took };
|
||||
e.cost_us = if flight.base().is_some() { e.cost_us.saturating_add(took) } else { took };
|
||||
}
|
||||
Err(why @ (Refused::Unreadable | Refused::TooLarge(_))) => {
|
||||
let (index, want) = {
|
||||
@@ -776,9 +785,11 @@ impl PlaneStore {
|
||||
}
|
||||
|
||||
/// Let the planes `flight` was asked to add to go ([`Refused::LetGo`]): the entry of `path`
|
||||
/// and the pictures' held planes no longer keep them (unless the entry holds others by now).
|
||||
/// and the pictures' held planes no longer keep them (unless the entry holds others by now),
|
||||
/// nor the flight (whose waiters may hold it a while yet).
|
||||
fn let_go(st: &mut State, path: &str, flight: &Flight) {
|
||||
let Some(base) = &flight.base else { return };
|
||||
let Some(base) = flight.base.lock().unwrap_or_else(PoisonError::into_inner).take() else { return };
|
||||
let base = &base;
|
||||
if let Some(e) = st.entries.iter_mut().find(|e| e.path == path && e.batch.as_ref().is_some_and(|b| Arc::ptr_eq(b, base))) {
|
||||
e.batch = None;
|
||||
e.bytes = 0;
|
||||
@@ -970,7 +981,7 @@ fn decode_claimed(path: &str, source: &ExrSource, flight: &Arc<Flight>, index: &
|
||||
CLAIM_SIGNAL.notify_all();
|
||||
}
|
||||
let mut handing = Handing { flight, outcome: None };
|
||||
let outcome = flight.hints.clone().run(|| decode(path, source, flight, flight.base.clone(), None));
|
||||
let outcome = flight.hints.clone().run(|| decode(path, source, flight, flight.base(), None));
|
||||
handing.outcome = Some(outcome.clone());
|
||||
drop(handing);
|
||||
match outcome {
|
||||
@@ -987,7 +998,7 @@ fn decode_claimed(path: &str, source: &ExrSource, flight: &Arc<Flight>, index: &
|
||||
|
||||
/// Run `flight`'s decode on the footage decode threads (here when this is one of them, or
|
||||
/// there are none) and block until it is done.
|
||||
fn run(path: &str, source: &ExrSource, flight: &Arc<Flight>, base: Option<Arc<Batch>>, opened: Option<Opened>) -> Result<Arc<Batch>, Refused> {
|
||||
fn run(path: &str, source: &ExrSource, flight: &Arc<Flight>, opened: Option<Opened>) -> Result<Arc<Batch>, Refused> {
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
if let Some(threads) = crate::pool::decode_threads()
|
||||
&& threads.current_thread_index().is_none()
|
||||
@@ -1005,7 +1016,7 @@ fn run(path: &str, source: &ExrSource, flight: &Arc<Flight>, base: Option<Arc<Ba
|
||||
run_hook(&STARTING, &path);
|
||||
// (claimed meanwhile by a decode thread: it decodes, and hands the outcome over)
|
||||
if queued.take(RUNNING) {
|
||||
let outcome = hints.run(|| decode(&path, &source, &queued, base, opened));
|
||||
let outcome = hints.run(|| decode(&path, &source, &queued, queued.base(), opened));
|
||||
// (let go of the flight before the asker goes on: the flight keeps the outcome
|
||||
// for its waiters, and a flight still held here would keep the planes looking in
|
||||
// use, so the store would drop another file's instead)
|
||||
@@ -1032,7 +1043,7 @@ fn run(path: &str, source: &ExrSource, flight: &Arc<Flight>, base: Option<Arc<Ba
|
||||
if !flight.take(RUNNING) {
|
||||
return flight.take_handed();
|
||||
}
|
||||
decode(path, source, flight, base, opened)
|
||||
decode(path, source, flight, flight.base(), opened)
|
||||
}
|
||||
|
||||
/// What a failed read of `path` means for the request.
|
||||
@@ -1162,7 +1173,7 @@ fn decode(path: &str, source: &ExrSource, flight: &Arc<Flight>, base: Option<Arc
|
||||
let fits = |planes: usize| crate::exr_read::check_total(w, h, planes, cap);
|
||||
let live = flight.live;
|
||||
let still_held = || {
|
||||
log::warn!("media: {path}: {live} OpenEXR channels let go are still in use");
|
||||
log::debug!("media: {path}: {live} OpenEXR channels let go are still in use");
|
||||
Refused::NoMemory(format!("its {w}×{h} OpenEXR channels don't fit the memory one file may use beside {live} of its channels still in use"))
|
||||
};
|
||||
let held = base.as_ref().map_or(0, |b| b.ids.len());
|
||||
|
||||
@@ -1887,8 +1887,8 @@ fn planes_let_go_but_still_held_count_toward_the_limit() {
|
||||
}
|
||||
|
||||
/// #3750 review: a decode thread that took over a decode whose held planes are let go asks
|
||||
/// again while the request that queued the decode still holds them (its flight): it is refused
|
||||
/// for now rather than read beside them; that request reads once they are gone.
|
||||
/// again while the request that queued the decode still holds that decode's flight: the flight
|
||||
/// no longer keeps the planes, so they are gone, not read beside, when the request is read.
|
||||
#[test]
|
||||
fn a_claimed_decode_is_not_read_beside_the_planes_it_let_go() {
|
||||
let path = "/3750-claimed-let-go.exr";
|
||||
@@ -1922,8 +1922,8 @@ fn a_claimed_decode_is_not_read_beside_the_planes_it_let_go() {
|
||||
let _ = CLAIM_SIGNAL.wait_timeout_while(claims, std::time::Duration::from_secs(10), |c| !c.iter().any(|c| c == p));
|
||||
});
|
||||
STARTING.lock().unwrap().push((path.to_string(), hold));
|
||||
// ...and the asker, handed the let-go, holds its flight (and the planes) until the decode
|
||||
// thread has asked again.
|
||||
// ...and the asker, handed the let-go, holds its flight until the decode thread has asked
|
||||
// again.
|
||||
{
|
||||
let done = done.clone();
|
||||
let wait: Hook = Arc::new(move |_: &str| {
|
||||
@@ -1941,8 +1941,8 @@ fn a_claimed_decode_is_not_read_beside_the_planes_it_let_go() {
|
||||
HANDED.lock().unwrap().retain(|(p, _)| p != path);
|
||||
set_read_error(path, None);
|
||||
assert!(CLAIMS.lock().unwrap().iter().any(|p| p == path), "claimed");
|
||||
assert!(matches!(&claimed, Err(Refused::NoMemory(why)) if why.contains("still in use")), "{claimed:?}");
|
||||
assert_eq!(asked.ok(), Some(1), "the asker reads the depth plane once the picture's planes are gone");
|
||||
assert_eq!(claimed.ok(), Some(1), "the decode thread reads the depth plane alone");
|
||||
assert_eq!(asked.ok(), Some(1), "so does the asker");
|
||||
assert!(old.upgrade().is_none());
|
||||
let alive = alive.lock().unwrap().clone();
|
||||
assert!(!alive.is_empty() && !alive.contains(&true), "no read beside the planes let go: {alive:?}");
|
||||
|
||||
Reference in new issue
Block a user