From 38dffd4aea498eeb0db91b9f2c462e8e2f837db3 Mon Sep 17 00:00:00 2001 From: nazar-gavrylyk Date: Fri, 6 Nov 2026 07:13:15 +0100 Subject: [PATCH 01/24] Disk cache: frames and layer buffers are compressed on the writer thread MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A render thread handed each frame to the disk cache already copied pixel by pixel and LZ4-compressed (~9 ms per 1080p frame) before it could start the next one, and layer buffers were encoded even when the queue then dropped them. The queue now holds raw RGBA (one memcpy via ColorImage::as_raw) and Arcs; the writer thread encodes them. Every frame of a 120-frame bench run now reaches disk (113–119 before). cache_bench prints the flush time. --- crates/render/src/cache.rs | 11 +-- crates/render/src/disk_cache.rs | 97 ++++++++++++++++++++------ crates/ui-egui/examples/cache_bench.rs | 4 +- crates/ui-egui/src/frames.rs | 6 +- 4 files changed, 85 insertions(+), 33 deletions(-) diff --git a/crates/render/src/cache.rs b/crates/render/src/cache.rs index 7bc52606f..4f72e79fa 100644 --- a/crates/render/src/cache.rs +++ b/crates/render/src/cache.rs @@ -361,10 +361,11 @@ pub trait LayerStore: Send + Sync { /// mid-render). An entry whose files changed, or damaged, is a miss and is dropped, so the /// buffer rendered again replaces it. `virtual_id` names files served from memory. fn get_layer(&self, key: u128, virtual_id: Option>) -> Option<(Arc, ReadFiles)>; - /// Keep a buffer that was slow to render, with the source files it was made of. `since`: the + /// Keep a buffer that was slow to render, with the source files it was made of (shared, so + /// a store can encode it off the render thread). `since`: the /// [`LayerStore::drop_generation`] when its render started (a buffer made of a file dropped /// after that is refused). - fn put_layer(&self, key: u128, buf: &Buf, files: &EntryFiles, since: u64); + fn put_layer(&self, key: u128, buf: &Arc, files: &EntryFiles, since: u64); /// How many times files were dropped from the store (File ▸ Reload Footage, /// [`crate::disk_cache::DiskCache::drop_files`]); `0` for a store that never drops. fn drop_generation(&self) -> u64 { @@ -540,7 +541,7 @@ impl LayerStore for PrefetchStore { } /// Queues the layer for writing to the persistent store the first time `key` is seen. - fn put_layer(&self, key: u128, buf: &Buf, files: &EntryFiles, _since: u64) { + fn put_layer(&self, key: u128, buf: &Arc, files: &EntryFiles, _since: u64) { let Ok(mut s) = self.st.lock() else { return }; if s.known.insert(key) { s.writes.push((key, crate::disk_cache::layer_entry(buf, files))); @@ -983,7 +984,7 @@ mod shared_insert_tests { assert!(store.get_layer(5, Some(&two)).is_none()); assert!(!store.has(5)); assert_eq!(store.take_misses(), [5]); - store.put_layer(5, &buf(), &EntryFiles::of(&files, Some(&two)).unwrap(), 0); + store.put_layer(5, &Arc::new(buf()), &EntryFiles::of(&files, Some(&two)).unwrap(), 0); assert_eq!(store.take_writes().len(), 1, "written over the changed entry"); // Through the layer cache: a hit reports its files; incomplete lists are not queued. let cache = LayerCache::default(); @@ -1065,7 +1066,7 @@ mod shared_insert_tests { fn get_layer(&self, key: u128, virtual_id: Option>) -> Option<(Arc, crate::reads::ReadFiles)> { super::LayerStore::get_layer(&*self.dc, key, virtual_id) } - fn put_layer(&self, key: u128, buf: &Buf, files: &crate::disk_cache::EntryFiles, since: u64) { + fn put_layer(&self, key: u128, buf: &Arc, files: &crate::disk_cache::EntryFiles, since: u64) { self.arrived.wait(); self.go_on.wait(); super::LayerStore::put_layer(&*self.dc, key, buf, files, since); diff --git a/crates/render/src/disk_cache.rs b/crates/render/src/disk_cache.rs index f509a5896..fa0a97696 100644 --- a/crates/render/src/disk_cache.rs +++ b/crates/render/src/disk_cache.rs @@ -264,11 +264,41 @@ impl LayerPrefetch { #[cfg_attr(target_arch = "wasm32", allow(dead_code))] enum Job { /// Kind, key, header format byte, payload. - Write(Kind, u128, u8, Vec), + Write(Kind, u128, u8, Payload), /// Delete the frame and layer entries made of these files ([`DiskCache::drop_files`]). Drop(Vec>), } +/// What a queued write holds. 8-bit frames and layer buffers are encoded (LZ4) on the writer +/// thread, not on the render thread that queued them: encoding a 1080p frame took ~9 ms of every +/// frame's render, and a full queue no longer throws away encoding work. +enum Payload { + Encoded(Vec), + /// 8-bit premultiplied RGBA, `width * height * 4` bytes, and the files it was made of. + Frame(u32, u32, Vec, EntryFiles), + Layer(Arc, EntryFiles), +} + +impl Payload { + /// Bytes it holds (the frame queue's budget). + #[cfg_attr(target_arch = "wasm32", allow(dead_code))] + fn len(&self) -> usize { + match self { + Payload::Encoded(p) | Payload::Frame(_, _, p, _) => p.len(), + Payload::Layer(b, _) => b.img.data.len().saturating_mul(std::mem::size_of::()), + } + } + + /// The entry's payload: its source files, then its pixels. + fn encode(self) -> Vec { + match self { + Payload::Encoded(p) => p, + Payload::Frame(w, h, rgba, files) => with_files(&files, encode_frame(w, h, &rgba)), + Payload::Layer(b, files) => with_files(&files, encode_layer(&b)), + } + } +} + struct Queue { /// Frame writes (taken first) and their payload bytes. frames: std::collections::VecDeque, @@ -771,20 +801,20 @@ impl DiskCache { /// Queue a write; `false` when it was dropped (a full queue). fn enqueue(&self, kind: Kind, key: u128, payload: Vec) -> bool { - self.enqueue_format(kind, key, FORMAT_RGBA8, payload, None) + self.enqueue_format(kind, key, FORMAT_RGBA8, Payload::Encoded(payload), None) } /// [`DiskCache::enqueue`] with the entry's header format byte. `made`: the source files of a /// frame or layer entry and the drop generation its reads started at /// ([`DiskCache::drop_generation`]); refused (`false`) when one of them was dropped since. - fn enqueue_format(&self, kind: Kind, key: u128, format: u8, payload: Vec, made: Option<(&EntryFiles, u64)>) -> bool { + fn enqueue_format(&self, kind: Kind, key: u128, format: u8, payload: Payload, made: Option<(&EntryFiles, u64)>) -> bool { let stale = |q: &Queue| made.is_some_and(|(files, since)| q.drops.stale(files.files().iter().map(|f| &f.path), since)); #[cfg(target_arch = "wasm32")] { if self.queue.0.lock().is_ok_and(|q| stale(&q)) { return false; } - self.write_format(kind, key, format, &payload); + self.write_format(kind, key, format, &payload.encode()); true } #[cfg(not(target_arch = "wasm32"))] @@ -1033,7 +1063,7 @@ impl DiskCache { } self.count_encode(); let payload = with_files(&files, encode_frame_f16(width, height, rgba)); - self.enqueue_format(Kind::Frame, key, FORMAT_F16_FILES, payload, Some((&files, since))) + self.enqueue_format(Kind::Frame, key, FORMAT_F16_FILES, Payload::Encoded(payload), Some((&files, since))) } /// Write a half-float frame synchronously (tests); a list that can't be stored is not. @@ -1050,6 +1080,22 @@ impl DiskCache { /// render started at). #[allow(clippy::too_many_arguments)] pub fn put_frame(&self, key: u128, width: u32, height: u32, rgba: &[u8], files: &ReadFiles, virtual_id: Option>, since: u64) -> bool { + self.put_frame_owned(key, width, height, rgba.to_vec(), files, virtual_id, since) + } + + /// [`DiskCache::put_frame`] of a frame the caller hands over (no copy; compressed on the + /// writer thread). + #[allow(clippy::too_many_arguments)] + pub fn put_frame_owned( + &self, + key: u128, + width: u32, + height: u32, + rgba: Vec, + files: &ReadFiles, + virtual_id: Option>, + since: u64, + ) -> bool { if rgba.len() as u64 != width as u64 * height as u64 * 4 { return false; } @@ -1057,13 +1103,13 @@ impl DiskCache { if self.contains(Kind::Frame, key) { return true; } - // (no room on the disk: refused before the pixels are compressed) + // (no room on the disk: refused before the pixels are queued and compressed) if !self.accepts_writes() { return false; } self.count_encode(); - let payload = with_files(&files, encode_frame(width, height, rgba)); - self.enqueue_format(Kind::Frame, key, FORMAT_RGBA8_FILES, payload, Some((&files, since))) + let made = files.clone(); + self.enqueue_format(Kind::Frame, key, FORMAT_RGBA8_FILES, Payload::Frame(width, height, rgba, files), Some((&made, since))) } /// A layer buffer (key: [`layer_disk_key`]) whose source files are unchanged, with those @@ -1078,13 +1124,14 @@ impl DiskCache { /// [`EntryFiles::of`]: a buffer whose files can't all be named must not be stored). Refused /// when one of them was dropped after `since`, the [`DiskCache::drop_generation`] its render /// started at. - pub fn put_layer(&self, key: u128, buf: &Buf, files: &EntryFiles, since: u64) { - // (no room on the disk: refused before the buffer is compressed) + /// The buffer is shared, and encoded on the writer thread. + pub fn put_layer(&self, key: u128, buf: &Arc, files: &EntryFiles, since: u64) { + // (no room on the disk: refused before the buffer is queued and compressed) if self.contains(Kind::Layer, key) || !self.accepts_writes() { return; } self.count_encode(); - let _ = self.enqueue_format(Kind::Layer, key, FORMAT_LAYER_FILES, with_files(files, encode_layer(buf)), Some((files, since))); + let _ = self.enqueue_format(Kind::Layer, key, FORMAT_LAYER_FILES, Payload::Layer(buf.clone(), files.clone()), Some((files, since))); } /// Write a layer buffer synchronously (tests); a list that can't be stored is not. @@ -1192,7 +1239,7 @@ fn writer(dc: std::sync::Weak, q: Arc<(Mutex, Condvar)>) { let Some(cache) = dc.upgrade() else { return }; let written = match job { Job::Write(kind, key, format, payload) => { - cache.write_format(kind, key, format, &payload); + cache.write_format(kind, key, format, &payload.encode()); (kind == Kind::Frame).then_some(key) } Job::Drop(paths) => { @@ -1292,7 +1339,7 @@ impl crate::cache::LayerStore for DiskCache { DiskCache::get_layer(self, key, virtual_id).map(|(b, f)| (Arc::new(b), f)) } /// Queues a layer write on the disk cache. - fn put_layer(&self, key: u128, buf: &Buf, files: &EntryFiles, since: u64) { + fn put_layer(&self, key: u128, buf: &Arc, files: &EntryFiles, since: u64) { DiskCache::put_layer(self, key, buf, files, since); } /// The current drop generation of the disk cache. @@ -1706,7 +1753,7 @@ mod tests { let dc = DiskCache::open(&dir, 1 << 30).unwrap(); assert!(dc.get_layer(42, None).is_none()); let b = buf(1, 33, 17); - dc.put_layer(42, &b, &EntryFiles::default(), 0); + dc.put_layer(42, &Arc::new(b.clone()), &EntryFiles::default(), 0); dc.flush(); let (back, files) = dc.get_layer(42, None).unwrap(); assert!(files.is_empty() && files.is_complete()); @@ -1715,10 +1762,14 @@ mod tests { let rgba: Vec = (0..20 * 10 * 4).map(|i| (i % 251) as u8).collect(); assert!(dc.put_frame(7, 20, 10, &rgba, &ReadFiles::default(), None, 0)); assert!(!dc.put_frame(8, 2, 2, &[0; 3], &ReadFiles::default(), None, 0), "a wrong-size frame is refused"); + // Handed over and compressed on the writer thread: the same pixels come back. + assert!(dc.put_frame_owned(9, 20, 10, rgba.clone(), &ReadFiles::default(), None, 0)); + assert!(!dc.put_frame_owned(10, 2, 2, vec![0; 3], &ReadFiles::default(), None, 0), "a wrong-size frame is refused"); dc.flush(); assert_eq!(dc.get_frame(7, None).map(|f| f.0).unwrap().rgba, rgba); + assert_eq!(dc.get_frame(9, None).map(|f| f.0).unwrap().rgba, rgba); let st = dc.stats(); - assert_eq!((st.entries, st.hits, st.misses, st.writes), (2, 2, 1, 2)); + assert_eq!((st.entries, st.hits, st.misses, st.writes), (3, 3, 1, 3)); // Wrong kind is a miss. assert!(dc.get_frame(42, None).map(|f| f.0).is_none()); } @@ -1780,7 +1831,7 @@ mod tests { let dir = tmpdir("burst"); let dc = DiskCache::open(&dir, 1 << 30).unwrap(); for k in 0..300 { - dc.put_layer(10_000 + k, &buf(1, 64, 64), &EntryFiles::default(), 0); + dc.put_layer(10_000 + k, &Arc::new(buf(1, 64, 64)), &EntryFiles::default(), 0); } let rgba = vec![7u8; 32 * 32 * 4]; for k in 0..300 { @@ -1950,7 +2001,7 @@ mod tests { let rgba = vec![7u8; 16 * 16 * 4]; assert!(!dc.put_frame(1, 16, 16, &rgba, &ReadFiles::default(), None, 0), "refused"); assert!(!dc.put_frame_f16(2, 16, 16, &[0x3c00u16; 16 * 16 * 4], &ReadFiles::default(), None, 0), "refused"); - dc.put_layer(3, &buf(3, 16, 16), &EntryFiles::default(), 0); + dc.put_layer(3, &Arc::new(buf(3, 16, 16)), &EntryFiles::default(), 0); dc.flush(); assert_eq!(dc.encodes(), 0, "nothing compressed"); assert_eq!(dc.stats().entries, 0); @@ -1960,7 +2011,7 @@ mod tests { assert_eq!(dc.stats().limit_bytes, 32 << 20); assert!(!dc.accepts_writes()); assert!(!dc.put_frame(1, 16, 16, &rgba, &ReadFiles::default(), None, 0), "refused"); - dc.put_layer(3, &buf(3, 16, 16), &EntryFiles::default(), 0); + dc.put_layer(3, &Arc::new(buf(3, 16, 16)), &EntryFiles::default(), 0); assert_eq!(dc.encodes(), 0, "nothing compressed"); // A small limit the setting asks for (not free space) is used. dc.set_disk_space(None); @@ -1971,7 +2022,7 @@ mod tests { dc.set_disk_space(Some((4096 * GB, 500 * GB))); assert!(dc.put_frame(1, 16, 16, &rgba, &ReadFiles::default(), None, 0)); assert!(dc.put_frame_f16(2, 16, 16, &[0x3c00u16; 16 * 16 * 4], &ReadFiles::default(), None, 0)); - dc.put_layer(3, &buf(3, 16, 16), &EntryFiles::default(), 0); + dc.put_layer(3, &Arc::new(buf(3, 16, 16)), &EntryFiles::default(), 0); dc.flush(); assert_eq!(dc.encodes(), 3); assert_eq!(dc.stats().entries, 3); @@ -2076,11 +2127,11 @@ mod tests { assert!(dc.get_frame(k, Some(&two)).is_none() && !dc.contains(Kind::Frame, k), "replaced: a miss, deleted"); // A layer buffer made of it. let lk = layer_disk_key(7); - dc.put_layer(lk, &buf(1, 4, 4), &EntryFiles::of(&files, Some(&one)).unwrap(), 0); + dc.put_layer(lk, &Arc::new(buf(1, 4, 4)), &EntryFiles::of(&files, Some(&one)).unwrap(), 0); dc.flush(); assert!(dc.get_layer(lk, Some(&one)).is_some_and(|(b, f)| b.img == buf(1, 4, 4).img && f == files && f.is_complete())); assert!(dc.get_layer(lk, None).is_none() && !dc.contains(Kind::Layer, lk), "no identity: a miss, deleted"); - dc.put_layer(lk, &buf(1, 4, 4), &EntryFiles::of(&files, Some(&one)).unwrap(), 0); + dc.put_layer(lk, &Arc::new(buf(1, 4, 4)), &EntryFiles::of(&files, Some(&one)).unwrap(), 0); dc.flush(); assert!(dc.get_layer(lk, Some(&two)).is_none() && !dc.contains(Kind::Layer, lk), "replaced: a miss, deleted"); // The browser's sealed layer entries carry the same list. @@ -2324,7 +2375,7 @@ mod tests { assert!(dc.dropped_since(&read(&[&a]), before) && !dc.dropped_since(&read(&[&b]), before)); assert!(!dc.put_frame(1, 1, 1, &[1; 4], &read(&[&a, &b]), None, before), "a frame from before the drop"); assert!(!dc.put_frame_f16(2, 1, 1, &[1; 4], &read(&[&a]), None, before)); - dc.put_layer(3, &buf(3, 2, 2), &EntryFiles::of(&read(&[&a]), None).unwrap(), before); + dc.put_layer(3, &Arc::new(buf(3, 2, 2)), &EntryFiles::of(&read(&[&a]), None).unwrap(), before); assert!(dc.put_frame(4, 1, 1, &[4; 4], &read(&[&b]), None, before), "b.png wasn't dropped"); dc.flush(); assert!(!dc.contains(Kind::Frame, 1) && !dc.contains(Kind::Frame, 2) && !dc.contains(Kind::Layer, 3)); @@ -2334,7 +2385,7 @@ mod tests { let after = dc.drop_generation(); assert!(after > before && !dc.dropped_since(&read(&[&a]), after)); assert!(dc.put_frame(5, 1, 1, &[5; 4], &read(&[&a]), None, after)); - dc.put_layer(6, &buf(6, 2, 2), &EntryFiles::of(&read(&[&a]), None).unwrap(), after); + dc.put_layer(6, &Arc::new(buf(6, 2, 2)), &EntryFiles::of(&read(&[&a]), None).unwrap(), after); dc.flush(); assert_eq!(dc.get_frame(5, None).map(|f| f.0.rgba), Some(vec![5; 4])); assert!(dc.get_layer(6, None).is_some()); diff --git a/crates/ui-egui/examples/cache_bench.rs b/crates/ui-egui/examples/cache_bench.rs index 828075249..7d57ea773 100644 --- a/crates/ui-egui/examples/cache_bench.rs +++ b/crates/ui-egui/examples/cache_bench.rs @@ -170,8 +170,10 @@ fn main() { println!("footage cache {} hits, {} misses, {} frames ({} MB) kept", p.hits, p.misses, p.frames, p.bytes >> 20); if let Some(dc) = &s.disk_cache { // (writes finish on the disk cache's own thread) + let t = Instant::now(); dc.flush(); + let flushed = t.elapsed().as_secs_f64(); let d = dc.stats(); - println!("disk cache {} written, {} dropped, {} entries ({} MB)", d.writes, d.dropped, d.entries, d.bytes >> 20); + println!("disk cache {} written, {} dropped, {} entries ({} MB), flushed in {flushed:.2} s", d.writes, d.dropped, d.entries, d.bytes >> 20); } } diff --git a/crates/ui-egui/src/frames.rs b/crates/ui-egui/src/frames.rs index 32f0dd161..800306385 100644 --- a/crates/ui-egui/src/frames.rs +++ b/crates/ui-egui/src/frames.rs @@ -571,10 +571,8 @@ fn put_disk_frame(dc: &DiskCache, dk: u128, img: &FrameImage, files: &ReadFiles, return false; } match img { - FrameImage::Cpu(img) => { - let rgba: Vec = img.pixels.iter().flat_map(|c| c.to_array()).collect(); - dc.put_frame(dk, img.size[0] as u32, img.size[1] as u32, &rgba, files, virtual_id, since) - } + // (one copy; the disk cache's writer thread compresses it) + FrameImage::Cpu(img) => dc.put_frame_owned(dk, img.size[0] as u32, img.size[1] as u32, img.as_raw().to_vec(), files, virtual_id, since), FrameImage::CpuF16(f) => dc.put_frame_f16(dk, f.width, f.height, f.bits(), files, virtual_id, since), FrameImage::Gpu(_) => false, } From 87bfaa8f1c3848d11f5d7ecdf723c6b3e96c38a6 Mon Sep 17 00:00:00 2001 From: nazar-gavrylyk Date: Fri, 6 Nov 2026 07:13:15 +0100 Subject: [PATCH 02/24] Preview: Cache Frames When Idle is on by default, as in After Effects A comp left alone for a second fills its RAM preview from the current time, so the first playback is cached instead of rendering frame by frame. Saved settings keep their value. Two UI tests that assumed the old default set it explicitly. --- crates/engine/src/settings/prefs/sections.rs | 2 +- crates/ui-egui/src/state.rs | 2 +- crates/ui-egui/tests/it/ui_comp_presets.rs | 2 ++ crates/ui-egui/tests/it/ui_preview_align.rs | 5 +++-- docs/preferences.md | 3 ++- 5 files changed, 9 insertions(+), 5 deletions(-) diff --git a/crates/engine/src/settings/prefs/sections.rs b/crates/engine/src/settings/prefs/sections.rs index 328f93bf4..9a9f4ab21 100644 --- a/crates/engine/src/settings/prefs/sections.rs +++ b/crates/engine/src/settings/prefs/sections.rs @@ -93,7 +93,7 @@ page!(Previews { /// Largest downsampling while playing back: `1/2`, `1/4`, `1/8`. adaptive_resolution_limit: String = "1/8".into(), show_internal_wireframes: bool = false, - cache_frames_when_idle: bool = false, + cache_frames_when_idle: bool = true, fast_previews: bool = false, /// `faster` or `moreAccurate`. zoom_quality: String = "moreAccurate".into(), diff --git a/crates/ui-egui/src/state.rs b/crates/ui-egui/src/state.rs index 612aa0a10..635b46acd 100644 --- a/crates/ui-egui/src/state.rs +++ b/crates/ui-egui/src/state.rs @@ -806,7 +806,7 @@ impl Default for UiState { home_learn: false, home_templates: false, align_to_selection: false, - cache_when_idle: false, + cache_when_idle: true, tracker_source: None, layer_panel: None, layer_view: None, diff --git a/crates/ui-egui/tests/it/ui_comp_presets.rs b/crates/ui-egui/tests/it/ui_comp_presets.rs index 3fda8a8cc..93cd02030 100644 --- a/crates/ui-egui/tests/it/ui_comp_presets.rs +++ b/crates/ui-egui/tests/it/ui_comp_presets.rs @@ -174,6 +174,8 @@ fn composition_settings_view_stays_put_when_it_scrolls() { #[test] fn composition_settings_refits_when_the_language_changes() { let mut s = Session::default(); + // Idle caching keeps repainting while it renders; this test waits for repaints to stop. + s.prefs.previews.cache_frames_when_idle = false; s.execute("comp.new", json!({"name": "Edit", "width": 2560, "height": 1440, "frameRate": 60, "duration": 4})).unwrap(); let mut h = Harness::builder() .with_size(egui::vec2(1600.0, 1000.0)) diff --git a/crates/ui-egui/tests/it/ui_preview_align.rs b/crates/ui-egui/tests/it/ui_preview_align.rs index 1469a5e9e..622df0397 100644 --- a/crates/ui-egui/tests/it/ui_preview_align.rs +++ b/crates/ui-egui/tests/it/ui_preview_align.rs @@ -238,8 +238,9 @@ fn cache_frames_when_idle_fills_the_work_area() { let mut h = harness(); let cid = h.state().session.active_comp_id().unwrap(); let cached = |h: &Harness<'_, EffectcraftApp>| h.state().frames.cached_frames(&h.state().shown_series(cid)).len(); - // Off: only a few frames ahead of the current time are prefetched (the work area is the - // whole 240 frames). + // Off (it is on by default): only a few frames ahead of the current time are prefetched (the + // work area is the whole 240 frames). + invoke(&mut h, "playback.cacheWhenIdle", json!({"value": false})); for _ in 0..120 { h.step(); std::thread::sleep(std::time::Duration::from_millis(2)); diff --git a/docs/preferences.md b/docs/preferences.md index 927b66788..de7e426bc 100644 --- a/docs/preferences.md +++ b/docs/preferences.md @@ -55,7 +55,8 @@ Commands (CLI, MCP, control channel): Adaptive Resolution drops to while you drag; a playing preview keeps the viewer's resolution) - `previews.cacheFramesWhenIdle`: Cache Frames When Idle (Composition ▸ Preview): after a second without input or edits, the viewer renders the work area into the RAM preview in the - background, from the current time on, until it is cached or the budget is full + background, from the current time on, until it is cached or the budget is full. On by default, + as in After Effects, so the first playback of a comp left alone for a second is already cached - `previews.fastPreviews`: Fast Previews (Draft 3D, Faster Effects) - `appearance.appearanceMode`: Appearance Mode (`auto` / `dark` / `light`). `auto` (Sync with System) follows the operating system's light or dark appearance while the app runs: on Linux the desktop app reads the XDG desktop portal (`org.freedesktop.appearance` `color-scheme`) once and then listens for its `SettingChanged` signal on a worker thread, repainting only when the value changes (nothing polls); `gsettings` runs at most once at start-up, by absolute path, when the portal gives no answer. Elsewhere it uses what the window system reports. With no answer it shows the dark theme. New installs stay `dark`; Sync with System is opt-in. The Tools bar's appearance button (a monitor, sun or moon) cycles Sync with System, Light and Dark (`view.theme.toggle`, Next Appearance Mode). - `appearance.darkTheme`: Dark Theme (`dark` / `darker`), shown in dark appearance From 42989edd8eb1adda172ad4d2b082801f6e18f982 Mon Sep 17 00:00:00 2001 From: nazar-gavrylyk Date: Fri, 6 Nov 2026 07:13:15 +0100 Subject: [PATCH 03/24] Gaussian Blur: the exact kernel for small blurs runs row by row MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The sampled Gaussian (sigma < 3) read every tap through a bounds-checked get, and its vertical pass walked a column per pixel. Taps inside a row are now one slice, and the vertical pass adds whole input rows in 128-pixel tiles. Same f64 sums in the same order: bit-identical output (tested against the per-pixel kernel). Each pass is 15–25% faster. --- crates/raster/src/blur.rs | 117 ++++++++++++++++++++++++++++++++++---- 1 file changed, 106 insertions(+), 11 deletions(-) diff --git a/crates/raster/src/blur.rs b/crates/raster/src/blur.rs index c8689a7b6..9b785959f 100644 --- a/crates/raster/src/blur.rs +++ b/crates/raster/src/blur.rs @@ -166,6 +166,9 @@ pub fn gaussian_is_exact(sigma_x: f64, sigma_y: f64) -> bool { sigma_x.max(sigma_y) < EXACT_SIGMA_BELOW } +/// Pixels per tile of [`gauss_pass`]'s vertical pass. +const TILE: usize = 128; + /// One separable pass of a sampled Gaussian (radius 3 sigma) along x or y. fn gauss_pass(img: &Image, sigma: f64, vertical: bool, repeat: bool) -> Image { // Below 1e-3 the kernel is the identity in f64; tiny sigmas would also make 2σ² underflow to 0 (NaN taps). @@ -176,20 +179,65 @@ fn gauss_pass(img: &Image, sigma: f64, vertical: bool, repeat: bool) -> Image { let mut k: Vec = (-r..=r).map(|i| (-((i * i) as f64) / (2.0 * sigma * sigma)).exp()).collect(); let total: f64 = k.iter().sum(); k.iter_mut().for_each(|v| *v /= total); - let w = img.width as usize; + let (w, h) = (img.width as usize, img.height as usize); let mut out = Image::new(img.width, img.height); - out.rows_mut().for_each(|(y, row)| { - for x in 0..w { - let mut acc = [0.0f64; 4]; - for (i, wt) in k.iter().enumerate() { - let d = i as i64 - r; - let (sx, sy) = if vertical { (x as i64, y as i64 + d) } else { (x as i64 + d, y as i64) }; - let p = if repeat { img.get_clamped(sx, sy) } else { img.get(sx, sy) }; - for c in 0..4 { - acc[c] += p[c] as f64 * wt; + // Every output pixel adds its taps in kernel order, in f64, as a per-pixel loop would: the + // result is the same to the bit. Taps outside a transparent edge are skipped (adding `+0.0` + // leaves a sum that started at `+0.0` unchanged). + if vertical { + // Row by row: each output row adds whole input rows (contiguous, vectorises), instead of + // walking a column per pixel (a strided read per tap). + out.data.par_chunks_mut(w).enumerate().for_each(|(y, row)| { + // 128 pixels at a time: the f64 sums (4 KB) stay in L1 across the taps. + for (t, out_tile) in row.chunks_mut(TILE).enumerate() { + let x0 = t * TILE; + let mut acc = [[0.0f64; 4]; TILE]; + for (i, &wt) in k.iter().enumerate() { + let sy = y as i64 + i as i64 - r; + let sy = if repeat { + sy.clamp(0, h as i64 - 1) + } else if sy < 0 || sy >= h as i64 { + continue; + } else { + sy + }; + let start = sy as usize * w + x0; + let Some(src) = img.data.get(start..start + out_tile.len()) else { continue }; + for (a, p) in acc.iter_mut().zip(src) { + for c in 0..4 { + a[c] += p[c] as f64 * wt; + } + } + } + for (o, a) in out_tile.iter_mut().zip(acc.iter()) { + *o = a.map(|v| v as f32); } } - row[x] = acc.map(|v| v as f32); + }); + return out; + } + let n = k.len(); + out.data.par_chunks_mut(w).zip(img.data.par_chunks(w)).for_each(|(row, src)| { + for (x, o) in row.iter_mut().enumerate() { + let mut acc = [0.0f64; 4]; + // Inside the row: the taps are one slice (no per-tap edge checks). + if let Some(taps) = x.checked_sub(r as usize).and_then(|x0| src.get(x0..x0 + n)) { + for (p, wt) in taps.iter().zip(&k) { + for c in 0..4 { + acc[c] += p[c] as f64 * wt; + } + } + } else { + for (i, wt) in k.iter().enumerate() { + let sx = x as i64 + i as i64 - r; + let at = if repeat { Some(sx.clamp(0, w as i64 - 1)) } else { Some(sx) }; + let Some(p) = at.and_then(|i| usize::try_from(i).ok()).and_then(|i| src.get(i)) else { continue }; + for c in 0..4 { + acc[c] += p[c] as f64 * wt; + } + } + } + *o = acc.map(|v| v as f32); } }); out @@ -264,6 +312,53 @@ pub fn radial_blur(img: &Image, center: (f64, f64), amount: f64, zoom: bool) -> mod tests { use super::*; + /// The sampled Gaussian pass as it was (one pixel at a time, bounds-checked taps): the + /// row-wise passes must match it to the bit. + fn gauss_pass_per_pixel(img: &Image, sigma: f64, vertical: bool, repeat: bool) -> Image { + let r = (sigma * 3.0).ceil() as i64; + let mut k: Vec = (-r..=r).map(|i| (-((i * i) as f64) / (2.0 * sigma * sigma)).exp()).collect(); + let total: f64 = k.iter().sum(); + k.iter_mut().for_each(|v| *v /= total); + let mut out = Image::new(img.width, img.height); + for y in 0..img.height as i64 { + for x in 0..img.width as i64 { + let mut acc = [0.0f64; 4]; + for (i, wt) in k.iter().enumerate() { + let d = i as i64 - r; + let (sx, sy) = if vertical { (x, y + d) } else { (x + d, y) }; + let p = if repeat { img.get_clamped(sx, sy) } else { img.get(sx, sy) }; + for c in 0..4 { + acc[c] += p[c] as f64 * wt; + } + } + out.set(x as u32, y as u32, acc.map(|v| v as f32)); + } + } + out + } + + #[test] + fn gauss_pass_is_bit_identical_to_the_per_pixel_kernel() { + // Sizes narrower and shorter than the kernel too (every tap near an edge), with + // negative, zero and HDR values. + for (w, h) in [(37, 23), (3, 2), (1, 9), (64, 1)] { + let mut img = Image::new(w, h); + for (i, p) in img.data.iter_mut().enumerate() { + let v = ((i * 7919) % 101) as f32 / 37.0 - 0.6; + *p = [v, -v * 0.5, if i % 5 == 0 { 0.0 } else { v * 3.0 }, (v + 0.6).min(1.0)]; + } + for sigma in [0.4, 1.3, 2.9] { + for vertical in [false, true] { + for repeat in [false, true] { + let (a, b) = (gauss_pass(&img, sigma, vertical, repeat), gauss_pass_per_pixel(&img, sigma, vertical, repeat)); + let bits = |i: &Image| i.data.iter().flat_map(|p| p.map(f32::to_bits)).collect::>(); + assert!(bits(&a) == bits(&b), "{w}×{h} sigma {sigma} vertical {vertical} repeat {repeat}"); + } + } + } + } + } + #[test] fn gaussian_preserves_mass() { let mut img = Image::new(64, 64); From b3dc49a0cf034d46042e21aa9b4fc5878a8e20bb Mon Sep 17 00:00:00 2001 From: nazar-gavrylyk Date: Fri, 6 Nov 2026 07:13:15 +0100 Subject: [PATCH 04/24] Render: plain footage is copied, converted and quantised in one pass A footage layer's frame was copied from the footage cache, converted from its colour profile, then quantised to the project depth in masked_source: three passes over a 33 MB 1080p float frame. Footage without masks now comes quantised from one fused pass (Pipe::copy_converted); other source() callers still get unquantised pixels. Bit-identical (tested at 8/16/32 bpc). --- crates/render/src/color.rs | 69 +++++++++++++++++++++++++++++++---- crates/render/src/pipeline.rs | 64 ++++++++++++++++++++++++-------- 2 files changed, 110 insertions(+), 23 deletions(-) diff --git a/crates/render/src/color.rs b/crates/render/src/color.rs index c423665d2..158dde42f 100644 --- a/crates/render/src/color.rs +++ b/crates/render/src/color.rs @@ -153,6 +153,26 @@ impl Pipe { p } + /// A copy of `src` converted by `c` and quantised to the bit depth, in one pass: one read and + /// one write of the frame instead of a copy, a conversion and a quantise pass (a 1080p float + /// frame is 33 MB, and these passes are limited by memory speed). The same pixels as + /// [`convert`] then [`Pipe::quantize`]. + pub fn copy_converted(&self, src: &Image, c: Option<&Conversion>) -> Image { + let levels = self.levels; + let data = src + .data + .par_iter() + .map(|&p| { + let mut p = c.map_or(p, |c| convert_px(p, c)); + if let Some(l) = levels { + quantize_px(&mut p, l, 1.0 / l); + } + p + }) + .collect(); + Image { width: src.width, height: src.height, data } + } + /// Clamp and quantise to the bit depth (no-op in 32 bpc). pub fn quantize(&self, img: &mut Image) { self.quantize_region(img, Region::Full); @@ -265,14 +285,18 @@ fn quantize_rect(img: &mut Image, r: PxRect, levels: f32) { /// Convert premultiplied pixels (on straight colour). pub fn convert(img: &mut Image, c: &Conversion) { - img.data.par_iter_mut().for_each(|p| { - let a = p[3]; - if a <= 0.0 { - return; - } - let o = c.apply([p[0] / a, p[1] / a, p[2] / a]); - *p = [o[0] * a, o[1] * a, o[2] * a, a]; - }); + img.data.par_iter_mut().for_each(|p| *p = convert_px(*p, c)); +} + +/// [`convert`] of one pixel. +#[inline] +fn convert_px(p: [f32; 4], c: &Conversion) -> [f32; 4] { + let a = p[3]; + if a <= 0.0 { + return p; + } + let o = c.apply([p[0] / a, p[1] / a, p[2] / a]); + [o[0] * a, o[1] * a, o[2] * a, a] } /// Compand: an exponential knee from 80%, approaching 1.0 (linear light, per channel). @@ -349,6 +373,35 @@ mod tests { } } + /// The fused footage pass gives the pixels of a copy, a conversion and a quantise pass, bit + /// for bit, at every depth, with and without a conversion. + #[test] + fn copy_converted_is_convert_then_quantize() { + let mut img = Image::new(17, 5); + for (i, p) in img.data.iter_mut().enumerate() { + *p = match i % 7 { + 0 => [0.3, -0.2, 0.71, 0.9], + 1 => [-0.0, 0.0, 0.0, 0.0], + 2 => [1.7, 0.4, f32::NAN, 0.5], + 3 => [0.0; 4], + _ => [i as f32 / 90.0, 0.5, 0.25, 1.0], + }; + } + for depth in [effectcraft_project::BitDepth::Bpc8, effectcraft_project::BitDepth::Bpc16, effectcraft_project::BitDepth::Bpc32] { + let pipe = Pipe::of(&ProjectSettings { bit_depth: depth, ..Default::default() }); + for c in [None, Some(Conversion::linearize(ColorSpace::Srgb))] { + let mut want = img.clone(); + if let Some(c) = &c { + convert(&mut want, c); + } + pipe.quantize(&mut want); + let got = pipe.copy_converted(&img, c.as_ref()); + let bits = |i: &Image| i.data.iter().map(|p| p.map(f32::to_bits)).collect::>(); + assert_eq!(bits(&got), bits(&want), "{depth:?} conversion {}", c.is_some()); + } + } + } + /// Both forms `round_half_up` picks between agree on everything quantising feeds it: /// 0..=levels, exact halves included. #[test] diff --git a/crates/render/src/pipeline.rs b/crates/render/src/pipeline.rs index 6265cb5f7..6979afc85 100644 --- a/crates/render/src/pipeline.rs +++ b/crates/render/src/pipeline.rs @@ -878,6 +878,12 @@ impl<'a> Renderer<'a> { /// Source pixels of a layer in layer space (before masks/effects). pub(crate) fn source(&self, ctx: &EvalCtx, layer: &Layer) -> Option { + self.source_with(ctx, layer, false) + } + + /// [`Renderer::source`]; with `quantized`, footage that [`Renderer::prequantized_footage`] + /// counts comes quantised to the project depth. + fn source_with(&self, ctx: &EvalCtx, layer: &Layer, quantized: bool) -> Option { let s = self.opts.scale; match &layer.source { LayerSource::Solid { item } => { @@ -910,7 +916,7 @@ impl<'a> Renderer<'a> { && let Some(nc) = self.project.comp(*item) { let img = self.footage_frame(ctx, layer, *item, pf, lt, None)?; - return Some(self.footage_buf(&img, pf, nc.width, nc.height)); + return Some(self.footage_buf(&img, pf, nc.width, nc.height, false)); } // Essential Properties overrides render the nested comp with this instance's // values. @@ -959,7 +965,7 @@ impl<'a> Renderer<'a> { } } let (img, pf) = self.footage_picture_of(ctx, layer, item, f)?; - Some(self.footage_buf(&img, pf, f.width, f.height)) + Some(self.footage_buf(&img, pf, f.width, f.height, quantized && Self::prequantized_footage(layer))) } // (the font files go to this render's log, which `ctx` may not carry: see // [`Renderer::with_reads`]) @@ -992,15 +998,42 @@ impl<'a> Renderer<'a> { /// A decoded frame of `f` as a layer buffer covering `w`×`h` layer pixels (the footage's /// nominal size; proxies are scaled up to it), converted from its colour profile. - fn footage_buf(&self, img: &Image, f: &Footage, w: u32, h: u32) -> Buf { - let mut buf = match self.footage_layout(img, w, h) { - FootageLayout::Native { scale } => Buf { img: img.clone(), offset: [0.0; 2], scale }, - FootageLayout::Resample { width, height, scale } => Buf { img: effectcraft_raster::resample(img, width, height), offset: [0.0; 2], scale }, - }; - if let Some(c) = self.pipe.media_in(f) { - color::convert(&mut buf.img, &c); + /// + /// `quantized`: also clamp and quantise it to the project depth, as [`Renderer::masked_source`] + /// would (fused with the copy of the cached frame: one pass over it instead of three). + fn footage_buf(&self, img: &Image, f: &Footage, w: u32, h: u32, quantized: bool) -> Buf { + let c = self.pipe.media_in(f); + match self.footage_layout(img, w, h) { + FootageLayout::Native { scale } => { + let img = match (quantized, &c) { + (true, _) => self.pipe.copy_converted(img, c.as_ref()), + (false, Some(c)) => { + let mut img = img.clone(); + color::convert(&mut img, c); + img + } + (false, None) => img.clone(), + }; + Buf { img, offset: [0.0; 2], scale } + } + FootageLayout::Resample { width, height, scale } => { + let mut img = effectcraft_raster::resample(img, width, height); + if let Some(c) = &c { + color::convert(&mut img, c); + } + if quantized { + self.pipe.quantize(&mut img); + } + Buf { img, offset: [0.0; 2], scale } + } } - buf + } + + /// The layer's source is footage that [`Renderer::source`] hands over already quantised + /// ([`Renderer::footage_buf`]): footage without masks (masks apply to unquantised pixels). + /// Collapsed layers may be vector footage, drawn elsewhere: not counted. + fn prequantized_footage(layer: &Layer) -> bool { + matches!(layer.source, LayerSource::Footage { .. }) && !layer.switches.collapse && layer.masks().is_none_or(|m| m.groups().next().is_none()) } /// How [`Renderer::footage_buf`] lays out a decoded frame `img` of footage `w`×`h` layer @@ -1159,13 +1192,14 @@ impl<'a> Renderer<'a> { /// Source → masks, clamped/quantised to the project depth. fn masked_source(&self, ctx: &EvalCtx, layer: &Layer) -> Option { - let mut buf = match self.content_times(ctx, layer) { - Some(times) => self.blurred_source(ctx, layer, ×)?, - None => self.source(ctx, layer)?, + let (mut buf, quantized) = match self.content_times(ctx, layer) { + Some(times) => (self.blurred_source(ctx, layer, ×)?, false), + None => (self.source_with(ctx, layer, true)?, Self::prequantized_footage(layer)), }; - // A solid is filled with a quantised colour: only masks can take it off the grid. + // A solid is filled with a quantised colour, and plain footage comes quantised: only + // masks can take them off the grid. let solid = matches!(layer.source, LayerSource::Solid { .. }); - if masks::apply(ctx, layer, &mut buf, self.mask_blur(ctx, layer)) || !solid { + if masks::apply(ctx, layer, &mut buf, self.mask_blur(ctx, layer)) || !(solid || quantized) { self.pipe.quantize(&mut buf.img); } Some(buf) From 6bb52b1cb1fec4ed661650a0d46c95d11e0165cc Mon Sep 17 00:00:00 2001 From: nazar-gavrylyk Date: Fri, 6 Nov 2026 07:32:19 +0100 Subject: [PATCH 05/24] cache_bench: --gpu measures the GPU compositor --gpu renders on a headless GPU of its own and reads the frames back, as the desktop viewer does; --backend cpu|gpu|auto picks the compositor (default auto, the viewer's Mercury GPU Acceleration). With auto the bench prints Auto's per-comp timing history: how many frames each side rendered and how long they took. --- crates/ui-egui/examples/cache_bench.rs | 35 +++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/crates/ui-egui/examples/cache_bench.rs b/crates/ui-egui/examples/cache_bench.rs index 7d57ea773..9367d9e5c 100644 --- a/crates/ui-egui/examples/cache_bench.rs +++ b/crates/ui-egui/examples/cache_bench.rs @@ -12,7 +12,10 @@ //! default 0.5: a viewer at half size), `--frames N` (frames to cache, default the whole comp), //! `--slots N` (frames in flight, at least 1, default one per frame thread as the app does when //! idle), -//! `--disk` (with a fresh disk cache in `target/cache-bench/disk`), `--no-adjustment`. +//! `--disk` (with a fresh disk cache in `target/cache-bench/disk`), `--no-adjustment`, +//! `--gpu` (composite on a GPU of its own, as the desktop app does with Mercury GPU Acceleration: +//! the GPU or the CPU, whichever Auto measures faster per comp, read back to the CPU; the viewer's +//! backend), `--backend cpu|gpu|auto` (with `--gpu`: force one compositor, default `auto`). use std::sync::Arc; use std::time::{Duration, Instant}; @@ -122,13 +125,25 @@ fn main() { let slots: usize = arg("--slots").and_then(|v| v.parse().ok()).map(|n: usize| n.max(1)).unwrap_or(frames.parallelism().max(1)); // Room for every frame: this measures speed, not eviction. frames.set_budget(64 << 30); - let opts = RenderOpts { scale, motion_blur: true, ..Default::default() }; + use effectcraft_engine::render::Backend; + let backend = match arg("--backend").as_deref() { + Some("cpu") => Backend::Cpu, + Some("gpu") => Backend::Gpu, + _ => Backend::Auto, + }; + let opts = RenderOpts { scale, motion_blur: true, backend, ..Default::default() }; + let gpu = std::env::args().any(|a| a == "--gpu").then(|| { + effectcraft_gpu::Gpu::headless().unwrap_or_else(|| { + eprintln!("--gpu: no usable GPU adapter"); + std::process::exit(2) + }) + }); let src = RenderSource { project: s.project.clone(), footage: s.footage.clone(), expr: s.expr.clone(), layer_cache: s.layer_cache.clone(), - gpu: None, + gpu, gpu_display: false, disk: s.disk_cache.clone(), }; @@ -142,7 +157,12 @@ fn main() { opts: opts_hash(&opts), }; - eprintln!("caching {n} frames of {}×{} at scale {scale} with {slots} in flight…", comp.width, comp.height); + eprintln!( + "caching {n} frames of {}×{} at scale {scale} with {slots} in flight{}…", + comp.width, + comp.height, + if src.gpu.is_some() { " on the GPU" } else { "" } + ); let (gop0, t0) = (effectcraft_media::gop_stats(), Instant::now()); let mut next = 0; loop { @@ -166,6 +186,13 @@ fn main() { gop.seeks - gop0.seeks, gop.hits - gop0.hits ); + if let Some(g) = &src.gpu + && let Some(p) = effectcraft_engine::render::Accelerator::auto_pick(g) + && let Some(a) = p.stats(effectcraft_engine::render::AutoKey::new(cid, scale, false)) + { + let ms = |v: Option| v.map_or("–".into(), |v| format!("{v:.1} ms")); + println!("auto (Mercury) GPU {} over {} frames, CPU {} over {} frames", ms(a.gpu_ms), a.gpu_frames, ms(a.cpu_ms), a.cpu_frames); + } let p = pool.stats(); println!("footage cache {} hits, {} misses, {} frames ({} MB) kept", p.hits, p.misses, p.frames, p.bytes >> 20); if let Some(dc) = &s.disk_cache { From 50a889737ef72bb77bc6d5682c32de2445d2577d Mon Sep 17 00:00:00 2001 From: nazar-gavrylyk Date: Fri, 6 Nov 2026 08:04:53 +0100 Subject: [PATCH 06/24] Auto: frames rendered side by side no longer all warm up the GPU MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A preview renders up to 12 frames at once, and Auto only learns a frame's time when it is done. A starting preview sent every lane to the GPU for its warm-up, and on an integrated GPU 2–4× slower than the CPU each first playback stalled. Now at most one GPU warm-up frame (and one re-probe frame per side) runs at once, a frame's time counts per frame running on its side at once, and the warm-up starts on the CPU, so the frames on screen come first. Iris Xe, 1080p: Auto went from 13–34% slower than CPU only to 0–17%. --- crates/render/src/auto.rs | 92 ++++++++++++++++++++++++++++----- crates/render/src/tests/auto.rs | 68 ++++++++++++++++++++++-- 2 files changed, 143 insertions(+), 17 deletions(-) diff --git a/crates/render/src/auto.rs b/crates/render/src/auto.rs index 39b3b02e6..b1138588b 100644 --- a/crates/render/src/auto.rs +++ b/crates/render/src/auto.rs @@ -10,6 +10,13 @@ //! (its first frame, with layer cache misses and texture uploads, is not counted; the fastest of //! the next ones starts the average), and the slower side is re-measured every //! [`AutoPick::REPROBE`] frames so the choice follows the comp as it changes. +//! +//! Frames render several at a time (a preview's lanes), and a frame is only measured once it +//! is done: so at most one GPU warm-up frame, or re-probe frame on either side, runs at once, +//! and the frames asked for meanwhile go to the other side. And a frame's time counts per +//! frame running on its side at once: twelve CPU frames that each take 200 ms side by side cost +//! about 17 ms a frame, against a GPU frame of 175 ms on its own. Otherwise every lane of a starting preview took a +//! warm-up frame on a GPU that turned out 2–4× slower (an integrated GPU), all at once. use std::collections::HashMap; use std::sync::Mutex; @@ -51,7 +58,37 @@ pub struct AutoStats { /// accelerator, see [`Accelerator::auto_pick`](crate::Accelerator::auto_pick)). #[derive(Default)] pub struct AutoPick { - stats: Mutex>, + stats: Mutex>, +} + +/// One key's history, when its running warm-up or re-probe frame started on each side, and +/// how many of its frames are running on each side (`[CPU, GPU]`). +#[derive(Default)] +struct Entry { + stats: AutoStats, + measuring: [Option; 2], + // ponytail: a frame that never reports back (a GPU failure path, a replaced remote frame) + // leaves its side counted as running, which only makes that side look cheaper; capped at + // MAX_RUNNING, cleared by `AutoPick::reset`. + running: [u32; 2], +} + +impl Entry { + /// A warm-up or re-probe frame is running on `gpu`'s side. It counts as done once + /// [`AutoPick::MEASURING_FOR`] has passed (a frame that failed without reporting back). + fn measuring(&self, gpu: bool) -> bool { + self.measuring[usize::from(gpu)].is_some_and(|t| t.elapsed() < AutoPick::MEASURING_FOR) + } + + /// Render the next warm-up or re-probe frame on `gpu`'s side, unless one is running there: + /// then the other side's (`!gpu`). + fn measure(&mut self, gpu: bool) -> bool { + if self.measuring(gpu) { + return !gpu; + } + self.measuring[usize::from(gpu)] = Some(web_time::Instant::now()); + gpu + } } impl AutoPick { @@ -66,19 +103,35 @@ impl AutoPick { pub const MARGIN: f64 = 0.85; /// Keys remembered before the history is reset. const MAX_KEYS: usize = 1024; + /// Frames counted as running on one side at most (see `Entry::running`). + const MAX_RUNNING: u32 = 256; + /// A warm-up or re-probe frame not reported back by then no longer holds its side. + const MEASURING_FOR: std::time::Duration = std::time::Duration::from_secs(10); - /// Render the frame for `key` on the GPU? + /// Render the frame for `key` on the GPU? Every frame asked for reports back with + /// [`AutoPick::record`] (or [`AutoPick::declined`]). pub fn choose(&self, key: AutoKey) -> bool { let Ok(mut m) = self.stats.lock() else { return true }; if m.len() > Self::MAX_KEYS && !m.contains_key(&key) { m.clear(); } - let s = m.entry(key).or_default(); - // Warm up both sides, alternating, GPU first. + let e = m.entry(key).or_default(); + let gpu = Self::pick(e); + let n = &mut e.running[usize::from(gpu)]; + *n = n.saturating_add(1).min(Self::MAX_RUNNING); + gpu + } + + fn pick(e: &mut Entry) -> bool { + let s = e.stats; + // Warm up both sides, alternating, CPU first: the first frames asked for are the + // ones on screen, and the CPU is the side that is never slow to start (no pipelines + // to build; an integrated GPU took 2 s for the frame at the playhead). let gpu_warm = s.declined || s.gpu_frames >= Self::WARMUP; - if !gpu_warm && s.gpu_frames <= s.cpu_frames { - return true; + if !gpu_warm && s.gpu_frames < s.cpu_frames { + return e.measure(true); } + // (CPU frames already render side by side, on lanes of their own.) if s.cpu_frames < Self::WARMUP { return false; } @@ -86,12 +139,12 @@ impl AutoPick { (Some(g), Some(c)) => c >= g * Self::MARGIN, (g, _) => g.is_some(), }; - // Periodically re-measure the slower side. - if s.since_probe >= Self::REPROBE { - s.since_probe = 0; - return !gpu_faster; + // Periodically re-measure the slower side (one frame at a time). + if s.since_probe >= Self::REPROBE && !e.measuring(!gpu_faster) { + e.stats.since_probe = 0; + return e.measure(!gpu_faster); } - s.since_probe += 1; + e.stats.since_probe = s.since_probe.saturating_add(1); gpu_faster } @@ -103,7 +156,13 @@ impl AutoPick { /// period). pub fn record(&self, key: AutoKey, gpu: bool, ms: f64) { let Ok(mut m) = self.stats.lock() else { return }; - let s = m.entry(key).or_default(); + let e = m.entry(key).or_default(); + e.measuring[usize::from(gpu)] = None; + // Per frame running on this side alongside it (itself included). + let n = &mut e.running[usize::from(gpu)]; + let ms = ms / f64::from((*n).max(1)); + *n = n.saturating_sub(1); + let s = &mut e.stats; let (avg, frames) = if gpu { (&mut s.gpu_ms, &mut s.gpu_frames) } else { (&mut s.cpu_ms, &mut s.cpu_frames) }; *frames = frames.saturating_add(1); if gpu { @@ -123,7 +182,12 @@ impl AutoPick { /// The GPU did not handle a frame for `key` (it rendered on the CPU instead). pub fn declined(&self, key: AutoKey) { if let Ok(mut m) = self.stats.lock() { - let s = m.entry(key).or_default(); + let e = m.entry(key).or_default(); + e.measuring[1] = None; + // The frame renders on the CPU instead (and reports back from there). + e.running[1] = e.running[1].saturating_sub(1); + e.running[0] = e.running[0].saturating_add(1).min(Self::MAX_RUNNING); + let s = &mut e.stats; s.declined = true; s.gpu_ms = None; s.since_probe = 0; @@ -132,7 +196,7 @@ impl AutoPick { /// The history of `key`. pub fn stats(&self, key: AutoKey) -> Option { - self.stats.lock().ok()?.get(&key).copied() + self.stats.lock().ok()?.get(&key).map(|e| e.stats) } /// Forget all history (settings that change both paths' costs). diff --git a/crates/render/src/tests/auto.rs b/crates/render/src/tests/auto.rs index 5cc503173..e7387eed3 100644 --- a/crates/render/src/tests/auto.rs +++ b/crates/render/src/tests/auto.rs @@ -229,6 +229,12 @@ fn healthy_gpu_decline_renders_cpu_once_outside_scope() { let src = ScopedFootage { in_scope: false.into(), reads: 0.into() }; let mut r = Renderer::new(&p, &src, RenderOpts { backend: Backend::Auto, ..Default::default() }); r.accel = Some(&a); + // Auto warms up CPU first: the first frame renders on the CPU without a GPU scope. + r.comp_frame_scoped(cid, Tick::ZERO, |_| -> Result, ()> { panic!("Auto's CPU warm-up frame must not enter GPU error scopes") }) + .unwrap(); + assert_eq!(src.reads.load(Ordering::SeqCst), 1); + assert_eq!(a.calls(), 0); + // The next one is the GPU's warm-up frame, which the accelerator declines. let frame = r .comp_frame_scoped(cid, Tick::ZERO, |a| { src.in_scope.store(true, Ordering::SeqCst); @@ -238,11 +244,67 @@ fn healthy_gpu_decline_renders_cpu_once_outside_scope() { }) .unwrap(); assert!(frame.get(100, 50)[3] > 0.99); - assert_eq!(src.reads.load(Ordering::SeqCst), 1); + assert_eq!(src.reads.load(Ordering::SeqCst), 2); assert_eq!(a.calls(), 1); - assert_eq!(a.pick.stats(AutoKey::new(cid, 1.0, false)).unwrap().cpu_frames, 1); + assert_eq!(a.pick.stats(AutoKey::new(cid, 1.0, false)).unwrap().cpu_frames, 2); // Auto's next frame stays on CPU, without entering a GPU scope at all. r.comp_frame_scoped(cid, Tick::ZERO, |_| -> Result, ()> { panic!("Auto CPU choice must not enter GPU error scopes") }).unwrap(); - assert_eq!(src.reads.load(Ordering::SeqCst), 2); + assert_eq!(src.reads.load(Ordering::SeqCst), 3); assert_eq!(a.effect_calls.load(Ordering::SeqCst), 0, "CPU effects must not use the GPU outside its scope"); } + +/// A preview starts with every lane asking at once, before any frame is measured: the first +/// frames (the ones on screen) render on the CPU; once one is measured, one lane takes a GPU +/// warm-up frame, and the next waits for its time. (Every lane took a GPU warm-up frame on an +/// integrated GPU 2–4× slower than the CPU, and each first playback stalled at its start.) +#[test] +fn auto_pick_warms_the_gpu_up_one_frame_at_a_time() { + let pick = AutoPick::default(); + let key = AutoKey::new(ItemId(3), 1.0, false); + let first: Vec = (0..12).map(|_| pick.choose(key)).collect(); + assert!(!first.iter().any(|g| *g), "{first:?}"); + pick.record(key, false, 10.0); + let next: Vec = (0..12).map(|_| pick.choose(key)).collect(); + assert_eq!(next.iter().filter(|g| **g).count(), 1, "{next:?}"); + // The CPU frames finish first: still no second GPU frame while the first runs. + for _ in 0..22 { + pick.record(key, false, 10.0); + } + assert!(!(0..12).any(|_| pick.choose(key)), "a second GPU warm-up frame while one runs"); + // Measured: the next warm-up frame may go to the GPU. + pick.record(key, true, 40.0); + assert!(pick.choose(key)); + // A re-probe of the slow side runs one frame at a time too. + let pick = AutoPick::default(); + let mut gpu_asked = 0; + for _ in 0..(4 * AutoPick::REPROBE) { + // (sequential warm-up, then GPU frames held: as lanes that never report back) + if pick.choose(key) { + gpu_asked += 1; + if pick.stats(key).is_none_or(|s| s.gpu_frames < AutoPick::WARMUP) { + pick.record(key, true, 40.0); + } + } else { + pick.record(key, false, 10.0); + } + } + assert!(gpu_asked <= AutoPick::WARMUP as usize + 1, "asked the GPU {gpu_asked} times"); +} + +/// Frames render side by side: twelve CPU frames of 210 ms each at once cost ~18 ms a frame, +/// so a lone GPU frame of 175 ms is the slower side (it measured faster and took the comp). +#[test] +fn auto_pick_counts_frames_running_side_by_side() { + let pick = AutoPick::default(); + let key = AutoKey::new(ItemId(4), 0.5, false); + // Twelve lanes: each starts its next frame as soon as one is done. + let mut lanes: std::collections::VecDeque = (0..12).map(|_| pick.choose(key)).collect(); + for _ in 0..240 { + let Some(gpu) = lanes.pop_front() else { break }; + pick.record(key, gpu, if gpu { 175.0 } else { 210.0 }); + lanes.push_back(pick.choose(key)); + } + let s = pick.stats(key).unwrap(); + assert!(s.cpu_ms.unwrap() < 30.0 && s.gpu_ms.unwrap() > 100.0, "{s:?}"); + assert!(lanes.iter().filter(|g| **g).count() <= 1, "{lanes:?}"); +} From 76e4008151759606b942f68cf3c19c96dd1e2498 Mon Sep 17 00:00:00 2001 From: nazar-gavrylyk Date: Fri, 6 Nov 2026 08:43:57 +0100 Subject: [PATCH 07/24] Footage: 8 bpc renders keep plain movie frames as 8-bit levels An 8 bpc render at Full quantises plain footage straight away, so the media pool now keeps those movie frames as the 8-bit levels it is quantised to (a quarter of the memory) and the renderer expands them into the layer buffer: the same pixels to the bit. Other depths, reduced renders, masks, colour profiles, fields, frame blending, Cineon, Invert Alpha, linear light and 10/12-bit or float movies keep the float path. First playback of plain 1080p clips at Full caches ~43% faster on the CPU, and the footage cache holds 4x the frames. --- crates/automation/src/confine.rs | 4 + crates/media/src/convert.rs | 103 ++++++++++++++++--- crates/media/src/pool.rs | 130 +++++++++++++++++++----- crates/raster/src/lib.rs | 46 ++++++++- crates/render/src/footage.rs | 11 ++ crates/render/src/pipeline.rs | 33 ++++++ crates/render/src/tests/footage_size.rs | 51 ++++++++++ 7 files changed, 334 insertions(+), 44 deletions(-) diff --git a/crates/automation/src/confine.rs b/crates/automation/src/confine.rs index a9fc4665f..371085938 100644 --- a/crates/automation/src/confine.rs +++ b/crates/automation/src/confine.rs @@ -388,6 +388,10 @@ impl FootageSource for ConfinedFootage { fn frame_with(&self, item: ItemId, footage: &Footage, t: Tick, size: Option<(u32, u32)>, float: FloatCurve) -> Option> { self.readable(footage).then(|| self.inner.frame_with(item, footage, t, size, float)).flatten() } + /// The frame as 8-bit levels, or `None` when the footage lies outside the allowed roots. + fn frame8(&self, item: ItemId, footage: &Footage, t: Tick, float: FloatCurve) -> Option> { + self.readable(footage).then(|| self.inner.frame8(item, footage, t, float)).flatten() + } /// The footage audio from the inner source, or `None` when the footage lies outside the allowed roots. fn audio(&self, item: ItemId, footage: &Footage, t: Tick, frames: usize, rate: u32) -> Option> { self.readable(footage).then(|| self.inner.audio(item, footage, t, frames, rate)).flatten() diff --git a/crates/media/src/convert.rs b/crates/media/src/convert.rs index a2b24b3ae..86fb088f9 100644 --- a/crates/media/src/convert.rs +++ b/crates/media/src/convert.rs @@ -7,8 +7,9 @@ //! are encoded in the file. Alpha follows the footage's [`AlphaMode`] interpretation. use effectcraft_project::{AlphaMode, Footage}; -use effectcraft_raster::{Image, Px}; +use effectcraft_raster::{Image, Px, quantize_px_u8}; use effectcraft_render::FloatCurve; +use effectcraft_render::disk_cache::Frame8; use filmcraft_color::{Matrix, Range}; use filmcraft_frame::{Chroma, PixelData, VideoFrame}; use rayon::prelude::*; @@ -80,14 +81,7 @@ pub(crate) fn frame_to_image_in(f: &VideoFrame, op: AlphaOp, mut buf: Vec) - return img; } match &f.data { - PixelData::Rgba8(d) => { - img.data.par_chunks_mut(w).enumerate().for_each(|(y, row)| { - let src = &d[y * w * 4..(y + 1) * w * 4]; - for (o, s) in row.iter_mut().zip(src.as_chunks::<4>().0.iter()) { - *o = op.apply([U8[s[0] as usize], U8[s[1] as usize], U8[s[2] as usize]], U8[s[3] as usize]); - } - }); - } + PixelData::Rgba8(d) => convert_rgba8(&mut img.data, d, w, op, |p| p), PixelData::RgbaF32(d) => { // linear premultiplied → encoded premultiplied img.data.par_chunks_mut(w).enumerate().for_each(|(y, row)| { @@ -104,7 +98,7 @@ pub(crate) fn frame_to_image_in(f: &VideoFrame, op: AlphaOp, mut buf: Vec) - let yuv = Yuv::new(f.color.matrix, f.color.range, 8); let amax = 255.0; convert_yuv( - &mut img, + &mut img.data, YuvSource { size: (w, h), chroma: *chroma, @@ -114,13 +108,14 @@ pub(crate) fn frame_to_image_in(f: &VideoFrame, op: AlphaOp, mut buf: Vec) - }, &yuv, op, + |p| p, ); } PixelData::Yuv16 { planes, chroma, bits, alpha } => { let yuv = Yuv::new(f.color.matrix, f.color.range, *bits); let amax = ((1u32 << *bits) - 1) as f32; convert_yuv( - &mut img, + &mut img.data, YuvSource { size: (w, h), chroma: *chroma, @@ -130,12 +125,52 @@ pub(crate) fn frame_to_image_in(f: &VideoFrame, op: AlphaOp, mut buf: Vec) - }, &yuv, op, + |p| p, ); } } img } +/// An 8-bit frame as [`frame_to_image_in`] converts it and an 8 bpc render then quantises it +/// ([`effectcraft_raster::quantize_px`]), kept as the 8-bit levels: expanded with +/// [`effectcraft_raster::expand_u8`] it is that quantised image to the bit, in a quarter of +/// the memory. `None` for frames with more than 8 bits (10/12-bit Y'CbCr, float). +pub(crate) fn frame_to_rgba8(f: &VideoFrame, op: AlphaOp) -> Option { + let (w, h) = (f.width as usize, f.height as usize); + let mut px = vec![[0u8; 4]; w * h]; + if w > 0 && h > 0 { + match &f.data { + PixelData::Rgba8(d) => convert_rgba8(&mut px, d, w, op, quantize_px_u8), + PixelData::Yuv8 { planes, chroma, alpha } => convert_yuv( + &mut px, + YuvSource { + size: (w, h), + chroma: *chroma, + planes: [&planes[0][..], &planes[1][..], &planes[2][..]], + alpha: alpha.as_deref().map(|a| &a[..]), + amax: 255.0, + }, + &Yuv::new(f.color.matrix, f.color.range, 8), + op, + quantize_px_u8, + ), + PixelData::RgbaF32(_) | PixelData::Yuv16 { .. } => return None, + } + } + Some(Frame8 { width: f.width, height: f.height, rgba: px.into_flattened() }) +} + +/// Straight or premultiplied 8-bit RGBA into `out` (one `O` per pixel, made by `store`). +fn convert_rgba8(out: &mut [O], d: &[u8], w: usize, op: AlphaOp, store: impl Fn(Px) -> O + Sync) { + out.par_chunks_mut(w).enumerate().for_each(|(y, row)| { + let Some(src) = d.get(y * w * 4..(y + 1) * w * 4) else { return }; + for (o, s) in row.iter_mut().zip(src.as_chunks::<4>().0.iter()) { + *o = store(op.apply([U8[s[0] as usize], U8[s[1] as usize], U8[s[2] as usize]], U8[s[3] as usize])); + } + }); +} + /// A movie frame turned the way its file says it is shown ([`crate::probe::video_turn`]: /// clockwise quarter turns; anything else leaves it as decoded). pub(crate) fn turned(img: Image, turn: image::metadata::Orientation) -> Image { @@ -311,13 +346,14 @@ struct YuvSource<'a, T> { amax: f32, } -fn convert_yuv(img: &mut Image, source: YuvSource<'_, T>, k: &Yuv, op: AlphaOp) { +/// Y'CbCr into `out` (one `O` per pixel, made by `store` from the converted pixel). +fn convert_yuv(out: &mut [O], source: YuvSource<'_, T>, k: &Yuv, op: AlphaOp, store: impl Fn(Px) -> O + Sync) { let YuvSource { size: (w, h), chroma, planes, alpha, amax } = source; let (sx, sy) = chroma.shifts(); let cw = w.div_ceil(1 << sx); let ch = h.div_ceil(1 << sy); let inv_amax = 1.0 / amax; - img.data.par_chunks_mut(w).enumerate().for_each_init( + out.par_chunks_mut(w).enumerate().for_each_init( || (vec![0f32; cw], vec![0f32; cw], vec![0f32; w], vec![0f32; w]), |(uc, vc, uf, vf), (y, row)| { // vertical chroma interpolation into normalised rows (Cb, Cr in −0.5..0.5) @@ -357,13 +393,13 @@ fn convert_yuv(img: &mut Image, source: YuvSource<'_, T>, k: &Yuv, op None => { for (x, o) in row.iter_mut().enumerate() { let c = rgb(x); - *o = [c[0], c[1], c[2], 1.0]; + *o = store([c[0], c[1], c[2], 1.0]); } } Some(al) => { let arow = &al[y * w..y * w + w]; for (x, o) in row.iter_mut().enumerate() { - *o = op.apply(rgb(x), (arow[x].f() * inv_amax).clamp(0.0, 1.0)); + *o = store(op.apply(rgb(x), (arow[x].f() * inv_amax).clamp(0.0, 1.0))); } } } @@ -552,6 +588,43 @@ mod tests { assert!((p[0] - 0.5).abs() < 1e-3 && (p[1] - 1.0).abs() < 1e-5 && p[2] > 1.5, "{p:?}"); } + /// The 8-bit levels of a frame expand to the float frame quantised to 8 bpc, bit for bit: + /// 4:2:0 and 4:4:4 Y'CbCr with and without alpha, and RGBA, under every alpha interpretation. + #[test] + fn rgba8_levels_are_the_quantised_float_frame() { + let (w, h) = (37usize, 18usize); + let noise = |n: usize, seed: usize| Arc::new((0..n).map(|i| ((i * 7919 + seed * 104_729) % 256) as u8).collect::>()); + let frame = |data| VideoFrame { + width: w as u32, + height: h as u32, + data, + color: filmcraft_color::ColorInfo::REC709, + par: (1, 1), + pts: filmcraft_time::Tick::ZERO, + }; + let yuv = |chroma: Chroma, alpha: bool| { + let n = w.div_ceil(1 << chroma.shifts().0) * h.div_ceil(1 << chroma.shifts().1); + frame(PixelData::Yuv8 { planes: [noise(w * h, 1), noise(n, 2), noise(n, 3)], chroma, alpha: alpha.then(|| noise(w * h, 4)) }) + }; + let frames = [yuv(Chroma::C420, false), yuv(Chroma::C420, true), yuv(Chroma::C444, true), frame(PixelData::Rgba8(noise(w * h * 4, 5)))]; + for f in &frames { + for mode in [AlphaMode::Straight, AlphaMode::Premultiplied, AlphaMode::Ignore] { + let op = AlphaOp::new(mode, [0.2, 0.5, 0.9]); + let q8 = frame_to_rgba8(f, op).expect("8-bit"); + let mut want = frame_to_image(f, op); + for p in &mut want.data { + effectcraft_raster::quantize_px(p, 255.0, 1.0 / 255.0); + } + assert_eq!((q8.width, q8.height), (want.width, want.height)); + for (got, want) in q8.rgba.as_chunks::<4>().0.iter().zip(&want.data) { + assert_eq!(effectcraft_raster::expand_u8(*got).map(f32::to_bits), want.map(f32::to_bits)); + } + } + } + let wide = frame(PixelData::RgbaF32(Arc::new(vec![0.5; w * h * 4]))); + assert!(frame_to_rgba8(&wide, AlphaOp::new(AlphaMode::Straight, [0.0; 3])).is_none()); + } + /// `cargo test -p effectcraft-media --lib bench -- --ignored --nocapture` #[test] #[ignore] diff --git a/crates/media/src/pool.rs b/crates/media/src/pool.rs index cd6676220..e35ff37fc 100644 --- a/crates/media/src/pool.rs +++ b/crates/media/src/pool.rs @@ -6,13 +6,14 @@ use std::sync::{Arc, Condvar, Mutex, OnceLock}; use effectcraft_project::{AlphaMode, Footage, FootageKind, ItemId}; use effectcraft_raster::{Image, Px}; +use effectcraft_render::disk_cache::Frame8; use effectcraft_render::reads::{FileStamp, ReadFile, ReadFiles, ReadLog}; use effectcraft_render::{FloatCurve, FootageSource}; use effectcraft_time::{FrameRate, TICKS_PER_SECOND, Tick}; use filmcraft_media::{FrameRequest, SharedSource}; use image::metadata::Orientation; -use crate::convert::{AlphaOp, dynamic_to_image, frame_to_image_in, frame_to_image_scaled, turned}; +use crate::convert::{AlphaOp, dynamic_to_image, frame_to_image_in, frame_to_image_scaled, frame_to_rgba8, turned}; use crate::downmix::Order; use crate::gif_anim::{self, Anim}; use crate::reads; @@ -53,10 +54,28 @@ struct Key { /// resampled (see `Inner::still_at_size`); also at its own size for a placed PDF / /// Illustrator page, which is drawn to fill it ([`Footage::placed_page`]); (0, 0) otherwise. nominal: (u32, u32), + /// Kept as 8-bit levels ([`FootageSource::frame8`]), not float. + q8: bool, +} + +/// A cached frame's pixels: float, or the 8-bit levels an 8 bpc render quantises it to. +#[derive(Clone)] +enum Pixels { + F32(Arc), + Q8(Arc), +} + +impl Pixels { + fn bytes(&self) -> usize { + match self { + Pixels::F32(img) => img.data.len() * std::mem::size_of::(), + Pixels::Q8(f) => f.rgba.len(), + } + } } struct Entry { - img: Arc, + img: Pixels, /// The files the frame was decoded from (see [`crate::reads`]). files: Arc, stamp: u64, @@ -112,8 +131,8 @@ struct Lru { } impl Lru { - /// The cached image and its read files for `k`, marking the entry as most recently used. - fn get(&mut self, k: &Key) -> Option<(Arc, Arc)> { + /// The cached frame and its read files for `k`, marking the entry as most recently used. + fn get(&mut self, k: &Key) -> Option<(Pixels, Arc)> { let e = self.map.get_mut(k)?; self.order.remove(&e.stamp); self.clock += 1; @@ -121,9 +140,9 @@ impl Lru { self.order.insert(e.stamp, k.clone()); Some((e.img.clone(), e.files.clone())) } - /// Stores an image with its read files under `k` and evicts the least recently used entries until the cache fits `budget` bytes. - fn insert(&mut self, k: Key, img: Arc, files: Arc, budget: usize) { - let bytes = img.data.len() * std::mem::size_of::() + 64; + /// Stores a frame with its read files under `k` and evicts the least recently used entries until the cache fits `budget` bytes. + fn insert(&mut self, k: Key, img: Pixels, files: Arc, budget: usize) { + let bytes = img.bytes() + 64; self.clock += 1; if let Some(old) = self.map.insert(k.clone(), Entry { img, files, stamp: self.clock, bytes }) { self.order.remove(&old.stamp); @@ -145,8 +164,10 @@ impl Lru { let Some((_, k)) = self.order.pop_first() else { break }; if let Some(e) = self.map.remove(&k) { self.bytes -= e.bytes; + // ponytail: float buffers only; add 8-bit spares if their allocation shows up in a profile if self.spare.len() < MAX_SPARE - && let Ok(img) = Arc::try_unwrap(e.img) + && let Pixels::F32(img) = e.img + && let Ok(img) = Arc::try_unwrap(img) { self.spare.push(img.data); } @@ -185,6 +206,8 @@ struct CacheState { inflight: HashSet, /// Last frame index requested per movie (sequential-playback detection). last: HashMap, i64>, + /// Movies whose frames can't be kept as 8-bit levels (more bits, or shown turned). + wide: HashSet>, } /// A movie/audio source by path ([`Opened`]), opened by the first caller while the others wait @@ -407,6 +430,7 @@ impl MediaPool { c.lru.remove(&k); } c.last.remove(path); + c.wide.remove(path); drop(c); crate::exr_channels::forget(path); } @@ -422,6 +446,21 @@ impl MediaPool { Inner::frame_at(&self.inner, footage, t, true, None, FloatCurve::Srgb) } + /// [`FootageSource::frame8`]: `None` (the renderer then asks for the float frame) for + /// anything but an 8-bit movie frame shown as decoded, and on errors. + fn frame8_of(&self, footage: &Footage, t: Tick, float: FloatCurve) -> Option> { + if footage.missing || footage.kind != FootageKind::Video || gif_anim::is_animated(footage) { + return None; + } + if lock(&self.inner.cache).wide.contains(footage.path.as_str()) { + return None; + } + match Inner::frame_px(&self.inner, footage, t, true, None, float, true) { + Ok(Pixels::Q8(f)) => Some(f), + _ => None, + } + } + /// [`FootageSource::frame`] / [`FootageSource::frame_at_size`]: `None` (logged) on errors. fn frame_sized(&self, _item: ItemId, footage: &Footage, t: Tick, size: Option<(u32, u32)>, float: FloatCurve) -> Option> { if footage.missing { @@ -775,7 +814,7 @@ impl Inner { } << 2) | (u8::from(footage.placed_page) << 4); let matte = if footage.alpha == AlphaMode::Premultiplied { footage.premul_color.map(f32::to_bits) } else { [0; 3] }; - let key = |path: &str, frame| Key { path: path.into(), frame, interp, matte, size: (0, 0), nominal: (0, 0) }; + let key = |path: &str, frame| Key { path: path.into(), frame, interp, matte, size: (0, 0), nominal: (0, 0), q8: false }; match footage.kind { FootageKind::Sequence if !footage.sequence.is_empty() => { let i = Self::frame_index(footage.frame_rate, t, footage.sequence_frames(), footage.loop_count); @@ -811,10 +850,22 @@ impl Inner { /// The frame of `footage` at `t`, resampled to `size` (see [`FootageSource::frame_at_size`]): /// a movie's made at that size directly, a still's resampled once and kept at that size. fn frame_at(self: &Arc, footage: &Footage, t: Tick, playback: bool, size: Option<(u32, u32)>, float: FloatCurve) -> Result> { + match Self::frame_px(self, footage, t, playback, size, float, false)? { + Pixels::F32(img) => Ok(img), + // (a float request's key never holds 8-bit levels) + Pixels::Q8(_) => Err(MediaError::Decode(format!("{}: 8-bit frame for a float request", footage.path))), + } + } + + /// [`Self::frame_at`], kept as 8-bit levels when `q8` is set and the frame has 8 bits (a + /// float frame under the float key otherwise, and the movie is remembered as `wide`). + #[allow(clippy::too_many_arguments)] + fn frame_px(self: &Arc, footage: &Footage, t: Tick, playback: bool, size: Option<(u32, u32)>, float: FloatCurve, q8: bool) -> Result { if !footage.has_video { return Err(MediaError::Unsupported(format!("{}: no video", footage.path))); } let mut loc = Self::locate(footage, t, float); + loc.key.q8 = q8; let movie = loc.media_t.is_some(); // A still asked for at its own size is the unsized frame (one decode, one entry). if let Some(size) = size.filter(|s| movie || *s != (footage.width, footage.height)) { @@ -854,7 +905,7 @@ impl Inner { drop(c); let (img, files) = reads::captured(|| self.decode(&loc, footage)); reads::note_all(&files); - return Ok(Arc::new(img?)); + return img; } c = self.done.wait(c).unwrap_or_else(std::sync::PoisonError::into_inner); } @@ -864,8 +915,14 @@ impl Inner { let _guard = Inflight { inner: self, key: loc.key.clone() }; let (img, files) = reads::captured(|| self.decode(&loc, footage)); reads::note_all(&files); - let img = Arc::new(img?); - lock(&self.cache).lru.insert(loc.key, img.clone(), Arc::new(files), self.budget()); + let img = img?; + let mut c = lock(&self.cache); + if q8 && let Pixels::F32(_) = img { + // not 8-bit: kept as the float frame it is, which the renderer asks for next + c.wide.insert(loc.key.path.clone()); + loc.key.q8 = false; + } + c.lru.insert(loc.key, img.clone(), Arc::new(files), self.budget()); Ok(img) } @@ -876,24 +933,25 @@ impl Inner { let next_t = t + footage.frame_rate.frame_duration(); let mut next = Self::locate(footage, next_t, float); next.key.size = loc.key.size; + next.key.q8 = loc.key.q8; if next.key != loc.key && !c.lru.map.contains_key(&next.key) && !c.inflight.contains(&next.key) { - self.prefetch(footage, next_t, size, float); + self.prefetch(footage, next_t, size, float, loc.key.q8); } } } #[cfg(not(target_arch = "wasm32"))] - fn prefetch(self: &Arc, footage: &Footage, t: Tick, size: Option<(u32, u32)>, float: FloatCurve) { + fn prefetch(self: &Arc, footage: &Footage, t: Tick, size: Option<(u32, u32)>, float: FloatCurve, q8: bool) { let (me, f) = (self.clone(), footage.clone()); self.prefetched.fetch_add(1, Ordering::Relaxed); // A plain thread, not a rayon job: the decode below parallelises with rayon itself. std::thread::spawn(move || { - let _ = me.frame_at(&f, t, false, size, float); + let _ = me.frame_px(&f, t, false, size, float, q8); }); } #[cfg(target_arch = "wasm32")] - fn prefetch(self: &Arc, _footage: &Footage, _t: Tick, _size: Option<(u32, u32)>, _float: FloatCurve) {} + fn prefetch(self: &Arc, _footage: &Footage, _t: Tick, _size: Option<(u32, u32)>, _float: FloatCurve, _q8: bool) {} /// A still (image file or sequence frame) at `loc.key.size` when one is set: resampled as the /// renderer would ([`FootageSource::frame_at_size`]), so the pool keeps the reduced frame and a @@ -910,9 +968,9 @@ impl Inner { effectcraft_raster::resample(&img, w, h) } - fn decode(&self, loc: &Loc, footage: &Footage) -> Result { + fn decode(&self, loc: &Loc, footage: &Footage) -> Result { match loc.media_t { - None => self.decode_still(loc, footage).map(|img| Self::still_at_size(img, loc, footage)), + None => self.decode_still(loc, footage).map(|img| Pixels::F32(Arc::new(Self::still_at_size(img, loc, footage)))), Some(_) => self.decode_media(loc, footage), } } @@ -959,15 +1017,16 @@ impl Inner { } } - fn decode_media(&self, loc: &Loc, footage: &Footage) -> Result { + fn decode_media(&self, loc: &Loc, footage: &Footage) -> Result { let op = AlphaOp::of(footage, loc.float); let path = &loc.key.path; + let f32 = |img| Pixels::F32(Arc::new(img)); match loc.media_t { - None => self.decode_still(loc, footage), + None => self.decode_still(loc, footage).map(f32), Some(mt) if gif_anim::is_animated(footage) => { let anim = self.gif(path)?; let frame = anim.frame_at(Tick(mt.0)).ok_or_else(|| MediaError::Decode(format!("{path}: no frames")))?; - Ok(dynamic_to_image(frame, op)) + Ok(f32(dynamic_to_image(frame, op))) } Some(mt) => { let Opened { src, turn, .. } = self.opened(path).ok_or_else(|| MediaError::Io(format!("{path}: cannot open")))?; @@ -985,10 +1044,16 @@ impl Inner { && (vf.width, vf.height) == side((footage.width, footage.height)) && let Some(img) = frame_to_image_scaled(&vf, w, h) { - return Ok(turned(img, turn)); + return Ok(f32(turned(img, turn))); + } + if loc.key.q8 + && turn == Orientation::NoTransforms + && let Some(f) = frame_to_rgba8(&vf, op) + { + return Ok(Pixels::Q8(Arc::new(f))); } let buf = lock(&self.cache).lru.take_spare(vf.width as usize * vf.height as usize); - Ok(turned(frame_to_image_in(&vf, op, buf), turn)) + Ok(f32(turned(frame_to_image_in(&vf, op, buf), turn))) } } } @@ -1038,6 +1103,9 @@ impl FootageSource for MediaPool { fn frame_with(&self, item: ItemId, footage: &Footage, t: Tick, size: Option<(u32, u32)>, float: FloatCurve) -> Option> { self.frame_sized(item, footage, t, size, float) } + fn frame8(&self, _item: ItemId, footage: &Footage, t: Tick, float: FloatCurve) -> Option> { + self.frame8_of(footage, t, float) + } fn problem(&self, footage: &Footage) -> Option { if footage.missing || footage.kind != FootageKind::Video { @@ -1275,8 +1343,8 @@ mod tests { #[test] fn lru_respects_budget_and_recycles() { let mut c = Lru::default(); - let img = || Arc::new(Image::new(16, 16)); // 4 KiB + overhead - let k = |i| Key { path: "a".into(), frame: i, interp: 0, matte: [0; 3], size: (0, 0), nominal: (0, 0) }; + let img = || Pixels::F32(Arc::new(Image::new(16, 16))); // 4 KiB + overhead + let k = |i| Key { path: "a".into(), frame: i, interp: 0, matte: [0; 3], size: (0, 0), nominal: (0, 0), q8: false }; let budget = 3 * (4096 + 64); for i in 0..10 { c.insert(k(i), img(), Arc::default(), budget); @@ -1292,6 +1360,18 @@ mod tests { assert!(c.take_spare(17).is_empty()); } + /// A frame kept as 8-bit levels is its own entry beside the float one, counted at 4 bytes + /// a pixel. + #[test] + fn lru_keeps_8_bit_frames_apart_at_a_quarter_of_the_bytes() { + let mut c = Lru::default(); + let k = |q8| Key { path: "a".into(), frame: 0, interp: 0, matte: [0; 3], size: (0, 0), nominal: (0, 0), q8 }; + c.insert(k(false), Pixels::F32(Arc::new(Image::new(16, 16))), Arc::default(), usize::MAX); + c.insert(k(true), Pixels::Q8(Arc::new(Frame8 { width: 16, height: 16, rgba: vec![0; 16 * 16 * 4] })), Arc::default(), usize::MAX); + assert!(matches!(c.get(&k(false)), Some((Pixels::F32(_), _))) && matches!(c.get(&k(true)), Some((Pixels::Q8(_), _)))); + assert_eq!(c.bytes, (16 * 16 * 16 + 64) + (16 * 16 * 4 + 64)); + } + /// #482: an OpenEXR file with HTJ2K compression (which the decoder lacks) says so on import /// and when its frame is read, instead of "no non-deep rgb channels" or a transparent frame. #[test] diff --git a/crates/raster/src/lib.rs b/crates/raster/src/lib.rs index fd2fd183b..e18a24c63 100644 --- a/crates/raster/src/lib.rs +++ b/crates/raster/src/lib.rs @@ -31,15 +31,36 @@ pub const MAX_PADDED_SIDE: u32 = 30_000; /// Clamp (colour ≤ alpha ≤ 1, NaN → 0) and round half up to `levels` steps (`inv` = 1 / levels): /// one pixel of an 8/16 bpc buffer. Values are non-negative after the clamp, so the integer /// conversion rounds like `round()` and vectorises. -#[allow(clippy::manual_clamp)] // max/min map NaN to 0; clamp would keep NaN #[inline(always)] pub fn quantize_px(p: &mut Px, levels: f32, inv: f32) { - let a = round_half_up(p[3].max(0.0).min(1.0) * levels) * inv; + *p = quantize_levels(*p, levels, inv).map(|n| n * inv); +} + +/// [`quantize_px`]'s steps: the whole number of `levels` each channel rounds to (0..=levels). +#[allow(clippy::manual_clamp)] // max/min map NaN to 0; clamp would keep NaN +#[inline(always)] +pub fn quantize_levels(p: Px, levels: f32, inv: f32) -> [f32; 4] { + let na = round_half_up(p[3].max(0.0).min(1.0) * levels); + let a = na * inv; let hi = [a, a, a, a]; + let mut n = [0.0, 0.0, 0.0, na]; for c in 0..3 { - p[c] = round_half_up(p[c].max(0.0).min(hi[c]) * levels) * inv; + n[c] = round_half_up(p[c].max(0.0).min(hi[c]) * levels); } - p[3] = a; + n +} + +/// [`quantize_px`] at 8 bpc, as bytes: [`expand_u8`] gives back exactly the pixel +/// [`quantize_px`] writes, so an 8 bpc buffer can be kept in a quarter of the memory. +#[inline(always)] +pub fn quantize_px_u8(p: Px) -> [u8; 4] { + quantize_levels(p, 255.0, 1.0 / 255.0).map(|n| n as u8) +} + +/// The 8 bpc pixel of [`quantize_px_u8`]'s bytes. +#[inline(always)] +pub fn expand_u8(n: [u8; 4]) -> Px { + n.map(|v| f32::from(v) * (1.0 / 255.0)) } /// `v` (0..=65 535) rounded half up. The same either way; the fastest on each target: x86-64's @@ -403,6 +424,23 @@ impl Mask { mod tests { use super::*; + /// Pixels kept as `quantize_px_u8` bytes expand to the 8 bpc pixel `quantize_px` writes, bit + /// for bit: every level, exact halves, out-of-range, negative, NaN and premultiplied colour + /// brighter than its alpha. + #[test] + fn u8_levels_expand_to_the_quantised_pixel() { + let mut values: Vec = (0..=255 * 4).map(|i| i as f32 / (255.0 * 4.0)).collect(); + values.extend([-0.0, -1.0, 1.0, 1.5, f32::NAN, f32::INFINITY, 0.5 / 255.0, 254.5 / 255.0]); + for &a in &[1.0, 0.5, 0.2, 0.0, 2.0, f32::NAN] { + for &v in &values { + let p = [v, 1.0 - v, v * 0.5, a]; + let mut want = p; + quantize_px(&mut want, 255.0, 1.0 / 255.0); + assert_eq!(expand_u8(quantize_px_u8(p)).map(f32::to_bits), want.map(f32::to_bits), "{p:?}"); + } + } + } + #[test] fn dither_keeps_exact_codes_and_averages_the_rest() { // A value on an 8-bit code is that code at every position. diff --git a/crates/render/src/footage.rs b/crates/render/src/footage.rs index 6b7fdcbf8..c14961d0f 100644 --- a/crates/render/src/footage.rs +++ b/crates/render/src/footage.rs @@ -37,6 +37,14 @@ pub trait FootageSource: Send + Sync { None => self.frame(item, footage, t), } } + /// [`FootageSource::frame_with`]'s full-size frame as 8-bit levels: each pixel is + /// [`effectcraft_raster::quantize_px_u8`] of the float pixel `frame_with` would give, so + /// [`effectcraft_raster::expand_u8`] makes that frame quantised to 8 bpc, bit for bit, from a + /// quarter of the memory (an 8 bpc render of 8-bit footage). `None` when the source has no + /// cheaper way than the float frame; the renderer then takes that. + fn frame8(&self, _item: ItemId, _footage: &Footage, _t: Tick, _float: FloatCurve) -> Option> { + None + } /// `frames` interleaved stereo sample frames of `item`'s audio from source time `t` at /// `rate` Hz (`None` when unavailable). The first sample is `t.to_units_floor(rate)`; /// a sample boundary uses the earliest representable tick that selects that sample. @@ -215,6 +223,9 @@ impl FootageSource for CancellableFootage<'_> { fn frame_with(&self, item: ItemId, footage: &Footage, t: Tick, size: Option<(u32, u32)>, float: FloatCurve) -> Option> { self.read(|| self.inner.frame_with(item, footage, t, size, float)) } + fn frame8(&self, item: ItemId, footage: &Footage, t: Tick, float: FloatCurve) -> Option> { + self.read(|| self.inner.frame8(item, footage, t, float)) + } } /// Footage that can be rasterised at any scale: SVG, PDF, Illustrator and EPS files, and a diff --git a/crates/render/src/pipeline.rs b/crates/render/src/pipeline.rs index 6979afc85..fd46260c2 100644 --- a/crates/render/src/pipeline.rs +++ b/crates/render/src/pipeline.rs @@ -964,6 +964,10 @@ impl<'a> Renderer<'a> { return Some(buf); } } + // 8 bpc plain footage: made from the footage's 8-bit levels, no float frame. + if quantized && let Some(buf) = self.footage_buf8(ctx, layer, item, self.proxy_for(item).unwrap_or(f), f, ctx.source_time(layer)) { + return Some(buf); + } let (img, pf) = self.footage_picture_of(ctx, layer, item, f)?; Some(self.footage_buf(&img, pf, f.width, f.height, quantized && Self::prequantized_footage(layer))) } @@ -1029,6 +1033,35 @@ impl<'a> Renderer<'a> { } } + /// [`Renderer::footage_buf`]'s quantised buffer made from the footage's 8-bit levels + /// ([`FootageSource::frame8`]), the same pixels to the bit, when nothing before the quantise + /// changes them: an 8 bpc render at Full size of plain footage (no masks, colour profile, + /// Separate Fields, Frame Blending, Cineon, Invert Alpha or linear light interpretation). + /// `None` otherwise, and when the source has no 8-bit frame. + #[allow(clippy::too_many_arguments)] + fn footage_buf8(&self, ctx: &EvalCtx, layer: &Layer, item: ItemId, pf: &Footage, f: &Footage, t: Tick) -> Option { + let plain = self.pipe.levels == Some(255.0) + && Self::prequantized_footage(layer) + && self.reduced_size(f.width, f.height).is_none() + && self.pipe.media_in(pf).is_none() + && pf.fields == effectcraft_project::FieldOrder::Off + && !(pf.invert_alpha || pf.linear_light_applies() || pf.cineon_applies()) + && frame_blend_mode(ctx, layer) == FrameBlend::Off; + if !plain { + return None; + } + let fr = self.footage.frame8(item, pf, t, self.pipe.float_curve(pf))?; + let n = (fr.width as usize).checked_mul(fr.height as usize)?; + let px = fr.rgba.as_chunks::<4>().0; + if px.len() != n { + return None; + } + use rayon::prelude::*; + let data = px.par_iter().map(|&p| effectcraft_raster::expand_u8(p)).collect(); + let k = if fr.width > 0 { f.width.max(1) as f64 / fr.width as f64 } else { 1.0 }; + Some(Buf { img: Image { width: fr.width, height: fr.height, data }, offset: [0.0; 2], scale: 1.0 / k }) + } + /// The layer's source is footage that [`Renderer::source`] hands over already quantised /// ([`Renderer::footage_buf`]): footage without masks (masks apply to unquantised pixels). /// Collapsed layers may be vector footage, drawn elsewhere: not counted. diff --git a/crates/render/src/tests/footage_size.rs b/crates/render/src/tests/footage_size.rs index 065941d8a..25c1a1814 100644 --- a/crates/render/src/tests/footage_size.rs +++ b/crates/render/src/tests/footage_size.rs @@ -106,3 +106,54 @@ fn still_layers_are_cached_across_times_and_movies_are_not() { assert_eq!(src.sizes.unwrap().into_inner().unwrap().len(), fetches + 3, "{kind:?}"); } } + +/// [`Clip`] that also keeps its frames as 8-bit levels, counting the times it's asked for them. +struct Clip8 { + asked: std::sync::atomic::AtomicUsize, +} + +impl FootageSource for Clip8 { + fn frame(&self, item: ItemId, f: &Footage, t: Tick) -> Option> { + let mut img = (*Clip { sizes: None }.frame(item, f, t)?).clone(); + // (partly transparent, premultiplied, as decoded with alpha) + img.data.iter_mut().enumerate().for_each(|(i, p)| { + let a = (i % 7) as f32 / 6.0; + *p = [p[0] * a, p[1] * a, p[2] * a, a]; + }); + Some(Arc::new(img)) + } + fn frame8(&self, item: ItemId, f: &Footage, t: Tick, _: crate::FloatCurve) -> Option> { + self.asked.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let img = self.frame(item, f, t)?; + let rgba = img.data.iter().flat_map(|p| effectcraft_raster::quantize_px_u8(*p)).collect(); + Some(Arc::new(crate::disk_cache::Frame8 { width: img.width, height: img.height, rgba })) + } +} + +/// An 8 bpc render at Full takes plain footage as 8-bit levels and renders the same pixels to +/// the bit; other depths, reduced renders and interpreted pixels never ask for them. +#[test] +fn footage_kept_as_8_bit_levels_renders_the_same_pixels() { + struct Float(Clip8); + impl FootageSource for Float { + fn frame(&self, item: ItemId, f: &Footage, t: Tick) -> Option> { + self.0.frame(item, f, t) + } + } + for (depth, invert, scale, used) in [ + (BitDepth::Bpc8, false, 1.0, true), + (BitDepth::Bpc8, false, 0.75, true), + (BitDepth::Bpc8, false, 0.5, false), + (BitDepth::Bpc8, true, 1.0, false), + (BitDepth::Bpc16, false, 1.0, false), + (BitDepth::Bpc32, false, 1.0, false), + ] { + let (p, cid) = project(depth, invert); + let opts = RenderOpts { scale, ..Default::default() }; + let q8 = Clip8 { asked: Default::default() }; + let frame = Renderer::new(&p, &q8, opts).comp_frame(cid, Tick::ZERO); + let plain = Renderer::new(&p, &Float(Clip8 { asked: Default::default() }), opts).comp_frame(cid, Tick::ZERO); + assert!(frame.data.iter().zip(&plain.data).all(|(a, b)| a.map(f32::to_bits) == b.map(f32::to_bits)), "{depth:?} {invert} {scale}"); + assert_eq!(q8.asked.into_inner() > 0, used, "{depth:?} {invert} {scale}"); + } +} From cc4f834a94a41527d4c6086ec54b9856547e45ea Mon Sep 17 00:00:00 2001 From: nazar-gavrylyk Date: Sat, 7 Nov 2026 19:12:55 +0100 Subject: [PATCH 08/24] cache_bench: --play measures a first playback in the app --play SECS runs the app headless (egui_kittest) on the bench comp, with the viewer at 100 %, Resolution Auto, nothing cached and Cache Frames When Idle off, presses play and steps it for SECS seconds of wall time. It prints the frames shown per second, the time the clock stalled on uncached frames, the frame reached and the share of passes at each viewer scale. --gpu composites on the GPU (Mercury Auto) as the desktop app does; --res N sets the comp's Resolution to every N-th pixel. --- crates/ui-egui/examples/cache_bench.rs | 66 ++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/crates/ui-egui/examples/cache_bench.rs b/crates/ui-egui/examples/cache_bench.rs index 9367d9e5c..9cfd5b26b 100644 --- a/crates/ui-egui/examples/cache_bench.rs +++ b/crates/ui-egui/examples/cache_bench.rs @@ -16,6 +16,12 @@ //! `--gpu` (composite on a GPU of its own, as the desktop app does with Mercury GPU Acceleration: //! the GPU or the CPU, whichever Auto measures faster per comp, read back to the CPU; the viewer's //! backend), `--backend cpu|gpu|auto` (with `--gpu`: force one compositor, default `auto`). +//! +//! `--play SECS` instead plays the comp in the app (headless, a viewer at 100 %, Resolution +//! Auto, nothing cached, Cache Frames When Idle off) for SECS seconds of wall time, as a first +//! playback, and prints the frames shown per second, the time stalled and the scale it played at. +//! With `--gpu` the app composites on the GPU (Mercury Auto), otherwise on the CPU; `--res N` +//! sets the comp's Resolution to every N-th pixel instead of Auto. use std::sync::Arc; use std::time::{Duration, Instant}; @@ -115,6 +121,10 @@ fn main() { cid } }; + if let Some(secs) = arg("--play").and_then(|v| v.parse().ok()) { + play(s, cid, secs, std::env::args().any(|a| a == "--gpu")); + return; + } let comp = s.project.comp(cid).expect("comp").clone(); let total = comp.frame_rate.frame_at(comp.duration); let n = arg("--frames").and_then(|v| v.parse().ok()).unwrap_or(total).min(total); @@ -204,3 +214,59 @@ fn main() { println!("disk cache {} written, {} dropped, {} entries ({} MB), flushed in {flushed:.2} s", d.writes, d.dropped, d.entries, d.bytes >> 20); } } + +/// `--play`: a first playback of `cid` in the app, as someone pressing play in it would see. +fn play(session: effectcraft_engine::Session, cid: ItemId, secs: f64, gpu: bool) { + use effectcraft_ui_egui::{ControlRequest, EffectcraftApp}; + let (tx, rx) = std::sync::mpsc::channel::(); + let builder = egui_kittest::Harness::builder().with_size(egui::vec2(1920.0, 1200.0)).with_pixels_per_point(1.0); + let builder = if gpu { builder.wgpu() } else { builder }; + let mut h = builder.build_eframe(move |_| EffectcraftApp::new(session).with_control(rx)); + { + let app = h.state_mut(); + let _ = app.session.execute("comp.open", json!({"comp": cid.0})); + app.ui.viewer.zoom = Some(1.0); + app.ui.cache_when_idle = false; + if let (Some(n), Some(c)) = (arg("--res").and_then(|v| v.parse().ok()), Arc::make_mut(&mut app.session.project).comp_mut(cid)) { + c.resolution = effectcraft_engine::project::Resolution::from_factor(n); + } + } + // Let the UI settle (fonts, the GPU compositor's pipelines) before the clock starts. + let ready = Instant::now() + Duration::from_secs(if gpu { 8 } else { 2 }); + while Instant::now() < ready { + h.step(); + std::thread::sleep(Duration::from_millis(10)); + } + h.state().frames.clear(); + let (reply, _r) = std::sync::mpsc::channel(); + let _ = tx.send(ControlRequest { method: "ui.playback".into(), params: json!({"action": "play"}), reply }); + let t0 = Instant::now(); + let (mut started, mut stalled, mut last, mut scales) = (None, 0.0, t0, std::collections::BTreeMap::::new()); + while t0.elapsed().as_secs_f64() < secs { + h.step(); + let now = Instant::now(); + let app = h.state(); + if app.playback.playing { + let t = *started.get_or_insert(now); + if app.playback.waiting && now > t { + stalled += (now - last).as_secs_f64(); + } + } + *scales.entry(app.shown_series(cid).scale).or_default() += 1; + last = now; + std::thread::sleep(Duration::from_millis(2)); + } + let app = h.state(); + let played = started.map_or(0.0, |t| t.elapsed().as_secs_f64()); + let shown = app.playback.shown; + let fr = app.session.project.comp(cid).map(|c| c.frame_rate); + let frame = fr.map_or(0, |fr| fr.frame_at(app.session.time())); + println!( + "played {shown} frames shown in {played:.2} s: {:.1} frames/s, stalled {stalled:.2} s, reached frame {frame}{}", + shown as f64 / played.max(1e-9), + if gpu { " (GPU)" } else { "" } + ); + let passes: u64 = scales.values().sum(); + let s: Vec = scales.iter().map(|(k, v)| format!("{:.3}: {:.0} %", *k as f64 / 1000.0, 100.0 * *v as f64 / passes.max(1) as f64)).collect(); + println!("viewer scale {}", s.join(", ")); +} From 893d9a0b04faf3f98b83fc3cd29f408fbbbf4fa3 Mon Sep 17 00:00:00 2001 From: nazar-gavrylyk Date: Sat, 7 Nov 2026 19:12:55 +0100 Subject: [PATCH 09/24] Preview: playback steps down a resolution when it can't keep up Settings > Previews > Lower Resolution When Playback Can't Keep Up (on by default): a preview at Resolution Auto that stalls on uncached frames and shows under 80 % of its frame rate for two seconds plays on at half the scale, and two seconds later at a quarter if still slow, no lower than the Adaptive Resolution Limit. Stopping goes back to the viewer's resolution and Cache Frames When Idle fills it in. An explicit Resolution, the comp's or the Preview panel's, is kept as set. First playback of a 1080p comp with a blur (cache_bench --play): 9.0 to 22.1 frames/s on the CPU (about 27 once at Half), 4.1 to 16.1 with Mercury Auto on an integrated GPU; plain clips that keep up stay at Full. --- crates/engine/src/settings/prefs/dialog.rs | 1 + crates/engine/src/settings/prefs/sections.rs | 3 ++ crates/ui-egui/src/app/playback.rs | 1 + crates/ui-egui/src/app/prefetch.rs | 28 +++++++++++ crates/ui-egui/src/app/viewer_frames.rs | 7 +++ crates/ui-egui/src/lib.rs | 3 ++ crates/ui-egui/tests/it/ui_preview_align.rs | 50 ++++++++++++++++++++ docs/preferences.md | 8 +++- 8 files changed, 100 insertions(+), 1 deletion(-) diff --git a/crates/engine/src/settings/prefs/dialog.rs b/crates/engine/src/settings/prefs/dialog.rs index 1e79cdd46..c23b2b5b1 100644 --- a/crates/engine/src/settings/prefs/dialog.rs +++ b/crates/engine/src/settings/prefs/dialog.rs @@ -170,6 +170,7 @@ pub fn pages() -> Vec { title: "Previews", items: vec![ s("previews.adaptiveResolutionLimit", "Adaptive Resolution Limit", Kind::Choice(ON_OFF_RES), true), + s("previews.adaptivePlayback", "Lower Resolution When Playback Can't Keep Up", B, true), s("previews.showInternalWireframes", "Show Internal Wireframes", B, true), s("previews.cacheFramesWhenIdle", "Cache Frames When Idle", B, true), s("previews.fastPreviews", "Fast Previews (Draft 3D, Faster Effects)", B, true), diff --git a/crates/engine/src/settings/prefs/sections.rs b/crates/engine/src/settings/prefs/sections.rs index 9a9f4ab21..ee86ce4c4 100644 --- a/crates/engine/src/settings/prefs/sections.rs +++ b/crates/engine/src/settings/prefs/sections.rs @@ -92,6 +92,9 @@ page!(CompositionPrefs { page!(Previews { /// Largest downsampling while playing back: `1/2`, `1/4`, `1/8`. adaptive_resolution_limit: String = "1/8".into(), + /// A preview at Resolution Auto that renders slower than real time steps down a resolution + /// (½, then ¼, no further than the Adaptive Resolution Limit) for the rest of that play. + adaptive_playback: bool = true, show_internal_wireframes: bool = false, cache_frames_when_idle: bool = true, fast_previews: bool = false, diff --git a/crates/ui-egui/src/app/playback.rs b/crates/ui-egui/src/app/playback.rs index 51dd04ee1..79be99544 100644 --- a/crates/ui-egui/src/app/playback.rs +++ b/crates/ui-egui/src/app/playback.rs @@ -35,6 +35,7 @@ impl EffectcraftApp { plan: Some(plan), caching: plan.cache_first && plan.video, res: preset.resolution.factor(preset.custom_resolution), + adaptive: None, overlays: preset.include_overlays, layer_controls: preset.include_layer_controls, restore_max, diff --git a/crates/ui-egui/src/app/prefetch.rs b/crates/ui-egui/src/app/prefetch.rs index 22e587643..3da62a17c 100644 --- a/crates/ui-egui/src/app/prefetch.rs +++ b/crates/ui-egui/src/app/prefetch.rs @@ -340,6 +340,8 @@ impl EffectcraftApp { self.session.set_time(fr.tick_of(f)); self.playback.frame_shown(now); } + // Dropping frames: a lower resolution may keep up (the rate shown says so). + self.step_down(now, pass.scale); } ctx.request_repaint(); true @@ -421,6 +423,7 @@ impl EffectcraftApp { } else { // Not cached yet: hold the clock at the current frame (cache first, then play). self.playback.waiting = true; + self.step_down(now, pass.scale); let next = plan.and_then(|p| p.frame_after(cur, 1)).unwrap_or(if cur + 1 > wb { wa } else { cur + 1 }); if self.frames.is_cached(&FrameKey { frame: next, ..series }) { self.session.set_time(fr.tick_of(next)); @@ -431,6 +434,31 @@ impl EffectcraftApp { ctx.request_repaint(); } + /// Settings ▸ Previews ▸ Lower Resolution When Playback Can't Keep Up: a preview at + /// Resolution Auto stalled on a frame that isn't cached and shows under 80 % of its frame + /// rate two seconds after it started or last stepped down (the first frames at a new scale, + /// and the GPU's warm-up frames, take a moment), so it plays on at half `scale`, no lower than + /// the Adaptive Resolution Limit. An explicit Resolution is kept. + // ponytail: it never steps back up during a play, and frames cached at the higher scale aren't + // shown once it stepped down; add both if loops over a partly cached comp need them. + fn step_down(&mut self, now: f64, scale: f64) { + let pb = &self.playback; + let since = pb.adaptive.map_or(pb.starting_since, |(_, t)| t); + let step = pb.plan.map_or(1, |p| p.step).max(1) as f64; + let lower = scale * 0.5; + if !self.session.prefs.previews.adaptive_playback + || pb.res.is_some() + || !self.viewer_res().is_auto() + || now - since < 2.0 + || pb.achieved_fps().is_some_and(|f| f * step >= pb.fps * 0.8) + || lower < self.session.prefs.adaptive_limit() - 1e-9 + { + return; + } + self.playback.adaptive = Some((lower, now)); + self.playback.shown_at.clear(); + } + /// While the project keeps changing (an edit within [`EDIT_QUIET`] seconds), how long until /// it counts as settled. fn editing(&mut self, now: f64) -> Option { diff --git a/crates/ui-egui/src/app/viewer_frames.rs b/crates/ui-egui/src/app/viewer_frames.rs index a1a0faaf5..fe3ad8abb 100644 --- a/crates/ui-egui/src/app/viewer_frames.rs +++ b/crates/ui-egui/src/app/viewer_frames.rs @@ -98,6 +98,13 @@ impl EffectcraftApp { // play back, and frames cached by a preview stay on screen when it stops. let res = self.viewer_res(); let full = res.scale(zoom, ppp); + // A preview that couldn't keep up stepped down (`step_down`). + if self.playback.playing + && res.is_auto() + && let Some((s, _)) = self.playback.adaptive + { + return full.min(s); + } // Fast Previews ▸ Adaptive Resolution / Fast Draft: lower resolution while dragging. let (_, k) = self.session.state.viewer.fast_previews.render(self.ui.viewer.interacting || self.ui.viewer.property_interacting); if k < 1.0 && res.is_auto() { diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 7a8805ec4..2c022595a 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -190,6 +190,9 @@ pub struct Playback { pub caching: bool, /// Resolution override while playing (pixel step), `None` = the viewer's. pub res: Option, + /// Settings ▸ Previews ▸ Lower Resolution When Playback Can't Keep Up: the lower render + /// scale this play stepped down to, and when (seconds). See `step_down`. + pub adaptive: Option<(f64, f64)>, /// Include Overlays / Include Layer Controls. pub overlays: bool, pub layer_controls: bool, diff --git a/crates/ui-egui/tests/it/ui_preview_align.rs b/crates/ui-egui/tests/it/ui_preview_align.rs index 622df0397..81b6c8073 100644 --- a/crates/ui-egui/tests/it/ui_preview_align.rs +++ b/crates/ui-egui/tests/it/ui_preview_align.rs @@ -316,3 +316,53 @@ fn cache_frames_when_idle_survives_an_inverted_work_area() { } assert!(!h.state().playback.playing); } + +/// Settings ▸ Previews ▸ Lower Resolution When Playback Can't Keep Up: a preview at Auto that +/// stalls for two seconds steps down to half its scale, two more to a quarter; an explicit Resolution and +/// the pref turned off keep the scale, and stopping goes back to the viewer's. +#[test] +fn playback_that_cannot_keep_up_steps_down_a_resolution() { + let stall = |h: &mut Harness<'_, EffectcraftApp>, secs: f64| { + for _ in 0..(secs / 0.1).round() as usize { + h.state_mut().clock_mut().advance(0.1); + h.step(); + } + }; + let play = |h: &mut Harness<'_, EffectcraftApp>| { + h.state_mut().use_manual_clock(); + invoke(h, "playback.toggle", json!({})); + h.step(); + assert!(h.state().playback.playing); + }; + let mut h = harness(); + // No room in the RAM preview: frames never stay cached and every pass stalls, as with a comp + // too heavy to render in real time. + h.state().frames.set_budget(0); + play(&mut h); + stall(&mut h, 1.5); + assert_eq!(h.state().viewer_scale(1.0, 1.0), 1.0, "it waits two seconds before it steps down"); + stall(&mut h, 0.7); + assert_eq!(h.state().viewer_scale(1.0, 1.0), 0.5); + stall(&mut h, 1.5); + assert_eq!(h.state().viewer_scale(1.0, 1.0), 0.5, "and two more before the next step"); + stall(&mut h, 0.7); + assert_eq!(h.state().viewer_scale(1.0, 1.0), 0.25); + invoke(&mut h, "playback.toggle", json!({})); + assert!(!h.state().playback.playing); + assert_eq!(h.state().viewer_scale(1.0, 1.0), 1.0, "stopped: the viewer's resolution again"); + + // An explicit Resolution is kept. + let cid = h.state().session.active_comp_id().unwrap(); + std::sync::Arc::make_mut(&mut h.state_mut().session.project).comp_mut(cid).unwrap().resolution = effectcraft_engine::project::Resolution::Full; + play(&mut h); + stall(&mut h, 2.5); + assert_eq!(h.state().viewer_scale(1.0, 1.0), 1.0); + invoke(&mut h, "playback.toggle", json!({})); + + // So is Auto with the pref off. + std::sync::Arc::make_mut(&mut h.state_mut().session.project).comp_mut(cid).unwrap().resolution = effectcraft_engine::project::Resolution::Auto; + h.state_mut().session.prefs.previews.adaptive_playback = false; + play(&mut h); + stall(&mut h, 2.5); + assert_eq!(h.state().viewer_scale(1.0, 1.0), 1.0); +} diff --git a/docs/preferences.md b/docs/preferences.md index de7e426bc..be7b8edb5 100644 --- a/docs/preferences.md +++ b/docs/preferences.md @@ -52,7 +52,13 @@ Commands (CLI, MCP, control channel): - `autoSave.saveOnRenderStart`: Save When Starting Render Queue - `composition.showRenderingProgress`: Show Rendering Progress in Info Panel and Flowchart - `previews.adaptiveResolutionLimit`: Adaptive Resolution Limit (the lowest resolution Fast Previews ▸ - Adaptive Resolution drops to while you drag; a playing preview keeps the viewer's resolution) + Adaptive Resolution drops to while you drag, and the lowest a playing preview steps down to) +- `previews.adaptivePlayback`: Lower Resolution When Playback Can't Keep Up. On by default. A + preview at Resolution Auto that stalls on frames and shows under 80 % of its frame rate steps + down a resolution (half the scale after two seconds, a quarter two seconds later if still slow, + no lower than the Adaptive Resolution Limit) for the rest of that play; stopping goes back to the viewer's resolution, and Cache + Frames When Idle fills it in. An explicit Resolution (the comp's or the Preview panel's) is + kept as set - `previews.cacheFramesWhenIdle`: Cache Frames When Idle (Composition ▸ Preview): after a second without input or edits, the viewer renders the work area into the RAM preview in the background, from the current time on, until it is cached or the budget is full. On by default, From 3709cccedf9e63cf23570ba3f27dee5d459fd18a Mon Sep 17 00:00:00 2001 From: nazar-gavrylyk Date: Sun, 8 Nov 2026 17:57:50 +0100 Subject: [PATCH 10/24] Preview: frame lanes are four threads wide, and cache_bench measures the cadence --- crates/ui-egui/examples/cache_bench.rs | 60 ++++++++++++++++++++--- crates/ui-egui/src/app/prefetch.rs | 9 ++-- crates/ui-egui/src/frames.rs | 34 ++++++++++++- crates/ui-egui/tests/it/ui_audio_scrub.rs | 8 +-- 4 files changed, 91 insertions(+), 20 deletions(-) diff --git a/crates/ui-egui/examples/cache_bench.rs b/crates/ui-egui/examples/cache_bench.rs index 9cfd5b26b..43e937c92 100644 --- a/crates/ui-egui/examples/cache_bench.rs +++ b/crates/ui-egui/examples/cache_bench.rs @@ -10,8 +10,8 @@ //! //! Options: `--project P --comp C` (a saved project's comp instead), `--scale S` (render scale, //! default 0.5: a viewer at half size), `--frames N` (frames to cache, default the whole comp), -//! `--slots N` (frames in flight, at least 1, default one per frame thread as the app does when -//! idle), +//! `--slots N` (frames in flight, at least 1, default one per lane as the app does when idle), +//! `--lane-width W` (threads per lane; the app's default when absent; `--play` takes it too), //! `--disk` (with a fresh disk cache in `target/cache-bench/disk`), `--no-adjustment`, //! `--gpu` (composite on a GPU of its own, as the desktop app does with Mercury GPU Acceleration: //! the GPU or the CPU, whichever Auto measures faster per comp, read back to the CPU; the viewer's @@ -37,6 +37,11 @@ fn arg(name: &str) -> Option { a.iter().position(|x| x == name).and_then(|i| a.get(i + 1).cloned()) } +/// `--lane-width W`: threads per frame lane (the app's default when absent). +fn lane_width() -> Option { + arg("--lane-width").and_then(|v| v.parse().ok()) +} + fn run(s: &mut Session, cmd: &str, p: Value) -> Value { s.execute(cmd, p).unwrap_or_else(|e| panic!("{cmd}: {e}")) } @@ -130,9 +135,9 @@ fn main() { let n = arg("--frames").and_then(|v| v.parse().ok()).unwrap_or(total).min(total); let scale: f64 = arg("--scale").and_then(|v| v.parse().ok()).unwrap_or(0.5); - let frames = Frames::default(); - // (as Cache Frames When Idle: one frame per frame thread, see `app/prefetch.rs`) - let slots: usize = arg("--slots").and_then(|v| v.parse().ok()).map(|n: usize| n.max(1)).unwrap_or(frames.parallelism().max(1)); + let frames = lane_width().map_or_else(Frames::default, Frames::with_lane_width); + // (as Cache Frames When Idle: one frame per lane, see `app/prefetch.rs`) + let slots: usize = arg("--slots").and_then(|v| v.parse().ok()).map(|n: usize| n.max(1)).unwrap_or(frames.lanes()); // Room for every frame: this measures speed, not eviction. frames.set_budget(64 << 30); use effectcraft_engine::render::Backend; @@ -168,9 +173,10 @@ fn main() { }; eprintln!( - "caching {n} frames of {}×{} at scale {scale} with {slots} in flight{}…", + "caching {n} frames of {}×{} at scale {scale} with {slots} in flight on {} lanes{}…", comp.width, comp.height, + frames.lanes(), if src.gpu.is_some() { " on the GPU" } else { "" } ); let (gop0, t0) = (effectcraft_media::gop_stats(), Instant::now()); @@ -224,6 +230,9 @@ fn play(session: effectcraft_engine::Session, cid: ItemId, secs: f64, gpu: bool) let mut h = builder.build_eframe(move |_| EffectcraftApp::new(session).with_control(rx)); { let app = h.state_mut(); + if let Some(w) = lane_width() { + app.frames = effectcraft_ui_egui::frames::Frames::with_lane_width(w); + } let _ = app.session.execute("comp.open", json!({"comp": cid.0})); app.ui.viewer.zoom = Some(1.0); app.ui.cache_when_idle = false; @@ -242,6 +251,10 @@ fn play(session: effectcraft_engine::Session, cid: ItemId, secs: f64, gpu: bool) let _ = tx.send(ControlRequest { method: "ui.playback".into(), params: json!({"action": "play"}), reply }); let t0 = Instant::now(); let (mut started, mut stalled, mut last, mut scales) = (None, 0.0, t0, std::collections::BTreeMap::::new()); + // When each new frame went on screen (seconds since play was pressed): the cadence. The + // current time, not `Playback::shown`: frames shown while the clock holds don't count there. + let fr = h.state().session.project.comp(cid).map(|c| c.frame_rate); + let (mut shown_at, mut last_frame) = (Vec::::new(), None); while t0.elapsed().as_secs_f64() < secs { h.step(); let now = Instant::now(); @@ -252,20 +265,51 @@ fn play(session: effectcraft_engine::Session, cid: ItemId, secs: f64, gpu: bool) stalled += (now - last).as_secs_f64(); } } + let frame = fr.map(|fr| fr.frame_at(app.session.time())); + if app.playback.playing && frame != last_frame { + last_frame = frame; + shown_at.push((now - t0).as_secs_f64()); + } *scales.entry(app.shown_series(cid).scale).or_default() += 1; last = now; std::thread::sleep(Duration::from_millis(2)); } let app = h.state(); let played = started.map_or(0.0, |t| t.elapsed().as_secs_f64()); - let shown = app.playback.shown; - let fr = app.session.project.comp(cid).map(|c| c.frame_rate); + let shown = shown_at.len(); let frame = fr.map_or(0, |fr| fr.frame_at(app.session.time())); println!( "played {shown} frames shown in {played:.2} s: {:.1} frames/s, stalled {stalled:.2} s, reached frame {frame}{}", shown as f64 / played.max(1e-9), if gpu { " (GPU)" } else { "" } ); + // Cadence: how evenly the frames went on screen (the intervals between consecutive shown + // frames after the first), and how long the first two took. + let mut gaps: Vec = shown_at.windows(2).map(|w| w[1] - w[0]).collect(); + gaps.sort_by(f64::total_cmp); + let at = |q: f64| gaps.get(((gaps.len() as f64 - 1.0) * q).round() as usize).copied().unwrap_or(0.0) * 1000.0; + let holds = gaps.iter().filter(|g| **g > 0.2).count(); + println!( + "cadence first frame {:.2} s, second {:.2} s; gaps median {:.0} ms, p95 {:.0} ms, max {:.0} ms, {holds} holds > 200 ms", + shown_at.first().copied().unwrap_or(0.0), + shown_at.get(1).copied().unwrap_or(0.0), + at(0.5), + at(0.95), + at(1.0) + ); + // The opening (the first 3 s, before an adaptive step down can have taken effect): how + // evenly the uncached frames at the starting resolution went on screen. + let mut early: Vec = shown_at.windows(2).filter(|w| w[1] <= 3.0).map(|w| w[1] - w[0]).collect(); + early.sort_by(f64::total_cmp); + let at = |q: f64| early.get(((early.len() as f64 - 1.0) * q).round() as usize).copied().unwrap_or(0.0) * 1000.0; + println!( + "opening (3 s) {} frames; gaps median {:.0} ms, p95 {:.0} ms, max {:.0} ms, {} holds > 200 ms", + early.len() + usize::from(!early.is_empty()), + at(0.5), + at(0.95), + at(1.0), + early.iter().filter(|g| **g > 0.2).count() + ); let passes: u64 = scales.values().sum(); let s: Vec = scales.iter().map(|(k, v)| format!("{:.3}: {:.0} %", *k as f64 / 1000.0, 100.0 * *v as f64 / passes.max(1) as f64)).collect(); println!("viewer scale {}", s.join(", ")); diff --git a/crates/ui-egui/src/app/prefetch.rs b/crates/ui-egui/src/app/prefetch.rs index 3da62a17c..f31df7094 100644 --- a/crates/ui-egui/src/app/prefetch.rs +++ b/crates/ui-egui/src/app/prefetch.rs @@ -549,13 +549,10 @@ impl EffectcraftApp { side(comp.width) * side(comp.height) * frames::frame_pixel_bytes(&self.session.project.settings) as f64 } + /// Frames rendered at once by a preview or idle caching: the lanes (in the browser the frame + /// workers; without them frames render one at a time on the UI thread, `Frames::pump`). fn frames_parallelism(&self) -> usize { - if cfg!(target_arch = "wasm32") { - // frame workers render one frame each; without them frames render one at a time - // on the UI thread (`Frames::pump`) - return self.frames.remote_slots().max(1); - } - self.frames.parallelism() + self.frames.lanes() } } diff --git a/crates/ui-egui/src/frames.rs b/crates/ui-egui/src/frames.rs index 800306385..2d51068b0 100644 --- a/crates/ui-egui/src/frames.rs +++ b/crates/ui-egui/src/frames.rs @@ -707,15 +707,36 @@ struct DiskCarry { #[cfg(not(target_arch = "wasm32"))] const MAX_DISK_SERIES: usize = 8; +/// Threads per lane (see [`Queue::lanes`]). A frame's passes are memory-bound and stop scaling +/// at about 4 threads, so a core per lane (12 frames at once, each on one thread) and 4 cores +/// per lane cache frames at the same rate (within 3 %, widths 1–4 measured on 12 cores); but the +/// 12 single-threaded frames start and finish together, so a preview that can't keep up shows +/// them in bursts, and its second frame waits for the whole batch (0.3–1.3 s). Three lanes of +/// four threads finish frames 3× as often and the second frame shows in 0.13–0.45 s. +/// Never fewer than two lanes: one would finish frames in a single file. +const LANE_WIDTH: usize = 4; + impl Default for Frames { fn default() -> Self { + Self::with_lane_width(LANE_WIDTH) + } +} + +impl Frames { + /// A frame cache whose lanes are `width` threads each (see [`LANE_WIDTH`]); `Default` uses + /// [`LANE_WIDTH`], benchmarks compare widths. + pub fn with_lane_width(width: usize) -> Self { + #[cfg(target_arch = "wasm32")] + let _ = width; #[cfg(not(target_arch = "wasm32"))] let threads = std::thread::available_parallelism().map(|n| n.get()).unwrap_or(4).clamp(2, 16); #[cfg(not(target_arch = "wasm32"))] - let lanes: Vec = (0..threads) + let width = width.clamp(1, (threads / 2).max(1)); + #[cfg(not(target_arch = "wasm32"))] + let lanes: Vec = (0..(threads / width).max(1)) .map_while(|i| { rayon::ThreadPoolBuilder::new() - .num_threads(1) + .num_threads(width) .thread_name(move |_| format!("ec-preview-{i}")) .panic_handler(|_| log::error!("a frame render panicked; that frame was skipped")) .build() @@ -765,6 +786,15 @@ impl Default for Frames { disk_series: Arc::default(), } } + + /// How many frames render at once on the lanes (a playing preview's prefetch, Cache Frames + /// When Idle); in the browser, the frame workers. + pub fn lanes(&self) -> usize { + #[cfg(target_arch = "wasm32")] + return self.remote_slots().max(1); + #[cfg(not(target_arch = "wasm32"))] + self.queue.lock().map_or(1, |q| q.lanes.max(1)) + } } /// Whether viewer frames of a project with `settings` are half floats ([`F16Frame`], rendered diff --git a/crates/ui-egui/tests/it/ui_audio_scrub.rs b/crates/ui-egui/tests/it/ui_audio_scrub.rs index ae724c9ad..d177d5654 100644 --- a/crates/ui-egui/tests/it/ui_audio_scrub.rs +++ b/crates/ui-egui/tests/it/ui_audio_scrub.rs @@ -215,7 +215,7 @@ fn preview_with_audio_holds_the_first_frame_and_sounds_from_it() { let gate = GatedFrames::default(); let renderer: Arc = Arc::new(gate.clone()); app.frames.set_remote(Some(renderer)); - let ahead = (app.frames.parallelism() * 2).max(4) as i64; + let ahead = (app.frames.lanes() * 2).max(4) as i64; app.session.execute("comp.settings", json!({"comp": cid.0, "duration": (4 * ahead) as f64 / 30.0})).unwrap(); let mut h = Harness::builder().with_size(egui::vec2(1400.0, 900.0)).with_step_dt(1.0 / 30.0).build_eframe(|_| app); h.state_mut().use_manual_clock(); @@ -258,7 +258,7 @@ fn preview_with_audio_sounds_without_caching_the_work_area() { h.state_mut().use_manual_clock(); h.run_steps(2); gate.discard(&mut h); - let ahead = (h.state().frames.parallelism() * 2).max(4) as i64; + let ahead = (h.state().frames.lanes() * 2).max(4) as i64; let now = h.state().now(); h.state_mut().play(now); let frames: Vec = (0..ahead).collect(); @@ -439,7 +439,7 @@ fn until_started(h: &mut Harness<'_, EffectcraftApp>) -> bool { fn startup_finishes_at_any_ram_preview_budget() { let probe = Fake::default(); let (app, _) = setup(true, &probe); - let ahead = (app.frames.parallelism() * 2).max(4) as i64; + let ahead = (app.frames.lanes() * 2).max(4) as i64; // A 64×36 frame is 9216 bytes; `fit` frames fit in (fit + ½) frames' worth / 0.9. let mut fits = vec![2, 3, 4, ahead - 1, ahead, ahead + 1]; fits.sort(); @@ -505,7 +505,7 @@ fn extended_viewer(fake: &Fake) -> (Harness<'static, EffectcraftApp>, ItemId, [u fn startup_finishes_when_extended_viewer_frames_are_larger() { let fake = Fake::default(); let (mut h, cid, size) = extended_viewer(&fake); - let ahead = (h.state().frames.parallelism() * 2).max(4); + let ahead = (h.state().frames.lanes() * 2).max(4); let (comp_bytes, real_bytes) = (160 * 120 * 4, size[0] * size[1] * 4); // Room for `ahead` comp-sized frames (and a half), which is fewer real ones. let budget = ((ahead as f64 + 0.5) * comp_bytes as f64 / 0.9) as usize; From b3000aac70842153e4dd4208cd55b44100a0b1a7 Mon Sep 17 00:00:00 2001 From: nazar-gavrylyk Date: Sun, 8 Nov 2026 17:57:50 +0100 Subject: [PATCH 11/24] Preview: playback runs at the pace frames arrive when it can't keep up --- crates/ui-egui/src/app/playback.rs | 3 + crates/ui-egui/src/app/prefetch.rs | 161 +++++++++++++++++--- crates/ui-egui/src/lib.rs | 8 + crates/ui-egui/tests/it/ui_preview_align.rs | 3 + docs/gaps.md | 2 +- docs/preferences.md | 3 +- 6 files changed, 156 insertions(+), 24 deletions(-) diff --git a/crates/ui-egui/src/app/playback.rs b/crates/ui-egui/src/app/playback.rs index 79be99544..679d8683d 100644 --- a/crates/ui-egui/src/app/playback.rs +++ b/crates/ui-egui/src/app/playback.rs @@ -46,6 +46,9 @@ impl EffectcraftApp { starting_since: now, first_cached: None, shown_at: Default::default(), + pace: 0.0, + pace_acc: None, + paced_at: None, }; if !self.playback.caching { if plan.video { diff --git a/crates/ui-egui/src/app/prefetch.rs b/crates/ui-egui/src/app/prefetch.rs index f31df7094..dbde3f085 100644 --- a/crates/ui-egui/src/app/prefetch.rs +++ b/crates/ui-egui/src/app/prefetch.rs @@ -35,6 +35,46 @@ fn between_picture_and_sound(cur: i64, f: i64, target: i64, looping: bool) -> bo if target >= cur || !looping { cur < f && f < target } else { f > cur || f < target } } +/// How long the pace takes to follow the rate frames arrive at (seconds, the time constant of +/// its smoothing): short enough to follow a change, long enough to play through a batch of +/// frames finishing together. +const PACE_TAU: f64 = 0.5; + +/// The rate a preview plays at when frames arrive slower than its frame rate: proportional to +/// the cached frames ahead (`lead` of `target`), so it settles where the clock takes frames as +/// fast as they arrive and the lead stays put; smoothed over [`PACE_TAU`] (`dt` seconds since +/// the last pass). Never above the frame rate, never under an eighth of it (slower reads as +/// stopped: the clock holds instead). No frame ahead: the previous pace, the clock holds anyway. +fn pace_for(prev: f64, lead: i64, target: i64, fps: f64, dt: f64) -> f64 { + if lead <= 0 || target <= 0 || fps <= 0.0 { + return prev; + } + let raw = (fps * lead as f64 / target as f64).clamp(fps / 8.0, fps); + let k = 1.0 - (-dt.max(0.0) / PACE_TAU).exp(); + prev + (raw - prev) * k +} + +impl PlaybackPass<'_> { + /// The frame `k` played frames after `from` (wrapping as the preview loops), `None` once a + /// preview that doesn't loop has reached its end. + fn frame_after(&self, from: i64, k: i64) -> Option { + match self.plan { + Some(p) => p.frame_after(from, k), + None => { + let FrameWindow { first: wa, last: wb } = self.window; + let f = from + k; + if f <= wb { + Some(f) + } else if self.looping { + Some(wa + (f - wa).rem_euclid((wb - wa + 1).max(1))) + } else { + None + } + } + } + } +} + impl EffectcraftApp { /// RAM preview: advance at the preview's frame rate when frames are cached; otherwise hold /// and keep rendering ahead (the green cache bar fills, then playback runs in real time). @@ -376,29 +416,42 @@ impl EffectcraftApp { } /// The wall clock advances the preview when no audio device drives it, and shows the target - /// frame (or holds when it is not cached yet). + /// frame (or holds when it is not cached yet). While frames arrive slower than the frame + /// rate it runs at their pace ([`pace_for`]): every frame shows, evenly, as After Effects + /// plays a preview it hasn't cached yet; real time again once the frames ahead are cached. fn advance_frame_clock(&mut self, pass: &PlaybackPass<'_>, step: i64) { - let PlaybackPass { ctx, now, plan, video, looping, frame_rate: fr, cur, series, .. } = *pass; - let FrameWindow { first: wa, last: wb } = pass.window; - let elapsed = now - self.playback.start_wall; + let PlaybackPass { ctx, now, plan, video, frame_rate: fr, cur, series, .. } = *pass; + let FrameWindow { last: wb, .. } = pass.window; // The plan carries the rate the preview was started with (and `play_with` copies it to // `Playback::fps`). Deriving the frame from the plan keeps a logic-only tick correct even // when a host/test supplies only the plan; a stale duplicate cannot hold playback open. let fps = plan.map_or(self.playback.fps, |p| p.fps); - let n = ((elapsed * fps) / step as f64).floor() as i64; - let target = match plan { - Some(p) => p.frame_after(self.playback.start_frame, n), - None => { - let t = self.playback.start_frame + n; - if t <= wb { - Some(t) - } else if looping { - Some(wa + (t - wa).rem_euclid((wb - wa + 1).max(1))) - } else { - None - } + let dt = self.playback.paced_at.map_or(0.0, |t| (now - t).max(0.0)); + self.playback.paced_at = Some(now); + if video { + // The lead counts out of twice the prefetch depth: the prefetch keeps that many + // frames coming, so a full lead means rendering keeps up. + let target = (pass.ahead * 2).max(1); + let lead = self.cached_lead(pass, target); + let prev = if self.playback.pace > 0.0 { self.playback.pace } else { fps }; + let pace = pace_for(prev, lead, target, fps, dt); + self.playback.pace = pace; + if pace < fps * 0.8 { + self.step_down(now, pass.scale); } - }; + if pace < fps * 0.98 { + self.advance_paced(pass, pace, dt); + return; + } + if self.playback.pace_acc.take().is_some() { + // Back to real time from here. + self.playback.start_wall = now; + self.playback.start_frame = cur; + } + } + let elapsed = now - self.playback.start_wall; + let n = ((elapsed * fps) / step as f64).floor() as i64; + let target = pass.frame_after(self.playback.start_frame, n); let Some(target) = target else { if video { self.session.set_time(fr.tick_of(wb)); @@ -424,7 +477,7 @@ impl EffectcraftApp { // Not cached yet: hold the clock at the current frame (cache first, then play). self.playback.waiting = true; self.step_down(now, pass.scale); - let next = plan.and_then(|p| p.frame_after(cur, 1)).unwrap_or(if cur + 1 > wb { wa } else { cur + 1 }); + let next = pass.frame_after(cur, 1).unwrap_or(cur); if self.frames.is_cached(&FrameKey { frame: next, ..series }) { self.session.set_time(fr.tick_of(next)); } @@ -434,11 +487,54 @@ impl EffectcraftApp { ctx.request_repaint(); } + /// The paced clock: it owes `pace` played frames a second (plus what the last passes left + /// over) and shows them one by one as far as they are cached; on a frame that isn't it holds + /// (owing one frame, so that frame shows the moment it arrives), and at the end of a preview + /// that doesn't loop it stops. + fn advance_paced(&mut self, pass: &PlaybackPass<'_>, pace: f64, dt: f64) { + let PlaybackPass { ctx, now, frame_rate: fr, cur, series, .. } = *pass; + let step = pass.plan.map_or(1, |p| p.step).max(1) as f64; + let mut owed = self.playback.pace_acc.unwrap_or(0.0) + dt * pace / step; + let mut f = cur; + while owed >= 1.0 { + match pass.frame_after(f, 1) { + Some(n) if self.frames.is_cached(&FrameKey { frame: n, ..series }) => { + f = n; + owed -= 1.0; + } + Some(_) => { + owed = 1.0; + break; + } + None => { + self.session.set_time(fr.tick_of(pass.window.last)); + self.stop(); + return; + } + } + } + self.playback.waiting = owed >= 1.0; + if f != cur { + self.session.set_time(fr.tick_of(f)); + self.playback.frame_shown(now); + } + self.playback.pace_acc = Some(owed); + ctx.request_repaint(); + } + + /// Cached frames among the next `max` in play order. Counted, not the consecutive run: one + /// late frame (a GPU warm-up frame, a heavy one) with frames cached beyond it holds the clock + /// for a moment when it gets there, rather than slowing the whole preview to a crawl. + fn cached_lead(&self, pass: &PlaybackPass<'_>, max: i64) -> i64 { + let cached = self.frames.cached_frames(&pass.series); + (1..=max).map_while(|k| pass.frame_after(pass.cur, k)).filter(|f| cached.binary_search(f).is_ok()).count() as i64 + } + /// Settings ▸ Previews ▸ Lower Resolution When Playback Can't Keep Up: a preview at - /// Resolution Auto stalled on a frame that isn't cached and shows under 80 % of its frame - /// rate two seconds after it started or last stepped down (the first frames at a new scale, - /// and the GPU's warm-up frames, take a moment), so it plays on at half `scale`, no lower than - /// the Adaptive Resolution Limit. An explicit Resolution is kept. + /// Resolution Auto plays under 80 % of its frame rate (its pace, or stalled on a frame that + /// isn't cached) two seconds after it started or last stepped down (the first frames at a + /// new scale, and the GPU's warm-up frames, take a moment), so it plays on at half `scale`, + /// no lower than the Adaptive Resolution Limit. An explicit Resolution is kept. // ponytail: it never steps back up during a play, and frames cached at the higher scale aren't // shown once it stepped down; add both if loops over a partly cached comp need them. fn step_down(&mut self, now: f64, scale: f64) { @@ -556,6 +652,27 @@ impl EffectcraftApp { } } +#[cfg(test)] +mod pace_tests { + use super::pace_for; + + /// The pace follows the cached lead: real time with a full lead, a third of it with a third, + /// never under an eighth, unchanged with nothing ahead, and smoothed over time. + #[test] + fn pace_follows_the_cached_lead() { + // Settled (a long dt): proportional to the lead. + assert!((pace_for(30.0, 12, 12, 30.0, 10.0) - 30.0).abs() < 1e-6); + assert!((pace_for(30.0, 4, 12, 30.0, 10.0) - 10.0).abs() < 1e-6); + assert!((pace_for(30.0, 1, 100, 30.0, 10.0) - 3.75).abs() < 1e-6, "never under an eighth"); + assert_eq!(pace_for(17.0, 0, 12, 30.0, 10.0), 17.0, "no frame ahead: unchanged"); + // Smoothing: after one time constant it has gone 63 % of the way. + let p = pace_for(30.0, 4, 12, 30.0, 0.5); + assert!((p - (30.0 - 20.0 * (1.0 - (-1.0f64).exp()))).abs() < 1e-6); + assert_eq!(pace_for(30.0, 4, 12, 30.0, 0.0), 30.0, "no time passed: unchanged"); + assert!(pace_for(10.0, 12, 12, 30.0, 0.5) > 10.0, "and up again as the lead fills"); + } +} + #[cfg(test)] mod startup_tests { use super::{between_picture_and_sound, startup_frames}; diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 2c022595a..2e911ea54 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -222,6 +222,14 @@ pub struct Playback { pub first_cached: Option, /// When the last frames were shown (seconds, the last second's worth): the achieved rate. pub shown_at: std::collections::VecDeque, + /// The rate the silent clock advances at (comp frames per second, at most `fps`; 0 = the + /// frame rate): below real time while frames arrive slower than that, so a first playback + /// shows them evenly instead of in bursts between freezes (see `pace_for`). + pub pace: f64, + /// Played frames the paced clock owes (fractional), `Some` while it runs below real time. + pub pace_acc: Option, + /// When the silent clock last ran (seconds), for the pace. + pub paced_at: Option, } impl Playback { diff --git a/crates/ui-egui/tests/it/ui_preview_align.rs b/crates/ui-egui/tests/it/ui_preview_align.rs index 81b6c8073..6b5525e09 100644 --- a/crates/ui-egui/tests/it/ui_preview_align.rs +++ b/crates/ui-egui/tests/it/ui_preview_align.rs @@ -285,6 +285,9 @@ fn playback_prefetches_past_the_cached_frames_ahead() { h.state_mut().use_manual_clock(); h.state_mut().session.set_time(Tick::ZERO); invoke(&mut h, "playback.toggle", json!({})); + // The first pass sees the frames ahead cached and starts the clock (a silent preview holds + // its first frame until a few are, as one with sound does). + h.step(); h.state_mut().clock_mut().advance(0.25); h.run_steps(2); assert!(h.state().playback.playing); diff --git a/docs/gaps.md b/docs/gaps.md index d9de84675..608ed60ed 100644 --- a/docs/gaps.md +++ b/docs/gaps.md @@ -61,7 +61,7 @@ are grouped by area; the issue tracker has the rest. | 2 | **Fidelity unmeasured** beyond scalar ease | [fidelity/](fidelity/README.md): 168 samples, one property type; `compare_frames` helper exists, no corpus | Every "done" feature may still differ; users find the differences | 60–100 h harness + P0/P1 corpus; more per feature | G1 · [target-app-parity.md](target-app-parity.md) | | 3 | **Open crash reports** | Open: #580, #501, #426, #341, #234, #879. Now closed: #591, #306, #511 | A crash loses work; outranks features ([AGENTS.md](../AGENTS.md)) | 10–25 h | G3 · this file | | 4 | ~~**No CI gate on pull requests**~~ closed | `.github/workflows/ci.yml` runs `cargo xtask ci` (fmt, clippy, tests, layers, assets, wasm) on every pull request and push to `main` through craft-common-actions' shared `gate.yml` (#2204) | — | done | G3 · this file | -| 5 | **Playback and scrubbing feel** | #595 stutter on cached frames (partly addressed: nearest-first RAM preview prefetch, cache bars at 1/3 resolution), #304, #504 / #335 audio late or crackling, #586 puppet lag (maintainers asked for details); #329 slow scrubbing (#784) and #554 closed | Preview is where motion designers spend their day | 25–50 h | G5 · [hardware-parity.md](hardware-parity.md) | +| 5 | **Playback and scrubbing feel** | #595 stutter on cached frames (partly addressed: nearest-first RAM preview prefetch, cache bars at 1/3 resolution; an uncached first playback plays at the rate frames render instead of freezing between bursts, with frame lanes four threads wide), #304, #504 / #335 audio late or crackling, #586 puppet lag (maintainers asked for details); #329 slow scrubbing (#784) and #554 closed | Preview is where motion designers spend their day | 25–50 h | G5 · [hardware-parity.md](hardware-parity.md) | | 6 | **Linux and Windows interaction bugs** | Open: #488 rename arrows (hardening landed, reporter asked for details), #492 anchor drag, #268 file drops on Wayland (winit), #613 Direct3D 12 compositor. Now closed: #491, #493, #467, #320, #302, #596 | Many reports come from Linux; some Windows GPUs fail to start the compositor | 25–45 h | G2 · [ui-parity.md](ui-parity.md) | | 7 | **Animation presets: 76 against 621** | After Effects ships 621 `.ffx` presets (Behaviors, Backgrounds, Shapes, Synthetics, Text, Transitions…); we ship 76 original presets (8 text-animator; 23 Transitions: wipes, clock and fan reveals, blinds, dissolves, card flips, CC wipes, fades, #757; 23 Backgrounds: drifting gradients, fog, cloud and smoke, plasma and magma, cells, hexagons, scrolling grids and checks, pulse rings, bokeh, light sweep, #1096, driven by expressions so they keep moving, not seamless loops; 22 Behaviors: wiggle, shake, float, orbit, bounce, drift, pendulum, spin, pulse, heartbeat, blink, strobe, flicker on transform and opacity, #1510) and read only our `.ecpreset`. Missing: Shapes, Synthetics, Image and more Text | Presets are how many designers start; tutorials assume them | 15–35 h (original presets; `.ffx` reading is out of clean-room scope) | G8 · [effects-parity.md](effects-parity.md) | | 8 | **Mask and shape bugs** | Open: #1004 mask vertex multi-select, #999 auto-pan, #1042 / #1033 vertex nodes under the bounding box, #959 / #960 shape group toggles and operators. Now closed: #510, #513, #527, #509, #303, #553, #932 (Convert To Bezier Path on a shape layer's parametric path) | Masks are core compositing | 10–20 h | G2 · [target-app-parity.md](target-app-parity.md) | diff --git a/docs/preferences.md b/docs/preferences.md index be7b8edb5..cad74aec2 100644 --- a/docs/preferences.md +++ b/docs/preferences.md @@ -54,7 +54,8 @@ Commands (CLI, MCP, control channel): - `previews.adaptiveResolutionLimit`: Adaptive Resolution Limit (the lowest resolution Fast Previews ▸ Adaptive Resolution drops to while you drag, and the lowest a playing preview steps down to) - `previews.adaptivePlayback`: Lower Resolution When Playback Can't Keep Up. On by default. A - preview at Resolution Auto that stalls on frames and shows under 80 % of its frame rate steps + preview at Resolution Auto that plays under 80 % of its frame rate (frames render slower than + that, so it plays them at the rate they arrive rather than freezing between them) steps down a resolution (half the scale after two seconds, a quarter two seconds later if still slow, no lower than the Adaptive Resolution Limit) for the rest of that play; stopping goes back to the viewer's resolution, and Cache Frames When Idle fills it in. An explicit Resolution (the comp's or the Preview panel's) is From 396495dfa0449560a33b8b2706ac539868d1ffa0 Mon Sep 17 00:00:00 2001 From: nazar-gavrylyk Date: Tue, 10 Nov 2026 14:13:18 +0100 Subject: [PATCH 12/24] Auto: frames that end without a time stop counting as running A GPU frame that failed or failed its health check, and a remote frame replaced before it finished, stayed counted as running on its side, so every later frame there was timed as one of several side by side and that side looked cheaper. AutoPick::abandoned releases them. --- crates/engine/src/background/remote.rs | 5 ++++- crates/render/src/auto.rs | 21 +++++++++++++++++---- crates/render/src/pipeline.rs | 8 +++++++- crates/render/src/tests/auto.rs | 20 ++++++++++++++++++++ crates/ui-egui/src/frames.rs | 4 ++++ 5 files changed, 52 insertions(+), 6 deletions(-) diff --git a/crates/engine/src/background/remote.rs b/crates/engine/src/background/remote.rs index aea210ec5..72b4598a2 100644 --- a/crates/engine/src/background/remote.rs +++ b/crates/engine/src/background/remote.rs @@ -430,7 +430,10 @@ impl FrameServer { } } // A frame still waiting for readbacks is replaced: end it, so every `frame_begin` has its `frame_end`. - if self.pending.take().is_some() { + if let Some(old) = self.pending.take() { + if let (Some((key, chose)), Some(pick)) = (old.auto, a.auto_pick()) { + pick.abandoned(key, chose); + } a.frame_end(); } a.frame_begin(); diff --git a/crates/render/src/auto.rs b/crates/render/src/auto.rs index b1138588b..fc7becb5d 100644 --- a/crates/render/src/auto.rs +++ b/crates/render/src/auto.rs @@ -67,9 +67,7 @@ pub struct AutoPick { struct Entry { stats: AutoStats, measuring: [Option; 2], - // ponytail: a frame that never reports back (a GPU failure path, a replaced remote frame) - // leaves its side counted as running, which only makes that side look cheaper; capped at - // MAX_RUNNING, cleared by `AutoPick::reset`. + // Lowered by `record`, `declined` and `abandoned`: every frame asked for ends in one. running: [u32; 2], } @@ -109,7 +107,7 @@ impl AutoPick { const MEASURING_FOR: std::time::Duration = std::time::Duration::from_secs(10); /// Render the frame for `key` on the GPU? Every frame asked for reports back with - /// [`AutoPick::record`] (or [`AutoPick::declined`]). + /// [`AutoPick::record`] (or [`AutoPick::declined`]), or ends with [`AutoPick::abandoned`]. pub fn choose(&self, key: AutoKey) -> bool { let Ok(mut m) = self.stats.lock() else { return true }; if m.len() > Self::MAX_KEYS && !m.contains_key(&key) { @@ -194,6 +192,21 @@ impl AutoPick { } } + /// A frame for `key` asked of `gpu`'s side ends without a time (it failed, or was replaced + /// before it finished). Without this its side stays counted as running, and every later + /// frame there is timed as one of several running side by side: that side looks cheaper. + // ponytail: clears the side's warm-up/probe mark even when another frame holds it; that + // only lets the next probe start early. + pub fn abandoned(&self, key: AutoKey, gpu: bool) { + if let Ok(mut m) = self.stats.lock() + && let Some(e) = m.get_mut(&key) + { + e.measuring[usize::from(gpu)] = None; + let n = &mut e.running[usize::from(gpu)]; + *n = n.saturating_sub(1); + } + } + /// The history of `key`. pub fn stats(&self, key: AutoKey) -> Option { self.stats.lock().ok()?.get(&key).map(|e| e.stats) diff --git a/crates/render/src/pipeline.rs b/crates/render/src/pipeline.rs index fd46260c2..a77f8661d 100644 --- a/crates/render/src/pipeline.rs +++ b/crates/render/src/pipeline.rs @@ -587,7 +587,13 @@ impl<'a> Renderer<'a> { let auto = self.frame_auto(a, comp_id, false); if auto.is_none_or(|(p, key)| p.choose(key)) { let t0 = web_time::Instant::now(); - if let Some(img) = scope(a)? { + let res = scope(a); + if res.is_err() + && let Some((p, key)) = auto + { + p.abandoned(key, true); + } + if let Some(img) = res? { if let Some((p, key)) = auto { p.record(key, true, t0.elapsed().as_secs_f64() * 1e3); } diff --git a/crates/render/src/tests/auto.rs b/crates/render/src/tests/auto.rs index e7387eed3..da3d9f348 100644 --- a/crates/render/src/tests/auto.rs +++ b/crates/render/src/tests/auto.rs @@ -308,3 +308,23 @@ fn auto_pick_counts_frames_running_side_by_side() { assert!(s.cpu_ms.unwrap() < 30.0 && s.gpu_ms.unwrap() > 100.0, "{s:?}"); assert!(lanes.iter().filter(|g| **g).count() <= 1, "{lanes:?}"); } + +/// A frame that ends without a time (a GPU failure, a remote frame replaced before it finished) +/// no longer counts as running: the frames after it are timed as one at a time. (It stayed +/// counted, and every later frame on its side was timed as half its cost.) +#[test] +fn auto_pick_abandoned_frames_stop_counting_as_running() { + let pick = AutoPick::default(); + let key = AutoKey::new(ItemId(5), 1.0, false); + let mut dropped = [false; 2]; + for _ in 0..16 { + let gpu = pick.choose(key); + if !std::mem::replace(&mut dropped[usize::from(gpu)], true) { + pick.abandoned(key, gpu); + continue; + } + pick.record(key, gpu, if gpu { 40.0 } else { 10.0 }); + } + let s = pick.stats(key).unwrap(); + assert_eq!((s.cpu_ms, s.gpu_ms), (Some(10.0), Some(40.0)), "{s:?}"); +} diff --git a/crates/ui-egui/src/frames.rs b/crates/ui-egui/src/frames.rs index 2d51068b0..01b9824d6 100644 --- a/crates/ui-egui/src/frames.rs +++ b/crates/ui-egui/src/frames.rs @@ -2019,6 +2019,7 @@ impl Worker { // Time the GPU's work, not just its recording. g.wait(); if g.context().check_health().is_err() { + p.abandoned(key, true); return self.render_job(job, false, files); } p.record(key, true, t0.elapsed().as_secs_f64() * 1000.0); @@ -2031,6 +2032,9 @@ impl Worker { } } Err(e) => { + if let Some((p, key)) = auto { + p.abandoned(key, true); + } self.gpu_attempt_rejected(g, &e); return self.render_job(job, false, files); } From dd1aff17e57653f529f9f130bcce073ad6b4bb48 Mon Sep 17 00:00:00 2001 From: nazar-gavrylyk Date: Tue, 10 Nov 2026 14:14:57 +0100 Subject: [PATCH 13/24] Footage: 8-bit levels are read while the render records its reads footage_buf8 asked for a movie's 8-bit levels outside Renderer::read, so the media pool noted the movie in no recording: a layer buffer made of it listed no files and was still marked complete, and could be served stale after the movie changed on disk. --- crates/render/src/pipeline.rs | 2 +- crates/render/src/tests/footage_size.rs | 37 +++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/crates/render/src/pipeline.rs b/crates/render/src/pipeline.rs index a77f8661d..27103f19a 100644 --- a/crates/render/src/pipeline.rs +++ b/crates/render/src/pipeline.rs @@ -1056,7 +1056,7 @@ impl<'a> Renderer<'a> { if !plain { return None; } - let fr = self.footage.frame8(item, pf, t, self.pipe.float_curve(pf))?; + let fr = self.read(|| self.footage.frame8(item, pf, t, self.pipe.float_curve(pf)))?; let n = (fr.width as usize).checked_mul(fr.height as usize)?; let px = fr.rgba.as_chunks::<4>().0; if px.len() != n { diff --git a/crates/render/src/tests/footage_size.rs b/crates/render/src/tests/footage_size.rs index 25c1a1814..fbf776617 100644 --- a/crates/render/src/tests/footage_size.rs +++ b/crates/render/src/tests/footage_size.rs @@ -157,3 +157,40 @@ fn footage_kept_as_8_bit_levels_renders_the_same_pixels() { assert_eq!(q8.asked.into_inner() > 0, used, "{depth:?} {invert} {scale}"); } } + +/// The 8-bit levels are read while the render records its reads: the movie is in the files the +/// frame (and a cached layer buffer) was made of. (Read outside, a buffer of a movie listed no +/// files and was still marked complete: changed on disk, it was served stale.) +#[test] +fn footage_kept_as_8_bit_levels_is_noted_as_read() { + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + struct Noted { + clip: Clip8, + recording: AtomicBool, + outside: AtomicUsize, + } + impl FootageSource for Noted { + fn frame(&self, item: ItemId, f: &Footage, t: Tick) -> Option> { + self.clip.frame(item, f, t) + } + fn frame8(&self, item: ItemId, f: &Footage, t: Tick, float: crate::FloatCurve) -> Option> { + if !self.recording.load(Ordering::SeqCst) { + self.outside.fetch_add(1, Ordering::SeqCst); + } + self.clip.frame8(item, f, t, float) + } + fn recording(&self, _: &crate::reads::ReadLog, f: &mut dyn FnMut()) { + self.recording.store(true, Ordering::SeqCst); + f(); + self.recording.store(false, Ordering::SeqCst); + } + } + let (p, cid) = project(BitDepth::Bpc8, false); + let src = Noted { clip: Clip8 { asked: Default::default() }, recording: AtomicBool::new(false), outside: AtomicUsize::new(0) }; + let log = crate::reads::ReadLog::new(); + let mut r = Renderer::new(&p, &src, RenderOpts::default()); + r.reads = Some(&log); + r.comp_frame(cid, Tick::ZERO); + assert!(src.clip.asked.load(Ordering::SeqCst) > 0, "the 8-bit levels were not asked for"); + assert_eq!(src.outside.load(Ordering::SeqCst), 0, "8-bit levels read outside the render's recording"); +} From bbc5f9889d3571717fd20995a5d283a00595fa01 Mon Sep 17 00:00:00 2001 From: nazar-gavrylyk Date: Tue, 10 Nov 2026 14:16:51 +0100 Subject: [PATCH 14/24] Preview: a cached end of a preview that doesn't loop plays in real time The paced clock counted the cached lead out of twice the prefetch depth even when fewer frames were left before the end, so a fully cached preview that doesn't loop slowed down over its last frames, and one shorter than that depth played slow throughout (and could step down). The lead now counts out of the frames left. --- crates/ui-egui/src/app/prefetch.rs | 45 +++++++++++++++++++++++------- 1 file changed, 35 insertions(+), 10 deletions(-) diff --git a/crates/ui-egui/src/app/prefetch.rs b/crates/ui-egui/src/app/prefetch.rs index dbde3f085..7f6494d3f 100644 --- a/crates/ui-egui/src/app/prefetch.rs +++ b/crates/ui-egui/src/app/prefetch.rs @@ -54,6 +54,15 @@ fn pace_for(prev: f64, lead: i64, target: i64, fps: f64, dt: f64) -> f64 { prev + (raw - prev) * k } +/// Cached frames (`cached`, sorted) among the next `max` frames `after(1..=max)` plays, and how +/// many frames it plays (`after` is `None` past the end). Counted, not the consecutive run: one +/// late frame (a GPU warm-up frame, a heavy one) with frames cached beyond it holds the clock +/// for a moment when it gets there, rather than slowing the whole preview to a crawl. +fn lead_of(after: impl Fn(i64) -> Option, cached: &[i64], max: i64) -> (i64, i64) { + let ahead: Vec = (1..=max).map_while(after).collect(); + (ahead.iter().filter(|f| cached.binary_search(f).is_ok()).count() as i64, ahead.len() as i64) +} + impl PlaybackPass<'_> { /// The frame `k` played frames after `from` (wrapping as the preview loops), `None` once a /// preview that doesn't loop has reached its end. @@ -430,9 +439,8 @@ impl EffectcraftApp { self.playback.paced_at = Some(now); if video { // The lead counts out of twice the prefetch depth: the prefetch keeps that many - // frames coming, so a full lead means rendering keeps up. - let target = (pass.ahead * 2).max(1); - let lead = self.cached_lead(pass, target); + // frames coming (or the frames left before the end), so a full lead means rendering keeps up. + let (lead, target) = self.cached_lead(pass, (pass.ahead * 2).max(1)); let prev = if self.playback.pace > 0.0 { self.playback.pace } else { fps }; let pace = pace_for(prev, lead, target, fps, dt); self.playback.pace = pace; @@ -522,12 +530,11 @@ impl EffectcraftApp { ctx.request_repaint(); } - /// Cached frames among the next `max` in play order. Counted, not the consecutive run: one - /// late frame (a GPU warm-up frame, a heavy one) with frames cached beyond it holds the clock - /// for a moment when it gets there, rather than slowing the whole preview to a crawl. - fn cached_lead(&self, pass: &PlaybackPass<'_>, max: i64) -> i64 { - let cached = self.frames.cached_frames(&pass.series); - (1..=max).map_while(|k| pass.frame_after(pass.cur, k)).filter(|f| cached.binary_search(f).is_ok()).count() as i64 + /// Cached frames among the next `max` in play order, and how many of those there are (fewer + /// near the end of a preview that doesn't loop: its cached tail is a full lead). See + /// [`lead_of`]. + fn cached_lead(&self, pass: &PlaybackPass<'_>, max: i64) -> (i64, i64) { + lead_of(|k| pass.frame_after(pass.cur, k), &self.frames.cached_frames(&pass.series), max) } /// Settings ▸ Previews ▸ Lower Resolution When Playback Can't Keep Up: a preview at @@ -654,7 +661,7 @@ impl EffectcraftApp { #[cfg(test)] mod pace_tests { - use super::pace_for; + use super::{lead_of, pace_for}; /// The pace follows the cached lead: real time with a full lead, a third of it with a third, /// never under an eighth, unchanged with nothing ahead, and smoothed over time. @@ -671,6 +678,24 @@ mod pace_tests { assert_eq!(pace_for(30.0, 4, 12, 30.0, 0.0), 30.0, "no time passed: unchanged"); assert!(pace_for(10.0, 12, 12, 30.0, 0.5) > 10.0, "and up again as the lead fills"); } + + /// Near the end of a preview that doesn't loop, the lead counts out of the frames left: a + /// cached tail plays in real time. (It counted out of the full depth, and a cached preview + /// slowed down over its last frames, a short one for its whole length.) + #[test] + fn a_cached_end_is_a_full_lead() { + let cached: Vec = (0..=20).collect(); + let to_20 = |cur: i64| move |k: i64| Some(cur + k).filter(|f| *f <= 20); + assert_eq!(lead_of(to_20(5), &cached, 12), (12, 12)); + assert_eq!(lead_of(to_20(16), &cached, 12), (4, 4)); + assert_eq!(lead_of(to_20(20), &cached, 12), (0, 0)); + let (lead, target) = lead_of(to_20(16), &cached, 12); + assert!((pace_for(30.0, lead, target, 30.0, 10.0) - 30.0).abs() < 1e-6); + // Frames left uncached still count against it. + assert_eq!(lead_of(to_20(16), &[17, 19], 12), (2, 4)); + // Looping: the walk wraps, always `max` frames. + assert_eq!(lead_of(|k| Some((18 + k) % 21), &cached, 12), (12, 12)); + } } #[cfg(test)] From e4c49f9bc08749ee59cd755c3246dbce94096ed3 Mon Sep 17 00:00:00 2001 From: nazar-gavrylyk Date: Tue, 10 Nov 2026 14:20:01 +0100 Subject: [PATCH 15/24] Disk cache: queued layer writes hold at most 1 GB Layer jobs keep their float buffer until the writer compresses it, so the 64-job limit could hold ~8.5 GB of evicted 4K buffers. Layer and matte writes now also have a byte budget, like frames; a single bigger buffer is still taken when nothing else is queued. --- crates/render/src/disk_cache.rs | 63 +++++++++++++++++++++++++++++---- 1 file changed, 56 insertions(+), 7 deletions(-) diff --git a/crates/render/src/disk_cache.rs b/crates/render/src/disk_cache.rs index fa0a97696..5c34348fe 100644 --- a/crates/render/src/disk_cache.rs +++ b/crates/render/src/disk_cache.rs @@ -58,6 +58,10 @@ const QUEUE_LIMIT: usize = 64; /// frames, so a fully cached comp came back with gaps in its blue bar. #[cfg(not(target_arch = "wasm32"))] const FRAME_QUEUE_BYTES: usize = 1 << 30; +/// Pending layer and matte bytes before new ones are dropped: a layer job holds its float +/// buffer until the writer compresses it (64 of them at 4K were ~8.5 GB). +#[cfg(not(target_arch = "wasm32"))] +const LAYER_QUEUE_BYTES: usize = 1 << 30; #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub enum Kind { @@ -304,8 +308,10 @@ struct Queue { frames: std::collections::VecDeque, #[cfg_attr(target_arch = "wasm32", allow(dead_code))] frame_bytes: usize, - /// Layer and matte writes. + /// Layer and matte writes, and their payload bytes. jobs: std::collections::VecDeque, + #[cfg_attr(target_arch = "wasm32", allow(dead_code))] + job_bytes: usize, busy: bool, /// Keys of frames queued or being written. pending_frames: std::collections::HashSet, @@ -360,7 +366,21 @@ impl Queue { } return Some(j); } - self.jobs.pop_front() + let j = self.jobs.pop_front()?; + if let Job::Write(_, _, _, p) = &j { + self.job_bytes = self.job_bytes.saturating_sub(p.len()); + } + Some(j) + } + + /// No room for a `kind` write of `len` bytes. A layer bigger than the whole budget is still + /// taken when nothing else is queued. + #[cfg(not(target_arch = "wasm32"))] + fn full(&self, kind: Kind, len: usize) -> bool { + match kind { + Kind::Frame => self.frame_bytes.saturating_add(len) > FRAME_QUEUE_BYTES, + _ => self.jobs.len() >= QUEUE_LIMIT || (self.job_bytes > 0 && self.job_bytes.saturating_add(len) > LAYER_QUEUE_BYTES), + } } fn is_empty(&self) -> bool { self.frames.is_empty() && self.jobs.is_empty() @@ -492,6 +512,7 @@ impl DiskCache { frames: Default::default(), frame_bytes: 0, jobs: Default::default(), + job_bytes: 0, busy: cfg!(not(target_arch = "wasm32")), pending_frames: Default::default(), drops: Drops::default(), @@ -680,6 +701,7 @@ impl DiskCache { #[cfg(target_arch = "wasm32")] { q.jobs.clear(); + q.job_bytes = 0; q.frames.clear(); q.pending_frames.clear(); } @@ -824,11 +846,7 @@ impl DiskCache { if stale(&q) { return false; } - let full = match kind { - Kind::Frame => q.frame_bytes.saturating_add(payload.len()) > FRAME_QUEUE_BYTES, - _ => q.jobs.len() >= QUEUE_LIMIT, - }; - if full { + if q.full(kind, payload.len()) { self.dropped.fetch_add(1, Ordering::Relaxed); return false; } @@ -837,6 +855,7 @@ impl DiskCache { q.frame_bytes = q.frame_bytes.saturating_add(payload.len()); q.frames.push_back(Job::Write(kind, key, format, payload)); } else { + q.job_bytes = q.job_bytes.saturating_add(payload.len()); q.jobs.push_back(Job::Write(kind, key, format, payload)); } cv.notify_all(); @@ -1824,6 +1843,36 @@ mod tests { assert_ne!(comp_content_key(&p, cid), key, "the expression reads the model's name"); } + /// Layer jobs hold float buffers until they are compressed: the queue holds at most + /// [`LAYER_QUEUE_BYTES`] of them (64 at 4K held ~8.5 GB), and a single bigger one only alone. + #[test] + fn layer_writes_queue_within_a_byte_budget() { + let mut q = Queue { + frames: Default::default(), + frame_bytes: 0, + jobs: Default::default(), + job_bytes: 0, + busy: true, + pending_frames: Default::default(), + drops: Drops::default(), + }; + let layer = LAYER_QUEUE_BYTES / 4 + 1; + for _ in 0..3 { + assert!(!q.full(Kind::Layer, layer)); + q.jobs.push_back(Job::Write(Kind::Layer, 0, 0, Payload::Encoded(vec![]))); + q.job_bytes += layer; + } + assert!(q.full(Kind::Layer, layer), "a fourth would pass the budget"); + assert!(!q.full(Kind::Frame, 1024), "frames have their own budget"); + let mut empty = Queue { jobs: Default::default(), job_bytes: 0, ..q }; + assert!(!empty.full(Kind::Layer, 2 * LAYER_QUEUE_BYTES), "a single big layer, alone"); + // Taking a job gives its bytes back. + empty.job_bytes = 3; + empty.jobs.push_back(Job::Write(Kind::Layer, 0, 0, Payload::Encoded(vec![0; 3]))); + assert!(empty.pop().is_some()); + assert_eq!(empty.job_bytes, 0); + } + /// A burst of layer writes never crowds out frames: every frame of a fully cached comp is /// on disk afterwards (the blue bar came back with gaps). #[test] From 21d54b2a1afbea7235dedaeaacb7451e1d79df4f Mon Sep 17 00:00:00 2001 From: nazar-gavrylyk Date: Tue, 10 Nov 2026 14:20:01 +0100 Subject: [PATCH 16/24] Gaussian Blur: the bit-identity test covers tiles past the first A 300-pixel-wide case runs the row-by-row pass over tiles at x 128 and 256 and a partial last tile. --- crates/raster/src/blur.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/crates/raster/src/blur.rs b/crates/raster/src/blur.rs index 9b785959f..d92148658 100644 --- a/crates/raster/src/blur.rs +++ b/crates/raster/src/blur.rs @@ -339,9 +339,10 @@ mod tests { #[test] fn gauss_pass_is_bit_identical_to_the_per_pixel_kernel() { - // Sizes narrower and shorter than the kernel too (every tap near an edge), with - // negative, zero and HDR values. - for (w, h) in [(37, 23), (3, 2), (1, 9), (64, 1)] { + // Sizes narrower and shorter than the kernel too (every tap near an edge), one wider + // than three tiles (tiles past the first, and a partial last one), with negative, zero + // and HDR values. + for (w, h) in [(37, 23), (3, 2), (1, 9), (64, 1), (300, 5)] { let mut img = Image::new(w, h); for (i, p) in img.data.iter_mut().enumerate() { let v = ((i * 7919) % 101) as f32 / 37.0 - 0.6; From e8b94271c46338c218676057d55058c1134ab423 Mon Sep 17 00:00:00 2001 From: nazar-gavrylyk Date: Tue, 10 Nov 2026 14:20:01 +0100 Subject: [PATCH 17/24] Footage: the 8-bit levels test checks the movie is in the frame's files The test source notes its file while a read is recorded, as the media pool does, and the test checks the render's read log lists it. --- crates/render/src/tests/footage_size.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/render/src/tests/footage_size.rs b/crates/render/src/tests/footage_size.rs index fbf776617..6c59f3fb2 100644 --- a/crates/render/src/tests/footage_size.rs +++ b/crates/render/src/tests/footage_size.rs @@ -179,7 +179,9 @@ fn footage_kept_as_8_bit_levels_is_noted_as_read() { } self.clip.frame8(item, f, t, float) } - fn recording(&self, _: &crate::reads::ReadLog, f: &mut dyn FnMut()) { + // (as the media pool does: the files of what `f` reads go to `reads`) + fn recording(&self, reads: &crate::reads::ReadLog, f: &mut dyn FnMut()) { + reads.note(crate::reads::ReadFile::new("clip.mov", crate::reads::FileStamp::Native { len: 1, modified: Some(1) })); self.recording.store(true, Ordering::SeqCst); f(); self.recording.store(false, Ordering::SeqCst); @@ -193,4 +195,5 @@ fn footage_kept_as_8_bit_levels_is_noted_as_read() { r.comp_frame(cid, Tick::ZERO); assert!(src.clip.asked.load(Ordering::SeqCst) > 0, "the 8-bit levels were not asked for"); assert_eq!(src.outside.load(Ordering::SeqCst), 0, "8-bit levels read outside the render's recording"); + assert!(log.files().files().iter().any(|f| &*f.path == "clip.mov"), "the movie is not in the frame's files"); } From db21c733b489965e8fb56bff0a11b542f35a47ed Mon Sep 17 00:00:00 2001 From: nazar-gavrylyk Date: Tue, 10 Nov 2026 15:07:57 +0100 Subject: [PATCH 18/24] Auto: a frame's ticket ends it on every exit AutoPick::ticket chooses a side and reports the frame's time once; dropped without one it ends the frame unmeasured. The viewer's frames, the renderer's top-level frames and frames rendered in passes use it, so an Auto CPU frame turned away by GPU admission (#3028), a cancelled export frame or any later exit no longer leaves its frame counted as running. The frame server keeps its choice across calls and still ends a replaced frame itself. --- crates/render/src/auto.rs | 52 +++++++++++++++++++++++++++++++-- crates/render/src/lib.rs | 2 +- crates/render/src/passes.rs | 12 ++++---- crates/render/src/pipeline.rs | 30 +++++++++---------- crates/render/src/tests/auto.rs | 32 ++++++++++++++++++++ crates/ui-egui/src/frames.rs | 27 ++++++++--------- 6 files changed, 116 insertions(+), 39 deletions(-) diff --git a/crates/render/src/auto.rs b/crates/render/src/auto.rs index fc7becb5d..ce714c1f0 100644 --- a/crates/render/src/auto.rs +++ b/crates/render/src/auto.rs @@ -67,7 +67,8 @@ pub struct AutoPick { struct Entry { stats: AutoStats, measuring: [Option; 2], - // Lowered by `record`, `declined` and `abandoned`: every frame asked for ends in one. + // Lowered by `record`, `declined` and `abandoned`: every frame asked for ends in one (an + // [`AutoTicket`] makes sure of it). running: [u32; 2], } @@ -106,8 +107,16 @@ impl AutoPick { /// A warm-up or re-probe frame not reported back by then no longer holds its side. const MEASURING_FOR: std::time::Duration = std::time::Duration::from_secs(10); + /// [`AutoPick::choose`] for a frame that ends where it started: the ticket reports back, and + /// a ticket dropped without a time (an error, a retry elsewhere, a panic, a cancelled future) + /// ends the frame with [`AutoPick::abandoned`]. + pub fn ticket(&self, key: AutoKey) -> AutoTicket<'_> { + AutoTicket { pick: self, key, gpu: self.choose(key), open: true } + } + /// Render the frame for `key` on the GPU? Every frame asked for reports back with - /// [`AutoPick::record`] (or [`AutoPick::declined`]), or ends with [`AutoPick::abandoned`]. + /// [`AutoPick::record`] (or [`AutoPick::declined`]), or ends with [`AutoPick::abandoned`] + /// ([`AutoPick::ticket`] does that on its own). pub fn choose(&self, key: AutoKey) -> bool { let Ok(mut m) = self.stats.lock() else { return true }; if m.len() > Self::MAX_KEYS && !m.contains_key(&key) { @@ -219,3 +228,42 @@ impl AutoPick { } } } + +/// One frame [`AutoPick`] chose a side for ([`AutoPick::ticket`]): it reports the frame's time +/// once, and ends the frame unmeasured when dropped without one. +#[must_use = "a ticket dropped at once ends its frame unmeasured"] +pub struct AutoTicket<'a> { + pick: &'a AutoPick, + key: AutoKey, + gpu: bool, + open: bool, +} + +impl AutoTicket<'_> { + /// The frame renders on the GPU. + pub fn gpu(&self) -> bool { + self.gpu + } + + /// The frame took `ms` on its side. + pub fn record(mut self, ms: f64) { + self.open = false; + self.pick.record(self.key, self.gpu, ms); + } + + /// The GPU did not take the frame: it renders on the CPU, and the ticket reports from there. + pub fn declined(&mut self) { + if self.gpu { + self.pick.declined(self.key); + self.gpu = false; + } + } +} + +impl Drop for AutoTicket<'_> { + fn drop(&mut self) { + if self.open { + self.pick.abandoned(self.key, self.gpu); + } + } +} diff --git a/crates/render/src/lib.rs b/crates/render/src/lib.rs index d1be87a69..f47443b18 100644 --- a/crates/render/src/lib.rs +++ b/crates/render/src/lib.rs @@ -26,7 +26,7 @@ pub mod three_d; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; -pub use auto::{AutoKey, AutoPick}; +pub use auto::{AutoKey, AutoPick, AutoTicket}; pub use cache::{CacheStats, LayerCache, LayerStore, PrefetchStore}; use effectcraft_color::BlendMode; use effectcraft_effects::{Buf, EffectCtx, EffectEnv, EffectHost, LayerPixels, Params}; diff --git a/crates/render/src/passes.rs b/crates/render/src/passes.rs index 37c94fff4..b8f94fc46 100644 --- a/crates/render/src/passes.rs +++ b/crates/render/src/passes.rs @@ -86,10 +86,10 @@ pub async fn comp_frame(r: &Renderer<'_>, comp: ItemId, t: Tick) -> Image { } else if opts.roi.is_none() && let Some(pick) = a.auto_pick() { - let key = AutoKey::new(comp, opts.scale, false); - let gpu = pick.choose(key); - opts.backend = if gpu { Backend::Gpu } else { Backend::Cpu }; - auto = Some((key, gpu)); + // (a frame cancelled or panicking between passes drops it: ended unmeasured) + let ticket = pick.ticket(AutoKey::new(comp, opts.scale, false)); + opts.backend = if ticket.gpu() { Backend::Gpu } else { Backend::Cpu }; + auto = Some(ticket); } } let t0 = web_time::Instant::now(); @@ -98,8 +98,8 @@ pub async fn comp_frame(r: &Renderer<'_>, comp: ItemId, t: Tick) -> Image { } else { in_passes(Some(a), |acc| Renderer { opts, accel: acc, ..*r }.comp_frame(comp, t)).await.value }; - if let (Some((key, chose)), Some(pick)) = (auto, a.auto_pick()) { - pick.record(key, chose, t0.elapsed().as_secs_f64() * 1e3); + if let Some(t) = auto { + t.record(t0.elapsed().as_secs_f64() * 1e3); } img } diff --git a/crates/render/src/pipeline.rs b/crates/render/src/pipeline.rs index 27103f19a..29ce6d69a 100644 --- a/crates/render/src/pipeline.rs +++ b/crates/render/src/pipeline.rs @@ -5,8 +5,9 @@ use super::fx_host::{FxHost, essential_overrides}; use super::opts::{Collapse3d, LayerTiming, MAX_THROUGH, RenderOpts, Through}; use super::reads::{ReadFiles, ReadLog}; use super::{ - Arc, AutoKey, AutoPick, BlendMode, Buf, Comp, EffectCtx, EffectEnv, EvalCtx, Footage, FootageKind, FrameBlend, FrameRate, GroupKind, Image, ItemId, - ItemKind, Layer, LayerCache, LayerSource, Params, Project, Quality, Region, Renderer, Tick, cache, color, masks, shapes, styles, text, three_d, vec2, + Arc, AutoKey, AutoPick, AutoTicket, BlendMode, Buf, Comp, EffectCtx, EffectEnv, EvalCtx, Footage, FootageKind, FrameBlend, FrameRate, GroupKind, Image, + ItemId, ItemKind, Layer, LayerCache, LayerSource, Params, Project, Quality, Region, Renderer, Tick, cache, color, masks, shapes, styles, text, three_d, + vec2, }; use effectcraft_keyframe::Value; use effectcraft_project::LayerId; @@ -584,29 +585,24 @@ impl<'a> Renderer<'a> { if self.depth == 0 && let Some(a) = self.active_accel() { - let auto = self.frame_auto(a, comp_id, false); - if auto.is_none_or(|(p, key)| p.choose(key)) { + // (an error returns with the ticket, which ends the frame unmeasured) + let mut ticket = self.frame_auto(a, comp_id, false).map(|(p, key)| p.ticket(key)); + if ticket.as_ref().is_none_or(AutoTicket::gpu) { let t0 = web_time::Instant::now(); - let res = scope(a); - if res.is_err() - && let Some((p, key)) = auto - { - p.abandoned(key, true); - } - if let Some(img) = res? { - if let Some((p, key)) = auto { - p.record(key, true, t0.elapsed().as_secs_f64() * 1e3); + if let Some(img) = scope(a)? { + if let Some(t) = ticket { + t.record(t0.elapsed().as_secs_f64() * 1e3); } return Ok(img); } - if let Some((p, key)) = auto { - p.declined(key); + if let Some(t) = &mut ticket { + t.declined(); } } let t0 = web_time::Instant::now(); let img = cpu.comp_frame_cpu(comp_id, t); - if let Some((p, key)) = auto { - p.record(key, false, t0.elapsed().as_secs_f64() * 1e3); + if let Some(t) = ticket { + t.record(t0.elapsed().as_secs_f64() * 1e3); } return Ok(img); } diff --git a/crates/render/src/tests/auto.rs b/crates/render/src/tests/auto.rs index da3d9f348..a53bb718a 100644 --- a/crates/render/src/tests/auto.rs +++ b/crates/render/src/tests/auto.rs @@ -328,3 +328,35 @@ fn auto_pick_abandoned_frames_stop_counting_as_running() { let s = pick.stats(key).unwrap(); assert_eq!((s.cpu_ms, s.gpu_ms), (Some(10.0), Some(40.0)), "{s:?}"); } + +/// A ticket that never reports (dropped on an error, a retry elsewhere, a cancelled export +/// future) ends its frame like [`AutoPick::abandoned`]; one the GPU declined reports from the +/// CPU. (Each exit that skipped `record` left its frame counted as running, and every later +/// frame on that side was timed as one of several side by side.) +#[test] +fn auto_tickets_end_their_frame_on_every_exit() { + let pick = AutoPick::default(); + let key = AutoKey::new(ItemId(6), 1.0, false); + let mut dropped = [false; 2]; + for _ in 0..16 { + let t = pick.ticket(key); + let gpu = t.gpu(); + if !std::mem::replace(&mut dropped[usize::from(gpu)], true) { + drop(t); + continue; + } + t.record(if gpu { 40.0 } else { 10.0 }); + } + let s = pick.stats(key).unwrap(); + assert_eq!((s.cpu_ms, s.gpu_ms), (Some(10.0), Some(40.0)), "{s:?}"); + // Declined: the frame renders on the CPU and reports there, one frame on it. + let key = AutoKey::new(ItemId(7), 1.0, false); + pick.ticket(key).record(10.0); + let mut t = pick.ticket(key); + assert!(t.gpu(), "the GPU's warm-up frame"); + t.declined(); + assert!(!t.gpu()); + t.record(10.0); + pick.ticket(key).record(10.0); + assert_eq!(pick.stats(key).unwrap().cpu_ms, Some(10.0), "timed as one frame on the CPU"); +} diff --git a/crates/ui-egui/src/frames.rs b/crates/ui-egui/src/frames.rs index 01b9824d6..c6f48ce0b 100644 --- a/crates/ui-egui/src/frames.rs +++ b/crates/ui-egui/src/frames.rs @@ -38,7 +38,7 @@ use std::sync::{Arc, Condvar, Mutex, PoisonError}; use effectcraft_engine::project::{Comp, Heard, ItemId, ItemKind, Node, Project, PropGroup}; use effectcraft_engine::render::disk_cache::{self, DiskCache}; use effectcraft_engine::render::reads::{ReadFiles, ReadLog}; -use effectcraft_engine::render::{CancellableFootage, ExprHost, FootageSource, LayerCache, RenderOpts, Renderer}; +use effectcraft_engine::render::{AutoTicket, CancellableFootage, ExprHost, FootageSource, LayerCache, RenderOpts, Renderer}; use effectcraft_engine::time::Tick; use effectcraft_gpu::{DisplayFrame, FrameError, Gpu}; use rayon::prelude::*; @@ -2009,32 +2009,32 @@ impl Worker { { // The browser cannot wait for the GPU to time it (and its CPU path is slow): always // the GPU there. - let auto = if cfg!(target_arch = "wasm32") { None } else { r.frame_auto(a, job.comp, true) }; - if auto.is_none_or(|(p, key)| p.choose(key)) { + // Every exit that doesn't record ends the frame unmeasured: the ticket is dropped + // before a retry (`take`), or on return. + let mut ticket = if cfg!(target_arch = "wasm32") { None } else { r.frame_auto(a, job.comp, true).map(|(p, key)| p.ticket(key)) }; + if ticket.as_ref().is_none_or(AutoTicket::gpu) { let t0 = web_time::Instant::now(); match gpu_attempt(g, || g.render_display(&r, job.comp, job.t)) { Ok(Some(f)) => { self.gpu_succeeded(); - if let Some((p, key)) = auto { + if let Some(t) = ticket.take() { // Time the GPU's work, not just its recording. g.wait(); if g.context().check_health().is_err() { - p.abandoned(key, true); + drop(t); return self.render_job(job, false, files); } - p.record(key, true, t0.elapsed().as_secs_f64() * 1000.0); + t.record(t0.elapsed().as_secs_f64() * 1000.0); } return FrameImage::Gpu(Arc::new(f)); } Ok(None) => { - if let Some((p, key)) = auto { - p.declined(key); + if let Some(t) = &mut ticket { + t.declined(); } } Err(e) => { - if let Some((p, key)) = auto { - p.abandoned(key, true); - } + drop(ticket.take()); self.gpu_attempt_rejected(g, &e); return self.render_job(job, false, files); } @@ -2046,12 +2046,13 @@ impl Worker { let img = match gpu_attempt(g, || r.comp_frame_cpu(job.comp, job.t)) { Ok(img) => cpu_frame(&img, &job.src.project, &job.opts), Err(e) => { + drop(ticket.take()); self.gpu_attempt_rejected(g, &e); return self.render_job(job, false, files); } }; - if let Some((p, key)) = auto { - p.record(key, false, t0.elapsed().as_secs_f64() * 1000.0); + if let Some(t) = ticket { + t.record(t0.elapsed().as_secs_f64() * 1000.0); } return img; } From b18bf9186a89fbd37cf42da753c7b32d20f0bfd6 Mon Sep 17 00:00:00 2001 From: nazar-gavrylyk Date: Tue, 10 Nov 2026 15:09:40 +0100 Subject: [PATCH 19/24] Preview: GPU admission's reasoning holds for lanes four threads wide Only a lane's drain thread waits for a GPU slot. A lane runs one drain at a time (it is idle, and started again, only once its drain found no job), and its other threads only help with that frame's parallel work, as Rayon helpers that never wait. The comments said lanes had one thread; the test now runs on a lane LANE_WIDTH wide and checks that its other threads decline while the drain thread waits. --- crates/ui-egui/src/frames.rs | 10 +++++++--- crates/ui-egui/src/frames/tests.rs | 17 ++++++++++++----- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/crates/ui-egui/src/frames.rs b/crates/ui-egui/src/frames.rs index c6f48ce0b..2b2b3d4b7 100644 --- a/crates/ui-egui/src/frames.rs +++ b/crates/ui-egui/src/frames.rs @@ -101,8 +101,12 @@ impl GpuRenderSlots { /// A Rayon helper may be executing a nested frame while an admitted render waits for /// its work. Blocking that helper on a permit would deadlock the render (#3028). fn for_render(&self, max: usize) -> Option> { - // Dedicated preview pools have one thread, protected from nested drains. They cannot - // be helpers of another admitted frame, so waiting preserves GPU preview throughput. + // Only a lane's drain thread waits: it renders one frame at a time, protected from + // nested drains, and a lane runs one drain at a time (it is idle again, and can be + // started anew, only once its drain found no job, so no frame of the lane is still + // running). The lane's other threads ([`LANE_WIDTH`]) only help with that frame's + // parallel work and are Rayon helpers here: they never wait. So no thread waits while + // an admitted frame waits for it, and waiting keeps the GPU preview's throughput. #[cfg(not(target_arch = "wasm32"))] if DRAINING_LANE.get() { return Some(self.acquire(max)); @@ -1720,7 +1724,7 @@ struct Worker { thread_local! { /// This pool thread is inside [`Worker::drain`] (rendering a frame). static DRAINING: std::cell::Cell = const { std::cell::Cell::new(false) }; - /// Only the single thread of a dedicated preview lane may wait for GPU admission. + /// Only a lane's drain thread may wait for GPU admission (see [`GpuRenderSlots::for_render`]). static DRAINING_LANE: std::cell::Cell = const { std::cell::Cell::new(false) }; } diff --git a/crates/ui-egui/src/frames/tests.rs b/crates/ui-egui/src/frames/tests.rs index 0a148b83c..baf6342e7 100644 --- a/crates/ui-egui/src/frames/tests.rs +++ b/crates/ui-egui/src/frames/tests.rs @@ -3325,18 +3325,19 @@ fn gpu_render_slots_never_block_rayon_helpers() { assert_eq!(*slots.busy.lock().unwrap(), 0); } -/// Dedicated preview lanes have one protected thread and can wait without creating the -/// shared-pool helper dependency cycle. They must keep rendering on a healthy GPU. +/// A lane's drain thread can wait without creating the shared-pool helper dependency cycle, +/// and must keep rendering on a healthy GPU; the lane's other threads ([`LANE_WIDTH`] wide) +/// help with its frame and never wait. #[cfg(not(target_arch = "wasm32"))] #[test] fn gpu_preview_lane_waits_for_admission_and_resets_its_marker() { let slots = GpuRenderSlots::new(); let first = slots.acquire(2); let second = slots.acquire(2); - let pool = rayon::ThreadPoolBuilder::new().num_threads(1).build().unwrap(); + let pool = rayon::ThreadPoolBuilder::new().num_threads(LANE_WIDTH).build().unwrap(); let (started, ready) = std::sync::mpsc::channel(); let (tx, rx) = std::sync::mpsc::channel(); - let (early, admitted) = std::thread::scope(|scope| { + let (early, admitted, helper) = std::thread::scope(|scope| { scope.spawn(|| { pool.install(|| { let _drain = Draining::enter(Some(0)).unwrap(); @@ -3347,13 +3348,19 @@ fn gpu_preview_lane_waits_for_admission_and_resets_its_marker() { }); }); ready.recv_timeout(std::time::Duration::from_secs(2)).unwrap(); + // Another thread of the same lane, while its drain thread waits: it declines. + let (htx, hrx) = std::sync::mpsc::channel(); + let (pool, slots) = (&pool, &slots); + scope.spawn(move || pool.install(|| htx.send(slots.for_render(2).is_none()).unwrap())); + let helper = hrx.recv_timeout(std::time::Duration::from_secs(2)); let early = rx.recv_timeout(std::time::Duration::from_millis(100)); drop((first, second)); let admitted = early.or_else(|_| rx.recv_timeout(std::time::Duration::from_secs(2))); - (early, admitted) + (early, admitted, helper) }); assert!(early.is_err(), "busy dedicated lane must wait instead of falling back to CPU"); assert_eq!(admitted, Ok(true)); + assert_eq!(helper, Ok(true), "a lane's helper thread must not wait on GPU permits"); pool.install(|| { assert!(!DRAINING.get()); assert!(!DRAINING_LANE.get()); From 513d5eaea7310ba72692c3ae59fd9e2e28e74e54 Mon Sep 17 00:00:00 2001 From: nazar-gavrylyk Date: Tue, 10 Nov 2026 15:10:24 +0100 Subject: [PATCH 20/24] Preview: Lower Resolution When Playback Can't Keep Up is off by default It never steps back up during a play, so a looping preview kept playing at a quarter of its resolution after its first loop. Without it the paced first loop caches the viewer's resolution and the next loops play in real time, as in After Effects. The setting stays for whoever prefers a lower first pass. --- crates/engine/src/settings/prefs/sections.rs | 7 +++++-- crates/ui-egui/tests/it/ui_preview_align.rs | 4 +++- docs/preferences.md | 4 +++- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/crates/engine/src/settings/prefs/sections.rs b/crates/engine/src/settings/prefs/sections.rs index ee86ce4c4..ca0e72bef 100644 --- a/crates/engine/src/settings/prefs/sections.rs +++ b/crates/engine/src/settings/prefs/sections.rs @@ -93,8 +93,11 @@ page!(Previews { /// Largest downsampling while playing back: `1/2`, `1/4`, `1/8`. adaptive_resolution_limit: String = "1/8".into(), /// A preview at Resolution Auto that renders slower than real time steps down a resolution - /// (½, then ¼, no further than the Adaptive Resolution Limit) for the rest of that play. - adaptive_playback: bool = true, + /// (½, then ¼, no further than the Adaptive Resolution Limit) for the rest of that play. Off + /// by default: it never steps back up during a play, so a looping preview kept playing at ¼ + /// after its first loop, where without it the first loop caches Full and the next ones play + /// in real time at Full, as in After Effects. + adaptive_playback: bool = false, show_internal_wireframes: bool = false, cache_frames_when_idle: bool = true, fast_previews: bool = false, diff --git a/crates/ui-egui/tests/it/ui_preview_align.rs b/crates/ui-egui/tests/it/ui_preview_align.rs index 6b5525e09..9d3119b21 100644 --- a/crates/ui-egui/tests/it/ui_preview_align.rs +++ b/crates/ui-egui/tests/it/ui_preview_align.rs @@ -322,7 +322,7 @@ fn cache_frames_when_idle_survives_an_inverted_work_area() { /// Settings ▸ Previews ▸ Lower Resolution When Playback Can't Keep Up: a preview at Auto that /// stalls for two seconds steps down to half its scale, two more to a quarter; an explicit Resolution and -/// the pref turned off keep the scale, and stopping goes back to the viewer's. +/// the pref turned off (its default) keep the scale, and stopping goes back to the viewer's. #[test] fn playback_that_cannot_keep_up_steps_down_a_resolution() { let stall = |h: &mut Harness<'_, EffectcraftApp>, secs: f64| { @@ -338,6 +338,8 @@ fn playback_that_cannot_keep_up_steps_down_a_resolution() { assert!(h.state().playback.playing); }; let mut h = harness(); + assert!(!h.state().session.prefs.previews.adaptive_playback, "off by default"); + h.state_mut().session.prefs.previews.adaptive_playback = true; // No room in the RAM preview: frames never stay cached and every pass stalls, as with a comp // too heavy to render in real time. h.state().frames.set_budget(0); diff --git a/docs/preferences.md b/docs/preferences.md index cad74aec2..07df169aa 100644 --- a/docs/preferences.md +++ b/docs/preferences.md @@ -53,7 +53,9 @@ Commands (CLI, MCP, control channel): - `composition.showRenderingProgress`: Show Rendering Progress in Info Panel and Flowchart - `previews.adaptiveResolutionLimit`: Adaptive Resolution Limit (the lowest resolution Fast Previews ▸ Adaptive Resolution drops to while you drag, and the lowest a playing preview steps down to) -- `previews.adaptivePlayback`: Lower Resolution When Playback Can't Keep Up. On by default. A +- `previews.adaptivePlayback`: Lower Resolution When Playback Can't Keep Up. Off by default (without it a first play renders + at the viewer's resolution at the rate frames arrive, and a looping preview plays its later loops + from the cache in real time, as in After Effects; with it the play stays lowered). A preview at Resolution Auto that plays under 80 % of its frame rate (frames render slower than that, so it plays them at the rate they arrive rather than freezing between them) steps down a resolution (half the scale after two seconds, a quarter two seconds later if still slow, From a10c7fa2cdafb15ea3012781ab6a5f421d7c211d Mon Sep 17 00:00:00 2001 From: nazar-gavrylyk Date: Tue, 10 Nov 2026 15:10:35 +0100 Subject: [PATCH 21/24] Footage: note that 8-bit and float frames of one movie decode apart In an 8 bpc comp, a movie used by both a plain and a masked layer has an 8-bit and a float entry in the media pool, decoded separately. Rare and only a speed cost, so it is noted where the 8-bit path starts rather than changed. --- crates/render/src/pipeline.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/render/src/pipeline.rs b/crates/render/src/pipeline.rs index 29ce6d69a..b1ec0ca02 100644 --- a/crates/render/src/pipeline.rs +++ b/crates/render/src/pipeline.rs @@ -1040,6 +1040,9 @@ impl<'a> Renderer<'a> { /// changes them: an 8 bpc render at Full size of plain footage (no masks, colour profile, /// Separate Fields, Frame Blending, Cineon, Invert Alpha or linear light interpretation). /// `None` otherwise, and when the source has no 8-bit frame. + // ponytail: the media pool keeps the 8-bit and the float frame as separate entries, so a + // movie shown by both a plain and a masked layer in one 8 bpc comp is decoded twice per + // frame; derive one from the other in the pool if a comp like that turns out common. #[allow(clippy::too_many_arguments)] fn footage_buf8(&self, ctx: &EvalCtx, layer: &Layer, item: ItemId, pf: &Footage, f: &Footage, t: Tick) -> Option { let plain = self.pipe.levels == Some(255.0) From e356e4adf5dbdf59def7fcbd9cd34a6888d307d6 Mon Sep 17 00:00:00 2001 From: nazar-gavrylyk Date: Tue, 10 Nov 2026 15:58:42 +0100 Subject: [PATCH 22/24] Preview: a stall after playing at full rate still steps down a resolution The step-down was skipped while the achieved frame rate read as keeping up, and that rate comes from the frames shown in the last second they were shown: through a stall it kept the rate from before it, so a preview that played a cached stretch and then stalled never stepped down. A rate whose last frame is more than a second old no longer counts. --- crates/ui-egui/src/app/prefetch.rs | 3 ++- crates/ui-egui/tests/it/ui_preview_align.rs | 24 +++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/crates/ui-egui/src/app/prefetch.rs b/crates/ui-egui/src/app/prefetch.rs index 7f6494d3f..a80ae98a9 100644 --- a/crates/ui-egui/src/app/prefetch.rs +++ b/crates/ui-egui/src/app/prefetch.rs @@ -553,7 +553,8 @@ impl EffectcraftApp { || pb.res.is_some() || !self.viewer_res().is_auto() || now - since < 2.0 - || pb.achieved_fps().is_some_and(|f| f * step >= pb.fps * 0.8) + // (a rate from before a stall: nothing shown for a second isn't keeping up) + || (pb.shown_at.back().is_some_and(|t| now - t <= 1.0) && pb.achieved_fps().is_some_and(|f| f * step >= pb.fps * 0.8)) || lower < self.session.prefs.adaptive_limit() - 1e-9 { return; diff --git a/crates/ui-egui/tests/it/ui_preview_align.rs b/crates/ui-egui/tests/it/ui_preview_align.rs index 9d3119b21..a9622d458 100644 --- a/crates/ui-egui/tests/it/ui_preview_align.rs +++ b/crates/ui-egui/tests/it/ui_preview_align.rs @@ -320,6 +320,30 @@ fn cache_frames_when_idle_survives_an_inverted_work_area() { assert!(!h.state().playback.playing); } +/// A preview that played at full rate and then stalls on frames that aren't cached still steps +/// down: the rate from before the stall doesn't count once nothing was shown for a second. (The +/// last second shown kept reading as full rate through the stall, and it never stepped down.) +#[test] +fn playback_that_stalls_after_keeping_up_steps_down() { + let mut h = harness(); + h.state_mut().session.prefs.previews.adaptive_playback = true; + h.state().frames.set_budget(0); + h.state_mut().use_manual_clock(); + invoke(&mut h, "playback.toggle", json!({})); + h.step(); + assert!(h.state().playback.playing); + // A second shown at the comp's rate, just before the stall. + let t0 = h.state().now(); + for i in 0..30 { + h.state_mut().playback.frame_shown(t0 - 1.0 + f64::from(i) / 30.0); + } + for _ in 0..25 { + h.state_mut().clock_mut().advance(0.1); + h.step(); + } + assert_eq!(h.state().viewer_scale(1.0, 1.0), 0.5); +} + /// Settings ▸ Previews ▸ Lower Resolution When Playback Can't Keep Up: a preview at Auto that /// stalls for two seconds steps down to half its scale, two more to a quarter; an explicit Resolution and /// the pref turned off (its default) keep the scale, and stopping goes back to the viewer's. From 114fbaed6335394be86e073badaede6c410e6bdb Mon Sep 17 00:00:00 2001 From: FredApps Date: Tue, 10 Nov 2026 17:22:44 +0100 Subject: [PATCH 23/24] Preview: the pace measures its lead over no more frames than the RAM preview holds The pace counted cached frames among the next ahead x 2, but the prefetch never requests more than the RAM preview holds (offsets 0..fit-1 from the current frame). With room for only a few frames the lead could never fill, so a preview that rendered every frame at once was slowed to a fraction of its frame rate, with Lower Resolution When Playback Can't Keep Up off (the default). The lead is now counted over at most the frames the prefetch keeps after the current one, and with none of those (room for at most two frames) the clock runs in real time as before pacing. --- crates/ui-egui/src/app/prefetch.rs | 48 ++++++++++++++++++--- crates/ui-egui/tests/it/ui_preview_align.rs | 41 ++++++++++++++++++ 2 files changed, 83 insertions(+), 6 deletions(-) diff --git a/crates/ui-egui/src/app/prefetch.rs b/crates/ui-egui/src/app/prefetch.rs index a80ae98a9..114f05b4c 100644 --- a/crates/ui-egui/src/app/prefetch.rs +++ b/crates/ui-egui/src/app/prefetch.rs @@ -54,6 +54,17 @@ fn pace_for(prev: f64, lead: i64, target: i64, fps: f64, dt: f64) -> f64 { prev + (raw - prev) * k } +/// How many frames after the current one the pace measures its lead over: twice the prefetch +/// depth `ahead`, but only frames the RAM preview can keep cached together with the current one. +/// `prefetch_playback` requests offsets `0..fit_ahead` from the current frame (the current frame +/// and `fit_ahead - 1` after it), so a lead counted over more frames than that could never fill: +/// a preview that renders every frame at once would still be slowed down to a fraction of its +/// frame rate. 0 when not even one frame after the current one is prefetched: the pace can't +/// measure anything, and the clock runs in real time (holding on frames that aren't cached). +fn pace_horizon(ahead: i64, fit_ahead: i64) -> i64 { + (ahead * 2).max(1).min(fit_ahead - 1).max(0) +} + /// Cached frames (`cached`, sorted) among the next `max` frames `after(1..=max)` plays, and how /// many frames it plays (`after` is `None` past the end). Counted, not the consecutive run: one /// late frame (a GPU warm-up frame, a heavy one) with frames cached beyond it holds the clock @@ -145,7 +156,7 @@ impl EffectcraftApp { if self.advance_audio_clock(&pass, step) { return; } - self.advance_frame_clock(&pass, step); + self.advance_frame_clock(&pass, step, fit_ahead); } /// While paused (scrubbing, editing) the viewer's frame comes first: prefetch only a few @@ -428,7 +439,7 @@ impl EffectcraftApp { /// frame (or holds when it is not cached yet). While frames arrive slower than the frame /// rate it runs at their pace ([`pace_for`]): every frame shows, evenly, as After Effects /// plays a preview it hasn't cached yet; real time again once the frames ahead are cached. - fn advance_frame_clock(&mut self, pass: &PlaybackPass<'_>, step: i64) { + fn advance_frame_clock(&mut self, pass: &PlaybackPass<'_>, step: i64, fit_ahead: i64) { let PlaybackPass { ctx, now, plan, video, frame_rate: fr, cur, series, .. } = *pass; let FrameWindow { last: wb, .. } = pass.window; // The plan carries the rate the preview was started with (and `play_with` copies it to @@ -439,10 +450,16 @@ impl EffectcraftApp { self.playback.paced_at = Some(now); if video { // The lead counts out of twice the prefetch depth: the prefetch keeps that many - // frames coming (or the frames left before the end), so a full lead means rendering keeps up. - let (lead, target) = self.cached_lead(pass, (pass.ahead * 2).max(1)); + // frames coming (or the frames left before the end), so a full lead means rendering + // keeps up. No more than the RAM preview holds ([`pace_horizon`]). + let horizon = pace_horizon(pass.ahead, fit_ahead); let prev = if self.playback.pace > 0.0 { self.playback.pace } else { fps }; - let pace = pace_for(prev, lead, target, fps, dt); + let pace = if horizon > 0 { + let (lead, target) = self.cached_lead(pass, horizon); + pace_for(prev, lead, target, fps, dt) + } else { + fps + }; self.playback.pace = pace; if pace < fps * 0.8 { self.step_down(now, pass.scale); @@ -662,7 +679,26 @@ impl EffectcraftApp { #[cfg(test)] mod pace_tests { - use super::{lead_of, pace_for}; + use super::{lead_of, pace_for, pace_horizon}; + + /// The lead is measured over no more frames than the RAM preview keeps ahead of the current + /// one, so a full cache plays in real time. (It counted out of twice the prefetch depth: with + /// room for 4 frames, at most 2 after the current one were ever cached, and a preview that + /// rendered at once played at a quarter of its frame rate.) + #[test] + fn the_pace_horizon_fits_in_the_ram_preview() { + // A roomy budget: twice the prefetch depth. + assert_eq!(pace_horizon(6, 400), 12); + // Room for 4 frames: the prefetch requests offsets 0..3, so 2 after the current one. + assert_eq!(pace_horizon(6, 3), 2); + let (lead, target) = lead_of(|k| Some(10 + k), &[10, 11, 12], pace_horizon(6, 3)); + assert!((pace_for(30.0, lead, target, 30.0, 10.0) - 30.0).abs() < 1e-6, "a full cache plays in real time"); + // Room for the current frame only (or nothing): no lead to measure. + assert_eq!(pace_horizon(6, 1), 0); + assert_eq!(pace_horizon(6, 0), 0); + // No prefetch depth: still the next frame. + assert_eq!(pace_horizon(0, 400), 1); + } /// The pace follows the cached lead: real time with a full lead, a third of it with a third, /// never under an eighth, unchanged with nothing ahead, and smoothed over time. diff --git a/crates/ui-egui/tests/it/ui_preview_align.rs b/crates/ui-egui/tests/it/ui_preview_align.rs index a9622d458..0baafc55c 100644 --- a/crates/ui-egui/tests/it/ui_preview_align.rs +++ b/crates/ui-egui/tests/it/ui_preview_align.rs @@ -395,3 +395,44 @@ fn playback_that_cannot_keep_up_steps_down_a_resolution() { stall(&mut h, 2.5); assert_eq!(h.state().viewer_scale(1.0, 1.0), 1.0); } + +/// A preview that renders every frame at once plays in real time even when the RAM preview holds +/// only a few frames. (The pace measured its lead over twice the prefetch depth, more frames than +/// a small RAM preview keeps ahead, so such a preview was slowed to a fraction of its frame rate.) +#[test] +fn a_preview_that_keeps_up_plays_in_real_time_with_a_small_ram_preview() { + let mut h = harness(); + let cid = h.state().session.active_comp_id().unwrap(); + let series = h.state().shown_series(cid); + let s = series.scale as f64 / 1000.0; + let frame = (320.0 * s).ceil() * (180.0 * s).ceil() * 4.0; + // Room for 4 frames (90 % of 5): the prefetch keeps the current frame and 2 after it. + h.state().frames.set_budget((frame * 5.0) as usize); + h.state_mut().use_manual_clock(); + h.state_mut().session.set_time(Tick::ZERO); + invoke(&mut h, "playback.toggle", json!({})); + let fr = h.state().session.active_comp().unwrap().frame_rate; + // Rendering keeps up: the frames ahead are rendered before the clock moves on. + let settle = |h: &mut Harness<'_, EffectcraftApp>| { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(20); + while std::time::Instant::now() < deadline { + h.step(); + let cur = fr.frame_at(h.state().session.time()); + let cached = h.state().frames.cached_frames(&series); + if (1..=2).all(|k| cached.contains(&((cur + k) % 240))) { + return; + } + std::thread::sleep(std::time::Duration::from_millis(1)); + } + panic!("frames ahead never cached"); + }; + for _ in 0..48 { + settle(&mut h); + h.state_mut().clock_mut().advance(1.0 / 24.0); + h.step(); + } + let pb = &h.state().playback; + assert!(pb.playing); + assert!(pb.pace >= 24.0 * 0.98, "plays at {} fps of 24", pb.pace); + assert!(fr.frame_at(h.state().session.time()) >= 40, "the clock kept real time: {:?}", h.state().session.time()); +} \ No newline at end of file From 9eb9ac003b2060fc433dc176a42d9dfccdfe7609 Mon Sep 17 00:00:00 2001 From: FredApps Date: Tue, 10 Nov 2026 17:56:26 +0100 Subject: [PATCH 24/24] Preview: end the small-RAM-preview pacing test with a newline (cargo fmt) --- crates/ui-egui/tests/it/ui_preview_align.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/ui-egui/tests/it/ui_preview_align.rs b/crates/ui-egui/tests/it/ui_preview_align.rs index 0baafc55c..afafa8a45 100644 --- a/crates/ui-egui/tests/it/ui_preview_align.rs +++ b/crates/ui-egui/tests/it/ui_preview_align.rs @@ -435,4 +435,4 @@ fn a_preview_that_keeps_up_plays_in_real_time_with_a_small_ram_preview() { assert!(pb.playing); assert!(pb.pace >= 24.0 * 0.98, "plays at {} fps of 24", pb.pace); assert!(fr.frame_at(h.state().session.time()) >= 40, "the clock kept real time: {:?}", h.state().session.time()); -} \ No newline at end of file +}