Preview: debounce GPU memory pressure and recover the frame budget (#3025)

Co-authored-by: FredApps <153173854+FredApps@users.noreply.github.com>

crates/ui-egui/src/frames.rs

@@ -39,10 +39,13 @@ use rayon::prelude::*;
 const MAX_KEEPERS: usize = 64;
 
 /// GPU frames kept for RAM preview (bytes of video memory) at first. wgpu can't tell how much
-/// video memory there is: each time the device runs out, the budget halves (to
-/// [`MIN_GPU_BUDGET`]) and the frame renders on the CPU (#106).
+/// video memory there is: allocation pressure halves the budget (to [`MIN_GPU_BUDGET`])
+/// and the frame renders on the CPU (#106). One burst is counted once; successful GPU
+/// renders gradually restore the budget after pressure subsides.
 const GPU_BUDGET: usize = 1 << 30;
 const MIN_GPU_BUDGET: usize = 64 << 20;
+const GPU_PRESSURE_COOLDOWN: std::time::Duration = std::time::Duration::from_secs(5);
+const GPU_RECOVERY_INTERVAL: std::time::Duration = std::time::Duration::from_secs(30);
 
 /// A rendered viewer frame: CPU pixels or a GPU texture.
 #[derive(Clone)]
@@ -151,6 +154,8 @@ struct Cache {
     budget: usize,
     /// Bytes of GPU frames to keep at most ([`GPU_BUDGET`]).
     gpu_budget: usize,
+    gpu_pressure: Option<web_time::Instant>,
+    gpu_adjusted: Option<web_time::Instant>,
     /// The playing preview, if any: evictions spare the frames it shows next.
     playhead: Option<Playhead>,
     /// Scale of the frame on screen (the newest urgent request): frames of other scales go first.
@@ -207,11 +212,28 @@ impl Cache {
             }
         }
     }
-    /// The GPU ran out of memory: keep half as many GPU frames from now on.
+    /// Several workers failing in one burst must not halve the shared budget repeatedly.
     fn gpu_out_of_memory(&mut self) {
-        self.gpu_budget = (self.gpu_budget / 2).max(MIN_GPU_BUDGET);
+        let now = web_time::Instant::now();
+        if self.gpu_pressure.is_none_or(|last| now.duration_since(last) >= GPU_PRESSURE_COOLDOWN) {
+            self.gpu_budget = (self.gpu_budget / 2).max(MIN_GPU_BUDGET);
+            self.gpu_adjusted = Some(now);
+        }
+        self.gpu_pressure = Some(now);
         self.evict_gpu();
     }
+    fn gpu_succeeded(&mut self) {
+        let now = web_time::Instant::now();
+        if self.gpu_pressure.is_some_and(|last| now.duration_since(last) >= GPU_RECOVERY_INTERVAL)
+            && self.gpu_adjusted.is_some_and(|last| now.duration_since(last) >= GPU_RECOVERY_INTERVAL)
+        {
+            self.gpu_budget = self.gpu_budget.saturating_add(MIN_GPU_BUDGET).min(GPU_BUDGET);
+            self.gpu_adjusted = Some(now);
+        }
+    }
+    fn gpu_ready(&self) -> bool {
+        self.gpu_pressure.is_none_or(|last| last.elapsed() >= GPU_PRESSURE_COOLDOWN)
+    }
     fn forget(&mut self, img: &FrameImage) {
         self.bytes -= img.bytes();
         if img.is_gpu() {
@@ -505,7 +527,7 @@ impl Default for Frames {
         #[cfg(target_arch = "wasm32")]
         let queue = Queue::default();
         Frames {
-            cache: Arc::new(Mutex::new(Cache { map: HashMap::new(), order: VecDeque::new(), bytes: 0, gpu_bytes: 0, budget: 3 << 30, gpu_budget: GPU_BUDGET, playhead: None, scale: None })),
+            cache: Arc::new(Mutex::new(Cache { map: HashMap::new(), order: VecDeque::new(), bytes: 0, gpu_bytes: 0, budget: 3 << 30, gpu_budget: GPU_BUDGET, gpu_pressure: None, gpu_adjusted: None, playhead: None, scale: None })),
             inflight: Arc::new(Mutex::new(HashSet::new())),
             queue: Arc::new(Mutex::new(queue)),
             #[cfg(not(target_arch = "wasm32"))]
@@ -914,6 +936,9 @@ impl Frames {
             c.order.clear();
             c.bytes = 0;
             c.gpu_bytes = 0;
+            c.gpu_budget = GPU_BUDGET;
+            c.gpu_pressure = None;
+            c.gpu_adjusted = None;
         }
         // Disk keys hash the footage files' size and time, but are remembered per revision; a
         // file reloaded from disk without a new revision (a re-rendered image sequence) must be
@@ -1411,9 +1436,19 @@ impl Worker {
         }
     }
 
+    fn gpu_succeeded(&self) {
+        if let Ok(mut c) = self.cache.lock() {
+            c.gpu_succeeded();
+        }
+    }
+
+    fn gpu_ready(&self) -> bool {
+        !self.gpu_retired.load(Ordering::Acquire) && self.cache.lock().is_ok_and(|c| c.gpu_ready())
+    }
+
     /// The frame on the GPU, kept there for the viewer, if the compositor can do it alone.
     fn render_gpu(&self, job: &Job) -> Option<FrameImage> {
-        if self.gpu_retired.load(Ordering::Acquire) {
+        if !self.gpu_ready() {
             return None;
         }
         let g = job.src.gpu.as_ref()?;
@@ -1426,7 +1461,12 @@ impl Worker {
         let f = g.within_memory(|| g.render_display(&r, job.comp, job.t));
         effectcraft_engine::render::Accelerator::frame_end(g);
         match f {
-            Ok(f) => f.map(|f| FrameImage::Gpu(Arc::new(f))),
+            Ok(f) => {
+                if f.is_some() {
+                    self.gpu_succeeded();
+                }
+                f.map(|f| FrameImage::Gpu(Arc::new(f)))
+            }
             Err(e) => {
                 self.gpu_attempt_failed(g, &e);
                 None
@@ -1439,7 +1479,7 @@ impl Worker {
         let footage = job.footage();
         let mut r = Renderer::new(&job.src.project, &footage, job.opts);
         r.expr = job.src.expr.as_deref();
-        let gpu_allowed = gpu && !self.gpu_retired.load(Ordering::Acquire);
+        let gpu_allowed = gpu && self.gpu_ready();
         // A failed GPU attempt may have populated intermediate entries before its scope
         // reported failure. The CPU retry must compute its own pixels from the snapshot.
         r.cache = gpu_allowed.then_some(job.src.layer_cache.as_ref());
@@ -1459,6 +1499,7 @@ impl Worker {
                 let t0 = web_time::Instant::now();
                 match g.within_memory(|| g.render_display(&r, job.comp, job.t)) {
                     Ok(Some(f)) => {
+                        self.gpu_succeeded();
                         if let Some((p, key)) = auto {
                             // Time the GPU's work, not just its recording.
                             g.wait();
@@ -1490,7 +1531,10 @@ impl Worker {
         if let Some(g) = gpu {
             // The GPU composites and the frame is read back.
             return match g.within_memory(|| r.comp_frame(job.comp, job.t)) {
-                Ok(img) => FrameImage::Cpu(Arc::new(to_color_image(&img))),
+                Ok(img) => {
+                    self.gpu_succeeded();
+                    FrameImage::Cpu(Arc::new(to_color_image(&img)))
+                }
                 Err(e) => {
                     self.gpu_attempt_failed(g, &e);
                     self.render_job(job, false)

crates/ui-egui/src/frames/tests.rs

@@ -469,18 +469,36 @@ fn a_newer_viewer_frame_drops_the_ones_still_waiting() {
     assert_eq!(f.inflight(), 3, "dropped frames aren't in flight: the viewer can ask again");
 }
 
-/// Each time the GPU runs out of memory, half as many GPU frames are kept, down to a floor.
+/// One OOM burst halves once; separate episodes back off to a floor and healthy GPU work recovers.
 #[test]
 fn running_out_of_video_memory_halves_the_gpu_frame_budget() {
     let f = Frames::default();
     let mut c = f.cache.lock().unwrap();
     assert_eq!(c.gpu_budget, GPU_BUDGET);
+    assert!(c.gpu_ready());
     c.gpu_out_of_memory();
+    assert!(!c.gpu_ready(), "allocation failures must not be retried for every frame");
     assert_eq!(c.gpu_budget, GPU_BUDGET / 2);
     for _ in 0..20 {
         c.gpu_out_of_memory();
     }
+    assert_eq!(c.gpu_budget, GPU_BUDGET / 2, "one burst must not collapse the shared budget");
+    for _ in 0..20 {
+        c.gpu_pressure = None;
+        c.gpu_out_of_memory();
+    }
     assert_eq!(c.gpu_budget, MIN_GPU_BUDGET);
+    let before = web_time::Instant::now() - GPU_RECOVERY_INTERVAL;
+    c.gpu_pressure = Some(before);
+    assert!(c.gpu_ready(), "GPU attempts resume after the cooldown");
+    c.gpu_adjusted = Some(before);
+    c.gpu_succeeded();
+    assert_eq!(c.gpu_budget, 2 * MIN_GPU_BUDGET);
+    c.gpu_succeeded();
+    assert_eq!(c.gpu_budget, 2 * MIN_GPU_BUDGET, "recovery is paced");
+    drop(c);
+    f.clear();
+    assert_eq!(f.cache.lock().unwrap().gpu_budget, GPU_BUDGET);
 }
 
 #[test]