diff --git a/crates/ui-egui/src/app/playback.rs b/crates/ui-egui/src/app/playback.rs index d40bc69ff..51dd04ee1 100644 --- a/crates/ui-egui/src/app/playback.rs +++ b/crates/ui-egui/src/app/playback.rs @@ -40,6 +40,7 @@ impl EffectcraftApp { restore_max, audio_frame: (!plan.video).then_some(plan.first), audio_held: false, + silent_project: None, starting: false, starting_since: now, first_cached: None, @@ -57,7 +58,7 @@ impl EffectcraftApp { } /// The preview includes audio at the comp's frame rate and the comp has something audible. - fn audio_plays(&self) -> bool { + pub(super) fn audio_plays(&self) -> bool { let (Some(pl), Some(cid)) = (self.playback.plan, self.session.active_comp_id()) else { return false }; let Some(c) = self.session.project.comp(cid) else { return false }; // A preview at another frame rate plays slower or faster than real time: silent. diff --git a/crates/ui-egui/src/app/prefetch.rs b/crates/ui-egui/src/app/prefetch.rs index 50638b708..22e587643 100644 --- a/crates/ui-egui/src/app/prefetch.rs +++ b/crates/ui-egui/src/app/prefetch.rs @@ -82,6 +82,7 @@ impl EffectcraftApp { let step = plan.map_or(1, |p| p.step); let fit_ahead = self.frames_that_fit(&c, scale) - 1; self.prefetch_playback(&PlaybackPass { cur: lead, ..pass }, fit_ahead); + self.wake_silent_preview(); self.pump_meters(ctx, now); if !self.playback.playing { self.cache_when_idle(ctx, &c, &series, now, input_active, (wa, wb)); @@ -201,6 +202,10 @@ impl EffectcraftApp { /// one second. With no source but an active meter, drive the meter to silence and request a repaint. fn pump_meters(&mut self, ctx: &egui::Context, now: f64) { if let Some(a) = &mut self.audio { + // An edit while playing (mute, unmute, solo, levels): the sound follows it (#2104). + if a.project().is_some_and(|p| !std::sync::Arc::ptr_eq(&p, &self.session.project)) { + a.follow_project(self.session.project.clone()); + } a.pump(); self.meter.update(a.feed.take_peaks(), now); } else if let Some(s) = &mut self.scrub { @@ -259,6 +264,29 @@ impl EffectcraftApp { false } + /// A silent preview whose project was edited so that it has sound now (a layer unmuted while + /// playing, #2104): video previews wait for cached frames; audio-only previews start from + /// their moving audio frame, not the stationary timeline. Checked once per project edit. + fn wake_silent_preview(&mut self) { + let pb = &self.playback; + if self.audio.is_some() || !pb.playing || pb.caching || pb.starting || pb.audio_held { + return; + } + if pb.silent_project.as_ref().is_some_and(|p| std::sync::Arc::ptr_eq(p, &self.session.project)) { + return; + } + self.playback.silent_project = Some(self.session.project.clone()); + if !self.audio_plays() { + return; + } + if self.playback.plan.is_some_and(|p| p.video) { + self.playback.audio_held = true; + } else if let Some(c) = self.session.active_comp() { + let frame = self.playback.audio_frame.unwrap_or(self.playback.start_frame); + self.start_audio(c.frame_rate.tick_of(frame)); + } + } + /// Held sound starts once the frames ahead are cached (see `Playback::audio_held`): the /// prefetch's `ahead` when the silent preview keeps real time, else the rest of the preview. fn start_held_audio(&mut self, pass: &PlaybackPass<'_>, fit: i64) { diff --git a/crates/ui-egui/src/audio.rs b/crates/ui-egui/src/audio.rs index fbfe4b8af..960aa71e2 100644 --- a/crates/ui-egui/src/audio.rs +++ b/crates/ui-egui/src/audio.rs @@ -18,12 +18,12 @@ //! **Scrubbing:** Ctrl/Cmd-dragging the current-time indicator plays a short snippet of the mix //! at each new frame ([`AudioScrub`], `playback.scrubAudio`), as in After Effects. -use std::collections::VecDeque; +use std::collections::{HashSet, VecDeque}; use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; use std::thread::JoinHandle; -use effectcraft_engine::project::ItemId; +use effectcraft_engine::project::{GroupKind, ItemId, ItemKind, Layer, LayerSource, Project, PropGroup}; use effectcraft_engine::time::{FrameRate, Tick}; use crate::frames::RenderSource; @@ -88,9 +88,11 @@ impl AudioFeed { pk[i & 1] = pk[i & 1].max(s.abs()); } real = n; + // Queue removal and the clock advance are one handoff: an edit must not restart + // behind samples already handed to the device. + self.consumed.fetch_add((real / 2) as u64, Ordering::Relaxed); } out[real..].iter_mut().for_each(|o| *o = 0.0); - self.consumed.fetch_add((real / 2) as u64, Ordering::Relaxed); for (c, v) in pk.into_iter().enumerate() { // Rust 1.99 renamed `fetch_update` to `try_update`; keep the old name while the // workspace supports Rust 1.95 (`rust-version`). @@ -106,6 +108,14 @@ impl AudioFeed { } } + /// Discard an old mix and snapshot the first unconsumed frame under the same queue lock + /// as [`Self::pull`], so a concurrent device callback cannot leave the cursor behind. + fn discard_pending(&self) -> Option { + let mut q = self.queue.lock().ok()?; + q.clear(); + Some(self.consumed()) + } + pub fn push(&self, samples: &[f32]) { if let Ok(mut q) = self.queue.lock() { q.extend(samples.iter().copied()); @@ -179,12 +189,24 @@ pub struct AudioPlayback { device: Box, stop: Arc, thread: Option>, - /// wasm32 (no threads): what the feeder thread would own, run by [`AudioPlayback::pump`]. - feeder: Option<(RenderSource, ItemId, i64, u32)>, + /// What the mixer reads and advances: the feeder thread, or [`AudioPlayback::pump`] on wasm32 + /// (no threads). + state: Arc>, + comp: ItemId, + mix: u32, pub rate: u32, pub span: PlaySpan, } +/// The mixer's position and its view of the project. +struct Feeder { + src: RenderSource, + /// The next comp sample index to mix. + cursor: i64, + /// Bumped when the project changes: a block mixed from an older project is dropped. + generation: u64, +} + /// Audio kept queued ahead of the device: ~200 ms (a feeder thread tops it up continuously); /// wasm32 tops it up once per UI frame, so it keeps ~350 ms to ride out slow frames. fn queue_target(rate: u32) -> usize { @@ -211,9 +233,10 @@ impl AudioPlayback { let span = PlaySpan::new(start, wa, wb, looping, rate); let feed = Arc::new(AudioFeed::default()); let stop = Arc::new(AtomicBool::new(false)); + let state = Arc::new(Mutex::new(Feeder { src, cursor: span.start, generation: 0 })); if cfg!(target_arch = "wasm32") { // The device takes samples only in `pump`, which fills the queue first. - let mut p = AudioPlayback { feed: feed.clone(), device, stop, thread: None, feeder: Some((src, comp, span.start, mix)), rate, span }; + let mut p = AudioPlayback { feed: feed.clone(), device, stop, thread: None, state, comp, mix, rate, span }; p.device.start(feed)?; p.pump(); return Ok(p); @@ -223,15 +246,14 @@ impl AudioPlayback { // as that took (#408). The device plays silence until the first block arrives, and the // clock doesn't count it. let thread = { - let (feed, stop) = (feed.clone(), stop.clone()); + let (feed, stop, state) = (feed.clone(), stop.clone(), state.clone()); std::thread::Builder::new() .name("ec-audio-feed".into()) .spawn(move || { - let mut cursor = span.start; let target = queue_target(rate); while !stop.load(Ordering::Relaxed) { if feed.queued_frames() < target && !feed.finished.load(Ordering::Relaxed) { - cursor = feed_block(&src, comp, &feed, &span, cursor, rate, mix); + feed_block(&state, comp, &feed, &span, rate, mix); } else { std::thread::sleep(std::time::Duration::from_millis(4)); } @@ -240,21 +262,54 @@ impl AudioPlayback { .map_err(|e| e.to_string())? }; device.start(feed.clone())?; - Ok(AudioPlayback { feed, device, stop, thread: Some(thread), feeder: None, rate, span }) + Ok(AudioPlayback { feed, device, stop, thread: Some(thread), state, comp, mix, rate, span }) } /// Every UI frame while playing: on wasm32 mix ahead into the feed (what the feeder thread /// does elsewhere), then let the device take what it needs. pub fn pump(&mut self) { - if let Some((src, comp, cursor, mix)) = &mut self.feeder { + if cfg!(target_arch = "wasm32") { let target = queue_target(self.rate); while self.feed.queued_frames() < target && !self.feed.finished.load(Ordering::Relaxed) { - *cursor = feed_block(src, *comp, &self.feed, &self.span, *cursor, self.rate, *mix); + feed_block(&self.state, self.comp, &self.feed, &self.span, self.rate, self.mix); } } self.device.pump(); } + /// The project the sound is mixed from. + pub fn project(&self) -> Option> { + self.state.lock().ok().map(|s| s.src.project.clone()) + } + + /// Mix from `project` from now on (the project was edited while playing, e.g. a layer's Audio + /// switch, #2104): what is queued came from the old project, so it is dropped and mixing + /// resumes at the first sample the device has not taken yet (also when the feeder had + /// already mixed up to the end of a non-looping span). + pub fn set_project(&mut self, project: Arc) { + let Ok(mut s) = self.state.lock() else { return }; + let Some(consumed) = self.feed.discard_pending() else { return }; + s.src.project = project; + s.generation = s.generation.wrapping_add(1); + self.feed.finished.store(false, Ordering::Relaxed); + // The display clock clamps to the last *audible* sample. A finished non-looping + // feeder must instead stay at the exclusive end, never replay that last sample. + let next = i128::from(self.span.start) + i128::from(consumed); + s.cursor = if !self.span.looping && next >= i128::from(self.span.wb) { self.span.wb } else { clock_sample(&self.span, consumed, 0) }; + } + + /// Follow an edited `project`: mixing continues from it, and the queued sound is dropped only + /// when the edit changed what is heard ([`audio_changed`]); a transform or colour edit leaves + /// the queue alone, so playback does not drop out. + pub fn follow_project(&mut self, project: Arc) { + let Some(old) = self.project() else { return }; + if audio_changed(&old, &project, self.comp) { + self.set_project(project); + } else if let Ok(mut s) = self.state.lock() { + s.src.project = project; + } + } + /// The comp sample index audible now. pub fn clock(&self) -> i64 { clock_sample(&self.span, self.feed.consumed(), self.device.latency_frames()) @@ -279,20 +334,95 @@ impl Drop for AudioPlayback { } } -/// Mix one block at `cursor` into the feed; returns the next cursor (wrapping or finishing at -/// the work-area end). -fn feed_block(src: &RenderSource, comp: ItemId, feed: &AudioFeed, span: &PlaySpan, cursor: i64, rate: u32, mix: u32) -> i64 { - let mut cursor = cursor; - if cursor >= span.wb { - if !span.looping { - feed.finished.store(true, Ordering::Relaxed); - return cursor; +/// Whether `new` sounds different from `old` for `comp`: a layer's timing, source, Audio / Solo / +/// Effects switch, Audio Levels or effects, or the same for a nested comp. Other edits (video +/// properties, names, labels) do not change the mix unless an audio expression can read them. +fn audio_changed(old: &Project, new: &Project, comp: ItemId) -> bool { + audio_changed_inner(old, new, comp, &mut HashSet::new()) +} + +/// Each shared precomp is compared once per depth. A depth limit alone still visits an +/// exponentially large number of paths in a small graph of reused precomps, blocking the UI. +fn audio_changed_inner(old: &Project, new: &Project, comp: ItemId, seen: &mut HashSet<(ItemId, usize)>) -> bool { + /// Compare one comp and its reachable audio sources, without revisiting a shared path depth. + fn comps_differ(old: &Project, new: &Project, comp: ItemId, depth: usize, seen: &mut HashSet<(ItemId, usize)>) -> bool { + if depth >= 16 || !seen.insert((comp, depth)) { + return false; } + let (Some(a), Some(b)) = (old.comp(comp), new.comp(comp)) else { return old.comp(comp).is_some() != new.comp(comp).is_some() }; + a.layers.len() != b.layers.len() + || a.layers.iter().zip(&b.layers).any(|(x, y)| { + x.id != y.id + || x.source != y.source + || x.start_time != y.start_time + || x.in_point != y.in_point + || x.out_point != y.out_point + || x.stretch != y.stretch + || x.switches.audio != y.switches.audio + || x.switches.solo != y.switches.solo + || x.switches.effects != y.switches.effects + || x.props.sub("audio") != y.props.sub("audio") + || audio_effects(x) != audio_effects(y) + || audio_has_expression(x) + || audio_has_expression(y) + || match &x.source { + LayerSource::Footage { item } => footage_differs(old, new, *item), + LayerSource::Comp { item } => comps_differ(old, new, *item, depth + 1, seen), + _ => false, + } + }) + } + /// The layer's audio effects (Effect > Audio), the only effects the mixer applies; a video effect + /// edit leaves them as they were. + fn audio_effects(l: &Layer) -> Vec<&PropGroup> { + let is_audio = |g: &&PropGroup| matches!(&g.kind, GroupKind::Effect { effect } if effectcraft_engine::effects::audio_fx::is_audio_effect(effect)); + l.effects().into_iter().flat_map(|fx| fx.groups()).filter(is_audio).collect() + } + /// Audio expressions can read any property in any comp (including video-only effects and + /// names). Without a complete dependency graph, keeping the old mix would be stale. + fn audio_has_expression(l: &Layer) -> bool { + l.switches.audio + && (l.props.sub("audio").and_then(|g| g.get("levels")).is_some_and(|p| p.has_expression()) + || (l.switches.effects && audio_effects(l).iter().any(|g| g.enabled && g.props().any(|p| p.has_expression())))) + } + /// The footage item's audio availability changed (relinked, missing, no audio). + fn footage_differs(old: &Project, new: &Project, item: ItemId) -> bool { + let f = |p: &Project| match p.item(item).map(|i| &i.kind) { + Some(ItemKind::Footage(f)) => Some((f.has_audio, f.missing, f.path.clone())), + _ => None, + }; + f(old) != f(new) + } + comps_differ(old, new, comp, 0, seen) +} + +/// Mix one block at the feeder's cursor into the feed and move the cursor on (wrapping or +/// finishing at the work-area end). The block is mixed without holding the lock, so a project +/// change never waits for a slow mix; a block mixed from the old project is dropped. +fn feed_block(state: &Mutex, comp: ItemId, feed: &AudioFeed, span: &PlaySpan, rate: u32, mix: u32) { + let Ok((src, mut cursor, generation)) = state.lock().map(|s| (s.src.clone(), s.cursor, s.generation)) else { + // Poisoned: nothing more can be mixed. + feed.finished.store(true, Ordering::Relaxed); + return; + }; + let mut block = None; + if cursor >= span.wb && span.looping { cursor = span.wa; } - let n = ((span.wb - cursor) as usize).min(BLOCK); - feed.push(&mix_block(src, comp, sample_time(cursor, rate), n, rate, mix)); - cursor + n as i64 + if cursor < span.wb { + let n = ((span.wb - cursor) as usize).min(BLOCK); + block = Some(mix_block(&src, comp, sample_time(cursor, rate), n, rate, mix)); + cursor += n as i64; + } + let Ok(mut s) = state.lock() else { return }; + if s.generation != generation { + return; + } + match block { + Some(b) => feed.push(&b), + None => feed.finished.store(true, Ordering::Relaxed), + } + s.cursor = cursor; } /// `n` stereo frames of `comp`'s mix from comp time `t` at the device `rate`, mixed at `mix` Hz @@ -559,6 +689,145 @@ mod tests { playback.stop(); } + /// #2104: muting a layer while the preview plays silences it: the queued sound is dropped + /// and everything mixed afterwards comes from the edited project. + #[test] + fn muting_a_layer_while_playing_silences_the_preview() { + use effectcraft_engine::color::Label; + use effectcraft_engine::project::{Comp, Footage, FootageKind, ItemKind, LayerSource, build}; + use effectcraft_engine::render::FootageSource; + use effectcraft_engine::render::cache::LayerCache; + use std::time::{Duration, Instant}; + + /// Footage audio that is a constant 0.5. + struct Steady; + impl FootageSource for Steady { + fn frame(&self, _: ItemId, _: &Footage, _: Tick) -> Option> { + None + } + fn audio(&self, _: ItemId, _: &Footage, _: Tick, frames: usize, _: u32) -> Option> { + Some(vec![0.5; frames * 2]) + } + } + struct Device; + impl AudioDevice for Device { + fn sample_rate(&self) -> u32 { + 48000 + } + fn start(&mut self, _: Arc) -> Result<(), String> { + Ok(()) + } + fn stop(&mut self) {} + } + + let mut p = Project::default(); + let footage = + Footage { path: "tone.flac".into(), kind: FootageKind::Audio, duration: Tick::from_seconds_f64(60.0), has_audio: true, ..Default::default() }; + let item = p.add_item("tone.flac", Label::SeaFoam, None, ItemKind::Footage(footage)); + let mut comp = Comp::new(64, 36, FrameRate::new(30, 1), Tick::from_seconds_f64(10.0)); + let layer = build::layer(&mut p, &comp, "tone.flac", LayerSource::Footage { item }, (0, 0), None); + comp.layers = vec![layer].into(); + let cid = p.add_item("Comp", Label::Sandstone, None, ItemKind::Comp(comp.into())); + let mut project = Arc::new(p); + let src = RenderSource { + project: project.clone(), + footage: Arc::new(Steady), + expr: None, + layer_cache: Arc::new(LayerCache::default()), + gpu: None, + gpu_display: false, + disk: None, + }; + let mut playback = + AudioPlayback::start(Box::new(Device), src, cid, Tick::ZERO, Tick::ZERO, Tick::from_seconds_f64(10.0), true, 0).expect("the preview starts"); + let t = Instant::now(); + while playback.feed.queued_frames() == 0 { + assert!(t.elapsed() < Duration::from_secs(30), "the feeder never delivered"); + std::thread::sleep(Duration::from_millis(10)); + } + // Mute the layer while the sound plays. + if let Some(c) = Arc::make_mut(&mut project).comp_mut(cid) { + c.layers[0].switches.audio = false; + } + playback.set_project(project.clone()); + assert!(playback.project().is_some_and(|p| Arc::ptr_eq(&p, &project))); + // The feeder tops the queue up again, now from the muted project. + while playback.feed.queued_frames() == 0 { + assert!(t.elapsed() < Duration::from_secs(30), "the feeder never delivered"); + std::thread::sleep(Duration::from_millis(10)); + } + let mut out = vec![1.0f32; playback.feed.queued_frames() * 2]; + playback.feed.pull(&mut out); + assert!(out.iter().all(|v| *v == 0.0), "the muted layer still plays"); + playback.stop(); + } + + /// #2104: only edits that change the mix drop the queued sound. + #[test] + fn only_audible_edits_change_the_audio() { + use effectcraft_engine::color::Label; + use effectcraft_engine::project::{Comp, Footage, FootageKind, build}; + + let mut p = Project::default(); + let footage = + Footage { path: "tone.flac".into(), kind: FootageKind::Audio, duration: Tick::from_seconds_f64(60.0), has_audio: true, ..Default::default() }; + let item = p.add_item("tone.flac", Label::SeaFoam, None, ItemKind::Footage(footage)); + let mut comp = Comp::new(64, 36, FrameRate::new(30, 1), Tick::from_seconds_f64(10.0)); + let layer = build::layer(&mut p, &comp, "tone.flac", LayerSource::Footage { item }, (0, 0), None); + comp.layers = vec![layer].into(); + let cid = p.add_item("Comp", Label::Sandstone, None, ItemKind::Comp(comp.into())); + let old = p; + // A video-only edit: the layer is hidden, its name and label change. + let mut video = old.clone(); + if let Some(l) = video.comp_mut(cid).and_then(|c| c.layers.first_mut()) { + l.switches.video = false; + l.name = "renamed".into(); + } + assert!(!audio_changed(&old, &video, cid), "a video edit must not drop the sound"); + // Muting, solo and a moved in point do change it. + for edit in 0..3 { + let mut edited = old.clone(); + if let Some(l) = edited.comp_mut(cid).and_then(|c| c.layers.first_mut()) { + match edit { + 0 => l.switches.audio = false, + 1 => l.switches.solo = true, + _ => l.in_point = Tick::from_seconds_f64(1.0), + } + } + assert!(audio_changed(&old, &edited, cid), "audible edit {edit} must be followed"); + } + } + + /// #2104: a video effect edit leaves the mix alone, an audio effect changes it. + #[test] + fn only_audio_effects_change_the_audio() { + use effectcraft_engine::color::Label; + use effectcraft_engine::project::build::{self, Ids}; + use effectcraft_engine::project::{Comp, Footage, FootageKind}; + + let mut p = Project::default(); + let footage = + Footage { path: "tone.flac".into(), kind: FootageKind::Audio, duration: Tick::from_seconds_f64(60.0), has_audio: true, ..Default::default() }; + let item = p.add_item("tone.flac", Label::SeaFoam, None, ItemKind::Footage(footage)); + let mut comp = Comp::new(64, 36, FrameRate::new(30, 1), Tick::from_seconds_f64(10.0)); + let layer = build::layer(&mut p, &comp, "tone.flac", LayerSource::Footage { item }, (0, 0), None); + comp.layers = vec![layer].into(); + let cid = p.add_item("Comp", Label::Sandstone, None, ItemKind::Comp(comp.into())); + let with_effect = |id: &str| { + let mut edited = p.clone(); + let spec = effectcraft_engine::effects::find(id).expect("the effect exists"); + let mut next = edited.next_id; + let g = effectcraft_engine::effects::instantiate(spec, &mut Ids(&mut next), spec.name, [64.0, 36.0]); + edited.next_id = next; + if let Some(fx) = edited.comp_mut(cid).and_then(|c| c.layers.first_mut()).and_then(|l| l.props.sub_mut("effects")) { + fx.children.push(g.into()); + } + edited + }; + assert!(!audio_changed(&p, &with_effect("ec.blur.gaussian"), cid), "a video effect must not drop the sound"); + assert!(audio_changed(&p, &with_effect("ec.audio.backwards"), cid), "an audio effect must be followed"); + } + #[test] fn meter_decays_and_holds_peaks() { let mut m = Meter::default(); @@ -573,3 +842,7 @@ mod tests { assert!(m.peak_db[0] < -10.0, "{}", m.peak_db[0]); } } + +#[cfg(test)] +#[path = "audio/live_tests.rs"] +mod live_tests; diff --git a/crates/ui-egui/src/audio/live_tests.rs b/crates/ui-egui/src/audio/live_tests.rs new file mode 100644 index 000000000..6ca45caac --- /dev/null +++ b/crates/ui-egui/src/audio/live_tests.rs @@ -0,0 +1,216 @@ +//! #2104: exercise the real mixer and queue without relying on thread timing or an audio device. + +use super::{AudioDevice, AudioFeed, AudioPlayback, Feeder, PlaySpan, feed_block}; +use crate::frames::RenderSource; +use effectcraft_engine::color::Label; +use effectcraft_engine::keyframe::Value; +use effectcraft_engine::project::{Comp, Expression, Footage, FootageKind, ItemId, ItemKind, Layer, LayerSource, Project, Property, build}; +use effectcraft_engine::render::{EvalCtx, ExprHost, FootageSource, LayerCache}; +use effectcraft_engine::time::{FrameRate, Tick}; +use std::sync::atomic::AtomicBool; +use std::sync::{Arc, Mutex, mpsc}; +use std::time::Duration; + +struct Device; +impl AudioDevice for Device { + fn sample_rate(&self) -> u32 { + 48000 + } + fn start(&mut self, _: Arc) -> Result<(), String> { + Ok(()) + } + fn stop(&mut self) {} +} + +/// A ramp identifies the exact source sample, exposing skips and repeats after an edit. +struct Ramp; +impl FootageSource for Ramp { + fn frame(&self, _: ItemId, _: &Footage, _: Tick) -> Option> { + None + } + fn audio(&self, _: ItemId, _: &Footage, t: Tick, frames: usize, rate: u32) -> Option> { + let first = t.to_units_floor(i64::from(rate)); + Some((0..frames).flat_map(|i| [(first + i as i64) as f32 / 128.0; 2]).collect()) + } +} + +/// Construct a paused feeder whose short nonzero span makes every source sample observable. +fn playback(footage: Arc) -> AudioPlayback { + let mut project = Project::default(); + let f = Footage { path: "ramp.wav".into(), kind: FootageKind::Audio, has_audio: true, ..Default::default() }; + let item = project.add_item("Ramp", Label::SeaFoam, None, ItemKind::Footage(f)); + let mut comp = Comp::new(64, 36, FrameRate::new(30, 1), Tick::from_seconds_f64(1.0)); + let layer = build::layer(&mut project, &comp, "Ramp", LayerSource::Footage { item }, (0, 0), None); + comp.layers = vec![layer].into(); + let cid = project.add_item("Comp", Label::Sandstone, None, ItemKind::Comp(comp.into())); + let src = RenderSource { + project: Arc::new(project), + footage, + expr: None, + layer_cache: Arc::new(LayerCache::default()), + gpu: None, + gpu_display: false, + disk: None, + }; + AudioPlayback { + feed: Arc::new(AudioFeed::default()), + device: Box::new(Device), + stop: Arc::new(AtomicBool::new(false)), + thread: None, + state: Arc::new(Mutex::new(Feeder { src, cursor: 24, generation: 0 })), + comp: cid, + mix: 48000, + rate: 48000, + span: PlaySpan { start: 24, wa: 16, wb: 64, looping: false }, + } +} + +/// Run exactly one production feeder block, without a background refill racing assertions. +fn mix(p: &AudioPlayback) { + feed_block(&p.state, p.comp, &p.feed, &p.span, p.rate, p.mix); +} + +/// Copy the active project and apply a layer edit through the same snapshot boundary as the UI. +fn edit(p: &AudioPlayback, update: impl FnOnce(&mut Layer)) -> Arc { + let mut project = p.project().unwrap(); + update(&mut Arc::make_mut(&mut project).comp_mut(p.comp).unwrap().layers[0]); + project +} + +/// Levels edits retain sample position across the loop boundary and the exclusive non-looping end. +#[test] +fn project_replacement_resumes_at_the_first_unconsumed_sample() { + for looping in [false, true] { + let mut p = playback(Arc::new(Ramp)); + p.span.looping = looping; + mix(&p); + let mut out = [0.0; 64]; + p.feed.pull(&mut out); + assert_eq!(out[0], 24.0 / 128.0); + assert_eq!(p.feed.consumed(), 32); + let updated = edit(&p, |l| l.props.sub_mut("audio").unwrap().get_mut("levels").unwrap().value = Value::Vec2([-6.020599913279624; 2])); + p.follow_project(updated); + assert_eq!(p.feed.queued_frames(), 0); + assert_eq!(p.state.lock().unwrap().cursor, 56); + mix(&p); + let mut tail = [0.0; 16]; + p.feed.pull(&mut tail); + assert!((tail[0] - 56.0 / 256.0).abs() < 1e-6); + assert!((tail[14] - 63.0 / 256.0).abs() < 1e-6); + p.set_project(p.project().unwrap()); + assert_eq!(p.state.lock().unwrap().cursor, if looping { 16 } else { 64 }); + mix(&p); + if looping { + let mut first = [0.0; 2]; + p.feed.pull(&mut first); + assert!((first[0] - 16.0 / 256.0).abs() < 1e-6, "resume at the work-area start"); + } else { + assert!(p.finished(), "an edit after the last sample must not replay it"); + assert_eq!(p.feed.queued_frames(), 0); + } + } +} + +struct Gated { + entered: mpsc::Sender<()>, + resume: Mutex>, +} +impl FootageSource for Gated { + fn frame(&self, _: ItemId, _: &Footage, _: Tick) -> Option> { + None + } + fn audio(&self, _: ItemId, _: &Footage, _: Tick, frames: usize, _: u32) -> Option> { + self.entered.send(()).unwrap(); + self.resume.lock().unwrap().recv_timeout(Duration::from_secs(10)).unwrap(); + Some(vec![0.5; frames * 2]) + } +} + +/// A slow mix must not block edits or publish samples from the superseded project generation. +#[test] +fn a_block_in_flight_cannot_publish_after_a_mute() { + let (entered, waiting) = mpsc::channel(); + let (resume, gate) = mpsc::channel(); + let mut p = playback(Arc::new(Gated { entered, resume: Mutex::new(gate) })); + let (state, feed, span, comp, rate) = (p.state.clone(), p.feed.clone(), p.span, p.comp, p.rate); + let worker = std::thread::spawn(move || feed_block(&state, comp, &feed, &span, rate, rate)); + waiting.recv_timeout(Duration::from_secs(10)).unwrap(); + // The edit completes while the mixer is blocked, without waiting on its work. + p.follow_project(edit(&p, |l| l.switches.audio = false)); + resume.send(()).unwrap(); + worker.join().unwrap(); + assert_eq!(p.feed.queued_frames(), 0, "the old generation cannot publish"); + assert_eq!(p.state.lock().unwrap().cursor, 24, "nor advance the replacement cursor"); + mix(&p); + let mut out = [1.0; 16]; + p.feed.pull(&mut out); + assert_eq!(out, [0.0; 16]); +} + +/// Models a levels expression that reads a video property through the renderer's expression API. +struct OpacityLevels; +impl ExprHost for OpacityLevels { + fn eval(&self, _: &EvalCtx, layer: &Layer, _: &Property, _: &Value) -> Result { + let opacity = layer.transform().unwrap().get("opacity").unwrap().value.as_f64(); + Ok(Value::Vec2([if opacity == 0.0 { -96.0 } else { 0.0 }; 2])) + } +} + +/// Video properties are irrelevant to a plain mix, but may be dependencies of audio expressions. +#[test] +fn video_edits_keep_the_queue_unless_audio_expressions_can_read_them() { + for expression in [false, true] { + let mut p = playback(Arc::new(Ramp)); + if expression { + let updated = edit(&p, |l| { + l.props.sub_mut("audio").unwrap().get_mut("levels").unwrap().expr = + Some(Expression { text: "[transform.opacity, transform.opacity]".into(), enabled: true }); + }); + let mut state = p.state.lock().unwrap(); + state.src.project = updated; + state.src.expr = Some(Arc::new(OpacityLevels)); + } + mix(&p); + assert_eq!(p.feed.queued_frames(), 40); + let updated = edit(&p, |l| l.props.sub_mut("transform").unwrap().get_mut("opacity").unwrap().value = Value::Scalar(0.0)); + p.follow_project(updated); + if expression { + assert_eq!(p.feed.queued_frames(), 0, "the expression can read the edited video property"); + mix(&p); + let mut out = [1.0; 16]; + p.feed.pull(&mut out); + assert_eq!(out, [0.0; 16]); + } else { + assert_eq!(p.feed.queued_frames(), 40, "an unrelated edit cannot interrupt audio"); + } + } +} + +/// Reused nested comps cost linear visits while edits to their descendants remain detectable. +#[test] +fn shared_nested_comps_are_compared_once_per_depth() { + let mut project = Project::default(); + let mut child = None; + for _ in 0..10 { + let mut comp = Comp::new(64, 36, FrameRate::new(30, 1), Tick::from_seconds_f64(1.0)); + if let Some(item) = child { + for _ in 0..10 { + let mut layer = build::layer(&mut project, &comp, "Nested", LayerSource::Comp { item }, (64, 36), None); + layer.switches.audio = false; + comp.layers.push(layer); + } + } + child = Some(project.add_item("Nested", Label::Sandstone, None, ItemKind::Comp(comp.into()))); + } + let root = child.unwrap(); + let mut edited = project.clone(); + edited.comp_mut(root).unwrap().layers[0].name = "Renamed".into(); + let mut seen = std::collections::HashSet::new(); + assert!(!super::audio_changed_inner(&project, &edited, root, &mut seen)); + assert_eq!(seen.len(), 10, "compare the ten comps, not a billion paths through them"); + let leaf = *seen.iter().find(|(_, depth)| *depth == 9).map(|(item, _)| item).unwrap(); + let mut layer = project.comp(root).unwrap().layers[0].clone(); + layer.source = LayerSource::Null; + edited.comp_mut(leaf).unwrap().layers.push(layer); + assert!(super::audio_changed(&project, &edited, root), "memoization must still find a nested edit"); +} diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index be14cd8df..7a8805ec4 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -203,6 +203,8 @@ pub struct Playback { /// sound plays it goes on through frames that aren't cached, and the picture drops frames to /// follow it (#2473). pub audio_held: bool, + /// The project last checked for a silent preview that became audible (#2104). + pub silent_project: Option>, /// Starting a preview with sound (#947): the playhead stays on the first frame while the /// frames ahead render (the prefetch window, at most what fits in the RAM preview), then /// sound and picture start together from it. Without this the picture played on silently diff --git a/crates/ui-egui/tests/it/ui_audio_scrub.rs b/crates/ui-egui/tests/it/ui_audio_scrub.rs index 80936042a..ae724c9ad 100644 --- a/crates/ui-egui/tests/it/ui_audio_scrub.rs +++ b/crates/ui-egui/tests/it/ui_audio_scrub.rs @@ -739,3 +739,47 @@ fn adding_markers_does_not_stop_the_sound() { } assert_eq!(h.state().session.project.comp(cid).unwrap().markers.len(), 6); } + +/// #2104: unmuting a preview that started silent opens sound at the moving preview position, +/// both with video and with Video off (where the timeline deliberately stays still). +#[test] +fn unmuting_a_silent_preview_starts_audio_at_its_current_frame() { + for video in [false, true] { + let fake = Fake::default(); + let (mut app, cid) = setup(true, &fake); + let layer = app.session.project.comp(cid).unwrap().layers[0].id.0; + app.session.execute("layer.setSwitch", json!({"layers": [layer], "switch": "audio", "value": false})).unwrap(); + let gate = GatedFrames::default(); + app.frames.set_remote(Some(Arc::new(gate.clone()))); + let mut h = Harness::builder().with_size(egui::vec2(1400.0, 900.0)).with_step_dt(1.0 / 30.0).build_eframe(|_| app); + h.state_mut().use_manual_clock(); + h.run_steps(2); + gate.render(&mut h, cid, &(0..60).collect::>()); + let now = h.state().now(); + let shortcut = if video { h.state().session.prefs.preview.current } else { effectcraft_engine::preview::PreviewShortcut::AltNumpad0 }; + h.state_mut().play_with(now, shortcut); + for _ in 0..6 { + step(&mut h); + } + assert!(h.state().audio.is_none(), "muted playback has no output"); + let current = if video { frame(&h, cid) } else { h.state().playback.audio_frame.unwrap() }; + assert!(current > 0, "the silent preview advanced"); + h.state_mut().session.execute("layer.setSwitch", json!({"layers": [layer], "switch": "audio", "value": true})).unwrap(); + gate.render(&mut h, cid, &(0..60).collect::>()); + h.step(); + assert_eq!(audio_frame(&h, cid), Some(current), "unmute starts at the moving preview frame, video={video}"); + if !video { + assert_eq!(frame(&h, cid), 0, "audio-only preview leaves the timeline stationary"); + } + let feed = fake.0.lock().unwrap().0.clone().expect("unmute opened the output"); + assert!(pull(&feed, 256).iter().any(|v| v.abs() > 0.01), "the unmuted tone reaches the output"); + // Mute and unmute again without stopping: edits reach the existing feed too. + h.state_mut().session.execute("layer.setSwitch", json!({"layers": [layer], "switch": "audio", "value": false})).unwrap(); + h.step(); + assert!(pull(&feed, 256).iter().all(|v| *v == 0.0), "mute silences the current output"); + h.state_mut().session.execute("layer.setSwitch", json!({"layers": [layer], "switch": "audio", "value": true})).unwrap(); + h.step(); + assert!(pull(&feed, 256).iter().any(|v| v.abs() > 0.01), "a second unmute reaches the same output"); + h.state_mut().stop(); + } +}