diff --git a/Cargo.lock b/Cargo.lock index e92eb3a16..9ad887965 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1633,6 +1633,17 @@ dependencies = [ "wgpu", ] +[[package]] +name = "effectcraft-helper-host" +version = "0.21.0" +dependencies = [ + "bytemuck", + "effectcraft-effects", + "log", + "serde", + "serde_json", +] + [[package]] name = "effectcraft-hevcenc" version = "0.21.0" @@ -1647,6 +1658,7 @@ dependencies = [ "effectcraft-engine", "effectcraft-export", "effectcraft-expr", + "effectcraft-helper-host", "effectcraft-interchange", "effectcraft-media", "effectcraft-pdf", @@ -1663,6 +1675,7 @@ dependencies = [ "log", "rayon", "serde_json", + "tempfile", ] [[package]] @@ -1958,6 +1971,7 @@ dependencies = [ "effectcraft-gpu", "effectcraft-host", "effectcraft-keyframe", + "effectcraft-media", "effectcraft-path", "effectcraft-project", "effectcraft-render", diff --git a/Cargo.toml b/Cargo.toml index 66702fa61..e512ecf15 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -52,6 +52,7 @@ effectcraft-script = { path = "crates/script" } effectcraft-hevcenc = { path = "crates/hevcenc" } effectcraft-av1enc = { path = "crates/av1enc" } effectcraft-plugin = { path = "crates/plugin" } +effectcraft-helper-host = { path = "crates/helper-host" } # foundation thiserror = "2" diff --git a/apps/effectcraft/src/main.rs b/apps/effectcraft/src/main.rs index 9e9c6863e..a70e03fcd 100644 --- a/apps/effectcraft/src/main.rs +++ b/apps/effectcraft/src/main.rs @@ -148,6 +148,24 @@ fn install_file_dialogs(app: &mut EffectcraftApp) -> Option { + for e in r["errors"].as_array().into_iter().flatten() { + log::warn!("effect helpers: {}", e.as_str().unwrap_or("error")); + } + } + Err(e) => log::warn!("effect helpers: {e}"), + } +} + /// Run a normal desktop launch: install logging and the panic hook, parse arguments (`--version` exits before the /// session log is attached), scan fonts, honor the launch guard's backend choice and open the eframe window. /// Builds the session, loads settings, plugins, the requested project/media and the optional control server, then @@ -253,7 +271,7 @@ fn main() -> eframe::Result { // config directory. Agent-driven runs (`--control`) skip crash recovery. if let Some(dir) = effectcraft_host::config_dir() { // WebAssembly effect plug-ins in /Plug-ins load before the menus are built. - let plugins = dir.join("Plug-ins"); + let plugins = dir.join(effectcraft_engine::commands::PLUGINS_DIR); if plugins.is_dir() { let _ = session.execute("effect.plugins.load", json!({"folder": plugins.to_string_lossy()})); // Packages installed from the Plug-in Library (/Plug-ins/Library). @@ -262,6 +280,7 @@ fn main() -> eframe::Result { log::warn!("plug-in library: {e}"); } } + start_helpers_if_opted_in(&mut session); session.config = Some(std::sync::Arc::new(effectcraft_engine::config::DirConfig::new(dir))); } session.load_settings(); @@ -503,6 +522,7 @@ impl eframe::App for Desktop { self.launch.exited(); self.window.save(self.app.session.config.as_deref()); self.app.on_exit(); + effectcraft_host::stop_helpers(); // Scripts in Scripts/Shutdown run with the project closed. let _ = self.app.session.execute("file.runShutdownScripts", json!({})); } diff --git a/contributors/people.toml b/contributors/people.toml index 2494d9238..defb83972 100644 --- a/contributors/people.toml +++ b/contributors/people.toml @@ -131,3 +131,6 @@ sync_github_name = false [people.JonRC] real_name = "Jonathan Cardoso" display_name = "Jonathan Cardoso" + +[people.nazar-gavrylyk] +sync_github_name = true diff --git a/crates/automation/src/confine.rs b/crates/automation/src/confine.rs index 0d7ecc209..a41dd17bd 100644 --- a/crates/automation/src/confine.rs +++ b/crates/automation/src/confine.rs @@ -11,7 +11,7 @@ //! folder. In a headless session the footage and other files a project or an imported file names //! are read only inside the roots ([`ConfinedFootage`], [`ConfinedServices`]). Commands that run //! code or open other applications (scripts, ScriptUI panels and the events that reach them, -//! Execute File, Reveal in Finder…) are refused while confined, as is granting scripts the file +//! Execute File, effect helpers, Reveal in Finder…) are refused while confined, as is granting scripts the file //! and network access the user withheld. use std::path::{Component, Path, PathBuf}; @@ -46,6 +46,10 @@ const REFUSED: &[&str] = &[ "file.installScript", "file.installScriptUIPanel", "file.uninstallScript", + // Copies a plug-in (and a bundle's ScriptUI panels) from a user-chosen path into the folders + // the app loads code from: an agent must not plant code that runs on every start, and its + // sources are checked no better than `file.installScript`'s. + "effect.plugins.install", // A ScriptUI panel's script runs when the panel opens, when an event reaches one of its // controls, and when one of its `app.scheduleTask` tasks comes due. "window.scriptPanel", @@ -56,6 +60,8 @@ const REFUSED: &[&str] = &[ "scriptui.mouse", "scriptui.close", "file.executeFile", + "effect.helpers.scan", + "effect.helpers.load", "file.revealInFinder", "layer.openSource", "help.revealLogFile", @@ -626,4 +632,25 @@ mod tests { assert!(c.check_command("file.createProxy", &serde_json::json!({"output": p(&outside, "proxy.mov")})).is_err()); let _ = std::fs::remove_dir_all(&base); } + + /// Installing a plug-in plants code the app runs at every start, so confinement refuses it + /// even when the file is inside a read root (like `file.installScript`). + #[test] + fn installing_plugins_is_refused_while_confined() { + let base = std::env::temp_dir().join(format!("ec-confine-plugins-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&base); + let read = base.join("in"); + std::fs::create_dir_all(&read).unwrap(); + let c = Confinement::new(std::slice::from_ref(&read), &[]).unwrap(); + let inside = read.join("a.wasm").to_string_lossy().into_owned(); + for params in [ + serde_json::json!({"path": inside}), + serde_json::json!({"folder": read.to_string_lossy()}), + serde_json::json!({"path": "a.wasm"}), + serde_json::json!({"bundle": true}), + ] { + assert!(c.check_command("effect.plugins.install", ¶ms).is_err(), "{params}"); + } + let _ = std::fs::remove_dir_all(&base); + } } diff --git a/crates/automation/src/tests.rs b/crates/automation/src/tests.rs index 20d6cc4e9..92b62da2f 100644 --- a/crates/automation/src/tests.rs +++ b/crates/automation/src/tests.rs @@ -1482,6 +1482,9 @@ fn automation_roots_confine_file_access() { let steps = json!([{"command": "library.install", "params": {"id": "x"}}]); denied(&mut s, "batch", json!({"steps": steps})); call_json(&mut s, "execute_command", json!({"command": "library.list"})); + // Effect helpers are native programs: starting them or loading through them is refused too. + denied(&mut s, "execute_command", json!({"command": "effect.helpers.scan"})); + denied(&mut s, "execute_command", json!({"command": "effect.helpers.load", "params": {"path": p(&read, "x.ofx")}})); for (id, params) in [ ("file.runStartupScripts", json!({})), ("file.runShutdownScripts", json!({})), diff --git a/crates/effects/src/audio_fx.rs b/crates/effects/src/audio_fx.rs index 2aab70309..c4cb50de2 100644 --- a/crates/effects/src/audio_fx.rs +++ b/crates/effects/src/audio_fx.rs @@ -193,7 +193,7 @@ pub fn specs() -> Vec { /// Is `id` an audio effect (applied by the mixdown rather than the compositor)? pub fn is_audio_effect(id: &str) -> bool { - id.starts_with("ec.audio.") + effectcraft_project::is_audio_effect(id) } /// Does the effect produce sound on its own (so a layer without audio becomes audible)? diff --git a/crates/effects/src/lib.rs b/crates/effects/src/lib.rs index 9130b061c..0138cd9ca 100644 --- a/crates/effects/src/lib.rs +++ b/crates/effects/src/lib.rs @@ -544,6 +544,12 @@ pub struct EffectEnv<'a> { /// comp's and the layer's motion blur switches are on (Transform's Use Composition's /// Shutter Angle). pub shutter: Option<(f64, f64, u32)>, + /// The layer's source time range in **layer time** seconds as (first, end): the layer time of + /// the first source frame and the time just past the last one (so the source lasts + /// `end - first` seconds and its last frame starts one frame before `end`). `None` when the + /// source has no duration (stills, solids, text, shapes) or the renderer does not know it. + /// Retimers clamp their source requests to it. + pub layer_span: Option<(f64, f64)>, } /// What an effect gets to render with. diff --git a/crates/effects/src/plugin.rs b/crates/effects/src/plugin.rs index 01b0b042c..5c973ac57 100644 --- a/crates/effects/src/plugin.rs +++ b/crates/effects/src/plugin.rs @@ -13,14 +13,21 @@ //! //! The contract (stable within an API version): //! * pixels are premultiplied RGBA `f32` (0–1, may exceed 1 in 32 bpc projects), row-major, -//! in the layer's buffer, which may be scaled (preview resolution) and padded; +//! row 0 on top, in the layer's buffer, which may be scaled (preview resolution) and padded; //! * parameters are flattened to `f64`s in declaration order: slider / angle / checkbox (0 or 1) //! / popup (0-based index) take one value, point two (**frame pixels**, already mapped to the -//! buffer), colour four (RGBA 0–1); +//! buffer), colour four (RGBA 0–1); API 2 adds point3 (x, y mapped like a point, z as is), +//! layer (one value: the layer id, -1 for none), and text / hidden strings, which take no +//! slot and are read with [`PluginParams::text`]; //! * `time` is layer time in seconds; `scale` is the buffer's pixels per layer pixel (scale //! pixel distances like blur radii by it); //! * rendering must be deterministic (same inputs, same output) and may run on several threads //! at once. +//! +//! API 2 (native plug-ins; the WebAssembly ABI stays at 1) adds [`EffectPlugin::padding`] and +//! [`EffectPlugin::render_v2`], which gets a [`PluginHost`] to read the effect's input or other +//! layers at other times, and the extra parameter kinds. External effect helpers +//! (`effectcraft-helper-host`) are built on it. use std::sync::{Arc, OnceLock, RwLock}; @@ -28,13 +35,19 @@ use effectcraft_keyframe::Value; use effectcraft_project::ParamUi; use serde::{Deserialize, Serialize}; -use crate::{Buf, EffectCtx, EffectSpec, ParamSpec, RenderFn}; +use crate::{Buf, EffectCtx, EffectSpec, ParamSpec, Params, RenderFn}; -/// The plug-in API version this build implements. -pub const PLUGIN_API_VERSION: u32 = 1; +/// The plug-in API version this build implements (manifests with API 1 or 2 are accepted). +pub const PLUGIN_API_VERSION: u32 = 2; +/// The plug-in API a WebAssembly module implements (its `ec_api_version`). API 2 is for native +/// and helper plug-ins; WebAssembly modules stay at 1. +pub const WASM_API_VERSION: u32 = 1; -/// The most plug-in effects one process can register. -pub const MAX_PLUGINS: usize = 256; +/// The most plug-in effects one process can register (commercial effect suites register a few hundred each). +pub const MAX_PLUGINS: usize = 1024; + +/// The most transparent pixels a plug-in may ask [`EffectPlugin::padding`] to add per side. +const MAX_PADDING: u32 = 4096; /// A plug-in parameter's kind, default and range. #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] @@ -53,6 +66,9 @@ pub enum PluginParamKind { slider_max: Option, #[serde(default = "two")] decimals: u8, + /// Whole numbers (API 2; integer parameters): shown with 0 decimals. + #[serde(default)] + integer: bool, }, Angle { #[serde(default)] @@ -78,6 +94,24 @@ pub enum PluginParamKind { #[serde(default = "white")] default: [f64; 4], }, + /// API 2: a 3D point (x, y in frame pixels like [`PluginParamKind::Point`], z as is). The + /// default is in layer pixels (not fractions). + Point3 { + #[serde(default)] + default: [f64; 3], + }, + /// API 2: another layer of the composition (flattens to its id, -1 for none). + Layer, + /// API 2: a string, shown as a text field; read with [`PluginParams::text`]. + Text { + #[serde(default)] + default: String, + }, + /// API 2: a string that is not shown (opaque plug-in data); read with [`PluginParams::text`]. + Hidden { + #[serde(default)] + default: String, + }, } fn hundred() -> f64 { @@ -98,10 +132,21 @@ impl PluginParamKind { pub fn width(&self) -> usize { match self { PluginParamKind::Point { .. } => 2, + PluginParamKind::Point3 { .. } => 3, PluginParamKind::Color { .. } => 4, + PluginParamKind::Text { .. } | PluginParamKind::Hidden { .. } => 0, _ => 1, } } + /// Whether the parameter holds a string (it flattens to no `f64`s but one entry of + /// [`PluginParams::texts`]). + pub fn is_text(&self) -> bool { + matches!(self, PluginParamKind::Text { .. } | PluginParamKind::Hidden { .. }) + } + /// Whether the kind needs plug-in API 2 (WebAssembly plug-ins can't carry it). + pub fn needs_v2(&self) -> bool { + matches!(self, PluginParamKind::Point3 { .. } | PluginParamKind::Layer | PluginParamKind::Text { .. } | PluginParamKind::Hidden { .. }) + } } /// One parameter of a plug-in effect. @@ -110,6 +155,10 @@ pub struct PluginParam { /// Stable id (the property's match name; letters, digits, `_`). pub id: String, pub name: String, + /// API 2: the group / page path the parameter belongs to (`Advanced/Edges`). Hosts may + /// ignore it. + #[serde(default)] + pub group: String, #[serde(flatten)] pub kind: PluginParamKind, } @@ -117,7 +166,7 @@ pub struct PluginParam { /// What a plug-in declares about itself. #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] pub struct PluginManifest { - /// The plug-in API version it was written for (must be [`PLUGIN_API_VERSION`]). + /// The plug-in API version it was written for (1, or 2 up to [`PLUGIN_API_VERSION`]). pub api: u32, /// Stable, reverse-domain effect id (`org.example.posterize`); `ec.` is reserved. pub id: String, @@ -137,9 +186,12 @@ pub struct PluginManifest { impl PluginManifest { /// Check the manifest against this API version. pub fn validate(&self) -> Result<(), String> { - const MAX_PARAMS: usize = 256; - if self.api != PLUGIN_API_VERSION { - return Err(format!("plug-in `{}` targets plug-in API {} but this build implements API {PLUGIN_API_VERSION}", self.id, self.api)); + // API 2 effects from large third-party suites can declare far more parameters. + let max_params: usize = if self.api >= 2 { 1024 } else { 256 }; + // Opaque plug-in data can be large (some effects keep look-up tables of a few MB). + const MAX_TEXT: usize = 64 << 20; + if !(1..=PLUGIN_API_VERSION).contains(&self.api) { + return Err(format!("plug-in `{}` targets plug-in API {} but this build implements API 1 to {PLUGIN_API_VERSION}", self.id, self.api)); } let ok_id = |s: &str| !s.is_empty() && s.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | '-')); if !ok_id(&self.id) || self.id.starts_with("ec.") { @@ -148,8 +200,8 @@ impl PluginManifest { if self.name.trim().is_empty() || self.category.trim().is_empty() { return Err(format!("plug-in `{}` needs a name and a category", self.id)); } - if self.params.len() > MAX_PARAMS { - return Err(format!("plug-in `{}` has {} parameters (at most {MAX_PARAMS})", self.id, self.params.len())); + if self.params.len() > max_params { + return Err(format!("plug-in `{}` has {} parameters (at most {max_params})", self.id, self.params.len())); } let mut seen = std::collections::HashSet::new(); for p in &self.params { @@ -161,6 +213,16 @@ impl PluginManifest { { return Err(format!("plug-in `{}`: popup `{}` needs options and a default index in range", self.id, p.id)); } + if let PluginParamKind::Point3 { default } = &p.kind + && default.iter().any(|v| !v.is_finite()) + { + return Err(format!("plug-in `{}`: point3 `{}` needs a finite default", self.id, p.id)); + } + if let PluginParamKind::Text { default } | PluginParamKind::Hidden { default } = &p.kind + && default.len() > MAX_TEXT + { + return Err(format!("plug-in `{}`: the default of `{}` is longer than {MAX_TEXT} bytes", self.id, p.id)); + } if let PluginParamKind::Slider { default, min, max, slider_min, slider_max, .. } = &p.kind { let (lo, hi) = (slider_min.unwrap_or(*min), slider_max.unwrap_or(*max)); if [*default, *min, *max, lo, hi].iter().any(|v| !v.is_finite()) || min > max || lo > hi || default < min || default > max { @@ -181,16 +243,78 @@ impl PluginManifest { pub struct PluginFrame<'a> { pub width: u32, pub height: u32, - /// Premultiplied RGBA, row-major, `width * height` pixels. + /// Premultiplied RGBA, row-major (row 0 = top), `width * height` pixels. pub pixels: &'a mut [[f32; 4]], /// Buffer pixels per layer pixel (preview resolution). pub scale: f64, + /// API 2: buffer-pixel position of layer pixel (0, 0) ([`Buf::offset`]; padding shifts it). + pub origin: [f64; 2], + /// API 2: the layer's bounds in layer pixels. + pub layer_size: [f64; 2], +} + +/// An image the host hands an API 2 plug-in (the input at another time, or another layer). +/// Same layout as [`PluginFrame`]: premultiplied RGBA, row 0 = top, `origin` = buffer pixels of +/// layer pixel (0, 0) in this image. +#[derive(Clone, Debug, Default)] +pub struct PluginImage { + pub width: u32, + pub height: u32, + pub pixels: Vec<[f32; 4]>, + pub origin: [f64; 2], + pub scale: f64, +} + +impl PluginImage { + fn from_buf(b: Buf) -> PluginImage { + PluginImage { width: b.img.width, height: b.img.height, pixels: b.img.data, origin: b.offset, scale: b.scale } + } +} + +/// Services available during an API 2 render ([`EffectPlugin::render_v2`]). +pub trait PluginHost { + /// This effect's input (its layer with masks and the effects above this one) at another + /// **layer time** in seconds, in its own extent (see [`PluginImage::origin`]; it is not + /// padded). `None` when unavailable (adjustment layers, no host). + fn input_at(&self, layer_time: f64) -> Option; + /// The layer chosen in Layer parameter `param_id` at **layer time** `layer_time` (converted + /// to composition time), effects and masks applied, resampled into this layer's buffer + /// (same size, origin and scale as the frame, centred when the layers differ in size). + /// `None` when unset or unavailable. + fn layer_input(&self, param_id: &str, layer_time: f64) -> Option; + /// All flattened parameter values (and texts) at another layer time. + fn params_at(&self, layer_time: f64) -> Option<(Vec, Vec)>; + /// Composition frame rate (30 when unknown). + fn frame_rate(&self) -> f64; + /// Layer time of the current render, seconds. + fn time(&self) -> f64; + /// A stable id of the applied effect instance: the same on every frame of one effect on one + /// layer, different for every layer and every application of the effect. Plug-ins that keep + /// state between frames pool their native instances by it. 0 when unknown. + fn instance_key(&self) -> u64 { + 0 + } + /// The layer's source time range in **layer time** seconds as (first, end): the first source + /// frame starts at `first` and the source ends at `end` (see [`crate::EffectEnv::layer_span`]). + /// `None` when the source has no duration or the host does not know it (retimers then + /// assume an unbounded range). + fn frame_range(&self) -> Option<(f64, f64)> { + None + } +} + +/// The [`PluginHost::instance_key`] of the effect `ctx` renders: its instance seed (stable per +/// effect instance), offset so it is never 0. +pub fn ctx_instance_key(ctx: &EffectCtx) -> u64 { + (1u64 << 32) | u64::from(ctx.seed) } /// Parameter values for one render, flattened (see the module docs) with typed accessors. pub struct PluginParams<'a> { pub manifest: &'a PluginManifest, pub flat: &'a [f64], + /// One string per text / hidden parameter, in declaration order. + pub texts: &'a [String], } impl PluginParams<'_> { @@ -214,10 +338,31 @@ impl PluginParams<'_> { } /// Point in frame pixels. pub fn point(&self, id: &str) -> [f64; 2] { - self.slot(id).map(|s| [s[0], s[1]]).unwrap_or([0.0; 2]) + self.slot(id).and_then(|s| Some([*s.first()?, *s.get(1)?])).unwrap_or([0.0; 2]) + } + /// 3D point: x, y in frame pixels, z as is. + pub fn point3(&self, id: &str) -> [f64; 3] { + self.slot(id).and_then(|s| Some([*s.first()?, *s.get(1)?, *s.get(2)?])).unwrap_or([0.0; 3]) } pub fn color(&self, id: &str) -> [f64; 4] { - self.slot(id).map(|s| [s[0], s[1], s[2], s[3]]).unwrap_or([1.0; 4]) + self.slot(id).and_then(|s| Some([*s.first()?, *s.get(1)?, *s.get(2)?, *s.get(3)?])).unwrap_or([1.0; 4]) + } + /// The layer chosen in a layer parameter. + pub fn layer(&self, id: &str) -> Option { + self.slot(id).and_then(|s| s.first().copied()).filter(|v| *v >= 0.0).map(|v| v as u64) + } + /// A text / hidden parameter's string (empty for other ids). + pub fn text(&self, id: &str) -> &str { + let mut at = 0; + for p in &self.manifest.params { + if p.kind.is_text() { + if p.id == id { + return self.texts.get(at).map(String::as_str).unwrap_or(""); + } + at += 1; + } + } + "" } } @@ -231,6 +376,18 @@ pub trait EffectPlugin: Send + Sync { fn source(&self) -> String { "built-in".into() } + /// API 2: transparent pixels to add on every side of the buffer before rendering (glows, + /// blurs, regions of definition larger than the input), in buffer pixels. `layer_size` is + /// the layer's size in layer pixels (the frame's [`PluginFrame::layer_size`]; 0 when unknown) + /// and `instance_key` the [`PluginHost::instance_key`] this render will have. + fn padding(&self, _params: &PluginParams, _time: f64, _scale: f64, _layer_size: [f64; 2], _instance_key: u64) -> u32 { + 0 + } + /// API 2: [`EffectPlugin::render`] with host services. The default calls `render`. May be + /// called from several threads at once. + fn render_v2(&self, frame: &mut PluginFrame, params: &PluginParams, time: f64, _host: &dyn PluginHost) -> Result<(), String> { + self.render(frame, params, time) + } } static SLOTS: [OnceLock>; MAX_PLUGINS] = [const { OnceLock::new() }; MAX_PLUGINS]; @@ -246,6 +403,30 @@ pub fn plugins() -> Vec<&'static EffectSpec> { plugin_specs().read().map(|v| v.clone()).unwrap_or_default() } +/// The last render error of each plug-in effect (cleared by its next good render), so the UI can +/// show it on the effect. +fn render_errors() -> &'static RwLock> { + static E: OnceLock>> = OnceLock::new(); + E.get_or_init(Default::default) +} + +/// Why the last render of plug-in effect `id` failed, if it did (the frame was left as it was). +pub fn plugin_error(id: &str) -> Option { + render_errors().read().ok()?.get(id).cloned() +} + +fn set_plugin_error(id: &str, error: Option) { + let Ok(mut m) = render_errors().write() else { return }; + match error { + Some(e) => { + m.insert(id.to_string(), e); + } + None => { + m.remove(id); + } + } +} + /// The plug-in behind a registered effect id. pub fn plugin(id: &str) -> Option> { SLOTS.iter().filter_map(|s| s.get()).find(|p| p.manifest().id == id).cloned() @@ -257,15 +438,25 @@ fn leak(s: &str) -> &'static str { fn param_spec(p: &PluginParam) -> ParamSpec { let (default, ui) = match &p.kind { - PluginParamKind::Slider { default, min, max, slider_min, slider_max, decimals } => ( + PluginParamKind::Slider { default, min, max, slider_min, slider_max, decimals, integer } => ( Value::Scalar(*default), - ParamUi::Slider { min: *min, max: *max, slider_min: slider_min.unwrap_or(*min), slider_max: slider_max.unwrap_or(*max), decimals: *decimals }, + ParamUi::Slider { + min: *min, + max: *max, + slider_min: slider_min.unwrap_or(*min), + slider_max: slider_max.unwrap_or(*max), + decimals: if *integer { 0 } else { *decimals }, + }, ), PluginParamKind::Angle { default } => (Value::Scalar(*default), ParamUi::Angle), PluginParamKind::Checkbox { default } => (Value::Bool(*default), ParamUi::Checkbox), PluginParamKind::Popup { options, default } => (Value::Enum(*default), ParamUi::Popup { options: options.clone() }), PluginParamKind::Point { default } => (Value::Vec2(*default), ParamUi::Point), PluginParamKind::Color { default } => (Value::Color(*default), ParamUi::Color), + PluginParamKind::Point3 { default } => (Value::Vec3(*default), ParamUi::Point3), + PluginParamKind::Layer => (Value::Layer(None), ParamUi::Layer), + PluginParamKind::Text { default } => (Value::Str(default.clone()), ParamUi::Text), + PluginParamKind::Hidden { default } => (Value::Str(default.clone()), ParamUi::Hidden), }; ParamSpec { id: leak(&p.id), name: leak(&p.name), default, ui } } @@ -303,55 +494,200 @@ fn trampoline(ctx: &EffectCtx, buf: Buf) -> Buf { run_slot(I, ctx, buf) } -macro_rules! trampolines { - ($($i:literal)*) => { [$(trampoline::<$i> as RenderFn),*] }; +/// Concatenate `A` runs of `B` render functions (`N` = `A * B`), for the table below. +const fn concat(parts: [[RenderFn; B]; A]) -> [RenderFn; N] { + let mut out = [trampoline::<0> as RenderFn; N]; + let mut i = 0; + while i < N { + out[i] = parts[i / B][i % B]; + i += 1; + } + out } -const TRAMPOLINES: [RenderFn; MAX_PLUGINS] = trampolines!( - 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 - 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 - 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 - 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 - 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 - 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 - 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 - 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 -); +/// 16 trampolines `base ..= base + 15`. +macro_rules! t16 { + ($b:expr) => { + [ + trampoline::<{ $b }> as RenderFn, + trampoline::<{ $b + 1 }> as RenderFn, + trampoline::<{ $b + 2 }> as RenderFn, + trampoline::<{ $b + 3 }> as RenderFn, + trampoline::<{ $b + 4 }> as RenderFn, + trampoline::<{ $b + 5 }> as RenderFn, + trampoline::<{ $b + 6 }> as RenderFn, + trampoline::<{ $b + 7 }> as RenderFn, + trampoline::<{ $b + 8 }> as RenderFn, + trampoline::<{ $b + 9 }> as RenderFn, + trampoline::<{ $b + 10 }> as RenderFn, + trampoline::<{ $b + 11 }> as RenderFn, + trampoline::<{ $b + 12 }> as RenderFn, + trampoline::<{ $b + 13 }> as RenderFn, + trampoline::<{ $b + 14 }> as RenderFn, + trampoline::<{ $b + 15 }> as RenderFn, + ] + }; +} + +/// Four runs of `$inner` (each `$n` long, starting `$n` apart) from `$b`: `$total` trampolines. +macro_rules! x4 { + ($inner:ident, $b:expr, $n:literal, $total:literal) => { + concat::<4, $n, $total>([$inner!($b), $inner!($b + $n), $inner!($b + 2 * $n), $inner!($b + 3 * $n)]) + }; +} +macro_rules! t64 { + ($b:expr) => { + x4!(t16, $b, 16, 64) + }; +} +macro_rules! t256 { + ($b:expr) => { + x4!(t64, $b, 64, 256) + }; +} +macro_rules! t1024 { + ($b:expr) => { + x4!(t256, $b, 256, 1024) + }; +} + +const TRAMPOLINES: [RenderFn; MAX_PLUGINS] = t1024!(0); /// Flatten an instance's evaluated parameters for the plug-in (points mapped to frame pixels). pub fn flatten(m: &PluginManifest, ctx: &EffectCtx, buf: &Buf) -> Vec { + flatten_params(m, ctx.params, buf.offset, buf.scale) +} + +/// [`flatten`] for any evaluated parameters, with points mapped by buffer `offset` and `scale`. +pub fn flatten_params(m: &PluginManifest, params: &Params, offset: [f64; 2], scale: f64) -> Vec { + let to_px = |q: [f64; 2]| (q[0] * scale + offset[0], q[1] * scale + offset[1]); let mut out = Vec::with_capacity(m.flat_len()); for p in &m.params { - let v = ctx.params.get(&p.id); + let v = params.get(&p.id); match &p.kind { PluginParamKind::Point { .. } => { - let q = v.map(Value::as_vec2).unwrap_or([0.0; 2]); - let (x, y) = buf.to_px(q); + let (x, y) = to_px(v.map(Value::as_vec2).unwrap_or([0.0; 2])); out.extend([x, y]); } + PluginParamKind::Point3 { default } => { + let q = v.map(Value::as_vec3).unwrap_or(*default); + let (x, y) = to_px([q[0], q[1]]); + out.extend([x, y, q[2]]); + } PluginParamKind::Color { .. } => { let c = v.map(Value::as_color).unwrap_or([1.0; 4]); out.extend(c.iter().map(|x| *x as f64)); } PluginParamKind::Checkbox { .. } => out.push(if v.map(Value::as_bool).unwrap_or(false) { 1.0 } else { 0.0 }), PluginParamKind::Popup { .. } => out.push(v.map(Value::as_enum).unwrap_or(0) as f64), + PluginParamKind::Layer => out.push(v.and_then(Value::as_layer).map(|l| l as f64).unwrap_or(-1.0)), + PluginParamKind::Text { .. } | PluginParamKind::Hidden { .. } => {} _ => out.push(v.map(Value::as_f64).unwrap_or(0.0)), } } out } +/// The strings of an instance's text / hidden parameters, in declaration order. +pub fn flatten_texts(m: &PluginManifest, params: &Params) -> Vec { + m.params + .iter() + .filter_map(|p| match &p.kind { + PluginParamKind::Text { default } | PluginParamKind::Hidden { default } => Some(match params.get(&p.id) { + Some(Value::Str(s)) => s.clone(), + _ => default.clone(), + }), + _ => None, + }) + .collect() +} + +/// The host services of one render, on top of the effect context and the (padded) buffer. +struct RenderHost<'a, 'b> { + ctx: &'a EffectCtx<'b>, + manifest: &'a PluginManifest, + width: u32, + height: u32, + offset: [f64; 2], + scale: f64, +} + +impl RenderHost<'_, '_> { + /// The frame's pixel grid without pixels (what [`crate::util::fit_layer`] measures). + fn grid(&self) -> Buf { + Buf { img: crate::Image { width: self.width, height: self.height, data: vec![] }, offset: self.offset, scale: self.scale } + } +} + +impl PluginHost for RenderHost<'_, '_> { + fn input_at(&self, layer_time: f64) -> Option { + let host = self.ctx.env.host?; + host.self_at(layer_time, self.ctx.env.effect_index).map(PluginImage::from_buf) + } + + fn layer_input(&self, param_id: &str, layer_time: f64) -> Option { + let ctx = self.ctx; + let lid = ctx.params.get(param_id)?.as_layer()?; + let other = if (layer_time - ctx.time).abs() < 1e-9 { + ctx.layer_param(param_id, true)? + } else { + let host = ctx.env.host?; + // Source / Masks show no effects; the default (Effects & Masks) does. + let with_effects = !matches!(ctx.params.get(&crate::layer_source_id(param_id)).map(Value::as_enum), Some(0 | 1)); + host.layer_at(lid, ctx.env.comp_time + (layer_time - ctx.time), with_effects)? + }; + let img = crate::util::fit_layer(ctx, &self.grid(), &other, false); + Some(PluginImage { width: img.width, height: img.height, pixels: img.data, origin: self.offset, scale: self.scale }) + } + + fn params_at(&self, layer_time: f64) -> Option<(Vec, Vec)> { + let p = self.ctx.env.host?.params_at(layer_time)?; + Some((flatten_params(self.manifest, &p, self.offset, self.scale), flatten_texts(self.manifest, &p))) + } + + fn frame_rate(&self) -> f64 { + self.ctx.fps() + } + + fn time(&self) -> f64 { + self.ctx.time + } + + fn instance_key(&self) -> u64 { + ctx_instance_key(self.ctx) + } + + fn frame_range(&self) -> Option<(f64, f64)> { + self.ctx.env.layer_span + } +} + fn run_slot(slot: usize, ctx: &EffectCtx, mut buf: Buf) -> Buf { let Some(plugin) = SLOTS[slot].get() else { return buf }; let m = plugin.manifest(); + let texts = flatten_texts(m, ctx.params); let flat = flatten(m, ctx, &buf); - let params = PluginParams { manifest: m, flat: &flat }; - let (width, height) = (buf.img.width, buf.img.height); - let before = buf.img.data.clone(); - let mut frame = PluginFrame { width, height, pixels: &mut buf.img.data, scale: buf.scale }; - if let Err(e) = plugin.render(&mut frame, ¶ms, ctx.time) { + let key = ctx_instance_key(ctx); + let pad = plugin.padding(&PluginParams { manifest: m, flat: &flat, texts: &texts }, ctx.time, buf.scale, ctx.layer_size, key).min(MAX_PADDING); + let before = buf.clone(); + // Padding moves the layer origin, which points are measured from. + let flat = if pad > 0 { + buf.pad(pad); + flatten(m, ctx, &buf) + } else { + flat + }; + let params = PluginParams { manifest: m, flat: &flat, texts: &texts }; + let (width, height, offset, scale) = (buf.img.width, buf.img.height, buf.offset, buf.scale); + let host = RenderHost { ctx, manifest: m, width, height, offset, scale }; + let mut frame = PluginFrame { width, height, pixels: &mut buf.img.data, scale, origin: offset, layer_size: ctx.layer_size }; + if let Err(e) = plugin.render_v2(&mut frame, ¶ms, ctx.time, &host) { log::warn!("plug-in `{}`: {e}", m.id); - buf.img.data = before; + set_plugin_error(&m.id, Some(e)); + return before; + } + if plugin_error(&m.id).is_some() { + set_plugin_error(&m.id, None); } buf } diff --git a/crates/engine/src/cmd.rs b/crates/engine/src/cmd.rs index ee2624fd8..076be70ec 100644 --- a/crates/engine/src/cmd.rs +++ b/crates/engine/src/cmd.rs @@ -19,7 +19,7 @@ pub mod content_fill; mod create; mod edit; mod effect; -pub use effect::{CryptoCoverage, CryptoProbeJob}; +pub use effect::{CryptoCoverage, CryptoProbeJob, PLUGINS_DIR}; pub(crate) use effect::{CryptoProbeSlot, crypto_coverage, crypto_probe_job, crypto_probe_ready, keep_crypto_probe, kept_crypto_probe}; pub mod essential; pub mod expr_tools; diff --git a/crates/engine/src/cmd/effect.rs b/crates/engine/src/cmd/effect.rs index f417c123f..ba9fefaba 100644 --- a/crates/engine/src/cmd/effect.rs +++ b/crates/engine/src/cmd/effect.rs @@ -24,11 +24,12 @@ fn plugins_list(s: &mut Session, _: &Value) -> Result { "author": m.as_ref().map(|m| m.author.clone()), "description": m.as_ref().map(|m| m.description.clone()), "source": p.map(|p| p.source()), + "error": plugin::plugin_error(spec.id), "params": spec.params.iter().map(|p| p.id).collect::>(), }) }) .collect(); - Ok(json!({"api": plugin::PLUGIN_API_VERSION, "wasm": s.plugin_loader.is_some(), "plugins": list})) + Ok(json!({"api": plugin::PLUGIN_API_VERSION, "wasm": s.plugin_loader.is_some(), "helpers": s.helpers.is_some(), "plugins": list})) } /// `effect.plugins.load`: load WebAssembly effect plug-ins (a file, or every `.wasm` in a @@ -70,6 +71,205 @@ fn plugins_load(s: &mut Session, p: &Value) -> Result { Ok(json!({"loaded": loaded, "errors": errors})) } +/// The settings-folder name the desktop app scans for WebAssembly effect plug-ins on start. +/// The scan uses this constant ([`apps/effectcraft/src/main.rs`]), so the two cannot drift. +pub const PLUGINS_DIR: &str = "Plug-ins"; + +/// The most one installed file may be: a plug-in is a compiled effect, a panel is script text. +/// Above this the install refuses rather than reading an unbounded file into memory and copying +/// it into a folder the app loads at every start. +const MAX_INSTALL_BYTES: u64 = 64 << 20; + +/// `effect.plugins.install`: install WebAssembly effect plug-ins so they load on every start, +/// not only in this session (Load Effect Plug-in keeps the file where it is and just loads it). +/// `path` installs one `.wasm`; `folder` installs a bundle in one go: every `.wasm` at its top +/// level is copied into the Plug-ins folder and every `.jsx` becomes a ScriptUI panel. What +/// installs also loads in this session at once, so the effect menu has it immediately. A `.wasm` +/// that fails to load is not copied: a broken plug-in must not fail every start. +fn plugins_install(s: &mut Session, p: &Value) -> Result { + let c = "effect.plugins.install"; + let loader = s.plugin_loader.ok_or_else(|| EngineError::Other("WebAssembly effect plug-ins are not available in this build".into()))?; + let cfg = s.config.clone().ok_or_else(|| EngineError::Other("this session has no settings folder to install plug-ins into".into()))?; + let root = cfg.dir().ok_or_else(|| EngineError::Other("this session has no settings folder to install plug-ins into".into()))?; + // The folder the Plug-in Library installs next to (`Session::plugins_dir`: the host's, else + // `Plug-ins` next to the settings), so both kinds of install live in one place. + let plug_dir = s.plugins_dir().unwrap_or_else(|| root.join(PLUGINS_DIR)); + let single = str_p(p, "path").is_some(); + let mut listing_errors: Vec = Vec::new(); + let sources: Vec = match (str_p(p, "path"), str_p(p, "folder")) { + (Some(path), _) => vec![std::path::PathBuf::from(path)], + (None, Some(dir)) => { + let read = std::fs::read_dir(dir).map_err(|e| bad(c, format!("cannot list {dir}: {e}")))?; + let mut v = Vec::new(); + for entry in read { + match entry { + Ok(e) => match e.path().metadata() { + Ok(m) if m.is_file() => v.push(e.path()), + // A vanished or unreadable entry is a bundle problem: report it rather + // than installing a bundle that silently misses files. (`is_file()` + // alone would treat unreadable metadata as "not a file" and skip it.) + Err(e) => listing_errors.push(json!({"error": format!("cannot read the bundle folder: {e}")})), + Ok(_) => {} + }, + Err(e) => listing_errors.push(json!({"error": format!("cannot read the bundle folder: {e}")})), + } + } + v.sort(); + v + } + _ => return Err(bad(c, "give `path` (a .wasm plug-in) or `folder` (a plug-in bundle)")), + }; + let mut errors = listing_errors; + let mut installed = Vec::new(); + let mut loaded = Vec::new(); + let mut panels = Vec::new(); + for src in &sources { + let name = src.file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default(); + if name.is_empty() || name.starts_with('.') { + // A bundle's hidden files (`.DS_Store`, `._x.wasm` resource forks) are skipped; a + // single install must not report success for a file it skipped. + if single { + return Err(bad(c, format!("`{}` is not a plug-in file name (hidden files are not installed)", src.display()))); + } + continue; + } + let is_wasm = src.extension().is_some_and(|x| x.eq_ignore_ascii_case("wasm")); + let is_panel = src.extension().is_some_and(|x| x.eq_ignore_ascii_case("jsx")); + if single { + if !is_wasm { + return Err(bad(c, "an effect plug-in install is a `.wasm` file")); + } + } else if !is_wasm && !is_panel { + continue; // a bundle may carry readmes, manifests and source: skip them + } + let body: std::result::Result<(), String> = (|| { + // The name the installed file gets, so it must be one the Scripts menu lists and + // `file.uninstallScript` accepts (like `file.installScript`'s): one bare component, + // and on Windows not a device name or one Windows rewrites. + if !super::scripts::is_bare_name(&name) || (cfg!(windows) && super::scripts::windows_unsafe_name(&name)) { + return Err(format!("`{name}` is not a file name that can be installed")); + } + // Never silently replace an installed file: a name collision is either a re-install + // the user did not ask to overwrite or a bundle that would shadow a working plug-in, + // and the update path is explicit (remove the old one first). + if is_panel { + let key = format!("{}/{name}", super::scripts::PANELS_DIR); + // Listed, not read: an existing panel whose text cannot be read is still a panel + // and must not be replaced (reading it would report it as absent). Compared + // ignoring case and checked on disk too: on Windows and macOS `my panel.jsx` is + // `My Panel.jsx`, and writing it would replace that panel. + let lower = name.to_lowercase(); + if cfg.list(super::scripts::PANELS_DIR).iter().any(|n| n.to_lowercase() == lower) || cfg.files().is_file(&root.join(&key)) { + return Err(format!("{name} is already installed as a ScriptUI panel; remove it from the Scripts/ScriptUI Panels folder first to update")); + } + let bytes = s.services.read_file_bounded(&src.to_string_lossy(), MAX_INSTALL_BYTES).map_err(|e| format!("cannot read it: {e}"))?; + let code = String::from_utf8(bytes).map_err(|_| "panels are UTF-8 text (.jsx)".to_string())?; + cfg.write(&key, &code).map_err(|e| format!("cannot install panel: {e}"))?; + panels.push(json!(name)); + return Ok(()); + } + let target = plug_dir.join(&name); + if cfg.files().is_file(&target) { + return Err(format!("{name} is already installed; remove it from the Plug-ins folder first to update")); + } + let bytes = s.services.read_file_bounded(&src.to_string_lossy(), MAX_INSTALL_BYTES).map_err(|e| format!("cannot read it: {e}"))?; + cfg.files().write(&target, &bytes).map_err(|e| format!("cannot write into the Plug-ins folder: {e}"))?; + // Load in-session now: a plug-in that cannot load is rolled back, so a broken + // plug-in never makes every start fail. + // Its source is the installed copy (what `effect.plugins.list` shows and the next + // start loads), not the file it was installed from. + match loader(&bytes, &target.to_string_lossy()) { + Ok(v) => { + loaded.push(v); + installed.push(json!(name)); + Ok(()) + } + Err(e) => match cfg.files().remove(&target) { + Ok(()) => Err(format!("it does not load, nothing was installed: {e}")), + // Never say "nothing was installed" when the invalid copy is still there: + // every start would report it and a re-install would say "already installed". + Err(re) => Err(format!( + "it does not load and its copy could not be removed from the Plug-ins folder ({re}): delete `{name}` there, or every start reports it; {e}" + )), + }, + } + })(); + match body { + Ok(()) => {} + Err(e) => { + if single { + return Err(bad(c, format!("{name}: {e}"))); + } + errors.push(json!({"file": name, "error": e})); + } + } + } + if !panels.is_empty() { + s.scripts_changed(); + } + if !installed.is_empty() || !panels.is_empty() { + let mut toast = format!("Installed {} effect plug-in(s)", installed.len()); + if !panels.is_empty() { + toast.push_str(&format!(" and {} ScriptUI panel(s)", panels.len())); + } + toast.push_str(": they load on every start"); + s.toast(toast); + } + Ok(json!({"installed": installed, "panels": panels, "loaded": loaded, "errors": errors})) +} + +/// External effect helpers need a host that runs them. +fn has_helpers(s: &Session) -> std::result::Result<(), String> { + if s.helpers.is_some() { Ok(()) } else { Err("effect helpers are not available in this build".into()) } +} + +/// The host's helper hooks, for the commands that start helpers. Helpers run native code, so a +/// script may start them only with Allow Scripts to Write Files and Access Network on (as for +/// File.execute()). +fn helper_hooks(s: &Session, c: &str) -> Result { + if s.running_for_script() && !s.prefs.scripting.allow_scripts_write_files { + return Err(bad( + c, + "scripts can't start effect helpers unless Settings ▸ Scripting & Expressions ▸ Allow Scripts to Write Files and Access Network is on", + )); + } + s.helpers.ok_or_else(|| EngineError::Other("effect helpers are not available in this build".into())) +} + +/// `effect.helpers.list`: the external effect helpers and their state. +fn helpers_list(s: &mut Session, _: &Value) -> Result { + Ok(match s.helpers { + Some(h) => (h.list)(), + None => json!({"helpers": [], "blocked": [], "available": false}), + }) +} + +/// `effect.helpers.scan`: start the installed helpers that aren't running and register their +/// effects. One bad helper doesn't stop the rest; `retry: true` turns blocked effects back on. +fn helpers_scan(s: &mut Session, p: &Value) -> Result { + let h = helper_hooks(s, "effect.helpers.scan")?; + let r = (h.scan)(b_p(p, "retry").unwrap_or(false)); + let n = r["loaded"].as_array().map_or(0, Vec::len); + if n > 0 { + s.toast(format!("{n} effect(s) from effect helpers")); + } + Ok(r) +} + +/// `effect.helpers.load`: load a file or folder through the helpers that accept it (an OpenFX +/// helper takes `.ofx` plug-ins, for example). +fn helpers_load(s: &mut Session, p: &Value) -> Result { + let c = "effect.helpers.load"; + let h = helper_hooks(s, c)?; + let path = str_p(p, "path").ok_or_else(|| bad(c, "missing `path`"))?; + let r = (h.load)(path).map_err(|e| EngineError::Other(format!("{path}: {e}")))?; + let n = r["loaded"].as_array().map_or(0, Vec::len); + if n > 0 { + s.toast(format!("Loaded {n} effect(s) through a helper")); + } + Ok(r) +} + fn apply(s: &mut Session, p: &Value) -> Result { let name = str_p(p, "effect").ok_or_else(|| bad("effect.apply", "missing `effect` (id or name)"))?; let spec = effectcraft_effects::lookup(name).ok_or_else(|| bad("effect.apply", format!("unknown effect `{name}`")))?; @@ -1140,7 +1340,7 @@ pub fn specs() -> Vec { crate::query!( "effect.plugins.list", "List Effect Plug-ins", - "{} → {api, wasm, plugins: [{id, name, category, version, author, source, params}]}", + "{} → {api, wasm, helpers, plugins: [{id, name, category, version, author, source, params, error}]}", plugins_list ), cmd!( @@ -1152,6 +1352,39 @@ pub fn specs() -> Vec { super::always, plugins_load ), + cmd!( + "effect.plugins.install", + "Install Effect Plug-in...", + [], + None, + "{path (.wasm plug-in: copied into the Plug-ins folder and loaded on every start) | folder (a plug-in bundle: every .wasm goes into the Plug-ins folder and every .jsx becomes a ScriptUI panel), bundle? (the menu: ask for a folder)}", + super::always, + plugins_install + ), + crate::query!( + "effect.helpers.list", + "List Effect Helpers", + "{} → {helpers: [{program, name, version, loads, running, effects, restarts, error}], blocked: [effect ids]}", + helpers_list + ), + cmd!( + "effect.helpers.scan", + "Scan for Effect Helpers", + [], + None, + "{retry?} → {paths, helpers, loaded, errors}: starts the effect helpers installed in helpers/ next to the app, helpers/ in the settings folder and EFFECTCRAFT_HELPER_PATH, and adds their effects; retry turns effects that kept stopping their helper back on", + has_helpers, + helpers_scan + ), + cmd!( + "effect.helpers.load", + "Load Plug-ins Through Helper...", + [], + None, + "{path (a file or folder an installed helper accepts, e.g. a folder of .ofx plug-ins for an OpenFX helper; the menu asks for a folder)} → {loaded: [{id, name, category, helper}], errors}", + has_helpers, + helpers_load + ), ] } diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 2e5f5aca4..6b680fbf5 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -200,6 +200,9 @@ pub struct Session { /// Loads WebAssembly effect plug-ins (set by the host that links `effectcraft-plugin` with /// its runtime): `effect.plugins.load`. pub plugin_loader: Option, + /// External effect helpers (native hosts link `effectcraft-helper-host`): + /// `effect.helpers.list`, `effect.helpers.scan`, `effect.helpers.load`. + pub helpers: Option, /// Renders a module without registering it (`library.preview`; set with `plugin_loader`). pub plugin_previewer: Option, /// Where effect plug-ins live (hosts set it; else `Plug-ins` next to the settings): the @@ -283,6 +286,18 @@ pub type ScriptRunner = fn(&mut Session, &ScriptRequest) -> Value; /// describes it (`{id, name, category, version, api, params}`). pub type PluginLoader = fn(&[u8], &str) -> std::result::Result; +/// What a host that runs external effect helpers provides (`docs/external-helpers.md`). +#[derive(Clone, Copy)] +pub struct HelperHooks { + /// Start the helpers installed in the host's search paths and register their effects + /// (`retry` forgets the block list first) → `{paths, helpers, loaded, errors}`. + pub scan: fn(bool) -> Value, + /// Load a file or folder through the helpers that accept it → `{loaded, errors}`. + pub load: fn(&str) -> std::result::Result, + /// The helpers and their state → `{helpers, blocked}`. + pub list: fn() -> Value, +} + /// Renders one WebAssembly effect module without registering it (the Plug-in Library preview): /// `(bytes, width, height, premultiplied RGBA pixels, times in seconds)` → (manifest JSON, one frame /// per time). @@ -343,6 +358,7 @@ impl Default for Session { snapshot: None, script: None, plugin_loader: None, + helpers: None, plugin_previewer: None, plugins_dir: None, library: Default::default(), diff --git a/crates/engine/src/menus.rs b/crates/engine/src/menus.rs index 15f55484a..ffff4cc04 100644 --- a/crates/engine/src/menus.rs +++ b/crates/engine/src/menus.rs @@ -1015,6 +1015,10 @@ Effect --- Manage Effects... | effect.manage Load Effect Plug-in... | effect.plugins.load + Install Effect Plug-in... | effect.plugins.install + Install Effect Plug-in Bundle (Folder)... | effect.plugins.install {"bundle": true} + Load Plug-ins Through Helper... | effect.helpers.load + Scan for Effect Helpers | effect.helpers.scan Plug-in Library... | prefs.open {"page":"library"} --- @effects diff --git a/crates/engine/src/services.rs b/crates/engine/src/services.rs index 2d425a4e6..5658df107 100644 --- a/crates/engine/src/services.rs +++ b/crates/engine/src/services.rs @@ -3,6 +3,17 @@ use super::config; /// Host services (file access) injected by the frontend. pub trait Services: Send + Sync { fn read_file(&self, path: &str) -> std::io::Result>; + /// [`Services::read_file`] for a copy that must not load an unbounded file: more than `max` + /// bytes is an error. The native one opens the file once, reads its size from that handle and + /// reads through a [`std::io::Read::take`] limit, so a file grown or swapped after the check + /// still cannot read past `max`. + fn read_file_bounded(&self, path: &str, max: u64) -> std::io::Result> { + let bytes = self.read_file(path)?; + if bytes.len() as u64 > max { + return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, format!("{path} is larger than the {max}-byte limit"))); + } + Ok(bytes) + } fn write_file(&self, path: &str, data: &[u8]) -> std::io::Result<()>; /// Whether a file exists (the background footage check after open). fn exists(&self, path: &str) -> bool { @@ -65,6 +76,26 @@ impl Services for FsServices { file.read_to_end(&mut bytes)?; Ok(bytes) } + fn read_file_bounded(&self, path: &str, max: u64) -> std::io::Result> { + use std::io::Read; + let file = std::fs::File::open(path)?; + let meta = file.metadata()?; + if !meta.is_file() { + return Err(std::io::Error::new(std::io::ErrorKind::InvalidInput, format!("{path} is not a file"))); + } + if meta.len() > max { + return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, format!("{path} is larger than the {max}-byte limit"))); + } + let mut bytes = Vec::new(); + bytes.try_reserve_exact(usize::try_from(meta.len()).unwrap_or(usize::MAX)).map_err(|e| std::io::Error::new(std::io::ErrorKind::OutOfMemory, e))?; + // The size above came from this handle, and the read is capped too: a file that grows + // after the check still cannot read past `max`. Saturating, so `u64::MAX` cannot overflow. + file.take(max.saturating_add(1)).read_to_end(&mut bytes)?; + if bytes.len() as u64 > max { + return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, format!("{path} is larger than the {max}-byte limit"))); + } + Ok(bytes) + } fn write_file(&self, path: &str, data: &[u8]) -> std::io::Result<()> { config::atomic_write(std::path::Path::new(path), data) } @@ -99,4 +130,17 @@ mod tests { assert_eq!(FsServices.read_file(&f.to_string_lossy()).unwrap(), b"data"); let _ = std::fs::remove_file(f); } + + /// The limit is inclusive and cannot overflow: `u64::MAX` reads the file, one byte too many + /// is an error, and nothing past it is read. + #[test] + fn fs_services_bounded_read_stops_at_the_limit() { + let f = std::env::temp_dir().join(format!("ec-services-bounded-{}.txt", std::process::id())); + std::fs::write(&f, b"data").unwrap(); + let p = f.to_string_lossy().to_string(); + assert_eq!(FsServices.read_file_bounded(&p, u64::MAX).unwrap(), b"data"); + assert_eq!(FsServices.read_file_bounded(&p, 4).unwrap(), b"data"); + assert!(FsServices.read_file_bounded(&p, 3).is_err()); + let _ = std::fs::remove_file(f); + } } diff --git a/crates/engine/src/system/library.rs b/crates/engine/src/system/library.rs index 46fa5757e..750ed600a 100644 --- a/crates/engine/src/system/library.rs +++ b/crates/engine/src/system/library.rs @@ -299,11 +299,12 @@ pub fn parse_catalog(bytes: &[u8]) -> std::result::Result<(Catalog, Vec) Ok((Catalog { schema: CATALOG_SCHEMA, packages }, warnings)) } -/// This build implements the release's plug-in API and is at least its `minApp`. +/// This build loads the release's WebAssembly plug-in API ([`WASM_API_VERSION`](crate::effects::plugin::WASM_API_VERSION): +/// the library lists `.wasm` modules only) and is at least its `minApp`. fn runs_here(r: &Release) -> bool { // Without a pre-release suffix (`0.18.0-dev` is 0.18.0). let app = env!("CARGO_PKG_VERSION").split(['-', '+']).next().unwrap_or_default(); - r.api == crate::effects::plugin::PLUGIN_API_VERSION && r.min_app.as_deref().is_none_or(|m| !newer(m, app)) + r.api == crate::effects::plugin::WASM_API_VERSION && r.min_app.as_deref().is_none_or(|m| !newer(m, app)) } /// The newest release this build can run (plug-in API and minimum app version). @@ -788,13 +789,13 @@ fn pick(s: &Session, c: &str, p: &Value) -> Result<(String, Package, Release)> { return Err(bad( c, format!( - "{} {} needs plug-in API {} and EffectCraft {} or later; this is EffectCraft {} with plug-in API {}", + "{} {} needs plug-in API {} and EffectCraft {} or later; this is EffectCraft {} with WebAssembly plug-in API {}", pkg.name, rel.version, rel.api, rel.min_app.as_deref().unwrap_or("(any)"), env!("CARGO_PKG_VERSION"), - crate::effects::plugin::PLUGIN_API_VERSION + crate::effects::plugin::WASM_API_VERSION ), )); } @@ -1111,7 +1112,7 @@ mod tests { #[test] fn the_newest_compatible_release_wins() { - let api = crate::effects::plugin::PLUGIN_API_VERSION; + let api = crate::effects::plugin::WASM_API_VERSION; let p = Package { versions: vec![ rel("1.0.0", api, None), @@ -1119,10 +1120,13 @@ mod tests { rel("1.1.0", api, None), rel("9.0.0", api, Some("999.0")), rel("3.0.0", api + 1, None), + rel("4.0.0", 0, None), ], ..Default::default() }; assert_eq!(latest_compatible(&p).map(|r| r.version.as_str()), Some("1.2.0")); + assert!(runs_here(&rel("1.0.0", 1, None)), "API 1 modules still run on a build with a newer plug-in API"); + assert!(!runs_here(&rel("1.0.0", 2, None)), "WebAssembly modules are API 1: an API 2 release can't load"); let j = package_json(&p, None, &Status::default()); assert_eq!(j["needsNewerApp"], true); assert!(latest_compatible(&Package::default()).is_none()); diff --git a/crates/engine/src/tests/effects.rs b/crates/engine/src/tests/effects.rs index 4b3b0188e..f5255c0a3 100644 --- a/crates/engine/src/tests/effects.rs +++ b/crates/engine/src/tests/effects.rs @@ -941,3 +941,25 @@ fn copied_effect_property_pastes_onto_the_same_effect() { assert_eq!(r["expressions"], 1, "{r}"); assert_eq!(layer(&s, b).props.prop("effects/#1/blurriness").unwrap().expr.as_ref().unwrap().text, "time * 10"); } + +/// Effect helpers run native code: a script starts them only with Allow Scripts to Write Files +/// and Access Network on (as for File.execute()); the user always can. +#[test] +fn scripts_start_effect_helpers_only_with_file_access() { + let mut s = Session { + helpers: Some(crate::HelperHooks { + scan: |_| json!({"paths": [], "helpers": [], "loaded": [], "errors": []}), + load: |_| Ok(json!({"loaded": [], "errors": []})), + list: || json!({"helpers": [], "blocked": []}), + }), + ..Default::default() + }; + for (id, params) in [("effect.helpers.scan", json!({})), ("effect.helpers.load", json!({"path": "/plug-ins/x.ofx"}))] { + let e = s.execute_for_script(id, params.clone()).unwrap_err().to_string(); + assert!(e.contains("Allow Scripts to Write Files"), "{id}: {e}"); + assert!(s.execute(id, params).is_ok(), "{id}: the user starts helpers"); + } + assert!(s.execute_for_script("effect.helpers.list", json!({})).is_ok(), "listing starts nothing"); + s.execute("prefs.set", json!({"key": "scripting.allowScriptsWriteFiles", "value": true})).unwrap(); + assert!(s.execute_for_script("effect.helpers.scan", json!({})).is_ok()); +} diff --git a/crates/helper-host/Cargo.toml b/crates/helper-host/Cargo.toml new file mode 100644 index 000000000..a593b5055 --- /dev/null +++ b/crates/helper-host/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "effectcraft-helper-host" +description = "External effect helpers: finds helper programs, talks to them over a versioned pipe + shared-file protocol and registers their effects (plug-in API 2). Safe Rust; a crashing helper is an error, not a crash" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[lints] +workspace = true + +[dependencies] +effectcraft-effects = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +log = { workspace = true } +bytemuck = { workspace = true } + +# The crate's integration tests (one binary, `tests/it/main.rs`): the whole protocol against a +# fake helper. The test binary is also the helper (started with `--effectcraft-helper`), so +# `cargo test` covers it on every platform with nothing to install; `harness = false` because +# libtest would reject the helper argument. +[[test]] +name = "it" +path = "tests/it/main.rs" +harness = false diff --git a/crates/helper-host/src/blocklist.rs b/crates/helper-host/src/blocklist.rs new file mode 100644 index 000000000..a7967edfc --- /dev/null +++ b/crates/helper-host/src/blocklist.rs @@ -0,0 +1,135 @@ +//! Effects that keep taking their helper down are turned off. +//! +//! Each time a render through a helper ends with the helper crashing or hanging, the effect is +//! charged; at [`CRASH_LIMIT`] it is blocked: its renders fail at once with a message instead of +//! restarting the helper again. Blocks are kept in `helper_blocklist.txt` in the state folder +//! ([`set_state_dir`]), one effect id per line, so they last across launches until a scan with +//! `retry` ([`clear`]). + +use std::collections::HashMap; +use std::io::Read; +use std::path::{Path, PathBuf}; +use std::sync::{Mutex, OnceLock}; + +use crate::client::lock; + +/// Crashes after which an effect is blocked. +pub const CRASH_LIMIT: u32 = 3; + +const FILE: &str = "helper_blocklist.txt"; +/// The most of the block list file read (tens of thousands of effect ids). +const MAX_FILE: u64 = 1 << 20; + +#[derive(Default)] +struct State { + dir: Option, + crashes: HashMap, + blocked: Vec, + loaded: bool, +} + +fn state() -> &'static Mutex { + static S: OnceLock> = OnceLock::new(); + S.get_or_init(Default::default) +} + +/// Where the block list lives (EffectCraft's settings folder); helpers get it as +/// `EFFECTCRAFT_HELPER_STATE_DIR` for their own records. +pub fn set_state_dir(dir: PathBuf) { + let mut s = lock(state()); + s.dir = Some(dir); + s.loaded = false; +} + +pub fn state_dir() -> Option { + lock(state()).dir.clone() +} + +fn load(s: &mut State) { + if s.loaded { + return; + } + s.loaded = true; + if let Some(text) = s.dir.as_ref().and_then(|d| read_capped(&d.join(FILE))) { + for id in text.lines().map(str::trim).filter(|l| !l.is_empty() && !l.starts_with('#')) { + if !s.blocked.iter().any(|b| b == id) { + s.blocked.push(id.to_string()); + } + } + } +} + +/// The file's text, at most [`MAX_FILE`] bytes of it. +fn read_capped(path: &Path) -> Option { + let mut text = String::new(); + std::fs::File::open(path).ok()?.take(MAX_FILE).read_to_string(&mut text).ok()?; + Some(text) +} + +fn save(s: &State) { + let Some(dir) = &s.dir else { return }; + let mut text = + String::from("# Effects turned off because they kept stopping their effect helper.\n# Delete a line (or scan with retry) to try one again.\n"); + for id in &s.blocked { + text.push_str(id); + text.push('\n'); + } + if let Err(e) = std::fs::create_dir_all(dir).and_then(|_| std::fs::write(dir.join(FILE), text)) { + log::warn!("can't save the helper block list: {e}"); + } +} + +pub fn is_blocked(id: &str) -> bool { + let mut s = lock(state()); + load(&mut s); + s.blocked.iter().any(|b| b == id) +} + +/// Charge `id` with a helper crash; returns whether it is now blocked. +pub fn note_crash(id: &str) -> bool { + let mut s = lock(state()); + load(&mut s); + let n = { + let n = s.crashes.entry(id.to_string()).or_insert(0); + *n = n.saturating_add(1); + *n + }; + if n >= CRASH_LIMIT && !s.blocked.iter().any(|b| b == id) { + s.blocked.push(id.to_string()); + save(&s); + } + n >= CRASH_LIMIT +} + +/// The blocked effect ids. +pub fn blocked() -> Vec { + let mut s = lock(state()); + load(&mut s); + s.blocked.clone() +} + +/// Forget every block and crash count. +pub fn clear() { + let mut s = lock(state()); + load(&mut s); + s.blocked.clear(); + s.crashes.clear(); + save(&s); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn effects_are_blocked_after_repeated_crashes() { + let id = "org.test.blocklist-unit"; + for _ in 1..CRASH_LIMIT { + assert!(!note_crash(id)); + } + assert!(note_crash(id)); + assert!(is_blocked(id) && blocked().iter().any(|b| b == id)); + clear(); + assert!(!is_blocked(id)); + } +} diff --git a/crates/helper-host/src/client.rs b/crates/helper-host/src/client.rs new file mode 100644 index 000000000..8379d4dd9 --- /dev/null +++ b/crates/helper-host/src/client.rs @@ -0,0 +1,686 @@ +//! EffectCraft's side of a helper: the child process, its pipes and frame exchange file, and +//! the [`EffectPlugin`] each remote effect is registered as. +//! +//! Nothing a helper does can take EffectCraft down: a helper that exits, hangs past its +//! timeout or answers nonsense is stopped, the render that hit it fails with a message (shown on +//! the effect, the frame is left as it was), and the next render starts the helper again. An +//! effect that took its helper down [`crate::blocklist::CRASH_LIMIT`] times is blocked. + +use std::io::{BufRead, BufReader, Read, Write}; +use std::path::{Path, PathBuf}; +use std::process::{Child, ChildStdin, Command, Stdio}; +use std::sync::mpsc::{Receiver, RecvTimeoutError, Sender, channel}; +use std::sync::{Arc, Condvar, Mutex, MutexGuard, PoisonError, TryLockError}; +use std::thread::ThreadId; +use std::time::{Duration, Instant}; + +use effectcraft_effects::plugin::{EffectPlugin, PluginFrame, PluginHost, PluginImage, PluginManifest, PluginParams}; + +use crate::blocklist; +use crate::frames::Exchange; +use crate::protocol::{HELPER_ARG, ImageRef, MAX_LINE, PROTOCOL_VERSION, RemoteEffect, RenderRequest, Reply, Request, parse_reply, request_line}; + +/// How deep renders through one helper may nest (an effect asking for its input, which another +/// effect of the same helper renders, ...). +const MAX_DEPTH: u32 = 16; +/// Restarts allowed per minute before a helper is left stopped until the next scan. +const MAX_RESTARTS_PER_MINUTE: usize = 5; +/// A helper idle for longer than this is pinged before it is used again. +const HEARTBEAT_AFTER: Duration = Duration::from_secs(30); +const HEARTBEAT_TIMEOUT: Duration = Duration::from_secs(10); +/// The longest piece of a helper's stderr logged as one line (longer lines are logged in parts). +const MAX_LOG_LINE: u64 = 64 << 10; + +/// [`set_render_timeout`] (milliseconds; 0 = not set). +static RENDER_TIMEOUT_MS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + +/// Override the render timeout (tests; `EFFECTCRAFT_HELPER_TIMEOUT` still wins). +pub fn set_render_timeout(t: Duration) { + RENDER_TIMEOUT_MS.store(u64::try_from(t.as_millis()).unwrap_or(u64::MAX), std::sync::atomic::Ordering::Relaxed); +} + +/// Timeout for one render step, `EFFECTCRAFT_HELPER_TIMEOUT` seconds (default 120: heavy +/// plug-ins take a while on their first frame). +fn render_timeout() -> Duration { + if let Some(s) = env_secs("EFFECTCRAFT_HELPER_TIMEOUT") { + return Duration::from_secs(s); + } + match RENDER_TIMEOUT_MS.load(std::sync::atomic::Ordering::Relaxed) { + 0 => Duration::from_secs(120), + ms => Duration::from_millis(ms), + } +} + +/// Timeout for start-up, `describe` and `load`, `EFFECTCRAFT_HELPER_LOAD_TIMEOUT` seconds +/// (default 600: scanning large plug-in suites is slow). +fn load_timeout() -> Duration { + Duration::from_secs(env_secs("EFFECTCRAFT_HELPER_LOAD_TIMEOUT").unwrap_or(600)) +} + +fn env_secs(k: &str) -> Option { + std::env::var(k).ok()?.trim().parse().ok().filter(|s| *s > 0) +} + +pub(crate) fn lock(m: &Mutex) -> MutexGuard<'_, T> { + m.lock().unwrap_or_else(PoisonError::into_inner) +} + +/// What the reader thread passes on. +enum Line { + Reply(Result), + Closed, +} + +/// A running helper process. +struct Conn { + child: Child, + /// Request lines for the writer thread, which owns the helper's stdin: a helper that stops + /// reading can block that thread, never the render waiting for it ([`Conn::send`]). + input: Sender>, + /// The writer thread's result for each line, in order. + written: Receiver>, + rx: Receiver, + exchange: Exchange, + /// The first free byte of the exchange file (images are stacked, see [`Helper::render`]). + top: u64, + last_used: Instant, + generation: u64, +} + +impl Conn { + /// Write one request, waiting at most `timeout` for the helper to take it. + fn send(&mut self, r: &Request, timeout: Duration) -> Result<(), Failure> { + let line = request_line(r).map_err(Failure::Broken)?; + let closed = || Failure::Broken("can't write to the helper: its input is closed".into()); + self.input.send(line.into_bytes()).map_err(|_| closed())?; + match self.written.recv_timeout(timeout) { + Ok(Ok(())) => Ok(()), + Ok(Err(e)) => Err(Failure::Broken(format!("can't write to the helper: {e}"))), + Err(RecvTimeoutError::Timeout) => { + Err(Failure::Broken(format!("the helper did not answer within {} s (it stopped reading its input)", timeout.as_secs()))) + } + Err(RecvTimeoutError::Disconnected) => Err(closed()), + } + } + + fn recv(&mut self, timeout: Duration) -> Result { + match self.rx.recv_timeout(timeout) { + Ok(Line::Reply(Ok(r))) => Ok(r), + Ok(Line::Reply(Err(e))) => Err(Failure::Broken(e)), + Ok(Line::Closed) | Err(RecvTimeoutError::Disconnected) => { + // Give the process a moment to finish exiting so its status can be reported. + std::thread::sleep(Duration::from_millis(50)); + let status = match self.child.try_wait() { + Ok(Some(s)) => s.to_string(), + _ => "closed its output".into(), + }; + Err(Failure::Broken(format!("the helper stopped ({status})"))) + } + Err(RecvTimeoutError::Timeout) => Err(Failure::Broken(format!("the helper did not answer within {} s", timeout.as_secs()))), + } + } + + fn call(&mut self, r: &Request, timeout: Duration) -> Result { + self.send(r, timeout)?; + self.recv(timeout) + } + + /// Reserve room for an image of `width × height` at the top of the exchange file. + fn alloc(&mut self, width: u32, height: u32, origin: [f64; 2], scale: f64) -> Result { + let at = ImageRef { offset: self.top, width, height, origin, scale }; + let bytes = at.bytes().ok_or_else(|| format!("image {width}×{height} is too large for a helper"))?; + // Keep images 64-byte aligned for helpers that map the file. + self.top = self.top.checked_add(bytes.div_ceil(64).saturating_mul(64)).ok_or("frame exchange file too large")?; + Ok(at) + } + + fn stop(mut self) { + // Not waited for: the process is killed right after, which also ends a blocked write. + if let Ok(line) = request_line(&Request::Shutdown) { + let _ = self.input.send(line.into_bytes()); + } + let _ = self.child.kill(); + let _ = self.child.wait(); + } +} + +/// Why a request failed. +enum Failure { + /// The helper answered with an error and is fine. + Refused(String), + /// The helper crashed, hung or broke the protocol: it is stopped. + Broken(String), +} + +/// Whose turn it is to talk to the helper: one thread at a time, re-entered by that thread for +/// nested renders. +#[derive(Default)] +struct Turn { + owner: Option, + depth: u32, +} + +struct TurnGuard<'a>(&'a Helper); + +impl Drop for TurnGuard<'_> { + fn drop(&mut self) { + let mut t = lock(&self.0.turn); + t.depth = t.depth.saturating_sub(1); + if t.depth == 0 { + t.owner = None; + self.0.turn_cv.notify_all(); + } + } +} + +/// What [`Helper::status`] reports. +#[derive(Clone, Debug, Default)] +pub struct HelperStatus { + pub program: PathBuf, + pub name: String, + pub version: String, + pub loads: Vec, + pub running: bool, + pub effects: usize, + pub restarts: usize, + pub last_error: Option, +} + +/// One helper program and, while it runs, its process. +pub struct Helper { + program: PathBuf, + info: Mutex, + conn: Mutex>, + turn: Mutex, + turn_cv: Condvar, + /// Paths loaded with [`Helper::load`], loaded again after a restart. + loaded: Mutex>, + restarts: Mutex>, + generation: Mutex, +} + +impl Helper { + /// Start a helper and list its effects. + pub fn start(program: &Path) -> Result<(Arc, Vec, Vec), String> { + let h = Arc::new(Helper { + program: program.to_path_buf(), + info: Mutex::new(HelperStatus { program: program.to_path_buf(), ..Default::default() }), + conn: Mutex::new(None), + turn: Mutex::new(Turn::default()), + turn_cv: Condvar::new(), + loaded: Mutex::new(vec![]), + restarts: Mutex::new(vec![]), + generation: Mutex::new(0), + }); + let (effects, errors) = { + let _turn = h.take_turn()?; + // A helper that crashes while listing its effects (one bad plug-in) is given a + // second and third chance: helpers remember what crashed them and skip it. + let mut last = String::new(); + let mut out = None; + for _ in 0..3 { + match h.with_conn(|c| c.call(&Request::Describe, load_timeout())) { + Ok(Reply::Effects { effects, errors }) => { + out = Some((effects, errors)); + break; + } + Ok(other) => last = format!("unexpected reply to describe: {other:?}"), + Err(Failure::Refused(e) | Failure::Broken(e)) => last = e, + } + } + out.ok_or_else(|| format!("{}: {last}", program.display()))? + }; + lock(&h.info).effects = effects.len(); + Ok((h, effects, errors)) + } + + pub fn program(&self) -> &Path { + &self.program + } + + pub fn status(&self) -> HelperStatus { + let mut s = lock(&self.info).clone(); + // A render holds the process for as long as the helper works on it: don't wait for it. + s.running = match self.conn.try_lock() { + Ok(c) => c.is_some(), + Err(TryLockError::Poisoned(c)) => c.into_inner().is_some(), + Err(TryLockError::WouldBlock) => true, + }; + s.restarts = lock(&self.restarts).len(); + s + } + + pub fn loads(&self, ext: &str) -> bool { + lock(&self.info).loads.iter().any(|e| e.eq_ignore_ascii_case(ext)) + } + + pub(crate) fn add_effects(&self, n: usize) { + let mut i = lock(&self.info); + i.effects = i.effects.saturating_add(n); + } + + /// Load effects from a path through the helper ([`Request::Load`]). + pub fn load(&self, path: &str) -> Result<(Vec, Vec), String> { + let _turn = self.take_turn()?; + let r = self.with_conn(|c| c.call(&Request::Load { path: path.to_string() }, load_timeout())); + match r { + Ok(Reply::Effects { effects, errors }) => { + let mut l = lock(&self.loaded); + if !l.iter().any(|p| p == path) { + l.push(path.to_string()); + } + Ok((effects, errors)) + } + Ok(Reply::Error { message }) => Err(message), + Ok(other) => Err(format!("unexpected reply to load: {other:?}")), + Err(Failure::Refused(e) | Failure::Broken(e)) => Err(e), + } + } + + /// Stop the process (it starts again when next needed). + pub fn stop(&self) { + if let Some(c) = lock(&self.conn).take() { + c.stop(); + } + } + + fn take_turn(&self) -> Result, String> { + let me = std::thread::current().id(); + // Bounded, so a thread that can't get the helper (its holder waiting for work that a + // blocked thread pool would have to do) fails instead of hanging forever. + let limit = load_timeout(); + let deadline = Instant::now().checked_add(limit); + let mut t = lock(&self.turn); + loop { + match t.owner { + None => { + t.owner = Some(me); + t.depth = 1; + break; + } + Some(o) if o == me => { + if t.depth >= MAX_DEPTH { + return Err("effects through this helper are nested too deeply".into()); + } + t.depth += 1; + break; + } + Some(_) => { + let left = deadline.map_or(limit, |d| d.saturating_duration_since(Instant::now())); + if left.is_zero() { + return Err(format!("the effect helper was busy for more than {} s", limit.as_secs())); + } + t = self.turn_cv.wait_timeout(t, left).unwrap_or_else(PoisonError::into_inner).0; + } + } + } + Ok(TurnGuard(self)) + } + + /// Run `f` on the running process, starting it first if needed. A [`Failure::Broken`] + /// stops the process. + fn with_conn(&self, f: impl FnOnce(&mut Conn) -> Result) -> Result { + let mut g = lock(&self.conn); + if g.is_none() { + *g = Some(self.spawn().map_err(|e| self.not_started(e))?); + } + let Some(c) = g.as_mut() else { return Err(Failure::Broken("the helper is not running".into())) }; + if c.last_used.elapsed() > HEARTBEAT_AFTER && !matches!(c.call(&Request::Ping, HEARTBEAT_TIMEOUT), Ok(Reply::Pong)) { + if let Some(dead) = g.take() { + dead.stop(); + } + self.note_error("the helper stopped answering; restarting it"); + *g = Some(self.spawn().map_err(|e| self.not_started(e))?); + } + let Some(c) = g.as_mut() else { return Err(Failure::Broken("the helper is not running".into())) }; + let r = f(c); + c.last_used = Instant::now(); + if let Err(Failure::Broken(e)) = &r { + self.note_error(e); + if let Some(dead) = g.take() { + dead.stop(); + } + } + r + } + + /// Run `f` on the process a render started on (generation `started`), without starting or + /// pinging one: mid-render the helper is waiting for the answer to its question. A + /// [`Failure::Broken`] stops the process. A process already gone was stopped by what broke it + /// (a nested render, charged there): not charged again to this render's effect. + fn with_running(&self, started: u64, f: impl FnOnce(&mut Conn) -> Result) -> Result { + let mut g = lock(&self.conn); + let Some(c) = g.as_mut().filter(|c| c.generation == started) else { + return Err(Failure::Refused("the helper stopped during the render".into())); + }; + let r = f(c); + c.last_used = Instant::now(); + if let Err(Failure::Broken(e)) = &r { + self.note_error(e); + if let Some(dead) = g.take() { + dead.stop(); + } + } + r + } + + /// A helper that could not be started (not installed any more, restarting too often, no + /// frame exchange file): reported, but not charged to the effect that asked for it, since + /// nothing it did stopped the helper ([`Failure::Refused`], not [`Failure::Broken`]). + fn not_started(&self, e: String) -> Failure { + self.note_error(&e); + Failure::Refused(e) + } + + fn note_error(&self, e: &str) { + log::warn!("helper {}: {e}", self.program.display()); + lock(&self.info).last_error = Some(e.to_string()); + } + + fn spawn(&self) -> Result { + { + let mut r = lock(&self.restarts); + r.retain(|t| t.elapsed() < Duration::from_secs(60)); + if r.len() >= MAX_RESTARTS_PER_MINUTE { + return Err("the helper keeps stopping; it stays off until the next scan (Effect ▸ Scan for Effect Helpers)".into()); + } + r.push(Instant::now()); + } + let exchange = Exchange::create().map_err(|e| format!("can't create the frame exchange file: {e}"))?; + let mut cmd = Command::new(&self.program); + cmd.arg(HELPER_ARG).stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::piped()); + if let Some(dir) = blocklist::state_dir() { + cmd.env("EFFECTCRAFT_HELPER_STATE_DIR", dir); + } + #[cfg(windows)] + { + use std::os::windows::process::CommandExt; + // CREATE_NO_WINDOW: no console window per helper. + cmd.creation_flags(0x0800_0000); + } + let mut child = cmd.spawn().map_err(|e| format!("can't start {}: {e}", self.program.display()))?; + let (Some(stdin), Some(stdout), stderr) = (child.stdin.take(), child.stdout.take(), child.stderr.take()) else { + let _ = child.kill(); + let _ = child.wait(); + return Err("the helper's pipes are unavailable".into()); + }; + let (tx, rx) = channel(); + let label = self.program.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default(); + let l2 = label.clone(); + let spawned = std::thread::Builder::new().name(format!("helper-out {label}")).spawn(move || { + let mut r = BufReader::new(stdout); + loop { + let mut buf = vec![]; + match (&mut r).take(MAX_LINE).read_until(b'\n', &mut buf) { + Ok(0) | Err(_) => { + let _ = tx.send(Line::Closed); + return; + } + Ok(_) => { + let line = String::from_utf8_lossy(&buf); + match parse_reply(&line) { + Some(reply) => { + if tx.send(Line::Reply(reply)).is_err() { + return; + } + } + None => log::info!("helper {l2}: {}", line.trim_end()), + } + } + } + } + }); + if let Err(e) = spawned { + let _ = child.kill(); + let _ = child.wait(); + return Err(format!("can't start a reader thread: {e}")); + } + let (input, lines) = channel::>(); + let (done, written) = channel(); + let writer = std::thread::Builder::new().name(format!("helper-in {label}")).spawn(move || { + let mut stdin: ChildStdin = stdin; + for line in lines { + let r = stdin.write_all(&line).and_then(|_| stdin.flush()).map_err(|e| e.to_string()); + let failed = r.is_err(); + if done.send(r).is_err() || failed { + return; + } + } + }); + if let Err(e) = writer { + let _ = child.kill(); + let _ = child.wait(); + return Err(format!("can't start a writer thread: {e}")); + } + if let Some(err) = stderr { + // Drained to the end whatever it holds (bytes that aren't UTF-8, endless lines): a + // helper blocked on a full stderr pipe would hang its render. + let _ = std::thread::Builder::new().name(format!("helper-err {label}")).spawn(move || { + let mut r = BufReader::new(err); + let mut buf = vec![]; + loop { + buf.clear(); + match (&mut r).take(MAX_LOG_LINE).read_until(b'\n', &mut buf) { + Ok(0) | Err(_) => return, + Ok(_) => log::info!("helper {label}: {}", String::from_utf8_lossy(&buf).trim_end()), + } + } + }); + } + let generation = { + let mut g = lock(&self.generation); + *g = g.saturating_add(1); + *g + }; + let mut c = Conn { child, input, written, rx, exchange, top: 0, last_used: Instant::now(), generation }; + let frames = c.exchange.path().to_string_lossy().into_owned(); + match c.call(&Request::Hello { version: PROTOCOL_VERSION, frames }, load_timeout()) { + Ok(Reply::Hello { version, name, helper_version, loads }) if version == PROTOCOL_VERSION => { + let mut i = lock(&self.info); + (i.name, i.version, i.loads) = (name, helper_version, loads); + } + Ok(Reply::Hello { version, .. }) => { + c.stop(); + return Err(format!("the helper speaks protocol {version}; this EffectCraft speaks {PROTOCOL_VERSION}")); + } + Ok(Reply::Error { message }) => { + c.stop(); + return Err(message); + } + Ok(other) => { + c.stop(); + return Err(format!("unexpected reply to hello: {other:?}")); + } + Err(Failure::Refused(e) | Failure::Broken(e)) => { + c.stop(); + return Err(e); + } + } + c.exchange.unlink(); + // After a restart the new process knows no effects yet: list them again (the first + // start lists them in `Helper::start`), then load what the user loaded before. + if generation > 1 && !matches!(c.call(&Request::Describe, load_timeout()), Ok(Reply::Effects { .. })) { + c.stop(); + return Err("the restarted helper could not list its effects".into()); + } + for path in lock(&self.loaded).clone() { + if !matches!(c.call(&Request::Load { path: path.clone() }, load_timeout()), Ok(Reply::Effects { .. })) { + log::warn!("helper {}: reloading {path} failed", self.program.display()); + } + } + Ok(c) + } + + fn padding(&self, effect: &str, params: &PluginParams, time: f64, scale: f64, layer_size: [f64; 2], instance: u64) -> Result { + let _turn = self.take_turn()?; + let req = Request::Padding { effect: effect.to_string(), instance, time, scale, layer_size, flat: params.flat.to_vec(), texts: params.texts.to_vec() }; + match self.with_conn(|c| c.call(&req, render_timeout())) { + Ok(Reply::Padding { pixels }) => Ok(pixels), + Ok(Reply::Error { message }) => Err(message), + Ok(other) => Err(format!("unexpected reply to padding: {other:?}")), + Err(Failure::Refused(e) | Failure::Broken(e)) => Err(e), + } + } + + /// Render one frame through the helper. Images live on a stack in the exchange file: the + /// frame first, then whatever the helper asks for, freed again when the render ends (nested + /// renders stack above). + fn render(&self, effect: &str, frame: &mut PluginFrame, params: &PluginParams, time: f64, host: &dyn PluginHost) -> Result<(), Failure> { + let _turn = self.take_turn().map_err(Failure::Refused)?; + // The generation of the process this render runs on: taken inside `with_conn`, which + // may start the process (after a crash or `stop`), so it is never that of a dead one. + let (at, base, started) = self.with_conn(|c| { + let base = c.top; + let at = c.alloc(frame.width, frame.height, frame.origin, frame.scale).map_err(Failure::Refused)?; + // Nothing is sent yet, so the helper is fine (a full disk, say): not its fault. + if let Err(e) = c.exchange.write(&at, frame.pixels) { + c.top = base; + return Err(Failure::Refused(e)); + } + Ok((at, base, c.generation)) + })?; + let req = Request::Render(RenderRequest { + effect: effect.to_string(), + instance: host.instance_key(), + time, + frame: at, + layer_size: frame.layer_size, + flat: params.flat.to_vec(), + texts: params.texts.to_vec(), + frame_rate: host.frame_rate(), + frame_range: host.frame_range(), + }); + let result = self.converse(req, frame, &at, host, started); + // Free this render's images (if the same process is still running). + if let Some(c) = lock(&self.conn).as_mut() + && c.generation == started + { + c.top = base; + } + result + } + + fn converse(&self, req: Request, frame: &mut PluginFrame, at: &ImageRef, host: &dyn PluginHost, started: u64) -> Result<(), Failure> { + let mut next = req; + loop { + let reply = self.with_running(started, |c| c.call(&next, render_timeout()))?; + next = match reply { + Reply::Done => return self.with_running(started, |c| c.exchange.read_into(at, frame.pixels).map_err(Failure::Broken)), + Reply::Error { message } => return Err(Failure::Refused(message)), + // Host callbacks run without holding the process, so they may render through + // this helper again (on this thread: the turn is re-entered). + Reply::InputAt { time } => self.send_image(host.input_at(time), started)?, + Reply::LayerInput { param, time } => self.send_image(host.layer_input(¶m, time), started)?, + Reply::ParamsAt { time } => Request::Params { values: host.params_at(time) }, + other => return Err(Failure::Broken(format!("unexpected reply during a render: {other:?}"))), + }; + } + } + + /// Put a callback's image in the exchange file and make the answer. The helper is waiting + /// for one, so an image that can't be passed on is answered with none (failing here would + /// leave it waiting, out of step with the next request). + fn send_image(&self, img: Option, started: u64) -> Result { + let Some(img) = img.filter(|i| u64::from(i.width).saturating_mul(u64::from(i.height)) == i.pixels.len() as u64) else { + return Ok(Request::Image { image: None }); + }; + self.with_running(started, |c| { + let base = c.top; + let written = c.alloc(img.width, img.height, img.origin, img.scale).and_then(|at| c.exchange.write(&at, &img.pixels).map(|()| at)); + Ok(match written { + Ok(at) => Request::Image { image: Some(at) }, + Err(e) => { + log::warn!("helper {}: {e}", self.program.display()); + c.top = base; + Request::Image { image: None } + } + }) + }) + } +} + +impl Drop for Helper { + fn drop(&mut self) { + self.stop(); + } +} + +/// A helper's effect, registered like any plug-in effect. +pub struct HelperEffect { + helper: Arc, + manifest: PluginManifest, + padding: bool, +} + +impl HelperEffect { + pub fn new(helper: Arc, remote: RemoteEffect) -> HelperEffect { + HelperEffect { helper, manifest: remote.manifest, padding: remote.padding } + } +} + +/// A host for [`EffectPlugin::render`] without services. +struct NoHost; + +impl PluginHost for NoHost { + fn input_at(&self, _: f64) -> Option { + None + } + fn layer_input(&self, _: &str, _: f64) -> Option { + None + } + fn params_at(&self, _: f64) -> Option<(Vec, Vec)> { + None + } + fn frame_rate(&self) -> f64 { + 30.0 + } + fn time(&self) -> f64 { + 0.0 + } +} + +impl EffectPlugin for HelperEffect { + fn manifest(&self) -> &PluginManifest { + &self.manifest + } + + fn render(&self, frame: &mut PluginFrame, params: &PluginParams, time: f64) -> Result<(), String> { + self.render_v2(frame, params, time, &NoHost) + } + + fn source(&self) -> String { + format!("helper: {}", self.helper.program.display()) + } + + fn padding(&self, params: &PluginParams, time: f64, scale: f64, layer_size: [f64; 2], instance_key: u64) -> u32 { + if !self.padding || blocklist::is_blocked(&self.manifest.id) { + return 0; + } + match self.helper.padding(&self.manifest.id, params, time, scale, layer_size, instance_key) { + Ok(p) => p, + Err(e) => { + log::warn!("helper effect `{}`: padding: {e}", self.manifest.id); + 0 + } + } + } + + fn render_v2(&self, frame: &mut PluginFrame, params: &PluginParams, time: f64, host: &dyn PluginHost) -> Result<(), String> { + let id = &self.manifest.id; + if blocklist::is_blocked(id) { + return Err(format!( + "{} is turned off: it stopped its helper {} times (Effect ▸ Scan for Effect Helpers with retry turns it back on)", + self.manifest.name, + blocklist::CRASH_LIMIT + )); + } + match self.helper.render(id, frame, params, time, host) { + Ok(()) => Ok(()), + Err(Failure::Refused(e)) => Err(e), + Err(Failure::Broken(e)) => { + blocklist::note_crash(id); + Err(format!("{}: the effect helper failed: {e}", self.manifest.name)) + } + } + } +} diff --git a/crates/helper-host/src/frames.rs b/crates/helper-host/src/frames.rs new file mode 100644 index 000000000..1570ce459 --- /dev/null +++ b/crates/helper-host/src/frames.rs @@ -0,0 +1,121 @@ +//! The frame exchange file: pixels shared between EffectCraft and a helper without going +//! through the control pipe. EffectCraft creates it (in `/dev/shm` when there is one, so it +//! stays in memory, else the temp folder) and both sides read and write it at the offsets the +//! messages name. Plain file I/O, no `unsafe`: the OS page cache makes it shared memory in +//! practice, and a helper is free to map it. + +use std::fs::{File, OpenOptions}; +use std::io::{Read, Seek, SeekFrom, Write}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; + +use crate::protocol::ImageRef; + +/// An open frame exchange file. +pub struct Exchange { + file: File, + path: PathBuf, + /// Remove the file when dropped (the side that created it). + owned: bool, +} + +impl Exchange { + /// Create a new, empty exchange file (EffectCraft's side). + pub fn create() -> std::io::Result { + static N: AtomicU64 = AtomicU64::new(0); + let shm = Path::new("/dev/shm"); + let dir = if cfg!(target_os = "linux") && shm.is_dir() { shm.to_path_buf() } else { std::env::temp_dir() }; + let nanos = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map(|d| d.subsec_nanos()).unwrap_or(0); + let path = dir.join(format!("effectcraft-helper-{}-{}-{nanos}.frames", std::process::id(), N.fetch_add(1, Ordering::Relaxed))); + let mut opts = OpenOptions::new(); + opts.read(true).write(true).create_new(true); + // Frames are the user's pixels and /dev/shm is shared: only this user may read them + // (the helper runs as the same user). + #[cfg(unix)] + std::os::unix::fs::OpenOptionsExt::mode(&mut opts, 0o600); + let file = opts.open(&path)?; + Ok(Exchange { file, path, owned: true }) + } + + /// Open an exchange file EffectCraft created (the helper's side). + pub fn open(path: &Path) -> std::io::Result { + let file = OpenOptions::new().read(true).write(true).open(path)?; + Ok(Exchange { file, path: path.to_path_buf(), owned: false }) + } + + pub fn path(&self) -> &Path { + &self.path + } + + /// Remove the file's name once both sides have it open (Unix keeps the open file usable), + /// so nothing is left behind if either side dies. + pub fn unlink(&mut self) { + if cfg!(unix) && self.owned && std::fs::remove_file(&self.path).is_ok() { + self.owned = false; + } + } + + /// Write `pixels` as the image `at` describes. + pub fn write(&mut self, at: &ImageRef, pixels: &[[f32; 4]]) -> Result<(), String> { + if at.pixels() != Some(pixels.len()) { + return Err(format!("image is {}×{} but has {} pixels", at.width, at.height, pixels.len())); + } + self.file.seek(SeekFrom::Start(at.offset)).map_err(io)?; + self.file.write_all(bytemuck::cast_slice(pixels)).map_err(io) + } + + /// Read the image `at` describes into `out` (which must have its size). + pub fn read_into(&mut self, at: &ImageRef, out: &mut [[f32; 4]]) -> Result<(), String> { + if at.pixels() != Some(out.len()) { + return Err(format!("image is {}×{} but the buffer has {} pixels", at.width, at.height, out.len())); + } + self.file.seek(SeekFrom::Start(at.offset)).map_err(io)?; + self.file.read_exact(bytemuck::cast_slice_mut(out)).map_err(io) + } + + /// Read the image `at` describes. + pub fn read(&mut self, at: &ImageRef) -> Result, String> { + let n = at.pixels().ok_or_else(|| format!("image {}×{} is too large", at.width, at.height))?; + let mut out = vec![[0.0f32; 4]; n]; + self.read_into(at, &mut out)?; + Ok(out) + } +} + +fn io(e: std::io::Error) -> String { + format!("frame exchange: {e}") +} + +impl Drop for Exchange { + fn drop(&mut self) { + if self.owned { + let _ = std::fs::remove_file(&self.path); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn both_sides_see_the_same_pixels() { + let mut a = Exchange::create().unwrap(); + let mut b = Exchange::open(a.path()).unwrap(); + // Owner-only: frames are the user's pixels and /dev/shm is shared. + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + assert_eq!(std::fs::metadata(a.path()).unwrap().permissions().mode() & 0o777, 0o600); + } + let at = ImageRef { offset: 64, width: 2, height: 1, origin: [0.0; 2], scale: 1.0 }; + a.write(&at, &[[0.1, 0.2, 0.3, 0.4], [1.0, 2.0, 3.0, 4.0]]).unwrap(); + assert_eq!(b.read(&at).unwrap()[1], [1.0, 2.0, 3.0, 4.0]); + assert!(b.write(&at, &[[0.0; 4]]).is_err(), "size mismatch is an error"); + let p = a.path().to_path_buf(); + a.unlink(); + drop(a); + drop(b); + assert!(!p.exists()); + } +} diff --git a/crates/helper-host/src/lib.rs b/crates/helper-host/src/lib.rs new file mode 100644 index 000000000..dd91f23b1 --- /dev/null +++ b/crates/helper-host/src/lib.rs @@ -0,0 +1,258 @@ +//! # effectcraft-helper-host (L3) +//! +//! **External effect helpers**: separate programs that offer effects to EffectCraft over a small, +//! versioned protocol ([`protocol`], `docs/external-helpers.md`, ADR +//! `docs/adr/0001-external-effect-helpers.md`). EffectCraft finds installed helpers +//! ([`find_helpers`]), starts each as a child process, lists its effects and registers every one +//! as an ordinary plug-in effect (plug-in API 2), so the UI, the property tree, keyframes, +//! scripting and the command surface need nothing new. Rendering sends the frame to the helper +//! through a shared exchange file ([`frames`]) and answers its questions (the input at another +//! time, another layer, parameters at another time) while it works. +//! +//! The point is containment: whatever native code a helper runs (an OpenFX plug-in host, for +//! example) runs in **its** process. If it crashes, hangs or talks nonsense, EffectCraft stops +//! it, shows the error on the effect, leaves the frame as it was and starts the helper again +//! for the next render; an effect that keeps taking its helper down is turned off +//! ([`blocklist`]). This crate is safe Rust (`unsafe_code = "forbid"`, like the rest of the +//! workspace), loads no native code and opens no network connection. +//! +//! [`serve`] is the helper side, for helper programs written in Rust. +//! +//! Native only: on wasm32 the crate is empty (there are no processes to start). + +#![cfg(not(target_arch = "wasm32"))] +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] + +pub mod blocklist; +pub mod client; +pub mod frames; +pub mod protocol; +pub mod serve; + +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex, OnceLock}; + +use effectcraft_effects::plugin::{plugin, register_plugin}; +use serde_json::{Value, json}; + +use crate::client::{Helper, HelperEffect, lock}; +use crate::protocol::RemoteEffect; + +/// Helper program names start with this (`effectcraft-helper-ofx`, plus `.exe` on Windows). +pub const PROGRAM_PREFIX: &str = "effectcraft-helper-"; + +/// The running helpers. +fn helpers() -> &'static Mutex>> { + static H: OnceLock>>> = OnceLock::new(); + H.get_or_init(Default::default) +} + +/// Where helpers are looked for: `helpers/` next to the executable, `helpers/` in the settings +/// folder (`config_dir`), and every entry of `EFFECTCRAFT_HELPER_PATH` (a path list of helper +/// programs or folders). +pub fn default_search_paths(config_dir: Option<&Path>) -> Vec { + let mut v = vec![]; + if let Some(dir) = std::env::current_exe().ok().and_then(|e| e.parent().map(Path::to_path_buf)) { + v.push(dir.join("helpers")); + // macOS app bundles: Contents/MacOS/effectcraft → Contents/Helpers. + if cfg!(target_os = "macos") + && let Some(contents) = dir.parent() + { + v.push(contents.join("Helpers")); + } + } + if let Some(c) = config_dir { + v.push(c.join("helpers")); + } + if let Some(p) = std::env::var_os("EFFECTCRAFT_HELPER_PATH") { + v.extend(std::env::split_paths(&p).filter(|p| !p.as_os_str().is_empty())); + } + v +} + +/// Whether a file name is a helper program's. +pub fn is_helper_name(name: &str) -> bool { + let stem = if cfg!(windows) { + match name.len().checked_sub(4).and_then(|i| name.get(i..).map(|ext| (i, ext))) { + Some((i, ext)) if ext.eq_ignore_ascii_case(".exe") => name.get(..i).unwrap_or(""), + _ => return false, + } + } else { + name + }; + stem.len() > PROGRAM_PREFIX.len() && stem.starts_with(PROGRAM_PREFIX) && !stem.contains('.') +} + +/// The helper programs in `paths`: files named `effectcraft-helper-*` in each folder and its +/// direct subfolders, and paths that are themselves such files (any name is accepted for an +/// explicit file). Sorted, without duplicates. +pub fn find_helpers(paths: &[PathBuf]) -> Vec { + fn named(p: &Path) -> bool { + p.is_file() && p.file_name().and_then(|n| n.to_str()).is_some_and(is_helper_name) + } + let mut out = vec![]; + for p in paths { + if p.is_file() { + out.push(p.clone()); + continue; + } + let Ok(rd) = std::fs::read_dir(p) else { continue }; + for e in rd.flatten() { + let path = e.path(); + if named(&path) { + out.push(path); + } else if path.is_dir() + && let Ok(sub) = std::fs::read_dir(&path) + { + out.extend(sub.flatten().map(|e| e.path()).filter(|p| named(p))); + } + } + } + out.sort(); + out.dedup(); + out +} + +/// Register a helper's effects. Effects already registered (an earlier scan) are kept as they +/// are; blocked effects are reported, not registered. +fn register(helper: &Arc, effects: Vec, errors: &mut Vec) -> Vec { + let mut loaded = vec![]; + let mut added = 0; + for e in effects { + let id = e.manifest.id.clone(); + if blocklist::is_blocked(&id) { + errors.push(format!("{id}: turned off because it kept stopping its helper (scan with retry to try again)")); + continue; + } + if plugin(&id).is_some() { + continue; + } + let (name, category) = (e.manifest.name.clone(), e.manifest.category.clone()); + match register_plugin(Arc::new(HelperEffect::new(helper.clone(), e))) { + Ok(_) => { + added += 1; + loaded.push(json!({"id": id, "name": name, "category": category, "helper": helper.program().to_string_lossy()})); + } + Err(err) => errors.push(format!("{id}: {err}")), + } + } + helper.add_effects(added); + loaded +} + +/// Start every helper found in `paths` that isn't running yet and register its effects. +/// `retry` forgets the block list first. Returns `{paths, helpers, loaded, errors}`. +pub fn scan_paths(paths: &[PathBuf], retry: bool) -> Value { + if retry { + blocklist::clear(); + } + let (mut started, mut loaded, mut errors) = (vec![], vec![], vec![]); + for program in find_helpers(paths) { + if lock(helpers()).iter().any(|h| h.program() == program) { + continue; + } + match Helper::start(&program) { + Ok((h, effects, errs)) => { + errors.extend(errs.into_iter().map(|e| format!("{}: {e}", program.display()))); + loaded.extend(register(&h, effects, &mut errors)); + started.push(json!(program.to_string_lossy())); + lock(helpers()).push(h); + } + Err(e) => errors.push(e), + } + } + json!({ + "paths": paths.iter().map(|p| p.to_string_lossy()).collect::>(), + "helpers": started, + "loaded": loaded, + "errors": errors, + }) +} + +/// Load a file or folder through the helpers that accept its extension (a folder goes to every +/// helper that loads anything). Returns `{loaded, errors}`. +pub fn load_path(path: &str) -> Result { + let p = Path::new(path); + let ext = p.extension().and_then(|e| e.to_str()).unwrap_or("").to_string(); + // `.ofx.bundle` folders and the like: match on any dotted part of the name. + let parts: Vec = p.file_name().and_then(|n| n.to_str()).map(|n| n.split('.').skip(1).map(str::to_string).collect()).unwrap_or_default(); + let list: Vec> = lock(helpers()).clone(); + let takers: Vec> = list + .into_iter() + .filter(|h| { + let s = h.status(); + if p.is_dir() && parts.is_empty() { !s.loads.is_empty() } else { h.loads(&ext) || parts.iter().any(|x| h.loads(x)) } + }) + .collect(); + if takers.is_empty() { + return Err(format!("no effect helper loads `{path}` (installed helpers: Effect ▸ Scan for Effect Helpers)")); + } + let (mut loaded, mut errors) = (vec![], vec![]); + for h in takers { + match h.load(path) { + Ok((effects, errs)) => { + errors.extend(errs); + loaded.extend(register(&h, effects, &mut errors)); + } + Err(e) => errors.push(e), + } + } + if loaded.is_empty() && !errors.is_empty() { + return Err(errors.join("; ")); + } + Ok(json!({"loaded": loaded, "errors": errors})) +} + +/// The helpers and their state: `{helpers: [{program, name, version, loads, running, effects, +/// restarts, error}], blocked}`. +pub fn list() -> Value { + let helpers: Vec = lock(helpers()) + .iter() + .map(|h| { + let s = h.status(); + json!({ + "program": s.program.to_string_lossy(), + "name": s.name, + "version": s.version, + "loads": s.loads, + "running": s.running, + "effects": s.effects, + "restarts": s.restarts, + "error": s.last_error, + }) + }) + .collect(); + json!({"helpers": helpers, "blocked": blocklist::blocked()}) +} + +/// Stop every helper process (they start again when needed). Called when the app quits; +/// helpers also exit on their own when EffectCraft's end of the pipe closes. +pub fn stop_all() { + for h in lock(helpers()).iter() { + h.stop(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn helper_programs_are_recognised_by_name() { + if cfg!(windows) { + assert!(is_helper_name("effectcraft-helper-ofx.exe")); + assert!(!is_helper_name("effectcraft-helper-ofx")); + } else { + assert!(is_helper_name("effectcraft-helper-ofx")); + assert!(!is_helper_name("effectcraft-helper-ofx.pdb")); + } + assert!(!is_helper_name("effectcraft-helper-")); + assert!(!is_helper_name("effectcraft")); + } + + #[test] + fn missing_folders_find_nothing_and_loading_without_helpers_is_an_error() { + assert!(find_helpers(&[PathBuf::from("/no/such/folder")]).is_empty()); + assert!(load_path("/no/such/thing.xyz").is_err()); + } +} diff --git a/crates/helper-host/src/protocol.rs b/crates/helper-host/src/protocol.rs new file mode 100644 index 000000000..48a891023 --- /dev/null +++ b/crates/helper-host/src/protocol.rs @@ -0,0 +1,207 @@ +//! The helper protocol, version [`PROTOCOL_VERSION`] (`docs/external-helpers.md`). +//! +//! Control messages are JSON, one per line. EffectCraft writes requests ([`Request`]) to the +//! helper's stdin; the helper writes replies ([`Reply`]) to its stdout, each line starting with +//! [`MAGIC`] so that anything else a helper (or a library it loads) prints is ignored. Pixels +//! never travel over the pipe: they live in the **frame exchange file** named in +//! [`Request::Hello`] and are addressed by [`ImageRef`]s (byte offset + size). + +use effectcraft_effects::plugin::PluginManifest; +use serde::{Deserialize, Serialize}; + +/// The protocol version this build speaks. A helper that answers `hello` with another version +/// is not used. +pub const PROTOCOL_VERSION: u32 = 1; + +/// Every reply line from a helper starts with this (anything before it on the line, and lines +/// without it, are the helper's own output and only logged). +pub const MAGIC: &str = "@ec-helper "; + +/// The argument EffectCraft starts helpers with. +pub const HELPER_ARG: &str = "--effectcraft-helper"; + +/// The longest control line either side accepts (bytes). Hidden parameters can carry a few MB. +pub const MAX_LINE: u64 = 256 << 20; + +/// The most pixels one image may have (16384 × 16384). +pub const MAX_PIXELS: u64 = 1 << 28; + +/// Bytes per pixel in the exchange file: premultiplied RGBA `f32`, native byte order. +pub const PIXEL_BYTES: u64 = 16; + +/// An image in the frame exchange file: `width * height` premultiplied RGBA `f32` pixels, row 0 +/// on top, starting `offset` bytes into the file. `origin` and `scale` are as in +/// [`effectcraft_effects::plugin::PluginImage`]. +#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)] +pub struct ImageRef { + pub offset: u64, + pub width: u32, + pub height: u32, + #[serde(default)] + pub origin: [f64; 2], + #[serde(default = "one")] + pub scale: f64, +} + +fn one() -> f64 { + 1.0 +} + +impl ImageRef { + /// Pixels in the image, `None` when it is larger than [`MAX_PIXELS`]. + pub fn pixels(&self) -> Option { + let n = u64::from(self.width).checked_mul(u64::from(self.height))?; + (n <= MAX_PIXELS).then_some(n).and_then(|n| usize::try_from(n).ok()) + } + /// Bytes the image takes in the exchange file. + pub fn bytes(&self) -> Option { + u64::try_from(self.pixels()?).ok()?.checked_mul(PIXEL_BYTES) + } +} + +/// One render of one effect instance. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct RenderRequest { + /// The effect id (as described by the helper). + pub effect: String, + /// A stable id of the applied effect instance ([`effectcraft_effects::plugin::PluginHost::instance_key`]). + pub instance: u64, + /// Layer time, seconds. + pub time: f64, + /// The buffer to process in place (already padded); the result is written back to it. + pub frame: ImageRef, + /// The layer's size in layer pixels. + pub layer_size: [f64; 2], + /// Flattened parameter values (plug-in API 2 rules). + pub flat: Vec, + /// Text / hidden parameter strings. + #[serde(default)] + pub texts: Vec, + pub frame_rate: f64, + /// The layer's source range in layer-time seconds, when it has one. + #[serde(default)] + pub frame_range: Option<(f64, f64)>, +} + +/// EffectCraft → helper. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(tag = "op", rename_all = "snake_case")] +pub enum Request { + /// First message: the protocol version and the frame exchange file to open (read + write). + Hello { version: u32, frames: String }, + /// List the effects the helper offers → [`Reply::Effects`]. + Describe, + /// Load more effects from a file or folder the user picked → [`Reply::Effects`] (only the + /// new ones). + Load { path: String }, + /// Transparent pixels to add around the buffer before [`Request::Render`] → [`Reply::Padding`]. + Padding { effect: String, instance: u64, time: f64, scale: f64, layer_size: [f64; 2], flat: Vec, texts: Vec }, + /// Render → callbacks ([`Reply::InputAt`], ...) answered with [`Request::Image`] / + /// [`Request::Params`], then [`Reply::Done`] or [`Reply::Error`]. + Render(RenderRequest), + /// The answer to [`Reply::InputAt`] / [`Reply::LayerInput`]: the image, or `None`. + Image { image: Option }, + /// The answer to [`Reply::ParamsAt`]: flattened values and texts, or `None`. + Params { values: Option<(Vec, Vec)> }, + /// The effect instance is gone; the helper may free what it keeps for it. No reply. + Destroy { instance: u64 }, + /// Heartbeat → [`Reply::Pong`]. + Ping, + /// Exit. No reply. (Closing stdin means the same.) + Shutdown, +} + +/// One effect a helper offers. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct RemoteEffect { + /// The plug-in API 2 manifest (id, name, category, parameters). + pub manifest: PluginManifest, + /// Whether to ask [`Request::Padding`] before every render (otherwise no padding). + #[serde(default)] + pub padding: bool, +} + +/// Helper → EffectCraft. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(tag = "op", rename_all = "snake_case")] +pub enum Reply { + /// Answer to [`Request::Hello`]. `loads` lists the file extensions [`Request::Load`] accepts + /// (`["ofx"]`), empty when the helper loads nothing on request. + Hello { + version: u32, + name: String, + #[serde(default)] + helper_version: String, + #[serde(default)] + loads: Vec, + }, + Effects { + effects: Vec, + #[serde(default)] + errors: Vec, + }, + Padding { + pixels: u32, + }, + /// During a render: the effect's input at another layer time. + InputAt { + time: f64, + }, + /// During a render: the layer chosen in a layer parameter, at a layer time. + LayerInput { + param: String, + time: f64, + }, + /// During a render: the parameter values at another layer time. + ParamsAt { + time: f64, + }, + /// The render finished; the result is in the request's frame. + Done, + /// The request failed (the helper keeps running). + Error { + message: String, + }, + Pong, +} + +/// A reply as one protocol line (with [`MAGIC`] and the newline). +pub fn reply_line(r: &Reply) -> Result { + serde_json::to_string(r).map(|s| format!("{MAGIC}{s}\n")).map_err(|e| e.to_string()) +} + +/// A request as one protocol line (with the newline). +pub fn request_line(r: &Request) -> Result { + serde_json::to_string(r).map(|s| format!("{s}\n")).map_err(|e| e.to_string()) +} + +/// Parse a helper output line: `Some` for a protocol line (after [`MAGIC`]), `None` for other +/// output. +pub fn parse_reply(line: &str) -> Option> { + let at = line.find(MAGIC)?; + let json = line.get(at + MAGIC.len()..)?.trim_end(); + Some(serde_json::from_str(json).map_err(|e| format!("bad reply from helper: {e}"))) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn replies_round_trip_and_other_output_is_ignored() { + let r = Reply::InputAt { time: 1.5 }; + let line = reply_line(&r).unwrap(); + assert_eq!(parse_reply(&line).unwrap().unwrap(), r); + // A plug-in that printed without a newline doesn't break the protocol line. + assert_eq!(parse_reply(&format!("loading...{line}")).unwrap().unwrap(), r); + assert!(parse_reply("plug-in chatter").is_none()); + assert!(parse_reply("@ec-helper {nope").unwrap().is_err()); + } + + #[test] + fn image_sizes_are_checked() { + let r = ImageRef { offset: 0, width: 4, height: 2, origin: [0.0; 2], scale: 1.0 }; + assert_eq!(r.bytes(), Some(128)); + assert_eq!(ImageRef { width: u32::MAX, height: u32::MAX, ..r }.pixels(), None); + } +} diff --git a/crates/helper-host/src/serve.rs b/crates/helper-host/src/serve.rs new file mode 100644 index 000000000..5eb82c236 --- /dev/null +++ b/crates/helper-host/src/serve.rs @@ -0,0 +1,240 @@ +//! The helper side of the protocol: [`serve`] answers EffectCraft's requests with any set of +//! [`EffectPlugin`]s, so a helper program only has to provide its effects ([`Provider`]). +//! The fake helper in the tests and the OpenFX helper both use it. +//! +//! During a render the plug-in's [`PluginHost`] calls become requests to EffectCraft +//! ([`Reply::InputAt`], ...). While a helper waits for the answer, EffectCraft may send a nested +//! [`Request::Render`] (an effect of this helper further up the same layer's stack); it is +//! served right away, on the same stack, before the wait goes on. + +use std::cell::RefCell; +use std::collections::HashMap; +use std::io::{BufRead, Read, Write}; +use std::path::Path; +use std::sync::Arc; + +use effectcraft_effects::plugin::{EffectPlugin, PluginFrame, PluginHost, PluginImage, PluginParams}; + +use crate::frames::Exchange; +use crate::protocol::{ImageRef, MAX_LINE, PROTOCOL_VERSION, RemoteEffect, RenderRequest, Reply, Request, reply_line}; + +/// How deep nested renders may go. +const MAX_DEPTH: u32 = 16; + +/// What a helper program offers. +pub trait Provider { + /// The helper's display name (`OpenFX`). + fn name(&self) -> String; + /// The helper's own version. + fn version(&self) -> String { + String::new() + } + /// File extensions [`Provider::load`] accepts (`["ofx"]`). + fn loads(&self) -> Vec { + vec![] + } + /// The effects available at start-up, and errors worth reporting. + fn effects(&mut self) -> (Vec>, Vec); + /// Load effects from a file or folder the user picked. + fn load(&mut self, _path: &str) -> Result<(Vec>, Vec), String> { + Err("this helper does not load files".into()) + } + /// Whether EffectCraft should ask for padding before each render of `effect`. + fn wants_padding(&self, _effect: &dyn EffectPlugin) -> bool { + true + } + /// The effect instance is gone. + fn destroy(&mut self, _instance: u64) {} +} + +/// Serve requests on stdin / stdout until EffectCraft says `shutdown` or closes the pipe. +pub fn serve(provider: &mut dyn Provider) -> Result<(), String> { + let stdin = std::io::stdin(); + let stdout = std::io::stdout(); + serve_io(provider, stdin.lock(), stdout.lock()) +} + +/// [`serve`] on any reader / writer. +pub fn serve_io(provider: &mut dyn Provider, input: impl BufRead, output: impl Write) -> Result<(), String> { + let mut s = Server { provider, input: Box::new(input), output: Box::new(output), exchange: None, effects: HashMap::new(), depth: 0 }; + loop { + let Some(req) = s.next_request()? else { return Ok(()) }; + if matches!(req, Request::Shutdown) { + return Ok(()); + } + s.handle(req)?; + } +} + +struct Server<'p> { + provider: &'p mut dyn Provider, + input: Box, + output: Box, + exchange: Option, + effects: HashMap>, + depth: u32, +} + +impl Server<'_> { + /// The next request; `None` when the pipe closed. + fn next_request(&mut self) -> Result, String> { + loop { + let mut buf = vec![]; + let n = (&mut self.input).take(MAX_LINE).read_until(b'\n', &mut buf).map_err(|e| e.to_string())?; + if n == 0 { + return Ok(None); + } + let line = String::from_utf8_lossy(&buf); + if line.trim().is_empty() { + continue; + } + match serde_json::from_str::(line.trim()) { + Ok(r) => return Ok(Some(r)), + Err(e) => self.send(&Reply::Error { message: format!("bad request: {e}") })?, + } + } + } + + fn send(&mut self, r: &Reply) -> Result<(), String> { + let line = reply_line(r)?; + self.output.write_all(line.as_bytes()).and_then(|_| self.output.flush()).map_err(|e| e.to_string()) + } + + fn describe(&mut self, list: Vec>, errors: Vec) -> Result<(), String> { + let mut effects = vec![]; + for e in list { + let manifest = e.manifest().clone(); + effects.push(RemoteEffect { padding: self.provider.wants_padding(e.as_ref()), manifest: manifest.clone() }); + self.effects.insert(manifest.id, e); + } + self.send(&Reply::Effects { effects, errors }) + } + + /// Answer one request (anything but `shutdown`). + fn handle(&mut self, req: Request) -> Result<(), String> { + match req { + Request::Hello { version, frames } => { + if version != PROTOCOL_VERSION { + return self.send(&Reply::Error { message: format!("this helper speaks protocol {PROTOCOL_VERSION}, not {version}") }); + } + match Exchange::open(Path::new(&frames)) { + Ok(x) => self.exchange = Some(x), + Err(e) => return self.send(&Reply::Error { message: format!("can't open the frame exchange file: {e}") }), + } + let (name, helper_version, loads) = (self.provider.name(), self.provider.version(), self.provider.loads()); + self.send(&Reply::Hello { version: PROTOCOL_VERSION, name, helper_version, loads }) + } + Request::Describe => { + let (list, errors) = self.provider.effects(); + self.describe(list, errors) + } + Request::Load { path } => match self.provider.load(&path) { + Ok((list, errors)) => self.describe(list, errors), + Err(message) => self.send(&Reply::Error { message }), + }, + Request::Padding { effect, instance, time, scale, layer_size, flat, texts } => { + let Some(e) = self.effects.get(&effect).cloned() else { + return self.send(&Reply::Error { message: format!("unknown effect `{effect}`") }); + }; + let params = PluginParams { manifest: e.manifest(), flat: &flat, texts: &texts }; + let pixels = e.padding(¶ms, time, scale, layer_size, instance); + self.send(&Reply::Padding { pixels }) + } + Request::Render(r) => match self.render(&r) { + Ok(()) => self.send(&Reply::Done), + Err(message) => self.send(&Reply::Error { message }), + }, + Request::Destroy { instance } => { + self.provider.destroy(instance); + Ok(()) + } + Request::Ping => self.send(&Reply::Pong), + Request::Image { .. } | Request::Params { .. } => self.send(&Reply::Error { message: "no callback is waiting for an answer".into() }), + Request::Shutdown => Ok(()), + } + } + + fn render(&mut self, r: &RenderRequest) -> Result<(), String> { + let e = self.effects.get(&r.effect).cloned().ok_or_else(|| format!("unknown effect `{}`", r.effect))?; + if self.depth >= MAX_DEPTH { + return Err("renders nested too deeply".into()); + } + let exchange = self.exchange.as_mut().ok_or("no frame exchange file (hello first)")?; + let mut pixels = exchange.read(&r.frame)?; + let params = PluginParams { manifest: e.manifest(), flat: &r.flat, texts: &r.texts }; + let mut frame = PluginFrame { + width: r.frame.width, + height: r.frame.height, + pixels: &mut pixels, + scale: r.frame.scale, + origin: r.frame.origin, + layer_size: r.layer_size, + }; + self.depth += 1; + let result = { + let host = Callbacks { server: RefCell::new(&mut *self), req: r }; + e.render_v2(&mut frame, ¶ms, r.time, &host) + }; + self.depth = self.depth.saturating_sub(1); + result?; + let exchange = self.exchange.as_mut().ok_or("no frame exchange file")?; + exchange.write(&r.frame, &pixels) + } + + /// Ask EffectCraft something during a render and wait for the answer, serving nested + /// requests meanwhile. + fn ask(&mut self, q: &Reply) -> Result { + self.send(q)?; + loop { + let req = self.next_request()?.ok_or("EffectCraft closed the pipe")?; + match req { + Request::Image { .. } | Request::Params { .. } => return Ok(req), + Request::Shutdown => return Err("EffectCraft asked the helper to shut down".into()), + other => self.handle(other)?, + } + } + } + + fn image(&mut self, q: &Reply) -> Option { + let Ok(Request::Image { image: Some(at) }) = self.ask(q) else { return None }; + self.read_image(&at) + } + + fn read_image(&mut self, at: &ImageRef) -> Option { + let pixels = self.exchange.as_mut()?.read(at).ok()?; + Some(PluginImage { width: at.width, height: at.height, pixels, origin: at.origin, scale: at.scale }) + } +} + +/// The [`PluginHost`] a helper's plug-in renders with: questions go back to EffectCraft. +struct Callbacks<'a, 'p> { + server: RefCell<&'a mut Server<'p>>, + req: &'a RenderRequest, +} + +impl PluginHost for Callbacks<'_, '_> { + fn input_at(&self, layer_time: f64) -> Option { + self.server.try_borrow_mut().ok()?.image(&Reply::InputAt { time: layer_time }) + } + fn layer_input(&self, param_id: &str, layer_time: f64) -> Option { + self.server.try_borrow_mut().ok()?.image(&Reply::LayerInput { param: param_id.to_string(), time: layer_time }) + } + fn params_at(&self, layer_time: f64) -> Option<(Vec, Vec)> { + match self.server.try_borrow_mut().ok()?.ask(&Reply::ParamsAt { time: layer_time }) { + Ok(Request::Params { values }) => values, + _ => None, + } + } + fn frame_rate(&self) -> f64 { + self.req.frame_rate + } + fn time(&self) -> f64 { + self.req.time + } + fn instance_key(&self) -> u64 { + self.req.instance + } + fn frame_range(&self) -> Option<(f64, f64)> { + self.req.frame_range + } +} diff --git a/crates/helper-host/tests/it/main.rs b/crates/helper-host/tests/it/main.rs new file mode 100644 index 000000000..b0338ee91 --- /dev/null +++ b/crates/helper-host/tests/it/main.rs @@ -0,0 +1,327 @@ +//! The helper protocol end to end against a fake helper, written in safe Rust with +//! [`effectcraft_helper_host::serve`]. This binary is both sides: started with +//! `--effectcraft-helper` it is the helper, otherwise it runs the checks, starting itself as +//! the helper. (`harness = false`: libtest would reject the helper argument.) + +use std::io::Write; +use std::sync::Arc; +use std::time::Duration; + +use effectcraft_effects::plugin::{EffectPlugin, PluginFrame, PluginHost, PluginImage, PluginManifest, PluginParam, PluginParamKind, PluginParams, plugin}; +use effectcraft_helper_host::protocol::{HELPER_ARG, Reply, reply_line}; +use effectcraft_helper_host::serve::{Provider, serve}; + +fn manifest(id: &str, params: Vec) -> PluginManifest { + PluginManifest { + api: 2, + id: id.into(), + name: id.rsplit('.').next().unwrap_or(id).into(), + category: "Helper Test".into(), + version: "1".into(), + author: String::new(), + description: String::new(), + params, + } +} + +fn amount() -> PluginParam { + PluginParam { + id: "amount".into(), + name: "Amount".into(), + group: String::new(), + kind: PluginParamKind::Slider { default: 1.0, min: 0.0, max: 1.0, slider_min: None, slider_max: None, decimals: 2, integer: false }, + } +} + +/// What each fake effect does. +#[derive(Clone, Copy, PartialEq)] +enum Kind { + /// RGB → amount·(1 − RGB). + Invert, + /// Asks for 3 pixels of padding; fills the frame with its padding as red. + Pad, + /// Replaces the frame with the input one frame earlier, and puts `amount` from + /// `params_at(0)` in alpha of pixel 0. + Echo, + /// Exits the process mid-render. + Crash, + /// Asks for the parameters before 0 (an answer far larger than a pipe holds), then never + /// reads it or answers. + Hang, + /// Writes a long line of bytes that aren't UTF-8 to stderr, then more, then inverts. + Noisy, +} + +struct Fake { + m: PluginManifest, + kind: Kind, +} + +impl EffectPlugin for Fake { + fn manifest(&self) -> &PluginManifest { + &self.m + } + fn render(&self, _: &mut PluginFrame, _: &PluginParams, _: f64) -> Result<(), String> { + Err("needs render_v2".into()) + } + fn padding(&self, _: &PluginParams, _: f64, _: f64, _: [f64; 2], _: u64) -> u32 { + if self.kind == Kind::Pad { 3 } else { 0 } + } + fn render_v2(&self, frame: &mut PluginFrame, params: &PluginParams, _time: f64, host: &dyn PluginHost) -> Result<(), String> { + match self.kind { + Kind::Invert => { + let a = params.f("amount") as f32; + for p in frame.pixels.iter_mut() { + for c in &mut p[..3] { + *c = a * (1.0 - *c); + } + } + } + Kind::Pad => frame.pixels.iter_mut().for_each(|p| *p = [3.0, 0.0, 0.0, 1.0]), + Kind::Echo => { + let prev = host.input_at(host.time() - 1.0 / host.frame_rate()).ok_or("no input at the previous frame")?; + if prev.pixels.len() != frame.pixels.len() { + return Err("previous frame has another size".into()); + } + frame.pixels.copy_from_slice(&prev.pixels); + let (flat, _) = host.params_at(0.0).ok_or("no params at 0")?; + if let (Some(p), Some(a)) = (frame.pixels.first_mut(), flat.first()) { + p[3] = *a as f32; + } + } + Kind::Crash => std::process::exit(3), + Kind::Hang => { + let mut out = std::io::stdout(); + out.write_all(reply_line(&Reply::ParamsAt { time: -1.0 })?.as_bytes()).and_then(|_| out.flush()).map_err(|e| e.to_string())?; + std::thread::sleep(Duration::from_secs(3600)); + } + Kind::Noisy => { + let mut err = std::io::stderr(); + let mut junk = vec![0xffu8; 1 << 20]; + junk.push(b'\n'); + err.write_all(&junk).map_err(|e| format!("stderr: {e}"))?; + // EffectCraft must still be reading. + std::thread::sleep(Duration::from_millis(300)); + err.write_all(b"still talking\n").map_err(|e| format!("stderr: {e}"))?; + frame.pixels.iter_mut().for_each(|p| *p = [1.0, 0.0, 0.0, 1.0]); + } + } + Ok(()) + } +} + +struct FakeProvider; + +impl Provider for FakeProvider { + fn name(&self) -> String { + "Fake".into() + } + fn effects(&mut self) -> (Vec>, Vec) { + let v: Vec> = + [("invert", Kind::Invert), ("pad", Kind::Pad), ("echo", Kind::Echo), ("crash", Kind::Crash), ("hang", Kind::Hang), ("noisy", Kind::Noisy)] + .into_iter() + .map(|(n, kind)| Arc::new(Fake { m: manifest(&format!("org.test.helper.{n}"), vec![amount()]), kind }) as Arc) + .collect(); + (v, vec!["fake helper: a warning".into()]) + } +} + +/// The host side of a render in these checks: the input at any time is a flat grey of that +/// time; with `nest`, asking for the input renders `invert` through the same helper first. +struct TestHost { + w: u32, + h: u32, + nest: bool, +} + +impl PluginHost for TestHost { + fn input_at(&self, t: f64) -> Option { + let mut pixels = vec![[t as f32, t as f32, t as f32, 1.0]; (self.w * self.h) as usize]; + if self.nest { + let inv = plugin("org.test.helper.invert")?; + let m = inv.manifest().clone(); + let mut f = + PluginFrame { width: self.w, height: self.h, pixels: &mut pixels, scale: 1.0, origin: [0.0; 2], layer_size: [self.w as f64, self.h as f64] }; + inv.render_v2(&mut f, &PluginParams { manifest: &m, flat: &[1.0], texts: &[] }, t, &TestHost { nest: false, ..*self }).ok()?; + } + Some(PluginImage { width: self.w, height: self.h, pixels, origin: [0.0; 2], scale: 1.0 }) + } + fn layer_input(&self, _: &str, _: f64) -> Option { + None + } + fn params_at(&self, t: f64) -> Option<(Vec, Vec)> { + // Before 0: megabytes, far more than a pipe holds (see `Kind::Hang`). + if t < 0.0 { + return Some((vec![0.5; 1 << 20], vec![])); + } + Some((vec![0.25], vec![])) + } + fn frame_rate(&self) -> f64 { + 10.0 + } + fn time(&self) -> f64 { + 1.0 + } + fn instance_key(&self) -> u64 { + 7 + } +} + +fn render(id: &str, pixels: &mut Vec<[f32; 4]>, w: u32, h: u32, amount: f64, nest: bool) -> Result<(), String> { + let p = plugin(id).ok_or_else(|| format!("{id} is not registered"))?; + let m = p.manifest().clone(); + let params = PluginParams { manifest: &m, flat: &[amount], texts: &[] }; + let host = TestHost { w, h, nest }; + let pad = p.padding(¶ms, 1.0, 1.0, [w as f64, h as f64], 7); + let (w, h) = (w + 2 * pad, h + 2 * pad); + pixels.resize((w * h) as usize, [0.0; 4]); + let mut f = PluginFrame { width: w, height: h, pixels, scale: 1.0, origin: [pad as f64; 2], layer_size: [w as f64, h as f64] }; + p.render_v2(&mut f, ¶ms, 1.0, &host) +} + +fn check(name: &str, ok: bool, detail: impl std::fmt::Debug) -> bool { + println!("{} {name}{}", if ok { "ok " } else { "FAIL" }, if ok { String::new() } else { format!(": {detail:?}") }); + ok +} + +fn run_checks() -> bool { + let exe = match std::env::current_exe() { + Ok(e) => e, + Err(e) => return check("current exe", false, e), + }; + let state = std::env::temp_dir().join(format!("ec-fake-helper-state-{}", std::process::id())); + effectcraft_helper_host::blocklist::set_state_dir(state.clone()); + let mut all = true; + + // Discovery + describe. + let r = effectcraft_helper_host::scan_paths(std::slice::from_ref(&exe), false); + let loaded = r["loaded"].as_array().map_or(0, Vec::len); + all &= check("scan registers the helper's effects", loaded == 6, &r); + all &= check("helper warnings are reported", r["errors"].to_string().contains("a warning"), &r); + all &= check( + "a second scan doesn't start it twice", + effectcraft_helper_host::scan_paths(std::slice::from_ref(&exe), false)["helpers"].as_array().is_some_and(Vec::is_empty), + "", + ); + let list = effectcraft_helper_host::list(); + all &= check("list names the helper", list["helpers"][0]["name"] == "Fake" && list["helpers"][0]["running"] == true, &list); + + // Plain render through the exchange file. + let mut px = vec![[0.25, 0.5, 1.0, 1.0]; 6]; + let r = render("org.test.helper.invert", &mut px, 3, 2, 1.0, false); + all &= check("invert renders", r.is_ok() && px.iter().all(|p| (p[0] - 0.75).abs() < 1e-6 && p[2].abs() < 1e-6 && p[3] == 1.0), (&r, &px)); + + // Output on stderr that isn't text, in long lines, is drained, so the helper never blocks on it + // or loses its stderr. + let mut px = vec![[0.0; 4]; 4]; + let r = render("org.test.helper.noisy", &mut px, 2, 2, 1.0, false); + all &= check("a helper's stderr is drained whatever it holds", r.is_ok() && px[0] == [1.0, 0.0, 0.0, 1.0], &r); + + // Padding. + let mut px = vec![[0.0; 4]; 4]; + let r = render("org.test.helper.pad", &mut px, 2, 2, 1.0, false); + all &= check("padding grows the frame", r.is_ok() && px.len() == 64 && px[0][0] == 3.0, (&r, px.len())); + + // Callbacks: the input at another time and parameters at another time. + let mut px = vec![[0.0; 4]; 4]; + let r = render("org.test.helper.echo", &mut px, 2, 2, 1.0, false); + all &= check("input_at / params_at round-trip", r.is_ok() && (px[1][0] - 0.9).abs() < 1e-6 && px[0][3] == 0.25, (&r, &px)); + + // Nested: answering input_at renders another effect of the same helper. + let mut px = vec![[0.0; 4]; 4]; + let r = render("org.test.helper.echo", &mut px, 2, 2, 1.0, true); + all &= check("nested renders through one helper", r.is_ok() && (px[1][0] - 0.1).abs() < 1e-5, (&r, &px)); + + // Several threads at once share the helper. + let threads: Vec<_> = (0..4) + .map(|i| { + std::thread::spawn(move || { + let mut px = vec![[0.0, 0.0, 0.0, 1.0]; 16]; + render("org.test.helper.invert", &mut px, 4, 4, 0.5, false).map(|_| (i, px[0][0])) + }) + }) + .collect(); + let results: Vec<_> = threads.into_iter().map(|t| t.join().map_err(|_| "thread panicked".to_string()).and_then(|r| r)).collect(); + all &= check("concurrent renders", results.iter().all(|r| matches!(r, Ok((_, v)) if (*v - 0.5).abs() < 1e-6)), &results); + + // A crash mid-render is an error, the frame is untouched, and the helper comes back. + let mut px = vec![[0.5; 4]; 4]; + let r = render("org.test.helper.crash", &mut px, 2, 2, 1.0, false); + all &= check("a crashing helper is an error", r.as_ref().is_err_and(|e| e.contains("helper")) && px.iter().all(|p| *p == [0.5; 4]), &r); + let mut px = vec![[0.0, 0.0, 0.0, 1.0]; 4]; + let r = render("org.test.helper.invert", &mut px, 2, 2, 1.0, false); + all &= check("the helper restarts for the next render", r.is_ok() && px[0][0] == 1.0, &r); + // A render that starts the stopped helper itself (no padding request first) succeeds. + effectcraft_helper_host::stop_all(); + let r = plugin("org.test.helper.invert").ok_or("invert is not registered").and_then(|p| { + let m = p.manifest().clone(); + let mut px = vec![[0.0, 0.0, 0.0, 1.0]; 4]; + let mut f = PluginFrame { width: 2, height: 2, pixels: &mut px, scale: 1.0, origin: [0.0; 2], layer_size: [2.0; 2] }; + let host = TestHost { w: 2, h: 2, nest: false }; + p.render_v2(&mut f, &PluginParams { manifest: &m, flat: &[1.0], texts: &[] }, 1.0, &host).map_err(|_| "render failed")?; + Ok(px[0][0]) + }); + all &= check("a stopped helper restarts for a render without padding", r == Ok(1.0), r); + + // Effects that keep crashing it are turned off (and the list survives in the state dir). + for _ in 0..3 { + let _ = render("org.test.helper.crash", &mut vec![[0.0; 4]; 4], 2, 2, 1.0, false); + } + let r = render("org.test.helper.crash", &mut vec![[0.0; 4]; 4], 2, 2, 1.0, false); + all &= check("repeat offenders are blocked", r.as_ref().is_err_and(|e| e.contains("turned off")), &r); + all &= check("the block list is saved", std::fs::read_to_string(state.join("helper_blocklist.txt")).is_ok_and(|t| t.contains("org.test.helper.crash")), ""); + let r = effectcraft_helper_host::scan_paths(&[], true); + all &= check("retry clears the block list", !effectcraft_helper_host::blocklist::is_blocked("org.test.helper.crash"), &r); + + // A hang is cut off by the timeout, also while EffectCraft is writing to a helper that + // stopped reading. + let t0 = std::time::Instant::now(); + let r = render("org.test.helper.hang", &mut vec![[0.0; 4]; 4], 2, 2, 1.0, false); + all &= check( + "a hanging helper times out", + r.as_ref().is_err_and(|e| e.contains("did not answer")) && t0.elapsed() < Duration::from_secs(30), + (&r, t0.elapsed()), + ); + + // The restarts above use up the helper's restarts for this minute, so it can't start again. + // That is not the fault of the effect being rendered: it fails, but is not charged and never + // turned off (it would be after CRASH_LIMIT charged failures). + let starts: Vec<_> = (0..4).map(|_| render("org.test.helper.invert", &mut vec![[0.0; 4]; 4], 2, 2, 1.0, false)).collect(); + all &= check( + "a helper that can't start doesn't block the effect", + starts.iter().all(|r| r.as_ref().is_err_and(|e| e.contains("keeps stopping"))) + && !effectcraft_helper_host::blocklist::is_blocked("org.test.helper.invert"), + &starts, + ); + + effectcraft_helper_host::stop_all(); + let _ = std::fs::remove_dir_all(&state); + all +} + +/// The one test this binary reports to test runners. +const TEST_NAME: &str = "helper_protocol"; + +fn main() { + let args: Vec = std::env::args().collect(); + if args.iter().any(|a| a == HELPER_ARG) { + if let Err(e) = serve(&mut FakeProvider) { + eprintln!("fake helper: {e}"); + std::process::exit(1); + } + return; + } + // The libtest listing protocol cargo-nextest uses (`--list --format terse`, with and without + // `--ignored`): one `name: test` line, then nextest runs ` helper_protocol --exact`. + if args.iter().any(|a| a == "--list") { + if !args.iter().any(|a| a == "--ignored") { + println!("{TEST_NAME}: test"); + } + return; + } + // Keep the hang check short. + effectcraft_helper_host::client::set_render_timeout(Duration::from_secs(3)); + if !run_checks() { + std::process::exit(1); + } +} diff --git a/crates/host/Cargo.toml b/crates/host/Cargo.toml index 80f61e4c3..70991b22e 100644 --- a/crates/host/Cargo.toml +++ b/crates/host/Cargo.toml @@ -18,13 +18,17 @@ effectcraft-expr = { workspace = true } effectcraft-script = { workspace = true } effectcraft-export = { workspace = true } effectcraft-plugin = { workspace = true } +serde_json = { workspace = true } # WebAssembly effect plug-ins run in wasmi on native builds. [target.'cfg(not(target_arch = "wasm32"))'.dependencies] effectcraft-plugin = { workspace = true, features = ["wasm"] } +# External effect helpers: separate processes, so their native code can't take the app down. +effectcraft-helper-host = { workspace = true } [dev-dependencies] serde_json = { workspace = true } +tempfile = { workspace = true } effectcraft-psd = { workspace = true } effectcraft-svg = { workspace = true } effectcraft-pdf = { workspace = true } diff --git a/crates/host/src/lib.rs b/crates/host/src/lib.rs index cf2308814..74fa8348d 100644 --- a/crates/host/src/lib.rs +++ b/crates/host/src/lib.rs @@ -95,6 +95,7 @@ pub fn session() -> Session { expr: Some(Arc::new(effectcraft_expr::Expressions)), expr_check: Some(effectcraft_expr::check_syntax), plugin_loader: effectcraft_plugin::wasm_available().then_some(effectcraft_plugin::loader as effectcraft_engine::PluginLoader), + helpers: helper_hooks(), plugin_previewer: effectcraft_plugin::wasm_available().then_some(effectcraft_plugin::preview as effectcraft_engine::PluginPreviewer), plugins_dir: config_dir().map(|d| d.join("Plug-ins")), ..Default::default() @@ -103,6 +104,31 @@ pub fn session() -> Session { s } +/// External effect helpers run as child processes: native builds only. +#[cfg(not(target_arch = "wasm32"))] +fn helper_hooks() -> Option { + use effectcraft_helper_host as hh; + // Effects that keep stopping their helper are remembered in the settings folder. + if let Some(dir) = config_dir() { + hh::blocklist::set_state_dir(dir); + } + fn scan(retry: bool) -> serde_json::Value { + hh::scan_paths(&hh::default_search_paths(config_dir().as_deref()), retry) + } + Some(effectcraft_engine::HelperHooks { scan, load: hh::load_path, list: hh::list }) +} + +#[cfg(target_arch = "wasm32")] +fn helper_hooks() -> Option { + None +} + +/// Stop the external effect helpers (when the app quits). +pub fn stop_helpers() { + #[cfg(not(target_arch = "wasm32"))] + effectcraft_helper_host::stop_all(); +} + /// The platform config directory for EffectCraft (`EFFECTCRAFT_CONFIG_DIR` overrides): /// `~/Library/Application Support/EffectCraft` (macOS), `%APPDATA%\EffectCraft` (Windows), /// `$XDG_CONFIG_HOME/effectcraft` or `~/.config/effectcraft` (Linux and others). A `portable.txt` diff --git a/crates/host/tests/it/plugins.rs b/crates/host/tests/it/plugins.rs index 278b54cc8..7e4839e6d 100644 --- a/crates/host/tests/it/plugins.rs +++ b/crates/host/tests/it/plugins.rs @@ -49,7 +49,7 @@ fn rust_plugins_apply_by_id() { version: "1".into(), author: String::new(), description: String::new(), - params: vec![PluginParam { id: "on".into(), name: "On".into(), kind: PluginParamKind::Checkbox { default: true } }], + params: vec![PluginParam { id: "on".into(), name: "On".into(), kind: PluginParamKind::Checkbox { default: true }, group: String::new() }], }; register_plugin(Arc::new(Swap { m })).unwrap(); let (mut s, l) = setup(); @@ -62,7 +62,7 @@ fn rust_plugins_apply_by_id() { assert!(p[0] > 0.99 && p[2] < 0.01, "{p:?}"); let list = s.execute("effect.plugins.list", json!({})).unwrap(); assert!(list["plugins"].as_array().unwrap().iter().any(|p| p["id"] == "org.test.swap" && p["category"] == "Channel")); - assert_eq!(list["api"], 1); + assert_eq!(list["api"], 2); assert!(s.execute("effect.list", json!({"filter": "swap red"})).unwrap().as_array().unwrap().len() == 1); } @@ -88,6 +88,154 @@ fn build_example() -> Option { Some(target.join("wasm32-unknown-unknown/release/posterize_bands.wasm")) } +/// A host session whose settings and Plug-ins folder are both in `config`: installs must never +/// reach the real settings folder the host session points its Plug-ins folder at. +fn install_session(config: &std::path::Path) -> Session { + let mut s = effectcraft_host::session(); + s.config = Some(Arc::new(effectcraft_engine::config::DirConfig::new(config))); + s.plugins_dir = Some(config.join("Plug-ins")); + s +} + +#[test] +fn plugin_bundle_install_copies_wasm_and_installs_panels() { + let config = tempfile::TempDir::new().unwrap(); + let config = config.path(); + let plug_dir = config.join("Plug-ins"); + let bundle = tempfile::TempDir::new().unwrap(); + let bundle = bundle.path(); + std::fs::write(bundle.join("a-good.wasm"), wat_plugin("org.test.host.install", 1, "i32.const 0")).unwrap(); + std::fs::write(bundle.join("My Panel.jsx"), "1".as_bytes()).unwrap(); + std::fs::write(bundle.join("notes.txt"), b"not a plug-in").unwrap(); + + let mut s = install_session(config); + let r = s.execute("effect.plugins.install", json!({"folder": bundle.to_string_lossy()})).unwrap(); + assert_eq!(r["installed"].as_array().unwrap().len(), 1, "{r}"); + assert_eq!(r["errors"].as_array().unwrap().len(), 0, "{r}"); + assert_eq!(r["panels"].as_array().unwrap()[0], json!("My Panel.jsx"), "{r}"); + // The plug-in is on disk where the start-up scan looks, and the panel is listable like an + // installed one. + assert!(plug_dir.join("a-good.wasm").is_file(), "{r}"); + assert!(s.config.as_ref().unwrap().read("Scripts/ScriptUI Panels/My Panel.jsx").as_deref() == Some("1"), "{r}"); + // The effect is usable in this session already. + s.execute("comp.new", json!({"name": "I", "width": 40, "height": 20, "duration": 1})).unwrap(); + let l = s.execute("layer.newSolid", json!({"color": [1.0, 0.5, 0.0], "width": 40, "height": 20})).unwrap()["layer"].as_u64().unwrap(); + s.execute("effect.apply", json!({"layer": l, "effect": "Host org.test.host.install"})).unwrap(); + + // A load failure rolls the copy back: nothing left behind to fail every start. + let bad = tempfile::TempDir::new().unwrap(); + let bad = bad.path(); + std::fs::write(bad.join("b-garbage.wasm"), b"\0asm\x01\0\0\0garbage").unwrap(); + let r2 = s.execute("effect.plugins.install", json!({"folder": bad.to_string_lossy()})).unwrap(); + assert_eq!(r2["errors"].as_array().unwrap().len(), 1, "{r2}"); + assert!(r2["installed"].as_array().unwrap().is_empty(), "{r2}"); + assert!(!plug_dir.join("b-garbage.wasm").exists(), "{r2}"); + + // A single-file install: non-wasm paths are refused, and good files land on disk. + assert!(s.execute("effect.plugins.install", json!({"path": bundle.join("My Panel.jsx").to_string_lossy()})).is_err()); + // A single-file success: the file lands in the Plug-ins folder and is reported installed. + let fresh = bundle.join("fresh.wasm"); + std::fs::write(&fresh, wat_plugin("org.test.host.fresh", 1, "i32.const 0")).unwrap(); + let r3 = s.execute("effect.plugins.install", json!({"path": fresh.to_string_lossy()})).unwrap(); + assert_eq!(r3["installed"].as_array().unwrap()[0], json!("fresh.wasm"), "{r3}"); + assert!(plug_dir.join("fresh.wasm").is_file(), "{r3}"); + // Again from `config`: an already-registered (in-session) id fails as a duplicate. + let again = config.join("keep.wasm"); + std::fs::copy(bundle.join("a-good.wasm"), &again).unwrap(); + assert!(s.execute("effect.plugins.install", json!({"path": again.to_string_lossy()})).is_err()); +} + +#[test] +fn plugin_install_never_replaces_an_installed_plugin() { + let config = tempfile::TempDir::new().unwrap(); + let config = config.path(); + let plug_dir = config.join("Plug-ins"); + let src = tempfile::TempDir::new().unwrap(); + let src = src.path(); + std::fs::write(src.join("a-good.wasm"), wat_plugin("org.test.host.replace", 1, "i32.const 0")).unwrap(); + + let mut s = install_session(config); + s.execute("effect.plugins.install", json!({"path": src.join("a-good.wasm").to_string_lossy()})).unwrap(); + let installed = std::fs::read(plug_dir.join("a-good.wasm")).unwrap(); + + // Same file name, a different plug-in: installing over the existing one must be refused, + // not quietly swap it — and the first plug-in's bytes on disk stay exactly what they were. + let attack = tempfile::TempDir::new().unwrap(); + let attack = attack.path(); + std::fs::write(attack.join("a-good.wasm"), wat_plugin("org.test.host.hack", 1, "i32.const 0")).unwrap(); + let r = s.execute("effect.plugins.install", json!({"folder": attack.to_string_lossy()})).unwrap(); + assert!(r["installed"].as_array().unwrap().is_empty(), "{r}"); + let errs: Vec = r["errors"].as_array().unwrap().iter().map(|e| e["error"].as_str().unwrap().to_string()).collect(); + assert!(errs.iter().any(|e| e.contains("already installed")), "{r}"); + assert_eq!(std::fs::read(plug_dir.join("a-good.wasm")).unwrap(), installed); + + // The same through a single-file install: a command error, disk untouched. + assert!(s.execute("effect.plugins.install", json!({"path": attack.join("a-good.wasm").to_string_lossy()})).is_err()); + assert_eq!(std::fs::read(plug_dir.join("a-good.wasm")).unwrap(), installed); + + // A ScriptUI panel is code too: a bundle must not replace an installed one silently either. + let p1 = tempfile::TempDir::new().unwrap(); + std::fs::write(p1.path().join("My Panel.jsx"), "1").unwrap(); + s.execute("effect.plugins.install", json!({"folder": p1.path().to_string_lossy()})).unwrap(); + let p2 = tempfile::TempDir::new().unwrap(); + std::fs::write(p2.path().join("My Panel.jsx"), "2").unwrap(); + let r = s.execute("effect.plugins.install", json!({"folder": p2.path().to_string_lossy()})).unwrap(); + assert!(r["panels"].as_array().unwrap().is_empty(), "{r}"); + let errs: Vec = r["errors"].as_array().unwrap().iter().map(|e| e["error"].as_str().unwrap().to_string()).collect(); + assert!(errs.iter().any(|e| e.contains("already installed as a ScriptUI panel")), "{r}"); + assert_eq!(s.config.as_ref().unwrap().read("Scripts/ScriptUI Panels/My Panel.jsx").as_deref(), Some("1")); +} + +/// A single install of a hidden file is an error, not a success that installed nothing; a panel +/// whose name differs from an installed one only in case is refused (on Windows and macOS writing +/// it would replace that panel); an installed plug-in's source is its installed copy. +#[test] +fn plugin_install_refuses_hidden_files_and_case_variant_panels() { + let config = tempfile::TempDir::new().unwrap(); + let config = config.path(); + let src = tempfile::TempDir::new().unwrap(); + let src = src.path(); + let mut s = install_session(config); + + let hidden = src.join(".hidden.wasm"); + std::fs::write(&hidden, wat_plugin("org.test.host.hidden", 1, "i32.const 0")).unwrap(); + assert!(s.execute("effect.plugins.install", json!({"path": hidden.to_string_lossy()})).is_err()); + assert!(!config.join("Plug-ins").join(".hidden.wasm").exists()); + + let p1 = tempfile::TempDir::new().unwrap(); + std::fs::write(p1.path().join("Case Panel.jsx"), "1").unwrap(); + s.execute("effect.plugins.install", json!({"folder": p1.path().to_string_lossy()})).unwrap(); + let p2 = tempfile::TempDir::new().unwrap(); + std::fs::write(p2.path().join("case panel.jsx"), "2").unwrap(); + let r = s.execute("effect.plugins.install", json!({"folder": p2.path().to_string_lossy()})).unwrap(); + assert!(r["panels"].as_array().unwrap().is_empty(), "{r}"); + assert_eq!(r["errors"].as_array().unwrap().len(), 1, "{r}"); + assert_eq!(s.config.as_ref().unwrap().read("Scripts/ScriptUI Panels/Case Panel.jsx").as_deref(), Some("1")); + + let good = src.join("sourced.wasm"); + std::fs::write(&good, wat_plugin("org.test.host.sourced", 1, "i32.const 0")).unwrap(); + s.execute("effect.plugins.install", json!({"path": good.to_string_lossy()})).unwrap(); + let list = s.execute("effect.plugins.list", json!({})).unwrap(); + let entry = list["plugins"].as_array().unwrap().iter().find(|p| p["id"] == "org.test.host.sourced").cloned().unwrap(); + assert_eq!(entry["source"], json!(config.join("Plug-ins").join("sourced.wasm").to_string_lossy()), "{entry}"); +} + +/// A huge file is refused before it is read into memory and copied into the folder the app loads +/// at every start. +#[test] +fn plugin_install_refuses_an_oversized_file() { + let config = tempfile::TempDir::new().unwrap(); + let src = tempfile::TempDir::new().unwrap(); + let big = src.path().join("big.wasm"); + // Sparse on every platform we build for: no 64 MiB written, just the length. + std::fs::File::create(&big).unwrap().set_len((64 << 20) + 1).unwrap(); + + let mut s = install_session(config.path()); + let Err(e) = s.execute("effect.plugins.install", json!({"path": big.to_string_lossy()})) else { panic!("an oversized plug-in was installed") }; + assert!(e.to_string().contains("byte limit"), "{e}"); + assert!(!config.path().join("Plug-ins/big.wasm").exists(), "{e}"); +} + #[test] fn example_wasm_plugin_loads_and_renders() { let Some(wasm) = build_example() else { return }; diff --git a/crates/media/src/lib.rs b/crates/media/src/lib.rs index e985985b7..31478f111 100644 --- a/crates/media/src/lib.rs +++ b/crates/media/src/lib.rs @@ -29,6 +29,8 @@ mod stream; mod wav_markers; mod webp_anim; +/// Video decoder counters (time in decoders, samples decoded, seeks, cache hits), for benchmarks. +pub use filmcraft_codecs::{GopStats, gop_stats}; pub use pool::{DEFAULT_BUDGET, MediaPool, PoolStats}; pub use probe::{DEFAULT_SEQUENCE_RATE, probe, probe_bytes, probe_model}; diff --git a/crates/plugin/src/lib.rs b/crates/plugin/src/lib.rs index 5a838dc64..7c85e144b 100644 --- a/crates/plugin/src/lib.rs +++ b/crates/plugin/src/lib.rs @@ -1,6 +1,7 @@ //! # effectcraft-plugin (L3) //! -//! Loads **WebAssembly effect plug-ins** (plug-in API v1, see `docs/plugins.md`) into the effect +//! Loads **WebAssembly effect plug-ins** (plug-in ABI v1, see `docs/plugins.md`; the native +//! plug-in API 2 additions such as layer parameters and host services are not part of it) into the effect //! registry ([`effectcraft_effects::plugin`]). A plug-in is a `.wasm` (or `.wat`) module with //! no imports that exports: //! @@ -25,7 +26,7 @@ #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] use effectcraft_effects::EffectSpec; -pub use effectcraft_effects::plugin::{EffectPlugin, PLUGIN_API_VERSION, PluginFrame, PluginManifest, PluginParams, register_plugin}; +pub use effectcraft_effects::plugin::{EffectPlugin, PLUGIN_API_VERSION, PluginFrame, PluginManifest, PluginParams, WASM_API_VERSION, register_plugin}; #[cfg(feature = "wasm")] mod wasm; @@ -59,7 +60,7 @@ pub fn loader(bytes: &[u8], source: &str) -> Result { "name": spec.name, "category": spec.category, "version": m.as_ref().map(|m| m.version.clone()).unwrap_or_default(), - "api": PLUGIN_API_VERSION, + "api": m.as_ref().map(|m| m.api).unwrap_or(PLUGIN_API_VERSION), "params": spec.params.len(), })) } @@ -82,13 +83,17 @@ pub fn preview(bytes: &[u8], width: u32, height: u32, pixels: &[[f32; 4]], times // Points default to fractions of the layer: the frame is the layer. PluginParamKind::Point { default } => flat.extend([default[0] * f64::from(width), default[1] * f64::from(height)]), PluginParamKind::Color { default } => flat.extend(default.iter().copied()), + // API 2 kinds: a WebAssembly module can't declare them (`WasmPlugin::new` refuses them). + PluginParamKind::Point3 { default } => flat.extend(default.iter().copied()), + PluginParamKind::Layer => flat.push(-1.0), + PluginParamKind::Text { .. } | PluginParamKind::Hidden { .. } => {} } } - let params = PluginParams { manifest: m, flat: &flat }; + let params = PluginParams { manifest: m, flat: &flat, texts: &[] }; let mut frames = Vec::with_capacity(times.len()); for &t in times { let mut px = pixels.to_vec(); - let mut frame = PluginFrame { width, height, pixels: &mut px, scale: 1.0 }; + let mut frame = PluginFrame { width, height, pixels: &mut px, scale: 1.0, origin: [0.0; 2], layer_size: [f64::from(width), f64::from(height)] }; p.render(&mut frame, ¶ms, t)?; frames.push(px); } diff --git a/crates/plugin/src/wasm.rs b/crates/plugin/src/wasm.rs index 1f61c6b44..18a34548f 100644 --- a/crates/plugin/src/wasm.rs +++ b/crates/plugin/src/wasm.rs @@ -4,7 +4,7 @@ use std::sync::Mutex; use wasmi::{Config, Engine, Instance, Linker, Memory, Module, Store, StoreLimits, StoreLimitsBuilder, TypedFunc, WasmParams, WasmResults}; -use crate::{EffectPlugin, PLUGIN_API_VERSION, PluginFrame, PluginManifest, PluginParams}; +use crate::{EffectPlugin, PluginFrame, PluginManifest, PluginParams, WASM_API_VERSION}; /// Instruction budget per frame: a fixed allowance plus some per pixel. Generous for per-pixel /// work, but a runaway loop stops instead of hanging the render. @@ -74,8 +74,8 @@ impl WasmPlugin { let mut inst = instantiate(&engine, &module)?; let version = inst.func::<(), i32>("ec_api_version")?; let v = version.call(&mut inst.store, ()).map_err(err)?; - if v != PLUGIN_API_VERSION as i32 { - return Err(format!("the module implements plug-in API {v}, this build implements API {PLUGIN_API_VERSION}")); + if u32::try_from(v).ok() != Some(WASM_API_VERSION) { + return Err(format!("the module implements plug-in API {v}, this build implements API {WASM_API_VERSION} for WebAssembly")); } let ptr = inst.func::<(), i32>("ec_manifest_ptr")?.call(&mut inst.store, ()).map_err(err)?; let len = inst.func::<(), i32>("ec_manifest_len")?.call(&mut inst.store, ()).map_err(err)?; @@ -86,6 +86,9 @@ impl WasmPlugin { inst.memory.read(&inst.store, ptr as u32 as usize, &mut text).map_err(|e| format!("manifest out of bounds: {e}"))?; let manifest: PluginManifest = serde_json::from_slice(&text).map_err(|e| format!("bad manifest JSON: {e}"))?; manifest.validate()?; + if let Some(p) = manifest.params.iter().find(|p| p.kind.needs_v2()) { + return Err(format!("parameter `{}`: WebAssembly plug-ins use plug-in API 1 parameter types (no point3, layer, text or hidden)", p.id)); + } Ok(WasmPlugin { manifest, source: source.to_string(), engine, module, pool: Mutex::new(vec![inst]) }) } diff --git a/crates/project/src/comp.rs b/crates/project/src/comp.rs index 63edc65be..d8d7cda16 100644 --- a/crates/project/src/comp.rs +++ b/crates/project/src/comp.rs @@ -1,7 +1,7 @@ use super::item::LayerId; use super::layer::{Layer, LayerSource}; use super::settings::{Marker, yes}; -use super::{Deserialize, FrameRate, Node, PropGroup, Serialize, Tick, Value, essential, styles}; +use super::{Deserialize, FrameRate, Node, PropGroup, Serialize, Tick, essential, styles}; /// 3D renderer of a composition. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] @@ -284,20 +284,22 @@ fn timing_markers(m: &[Marker]) -> impl Iterator + '_ { } impl Comp { - /// Draws the same pixels as `o`: equal but for its layers' Audio, Lock and Shy switches, Hide - /// Shy Layers and the viewer's Resolution. + /// Draws the same pixels as `o`: equal but for its layers' Lock and Shy switches, Hide Shy + /// Layers, the viewer's Resolution, the work area (a preview and render range only) and, as + /// nothing draws their audio, its layers' Audio switches, Audio groups and audio effects. pub fn same_pixels(&self, o: &Comp) -> bool { - self.same_pixels_with(o, true) + self.same_pixels_with(o, true, &|_| false) } /// [`Self::same_pixels`], with the comp's and its layers' markers counting only when /// `with_markers`: only expressions read them, so while none does, adding or moving a marker /// (even during playback) leaves every rendered frame valid. Protected time markers with a - /// duration set the timing (Responsive Design), so those always count. - pub fn same_pixels_with(&self, o: &Comp, with_markers: bool) -> bool { + /// duration set the timing (Responsive Design), so those always count. A layer's audio counts + /// when `heard` says something draws it ([`crate::Project::heard_layers`]). + pub fn same_pixels_with(&self, o: &Comp, with_markers: bool, heard: &dyn Fn(LayerId) -> bool) -> bool { self.same_settings(o, with_markers) && self.layers.len() == o.layers.len() - && self.layers.iter().zip(&o.layers).all(|(a, b)| a.same_pixels_with(b, with_markers)) + && self.layers.iter().zip(&o.layers).all(|(a, b)| a.same_pixels_with(b, with_markers, heard(a.id))) } /// [`Self::same_pixels_with`] but for the layers. @@ -311,7 +313,7 @@ impl Comp { duration, display_start, background, - work_area, + work_area: _, layers: _, markers, shutter_angle, @@ -338,7 +340,6 @@ impl Comp { && *duration == o.duration && *display_start == o.display_start && *background == o.background - && *work_area == o.work_area && (if with_markers { *markers == o.markers } else { timing_markers(markers).eq(timing_markers(&o.markers)) }) && *shutter_angle == o.shutter_angle && *shutter_phase == o.shutter_phase @@ -356,54 +357,139 @@ impl Comp { && *essential == o.essential } - /// The comp-time span (`in_point..out_point` of every layer changed, added or removed, - /// widened by the shutter) outside which `self` draws the same pixels as `o`, so cached - /// frames there stay valid (#1641). `None` when the edit can reach any frame: a comp setting, - /// a layer reorder, Solo turning on or off, an expression (it reads anything), or a changed - /// layer another layer parents to, mattes with or picks in an effect. - pub fn changed_span(&self, o: &Comp) -> Option<(Tick, Tick)> { - fn reads(g: &PropGroup, changed: &[LayerId]) -> bool { + /// The comp-time spans where `self` may draw other pixels than `o` after an edit of some of + /// its layers (moved, trimmed, split, edited, added, removed): where the active layers differ + /// or a changed one draws differently, widened by the shutter. A trim reaches only the part it + /// cut, a split only the copy's half (#1641). A layer that reads a changed one (its parent, + /// track matte, a layer it picks in an effect or tracker) draws differently wherever it is + /// active, and so does a layer with an expression that may read other layers or items when anything + /// changed: here or, `elsewhere`, outside these layers (markers, layers that draw nothing, + /// other comps and items). `None` when the edit can reach any frame: a comp setting, a layer + /// reorder, Solo turning on or off, an environment layer. A layer's audio counts when `heard` + /// says something draws it, a layer's markers when `with_markers`. `touched` layers draw differently in spans the caller adds (a + /// precomp whose comp changed): they reach the layers that read them, not their own frames. + pub fn changed_spans( + &self, + o: &Comp, + with_markers: bool, + heard: &dyn Fn(LayerId) -> bool, + elsewhere: bool, + touched: &[LayerId], + ) -> Option> { + // An expression that may read outside its own layer: it names another comp, layer, + // footage, the project or its precomp's source, or its layer's `index` (a layer added or + // removed above shifts it). Its parent and the layers it picks in an effect are read + // through `referenced_layers` below. + // ponytail: text heuristic; misses another layer reached through a variable from a call + // without those words. Upgrade: record reads in the expression host. + fn reads_outside(g: &PropGroup) -> bool { g.children.iter().any(|c| match c { - Node::Prop(p) => { - p.expr.is_some() - || std::iter::once(&p.value) - .chain(p.keys.iter().map(|k| &k.value)) - .any(|v| matches!(v, Value::Layer(Some(id)) if changed.contains(&LayerId(*id)))) - } - Node::Group(g) => reads(g, changed), + Node::Prop(p) => p.expr.as_ref().is_some_and(|e| { + let t = e.text.to_lowercase().replace("thislayer", ""); + ["comp", "layer", "footage", "project", "source", "index"].iter().any(|w| t.contains(w)) + }), + Node::Group(g) => reads_outside(g), }) } - // Solo hides every other layer at every frame. - if !self.same_settings(o, true) || self.any_solo() != o.any_solo() { + fn any_expression(g: &PropGroup) -> bool { + g.children.iter().any(|c| match c { + Node::Prop(p) => p.expr.is_some(), + Node::Group(g) => any_expression(g), + }) + } + // Solo hides every other layer at every frame. (Comp markers count through `elsewhere`: + // an expression reading them names `thisComp`.) + if !self.same_settings(o, false) || self.any_solo() != o.any_solo() { return None; } let common = |a: &Comp, b: &Comp| a.layers.iter().map(|l| l.id).filter(|id| b.layer(*id).is_some()).collect::>(); if common(self, o) != common(o, self) { return None; } + // Changed, added or removed layers (each from both sides: its old and its new span), and + // those that draw the same apart from their span (a trim or split). Ids count: they seed + // randomness (noise, text randomisers), so a split's copy may draw differently. let mut changed = Vec::new(); - let mut span: Option<(Tick, Tick)> = None; - // Each changed layer comes up from both sides: its old and its new span. + let mut respanned = Vec::new(); + let mut keyed = Vec::new(); + let mut cuts = Vec::new(); for (l, other) in self.layers.iter().map(|l| (l, o.layer(l.id))).chain(o.layers.iter().map(|l| (l, self.layer(l.id)))) { - if other.is_some_and(|b| l.same_pixels(b)) { + if other.is_some_and(|b| l.same_pixels_with(b, with_markers, heard(l.id))) { continue; } + if l.environment || l.environment_background { + return None; + } + // (an expression on the layer can read its own in and out points and name, `inPoint`, + // `thisLayer.name`: then a trim or rename reaches all of it) + if let Some(b) = other + && !any_expression(&l.props) + && !any_expression(&b.props) + && (Layer { in_point: b.in_point, out_point: b.out_point, name: b.name.clone(), label: b.label, comment: b.comment.clone(), ..l.clone() }) + .same_pixels_with(b, with_markers, heard(l.id)) + { + respanned.push(l.id); + } else if let Some(b) = other + && let Some(spans) = l.keyframe_spans(b, with_markers, heard(l.id)) + { + // (drawn the same outside the keys it changed) + respanned.push(l.id); + keyed.extend(spans); + } changed.push(l.id); - let (s, e) = (l.in_point.min(l.out_point), l.in_point.max(l.out_point)); - span = Some(span.map_or((s, e), |(a, b)| (a.min(s), b.max(e)))); + cuts.extend([l.in_point, l.out_point]); } - let reaches = |l: &Layer| { - l.parent.is_some_and(|p| changed.contains(&p)) || l.track_matte.as_ref().is_some_and(|m| changed.contains(&m.layer)) || reads(&l.props, &changed) - }; - if self.layers.iter().chain(&o.layers).any(reaches) { - return None; + // The layers the change reaches besides: expressions, then whatever reads a reached layer. + let both = || self.layers.iter().chain(&o.layers); + let mut reached: Vec = Vec::new(); + if !changed.is_empty() || elsewhere || !touched.is_empty() { + reached.extend(both().filter(|l| reads_outside(&l.props)).map(|l| l.id)); } - // A frame at `t` samples cameras, lights and transforms over its shutter, - // `t + (phase + angle * 0..=1) / 360` frames, and draws the layers active at `t` itself. + // (each pass adds a layer or stops: at most one pass per layer) + loop { + let read = |l: &&Layer| { + !reached.contains(&l.id) + && l.referenced_layers().iter().any(|r| *r != l.id && (changed.contains(r) || reached.contains(r) || touched.contains(r))) + }; + let more: Vec = both().filter(read).map(|l| l.id).collect(); + if more.is_empty() { + break; + } + reached.extend(more); + } + reached.sort_unstable(); + reached.dedup(); + for l in both().filter(|l| reached.contains(&l.id)) { + if l.environment || l.environment_background { + return None; + } + cuts.extend([l.in_point, l.out_point]); + } + let (before, after) = self.shutter_margin(); + // Between two cuts the same layers are active; outside them no changed layer is. + cuts.sort_unstable(); + cuts.dedup(); + let draws_same = |l: &Layer| (!changed.contains(&l.id) || respanned.contains(&l.id)) && !reached.contains(&l.id); + let spans = cuts + .windows(2) + .filter_map(|w| { + let (&s, &e) = (w.first()?, w.last()?); + let active = |c: &Comp| c.layers.iter().filter(|l| l.is_active_at(s)).map(|l| l.id).collect::>(); + let same = active(self) == active(o) && self.layers.iter().filter(|l| l.is_active_at(s)).all(draws_same); + (!same).then(|| (s - before, e + after)) + }) + .chain(keyed.into_iter().map(|(s, e)| (s - before, e + after))) + .collect(); + Some(spans) + } + + /// How far a change at a time reaches the frames before and after it through the shutter: a + /// frame at `t` samples cameras, lights and transforms over its shutter, + /// `t + (phase + angle * 0..=1) / 360` frames, and draws the layers active at `t` itself. + pub fn shutter_margin(&self) -> (Tick, Tick) { let (a, p) = (self.shutter_angle / 360.0, self.shutter_phase / 360.0); let fd = self.frame_duration().seconds(); - let (before, after) = (Tick::from_seconds_f64(p.max(p + a).max(0.0) * fd), Tick::from_seconds_f64(-p.min(p + a).min(0.0) * fd)); - span.map(|(s, e)| (s - before, e + after)) + (Tick::from_seconds_f64(p.max(p + a).max(0.0) * fd), Tick::from_seconds_f64(-p.min(p + a).min(0.0) * fd)) } /// Some layer is soloed: only the soloed layers draw. @@ -411,18 +497,21 @@ impl Comp { self.layers.iter().any(|l| l.switches.solo && l.source.is_av()) } - /// The comp with those switches and the Resolution at their defaults (borrowed when they - /// already are), for keys that hash the whole comp. + /// The comp with those switches, the Resolution and the work area at their defaults (borrowed + /// when they already are), for keys that hash the whole comp. pub fn pixel_form(&self) -> std::borrow::Cow<'_, Comp> { - self.pixel_form_with(true) + self.pixel_form_with(true, &|_| false) } /// [`Self::pixel_form`], also without the comp's plain markers and its layers' markers when not - /// `with_markers` (see [`Self::same_pixels_with`] and [`Project::reads_markers`]). - pub fn pixel_form_with(&self, with_markers: bool) -> std::borrow::Cow<'_, Comp> { + /// `with_markers` (see [`Self::same_pixels_with`] and [`Project::reads_markers`]), and with the + /// audio of the layers `heard` doesn't name at its defaults ([`Layer::picture_form`]). + pub fn pixel_form_with(&self, with_markers: bool, heard: &dyn Fn(LayerId) -> bool) -> std::borrow::Cow<'_, Comp> { + let pictures: Vec> = self.layers.iter().map(|l| l.picture_form(heard(l.id))).collect(); if !self.hide_shy && self.resolution.is_auto() - && self.layers.iter().all(|l| l.switches == l.switches.pixels()) + && self.work_area == (Tick::ZERO, Tick::ZERO) + && pictures.iter().all(Option::is_none) && (with_markers || (self.markers.iter().all(is_timing_marker) && self.layers.iter().all(|l| l.markers.is_empty()))) { return std::borrow::Cow::Borrowed(self); @@ -430,11 +519,14 @@ impl Comp { let mut c = self.clone(); c.hide_shy = false; c.resolution = Resolution::Auto; + c.work_area = (Tick::ZERO, Tick::ZERO); if !with_markers { c.markers.retain(is_timing_marker); } - for l in &mut c.layers { - l.switches = l.switches.pixels(); + for (l, picture) in c.layers.iter_mut().zip(pictures) { + if let Some(p) = picture { + *l = p; + } if !with_markers { l.markers.clear(); } diff --git a/crates/project/src/dirty.rs b/crates/project/src/dirty.rs new file mode 100644 index 000000000..a83ca35d9 --- /dev/null +++ b/crates/project/src/dirty.rs @@ -0,0 +1,112 @@ +//! Which frames of a comp an edit can change: frame caches keep the others. + +use super::comp::Comp; +use super::item::{Heard, ItemId}; +use super::layer::{Layer, LayerSource, reads_other_frames}; +use super::{Arc, GroupKind, Node, Project, Tick}; + +/// Precomps nested deeper than this count as changed wherever they show. +const MAX_DEPTH: usize = 32; + +type Seen = std::collections::HashMap>>; + +impl Project { + /// The comp-time spans of `comp` where `self` may draw other pixels than `old` (the project + /// before an edit): [`Comp::changed_spans`] of its layers, and for a precomp whose comp + /// changed, the spans that comp changed in, where the precomp shows them (#1641). Layers that + /// can't change the picture ([`Project::silent_layers`]) don't count, nor does the audio of + /// layers no effect draws it from (`heard`, best of both states). Expressions can read + /// anything, so any change (`outside` these comps too: another comp, an item's name) reaches + /// the layers that have one. `None` when the edit can reach any frame. + pub fn dirty_spans(&self, old: &Project, comp: ItemId, heard: &Heard, outside: bool) -> Option> { + self.dirty_spans_at(old, comp, heard, outside, 0, &mut std::collections::HashMap::new()) + } + + /// `seen`: the spans of the comps already walked in this call, so a precomp shared by many + /// layers is walked once. + fn dirty_spans_at(&self, old: &Project, comp: ItemId, heard: &Heard, outside: bool, depth: usize, seen: &mut Seen) -> Option> { + if depth > MAX_DEPTH { + return None; + } + if let Some(spans) = seen.get(&comp) { + return spans.clone(); + } + let spans = self.comp_dirty_spans(old, comp, heard, outside, depth, seen); + seen.insert(comp, spans.clone()); + spans + } + + fn comp_dirty_spans(&self, old: &Project, comp: ItemId, heard: &Heard, outside: bool, depth: usize, seen: &mut Seen) -> Option> { + let (new_c, old_c) = (self.comp_arc(comp)?, old.comp_arc(comp)?); + // Expressions may read what changed elsewhere, even from comps that didn't change. + let reads_elsewhere = heard.expressions && outside; + let quiet = self.silent_layers(comp, &new_c, heard); + // An edit of this comp is outside the comps it nests: their expressions may read it + // (`comp("Main").layer("Controls")`, at their own time). + let nested_outside = outside || !Arc::ptr_eq(&new_c, &old_c); + // Precomps (drawing the same comp before and after) whose comp changed: where they show it. + let mut touched = Vec::new(); + let mut nested = Vec::new(); + for l in new_c.layers.iter().filter(|l| !quiet.contains(&l.id)) { + let LayerSource::Comp { item } = l.source else { continue }; + if old_c.layer(l.id).is_none_or(|before| before.source != l.source) { + continue; + } + // (an unchanged `Arc` says nothing of the comps nested in it: walk it anyway) + let (Some(n), Some(_)) = (self.comp_arc(item), old.comp_arc(item)) else { continue }; + let inner = self.dirty_spans_at(old, item, heard, nested_outside, depth + 1, seen); + if inner.as_ref().is_some_and(Vec::is_empty) { + continue; + } + touched.push(l.id); + nested.extend(shown(l, &new_c, &n, inner)); + } + if Arc::ptr_eq(&new_c, &old_c) && touched.is_empty() && !reads_elsewhere { + return Some(vec![]); + } + let picture = |p: &Project, c: &Comp| { + let mut c = c.clone(); + let quiet = p.silent_layers(comp, &c, heard); + c.layers.retain(|l| !quiet.contains(&l.id)); + c + }; + let (a, b) = (picture(old, &old_c), picture(self, &new_c)); + let elsewhere = heard.expressions && (outside || !new_c.same_pixels_with(&old_c, true, &|l| heard.contains(comp, l)) || !touched.is_empty()); + let mut spans = b.changed_spans(&a, heard.expressions, &|l| heard.contains(comp, l), elsewhere, &touched)?; + spans.extend(nested); + Some(spans) + } +} + +/// The comp-time spans of `outer` where its precomp layer `l` shows the spans `inner` of its comp +/// `n` (`None`: all of it), widened by `outer`'s shutter. A time remap, Responsive Design +/// timing or an effect reading the layer at other times (Echo, Posterize Time…: [`reads_other_frames`], +/// or an effect picking the layer itself) can show any nested time anywhere: then the layer's +/// whole span. +fn shown(l: &Layer, outer: &Comp, n: &Comp, inner: Option>) -> Vec<(Tick, Tick)> { + let (before, after) = outer.shutter_margin(); + let whole = vec![(l.in_point - before, l.out_point + after)]; + let responsive = (l.stretch - 100.0).abs() > 1e-9 && n.markers.iter().any(|m| m.protected && m.duration > Tick::ZERO); + let Some(inner) = inner else { return whole }; + if l.props.get("timeRemap").is_some() || responsive || reads_itself_elsewhen(l) { + return whole; + } + // (a nested frame either side: Preserve Frame Rate holds nested frames, Frame Blending mixes + // neighbours) + let fd = n.frame_duration(); + inner + .into_iter() + .filter_map(|(s, e)| { + let (x, y) = (l.comp_time(s - fd), l.comp_time(e + fd)); + let (lo, hi) = (x.min(y).max(l.in_point), x.max(y).min(l.out_point)); + (lo <= hi).then(|| (lo - before, hi + after)) + }) + .collect() +} + +/// Some effect of `l` reads the layer at other times ([`reads_other_frames`]) or picks the layer +/// itself (a layer parameter set to its own layer). +fn reads_itself_elsewhen(l: &Layer) -> bool { + let time_fx = |c: &Node| matches!(c, Node::Group(f) if matches!(&f.kind, GroupKind::Effect { effect } if reads_other_frames(effect))); + l.picked_layers().contains(&l.id) || l.props.group("effects").is_some_and(|e| e.children.iter().any(time_fx)) +} diff --git a/crates/project/src/item.rs b/crates/project/src/item.rs index cc55397ec..cb43de143 100644 --- a/crates/project/src/item.rs +++ b/crates/project/src/item.rs @@ -131,6 +131,78 @@ impl Project { self.comps().any(|(_, c)| c.layers.iter().any(|l| any_expr(&l.props))) } + /// The layers whose audio something draws ([`Heard`]): those a layer picks in an effect (an + /// Audio Spectrum or Audio Waveform's Audio Layer, even on itself), and the layers inside a + /// picked precomp, whose audio it mixes. With an expression anywhere (it can read any layer's + /// Audio Levels) every layer may be heard ([`Heard::contains`]). + pub fn heard_layers(&self) -> Heard { + let mut layers = std::collections::BTreeSet::new(); + let mut stack: Vec<(ItemId, LayerId)> = + self.comps().flat_map(|(id, c)| c.layers.iter().flat_map(move |l| l.picked_layers().into_iter().map(move |r| (*id, r)))).collect(); + // (bounded: each (comp, layer) is visited once) + while let Some((cid, lid)) = stack.pop() { + if !layers.insert((cid, lid)) { + continue; + } + let inner = self.comp(cid).and_then(|c| c.layer(lid)).and_then(|l| match l.source { + LayerSource::Comp { item } => Some(item), + _ => None, + }); + if let Some(inner) = inner + && let Some(c) = self.comp(inner) + { + stack.extend(c.layers.iter().map(|l| (inner, l.id))); + } + } + Heard { expressions: self.reads_markers(), audio_exprs: self.reads_audio(), layers } + } + + /// Some expression may read a layer's audio: its text names it (`audio`, `"Audio Levels"`, + /// `"ADBE Audio Group"`) or walks properties by index (`layer(1)(3)`). + // ponytail: text heuristic; misses an index walk through a variable (`g = layer(1); g(3)`). + // Upgrade: record audio reads in the expression host. + fn reads_audio(&self) -> bool { + fn reads(text: &str) -> bool { + let t: String = text.chars().filter(|c| !c.is_whitespace()).collect::().to_lowercase(); + t.contains("audio") || t.match_indices(")(").any(|(i, _)| t.get(i + 2..).and_then(|r| r.chars().next()).is_some_and(|c| c.is_ascii_digit())) + } + fn any(g: &crate::PropGroup) -> bool { + g.children.iter().any(|c| match c { + crate::Node::Prop(p) => p.expr.as_ref().is_some_and(|e| reads(&e.text)), + crate::Node::Group(sub) => any(sub), + }) + } + self.comps().any(|(_, c)| c.layers.iter().any(|l| any(&l.props))) + } + + /// Layers of `comp` (item `id`) that can't change its pixels: footage layers of files without + /// video (music, voice-over) and layers with the Video switch off, unless another layer reads + /// them (a hidden track matte) or something draws their audio ([`Self::heard_layers`]: the + /// Audio Spectrum's Audio Layer, also from a comp that nests this one). Cameras, lights, + /// nulls, environment layers and soloed layers (Solo hides every other layer) are never + /// silent. Expressions, which can read any layer, are the caller's concern. + pub fn silent_layers(&self, id: ItemId, comp: &Comp, heard: &Heard) -> std::collections::BTreeSet { + let read: std::collections::BTreeSet = + comp.layers.iter().flat_map(|l| l.referenced_layers().into_iter().filter(move |r| *r != l.id)).collect(); + comp.layers + .iter() + .filter(|l| { + let no_picture = match &l.source { + LayerSource::Footage { item } => matches!(self.item(*item).map(|i| &i.kind), Some(ItemKind::Footage(f)) if !f.has_video), + _ => false, + }; + // (environment layers light Advanced 3D scenes without being drawn) + (no_picture || l.source.is_av() && !l.switches.video) + && !l.switches.solo + && !l.environment + && !l.environment_background + && !read.contains(&l.id) + && !heard.picked(id, l.id) + }) + .map(|l| l.id) + .collect() + } + /// Allocate an id (items, layers and property uids share one counter). pub fn alloc(&mut self) -> u64 { let id = self.next_id; @@ -324,3 +396,36 @@ impl Project { Ok((p, notes)) } } + +/// The layers whose audio something draws ([`Project::heard_layers`]): their Audio switch, Audio +/// group and audio effects change pixels. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct Heard { + /// Some layer has an expression (it can read anything). + pub expressions: bool, + /// Some expression may read a layer's audio ([`Project::reads_audio`]). + audio_exprs: bool, + layers: std::collections::BTreeSet<(ItemId, LayerId)>, +} + +impl Heard { + /// Something may draw the audio of layer `layer` of comp `comp`: a layer picks it + /// ([`Self::picked`]) or an expression may read it. + pub fn contains(&self, comp: ItemId, layer: LayerId) -> bool { + self.audio_exprs || self.picked(comp, layer) + } + + /// A layer picks layer `layer` of comp `comp` in an effect (or a precomp of it), so draws its + /// audio. Expressions aside: they draw in their own layers. + pub fn picked(&self, comp: ItemId, layer: LayerId) -> bool { + self.layers.contains(&(comp, layer)) + } + + /// Heard in `self` or in `o` (the states before and after an edit). + pub fn union(mut self, o: &Heard) -> Heard { + self.expressions |= o.expressions; + self.audio_exprs |= o.audio_exprs; + self.layers.extend(o.layers.iter().copied()); + self + } +} diff --git a/crates/project/src/layer.rs b/crates/project/src/layer.rs index 3d7676b28..34eeb50c0 100644 --- a/crates/project/src/layer.rs +++ b/crates/project/src/layer.rs @@ -1,6 +1,6 @@ use super::item::{ItemId, LayerId}; use super::settings::Marker; -use super::{BlendMode, Deserialize, Label, ParamUi, PropGroup, Property, Serialize, Tick}; +use super::{BlendMode, Deserialize, GroupKind, Label, Node, ParamUi, PropGroup, Property, Serialize, Tick, Value}; #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] pub enum LightKind { @@ -182,25 +182,119 @@ impl Default for Switches { } } -// What a frame's pixels depend on. Caches key on it: the Audio, Lock and Shy switches and Hide -// Shy Layers don't change pixels, so toggling them keeps cached frames (#103). +// What a frame's pixels depend on. Caches key on it: the Lock and Shy switches and Hide Shy +// Layers never change pixels, and the Audio switch, the Audio group and audio effects change them +// only when something draws the layer's audio (a waveform or spectrum picks it, see +// [`crate::Project::heard_layers`]), so editing those keeps cached frames (#103). + +/// Is `id` an audio effect (applied by the audio mix rather than the compositor)? +pub fn is_audio_effect(id: &str) -> bool { + id.starts_with("ec.audio.") +} + +/// Does effect `id` read its own layer at other times (`EffectHost::self_at`, `layer_at` on +/// itself): the time effects (Echo, Posterize Time, Timewarp…), simulations, temporal smoothing +/// and analysis (Remove Grain, Auto Levels, Color Stabilizer's reference frame, Warp Stabilizer, +/// Roto Brush…) and Clone strokes with a time offset? On a precomp layer, a change at one nested +/// time then reaches frames of the layer far from where it shows. Plug-in effects (every id +/// outside `ec.`) count too: plug-in API 2 reads the input at other times (`PluginHost`). +// ponytail: hand-kept list of the effects that call `self_at`; a new one must be added here. +pub fn reads_other_frames(id: &str) -> bool { + !id.starts_with("ec.") || id.starts_with("ec.time.") || id.starts_with("ec.sim.") || READS_OTHER_FRAMES.contains(&id) +} + +/// The effects outside `ec.time.` and `ec.sim.` that read their own layer at other times. +const READS_OTHER_FRAMES: &[&str] = &[ + "ec.noise.removegrain", + "ec.color.autolevels", + "ec.color.autocontrast", + "ec.color.autocolor", + "ec.color.shadowhighlight", + "ec.color.colorstabilizer", + "ec.matte.refinesoft", + "ec.matte.refinehard", + "ec.matte.rotobrush", + "ec.matte.objectmatte", + "ec.blur.camerashakedeblur", + "ec.distort.rollingshutterrepair", + "ec.distort.warpstabilizer", + "ec.key.ccsimplewireremoval", + "ec.paint.paint", +]; impl Switches { /// These switches with Audio, Lock and Shy at their defaults. pub fn pixels(&self) -> Switches { Switches { audio: true, locked: false, shy: false, ..self.clone() } } + + /// The switches that change the pixels of a layer: [`Self::pixels`], keeping Audio when + /// something draws the layer's audio (`heard`). + pub fn picture(&self, heard: bool) -> Switches { + Switches { audio: if heard { self.audio } else { true }, ..self.pixels() } + } +} + +/// A node only the audio mix reads: the layer's Audio group, an audio effect. +fn audio_only(n: &Node, root: bool) -> bool { + match n { + Node::Group(g) => (root && g.match_id == "audio") || matches!(&g.kind, GroupKind::Effect { effect } if is_audio_effect(effect)), + Node::Prop(_) => false, + } +} + +/// `a` and `b` are equal but for their audio-only nodes ([`audio_only`]: the root's Audio group, +/// the effects in its Effects group that are audio effects). +fn same_picture_props(a: &PropGroup, b: &PropGroup) -> bool { + fn head(a: &PropGroup, b: &PropGroup) -> bool { + // Every field (no `..`: a new field must be considered here). + let PropGroup { uid, match_id, name, kind, enabled, label, children: _ } = a; + *uid == b.uid && *match_id == b.match_id && *name == b.name && *kind == b.kind && *enabled == b.enabled && *label == b.label + } + fn children(a: &PropGroup, b: &PropGroup, root: bool) -> bool { + let mut x = a.children.iter().filter(|n| !audio_only(n, root)); + let mut y = b.children.iter().filter(|n| !audio_only(n, root)); + loop { + match (x.next(), y.next()) { + (None, None) => return true, + (Some(Node::Group(g)), Some(Node::Group(h))) if root && g.match_id == "effects" && h.match_id == "effects" => { + if !(head(g, h) && children(g, h, false)) { + return false; + } + } + (Some(m), Some(n)) if m == n => {} + _ => return false, + } + } + } + head(a, b) && children(a, b, true) +} + +/// `g` (a layer's property tree) without its audio-only nodes ([`same_picture_props`]). +fn picture_props(g: &PropGroup) -> PropGroup { + let mut g = g.clone(); + g.children.retain(|n| !audio_only(n, true)); + for n in &mut g.children { + if let Node::Group(e) = n + && e.match_id == "effects" + { + e.children.retain(|n| !audio_only(n, false)); + } + } + g } impl Layer { - /// Draws the same pixels as `o`: equal but for the Audio, Lock and Shy switches. + /// Draws the same pixels as `o`: equal but for the Audio, Lock and Shy switches and what only + /// the audio mix reads (nothing draws its audio). pub fn same_pixels(&self, o: &Layer) -> bool { - self.same_pixels_with(o, true) + self.same_pixels_with(o, true, false) } /// [`Self::same_pixels`], with the markers counting only when `with_markers` (they are read by - /// expressions alone, so a project without any can edit them freely). - pub fn same_pixels_with(&self, o: &Layer, with_markers: bool) -> bool { + /// expressions alone, so a project without any can edit them freely) and the audio (the Audio + /// switch, the Audio group, audio effects) only when `heard`: something draws it. + pub fn same_pixels_with(&self, o: &Layer, with_markers: bool, heard: bool) -> bool { // Every field (no `..`: a new field must be considered here); the property tree last. let Layer { id, @@ -233,7 +327,7 @@ impl Layer { && *in_point == o.in_point && *out_point == o.out_point && *stretch == o.stretch - && switches.pixels() == o.switches.pixels() + && switches.picture(heard) == o.switches.picture(heard) && *blend_mode == o.blend_mode && *preserve_transparency == o.preserve_transparency && *track_matte == o.track_matte @@ -243,7 +337,100 @@ impl Layer { && *auto_orient == o.auto_orient && *environment == o.environment && *environment_background == o.environment_background - && *props == o.props + && (if heard { *props == o.props } else { same_picture_props(props, &o.props) }) + } + + /// The comp-time spans where `self` draws differently than `o` when the two differ only in + /// the keyframes of Transform properties (a key's value, ease or interpolation moved in the + /// Timeline or Graph Editor): from two keys before the first changed key to two after the + /// last (auto-Bezier and continuous tangents reach the neighbouring segments), to the + /// layer's in or out point past the first or last key (values hold there). `None` for any + /// other difference, a static value, an expression on the layer (it can read the keyed + /// property at other times), roving or muted keys, a time remap or a time effect (it reads other + /// times): those reach the layer's whole span. + pub fn keyframe_spans(&self, o: &Layer, with_markers: bool, heard: bool) -> Option> { + let transform = |l: &Layer| l.props.group("transform").cloned(); + let (t1, t0) = (transform(self)?, transform(o)?); + let reads_time = |l: &Layer| { + l.props.get("timeRemap").is_some() + || l.props.group("effects").is_some_and(|e| { + e.children.iter().any(|n| matches!(n, Node::Group(f) if matches!(&f.kind, GroupKind::Effect { effect } if reads_other_frames(effect)))) + }) + }; + let has_expr = |l: &Layer| { + let mut any = false; + l.props.walk("", &mut |_, p| any |= p.has_expression()); + any + }; + if reads_time(self) || reads_time(o) || has_expr(self) || has_expr(o) { + return None; + } + // Everything but the Transform group draws the same. + let rest = |l: &Layer| { + let mut l = l.clone(); + l.props.children.retain(|n| !matches!(n, Node::Group(g) if g.match_id == "transform")); + l + }; + let head = |g: &PropGroup| PropGroup { children: vec![], ..g.clone() }; + if !rest(self).same_pixels_with(&rest(o), with_markers, heard) || head(&t1) != head(&t0) || t1.children.len() != t0.children.len() { + return None; + } + let mut spans = Vec::new(); + for (n1, n0) in t1.children.iter().zip(&t0.children) { + if n1 == n0 { + continue; + } + let (Node::Prop(p1), Node::Prop(p0)) = (n1, n0) else { return None }; + let bare = |p: &Property| Property { keys: vec![], value: Value::Scalar(0.0), ..p.clone() }; + // (muted keys are skipped: the keys around an edit that play can be any distance away) + if bare(p1) != bare(p0) || p1.keys.is_empty() || p0.keys.is_empty() || p1.keys.iter().chain(&p0.keys).any(|k| k.roving || k.muted) { + return None; + } + // Keys only one side has as they are (added, removed, edited, moved). + let changed: Vec = + p1.keys.iter().filter(|k| !p0.keys.contains(k)).chain(p0.keys.iter().filter(|k| !p1.keys.contains(k))).map(|k| k.time).collect(); + let (Some(first), Some(last)) = (changed.iter().min(), changed.iter().max()) else { continue }; + // Two keys before the first changed one and two after the last, on either side (the + // first or last key when there are fewer: values hold beyond it). Past the first or + // last key the change reaches the in or out point (`None`). + let mut lo = Some(Tick(i64::MAX)); + let mut hi = Some(Tick(i64::MIN)); + for keys in [&p1.keys, &p0.keys] { + let before = keys.iter().filter(|k| k.time < *first).count(); + let back = if before == 0 { None } else { keys.get(before.saturating_sub(2)) }; + lo = lo.zip(back).map(|(a, k)| a.min(k.time)); + let upto = keys.iter().filter(|k| k.time <= *last).count(); + let ahead = if upto >= keys.len() { None } else { keys.get(upto.saturating_add(1).min(keys.len().saturating_sub(1))) }; + hi = hi.zip(ahead).map(|(a, k)| a.max(k.time)); + } + let (x, y) = (lo.map_or(self.in_point, |t| self.comp_time(t)), hi.map_or(self.out_point, |t| self.comp_time(t))); + let (x, y) = (x.min(y).max(self.in_point), x.max(y).min(self.out_point)); + if x <= y { + spans.push((x, y)); + } + } + Some(spans) + } + + /// This layer with what doesn't change its pixels at its defaults (see + /// [`Self::same_pixels_with`]), for keys that hash it; `None` when it already is. + pub fn picture_form(&self, heard: bool) -> Option { + let switches = self.switches.picture(heard); + let audio = !heard + && (self.props.children.iter().any(|n| audio_only(n, true)) + || self + .props + .children + .iter() + .any(|n| matches!(n, Node::Group(e) if e.match_id == "effects" && e.children.iter().any(|n| audio_only(n, false))))); + if switches == self.switches && !audio { + return None; + } + let mut l = Layer { switches, ..self.clone() }; + if audio { + l.props = picture_props(&self.props); + } + Some(l) } } @@ -326,6 +513,44 @@ pub struct Layer { } impl Layer { + /// The layers this layer reads: its parent, track matte, layer parameters (Displacement Map, + /// Audio Spectrum's Audio Layer…) and tracker targets. + pub fn referenced_layers(&self) -> Vec { + fn walk(g: &PropGroup, depth: usize, out: &mut Vec) { + if depth > 64 { + return; + } + if let GroupKind::Tracker { settings } = &g.kind + && let Some(t) = settings.target + { + out.push(t); + } + for c in &g.children { + match c { + Node::Prop(p) => { + for v in std::iter::once(&p.value).chain(p.keys.iter().map(|k| &k.value)) { + if let Value::Layer(Some(id)) = v { + out.push(LayerId(*id)); + } + } + } + Node::Group(sub) => walk(sub, depth + 1, out), + } + } + } + let mut out: Vec = self.parent.iter().copied().chain(self.track_matte.iter().map(|m| m.layer)).collect(); + walk(&self.props, 0, &mut out); + out + } + + /// Layers this one picks in an effect or tracker (an Audio Spectrum's Audio Layer, a + /// Displacement Map's map layer…), itself included: [`Self::referenced_layers`] without the + /// parent and the track matte. + pub fn picked_layers(&self) -> Vec { + let skip = usize::from(self.parent.is_some()) + usize::from(self.track_matte.is_some()); + self.referenced_layers().into_iter().skip(skip).collect() + } + pub fn is_active_at(&self, t: Tick) -> bool { t >= self.in_point && t < self.out_point } diff --git a/crates/project/src/lib.rs b/crates/project/src/lib.rs index 67b846cd6..079b81765 100644 --- a/crates/project/src/lib.rs +++ b/crates/project/src/lib.rs @@ -30,6 +30,7 @@ pub use effectcraft_keyframe as keyframe; pub use effectcraft_keyframe::{Keyframe, Value}; mod comp; +mod dirty; mod footage; mod item; mod layer; @@ -39,8 +40,11 @@ mod settings; pub use comp::{Comp, Guide, Renderer, Resolution}; pub use footage::{AlphaMode, CineonSettings, CineonUnits, FieldOrder, Footage, FootageKind, Proxy, Solid, SourceLayer, VIDEO_EXTENSIONS, cineon_log_to_lin}; pub use item::NO_LAYER; -pub use item::{Item, ItemId, ItemKind, LayerId}; -pub use layer::{AutoOrient, FrameBlend, Layer, LayerSource, LightKind, MatteKind, PrimitiveKind, Quality, Sampling, Switches, TrackMatte}; +pub use item::{Heard, Item, ItemId, ItemKind, LayerId}; +pub use layer::{ + AutoOrient, FrameBlend, Layer, LayerSource, LightKind, MatteKind, PrimitiveKind, Quality, Sampling, Switches, TrackMatte, is_audio_effect, + reads_other_frames, +}; pub use layers::Layers; pub use settings::{ APP_VERSION, BitDepth, ColorEngine, CuePoint, HdrMode, Marker, ProjectError, ProjectSettings, SCHEMA_EFFECT_SPACE, SCHEMA_VERSION, SavedView, diff --git a/crates/project/src/tests.rs b/crates/project/src/tests.rs index ed0d05588..bb6def2c9 100644 --- a/crates/project/src/tests.rs +++ b/crates/project/src/tests.rs @@ -410,3 +410,84 @@ fn upgrading_deeply_nested_shape_groups_fits_a_small_stack() { let ids: Vec<&str> = g.children.iter().map(|c| c.match_id()).collect(); assert_eq!(ids, ["size", "detail", "points", "speed", "correlation", "phase", "spatialPhase", "seed"]); } + +/// Music in a precomp is heard, and so not silent, when a layer of a comp nesting it picks the +/// precomp layer (an Audio Waveform drawing the precomp's mix). +#[test] +fn music_in_a_precomp_a_waveform_picks_is_not_silent() { + use crate::{Footage, FootageKind, GroupKind}; + let (mut p, inner) = project_with_layer(); + let song = p.add_item( + "song.wav", + effectcraft_color::Label::None, + None, + ItemKind::Footage(Footage { kind: FootageKind::Audio, has_video: false, has_audio: true, ..Default::default() }), + ); + let comp = p.comp(inner).unwrap().clone(); + let music = build::layer(&mut p, &comp, "Music", LayerSource::Footage { item: song }, (1920, 1080), None); + let music_id = music.id; + p.comp_mut(inner).unwrap().layers.push(music); + let outer = p.add_item("Outer", effectcraft_color::Label::None, None, ItemKind::Comp(comp.clone().into())); + let pre = build::layer(&mut p, &comp, "Pre", LayerSource::Comp { item: inner }, (1920, 1080), None); + let pre_id = pre.id; + p.comp_mut(outer).unwrap().layers.push(pre); + let silent = |p: &Project| p.silent_layers(inner, p.comp(inner).unwrap(), &p.heard_layers()).contains(&music_id); + assert!(silent(&p), "nothing draws the precomp's audio"); + let mut n = 1_000_000; + let mut ids = build::Ids(&mut n); + let mut g = ids.group("waveform", "Audio Waveform"); + g.kind = GroupKind::Effect { effect: "ec.generate.audiowaveform".into() }; + g.children.push(ids.prop("audioLayer", "Audio Layer", Value::Layer(Some(pre_id.0))).into()); + let solid = p.comp(outer).unwrap().layers[0].id; + p.comp_mut(outer).unwrap().layer_mut(solid).unwrap().props.group_mut("effects").unwrap().children.push(g.into()); + assert!(!silent(&p), "the outer comp's waveform draws the music"); +} + +/// Layers that can't change the picture: music, hidden layers nothing reads. A hidden matte is +/// read by the layer it mattes; nulls and cameras are never silent. +#[test] +fn silent_layers_are_music_and_unread_hidden_layers() { + use crate::{Footage, FootageKind, LayerId, MatteKind, TrackMatte}; + let (mut p, cid) = project_with_layer(); + let comp = p.comp(cid).unwrap().clone(); + let song = p.add_item( + "song.wav", + effectcraft_color::Label::None, + None, + ItemKind::Footage(Footage { kind: FootageKind::Audio, has_video: false, has_audio: true, ..Default::default() }), + ); + let add = |p: &mut Project, name: &str, source: LayerSource| { + let l = build::layer(p, &comp, name, source, (1920, 1080), None); + let id = l.id; + p.comp_mut(cid).unwrap().layers.push(l); + id + }; + let music = add(&mut p, "Music", LayerSource::Footage { item: song }); + let solid = p.comp(cid).unwrap().layers[0].source.clone(); + let hidden = add(&mut p, "Hidden", solid.clone()); + let matte = add(&mut p, "Matte", solid); + let null = add(&mut p, "Null", LayerSource::Null); + let first = p.comp(cid).unwrap().layers[0].id; + let c = p.comp_mut(cid).unwrap(); + for id in [hidden, matte, null] { + c.layer_mut(id).unwrap().switches.video = false; + } + c.layer_mut(first).unwrap().track_matte = Some(TrackMatte { layer: matte, kind: MatteKind::Alpha }); + let c = p.comp(cid).unwrap(); + assert_eq!(p.silent_layers(cid, c, &p.heard_layers()).into_iter().collect::>(), vec![music, hidden]); + // Solo hides every other layer, so a soloed music or hidden layer changes the picture. + for (soloed, rest) in [(music, hidden), (hidden, music)] { + let mut q = p.clone(); + q.comp_mut(cid).unwrap().layer_mut(soloed).unwrap().switches.solo = true; + assert_eq!(q.silent_layers(cid, q.comp(cid).unwrap(), &q.heard_layers()).into_iter().collect::>(), vec![rest]); + } +} + +/// Plug-in effects read their input at other times (plug-in API 2), so a precomp under one is +/// re-rendered over its whole span; built-in effects only when they are on the list. +#[test] +fn plug_in_effects_read_other_frames() { + assert!(crate::reads_other_frames("org.example.glow")); + assert!(crate::reads_other_frames("ec.time.echo")); + assert!(!crate::reads_other_frames("ec.blur.gaussian")); +} diff --git a/crates/render/src/disk_cache.rs b/crates/render/src/disk_cache.rs index 9cf5694f0..e05ff739f 100644 --- a/crates/render/src/disk_cache.rs +++ b/crates/render/src/disk_cache.rs @@ -33,9 +33,14 @@ use effectcraft_raster::Image; /// File format version (part of the folder layout). const VERSION: &str = "v1"; const MAGIC: &[u8; 4] = b"ECDC"; -/// Pending writes before new ones are dropped. +/// Pending layer and matte writes before new ones are dropped. #[cfg(not(target_arch = "wasm32"))] const QUEUE_LIMIT: usize = 64; +/// Pending frame bytes before new frames are dropped. Frames have their own lane, written +/// before layer buffers: a burst of large layer writes used to fill the shared queue and drop +/// frames, so a fully cached comp came back with gaps in its blue bar. +#[cfg(not(target_arch = "wasm32"))] +const FRAME_QUEUE_BYTES: usize = 1 << 30; #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub enum Kind { @@ -225,8 +230,30 @@ enum Job { } struct Queue { + /// Frame writes (taken first) and their payload bytes. + frames: std::collections::VecDeque, + #[cfg_attr(target_arch = "wasm32", allow(dead_code))] + frame_bytes: usize, + /// Layer and matte writes. jobs: std::collections::VecDeque, busy: bool, + /// Keys of frames queued or being written. + pending_frames: std::collections::HashSet, +} + +impl Queue { + #[cfg(not(target_arch = "wasm32"))] + fn pop(&mut self) -> Option { + if let Some(j) = self.frames.pop_front() { + let Job::Write(_, _, p) = &j; + self.frame_bytes = self.frame_bytes.saturating_sub(p.len()); + return Some(j); + } + self.jobs.pop_front() + } + fn is_empty(&self) -> bool { + self.frames.is_empty() && self.jobs.is_empty() + } } pub struct DiskCache { @@ -320,7 +347,16 @@ impl DiskCache { dropped: AtomicU64::new(0), failed: AtomicU64::new(0), tmp_counter: AtomicU64::new(0), - queue: Arc::new((Mutex::new(Queue { jobs: Default::default(), busy: cfg!(not(target_arch = "wasm32")) }), Condvar::new())), + queue: Arc::new(( + Mutex::new(Queue { + frames: Default::default(), + frame_bytes: 0, + jobs: Default::default(), + busy: cfg!(not(target_arch = "wasm32")), + pending_frames: Default::default(), + }), + Condvar::new(), + )), min_layer_ms: AtomicU64::new(30), }); dc.rescan(); @@ -428,6 +464,11 @@ impl DiskCache { self.index.lock().is_ok_and(|g| g.contains(kind, key)) } + /// A frame write is queued or in progress. + pub fn frame_pending(&self, key: u128) -> bool { + self.queue.0.lock().is_ok_and(|q| q.pending_frames.contains(&key)) + } + /// Keys of one kind (unordered). pub fn keys(&self, kind: Kind) -> Vec { self.index.lock().map(|g| g.keys(kind)).unwrap_or_default() @@ -467,8 +508,10 @@ impl DiskCache { #[cfg(target_arch = "wasm32")] { q.jobs.clear(); + q.frames.clear(); + q.pending_frames.clear(); } - while !q.jobs.is_empty() || q.busy { + while !q.is_empty() || q.busy { q = match cv.wait(q) { Ok(q) => q, Err(_) => return, @@ -515,21 +558,34 @@ impl DiskCache { } } - fn enqueue(&self, kind: Kind, key: u128, payload: Vec) { + /// Queue a write; `false` when it was dropped (a full queue). + fn enqueue(&self, kind: Kind, key: u128, payload: Vec) -> bool { #[cfg(target_arch = "wasm32")] { self.write_now(kind, key, &payload); + true } #[cfg(not(target_arch = "wasm32"))] { let (m, cv) = &*self.queue; - let Ok(mut q) = m.lock() else { return }; - if q.jobs.len() >= QUEUE_LIMIT { + let Ok(mut q) = m.lock() else { return false }; + let full = match kind { + Kind::Frame => q.frame_bytes.saturating_add(payload.len()) > FRAME_QUEUE_BYTES, + _ => q.jobs.len() >= QUEUE_LIMIT, + }; + if full { self.dropped.fetch_add(1, Ordering::Relaxed); - return; + return false; + } + if kind == Kind::Frame { + q.pending_frames.insert(key); + q.frame_bytes = q.frame_bytes.saturating_add(payload.len()); + q.frames.push_back(Job::Write(kind, key, payload)); + } else { + q.jobs.push_back(Job::Write(kind, key, payload)); } - q.jobs.push_back(Job::Write(kind, key, payload)); cv.notify_all(); + true } } @@ -595,12 +651,13 @@ impl DiskCache { decode_frame(&p) } - /// Queue a frame for writing (8-bit premultiplied RGBA, `width * height * 4` bytes). - pub fn put_frame(&self, key: u128, width: u32, height: u32, rgba: &[u8]) { - if self.contains(Kind::Frame, key) || rgba.len() != width as usize * height as usize * 4 { - return; + /// Queue a frame for writing (8-bit premultiplied RGBA, `width * height * 4` bytes). `true` + /// when it is on disk or queued; `false` when refused (wrong size, a full queue). + pub fn put_frame(&self, key: u128, width: u32, height: u32, rgba: &[u8]) -> bool { + if rgba.len() != width as usize * height as usize * 4 { + return false; } - self.enqueue(Kind::Frame, key, encode_frame(width, height, rgba)); + self.contains(Kind::Frame, key) || self.enqueue(Kind::Frame, key, encode_frame(width, height, rgba)) } pub fn get_layer(&self, key: u128) -> Option { @@ -613,7 +670,7 @@ impl DiskCache { if self.contains(Kind::Layer, key) { return; } - self.enqueue(Kind::Layer, key, encode_layer(buf)); + let _ = self.enqueue(Kind::Layer, key, encode_layer(buf)); } /// Write a layer buffer synchronously (tests). @@ -634,7 +691,39 @@ impl DiskCache { if self.contains(Kind::Matte, k) { return; } - self.enqueue(Kind::Matte, k, lz4_flex::compress_prepend_size(&seg.to_bytes())); + let _ = self.enqueue(Kind::Matte, k, lz4_flex::compress_prepend_size(&seg.to_bytes())); + } + + /// Make the frame at key `from` the frame at key `to` too (an edit that can't change it gave + /// its comp a new content key): a hard link to the same file, or a copy where the file system + /// has no links. `false` when `from` isn't cached, `to` already is, or the file system + /// refused. (Entries don't contain their key, so the same file serves both.) + pub fn link_frame(&self, from: u128, to: u128) -> bool { + if !self.contains(Kind::Frame, from) || self.contains(Kind::Frame, to) { + return false; + } + let (src, dst) = (self.path(Kind::Frame, from), self.path(Kind::Frame, to)); + let Some(parent) = dst.parent() else { return false }; + let n = self.tmp_counter.fetch_add(1, Ordering::Relaxed); + let tmp = parent.join(format!("{to:032x}.tmp.{}.{n}", std::process::id())); + let linked = std::fs::create_dir_all(parent) + .and_then(|()| std::fs::hard_link(&src, &tmp).or_else(|_| std::fs::copy(&src, &tmp).map(|_| ()))) + .and_then(|()| std::fs::metadata(&tmp)) + .and_then(|m| std::fs::rename(&tmp, &dst).map(|()| m.len())); + let bytes = match linked { + Ok(bytes) => bytes, + Err(_) => { + // (gone meanwhile: another process evicted it) + let _ = std::fs::remove_file(&tmp); + return false; + } + }; + // (a link shares its file, but counts in full: the cache only ever errs small) + self.evict_to(self.limit_bytes().saturating_sub(bytes)); + if let Ok(mut g) = self.index.lock() { + g.insert(Kind::Frame, to, bytes); + } + true } /// Write a frame synchronously (tests). @@ -659,7 +748,7 @@ fn writer(dc: std::sync::Weak, q: Arc<(Mutex, Condvar)>) { let job = { let Ok(mut g) = m.lock() else { return }; loop { - if let Some(j) = g.jobs.pop_front() { + if let Some(j) = g.pop() { g.busy = true; break j; } @@ -673,12 +762,14 @@ fn writer(dc: std::sync::Weak, q: Arc<(Mutex, Condvar)>) { } }; let Some(cache) = dc.upgrade() else { return }; - match job { - Job::Write(kind, key, payload) => cache.write_now(kind, key, &payload), - } + let Job::Write(kind, key, payload) = job; + cache.write_now(kind, key, &payload); drop(cache); if let Ok(mut g) = m.lock() { g.busy = false; + if kind == Kind::Frame { + g.pending_frames.remove(&key); + } } cv.notify_all(); } @@ -841,7 +932,7 @@ fn decode_layer(p: &[u8]) -> Option { // ---------------------------------------------------------------- content keys /// Bump when rendering changes in a way that makes old disk entries wrong. -const RENDER_VERSION: &str = "effectcraft-render-5"; +const RENDER_VERSION: &str = "effectcraft-render-6"; fn hash_debug(h: &mut Hash128, v: &impl std::fmt::Debug) { h.write(format!("{v:?}").as_bytes()); @@ -889,8 +980,26 @@ pub fn footage_salt(project: &effectcraft_project::Project) -> u64 { pub fn footage_salt_with(project: &effectcraft_project::Project, virtual_id: Option>) -> u64 { let mut h = Hash128::default(); h.write(RENDER_VERSION.as_bytes()); + // Footage another layer reads (Audio Spectrum's Audio Layer); a read precomp can hold any. + let mut read = std::collections::BTreeSet::new(); + let mut read_comp = false; + for (_, c) in project.comps() { + for l in c.layers.iter() { + for r in l.referenced_layers().into_iter().filter(|r| *r != l.id) { + match c.layer(r).map(|x| &x.source) { + Some(effectcraft_project::LayerSource::Footage { item }) => _ = read.insert(*item), + Some(effectcraft_project::LayerSource::Comp { .. }) => read_comp = true, + _ => {} + } + } + } + } for (id, it) in &project.items { - if let effectcraft_project::ItemKind::Footage(f) = &it.kind { + // (files without video make layer buffers only through a layer that reads them: + // importing music keeps the keys) + if let effectcraft_project::ItemKind::Footage(f) = &it.kind + && (f.has_video || read_comp || read.contains(id)) + { h.write_u64(id.0); hash_debug(&mut h, f); footage_stamp(&mut h, f, virtual_id); @@ -924,34 +1033,46 @@ fn comp_content_key_for_renderer( let mut h = Hash128::default(); h.write(renderer.as_bytes()); hash_debug(&mut h, &project.settings); + // Expressions can read anything (every item counts); without them, layers that can't + // change the picture (music, hidden layers nothing reads) and their files don't count. + let expressions = project.reads_markers(); + let heard = project.heard_layers(); + let silent = |id, c: &effectcraft_project::Comp| if expressions { Default::default() } else { project.silent_layers(id, c, &heard) }; let mut seen = std::collections::BTreeSet::new(); let mut stack = vec![comp]; - let mut expressions = false; while let Some(id) = stack.pop() { if !seen.insert(id) { continue; } let Some(it) = project.item(id) else { continue }; if let ItemKind::Comp(c) = &it.kind { - for l in &c.layers { + let quiet = silent(id, c); + for l in c.layers.iter().filter(|l| !quiet.contains(&l.id)) { if let LayerSource::Comp { item } | LayerSource::Footage { item } | LayerSource::Solid { item } = l.source { stack.push(item); } - if !expressions { - l.props.walk("", &mut |_, p| expressions |= p.expr.is_some()); - } } } } // Markers are pixels only to the expressions that read them (#1083). - let markers = project.reads_markers(); + let markers = expressions; let ids: Vec = if expressions { project.items.keys().copied().collect() } else { seen.into_iter().collect() }; for id in ids { let Some(it) = project.item(id) else { continue }; h.write_u64(id.0); h.write(it.name.as_bytes()); match &it.kind { - ItemKind::Comp(c) => hash_debug(&mut h, &c.pixel_form_with(markers)), + ItemKind::Comp(c) => { + let quiet = silent(id, c); + let heard = |l| heard.contains(id, l); + if quiet.is_empty() { + hash_debug(&mut h, &c.pixel_form_with(markers, &heard)); + } else { + let mut pic = (**c).clone(); + pic.layers.retain(|l| !quiet.contains(&l.id)); + hash_debug(&mut h, &pic.pixel_form_with(markers, &heard)); + } + } k => hash_debug(&mut h, k), } if let ItemKind::Footage(f) = &it.kind { @@ -986,6 +1107,26 @@ mod tests { d } + /// A linked frame reads back as the original under its new key, survives the original's + /// eviction, and is never linked over an existing entry or from a missing one. + #[test] + fn link_frame_serves_a_frame_under_a_new_key() { + let dc = DiskCache::open(tmpdir("link"), 1 << 30).unwrap(); + let rgba: Vec = (0..16).collect(); + dc.put_frame_now(1, 2, 2, &rgba); + assert!(dc.link_frame(1, 2)); + assert_eq!(dc.get_frame(2).map(|f| f.rgba), Some(rgba.clone())); + assert!(!dc.link_frame(1, 2), "already there"); + assert!(!dc.link_frame(3, 4), "nothing to link"); + assert!(!dc.contains(Kind::Frame, 4)); + dc.put_frame_now(5, 2, 2, &[7; 16]); + assert!(!dc.link_frame(5, 2), "an existing entry stays"); + assert_eq!(dc.get_frame(2).map(|f| f.rgba), Some(rgba.clone())); + let _ = std::fs::remove_file(dc.path(Kind::Frame, 1)); + dc.forget(Kind::Frame, 1); + assert_eq!(dc.get_frame(2).map(|f| f.rgba), Some(rgba), "the link outlives the original"); + } + fn buf(seed: u32, w: u32, h: u32) -> Buf { let mut img = Image::new(w, h); for (i, p) in img.data.iter_mut().enumerate() { @@ -1073,7 +1214,8 @@ mod tests { assert_eq!(back.img, b.img); assert_eq!((back.offset, back.scale), (b.offset, b.scale)); let rgba: Vec = (0..20 * 10 * 4).map(|i| (i % 251) as u8).collect(); - dc.put_frame(7, 20, 10, &rgba); + assert!(dc.put_frame(7, 20, 10, &rgba)); + assert!(!dc.put_frame(8, 2, 2, &[0; 3]), "a wrong-size frame is refused"); dc.flush(); assert_eq!(dc.get_frame(7).unwrap().rgba, rgba); let st = dc.stats(); @@ -1082,6 +1224,51 @@ mod tests { assert!(dc.get_frame(42).is_none()); } + /// Music doesn't change a comp's disk keys (adding it to the timeline kept no blue frame); + /// hiding a clip does. Importing music keeps the layer-cache salt. + #[test] + fn music_keeps_the_disk_keys() { + use effectcraft_project::{Comp, Footage, FootageKind, ItemKind, LayerSource, Project, Solid, build}; + use effectcraft_time::{FrameRate, Tick}; + let mut p = Project::default(); + let comp = Comp::new(64, 64, FrameRate::FPS_30, Tick::from_seconds_f64(2.0)); + let cid = p.add_item("Main", Default::default(), None, ItemKind::Comp(comp.clone().into())); + let sid = p.add_item("Solid", Default::default(), None, ItemKind::Solid(Solid { color: [1.0, 0.0, 0.0], width: 64, height: 64, pixel_aspect: 1.0 })); + let clip = build::layer(&mut p, &comp, "Clip", LayerSource::Solid { item: sid }, (64, 64), None); + let clip_id = clip.id; + p.comp_mut(cid).unwrap().layers.push(clip); + let (key, salt) = (comp_content_key(&p, cid), footage_salt(&p)); + let song = p.add_item( + "song.wav", + Default::default(), + None, + ItemKind::Footage(Footage { kind: FootageKind::Audio, has_video: false, has_audio: true, ..Default::default() }), + ); + assert_eq!(footage_salt(&p), salt, "importing music"); + let music = build::layer(&mut p, &comp, "Music", LayerSource::Footage { item: song }, (64, 64), None); + p.comp_mut(cid).unwrap().layers.insert(0, music); + assert_eq!(comp_content_key(&p, cid), key, "music on the timeline"); + p.comp_mut(cid).unwrap().layer_mut(clip_id).unwrap().switches.video = false; + assert_ne!(comp_content_key(&p, cid), key, "a hidden clip"); + } + + /// A burst of layer writes never crowds out frames: every frame of a fully cached comp is + /// on disk afterwards (the blue bar came back with gaps). + #[test] + fn frames_are_kept_when_layer_writes_fill_the_queue() { + let dir = tmpdir("burst"); + let dc = DiskCache::open(&dir, 1 << 30).unwrap(); + for k in 0..300 { + dc.put_layer(10_000 + k, &buf(1, 64, 64)); + } + let rgba = vec![7u8; 32 * 32 * 4]; + for k in 0..300 { + dc.put_frame(k, 32, 32, &rgba); + } + dc.flush(); + assert!((0..300).all(|k| dc.contains(Kind::Frame, k)), "every frame written"); + } + #[test] fn previous_renderer_frames_are_not_reused() { use effectcraft_project::{Comp, ItemKind, Project}; @@ -1091,7 +1278,7 @@ mod tests { let comp = Comp::new(2, 2, FrameRate::FPS_30, Tick::from_seconds_f64(1.0)); let id = project.add_item("Cached composition", effectcraft_color::Label::None, None, ItemKind::Comp(comp.into())); let key = frame_key(comp_content_key(&project, id), 0, 1000, 0); - let previous_key = frame_key(comp_content_key_for_renderer(&project, id, None, "effectcraft-render-4"), 0, 1000, 0); + let previous_key = frame_key(comp_content_key_for_renderer(&project, id, None, "effectcraft-render-5"), 0, 1000, 0); let dir = tmpdir("previous-renderer"); { let cache = DiskCache::open(&dir, 1 << 30).unwrap(); @@ -1267,6 +1454,32 @@ mod tests { } } + /// An audio file nothing reads stays out of the layer salt; one a layer reads (Audio + /// Spectrum) is in it, so overwriting it doesn't reuse that layer's buffers. + #[test] + fn read_audio_counts_in_the_footage_salt() { + use effectcraft_project::build; + use effectcraft_project::{Comp, Footage, FootageKind, ItemKind, LayerSource, MatteKind, Project, TrackMatte}; + use effectcraft_time::{FrameRate, Tick}; + + let mut p = Project::default(); + let f = Footage { path: "/song.wav".into(), kind: FootageKind::Audio, has_video: false, has_audio: true, ..Default::default() }; + let song = p.add_item("song.wav", effectcraft_color::Label::None, None, ItemKind::Footage(f)); + let mut comp = Comp::new(16, 9, FrameRate::FPS_30, Tick::from_seconds_f64(1.0)); + let music = build::layer(&mut p, &comp, "music", LayerSource::Footage { item: song }, (16, 9), None); + let mut reader = build::layer(&mut p, &comp, "reader", LayerSource::Null, (16, 9), None); + let music_id = music.id; + comp.layers = vec![reader.clone(), music].into(); + let cid = p.add_item("C", effectcraft_color::Label::None, None, ItemKind::Comp(comp.into())); + + let id = |mark: u64| move |path: &str| (path == "/song.wav").then_some((8, mark)); + let (one, two) = (id(1), id(2)); + assert_eq!(footage_salt_with(&p, Some(&one)), footage_salt_with(&p, Some(&two)), "unread music stays out"); + reader.track_matte = Some(TrackMatte { layer: music_id, kind: MatteKind::Alpha }); + *p.comp_mut(cid).unwrap().layers.first_mut().unwrap() = reader; + assert_ne!(footage_salt_with(&p, Some(&one)), footage_salt_with(&p, Some(&two)), "read audio counts"); + } + /// A path with no native metadata takes its key from the caller's size and content hash (#1253). #[test] fn virtual_file_bytes_change_content_keys() { diff --git a/crates/render/src/eval.rs b/crates/render/src/eval.rs index 81fe45d6c..f4d2cbb37 100644 --- a/crates/render/src/eval.rs +++ b/crates/render/src/eval.rs @@ -384,6 +384,24 @@ fn effect_bounds_of(comp: &Comp, size: (u32, u32)) -> ([f64; 2], [f64; 2]) { } } +/// The layer's source time range in layer time seconds as (first, end) for +/// [`effectcraft_effects::EffectEnv::layer_span`]: footage and nested comps last their item's +/// duration from layer time 0. With Time Remap layer time is no longer source time, so the layer's +/// visible span (in to out point, in layer time) is reported instead. `None` for sources without +/// a duration (stills, solids, text, shapes...). +pub fn layer_span(project: &Project, layer: &Layer) -> Option<(f64, f64)> { + let span = if layer.props.get("timeRemap").is_some() { + let (a, b) = (layer.layer_time(layer.in_point).seconds(), layer.layer_time(layer.out_point).seconds()); + (a.min(b), a.max(b)) + } else { + match &layer.source { + LayerSource::Footage { item } | LayerSource::Comp { item } => (0.0, project.item(*item)?.duration()?.seconds()), + _ => return None, + } + }; + (span.0.is_finite() && span.1.is_finite() && span.1 > span.0).then_some(span) +} + /// Size of a layer's source in layer pixels. pub fn source_size(project: &Project, layer: &Layer) -> (u32, u32) { match &layer.source { diff --git a/crates/render/src/pipeline.rs b/crates/render/src/pipeline.rs index a4927144f..ddf671dc4 100644 --- a/crates/render/src/pipeline.rs +++ b/crates/render/src/pipeline.rs @@ -615,6 +615,7 @@ impl<'a> Renderer<'a> { working_linear: self.pipe.linear, float_depth: self.pipe.levels.is_none(), shutter: self.mb_on(ctx, layer).then_some((ctx.comp.shutter_angle, ctx.comp.shutter_phase, ctx.comp.motion_blur_samples)), + layer_span: crate::eval::layer_span(self.project, layer), }; // Video effects in stack order (index, group, spec); disabled and audio effects skipped. let stack: Vec<(usize, &effectcraft_project::PropGroup, &'static effectcraft_effects::EffectSpec)> = fx diff --git a/crates/render/src/tests/remap.rs b/crates/render/src/tests/remap.rs index 67b6ebef7..8d3cd8b63 100644 --- a/crates/render/src/tests/remap.rs +++ b/crates/render/src/tests/remap.rs @@ -63,3 +63,24 @@ fn time_remap_picks_the_source_frame() { assert!(img.get(90, 50)[3] > 0.99, "frozen at the end position"); assert!(img.get(10, 50)[3] < 0.01); } + +#[test] +fn layer_span_is_the_source_duration_in_layer_time() { + let mut p = Project::default(); + let outer = Comp::new(100, 100, FrameRate::FPS_30, s(4.0)); + let inner = Comp::new(100, 100, FrameRate::FPS_30, s(2.5)); + let iid = p.add_item("Inner", Label::Sandstone, None, ItemKind::Comp(inner.into())); + let sid = p.add_item("S", Label::Red, None, ItemKind::Solid(Solid { color: [1.0, 1.0, 1.0], width: 10, height: 10, pixel_aspect: 1.0 })); + let mut pre = build::layer(&mut p, &outer, "Inner", LayerSource::Comp { item: iid }, (100, 100), None); + let sol = build::layer(&mut p, &outer, "S", LayerSource::Solid { item: sid }, (10, 10), None); + let (first, end) = crate::eval::layer_span(&p, &pre).expect("a nested comp has a duration"); + assert!(first.abs() < 1e-9 && (end - 2.5).abs() < 1e-9); + assert_eq!(crate::eval::layer_span(&p, &sol), None, "solids have no duration"); + // Time Remap: layer time is not source time any more, the visible span is reported. + let mut next = p.next_id; + let tr = Ids(&mut next).prop("timeRemap", "Time Remap", Value::Scalar(0.0)); + p.next_id = next; + pre.props.children.push(Node::Prop(tr)); + let (first, end) = crate::eval::layer_span(&p, &pre).expect("a remapped layer has a visible span"); + assert!(end > first); +} diff --git a/crates/ui-egui/Cargo.toml b/crates/ui-egui/Cargo.toml index e150b2f2f..59d437b26 100644 --- a/crates/ui-egui/Cargo.toml +++ b/crates/ui-egui/Cargo.toml @@ -48,6 +48,7 @@ egui_kittest = { workspace = true, features = ["eframe", "wgpu"] } pollster = "1" effectcraft-segment = { workspace = true } effectcraft-host = { workspace = true } +effectcraft-media = { workspace = true } exr = { workspace = true } [build-dependencies] diff --git a/crates/ui-egui/examples/cache_bench.rs b/crates/ui-egui/examples/cache_bench.rs new file mode 100644 index 000000000..828075249 --- /dev/null +++ b/crates/ui-egui/examples/cache_bench.rs @@ -0,0 +1,177 @@ +//! Background caching benchmark: how fast the RAM preview fills a comp, and where the time goes. +//! +//! `cargo run --release -p effectcraft-ui-egui --example cache_bench -- [options]` +//! +//! Without `--project` it builds a comp like a typical edit: eight 1080p30 H.264 clips one after +//! another (encoded once into `target/cache-bench/media` with the Render Queue, no committed +//! media) under an adjustment layer with Gaussian Blur. It then fills the comp's RAM preview the +//! way idle caching does, through [`Frames`], and prints frames per second with the decoder, +//! footage-cache and disk-cache counters. +//! +//! Options: `--project P --comp C` (a saved project's comp instead), `--scale S` (render scale, +//! default 0.5: a viewer at half size), `--frames N` (frames to cache, default the whole comp), +//! `--slots N` (frames in flight, at least 1, default one per frame thread as the app does when +//! idle), +//! `--disk` (with a fresh disk cache in `target/cache-bench/disk`), `--no-adjustment`. + +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use effectcraft_engine::Session; +use effectcraft_engine::project::ItemId; +use effectcraft_engine::render::RenderOpts; +use effectcraft_ui_egui::frames::{FrameKey, Frames, RenderSource, opts_hash}; +use serde_json::{Value, json}; + +fn arg(name: &str) -> Option { + let a: Vec = std::env::args().collect(); + a.iter().position(|x| x == name).and_then(|i| a.get(i + 1).cloned()) +} + +fn run(s: &mut Session, cmd: &str, p: Value) -> Value { + s.execute(cmd, p).unwrap_or_else(|e| panic!("{cmd}: {e}")) +} + +/// Eight copies of one generated clip (separate files: separate decoders, as with real clips). +fn clips(dir: &std::path::Path) -> Vec { + let media = dir.join("media"); + std::fs::create_dir_all(&media).expect("media folder"); + let first = media.join("clip0.mp4"); + if !first.exists() { + eprintln!("encoding the source clip (once)…"); + let mut s = effectcraft_host::session(); + run(&mut s, "comp.new", json!({"name": "Src", "width": 1920, "height": 1080, "frameRate": 30, "duration": 4.25})); + let l = run(&mut s, "layer.newSolid", json!({"color": "#808080"}))["layer"].clone(); + run(&mut s, "effect.apply", json!({"layers": [l], "effect": "Fractal Noise"})); + run(&mut s, "prop.addKey", json!({"layer": l, "path": "effects/#1/evolution", "time": 0.0, "value": 0.0})); + run(&mut s, "prop.addKey", json!({"layer": l, "path": "effects/#1/evolution", "time": 4.25, "value": 720.0})); + run(&mut s, "renderQueue.add", json!({"comp": "Src", "format": "h264", "output": first.to_string_lossy()})); + run(&mut s, "renderQueue.render", json!({"wait": true})); + } + (0..8) + .map(|i| { + let p = media.join(format!("clip{i}.mp4")); + if !p.exists() { + std::fs::copy(&first, &p).expect("copy clip"); + } + p.to_string_lossy().to_string() + }) + .collect() +} + +fn main() { + let dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../target/cache-bench"); + let mut s = effectcraft_host::session(); + let pool = Arc::new(effectcraft_media::MediaPool::new()); + s.footage = pool.clone(); + if std::env::args().any(|a| a == "--disk") { + let disk = dir.join("disk"); + if let Err(e) = std::fs::remove_dir_all(&disk) + && e.kind() != std::io::ErrorKind::NotFound + { + eprintln!("can't clear the disk cache at {}: {e}", disk.display()); + std::process::exit(2); + } + s.prefs.disk.disk_cache_enabled = true; + s.prefs.disk.disk_cache_folder = disk.to_string_lossy().to_string(); + s.configure_disk_cache(); + } + let cid = match arg("--project") { + Some(p) => { + run(&mut s, "file.open", json!({"path": p})); + let comp = match arg("--comp") { + Some(c) => s.project.comps().find(|(id, _)| s.project.item(**id).is_some_and(|it| it.name == c) || id.0.to_string() == c).map(|(id, _)| *id), + None => s.active_comp_id(), + }; + comp.unwrap_or_else(|| { + eprintln!("no such composition: {}", arg("--comp").unwrap_or_else(|| "(no active comp; pass --comp)".into())); + std::process::exit(2) + }) + } + None => { + let paths = clips(&dir); + let cid = ItemId( + run(&mut s, "comp.new", json!({"name": "Edit", "width": 1920, "height": 1080, "frameRate": 30, "duration": 34}))["comp"] + .as_u64() + .expect("comp"), + ); + let items = run(&mut s, "file.import", json!({"paths": paths}))["items"].clone(); + for (i, item) in items.as_array().expect("items").iter().enumerate() { + let l = run(&mut s, "layer.addItem", json!({"item": item}))["layer"].as_u64().expect("layer"); + let c = Arc::make_mut(&mut s.project).comp_mut(cid).expect("comp"); + let fr = c.frame_rate; + let l = c.layer_mut(effectcraft_engine::project::LayerId(l)).expect("layer"); + let at = fr.tick_of(i as i64 * 1275 / 10); + (l.start_time, l.in_point, l.out_point) = (at, at, fr.tick_of((i as i64 + 1) * 1275 / 10)); + } + if !std::env::args().any(|a| a == "--no-adjustment") { + let adj = run(&mut s, "layer.newAdjustment", json!({}))["layer"].clone(); + run(&mut s, "effect.apply", json!({"layers": [adj], "effect": "Gaussian Blur"})); + run(&mut s, "prop.set", json!({"layer": adj, "path": "effects/#1/blurriness", "value": 4.0})); + } + cid + } + }; + let comp = s.project.comp(cid).expect("comp").clone(); + let total = comp.frame_rate.frame_at(comp.duration); + let n = arg("--frames").and_then(|v| v.parse().ok()).unwrap_or(total).min(total); + let scale: f64 = arg("--scale").and_then(|v| v.parse().ok()).unwrap_or(0.5); + + let frames = Frames::default(); + // (as Cache Frames When Idle: one frame per frame thread, see `app/prefetch.rs`) + let slots: usize = arg("--slots").and_then(|v| v.parse().ok()).map(|n: usize| n.max(1)).unwrap_or(frames.parallelism().max(1)); + // Room for every frame: this measures speed, not eviction. + frames.set_budget(64 << 30); + let opts = RenderOpts { scale, motion_blur: true, ..Default::default() }; + let src = RenderSource { + project: s.project.clone(), + footage: s.footage.clone(), + expr: s.expr.clone(), + layer_cache: s.layer_cache.clone(), + gpu: None, + gpu_display: false, + disk: s.disk_cache.clone(), + }; + let series = FrameKey { + revision: 1, + content: frames.content_of(&src.project, 1, cid), + comp: cid.0, + frame: 0, + scale: (scale * 1000.0).round() as u32, + view: 0, + opts: opts_hash(&opts), + }; + + eprintln!("caching {n} frames of {}×{} at scale {scale} with {slots} in flight…", comp.width, comp.height); + let (gop0, t0) = (effectcraft_media::gop_stats(), Instant::now()); + let mut next = 0; + loop { + while next < n && frames.inflight() < slots { + frames.request_background(&src, FrameKey { frame: next, ..series }, cid, comp.frame_rate.tick_of(next), opts); + next += 1; + } + if next >= n && frames.inflight() == 0 { + break; + } + std::thread::sleep(Duration::from_millis(1)); + } + let secs = t0.elapsed().as_secs_f64(); + let gop = effectcraft_media::gop_stats(); + let cached = frames.cached_frames(&series).len(); + println!("frames cached {cached}/{n} in {secs:.2} s: {:.1} frames/s, {:.1} ms each", cached as f64 / secs, secs * 1000.0 / cached.max(1) as f64); + println!( + "video decoding {:.2} s in decoders, {} samples decoded, {} seeks, {} cache hits", + (gop.decode_ns - gop0.decode_ns) as f64 / 1e9, + gop.decoded - gop0.decoded, + gop.seeks - gop0.seeks, + gop.hits - gop0.hits + ); + let p = pool.stats(); + println!("footage cache {} hits, {} misses, {} frames ({} MB) kept", p.hits, p.misses, p.frames, p.bytes >> 20); + if let Some(dc) = &s.disk_cache { + // (writes finish on the disk cache's own thread) + dc.flush(); + let d = dc.stats(); + println!("disk cache {} written, {} dropped, {} entries ({} MB)", d.writes, d.dropped, d.entries, d.bytes >> 20); + } +} diff --git a/crates/ui-egui/src/app/prefetch.rs b/crates/ui-egui/src/app/prefetch.rs index 54bf07250..f00d44014 100644 --- a/crates/ui-egui/src/app/prefetch.rs +++ b/crates/ui-egui/src/app/prefetch.rs @@ -399,6 +399,12 @@ impl EffectcraftApp { ctx.request_repaint_after(std::time::Duration::from_secs_f64(1.0 - quiet)); return; } + // Frames an edit kept in RAM are on disk under the comp's previous content only. + // Again soon while frames wait to be written (a refused write is retried then). + #[cfg(not(target_arch = "wasm32"))] + if self.frames.persist_missing(&self.render_source(), series, &self.frame_opts(ItemId(series.comp), series.scale as f64 / 1000.0)) { + ctx.request_repaint_after(std::time::Duration::from_millis(500)); + } // An empty work area has nothing to cache. let n = wb - wa + 1; if n < 1 { diff --git a/crates/ui-egui/src/app/viewer_frames.rs b/crates/ui-egui/src/app/viewer_frames.rs index f87db615c..98a78ff75 100644 --- a/crates/ui-egui/src/app/viewer_frames.rs +++ b/crates/ui-egui/src/app/viewer_frames.rs @@ -105,6 +105,15 @@ impl EffectcraftApp { self.frame_series(comp, scale as f64 / 1000.0) } + /// The frame series a preview with options `o` plays: at its Resolution, or the one the + /// viewer shows with Auto (it follows the zoom). The Preview panel's cache bar counts it. + pub fn preview_series(&self, comp: ItemId, o: &effectcraft_engine::preview::PreviewPreset) -> FrameKey { + match o.resolution.factor(o.custom_resolution) { + Some(n) => self.frame_series(comp, 1.0 / f64::from(n)), + None => self.shown_series(comp), + } + } + /// Hash of the comp viewer's 3D view camera (0 for the active camera view). pub fn view_hash(&self, comp: ItemId) -> u64 { use std::hash::{Hash, Hasher}; diff --git a/crates/ui-egui/src/frames.rs b/crates/ui-egui/src/frames.rs index 9aad2a81c..788af49e8 100644 --- a/crates/ui-egui/src/frames.rs +++ b/crates/ui-egui/src/frames.rs @@ -28,7 +28,7 @@ use std::collections::{BTreeSet, HashMap, HashSet, VecDeque}; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; -use effectcraft_engine::project::{Comp, ItemId, ItemKind, LayerSource, Node, Project, PropGroup}; +use effectcraft_engine::project::{Comp, Heard, ItemId, ItemKind, LayerSource, Node, Project, PropGroup}; use effectcraft_engine::render::disk_cache::{self, DiskCache}; use effectcraft_engine::render::{CancellableFootage, ExprHost, FootageSource, LayerCache, RenderOpts, Renderer}; use effectcraft_engine::time::Tick; @@ -337,12 +337,26 @@ fn content_key(keys: &ContentKeys, project: &Project, revision: u64, comp: u64, /// hands out), and by value the project settings and the footage, solids and proxies it uses. /// With expressions in its comps (which can read anything) every item counts. pub fn comp_content(project: &Project, comp: ItemId) -> u64 { - comp_content_with(project, comp, project, |_, c| c) + comp_content_with(project, comp, project, |_, c| c, true) +} + +/// Frame ranges (inclusive) of `comp` in `new` that may draw differently than in `old` +/// ([`Project::dirty_spans`]), a frame wider either side for rounding. +fn dirty_frames(old: &Project, new: &Project, comp: ItemId, heard: &Heard, outside: bool) -> Option> { + let fr = new.comp(comp)?.frame_rate; + Some(new.dirty_spans(old, comp, heard, outside)?.into_iter().map(|(s, e)| (fr.frame_at(s).saturating_sub(1), fr.frame_at(e).saturating_add(1))).collect()) } /// [`comp_content`] with each comp's version selected for both its sources and its address. -/// Sources removed by undo are read from `base`. -fn comp_content_with<'a>(project: &'a Project, comp: ItemId, base: &'a Project, version: impl Fn(ItemId, &'a Arc) -> &'a Arc) -> u64 { +/// Sources removed by undo are read from `base`. Only the comps and items it draws count unless +/// `everything` and an expression (which can read any item) is among them. +fn comp_content_with<'a>( + project: &'a Project, + comp: ItemId, + base: &'a Project, + version: impl Fn(ItemId, &'a Arc) -> &'a Arc, + everything: bool, +) -> u64 { use std::hash::{Hash, Hasher}; fn has_expression(g: &PropGroup) -> bool { g.children.iter().any(|c| match c { @@ -366,7 +380,7 @@ fn comp_content_with<'a>(project: &'a Project, comp: ItemId, base: &'a Project, if let LayerSource::Comp { item } | LayerSource::Footage { item } | LayerSource::Solid { item } = l.source { stack.push(item); } - expressions = expressions || has_expression(&l.props); + expressions = expressions || (everything && has_expression(&l.props)); } } let ids: Vec = if expressions { project.items.keys().copied().collect() } else { seen.into_iter().collect() }; @@ -505,8 +519,41 @@ pub struct Frames { /// Where frames render instead of the UI thread (wasm32), and how many it is rendering. remote: Option>, remote_busy: Arc>, + /// Disk keys [`Frames::persist_missing`] is converting (the disk cache tracks them once + /// queued, so an evicted or refused frame is written again). + #[cfg(not(target_arch = "wasm32"))] + persisted: Arc>>, + /// The disk limit and how often each disk key was written under it: a frame is rewritten + /// once after an eviction, not again (a disk cache smaller than the RAM preview, or a frame + /// that can't fit, would otherwise be rewritten on every idle pass). A new limit resets it. + #[cfg(not(target_arch = "wasm32"))] + written: Arc)>>, + /// Edits whose kept frames are still to be found on disk under the new content key + /// ([`Frames::carry_disk`]), oldest first. + #[cfg(not(target_arch = "wasm32"))] + disk_carries: Arc>>, + /// A carry job is scheduled or running: at most one at a time, draining the edits in order. + #[cfg(not(target_arch = "wasm32"))] + carry_scheduled: Arc, + /// The disk series (scale, view and options hash) each comp's frames are cached in. + #[cfg(not(target_arch = "wasm32"))] + disk_series: Arc>>>, } +/// An edit's kept frames, to find on disk under the comp's new content key: those of `comp` in +/// `from` outside `dirty` (inclusive frame ranges) also draw `to`'s. +#[cfg(not(target_arch = "wasm32"))] +struct DiskCarry { + comp: u64, + from: Arc, + to: Arc, + dirty: Vec<(i64, i64)>, +} + +/// Disk series remembered per comp (see [`Frames::disk_series`]). +#[cfg(not(target_arch = "wasm32"))] +const MAX_DISK_SERIES: usize = 8; + impl Default for Frames { fn default() -> Self { #[cfg(not(target_arch = "wasm32"))] @@ -554,6 +601,16 @@ impl Default for Frames { keepers: Mutex::default(), remote: None, remote_busy: Arc::default(), + #[cfg(not(target_arch = "wasm32"))] + persisted: Arc::default(), + #[cfg(not(target_arch = "wasm32"))] + written: Arc::default(), + #[cfg(not(target_arch = "wasm32"))] + disk_carries: Arc::default(), + #[cfg(not(target_arch = "wasm32"))] + carry_scheduled: Arc::default(), + #[cfg(not(target_arch = "wasm32"))] + disk_series: Arc::default(), } } } @@ -789,29 +846,51 @@ impl Frames { let Ok(mut bases) = self.bases.lock() else { return (plain, None) }; // Markers are pixels only to the expressions that read them (#1083). let markers = project.reads_markers(); + // Whose audio is drawn, before or after the edit (a waveform added or removed counts). + let heard = bases.get(&comp.0).map_or_else(|| project.heard_layers(), |(_, base)| project.heard_layers().union(&base.heard_layers())); if let Some((base_id, base)) = bases.get(&comp.0) && *base_id != plain - && comp_content_with(project, comp, project, |id, c| match base.item(id).map(|i| &i.kind) { - Some(ItemKind::Comp(b)) if Arc::ptr_eq(b, c) || b.same_pixels_with(c, markers) => b, - _ => c, - }) == *base_id + && comp_content_with( + project, + comp, + project, + |id, c| match base.item(id).map(|i| &i.kind) { + Some(ItemKind::Comp(b)) if Arc::ptr_eq(b, c) || b.same_pixels_with(c, markers, &|l| heard.contains(id, l)) => b, + _ => c, + }, + true, + ) == *base_id { return (*base_id, Some(base.clone())); } - // Only layers of this comp changed: its frames outside their span (and a frame either - // side for rounding) keep drawing the same pixels, so a cut or trim recaches only the - // layers it touched (#1641). + // Only comps changed (not the footage, solids or settings they draw): its frames outside + // the spans the edit reaches (and a frame either side for rounding) keep drawing the same + // pixels, so a nudge, cut or trim, here or in a precomp, recaches only the frames it + // touched (#1641). Expressions can read anything at any time, so with them a change + // elsewhere in the project (another comp, an item's name) reaches the frames of the + // layers that have one. + let base_comp = |id: ItemId| bases.get(&comp.0).and_then(|(_, base)| base.item(id)).map(|i| &i.kind); + let base_tree = |id, c| match base_comp(id) { + Some(ItemKind::Comp(b)) if id == comp || Arc::ptr_eq(b, c) || b.same_pixels_with(c, markers, &|l| heard.contains(id, l)) => b, + _ => c, + }; + let base_comps = |id, c| match base_comp(id) { + Some(ItemKind::Comp(b)) => b, + _ => c, + }; if let Some((base_id, base)) = bases.get(&comp.0) - && let (Some(old), Some(new)) = (base.comp(comp), project.comp(comp)) - && comp_content_with(project, comp, base, |id, c| match base.item(id).map(|i| &i.kind) { - Some(ItemKind::Comp(b)) if id == comp || Arc::ptr_eq(b, c) || b.same_pixels_with(c, markers) => b, - _ => c, - }) == *base_id - && let Some((s, e)) = new.changed_span(old) + && comp_content_with(project, comp, base, base_comps, false) == comp_content_with(base, comp, base, |_, c| c, false) + && let Some(dirty) = dirty_frames(base, project, comp, &heard, comp_content_with(project, comp, base, base_tree, true) != *base_id) && let Ok(mut cache) = self.cache.lock() { - let (first, last) = (new.frame_rate.frame_at(s).saturating_sub(1), new.frame_rate.frame_at(e).saturating_add(1)); - cache.carry(*base_id, plain, |f| f < first || f > last); + cache.carry(*base_id, plain, |f| !dirty.iter().any(|(lo, hi)| (*lo..=*hi).contains(&f))); + #[cfg(not(target_arch = "wasm32"))] + if let Ok(mut q) = self.disk_carries.lock() { + if q.len() >= MAX_KEEPERS { + q.pop_front(); + } + q.push_back(DiskCarry { comp: comp.0, from: base.clone(), to: project.clone(), dirty }); + } } if bases.len() > MAX_KEEPERS { bases.clear(); @@ -857,6 +936,11 @@ impl Frames { } let epoch = self.virtual_epoch.load(Ordering::Relaxed); #[cfg(not(target_arch = "wasm32"))] + { + self.note_disk_series(base.comp, base.scale, base.view ^ opts_hash(opts)); + self.carry_disk(src); + } + #[cfg(not(target_arch = "wasm32"))] let content = { let known = self.content_keys.lock().unwrap_or_else(std::sync::PoisonError::into_inner).get(&(base.revision, base.comp, epoch)).copied(); let Some(content) = known else { @@ -872,6 +956,145 @@ impl Frames { (start..end).filter(|f| !ram.contains(f)).filter(|f| contains(disk_cache::frame_key(content, *f, base.scale, base.view ^ oh))).collect() } + /// Write the RAM frames of `base`'s series that the disk cache lacks: frames an edit kept + /// (written under the comp's previous content key) and writes the disk cache dropped. They + /// are converted on the frame pool, each queued once. `true` while there may be more to + /// write (a content key being computed, frames being converted): call again soon, since a + /// refused write is only retried by a later call. Disk keys hash `opts` the way the worker + /// writes them (`base.opts` also carries the Roto Brush model stamp). + #[cfg(not(target_arch = "wasm32"))] + pub fn persist_missing(&self, src: &RenderSource, base: &FrameKey, opts: &RenderOpts) -> bool { + let Some(dc) = src.disk.clone() else { return false }; + let oh = opts_hash(opts); + self.note_disk_series(base.comp, base.scale, base.view ^ oh); + self.carry_disk(src); + let epoch = self.virtual_epoch.load(Ordering::Relaxed); + let known = self.content_keys.lock().unwrap_or_else(std::sync::PoisonError::into_inner).get(&(base.revision, base.comp, epoch)).copied(); + let Some(content) = known else { + self.request_content_key(src.project.clone(), base.revision, base.comp); + return true; + }; + let (missing, converting): (Vec<(u128, Arc)>, bool) = { + let (Ok(c), Ok(mut queued), Ok(mut written)) = (self.cache.lock(), self.persisted.lock(), self.written.lock()) else { return false }; + if queued.len() > 100_000 { + queued.clear(); + } + let limit = dc.limit_bytes(); + if written.0 != limit || written.1.len() > 100_000 { + *written = (limit, HashMap::new()); + } + let missing = c + .map + .iter() + .filter(|(k, _)| k.same_series(base)) + .filter_map(|(k, img)| { + let FrameImage::Cpu(img) = img else { return None }; + let dk = disk_cache::frame_key(content, k.frame, k.scale, k.view ^ oh); + let tries = written.1.get(&dk).copied().unwrap_or(0); + (tries < 2 && !dc.contains(disk_cache::Kind::Frame, dk) && !dc.frame_pending(dk) && queued.insert(dk)).then(|| (dk, img.clone())) + }) + .collect(); + (missing, !queued.is_empty()) + }; + if missing.is_empty() { + return converting; + } + let (persisted, written) = (self.persisted.clone(), self.written.clone()); + let work = move || { + for (dk, img) in missing { + let rgba: Vec = img.pixels.iter().flat_map(|c| c.to_array()).collect(); + // (accepted: the disk cache tracks it until written, and it counts as a try; + // refused by a full queue: retried, without counting) + if dc.put_frame(dk, img.size[0] as u32, img.size[1] as u32, &rgba) + && let Ok(mut w) = written.lock() + { + let n = w.1.entry(dk).or_default(); + *n = n.saturating_add(1); + } + if let Ok(mut q) = persisted.lock() { + q.remove(&dk); + } + } + }; + match &self.pool { + Some(pool) => pool.spawn(work), + None => rayon::spawn(work), + } + true + } + + /// Remember that `comp`'s frames are cached on disk at `scale` with view and options hash + /// `view` (the most recent [`MAX_DISK_SERIES`]). + #[cfg(not(target_arch = "wasm32"))] + fn note_disk_series(&self, comp: u64, scale: u32, view: u64) { + if let Ok(mut m) = self.disk_series.lock() { + let v = m.entry(comp).or_default(); + if v.first() != Some(&(scale, view)) { + v.retain(|s| *s != (scale, view)); + v.insert(0, (scale, view)); + v.truncate(MAX_DISK_SERIES); + } + } + } + + /// Find the frames edits kept ([`Frames::identity`]) on disk under their comps' new content + /// keys: link each kept frame of every disk series of the comp to its new key, on the frame + /// pool, oldest edit first. + #[cfg(not(target_arch = "wasm32"))] + fn carry_disk(&self, src: &RenderSource) { + let Some(dc) = src.disk.clone() else { return }; + let pending = |q: &Mutex>| q.lock().is_ok_and(|q| !q.is_empty()); + // (called on every paint: one job drains every carry queued meanwhile) + if !pending(&self.disk_carries) || self.carry_scheduled.swap(true, Ordering::AcqRel) { + return; + } + let (carries, scheduled, series, ctx) = (self.disk_carries.clone(), self.carry_scheduled.clone(), self.disk_series.clone(), self.ctx.clone()); + let virtual_id = self.virtual_id.clone(); + let work = move || { + let mut linked = false; + loop { + let todo: Vec = carries.lock().map(|mut q| q.drain(..).collect()).unwrap_or_default(); + if todo.is_empty() { + scheduled.store(false, Ordering::Release); + // (a carry queued after the drain and before the store: keep it, unless a new job took it) + if pending(&carries) && !scheduled.swap(true, Ordering::AcqRel) { + continue; + } + break; + } + for c in todo { + // (a panicking carry is skipped: the others still carry, and the flag is + // still cleared below) + let carry = || { + let Some(series) = series.lock().ok().and_then(|m| m.get(&c.comp).cloned()) else { return false }; + let comp = ItemId(c.comp); + let n = c.to.comp(comp).map_or(0, |k| k.frame_rate.frame_at(k.duration)).clamp(0, 100_000); + let key = |p: &Project| disk_cache::comp_content_key_with(p, comp, virtual_id.as_deref()); + let (a, b) = (key(&c.from), key(&c.to)); + if a == b { + return false; + } + let mut linked = false; + for f in (0..n).filter(|f| !c.dirty.iter().any(|(lo, hi)| (*lo..=*hi).contains(f))) { + for (scale, view) in &series { + linked |= dc.link_frame(disk_cache::frame_key(a, f, *scale, *view), disk_cache::frame_key(b, f, *scale, *view)); + } + } + linked + }; + linked |= std::panic::catch_unwind(std::panic::AssertUnwindSafe(carry)).unwrap_or(false); + } + } + if linked && let Some(ctx) = &ctx { + ctx.request_repaint(); + } + }; + match &self.pool { + Some(pool) => pool.spawn(work), + None => rayon::spawn(work), + } + } + #[cfg(not(target_arch = "wasm32"))] fn request_content_key(&self, project: Arc, revision: u64, comp: u64) { let mut jobs = self.content_key_jobs.lock().unwrap_or_else(std::sync::PoisonError::into_inner); diff --git a/crates/ui-egui/src/frames/tests.rs b/crates/ui-egui/src/frames/tests.rs index ace51b0bb..4099aa4c6 100644 --- a/crates/ui-egui/src/frames/tests.rs +++ b/crates/ui-egui/src/frames/tests.rs @@ -908,8 +908,8 @@ fn audio_lock_and_shy_switches_keep_the_frames() { assert_ne!(at(&s), o1); } -/// A cut or trim recaches only the frames of the layers it touched (#1641), and none are -/// kept when the change reaches other layers (here a child of the trimmed layer). +/// A cut or trim recaches only the frames of the layers it touched (#1641), and those of the +/// layers the change reaches (here a child of the trimmed layer). #[test] fn an_edit_keeps_the_frames_outside_the_layers_it_changed() { use effectcraft_engine::Session; @@ -935,14 +935,18 @@ fn an_edit_keeps_the_frames_outside_the_layers_it_changed() { s.execute("edit.splitLayer", json!({"layers": [a]})).unwrap(); let after = f.content_of(&s.project, s.revision, comp); assert_ne!(after, before); - let kept: Vec = (fr.frame_at(out) + 2..frames.len() as i64).collect(); - assert_eq!(cached(after), kept, "the cut layer's frames (and one either side) render again; the rest stay"); + let kept: Vec = (0..fr.frame_at(Tick::from_seconds_f64(2.0)) - 1).chain(fr.frame_at(out) + 2..frames.len() as i64).collect(); + assert_eq!(cached(after), kept, "the copy's half (and one frame either side) renders again; the rest stay"); s.execute("layer.setParent", json!({"layers": [b], "parent": a})).unwrap(); let parented = cache_all(&s); s.execute("layer.timing", json!({"layers": [a], "out": 1})).unwrap(); - assert_eq!(cached(f.content_of(&s.project, s.revision, comp)), Vec::::new(), "the trimmed layer's child moves everywhere"); - assert_eq!(cached(parented).len(), frames.len(), "the earlier frames stay for undo"); + // The cut (1–2 s) and the trimmed layer's child (4–10 s) render again, a frame either side; + // the split's copy (2–4 s) stays. + assert_eq!(cached(f.content_of(&s.project, s.revision, comp)), (0..=27).chain(62..=117).collect::>(), "the trimmed layer's child moves where it is"); + s.execute("edit.undo", json!({})).unwrap(); + assert_eq!(f.content_of(&s.project, s.revision, comp), parented); + assert_eq!(cached(parented).len(), frames.len(), "undo finds every earlier frame"); } /// Every frame an edit keeps renders as a fresh render of the edited project does: Solo @@ -981,6 +985,16 @@ fn the_frames_an_edit_keeps_match_a_fresh_render() { assert_eq!(kept(&mut s, comp, solo(true)), Vec::::new()); assert_eq!(kept(&mut s, comp, solo(false)), Vec::::new()); + // The work area is a preview and render range: setting it (B, N) keeps every frame, on + // disk too. + let n = s.project.comp(comp).map(|c| c.frame_rate.frame_at(c.duration)).unwrap(); + let disk = |s: &Session| effectcraft_engine::render::disk_cache::comp_content_key_with(&s.project, comp, None); + let d0 = disk(&s); + let work_area = |start: f64, end: f64| json!({"cmd": "comp.workArea", "params": {"comp": comp.0, "start": start, "end": end}}); + assert_eq!(kept(&mut s, comp, work_area(1.0, 4.0)), (0..n).collect::>()); + assert_eq!(kept(&mut s, comp, work_area(2.0, 3.0)), (0..n).collect::>()); + assert_eq!(disk(&s), d0, "the disk cache keeps them"); + // A 2–3 s camera over a 4 s motion-blurred 3D layer, shutter 720° at phase 360°: a frame // samples 1 to 3 frames ahead, so moving the camera reaches the 3 frames before it. let comp = ItemId(id( @@ -1113,3 +1127,1137 @@ fn timing_markers_and_expressions_still_change_the_frames() { s.execute("layer.addMarker", json!({"comp": outer.0, "layers": [solid], "time": 0.75})).unwrap(); assert_ne!(at(&s), o2, "an expression can read the marker, so the frames change"); } + +/// The oracle for [`Comp::changed_spans`]: every frame an edit keeps renders the same in the edited +/// project. Moves, trims, splits, deletes and duplicates of moving clips, under adjustment +/// layers, with seeded noise, motion blur, mattes, parents and effects that read other times. +#[test] +fn kept_frames_render_the_same() { + use effectcraft_engine::Session; + use effectcraft_engine::render::render_frame; + use serde_json::{Value as Json, json}; + fn run(s: &mut Session, cmd: &str, p: Json) -> Json { + s.execute(cmd, p).unwrap_or_else(|e| panic!("{cmd}: {e}")) + } + /// Set a layer's start, in and out (frames). + fn timing(s: &mut Session, cid: ItemId, l: u64, start: i64, inp: i64, out: i64) { + let c = Arc::make_mut(&mut s.project).comp_mut(cid).unwrap(); + let fr = c.frame_rate; + let l = c.layer_mut(effectcraft_engine::project::LayerId(l)).unwrap(); + (l.start_time, l.in_point, l.out_point) = (fr.tick_of(start), fr.tick_of(inp), fr.tick_of(out)); + } + /// A 10×10 solid moving across the comp. + fn solid(s: &mut Session, color: &str) -> u64 { + let l = run(s, "layer.newSolid", json!({"color": color, "width": 10, "height": 10}))["layer"].as_u64().unwrap(); + run(s, "prop.addKey", json!({"layer": l, "path": "transform/position", "time": 0.0, "value": [0.0, 4.0, 0.0]})); + run(s, "prop.addKey", json!({"layer": l, "path": "transform/position", "time": 1.5, "value": [16.0, 12.0, 0.0]})); + l + } + fn split(s: &mut Session, cid: ItemId, l: u64, at: i64) { + let fr = s.project.comp(cid).unwrap().frame_rate; + s.set_time(fr.tick_of(at)); + run(s, "edit.splitLayer", json!({"layers": [l]})); + } + fn adjustment(s: &mut Session, cid: ItemId, effect: &str) -> u64 { + let l = run(s, "layer.newAdjustment", json!({}))["layer"].as_u64().unwrap(); + run(s, "effect.apply", json!({"layers": [l], "effect": effect})); + timing(s, cid, l, 0, 10, 35); + l + } + /// Echo reaching 9 frames back, three times (past the shutter margin). + fn long_echo(s: &mut Session, l: u64) { + run(s, "effect.apply", json!({"layers": [l], "effect": "Echo"})); + run(s, "prop.set", json!({"layer": l, "path": "effects/#1/echoTime", "value": -0.3})); + run(s, "prop.set", json!({"layer": l, "path": "effects/#1/numberOfEchoes", "value": 3.0})); + } + // Clip a on frames 0–20, clip b on 20–40 of a 45-frame comp. + type Step = fn(&mut Session, ItemId, &mut Vec); + let none: Step = |_, _, _| {}; + /// An audio-only file (music) and a layer of it over the whole comp. + fn music(s: &mut Session, cid: ItemId) -> u64 { + use effectcraft_engine::project::{Footage, FootageKind}; + let song = Footage { kind: FootageKind::Audio, has_video: false, has_audio: true, ..Default::default() }; + let song = Arc::make_mut(&mut s.project).add_item("song.wav", Default::default(), None, ItemKind::Footage(song)); + let l = run(s, "layer.addItem", json!({"item": song.0}))["layer"].as_u64().unwrap(); + timing(s, cid, l, 0, 0, 45); + l + } + /// What an edit keeps: every frame, some, or none. + #[derive(Debug, PartialEq)] + enum Keep { + All, + Part, + Nothing, + } + let scenarios: Vec<(&str, Step, Step, Keep)> = vec![ + ("nudge", none, |s, c, l| timing(s, c, l[1], 21, 21, 41), Keep::Part), + ("trim out", none, |s, c, l| timing(s, c, l[1], 20, 20, 32), Keep::Part), + ("trim in", none, |s, c, l| timing(s, c, l[1], 20, 28, 40), Keep::Part), + ("split", none, |s, c, l| split(s, c, l[1], 30), Keep::Part), + ("delete", none, |s, _, l| _ = run(s, "edit.clear", json!({"layers": [l[1]]})), Keep::Part), + ("duplicate", none, |s, _, l| _ = run(s, "edit.duplicate", json!({"layers": [l[0]]})), Keep::Part), + ("adjustment trim", |s, c, l| l.push(adjustment(s, c, "Gaussian Blur")), |s, c, l| timing(s, c, l[2], 0, 10, 25), Keep::Part), + ("adjustment split", |s, c, l| l.push(adjustment(s, c, "Invert")), |s, c, l| split(s, c, l[2], 22), Keep::Part), + ( + "adjustment fx off", + |s, c, l| l.push(adjustment(s, c, "Invert")), + |s, _, l| _ = run(s, "layer.setSwitch", json!({"layers": [l[2]], "switch": "fx", "value": false})), + Keep::Part, + ), + // Music changes no picture; neither does it when imported with the same action. + ("add music", none, |s, c, _| _ = music(s, c), Keep::All), + ("trim music", |s, c, l| l.push(music(s, c)), |s, c, l| timing(s, c, l[2], 0, 0, 20), Keep::All), + ("move music", |s, c, l| l.push(music(s, c)), |s, c, l| timing(s, c, l[2], 5, 5, 45), Keep::All), + ("delete music", |s, c, l| l.push(music(s, c)), |s, _, l| _ = run(s, "edit.clear", json!({"layers": [l[2]]})), Keep::All), + ("hide a clip", none, |s, _, l| _ = run(s, "layer.setSwitch", json!({"layers": [l[1]], "switch": "video", "value": false})), Keep::Part), + // Music another layer reads isn't silent: like any layer another one reads, editing + // it re-renders everything. + ( + "trim music an audio spectrum reads", + |s, c, l| { + let m = music(s, c); + run(s, "effect.apply", json!({"layers": [l[0]], "effect": "Audio Spectrum"})); + run(s, "prop.set", json!({"layer": l[0], "path": "effects/#1/audioLayer", "value": m})); + l.push(m); + }, + |s, c, l| timing(s, c, l[2], 0, 0, 20), + Keep::Nothing, + ), + ( + "noise split", + |s, _, l| _ = run(s, "effect.apply", json!({"layers": [l[1]], "effect": "Fractal Noise"})), + |s, c, l| split(s, c, l[1], 30), + Keep::Part, + ), + ( + "motion blur trim", + |s, c, l| { + Arc::make_mut(&mut s.project).comp_mut(c).unwrap().enable_motion_blur = true; + run(s, "layer.setSwitch", json!({"layers": [l[1]], "switch": "motionBlur", "value": true})); + }, + |s, c, l| timing(s, c, l[1], 20, 20, 32), + Keep::Part, + ), + ( + "matted clip trim", + |s, c, l| { + let m = solid(s, "#808080"); + timing(s, c, m, 0, 0, 45); + run(s, "layer.setTrackMatte", json!({"layer": l[1], "matte": m})); + }, + |s, c, l| timing(s, c, l[1], 20, 20, 32), + Keep::Part, + ), + ( + "parent move", + |s, _, l| _ = run(s, "layer.setParent", json!({"layers": [l[0]], "parent": l[1]})), + |s, c, l| timing(s, c, l[1], 21, 21, 41), + // (the child re-renders where it is active, the rest stays) + Keep::Part, + ), + ("echo trim in", |s, _, l| long_echo(s, l[1]), |s, c, l| timing(s, c, l[1], 20, 28, 40), Keep::Part), + ("echo trim out", |s, _, l| long_echo(s, l[1]), |s, c, l| timing(s, c, l[1], 20, 20, 32), Keep::Part), + ( + "posterize time trim in", + |s, _, l| { + run(s, "effect.apply", json!({"layers": [l[1]], "effect": "Posterize Time"})); + run(s, "prop.set", json!({"layer": l[1], "path": "effects/#1/frameRate", "value": 3.0})); + }, + |s, c, l| timing(s, c, l[1], 20, 28, 40), + Keep::Part, + ), + ( + "time displacement trim in", + |s, _, l| _ = run(s, "effect.apply", json!({"layers": [l[1]], "effect": "Time Displacement"})), + |s, c, l| timing(s, c, l[1], 20, 28, 40), + Keep::Part, + ), + // (Echo on an adjustment layer doesn't read other times yet, `FxHost::self_at`; + // when it does, an edit under it reaches frames after the edit.) + ( + "echo adjustment nudge", + |s, _, l| { + let a = run(s, "layer.newAdjustment", json!({}))["layer"].as_u64().unwrap(); + long_echo(s, a); + l.push(a); + }, + |s, c, l| timing(s, c, l[1], 21, 21, 41), + Keep::Part, + ), + ]; + for (name, setup, edit, expect) in scenarios { + let mut s = Session::default(); + let cid = ItemId(run(&mut s, "comp.new", json!({"name": "A", "width": 16, "height": 16, "duration": 1.5}))["comp"].as_u64().unwrap()); + let mut l = vec![solid(&mut s, "#ff0000"), solid(&mut s, "#00ff00")]; + timing(&mut s, cid, l[0], 0, 0, 20); + timing(&mut s, cid, l[1], 20, 20, 40); + setup(&mut s, cid, &mut l); + let p0 = s.project.clone(); + edit(&mut s, cid, &mut l); + let p1 = s.project.clone(); + let fr = p1.comp(cid).unwrap().frame_rate; + let frames = fr.frame_at(p1.comp(cid).unwrap().duration); + let dirty = dirty_frames(&p0, &p1, cid, &p1.heard_layers().union(&p0.heard_layers()), false); + let kept: Vec = match &dirty { + Some(d) => (0..frames).filter(|f| !d.iter().any(|(lo, hi)| (*lo..=*hi).contains(f))).collect(), + None => vec![], + }; + let got = match kept.len() as i64 { + 0 => Keep::Nothing, + n if n == frames => Keep::All, + _ => Keep::Part, + }; + assert_eq!(got, expect, "{name}: dirty {dirty:?}"); + for f in kept { + let (a, b) = (render_frame(&p0, cid, fr.tick_of(f), 1.0), render_frame(&p1, cid, fr.tick_of(f), 1.0)); + assert!(a.data == b.data, "{name}: kept frame {f} renders differently after the edit"); + } + } +} + +/// An edit keeps the cached frames it can't reach: nudging a clip re-renders only where it +/// was and is, a trim only the part it cut, a split only the copy's half (its new ids seed +/// randomness). Undo finds them all again; a clip another layer reads (its parent here) also +/// drops the frames of that layer. +#[test] +fn an_edit_keeps_the_frames_it_cant_reach() { + use effectcraft_engine::Session; + use effectcraft_engine::project::{Layer, LayerId}; + use serde_json::json; + type Edit = fn(&mut Comp, LayerId, &dyn Fn(i64) -> Tick); + let nudge: Edit = |c, b, t| { + let l = c.layer_mut(b).unwrap(); + (l.start_time, l.in_point, l.out_point) = (l.start_time + t(1), l.in_point + t(1), l.out_point + t(1)); + }; + let trim: Edit = |c, b, t| c.layer_mut(b).unwrap().out_point = t(75); + let split: Edit = |c, b, t| { + let i = c.layers.iter().position(|l| l.id == b).unwrap(); + let mut copy: Layer = c.layers[i].clone(); + copy.id = LayerId(1000); + copy.props.reassign_uids(&mut 1000); + copy.in_point = t(75); + c.layers[i].out_point = t(75); + c.layers.insert(i, copy); + }; + let cut: Vec = (0..=72).chain(92..=95).collect(); + for (edit, kept) in [(nudge, (0..=57).chain(93..=95).collect::>()), (trim, cut.clone()), (split, cut)] { + for parented in [false, true] { + let mut s = Session::default(); + let cid = ItemId(s.execute("comp.new", json!({"name": "A", "width": 8, "height": 8, "duration": 4})).unwrap()["comp"].as_u64().unwrap()); + s.execute("layer.newSolid", json!({})).unwrap(); + s.execute("layer.newSolid", json!({})).unwrap(); + let fr = s.project.comp(cid).unwrap().frame_rate; + let t = |f: i64| fr.tick_of(f); + let b = { + let c = Arc::make_mut(&mut s.project).comp_mut(cid).unwrap(); + let (a, b): (LayerId, LayerId) = (c.layers[0].id, c.layers[1].id); + let l = c.layer_mut(a).unwrap(); + (l.in_point, l.out_point) = (t(0), t(30)); + l.parent = parented.then_some(b); + let l = c.layer_mut(b).unwrap(); + (l.start_time, l.in_point, l.out_point) = (t(60), t(60), t(90)); + b + }; + let p0 = s.project.clone(); + let f = Frames::default(); + let k = |content: u64, frame: i64| FrameKey { content, comp: cid.0, ..key(0, frame, 0) }; + let id0 = f.content_of(&p0, 1, cid); + for fr in 0..=95 { + insert(&f, k(id0, fr)); + } + let mut p1 = p0.clone(); + edit(Arc::make_mut(&mut p1).comp_mut(cid).unwrap(), b, &t); + let id1 = f.content_of(&p1, 2, cid); + assert_ne!(id0, id1); + let got = f.cached_frames(&k(id1, 0)); + // (the child, 0–30, renders again too) + assert_eq!(got, if parented { kept.iter().copied().filter(|f| *f > 31).collect() } else { kept.clone() }); + assert_eq!(f.content_of(&p0, 3, cid), id0, "undo"); + assert_eq!(f.cached_frames(&k(id0, 0)), (0..=95).collect::>()); + } + } +} + +/// A saved and reopened project finds its frames on disk: the content key is the same. +#[test] +fn disk_keys_survive_save_and_reopen() { + use serde_json::json; + let dir = std::env::temp_dir().join(format!("ec-disk-keys-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let png = dir.join("still.png"); + image::RgbaImage::from_pixel(4, 4, image::Rgba([200, 100, 50, 255])).save(&png).unwrap(); + let mut s = effectcraft_host::session(); + let cid = ItemId(s.execute("comp.new", json!({"name": "Keys", "width": 16, "height": 16, "duration": 1})).unwrap()["comp"].as_u64().unwrap()); + let item = s.execute("file.import", json!({"paths": [png.to_string_lossy()]})).unwrap()["items"][0].clone(); + s.execute("layer.addItem", json!({"item": item})).unwrap(); + s.execute("layer.newSolid", json!({"color": "#00ff00", "width": 8, "height": 8})).unwrap(); + let adj = s.execute("layer.newAdjustment", json!({})).unwrap()["layer"].clone(); + s.execute("effect.apply", json!({"layers": [adj], "effect": "Gaussian Blur"})).unwrap(); + let before = disk_cache::comp_content_key(&s.project, cid); + let path = dir.join("keys.ecproj"); + s.execute("file.saveAs", json!({"path": path.to_string_lossy()})).unwrap(); + let mut again = effectcraft_host::session(); + again.execute("file.open", json!({"path": path.to_string_lossy()})).unwrap(); + assert_eq!(disk_cache::comp_content_key(&again.project, cid), before); + let _ = std::fs::remove_dir_all(&dir); +} + +/// Frames in RAM that the disk lacks (an edit kept them; a write was dropped) are written +/// when idle, so the reopened project shows them all as the blue bar. The RAM series carries a +/// Roto Brush model stamp in its opts; the disk keys are the worker's, without it. +#[test] +fn ram_frames_missing_on_disk_are_written() { + use effectcraft_engine::Session; + use serde_json::json; + let dir = std::env::temp_dir().join(format!("ec-persist-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + let disk = DiskCache::open(&dir, 1 << 30).unwrap(); + let mut s = Session::default(); + let cid = ItemId(s.execute("comp.new", json!({"name": "A", "width": 8, "height": 8, "duration": 1})).unwrap()["comp"].as_u64().unwrap()); + s.execute("layer.newSolid", json!({})).unwrap(); + let src = RenderSource { + project: s.project.clone(), + footage: Arc::new(effectcraft_engine::render::NoFootage), + expr: None, + layer_cache: Arc::default(), + gpu: None, + gpu_display: false, + disk: Some(disk.clone()), + }; + let f = Frames::default(); + let series = + FrameKey { revision: 1, content: f.content_of(&src.project, 1, cid), comp: cid.0, opts: opts_hash(&RenderOpts::default()) ^ 1, ..key(0, 0, 0) }; + for fr in 0..10 { + insert(&f, FrameKey { frame: fr, ..series }); + } + let on_disk = |fr: i64| { + disk.contains( + disk_cache::Kind::Frame, + frame_disk_key( + &f.content_keys, + &src.project, + &FrameKey { frame: fr, ..series }, + opts_hash(&RenderOpts::default()), + f.virtual_epoch.load(Ordering::Relaxed), + None, + ), + ) + }; + let persist = || { + // The content key is computed on the frame pool first. + for _ in 0..1000 { + f.persist_missing(&src, &series, &RenderOpts::default()); + disk.flush(); + // (`flush` waits for the disk writes, not for the conversion to forget its keys) + if (0..10).all(on_disk) && f.persisted.lock().unwrap().is_empty() { + return true; + } + std::thread::sleep(std::time::Duration::from_millis(5)); + } + false + }; + let evict = || { + disk.set_max_bytes(1); + disk.set_max_bytes(1 << 30); + assert!(!(0..10).any(on_disk)); + }; + assert!(persist(), "every RAM frame written"); + assert!(!f.persist_missing(&src, &series, &RenderOpts::default()), "nothing left to write: no more idle passes"); + // Evicted from disk: the RAM frames are written again, once. + evict(); + assert!(persist(), "evicted frames written again"); + // Reopened: a new preview cache over the same disk shows every frame in blue. + let reopened = Frames::default(); + let reopened_series = FrameKey { content: reopened.content_of(&src.project, 1, cid), ..series }; + let mut blue = vec![]; + for _ in 0..1000 { + blue = reopened.disk_frames(&src, &reopened_series, 0..10, &RenderOpts::default()); + if blue.len() == 10 { + break; + } + std::thread::sleep(std::time::Duration::from_millis(5)); + } + assert_eq!(blue, (0..10).collect::>()); + // Evicted again (a disk cache too small for them): not rewritten on every idle pass. + evict(); + assert!(!f.persist_missing(&src, &series, &RenderOpts::default()), "evicted twice: no more idle passes"); + disk.flush(); + assert!(!(0..10).any(on_disk)); + let _ = std::fs::remove_dir_all(&dir); +} + +/// Audio changes pixels only where something draws it: a clip's Audio Levels, audio effects and +/// Audio switch keep every frame, in RAM and on disk, until a waveform picks the clip (on another +/// layer, on itself, or through a precomp of it); then the frames of the reader render again. +#[test] +fn audio_edits_keep_the_frames_until_something_draws_the_audio() { + use effectcraft_engine::Session; + use effectcraft_engine::project::{Footage, GroupKind, LayerId, Value, build}; + use effectcraft_engine::render::disk_cache::comp_content_key_with; + use serde_json::json; + let mut s = Session::default(); + let cid = ItemId(s.execute("comp.new", json!({"name": "A", "width": 8, "height": 8, "duration": 4, "frameRate": 30})).unwrap()["comp"].as_u64().unwrap()); + s.execute("layer.newSolid", json!({})).unwrap(); + let mut p = (*s.project).clone(); + let clip = + p.add_item("clip.mov", effectcraft_color::Label::None, None, ItemKind::Footage(Footage { has_video: true, has_audio: true, ..Default::default() })); + let comp = p.comp(cid).unwrap().clone(); + let fr = comp.frame_rate; + let t = move |f: i64| fr.tick_of(f); + let v = build::layer(&mut p, &comp, "Clip", LayerSource::Footage { item: clip }, (8, 8), None); + let (clip_layer, solid) = (v.id, comp.layers[0].id); + p.comp_mut(cid).unwrap().layers.push(v); + for (id, a, b) in [(solid, 0, 30), (clip_layer, 90, 120)] { + let l = p.comp_mut(cid).unwrap().layer_mut(id).unwrap(); + (l.start_time, l.in_point, l.out_point) = (t(a), t(a), t(b)); + } + // The outer comp shows the whole of A. + let outer = p.add_item("Outer", effectcraft_color::Label::None, None, ItemKind::Comp(Arc::new(Comp { layers: Default::default(), ..comp.clone() }))); + let oc = p.comp(outer).unwrap().clone(); + let mut pre = build::layer(&mut p, &oc, "Pre", LayerSource::Comp { item: cid }, (8, 8), None); + (pre.start_time, pre.in_point, pre.out_point) = (t(0), t(0), t(120)); + let (pre_layer, reader) = (pre.id, oc.layers.first().map(|l| l.id)); + assert!(reader.is_none()); + p.comp_mut(outer).unwrap().layers.push(pre); + // An effect on `on` (of comp `c`) that picks layer `pick`. + let pick = |p: &mut Project, c: ItemId, on: LayerId, pick: LayerId| { + let mut n = 1_000_000; + let mut ids = build::Ids(&mut n); + let mut g = ids.group("waveform", "Audio Waveform"); + g.kind = GroupKind::Effect { effect: "ec.generate.audiowaveform".into() }; + g.children.push(ids.prop("audioLayer", "Audio Layer", Value::Layer(Some(pick.0))).into()); + p.comp_mut(c).unwrap().layer_mut(on).unwrap().props.group_mut("effects").unwrap().children.push(g.into()); + }; + let levels = |p: &mut Project| { + p.comp_mut(cid).unwrap().layer_mut(clip_layer).unwrap().props.prop_mut("audio/levels").unwrap().set_value_at(t(0), Value::Vec2([-6.0, -6.0])) + }; + let backwards = |p: &mut Project| { + let mut n = 2_000_000; + let mut g = build::Ids(&mut n).group("backwards", "Backwards"); + g.kind = GroupKind::Effect { effect: "ec.audio.backwards".into() }; + p.comp_mut(cid).unwrap().layer_mut(clip_layer).unwrap().props.group_mut("effects").unwrap().children.push(g.into()); + }; + let mute = |p: &mut Project| p.comp_mut(cid).unwrap().layer_mut(clip_layer).unwrap().switches.audio = false; + // The frames of `view` an edit keeps, and whether its disk key stays. + let kept = |p0: &Project, view: ItemId, edit: &dyn Fn(&mut Project)| { + let p0 = Arc::new(p0.clone()); + let f = Frames::default(); + let k = |content: u64, frame: i64| FrameKey { content, comp: view.0, ..key(0, frame, 0) }; + let id0 = f.content_of(&p0, 1, view); + (0..120).for_each(|n| insert(&f, k(id0, n))); + let mut p1 = p0.clone(); + edit(Arc::make_mut(&mut p1)); + let id1 = f.content_of(&p1, 2, view); + (f.cached_frames(&k(id1, 0)), comp_content_key_with(&p0, view, None) == comp_content_key_with(&p1, view, None)) + }; + let all: Vec = (0..120).collect(); + let edits: [(&str, &dyn Fn(&mut Project)); 3] = [("levels", &levels), ("audio effect", &backwards), ("audio switch", &mute)]; + for (name, edit) in edits { + assert_eq!(kept(&p, cid, edit), (all.clone(), true), "{name}: nothing draws the clip's audio"); + assert_eq!(kept(&p, outer, edit), (all.clone(), true), "{name}: nor through the precomp"); + + let mut q = p.clone(); + pick(&mut q, cid, solid, clip_layer); + let (frames, disk) = kept(&q, cid, edit); + assert!(!disk, "{name}: the solid draws the clip's audio"); + assert!(frames.contains(&60) && !frames.contains(&10) && !frames.contains(&100), "{name}: the reader's and the clip's frames go: {frames:?}"); + + let mut q = p.clone(); + pick(&mut q, cid, clip_layer, clip_layer); + let (frames, disk) = kept(&q, cid, edit); + assert!(!disk, "{name}: the clip draws its own audio"); + assert!(frames.contains(&10) && frames.contains(&60) && !frames.contains(&100), "{name}: the clip's frames go: {frames:?}"); + + let mut q = p.clone(); + let r = q.comp(cid).unwrap().layers[0].clone(); + let reader = LayerId(r.id.0 + 500_000); + q.comp_mut(outer).unwrap().layers.insert(0, effectcraft_engine::project::Layer { id: reader, ..r }); + pick(&mut q, outer, reader, pre_layer); + let (frames, disk) = kept(&q, outer, edit); + assert!( + !disk && frames.contains(&60) && !frames.contains(&10) && !frames.contains(&100), + "{name}: a waveform in the outer comp draws the precomp's audio: {frames:?}" + ); + } +} + +/// An expression reaches only the frames of its own layer: with one in the comp, an edit (here +/// or elsewhere in the project: a trim, a rename, a marker, another comp) re-renders that layer's +/// frames besides the edit's own, and every frame kept renders as before, expressions evaluated, +/// also when the expression reads the edited layer. +#[test] +fn an_expression_reaches_only_the_frames_of_its_layer() { + use effectcraft_engine::project::{LayerId, Marker}; + use effectcraft_engine::render::NoFootage; + use serde_json::json; + let mut s = effectcraft_host::session(); + let cid = ItemId(s.execute("comp.new", json!({"name": "A", "width": 16, "height": 16, "duration": 1.5})).unwrap()["comp"].as_u64().unwrap()); + let other = ItemId(s.execute("comp.new", json!({"name": "Other", "width": 8, "height": 8, "duration": 1})).unwrap()["comp"].as_u64().unwrap()); + s.execute("comp.open", json!({"comp": cid.0})).ok(); + let mut solid = |name: &str| { + let l = s.execute("layer.newSolid", json!({"comp": cid.0, "color": "#ff0000", "width": 10, "height": 10})).unwrap()["layer"].as_u64().unwrap(); + s.execute("layer.rename", json!({"comp": cid.0, "layer": l, "name": name})).unwrap(); + for (t, x) in [(0.0, 0.0), (1.5, 16.0)] { + s.execute("prop.addKey", json!({"comp": cid.0, "layer": l, "path": "transform/position", "time": t, "value": [x, 8.0, 0.0]})).unwrap(); + } + LayerId(l) + }; + let (a, b, c) = (solid("A"), solid("B"), solid("C")); + s.execute( + "prop.setExpression", + json!({"comp": cid.0, "layer": c.0, "path": "transform/position", "expression": "thisComp.layer(\"B\").transform.position + [0, time * 4]"}), + ) + .unwrap(); + let fr = s.project.comp(cid).unwrap().frame_rate; + let n = fr.frame_at(s.project.comp(cid).unwrap().duration); + let mut p0 = (*s.project).clone(); + for (id, i, o) in [(a, 0, 20), (b, 20, 40), (c, 40, 45)] { + let l = p0.comp_mut(cid).unwrap().layer_mut(id).unwrap(); + (l.start_time, l.in_point, l.out_point) = (fr.tick_of(i), fr.tick_of(i), fr.tick_of(o)); + } + let p0 = Arc::new(p0); + let host = s.expr.clone().unwrap(); + let render = |p: &Project, i: i64| { + let mut r = Renderer::new(p, &NoFootage, RenderOpts::default()); + r.expr = Some(&*host); + r.comp_frame(cid, fr.tick_of(i)) + }; + type Edit = fn(&mut Project, ItemId, ItemId, [LayerId; 3]); + let edits: [(&str, Edit, i64); 4] = [ + ("trim B", |p, c, _, l| p.comp_mut(c).unwrap().layer_mut(l[1]).unwrap().out_point = p.comp(c).unwrap().frame_rate.tick_of(32), 30), + ("rename A", |p, c, _, l| p.comp_mut(c).unwrap().layer_mut(l[0]).unwrap().name = "Renamed".into(), 38), + ("comp marker", |p, c, _, _| p.comp_mut(c).unwrap().markers.push(Marker { time: Tick::ZERO, ..Default::default() }), 38), + ("another comp", |p, _, o, _| p.comp_mut(o).unwrap().width = 9, 38), + ]; + for (name, edit, kept_from) in edits { + let f = Frames::default(); + let k = |content: u64, frame: i64| FrameKey { content, comp: cid.0, ..key(0, frame, 0) }; + let id0 = f.content_of(&p0, 1, cid); + (0..n).for_each(|i| insert(&f, k(id0, i))); + let mut p1 = p0.clone(); + edit(Arc::make_mut(&mut p1), cid, other, [a, b, c]); + let kept = f.cached_frames(&k(f.content_of(&p1, 2, cid), 0)); + // C (frames 40–45) reads anything: it renders again, the frames before it (and before a + // trim of B) stay. + assert!((0..kept_from).all(|i| kept.contains(&i)) && !kept.contains(&42), "{name}: {kept:?}"); + for i in kept { + assert_eq!(render(&p0, i), render(&p1, i), "{name}: kept frame {i} draws other pixels now"); + } + } +} + +/// An edit inside a precomp re-renders only the frames where the precomp shows what it changed: +/// shifted by the precomp layer's start, scaled by its stretch, and its whole span under a time +/// remap. Every frame kept renders as before. +#[test] +fn an_edit_in_a_precomp_reaches_only_where_it_shows() { + use effectcraft_engine::Session; + use effectcraft_engine::project::{LayerId, Value, build}; + use effectcraft_engine::render::render_frame; + use serde_json::json; + let mut s = Session::default(); + let id = |v: serde_json::Value, k: &str| v[k].as_u64().unwrap(); + let inner = ItemId(id(s.execute("comp.new", json!({"name": "Inner", "width": 16, "height": 16, "duration": 4, "frameRate": 30})).unwrap(), "comp")); + let solid = |s: &mut Session| { + let l = id(s.execute("layer.newSolid", json!({"comp": inner.0, "color": "#ff0000", "width": 6, "height": 6})).unwrap(), "layer"); + for (t, x) in [(0.0, 0.0), (4.0, 16.0)] { + s.execute("prop.addKey", json!({"comp": inner.0, "layer": l, "path": "transform/position", "time": t, "value": [x, 8.0, 0.0]})).unwrap(); + } + LayerId(l) + }; + let (a, b) = (solid(&mut s), solid(&mut s)); + let outer = ItemId(id(s.execute("comp.new", json!({"name": "Outer", "width": 16, "height": 16, "duration": 6, "frameRate": 30})).unwrap(), "comp")); + let pre = LayerId(id(s.execute("layer.addItem", json!({"comp": outer.0, "item": inner.0})).unwrap(), "layer")); + let fr = s.project.comp(outer).unwrap().frame_rate; + let t = |f: i64| fr.tick_of(f); + let mut p0 = (*s.project).clone(); + for (l, i, o) in [(a, 0, 30), (b, 30, 60)] { + let l = p0.comp_mut(inner).unwrap().layer_mut(l).unwrap(); + (l.start_time, l.in_point, l.out_point) = (t(i), t(i), t(o)); + } + let n = fr.frame_at(p0.comp(outer).unwrap().duration); + // The precomp starts at frame 15; `layout` sets its stretch and time remap. + let place = |p: &mut Project, stretch: f64, remap: bool| { + let l = p.comp_mut(outer).unwrap().layer_mut(pre).unwrap(); + (l.start_time, l.in_point, l.out_point, l.stretch) = (t(15), t(15), t(15) + Tick((t(120).0 as f64 * stretch / 100.0) as i64), stretch); + if remap { + let mut uid = 9_000_000; + l.props.children.push(build::Ids(&mut uid).prop("timeRemap", "Time Remap", Value::Scalar(1.5)).into()); + } + }; + // Trim B (inner frames 30–60) to end at 45. + let trim = |p: &mut Project| p.comp_mut(inner).unwrap().layer_mut(b).unwrap().out_point = t(45); + for (name, stretch, remap, kept_frames, dirty_frames) in [ + ("offset", 100.0, false, vec![30, 55, 80, 130], vec![60, 75]), + ("stretched", 200.0, false, vec![30, 100, 140, 179], vec![105, 135]), + ("time remap", 100.0, true, vec![5], vec![30, 100]), + ] { + let mut q = p0.clone(); + place(&mut q, stretch, remap); + let q = Arc::new(q); + let f = Frames::default(); + let k = |content: u64, frame: i64| FrameKey { content, comp: outer.0, ..key(0, frame, 0) }; + let id0 = f.content_of(&q, 1, outer); + (0..n).for_each(|i| insert(&f, k(id0, i))); + let mut q1 = q.clone(); + trim(Arc::make_mut(&mut q1)); + let kept = f.cached_frames(&k(f.content_of(&q1, 2, outer), 0)); + assert!(kept_frames.iter().all(|i| kept.contains(i)) && !dirty_frames.iter().any(|i| kept.contains(i)), "{name}: {kept:?}"); + for i in kept { + assert!( + render_frame(&q, outer, t(i), 1.0).data == render_frame(&q1, outer, t(i), 1.0).data, + "{name}: kept frame {i} renders differently after the edit" + ); + } + } +} + +/// Frames only on disk (the blue bar) outlive an edit that can't change them: a rename keeps them +/// all, a trim all but the frames it cut, and undo finds the earlier ones again. +#[test] +fn disk_frames_outlive_an_edit_that_cant_change_them() { + use effectcraft_engine::Session; + use effectcraft_engine::project::LayerId; + use serde_json::json; + let dir = std::env::temp_dir().join(format!("ec-disk-carry-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + let disk = DiskCache::open(&dir, 1 << 30).unwrap(); + let mut s = Session::default(); + let cid = ItemId(s.execute("comp.new", json!({"name": "A", "width": 8, "height": 8, "duration": 1, "frameRate": 30})).unwrap()["comp"].as_u64().unwrap()); + let a = LayerId(s.execute("layer.newSolid", json!({"color": "#ff0000"})).unwrap()["layer"].as_u64().unwrap()); + let b = LayerId(s.execute("layer.newSolid", json!({"color": "#00ff00"})).unwrap()["layer"].as_u64().unwrap()); + let fr = s.project.comp(cid).unwrap().frame_rate; + let mut p0 = (*s.project).clone(); + for (l, i, o) in [(a, 0, 15), (b, 15, 30)] { + let l = p0.comp_mut(cid).unwrap().layer_mut(l).unwrap(); + (l.start_time, l.in_point, l.out_point) = (fr.tick_of(i), fr.tick_of(i), fr.tick_of(o)); + } + let p0 = Arc::new(p0); + let opts = RenderOpts::default(); + let oh = opts_hash(&opts); + let src = |p: &Arc| RenderSource { + project: p.clone(), + footage: Arc::new(effectcraft_engine::render::NoFootage), + expr: None, + layer_cache: Arc::default(), + gpu: None, + gpu_display: false, + disk: Some(disk.clone()), + }; + // Every frame of `p0` on disk, none in RAM. + let content0 = disk_cache::comp_content_key_with(&p0, cid, None); + for f in 0..30 { + disk.put_frame_now(disk_cache::frame_key(content0, f, 1000, oh), 1, 1, &[1, 2, 3, 255]); + } + let rename: fn(&mut Project, ItemId, LayerId) = |p, c, l| p.comp_mut(c).unwrap().layer_mut(l).unwrap().name = "Renamed".into(); + let trim: fn(&mut Project, ItemId, LayerId) = |p, c, l| { + let fr = p.comp(c).unwrap().frame_rate; + p.comp_mut(c).unwrap().layer_mut(l).unwrap().out_point = fr.tick_of(22); + }; + for (name, edit, layer, kept) in [("rename", rename, a, (0..30).collect::>()), ("trim", trim, b, (0..20).collect())] { + let f = Frames::default(); + let series = |p: &Arc, rev: u64| FrameKey { revision: rev, content: f.content_of(p, rev, cid), comp: cid.0, opts: oh, ..key(0, 0, 0) }; + // Blue frames of `p` once the disk carries and the content key are done. + let blue = |p: &Arc, rev: u64, want: &[i64]| { + let base = series(p, rev); + let mut got = vec![]; + for _ in 0..1000 { + got = f.disk_frames(&src(p), &base, 0..30, &opts); + if got == want { + break; + } + std::thread::sleep(std::time::Duration::from_millis(5)); + } + got + }; + assert_eq!(blue(&p0, 1, &(0..30).collect::>()), (0..30).collect::>(), "{name}: before"); + let mut p1 = p0.clone(); + edit(Arc::make_mut(&mut p1), cid, layer); + assert_eq!(blue(&p1, 2, &kept), kept, "{name}: after"); + assert_eq!(blue(&p0, 3, &(0..30).collect::>()), (0..30).collect::>(), "{name}: undo"); + // A drag: an edit per paint, each paint asking for the blue bar. One carry job drains + // them in order, and none is left scheduled. + let mut p = p0.clone(); + for rev in 4..20 { + Arc::make_mut(&mut p).comp_mut(cid).unwrap().layer_mut(a).unwrap().name = format!("Drag {rev}"); + let _ = f.disk_frames(&src(&p), &series(&p, rev), 0..30, &opts); + } + assert_eq!(blue(&p, 19, &(0..30).collect::>()), (0..30).collect::>(), "{name}: dragged"); + for _ in 0..1000 { + if !f.carry_scheduled.load(Ordering::Acquire) { + break; + } + std::thread::sleep(std::time::Duration::from_millis(5)); + } + assert!(!f.carry_scheduled.load(Ordering::Acquire) && f.disk_carries.lock().unwrap().is_empty(), "{name}: carry job finished"); + } + let _ = std::fs::remove_dir_all(&dir); +} + +/// A keyframe edit re-renders only around the keys it changed: two keys either side (auto-Bezier +/// and continuous tangents reach the neighbouring segments), and from the layer's in point when +/// the first key changes. A static value or an expression still reaches the whole layer. Every +/// frame kept renders as before. +#[test] +fn a_keyframe_edit_reaches_only_the_keys_around_it() { + use effectcraft_engine::Session; + use effectcraft_engine::project::{LayerId, Value}; + use effectcraft_engine::render::render_frame; + use effectcraft_keyframe::Interp; + use serde_json::json; + let mut s = Session::default(); + let cid = ItemId(s.execute("comp.new", json!({"name": "A", "width": 16, "height": 16, "duration": 4, "frameRate": 30})).unwrap()["comp"].as_u64().unwrap()); + let l = s.execute("layer.newSolid", json!({"color": "#ff0000", "width": 4, "height": 4})).unwrap()["layer"].as_u64().unwrap(); + let fr = s.project.comp(cid).unwrap().frame_rate; + // Keys at frames 0, 20, 40, 60, 80, 100, zig-zagging, all auto-Bezier and continuous. + for (i, f) in (0..6).map(|i| (i, i * 20)) { + let y = if i % 2 == 0 { 3.0 } else { 13.0 }; + s.execute("prop.addKey", json!({"layer": l, "path": "transform/position", "time": fr.tick_of(f).seconds(), "value": [2.0 + i as f64 * 2.0, y, 0.0]})) + .unwrap(); + } + let mut p0 = (*s.project).clone(); + for k in &mut p0.comp_mut(cid).unwrap().layer_mut(LayerId(l)).unwrap().props.prop_mut("transform/position").unwrap().keys { + (k.in_interp, k.out_interp, k.auto_bezier, k.continuous, k.spatial_continuous) = (Interp::Bezier, Interp::Bezier, true, true, true); + } + let p0 = Arc::new(p0); + let n = 120; + type Edit = fn(&mut effectcraft_engine::project::Property); + let edits: [(&str, Edit, Vec, Vec); 5] = [ + ("middle value", |p| p.keys[3].value = Value::Vec3([8.0, 8.0, 0.0]), vec![0, 17, 103, 119], vec![22, 60, 98]), + ("middle interpolation", |p| p.keys[3].out_interp = Interp::Hold, vec![0, 17, 103, 119], vec![22, 60, 98]), + ("first value", |p| p.keys[0].value = Value::Vec3([1.0, 1.0, 0.0]), vec![43, 119], vec![0, 20, 39]), + ("static", |p| p.keys.clear(), vec![], vec![0, 60, 119]), + ("expression", |p| p.expr = Some(effectcraft_engine::project::Expression { text: "value".into(), enabled: true }), vec![], vec![0, 60, 119]), + ]; + for (name, edit, kept_frames, dirty_frames) in edits { + let f = Frames::default(); + let k = |content: u64, frame: i64| FrameKey { content, comp: cid.0, ..key(0, frame, 0) }; + let id0 = f.content_of(&p0, 1, cid); + (0..n).for_each(|i| insert(&f, k(id0, i))); + let mut p1 = p0.clone(); + edit(Arc::make_mut(&mut p1).comp_mut(cid).unwrap().layer_mut(LayerId(l)).unwrap().props.prop_mut("transform/position").unwrap()); + let kept = f.cached_frames(&k(f.content_of(&p1, 2, cid), 0)); + assert!(kept_frames.iter().all(|i| kept.contains(i)) && !dirty_frames.iter().any(|i| kept.contains(i)), "{name}: {kept:?}"); + let same = |i: i64| render_frame(&p0, cid, fr.tick_of(i), 1.0).data == render_frame(&p1, cid, fr.tick_of(i), 1.0).data; + // (the edit shows: the check below compares something) + assert!(name == "expression" || !(0..n).all(same), "{name}: the edit changes no pixels"); + for i in kept { + assert!(same(i), "{name}: kept frame {i} renders differently after the edit"); + } + } +} + +/// Music's Audio Levels keep every frame, in RAM and on disk, while the project's expressions +/// can't read audio (a wiggle); an expression naming the music's Audio Levels re-renders its layer. +#[test] +fn music_volume_keeps_the_frames_with_expressions_that_dont_read_audio() { + use effectcraft_engine::Session; + use effectcraft_engine::project::{Expression, Footage, FootageKind, LayerId, Value, build}; + use effectcraft_engine::render::disk_cache::comp_content_key_with; + use serde_json::json; + let mut s = Session::default(); + let cid = ItemId(s.execute("comp.new", json!({"name": "A", "width": 8, "height": 8, "duration": 4, "frameRate": 30})).unwrap()["comp"].as_u64().unwrap()); + let solid = LayerId(s.execute("layer.newSolid", json!({})).unwrap()["layer"].as_u64().unwrap()); + let mut p = (*s.project).clone(); + let song = p.add_item( + "song.wav", + effectcraft_color::Label::None, + None, + ItemKind::Footage(Footage { kind: FootageKind::Audio, has_video: false, has_audio: true, ..Default::default() }), + ); + let comp = p.comp(cid).unwrap().clone(); + let music = build::layer(&mut p, &comp, "Music", LayerSource::Footage { item: song }, (8, 8), None); + let music_id = music.id; + p.comp_mut(cid).unwrap().layers.push(music); + let kept = |text: &str| { + let mut p0 = p.clone(); + p0.comp_mut(cid).unwrap().layer_mut(solid).unwrap().props.prop_mut("transform/position").unwrap().expr = + Some(Expression { text: text.into(), enabled: true }); + let p0 = Arc::new(p0); + let f = Frames::default(); + let k = |content: u64, frame: i64| FrameKey { content, comp: cid.0, ..key(0, frame, 0) }; + let id0 = f.content_of(&p0, 1, cid); + (0..120).for_each(|n| insert(&f, k(id0, n))); + let mut p1 = p0.clone(); + Arc::make_mut(&mut p1).comp_mut(cid).unwrap().layer_mut(music_id).unwrap().props.prop_mut("audio/levels").unwrap().value = Value::Vec2([-6.0, -6.0]); + let id1 = f.content_of(&p1, 2, cid); + (f.cached_frames(&k(id1, 0)).len(), comp_content_key_with(&p0, cid, None) == comp_content_key_with(&p1, cid, None)) + }; + assert_eq!(kept("wiggle(2, 20)"), (120, true), "a wiggle reads no audio"); + let (frames, disk) = kept("value + thisComp.layer(\"Music\")(\"Audio\")(\"Audio Levels\")[0]"); + assert!(frames < 120 && !disk, "the expression reads the music's levels: {frames} frames kept, disk kept {disk}"); +} + +/// A Scale edit with only two keys re-renders from the other key on (values hold past it), not +/// the layer's whole length; a full-length layer whose expression reads only itself (a wiggle) +/// keeps its frames. Every frame kept renders as before. +#[test] +fn a_two_key_edit_beside_a_wiggle_keeps_the_other_frames() { + use effectcraft_engine::project::{Expression, LayerId, Value}; + use effectcraft_engine::render::NoFootage; + use serde_json::json; + let mut s = effectcraft_host::session(); + let cid = ItemId(s.execute("comp.new", json!({"name": "A", "width": 16, "height": 16, "duration": 4, "frameRate": 30})).unwrap()["comp"].as_u64().unwrap()); + let wig = LayerId(s.execute("layer.newSolid", json!({"color": "#0000ff", "width": 4, "height": 4})).unwrap()["layer"].as_u64().unwrap()); + let l = LayerId(s.execute("layer.newSolid", json!({"color": "#ff0000", "width": 4, "height": 4})).unwrap()["layer"].as_u64().unwrap()); + let fr = s.project.comp(cid).unwrap().frame_rate; + for (f, v) in [(20, 50.0), (80, 150.0)] { + s.execute("prop.addKey", json!({"layer": l.0, "path": "transform/scale", "time": fr.tick_of(f).seconds(), "value": [v, v, 100.0]})).unwrap(); + } + let mut p0 = (*s.project).clone(); + p0.comp_mut(cid).unwrap().layer_mut(wig).unwrap().props.prop_mut("transform/position").unwrap().expr = + Some(Expression { text: "wiggle(2, 3)".into(), enabled: true }); + let p0 = Arc::new(p0); + let host = s.expr.clone().unwrap(); + let render = |p: &Project, i: i64| { + let mut r = Renderer::new(p, &NoFootage, RenderOpts::default()); + r.expr = Some(&*host); + r.comp_frame(cid, fr.tick_of(i)) + }; + // (key index, frames that must stay, frames that must go) + for (n, keep, go) in [(1, 0..18, [21, 60, 119]), (0, 82..120, [0, 20, 79])] { + let f = Frames::default(); + let k = |content: u64, frame: i64| FrameKey { content, comp: cid.0, ..key(0, frame, 0) }; + let id0 = f.content_of(&p0, 1, cid); + (0..120).for_each(|i| insert(&f, k(id0, i))); + let mut p1 = p0.clone(); + Arc::make_mut(&mut p1).comp_mut(cid).unwrap().layer_mut(l).unwrap().props.prop_mut("transform/scale").unwrap().keys[n].value = + Value::Vec3([90.0, 90.0, 100.0]); + let kept = f.cached_frames(&k(f.content_of(&p1, 2, cid), 0)); + assert!(keep.clone().all(|i| kept.contains(&i)) && !go.iter().any(|i| kept.contains(i)), "key {n}: {kept:?}"); + for i in kept { + assert_eq!(render(&p0, i), render(&p1, i), "key {n}: kept frame {i} draws other pixels now"); + } + } +} + +/// An expression that reads its own layer's markers (no other layer or comp named) re-renders +/// that layer when one of its markers changes. Every frame kept renders as before. +#[test] +fn a_marker_read_by_its_own_layer_re_renders_it() { + use effectcraft_engine::render::NoFootage; + use serde_json::json; + let mut s = effectcraft_host::session(); + let cid = ItemId(s.execute("comp.new", json!({"name": "A", "width": 16, "height": 16, "duration": 4, "frameRate": 30})).unwrap()["comp"].as_u64().unwrap()); + let l = s.execute("layer.newSolid", json!({"color": "#ff0000", "width": 4, "height": 4})).unwrap()["layer"].as_u64().unwrap(); + s.execute("prop.setExpression", json!({"layer": l, "path": "transform/opacity", "expression": "marker.numKeys > 0 ? 0 : 100"})).unwrap(); + let p0 = s.project.clone(); + s.execute("markers.set", json!({"comp": cid.0, "layer": l, "new": true, "time": 3.0})).unwrap(); + let p1 = s.project.clone(); + let host = s.expr.clone().unwrap(); + let fr = p0.comp(cid).unwrap().frame_rate; + let render = |p: &Project, i: i64| { + let mut r = Renderer::new(p, &NoFootage, RenderOpts::default()); + r.expr = Some(&*host); + r.comp_frame(cid, fr.tick_of(i)) + }; + let f = Frames::default(); + let k = |content: u64, frame: i64| FrameKey { content, comp: cid.0, ..key(0, frame, 0) }; + let id0 = f.content_of(&p0, 1, cid); + (0..120).for_each(|i| insert(&f, k(id0, i))); + let kept = f.cached_frames(&k(f.content_of(&p1, 2, cid), 0)); + assert!(!kept.contains(&0), "the layer's frames must go: {kept:?}"); + for i in kept { + assert_eq!(render(&p0, i), render(&p1, i), "kept frame {i} draws other pixels now"); + } +} + +/// An expression that reads its own layer's audio re-renders that layer when its volume +/// changes. +#[test] +fn a_volume_read_by_its_own_layer_re_renders_it() { + use effectcraft_engine::Session; + use effectcraft_engine::project::{Expression, Footage, FootageKind, Value, build}; + use serde_json::json; + let mut s = Session::default(); + let cid = ItemId(s.execute("comp.new", json!({"name": "A", "width": 8, "height": 8, "duration": 4, "frameRate": 30})).unwrap()["comp"].as_u64().unwrap()); + let mut p = (*s.project).clone(); + let clip = p.add_item( + "clip.mp4", + effectcraft_color::Label::None, + None, + ItemKind::Footage(Footage { kind: FootageKind::Video, has_video: true, has_audio: true, width: 8, height: 8, ..Default::default() }), + ); + let comp = p.comp(cid).unwrap().clone(); + let mut layer = build::layer(&mut p, &comp, "Clip", LayerSource::Footage { item: clip }, (8, 8), None); + layer.props.prop_mut("transform/opacity").unwrap().expr = + Some(Expression { text: "100 + thisLayer(\"Audio\")(\"Audio Levels\")[0]".into(), enabled: true }); + let id = layer.id; + p.comp_mut(cid).unwrap().layers.push(layer); + let p0 = Arc::new(p); + let f = Frames::default(); + let k = |content: u64, frame: i64| FrameKey { content, comp: cid.0, ..key(0, frame, 0) }; + let id0 = f.content_of(&p0, 1, cid); + (0..120).for_each(|i| insert(&f, k(id0, i))); + let mut p1 = p0.clone(); + Arc::make_mut(&mut p1).comp_mut(cid).unwrap().layer_mut(id).unwrap().props.prop_mut("audio/levels").unwrap().value = Value::Vec2([-6.0, -6.0]); + let kept = f.cached_frames(&k(f.content_of(&p1, 2, cid), 0)); + assert!(!kept.contains(&0), "the clip's frames must go: {kept:?}"); +} + +/// An expression on the keyed layer can read the keyed property at other times (Rotation +/// reading Position a second earlier): a key edit re-renders the layer's whole length. Every +/// frame kept renders as before. +#[test] +fn a_keyframe_edit_read_at_other_times_by_its_layer_re_renders_it() { + use effectcraft_engine::project::{LayerId, Value}; + use effectcraft_engine::render::NoFootage; + use serde_json::json; + let mut s = effectcraft_host::session(); + let cid = ItemId(s.execute("comp.new", json!({"name": "A", "width": 16, "height": 16, "duration": 4, "frameRate": 30})).unwrap()["comp"].as_u64().unwrap()); + let l = LayerId(s.execute("layer.newSolid", json!({"color": "#ff0000", "width": 4, "height": 4})).unwrap()["layer"].as_u64().unwrap()); + let fr = s.project.comp(cid).unwrap().frame_rate; + for (f, x) in [(20, 2.0), (80, 14.0)] { + s.execute("prop.addKey", json!({"layer": l.0, "path": "transform/position", "time": fr.tick_of(f).seconds(), "value": [x, 8.0, 0.0]})).unwrap(); + } + s.execute("prop.setExpression", json!({"layer": l.0, "path": "transform/rotation", "expression": "transform.position.valueAtTime(time - 1)[0] * 3"})) + .unwrap(); + let p0 = s.project.clone(); + let host = s.expr.clone().unwrap(); + let render = |p: &Project, i: i64| { + let mut r = Renderer::new(p, &NoFootage, RenderOpts::default()); + r.expr = Some(&*host); + r.comp_frame(cid, fr.tick_of(i)) + }; + let f = Frames::default(); + let k = |content: u64, frame: i64| FrameKey { content, comp: cid.0, ..key(0, frame, 0) }; + let id0 = f.content_of(&p0, 1, cid); + (0..120).for_each(|i| insert(&f, k(id0, i))); + let mut p1 = p0.clone(); + Arc::make_mut(&mut p1).comp_mut(cid).unwrap().layer_mut(l).unwrap().props.prop_mut("transform/position").unwrap().keys[0].value = + Value::Vec3([8.0, 2.0, 0.0]); + let kept = f.cached_frames(&k(f.content_of(&p1, 2, cid), 0)); + assert!(!kept.contains(&100), "Rotation at frame 100 reads Position at frame 70: {kept:?}"); + for i in kept { + assert_eq!(render(&p0, i), render(&p1, i), "kept frame {i} draws other pixels now"); + } +} + +/// An edit two precomps down (Outer → Mid → Inner) re-renders Outer where it shows, although +/// Mid's own comp didn't change. Every frame kept renders as before. +#[test] +fn an_edit_two_precomps_down_reaches_the_outer_comp() { + use effectcraft_engine::Session; + use effectcraft_engine::project::LayerId; + use effectcraft_engine::render::render_frame; + use serde_json::json; + let mut s = Session::default(); + let id = |v: serde_json::Value, k: &str| v[k].as_u64().unwrap(); + let comp = |s: &mut Session, name: &str| { + ItemId(id(s.execute("comp.new", json!({"name": name, "width": 16, "height": 16, "duration": 4, "frameRate": 30})).unwrap(), "comp")) + }; + let inner = comp(&mut s, "Inner"); + let solid = |s: &mut Session, color: &str| { + LayerId(id(s.execute("layer.newSolid", json!({"comp": inner.0, "color": color, "width": 6, "height": 6})).unwrap(), "layer")) + }; + let (a, b) = (solid(&mut s, "#ff0000"), solid(&mut s, "#00ff00")); + let mid = comp(&mut s, "Mid"); + s.execute("layer.addItem", json!({"comp": mid.0, "item": inner.0})).unwrap(); + let outer = comp(&mut s, "Outer"); + s.execute("layer.addItem", json!({"comp": outer.0, "item": mid.0})).unwrap(); + let fr = s.project.comp(outer).unwrap().frame_rate; + let t = |f: i64| fr.tick_of(f); + let mut p0 = (*s.project).clone(); + for (l, i, o) in [(a, 0, 30), (b, 30, 60)] { + let l = p0.comp_mut(inner).unwrap().layer_mut(l).unwrap(); + (l.start_time, l.in_point, l.out_point) = (t(i), t(i), t(o)); + } + let p0 = Arc::new(p0); + let f = Frames::default(); + let k = |content: u64, frame: i64| FrameKey { content, comp: outer.0, ..key(0, frame, 0) }; + let id0 = f.content_of(&p0, 1, outer); + (0..120).for_each(|i| insert(&f, k(id0, i))); + let mut p1 = p0.clone(); + Arc::make_mut(&mut p1).comp_mut(inner).unwrap().layer_mut(b).unwrap().out_point = t(45); + let kept = f.cached_frames(&k(f.content_of(&p1, 2, outer), 0)); + assert!(kept.contains(&10) && kept.contains(&100) && !kept.contains(&50), "{kept:?}"); + for i in kept { + assert!(render_frame(&p0, outer, t(i), 1.0).data == render_frame(&p1, outer, t(i), 1.0).data, "kept frame {i} renders differently after the edit"); + } +} + +/// An expression reading its own layer's in point (a fade from `inPoint`) re-renders the layer +/// when it is trimmed, and one reading its `index` re-renders it when a short layer is added +/// above it. Every frame kept renders as before. +#[test] +fn expressions_reading_their_own_layer_re_render_on_a_trim_or_a_layer_above() { + use effectcraft_engine::Session; + use effectcraft_engine::project::LayerId; + use effectcraft_engine::render::NoFootage; + use serde_json::json; + /// Trim the layer to start at frame 10. + fn trim(s: &mut Session, cid: ItemId, l: LayerId) { + let fr = s.project.comp(cid).unwrap().frame_rate; + Arc::make_mut(&mut s.project).comp_mut(cid).unwrap().layer_mut(l).unwrap().in_point = fr.tick_of(10); + } + /// Add a solid above the layer on frames 0–5. + fn layer_above(s: &mut Session, cid: ItemId, _: LayerId) { + let fr = s.project.comp(cid).unwrap().frame_rate; + let a = s.execute("layer.newSolid", json!({"comp": cid.0, "color": "#00ff00", "width": 2, "height": 2})).unwrap()["layer"].as_u64().unwrap(); + let a = Arc::make_mut(&mut s.project).comp_mut(cid).unwrap().layer_mut(LayerId(a)).unwrap(); + (a.start_time, a.in_point, a.out_point) = (fr.tick_of(0), fr.tick_of(0), fr.tick_of(5)); + } + type Edit = fn(&mut Session, ItemId, LayerId); + let edits: [(&str, &str, &str, Edit); 2] = + [("trim", "transform/opacity", "linear(time, inPoint, inPoint + 2, 20, 100)", trim), ("layer above", "transform/rotation", "index * 20", layer_above)]; + for (name, path, expression, edit) in edits { + let mut s = effectcraft_host::session(); + let new = json!({"name": "A", "width": 16, "height": 16, "duration": 4, "frameRate": 30}); + let cid = ItemId(s.execute("comp.new", new).unwrap()["comp"].as_u64().unwrap()); + let l = LayerId(s.execute("layer.newSolid", json!({"comp": cid.0, "color": "#ff0000", "width": 6, "height": 6})).unwrap()["layer"].as_u64().unwrap()); + s.execute("prop.setExpression", json!({"comp": cid.0, "layer": l.0, "path": path, "expression": expression})).unwrap(); + let p0 = s.project.clone(); + edit(&mut s, cid, l); + let p1 = s.project.clone(); + let fr = p0.comp(cid).unwrap().frame_rate; + let host = s.expr.clone().unwrap(); + let render = |p: &Project, i: i64| { + let mut r = Renderer::new(p, &NoFootage, RenderOpts::default()); + r.expr = Some(&*host); + r.comp_frame(cid, fr.tick_of(i)) + }; + assert_ne!(render(&p0, 60), render(&p1, 60), "{name}: the edit changes frame 60"); + let f = Frames::default(); + let k = |content: u64, frame: i64| FrameKey { content, comp: cid.0, ..key(0, frame, 0) }; + let id0 = f.content_of(&p0, 1, cid); + (0..120).for_each(|i| insert(&f, k(id0, i))); + let kept = f.cached_frames(&k(f.content_of(&p1, 2, cid), 0)); + assert!(!kept.contains(&60), "{name}: frame 60 must go: {kept:?}"); + for i in kept { + assert_eq!(render(&p0, i), render(&p1, i), "{name}: kept frame {i} draws other pixels now"); + } + } +} + +/// An expression inside a precomp reading a layer of the comp nesting it (a hidden Controls +/// layer, `comp("Main").layer("Controls")`) re-renders where the precomp shows it when that +/// layer changes. Every frame kept renders as before. +#[test] +fn an_edit_read_by_a_precomp_expression_reaches_the_precomp() { + use effectcraft_engine::project::LayerId; + use effectcraft_engine::render::NoFootage; + use serde_json::json; + let mut s = effectcraft_host::session(); + let id = |v: serde_json::Value, k: &str| v[k].as_u64().unwrap(); + let new_comp = |s: &mut effectcraft_engine::Session, name: &str| { + ItemId(id(s.execute("comp.new", json!({"name": name, "width": 16, "height": 16, "duration": 4, "frameRate": 30})).unwrap(), "comp")) + }; + let inner = new_comp(&mut s, "Inner"); + let dot = id(s.execute("layer.newSolid", json!({"comp": inner.0, "color": "#ff0000", "width": 6, "height": 6})).unwrap(), "layer"); + s.execute( + "prop.setExpression", + json!({"comp": inner.0, "layer": dot, "path": "transform/opacity", "expression": "comp(\"Main\").layer(\"Controls\").transform.rotation"}), + ) + .unwrap(); + let main = new_comp(&mut s, "Main"); + s.execute("layer.addItem", json!({"comp": main.0, "item": inner.0})).unwrap(); + let controls = json!({"comp": main.0, "name": "Controls", "color": "#0000ff", "width": 2, "height": 2}); + let controls = LayerId(id(s.execute("layer.newSolid", controls).unwrap(), "layer")); + // (hidden, and short: the precomp reads it at every frame all the same) + let fr = s.project.comp(main).unwrap().frame_rate; + let c = Arc::make_mut(&mut s.project).comp_mut(main).unwrap().layer_mut(controls).unwrap(); + (c.switches.video, c.start_time, c.in_point, c.out_point) = (false, fr.tick_of(0), fr.tick_of(0), fr.tick_of(10)); + let p0 = s.project.clone(); + s.execute("prop.set", json!({"comp": main.0, "layer": controls.0, "path": "transform/rotation", "value": 60.0})).unwrap(); + let p1 = s.project.clone(); + let host = s.expr.clone().unwrap(); + let render = |p: &Project, i: i64| { + let mut r = Renderer::new(p, &NoFootage, RenderOpts::default()); + r.expr = Some(&*host); + r.comp_frame(main, fr.tick_of(i)) + }; + assert_ne!(render(&p0, 60), render(&p1, 60), "the dot's opacity follows Controls"); + let f = Frames::default(); + let k = |content: u64, frame: i64| FrameKey { content, comp: main.0, ..key(0, frame, 0) }; + let id0 = f.content_of(&p0, 1, main); + (0..120).for_each(|i| insert(&f, k(id0, i))); + let kept = f.cached_frames(&k(f.content_of(&p1, 2, main), 0)); + assert!(!kept.contains(&60), "the precomp's frames must go: {kept:?}"); + for i in kept { + assert_eq!(render(&p0, i), render(&p1, i), "kept frame {i} draws other pixels now"); + } +} + +/// An edit inside a precomp whose layer runs Echo (it reads the precomp at earlier times) +/// re-renders the precomp layer's whole span, not only where the precomp shows the edit. Every +/// frame kept renders as before. +#[test] +fn an_edit_in_a_precomp_under_a_time_effect_reaches_its_whole_layer() { + use effectcraft_engine::Session; + use effectcraft_engine::project::LayerId; + use effectcraft_engine::render::render_frame; + use serde_json::json; + let mut s = Session::default(); + let id = |v: serde_json::Value, k: &str| v[k].as_u64().unwrap(); + let inner = ItemId(id(s.execute("comp.new", json!({"name": "Inner", "width": 16, "height": 16, "duration": 4, "frameRate": 30})).unwrap(), "comp")); + let solid = |s: &mut Session, color: &str| { + LayerId(id(s.execute("layer.newSolid", json!({"comp": inner.0, "color": color, "width": 6, "height": 6})).unwrap(), "layer")) + }; + let (a, b) = (solid(&mut s, "#ff0000"), solid(&mut s, "#00ff00")); + let outer = ItemId(id(s.execute("comp.new", json!({"name": "Outer", "width": 16, "height": 16, "duration": 4, "frameRate": 30})).unwrap(), "comp")); + let pre = id(s.execute("layer.addItem", json!({"comp": outer.0, "item": inner.0})).unwrap(), "layer"); + // Echo reaching 9 frames back, once (more echoes of one solid add up past full and look alike). + s.execute("effect.apply", json!({"comp": outer.0, "layers": [pre], "effect": "Echo"})).unwrap(); + s.execute("prop.set", json!({"comp": outer.0, "layer": pre, "path": "effects/#1/echoTime", "value": -0.3})).unwrap(); + let fr = s.project.comp(outer).unwrap().frame_rate; + let t = |f: i64| fr.tick_of(f); + let mut p0 = (*s.project).clone(); + for (l, i, o) in [(a, 0, 30), (b, 30, 60)] { + let l = p0.comp_mut(inner).unwrap().layer_mut(l).unwrap(); + (l.start_time, l.in_point, l.out_point) = (t(i), t(i), t(o)); + } + let p0 = Arc::new(p0); + let f = Frames::default(); + let k = |content: u64, frame: i64| FrameKey { content, comp: outer.0, ..key(0, frame, 0) }; + let id0 = f.content_of(&p0, 1, outer); + (0..120).for_each(|i| insert(&f, k(id0, i))); + let mut p1 = p0.clone(); + Arc::make_mut(&mut p1).comp_mut(inner).unwrap().layer_mut(b).unwrap().out_point = t(45); + let same = |i: i64| render_frame(&p0, outer, t(i), 1.0).data == render_frame(&p1, outer, t(i), 1.0).data; + assert!(!same(65), "Echo at frame 65 shows frame 56, which the trim removed"); + let kept = f.cached_frames(&k(f.content_of(&p1, 2, outer), 0)); + assert!(!kept.contains(&65), "{kept:?}"); + for i in kept { + assert!(same(i), "kept frame {i} renders differently after the edit"); + } +} + +/// A keyframe edit beside muted keys reaches back to the nearest keys that play: muted keys are +/// skipped, so the segment the edited key ends can start far before the two keys before it. +/// Every frame kept renders as before. +#[test] +fn a_keyframe_edit_beside_muted_keys_reaches_the_keys_that_play() { + use effectcraft_engine::Session; + use effectcraft_engine::project::{LayerId, Value}; + use effectcraft_engine::render::render_frame; + use serde_json::json; + let mut s = Session::default(); + let new = json!({"name": "A", "width": 16, "height": 16, "duration": 4, "frameRate": 30}); + let cid = ItemId(s.execute("comp.new", new).unwrap()["comp"].as_u64().unwrap()); + let l = LayerId(s.execute("layer.newSolid", json!({"color": "#ff0000", "width": 4, "height": 4})).unwrap()["layer"].as_u64().unwrap()); + let fr = s.project.comp(cid).unwrap().frame_rate; + // Keys at frames 0, 20, 40, 60, 80, 100; the ones at 20 and 40 muted. + for i in 0..6 { + let add = json!({"layer": l.0, "path": "transform/position", "time": fr.tick_of(i * 20).seconds(), "value": [2.0 + i as f64 * 2.0, 8.0, 0.0]}); + s.execute("prop.addKey", add).unwrap(); + } + let mut p0 = (*s.project).clone(); + for k in p0.comp_mut(cid).unwrap().layer_mut(l).unwrap().props.prop_mut("transform/position").unwrap().keys.iter_mut().skip(1).take(2) { + k.muted = true; + } + let p0 = Arc::new(p0); + let mut p1 = p0.clone(); + Arc::make_mut(&mut p1).comp_mut(cid).unwrap().layer_mut(l).unwrap().props.prop_mut("transform/position").unwrap().keys[3].value = + Value::Vec3([8.0, 2.0, 0.0]); + let same = |i: i64| render_frame(&p0, cid, fr.tick_of(i), 1.0).data == render_frame(&p1, cid, fr.tick_of(i), 1.0).data; + assert!(!same(10), "the edited key's segment starts at frame 0"); + let f = Frames::default(); + let k = |content: u64, frame: i64| FrameKey { content, comp: cid.0, ..key(0, frame, 0) }; + let id0 = f.content_of(&p0, 1, cid); + (0..120).for_each(|i| insert(&f, k(id0, i))); + let kept = f.cached_frames(&k(f.content_of(&p1, 2, cid), 0)); + assert!(!kept.contains(&10), "{kept:?}"); + for i in kept { + assert!(same(i), "kept frame {i} renders differently after the edit"); + } +} diff --git a/crates/ui-egui/src/i18n.rs b/crates/ui-egui/src/i18n.rs index 8ea88eaee..fe9466d3e 100644 --- a/crates/ui-egui/src/i18n.rs +++ b/crates/ui-egui/src/i18n.rs @@ -874,6 +874,8 @@ const SIMPLIFIED_CHINESE: &[(&str, &str, &str)] = &[ ("effect.removeAll", "Remove All", "全部移除"), ("effect.manage", "Manage Effects...", "管理效果..."), ("effect.plugins.load", "Load Effect Plug-in...", "加载效果插件..."), + ("effect.helpers.load", "Load Plug-ins Through Helper...", "通过辅助程序加载插件..."), + ("effect.helpers.scan", "Scan for Effect Helpers", "扫描效果辅助程序"), ("", "Animation", "动画"), ("anim.savePreset", "Save Animation Preset...", "保存动画预设..."), ("anim.applyPreset", "Apply Animation Preset...", "将动画预设应用于..."), @@ -1509,6 +1511,8 @@ const SWEDISH: &[(&str, &str, &str)] = &[ ("effect.removeAll", "Remove All", "Ta bort alla"), ("effect.manage", "Manage Effects...", "Hantera effekter..."), ("effect.plugins.load", "Load Effect Plug-in...", "Läs in effektplugin..."), + ("effect.helpers.load", "Load Plug-ins Through Helper...", "Läs in plugin via hjälpprogram..."), + ("effect.helpers.scan", "Scan for Effect Helpers", "Sök efter effekthjälpprogram"), ("", "Animation", "Animering"), ("anim.savePreset", "Save Animation Preset...", "Spara animeringsförinställning..."), ("anim.applyPreset", "Apply Animation Preset...", "Använd animeringsförinställning..."), @@ -2150,6 +2154,8 @@ const TRADITIONAL_CHINESE: &[(&str, &str, &str)] = &[ ("effect.removeAll", "Remove All", "全部移除"), ("effect.manage", "Manage Effects...", "管理效果..."), ("effect.plugins.load", "Load Effect Plug-in...", "載入效果外掛程式..."), + ("effect.helpers.load", "Load Plug-ins Through Helper...", "透過輔助程式載入外掛程式..."), + ("effect.helpers.scan", "Scan for Effect Helpers", "掃描效果輔助程式"), ("", "Animation", "動畫"), ("anim.savePreset", "Save Animation Preset...", "儲存動畫預設集..."), ("anim.applyPreset", "Apply Animation Preset...", "套用動畫預設集..."), @@ -2786,6 +2792,8 @@ const JAPANESE: &[(&str, &str, &str)] = &[ ("effect.removeAll", "Remove All", "すべてを削除"), ("effect.manage", "Manage Effects...", "エフェクトを管理..."), ("effect.plugins.load", "Load Effect Plug-in...", "エフェクトプラグインを読み込み..."), + ("effect.helpers.load", "Load Plug-ins Through Helper...", "ヘルパー経由でプラグインを読み込み..."), + ("effect.helpers.scan", "Scan for Effect Helpers", "エフェクトヘルパーをスキャン"), ("", "Animation", "アニメーション"), ("anim.savePreset", "Save Animation Preset...", "アニメーションプリセットを保存..."), ("anim.applyPreset", "Apply Animation Preset...", "アニメーションプリセットを適用..."), @@ -3425,6 +3433,8 @@ const UKRAINIAN: &[(&str, &str, &str)] = &[ ("effect.removeAll", "Remove All", "Вилучити все"), ("effect.manage", "Manage Effects...", "Керування ефектами..."), ("effect.plugins.load", "Load Effect Plug-in...", "Завантажити плагін ефекту..."), + ("effect.helpers.load", "Load Plug-ins Through Helper...", "Завантажити плагіни через помічника..."), + ("effect.helpers.scan", "Scan for Effect Helpers", "Шукати помічників ефектів"), ("", "Animation", "Анімація"), ("anim.savePreset", "Save Animation Preset...", "Зберегти набір анімації..."), ("anim.applyPreset", "Apply Animation Preset...", "Застосувати набір анімації..."), @@ -4065,6 +4075,8 @@ const SPANISH: &[(&str, &str, &str)] = &[ ("effect.removeAll", "Remove All", "Quitar todo"), ("effect.manage", "Manage Effects...", "Administrar efectos..."), ("effect.plugins.load", "Load Effect Plug-in...", "Cargar complemento de efecto..."), + ("effect.helpers.load", "Load Plug-ins Through Helper...", "Cargar complementos mediante asistente..."), + ("effect.helpers.scan", "Scan for Effect Helpers", "Buscar asistentes de efectos"), ("", "Animation", "Animación"), ("anim.savePreset", "Save Animation Preset...", "Guardar ajuste preestablecido de animación..."), ("anim.applyPreset", "Apply Animation Preset...", "Aplicar ajuste preestablecido de animación..."), @@ -4703,6 +4715,8 @@ const PORTUGUESE: &[(&str, &str, &str)] = &[ ("effect.removeAll", "Remove All", "Remover todos"), ("effect.manage", "Manage Effects...", "Gerenciar efeitos..."), ("effect.plugins.load", "Load Effect Plug-in...", "Carregar plug-in de efeito..."), + ("effect.helpers.load", "Load Plug-ins Through Helper...", "Carregar plug-ins por meio do auxiliar..."), + ("effect.helpers.scan", "Scan for Effect Helpers", "Procurar auxiliares de efeitos"), ("", "Animation", "Animação"), ("anim.savePreset", "Save Animation Preset...", "Salvar predefinição de animação..."), ("anim.applyPreset", "Apply Animation Preset...", "Aplicar predefinição de animação..."), @@ -5348,6 +5362,8 @@ const RUSSIAN: &[(&str, &str, &str)] = &[ ("effect.removeAll", "Remove All", "Удалить все"), ("effect.manage", "Manage Effects...", "Управление эффектами..."), ("effect.plugins.load", "Load Effect Plug-in...", "Загрузить плагин эффектов..."), + ("effect.helpers.load", "Load Plug-ins Through Helper...", "Загрузить плагины через помощника..."), + ("effect.helpers.scan", "Scan for Effect Helpers", "Найти помощников эффектов"), ("", "Animation", "Анимация"), ("anim.savePreset", "Save Animation Preset...", "Сохранить заготовку анимации..."), ("anim.applyPreset", "Apply Animation Preset...", "Применить заготовку анимации..."), @@ -5986,6 +6002,8 @@ const FRENCH: &[(&str, &str, &str)] = &[ ("effect.removeAll", "Remove All", "Tout supprimer"), ("effect.manage", "Manage Effects...", "Gérer les effets..."), ("effect.plugins.load", "Load Effect Plug-in...", "Charger un plug-in d'effet..."), + ("effect.helpers.load", "Load Plug-ins Through Helper...", "Charger des plug-ins via un assistant..."), + ("effect.helpers.scan", "Scan for Effect Helpers", "Rechercher les assistants d'effets"), ("", "Animation", "Animation"), ("anim.savePreset", "Save Animation Preset...", "Enregistrer une animation prédéfinie..."), ("anim.applyPreset", "Apply Animation Preset...", "Appliquer une animation prédéfinie..."), @@ -6624,6 +6642,8 @@ const GERMAN: &[(&str, &str, &str)] = &[ ("effect.removeAll", "Remove All", "Alle entfernen"), ("effect.manage", "Manage Effects...", "Effekte verwalten..."), ("effect.plugins.load", "Load Effect Plug-in...", "Effekt-Plug-in laden..."), + ("effect.helpers.load", "Load Plug-ins Through Helper...", "Plug-ins über Hilfsprogramm laden..."), + ("effect.helpers.scan", "Scan for Effect Helpers", "Nach Effekt-Hilfsprogrammen suchen"), ("", "Animation", "Animation"), ("anim.savePreset", "Save Animation Preset...", "Animationsvorgabe speichern..."), ("anim.applyPreset", "Apply Animation Preset...", "Animationsvorgabe anwenden..."), @@ -7261,6 +7281,8 @@ const POLISH: &[(&str, &str, &str)] = &[ ("effect.removeAll", "Remove All", "Usuń wszystkie"), ("effect.manage", "Manage Effects...", "Zarządzaj efektami..."), ("effect.plugins.load", "Load Effect Plug-in...", "Wczytaj wtyczkę efektu..."), + ("effect.helpers.load", "Load Plug-ins Through Helper...", "Wczytaj wtyczki przez program pomocniczy..."), + ("effect.helpers.scan", "Scan for Effect Helpers", "Wyszukaj programy pomocnicze efektów"), ("", "Animation", "Animacja"), ("anim.savePreset", "Save Animation Preset...", "Zapisz ustawienie animacji..."), ("anim.applyPreset", "Apply Animation Preset...", "Zastosuj ustawienie animacji..."), @@ -7898,6 +7920,8 @@ const CZECH: &[(&str, &str, &str)] = &[ ("effect.removeAll", "Remove All", "Odstranit vše"), ("effect.manage", "Manage Effects...", "Spravovat efekty..."), ("effect.plugins.load", "Load Effect Plug-in...", "Načíst zásuvný modul efektu..."), + ("effect.helpers.load", "Load Plug-ins Through Helper...", "Načíst zásuvné moduly přes pomocný program..."), + ("effect.helpers.scan", "Scan for Effect Helpers", "Vyhledat pomocné programy efektů"), ("", "Animation", "Animace"), ("anim.savePreset", "Save Animation Preset...", "Uložit přednastavení animace..."), ("anim.applyPreset", "Apply Animation Preset...", "Použít přednastavení animace..."), @@ -8532,6 +8556,8 @@ const GEORGIAN: &[(&str, &str, &str)] = &[ ("effect.removeAll", "Remove All", "ყველას წაშლა"), ("effect.manage", "Manage Effects...", "ეფექტების მართვა..."), ("effect.plugins.load", "Load Effect Plug-in...", "ეფექტის პლაგინის ჩატვირთვა..."), + ("effect.helpers.load", "Load Plug-ins Through Helper...", "პლაგინების ჩატვირთვა დამხმარე პროგრამით..."), + ("effect.helpers.scan", "Scan for Effect Helpers", "ეფექტის დამხმარე პროგრამების ძიება"), ("", "Animation", "ანიმაცია"), ("anim.savePreset", "Save Animation Preset...", "ანიმაციის პრესეტის შენახვა..."), ("anim.applyPreset", "Apply Animation Preset...", "ანიმაციის პრესეტის გამოყენება..."), diff --git a/crates/ui-egui/src/i18n/ui.rs b/crates/ui-egui/src/i18n/ui.rs index f324041d6..2546a473e 100644 --- a/crates/ui-egui/src/i18n/ui.rs +++ b/crates/ui-egui/src/i18n/ui.rs @@ -240,7 +240,7 @@ const JAPANESE: &[(&str, &str)] = &[ ("Browser Storage", "ブラウザーストレージ"), ("Browse…", "参照…"), ("Brush Dynamics", "ブラシのダイナミクス"), - ("Caching frames…", "フレームをキャッシュ中…"), + ("Caching frames", "フレームをキャッシュ中"), ("Cancel", "キャンセル"), ("Import After Effects Project", "After Effectsプロジェクトを読み込み"), ("Some parts of this project cannot be imported exactly (left out or changed):", "このプロジェクトの一部は正確に読み込めません(省略または変更されます):"), @@ -1371,7 +1371,7 @@ const SIMPLIFIED_CHINESE: &[(&str, &str)] = &[ ("Browser Storage", "浏览器存储"), ("Browse…", "浏览…"), ("Brush Dynamics", "画笔动态"), - ("Caching frames…", "正在缓存帧…"), + ("Caching frames", "正在缓存帧"), ("Cancel", "取消"), ("Import After Effects Project", "导入 After Effects 项目"), ("Some parts of this project cannot be imported exactly (left out or changed):", "此项目的某些部分无法准确导入(将被省略或更改):"), @@ -2481,7 +2481,7 @@ const SWEDISH: &[(&str, &str)] = &[ ("Browser Storage", "Webbläsarlagring"), ("Browse…", "Bläddra…"), ("Brush Dynamics", "Penseldynamik"), - ("Caching frames…", "Cachelagrar bildrutor…"), + ("Caching frames", "Cachelagrar bildrutor"), ("Cancel", "Avbryt"), ("Import After Effects Project", "Importera After Effects-projekt"), ( @@ -3745,7 +3745,7 @@ const TRADITIONAL_CHINESE: &[(&str, &str)] = &[ ("Browser Storage", "瀏覽器儲存"), ("Browse…", "瀏覽…"), ("Brush Dynamics", "筆刷動態"), - ("Caching frames…", "正在快取幀…"), + ("Caching frames", "正在快取幀"), ("Cancel", "取消"), ("Import After Effects Project", "匯入 After Effects 專案"), ("Some parts of this project cannot be imported exactly (left out or changed):", "此專案的部分內容無法準確匯入(將被省略或變更):"), @@ -4803,7 +4803,7 @@ const UKRAINIAN: &[(&str, &str)] = &[ ("Browser Storage", "Сховище браузера"), ("Browse…", "Огляд…"), ("Brush Dynamics", "Динаміка пензля"), - ("Caching frames…", "Кешування кадрів…"), + ("Caching frames", "Кешування кадрів"), ("Cancel", "Скасувати"), ("Import After Effects Project", "Імпортувати проєкт After Effects"), ( @@ -5378,7 +5378,7 @@ const PORTUGUESE: &[(&str, &str)] = &[ ("Browser Storage", "Armazenamento do navegador"), ("Browse…", "Procurar…"), ("Brush Dynamics", "Dinâmica do pincel"), - ("Caching frames…", "Armazenando quadros em cache…"), + ("Caching frames", "Armazenando quadros em cache"), ("Cancel", "Cancelar"), ("Import After Effects Project", "Importar projeto do After Effects"), ( @@ -6144,7 +6144,7 @@ const POLISH: &[(&str, &str)] = &[ ("Browser Storage", "Pamięć przeglądarki"), ("Browse…", "Przeglądaj…"), ("Brush Dynamics", "Dynamika pędzla"), - ("Caching frames…", "Buforowanie klatek…"), + ("Caching frames", "Buforowanie klatek"), ("Cancel", "Anuluj"), ("Import After Effects Project", "Importuj projekt After Effects"), ( @@ -6894,7 +6894,7 @@ const CZECH: &[(&str, &str)] = &[ ("Browser Storage", "Úložiště prohlížeče"), ("Browse…", "Procházet…"), ("Brush Dynamics", "Dynamika štětce"), - ("Caching frames…", "Ukládání snímků do mezipaměti…"), + ("Caching frames", "Ukládání snímků do mezipaměti"), ("Cancel", "Zrušit"), ("Import After Effects Project", "Importovat projekt After Effects"), ( @@ -7463,7 +7463,7 @@ const SPANISH: &[(&str, &str)] = &[ ("Browser Storage", "Almacenamiento del navegador"), ("Browse…", "Examinar…"), ("Brush Dynamics", "Dinámica del pincel"), - ("Caching frames…", "Almacenando fotogramas en caché…"), + ("Caching frames", "Almacenando fotogramas en caché"), ("Cancel", "Cancelar"), ("Import After Effects Project", "Importar proyecto de After Effects"), ( @@ -8023,7 +8023,7 @@ const GEORGIAN: &[(&str, &str)] = &[ ("Browser Storage", "ბრაუზერის მეხსიერება"), ("Browse…", "არჩევა…"), ("Brush Dynamics", "ფუნჯის დინამიკა"), - ("Caching frames…", "კადრების ქეშირება…"), + ("Caching frames", "კადრების ქეშირება"), ("Cancel", "გაუქმება"), ("Import After Effects Project", "After Effects-ის პროექტის იმპორტი"), ( @@ -8991,7 +8991,7 @@ const FRENCH: &[(&str, &str)] = &[ ("Browser Storage", "Stockage du navigateur"), ("Browse…", "Parcourir…"), ("Brush Dynamics", "Dynamique du pinceau"), - ("Caching frames…", "Mise en cache des images…"), + ("Caching frames", "Mise en cache des images"), ("Cancel", "Annuler"), ("Import After Effects Project", "Importer un projet After Effects"), ( diff --git a/crates/ui-egui/src/menus/bar.rs b/crates/ui-egui/src/menus/bar.rs index ec9ab9170..dc4478a47 100644 --- a/crates/ui-egui/src/menus/bar.rs +++ b/crates/ui-egui/src/menus/bar.rs @@ -115,10 +115,11 @@ pub fn menu_bar(app: &mut EffectcraftApp, ui: &mut egui::Ui) { egui::Popup::close_id(&ctx, top); } } - if let Some((id, params)) = clicked.map(|(id, params)| menu_choice(app, id, params)) - && let Err(e) = invoke(app, &ctx, &id, if params.is_null() { json!({}) } else { params }) - { - app.ui.status = e; + if let Some((id, params)) = clicked.map(|(id, params)| menu_choice(app, id, params)) { + match invoke(app, &ctx, &id, if params.is_null() { json!({}) } else { params }) { + Ok(result) => super::frontend::show_result_errors(app, &result), + Err(e) => app.ui.status = e, + } } // What the next frame starts from: the open title, and a title press still in progress. let open = nav.open_index(&ctx); diff --git a/crates/ui-egui/src/menus/frontend.rs b/crates/ui-egui/src/menus/frontend.rs index 45653da00..f22b0f079 100644 --- a/crates/ui-egui/src/menus/frontend.rs +++ b/crates/ui-egui/src/menus/frontend.rs @@ -583,6 +583,9 @@ pub(super) fn file_dialog(app: &mut EffectcraftApp, id: &str, params: &Value) -> "file.runScript" if params.get("name").is_none() && params.get("recent").is_none() => ("path", Ask::Open(&["jsx", "js", "jsonl", "json", "txt"])), "file.installScript" | "file.installScriptUIPanel" => ("path", Ask::Open(&["jsx", "js"])), "effect.plugins.load" if params.get("folder").is_none() => ("path", Ask::Open(&["wasm", "wat"])), + "effect.plugins.install" if params.get("bundle").is_some() => ("folder", Ask::Folder), + "effect.plugins.install" if params.get("folder").is_none() && params.get("path").is_none() => ("path", Ask::Open(&["wasm"])), + "effect.helpers.load" => ("path", Ask::Folder), "file.replaceFootage" => ("path", Ask::Import), "file.relinkFootage" => ("folder", Ask::Folder), "file.collectFiles" => ("folder", Ask::Save("Collected Files".into())), @@ -679,11 +682,34 @@ fn picked_value(key: &str, paths: Vec) -> Option { /// command with the picked path (nothing when it was cancelled). pub(crate) fn dialog_answered(app: &mut EffectcraftApp, answer: crate::DialogAnswer) { let Some(v) = picked_value(answer.key, answer.paths) else { return }; - if let Err(e) = run_with_picked(app, &answer.command, answer.key, &answer.params, v) { - app.ui.status = e; + match run_with_picked(app, &answer.command, answer.key, &answer.params, v) { + Ok(result) => show_result_errors(app, &result), + Err(e) => app.ui.status = e, } } +/// A successful command can still report per-item problems in an `errors` array (Effect ▸ +/// Install Effect Plug-in Bundle does): show a short, readable report in the status line, +/// without turning the success into an error. The full structured array stays in the result +/// for scripts; the status line is single-line and short-lived, so it lists the first few +/// problems and counts the rest instead of printing raw JSON. +pub(crate) fn show_result_errors(app: &mut EffectcraftApp, result: &Value) { + let Some(errors) = result.get("errors").and_then(Value::as_array).filter(|e| !e.is_empty()) else { return }; + let parts: Vec = errors + .iter() + .map(|e| match (e.get("file").and_then(Value::as_str), e.get("error").and_then(Value::as_str)) { + (Some(file), Some(error)) => format!("{file}: {error}"), + (_, Some(error)) => error.to_string(), + _ => e.to_string(), + }) + .collect(); + let mut msg = parts.iter().take(3).cloned().collect::>().join("; "); + if parts.len() > 3 { + msg.push_str(&format!("; +{} more", parts.len() - 3)); + } + app.ui.status = msg; +} + /// Run command `id` with the path picked for it in parameter `key`. fn run_with_picked(app: &mut EffectcraftApp, id: &str, key: &str, params: &Value, v: Value) -> Result { let mut p = params.as_object().cloned().unwrap_or_default(); diff --git a/crates/ui-egui/src/menus/tests.rs b/crates/ui-egui/src/menus/tests.rs index 0ed3fe7ad..0eaa957ac 100644 --- a/crates/ui-egui/src/menus/tests.rs +++ b/crates/ui-egui/src/menus/tests.rs @@ -637,3 +637,51 @@ fn menu_items_cover_the_tree() { assert_eq!(menus().last(), Some(&"Help")); assert!(effectcraft_engine::menus::entries().len() > 500); } + +/// A successful command that reports per-item problems (`errors`) shows a compact, readable +/// report in the status line: `file: error; …`, first three and a count, never raw JSON. +#[test] +fn bundle_result_errors_show_compactly_in_the_status_line() { + let mut app = EffectcraftApp::new(effectcraft_engine::Session::default()); + let result = json!({"installed": [], "errors": [ + {"file": "a.wasm", "error": "already installed"}, + {"file": "b.wasm", "error": "does not load"}, + {"file": "c.wasm", "error": "cannot read it"}, + {"error": "cannot read the bundle folder"}, + ]}); + super::frontend::show_result_errors(&mut app, &result); + assert_eq!(app.ui.status, "a.wasm: already installed; b.wasm: does not load; c.wasm: cannot read it; +1 more"); + // A clean success leaves the status line alone. + super::frontend::show_result_errors(&mut app, &json!({"installed": ["x.wasm"], "errors": []})); + assert_eq!(app.ui.status, "a.wasm: already installed; b.wasm: does not load; c.wasm: cannot read it; +1 more"); +} + +/// Effect ▸ Install Effect Plug-in… picks its dialog by params: a bundle asks for a folder, +/// a plain install asks for a `.wasm`, and with both given inline nothing asks at all. +#[test] +fn effect_plugin_install_dialog_router() { + use std::sync::{Arc, Mutex}; + let mut app = EffectcraftApp::new(effectcraft_engine::Session::default()); + let ctx = egui::Context::default(); + let pending = Arc::new(Mutex::new(Vec::new())); + let queue = pending.clone(); + app.hooks.file_dialog_async = Some(Box::new(move |dialog, _done| queue.lock().unwrap().push(dialog))); + // The bundle menu entry: a folder picker. + assert_eq!(invoke(&mut app, &ctx, "effect.plugins.install", json!({"bundle": true})).unwrap(), json!({"dialog": "effect.plugins.install"})); + assert_eq!(pending.lock().unwrap().pop().unwrap(), crate::FileDialog::Folder, "the bundle asks for a folder"); + // The plain command: a `.wasm` file picker. + assert_eq!(invoke(&mut app, &ctx, "effect.plugins.install", json!({})).unwrap(), json!({"dialog": "effect.plugins.install"})); + let crate::FileDialog::Open { exts, many } = pending.lock().unwrap().pop().unwrap() else { panic!("not an open dialog") }; + assert_eq!(exts, vec!["wasm".to_string()], "{exts:?}"); + assert!(!many); + // `path` given inline overrides the bundle ask: the command runs directly, opening no dialog + // (this session has no plug-in loader, so it errors — but nothing was asked). + let r = invoke(&mut app, &ctx, "effect.plugins.install", json!({"bundle": true, "path": "x.wasm"})); + assert!(r.is_err(), "{r:?}"); + assert!(pending.lock().unwrap().is_empty(), "no dialog for an inline path"); + // A supplied `folder` is a bundle source too: asking for a file would let the install take + // the picked file and ignore the folder, so nothing is asked. + let r = invoke(&mut app, &ctx, "effect.plugins.install", json!({"folder": "some/where"})); + assert!(r.is_err(), "{r:?}"); + assert!(pending.lock().unwrap().is_empty(), "no dialog for an inline folder"); +} diff --git a/crates/ui-egui/src/panels/effect_controls/panel.rs b/crates/ui-egui/src/panels/effect_controls/panel.rs index 4b3306e3d..0857dfa3f 100644 --- a/crates/ui-egui/src/panels/effect_controls/panel.rs +++ b/crates/ui-egui/src/panels/effect_controls/panel.rs @@ -127,6 +127,17 @@ fn drop_payload(ui: &egui::Ui, rect: Rect, p: &egui::Painter, t: &Tokens, layer_ } } +/// A plug-in effect whose last render failed (an effect helper that stopped, ...): a warning sign +/// at `at`, after the effect's name, with the message on hover. The frame shows the input as is. +fn render_error_mark(app: &mut EffectcraftApp, ui: &mut egui::Ui, bp: &egui::Painter, effect: &str, g: &PropGroup, at: egui::Pos2) { + let Some(err) = effectcraft_engine::effects::plugin::plugin_error(effect) else { return }; + let wr = bp.text(at, Align2::LEFT_CENTER, "⚠", Tokens::semibold(12.0), app.tokens.warning); + let resp = ui.interact(wr.expand(2.0), egui::Id::unique(("ec-fx-error", g.uid)), Sense::hover()); + app.auto.add(&format!("effectControls.effect.{}.error", g.uid), wr, &err); + resp.on_hover_text(err); + ui.ctx().request_repaint_after(std::time::Duration::from_millis(500)); +} + fn about_button(app: &mut EffectcraftApp, ui: &mut egui::Ui, bp: &egui::Painter, t: &Tokens, effect: &str, g: &PropGroup, r: Rect) { // About…: the effect's name and category (AE's About dialog). let about = Rect::from_min_size(pos2(r.max.x - 50.0, r.center().y - 9.0), vec2(44.0, 18.0)); @@ -246,6 +257,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { } else { let name = bp.text(pos2(tw.max.x + 6.0, r.center().y), Align2::LEFT_CENTER, &shown, Tokens::semibold(12.0), t.text); app.auto.add(&format!("effectControls.effect.{}.name", g.uid), name, &g.name); + render_error_mark(app, ui, &bp, effect, g, pos2(name.max.x + 8.0, r.center().y)); } let rresp = ui.interact(reset, egui::Id::unique(("ec-reset", g.uid)), Sense::click()); bp.text(reset.center(), Align2::CENTER_CENTER, crate::i18n::tr("Reset"), Tokens::ui(11.5), if rresp.hovered() { t.hot_text } else { t.text_dim }); diff --git a/crates/ui-egui/src/panels/misc.rs b/crates/ui-egui/src/panels/misc.rs index fb3b9063c..0c87721db 100644 --- a/crates/ui-egui/src/panels/misc.rs +++ b/crates/ui-egui/src/panels/misc.rs @@ -240,7 +240,7 @@ pub fn preview(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { Some(pl) => pl, None => effectcraft_engine::preview::plan(&o, &c, app.session.time()), }; - let cached_set = app.frames.cached_frames(&app.shown_series(cid)); + let cached_set = app.frames.cached_frames(&app.preview_series(cid, &o)); let total = pl.frames().count(); let cached = pl.frames().filter(|f| cached_set.binary_search(f).is_ok()).count(); let bar = Rect::from_min_size(pos2(rect.min.x + 12.0, yy), vec2(rect.width() - 24.0, 6.0)); diff --git a/crates/ui-egui/src/panels/viewer.rs b/crates/ui-egui/src/panels/viewer.rs index bf921535a..3da62d82b 100644 --- a/crates/ui-egui/src/panels/viewer.rs +++ b/crates/ui-egui/src/panels/viewer.rs @@ -1896,10 +1896,24 @@ fn phase_tail(app: &mut EffectcraftApp, ui: &mut egui::Ui, ectx: &EvalCtx, v: &m crate::panels::text_range_handles::viewer_hook(app, ui, &painter, &map, ectx, &|c, l| l2c(c, l).0); // Status chips (render time / caching). + // The caching chip stays up a moment after the last wait, so gaps between uncached frames + // don't make it flicker. + // ponytail: fixed 0.6 s hold; tie it to the prefetch queue being empty if gaps outlast it. + let chip_id = egui::Id::unique("viewer-caching-chip"); + let now = ctx.input(|i| i.time); if app.playback.playing && app.playback.waiting { - let r = Rect::from_min_size(area.min + vec2(10.0, 10.0), vec2(150.0, 22.0)); - painter.rect_filled(r, 11.0, Color32::from_black_alpha(170)); - painter.text(r.center(), Align2::CENTER_CENTER, crate::i18n::tr("Caching frames…"), Tokens::ui(11.5), t.cache_green); + ctx.data_mut(|d| d.insert_temp(chip_id, now + 0.6)); + } + let until = ctx.data(|d| d.get_temp::(chip_id)).unwrap_or(0.0); + if app.playback.playing && now < until { + if !app.playback.waiting { + ctx.request_repaint_after(std::time::Duration::from_secs_f64(until - now)); + } + let text = painter.layout_no_wrap(crate::i18n::tr("Caching frames").to_string(), Tokens::ui(10.5), t.text_dim); + let r = Rect::from_min_size(area.min + vec2(8.0, 8.0), vec2(text.size().x + 30.0, 18.0)); + painter.rect_filled(r, 9.0, Color32::from_black_alpha(140)); + ui.put(Rect::from_center_size(pos2(r.min.x + 12.0, r.center().y), vec2(10.0, 10.0)), egui::Spinner::new().size(10.0).color(t.cache_green)); + painter.galley(pos2(r.min.x + 22.0, r.center().y - text.size().y / 2.0), text, t.text_dim); } if app.video_preview_off().is_some() { let r = Rect::from_min_size(area.min + vec2(10.0, 10.0), vec2(330.0, 22.0)); diff --git a/docs/adr/0001-external-effect-helpers.md b/docs/adr/0001-external-effect-helpers.md new file mode 100644 index 000000000..732fae877 --- /dev/null +++ b/docs/adr/0001-external-effect-helpers.md @@ -0,0 +1,73 @@ +# ADR 0001: External effect helpers + +- Status: proposed +- Date: 2026-10-11 +- Context: review of storytold/effectcraft#508 (an in-process OpenFX host) + +## Context + +People want to use effects that already exist as native plug-ins: OpenFX suites (Sapphire, +ReelSmart Motion Blur, Twixtor, ...) and, later, other native formats. storytold/effectcraft#508 +hosted OpenFX in-process. That needs `unsafe` to call a C ABI and runs closed-source native +code in EffectCraft's own process, which conflicts with two non-negotiables: + +- **Never crash** (AGENTS.md): one null dereference in a third-party plug-in takes the whole + app, and the user's unsaved work, down with it. `catch_unwind` can't catch a segfault, and a + blocklist only helps after the first crash. +- **No `unsafe`** in the workspace (`unsafe_code = "forbid"`). + +## Decision + +EffectCraft never loads native plug-in code into its own process. Native effects are served by +**external effect helpers**: separate programs that EffectCraft starts as child processes and +talks to over a small, versioned protocol. The core gains one generic, safe-Rust crate, +`effectcraft-helper-host` (L3); format-specific hosts (OpenFX, ...) live in helper programs +that are built, released and installed separately. + +- **Crash containment.** The process boundary contains everything a helper does. A helper + that exits, hangs past a timeout or breaks the protocol is stopped. The render that hit it + fails with a message shown on the effect, the frame stays as it was, and the helper starts + again for the next render. An effect that takes its helper down three times is turned off and + remembered in `helper_blocklist.txt` until the user retries. +- **Same effects surface.** Each remote effect is registered through the existing plug-in API 2 + (`EffectPlugin`), so the UI, the property tree, keyframes, expressions, scripting, the CLI, the + control channel and MCP need nothing new. +- **Protocol** (`docs/external-helpers.md`): JSON lines over the helper's stdin / stdout + (`hello` → `describe` → `render` with callbacks → `done`, plus `padding`, `load`, `destroy`, + `ping`, `shutdown`). Pixels go through a **frame exchange file** (in `/dev/shm` on Linux, the + temp folder elsewhere) addressed by offsets, never through the pipe. Callbacks (`input_at`, + `layer_input`, `params_at`) let temporal effects (motion blur, retiming) read other frames + while rendering; nested renders through the same helper are supported. +- **Discovery.** EffectCraft looks in `helpers/` next to the executable (and + `Contents/Helpers` in a macOS bundle), `helpers/` in the settings folder, and + `EFFECTCRAFT_HELPER_PATH`. Helper programs are named `effectcraft-helper-*`. +- **Security.** Helpers are local programs the user installed, just like the plug-ins they host. + EffectCraft runs only programs from those locations, passes them nothing beyond the protocol + (no network, no project files), and runs no helper on the web build. + +## Alternatives considered + +- **In-process host behind a feature flag** (#508). Rejected: a crash is still a crash, and it + needs `unsafe` in the workspace. +- **`ipc-channel` for transport.** Its wire format and shared-memory handles are Rust-specific + (and differ per OS), so a helper written in C++ against the OpenFX reference host couldn't + speak it. The chosen transport, JSON lines plus a plain file, works from any language without + a library. +- **Memory-mapped shared memory on the EffectCraft side.** Mapping needs `unsafe` (or a crate + whose safe API is unsound if the other process truncates the mapping). Positional reads and + writes on a file in the page cache cost one copy per frame each way and stay safe. A helper is + free to map the file. +- **WebAssembly-compiled plug-ins.** Not available for closed-source native plug-ins. + +## Consequences + +- Native effects cost one copy of the frame each way plus process round trips. That's fine for + CPU plug-ins working on whole frames. +- Renders through one helper are serialized (one process, one conversation at a time). A pool + of helper processes is a possible later improvement. +- Out of scope for the core: any OpenFX (or other native-format) code. The OpenFX helper is a + separate program in its own repository (an example: [ec-extension-ofx](https://github.com/glumbis/ec-extension-ofx), unofficial) that reuses the + existing Rust OpenFX host behind `serve()`. A C++ helper built on the OpenFX HostSupport + library is an equally valid way to implement the same protocol. +- Later: the ArtCraft launcher can install helpers into `helpers/` the same way it installs + apps. diff --git a/docs/architecture.md b/docs/architecture.md index 49228d5ad..4400aeb01 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -44,6 +44,7 @@ cpal or muda. Everything in L0 to L4, the egui UI and the web app also build for | L3 | `aep` | After Effects projects in one crate (like `lottie`): `container` reads and writes the `.aep` container (RIFX `Egg!`) as a lossless chunk tree with a byte-identical round trip, bounded for hostile input, reads the `.aepx` XML form into the same tree, and has a typed view of items, layers, properties, effects, masks, shapes, text, markers, footage, cameras, lights and layer styles, from our own observations of generated files ([aep-format.md](aep-format.md)); `import` maps the typed view into a project (items, settings, compositions, every layer kind, Transform, keyframes, expressions, masks, text, shapes, cameras, lights, mattes, markers, time remapping, layer styles, the effects of the importer's table, footage) with a report of everything not imported ([aep-import.md](aep-import.md)) | | L3 | `lottie` | Lottie JSON / dotLottie import and export (layers, precomps, eased and spatial keyframes, shapes, masks, mattes) with a warnings list for what Lottie cannot express | | L3 | `plugin` | Effect plug-ins: loads sandboxed WebAssembly effect modules (plug-in API v1; the wasmi interpreter behind the `wasm` feature, on for native hosts) into the effect registry ([plugins.md](plugins.md)) | +| L3 | `helper-host` | External effect helpers: finds helper programs, runs each as a child process and registers its effects (plug-in API 2) over a versioned pipe + shared-file protocol; a crashing helper is an error on the effect, never a crash ([external-helpers.md](external-helpers.md), [ADR 0001](adr/0001-external-effect-helpers.md)). Native only | | L4 | `engine` | `Session`: project, branching undo history, editor state, the command registry and menus, the ScriptUI window model (`scriptui`) | | L4 | `script` | Scripting: JavaScript (boa) with an After Effects-style object model (`app.project`, comps, layers, properties, render queue) and ScriptUI (dialogs, palettes, dockable panels; script-host threads so dialogs block `show()`), whose edits run engine commands; AE match names | | L4 | `host` | A fully wired `Session` (media, expressions, scripting, WebAssembly plug-ins, exporter) for the frontends | diff --git a/docs/external-helpers.md b/docs/external-helpers.md new file mode 100644 index 000000000..3c513df95 --- /dev/null +++ b/docs/external-helpers.md @@ -0,0 +1,98 @@ +# External effect helpers + +An **external effect helper** is a separate program that offers effects to EffectCraft. Each +helper runs as a child process and talks to EffectCraft over the protocol below. EffectCraft +registers every effect a helper lists as an ordinary plug-in effect: it shows up in Effects & +Presets, has normal properties (keyframes, expressions, Effect Controls) and renders through +the helper. Helpers exist to run native code, such as an OpenFX plug-in host, **outside** +EffectCraft's process. If a helper crashes or hangs, EffectCraft shows the error on the effect +and carries on. See [ADR 0001](adr/0001-external-effect-helpers.md) for why. + +## Using helpers + +| What | How | +|---|---| +| Install | Put the helper program (`effectcraft-helper-`, `.exe` on Windows) in `helpers/` next to EffectCraft, in `helpers/` in the [settings folder](preferences.md), or in a folder named in `EFFECTCRAFT_HELPER_PATH` (a path list; entries can be programs or folders). Subfolders one level down are searched too. | +| Start | Helpers run native code, so nothing starts them unless the user asks: Effect ▸ Scan for Effect Helpers (`effect.helpers.scan`) starts them. To start them every time EffectCraft launches, opt in with `EFFECTCRAFT_HELPERS=1`. It is an environment variable, not a setting, so writing to the settings folder can't turn it on. Agents confined with `--automation-read-root` / `--automation-write-root` can't run `effect.helpers.scan` or `effect.helpers.load`, and scripts can run them only with Allow Scripts to Write Files and Access Network on. | +| Load more plug-ins | Effect ▸ Load Plug-ins Through Helper… (`effect.helpers.load {path}`) hands a file or folder to the helpers that accept it (an OpenFX helper takes `.ofx` plug-ins and `.ofx.bundle` folders). | +| Inspect | `effect.helpers.list` → `{helpers: [{program, name, version, loads, running, effects, restarts, error}], blocked}`; `effect.plugins.list` shows each effect's `source` (`helper: …`) and last render `error`. | +| Errors | A failed render leaves the frame as it was and puts ⚠ after the effect's name in Effect Controls (hover for the message). The next render starts the helper again. An effect that stops its helper 3 times is turned off and listed in `helper_blocklist.txt` in the settings folder. `effect.helpers.scan {"retry": true}` turns it back on. A helper that stops 5 times in a minute stays off until the next scan. | +| Timeouts | `EFFECTCRAFT_HELPER_TIMEOUT` (seconds per render step, default 120) and `EFFECTCRAFT_HELPER_LOAD_TIMEOUT` (start-up, describe, load; default 600). A helper idle for 30 s is pinged before it is used again. | + +Helpers are not available in the web build. + +## Protocol (version 1) + +EffectCraft starts the helper with the single argument `--effectcraft-helper`, with stdin, +stdout and stderr piped. The environment variable `EFFECTCRAFT_HELPER_STATE_DIR` names a folder +where the helper may keep its own records (for example, plug-ins that crashed it while +loading). + +- **Requests** (EffectCraft → helper): one JSON object per line on stdin. +- **Replies** (helper → EffectCraft): one JSON object per line on stdout, prefixed with + `@ec-helper ` (with the space). Lines without the prefix, or text before it, are logged and + otherwise ignored, so plug-ins that print to stdout don't break the protocol. stderr is + logged (whatever bytes it holds). +- Every object has an `"op"`. Lines are at most 256 MB. +- The helper exits when stdin closes or on `shutdown`. + +### Frame exchange file + +Pixels never go through the pipe. EffectCraft creates a file (in `/dev/shm` on Linux when +available, otherwise the temp folder) and names it in `hello`. Images are addressed by an +**image ref**: + +```json +{"offset": 0, "width": 1920, "height": 1080, "origin": [0, 0], "scale": 1} +``` + +The image is `width × height` pixels starting `offset` bytes into the file. Each pixel is +premultiplied RGBA as four `f32`s in native byte order (16 bytes), rows top to bottom, and an +image is at most 2²⁸ pixels. `origin` is the buffer position of layer pixel (0, 0); `scale` is +buffer pixels per layer pixel. EffectCraft decides every offset, so the helper only reads and +writes where it's told. On Unix, EffectCraft unlinks the file once the helper has opened it, so +nothing is left behind if either side dies. Helpers may map the file instead of reading it. + +### Messages + +| Request | Reply | | +|---|---|---| +| `{"op":"hello","version":1,"frames":""}` | `{"op":"hello","version":1,"name":"OpenFX","helper_version":"1.0","loads":["ofx"]}` | First message. The helper opens `frames` read/write. `loads` lists the extensions `load` accepts. A different `version` means EffectCraft won't use the helper. | +| `{"op":"describe"}` | `{"op":"effects","effects":[{"manifest":{…},"padding":true}],"errors":["…"]}` | `manifest` is a [plug-in API 2 manifest](plugins.md#plug-in-api-2-native-plug-ins) (id, name, category, params). `padding: true` means EffectCraft asks `padding` before each render. | +| `{"op":"load","path":"…"}` | `effects` (only the new ones) or `error` | A file or folder the user picked. Replayed after a restart. | +| `{"op":"padding","effect":"…","instance":7,"time":1.0,"scale":1.0,"layer_size":[w,h],"flat":[…],"texts":[…]}` | `{"op":"padding","pixels":3}` | Transparent pixels to add on each side before the render (at most 4096). | +| `{"op":"render","effect":"…","instance":7,"time":1.0,"frame":,"layer_size":[w,h],"flat":[…],"texts":[…],"frame_rate":30,"frame_range":[0,10]}` | callbacks, then `{"op":"done"}` or `{"op":"error","message":"…"}` | Process `frame` in place: read it, write the result to the same ref, reply `done`. `flat` / `texts` are the parameter values flattened by plug-in API 2 rules. `instance` identifies the applied effect (stable across frames). `frame_range` is the layer's source range in layer-time seconds, or `null`. | +| `{"op":"destroy","instance":7}` | none | The instance is gone; free what it holds. | +| `{"op":"ping"}` | `{"op":"pong"}` | Heartbeat. | +| `{"op":"shutdown"}` | none | Exit. | + +**Callbacks during a render.** Before it replies `done`, the helper may ask: + +| Helper asks | EffectCraft answers | +|---|---| +| `{"op":"input_at","time":t}` | `{"op":"image","image":\|null}`: the effect's input (the layer, its masks, the effects above) at layer time `t`, in its own extent. | +| `{"op":"layer_input","param":"","time":t}` | `image`: that layer at layer time `t` with effects and masks, resampled into this frame's grid. | +| `{"op":"params_at","time":t}` | `{"op":"params","values":[[flat…],[texts…]]\|null}` | + +While a helper waits for an answer, EffectCraft may send another `render` (or `padding`). This +happens when producing the answer needs an effect of the same helper: for example, motion blur +asking for the previous frame of a layer that also has a retime effect from the same helper. +The helper must serve it completely (with its own callbacks, nested up to 16 deep) and then +keep waiting. Images for callbacks are placed after the frame in the exchange file and freed +when the render ends. + +### Errors and restarts + +`error` replies are per request: the helper carries on. EffectCraft stops (kills) the helper +when it exits, closes stdout, sends a reply that doesn't parse or doesn't fit, or misses a +timeout, or stops reading its stdin for longer than the timeout. The request then fails. The next request starts a fresh process, runs `hello`, replays +earlier `load`s and recreates instances lazily: the helper sees a `render` for an `instance` it +doesn't know and creates it. + +## Writing a helper + +Any language works: read lines, write prefixed lines, read and write a file. Rust helpers can +use `effectcraft_helper_host::serve::serve` with a `Provider` that lists `EffectPlugin`s; +`crates/helper-host/tests/it/main.rs` is a complete example and the protocol test that runs +in `cargo xtask ci`. An unofficial OpenFX helper built this way lives in its own repository, +[ec-extension-ofx](https://github.com/glumbis/ec-extension-ofx). diff --git a/docs/gaps.md b/docs/gaps.md index 291535cae..f3ddeee8e 100644 --- a/docs/gaps.md +++ b/docs/gaps.md @@ -309,6 +309,11 @@ Issues #41–#47 (all from the Linux AppImage 0.1.1, 5 October): panning the vie Settings ▸ Plug-in Library installs, updates and removes community WebAssembly effects listed in one reviewed catalog ([plugins.md](plugins.md#plug-in-library), #376); the catalog repository still has to be created and filled. +- Native plug-in formats (OpenFX first) run only in **external effect helpers**, separate + processes behind a safe, versioned protocol ([external-helpers.md](external-helpers.md), + [ADR 0001](adr/0001-external-effect-helpers.md)): the core gains no native code and no + `unsafe`, and a crashing plug-in is an error on the effect. The helpers themselves ship + separately. - Features that come from third parties in After Effects are extensions in EffectCraft too, built on the same extension points as After Effects: scripts, ScriptUI panels and effect plug-ins ([plugins.md](plugins.md#extension-points-compared-with-after-effects) compares them). ScriptUI diff --git a/docs/plugins.md b/docs/plugins.md index 41a1756d9..3fd8cf944 100644 --- a/docs/plugins.md +++ b/docs/plugins.md @@ -152,7 +152,7 @@ example of a keyframe tool. ## Effect plug-ins -EffectCraft loads third-party effects through a small, versioned plug-in API (**API version 1**). +EffectCraft loads third-party effects through a small, versioned plug-in API (**API version 2**; WebAssembly ABI 1). A plug-in effect behaves exactly like a built-in one: it is listed in Effects & Presets (and the Effect menu) under its category, applied by id (`effect.apply {"effect": "org.example.posterize"}`) or display name, its parameters are ordinary properties (keyframes, expressions, Effect Controls, @@ -163,7 +163,7 @@ There are two ways to write one: | | WebAssembly plug-in | Rust plug-in | |---|---|---| | What | a `.wasm` (or `.wat`) module | a type implementing `effectcraft_effects::plugin::EffectPlugin` | -| Loaded by | Effect ▸ Load Effect Plug-in…, `effect.plugins.load {path \| folder}`, or the `Plug-ins` folder next to the settings at start-up | `register_plugin(Arc::new(MyEffect))` in an app that links EffectCraft's crates | +| Loaded by | Effect ▸ Install Effect Plug-in… (copied into the `Plug-ins` folder next to the settings and loaded on every start-up), Effect ▸ Load Effect Plug-in… / `effect.plugins.load {path \| folder}` (loads in this session only), the `Plug-ins` folder next to the settings at start-up, or `effect.plugins.install {path \| folder}` | `register_plugin(Arc::new(MyEffect))` in an app that links EffectCraft's crates | | Sandbox | yes: no imports (no files, clock or network), fuel-limited, at most 1 GiB of memory, deterministic floats | none (it is your code) | | Platforms | desktop and CLI (the wasmi interpreter, crate feature `effectcraft-plugin/wasm`); not the web build | everywhere | @@ -241,11 +241,59 @@ plain Rust (no dependencies): ```sh cd examples/plugins/posterize-bands cargo build --release --target wasm32-unknown-unknown -# then, in EffectCraft: Effect ▸ Load Effect Plug-in… → target/wasm32-unknown-unknown/release/posterize_bands.wasm +# then, in EffectCraft: Effect ▸ Install Effect Plug-in… +# → target/wasm32-unknown-unknown/release/posterize_bands.wasm (copied into Plug-ins, loads on start-up) +# or in-session only: Effect ▸ Load Effect Plug-in… # or headless: effectcraft-cli exec effect.plugins.load '{"path": "target/wasm32-unknown-unknown/release/posterize_bands.wasm"}' ``` +**Bundles.** `effect.plugins.install {"folder": ...}` installs a whole plug-in folder in one +go: every `.wasm` at its top level is copied into the `Plug-ins` folder (and loads at once), +and every `.jsx` at its top level is installed as a ScriptUI panel — so a third-party plug-in +ships as one folder the user installs once. A `.wasm` that does not load is not copied (the +install rolls it back), and per-file problems in a folder install come back in `errors` +instead of failing the whole bundle. + +**What an install does and does not do.** It is a security boundary: it copies code into a +folder the app runs at every start, so: + +* Only `.wasm` (`path` / a bundle's `.wasm`) and `.jsx` (a bundle's panel) are installed; other + files in a bundle (readmes, manifests, sources) are skipped. Files are named by their own + entry name, so a bundle cannot carry a path, a `..` component or a name that escapes the + destination folder. +* Nothing is overwritten. An existing plug-in or panel with the same file name makes that file + fail with "already installed": delete it from `Plug-ins` / `Scripts/ScriptUI Panels` first to + update it (versioning the file name is the other option). A bundle is installed file by file, + so one collision does not undo the rest. +* Files over 64 MiB are refused before they are read, rather than copied into a startup folder. +* A `.wasm` is loaded once, in the same sandbox it runs in later (see below), so a module that + cannot load is reported instead of failing every start. That load runs the module's start-up + code and its metadata exports; `ec_render` is never called, and a panel's script is not run at + install time (it runs when the panel opens, as with any installed panel). The copy made before + the load is then removed again — best effort: if removing it fails, the command says so and + names the file, and you delete it from `Plug-ins` yourself. +* A `.wasm` is uninstalled by deleting it from `Plug-ins` (the next start no longer loads it): + there is no `effect.plugins.uninstall`. A panel installed here is an installed script, so + `file.uninstallScript {name}` removes it. Replacing either means removing it first. +* Hidden files (names starting with `.`, such as macOS `._x.wasm` copies) are skipped in a bundle + and refused as a single install. A panel name that differs from an installed panel's only in + letter case counts as the same name. +* Installed plug-ins share the `Plug-ins` folder with the [Plug-in Library](#plug-in-library) + (which keeps its packages in `Plug-ins/Library`). An effect id can be registered only once: a + plug-in whose id is already loaded, from either kind of install, is refused. At start-up the + files installed here load before the library's packages, so if both carry the same id (for + example after files were copied by hand), the file here wins and the library package reports + the duplicate. + +**The sandbox.** A plug-in runs in the [wasmi](https://github.com/wasmi-labs/wasmi) interpreter +with no imports (no file, network or clock access), deterministic floats, a fuel budget per +frame and a 1 GiB memory cap, so a plug-in cannot escape the sandbox, hang the app or render +differently on another machine. A ScriptUI panel is different: it is a script with the app's +scripting object model, so it can read and write project data through commands; file and +network access for scripts follow Settings ▸ Scripting & Expressions, exactly as for a script +installed with File ▸ Scripts ▸ Install ScriptUI Panel…. + Any language that compiles to WebAssembly works (C, Zig, AssemblyScript…) as long as the module has no imports. `crates/plugin/src/tests.rs` has a plug-in written directly in WebAssembly text. @@ -270,8 +318,39 @@ impl EffectPlugin for Swap { register_plugin(Arc::new(Swap(manifest)))?; // before the UI builds its menus ``` -`PluginParams` reads values by parameter id (`f`, `b`, `point`, `color`). At most 256 plug-in -effects can be registered per process. +`PluginParams` reads values by parameter id (`f`, `b`, `point`, `color`, and in API 2 `point3`, +`layer`, `text`). At most 1024 plug-in effects can be registered per process. + +## Plug-in API 2 (native plug-ins) + +API 2 is for Rust plug-ins and for hosts that bring effects from elsewhere (such as +[external effect helpers](external-helpers.md)); the WebAssembly ABI stays at 1. Manifests with `"api": 1` or `2` are accepted, +`effect.plugins.list` reports `api: 2`, and up to 1024 plug-in effects can be registered. + +* **New parameter types**: `point3` (`default` `[x, y, z]`; x, y arrive in buffer pixels, z as is), + `layer` (another layer of the composition; flattens to its id, -1 for none, read with + `PluginParams::layer`), `text` (a string field) and `hidden` (a string that is not shown, for + opaque plug-in data). `text` / `hidden` take no slot in `flat`; their strings are in + `PluginParams::texts`, one per such parameter in declaration order, read with + `PluginParams::text(id)`. `slider` gains `"integer": true` (shown with 0 decimals) and every + parameter an optional `"group"` path (`"Advanced/Edges"`; informational). WebAssembly modules + can't use `point3`, `layer`, `text` or `hidden`. +* **`PluginFrame`** also carries `origin` (buffer pixels of layer pixel (0, 0); padding shifts it) + and `layer_size` (layer pixels). +* **`EffectPlugin::padding(params, time, scale, layer_size, instance_key) -> u32`** asks for that + many transparent buffer pixels on every side before rendering (glows, regions of definition + larger than the input); at most 4096. `layer_size` is the layer's size in layer pixels (so + sizes relative to the frame can be measured) and `instance_key` identifies the applied effect + instance (see `PluginHost::instance_key`). +* **`EffectPlugin::render_v2(frame, params, time, host)`** (default: calls `render`) gets a + `PluginHost` with `input_at(layer_time)` (the effect's input at another time, in its own extent; + the effects above this one are applied), `layer_input(param_id, layer_time)` (a layer parameter's + layer, effects and masks applied, resampled into the frame's grid), `params_at(layer_time)`, + `frame_rate()`, `time()`, `instance_key()` (a stable id of the applied effect instance: the + same on every frame, different per layer and per application; plug-ins that keep state between + frames pool their native instances by it) and `frame_range()` (the layer's source time range in + layer-time seconds, `None` when it has none). Images are `PluginImage { width, height, pixels, origin, scale }`. + It may run on several threads at once. ### Compatibility diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 81e30ea70..24023ee88 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -58,6 +58,7 @@ const LAYERS: &[(&str, u8)] = &[ ("lottie", 3), ("format", 3), ("plugin", 3), + ("helper-host", 3), ("interchange", 3), ("engine", 4), ("script", 4),