effectcraft/crates/ui-egui/src/control.rs

//! Programmatic control of the running app (agents, tests, MCP bridge).
//!
//! Transport-agnostic: a transport thread (TCP in the desktop app) sends [`ControlRequest`]s; the
//! UI thread handles them between frames and replies with `{"ok":…, "result"|"error":…}`.
//!
//! Methods:
//! - `engine.execute {command, params}` / `ui.menu.invoke {id, params}`: run any engine or UI command
//! - `engine.commands`: list engine commands; `ui.menu.list`: menu tree with enablement
//! - `ui.inspect`: UI state + summary; `ui.elements {prefix?}`: registered widgets (id, label, rect)
//! - `ui.set {tool?, workspace?, theme?, focused?, viewer?: {zoom, res, pan, grid, …}, timeline?: {pps, start, graphEditor, showModes}}`
//! - `ui.panel.show {panel}` / `ui.panel.close {panel}`
//! - `ui.click {id | x,y, button?, count?, modifiers?}` / `ui.move {x,y}` / `ui.scroll {x,y,dx,dy,unit?,source?,pinch?}`
//! - `ui.drag {from:{id|x,y}, to:{id|x,y}, steps?, modifiers?}`: synthetic press-move-release
//! - `ui.key {key, command?, shift?, alt?, ctrl?}` / `ui.type {text}`
//! - `ui.viewer.locate {x, y}`: screen point of a comp pixel; `ui.viewer.hit {x, y}`: comp pixel at a screen point
//! - `ui.timeline.locate {layer, time?}`: screen point of a layer bar (at a comp time)
//! - `ui.playback {action: play|stop|toggle}`
//! - `ui.screenshot {path?, panel?, id?}`: PNG of the window (or one panel / element, resolved when the capture is issued)
//! - `render.frame {comp?, time?, max_side?, path?, base64?}`: PNG of a comp frame rendered by the session
//! - `ui.resize {width, height}` / `ui.focus` / `app.quit {force?}` (a modified project asks to save first unless `force`)
//!
//! The full reference with examples is `docs/control-protocol.md`.

use std::sync::mpsc::Sender;

use serde_json::{Value, json};

use crate::EffectcraftApp;
use crate::dock::PanelKind;
use crate::state::{Resolution, Tool};

/// Prefix marking errors that may resolve after another frame.
pub const RETRY: &str = "\u{1}";

pub struct ControlRequest {
    pub method: String,
    pub params: Value,
    pub reply: Sender<Value>,
}

impl ControlRequest {
    /// Creates a request for `method` with `params`, returning it with the channel that will
    /// receive the UI thread's reply.
    pub fn new(method: impl Into<String>, params: Value) -> (Self, std::sync::mpsc::Receiver<Value>) {
        let (tx, rx) = std::sync::mpsc::channel();
        (Self { method: method.into(), params, reply: tx }, rx)
    }
}

pub enum Outcome {
    Done(Value),
    /// Try again on a later frame (e.g. the element is not on screen yet).
    Retry(String),
    /// Reply once queued synthetic input has been processed.
    AfterInput,
    Screenshot {
        path: Option<String>,
        /// The panel/element identity to capture, resolved again when the capture is issued.
        target: crate::ShotTarget,
    },
    /// Reply once background jobs end (a `wait: true` command whose job runs in a worker).
    AwaitJobs(JobWaiter),
}

/// A `wait: true` engine command whose job was sent to the session's offload (the browser's
/// Web Workers) instead of blocking the UI thread: the reply goes out when the job ends.
pub struct JobWaiter {
    pub command: String,
    pub params: Value,
    /// The command's own result (with `wait: false`).
    pub result: Value,
    /// Ids of the jobs it started (`jobs.list` ids: `render`, `track`, `roto`…).
    pub jobs: Vec<String>,
    /// Absolute completion-log position when it started (including evicted entries).
    pub log_cursor: usize,
}

/// Commands that block with `wait` (and its default).
fn waits(command: &str, params: &Value) -> bool {
    let default = command == "renderQueue.render";
    params.get("wait").and_then(Value::as_bool).unwrap_or(default)
}

/// Run an engine command. A waiting command (`wait: true`) whose job the session offloads
/// runs with `wait: false` instead, and the reply waits for the job ([`Outcome::AwaitJobs`]),
/// so the browser's UI keeps drawing while it runs.
fn execute_engine(app: &mut EffectcraftApp, id: &str, params: Value) -> Result<Outcome, String> {
    if !app.session.offloads() || !waits(id, &params) {
        return app.session.execute_checked(id, params).map(ok).map_err(|e| e.to_string());
    }
    let before: Vec<String> = app.session.jobs().into_iter().map(|j| j.id).collect();
    let log_cursor = app.session.offload_log_start.saturating_add(app.session.offload_log.len());
    let mut p = params.clone();
    p["wait"] = json!(false);
    let result = app.session.execute_checked(id, p).map_err(|e| e.to_string())?;
    let jobs: Vec<String> = app.session.jobs().into_iter().map(|j| j.id).filter(|j| !before.contains(j)).collect();
    if jobs.is_empty() {
        return Ok(ok(result));
    }
    Ok(Outcome::AwaitJobs(JobWaiter { command: id.to_string(), params, result, jobs, log_cursor }))
}

impl JobWaiter {
    /// Whether its jobs are still running.
    pub fn pending(&self, app: &EffectcraftApp) -> bool {
        app.session.jobs().iter().any(|j| self.jobs.contains(&j.id))
    }

    /// The reply: what the blocking command would have returned (`running: false`, the final
    /// progress, render items and analysis status), or the job's error.
    pub fn finish(self, app: &mut EffectcraftApp) -> Value {
        let s = &mut app.session;
        s.poll_render();
        let error = s
            .offload_log
            .iter()
            .skip(self.log_cursor.saturating_sub(s.offload_log_start))
            .filter(|(kind, _)| self.jobs.iter().any(|id| id == kind.job_id()))
            .find_map(|(_, e)| e.as_ref());
        if let Some(e) = error {
            return json!({"ok": false, "error": e});
        }
        let mut r = self.result;
        if r.get("running").is_some() {
            r["running"] = json!(false);
        }
        if self.command == "renderQueue.render" {
            let ids: Vec<Value> = r["items"].as_array().map(|a| a.iter().map(|i| i["id"].clone()).collect()).unwrap_or_default();
            if let Ok(list) = s.execute("renderQueue.list", json!({})) {
                r["items"] =
                    json!(list["items"].as_array().map(|a| a.iter().filter(|i| ids.contains(&i["id"])).cloned().collect::<Vec<_>>()).unwrap_or_default());
            }
            r["rendering"] = json!(s.is_rendering());
        }
        let prefix = self.command.split('.').next().unwrap_or("");
        if matches!(prefix, "warp" | "camera" | "roto")
            && r.get("layer").is_some()
            && let Ok(st) = s.execute(&format!("{prefix}.status"), json!({"layer": r["layer"], "effect": r["effect"]}))
        {
            r["status"] = st;
        }
        if r.get("progress").is_some() {
            r["progress"] = Value::Null;
        }
        json!({"ok": true, "result": r})
    }
}

fn ok(v: Value) -> Outcome {
    Outcome::Done(json!({"ok": true, "result": v}))
}
fn err(e: impl std::fmt::Display) -> Outcome {
    Outcome::Done(json!({"ok": false, "error": e.to_string()}))
}

fn modifiers(p: &Value) -> egui::Modifiers {
    let m = p.get("modifiers").unwrap_or(p);
    let b = |k: &str| m.get(k).and_then(Value::as_bool).unwrap_or(false);
    egui::Modifiers { alt: b("alt"), ctrl: b("ctrl"), shift: b("shift"), mac_cmd: cfg!(target_os = "macos") && b("command"), command: b("command") }
}

/// Resolve a point from `{id}` (element centre) or `{x, y}`.
fn point(app: &EffectcraftApp, p: &Value) -> Result<egui::Pos2, String> {
    if let Some(id) = p.get("id").and_then(Value::as_str) {
        let e = app.auto.find(id).ok_or_else(|| format!("{RETRY}no element `{id}` (see ui.elements)"))?;
        let fx = p.get("fx").and_then(Value::as_f64).unwrap_or(0.5) as f32;
        let fy = p.get("fy").and_then(Value::as_f64).unwrap_or(0.5) as f32;
        return Ok(egui::pos2(e.rect[0] + e.rect[2] * fx, e.rect[1] + e.rect[3] * fy));
    }
    let x = p.get("x").and_then(Value::as_f64).ok_or("need `id` or `x`,`y`")?;
    let y = p.get("y").and_then(Value::as_f64).ok_or("need `y`")?;
    Ok(egui::pos2(x as f32, y as f32))
}

/// Dispatches one control request on the UI thread. Replies are `{"ok": true, "result": …}`
/// or `{"ok": false, "error": …}`; an unknown method is an error. Some methods answer with
/// [`Outcome::Retry`], [`Outcome::AfterInput`], [`Outcome::Screenshot`] or
/// [`Outcome::AwaitJobs`] instead of replying in this call.
pub fn handle(app: &mut EffectcraftApp, ctx: &egui::Context, req: &ControlRequest, now: f64) -> Outcome {
    let p = &req.params;
    match req.method.as_str() {
        "engine.execute" | "ui.menu.invoke" => execute_command(app, ctx, &req.method, p),
        "engine.commands" => engine_commands(app, p),
        "ui.menu.list" => ok(serde_json::to_value(crate::menus::menu_items(app)).unwrap_or_default()),
        "ui.inspect" => ok(inspect(app, ctx)),
        "ui.elements" => elements(app, p),
        "ui.set" => set_ui(app, ctx, p),
        "ui.panel.show" | "ui.panel.close" => panel(app, &req.method, p),
        "ui.click" | "ui.move" => click_or_move(app, &req.method, p),
        "ui.drag" => drag(app, p),
        "ui.scroll" => scroll(app, p),
        "ui.key" => key(app, p),
        "ui.type" => type_text(app, p),
        "ui.viewer.locate" => viewer_locate(ctx, p),
        "ui.viewer.hit" => viewer_hit(app, ctx, p),
        "ui.timeline.locate" => timeline_locate(app, p),
        "ui.playback" => playback(app, p, now),
        "ui.screenshot" => screenshot(app, ctx, p),
        "render.frame" => render_frame(app, p),
        "ui.resize" => resize(ctx, p),
        "ui.focus" => focus(ctx),
        "app.quit" => quit(app, ctx, p),
        m => err(format!("unknown method `{m}`")),
    }
}

/// Runs `engine.execute` or `ui.menu.invoke`. `engine.execute` with a known engine command id
/// goes through [`execute_engine`] (so a `wait: true` job can be offloaded); every other id,
/// and every `ui.menu.invoke`, goes through the menu dispatcher. A failure is copied to the
/// status line.
fn execute_command(app: &mut EffectcraftApp, ctx: &egui::Context, method: &str, p: &Value) -> Outcome {
    let s = |k: &str| p.get(k).and_then(Value::as_str);
    let Some(id) = s("command").or(s("id")) else { return err("missing `command`") };
    let params = p.get("params").cloned().filter(|v| !v.is_null()).unwrap_or(json!({}));
    // `engine.execute` runs engine commands exactly as headless (never opens a dialog, e.g.
    // `comp.new {}` creates a default comp); UI-only ids (`tool.*`, `view.*`, `window.*`,
    // `playback.*` …) and `ui.menu.invoke` go through the menu dispatcher like a click.
    let r = if method == "engine.execute" && effectcraft_engine::find_command(id).is_some() {
        execute_engine(app, id, params)
    } else {
        crate::menus::invoke(app, ctx, id, params).map(ok)
    };
    match r {
        Ok(o) => o,
        Err(e) => {
            app.ui.status = e.clone();
            err(e)
        }
    }
}

/// Replies with `command.list`'s result, forwarding `filter` and `enabledOnly`.
fn engine_commands(app: &mut EffectcraftApp, p: &Value) -> Outcome {
    match app.session.execute("command.list", json!({"filter": p.get("filter"), "enabledOnly": p.get("enabledOnly")})) {
        Ok(v) => ok(v),
        Err(e) => err(e),
    }
}

/// Replies with the automation elements whose id starts with `prefix` (all when empty), as
/// `{id, label, rect}` records.
fn elements(app: &EffectcraftApp, p: &Value) -> Outcome {
    let prefix = p.get("prefix").and_then(Value::as_str).unwrap_or("");
    ok(serde_json::to_value(app.auto.query(prefix)).unwrap_or_default())
}

/// Explicit request: activate the app and take keyboard focus.
fn focus(ctx: &egui::Context) -> Outcome {
    ctx.send_viewport_cmd(egui::ViewportCommand::Focus);
    ok(Value::Null)
}

/// Applies `ui.set` in order: tool, workspace, theme, focused panel, viewer settings, then the
/// layout fields ([`set_ui_layout`]). An unknown tool, workspace or theme name, or an invalid
/// viewer value, replies with an error after the earlier fields were already applied.
fn set_ui(app: &mut EffectcraftApp, ctx: &egui::Context, p: &Value) -> Outcome {
    let s = |k: &str| p.get(k).and_then(Value::as_str);
    if let Some(t) = s("tool") {
        match Tool::from_name(t) {
            Some(t) => app.ui.tool = t,
            None => return err(format!("unknown tool `{t}`")),
        }
    }
    if let Some(w) = s("workspace") {
        match crate::dock::WORKSPACES.iter().find(|x| x.eq_ignore_ascii_case(w)) {
            Some(w) => app.set_workspace(w),
            None => return err(format!("unknown workspace `{w}`")),
        }
    }
    if let Some(th) = s("theme") {
        match crate::theme::ThemeKind::from_name(th) {
            Some(k) => app.set_theme(ctx, k),
            None => return err("unknown theme (dark, darker, light)"),
        }
    }
    if let Some(f) = s("focused").and_then(PanelKind::from_name) {
        app.ui.focused = f;
    }
    if let Some(v) = p.get("viewer")
        && let Err(e) = apply_viewer(app, v)
    {
        return err(e);
    }
    set_ui_layout(app, p)
}

/// The remaining `ui.set` fields, applied in the same order after the viewer settings.
fn set_ui_layout(app: &mut EffectcraftApp, p: &Value) -> Outcome {
    if let Some(tl) = p.get("timeline") {
        apply_timeline(app, tl);
    }
    if let Some(b) = p.get("menuBar").and_then(Value::as_bool) {
        app.ui.show_menu_bar = b;
    }
    // Project panel optional columns, e.g. ["type", "duration", "path", "comment"].
    if let Some(Value::Array(a)) = p.get("projectColumns") {
        apply_project_columns(app, a);
    }
    // Wiggler / Smoother / Motion Sketch settings (fields of `AnimToolsState`).
    if let Some(Value::Object(m)) = p.get("animTools")
        && let Err(e) = apply_anim_tools(app, m)
    {
        return err(e);
    }
    if let Some(b) = p.get("home").and_then(Value::as_bool) {
        app.ui.start_screen = b;
    }
    // Home screen tab: "home", "templates" or "learn".
    if let Some(tab) = p.get("homeTab").and_then(Value::as_str) {
        app.ui.home_learn = tab == "learn";
        app.ui.home_templates = tab == "templates";
    }
    ok(Value::Null)
}

/// Applies viewer `zoom`, the active comp's `res` (stored in the project), `pan` and the
/// `grid`, `rulers`, `safeMargins` and `transparencyGrid` toggles. An unknown resolution name
/// is an error; a resolution change without an active comp is ignored.
fn apply_viewer(app: &mut EffectcraftApp, v: &Value) -> Result<(), String> {
    if let Some(z) = v.get("zoom") {
        app.ui.viewer.zoom = z.as_f64().map(|z| z as f32);
    }
    // The active comp's Resolution (`view.res.*`), saved with the project.
    if let Some(r) = v.get("res").and_then(Value::as_str) {
        let Some(res) = Resolution::ALL.into_iter().find(|x| x.label().eq_ignore_ascii_case(r)) else {
            return Err(format!("unknown resolution `{r}` (auto, full, half, third, quarter)"));
        };
        if let Some(cid) = app.session.active_comp_id()
            && let Err(e) = app.session.set_resolution(cid, res)
        {
            return Err(e.to_string());
        }
    }
    if let Some(Value::Array(a)) = v.get("pan") {
        app.ui.viewer.pan = [a.first().and_then(Value::as_f64).unwrap_or(0.0) as f32, a.get(1).and_then(Value::as_f64).unwrap_or(0.0) as f32];
    }
    for (k, slot) in [
        ("grid", &mut app.ui.viewer.grid),
        ("rulers", &mut app.ui.viewer.rulers),
        ("safeMargins", &mut app.ui.viewer.safe_margins),
        ("transparencyGrid", &mut app.ui.viewer.transparency_grid),
    ] {
        if let Some(b) = v.get(k).and_then(Value::as_bool) {
            *slot = b;
        }
    }
    Ok(())
}

/// Applies timeline `pps`, `start`, `graphEditor`, `showModes`, `openLayers` and `openGroups`
/// to the UI state, leaving absent fields unchanged.
fn apply_timeline(app: &mut EffectcraftApp, tl: &Value) {
    if let Some(v) = tl.get("pps") {
        app.ui.timeline.pps = v.as_f64();
    }
    if let Some(v) = tl.get("start").and_then(Value::as_f64) {
        app.ui.timeline.start = v;
    }
    if let Some(v) = tl.get("graphEditor").and_then(Value::as_bool) {
        app.ui.timeline.graph_editor = v;
    }
    if let Some(v) = tl.get("showModes").and_then(Value::as_bool) {
        app.ui.timeline.show_modes = v;
    }
    if let Some(Value::Array(a)) = tl.get("openLayers") {
        app.ui.timeline.open_layers = a.iter().filter_map(Value::as_u64).collect();
    }
    if let Some(Value::Array(a)) = tl.get("openGroups") {
        app.ui.timeline.open_groups = a.iter().filter_map(Value::as_u64).collect();
    }
}

/// Replaces the Project panel's optional columns with the names in `a`, in the canonical
/// `COLUMNS` order; unknown names are kept, sorted after the known ones.
fn apply_project_columns(app: &mut EffectcraftApp, a: &[Value]) {
    let mut cols = vec![];
    for k in a.iter().filter_map(Value::as_str) {
        crate::panels::project::set_column(&mut cols, k, true);
    }
    app.ui.project_columns = cols;
}

/// Merges `m` over the current Wiggler/Smoother/Motion Sketch settings by round-tripping them
/// through JSON. A value of the wrong type is an error and leaves the settings unchanged.
fn apply_anim_tools(app: &mut EffectcraftApp, m: &serde_json::Map<String, Value>) -> Result<(), String> {
    let mut cur = serde_json::to_value(&app.ui.anim_tools).unwrap_or_default();
    for (k, v) in m {
        cur[k] = v.clone();
    }
    match serde_json::from_value(cur) {
        Ok(v) => {
            app.ui.anim_tools = v;
            Ok(())
        }
        Err(e) => Err(format!("animTools: {e}")),
    }
}

/// Shows (`ui.panel.show`) or closes (`ui.panel.close`) the named panel; an unknown panel name
/// is an error.
fn panel(app: &mut EffectcraftApp, method: &str, p: &Value) -> Outcome {
    let s = |k: &str| p.get(k).and_then(Value::as_str);
    let Some(panel) = s("panel").and_then(PanelKind::from_name) else { return err("unknown `panel`") };
    if method == "ui.panel.show" {
        app.show_panel(panel);
    } else {
        app.close_panel(panel);
    }
    ok(Value::Null)
}

/// The number of synthetic events a gesture queues: `2 * count + 3` for `ui.click` (`count`
/// 1..=512), or 3 for `ui.move`. An out-of-range `count` or an overflow is an error.
fn click_gesture_events(method: &str, p: &Value) -> Result<usize, String> {
    if method != "ui.click" {
        return Ok(3);
    }
    const MAX_CLICK_COUNT: u64 = 512;
    let count = p.get("count").and_then(Value::as_u64).unwrap_or(1);
    if count == 0 || count > MAX_CLICK_COUNT {
        return Err(format!("`count` must be between 1 and {MAX_CLICK_COUNT}"));
    }
    usize::try_from(count).ok().and_then(|n| n.checked_mul(2)).and_then(|n| n.checked_add(3)).ok_or_else(|| "click event count overflow".to_string())
}

/// Queues a click or a plain move at `{id}`'s centre (`fx`/`fy` within its rect, default 0.5)
/// or at `{x, y}`, holding the request's modifiers across the whole gesture. An element that is
/// not on screen returns [`Outcome::Retry`]; the gesture is size-checked against the 4096-event
/// queue limit before anything is pushed. Replies [`Outcome::AfterInput`].
fn click_or_move(app: &mut EffectcraftApp, method: &str, p: &Value) -> Outcome {
    let s = |k: &str| p.get(k).and_then(Value::as_str);
    let pos = match point(app, p) {
        Ok(p) => p,
        Err(e) if e.starts_with(RETRY) => return Outcome::Retry(e.trim_start_matches(RETRY).to_string()),
        Err(e) => return err(e),
    };
    let m = modifiers(p);
    // Validate the complete gesture before mutating the queue; the loopback control
    // channel is externally reachable and must not amplify tiny requests into huge work.
    const MAX_PENDING_EVENTS: usize = 4096;
    let pending = app.synthetic.len();
    let gesture_events = match click_gesture_events(method, p) {
        Ok(events) => events,
        Err(e) => return err(e),
    };
    if pending.checked_add(gesture_events).is_none_or(|n| n > MAX_PENDING_EVENTS) {
        return err("synthetic input queue is full");
    }
    // egui keeps the modifiers from `ModifiersChanged` (not the ones on a pointer event)
    // and the synthetic events arrive one per frame: hold them for the whole gesture.
    app.synthetic.push(egui::Event::ModifiersChanged(m));
    app.synthetic.push(egui::Event::PointerMoved(pos));
    if method == "ui.click" {
        let button = match s("button") {
            Some("right") => egui::PointerButton::Secondary,
            Some("middle") => egui::PointerButton::Middle,
            _ => egui::PointerButton::Primary,
        };
        let count = p.get("count").and_then(Value::as_u64).unwrap_or(1);
        for _ in 0..count {
            app.synthetic.push(egui::Event::PointerButton { pos, button, pressed: true, modifiers: m });
            app.synthetic.push(egui::Event::PointerButton { pos, button, pressed: false, modifiers: m });
        }
    }
    app.synthetic.push(egui::Event::ModifiersChanged(egui::Modifiers::NONE));
    Outcome::AfterInput
}

/// Queues a primary-button drag from `from` to `to` over `steps` interpolated moves (default
/// 12, at least 2, at most 2048), with the request's modifiers. Both endpoints are resolved
/// first (an unseen element returns [`Outcome::Retry`]) and the event count is checked against
/// the 4096-event queue limit. Replies [`Outcome::AfterInput`].
fn drag(app: &mut EffectcraftApp, p: &Value) -> Outcome {
    let (Some(from), Some(to)) = (p.get("from"), p.get("to")) else { return err("need `from` and `to`") };
    let (a, b) = match (point(app, from), point(app, to)) {
        (Ok(a), Ok(b)) => (a, b),
        (Err(e), _) | (_, Err(e)) if e.starts_with(RETRY) => return Outcome::Retry(e.trim_start_matches(RETRY).to_string()),
        (Err(e), _) | (_, Err(e)) => return err(e),
    };
    let steps = p.get("steps").and_then(Value::as_u64).unwrap_or(12).max(2);
    const MAX_PENDING_EVENTS: usize = 4096;
    const MAX_DRAG_STEPS: u64 = 2048;
    if steps > MAX_DRAG_STEPS {
        return err(format!("`steps` must be at most {MAX_DRAG_STEPS}"));
    }
    let Some(gesture_events) = usize::try_from(steps).ok().and_then(|n| n.checked_add(5)) else { return err("drag event count overflow") };
    if app.synthetic.len().checked_add(gesture_events).is_none_or(|n| n > MAX_PENDING_EVENTS) {
        return err("synthetic input queue is full");
    }
    let m = modifiers(p);
    app.synthetic.push(egui::Event::ModifiersChanged(m));
    app.synthetic.push(egui::Event::PointerMoved(a));
    app.synthetic.push(egui::Event::PointerButton { pos: a, button: egui::PointerButton::Primary, pressed: true, modifiers: m });
    for i in 1..=steps {
        let f = i as f32 / steps as f32;
        app.synthetic.push(egui::Event::PointerMoved(a + (b - a) * f));
    }
    app.synthetic.push(egui::Event::PointerButton { pos: b, button: egui::PointerButton::Primary, pressed: false, modifiers: m });
    app.synthetic.push(egui::Event::ModifiersChanged(egui::Modifiers::NONE));
    Outcome::AfterInput
}

/// Queues a move to the target point, a wheel event of `dx`, `dy` (`unit`: points or lines) with
/// the request's modifiers, a pinch's zoom factor, and a released Extra1 button. Replies
/// [`Outcome::AfterInput`].
fn scroll(app: &mut EffectcraftApp, p: &Value) -> Outcome {
    let pos = match point(app, p) {
        Ok(p) => p,
        Err(e) => return err(e),
    };
    let dx = p.get("dx").and_then(Value::as_f64).unwrap_or(0.0) as f32;
    let dy = p.get("dy").and_then(Value::as_f64).unwrap_or(0.0) as f32;
    let line = p.get("unit").and_then(Value::as_str) == Some("line");
    let unit = if line { egui::MouseWheelUnit::Line } else { egui::MouseWheelUnit::Point };
    // Without a `source`, a scroll in lines is a mouse wheel and one in points comes from an
    // unnamed device, as on Windows, Linux and the web.
    let source = match p.get("source").and_then(Value::as_str) {
        Some("trackpad") => egui::MouseWheelSource::Trackpad,
        Some("wheel") => egui::MouseWheelSource::Wheel,
        _ if line => egui::MouseWheelSource::Wheel,
        _ => egui::MouseWheelSource::Unknown,
    };
    app.synthetic.push(egui::Event::PointerMoved(pos));
    if dx != 0.0 || dy != 0.0 {
        app.synthetic.push(egui::Event::MouseWheel { unit, delta: egui::vec2(dx, dy), modifiers: modifiers(p), phase: egui::TouchPhase::Move, source });
    }
    if let Some(pinch) = p.get("pinch").and_then(Value::as_f64).filter(|z| z.is_finite() && *z > 0.0) {
        app.synthetic.push(egui::Event::Zoom(pinch as f32));
    }
    app.synthetic.push(egui::Event::PointerButton {
        pos: egui::pos2(-100.0, -100.0),
        button: egui::PointerButton::Extra1,
        pressed: false,
        modifiers: Default::default(),
    });
    Outcome::AfterInput
}

/// Queues a press and release of `key`, whose name may carry a shortcut (`Cmd+Shift+Z`); the
/// separate `modifiers` flags are added. A `Physical+` prefix makes the key positional. An
/// unknown key name is an error.
fn key(app: &mut EffectcraftApp, p: &Value) -> Outcome {
    let s = |k: &str| p.get(k).and_then(Value::as_str);
    let Some(name) = s("key") else { return err("missing `key`") };
    let Some((mut m, key)) = crate::menus::parse_shortcut(name) else { return err(format!("unknown key `{name}`")) };
    let extra = modifiers(p);
    m |= extra;
    let physical_key = name.starts_with("Physical+").then_some(key);
    app.synthetic.push(egui::Event::Key { key, physical_key, pressed: true, repeat: false, modifiers: m });
    app.synthetic.push(egui::Event::Key { key, physical_key, pressed: false, repeat: false, modifiers: m });
    Outcome::AfterInput
}

/// Queues a text event with `text`, followed by a released F35 key. Missing `text` is an
/// error.
fn type_text(app: &mut EffectcraftApp, p: &Value) -> Outcome {
    let Some(text) = p.get("text").and_then(Value::as_str) else { return err("missing `text`") };
    app.synthetic.push(egui::Event::Text(text.to_string()));
    app.synthetic.push(egui::Event::Key { key: egui::Key::F35, physical_key: None, pressed: false, repeat: false, modifiers: Default::default() });
    Outcome::AfterInput
}

/// Converts comp pixel `x`, `y` to a screen point with the viewer's current transform, or
/// returns [`Outcome::Retry`] while the viewer is not visible.
fn viewer_locate(ctx: &egui::Context, p: &Value) -> Outcome {
    let (Some(x), Some(y)) = (p.get("x").and_then(Value::as_f64), p.get("y").and_then(Value::as_f64)) else {
        return err("need `x`, `y` (comp pixels)");
    };
    match crate::panels::viewer::comp_to_screen(ctx, [x as f32, y as f32]) {
        Some(pt) => ok(json!({"x": pt.x, "y": pt.y})),
        None => Outcome::Retry("the composition viewer is not visible".into()),
    }
}

/// Converts a point (`{id}`, or `{x, y}`) to comp pixels; replies with an error when the
/// viewer is not visible.
fn viewer_hit(app: &mut EffectcraftApp, ctx: &egui::Context, p: &Value) -> Outcome {
    let pos = match point(app, p) {
        Ok(p) => p,
        Err(e) => return err(e),
    };
    match crate::panels::viewer::screen_to_comp(ctx, pos) {
        Some(c) => ok(json!({"x": c[0], "y": c[1]})),
        None => err("the composition viewer is not visible"),
    }
}

/// Returns the screen point at `time` seconds on the ruler row of layer `layer`'s bar;
/// without a numeric `time`, returns the bar's centre. Returns [`Outcome::Retry`] when the bar
/// is unavailable, or a numeric `time` is supplied but the ruler is unavailable or malformed.
fn timeline_locate(app: &EffectcraftApp, p: &Value) -> Outcome {
    let Some(l) = p.get("layer").and_then(Value::as_u64) else { return err("need `layer` (id)") };
    match crate::panels::timeline::locate(app, l, p.get("time").and_then(Value::as_f64)) {
        Some((x, y)) => ok(json!({"x": x, "y": y})),
        None => Outcome::Retry("layer not visible in the timeline".into()),
    }
}

/// Applies `action` (`play`, `stop`, `status`, else toggle) at `now` and replies with the
/// playing state, current time in seconds, whether audio is present and the meter's level and
/// peak in dB.
fn playback(app: &mut EffectcraftApp, p: &Value, now: f64) -> Outcome {
    let action = p.get("action").and_then(Value::as_str).unwrap_or("toggle");
    match action {
        "play" => app.play(now),
        "stop" => app.stop(),
        "status" => {}
        _ => app.toggle_play(now),
    }
    ok(
        json!({"playing": app.playback.playing, "time": app.session.time().seconds(), "audio": app.audio.is_some(), "levelsDb": app.meter.level_db, "peaksDb": app.meter.peak_db}),
    )
}

/// Resolves the capture target — a named panel, an element id, or the whole window — and
/// returns [`Outcome::Retry`] while a requested panel or element is not registered, so a
/// missing target is never silently captured as the window. Requests a repaint without raising
/// or focusing the window and returns [`Outcome::Screenshot`]; the target is resolved again
/// when the capture is issued.
fn screenshot(app: &mut EffectcraftApp, ctx: &egui::Context, p: &Value) -> Outcome {
    let s = |k: &str| p.get(k).and_then(Value::as_str);
    // A file the project reads is never the output.
    if let Some(path) = s("path")
        && let Err(e) = app.session.refuse_export(path)
    {
        return err(e);
    }
    // The panel/element is an identity, not a rectangle: it is resolved when the capture
    // is issued (after the UI settles), so a panel that moves meanwhile is not cropped
    // with its old rectangle.
    let target = match s("panel") {
        Some(name) => match PanelKind::from_name(name) {
            Some(pk) => crate::ShotTarget::Panel(pk),
            None => return err(format!("unknown panel `{name}`")),
        },
        None => s("id").map_or(crate::ShotTarget::Window, |id| crate::ShotTarget::Id(id.to_string())),
    };
    // Asking for one element and silently capturing the whole window would hide a missing
    // panel: wait for it like any other not-yet-laid-out control.
    let missing = match &target {
        crate::ShotTarget::Panel(pk) => app.auto.find(&format!("panel.{}", pk.id())).is_none(),
        crate::ShotTarget::Id(id) => app.auto.find(id).is_none(),
        crate::ShotTarget::Window | crate::ShotTarget::Rect(_) => false,
    };
    if missing {
        return Outcome::Retry(match &target {
            crate::ShotTarget::Panel(pk) => format!("the {} panel is not on screen", pk.title()),
            crate::ShotTarget::Id(id) => format!("no element `{id}` is on screen"),
            crate::ShotTarget::Window | crate::ShotTarget::Rect(_) => "the element is not on screen".into(),
        });
    }
    // Request a repaint without raising the window or taking keyboard focus.
    app.raise_for_control(ctx);
    Outcome::Screenshot { path: s("path").map(str::to_string), target }
}

/// Asks the viewport for an inner size of `width`×`height` pixels (default 1600×1000).
fn resize(ctx: &egui::Context, p: &Value) -> Outcome {
    let w = p.get("width").and_then(Value::as_f64).unwrap_or(1600.0) as f32;
    let h = p.get("height").and_then(Value::as_f64).unwrap_or(1000.0) as f32;
    ctx.send_viewport_cmd(egui::ViewportCommand::InnerSize(egui::vec2(w, h)));
    ok(Value::Null)
}

/// Sends the viewport close request. `force: true` marks the unsaved-changes state as
/// quitting, so a modified project closes without the save prompt.
fn quit(app: &mut EffectcraftApp, ctx: &egui::Context, p: &Value) -> Outcome {
    // A modified project asks to save first unless `force: true`.
    if p.get("force").and_then(Value::as_bool).unwrap_or(false) {
        app.dialog_state.unsaved.quitting = true;
    }
    ctx.send_viewport_cmd(egui::ViewportCommand::Close);
    ok(Value::Null)
}

/// `render.frame {comp?, time?, max_side?, path?, base64?, write?}`: render a comp frame through the session
/// (independent of the viewer zoom/resolution) and return it as PNG, written to `path` (default a
/// temp file) or inline as base64 when `base64: true`. `write: true` with `base64: true` does both.
fn render_frame(app: &EffectcraftApp, p: &Value) -> Outcome {
    let s = &app.session;
    // A file the project reads is never the output (checked before rendering).
    if let Some(path) = p.get("path").and_then(Value::as_str)
        && let Err(e) = s.refuse_export(path)
    {
        return err(e);
    }
    let cid = match s.resolve_comp(p.get("comp")) {
        Ok(c) => c,
        Err(e) => return err(e),
    };
    let t = p.get("time").or(p.get("seconds")).and_then(Value::as_f64).map(effectcraft_time::Tick::from_seconds_f64).unwrap_or_else(|| s.time());
    let max_side = p.get("max_side").or(p.get("maxSide")).and_then(Value::as_u64).unwrap_or(0) as u32;
    let transparent = p.get("transparent").and_then(Value::as_bool).unwrap_or(false);
    let (w, h, rgba) = match s.render_rgba8_alpha(cid, t, max_side, transparent) {
        Ok(r) => r,
        Err(e) => return err(e),
    };
    let png = match encode_png(&rgba, w, h) {
        Ok(p) => p,
        Err(e) => return err(e),
    };
    let mut out = json!({"comp": cid.0, "time": t.seconds(), "width": w, "height": h});
    // (the web has no file system: inline unless a path is given)
    let inline = p.get("base64").and_then(Value::as_bool).unwrap_or(cfg!(target_arch = "wasm32") && p.get("path").is_none());
    if inline && !p.get("write").and_then(Value::as_bool).unwrap_or(false) {
        out["png"] = json!(base64(&png));
        return ok(out);
    }
    let path = p
        .get("path")
        .and_then(Value::as_str)
        .map(str::to_string)
        .unwrap_or_else(|| std::env::temp_dir().join(format!("effectcraft-frame-{}.png", std::process::id())).to_string_lossy().to_string());
    if inline {
        out["png"] = json!(base64(&png));
    }
    match s.write_output(&path, &png) {
        Ok(()) => {
            out["path"] = json!(path);
            ok(out)
        }
        Err(e) => err(format!("cannot write {path}: {e}")),
    }
}

/// Standard base64 (RFC 4648, padded).
pub fn base64(data: &[u8]) -> String {
    const A: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
    let mut s = String::with_capacity(data.len().div_ceil(3) * 4);
    for c in data.chunks(3) {
        let n = (c[0] as u32) << 16 | (*c.get(1).unwrap_or(&0) as u32) << 8 | *c.get(2).unwrap_or(&0) as u32;
        for i in 0..4 {
            if i <= c.len() {
                s.push(A[(n >> (18 - 6 * i) & 63) as usize] as char);
            } else {
                s.push('=');
            }
        }
    }
    s
}

/// Builds the `ui.inspect` reply: window size and pixels per point, fps, UI state with the
/// active comp's viewer resolution, playback and time, active comp, selected layer/key counts,
/// registered element count, keyboard focus (egui id, rect, automation ids and rect
/// candidates), fresh window focus, workspace storage error, open dialog and last frame time.
pub fn inspect(app: &EffectcraftApp, ctx: &egui::Context) -> Value {
    // A logic-only tick refreshes the viewport information but not `InputState`'s screen rect.
    let size = ctx.input(|i| i.viewport().inner_rect).map_or_else(|| ctx.content_rect().size(), |r| r.size());
    let keyboard_focus = ctx.memory(|m| m.focused()).map(|id| {
        let response = ctx.read_response(id);
        let rect = response.as_ref().map(|r| [r.rect.min.x, r.rect.min.y, r.rect.width(), r.rect.height()]);
        // The registry names screen elements independently of egui's internal ids. Report
        // exact rectangle matches rather than guessing which enclosing panel owns focus.
        let candidates: Vec<&str> = app
            .auto
            .previous
            .iter()
            .filter(|e| rect.is_some_and(|r| e.rect.iter().zip(r).all(|(a, b)| (*a - b).abs() < 0.01)))
            .map(|e| e.id.as_str())
            .collect();
        let automation_id = app.auto.elements.iter().chain(app.auto.previous.iter()).find(|e| e.response_id == Some(id)).map(|e| e.id.as_str());
        json!({"id": format!("{:016x}", id.value()), "rect": rect, "automationId": automation_id, "registryCandidates": candidates})
    });
    let mut ui = json!(app.ui);
    // The viewer's Resolution is the active comp's (kept in the project).
    if let Some(v) = ui.get_mut("viewer").and_then(Value::as_object_mut) {
        v.insert("res".into(), json!(app.viewer_res()));
    }
    json!({
        "window": [size.x, size.y],
        "pixelsPerPoint": ctx.pixels_per_point(),
        "fps": app.fps,
        "ui": ui,
        "playing": app.playback.playing,
        "time": app.session.time().seconds(),
        "activeComp": app.session.state.active_comp.map(|i| i.0),
        "selectedLayers": app.session.state.selected_layers.iter().map(|c| c.0).collect::<Vec<_>>(),
        "selectedKeys": app.session.state.selected_keys.len(),
        "elements": app.auto.previous.len(),
        "keyboardFocus": keyboard_focus,
        // Fresh viewport focus only: in a logic-only tick `InputState::focused` still holds the
        // previous full pass's value, and the app field is that same fresh value.
        "windowFocused": ctx.input(|i| i.viewport().focused).unwrap_or(app.focused),
        "workspaceStorageError": app.workspace_store.error,
        "dialog": app.dialog.map(|d| format!("{d:?}")),
        "renderMs": app.frames.last_ms.lock().map(|v| *v).unwrap_or(0.0),
    })
}

/// Save a screenshot (optionally cropped to a rect in points) as PNG; returns the reply JSON.
///
/// `crop` is relative to the image's own origin (the issued surface's top-left corner), and
/// `scale` is the returned image's pixels per point in x and y, derived from the image size and
/// the issued rect. The image can be smaller than requested when the adapter-limit fit (E2)
/// applies, so its own scale — never the issued or current pixels-per-point — cuts the crop:
/// any other scale reads a shifted or rescaled region.
pub fn save_screenshot(image: &egui::ColorImage, path: Option<&str>, crop: Option<[f32; 4]>, scale: [f64; 2]) -> Value {
    let [w, h] = image.size;
    let [sx, sy] = scale;
    if w == 0 || h == 0 || !sx.is_finite() || sx <= 0.0 || !sy.is_finite() || sy <= 0.0 {
        return json!({"ok": false, "error": "the screenshot has no pixels to save"});
    }
    if image.pixels.len() != w.saturating_mul(h) {
        return json!({"ok": false, "error": "the screenshot's pixels do not match its dimensions"});
    }
    let px = |v: f64, s: f64| (v * s).max(0.0) as usize;
    let (x0, y0, cw, ch) = match crop {
        Some([x, y, cw, ch]) if [x, y, cw, ch].iter().all(|v| v.is_finite()) => {
            let x0 = px(x as f64, sx).min(w);
            let y0 = px(y as f64, sy).min(h);
            (x0, y0, px(cw as f64, sx).min(w - x0), px(ch as f64, sy).min(h - y0))
        }
        Some(_) => return json!({"ok": false, "error": "the screenshot crop is not a finite rectangle"}),
        None => (0, 0, w, h),
    };
    let mut rgba = Vec::with_capacity(cw.saturating_mul(ch).saturating_mul(4));
    for y in y0..y0 + ch {
        for x in x0..x0 + cw {
            let Some(c) = image.pixels.get(y * w + x) else {
                return json!({"ok": false, "error": "the screenshot's pixels do not match its dimensions"});
            };
            rgba.extend_from_slice(&[c.r(), c.g(), c.b(), 255]);
        }
    }
    if path.is_none() && cfg!(target_arch = "wasm32") {
        // no file system on the web: inline PNG
        return match encode_png(&rgba, cw as u32, ch as u32) {
            Ok(png) => json!({"ok": true, "result": {"png": base64(&png), "width": cw, "height": ch}}),
            Err(e) => json!({"ok": false, "error": e}),
        };
    }
    let path = path.map(str::to_string).unwrap_or_else(|| std::env::temp_dir().join("effectcraft-screenshot.png").to_string_lossy().to_string());
    match encode_png(&rgba, cw as u32, ch as u32) {
        Ok(png) => match std::fs::write(&path, png) {
            Ok(()) => json!({"ok": true, "result": {"path": path, "width": cw, "height": ch}}),
            Err(e) => json!({"ok": false, "error": e.to_string()}),
        },
        Err(e) => json!({"ok": false, "error": e}),
    }
}

/// Minimal PNG encoder (stored deflate blocks, no compression dependency).
pub fn encode_png(rgba: &[u8], w: u32, h: u32) -> Result<Vec<u8>, String> {
    fn chunk(out: &mut Vec<u8>, ty: &[u8], data: &[u8]) {
        out.extend_from_slice(&(data.len() as u32).to_be_bytes());
        let mut c = Vec::with_capacity(4 + data.len());
        c.extend_from_slice(ty);
        c.extend_from_slice(data);
        out.extend_from_slice(&c);
        out.extend_from_slice(&crc(&c).to_be_bytes());
    }
    fn crc(d: &[u8]) -> u32 {
        let mut c = 0xFFFF_FFFFu32;
        for &b in d {
            c ^= b as u32;
            for _ in 0..8 {
                c = if c & 1 != 0 { (c >> 1) ^ 0xEDB8_8320 } else { c >> 1 };
            }
        }
        !c
    }
    let mut raw = Vec::with_capacity((w as usize * 4 + 1) * h as usize);
    for y in 0..h as usize {
        raw.push(0);
        raw.extend_from_slice(&rgba[y * w as usize * 4..(y + 1) * w as usize * 4]);
    }
    // zlib stream with stored blocks
    let mut z = vec![0x78, 0x01];
    for (i, block) in raw.chunks(65535).enumerate() {
        let last = (i + 1) * 65535 >= raw.len();
        z.push(last as u8);
        z.extend_from_slice(&(block.len() as u16).to_le_bytes());
        z.extend_from_slice(&(!(block.len() as u16)).to_le_bytes());
        z.extend_from_slice(block);
    }
    let (mut a, mut b) = (1u32, 0u32);
    for &x in &raw {
        a = (a + x as u32) % 65521;
        b = (b + a) % 65521;
    }
    z.extend_from_slice(&((b << 16) | a).to_be_bytes());
    let mut out = vec![0x89, b'P', b'N', b'G', 0x0D, 0x0A, 0x1A, 0x0A];
    let mut ihdr = Vec::new();
    ihdr.extend_from_slice(&w.to_be_bytes());
    ihdr.extend_from_slice(&h.to_be_bytes());
    ihdr.extend_from_slice(&[8, 6, 0, 0, 0]);
    chunk(&mut out, b"IHDR", &ihdr);
    chunk(&mut out, b"IDAT", &z);
    chunk(&mut out, b"IEND", &[]);
    Ok(out)
}

#[cfg(test)]
mod tests {
    #[test]
    fn base64_rfc4648_vectors() {
        for (i, o) in [("", ""), ("f", "Zg=="), ("fo", "Zm8="), ("foo", "Zm9v"), ("foob", "Zm9vYg=="), ("fooba", "Zm9vYmE="), ("foobar", "Zm9vYmFy")] {
            assert_eq!(super::base64(i.as_bytes()), o);
        }
    }
}