Compare commits
24
Commits
No files matched your search
@@ -47,6 +47,10 @@ jobs:
|
||||
- name: XML, desktop entry, AppStream, Flatpak manifest
|
||||
run: |
|
||||
xmllint --noout packaging/windows/filmcraft.wxs packaging/macos/*.plist* packaging/linux/*.xml*
|
||||
# Windows Installer needs Icon ids ending in .ico or .exe, or shortcuts show a blank icon.
|
||||
if grep -oE '<Icon Id="[^"]*"' packaging/windows/filmcraft.wxs | grep -vE '\.(ico|exe)"$'; then
|
||||
echo "::error::Icon Id must end in .ico or .exe"; exit 1
|
||||
fi
|
||||
desktop-file-validate packaging/linux/ai.storyteller.filmcraft.desktop
|
||||
sed -e 's/@VERSION@/0.0.0/' -e 's/@DATE@/2026-01-01/' packaging/linux/ai.storyteller.filmcraft.metainfo.xml.in >"$RUNNER_TEMP/ai.storyteller.filmcraft.metainfo.xml"
|
||||
appstreamcli validate --no-net --explain "$RUNNER_TEMP/ai.storyteller.filmcraft.metainfo.xml"
|
||||
|
||||
@@ -221,8 +221,9 @@ jobs:
|
||||
- name: Install wasm-bindgen CLI
|
||||
run: |
|
||||
# Must match the wasm-bindgen crate version exactly (WASM_BINDGEN in xtask/src/main.rs).
|
||||
command -v wasm-bindgen >/dev/null && wasm-bindgen --version | grep -q "$WASM_BINDGEN_VERSION" \
|
||||
|| cargo install wasm-bindgen-cli --version "$WASM_BINDGEN_VERSION" --locked
|
||||
if ! { command -v wasm-bindgen >/dev/null && wasm-bindgen --version | grep -q "$WASM_BINDGEN_VERSION"; }; then
|
||||
cargo install wasm-bindgen-cli --version "$WASM_BINDGEN_VERSION" --locked
|
||||
fi
|
||||
wasm-bindgen --version
|
||||
- name: Apply version override
|
||||
if: inputs.version != ''
|
||||
|
||||
@@ -120,6 +120,12 @@ See [§0](#0-never-crash). It outranks every other engineering rule.
|
||||
See `CLAUDE.md`: pure Rust, dependency layering (`cargo xtask layers`), exact `Tick` time, everything is
|
||||
a command, everything is agent-drivable, and the quality gates (`cargo xtask ci`) before every commit.
|
||||
|
||||
Shared real-file test corpora (Photoshop-authored PSDs, etc.) live in
|
||||
[`storytold/photocraft-corpus`](https://github.com/storytold/photocraft-corpus), explained in
|
||||
[craftrules `standards/test-corpora.md`](https://github.com/storytold/craftrules/blob/main/standards/test-corpora.md).
|
||||
Never commit large binary fixtures to this repo; fetch them pinned by commit and sha256-verified,
|
||||
as PhotoCraft does with `cargo xtask corpus`.
|
||||
|
||||
## See also
|
||||
|
||||
- [CONTRIBUTING.md](CONTRIBUTING.md) and [docs/contributing.md](docs/contributing.md): setup, gates, commits, how to add things
|
||||
|
||||
Generated
+46
-46
@@ -1655,7 +1655,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "filmcraft"
|
||||
version = "0.2.0"
|
||||
version = "0.2.1"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"cpal",
|
||||
@@ -1677,7 +1677,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "filmcraft-aac"
|
||||
version = "0.2.0"
|
||||
version = "0.2.1"
|
||||
dependencies = [
|
||||
"filmcraft-bitstream",
|
||||
"filmcraft-testkit",
|
||||
@@ -1687,7 +1687,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "filmcraft-ac3"
|
||||
version = "0.2.0"
|
||||
version = "0.2.1"
|
||||
dependencies = [
|
||||
"filmcraft-testkit",
|
||||
"thiserror 2.0.21",
|
||||
@@ -1695,14 +1695,14 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "filmcraft-audio-dsp"
|
||||
version = "0.2.0"
|
||||
version = "0.2.1"
|
||||
dependencies = [
|
||||
"filmcraft-testkit",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "filmcraft-automation"
|
||||
version = "0.2.0"
|
||||
version = "0.2.1"
|
||||
dependencies = [
|
||||
"base64 0.22.1",
|
||||
"filmcraft-engine",
|
||||
@@ -1718,7 +1718,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "filmcraft-av1"
|
||||
version = "0.2.0"
|
||||
version = "0.2.1"
|
||||
dependencies = [
|
||||
"filmcraft-bitstream",
|
||||
"filmcraft-testkit",
|
||||
@@ -1729,14 +1729,14 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "filmcraft-bitstream"
|
||||
version = "0.2.0"
|
||||
version = "0.2.1"
|
||||
dependencies = [
|
||||
"proptest",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "filmcraft-captions"
|
||||
version = "0.2.0"
|
||||
version = "0.2.1"
|
||||
dependencies = [
|
||||
"filmcraft-color",
|
||||
"filmcraft-project",
|
||||
@@ -1749,11 +1749,11 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "filmcraft-cfb"
|
||||
version = "0.2.0"
|
||||
version = "0.2.1"
|
||||
|
||||
[[package]]
|
||||
name = "filmcraft-cli"
|
||||
version = "0.2.0"
|
||||
version = "0.2.1"
|
||||
dependencies = [
|
||||
"filmcraft-automation",
|
||||
"filmcraft-codecs",
|
||||
@@ -1770,7 +1770,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "filmcraft-codecs"
|
||||
version = "0.2.0"
|
||||
version = "0.2.1"
|
||||
dependencies = [
|
||||
"filmcraft-aac",
|
||||
"filmcraft-ac3",
|
||||
@@ -1802,7 +1802,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "filmcraft-color"
|
||||
version = "0.2.0"
|
||||
version = "0.2.1"
|
||||
dependencies = [
|
||||
"proptest",
|
||||
"serde",
|
||||
@@ -1810,7 +1810,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "filmcraft-dnx"
|
||||
version = "0.2.0"
|
||||
version = "0.2.1"
|
||||
dependencies = [
|
||||
"filmcraft-bitstream",
|
||||
"filmcraft-testkit",
|
||||
@@ -1820,7 +1820,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "filmcraft-edit"
|
||||
version = "0.2.0"
|
||||
version = "0.2.1"
|
||||
dependencies = [
|
||||
"filmcraft-media",
|
||||
"filmcraft-project",
|
||||
@@ -1832,7 +1832,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "filmcraft-engine"
|
||||
version = "0.2.0"
|
||||
version = "0.2.1"
|
||||
dependencies = [
|
||||
"filmcraft-audio-dsp",
|
||||
"filmcraft-captions",
|
||||
@@ -1863,7 +1863,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "filmcraft-export"
|
||||
version = "0.2.0"
|
||||
version = "0.2.1"
|
||||
dependencies = [
|
||||
"filmcraft-aac",
|
||||
"filmcraft-audio-dsp",
|
||||
@@ -1892,7 +1892,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "filmcraft-format"
|
||||
version = "0.2.0"
|
||||
version = "0.2.1"
|
||||
dependencies = [
|
||||
"filmcraft-media",
|
||||
"filmcraft-project",
|
||||
@@ -1904,7 +1904,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "filmcraft-frame"
|
||||
version = "0.2.0"
|
||||
version = "0.2.1"
|
||||
dependencies = [
|
||||
"filmcraft-color",
|
||||
"filmcraft-time",
|
||||
@@ -1914,7 +1914,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "filmcraft-geom"
|
||||
version = "0.2.0"
|
||||
version = "0.2.1"
|
||||
dependencies = [
|
||||
"proptest",
|
||||
"serde",
|
||||
@@ -1922,7 +1922,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "filmcraft-golden"
|
||||
version = "0.2.0"
|
||||
version = "0.2.1"
|
||||
dependencies = [
|
||||
"filmcraft-color",
|
||||
"filmcraft-frame",
|
||||
@@ -1939,7 +1939,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "filmcraft-gpu"
|
||||
version = "0.2.0"
|
||||
version = "0.2.1"
|
||||
dependencies = [
|
||||
"filmcraft-color",
|
||||
"filmcraft-frame",
|
||||
@@ -1956,7 +1956,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "filmcraft-h264"
|
||||
version = "0.2.0"
|
||||
version = "0.2.1"
|
||||
dependencies = [
|
||||
"filmcraft-bitstream",
|
||||
"filmcraft-testkit",
|
||||
@@ -1966,7 +1966,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "filmcraft-h264enc"
|
||||
version = "0.2.0"
|
||||
version = "0.2.1"
|
||||
dependencies = [
|
||||
"filmcraft-bitstream",
|
||||
"filmcraft-testkit",
|
||||
@@ -1976,7 +1976,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "filmcraft-hevc"
|
||||
version = "0.2.0"
|
||||
version = "0.2.1"
|
||||
dependencies = [
|
||||
"filmcraft-bitstream",
|
||||
"filmcraft-testkit",
|
||||
@@ -1986,7 +1986,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "filmcraft-interchange"
|
||||
version = "0.2.0"
|
||||
version = "0.2.1"
|
||||
dependencies = [
|
||||
"filmcraft-cfb",
|
||||
"filmcraft-color",
|
||||
@@ -2003,7 +2003,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "filmcraft-isobmff"
|
||||
version = "0.2.0"
|
||||
version = "0.2.1"
|
||||
dependencies = [
|
||||
"filmcraft-bitstream",
|
||||
"filmcraft-testkit",
|
||||
@@ -2014,7 +2014,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "filmcraft-matroska"
|
||||
version = "0.2.0"
|
||||
version = "0.2.1"
|
||||
dependencies = [
|
||||
"filmcraft-testkit",
|
||||
"serde_json",
|
||||
@@ -2022,7 +2022,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "filmcraft-media"
|
||||
version = "0.2.0"
|
||||
version = "0.2.1"
|
||||
dependencies = [
|
||||
"filmcraft-color",
|
||||
"filmcraft-frame",
|
||||
@@ -2035,7 +2035,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "filmcraft-mpeg2v"
|
||||
version = "0.2.0"
|
||||
version = "0.2.1"
|
||||
dependencies = [
|
||||
"filmcraft-testkit",
|
||||
"rayon",
|
||||
@@ -2044,7 +2044,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "filmcraft-mpegts"
|
||||
version = "0.2.0"
|
||||
version = "0.2.1"
|
||||
dependencies = [
|
||||
"filmcraft-testkit",
|
||||
"thiserror 2.0.21",
|
||||
@@ -2052,28 +2052,28 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "filmcraft-mxf"
|
||||
version = "0.2.0"
|
||||
version = "0.2.1"
|
||||
dependencies = [
|
||||
"filmcraft-testkit",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "filmcraft-ogg"
|
||||
version = "0.2.0"
|
||||
version = "0.2.1"
|
||||
dependencies = [
|
||||
"filmcraft-testkit",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "filmcraft-opus"
|
||||
version = "0.2.0"
|
||||
version = "0.2.1"
|
||||
dependencies = [
|
||||
"filmcraft-testkit",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "filmcraft-platform"
|
||||
version = "0.2.0"
|
||||
version = "0.2.1"
|
||||
dependencies = [
|
||||
"filmcraft-codecs",
|
||||
"filmcraft-color",
|
||||
@@ -2091,7 +2091,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "filmcraft-project"
|
||||
version = "0.2.0"
|
||||
version = "0.2.1"
|
||||
dependencies = [
|
||||
"filmcraft-color",
|
||||
"filmcraft-geom",
|
||||
@@ -2105,7 +2105,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "filmcraft-prores"
|
||||
version = "0.2.0"
|
||||
version = "0.2.1"
|
||||
dependencies = [
|
||||
"filmcraft-bitstream",
|
||||
"filmcraft-testkit",
|
||||
@@ -2115,7 +2115,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "filmcraft-render"
|
||||
version = "0.2.0"
|
||||
version = "0.2.1"
|
||||
dependencies = [
|
||||
"filmcraft-audio-dsp",
|
||||
"filmcraft-captions",
|
||||
@@ -2135,7 +2135,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "filmcraft-scopes"
|
||||
version = "0.2.0"
|
||||
version = "0.2.1"
|
||||
dependencies = [
|
||||
"filmcraft-color",
|
||||
"serde",
|
||||
@@ -2144,7 +2144,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "filmcraft-speech"
|
||||
version = "0.2.0"
|
||||
version = "0.2.1"
|
||||
dependencies = [
|
||||
"candle-core",
|
||||
"candle-nn",
|
||||
@@ -2162,14 +2162,14 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "filmcraft-testkit"
|
||||
version = "0.2.0"
|
||||
version = "0.2.1"
|
||||
dependencies = [
|
||||
"png",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "filmcraft-text"
|
||||
version = "0.2.0"
|
||||
version = "0.2.1"
|
||||
dependencies = [
|
||||
"filmcraft-color",
|
||||
"harfrust",
|
||||
@@ -2180,7 +2180,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "filmcraft-time"
|
||||
version = "0.2.0"
|
||||
version = "0.2.1"
|
||||
dependencies = [
|
||||
"proptest",
|
||||
"serde",
|
||||
@@ -2188,7 +2188,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "filmcraft-ui-egui"
|
||||
version = "0.2.0"
|
||||
version = "0.2.1"
|
||||
dependencies = [
|
||||
"eframe",
|
||||
"egui",
|
||||
@@ -2221,7 +2221,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "filmcraft-vp9"
|
||||
version = "0.2.0"
|
||||
version = "0.2.1"
|
||||
dependencies = [
|
||||
"filmcraft-bitstream",
|
||||
"rayon",
|
||||
@@ -2230,7 +2230,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "filmcraft-web"
|
||||
version = "0.2.0"
|
||||
version = "0.2.1"
|
||||
dependencies = [
|
||||
"eframe",
|
||||
"egui",
|
||||
@@ -7211,7 +7211,7 @@ checksum = "e450f9b2ed1dff33c94c12589a87338689467b9c4f5d8a5710bd09a847d2c8a7"
|
||||
|
||||
[[package]]
|
||||
name = "xtask"
|
||||
version = "0.2.0"
|
||||
version = "0.2.1"
|
||||
dependencies = [
|
||||
"filmcraft-testkit",
|
||||
"serde_json",
|
||||
|
||||
+1
-1
@@ -4,7 +4,7 @@ members = ["crates/*", "apps/*", "xtask"]
|
||||
default-members = ["crates/*", "apps/*", "xtask"]
|
||||
|
||||
[workspace.package]
|
||||
version = "0.2.0"
|
||||
version = "0.2.1"
|
||||
edition = "2024"
|
||||
license = "MIT OR Apache-2.0"
|
||||
rust-version = "1.95"
|
||||
|
||||
@@ -184,7 +184,7 @@ No FFmpeg inside. The video codecs, AAC, Opus and the containers are our own Rus
|
||||
| **Matroska / WebM** | ✓ | | Lacing, Cues, header stripping, HDR colour metadata |
|
||||
| **Stills** | ✓ | ✓ | Import PNG, JPEG, GIF, WebP, TIFF and BMP; export PNG sequences and animated GIF |
|
||||
|
||||
MXF and hardware decode are next ([roadmap](ROADMAP.md)).
|
||||
Also imported: MPEG-2 / MPEG-1 video, AC-3, MP2, MXF (OP1a / OP-Atom), MPEG transport and program streams (AVCHD, broadcast, DVD), Ogg and image sequences. Not yet: HEVC and AV1 export, camera RAW, E-AC-3. Hardware decoding works on macOS (VideoToolbox), with Linux and Windows to follow ([#30](https://github.com/storytold/filmcraft/issues/30)).
|
||||
|
||||
<br>
|
||||
|
||||
@@ -266,11 +266,26 @@ The control protocol is documented in [docs/control-protocol.md](docs/control-pr
|
||||
| [docs/control-protocol.md](docs/control-protocol.md) | Control-channel and MCP reference |
|
||||
| [docs/project-files.md](docs/project-files.md) | `.fcproj` format, schema migrations, auto-save and crash recovery |
|
||||
| [docs/graphics.md](docs/graphics.md) · [docs/captions.md](docs/captions.md) | Text engine, graphic clips and tools; caption tracks and formats |
|
||||
| [ROADMAP.md](ROADMAP.md) | Milestones and estimates |
|
||||
| [ROADMAP.md](ROADMAP.md) | Honest assessment, what's missing, milestones and estimates |
|
||||
|
||||
## Status
|
||||
|
||||
FilmCraft is young and moving fast: roughly 45% of the way to Premiere Pro parity, with editing, trimming, colour, keyframes, titles, captions, mixing, codecs and export working today. [ROADMAP.md](ROADMAP.md) tracks every milestone with estimates.
|
||||
FilmCraft is young and moving fast. Editing, trimming, multicam, colour, keyframes, effects, titles, captions, mixing, codecs and export work today.
|
||||
|
||||
We track two numbers ([ROADMAP.md](ROADMAP.md#honest-assessment-2026-10-05)):
|
||||
|
||||
- **Feature checklist: ~87%.** Premiere Pro's menu items, effects, transitions, panels and formats that exist in FilmCraft.
|
||||
- **Ready for real work: ~50–60%.** Our honest estimate of how close FilmCraft is to replacing Premiere on real projects.
|
||||
|
||||
The biggest gaps today:
|
||||
|
||||
- **Speed on big footage.** Hardware decoding works on macOS; Windows and Linux have no hardware path yet. Blend modes and the most common effects run on the GPU, but Lumetri, keys, export and encoding still run on the CPU ([#30](https://github.com/storytold/filmcraft/issues/30)).
|
||||
- **No plugins.** No VST3 / Audio Units or OpenFX hosting.
|
||||
- **Delivery codecs.** H.264 is our only delivery-codec export; no HEVC or AV1 export yet.
|
||||
- **Real-world media and platforms.** Our decoders are bit-exact on conformance streams, but camera and phone files in the wild are less tested. Windows and Linux get far less testing than macOS.
|
||||
- **AI features.** Few so far; speech to text is optional and off by default.
|
||||
|
||||
Bug reports with real footage are the most useful thing you can send us: [open an issue](https://github.com/storytold/filmcraft/issues) or tell us in [Discord](https://discord.gg/artcraft).
|
||||
|
||||
## Architecture
|
||||
|
||||
|
||||
+90
-6
@@ -1,10 +1,88 @@
|
||||
# FilmCraft Roadmap
|
||||
|
||||
Progress toward feature parity with Adobe Premiere Pro, with estimates. Updated as milestones land.
|
||||
Progress toward parity with Adobe Premiere Pro, with estimates. Updated as milestones land.
|
||||
|
||||
**Last updated:** 2026-10-02 · **Overall parity:** ~87% (measured scorecard below) · **Code:** 34 crates, 1605 tests
|
||||
**Last updated:** 2026-10-05 · **Feature checklist:** ~87% · **Ready for real work:** ~50–60% · **Code:** 35 crates, 1605+ tests
|
||||
|
||||
## Parity scorecard
|
||||
Two numbers, because they answer different questions. The **feature checklist** counts whether
|
||||
Premiere's menu items, effects, panels and formats exist in FilmCraft. **Ready for real work** is
|
||||
our honest estimate of how close FilmCraft is to replacing Premiere for an editor on real
|
||||
projects: depth, correctness, robustness, speed and the pro ecosystem. The gap between the two
|
||||
is the work that matters most now. Agents: read [Where we are lacking](#where-we-are-lacking)
|
||||
before choosing work.
|
||||
|
||||
## Honest assessment (2026-10-05)
|
||||
|
||||
### How the checklist number is made, and what it doesn't tell you
|
||||
|
||||
- **Partly measured.** The menu figure comes from diffing a dump of Premiere 26.5's menus against
|
||||
`filmcraft-cli commands`. The effects, transitions and audio-effects counts are checked by unit
|
||||
tests against Premiere's Effects panel tree (`premiere_26_video_effects_catalogue`,
|
||||
`premiere_audio_set_is_complete_and_foldered`, `premiere_26_transition_tree`).
|
||||
- **Partly agent-reported.** The other areas' percentages are written by the agents doing the work.
|
||||
No automated parity check runs in CI (PhotoCraft has `cargo xtask parity`; we don't yet), so
|
||||
nothing stops a number drifting upward.
|
||||
- **Presence, not quality.** An item counts once it exists. An effect described as an
|
||||
"approximation" counts the same as an exact one. Nothing compares our rendering with
|
||||
Premiere's.
|
||||
- **Weights understate what users hit first.** Performance carries 5%. Plugins and AI features
|
||||
aren't in the scorecard at all.
|
||||
|
||||
### By dimension
|
||||
|
||||
| Dimension | Checklist says | Honest estimate | Evidence |
|
||||
|---|---|---|---|
|
||||
| **Feature breadth** (menus, commands, panels) | ~86–95% | **~85%** | Menu diff against Premiere 26.5; effect, transition and audio-effect catalogues complete by test. The long tail is mostly there. |
|
||||
| **Correctness in depth** | not tracked | **~60–70%** | A first-time contributor found four real bugs in core paths within hours (#5–#8: AIFF import failed, Slide left linked audio behind, transitions didn't follow ripple trims, export ignored start/end times). User reports: Color Matte always grey with no picker (#29); ALSA audio error on Linux (#23). Settings are "most wired", not all. |
|
||||
| **Codecs and media** | ~90% | **~75%** | Our decoders are bit-exact on conformance streams. Real camera and phone media (variable frame rate, damaged files, unusual containers) is far less tested. No camera RAW (RED, BRAW, ARRIRAW), no E-AC-3. |
|
||||
| **Export** | ~92% | **~70%** | Delivery: H.264 only (ProRes, DNxHR and image sequences cover mastering). No HEVC or AV1 export. AAF and OMF have never been validated in Avid Media Composer or Pro Tools. Export renders and encodes on the CPU only. |
|
||||
| **Performance and hardware** | ~62% | **~35–40%** | Premiere runs effects, decoding and encoding on the GPU and the hardware media engines. Here, hardware decode works on macOS only (VideoToolbox, #33). The GPU does compositing, blend modes (#32) and 31 common effects (GPU2): colour adjustments, crop/flip/transform, blurs, sharpen. Lumetri, keys, masks, the remaining effects, export and encoding are CPU. Linux and Windows have no hardware path. 8K and AV1 aren't real-time. The effect list shows Premiere's "GPU accelerated" badge, but only 31 effect ids run on the GPU here; the rest, Lumetri included, run on the CPU. |
|
||||
| **Stability** | not tracked | **improving, unproven** | The never-crash pass (no panics in product code, last-resort guards) landed on 2026-10-04. There's no field record from real users' projects yet. |
|
||||
| **Plugins and pro ecosystem** | not tracked | **~0–10%** | No audio plugin hosting (VST3 / Audio Units) and no third-party video effects (OpenFX). For many professional editors this alone rules FilmCraft out. Team Projects and Productions are out of scope by design. |
|
||||
| **AI features** | partial | **~25–35%** | Speech to text exists but is off in default builds (the `whisper` feature). Enhance Speech is a DSP chain, not a model. Auto Reframe is approximate. No Generative Extend, and no media-intelligence search or auto colour. |
|
||||
| **Platforms** | not tracked | **macOS solid; Windows and Linux thin** | Development and testing happen on macOS. Windows and Linux are built for releases but barely exercised at runtime. The web build is a compile target; the browser app shell isn't finished. |
|
||||
| **UI fidelity** | ~85% | **~80%** | Agents compare with Premiere reference screenshots; this hasn't been checked independently. |
|
||||
|
||||
### Where we are lacking
|
||||
|
||||
In priority order. Agents should prefer this work over adding more checklist items.
|
||||
|
||||
1. **Hardware acceleration** (#30), the most visible gap to users:
|
||||
- hardware decode on Linux (VA-API) and Windows (Media Foundation);
|
||||
- zero-copy decoded frames into wgpu;
|
||||
- the remaining effects on the GPU: Lumetri, keys, Vignette, Video Limiter, masks (31 common effects are done);
|
||||
- GPU export, then hardware encode (H.264 / HEVC).
|
||||
2. **A measured parity number.** Add `cargo xtask parity`, run in CI. It should cover menus,
|
||||
commands, effects, transitions, panels, preferences and formats against the Premiere snapshot in
|
||||
`plan/premiere/`. It should report presence and fidelity separately (exact, approximate, stub).
|
||||
3. **Real-world media corpus.** Phone, camera and screen recordings, variable frame rate, damaged
|
||||
files, long GOPs, odd containers. Each gets an import → edit → export round-trip test with an
|
||||
ffprobe / ffmpeg oracle.
|
||||
4. **Workflow acceptance tests.** Scripted MCP sessions that do real jobs end to end: assemble and
|
||||
trim a scene, colour it, mix and loudness-normalise it, caption it, deliver it, round-trip
|
||||
through XML / AAF.
|
||||
5. **Windows and Linux at runtime.** CI that runs the test suite and a headless playback/export
|
||||
smoke test on both, not just release builds. Fix the Linux audio path (#23).
|
||||
6. **Delivery codecs.** HEVC and AV1 export: hardware encoders first, and `rav1e` (BSD-2) for AV1.
|
||||
7. **Plugin hosting** (VST3 / Audio Units, then OpenFX). Needs FFI, so it falls under the isolated
|
||||
`unsafe` crate rule (AGENTS.md §0.3). **Owner decision needed** before starting.
|
||||
8. **AI features** with openly licensed local models: transcription on by default, text-based
|
||||
editing, speech enhancement, reframing.
|
||||
9. **The web app shell**: file access, WebCodecs, audio.
|
||||
|
||||
### Where we're going
|
||||
|
||||
**Next:** close items 1–5. Hardware acceleration fixes the gap users notice first (CPU-bound 4K playback and export). The
|
||||
parity tool, media corpus and workflow tests make the progress number trustworthy. Windows and
|
||||
Linux CI makes "runs everywhere" true.
|
||||
|
||||
**After that:** delivery codecs, plugin hosting (pending the owner's decision) and AI features,
|
||||
measured with the same tools.
|
||||
|
||||
**Goal:** "ready for real work" at 85%+, measured rather than estimated, on all three desktop
|
||||
platforms.
|
||||
|
||||
## Feature checklist scorecard
|
||||
|
||||
Measured against Premiere Pro 26.5 on this machine. Menu items: the native menu bar dump, minus
|
||||
Adobe-cloud-only items (Team Projects, Productions, Firefly, Stock, Dynamic Link, account/help pages),
|
||||
@@ -22,11 +100,14 @@ matched against `filmcraft-cli commands` and the UI command table. Effects: the
|
||||
| Graphics and captions | 8% | text engine, shapes, per-character styles, our own graphics templates (.fcgt, 8 built-ins, export/install/edit), rolls/crawls, responsive pins + time, captions SRT/VTT/SCC/MCC/STL/TTML/DFXP, transcripts | ~85% |
|
||||
| Export | 8% | own H.264/AAC, ProRes, DNxHR, MXF OP1a/OP-Atom, image sequences, GIF, WAV, 5.1; preset library + user presets, full Export-mode settings, queue, Quick Export; AAF (Edit Protocol) import/export, OMF 2.0 export; no AAF/OMF validation against Avid/Pro Tools yet | ~92% |
|
||||
| Preferences and project management | 5% | Settings dialog with 16 categories (most settings wired), project settings, scratch disks, search bins, templates | ~80% |
|
||||
| Performance | 5% | 1080p real-time, 3×1080p; 4K H.264 real-time at load ≤140 (−23% decoder cycles, opt-in draft decoding at 1/2–1/4); HEVC 2.5× cheaper, catch-up decoding, `cargo xtask bench` + `perf.stats`; 8K and AV1 not yet real-time on a loaded machine | ~62% |
|
||||
| Performance | 5% | 1080p real-time, 3×1080p; 4K H.264 real-time at load ≤140 (−23% decoder cycles, opt-in draft decoding at 1/2–1/4); HEVC 2.5× cheaper, catch-up decoding, `cargo xtask bench` + `perf.stats`; blend modes on the GPU (#32); 8K and AV1 not yet real-time on a loaded machine; no hardware path on Windows/Linux (see the honest assessment: ~35–40% against Premiere) | ~62% |
|
||||
| **Weighted total** | | | **~87%** |
|
||||
|
||||
## Estimate to parity
|
||||
|
||||
This estimate is for the **feature checklist**. Closing the honest-assessment gaps (above) is the
|
||||
"Robust on real-world material" row, plus plugins and AI features, which it doesn't include.
|
||||
|
||||
Measured throughput in the last work block (2026-10-01 night → 10-02): five Opus 5.5 agents in
|
||||
parallel for ~4.5 wall-clock hours (~18 agent-hours including integration) moved parity by ~4 points
|
||||
(≈4–5 agent-hours per point), on a machine that was heavily overloaded (load 150–300 on 14 cores) and
|
||||
@@ -62,7 +143,7 @@ Status: ✅ done · 🟡 in progress · ⬜ not started. Estimates are remaining
|
||||
| M2 | H.264 decoder | ✅ | Own decoder, bit-exact on 37+ streams, 500–600 fps 1080p | — | — |
|
||||
| M3 | Editing core | 🟡 | Edit algebra (insert/overwrite/razor/lift/extract/ripple/roll/slip/slide/rate-stretch/nest/paste), tools, markers, trim mode + Trim Monitor + dynamic J/K/L trimming, Keyboard Shortcuts editor with FilmCraft/Premiere/FCP/Avid presets; Edit/Clip/File menu commands (Label colours and Select Label Group, Paste/Remove Attributes, Select All Matching, Remove Unused, Consolidate Duplicates, Sequence From Clip, Bin From Selection, Offline File, Close Project, Make/Edit Subclip, Modify Audio Channels/Timecode, Frame Hold Options/Add Frame Hold/Insert Frame Hold Segment, Time Interpolation with frame blending, Fit/Fill frame, Breakout to Mono, Extract Audio, Replace With Clip; M3.11: Scene Edit Detection (pure-Rust cut detection, background job), Normalize Mix Track, Simplify Sequence, Transcribe Sequence, Find/Find Next and search bins, Automate to Sequence, Edit Original, Edit Offline, Source Settings, Update Metadata (XMP), Generate Audio Waveform, Project Settings General/Scratch Disks, Get Media File Properties, Save as Template, Selection as FilmCraft Project, Avid Log Exchange export, Flash Cue markers, Dynamic Audio Waveforms, Reveal Log Files, System Compatibility Report); M3.12 keyboard parity: Premiere's default keyboard (~115 keyboard-only commands: edit-point navigation on targeted/any track, select clip at playhead/next/previous, extend edit to playhead, nudge/slip/slide selection, target and source-patch toggles, clip volume ±1 dB/many, frame maximize/full screen/panel cycling, monitor zoom, track heights, Project and Text panel keyboard navigation, text size/leading/alignment, Export Frame, poster frames; see docs/keyboard.md); multicam (Create Multi-Camera Source Sequence, Multi-Camera view with live switching on 1–9, angle switching, Enable/Flatten, Edit Cameras) and sync (Synchronize, Merge Clips; In/Out/timecode/marker/audio — GCC-PHAT, sample-accurate) | Multicam paging >16 angles and grid thumbnails, optical flow (renders as frame blending), 39 Premiere default shortcuts skipped with reasons in docs/keyboard.md (Productions, AI/cloud tools, work area bar, Production/Search panels) | 6–10 |
|
||||
| M4 | Playback | 🟡 | Audio-clock master, prefetch with cancellation, J/K/L, correct dropped-frame stats, playback resolution, render bar + content-hashed render previews, App Nap opt-out; 1080p H.264 and 3 stacked 1080p streams play with 0 dropped frames | 4K under load, 8K, frame-threaded AV1 decode, reduced-resolution decode for multicam grids | 6–10 |
|
||||
| M5 | Effects, keyframes, GPU | 🟡 | ~60 CPU effects, 30 transitions, keyframes + value/velocity graphs, wgpu compositor, effect + opacity masks (ellipse/polygon/pen, feather, expansion, modes; CPU/WGSL parity), mask tracking (Lucas–Kanade + RANSAC), adjustment layers, effect presets (built-in + user, JSON import/export) | Full ~150-effect catalogue, WGSL parity for all effects and masks in the live GPU path, Warp Stabilizer, Morph Cut | 20–30 |
|
||||
| M5 | Effects, keyframes, GPU | 🟡 | All 93 Premiere 26 video effects and 84 transitions (CPU), blend modes and 31 common effects on the GPU (#32, GPU2), keyframes + value/velocity graphs, wgpu compositor, effect + opacity masks (ellipse/polygon/pen, feather, expansion, modes; CPU/WGSL parity), mask tracking (Lucas–Kanade + RANSAC), adjustment layers, effect presets (built-in + user, JSON import/export) | Effects on the GPU (in progress, #30), GPU masks in the live path, GPU export, exact Warp Stabilizer (3-D) and Morph Cut | 20–30 |
|
||||
| M6 | Export | ✅ | Own H.264 encoder (High/Main/Baseline, B-frames, VBR/CBR/2-pass) → MP4 or QuickTime + own AAC; ProRes, DNxHR, MJPEG, PNG/TIFF/BMP sequences (Premiere-style numbering), GIF, WAV, AIFF; background jobs; M6.5: Export mode parity (frame size / rate / scaling / pixel aspect, profile / level, CBR / VBR 1- and 2-pass, keyframe distance, audio codec / rate / channels / bitrate / sample size, multiplexer, burn-in or sidecar captions, image / name / timecode overlays, video limiter, loudness normalization with true-peak limiter, metadata, ranges, estimated size, summary), 24 built-in presets + user presets with favourites and import/export (Preset Manager), export queue (reorder, cancel, retry, several sequences / ranges), Quick Export; `export.*` commands and `filmcraft-cli export --preset` | Interlaced encoding, 5.1 audio, smart render, publishing destinations | 6–8 |
|
||||
| M7 | Audio | 🟡 | Mixer graph (tracks → submixes → Mix, pre/post-fader inserts and sends, latency-compensated, sample-accurate, ~6× realtime for 24 tracks × 3 effects on one core), Audio Track Mixer + Audio Clip Mixer panels, track automation (Off/Read/Latch/Touch/Write, recorded live while playing, thinned to keyframes, timeline lanes with pen editing), solo/solo-safe, channel mapping basics, peak + BS.1770 loudness meters (match ffmpeg), DSP crate with 16 clip/track effects, Audio Gain (set/adjust/normalize), Constant Power / Constant Gain / Exponential Fade, Essential Sound (types, Loudness auto-match, Repair incl. DeEss/DeReverb, Clarity, Creative, Ducking keyframes, presets; full Dialogue chain 22× realtime) | Music duration remix, ML speech enhancement, 5.1 panner and multichannel buses, voice-over record, effect editor windows (EQ curve), remaining effects (multiband, convolution reverb), clip-mixer automation recording | 6–9 |
|
||||
| M8 | Colour | 🟡 | Lumetri: basic, creative + looks, RGB & hue curves, wheels, HSL secondary, vignette, section bypass; Input LUT / Look LUT (.cube 1D/3D/shaper, .3dl; tetrahedral CPU + WGSL; project LUT library; built-in camera conversions); Colour Match (Oklab tonal-range statistics, skin protection, solved in Lumetri wheels); colour management: Rec. 709 / Rec. 2100 PQ / HLG working spaces + wide gamut, Interpret Footage colour space (S-Log3, V-Log, Canon Log 2/3, LogC3/4, Apple Log, D-Log from published specs), metadata auto-detect, BT.2390 tone mapping, gamut mapping, HDR export signalling (VUI/colr/mdcv/clli/SEI, ffprobe-verified); Lumetri Scopes: Vectorscope YUV (75/100 % targets, skin-tone line) and HLS, Histogram, Parade (RGB/YUV/RGB-White), Waveform (RGB/Luma/YC/YC no Chroma), presets, Rec. 601/709/2100, 8-bit/float/HDR (cd/m²) scales, multi-scope grid, `scopes.read` | HDR-aware Lumetri maths, macOS EDR monitors, mastering metadata → tone-map peak, D-Log M (no published formula), HSL Secondary refine | 3–5 |
|
||||
@@ -73,10 +154,13 @@ Status: ✅ done · 🟡 in progress · ⬜ not started. Estimates are remaining
|
||||
|
||||
## Running now
|
||||
|
||||
- Nothing; next: remaining Premiere menu items (Settings categories, Scene Edit Detection, Normalize Mix Track, OMF/AAF, Find), AV1 single-thread re-measure on an idle machine
|
||||
- **Hardware acceleration (#30):** landed: blend modes on the GPU (#32); VideoToolbox hardware decode + the `platform` FFI crate (#33); 31 common effects on the GPU (GPU2). Next: VA-API (Linux) and Media Foundation (Windows) decode, zero-copy upload, Lumetri on the GPU.
|
||||
- **Next:** [Where we are lacking](#where-we-are-lacking) items 1–5. GPU export waits for the export-crate work (#19 and the never-crash follow-up) to land.
|
||||
|
||||
## Log
|
||||
|
||||
- **2026-10-05:** honest assessment added (checklist ~87% vs ready for real work ~50–60%). Hardware acceleration started (#30): blend modes on the GPU (#32); VideoToolbox decode + `platform` crate, the one crate allowed `unsafe` (#33). Community fixes landed: AIFF import, export start/end ranges, transitions following ripple trims, Slide moving linked audio (#5–#8).
|
||||
|
||||
- **2026-10-04 (later):** interchange + MXF export (M11.4/M6.6): own compound-file container (`crates/cfb`), AAF Edit Protocol import/export (embedded/linked/consolidated audio, video mixdown, handles, breakout to mono), OMF 2.0 export (Bento), MXF OP1a/OP-Atom writer (DNxHR/ProRes/H.264/PCM) as export formats. 1605 tests.
|
||||
|
||||
- **2026-10-04:** MPEG-2 (M9.15): own MPEG-2/MPEG-1 video decoder (4:2:2, interlaced, field pictures; IEEE 1180 IDCT), MPEG TS/PS demux (AVCHD .mts, .m2ts, .ts, .mpg/.vob/.mod), own AC-3 decoder (ATSC A/52), MP2, LATM AAC, Blu-ray/DVD LPCM, MPEG-2 in MXF/MOV/MP4/MKV. 1562 tests.
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
//! with unsaved changes the app asks to recover them; `--recover` recovers the newest without
|
||||
//! asking, `--no-recover` starts without asking (the changes stay available via File ▸ Recover
|
||||
//! Unsaved Changes…).
|
||||
|
||||
#![cfg_attr(all(target_os = "windows", not(debug_assertions)), windows_subsystem = "windows")]
|
||||
#![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable))]
|
||||
|
||||
mod app_nap;
|
||||
|
||||
@@ -11,6 +11,7 @@
|
||||
#![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable))]
|
||||
|
||||
pub mod bridge;
|
||||
pub mod long_job;
|
||||
pub mod server;
|
||||
|
||||
pub use bridge::BridgeClient;
|
||||
|
||||
@@ -0,0 +1,183 @@
|
||||
//! Long exports over MCP (docs/agents.md § Long exports): `file.exportMedia` with `wait: true`
|
||||
//! runs as a background engine job; the server reports its progress as MCP
|
||||
//! `notifications/progress` (when the request carried a `progressToken`) and stops it on
|
||||
//! `notifications/cancelled`, deleting the partial output. The session lock is only held for
|
||||
//! short polls, so other requests are answered while the export runs.
|
||||
|
||||
use std::path::Path;
|
||||
use std::time::{Duration, SystemTime};
|
||||
|
||||
use rmcp::RoleServer;
|
||||
use rmcp::model::{CallToolResult, ContentBlock as Content, ProgressNotificationParam, ProgressToken};
|
||||
use rmcp::service::{Peer, RequestContext};
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use crate::server::FilmcraftMcp;
|
||||
|
||||
/// Engine commands that block until a render is written when called with `wait: true`.
|
||||
pub const LONG_COMMANDS: &[&str] = &["file.exportMedia"];
|
||||
|
||||
/// How often the job is polled (and at most how often progress is reported).
|
||||
const POLL: Duration = Duration::from_millis(100);
|
||||
|
||||
/// Whether `command_run {id, params}` is a long call this module runs.
|
||||
pub fn is_long(id: &str, params: &Value) -> bool {
|
||||
LONG_COMMANDS.contains(&id) && params.get("wait").and_then(Value::as_bool) == Some(true)
|
||||
}
|
||||
|
||||
/// Sends `notifications/progress` for one request; progress only ever increases.
|
||||
struct Reporter {
|
||||
peer: Peer<RoleServer>,
|
||||
token: Option<ProgressToken>,
|
||||
last: f64,
|
||||
}
|
||||
|
||||
impl Reporter {
|
||||
async fn report(&mut self, done: f64, total: f64, message: &str) {
|
||||
let Some(token) = self.token.clone() else { return };
|
||||
if total <= 0.0 || done <= self.last {
|
||||
return;
|
||||
}
|
||||
self.last = done;
|
||||
let mut p = ProgressNotificationParam::new(token, done).with_total(total);
|
||||
if !message.is_empty() {
|
||||
p = p.with_message(message);
|
||||
}
|
||||
let _ = self.peer.notify_progress(p).await;
|
||||
}
|
||||
}
|
||||
|
||||
impl FilmcraftMcp {
|
||||
/// Run `id` (a [`LONG_COMMANDS`] entry, `wait: true`) as a job, reporting progress and
|
||||
/// honouring cancellation.
|
||||
pub(crate) async fn run_long(&self, id: &str, mut params: Value, context: &RequestContext<RoleServer>) -> CallToolResult {
|
||||
let started = SystemTime::now();
|
||||
if let Some(p) = params.as_object_mut() {
|
||||
p.insert("wait".into(), json!(false));
|
||||
}
|
||||
let start = match self.run(id, params).await {
|
||||
Ok(v) => v,
|
||||
Err(e) => return CallToolResult::error(vec![Content::text(e.to_string())]),
|
||||
};
|
||||
let Some(job) = start.get("job").and_then(Value::as_u64) else {
|
||||
return CallToolResult::error(vec![Content::text(format!("{id} started no job: {start}"))]);
|
||||
};
|
||||
let path = start.get("path").and_then(Value::as_str).unwrap_or_default().to_string();
|
||||
let mut rep = Reporter { peer: context.peer.clone(), token: context.meta.get_progress_token(), last: 0.0 };
|
||||
loop {
|
||||
let state = self.job_state(job).await;
|
||||
let finished = state.as_ref().is_none_or(|j| j["finished"].as_bool() == Some(true));
|
||||
if let Some(j) = &state {
|
||||
let (done, total) = (j["done"].as_f64().unwrap_or(0.0), j["total"].as_f64().unwrap_or(0.0));
|
||||
rep.report(done, total, j["status"].as_str().unwrap_or_default()).await;
|
||||
}
|
||||
if finished {
|
||||
let result = state.map(|j| j["result"].clone()).unwrap_or(Value::Null);
|
||||
if let Some(e) = result.get("error").and_then(Value::as_str) {
|
||||
return CallToolResult::error(vec![Content::text(format!("export failed: {e}"))]);
|
||||
}
|
||||
let mut out = start;
|
||||
if let Some(o) = out.as_object_mut() {
|
||||
o.insert("result".into(), result);
|
||||
}
|
||||
return CallToolResult::success(vec![Content::text(serde_json::to_string_pretty(&out).unwrap_or_default())]);
|
||||
}
|
||||
tokio::select! {
|
||||
_ = context.ct.cancelled() => break,
|
||||
_ = tokio::time::sleep(POLL) => {}
|
||||
}
|
||||
}
|
||||
// Cancelled: stop the encode at the next batch, wait for the worker, delete the partial file.
|
||||
let _ = self.run("jobs.cancel", json!({"job": job})).await;
|
||||
for _ in 0..600 {
|
||||
if self.job_state(job).await.is_none_or(|j| j["finished"].as_bool() == Some(true)) {
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(POLL).await;
|
||||
}
|
||||
remove_partial(&path, started);
|
||||
CallToolResult::error(vec![Content::text("cancelled")])
|
||||
}
|
||||
|
||||
/// The job's `jobs.list` entry.
|
||||
async fn job_state(&self, job: u64) -> Option<Value> {
|
||||
let jobs = self.run("jobs.list", json!({})).await.ok()?;
|
||||
jobs.as_array()?.iter().find(|j| j["id"].as_u64() == Some(job)).cloned()
|
||||
}
|
||||
}
|
||||
|
||||
/// Delete what an interrupted export wrote, and nothing else: the output file itself, the numbered
|
||||
/// stills of an image sequence (`<stem><digits>.<same extension>`) and the caption sidecar
|
||||
/// (`<stem>.srt` / `<stem>.vtt`), each only if it was written since `since`. Other files that
|
||||
/// merely share the name's stem (`<stem>.aep`, `<stem>2.psd`, …) are never touched, even when they
|
||||
/// were saved during the export.
|
||||
pub fn remove_partial(path: &str, since: SystemTime) {
|
||||
let p = Path::new(path);
|
||||
let (Some(dir), Some(stem), Some(file)) = (p.parent(), p.file_stem().and_then(|s| s.to_str()), p.file_name().and_then(|s| s.to_str())) else {
|
||||
return;
|
||||
};
|
||||
let ext = p.extension().and_then(|s| s.to_str()).unwrap_or_default();
|
||||
let dir = if dir.as_os_str().is_empty() { Path::new(".") } else { dir };
|
||||
let Ok(entries) = std::fs::read_dir(dir) else { return };
|
||||
// file times are coarser than the clock
|
||||
let since = since.checked_sub(Duration::from_secs(1)).unwrap_or(since);
|
||||
for e in entries.flatten() {
|
||||
let name = e.file_name();
|
||||
let Some(name) = name.to_str() else { continue };
|
||||
if is_export_output(name, file, stem, ext) && e.metadata().is_ok_and(|m| m.is_file() && m.modified().is_ok_and(|t| t >= since)) {
|
||||
let _ = std::fs::remove_file(e.path());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether `name` is a file an export to `file` (`stem` + `.ext`) writes.
|
||||
fn is_export_output(name: &str, file: &str, stem: &str, ext: &str) -> bool {
|
||||
if name == file {
|
||||
return true;
|
||||
}
|
||||
let Some(rest) = name.strip_prefix(stem) else { return false };
|
||||
if let Some(sidecar) = rest.strip_prefix('.') {
|
||||
return sidecar.eq_ignore_ascii_case("srt") || sidecar.eq_ignore_ascii_case("vtt");
|
||||
}
|
||||
// image sequence: one or more digits, then the output's own extension
|
||||
let digits = rest.len() - rest.trim_start_matches(|c: char| c.is_ascii_digit()).len();
|
||||
digits > 0 && !ext.is_empty() && rest.get(digits..).is_some_and(|r| r.strip_prefix('.').is_some_and(|x| x == ext))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn only_the_exports_own_files_are_removed() {
|
||||
let dir = std::env::temp_dir().join(format!("fc-remove-partial-{}", std::process::id()));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let started = SystemTime::now();
|
||||
let ours = ["trailer.png", "trailer000.png", "trailer123.png", "trailer.srt", "trailer.vtt"];
|
||||
// same stem, written during the export, but not the export's
|
||||
let theirs = ["trailer.aep", "trailer2.psd", "trailer.docx", "trailer000.jpg", "trailers.png", "trailer.png.bak", "other.png"];
|
||||
for f in ours.iter().chain(theirs.iter()) {
|
||||
std::fs::write(dir.join(f), b"x").unwrap();
|
||||
}
|
||||
remove_partial(&dir.join("trailer.png").to_string_lossy(), started);
|
||||
for f in ours {
|
||||
assert!(!dir.join(f).exists(), "{f} should be removed");
|
||||
}
|
||||
for f in theirs {
|
||||
assert!(dir.join(f).exists(), "{f} must not be removed");
|
||||
}
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn files_older_than_the_export_are_kept() {
|
||||
let dir = std::env::temp_dir().join(format!("fc-remove-partial-old-{}", std::process::id()));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
std::fs::write(dir.join("clip.mp4"), b"x").unwrap();
|
||||
remove_partial(&dir.join("clip.mp4").to_string_lossy(), SystemTime::now() + Duration::from_secs(60));
|
||||
assert!(dir.join("clip.mp4").exists());
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
}
|
||||
+546
-28
@@ -1,6 +1,6 @@
|
||||
//! The MCP server. Tool names are snake_case.
|
||||
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::sync::{Arc, Mutex, PoisonError};
|
||||
|
||||
use base64::Engine as _;
|
||||
use filmcraft_engine::Session;
|
||||
@@ -19,6 +19,19 @@ pub enum Backend {
|
||||
Bridge(Arc<BridgeClient>),
|
||||
}
|
||||
|
||||
/// A panic payload as text.
|
||||
fn panic_message(p: Box<dyn std::any::Any + Send>) -> String {
|
||||
p.downcast_ref::<&str>().map(|s| s.to_string()).or_else(|| p.downcast_ref::<String>().cloned()).unwrap_or_else(|| "panic".into())
|
||||
}
|
||||
|
||||
/// A failed blocking task: a panic is reported as an internal error and the session stays usable.
|
||||
fn join_error(e: tokio::task::JoinError) -> AutomationError {
|
||||
match e.try_into_panic() {
|
||||
Ok(p) => AutomationError::Other(format!("internal error: {}", panic_message(p))),
|
||||
Err(e) => AutomationError::Other(e.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct FilmcraftMcp {
|
||||
backend: Arc<Backend>,
|
||||
@@ -44,6 +57,24 @@ pub struct RunParams {
|
||||
pub params: Option<Value>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, JsonSchema)]
|
||||
pub struct BatchStep {
|
||||
/// Command id.
|
||||
pub id: String,
|
||||
/// Parameters object.
|
||||
#[serde(default)]
|
||||
pub params: Option<Value>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, JsonSchema)]
|
||||
pub struct BatchParams {
|
||||
/// Commands to run in order.
|
||||
pub steps: Vec<BatchStep>,
|
||||
/// Stop at the first failing step (default true).
|
||||
#[serde(default)]
|
||||
pub stop_on_error: Option<bool>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Deserialize, JsonSchema)]
|
||||
pub struct RenderParams {
|
||||
/// Timeline time in seconds (default: the playhead).
|
||||
@@ -148,9 +179,60 @@ impl FilmcraftMcp {
|
||||
}
|
||||
|
||||
pub async fn serve_stdio(self) -> Result<(), AutomationError> {
|
||||
let running = self.serve(rmcp::transport::stdio()).await.map_err(|e| AutomationError::Other(format!("MCP init: {e}")))?;
|
||||
running.waiting().await.map_err(|e| AutomationError::Other(e.to_string()))?;
|
||||
Ok(())
|
||||
self.serve_io(tokio::io::stdin(), tokio::io::stdout()).await
|
||||
}
|
||||
|
||||
/// Serve MCP over any line stream. A line that isn't JSON gets a JSON-RPC parse error
|
||||
/// (`-32700`, id null) instead of being dropped silently; the rest go to the rmcp service. One
|
||||
/// writer task owns `output`, so replies never interleave.
|
||||
pub async fn serve_io(
|
||||
self,
|
||||
input: impl tokio::io::AsyncRead + Unpin + Send + 'static,
|
||||
output: impl tokio::io::AsyncWrite + Unpin + Send + 'static,
|
||||
) -> Result<(), AutomationError> {
|
||||
use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};
|
||||
let (mut to_service, service_in) = tokio::io::duplex(1 << 20);
|
||||
let (service_out, from_service) = tokio::io::duplex(1 << 20);
|
||||
let (err_tx, mut err_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
|
||||
let reader = tokio::spawn(async move {
|
||||
let mut lines = BufReader::new(input).lines();
|
||||
while let Ok(Some(line)) = lines.next_line().await {
|
||||
if line.trim().is_empty() {
|
||||
continue;
|
||||
}
|
||||
match serde_json::from_str::<Value>(&line) {
|
||||
Ok(_) => {
|
||||
if to_service.write_all(format!("{line}\n").as_bytes()).await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
let reply = json!({"jsonrpc": "2.0", "id": null, "error": {"code": -32700, "message": format!("parse error: {e}")}});
|
||||
let _ = err_tx.send(reply.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
// EOF: closing the pipe ends the service
|
||||
let _ = to_service.shutdown().await;
|
||||
});
|
||||
let writer = tokio::spawn(async move {
|
||||
let mut output = output;
|
||||
let mut replies = BufReader::new(from_service).lines();
|
||||
loop {
|
||||
let line = tokio::select! {
|
||||
l = replies.next_line() => match l { Ok(Some(l)) => l, _ => break },
|
||||
Some(e) = err_rx.recv() => e,
|
||||
};
|
||||
if output.write_all(format!("{line}\n").as_bytes()).await.is_err() || output.flush().await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
let running = self.serve((service_in, service_out)).await.map_err(|e| AutomationError::Other(format!("MCP init: {e}")))?;
|
||||
let r = running.waiting().await.map_err(|e| AutomationError::Other(e.to_string()));
|
||||
reader.abort();
|
||||
let _ = writer.await;
|
||||
r.map(|_| ())
|
||||
}
|
||||
|
||||
/// Run an engine command on whichever backend.
|
||||
@@ -160,16 +242,24 @@ impl FilmcraftMcp {
|
||||
let s = s.clone();
|
||||
let id = id.to_string();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let mut g = s.lock().map_err(|_| AutomationError::Other("session lock poisoned".into()))?;
|
||||
// A command that panicked earlier poisoned the lock; the session is still usable.
|
||||
let mut g = s.lock().unwrap_or_else(PoisonError::into_inner);
|
||||
g.execute(&id, params).map_err(AutomationError::from)
|
||||
})
|
||||
.await
|
||||
.map_err(|e| AutomationError::Other(e.to_string()))?
|
||||
.map_err(join_error)?
|
||||
}
|
||||
Backend::Bridge(b) => b.call("engine.execute", json!({"command": id, "params": params})).await,
|
||||
}
|
||||
}
|
||||
|
||||
/// The project tree plus the active sequence (null when there is none).
|
||||
async fn document(&self) -> Result<Value, AutomationError> {
|
||||
let project = self.run("project.inspect", json!({})).await?;
|
||||
let sequence = self.run("sequence.inspect", json!({})).await.ok();
|
||||
Ok(json!({"project": project, "sequence": sequence}))
|
||||
}
|
||||
|
||||
fn bridge_client(&self) -> Option<Arc<BridgeClient>> {
|
||||
match &*self.backend {
|
||||
Backend::Bridge(b) => Some(b.clone()),
|
||||
@@ -190,7 +280,9 @@ impl FilmcraftMcp {
|
||||
#[tool_router]
|
||||
impl FilmcraftMcp {
|
||||
#[tool(
|
||||
description = "List every command (id, label, menu path, shortcut, parameter doc, enabled now). Menus, shortcuts, panels and timeline gestures all map to these ids."
|
||||
title = "List commands",
|
||||
description = "List every command (id, label, menu path, shortcut, parameter doc, enabled now). Menus, shortcuts, panels and timeline gestures all map to these ids.",
|
||||
annotations(read_only_hint = true, destructive_hint = false, idempotent_hint = true, open_world_hint = false)
|
||||
)]
|
||||
async fn command_list(&self, Parameters(p): Parameters<ListParams>) -> Result<CallToolResult, McpError> {
|
||||
let r = self.run("command.list", json!({})).await.map(|v| {
|
||||
@@ -213,38 +305,99 @@ impl FilmcraftMcp {
|
||||
}
|
||||
|
||||
#[tool(
|
||||
description = "Run a command by id with JSON params, e.g. {\"id\":\"timeline.razor\",\"params\":{\"seconds\":3.5}} or {\"id\":\"effects.apply\",\"params\":{\"effect\":\"Gaussian Blur\"}}. Undoable edits land in History."
|
||||
title = "Run a command",
|
||||
description = "Run a command by id with JSON params, e.g. {\"id\":\"timeline.razor\",\"params\":{\"seconds\":3.5}} or {\"id\":\"effects.apply\",\"params\":{\"effect\":\"Gaussian Blur\"}}. Undoable edits land in History.",
|
||||
annotations(read_only_hint = false, destructive_hint = true, idempotent_hint = false, open_world_hint = false)
|
||||
)]
|
||||
async fn command_run(&self, Parameters(p): Parameters<RunParams>) -> Result<CallToolResult, McpError> {
|
||||
wrap(self.run(&p.id, p.params.unwrap_or(json!({}))).await)
|
||||
}
|
||||
|
||||
#[tool(description = "Project tree (bins and items with ids, types, durations), active sequence, source clip.")]
|
||||
#[tool(
|
||||
title = "Run several commands",
|
||||
description = "Run several commands in order: {steps: [{id, params?}], stop_on_error?: true}. Returns {completed, failed, results: [{ok, result | error}]}; each edit is its own undo step.",
|
||||
annotations(read_only_hint = false, destructive_hint = true, idempotent_hint = false, open_world_hint = false)
|
||||
)]
|
||||
async fn command_batch(&self, Parameters(p): Parameters<BatchParams>) -> Result<CallToolResult, McpError> {
|
||||
let stop = p.stop_on_error.unwrap_or(true);
|
||||
let (mut completed, mut failed, mut results) = (0u32, 0u32, Vec::new());
|
||||
for st in p.steps {
|
||||
match self.run(&st.id, st.params.unwrap_or(json!({}))).await {
|
||||
Ok(v) => {
|
||||
completed += 1;
|
||||
results.push(json!({"ok": true, "result": v}));
|
||||
}
|
||||
Err(e) => {
|
||||
failed += 1;
|
||||
results.push(json!({"ok": false, "error": e.to_string()}));
|
||||
if stop {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
let v = json!({"completed": completed, "failed": failed, "results": results});
|
||||
Ok(if failed > 0 { CallToolResult::error(vec![Content::text(v.to_string())]) } else { ok_json(&v) })
|
||||
}
|
||||
|
||||
#[tool(
|
||||
title = "Inspect the document",
|
||||
description = "The project as JSON: the tree (as project_inspect) plus the active sequence (as sequence_inspect; null when there is none).",
|
||||
annotations(read_only_hint = true, destructive_hint = false, idempotent_hint = true, open_world_hint = false)
|
||||
)]
|
||||
async fn doc_inspect(&self) -> Result<CallToolResult, McpError> {
|
||||
wrap(self.document().await)
|
||||
}
|
||||
|
||||
#[tool(
|
||||
title = "Render a preview",
|
||||
description = "The active sequence's frame at `seconds` (default: the playhead) as PNG, same as render_frame (bridge mode: Program monitor screenshot).",
|
||||
annotations(read_only_hint = true, destructive_hint = false, idempotent_hint = true, open_world_hint = false)
|
||||
)]
|
||||
async fn render_preview(&self, p: Parameters<RenderParams>) -> Result<CallToolResult, McpError> {
|
||||
self.render_frame(p).await
|
||||
}
|
||||
|
||||
#[tool(
|
||||
title = "Inspect the project tree",
|
||||
description = "Project tree (bins and items with ids, types, durations), active sequence, source clip.",
|
||||
annotations(read_only_hint = true, destructive_hint = false, idempotent_hint = true, open_world_hint = false)
|
||||
)]
|
||||
async fn project_inspect(&self) -> Result<CallToolResult, McpError> {
|
||||
wrap(self.run("project.inspect", json!({})).await)
|
||||
}
|
||||
|
||||
#[tool(
|
||||
description = "The active sequence as JSON: settings, tracks, clips (ids, start/duration in ticks and frames, effects), transitions, markers, in/out, playhead, selection. 254016000000 ticks = 1 second."
|
||||
title = "Inspect the active sequence",
|
||||
description = "The active sequence as JSON: settings, tracks, clips (ids, start/duration in ticks and frames, effects), transitions, markers, in/out, playhead, selection. 254016000000 ticks = 1 second.",
|
||||
annotations(read_only_hint = true, destructive_hint = false, idempotent_hint = true, open_world_hint = false)
|
||||
)]
|
||||
async fn sequence_inspect(&self) -> Result<CallToolResult, McpError> {
|
||||
wrap(self.run("sequence.inspect", json!({})).await)
|
||||
}
|
||||
|
||||
#[tool(description = "Import media files by absolute path (MP4/MOV, WAV/MP3/FLAC/AIFF/Ogg, PNG/JPEG/…).")]
|
||||
#[tool(
|
||||
title = "Import media",
|
||||
description = "Import media files by absolute path (MP4/MOV, WAV/MP3/FLAC/AIFF/Ogg, PNG/JPEG/…).",
|
||||
annotations(read_only_hint = false, destructive_hint = false, idempotent_hint = false, open_world_hint = false)
|
||||
)]
|
||||
async fn media_import(&self, Parameters(p): Parameters<TypeParams>) -> Result<CallToolResult, McpError> {
|
||||
let paths: Vec<&str> = p.text.split('\n').map(str::trim).filter(|s| !s.is_empty()).collect();
|
||||
wrap(self.run("file.import", json!({"paths": paths})).await)
|
||||
}
|
||||
|
||||
#[tool(description = "Render the active sequence's frame (at `seconds` or the playhead) and return it as PNG.")]
|
||||
#[tool(
|
||||
title = "Render a frame",
|
||||
description = "Render the active sequence's frame (at `seconds` or the playhead) and return it as PNG.",
|
||||
annotations(read_only_hint = true, destructive_hint = false, idempotent_hint = true, open_world_hint = false)
|
||||
)]
|
||||
async fn render_frame(&self, Parameters(p): Parameters<RenderParams>) -> Result<CallToolResult, McpError> {
|
||||
let max = p.max_side.unwrap_or(960);
|
||||
match &*self.backend {
|
||||
Backend::Headless(s) => {
|
||||
let s = s.clone();
|
||||
let r = tokio::task::spawn_blocking(move || -> Result<Vec<u8>, AutomationError> {
|
||||
let mut g = s.lock().map_err(|_| AutomationError::Other("lock".into()))?;
|
||||
let mut g = s.lock().unwrap_or_else(PoisonError::into_inner);
|
||||
if let Some(sec) = p.seconds {
|
||||
g.set_playhead(filmcraft_time::Tick::from_seconds_f64(sec));
|
||||
}
|
||||
@@ -255,7 +408,7 @@ impl FilmcraftMcp {
|
||||
png_rgba(img.w as u32, img.h as u32, img.over_black_rgba8(), max)
|
||||
})
|
||||
.await
|
||||
.map_err(|e| McpError::internal_error(e.to_string(), None))?;
|
||||
.unwrap_or_else(|e| Err(join_error(e)));
|
||||
Ok(match r {
|
||||
Ok(png) => png_result(&png, "rendered frame"),
|
||||
Err(e) => fail(e),
|
||||
@@ -272,17 +425,29 @@ impl FilmcraftMcp {
|
||||
}
|
||||
}
|
||||
|
||||
#[tool(description = "Live app: UI state (tool, workspace, panels, timeline zoom, playback, selection, fps).")]
|
||||
#[tool(
|
||||
title = "Live app: UI state",
|
||||
description = "Live app: UI state (tool, workspace, panels, timeline zoom, playback, selection, fps).",
|
||||
annotations(read_only_hint = true, destructive_hint = false, idempotent_hint = true, open_world_hint = false)
|
||||
)]
|
||||
async fn ui_inspect(&self) -> Result<CallToolResult, McpError> {
|
||||
self.ui("ui.inspect", json!({})).await
|
||||
}
|
||||
|
||||
#[tool(description = "Live app: every on-screen interactive element with id, label and rect (points). Use ids with ui_click/ui_drag.")]
|
||||
#[tool(
|
||||
title = "Live app: on-screen elements",
|
||||
description = "Live app: every on-screen interactive element with id, label and rect (points). Use ids with ui_click/ui_drag.",
|
||||
annotations(read_only_hint = true, destructive_hint = false, idempotent_hint = true, open_world_hint = false)
|
||||
)]
|
||||
async fn ui_elements(&self, Parameters(p): Parameters<ElementsParams>) -> Result<CallToolResult, McpError> {
|
||||
self.ui("ui.elements", json!({"prefix": p.prefix.unwrap_or_default()})).await
|
||||
}
|
||||
|
||||
#[tool(description = "Live app: click an element by id or at x,y (points). Supports right/middle button, double-click (count=2) and modifiers.")]
|
||||
#[tool(
|
||||
title = "Live app: click",
|
||||
description = "Live app: click an element by id or at x,y (points). Supports right/middle button, double-click (count=2) and modifiers.",
|
||||
annotations(read_only_hint = false, destructive_hint = true, idempotent_hint = false, open_world_hint = false)
|
||||
)]
|
||||
async fn ui_click(&self, Parameters(p): Parameters<ClickParams>) -> Result<CallToolResult, McpError> {
|
||||
let mut v = json!({"button": p.button, "count": p.count, "modifiers": p.modifiers});
|
||||
if let Some(id) = p.id {
|
||||
@@ -295,23 +460,37 @@ impl FilmcraftMcp {
|
||||
}
|
||||
|
||||
#[tool(
|
||||
description = "Live app: press-drag-release from one point/element to another (move clips, trim edges, scrub, resize panels, drop project items onto tracks)."
|
||||
title = "Live app: drag",
|
||||
description = "Live app: press-drag-release from one point/element to another (move clips, trim edges, scrub, resize panels, drop project items onto tracks).",
|
||||
annotations(read_only_hint = false, destructive_hint = true, idempotent_hint = false, open_world_hint = false)
|
||||
)]
|
||||
async fn ui_drag(&self, Parameters(p): Parameters<DragParams>) -> Result<CallToolResult, McpError> {
|
||||
self.ui("ui.drag", json!({"from": p.from, "to": p.to, "steps": p.steps, "modifiers": p.modifiers})).await
|
||||
}
|
||||
|
||||
#[tool(description = "Live app: press a key or shortcut (e.g. `Space`, `L`, `Cmd+K`, `Shift+Delete`, `Cmd+Z`).")]
|
||||
#[tool(
|
||||
title = "Live app: press a key",
|
||||
description = "Live app: press a key or shortcut (e.g. `Space`, `L`, `Cmd+K`, `Shift+Delete`, `Cmd+Z`).",
|
||||
annotations(read_only_hint = false, destructive_hint = true, idempotent_hint = false, open_world_hint = false)
|
||||
)]
|
||||
async fn ui_key(&self, Parameters(p): Parameters<KeyParams>) -> Result<CallToolResult, McpError> {
|
||||
self.ui("ui.key", json!({"key": p.key})).await
|
||||
}
|
||||
|
||||
#[tool(description = "Live app: type text into the focused field.")]
|
||||
#[tool(
|
||||
title = "Live app: type text",
|
||||
description = "Live app: type text into the focused field.",
|
||||
annotations(read_only_hint = false, destructive_hint = true, idempotent_hint = false, open_world_hint = false)
|
||||
)]
|
||||
async fn ui_type(&self, Parameters(p): Parameters<TypeParams>) -> Result<CallToolResult, McpError> {
|
||||
self.ui("ui.type", json!({"text": p.text})).await
|
||||
}
|
||||
|
||||
#[tool(description = "Live app: screenshot of the window or one panel (PNG).")]
|
||||
#[tool(
|
||||
title = "Live app: screenshot",
|
||||
description = "Live app: screenshot of the window or one panel (PNG).",
|
||||
annotations(read_only_hint = true, destructive_hint = false, idempotent_hint = true, open_world_hint = false)
|
||||
)]
|
||||
async fn ui_screenshot(&self, Parameters(p): Parameters<ShotParams>) -> Result<CallToolResult, McpError> {
|
||||
match self.bridge_client() {
|
||||
Some(b) => self.screenshot(&b, p.panel, p.max_side.unwrap_or(1600)).await,
|
||||
@@ -320,7 +499,9 @@ impl FilmcraftMcp {
|
||||
}
|
||||
|
||||
#[tool(
|
||||
description = "Live app: call any control-channel method directly (ui.set, ui.scroll, ui.timeline.hit, ui.timeline.locate, ui.playback, ui.panel.show, ui.menu.list, …)."
|
||||
title = "Live app: control-channel call",
|
||||
description = "Live app: call any control-channel method directly (ui.set, ui.scroll, ui.timeline.hit, ui.timeline.locate, ui.playback, ui.panel.show, ui.menu.list, …).",
|
||||
annotations(read_only_hint = false, destructive_hint = true, idempotent_hint = false, open_world_hint = false)
|
||||
)]
|
||||
async fn ui_control(&self, Parameters(p): Parameters<ControlParams>) -> Result<CallToolResult, McpError> {
|
||||
self.ui(&p.method, p.params.unwrap_or(json!({}))).await
|
||||
@@ -350,12 +531,104 @@ impl FilmcraftMcp {
|
||||
}
|
||||
}
|
||||
|
||||
#[tool_handler(
|
||||
router = self.tool_router,
|
||||
name = "filmcraft",
|
||||
instructions = "FilmCraft video editor (Premiere Pro-class). Every edit is an engine command: `command_list` to discover ids/params, `command_run` to execute (undoable). `project_inspect`/`sequence_inspect` return ids you can pass to commands; `render_frame` shows the result. In bridge mode the `ui_*` tools drive the live app: `ui_elements` lists clickable ids, `ui_click`/`ui_drag`/`ui_key` operate it, `ui_screenshot` shows it. Time is in ticks: 254016000000 per second (commands also accept `seconds`, `frame` or `timecode`)."
|
||||
)]
|
||||
impl ServerHandler for FilmcraftMcp {}
|
||||
impl FilmcraftMcp {
|
||||
/// The first argument `tool` doesn't take, as a message naming the ones it does.
|
||||
fn unknown_arg(&self, tool: &str, args: Option<&rmcp::model::JsonObject>) -> Option<String> {
|
||||
let def = self.tool_router.get(tool)?;
|
||||
let props = def.input_schema.get("properties").and_then(Value::as_object).cloned().unwrap_or_default();
|
||||
let bad = args?.keys().find(|k| !props.contains_key(*k))?;
|
||||
let accepted: Vec<&str> = props.keys().map(String::as_str).collect();
|
||||
let accepted = if accepted.is_empty() { "no arguments".to_string() } else { accepted.join(", ") };
|
||||
Some(format!("unknown argument \"{bad}\" for {tool}; expected: {accepted}"))
|
||||
}
|
||||
}
|
||||
|
||||
const INSTRUCTIONS: &str = "FilmCraft video editor (Premiere Pro-class). Every edit is an engine command: `command_list` to discover ids/params, `command_run` to execute (undoable; `command_batch` runs several). `doc_inspect` (or `project_inspect`/`sequence_inspect`) returns ids you can pass to commands; `render_preview` shows the result. In bridge mode the `ui_*` tools drive the live app: `ui_elements` lists clickable ids, `ui_click`/`ui_drag`/`ui_key` operate it, `ui_screenshot` shows it. Time is in ticks: 254016000000 per second (commands also accept `seconds`, `frame` or `timecode`).";
|
||||
|
||||
/// Resources: the project (as `doc_inspect`) and the command catalog (as `command_list`).
|
||||
const DOCUMENT_URI: &str = "filmcraft://document";
|
||||
const COMMANDS_URI: &str = "filmcraft://commands";
|
||||
|
||||
/// MCP 2026-07-28 clients (current Claude Code) require `ttlMs` and `cacheScope` on list and read
|
||||
/// results; rmcp fills `resultType` and leaves the cache hints to the server, as the generated
|
||||
/// `tools/list` does. The resource list never changes; the document does, so reads aren't cached.
|
||||
fn cache_hints(context: &rmcp::service::RequestContext<rmcp::RoleServer>, ttl_ms: u64) -> Option<(u64, rmcp::model::CacheScope)> {
|
||||
context.protocol_version().is_some_and(|v| v >= rmcp::model::ProtocolVersion::V_2026_07_28).then_some((ttl_ms, rmcp::model::CacheScope::Private))
|
||||
}
|
||||
|
||||
#[tool_handler(router = self.tool_router)]
|
||||
impl ServerHandler for FilmcraftMcp {
|
||||
fn get_info(&self) -> rmcp::model::ServerConfig {
|
||||
rmcp::model::ServerConfig::new(rmcp::model::ServerCapabilities::builder().enable_tools().enable_resources().build())
|
||||
.with_server_info(rmcp::model::Implementation::new("filmcraft", env!("CARGO_PKG_VERSION")))
|
||||
.with_instructions(INSTRUCTIONS.to_string())
|
||||
}
|
||||
|
||||
/// Strict arguments and long exports in front of the tool router.
|
||||
async fn call_tool(
|
||||
&self,
|
||||
request: rmcp::model::CallToolRequestParams,
|
||||
context: rmcp::service::RequestContext<rmcp::RoleServer>,
|
||||
) -> Result<rmcp::model::CallToolResponse, McpError> {
|
||||
if let Some(m) = self.unknown_arg(&request.name, request.arguments.as_ref()) {
|
||||
return Err(McpError::invalid_params(m, None));
|
||||
}
|
||||
// A blocking export reports progress and can be cancelled (docs/agents.md § Long exports).
|
||||
if request.name == "command_run"
|
||||
&& matches!(&*self.backend, Backend::Headless(_))
|
||||
&& let Some(a) = &request.arguments
|
||||
&& let Some(id) = a.get("id").and_then(Value::as_str)
|
||||
{
|
||||
let params = a.get("params").cloned().filter(Value::is_object).unwrap_or_else(|| json!({}));
|
||||
if crate::long_job::is_long(id, ¶ms) {
|
||||
return Ok(self.run_long(id, params, &context).await.into());
|
||||
}
|
||||
}
|
||||
let tcc = rmcp::handler::server::tool::ToolCallContext::new(self, request, context);
|
||||
self.tool_router.call(tcc).await
|
||||
}
|
||||
|
||||
async fn list_resources(
|
||||
&self,
|
||||
_request: Option<rmcp::model::PaginatedRequestParams>,
|
||||
context: rmcp::service::RequestContext<rmcp::RoleServer>,
|
||||
) -> Result<rmcp::model::ListResourcesResult, McpError> {
|
||||
use rmcp::model::Resource;
|
||||
let mut r = rmcp::model::ListResourcesResult::with_all_items(vec![
|
||||
Resource::new(DOCUMENT_URI, "document")
|
||||
.with_title("Project")
|
||||
.with_description("The project tree and the active sequence (same as doc_inspect).")
|
||||
.with_mime_type("application/json"),
|
||||
Resource::new(COMMANDS_URI, "commands")
|
||||
.with_title("Command catalog")
|
||||
.with_description("Every command with id, label, menu path, shortcut, params and enabled state (same as command_list).")
|
||||
.with_mime_type("application/json"),
|
||||
]);
|
||||
if let Some((ttl, scope)) = cache_hints(&context, 600_000) {
|
||||
r = r.with_ttl_ms(ttl).with_cache_scope(scope);
|
||||
}
|
||||
Ok(r)
|
||||
}
|
||||
|
||||
async fn read_resource(
|
||||
&self,
|
||||
request: rmcp::model::ReadResourceRequestParams,
|
||||
context: rmcp::service::RequestContext<rmcp::RoleServer>,
|
||||
) -> Result<rmcp::model::ReadResourceResponse, McpError> {
|
||||
let v = match request.uri.as_str() {
|
||||
DOCUMENT_URI => self.document().await,
|
||||
COMMANDS_URI => self.run("command.list", json!({})).await,
|
||||
other => return Err(McpError::resource_not_found(format!("resource not found: {other}"), None)),
|
||||
}
|
||||
.map_err(|e| McpError::internal_error(e.to_string(), None))?;
|
||||
let text = serde_json::to_string_pretty(&v).map_err(|e| McpError::internal_error(e.to_string(), None))?;
|
||||
let mut r = rmcp::model::ReadResourceResult::new(vec![rmcp::model::ResourceContents::text(text, request.uri).with_mime_type("application/json")]);
|
||||
if let Some((ttl, scope)) = cache_hints(&context, 0) {
|
||||
r = r.with_ttl_ms(ttl).with_cache_scope(scope);
|
||||
}
|
||||
Ok(r.into())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
@@ -371,4 +644,249 @@ mod tests {
|
||||
let r = m.render_frame(Parameters(RenderParams { seconds: Some(1.0), max_side: Some(320) })).await.unwrap();
|
||||
assert_ne!(r.is_error, Some(true));
|
||||
}
|
||||
|
||||
/// One request/response exchange over `serve_io`, as an MCP client sees it.
|
||||
struct Client {
|
||||
tx: tokio::io::DuplexStream,
|
||||
rx: tokio::io::Lines<tokio::io::BufReader<tokio::io::DuplexStream>>,
|
||||
}
|
||||
|
||||
impl Client {
|
||||
fn start(s: Session) -> Self {
|
||||
use tokio::io::AsyncBufReadExt;
|
||||
let (tx, server_in) = tokio::io::duplex(1 << 22);
|
||||
let (server_out, rx) = tokio::io::duplex(1 << 22);
|
||||
tokio::spawn(FilmcraftMcp::headless(s).serve_io(server_in, server_out));
|
||||
Self { tx, rx: tokio::io::BufReader::new(rx).lines() }
|
||||
}
|
||||
async fn send(&mut self, line: &str) {
|
||||
use tokio::io::AsyncWriteExt;
|
||||
self.tx.write_all(format!("{line}\n").as_bytes()).await.unwrap();
|
||||
}
|
||||
async fn next(&mut self) -> Value {
|
||||
serde_json::from_str(&self.rx.next_line().await.unwrap().unwrap()).unwrap()
|
||||
}
|
||||
async fn ask(&mut self, msg: Value) -> Value {
|
||||
self.send(&msg.to_string()).await;
|
||||
self.next().await
|
||||
}
|
||||
async fn init(&mut self) -> Value {
|
||||
let r = self.ask(json!({"jsonrpc":"2.0","id":0,"method":"initialize","params":{"protocolVersion":"2025-06-18","capabilities":{},"clientInfo":{"name":"t","version":"1"}}})).await;
|
||||
self.send(&json!({"jsonrpc":"2.0","method":"notifications/initialized"}).to_string()).await;
|
||||
r
|
||||
}
|
||||
async fn call(&mut self, id: u64, name: &str, arguments: Value) -> Value {
|
||||
self.ask(json!({"jsonrpc":"2.0","id":id,"method":"tools/call","params":{"name":name,"arguments":arguments}})).await
|
||||
}
|
||||
}
|
||||
|
||||
fn demo() -> Session {
|
||||
let mut s = Session::default();
|
||||
s.execute("file.openDemoProject", json!({})).unwrap();
|
||||
s
|
||||
}
|
||||
|
||||
/// Every listed tool has a title and the four hints, and the core tools are listed.
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn tools_are_titled_and_annotated() {
|
||||
let mut c = Client::start(demo());
|
||||
let init = c.init().await;
|
||||
assert_eq!(init["result"]["serverInfo"]["name"], "filmcraft");
|
||||
let tools = c.ask(json!({"jsonrpc":"2.0","id":1,"method":"tools/list"})).await["result"]["tools"].as_array().cloned().unwrap();
|
||||
for t in &tools {
|
||||
assert!(t["title"].is_string(), "{} has no title", t["name"]);
|
||||
for h in ["readOnlyHint", "destructiveHint", "idempotentHint", "openWorldHint"] {
|
||||
assert!(t["annotations"][h].is_boolean(), "{} lacks {h}", t["name"]);
|
||||
}
|
||||
}
|
||||
let names: Vec<&str> = tools.iter().filter_map(|t| t["name"].as_str()).collect();
|
||||
for want in ["command_list", "command_run", "command_batch", "doc_inspect", "render_preview", "project_inspect", "render_frame", "ui_click"] {
|
||||
assert!(names.contains(&want), "{want} not listed");
|
||||
}
|
||||
let ro = |n: &str| tools.iter().find(|t| t["name"] == n).map(|t| t["annotations"]["readOnlyHint"].clone());
|
||||
assert_eq!((ro("command_list"), ro("command_run")), (Some(json!(true)), Some(json!(false))));
|
||||
}
|
||||
|
||||
/// Strict arguments, malformed JSON, batch, doc_inspect and render_preview over the raw line
|
||||
/// protocol.
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn strict_arguments_and_parse_errors() {
|
||||
let mut c = Client::start(demo());
|
||||
c.init().await;
|
||||
let r = c.call(1, "command_list", json!({"filter": "razor", "enabled_only": true})).await;
|
||||
assert!(r["result"]["content"][0]["text"].as_str().unwrap().contains("timeline.razor"), "{r}");
|
||||
// an unknown argument names itself and the accepted ones
|
||||
let r = c.call(2, "command_list", json!({"filtr": "x"})).await;
|
||||
assert_eq!(r["error"]["code"], -32602, "{r}");
|
||||
let m = r["error"]["message"].as_str().unwrap();
|
||||
assert!(m.contains("\"filtr\"") && m.contains("filter") && m.contains("enabled_only"), "{m}");
|
||||
let r = c.call(3, "doc_inspect", json!({"verbose": true})).await;
|
||||
assert_eq!(r["error"]["code"], -32602, "{r}");
|
||||
// command params stay free-form
|
||||
let r = c.call(4, "command_run", json!({"id": "playhead.set", "params": {"seconds": 0.5}})).await;
|
||||
assert_eq!(r["result"]["isError"], false, "{r}");
|
||||
// malformed JSON: a parse error, and the server keeps serving
|
||||
c.send("{\"jsonrpc\": \"2.0\", \"id\": 5, BROKEN").await;
|
||||
let r = c.next().await;
|
||||
assert_eq!((r["error"]["code"].as_i64(), r["id"].is_null()), (Some(-32700), true), "{r}");
|
||||
let r = c.ask(json!({"jsonrpc":"2.0","id":6,"method":"ping"})).await;
|
||||
assert!(r["result"].is_object(), "{r}");
|
||||
// batch: stops at the first failure and reports both
|
||||
let r =
|
||||
c.call(7, "command_batch", json!({"steps": [{"id": "playhead.set", "params": {"seconds": 0.5}}, {"id": "no.such"}, {"id": "playhead.set"}]})).await;
|
||||
assert_eq!(r["result"]["isError"], true, "{r}");
|
||||
let v: Value = serde_json::from_str(r["result"]["content"][0]["text"].as_str().unwrap()).unwrap();
|
||||
assert_eq!((v["completed"].as_u64(), v["failed"].as_u64(), v["results"].as_array().map(Vec::len)), (Some(1), Some(1), Some(2)), "{v}");
|
||||
let r = c.call(8, "doc_inspect", json!({})).await;
|
||||
let v: Value = serde_json::from_str(r["result"]["content"][0]["text"].as_str().unwrap()).unwrap();
|
||||
assert!(v["project"].is_object() && v["sequence"].is_object(), "{v}");
|
||||
let r = c.call(9, "render_preview", json!({"max_side": 64})).await;
|
||||
assert_eq!(r["result"]["content"][0]["type"], "image", "{r}");
|
||||
}
|
||||
|
||||
/// `filmcraft://document` and `filmcraft://commands` are listed and readable as JSON.
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn resources_document_and_commands() {
|
||||
let mut c = Client::start(demo());
|
||||
let init = c.init().await;
|
||||
assert!(init["result"]["capabilities"]["resources"].is_object(), "{init}");
|
||||
assert!(init["result"]["instructions"].as_str().unwrap().contains("doc_inspect"), "{init}");
|
||||
let r = c.ask(json!({"jsonrpc":"2.0","id":1,"method":"resources/list"})).await;
|
||||
let uris: Vec<&str> = r["result"]["resources"].as_array().unwrap().iter().filter_map(|x| x["uri"].as_str()).collect();
|
||||
assert_eq!(uris, [DOCUMENT_URI, COMMANDS_URI], "{r}");
|
||||
let r = c.ask(json!({"jsonrpc":"2.0","id":2,"method":"resources/read","params":{"uri":DOCUMENT_URI}})).await;
|
||||
assert_eq!(r["result"]["contents"][0]["mimeType"], "application/json", "{r}");
|
||||
let v: Value = serde_json::from_str(r["result"]["contents"][0]["text"].as_str().unwrap()).unwrap();
|
||||
assert!(v["project"].is_object() && v["sequence"].is_object(), "{v}");
|
||||
let r = c.ask(json!({"jsonrpc":"2.0","id":3,"method":"resources/read","params":{"uri":COMMANDS_URI}})).await;
|
||||
assert!(r["result"]["contents"][0]["text"].as_str().unwrap().contains("timeline.razor"), "{r}");
|
||||
let r = c.ask(json!({"jsonrpc":"2.0","id":4,"method":"resources/read","params":{"uri":"filmcraft://nope"}})).await;
|
||||
assert!(r["error"]["code"].is_i64(), "{r}");
|
||||
}
|
||||
|
||||
/// MCP 2026-07-28 clients (current Claude Code) negotiate per request and reject list and read
|
||||
/// results without `resultType`, `ttlMs` and `cacheScope`; sessions that negotiated an older
|
||||
/// revision through `initialize` get the old shape.
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn modern_clients_get_result_type_and_cache_hints() {
|
||||
let mut c = Client::start(Session::default());
|
||||
let meta = json!({
|
||||
"io.modelcontextprotocol/protocolVersion": "2026-07-28",
|
||||
"io.modelcontextprotocol/clientInfo": {"name": "t", "version": "1"},
|
||||
"io.modelcontextprotocol/clientCapabilities": {},
|
||||
});
|
||||
let r = c.ask(json!({"jsonrpc":"2.0","id":1,"method":"server/discover","params":{"_meta":meta}})).await;
|
||||
assert!(r["result"]["supportedVersions"].as_array().unwrap().contains(&json!("2026-07-28")), "{r}");
|
||||
for (id, method, params) in [
|
||||
(2, "tools/list", json!({"_meta": meta})),
|
||||
(3, "resources/list", json!({"_meta": meta})),
|
||||
(4, "resources/read", json!({"_meta": meta, "uri": DOCUMENT_URI})),
|
||||
] {
|
||||
let r = c.ask(json!({"jsonrpc":"2.0","id":id,"method":method,"params":params})).await;
|
||||
assert_eq!(r["result"]["resultType"], "complete", "{method}: {r}");
|
||||
assert!(r["result"]["ttlMs"].is_u64(), "{method}: {r}");
|
||||
assert!(matches!(r["result"]["cacheScope"].as_str(), Some("public" | "private")), "{method}: {r}");
|
||||
}
|
||||
let mut c = Client::start(Session::default());
|
||||
c.init().await;
|
||||
for (id, method, params) in [(2, "tools/list", json!({})), (3, "resources/list", json!({})), (4, "resources/read", json!({"uri": DOCUMENT_URI}))] {
|
||||
let r = c.ask(json!({"jsonrpc":"2.0","id":id,"method":method,"params":params})).await;
|
||||
assert!(r["result"].is_object() && r["result"].get("resultType").is_none() && r["result"].get("ttlMs").is_none(), "{method}: {r}");
|
||||
}
|
||||
}
|
||||
|
||||
/// docs/agents.md § Long exports: a blocking export reports `notifications/progress` for its
|
||||
/// token (none without one), answers `ping` meanwhile, and `notifications/cancelled` stops
|
||||
/// it, deletes the partial file and sends no response.
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn export_progress_and_cancel() {
|
||||
let dir = std::env::temp_dir().join(format!("filmcraft-mcp-progress-{}", std::process::id()));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let mut c = Client::start(demo());
|
||||
c.init().await;
|
||||
let export = |id: u64, name: &str, token: Option<Value>, seconds: f64| {
|
||||
let path = dir.join(name).to_string_lossy().to_string();
|
||||
let params = json!({"path": path, "format": "h264", "width": 640, "height": 360, "audio": false, "range": "custom", "startSeconds": 0, "endSeconds": seconds, "wait": true});
|
||||
let mut call =
|
||||
json!({"jsonrpc":"2.0","id":id,"method":"tools/call","params":{"name":"command_run","arguments":{"id":"file.exportMedia","params":params}}});
|
||||
if let Some(t) = token {
|
||||
call["params"]["_meta"] = json!({"progressToken": t});
|
||||
}
|
||||
call.to_string()
|
||||
};
|
||||
// no token: no notifications, the same response as before
|
||||
c.send(&export(1, "quiet.mp4", None, 0.5)).await;
|
||||
let m = c.next().await;
|
||||
assert_eq!((m["id"].as_u64(), &m["result"]["isError"]), (Some(1), &json!(false)), "{m}");
|
||||
// progress, strictly increasing, with a total; ping answered while the export runs
|
||||
c.send(&export(2, "a.mp4", Some(json!("tok")), 1.0)).await;
|
||||
let (mut last, mut notes, mut pinged, mut pong) = (-1.0, 0, false, false);
|
||||
let done = loop {
|
||||
let m = c.next().await;
|
||||
if m["method"] == "notifications/progress" {
|
||||
assert_eq!(m["params"]["progressToken"], "tok", "{m}");
|
||||
let p = m["params"]["progress"].as_f64().unwrap();
|
||||
assert!(p > last && m["params"]["total"].as_f64().unwrap() >= p, "{m}");
|
||||
last = p;
|
||||
notes += 1;
|
||||
if !pinged {
|
||||
pinged = true;
|
||||
c.send(r#"{"jsonrpc":"2.0","id":3,"method":"ping"}"#).await;
|
||||
}
|
||||
} else if m["id"] == 3 {
|
||||
pong = true;
|
||||
} else if m["id"] == 2 {
|
||||
break m;
|
||||
}
|
||||
};
|
||||
assert_eq!(done["result"]["isError"], false, "{done}");
|
||||
assert!(notes >= 2 && pong, "{notes} notifications, ping answered during the export: {pong}");
|
||||
assert!(dir.join("a.mp4").exists());
|
||||
// cancel after the first notification: no response, no partial file
|
||||
c.send(&export(4, "b.mp4", Some(json!(7)), 8.0)).await;
|
||||
loop {
|
||||
let m = c.next().await;
|
||||
assert_ne!(m["id"], 4, "finished before the cancel: {m}");
|
||||
if m["method"] == "notifications/progress" {
|
||||
break;
|
||||
}
|
||||
}
|
||||
c.send(r#"{"jsonrpc":"2.0","method":"notifications/cancelled","params":{"requestId":4}}"#).await;
|
||||
let t0 = std::time::Instant::now();
|
||||
while dir.join("b.mp4").exists() || t0.elapsed() < std::time::Duration::from_millis(500) {
|
||||
assert!(t0.elapsed().as_secs() < 30, "partial file still there");
|
||||
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
|
||||
}
|
||||
c.send(r#"{"jsonrpc":"2.0","id":5,"method":"tools/call","params":{"name":"command_run","arguments":{"id":"jobs.list"}}}"#).await;
|
||||
let m = loop {
|
||||
let m = c.next().await;
|
||||
assert_ne!(m["id"], 4, "no response for a cancelled request: {m}");
|
||||
if m["id"] == 5 {
|
||||
break m;
|
||||
}
|
||||
};
|
||||
let jobs: Value = serde_json::from_str(m["result"]["content"][0]["text"].as_str().unwrap()).unwrap();
|
||||
let last = jobs.as_array().and_then(|a| a.last()).cloned().unwrap_or_default();
|
||||
assert_eq!(last["result"]["error"], "cancelled", "{jobs}");
|
||||
assert!(!dir.join("b.mp4").exists());
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
/// A command that panicked used to poison the session lock, and every later call failed with
|
||||
/// "session lock poisoned" until the server was restarted.
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn a_poisoned_session_lock_keeps_serving() {
|
||||
let m = FilmcraftMcp::headless(demo());
|
||||
let Backend::Headless(s) = &*m.backend else { panic!("headless backend") };
|
||||
let s = s.clone();
|
||||
let _ = std::thread::spawn(move || {
|
||||
let _g = s.lock().unwrap();
|
||||
panic!("a command panicked");
|
||||
})
|
||||
.join();
|
||||
let v = m.run("sequence.inspect", json!({})).await.unwrap();
|
||||
assert!(v.is_object(), "{v}");
|
||||
let r = m.render_frame(Parameters(RenderParams { seconds: Some(0.5), max_side: Some(64) })).await.unwrap();
|
||||
assert_ne!(r.is_error, Some(true));
|
||||
}
|
||||
}
|
||||
+70
-15
@@ -184,28 +184,45 @@ struct Timing {
|
||||
tick_rate: i64,
|
||||
}
|
||||
|
||||
fn positive_integer(s: &str) -> Option<i64> {
|
||||
let s = s.trim();
|
||||
if s.is_empty() || !s.bytes().all(|b| b.is_ascii_digit()) {
|
||||
return None;
|
||||
}
|
||||
s.parse::<i64>().ok().filter(|v| *v > 0)
|
||||
}
|
||||
|
||||
fn parse_ratio(s: &str) -> Option<(i64, i64)> {
|
||||
let mut it = s.split_whitespace().map(|x| x.parse::<i64>().ok());
|
||||
let mut it = s.split_whitespace().map(positive_integer);
|
||||
let a = it.next()??;
|
||||
let b = it.next().flatten().unwrap_or(1);
|
||||
(a > 0 && b > 0).then_some((a, b))
|
||||
let b = it.next()??;
|
||||
it.next().is_none().then_some((a, b))
|
||||
}
|
||||
|
||||
fn attr<'a>(n: roxmltree::Node<'a, '_>, local: &str) -> Option<&'a str> {
|
||||
n.attributes().find(|a| a.name() == local).map(|a| a.value())
|
||||
}
|
||||
|
||||
fn timing(root: roxmltree::Node) -> Timing {
|
||||
let base = attr(root, "frameRate").and_then(|v| v.trim().parse::<i64>().ok()).filter(|v| *v > 0).unwrap_or(30);
|
||||
let (mn, md) = attr(root, "frameRateMultiplier").and_then(parse_ratio).unwrap_or((1, 1));
|
||||
let rate = FrameRate::new(base * mn, md);
|
||||
let sub = attr(root, "subFrameRate").and_then(|v| v.trim().parse::<i64>().ok()).filter(|v| *v > 0).unwrap_or(1);
|
||||
let tick_rate = attr(root, "tickRate").and_then(|v| v.trim().parse::<i64>().ok()).filter(|v| *v > 0).unwrap_or(if attr(root, "frameRate").is_some() {
|
||||
base * sub
|
||||
} else {
|
||||
1
|
||||
});
|
||||
Timing { rate, base, sub_frame_rate: sub, tick_rate }
|
||||
fn timing(root: roxmltree::Node) -> Result<Timing> {
|
||||
let invalid =
|
||||
|name: &str| Error::Syntax { line: root.document().text_pos_at(root.range().start).row as usize, msg: format!("invalid TTML timing parameter {name}") };
|
||||
let integer = |name: &str, default| match attr(root, name) {
|
||||
Some(v) => positive_integer(v).ok_or_else(|| invalid(name)),
|
||||
None => Ok(default),
|
||||
};
|
||||
let base = integer("frameRate", 30)?;
|
||||
let (mn, md) = match attr(root, "frameRateMultiplier") {
|
||||
Some(v) => parse_ratio(v).ok_or_else(|| invalid("frameRateMultiplier"))?,
|
||||
None => (1, 1),
|
||||
};
|
||||
let rate = FrameRate::new(base.checked_mul(mn).ok_or_else(|| invalid("frameRateMultiplier"))?, md);
|
||||
let sub = integer("subFrameRate", 1)?;
|
||||
let tick_rate = match attr(root, "tickRate") {
|
||||
Some(v) => positive_integer(v).ok_or_else(|| invalid("tickRate"))?,
|
||||
None if attr(root, "frameRate").is_some() => base.checked_mul(sub).ok_or_else(|| invalid("subFrameRate"))?,
|
||||
None => 1,
|
||||
};
|
||||
Ok(Timing { rate, base, sub_frame_rate: sub, tick_rate })
|
||||
}
|
||||
|
||||
/// Seconds as an exact decimal string → ticks.
|
||||
@@ -383,7 +400,7 @@ pub fn parse(text: &str) -> Result<Document> {
|
||||
if root.tag_name().name() != "tt" {
|
||||
return Err(Error::NotFormat("TTML"));
|
||||
}
|
||||
let tm = timing(root);
|
||||
let tm = timing(root)?;
|
||||
let mut styles = std::collections::HashMap::new();
|
||||
for s in root.descendants().filter(|n| n.is_element() && n.tag_name().name() == "style") {
|
||||
if let Some(id) = s.attributes().find(|a| a.name() == "id").map(|a| a.value()) {
|
||||
@@ -478,6 +495,44 @@ mod tests {
|
||||
Timing { rate, base: rate.timecode_base(), sub_frame_rate: 1, tick_rate: 1 }
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_timing_parameters_return_an_error() {
|
||||
for params in [
|
||||
r#"ttp:frameRate="0""#,
|
||||
r#"ttp:frameRate="-25""#,
|
||||
r#"ttp:frameRate="""#,
|
||||
r#"ttp:subFrameRate="0""#,
|
||||
r#"ttp:tickRate="-1""#,
|
||||
r#"ttp:frameRateMultiplier="1 0""#,
|
||||
r#"ttp:frameRateMultiplier="1""#,
|
||||
r#"ttp:frameRateMultiplier="1 2 3""#,
|
||||
] {
|
||||
let xml = format!(
|
||||
r#"<tt xmlns="http://www.w3.org/ns/ttml" xmlns:ttp="http://www.w3.org/ns/ttml#parameter" {params}><body><div><p begin="1s" end="2s">Hello</p></div></body></tt>"#
|
||||
);
|
||||
assert!(crate::parse(xml.as_bytes(), crate::Format::Ttml).is_err(), "{params}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_timing_parameters_keep_defaults() {
|
||||
for (params, begin, end) in [
|
||||
("", "1s", "2s"),
|
||||
(r#"ttp:frameRate="25""#, "25f", "50f"),
|
||||
(r#"ttp:frameRate="30" ttp:subFrameRate="2""#, "60t", "120t"),
|
||||
(r#"ttp:frameRate="30" ttp:subFrameRate="2" ttp:tickRate="10""#, "10t", "20t"),
|
||||
(r#"ttp:frameRate="25" ttp:frameRateMultiplier="2 1""#, "50f", "100f"),
|
||||
] {
|
||||
let xml = format!(
|
||||
r#"<tt xmlns="http://www.w3.org/ns/ttml" xmlns:ttp="http://www.w3.org/ns/ttml#parameter" {params}><body><div><p begin="{begin}" end="{end}">Hello</p></div></body></tt>"#
|
||||
);
|
||||
let doc = crate::parse(xml.as_bytes(), crate::Format::Ttml).unwrap();
|
||||
assert_eq!(doc.cues.len(), 1, "{params}");
|
||||
assert_eq!(doc.cues[0].start, Tick(TICKS_PER_SECOND), "{params}");
|
||||
assert_eq!(doc.cues[0].end, Tick(2 * TICKS_PER_SECOND), "{params}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn time_expressions() {
|
||||
let t = tm(FrameRate::FPS_25);
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
//! An export depends only on the project and the settings: the same file on every machine, however
|
||||
//! many cores render it and however the work is cut into steps.
|
||||
|
||||
use super::*;
|
||||
use crate::tests::project;
|
||||
|
||||
fn tmp(name: &str) -> String {
|
||||
let d = std::env::temp_dir().join(format!("fc-export-det-{}", std::process::id()));
|
||||
std::fs::create_dir_all(&d).unwrap();
|
||||
d.join(name).to_string_lossy().to_string()
|
||||
}
|
||||
|
||||
/// Export on a machine with `threads` cores: `export` renders one frame per core of the pool it
|
||||
/// runs in.
|
||||
fn export_with_cores(threads: usize, settings: &ExportSettings) -> Vec<u8> {
|
||||
let (p, seq, m) = project();
|
||||
let pool = rayon::ThreadPoolBuilder::new().num_threads(threads).build().unwrap();
|
||||
pool.install(|| export(&p, seq, settings, &m, &Progress::default())).unwrap();
|
||||
std::fs::read(&settings.path).unwrap()
|
||||
}
|
||||
|
||||
/// Export a step at a time with `batch` frames per step (the web app uses 1).
|
||||
fn export_stepped(batch: i64, settings: &ExportSettings) -> Vec<u8> {
|
||||
let (p, seq, m) = project();
|
||||
let prog = Progress::default();
|
||||
let mut ex = Exporter::new(p, seq, settings, &prog).unwrap();
|
||||
ex.set_batch(batch);
|
||||
while !matches!(ex.step(&m, &prog).unwrap(), Step::Done(_)) {}
|
||||
std::fs::read(&settings.path).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn same_file_whatever_the_core_count() {
|
||||
// 24 frames: one full interleave group and a partial one
|
||||
for (format, ext) in [(Format::ProRes, "mov"), (Format::H264, "mp4"), (Format::Mjpeg, "mov")] {
|
||||
let settings = |name: &str| ExportSettings { format, path: tmp(&format!("{name}.{ext}")), ..Default::default() };
|
||||
let reference = export_with_cores(2, &settings(&format!("{}-2", format.id())));
|
||||
for threads in [3, 5, 16] {
|
||||
let other = export_with_cores(threads, &settings(&format!("{}-{threads}", format.id())));
|
||||
assert!(reference == other, "{}: {threads} cores give a different file than 2 cores", format.label());
|
||||
}
|
||||
for batch in [1, 7] {
|
||||
let other = export_stepped(batch, &settings(&format!("{}-b{batch}", format.id())));
|
||||
assert!(reference == other, "{}: {batch} frame(s) per step give a different file", format.label());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Slice NAL units (types 1 and 5) in a length-prefixed (4-byte) H.264 sample.
|
||||
fn slices_in(sample: &[u8]) -> usize {
|
||||
let (mut n, mut i) = (0, 0);
|
||||
while i + 4 <= sample.len() {
|
||||
let len = u32::from_be_bytes([sample[i], sample[i + 1], sample[i + 2], sample[i + 3]]) as usize;
|
||||
if let Some(&h) = sample.get(i + 4)
|
||||
&& matches!(h & 0x1f, 1 | 5)
|
||||
{
|
||||
n += 1;
|
||||
}
|
||||
i += 4 + len;
|
||||
}
|
||||
n
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn h264_slices_follow_the_frame_size_not_the_cores() {
|
||||
// 4096 lines = 256 macroblock rows: one slice per four rows on every machine, also on machines
|
||||
// with fewer than 64 cores (the encoder's default caps the slices at the core count)
|
||||
let path = tmp("tall.mp4");
|
||||
let s = ExportSettings { format: Format::H264, path: path.clone(), frame_size: Some((64, 4096)), include_audio: false, ..Default::default() };
|
||||
let (p, seq, m) = project();
|
||||
export(&p, seq, &s, &m, &Progress::default()).unwrap();
|
||||
let bytes = std::fs::read(&path).unwrap();
|
||||
let file = filmcraft_isobmff::open(bytes.as_slice()).unwrap();
|
||||
let vt = file.track_of_kind(filmcraft_isobmff::TrackKind::Video).unwrap();
|
||||
assert_eq!(file.tracks[vt].samples.len(), 24);
|
||||
for i in 0..file.tracks[vt].samples.len() {
|
||||
let sample = file.read_sample(bytes.as_slice(), vt, i).unwrap();
|
||||
assert_eq!(slices_in(&sample), 64, "picture {i}");
|
||||
}
|
||||
}
|
||||
+47
-18
@@ -11,6 +11,10 @@
|
||||
//! When a source's bytes are not available yet (asynchronous web reads mark
|
||||
//! [`filmcraft_media::pending`]), the batch is rendered again on the next call: nothing is encoded
|
||||
//! from frames or audio with missing media.
|
||||
//!
|
||||
//! The file does not depend on how many frames a step renders (one per core natively, one on the
|
||||
//! web): packets go to the muxer in fixed groups of [`INTERLEAVE`] output frames, each followed by
|
||||
//! the audio up to its end, so the same project and settings give the same bytes on any machine.
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::Ordering;
|
||||
@@ -30,6 +34,11 @@ use crate::{
|
||||
audio_factories, export_range, frame_span, video_factories,
|
||||
};
|
||||
|
||||
/// Output frames per interleaved group: the video packets of these frames, then the audio up to
|
||||
/// their end, form one chunk of each track. Fixed (not the batch size, which follows the core
|
||||
/// count) so the file layout is the same on every machine.
|
||||
pub const INTERLEAVE: i64 = 16;
|
||||
|
||||
/// What one [`Exporter::step`] did.
|
||||
#[derive(Debug)]
|
||||
pub enum Step {
|
||||
@@ -58,7 +67,9 @@ pub struct Exporter {
|
||||
measured: bool,
|
||||
/// The first pass of a two-pass encode is running.
|
||||
first_pass: bool,
|
||||
/// Created after the first batch (encoders may finalise their codec config then).
|
||||
/// Encoded video packets not written yet (the current [`INTERLEAVE`] group).
|
||||
queued: Vec<EncodedPacket>,
|
||||
/// Created after the first group was encoded (encoders may finalise their codec config then).
|
||||
mux: Option<Mp4Writer<Out>>,
|
||||
/// MXF exports write through this instead of `mux`.
|
||||
mxf: Option<MxfMux>,
|
||||
@@ -140,6 +151,7 @@ impl Exporter {
|
||||
audio,
|
||||
measured,
|
||||
first_pass: two_pass,
|
||||
queued: Vec::new(),
|
||||
mux: None,
|
||||
mxf: None,
|
||||
seq,
|
||||
@@ -177,7 +189,7 @@ impl Exporter {
|
||||
if n == 0 {
|
||||
return Ok(());
|
||||
}
|
||||
let mux = self.mux.as_mut().expect("mux created");
|
||||
let mux = self.mux.as_mut().ok_or_else(|| ExportError::Encode("internal: the container writer was not created".into()))?;
|
||||
match self.aenc.as_mut() {
|
||||
Some(a) => {
|
||||
for au in a.encode(&buf)? {
|
||||
@@ -209,7 +221,7 @@ impl Exporter {
|
||||
if let Some(m) = self.mxf.as_mut() {
|
||||
return m.write_video(packets);
|
||||
}
|
||||
let mux = self.mux.as_mut().expect("mux created");
|
||||
let mux = self.mux.as_mut().ok_or_else(|| ExportError::Encode("internal: the container writer was not created".into()))?;
|
||||
for p in packets {
|
||||
mux.write_sample(self.vt, WriteSample { data: &p.data, duration: p.duration, composition_offset: p.composition_offset, is_sync: p.key })
|
||||
.map_err(|e| ExportError::Io(e.to_string()))?;
|
||||
@@ -270,7 +282,7 @@ impl Exporter {
|
||||
if !self.settings.part_of_batch {
|
||||
progress.set_status("Measuring loudness");
|
||||
}
|
||||
let a = self.audio.as_mut().expect("audio");
|
||||
let a = self.audio.as_mut().ok_or_else(|| ExportError::Encode("internal: the audio pipeline was not created".into()))?;
|
||||
a.measure(&self.settings, sources, &|| progress.cancel.load(Ordering::Relaxed))?;
|
||||
if filmcraft_media::pending::take() {
|
||||
return Ok(Step::Pending);
|
||||
@@ -295,22 +307,33 @@ impl Exporter {
|
||||
self.next = end;
|
||||
return Ok(Step::Progress);
|
||||
}
|
||||
let mixed = match self.audio.as_ref().map(|a| a.sr) {
|
||||
Some(sr) if (self.mux.is_none() && self.mxf.is_none()) || self.at.is_some() || self.mxf.is_some() => {
|
||||
let until = self.sample_at_frame(end, sr);
|
||||
self.audio.as_mut().and_then(|a| a.pull(until, sources))
|
||||
// the audio of every interleave group that ends in this batch (before encoding, so a
|
||||
// pending source can still make the batch run again)
|
||||
let cuts: Vec<i64> = (f + 1..=end).filter(|&g| (g - self.f0) % INTERLEAVE == 0).collect();
|
||||
let mut mixed = Vec::with_capacity(cuts.len());
|
||||
if let Some(sr) = self.audio.as_ref().map(|a| a.sr) {
|
||||
for &g in &cuts {
|
||||
let until = self.sample_at_frame(g, sr);
|
||||
mixed.push(self.audio.as_mut().and_then(|a| a.pull(until, sources)));
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
}
|
||||
if filmcraft_media::pending::take() {
|
||||
return Ok(Step::Pending);
|
||||
}
|
||||
let packets = self.encode(&frames, f)?;
|
||||
if self.mux.is_none() && self.mxf.is_none() {
|
||||
self.open_mux()?;
|
||||
let mut mixed = mixed.into_iter();
|
||||
for (k, frame) in frames.iter().enumerate() {
|
||||
let fi = f + k as i64;
|
||||
let packets = self.encode(std::slice::from_ref(frame), fi)?;
|
||||
self.queued.extend(packets);
|
||||
if cuts.contains(&(fi + 1)) {
|
||||
if self.mux.is_none() && self.mxf.is_none() {
|
||||
self.open_mux()?;
|
||||
}
|
||||
let packets = std::mem::take(&mut self.queued);
|
||||
self.write_video(packets)?;
|
||||
self.write_audio(mixed.next().flatten())?;
|
||||
}
|
||||
}
|
||||
self.write_video(packets)?;
|
||||
self.write_audio(mixed)?;
|
||||
progress.done.fetch_add((end - f) as u64, Ordering::Relaxed);
|
||||
self.next = end;
|
||||
return Ok(Step::Progress);
|
||||
@@ -332,12 +355,13 @@ impl Exporter {
|
||||
if self.mux.is_none() && self.mxf.is_none() {
|
||||
self.open_mux()?;
|
||||
}
|
||||
let tail = self.venc.flush()?;
|
||||
let mut tail = std::mem::take(&mut self.queued);
|
||||
tail.extend(self.venc.flush()?);
|
||||
self.write_video(tail)?;
|
||||
self.write_audio(mixed)?;
|
||||
if let (Some(at), Some(a)) = (self.at, self.aenc.as_mut()) {
|
||||
let fs = a.frame_size();
|
||||
let mux = self.mux.as_mut().expect("mux created");
|
||||
let mux = self.mux.as_mut().ok_or_else(|| ExportError::Encode("internal: the container writer was not created".into()))?;
|
||||
for au in a.flush()? {
|
||||
mux.write_sample(at, WriteSample { data: &au, duration: fs, composition_offset: 0, is_sync: true })
|
||||
.map_err(|e| ExportError::Io(e.to_string()))?;
|
||||
@@ -346,7 +370,12 @@ impl Exporter {
|
||||
let (bytes, extra_files) = match self.mxf.take() {
|
||||
Some(m) => m.finish(&self.settings)?,
|
||||
None => {
|
||||
let w = self.mux.take().expect("mux created").finish().map_err(|e| ExportError::Io(e.to_string()))?;
|
||||
let w = self
|
||||
.mux
|
||||
.take()
|
||||
.ok_or_else(|| ExportError::Encode("internal: the container writer was not created".into()))?
|
||||
.finish()
|
||||
.map_err(|e| ExportError::Io(e.to_string()))?;
|
||||
(w.finish(&self.settings)?, Vec::new())
|
||||
}
|
||||
};
|
||||
|
||||
@@ -11,6 +11,7 @@
|
||||
//! [`ExportSettings`] carries every Export-mode setting (frame size, rate, bitrate encoding, audio
|
||||
//! format, multiplexer, captions, effects, metadata) as serde data; [`presets`] defines the
|
||||
//! built-in presets.
|
||||
#![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable))]
|
||||
|
||||
use std::io::Write;
|
||||
|
||||
@@ -916,6 +917,13 @@ pub fn rgbf_to_yuv420_8(rgb: &[f32], w: usize, h: usize, kr: f32, kb: f32, y: &m
|
||||
});
|
||||
}
|
||||
|
||||
/// Slices per H.264 picture: one per four macroblock rows. The encoder's default follows the core
|
||||
/// count, which would make the stream (and every decoded picture) depend on the machine; this is
|
||||
/// what it picks on a machine with enough cores.
|
||||
pub(crate) fn h264_slices(height: u32) -> usize {
|
||||
(height.div_ceil(16) as usize).div_ceil(4).max(1)
|
||||
}
|
||||
|
||||
fn h264_factory(format: Format, w: u32, h: u32, rate: FrameRate, s: &ExportSettings) -> Option<Result<Box<dyn VideoEncoder>>> {
|
||||
if format != Format::H264 {
|
||||
return None;
|
||||
@@ -925,6 +933,7 @@ fn h264_factory(format: Format, w: u32, h: u32, rate: FrameRate, s: &ExportSetti
|
||||
cfg.format = if s.format.is_mxf() { filmcraft_h264enc::PacketFormat::AnnexB } else { filmcraft_h264enc::PacketFormat::LengthPrefixed };
|
||||
cfg.aud = s.format.is_mxf();
|
||||
cfg.keyint = s.keyframe_distance.filter(|k| *k > 0).unwrap_or_else(|| (rate.num as f64 / rate.den as f64 * 2.0).round().max(1.0) as u32);
|
||||
cfg.slices = h264_slices(h);
|
||||
let kbps = s.bitrate_kbps.max(100);
|
||||
let max = s.max_bitrate_kbps.filter(|m| *m >= kbps).unwrap_or(kbps * 3 / 2);
|
||||
cfg.rate = match s.bitrate_mode {
|
||||
@@ -1108,7 +1117,10 @@ pub fn export(project: &Arc<Project>, seq: ItemId, settings: &ExportSettings, so
|
||||
}
|
||||
(total, count)
|
||||
}
|
||||
Format::H264 | Format::ProRes | Format::DnxHr | Format::Mjpeg | Format::MxfOp1a | Format::MxfOpAtom => unreachable!("stepped export"),
|
||||
Format::H264 | Format::ProRes | Format::DnxHr | Format::Mjpeg | Format::MxfOp1a | Format::MxfOpAtom => {
|
||||
// Handled by the stepped exporter above; reaching here would be a dispatch bug.
|
||||
return Err(ExportError::Unsupported(format!("{:?} must run as a stepped export", settings.format)));
|
||||
}
|
||||
};
|
||||
let secs = t0.elapsed().as_secs_f64();
|
||||
if !settings.part_of_batch {
|
||||
@@ -1129,3 +1141,6 @@ mod surround_tests;
|
||||
|
||||
#[cfg(test)]
|
||||
mod mxf_tests;
|
||||
|
||||
#[cfg(test)]
|
||||
mod determinism_tests;
|
||||
@@ -781,7 +781,7 @@ fn gpu_matches_cpu_on_golden_scenes() {
|
||||
/// GPU rasterises, and with blend modes those edge pixels are a visible share of a 320×180 frame.
|
||||
fn interior_diff(plan: &filmcraft_render::plan::FramePlan, a: &Rgba8, b: &Rgba8) -> (u32, f64) {
|
||||
let filmcraft_render::plan::FramePlan::Layers { width, layers, .. } = plan else { return (u32::MAX, f64::MAX) };
|
||||
let inv: Vec<_> = layers.iter().filter_map(|l| Some((l.matrix.inverse()?, l.frame.width as f64, l.frame.height as f64))).collect();
|
||||
let inv: Vec<_> = layers.iter().filter_map(|l| Some((l.matrix.inverse()?, l.size().0 as f64, l.size().1 as f64))).collect();
|
||||
let mut d: Vec<u32> = Vec::new();
|
||||
for (i, (pa, pb)) in a.px.chunks(4).zip(b.px.chunks(4)).enumerate() {
|
||||
let (x, y) = ((i % width) as f64 + 0.5, (i / width) as f64 + 0.5);
|
||||
@@ -880,3 +880,96 @@ fn gpu_transition_of_blended_clips_matches_cpu_render() {
|
||||
eprintln!("Screen clips in a cross dissolve: GPU vs CPU {d}");
|
||||
assert!(d.p99 <= 6 && d.mean_abs * 4.0 / 3.0 < 1.5, "{d}");
|
||||
}
|
||||
|
||||
/// Standard effect chains run on the GPU: every clip below plans as a GPU layer with its effects
|
||||
/// (no CPU-rendered image), including keyframed parameters, transformed clips, partial opacity
|
||||
/// and blend modes, and matches the CPU render with the GPU parity tolerance (p99 ≤ 6, mean < 1.5
|
||||
/// away from layer outlines) at three times. A clip with an effect the GPU stage lacks (Emboss)
|
||||
/// still falls back to a CPU layer image.
|
||||
#[test]
|
||||
fn gpu_effect_chains_match_cpu_render() {
|
||||
let mut b = Builder::new(5);
|
||||
let (bg, dunes, aurora, bars, sunset) =
|
||||
(b.demo(DemoScene::OceanSunset), b.demo(DemoScene::Dunes), b.demo(DemoScene::Aurora), b.media(Generator::BarsAndTone), b.demo(DemoScene::OceanSunset));
|
||||
let v1 = b.place(0, bg, 0, 48);
|
||||
b.effect(v1, "brightness_contrast", &[("brightness", fl(-10.0)), ("contrast", fl(20.0))]);
|
||||
b.effect(v1, "gaussian_blur", &[("repeat_edge", ParamValue::Bool(true))]);
|
||||
// keyframed blur: evaluated at the frame's time
|
||||
let p = b.clip(v1).effects.last_mut().and_then(|e| e.params.get_mut("blurriness")).expect("blurriness");
|
||||
p.put_keyframe(Tick::ZERO, fl(2.0));
|
||||
p.put_keyframe(RATE.tick_of(24), fl(20.0));
|
||||
let v2 = b.place(1, dunes, 0, 48);
|
||||
b.fixed(v2, "motion", &[("scale", fl(60.0)), ("rotation", fl(12.0)), ("position", pt(130.0, 80.0))]);
|
||||
b.fixed(v2, "opacity", &[("opacity", fl(70.0)), ("blend", blend("Screen"))]);
|
||||
b.effect(v2, "tint", &[("amount", fl(60.0))]);
|
||||
b.effect(v2, "unsharp_mask", &[("amount", fl(120.0)), ("radius", fl(2.0))]);
|
||||
b.effect(v2, "crop", &[("left", fl(8.0)), ("bottom", fl(12.0)), ("feather", fl(6.0))]);
|
||||
let v3 = b.place(2, aurora, 0, 48);
|
||||
b.fixed(v3, "motion", &[("scale", fl(45.0)), ("rotation", fl(-20.0)), ("position", pt(210.0, 110.0))]);
|
||||
b.fixed(v3, "opacity", &[("opacity", fl(85.0))]);
|
||||
b.effect(v3, "levels", &[("in_black", fl(20.0)), ("gamma", fl(130.0))]);
|
||||
b.effect(v3, "mirror", &[("angle", fl(60.0))]);
|
||||
b.effect(v3, "directional_blur", &[("length", fl(4.0)), ("direction", fl(45.0))]);
|
||||
b.effect(v3, "transform", &[("rotation", fl(10.0)), ("scale_height", fl(110.0))]);
|
||||
let v4 = b.place(3, bars, 0, 48);
|
||||
b.fixed(v4, "motion", &[("scale", fl(30.0)), ("position", pt(250.0, 50.0))]);
|
||||
b.fixed(v4, "opacity", &[("blend", blend("Multiply"))]);
|
||||
b.effect(v4, "posterize", &[("levels", fl(5.0))]);
|
||||
b.effect(v4, "invert", &[("blend", fl(40.0))]);
|
||||
b.effect(v4, "offset", &[("shift", pt(400.0, 120.0))]);
|
||||
b.effect(v4, "horizontal_flip", &[]);
|
||||
b.effect(v4, "black_white", &[]);
|
||||
let v5 = b.place(4, sunset, 0, 48);
|
||||
b.fixed(v5, "motion", &[("scale", fl(25.0)), ("position", pt(60.0, 140.0))]);
|
||||
b.effect(v5, "emboss", &[]);
|
||||
let mut s = b.at(12);
|
||||
let plan = filmcraft_render::plan::plan_frame(&s.project, s.seq, RATE.tick_of(s.frame), RenderOptions::default(), &s.sources);
|
||||
let filmcraft_render::plan::FramePlan::Layers { layers, .. } = &plan else { panic!("planned as a CPU image") };
|
||||
assert_eq!(layers.len(), 5);
|
||||
for (i, l) in layers.iter().take(4).enumerate() {
|
||||
let fx = l.fx.as_ref().unwrap_or_else(|| panic!("V{}: not a GPU effect layer", i + 1));
|
||||
assert!(!fx.ops.is_empty());
|
||||
}
|
||||
assert!(layers[4].fx.is_none(), "Emboss is rendered on the CPU");
|
||||
let Some((dev, q)) = device() else {
|
||||
eprintln!("SKIPPED (gpu effect parity): no GPU adapter");
|
||||
return;
|
||||
};
|
||||
let mut c = filmcraft_gpu::GpuCompositor::new(&dev, &q);
|
||||
let mut failures = Vec::new();
|
||||
for frame in [0, 12, 30] {
|
||||
s.frame = frame;
|
||||
let plan = filmcraft_render::plan::plan_frame(&s.project, s.seq, RATE.tick_of(frame), RenderOptions::default(), &s.sources);
|
||||
c.composite(&plan);
|
||||
let (gw, gh, mut px) = c.read_output().expect("GPU readback");
|
||||
px.as_chunks_mut::<4>().0.iter_mut().for_each(|p| p[3] = 255);
|
||||
let gpu = Rgba8::new(gw, gh, px);
|
||||
let cpu = render_cpu(&s);
|
||||
let d = diff(&cpu, &gpu).unwrap();
|
||||
let (p99, mean) = interior_diff(&plan, &cpu, &gpu);
|
||||
eprintln!("effect chains @ {frame}: GPU vs CPU interior p99 {p99}, mean {mean:.3}; whole frame {d}");
|
||||
if p99 > 6 || mean >= 1.5 {
|
||||
let dir = filmcraft_testkit::golden::failures_dir();
|
||||
let _ = filmcraft_testkit::golden::write_png(&dir.join(format!("fx_chains_{frame}.gpu.png")), &gpu);
|
||||
let _ = filmcraft_testkit::golden::write_png(&dir.join(format!("fx_chains_{frame}.cpu.png")), &cpu);
|
||||
failures.push(format!("frame {frame}: interior p99 {p99}, mean {mean}; {d}; images in {}", dir.display()));
|
||||
}
|
||||
}
|
||||
// ½ playback resolution: the working images (and the effects' pixel radii) shrink with it
|
||||
let opts = RenderOptions { scale: 0.5, ..RenderOptions::default() };
|
||||
let plan = filmcraft_render::plan::plan_frame(&s.project, s.seq, RATE.tick_of(s.frame), opts, &s.sources);
|
||||
let filmcraft_render::plan::FramePlan::Layers { layers, .. } = &plan else { panic!("½: planned as a CPU image") };
|
||||
assert!(layers.iter().take(4).all(|l| l.fx.is_some()));
|
||||
c.composite(&plan);
|
||||
let (gw, gh, mut px) = c.read_output().expect("GPU readback");
|
||||
px.as_chunks_mut::<4>().0.iter_mut().for_each(|p| p[3] = 255);
|
||||
let gpu = Rgba8::new(gw, gh, px);
|
||||
let img = render_sequence(&s.project, s.seq, RATE.tick_of(s.frame), opts, &s.sources);
|
||||
let cpu = Rgba8::new(img.w as u32, img.h as u32, img.over_black_rgba8());
|
||||
let (p99, mean) = interior_diff(&plan, &cpu, &gpu);
|
||||
eprintln!("effect chains @ ½: GPU vs CPU interior p99 {p99}, mean {mean:.3}; whole frame {}", diff(&cpu, &gpu).unwrap());
|
||||
if p99 > 6 || mean >= 1.5 {
|
||||
failures.push(format!("½ resolution: interior p99 {p99}, mean {mean}"));
|
||||
}
|
||||
assert!(failures.is_empty(), "GPU effect chains differ from the CPU render:\n{}", failures.join("\n"));
|
||||
}
|
||||
@@ -10,13 +10,13 @@ use super::*;
|
||||
use filmcraft_geom::{Affine, Vec2};
|
||||
use filmcraft_render::plan::execute_cpu;
|
||||
|
||||
fn device() -> Option<(wgpu::Device, wgpu::Queue)> {
|
||||
pub(crate) fn device() -> Option<(wgpu::Device, wgpu::Queue)> {
|
||||
let instance = wgpu::Instance::default();
|
||||
let adapter = pollster::block_on(instance.request_adapter(&wgpu::RequestAdapterOptions::default())).ok()?;
|
||||
pollster::block_on(adapter.request_device(&wgpu::DeviceDescriptor::default())).ok()
|
||||
}
|
||||
|
||||
fn yuv_frame(w: u32, h: u32) -> Arc<VideoFrame> {
|
||||
pub(crate) fn yuv_frame(w: u32, h: u32) -> Arc<VideoFrame> {
|
||||
let (cw, ch) = (w.div_ceil(2), h.div_ceil(2));
|
||||
let y: Vec<u8> = (0..w * h).map(|i| (16 + ((i % w) * 219 / w)) as u8).collect();
|
||||
let u: Vec<u8> = (0..cw * ch).map(|i| (64 + (i / cw) * 128 / ch) as u8).collect();
|
||||
@@ -32,7 +32,7 @@ fn yuv_frame(w: u32, h: u32) -> Arc<VideoFrame> {
|
||||
}
|
||||
|
||||
/// Decode an IEEE half.
|
||||
fn f16_to_f32(h: u16) -> f32 {
|
||||
pub(crate) fn f16_to_f32(h: u16) -> f32 {
|
||||
let s = if h & 0x8000 != 0 { -1.0 } else { 1.0 };
|
||||
let e = ((h >> 10) & 0x1f) as i32;
|
||||
let m = (h & 0x3ff) as f32;
|
||||
@@ -44,7 +44,7 @@ fn f16_to_f32(h: u16) -> f32 {
|
||||
}
|
||||
|
||||
/// Read the linear premultiplied accumulator back as f32 RGBA (what the CPU plan executor returns).
|
||||
fn read_accum(c: &GpuCompositor) -> Vec<f32> {
|
||||
pub(crate) fn read_accum(c: &GpuCompositor) -> Vec<f32> {
|
||||
let (tex, _, (w, h)) = c.accum.as_ref().expect("accumulator");
|
||||
let row = (w * 8).div_ceil(256) * 256;
|
||||
let buf = c.device.create_buffer(&wgpu::BufferDescriptor {
|
||||
@@ -74,7 +74,7 @@ fn read_accum(c: &GpuCompositor) -> Vec<f32> {
|
||||
|
||||
/// Per-pixel max channel difference of two RGBA8 images over the pixels `keep` selects: (p99,
|
||||
/// mean, max).
|
||||
fn stats8(a: &[u8], b: &[u8], keep: &[bool]) -> (u32, f64, u32) {
|
||||
pub(crate) fn stats8(a: &[u8], b: &[u8], keep: &[bool]) -> (u32, f64, u32) {
|
||||
let mut d: Vec<u32> = a
|
||||
.chunks(4)
|
||||
.zip(b.chunks(4))
|
||||
@@ -88,9 +88,9 @@ fn stats8(a: &[u8], b: &[u8], keep: &[bool]) -> (u32, f64, u32) {
|
||||
|
||||
/// Pixels at least 1.5 px away from every layer's quad outline (the CPU resampler fades a layer's
|
||||
/// edge over a pixel beyond it, the GPU rasterises the quad: compared separately).
|
||||
fn interior(plan: &FramePlan) -> Vec<bool> {
|
||||
pub(crate) fn interior(plan: &FramePlan) -> Vec<bool> {
|
||||
let FramePlan::Layers { width, height, layers } = plan else { return Vec::new() };
|
||||
let inv: Vec<(Affine, f64, f64)> = layers.iter().filter_map(|l| Some((l.matrix.inverse()?, l.frame.width as f64, l.frame.height as f64))).collect();
|
||||
let inv: Vec<(Affine, f64, f64)> = layers.iter().filter_map(|l| Some((l.matrix.inverse()?, l.size().0 as f64, l.size().1 as f64))).collect();
|
||||
let mut keep = vec![true; width * height];
|
||||
for (i, k) in keep.iter_mut().enumerate() {
|
||||
let (x, y) = ((i % width) as f64 + 0.5, (i / width) as f64 + 0.5);
|
||||
@@ -108,7 +108,7 @@ fn interior(plan: &FramePlan) -> Vec<bool> {
|
||||
|
||||
/// Linear premultiplied RGBA f32 test frame: a colour ramp with black and white bands; alpha 0 in
|
||||
/// the left tenth, 1 in the top third, partial below.
|
||||
fn ramp_layer(w: u32, h: u32, phase: u32) -> Arc<VideoFrame> {
|
||||
pub(crate) fn ramp_layer(w: u32, h: u32, phase: u32) -> Arc<VideoFrame> {
|
||||
let mut px = Vec::with_capacity((w * h * 4) as usize);
|
||||
for y in 0..h {
|
||||
for x in 0..w {
|
||||
@@ -162,19 +162,21 @@ fn gpu_blend_modes_match_cpu() {
|
||||
layers: vec![
|
||||
// opaque YUV on the left, a partially transparent ramp on the right; the top-right
|
||||
// corner stays empty (destination alpha 0)
|
||||
PlanLayer { frame: yuv_frame(640, 360), matrix: Affine::scale(0.35, 0.5), opacity: 1.0, blend: Blend::Normal },
|
||||
PlanLayer { frame: base.clone(), matrix: Affine::translate(200.0, 40.0), opacity: 0.9, blend: Blend::Normal },
|
||||
PlanLayer { frame: yuv_frame(640, 360), matrix: Affine::scale(0.35, 0.5), opacity: 1.0, blend: Blend::Normal, fx: None },
|
||||
PlanLayer { frame: base.clone(), matrix: Affine::translate(200.0, 40.0), opacity: 0.9, blend: Blend::Normal, fx: None },
|
||||
PlanLayer {
|
||||
frame: top.clone(),
|
||||
matrix: Affine::motion(Vec2::new(170.0, 95.0), Vec2::new(1.1, 1.1), 17.0, Vec2::new(80.0, 60.0)),
|
||||
opacity: 0.8,
|
||||
blend: mode,
|
||||
fx: None,
|
||||
},
|
||||
PlanLayer {
|
||||
frame: demo.clone(),
|
||||
matrix: Affine::motion(Vec2::new(250.0, 70.0), Vec2::new(0.45, 0.45), -9.0, Vec2::new(160.0, 90.0)),
|
||||
opacity: 0.6,
|
||||
blend: mode,
|
||||
fx: None,
|
||||
},
|
||||
],
|
||||
};
|
||||
@@ -233,8 +235,8 @@ fn gpu_blend_edge_cases_match_cpu() {
|
||||
width: n,
|
||||
height: n,
|
||||
layers: vec![
|
||||
PlanLayer { frame: back.clone(), matrix: Affine::IDENTITY, opacity: 1.0, blend: Blend::Normal },
|
||||
PlanLayer { frame: src.clone(), matrix: Affine::IDENTITY, opacity, blend: mode },
|
||||
PlanLayer { frame: back.clone(), matrix: Affine::IDENTITY, opacity: 1.0, blend: Blend::Normal, fx: None },
|
||||
PlanLayer { frame: src.clone(), matrix: Affine::IDENTITY, opacity, blend: mode, fx: None },
|
||||
],
|
||||
};
|
||||
let cpu = execute_cpu(&plan);
|
||||
@@ -274,8 +276,8 @@ fn gpu_dissolve_pattern_is_exact() {
|
||||
width: w,
|
||||
height: h,
|
||||
layers: vec![
|
||||
PlanLayer { frame: red.clone(), matrix: Affine::scale(w as f64, h as f64), opacity: 1.0, blend: Blend::Normal },
|
||||
PlanLayer { frame: white.clone(), matrix: Affine::scale(w as f64, h as f64), opacity: op, blend: Blend::Dissolve },
|
||||
PlanLayer { frame: red.clone(), matrix: Affine::scale(w as f64, h as f64), opacity: 1.0, blend: Blend::Normal, fx: None },
|
||||
PlanLayer { frame: white.clone(), matrix: Affine::scale(w as f64, h as f64), opacity: op, blend: Blend::Dissolve, fx: None },
|
||||
],
|
||||
};
|
||||
let cpu = execute_cpu(&plan).over_black_rgba8();
|
||||
@@ -301,7 +303,7 @@ fn normal_fast_path_and_blend_layers_off_output() {
|
||||
};
|
||||
let mut c = GpuCompositor::new(&dev, &q);
|
||||
let grey = Arc::new(VideoFrame::rgba_f32(4, 4, vec![0.2; 64]));
|
||||
let layer = |m: Affine, blend: Blend| PlanLayer { frame: grey.clone(), matrix: m, opacity: 1.0, blend };
|
||||
let layer = |m: Affine, blend: Blend| PlanLayer { frame: grey.clone(), matrix: m, opacity: 1.0, blend, fx: None };
|
||||
let normal = FramePlan::Layers { width: 32, height: 16, layers: vec![layer(Affine::scale(8.0, 4.0), Blend::Normal)] };
|
||||
c.composite(&normal);
|
||||
assert!(c.backdrop.is_none());
|
||||
@@ -327,7 +329,7 @@ fn normal_fast_path_and_blend_layers_off_output() {
|
||||
|
||||
#[test]
|
||||
fn quad_bounds_clamps_and_handles_non_finite() {
|
||||
let l = |m: Affine| PlanLayer { frame: Arc::new(VideoFrame::rgba_f32(10, 10, vec![0.0; 400])), matrix: m, opacity: 1.0, blend: Blend::Screen };
|
||||
let l = |m: Affine| PlanLayer { frame: Arc::new(VideoFrame::rgba_f32(10, 10, vec![0.0; 400])), matrix: m, opacity: 1.0, blend: Blend::Screen, fx: None };
|
||||
assert_eq!(quad_bounds(&l(Affine::translate(5.0, 5.0)), 100, 100), Some((4, 4, 12, 12)));
|
||||
assert_eq!(quad_bounds(&l(Affine::translate(-5.0, 95.0)), 100, 100), Some((0, 94, 6, 6)));
|
||||
assert_eq!(quad_bounds(&l(Affine::translate(f64::NAN, 0.0)), 100, 100), Some((0, 0, 100, 100)));
|
||||
|
||||
@@ -0,0 +1,418 @@
|
||||
//! The GPU effect stage: standard effects of a plan layer ([`LayerFx`]) run in WGSL (`fx.wgsl`).
|
||||
//!
|
||||
//! A layer with effects is first drawn from its source (YUV planes or RGBA, through the
|
||||
//! compositor's own `fs` shader, so decoding is the same as for a plain layer) into a working
|
||||
//! texture of the size the CPU decodes it at; each [`FxOp`] then runs as compute passes that
|
||||
//! ping-pong between two working textures (three for Unsharp Mask, which keeps the original);
|
||||
//! the last one is drawn into the accumulator with the layer's matrix, opacity and blend mode
|
||||
//! like any RGBA layer. Working textures are `Rgba32Float` (the CPU reference's precision) and
|
||||
//! pooled per size; layers without effects never touch any of this.
|
||||
//!
|
||||
//! Box blurs (Gaussian Blur, Camera Blur, the blur inside Unsharp / Sharpen) run per pixel for
|
||||
//! radii up to [`BOX_PER_PIXEL_MAX`] and as a running sum per row / column above, so a huge
|
||||
//! radius costs O(1) per pixel like on the CPU (radii are capped by `gaussian_boxes`).
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
|
||||
use filmcraft_render::gpufx::FxOp;
|
||||
use filmcraft_render::plan::LayerFx;
|
||||
|
||||
/// Working texture format.
|
||||
pub(crate) const FX_FORMAT: wgpu::TextureFormat = wgpu::TextureFormat::Rgba32Float;
|
||||
/// Box radii up to this many pixels are summed per output pixel; larger ones as a running sum.
|
||||
pub const BOX_PER_PIXEL_MAX: u32 = 32;
|
||||
|
||||
const OP_BRIGHTNESS_CONTRAST: u32 = 1;
|
||||
const OP_PROC_AMP: u32 = 2;
|
||||
const OP_TINT: u32 = 3;
|
||||
const OP_BLACK_WHITE: u32 = 4;
|
||||
const OP_COLOR_BALANCE: u32 = 5;
|
||||
const OP_LEAVE_COLOR: u32 = 6;
|
||||
const OP_CHANGE_TO_COLOR: u32 = 7;
|
||||
const OP_COLOR_PASS: u32 = 8;
|
||||
const OP_GAMMA: u32 = 9;
|
||||
const OP_LEVELS: u32 = 10;
|
||||
const OP_EXTRACT: u32 = 11;
|
||||
const OP_INVERT: u32 = 12;
|
||||
const OP_INVERT_ALPHA: u32 = 13;
|
||||
const OP_POSTERIZE: u32 = 14;
|
||||
const OP_ASC_CDL: u32 = 15;
|
||||
const OP_CHANNEL_MIX: u32 = 16;
|
||||
const OP_COLOR_REPLACE: u32 = 17;
|
||||
const OP_ALPHA_ADJUST: u32 = 18;
|
||||
const OP_BOX: u32 = 20;
|
||||
const OP_DIRECTIONAL: u32 = 22;
|
||||
const OP_UNSHARP: u32 = 23;
|
||||
const OP_CROP: u32 = 24;
|
||||
const OP_RESAMPLE: u32 = 25;
|
||||
const OP_HFLIP: u32 = 26;
|
||||
const OP_VFLIP: u32 = 27;
|
||||
const OP_MIRROR: u32 = 28;
|
||||
const OP_OFFSET: u32 = 29;
|
||||
|
||||
type Target = (wgpu::Texture, wgpu::TextureView);
|
||||
|
||||
pub(crate) struct FxStage {
|
||||
px: wgpu::ComputePipeline,
|
||||
run: wgpu::ComputePipeline,
|
||||
bgl: wgpu::BindGroupLayout,
|
||||
/// Draws a layer's source into a working texture (`composite.wgsl` `fs`, no blending).
|
||||
source: wgpu::RenderPipeline,
|
||||
pool: HashMap<(u32, u32), Vec<Target>>,
|
||||
used: HashSet<(u32, u32)>,
|
||||
}
|
||||
|
||||
/// One compute pass of a job.
|
||||
struct Dispatch {
|
||||
run: bool,
|
||||
bg: wgpu::BindGroup,
|
||||
groups: (u32, u32),
|
||||
}
|
||||
|
||||
/// The recorded work of one layer's effects (bind groups made; recorded into the frame's encoder
|
||||
/// right before the layer is drawn, so layers can share pooled textures).
|
||||
pub(crate) struct FxJob {
|
||||
source_bg: wgpu::BindGroup,
|
||||
source_target: wgpu::TextureView,
|
||||
dispatches: Vec<Dispatch>,
|
||||
/// The texture holding the result.
|
||||
pub(crate) result: wgpu::TextureView,
|
||||
pub(crate) result_texture: wgpu::Texture,
|
||||
}
|
||||
|
||||
/// One pass before texture routing: entry point, op code, integer and float parameters.
|
||||
struct Step {
|
||||
run: bool,
|
||||
code: u32,
|
||||
i1: [u32; 4],
|
||||
p: [f32; 16],
|
||||
/// Unsharp combine: reads the blurred image and the original.
|
||||
combine: bool,
|
||||
/// A blur pass of Unsharp Mask (the original stays untouched until the combine).
|
||||
unsharp_blur: bool,
|
||||
}
|
||||
|
||||
fn step(code: u32, i1: [u32; 4], p: &[f32]) -> Step {
|
||||
let mut q = [0.0; 16];
|
||||
for (d, s) in q.iter_mut().zip(p) {
|
||||
*d = *s;
|
||||
}
|
||||
Step { run: false, code, i1, p: q, combine: false, unsharp_blur: false }
|
||||
}
|
||||
|
||||
/// Box passes along x (vertical = false) or y for the radii (radius 0 is a no-op on the CPU).
|
||||
fn boxes(radii: &[u32], vertical: bool, repeat: bool, out: &mut Vec<Step>) {
|
||||
for &r in radii.iter().filter(|r| **r > 0) {
|
||||
let mut s = step(OP_BOX, [r, repeat as u32, vertical as u32, 0], &[]);
|
||||
s.run = r > BOX_PER_PIXEL_MAX;
|
||||
out.push(s);
|
||||
}
|
||||
}
|
||||
|
||||
/// The passes of an op (empty: no-op). Unsharp's blur passes come first, then the combine.
|
||||
fn steps(op: &FxOp) -> Vec<Step> {
|
||||
let mut out = Vec::new();
|
||||
match op {
|
||||
FxOp::BrightnessContrast { br, co } => out.push(step(OP_BRIGHTNESS_CONTRAST, [0; 4], &[*br, *co])),
|
||||
FxOp::ProcAmp { br, co, hue, sat } => out.push(step(OP_PROC_AMP, [0; 4], &[*br, *co, *hue, *sat])),
|
||||
FxOp::Tint { black, white, amount } => {
|
||||
out.push(step(OP_TINT, [0; 4], &[black[0], black[1], black[2], *amount, white[0], white[1], white[2]]));
|
||||
}
|
||||
FxOp::BlackWhite => out.push(step(OP_BLACK_WHITE, [0; 4], &[])),
|
||||
FxOp::ColorBalance { sh, md, hi, preserve } => {
|
||||
out.push(step(OP_COLOR_BALANCE, [*preserve as u32, 0, 0, 0], &[sh[0], sh[1], sh[2], 0.0, md[0], md[1], md[2], 0.0, hi[0], hi[1], hi[2]]))
|
||||
}
|
||||
FxOp::LeaveColor { amount, key_hue, tol, soft } => out.push(step(OP_LEAVE_COLOR, [0; 4], &[*amount, *key_hue, *tol, *soft])),
|
||||
FxOp::ChangeToColor { from_hue, to_hue, tol, soft } => out.push(step(OP_CHANGE_TO_COLOR, [0; 4], &[*from_hue, *to_hue, *tol, *soft])),
|
||||
FxOp::ColorPass { key, sim, reverse } => out.push(step(OP_COLOR_PASS, [*reverse as u32, 0, 0, 0], &[key[0], key[1], key[2], *sim])),
|
||||
FxOp::Gamma { g } => out.push(step(OP_GAMMA, [0; 4], &[*g])),
|
||||
FxOp::Levels { ib, iw, ob, ow, g } => out.push(step(OP_LEVELS, [0; 4], &[*ib, *iw, *ob, *ow, *g])),
|
||||
FxOp::Extract { lo, hi, soft, invert } => out.push(step(OP_EXTRACT, [*invert as u32, 0, 0, 0], &[*lo, *hi, *soft])),
|
||||
FxOp::Invert { channel: 4, blend } => out.push(step(OP_INVERT_ALPHA, [0; 4], &[*blend])),
|
||||
FxOp::Invert { channel, blend } => out.push(step(OP_INVERT, [*channel, 0, 0, 0], &[*blend])),
|
||||
FxOp::Posterize { n } => out.push(step(OP_POSTERIZE, [0; 4], &[*n])),
|
||||
FxOp::Gaussian { rx, ry, repeat } => {
|
||||
boxes(rx, false, *repeat, &mut out);
|
||||
boxes(ry, true, *repeat, &mut out);
|
||||
}
|
||||
FxOp::DirectionalBlur { dx, dy, steps } => {
|
||||
if *steps >= 2 {
|
||||
out.push(step(OP_DIRECTIONAL, [*steps, 0, 0, 0], &[*dx, *dy]));
|
||||
}
|
||||
}
|
||||
FxOp::Unsharp { rx, ry, amount, threshold } => {
|
||||
boxes(rx, false, true, &mut out);
|
||||
boxes(ry, true, true, &mut out);
|
||||
for s in &mut out {
|
||||
s.unsharp_blur = true;
|
||||
}
|
||||
let mut s = step(OP_UNSHARP, [0; 4], &[*amount, *threshold]);
|
||||
s.combine = true;
|
||||
out.push(s);
|
||||
}
|
||||
FxOp::Crop { x0, x1, y0, y1, feather } => out.push(step(OP_CROP, [0; 4], &[*x0, *x1, *y0, *y1, *feather])),
|
||||
FxOp::Resample(r) => {
|
||||
let op = if (r.opacity - 1.0).abs() < 1e-6 { 1.0 } else { r.opacity };
|
||||
match &r.inv {
|
||||
Some(m) => out.push(step(OP_RESAMPLE, r.rect, &[m.a as f32, m.b as f32, m.c as f32, m.d as f32, m.e as f32, m.f as f32, op, 1.0])),
|
||||
None => out.push(step(OP_RESAMPLE, [0; 4], &[0.0; 8])),
|
||||
}
|
||||
}
|
||||
FxOp::HFlip => out.push(step(OP_HFLIP, [0; 4], &[])),
|
||||
FxOp::VFlip => out.push(step(OP_VFLIP, [0; 4], &[])),
|
||||
FxOp::Mirror { cx, cy, nx, ny } => out.push(step(OP_MIRROR, [0; 4], &[*cx, *cy, *nx, *ny])),
|
||||
FxOp::Offset { dx, dy, blend } => out.push(step(OP_OFFSET, [0; 4], &[*dx, *dy, *blend])),
|
||||
FxOp::AscCdl { slope: s, offset: o, power: p, sat } => {
|
||||
out.push(step(OP_ASC_CDL, [0; 4], &[s[0], s[1], s[2], *sat, o[0], o[1], o[2], 0.0, p[0], p[1], p[2]]));
|
||||
}
|
||||
FxOp::ChannelMix { m } => {
|
||||
let p: Vec<f32> = m.iter().flatten().copied().collect();
|
||||
out.push(step(OP_CHANNEL_MIX, [0; 4], &p));
|
||||
}
|
||||
FxOp::ColorReplace { sim, solid, target: t, replace: r, replace_hsl: h } => {
|
||||
out.push(step(OP_COLOR_REPLACE, [*solid as u32, 0, 0, 0], &[t[0], t[1], t[2], *sim, r[0], r[1], r[2], 0.0, h[0], h[1], h[2]]))
|
||||
}
|
||||
FxOp::AlphaAdjust { opacity, ignore, invert, mask_only } => {
|
||||
out.push(step(OP_ALPHA_ADJUST, [*ignore as u32, *invert as u32, *mask_only as u32, 0], &[*opacity]));
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
impl FxStage {
|
||||
/// Whether `device` can run the stage: compute shaders with 16×16 workgroups and a storage
|
||||
/// texture (WebGL2-class devices can't; they render effects on the CPU).
|
||||
pub(crate) fn supported(device: &wgpu::Device) -> bool {
|
||||
let l = device.limits();
|
||||
l.max_storage_textures_per_shader_stage >= 1
|
||||
&& l.max_compute_invocations_per_workgroup >= 256
|
||||
&& l.max_compute_workgroup_size_x >= 64
|
||||
&& l.max_compute_workgroup_size_y >= 16
|
||||
&& l.max_compute_workgroups_per_dimension >= 1
|
||||
}
|
||||
|
||||
pub(crate) fn new(device: &wgpu::Device, composite: &wgpu::ShaderModule, layer_bgl: &wgpu::BindGroupLayout) -> Self {
|
||||
let shader = device.create_shader_module(wgpu::ShaderModuleDescriptor {
|
||||
label: Some("filmcraft-fx"),
|
||||
source: wgpu::ShaderSource::Wgsl(include_str!("fx.wgsl").into()),
|
||||
});
|
||||
let tex = |binding| wgpu::BindGroupLayoutEntry {
|
||||
binding,
|
||||
visibility: wgpu::ShaderStages::COMPUTE,
|
||||
ty: wgpu::BindingType::Texture {
|
||||
sample_type: wgpu::TextureSampleType::Float { filterable: false },
|
||||
view_dimension: wgpu::TextureViewDimension::D2,
|
||||
multisampled: false,
|
||||
},
|
||||
count: None,
|
||||
};
|
||||
let bgl = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
|
||||
label: Some("fx"),
|
||||
entries: &[
|
||||
wgpu::BindGroupLayoutEntry {
|
||||
binding: 0,
|
||||
visibility: wgpu::ShaderStages::COMPUTE,
|
||||
ty: wgpu::BindingType::Buffer { ty: wgpu::BufferBindingType::Uniform, has_dynamic_offset: false, min_binding_size: None },
|
||||
count: None,
|
||||
},
|
||||
tex(1),
|
||||
wgpu::BindGroupLayoutEntry {
|
||||
binding: 2,
|
||||
visibility: wgpu::ShaderStages::COMPUTE,
|
||||
ty: wgpu::BindingType::StorageTexture {
|
||||
access: wgpu::StorageTextureAccess::WriteOnly,
|
||||
format: FX_FORMAT,
|
||||
view_dimension: wgpu::TextureViewDimension::D2,
|
||||
},
|
||||
count: None,
|
||||
},
|
||||
tex(3),
|
||||
],
|
||||
});
|
||||
let pl = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { label: Some("fx"), bind_group_layouts: &[Some(&bgl)], immediate_size: 0 });
|
||||
let compute = |entry: &str| {
|
||||
device.create_compute_pipeline(&wgpu::ComputePipelineDescriptor {
|
||||
label: Some("fx"),
|
||||
layout: Some(&pl),
|
||||
module: &shader,
|
||||
entry_point: Some(entry),
|
||||
compilation_options: Default::default(),
|
||||
cache: None,
|
||||
})
|
||||
};
|
||||
let spl = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor {
|
||||
label: Some("fx-source"),
|
||||
bind_group_layouts: &[Some(layer_bgl)],
|
||||
immediate_size: 0,
|
||||
});
|
||||
let source = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor {
|
||||
label: Some("fx-source"),
|
||||
layout: Some(&spl),
|
||||
vertex: wgpu::VertexState { module: composite, entry_point: Some("vs"), compilation_options: Default::default(), buffers: &[] },
|
||||
primitive: wgpu::PrimitiveState::default(),
|
||||
depth_stencil: None,
|
||||
multisample: wgpu::MultisampleState::default(),
|
||||
fragment: Some(wgpu::FragmentState {
|
||||
module: composite,
|
||||
entry_point: Some("fs"),
|
||||
compilation_options: Default::default(),
|
||||
targets: &[Some(wgpu::ColorTargetState { format: FX_FORMAT, blend: None, write_mask: wgpu::ColorWrites::ALL })],
|
||||
}),
|
||||
multiview_mask: None,
|
||||
cache: None,
|
||||
});
|
||||
Self { px: compute("fx_px"), run: compute("fx_run"), bgl, source, pool: HashMap::new(), used: HashSet::new() }
|
||||
}
|
||||
|
||||
/// Start a frame: pooled textures of sizes no layer uses by [`end_frame`](Self::end_frame)
|
||||
/// are released then.
|
||||
pub(crate) fn begin_frame(&mut self) {
|
||||
self.used.clear();
|
||||
}
|
||||
|
||||
pub(crate) fn end_frame(&mut self) {
|
||||
let used = &self.used;
|
||||
self.pool.retain(|k, _| used.contains(k));
|
||||
}
|
||||
|
||||
/// No working textures held.
|
||||
#[cfg(test)]
|
||||
pub(crate) fn is_idle(&self) -> bool {
|
||||
self.pool.is_empty()
|
||||
}
|
||||
|
||||
fn targets(&mut self, device: &wgpu::Device, size: (u32, u32), n: usize) -> Vec<Target> {
|
||||
self.used.insert(size);
|
||||
let v = self.pool.entry(size).or_default();
|
||||
while v.len() < n {
|
||||
let t = device.create_texture(&wgpu::TextureDescriptor {
|
||||
label: Some("filmcraft-fx"),
|
||||
size: wgpu::Extent3d { width: size.0, height: size.1, depth_or_array_layers: 1 },
|
||||
mip_level_count: 1,
|
||||
sample_count: 1,
|
||||
dimension: wgpu::TextureDimension::D2,
|
||||
format: FX_FORMAT,
|
||||
usage: wgpu::TextureUsages::RENDER_ATTACHMENT
|
||||
| wgpu::TextureUsages::TEXTURE_BINDING
|
||||
| wgpu::TextureUsages::STORAGE_BINDING
|
||||
| wgpu::TextureUsages::COPY_SRC,
|
||||
view_formats: &[],
|
||||
});
|
||||
let view = t.create_view(&Default::default());
|
||||
v.push((t, view));
|
||||
}
|
||||
v.iter().take(n).cloned().collect()
|
||||
}
|
||||
|
||||
/// Prepare the passes of `fx`, whose source draw uses `source_bg` (a layer bind group whose
|
||||
/// uniforms map the frame onto the working image). None when the working image does not fit
|
||||
/// in a texture (the layer is then drawn without its effects — never larger than the source
|
||||
/// frame, which would not have uploaded either).
|
||||
pub(crate) fn job(
|
||||
&mut self,
|
||||
device: &wgpu::Device,
|
||||
queue: &wgpu::Queue,
|
||||
fx: &LayerFx,
|
||||
source_bg: wgpu::BindGroup,
|
||||
dummy: &wgpu::TextureView,
|
||||
) -> Option<FxJob> {
|
||||
let (w, h) = (fx.size.0.max(1), fx.size.1.max(1));
|
||||
let max = device.limits().max_texture_dimension_2d;
|
||||
if w > max || h > max {
|
||||
return None;
|
||||
}
|
||||
let all: Vec<Step> = fx.ops.iter().flat_map(steps).collect();
|
||||
let n = if all.iter().any(|s| s.combine) { 3 } else { 2 };
|
||||
let targets = self.targets(device, (w, h), n);
|
||||
let views: Vec<wgpu::TextureView> = targets.iter().map(|t| t.1.clone()).collect();
|
||||
let mut dispatches = Vec::with_capacity(all.len());
|
||||
let mut cur = 0usize;
|
||||
// Unsharp: the image its blur started from (kept while its blur passes run)
|
||||
let mut orig: Option<usize> = None;
|
||||
for s in &all {
|
||||
// (an Unsharp without blur passes, radius 0, combines the original with itself)
|
||||
if (s.unsharp_blur || s.combine) && orig.is_none() {
|
||||
orig = Some(cur);
|
||||
}
|
||||
let keep = if s.unsharp_blur || s.combine { orig } else { None };
|
||||
let dst = (0..n).find(|k| *k != cur && Some(*k) != keep)?;
|
||||
let aux = match (s.combine, orig) {
|
||||
(true, Some(o)) => views.get(o)?,
|
||||
_ => dummy,
|
||||
};
|
||||
let mut bytes = Vec::with_capacity(96);
|
||||
for v in [s.code, w, h, 0].iter().chain(&s.i1) {
|
||||
bytes.extend_from_slice(&v.to_le_bytes());
|
||||
}
|
||||
for v in &s.p {
|
||||
bytes.extend_from_slice(&v.to_le_bytes());
|
||||
}
|
||||
let buf = device.create_buffer(&wgpu::BufferDescriptor {
|
||||
label: Some("fx-u"),
|
||||
size: bytes.len() as u64,
|
||||
usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST,
|
||||
mapped_at_creation: false,
|
||||
});
|
||||
queue.write_buffer(&buf, 0, &bytes);
|
||||
let bg = device.create_bind_group(&wgpu::BindGroupDescriptor {
|
||||
label: Some("fx"),
|
||||
layout: &self.bgl,
|
||||
entries: &[
|
||||
wgpu::BindGroupEntry { binding: 0, resource: buf.as_entire_binding() },
|
||||
wgpu::BindGroupEntry { binding: 1, resource: wgpu::BindingResource::TextureView(views.get(cur)?) },
|
||||
wgpu::BindGroupEntry { binding: 2, resource: wgpu::BindingResource::TextureView(views.get(dst)?) },
|
||||
wgpu::BindGroupEntry { binding: 3, resource: wgpu::BindingResource::TextureView(aux) },
|
||||
],
|
||||
});
|
||||
let groups = if s.run {
|
||||
let lines = if s.i1[2] != 0 { w } else { h };
|
||||
(lines.div_ceil(64), 1)
|
||||
} else {
|
||||
(w.div_ceil(16), h.div_ceil(16))
|
||||
};
|
||||
dispatches.push(Dispatch { run: s.run, bg, groups });
|
||||
cur = dst;
|
||||
if s.combine {
|
||||
orig = None;
|
||||
}
|
||||
}
|
||||
Some(FxJob {
|
||||
source_bg,
|
||||
source_target: views.first()?.clone(),
|
||||
dispatches,
|
||||
result: views.get(cur)?.clone(),
|
||||
result_texture: targets.get(cur)?.0.clone(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Record a job (source draw, then its compute passes) into `enc`.
|
||||
pub(crate) fn record(&self, enc: &mut wgpu::CommandEncoder, job: &FxJob) {
|
||||
{
|
||||
let mut pass = enc.begin_render_pass(&wgpu::RenderPassDescriptor {
|
||||
label: Some("fx-source"),
|
||||
color_attachments: &[Some(wgpu::RenderPassColorAttachment {
|
||||
view: &job.source_target,
|
||||
depth_slice: None,
|
||||
resolve_target: None,
|
||||
ops: wgpu::Operations { load: wgpu::LoadOp::Clear(wgpu::Color::TRANSPARENT), store: wgpu::StoreOp::Store },
|
||||
})],
|
||||
depth_stencil_attachment: None,
|
||||
timestamp_writes: None,
|
||||
occlusion_query_set: None,
|
||||
multiview_mask: None,
|
||||
});
|
||||
pass.set_pipeline(&self.source);
|
||||
pass.set_bind_group(0, &job.source_bg, &[]);
|
||||
pass.draw(0..6, 0..1);
|
||||
}
|
||||
if job.dispatches.is_empty() {
|
||||
return;
|
||||
}
|
||||
let mut pass = enc.begin_compute_pass(&wgpu::ComputePassDescriptor { label: Some("fx"), timestamp_writes: None });
|
||||
for d in &job.dispatches {
|
||||
pass.set_pipeline(if d.run { &self.run } else { &self.px });
|
||||
pass.set_bind_group(0, &d.bg, &[]);
|
||||
pass.dispatch_workgroups(d.groups.0, d.groups.1, 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,525 @@
|
||||
// FilmCraft GPU effect stage: the standard effects of `filmcraft_render::gpufx::FxOp`, one
|
||||
// compute pass each, reading the working image (`src`, linear premultiplied RGBA f32) and writing
|
||||
// the next one (`dst`). The math mirrors `FxOp::apply` (the CPU reference) operation by
|
||||
// operation; parameters arrive evaluated (keyframes, defaults and clamps applied on the CPU).
|
||||
|
||||
struct U {
|
||||
i0: vec4<u32>, // op, width, height, unused
|
||||
i1: vec4<u32>, // integer parameters (box radius, repeat, steps, channel, rect…)
|
||||
p0: vec4<f32>,
|
||||
p1: vec4<f32>,
|
||||
p2: vec4<f32>,
|
||||
p3: vec4<f32>,
|
||||
};
|
||||
|
||||
@group(0) @binding(0) var<uniform> u: U;
|
||||
@group(0) @binding(1) var src: texture_2d<f32>;
|
||||
@group(0) @binding(2) var dst: texture_storage_2d<rgba32float, write>;
|
||||
// Unsharp Mask: the image before blurring.
|
||||
@group(0) @binding(3) var aux: texture_2d<f32>;
|
||||
|
||||
const OP_BRIGHTNESS_CONTRAST: u32 = 1u;
|
||||
const OP_PROC_AMP: u32 = 2u;
|
||||
const OP_TINT: u32 = 3u;
|
||||
const OP_BLACK_WHITE: u32 = 4u;
|
||||
const OP_COLOR_BALANCE: u32 = 5u;
|
||||
const OP_LEAVE_COLOR: u32 = 6u;
|
||||
const OP_CHANGE_TO_COLOR: u32 = 7u;
|
||||
const OP_COLOR_PASS: u32 = 8u;
|
||||
const OP_GAMMA: u32 = 9u;
|
||||
const OP_LEVELS: u32 = 10u;
|
||||
const OP_EXTRACT: u32 = 11u;
|
||||
const OP_INVERT: u32 = 12u;
|
||||
const OP_INVERT_ALPHA: u32 = 13u;
|
||||
const OP_POSTERIZE: u32 = 14u;
|
||||
const OP_ASC_CDL: u32 = 15u;
|
||||
const OP_CHANNEL_MIX: u32 = 16u;
|
||||
const OP_COLOR_REPLACE: u32 = 17u;
|
||||
const OP_ALPHA_ADJUST: u32 = 18u;
|
||||
const OP_BOX: u32 = 20u;
|
||||
const OP_DIRECTIONAL: u32 = 22u;
|
||||
const OP_UNSHARP: u32 = 23u;
|
||||
const OP_CROP: u32 = 24u;
|
||||
const OP_RESAMPLE: u32 = 25u;
|
||||
const OP_HFLIP: u32 = 26u;
|
||||
const OP_VFLIP: u32 = 27u;
|
||||
const OP_MIRROR: u32 = 28u;
|
||||
const OP_OFFSET: u32 = 29u;
|
||||
|
||||
fn size() -> vec2<i32> {
|
||||
return vec2<i32>(i32(u.i0.y), i32(u.i0.z));
|
||||
}
|
||||
|
||||
fn ld(p: vec2<i32>) -> vec4<f32> {
|
||||
return textureLoad(src, p, 0);
|
||||
}
|
||||
|
||||
// `Image::get_or_clear`
|
||||
fn ld_or_clear(p: vec2<i32>) -> vec4<f32> {
|
||||
let s = size();
|
||||
if p.x < 0 || p.y < 0 || p.x >= s.x || p.y >= s.y {
|
||||
return vec4(0.0);
|
||||
}
|
||||
return textureLoad(src, p, 0);
|
||||
}
|
||||
|
||||
// `Image::sample_bilinear`: pixel centres at +0.5, transparent outside.
|
||||
fn sample_bilinear(x: f32, y: f32) -> vec4<f32> {
|
||||
let fx = x - 0.5;
|
||||
let fy = y - 0.5;
|
||||
let x0 = floor(fx);
|
||||
let y0 = floor(fy);
|
||||
let tx = fx - x0;
|
||||
let ty = fy - y0;
|
||||
let i = vec2<i32>(i32(x0), i32(y0));
|
||||
let a = ld_or_clear(i);
|
||||
let b = ld_or_clear(i + vec2(1, 0));
|
||||
let c = ld_or_clear(i + vec2(0, 1));
|
||||
let d = ld_or_clear(i + vec2(1, 1));
|
||||
let top = a + (b - a) * tx;
|
||||
let bot = c + (d - c) * tx;
|
||||
return top + (bot - top) * ty;
|
||||
}
|
||||
|
||||
// `Image::sample_bilinear_clamped`
|
||||
fn sample_clamped(x: f32, y: f32) -> vec4<f32> {
|
||||
let s = size();
|
||||
let fx = clamp(x - 0.5, 0.0, f32(s.x) - 1.0);
|
||||
let fy = clamp(y - 0.5, 0.0, f32(s.y) - 1.0);
|
||||
let x0 = i32(floor(fx));
|
||||
let y0 = i32(floor(fy));
|
||||
let x1 = min(x0 + 1, s.x - 1);
|
||||
let y1 = min(y0 + 1, s.y - 1);
|
||||
let tx = fx - f32(x0);
|
||||
let ty = fy - f32(y0);
|
||||
let a = ld(vec2(x0, y0));
|
||||
let b = ld(vec2(x1, y0));
|
||||
let c = ld(vec2(x0, y1));
|
||||
let d = ld(vec2(x1, y1));
|
||||
let top = a + (b - a) * tx;
|
||||
let bot = c + (d - c) * tx;
|
||||
return top + (bot - top) * ty;
|
||||
}
|
||||
|
||||
// ---- colour helpers (`filmcraft_color`, `effects::enc` / `dec`)
|
||||
|
||||
fn linear_to_srgb1(v: f32) -> f32 {
|
||||
if v <= 0.0031308 {
|
||||
return v * 12.92;
|
||||
}
|
||||
return 1.055 * pow(v, 1.0 / 2.4) - 0.055;
|
||||
}
|
||||
|
||||
fn srgb_to_linear1(v: f32) -> f32 {
|
||||
if v <= 0.04045 {
|
||||
return v / 12.92;
|
||||
}
|
||||
return pow((v + 0.055) / 1.055, 2.4);
|
||||
}
|
||||
|
||||
fn enc(c: vec3<f32>) -> vec3<f32> {
|
||||
let m = max(c, vec3(0.0));
|
||||
return vec3(linear_to_srgb1(m.x), linear_to_srgb1(m.y), linear_to_srgb1(m.z));
|
||||
}
|
||||
|
||||
fn dec(c: vec3<f32>) -> vec3<f32> {
|
||||
let m = clamp(c, vec3(0.0), vec3(1.0));
|
||||
return vec3(srgb_to_linear1(m.x), srgb_to_linear1(m.y), srgb_to_linear1(m.z));
|
||||
}
|
||||
|
||||
fn luma709(c: vec3<f32>) -> f32 {
|
||||
return 0.2126 * c.x + 0.7152 * c.y + 0.0722 * c.z;
|
||||
}
|
||||
|
||||
// `f32::powf` for x >= 0 (WGSL leaves pow(0, y) undefined).
|
||||
fn powf(x: f32, y: f32) -> f32 {
|
||||
if x == 0.0 {
|
||||
if y == 0.0 {
|
||||
return 1.0;
|
||||
}
|
||||
if y > 0.0 {
|
||||
return 0.0;
|
||||
}
|
||||
return 3.0e38;
|
||||
}
|
||||
return pow(x, y);
|
||||
}
|
||||
|
||||
// `f32::round`: halves away from zero (WGSL `round` rounds them to even).
|
||||
fn round_away(x: f32) -> f32 {
|
||||
let a = abs(x);
|
||||
let f = floor(a);
|
||||
let r = select(f, f + 1.0, a - f >= 0.5);
|
||||
return select(r, -r, x < 0.0);
|
||||
}
|
||||
|
||||
// `rem_euclid(1.0)`
|
||||
fn fract_euclid(x: f32) -> f32 {
|
||||
return x - floor(x);
|
||||
}
|
||||
|
||||
// `vfx::smoothstep` (not WGSL's: a degenerate edge pair divides by 1e-6)
|
||||
fn smoothstep_fx(e0: f32, e1: f32, x: f32) -> f32 {
|
||||
let t = clamp((x - e0) / max(e1 - e0, 1e-6), 0.0, 1.0);
|
||||
return t * t * (3.0 - 2.0 * t);
|
||||
}
|
||||
|
||||
fn rgb_to_hsl(c: vec3<f32>) -> vec3<f32> {
|
||||
let r = c.x; let g = c.y; let b = c.z;
|
||||
let mx = max(max(r, g), b);
|
||||
let mn = min(min(r, g), b);
|
||||
let l = (mx + mn) / 2.0;
|
||||
if abs(mx - mn) < 1e-7 {
|
||||
return vec3(0.0, 0.0, l);
|
||||
}
|
||||
let d = mx - mn;
|
||||
var s: f32;
|
||||
if l > 0.5 {
|
||||
s = d / (2.0 - mx - mn);
|
||||
} else {
|
||||
s = d / (mx + mn);
|
||||
}
|
||||
var h: f32;
|
||||
if mx == r {
|
||||
h = (g - b) / d + select(0.0, 6.0, g < b);
|
||||
} else if mx == g {
|
||||
h = (b - r) / d + 2.0;
|
||||
} else {
|
||||
h = (r - g) / d + 4.0;
|
||||
}
|
||||
return vec3(h / 6.0, s, l);
|
||||
}
|
||||
|
||||
fn hsl_channel(p: f32, q: f32, t0: f32) -> f32 {
|
||||
let t = fract_euclid(t0);
|
||||
if t < 1.0 / 6.0 {
|
||||
return p + (q - p) * 6.0 * t;
|
||||
}
|
||||
if t < 0.5 {
|
||||
return q;
|
||||
}
|
||||
if t < 2.0 / 3.0 {
|
||||
return p + (q - p) * (2.0 / 3.0 - t) * 6.0;
|
||||
}
|
||||
return p;
|
||||
}
|
||||
|
||||
fn hsl_to_rgb(h: f32, s: f32, l: f32) -> vec3<f32> {
|
||||
if s <= 0.0 {
|
||||
return vec3(l);
|
||||
}
|
||||
var q: f32;
|
||||
if l < 0.5 {
|
||||
q = l * (1.0 + s);
|
||||
} else {
|
||||
q = l + s - l * s;
|
||||
}
|
||||
let p = 2.0 * l - q;
|
||||
return vec3(hsl_channel(p, q, h + 1.0 / 3.0), hsl_channel(p, q, h), hsl_channel(p, q, h - 1.0 / 3.0));
|
||||
}
|
||||
|
||||
// ---- per-pixel colour effects on straight colour (`Image::map_rgb`)
|
||||
|
||||
fn color_op(op: u32, c: vec3<f32>) -> vec3<f32> {
|
||||
switch op {
|
||||
case OP_BRIGHTNESS_CONTRAST: {
|
||||
let br = u.p0.x; let co = u.p0.y;
|
||||
return dec((enc(c) - 0.5) * co + 0.5 + br);
|
||||
}
|
||||
case OP_PROC_AMP: {
|
||||
let br = u.p0.x; let co = u.p0.y; let hue = u.p0.z; let sat = u.p0.w;
|
||||
var hsl = rgb_to_hsl(enc(c));
|
||||
hsl.x = fract_euclid(hsl.x + hue);
|
||||
hsl.y = clamp(hsl.y * sat, 0.0, 1.0);
|
||||
return dec((hsl_to_rgb(hsl.x, hsl.y, hsl.z) - 0.5) * co + 0.5 + br);
|
||||
}
|
||||
case OP_TINT: {
|
||||
let bl = u.p0.xyz; let wh = u.p1.xyz; let amt = u.p0.w;
|
||||
let l = linear_to_srgb1(max(luma709(c), 0.0));
|
||||
let t = bl + (wh - bl) * l;
|
||||
let e = enc(c);
|
||||
return dec(e + (t - e) * amt);
|
||||
}
|
||||
case OP_BLACK_WHITE: {
|
||||
return vec3(luma709(c));
|
||||
}
|
||||
case OP_COLOR_BALANCE: {
|
||||
let e = enc(c);
|
||||
let l = 0.2126 * e.x + 0.7152 * e.y + 0.0722 * e.z;
|
||||
let ws = (1.0 - l) * (1.0 - l);
|
||||
let wh = l * l;
|
||||
let wm = 1.0 - ws - wh;
|
||||
var o = e + u.p0.xyz * ws + u.p1.xyz * max(wm, 0.0) + u.p2.xyz * wh;
|
||||
if u.i1.x != 0u {
|
||||
let l2 = 0.2126 * o.x + 0.7152 * o.y + 0.0722 * o.z;
|
||||
o = o + (l - l2);
|
||||
}
|
||||
return dec(o);
|
||||
}
|
||||
case OP_LEAVE_COLOR: {
|
||||
let amt = u.p0.x; let kh = u.p0.y; let tol = u.p0.z; let soft = u.p0.w;
|
||||
let h = rgb_to_hsl(enc(c)).x;
|
||||
let d = min(abs(h - kh), 1.0 - abs(h - kh)) * 2.0;
|
||||
let keep = 1.0 - clamp((d - tol) / soft, 0.0, 1.0);
|
||||
let l = luma709(c);
|
||||
let k = amt * (1.0 - keep);
|
||||
return c + (l - c) * k;
|
||||
}
|
||||
case OP_CHANGE_TO_COLOR: {
|
||||
let fh = u.p0.x; let th = u.p0.y; let tol = u.p0.z; let soft = u.p0.w;
|
||||
var hsl = rgb_to_hsl(enc(c));
|
||||
let d = min(abs(hsl.x - fh), 1.0 - abs(hsl.x - fh));
|
||||
let w = 1.0 - clamp((d - tol) / soft, 0.0, 1.0);
|
||||
hsl.x = fract_euclid(hsl.x + (th - fh) * w);
|
||||
return dec(hsl_to_rgb(hsl.x, hsl.y, hsl.z));
|
||||
}
|
||||
case OP_COLOR_PASS: {
|
||||
let e = enc(c);
|
||||
let k = e - u.p0.xyz;
|
||||
let d = sqrt(k.x * k.x + k.y * k.y + k.z * k.z);
|
||||
let passes = (d <= u.p0.w * 1.2) != (u.i1.x != 0u);
|
||||
if passes {
|
||||
return c;
|
||||
}
|
||||
return vec3(luma709(c));
|
||||
}
|
||||
case OP_GAMMA: {
|
||||
let e = enc(c);
|
||||
let g = u.p0.x;
|
||||
return dec(vec3(powf(max(e.x, 0.0), g), powf(max(e.y, 0.0), g), powf(max(e.z, 0.0), g)));
|
||||
}
|
||||
case OP_LEVELS: {
|
||||
let ib = u.p0.x; let iw = u.p0.y; let ob = u.p0.z; let ow = u.p0.w; let g = u.p1.x;
|
||||
let v = clamp((enc(c) - ib) / (iw - ib), vec3(0.0), vec3(1.0));
|
||||
return dec(ob + vec3(powf(v.x, g), powf(v.y, g), powf(v.z, g)) * (ow - ob));
|
||||
}
|
||||
case OP_EXTRACT: {
|
||||
let lo = u.p0.x; let hi = u.p0.y; let soft = u.p0.z;
|
||||
let l = linear_to_srgb1(max(luma709(c), 0.0));
|
||||
let inside = min(clamp((l - lo) / soft, 0.0, 1.0), clamp((hi - l) / soft, 0.0, 1.0));
|
||||
return vec3(select(inside, 1.0 - inside, u.i1.x != 0u));
|
||||
}
|
||||
case OP_INVERT: {
|
||||
let ch = u.i1.x;
|
||||
let blend = u.p0.x;
|
||||
let e = enc(c);
|
||||
var o = e;
|
||||
if ch == 0u || ch == 1u { o.x = 1.0 - e.x; }
|
||||
if ch == 0u || ch == 2u { o.y = 1.0 - e.y; }
|
||||
if ch == 0u || ch == 3u { o.z = 1.0 - e.z; }
|
||||
return dec(o + (e - o) * blend);
|
||||
}
|
||||
case OP_POSTERIZE: {
|
||||
let n = u.p0.x;
|
||||
let e = enc(c) * n;
|
||||
return dec(vec3(round_away(e.x), round_away(e.y), round_away(e.z)) / n);
|
||||
}
|
||||
case OP_ASC_CDL: {
|
||||
let e = enc(c);
|
||||
let s = u.p0.xyz; let o = u.p1.xyz; let pw = max(u.p2.xyz, vec3(0.0)); let sat = u.p0.w;
|
||||
let q = clamp(e * s + o, vec3(0.0), vec3(1.0));
|
||||
let v = vec3(powf(q.x, pw.x), powf(q.y, pw.y), powf(q.z, pw.z));
|
||||
let l = luma709(v);
|
||||
return dec(clamp(l + sat * (v - l), vec3(0.0), vec3(1.0)));
|
||||
}
|
||||
case OP_CHANNEL_MIX: {
|
||||
let v = enc(c);
|
||||
let r = u.p0; let g = u.p1; let b = u.p2;
|
||||
return dec(clamp(vec3(r.x * v.x + r.y * v.y + r.z * v.z + r.w, g.x * v.x + g.y * v.y + g.z * v.z + g.w, b.x * v.x + b.y * v.y + b.z * v.z + b.w), vec3(0.0), vec3(1.0)));
|
||||
}
|
||||
case OP_COLOR_REPLACE: {
|
||||
let v = enc(c);
|
||||
let t = u.p0.xyz; let sim = u.p0.w;
|
||||
let dd = v - t;
|
||||
let d = sqrt(dd.x * dd.x + dd.y * dd.y + dd.z * dd.z);
|
||||
let k = 1.0 - smoothstep_fx(sim * 0.85, max(sim, 1e-4), d);
|
||||
if k <= 0.0 {
|
||||
return c;
|
||||
}
|
||||
var repl = u.p1.xyz;
|
||||
if u.i1.x == 0u {
|
||||
let rh = u.p2.xyz;
|
||||
repl = hsl_to_rgb(rh.x, rh.y, rgb_to_hsl(v).z);
|
||||
}
|
||||
return dec(v + (repl - v) * k);
|
||||
}
|
||||
default: {
|
||||
return c;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// One output pixel of every op but the running-sum box blur.
|
||||
fn pixel(op: u32, p: vec2<i32>) -> vec4<f32> {
|
||||
let s = size();
|
||||
let pc = vec2<f32>(p) + 0.5;
|
||||
switch op {
|
||||
case OP_INVERT_ALPHA: {
|
||||
let o = ld(p);
|
||||
let a = o.a;
|
||||
let na = 1.0 - a;
|
||||
let k = select(0.0, na / a, a > 1e-6);
|
||||
let blend = u.p0.x;
|
||||
return vec4(o.rgb * k, na * (1.0 - blend) + a * blend);
|
||||
}
|
||||
case OP_ALPHA_ADJUST: {
|
||||
let o = ld(p);
|
||||
let c = select(o.rgb / o.a, vec3(0.0), o.a <= 1e-6);
|
||||
var a = select(o.a, 1.0, u.i1.x != 0u);
|
||||
if u.i1.y != 0u {
|
||||
a = 1.0 - a;
|
||||
}
|
||||
a = clamp(a * u.p0.x, 0.0, 1.0);
|
||||
if u.i1.z != 0u {
|
||||
return vec4(vec3(srgb_to_linear1(a)), 1.0);
|
||||
}
|
||||
return vec4(c * a, a);
|
||||
}
|
||||
case OP_BOX: {
|
||||
// one box pass along x (i1.z = 0) or y (1): mean over 2r+1 pixels, clamped to the
|
||||
// edge (repeat) or skipping pixels outside, as `effects::box_rows`
|
||||
let r = i32(u.i1.x);
|
||||
let repeat = u.i1.y != 0u;
|
||||
let vertical = u.i1.z != 0u;
|
||||
let n = select(s.x, s.y, vertical);
|
||||
let x = select(p.x, p.y, vertical);
|
||||
var acc = vec4(0.0);
|
||||
for (var i = x - r; i <= x + r; i++) {
|
||||
if repeat || (i >= 0 && i < n) {
|
||||
let j = clamp(i, 0, n - 1);
|
||||
acc += ld(select(vec2(j, p.y), vec2(p.x, j), vertical));
|
||||
}
|
||||
}
|
||||
return acc * (1.0 / f32(2 * r + 1));
|
||||
}
|
||||
case OP_DIRECTIONAL: {
|
||||
let steps = u.i1.x;
|
||||
let dx = u.p0.x; let dy = u.p0.y;
|
||||
var acc = vec4(0.0);
|
||||
for (var i = 0u; i < steps; i++) {
|
||||
let t = f32(i) / f32(steps - 1u) - 0.5;
|
||||
acc += sample_clamped(pc.x + dx * t, pc.y + dy * t);
|
||||
}
|
||||
return acc / f32(steps);
|
||||
}
|
||||
case OP_UNSHARP: {
|
||||
// src: the blurred image, aux: the original
|
||||
var o = textureLoad(aux, p, 0);
|
||||
let bq = ld(p);
|
||||
let amount = u.p0.x; let th = u.p0.y;
|
||||
for (var k = 0; k < 3; k++) {
|
||||
let d = o[k] - bq[k];
|
||||
if abs(d) >= th * o.a {
|
||||
o[k] = max(o[k] + d * amount, 0.0);
|
||||
}
|
||||
}
|
||||
return o;
|
||||
}
|
||||
case OP_CROP: {
|
||||
let x0 = u.p0.x; let x1 = u.p0.y; let y0 = u.p0.z; let y1 = u.p0.w; let fe = u.p1.x;
|
||||
let d = min(min(pc.x - x0, x1 - pc.x), min(pc.y - y0, y1 - pc.y));
|
||||
var a: f32;
|
||||
if fe > 0.0 {
|
||||
a = clamp(d / fe, 0.0, 1.0);
|
||||
} else {
|
||||
a = clamp(d + 0.5, 0.0, 1.0);
|
||||
}
|
||||
let o = ld(p);
|
||||
return select(o, o * a, a < 1.0);
|
||||
}
|
||||
case OP_RESAMPLE: {
|
||||
// inverse map p0 = (a b c d), p1 = (e f opacity valid); rect i1 = x0 x1 y0 y1
|
||||
if u.p1.w == 0.0 || u32(p.x) < u.i1.x || u32(p.x) >= u.i1.y || u32(p.y) < u.i1.z || u32(p.y) >= u.i1.w {
|
||||
return vec4(0.0);
|
||||
}
|
||||
let m0 = u.p0; let m1 = u.p1;
|
||||
let uu = m0.x * pc.x + m0.z * pc.y + m1.x;
|
||||
let vv = m0.y * pc.x + m0.w * pc.y + m1.y;
|
||||
if uu < -1.0 || vv < -1.0 || uu > f32(s.x) + 1.0 || vv > f32(s.y) + 1.0 {
|
||||
return vec4(0.0);
|
||||
}
|
||||
return sample_bilinear(uu, vv) * m1.z;
|
||||
}
|
||||
case OP_HFLIP: {
|
||||
return ld(vec2(s.x - 1 - p.x, p.y));
|
||||
}
|
||||
case OP_VFLIP: {
|
||||
return ld(vec2(p.x, s.y - 1 - p.y));
|
||||
}
|
||||
case OP_MIRROR: {
|
||||
let c = u.p0.xy; let nrm = u.p0.zw;
|
||||
let d = (pc.x - c.x) * nrm.x + (pc.y - c.y) * nrm.y;
|
||||
if d > 0.0 {
|
||||
return sample_bilinear(pc.x - 2.0 * d * nrm.x, pc.y - 2.0 * d * nrm.y);
|
||||
}
|
||||
return ld(p);
|
||||
}
|
||||
case OP_OFFSET: {
|
||||
let w = f32(s.x); let h = f32(s.y);
|
||||
// dx, dy in 0..w / 0..h, so the shifted coordinate is within (-w, w + 0.5)
|
||||
var uu = pc.x - u.p0.x;
|
||||
var vv = pc.y - u.p0.y;
|
||||
if uu < 0.0 { uu += w; }
|
||||
if uu >= w { uu -= w; }
|
||||
if vv < 0.0 { vv += h; }
|
||||
if vv >= h { vv -= h; }
|
||||
let q = sample_clamped(uu, vv);
|
||||
let o = ld(p);
|
||||
return q + (o - q) * u.p0.z;
|
||||
}
|
||||
default: {
|
||||
let o = ld(p);
|
||||
let a = o.a;
|
||||
if a <= 1e-6 {
|
||||
return o;
|
||||
}
|
||||
return vec4(color_op(op, o.rgb / a) * a, a);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@compute @workgroup_size(16, 16)
|
||||
fn fx_px(@builtin(global_invocation_id) id: vec3<u32>) {
|
||||
if id.x >= u.i0.y || id.y >= u.i0.z {
|
||||
return;
|
||||
}
|
||||
let p = vec2<i32>(id.xy);
|
||||
textureStore(dst, p, pixel(u.i0.x, p));
|
||||
}
|
||||
|
||||
// A box pass as a running sum along one row (i1.z = 0) or column (1) per invocation: O(1) per
|
||||
// pixel for any radius, the arithmetic of `effects::box_rows`.
|
||||
@compute @workgroup_size(64)
|
||||
fn fx_run(@builtin(global_invocation_id) id: vec3<u32>) {
|
||||
let s = size();
|
||||
let vertical = u.i1.z != 0u;
|
||||
let n = select(s.x, s.y, vertical);
|
||||
let lines = select(s.y, s.x, vertical);
|
||||
let line = i32(id.x);
|
||||
if line >= lines {
|
||||
return;
|
||||
}
|
||||
let r = i32(u.i1.x);
|
||||
let repeat = u.i1.y != 0u;
|
||||
let inv = 1.0 / f32(2 * r + 1);
|
||||
var acc = vec4(0.0);
|
||||
for (var i = -r; i <= r; i++) {
|
||||
if repeat || (i >= 0 && i < n) {
|
||||
let j = clamp(i, 0, n - 1);
|
||||
acc += ld(select(vec2(j, line), vec2(line, j), vertical));
|
||||
}
|
||||
}
|
||||
for (var x = 0; x < n; x++) {
|
||||
textureStore(dst, select(vec2(x, line), vec2(line, x), vertical), acc * inv);
|
||||
let out_i = x - r;
|
||||
let in_i = x + r + 1;
|
||||
if repeat || out_i >= 0 {
|
||||
let j = clamp(out_i, 0, n - 1);
|
||||
acc -= ld(select(vec2(j, line), vec2(line, j), vertical));
|
||||
}
|
||||
if repeat || in_i < n {
|
||||
let j = clamp(in_i, 0, n - 1);
|
||||
acc += ld(select(vec2(j, line), vec2(line, j), vertical));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,491 @@
|
||||
//! GPU-vs-CPU parity of the effect stage (`filmcraft_render::gpufx::FxOp::apply` is the
|
||||
//! reference).
|
||||
//!
|
||||
//! Tolerances:
|
||||
//! - Exact checks ([`gpu_effects_match_cpu_exactly`]): the working image read back as f32 from a
|
||||
//! half-float-representable source (so the source draw is lossless) must match the CPU op to
|
||||
//! [`EXACT`] per value (f32 arithmetic and the GPU's `pow` / `sqrt` differ in the last bits).
|
||||
//! Step functions (Posterize, Color Pass, Leave Color and Change to Color keys, Extract) may
|
||||
//! flip a pixel that lands on a threshold; at most [`FLIPS`] of the values may differ by more.
|
||||
//! - Composited plans ([`gpu_effect_layers_match_cpu_plan`]): the compositor's criterion (p99 of
|
||||
//! the per-pixel max 8-bit channel difference ≤ 6, mean < 1.5) away from layer outlines, as for
|
||||
//! the blend modes.
|
||||
|
||||
use super::blend_tests::{device, interior, ramp_layer, stats8, yuv_frame};
|
||||
use super::*;
|
||||
use filmcraft_geom::{Affine, Vec2};
|
||||
use filmcraft_project::{EffectInstance, ParamValue, find_effect};
|
||||
use filmcraft_render::effects::FxCtx;
|
||||
use filmcraft_render::gpufx::FxOp;
|
||||
use filmcraft_render::plan::{LayerFx, execute_cpu};
|
||||
use filmcraft_time::Tick;
|
||||
|
||||
const EXACT: f32 = 1e-4;
|
||||
const FLIPS: f64 = 0.01;
|
||||
|
||||
fn cx(t: Tick, px_scale: f32) -> FxCtx<'static> {
|
||||
FxCtx { t, px_scale, seconds: 0.0, timecode: "", clip_name: "", project: None, env: None, working: filmcraft_color::WorkingSpace::Rec709 }
|
||||
}
|
||||
|
||||
fn effect(id: &str, params: &[(&str, ParamValue)]) -> EffectInstance {
|
||||
let mut e = find_effect(id).unwrap_or_else(|| panic!("{id}")).instance();
|
||||
for (k, v) in params {
|
||||
e.params.get_mut(*k).unwrap_or_else(|| panic!("{id}.{k}")).value = v.clone();
|
||||
}
|
||||
e
|
||||
}
|
||||
|
||||
fn fl(v: f64) -> ParamValue {
|
||||
ParamValue::Float(v)
|
||||
}
|
||||
fn col(r: f32, g: f32, b: f32) -> ParamValue {
|
||||
ParamValue::Color([r, g, b, 1.0])
|
||||
}
|
||||
fn pt(x: f64, y: f64) -> ParamValue {
|
||||
ParamValue::Vec2(Vec2::new(x, y))
|
||||
}
|
||||
|
||||
fn half(v: f32) -> f32 {
|
||||
super::blend_tests::f16_to_f32(f32_to_f16(v))
|
||||
}
|
||||
|
||||
/// Linear premultiplied test picture, every value representable in half floats: hue sweep along
|
||||
/// x, lightness along y, a grey ramp, black and white columns, super-whites (up to 1.6), and
|
||||
/// alpha 1 / partial / 0 bands.
|
||||
fn picture(w: u32, h: u32) -> Vec<f32> {
|
||||
let mut px = Vec::with_capacity((w * h * 4) as usize);
|
||||
for y in 0..h {
|
||||
for x in 0..w {
|
||||
let fx = x as f32 / (w - 1) as f32;
|
||||
let fy = y as f32 / (h - 1) as f32;
|
||||
let c = match x % 11 {
|
||||
0 => [0.0, 0.0, 0.0],
|
||||
1 => [1.0, 1.0, 1.0],
|
||||
2 => [fy, fy, fy],
|
||||
3 => [1.6 * fy, 1.2 * fy, 0.3],
|
||||
_ => {
|
||||
let rgb = filmcraft_color::hsl_to_rgb(fx, 0.25 + 0.75 * ((y % 5) as f32 / 4.0), 0.1 + 0.8 * fy);
|
||||
rgb.map(filmcraft_color::srgb_to_linear)
|
||||
}
|
||||
};
|
||||
let a = match y % 7 {
|
||||
0 => 0.0,
|
||||
1 | 2 => 0.5 + 0.4 * fx,
|
||||
3 => 0.25,
|
||||
_ => 1.0,
|
||||
};
|
||||
px.extend_from_slice(&[c[0] * a, c[1] * a, c[2] * a, a].map(half));
|
||||
}
|
||||
}
|
||||
px
|
||||
}
|
||||
|
||||
/// Max |cpu − gpu| (finite values required) and the share of values off by more than 0.02.
|
||||
fn compare(cpu: &[f32], gpu: &[f32]) -> (f32, f64) {
|
||||
assert_eq!(cpu.len(), gpu.len());
|
||||
let mut worst = 0f32;
|
||||
let mut flips = 0usize;
|
||||
for (a, b) in cpu.iter().zip(gpu) {
|
||||
assert!(b.is_finite(), "non-finite GPU value");
|
||||
let d = (a - b).abs() / (1.0 + a.abs());
|
||||
if d > 0.02 {
|
||||
flips += 1;
|
||||
} else {
|
||||
worst = worst.max(d);
|
||||
}
|
||||
}
|
||||
(worst, flips as f64 / cpu.len() as f64)
|
||||
}
|
||||
|
||||
/// Effect cases: (effect id, parameters, whether a threshold may flip pixels).
|
||||
fn cases() -> Vec<(&'static str, Vec<(&'static str, ParamValue)>, bool)> {
|
||||
vec![
|
||||
("brightness_contrast", vec![("brightness", fl(30.0)), ("contrast", fl(40.0))], false),
|
||||
("brightness_contrast", vec![("brightness", fl(-100.0)), ("contrast", fl(-100.0))], false),
|
||||
("brightness_contrast", vec![("brightness", fl(100.0)), ("contrast", fl(100.0))], false),
|
||||
("brightness_contrast", vec![("brightness", fl(-20.0)), ("contrast", fl(900.0))], false),
|
||||
("proc_amp", vec![("brightness", fl(10.0)), ("contrast", fl(130.0)), ("hue", fl(200.0)), ("saturation", fl(150.0))], false),
|
||||
("proc_amp", vec![("hue", fl(-400.0)), ("saturation", fl(0.0))], false),
|
||||
("tint", vec![], false),
|
||||
("tint", vec![("black", col(0.2, 0.1, 0.5)), ("white", col(1.0, 0.9, 0.3)), ("amount", fl(60.0))], false),
|
||||
("black_white", vec![], false),
|
||||
("color_balance", vec![("shadow_r", fl(50.0)), ("mid_g", fl(-40.0)), ("hi_b", fl(80.0)), ("shadow_b", fl(-100.0)), ("hi_r", fl(100.0))], false),
|
||||
("color_balance", vec![("mid_r", fl(70.0)), ("hi_g", fl(-60.0)), ("preserve", ParamValue::Bool(true))], false),
|
||||
("leave_color", vec![("amount", fl(80.0)), ("tolerance", fl(20.0)), ("softness", fl(30.0))], true),
|
||||
("leave_color", vec![("amount", fl(100.0)), ("color", col(0.1, 0.8, 0.2)), ("tolerance", fl(0.0)), ("softness", fl(0.0))], true),
|
||||
("change_to_color", vec![], true),
|
||||
("change_to_color", vec![("hue_tol", fl(30.0)), ("softness", fl(10.0)), ("to", col(0.9, 0.9, 0.1))], true),
|
||||
("color_pass", vec![("similarity", fl(30.0))], true),
|
||||
("color_pass", vec![("similarity", fl(45.0)), ("reverse", ParamValue::Bool(true)), ("color", col(0.2, 0.4, 0.9))], true),
|
||||
("gamma_correction", vec![("gamma", fl(5.0))], false),
|
||||
("gamma_correction", vec![("gamma", fl(28.0))], false),
|
||||
("gamma_correction", vec![("gamma", fl(0.0))], false),
|
||||
("gamma_correction", vec![("gamma", fl(-3.0))], false),
|
||||
("levels", vec![("in_black", fl(20.0)), ("in_white", fl(230.0)), ("out_black", fl(10.0)), ("out_white", fl(240.0)), ("gamma", fl(150.0))], false),
|
||||
("levels", vec![("in_black", fl(200.0)), ("in_white", fl(0.0)), ("gamma", fl(5.0)), ("out_white", fl(-50.0))], true),
|
||||
("extract", vec![("black", fl(40.0)), ("white", fl(200.0)), ("softness", fl(30.0))], true),
|
||||
("extract", vec![("black", fl(90.0)), ("white", fl(160.0)), ("invert", ParamValue::Bool(true))], true),
|
||||
("invert", vec![("blend", fl(30.0))], false),
|
||||
("invert", vec![("channel", ParamValue::Choice(1))], false),
|
||||
("invert", vec![("channel", ParamValue::Choice(2)), ("blend", fl(70.0))], false),
|
||||
("invert", vec![("channel", ParamValue::Choice(3))], false),
|
||||
("invert", vec![("channel", ParamValue::Choice(4)), ("blend", fl(20.0))], false),
|
||||
("invert", vec![("channel", ParamValue::Choice(9))], false),
|
||||
("posterize", vec![("levels", fl(2.0))], true),
|
||||
("posterize", vec![("levels", fl(7.0))], true),
|
||||
("posterize", vec![("levels", fl(255.0))], true),
|
||||
("gaussian_blur", vec![("blurriness", fl(0.0))], false),
|
||||
("gaussian_blur", vec![("blurriness", fl(5.0))], false),
|
||||
("gaussian_blur", vec![("blurriness", fl(5.0)), ("repeat_edge", ParamValue::Bool(true))], false),
|
||||
("gaussian_blur", vec![("blurriness", fl(80.0))], false),
|
||||
("gaussian_blur", vec![("blurriness", fl(80.0)), ("repeat_edge", ParamValue::Bool(true))], false),
|
||||
("gaussian_blur", vec![("blurriness", fl(12.0)), ("dimensions", ParamValue::Choice(1))], false),
|
||||
("gaussian_blur", vec![("blurriness", fl(12.0)), ("dimensions", ParamValue::Choice(2)), ("repeat_edge", ParamValue::Bool(true))], false),
|
||||
("gaussian_blur", vec![("blurriness", fl(3000.0)), ("repeat_edge", ParamValue::Bool(true))], false),
|
||||
("gaussian_blur", vec![("blurriness", fl(1e9))], false),
|
||||
("gaussian_blur", vec![("blurriness", fl(-50.0))], false),
|
||||
("camera_blur", vec![("percent", fl(20.0))], false),
|
||||
("directional_blur", vec![("length", fl(10.0)), ("direction", fl(30.0))], false),
|
||||
("directional_blur", vec![("length", fl(0.2))], false),
|
||||
("directional_blur", vec![("length", fl(1000.0)), ("direction", fl(-100.0))], false),
|
||||
("sharpen", vec![("amount", fl(80.0))], false),
|
||||
("sharpen", vec![("amount", fl(4000.0))], false),
|
||||
("unsharp_mask", vec![("amount", fl(150.0)), ("radius", fl(3.0)), ("threshold", fl(10.0))], true),
|
||||
("unsharp_mask", vec![("radius", fl(0.1))], false),
|
||||
("unsharp_mask", vec![("amount", fl(300.0)), ("radius", fl(250.0))], false),
|
||||
("crop", vec![("left", fl(10.0)), ("top", fl(5.0)), ("right", fl(20.0)), ("bottom", fl(15.0))], false),
|
||||
("crop", vec![("left", fl(10.0)), ("top", fl(5.0)), ("right", fl(20.0)), ("bottom", fl(15.0)), ("feather", fl(12.0))], false),
|
||||
("crop", vec![("left", fl(12.0)), ("top", fl(5.0)), ("right", fl(20.0)), ("zoom", ParamValue::Bool(true))], false),
|
||||
("crop", vec![("left", fl(70.0)), ("right", fl(70.0)), ("feather", fl(5000.0))], false),
|
||||
("edge_feather", vec![("amount", fl(30.0))], false),
|
||||
("transform", vec![("scale_height", fl(150.0)), ("rotation", fl(30.0)), ("position", pt(30.0, 20.0)), ("opacity", fl(60.0))], false),
|
||||
(
|
||||
"transform",
|
||||
vec![
|
||||
("uniform_scale", ParamValue::Bool(false)),
|
||||
("scale_width", fl(80.0)),
|
||||
("scale_height", fl(120.0)),
|
||||
("skew", fl(20.0)),
|
||||
("skew_axis", fl(45.0)),
|
||||
("anchor", pt(10.0, 30.0)),
|
||||
],
|
||||
false,
|
||||
),
|
||||
("horizontal_flip", vec![], false),
|
||||
("vertical_flip", vec![], false),
|
||||
("mirror", vec![("center", pt(20.0, 10.0)), ("angle", fl(30.0))], false),
|
||||
("mirror", vec![("angle", fl(200.0))], false),
|
||||
("offset", vec![("shift", pt(10.0, 30.0)), ("blend", fl(25.0))], false),
|
||||
("offset", vec![("shift", pt(-1e5, 7.25))], false),
|
||||
("asc_cdl", vec![("r_slope", fl(1.4)), ("g_offset", fl(-0.1)), ("b_power", fl(2.2)), ("saturation", fl(1.6))], false),
|
||||
("asc_cdl", vec![("r_power", fl(0.0)), ("g_slope", fl(0.0)), ("saturation", fl(0.0))], false),
|
||||
("channel_mix", vec![("rr", fl(50.0)), ("rg", fl(60.0)), ("gb", fl(-80.0)), ("bc", fl(30.0))], false),
|
||||
("channel_mix", vec![("rb", fl(120.0)), ("monochrome", ParamValue::Bool(true))], false),
|
||||
("color_replace", vec![("similarity", fl(40.0))], true),
|
||||
("color_replace", vec![("similarity", fl(25.0)), ("solid", ParamValue::Bool(true)), ("target", col(0.3, 0.6, 0.2))], true),
|
||||
("color_replace", vec![("similarity", fl(0.0))], true),
|
||||
("alpha_adjust", vec![("opacity", fl(60.0))], false),
|
||||
("alpha_adjust", vec![("ignore", ParamValue::Bool(true)), ("invert", ParamValue::Bool(true))], false),
|
||||
("alpha_adjust", vec![("invert", ParamValue::Bool(true)), ("mask_only", ParamValue::Bool(true)), ("opacity", fl(150.0))], false),
|
||||
]
|
||||
}
|
||||
|
||||
/// Every effect case on the GPU against its CPU reference, on the working image read back as f32.
|
||||
#[test]
|
||||
fn gpu_effects_match_cpu_exactly() {
|
||||
let Some((dev, q)) = device() else {
|
||||
eprintln!("no GPU adapter; skipping");
|
||||
return;
|
||||
};
|
||||
let mut c = GpuCompositor::new(&dev, &q);
|
||||
let (w, h) = (67u32, 41u32);
|
||||
let px = picture(w, h);
|
||||
let frame = VideoFrame::rgba_f32(w, h, px.clone());
|
||||
let mut failures = Vec::new();
|
||||
for (id, params, steps) in cases() {
|
||||
let e = effect(id, ¶ms);
|
||||
let op = FxOp::eval(&e, &cx(Tick::ZERO, 1.0), w as usize, h as usize).unwrap_or_else(|| panic!("{id} has no GPU op"));
|
||||
assert!(op.gpu_ok(), "{id} {params:?}: not GPU-capable");
|
||||
let mut cpu = filmcraft_render::Image { w: w as usize, h: h as usize, px: px.clone() };
|
||||
op.apply(&mut cpu);
|
||||
let fx = LayerFx { size: (w, h), decimation: 1, ops: vec![op] };
|
||||
let (_, _, gpu) = c.effect_image(&frame, &fx).expect("effect image");
|
||||
let (worst, flips) = compare(&cpu.px, &gpu);
|
||||
eprintln!("{id} {params:?}: max rel diff {worst:.2e}, {:.3}% flipped", flips * 100.0);
|
||||
if worst > EXACT || flips > if steps { FLIPS } else { 0.0 } {
|
||||
failures.push(format!("{id} {params:?}: {worst:.2e}, flips {flips:.4}"));
|
||||
}
|
||||
}
|
||||
assert!(failures.is_empty(), "GPU effects differ from the CPU:\n{}", failures.join("\n"));
|
||||
}
|
||||
|
||||
/// Chains run in order (and Unsharp keeps its original while blurring, between other blurs).
|
||||
#[test]
|
||||
fn gpu_effect_chains_match_cpu() {
|
||||
let Some((dev, q)) = device() else {
|
||||
eprintln!("no GPU adapter; skipping");
|
||||
return;
|
||||
};
|
||||
let mut c = GpuCompositor::new(&dev, &q);
|
||||
let (w, h) = (96u32, 54u32);
|
||||
let px = picture(w, h);
|
||||
let frame = VideoFrame::rgba_f32(w, h, px.clone());
|
||||
let chains: Vec<Vec<EffectInstance>> = vec![
|
||||
vec![
|
||||
effect("brightness_contrast", &[("brightness", fl(20.0)), ("contrast", fl(30.0))]),
|
||||
effect("gaussian_blur", &[("blurriness", fl(8.0))]),
|
||||
effect("tint", &[("amount", fl(50.0))]),
|
||||
],
|
||||
vec![
|
||||
effect("gaussian_blur", &[("blurriness", fl(90.0)), ("repeat_edge", ParamValue::Bool(true))]),
|
||||
effect("unsharp_mask", &[("amount", fl(200.0)), ("radius", fl(4.0))]),
|
||||
effect("gaussian_blur", &[("blurriness", fl(3.0)), ("dimensions", ParamValue::Choice(1))]),
|
||||
effect("sharpen", &[("amount", fl(50.0))]),
|
||||
],
|
||||
vec![
|
||||
effect("crop", &[("left", fl(10.0)), ("bottom", fl(20.0)), ("feather", fl(4.0))]),
|
||||
effect("transform", &[("rotation", fl(-15.0)), ("scale_height", fl(110.0))]),
|
||||
effect("horizontal_flip", &[]),
|
||||
effect("levels", &[("in_black", fl(15.0)), ("gamma", fl(80.0))]),
|
||||
effect("offset", &[("shift", pt(70.0, 10.0))]),
|
||||
effect("invert", &[("channel", ParamValue::Choice(4))]),
|
||||
effect("directional_blur", &[("length", fl(6.0)), ("direction", fl(90.0))]),
|
||||
],
|
||||
];
|
||||
for chain in chains {
|
||||
let ops: Vec<FxOp> = chain.iter().filter_map(|e| FxOp::eval(e, &cx(Tick::ZERO, 1.0), w as usize, h as usize)).collect();
|
||||
assert!(ops.iter().all(FxOp::gpu_ok));
|
||||
let mut cpu = filmcraft_render::Image { w: w as usize, h: h as usize, px: px.clone() };
|
||||
for op in &ops {
|
||||
op.apply(&mut cpu);
|
||||
}
|
||||
let names: Vec<&str> = chain.iter().map(|e| e.effect.as_str()).collect();
|
||||
let (_, _, gpu) = c.effect_image(&frame, &LayerFx { size: (w, h), decimation: 1, ops }).expect("effect image");
|
||||
let (worst, flips) = compare(&cpu.px, &gpu);
|
||||
eprintln!("{names:?}: max rel diff {worst:.2e}, {:.3}% flipped", flips * 100.0);
|
||||
assert!(worst < EXACT * 2.0 && flips == 0.0, "{names:?}: {worst}, {flips}");
|
||||
}
|
||||
}
|
||||
|
||||
/// Keyframed parameters are evaluated at the layer's time on the CPU: the GPU result follows them.
|
||||
#[test]
|
||||
fn keyframed_effects_follow_time() {
|
||||
let Some((dev, q)) = device() else {
|
||||
eprintln!("no GPU adapter; skipping");
|
||||
return;
|
||||
};
|
||||
let mut c = GpuCompositor::new(&dev, &q);
|
||||
let (w, h) = (48u32, 32u32);
|
||||
let px = picture(w, h);
|
||||
let frame = VideoFrame::rgba_f32(w, h, px.clone());
|
||||
let mut blur = effect("gaussian_blur", &[]);
|
||||
let mut bc = effect("brightness_contrast", &[]);
|
||||
let p = blur.params.get_mut("blurriness").expect("blurriness");
|
||||
p.put_keyframe(Tick::ZERO, fl(0.0));
|
||||
p.put_keyframe(Tick::from_seconds_f64(1.0), fl(30.0));
|
||||
let p = bc.params.get_mut("brightness").expect("brightness");
|
||||
p.put_keyframe(Tick::ZERO, fl(-50.0));
|
||||
p.put_keyframe(Tick::from_seconds_f64(1.0), fl(60.0));
|
||||
let mut last: Option<Vec<f32>> = None;
|
||||
for s in [0.0, 0.3, 0.75, 1.0] {
|
||||
let cx = cx(Tick::from_seconds_f64(s), 1.0);
|
||||
let ops: Vec<FxOp> = [&bc, &blur].iter().filter_map(|e| FxOp::eval(e, &cx, w as usize, h as usize)).collect();
|
||||
let mut cpu = filmcraft_render::Image { w: w as usize, h: h as usize, px: px.clone() };
|
||||
for op in &ops {
|
||||
op.apply(&mut cpu);
|
||||
}
|
||||
let (_, _, gpu) = c.effect_image(&frame, &LayerFx { size: (w, h), decimation: 1, ops }).expect("effect image");
|
||||
let (worst, flips) = compare(&cpu.px, &gpu);
|
||||
assert!(worst < EXACT && flips == 0.0, "t = {s}: {worst}, {flips}");
|
||||
assert_ne!(last.as_ref(), Some(&gpu), "t = {s}: the keyframes change the picture");
|
||||
last = Some(gpu);
|
||||
}
|
||||
}
|
||||
|
||||
/// Hostile values: non-finite parameters stay on the CPU (`gpu_ok` is false), huge radii and
|
||||
/// sizes are capped as on the CPU and run in bounded time on the GPU.
|
||||
#[test]
|
||||
fn hostile_parameters_are_bounded() {
|
||||
let (w, h) = (40usize, 24usize);
|
||||
let nan = [
|
||||
effect("brightness_contrast", &[("brightness", fl(f64::NAN))]),
|
||||
effect("gaussian_blur", &[("blurriness", fl(f64::INFINITY))]),
|
||||
effect("directional_blur", &[("length", fl(f64::NAN))]),
|
||||
effect("transform", &[("rotation", fl(f64::NAN))]),
|
||||
effect("crop", &[("left", fl(f64::NAN))]),
|
||||
effect("offset", &[("shift", pt(f64::INFINITY, 0.0))]),
|
||||
effect("levels", &[("in_black", fl(f64::NEG_INFINITY))]),
|
||||
effect("transform", &[("scale_height", fl(1e300))]),
|
||||
// minifying resamples (the CPU's mip path) stay on the CPU
|
||||
effect("transform", &[("scale_height", fl(20.0))]),
|
||||
effect("crop", &[("left", fl(-300.0)), ("zoom", ParamValue::Bool(true))]),
|
||||
];
|
||||
for e in &nan {
|
||||
let op = FxOp::eval(e, &cx(Tick::ZERO, 1.0), w, h);
|
||||
// gaussian with an infinite radius is capped (no blur), which the GPU runs
|
||||
if e.effect == "gaussian_blur" {
|
||||
assert!(op.as_ref().is_some_and(FxOp::gpu_ok));
|
||||
continue;
|
||||
}
|
||||
assert!(op.as_ref().is_some_and(|o| !o.gpu_ok()), "{}: {op:?}", e.effect);
|
||||
// and the CPU reference copes
|
||||
let mut img = filmcraft_render::Image::filled(w, h, [0.3, 0.2, 0.1, 1.0]);
|
||||
if let Some(op) = op {
|
||||
op.apply(&mut img);
|
||||
}
|
||||
}
|
||||
// huge radii: box radii capped by the image size
|
||||
let op = FxOp::eval(&effect("gaussian_blur", &[("blurriness", fl(1e30))]), &cx(Tick::ZERO, 1.0), w, h);
|
||||
let Some(FxOp::Gaussian { rx, ry, .. }) = op else { panic!("{op:?}") };
|
||||
assert!(rx.iter().chain(&ry).all(|r| *r as usize <= 4 * w.max(h).max(8)), "{rx:?} {ry:?}");
|
||||
let Some((dev, q)) = device() else {
|
||||
eprintln!("no GPU adapter; skipping");
|
||||
return;
|
||||
};
|
||||
let mut c = GpuCompositor::new(&dev, &q);
|
||||
let frame = VideoFrame::rgba_f32(w as u32, h as u32, picture(w as u32, h as u32));
|
||||
let ops = vec![
|
||||
FxOp::eval(&effect("gaussian_blur", &[("blurriness", fl(1e30))]), &cx(Tick::ZERO, 1.0), w, h).expect("op"),
|
||||
FxOp::eval(&effect("unsharp_mask", &[("radius", fl(1e30)), ("amount", fl(1e30))]), &cx(Tick::ZERO, 1.0), w, h).expect("op"),
|
||||
FxOp::eval(&effect("directional_blur", &[("length", fl(1e30))]), &cx(Tick::ZERO, 1.0), w, h).expect("op"),
|
||||
];
|
||||
let t0 = std::time::Instant::now();
|
||||
let (_, _, gpu) = c.effect_image(&frame, &LayerFx { size: (w as u32, h as u32), decimation: 1, ops }).expect("effect image");
|
||||
assert!(t0.elapsed().as_secs() < 30);
|
||||
assert_eq!(gpu.len(), w * h * 4);
|
||||
// a working image larger than any texture is refused, not attempted
|
||||
assert!(c.effect_image(&frame, &LayerFx { size: (1 << 20, 4), decimation: 1, ops: vec![FxOp::BlackWhite] }).is_none());
|
||||
}
|
||||
|
||||
/// Layers with effect chains in a composited plan — YUV and RGBA sources, a decimated working
|
||||
/// image, rotated / scaled placement, partial opacity and alpha, blend modes — match the CPU plan
|
||||
/// executor.
|
||||
#[test]
|
||||
fn gpu_effect_layers_match_cpu_plan() {
|
||||
let Some((dev, q)) = device() else {
|
||||
eprintln!("no GPU adapter; skipping");
|
||||
return;
|
||||
};
|
||||
let mut c = GpuCompositor::new(&dev, &q);
|
||||
let (w, h) = (320usize, 180usize);
|
||||
let ops = |list: &[EffectInstance], lw: u32, lh: u32, px_scale: f32| -> Vec<FxOp> {
|
||||
list.iter().filter_map(|e| FxOp::eval(e, &cx(Tick::ZERO, px_scale), lw as usize, lh as usize)).collect()
|
||||
};
|
||||
let chain_a = [
|
||||
effect("brightness_contrast", &[("brightness", fl(15.0)), ("contrast", fl(25.0))]),
|
||||
effect("gaussian_blur", &[("blurriness", fl(6.0))]),
|
||||
effect("tint", &[("amount", fl(40.0))]),
|
||||
];
|
||||
let chain_b = [
|
||||
effect("crop", &[("left", fl(8.0)), ("right", fl(12.0)), ("feather", fl(6.0))]),
|
||||
effect("unsharp_mask", &[("amount", fl(120.0)), ("radius", fl(2.0))]),
|
||||
effect("color_balance", &[("hi_b", fl(60.0)), ("shadow_r", fl(40.0))]),
|
||||
];
|
||||
let chain_c = [effect("mirror", &[("angle", fl(90.0))]), effect("gaussian_blur", &[("blurriness", fl(50.0)), ("dimensions", ParamValue::Choice(1))])];
|
||||
let ramp = ramp_layer(160, 120, 0);
|
||||
for mode in [Blend::Normal, Blend::Screen, Blend::Multiply, Blend::Overlay] {
|
||||
let plan = FramePlan::Layers {
|
||||
width: w,
|
||||
height: h,
|
||||
layers: vec![
|
||||
// YUV source decoded at half size (decimation 2) with effects, filling the output
|
||||
PlanLayer {
|
||||
frame: yuv_frame(640, 360),
|
||||
matrix: Affine::IDENTITY,
|
||||
opacity: 1.0,
|
||||
blend: Blend::Normal,
|
||||
fx: Some(Arc::new(LayerFx { size: (320, 180), decimation: 2, ops: ops(&chain_c, 320, 180, 0.5) })),
|
||||
},
|
||||
PlanLayer {
|
||||
frame: ramp.clone(),
|
||||
matrix: Affine::motion(Vec2::new(170.0, 95.0), Vec2::new(1.1, 1.1), 17.0, Vec2::new(80.0, 60.0)),
|
||||
opacity: 0.8,
|
||||
blend: mode,
|
||||
fx: Some(Arc::new(LayerFx { size: (160, 120), decimation: 1, ops: ops(&chain_a, 160, 120, 1.0) })),
|
||||
},
|
||||
PlanLayer::new(ramp.clone(), Affine::translate(10.0, 20.0), 0.5, Blend::Normal),
|
||||
PlanLayer {
|
||||
frame: yuv_frame(320, 180),
|
||||
matrix: Affine::motion(Vec2::new(250.0, 70.0), Vec2::new(0.45, 0.45), -9.0, Vec2::new(160.0, 90.0)),
|
||||
opacity: 0.7,
|
||||
blend: mode,
|
||||
fx: Some(Arc::new(LayerFx { size: (320, 180), decimation: 1, ops: ops(&chain_b, 320, 180, 1.0) })),
|
||||
},
|
||||
],
|
||||
};
|
||||
let cpu = execute_cpu(&plan).over_black_rgba8();
|
||||
c.composite(&plan);
|
||||
let (_, _, gpu) = c.read_output().expect("readback");
|
||||
let keep = interior(&plan);
|
||||
let (p99, mean, max) = stats8(&cpu, &gpu, &keep);
|
||||
let (ap99, amean, _) = stats8(&cpu, &gpu, &vec![true; keep.len()]);
|
||||
eprintln!("{mode:?}: interior 8-bit p99 {p99} mean {mean:.3} max {max}; with edges p99 {ap99} mean {amean:.3}");
|
||||
assert!(p99 <= 6 && mean < 1.5, "{mode:?}: p99 {p99}, mean {mean}");
|
||||
}
|
||||
}
|
||||
|
||||
/// Plans without effects never create working textures.
|
||||
#[test]
|
||||
fn plain_layers_skip_the_effect_stage() {
|
||||
let Some((dev, q)) = device() else {
|
||||
eprintln!("no GPU adapter; skipping");
|
||||
return;
|
||||
};
|
||||
let mut c = GpuCompositor::new(&dev, &q);
|
||||
let plan = FramePlan::Layers { width: 64, height: 36, layers: vec![PlanLayer::new(yuv_frame(64, 36), Affine::IDENTITY, 1.0, Blend::Normal)] };
|
||||
c.composite(&plan);
|
||||
assert!(c.fx.as_ref().is_some_and(fx::FxStage::is_idle));
|
||||
let fx = LayerFx { size: (64, 36), decimation: 1, ops: vec![FxOp::BlackWhite] };
|
||||
let with = FramePlan::Layers {
|
||||
width: 64,
|
||||
height: 36,
|
||||
layers: vec![PlanLayer { fx: Some(Arc::new(fx)), ..PlanLayer::new(yuv_frame(64, 36), Affine::IDENTITY, 1.0, Blend::Normal) }],
|
||||
};
|
||||
c.composite(&with);
|
||||
assert!(!c.fx.as_ref().is_some_and(fx::FxStage::is_idle));
|
||||
// and they are released once no layer needs them
|
||||
c.composite(&plan);
|
||||
assert!(c.fx.as_ref().is_some_and(fx::FxStage::is_idle));
|
||||
}
|
||||
|
||||
/// Without an effect stage (devices without compute shaders) the compositor renders a layer's
|
||||
/// effects on the CPU itself: the picture is the same.
|
||||
#[test]
|
||||
fn layers_with_effects_fall_back_to_the_cpu_without_a_stage() {
|
||||
let Some((dev, q)) = device() else {
|
||||
eprintln!("no GPU adapter; skipping");
|
||||
return;
|
||||
};
|
||||
let ops: Vec<FxOp> = [effect("tint", &[("amount", fl(70.0))]), effect("gaussian_blur", &[("blurriness", fl(10.0))])]
|
||||
.iter()
|
||||
.filter_map(|e| FxOp::eval(e, &cx(Tick::ZERO, 1.0), 160, 120))
|
||||
.collect();
|
||||
let plan = FramePlan::Layers {
|
||||
width: 200,
|
||||
height: 150,
|
||||
layers: vec![
|
||||
PlanLayer::new(yuv_frame(200, 150), Affine::IDENTITY, 1.0, Blend::Normal),
|
||||
PlanLayer {
|
||||
fx: Some(Arc::new(LayerFx { size: (160, 120), decimation: 1, ops })),
|
||||
..PlanLayer::new(
|
||||
ramp_layer(160, 120, 2),
|
||||
Affine::motion(Vec2::new(100.0, 75.0), Vec2::new(0.9, 0.9), 8.0, Vec2::new(80.0, 60.0)),
|
||||
0.8,
|
||||
Blend::Screen,
|
||||
)
|
||||
},
|
||||
],
|
||||
};
|
||||
let cpu = execute_cpu(&plan).over_black_rgba8();
|
||||
for stage in [true, false] {
|
||||
let mut c = GpuCompositor::new(&dev, &q);
|
||||
if !stage {
|
||||
c.fx = None;
|
||||
}
|
||||
c.composite(&plan);
|
||||
let (_, _, gpu) = c.read_output().expect("readback");
|
||||
let (p99, mean, _) = stats8(&cpu, &gpu, &interior(&plan));
|
||||
eprintln!("effect stage {stage}: interior p99 {p99}, mean {mean:.3}");
|
||||
assert!(p99 <= 6 && mean < 1.5, "stage {stage}: p99 {p99}, mean {mean}");
|
||||
}
|
||||
}
|
||||
+207
-32
@@ -14,6 +14,12 @@
|
||||
//! with the CPU reference's math (`filmcraft_render::blend::composite`: sRGB-encoded straight
|
||||
//! colour, W3C formulas, back to linear) and writes the result. Only those layers pay the copy.
|
||||
//!
|
||||
//! A layer carrying standard effects ([`filmcraft_render::plan::LayerFx`]) goes through the
|
||||
//! effect stage ([`fx`]) first: its source is drawn into an `Rgba32Float` working image, the
|
||||
//! effects run as compute passes in WGSL with the CPU reference's math
|
||||
//! (`filmcraft_render::gpufx::FxOp::apply`), and the result is drawn like any RGBA layer. Layers
|
||||
//! without effects take the single draw above.
|
||||
//!
|
||||
//! Layers that need converting before upload (linear f32 RGBA from CPU-rendered layers, 16-bit
|
||||
//! YUV such as ProRes) are converted to half floats by [`prepare`], which frame workers run off the
|
||||
//! UI thread; [`GpuCompositor::composite_prepared`] then only copies bytes into textures.
|
||||
@@ -32,6 +38,7 @@ use filmcraft_frame::{Chroma, PixelData, VideoFrame};
|
||||
use filmcraft_render::Blend;
|
||||
use filmcraft_render::plan::{FramePlan, PlanLayer};
|
||||
|
||||
pub mod fx;
|
||||
pub mod lut;
|
||||
pub mod mask;
|
||||
pub use lut::GpuLut;
|
||||
@@ -70,6 +77,9 @@ pub struct GpuCompositor {
|
||||
uploads: HashMap<(usize, u32, u32), Uploaded>,
|
||||
clock: u64,
|
||||
dummy: wgpu::TextureView,
|
||||
/// The effect stage (None on devices without compute shaders or storage textures: layers
|
||||
/// with effects are then rendered on the CPU here).
|
||||
fx: Option<fx::FxStage>,
|
||||
/// Total bytes uploaded (stats).
|
||||
pub uploaded_bytes: u64,
|
||||
}
|
||||
@@ -307,6 +317,7 @@ impl GpuCompositor {
|
||||
view_formats: &[],
|
||||
});
|
||||
let dummy = dummy_tex.create_view(&Default::default());
|
||||
let fx = fx::FxStage::supported(device).then(|| fx::FxStage::new(device, &shader, &layer_bgl));
|
||||
Self {
|
||||
device: device.clone(),
|
||||
queue: queue.clone(),
|
||||
@@ -322,6 +333,7 @@ impl GpuCompositor {
|
||||
uploads: HashMap::new(),
|
||||
clock: 0,
|
||||
dummy,
|
||||
fx,
|
||||
uploaded_bytes: 0,
|
||||
}
|
||||
}
|
||||
@@ -446,18 +458,21 @@ impl GpuCompositor {
|
||||
key
|
||||
}
|
||||
|
||||
fn uniforms(&self, l: &PlanLayer, key: (usize, u32, u32), out: (u32, u32)) -> [f32; 28] {
|
||||
let up = &self.uploads[&key];
|
||||
let f = &l.frame;
|
||||
let m = &l.matrix;
|
||||
let (kr, kb) = f.color.matrix.kr_kb();
|
||||
let bits_scale = up.code_scale / 255.0; // code units relative to 8-bit
|
||||
let (yo, ys, co, cs) = match (f.color.range, up.kind) {
|
||||
/// The source description of an uploaded frame.
|
||||
fn src_info(&self, f: &VideoFrame, key: (usize, u32, u32)) -> Option<SrcInfo> {
|
||||
let up = self.uploads.get(&key)?;
|
||||
Some(SrcInfo { kind: up.kind, code_scale: up.code_scale, chroma: up.chroma, size: (f.width, f.height), color: f.color })
|
||||
}
|
||||
|
||||
fn uniforms(src: &SrcInfo, m: &filmcraft_geom::Affine, opacity: f32, blend: Blend, out: (u32, u32)) -> [f32; 28] {
|
||||
let (kr, kb) = src.color.matrix.kr_kb();
|
||||
let bits_scale = src.code_scale / 255.0; // code units relative to 8-bit
|
||||
let (yo, ys, co, cs) = match (src.color.range, src.kind) {
|
||||
(Range::Limited, 2) => (16.0 * bits_scale, 219.0 * bits_scale, 128.0 * bits_scale, 224.0 * bits_scale),
|
||||
(Range::Full, 2) => (0.0, up.code_scale - 1.0, up.code_scale / 2.0, up.code_scale - 1.0),
|
||||
(Range::Full, 2) => (0.0, src.code_scale - 1.0, src.code_scale / 2.0, src.code_scale - 1.0),
|
||||
_ => (0.0, 1.0, 0.0, 1.0),
|
||||
};
|
||||
let transfer = match f.color.transfer {
|
||||
let transfer = match src.color.transfer {
|
||||
Transfer::Linear => 1.0,
|
||||
Transfer::Pq => 2.0,
|
||||
Transfer::Hlg => 3.0,
|
||||
@@ -478,12 +493,12 @@ impl GpuCompositor {
|
||||
m.f as f32,
|
||||
out.0 as f32,
|
||||
out.1 as f32,
|
||||
f.width as f32,
|
||||
f.height as f32,
|
||||
up.chroma.0 as f32,
|
||||
up.chroma.1 as f32,
|
||||
l.opacity,
|
||||
up.kind as f32,
|
||||
src.size.0 as f32,
|
||||
src.size.1 as f32,
|
||||
src.chroma.0 as f32,
|
||||
src.chroma.1 as f32,
|
||||
opacity,
|
||||
src.kind as f32,
|
||||
taps,
|
||||
transfer,
|
||||
yo,
|
||||
@@ -492,15 +507,27 @@ impl GpuCompositor {
|
||||
cs,
|
||||
kr,
|
||||
kb,
|
||||
up.code_scale,
|
||||
src.code_scale,
|
||||
footprint.max(1.0),
|
||||
l.blend.index() as f32,
|
||||
blend.index() as f32,
|
||||
0.0,
|
||||
0.0,
|
||||
0.0,
|
||||
]
|
||||
}
|
||||
|
||||
fn uniform_buffer(&self, u: &[f32; 28]) -> wgpu::Buffer {
|
||||
let bytes: Vec<u8> = u.iter().flat_map(|v| v.to_le_bytes()).collect();
|
||||
let buf = self.device.create_buffer(&wgpu::BufferDescriptor {
|
||||
label: Some("layer-u"),
|
||||
size: bytes.len() as u64,
|
||||
usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST,
|
||||
mapped_at_creation: false,
|
||||
});
|
||||
self.queue.write_buffer(&buf, 0, &bytes);
|
||||
buf
|
||||
}
|
||||
|
||||
/// Composite a plan; returns the output view (sRGB, over black) and its size.
|
||||
pub fn composite(&mut self, plan: &FramePlan) -> (wgpu::TextureView, (u32, u32)) {
|
||||
self.composite_prepared(plan, None)
|
||||
@@ -523,7 +550,7 @@ impl GpuCompositor {
|
||||
},
|
||||
None => VideoFrame::rgba_f32(img.w as u32, img.h as u32, img.px.clone()),
|
||||
};
|
||||
owned = [PlanLayer { frame: Arc::new(frame), matrix: filmcraft_geom::Affine::IDENTITY, opacity: 1.0, blend: Blend::Normal }];
|
||||
owned = [PlanLayer { frame: Arc::new(frame), matrix: filmcraft_geom::Affine::IDENTITY, opacity: 1.0, blend: Blend::Normal, fx: None }];
|
||||
(img.w as u32, img.h as u32, &owned[..])
|
||||
}
|
||||
};
|
||||
@@ -535,33 +562,100 @@ impl GpuCompositor {
|
||||
None
|
||||
};
|
||||
let out_view = Self::target(&self.device, &mut self.output, w, h, OUTPUT_FORMAT, wgpu::TextureUsages::COPY_SRC);
|
||||
// Layers whose effects the GPU stage can't run (no stage on this device, or a working image
|
||||
// larger than a texture) are rendered on the CPU here and drawn as plain RGBA layers.
|
||||
let max_side = self.device.limits().max_texture_dimension_2d;
|
||||
let cpu_fx: Vec<Option<PlanLayer>> = layers
|
||||
.iter()
|
||||
.map(|l| {
|
||||
let lfx = l.fx.as_ref()?;
|
||||
if self.fx.is_some() && lfx.size.0.max(lfx.size.1) <= max_side {
|
||||
return None;
|
||||
}
|
||||
let img = filmcraft_render::plan::effect_image(&l.frame, lfx);
|
||||
Some(PlanLayer::new(Arc::new(VideoFrame::rgba_f32(img.w as u32, img.h as u32, img.px)), l.matrix, l.opacity, l.blend))
|
||||
})
|
||||
.collect();
|
||||
let resolved: Vec<&PlanLayer> = layers.iter().zip(&cpu_fx).map(|(l, c)| c.as_ref().unwrap_or(l)).collect();
|
||||
let layers = resolved;
|
||||
let keys: Vec<(usize, u32, u32)> = layers
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, l)| {
|
||||
let p = prep.and_then(|p| if matches!(plan, FramePlan::Image(_)) { p.image.as_ref() } else { p.layers.get(i).and_then(Option::as_ref) });
|
||||
// (texels converted beforehand belong to the plan's own frame)
|
||||
let p = if cpu_fx.get(i).is_some_and(Option::is_some) { None } else { p };
|
||||
self.upload(&l.frame, p)
|
||||
})
|
||||
.collect();
|
||||
// (bind group, None for a fixed-function layer or Some(region of the accumulator to copy
|
||||
// into the backdrop) for a layer that reads the destination; Some(None): off the output)
|
||||
let mut bind_groups = Vec::with_capacity(layers.len());
|
||||
// the effect stage of each layer with effects (recorded right before its draw)
|
||||
let mut jobs: Vec<Option<fx::FxJob>> = Vec::with_capacity(layers.len());
|
||||
if let Some(f) = self.fx.as_mut() {
|
||||
f.begin_frame();
|
||||
}
|
||||
for (l, k) in layers.iter().zip(&keys) {
|
||||
let u = self.uniforms(l, *k, (w, h));
|
||||
let bytes: Vec<u8> = u.iter().flat_map(|v| v.to_le_bytes()).collect();
|
||||
let buf = self.device.create_buffer(&wgpu::BufferDescriptor {
|
||||
label: Some("layer-u"),
|
||||
size: bytes.len() as u64,
|
||||
usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST,
|
||||
mapped_at_creation: false,
|
||||
// (every frame was just uploaded; a missing upload draws transparent texels)
|
||||
let src = self.src_info(&l.frame, *k).unwrap_or(SrcInfo {
|
||||
kind: 1,
|
||||
code_scale: 1.0,
|
||||
chroma: (1, 1),
|
||||
size: (l.frame.width, l.frame.height),
|
||||
color: l.frame.color,
|
||||
});
|
||||
self.queue.write_buffer(&buf, 0, &bytes);
|
||||
let up = &self.uploads[k];
|
||||
let views = self.uploads.get(k).map_or_else(|| [self.dummy.clone(), self.dummy.clone(), self.dummy.clone()], |up| up.views.clone());
|
||||
let job = match &l.fx {
|
||||
Some(lfx) => {
|
||||
// the source drawn onto the working image (box-decimated by n, as the CPU decodes)
|
||||
let n = lfx.decimation.max(1) as f64;
|
||||
let su = Self::uniforms(&src, &filmcraft_geom::Affine::scale(1.0 / n, 1.0 / n), 1.0, Blend::Normal, (lfx.size.0.max(1), lfx.size.1.max(1)));
|
||||
let sbuf = self.uniform_buffer(&su);
|
||||
let sbg = self.device.create_bind_group(&wgpu::BindGroupDescriptor {
|
||||
label: Some("fx-source"),
|
||||
layout: &self.layer_bgl,
|
||||
entries: &[
|
||||
wgpu::BindGroupEntry { binding: 0, resource: sbuf.as_entire_binding() },
|
||||
wgpu::BindGroupEntry { binding: 1, resource: wgpu::BindingResource::TextureView(&views[0]) },
|
||||
wgpu::BindGroupEntry { binding: 2, resource: wgpu::BindingResource::TextureView(&views[1]) },
|
||||
wgpu::BindGroupEntry { binding: 3, resource: wgpu::BindingResource::TextureView(&views[2]) },
|
||||
],
|
||||
});
|
||||
match self.fx.as_mut() {
|
||||
Some(f) => f.job(&self.device, &self.queue, lfx, sbg, &self.dummy),
|
||||
None => None,
|
||||
}
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
// what the layer draws: the effect result (linear premultiplied RGBA) or the frame
|
||||
let (u, tex) = match &job {
|
||||
Some(j) => {
|
||||
let s = SrcInfo { kind: 1, code_scale: 1.0, chroma: l.size(), size: l.size(), color: l.frame.color };
|
||||
(Self::uniforms(&s, &l.matrix, l.opacity, l.blend, (w, h)), [j.result.clone(), self.dummy.clone(), self.dummy.clone()])
|
||||
}
|
||||
None => {
|
||||
// (a layer with effects only gets here if its job failed, which the CPU
|
||||
// fallback above rules out; it is then drawn without them, where the working
|
||||
// image would be)
|
||||
let m = match &l.fx {
|
||||
Some(lfx) => {
|
||||
let n = 1.0 / lfx.decimation.max(1) as f64;
|
||||
l.matrix.then_apply(&filmcraft_geom::Affine::scale(n, n))
|
||||
}
|
||||
None => l.matrix,
|
||||
};
|
||||
(Self::uniforms(&src, &m, l.opacity, l.blend, (w, h)), views)
|
||||
}
|
||||
};
|
||||
jobs.push(job);
|
||||
let buf = self.uniform_buffer(&u);
|
||||
let mut entries = vec![
|
||||
wgpu::BindGroupEntry { binding: 0, resource: buf.as_entire_binding() },
|
||||
wgpu::BindGroupEntry { binding: 1, resource: wgpu::BindingResource::TextureView(&up.views[0]) },
|
||||
wgpu::BindGroupEntry { binding: 2, resource: wgpu::BindingResource::TextureView(&up.views[1]) },
|
||||
wgpu::BindGroupEntry { binding: 3, resource: wgpu::BindingResource::TextureView(&up.views[2]) },
|
||||
wgpu::BindGroupEntry { binding: 1, resource: wgpu::BindingResource::TextureView(&tex[0]) },
|
||||
wgpu::BindGroupEntry { binding: 2, resource: wgpu::BindingResource::TextureView(&tex[1]) },
|
||||
wgpu::BindGroupEntry { binding: 3, resource: wgpu::BindingResource::TextureView(&tex[2]) },
|
||||
];
|
||||
let (layout, region) = match (&backdrop_view, l.blend.reads_destination()) {
|
||||
(Some(b), true) => {
|
||||
@@ -573,6 +667,9 @@ impl GpuCompositor {
|
||||
let bg = self.device.create_bind_group(&wgpu::BindGroupDescriptor { label: Some("layer"), layout, entries: &entries });
|
||||
bind_groups.push((bg, region));
|
||||
}
|
||||
if let Some(f) = self.fx.as_mut() {
|
||||
f.end_frame();
|
||||
}
|
||||
let final_bg = self.device.create_bind_group(&wgpu::BindGroupDescriptor {
|
||||
label: Some("final"),
|
||||
layout: &self.final_bgl,
|
||||
@@ -586,6 +683,11 @@ impl GpuCompositor {
|
||||
let mut cleared = false;
|
||||
let mut i = 0;
|
||||
while i < bind_groups.len() || !cleared {
|
||||
// a layer's effects run right before it is drawn (pooled working textures are reused
|
||||
// by the next layer with effects)
|
||||
if let (Some(Some(job)), Some(f)) = (jobs.get(i), self.fx.as_ref()) {
|
||||
f.record(&mut enc, job);
|
||||
}
|
||||
if let Some((bg, Some(region))) = bind_groups.get(i) {
|
||||
i += 1;
|
||||
let (Some(rect), Some(src), Some(dst)) = (region, &accum_tex, &backdrop_tex) else { continue };
|
||||
@@ -605,7 +707,10 @@ impl GpuCompositor {
|
||||
pass.draw(0..6, 0..1);
|
||||
continue;
|
||||
}
|
||||
let end = bind_groups.iter().skip(i).position(|(_, r)| r.is_some()).map_or(bind_groups.len(), |n| i + n);
|
||||
// a run of fixed-function layers, up to the next one that reads the destination or
|
||||
// has effects to run first
|
||||
let breaks = |j: &usize| bind_groups.get(*j).is_some_and(|(_, r)| r.is_some()) || jobs.get(*j).is_some_and(Option::is_some);
|
||||
let end = (i + 1..bind_groups.len()).find(breaks).unwrap_or(bind_groups.len()).max(i);
|
||||
let load = if cleared { wgpu::LoadOp::Load } else { wgpu::LoadOp::Clear(wgpu::Color::TRANSPARENT) };
|
||||
let mut pass = accum_pass(&mut enc, &accum_view, load);
|
||||
cleared = true;
|
||||
@@ -638,6 +743,64 @@ impl GpuCompositor {
|
||||
(out_view, (w, h))
|
||||
}
|
||||
|
||||
/// Run the effect stage of a layer alone: `frame` decoded into its working image and `fx`
|
||||
/// applied, read back as linear premultiplied RGBA f32 (what `LayerFx` ops produce on the
|
||||
/// CPU from the same frame). None when the working image does not fit in a texture.
|
||||
pub fn effect_image(&mut self, frame: &VideoFrame, fx: &filmcraft_render::plan::LayerFx) -> Option<(u32, u32, Vec<f32>)> {
|
||||
let key = self.upload(frame, None);
|
||||
let src = self.src_info(frame, key)?;
|
||||
let views = self.uploads.get(&key)?.views.clone();
|
||||
let n = fx.decimation.max(1) as f64;
|
||||
let (w, h) = (fx.size.0.max(1), fx.size.1.max(1));
|
||||
let su = Self::uniforms(&src, &filmcraft_geom::Affine::scale(1.0 / n, 1.0 / n), 1.0, Blend::Normal, (w, h));
|
||||
let sbuf = self.uniform_buffer(&su);
|
||||
let sbg = self.device.create_bind_group(&wgpu::BindGroupDescriptor {
|
||||
label: Some("fx-source"),
|
||||
layout: &self.layer_bgl,
|
||||
entries: &[
|
||||
wgpu::BindGroupEntry { binding: 0, resource: sbuf.as_entire_binding() },
|
||||
wgpu::BindGroupEntry { binding: 1, resource: wgpu::BindingResource::TextureView(&views[0]) },
|
||||
wgpu::BindGroupEntry { binding: 2, resource: wgpu::BindingResource::TextureView(&views[1]) },
|
||||
wgpu::BindGroupEntry { binding: 3, resource: wgpu::BindingResource::TextureView(&views[2]) },
|
||||
],
|
||||
});
|
||||
let stage = self.fx.as_mut()?;
|
||||
stage.begin_frame();
|
||||
let job = stage.job(&self.device, &self.queue, fx, sbg, &self.dummy)?;
|
||||
let row = (w * 16).div_ceil(256) * 256;
|
||||
let buf = self.device.create_buffer(&wgpu::BufferDescriptor {
|
||||
label: Some("fx-readback"),
|
||||
size: (row * h) as u64,
|
||||
usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ,
|
||||
mapped_at_creation: false,
|
||||
});
|
||||
let mut enc = self.device.create_command_encoder(&Default::default());
|
||||
stage.record(&mut enc, &job);
|
||||
enc.copy_texture_to_buffer(
|
||||
wgpu::TexelCopyTextureInfo { texture: &job.result_texture, mip_level: 0, origin: wgpu::Origin3d::ZERO, aspect: wgpu::TextureAspect::All },
|
||||
wgpu::TexelCopyBufferInfo { buffer: &buf, layout: wgpu::TexelCopyBufferLayout { offset: 0, bytes_per_row: Some(row), rows_per_image: Some(h) } },
|
||||
wgpu::Extent3d { width: w, height: h, depth_or_array_layers: 1 },
|
||||
);
|
||||
self.queue.submit([enc.finish()]);
|
||||
stage.end_frame();
|
||||
let slice = buf.slice(..);
|
||||
let (tx, rx) = std::sync::mpsc::channel();
|
||||
slice.map_async(wgpu::MapMode::Read, move |r| {
|
||||
let _ = tx.send(r);
|
||||
});
|
||||
let _ = self.device.poll(wgpu::PollType::wait_indefinitely());
|
||||
rx.recv().ok()?.ok()?;
|
||||
let data = slice.get_mapped_range().ok()?;
|
||||
let mut out = Vec::with_capacity((w * h * 4) as usize);
|
||||
for y in 0..h {
|
||||
let r = data.get((y * row) as usize..(y * row + w * 16) as usize)?;
|
||||
out.extend(r.as_chunks::<4>().0.iter().map(|b| f32::from_le_bytes(*b)));
|
||||
}
|
||||
drop(data);
|
||||
buf.unmap();
|
||||
Some((w, h, out))
|
||||
}
|
||||
|
||||
/// Read the output back as RGBA8 (tests / screenshots / thumbnails).
|
||||
pub fn read_output(&self) -> Option<(u32, u32, Vec<u8>)> {
|
||||
let (tex, _, (w, h)) = self.output.as_ref()?;
|
||||
@@ -673,6 +836,16 @@ impl GpuCompositor {
|
||||
}
|
||||
}
|
||||
|
||||
/// What a layer's shader samples: the texture kind (0 RGBA8 sRGB, 1 linear premultiplied RGBA
|
||||
/// float, 2 YUV planes), code scale, chroma plane size, picture size and colour description.
|
||||
struct SrcInfo {
|
||||
kind: u32,
|
||||
code_scale: f32,
|
||||
chroma: (u32, u32),
|
||||
size: (u32, u32),
|
||||
color: filmcraft_color::ColorInfo,
|
||||
}
|
||||
|
||||
/// A render pass drawing into the accumulator.
|
||||
fn accum_pass<'e>(enc: &'e mut wgpu::CommandEncoder, view: &wgpu::TextureView, load: wgpu::LoadOp<wgpu::Color>) -> wgpu::RenderPass<'e> {
|
||||
enc.begin_render_pass(&wgpu::RenderPassDescriptor {
|
||||
@@ -694,7 +867,7 @@ fn accum_pass<'e>(enc: &'e mut wgpu::CommandEncoder, view: &wgpu::TextureView, l
|
||||
/// output with a one-pixel margin; None when it touches none. A non-finite matrix covers it all.
|
||||
fn quad_bounds(l: &PlanLayer, w: u32, h: u32) -> Option<(u32, u32, u32, u32)> {
|
||||
let m = &l.matrix;
|
||||
let (fw, fh) = (l.frame.width as f64, l.frame.height as f64);
|
||||
let (fw, fh) = (l.size().0 as f64, l.size().1 as f64);
|
||||
let pts = [(0.0, 0.0), (fw, 0.0), (0.0, fh), (fw, fh)].map(|(x, y)| (m.a * x + m.c * y + m.e, m.b * x + m.d * y + m.f));
|
||||
if pts.iter().any(|p| !p.0.is_finite() || !p.1.is_finite()) {
|
||||
return Some((0, 0, w, h));
|
||||
@@ -713,4 +886,6 @@ fn quad_bounds(l: &PlanLayer, w: u32, h: u32) -> Option<(u32, u32, u32, u32)> {
|
||||
#[cfg(test)]
|
||||
mod blend_tests;
|
||||
#[cfg(test)]
|
||||
mod fx_tests;
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -45,12 +45,13 @@ fn gpu_matches_cpu_plan() {
|
||||
width: w,
|
||||
height: h,
|
||||
layers: vec![
|
||||
PlanLayer { frame: yuv_frame(640, 360), matrix: Affine::scale(0.5, 0.5), opacity: 1.0, blend: Blend::Normal },
|
||||
PlanLayer { frame: yuv_frame(640, 360), matrix: Affine::scale(0.5, 0.5), opacity: 1.0, blend: Blend::Normal, fx: None },
|
||||
PlanLayer {
|
||||
frame: rgba,
|
||||
matrix: Affine::motion(Vec2::new(200.0, 100.0), Vec2::new(0.4, 0.4), 12.0, Vec2::new(160.0, 90.0)),
|
||||
opacity: 0.7,
|
||||
blend: Blend::Normal,
|
||||
fx: None,
|
||||
},
|
||||
],
|
||||
};
|
||||
@@ -114,8 +115,8 @@ fn prepared_upload_matches_inline_conversion() {
|
||||
width: w as usize,
|
||||
height: h as usize,
|
||||
layers: vec![
|
||||
PlanLayer { frame: yuv16, matrix: Affine::IDENTITY, opacity: 1.0, blend: Blend::Normal },
|
||||
PlanLayer { frame: rgbaf, matrix: Affine::scale(0.5, 0.5), opacity: 0.8, blend: Blend::Normal },
|
||||
PlanLayer { frame: yuv16, matrix: Affine::IDENTITY, opacity: 1.0, blend: Blend::Normal, fx: None },
|
||||
PlanLayer { frame: rgbaf, matrix: Affine::scale(0.5, 0.5), opacity: 0.8, blend: Blend::Normal, fx: None },
|
||||
],
|
||||
};
|
||||
let image = FramePlan::Image(filmcraft_render::Image { w: w as usize, h: h as usize, px: f32_layer });
|
||||
@@ -144,7 +145,8 @@ fn upload_cache_keeps_buffers_alive() {
|
||||
let mut c = GpuCompositor::new(&dev, &q);
|
||||
let px = Arc::new(vec![200u8; 16 * 8 * 4]);
|
||||
let frame = Arc::new(VideoFrame { width: 16, height: 8, data: PixelData::Rgba8(px.clone()), ..(*yuv_frame(16, 8)).clone() });
|
||||
let plan = FramePlan::Layers { width: 16, height: 8, layers: vec![PlanLayer { frame, matrix: Affine::IDENTITY, opacity: 1.0, blend: Blend::Normal }] };
|
||||
let plan =
|
||||
FramePlan::Layers { width: 16, height: 8, layers: vec![PlanLayer { frame, matrix: Affine::IDENTITY, opacity: 1.0, blend: Blend::Normal, fx: None }] };
|
||||
c.composite(&plan);
|
||||
drop(plan);
|
||||
assert!(Arc::strong_count(&px) > 1, "cached upload must own its pixel buffer");
|
||||
|
||||
+46
-303
@@ -34,10 +34,10 @@ pub struct FxCtx<'a> {
|
||||
pub working: filmcraft_color::WorkingSpace,
|
||||
}
|
||||
|
||||
fn f(e: &EffectInstance, id: &str, cx: &FxCtx) -> f32 {
|
||||
pub(crate) fn f(e: &EffectInstance, id: &str, cx: &FxCtx) -> f32 {
|
||||
e.f64_at(id, cx.t) as f32
|
||||
}
|
||||
fn b(e: &EffectInstance, id: &str) -> bool {
|
||||
pub(crate) fn b(e: &EffectInstance, id: &str) -> bool {
|
||||
e.param(id).and_then(|p| p.value.as_bool()).unwrap_or(false)
|
||||
}
|
||||
/// A section switch (missing in older projects = on).
|
||||
@@ -50,17 +50,17 @@ fn text<'e>(e: &'e EffectInstance, id: &str) -> &'e str {
|
||||
_ => "",
|
||||
}
|
||||
}
|
||||
fn choice(e: &EffectInstance, id: &str) -> u32 {
|
||||
pub(crate) fn choice(e: &EffectInstance, id: &str) -> u32 {
|
||||
match e.param(id).map(|p| &p.value) {
|
||||
Some(ParamValue::Choice(c)) => *c,
|
||||
_ => 0,
|
||||
}
|
||||
}
|
||||
fn color(e: &EffectInstance, id: &str, cx: &FxCtx) -> [f32; 4] {
|
||||
pub(crate) fn color(e: &EffectInstance, id: &str, cx: &FxCtx) -> [f32; 4] {
|
||||
e.param(id).and_then(|p| p.value_at(cx.t).as_color()).unwrap_or([1.0; 4])
|
||||
}
|
||||
/// A point param in working-image pixels; NaN components default to the image centre.
|
||||
fn point(e: &EffectInstance, id: &str, cx: &FxCtx, img: &Image) -> Vec2 {
|
||||
pub(crate) fn point(e: &EffectInstance, id: &str, cx: &FxCtx, img: &Image) -> Vec2 {
|
||||
let v = e.param(id).map(|p| p.vec2_at(cx.t)).unwrap_or(Vec2::new(f64::NAN, f64::NAN));
|
||||
Vec2::new(
|
||||
if v.x.is_nan() { img.w as f64 / 2.0 } else { v.x * cx.px_scale as f64 },
|
||||
@@ -93,196 +93,13 @@ pub fn apply(img: &mut Image, e: &EffectInstance, cx: &FxCtx) {
|
||||
if crate::vfx::apply(img, e, cx) {
|
||||
return;
|
||||
}
|
||||
// effects with a GPU implementation: evaluated parameters + CPU reference (`gpufx`)
|
||||
if let Some(op) = crate::gpufx::FxOp::eval(e, cx, img.w, img.h) {
|
||||
op.apply(img);
|
||||
return;
|
||||
}
|
||||
match e.effect.as_str() {
|
||||
"brightness_contrast" => {
|
||||
let br = f(e, "brightness", cx) / 100.0 * 0.4;
|
||||
let co = 1.0 + f(e, "contrast", cx) / 100.0;
|
||||
img.map_rgb(|c, _, _| {
|
||||
let c = enc(c);
|
||||
dec(c.map(|v| (v - 0.5) * co + 0.5 + br))
|
||||
});
|
||||
}
|
||||
"proc_amp" => {
|
||||
let br = f(e, "brightness", cx) / 100.0 * 0.4;
|
||||
let co = f(e, "contrast", cx) / 100.0;
|
||||
let hue = f(e, "hue", cx) / 360.0;
|
||||
let sat = f(e, "saturation", cx) / 100.0;
|
||||
img.map_rgb(|c, _, _| {
|
||||
let c = enc(c);
|
||||
let mut hsl = rgb_to_hsl(c[0], c[1], c[2]);
|
||||
hsl[0] = (hsl[0] + hue).rem_euclid(1.0);
|
||||
hsl[1] = (hsl[1] * sat).clamp(0.0, 1.0);
|
||||
let c = hsl_to_rgb(hsl[0], hsl[1], hsl[2]);
|
||||
dec(c.map(|v| (v - 0.5) * co + 0.5 + br))
|
||||
});
|
||||
}
|
||||
"tint" => {
|
||||
let bl = color(e, "black", cx);
|
||||
let wh = color(e, "white", cx);
|
||||
let amt = f(e, "amount", cx) / 100.0;
|
||||
img.map_rgb(|c, _, _| {
|
||||
let l = linear_to_srgb(luma709(c[0], c[1], c[2]).max(0.0));
|
||||
let t = [bl[0] + (wh[0] - bl[0]) * l, bl[1] + (wh[1] - bl[1]) * l, bl[2] + (wh[2] - bl[2]) * l];
|
||||
let e = enc(c);
|
||||
dec([e[0] + (t[0] - e[0]) * amt, e[1] + (t[1] - e[1]) * amt, e[2] + (t[2] - e[2]) * amt])
|
||||
});
|
||||
}
|
||||
"black_white" => img.map_rgb(|c, _, _| {
|
||||
let l = luma709(c[0], c[1], c[2]);
|
||||
[l, l, l]
|
||||
}),
|
||||
"color_balance" => {
|
||||
let g = |k: &str| f(e, k, cx) / 100.0 * 0.25;
|
||||
let sh = [g("shadow_r"), g("shadow_g"), g("shadow_b")];
|
||||
let md = [g("mid_r"), g("mid_g"), g("mid_b")];
|
||||
let hi = [g("hi_r"), g("hi_g"), g("hi_b")];
|
||||
let preserve = b(e, "preserve");
|
||||
img.map_rgb(|c, _, _| {
|
||||
let c = enc(c);
|
||||
let l = 0.2126 * c[0] + 0.7152 * c[1] + 0.0722 * c[2];
|
||||
let ws = (1.0 - l).powi(2);
|
||||
let wh = l.powi(2);
|
||||
let wm = 1.0 - ws - wh;
|
||||
let mut o = [0.0; 3];
|
||||
for k in 0..3 {
|
||||
o[k] = c[k] + sh[k] * ws + md[k] * wm.max(0.0) + hi[k] * wh;
|
||||
}
|
||||
if preserve {
|
||||
let l2 = 0.2126 * o[0] + 0.7152 * o[1] + 0.0722 * o[2];
|
||||
let d = l - l2;
|
||||
o = o.map(|v| v + d);
|
||||
}
|
||||
dec(o)
|
||||
});
|
||||
}
|
||||
"leave_color" => {
|
||||
let amt = f(e, "amount", cx) / 100.0;
|
||||
let key = color(e, "color", cx);
|
||||
let tol = f(e, "tolerance", cx) / 100.0;
|
||||
let soft = f(e, "softness", cx) / 100.0 + 1e-4;
|
||||
let kh = rgb_to_hsl(key[0], key[1], key[2])[0];
|
||||
img.map_rgb(|c, _, _| {
|
||||
let ec = enc(c);
|
||||
let h = rgb_to_hsl(ec[0], ec[1], ec[2])[0];
|
||||
let d = (h - kh).abs().min(1.0 - (h - kh).abs()) * 2.0;
|
||||
let keep = 1.0 - ((d - tol) / soft).clamp(0.0, 1.0);
|
||||
let l = luma709(c[0], c[1], c[2]);
|
||||
let k = amt * (1.0 - keep);
|
||||
[c[0] + (l - c[0]) * k, c[1] + (l - c[1]) * k, c[2] + (l - c[2]) * k]
|
||||
});
|
||||
}
|
||||
"change_to_color" => {
|
||||
let from = color(e, "from", cx);
|
||||
let to = color(e, "to", cx);
|
||||
let tol = f(e, "hue_tol", cx) / 100.0;
|
||||
let soft = f(e, "softness", cx) / 100.0 * 0.3 + 1e-4;
|
||||
let fh = rgb_to_hsl(from[0], from[1], from[2])[0];
|
||||
let th = rgb_to_hsl(to[0], to[1], to[2])[0];
|
||||
img.map_rgb(|c, _, _| {
|
||||
let ec = enc(c);
|
||||
let mut hsl = rgb_to_hsl(ec[0], ec[1], ec[2]);
|
||||
let d = (hsl[0] - fh).abs().min(1.0 - (hsl[0] - fh).abs());
|
||||
let w = 1.0 - ((d - tol) / soft).clamp(0.0, 1.0);
|
||||
hsl[0] = (hsl[0] + (th - fh) * w).rem_euclid(1.0);
|
||||
dec(hsl_to_rgb(hsl[0], hsl[1], hsl[2]))
|
||||
});
|
||||
}
|
||||
"color_pass" => {
|
||||
let key = color(e, "color", cx);
|
||||
let sim = f(e, "similarity", cx) / 100.0;
|
||||
let rev = b(e, "reverse");
|
||||
img.map_rgb(|c, _, _| {
|
||||
let ec = enc(c);
|
||||
let d = ((ec[0] - key[0]).powi(2) + (ec[1] - key[1]).powi(2) + (ec[2] - key[2]).powi(2)).sqrt();
|
||||
let pass = (d <= sim * 1.2) != rev;
|
||||
if pass {
|
||||
c
|
||||
} else {
|
||||
let l = luma709(c[0], c[1], c[2]);
|
||||
[l, l, l]
|
||||
}
|
||||
});
|
||||
}
|
||||
"gamma_correction" => {
|
||||
let g = f(e, "gamma", cx) / 10.0;
|
||||
img.map_rgb(|c, _, _| dec(enc(c).map(|v| v.max(0.0).powf(g))));
|
||||
}
|
||||
"levels" => {
|
||||
let ib = f(e, "in_black", cx) / 255.0;
|
||||
let iw = (f(e, "in_white", cx) / 255.0).max(ib + 1e-3);
|
||||
let ob = f(e, "out_black", cx) / 255.0;
|
||||
let ow = f(e, "out_white", cx) / 255.0;
|
||||
let g = 100.0 / f(e, "gamma", cx).max(1.0);
|
||||
img.map_rgb(|c, _, _| dec(enc(c).map(|v| ob + (((v - ib) / (iw - ib)).clamp(0.0, 1.0)).powf(g) * (ow - ob))));
|
||||
}
|
||||
"extract" => {
|
||||
let lo = f(e, "black", cx) / 255.0;
|
||||
let hi = f(e, "white", cx) / 255.0;
|
||||
let soft = f(e, "softness", cx) / 100.0 * 0.2 + 1e-4;
|
||||
let inv = b(e, "invert");
|
||||
img.map_rgb(|c, _, _| {
|
||||
let l = linear_to_srgb(luma709(c[0], c[1], c[2]).max(0.0));
|
||||
let inside = ((l - lo) / soft).clamp(0.0, 1.0).min(((hi - l) / soft).clamp(0.0, 1.0));
|
||||
let v = if inv { 1.0 - inside } else { inside };
|
||||
[v, v, v]
|
||||
});
|
||||
}
|
||||
"invert" => {
|
||||
let ch = choice(e, "channel");
|
||||
let blend = f(e, "blend", cx) / 100.0;
|
||||
if ch == 4 {
|
||||
img.px.par_chunks_mut(4).for_each(|p| {
|
||||
let a = p[3];
|
||||
let na = 1.0 - a;
|
||||
let k = if a > 1e-6 { na / a } else { 0.0 };
|
||||
for c in &mut p[..3] {
|
||||
*c *= k;
|
||||
}
|
||||
p[3] = na * (1.0 - blend) + a * blend;
|
||||
});
|
||||
} else {
|
||||
img.map_rgb(|c, _, _| {
|
||||
let ec = enc(c);
|
||||
let mut o = ec;
|
||||
for k in 0..3 {
|
||||
if ch == 0 || ch as usize == k + 1 {
|
||||
o[k] = 1.0 - ec[k];
|
||||
}
|
||||
}
|
||||
dec([o[0] + (ec[0] - o[0]) * blend, o[1] + (ec[1] - o[1]) * blend, o[2] + (ec[2] - o[2]) * blend])
|
||||
});
|
||||
}
|
||||
}
|
||||
"posterize" => {
|
||||
let n = f(e, "levels", cx).max(2.0) - 1.0;
|
||||
img.map_rgb(|c, _, _| dec(enc(c).map(|v| (v * n).round() / n)));
|
||||
}
|
||||
"lumetri" => lumetri(img, e, cx),
|
||||
"gaussian_blur" => {
|
||||
let r = f(e, "blurriness", cx) * cx.px_scale * 0.5;
|
||||
let dims = choice(e, "dimensions");
|
||||
let repeat = b(e, "repeat_edge");
|
||||
gaussian(img, if dims == 2 { 0.0 } else { r }, if dims == 1 { 0.0 } else { r }, repeat);
|
||||
}
|
||||
"camera_blur" => {
|
||||
let r = f(e, "percent", cx) * cx.px_scale * 0.3;
|
||||
gaussian(img, r, r, true);
|
||||
}
|
||||
"directional_blur" => {
|
||||
let len = f(e, "length", cx) * cx.px_scale * 2.0;
|
||||
let dir = (f(e, "direction", cx) as f64).to_radians();
|
||||
directional_blur(img, len, dir);
|
||||
}
|
||||
"sharpen" => {
|
||||
let amt = f(e, "amount", cx) / 100.0;
|
||||
unsharp(img, 1.0 * cx.px_scale.max(0.35), amt, 0.0);
|
||||
}
|
||||
"unsharp_mask" => {
|
||||
let amt = f(e, "amount", cx) / 100.0;
|
||||
let r = f(e, "radius", cx) * cx.px_scale;
|
||||
let th = f(e, "threshold", cx) / 255.0;
|
||||
unsharp(img, r, amt, th);
|
||||
}
|
||||
"median" => {
|
||||
let r = (f(e, "radius", cx) * cx.px_scale).round().clamp(0.0, 4.0) as isize;
|
||||
if r > 0 {
|
||||
@@ -389,100 +206,6 @@ pub fn apply(img: &mut Image, e: &EffectInstance, cx: &FxCtx) {
|
||||
img.map_rgb(|o, _, _| [lc[0] + (o[0] - lc[0]) * blend, lc[1] + (o[1] - lc[1]) * blend, lc[2] + (o[2] - lc[2]) * blend]);
|
||||
}
|
||||
}
|
||||
"crop" => {
|
||||
let l = f(e, "left", cx) / 100.0;
|
||||
let t = f(e, "top", cx) / 100.0;
|
||||
let r = f(e, "right", cx) / 100.0;
|
||||
let bt = f(e, "bottom", cx) / 100.0;
|
||||
let feather = f(e, "feather", cx) * cx.px_scale;
|
||||
if b(e, "zoom") && l + r < 0.99 && t + bt < 0.99 {
|
||||
let src = img.clone();
|
||||
let (w, h) = (img.w as f64, img.h as f64);
|
||||
let m =
|
||||
Affine::scale(1.0 / (1.0 - (l + r) as f64), 1.0 / (1.0 - (t + bt) as f64)).then_apply(&Affine::translate(-(l as f64) * w, -(t as f64) * h));
|
||||
*img = src.transformed(img.w, img.h, &m);
|
||||
} else {
|
||||
crop(img, l, t, r, bt, feather);
|
||||
}
|
||||
}
|
||||
"horizontal_flip" => {
|
||||
let w = img.w;
|
||||
img.px.par_chunks_mut(w * 4).for_each(|row| {
|
||||
for x in 0..w / 2 {
|
||||
for k in 0..4 {
|
||||
row.swap(x * 4 + k, (w - 1 - x) * 4 + k);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
"vertical_flip" => {
|
||||
let (w, h) = (img.w, img.h);
|
||||
for y in 0..h / 2 {
|
||||
let (a, bb) = img.px.split_at_mut((h - 1 - y) * w * 4);
|
||||
a[y * w * 4..(y + 1) * w * 4].swap_with_slice(&mut bb[..w * 4]);
|
||||
}
|
||||
}
|
||||
"edge_feather" => {
|
||||
let amt = f(e, "amount", cx) / 100.0 * (img.w.min(img.h) as f32) * 0.5;
|
||||
crop(img, 0.0, 0.0, 0.0, 0.0, amt);
|
||||
}
|
||||
"transform" => {
|
||||
let anchor = point(e, "anchor", cx, img);
|
||||
let pos = point(e, "position", cx, img);
|
||||
let sh = f(e, "scale_height", cx) as f64 / 100.0;
|
||||
let sw = if b(e, "uniform_scale") { sh } else { f(e, "scale_width", cx) as f64 / 100.0 };
|
||||
let rot = f(e, "rotation", cx) as f64;
|
||||
let skew = (f(e, "skew", cx) as f64).to_radians().tan();
|
||||
let skew_axis = f(e, "skew_axis", cx) as f64;
|
||||
let op = f(e, "opacity", cx) / 100.0;
|
||||
let sk = Affine::rotate_deg(skew_axis)
|
||||
.then_apply(&Affine { a: 1.0, b: 0.0, c: skew, d: 1.0, e: 0.0, f: 0.0 })
|
||||
.then_apply(&Affine::rotate_deg(-skew_axis));
|
||||
let m = Affine::translate(pos.x, pos.y)
|
||||
.then_apply(&Affine::rotate_deg(rot))
|
||||
.then_apply(&sk)
|
||||
.then_apply(&Affine::scale(sw, sh))
|
||||
.then_apply(&Affine::translate(-anchor.x, -anchor.y));
|
||||
let mut out = img.transformed(img.w, img.h, &m);
|
||||
out.scale_alpha(op);
|
||||
*img = out;
|
||||
}
|
||||
"mirror" => {
|
||||
let c = point(e, "center", cx, img);
|
||||
let ang = (f(e, "angle", cx) as f64).to_radians();
|
||||
let (nx, ny) = (ang.cos(), ang.sin());
|
||||
let src = img.clone();
|
||||
let w = img.w;
|
||||
img.px.par_chunks_mut(w * 4).enumerate().for_each(|(y, row)| {
|
||||
for x in 0..w {
|
||||
let (px, py) = (x as f64 + 0.5, y as f64 + 0.5);
|
||||
let d = (px - c.x) * nx + (py - c.y) * ny;
|
||||
if d > 0.0 {
|
||||
let (rx, ry) = (px - 2.0 * d * nx, py - 2.0 * d * ny);
|
||||
row[x * 4..x * 4 + 4].copy_from_slice(&src.sample_bilinear(rx as f32, ry as f32));
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
"offset" => {
|
||||
let s = point(e, "shift", cx, img);
|
||||
let (dx, dy) = (s.x - img.w as f64 / 2.0, s.y - img.h as f64 / 2.0);
|
||||
let blend = f(e, "blend", cx) / 100.0;
|
||||
let src = img.clone();
|
||||
let (w, h) = (img.w as f64, img.h as f64);
|
||||
let wi = img.w;
|
||||
img.px.par_chunks_mut(wi * 4).enumerate().for_each(|(y, row)| {
|
||||
for x in 0..wi {
|
||||
let u = (x as f64 + 0.5 - dx).rem_euclid(w);
|
||||
let v = (y as f64 + 0.5 - dy).rem_euclid(h);
|
||||
let p = src.sample_bilinear_clamped(u as f32, v as f32);
|
||||
let o = src.get(x, y);
|
||||
for k in 0..4 {
|
||||
row[x * 4 + k] = p[k] + (o[k] - p[k]) * blend;
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
"twirl" => {
|
||||
let c = point(e, "center", cx, img);
|
||||
let ang = (f(e, "angle", cx) as f64).to_radians();
|
||||
@@ -782,8 +505,12 @@ pub(crate) fn warp_opt(img: &mut Image, f: impl Fn(f64, f64) -> Option<(f64, f64
|
||||
|
||||
pub(crate) fn crop(img: &mut Image, l: f32, t: f32, r: f32, b: f32, feather: f32) {
|
||||
let (w, h) = (img.w as f32, img.h as f32);
|
||||
let (x0, x1, y0, y1) = (l * w, w * (1.0 - r), t * h, h * (1.0 - b));
|
||||
let fe = feather.max(0.0);
|
||||
crop_px(img, l * w, w * (1.0 - r), t * h, h * (1.0 - b), feather.max(0.0));
|
||||
}
|
||||
|
||||
/// Fade alpha towards the edges of the rectangle x0..x1 × y0..y1 (working pixels): over `fe`
|
||||
/// pixels inside it, or a half-pixel antialiased edge when `fe` is 0.
|
||||
pub(crate) fn crop_px(img: &mut Image, x0: f32, x1: f32, y0: f32, y1: f32, fe: f32) {
|
||||
let wi = img.w;
|
||||
img.px.par_chunks_mut(wi * 4).enumerate().for_each(|(y, row)| {
|
||||
let py = y as f32 + 0.5;
|
||||
@@ -1047,26 +774,42 @@ fn transpose(img: &Image) -> Image {
|
||||
|
||||
/// Gaussian blur via 3 box passes per axis (O(1) per pixel for any radius).
|
||||
pub fn gaussian(img: &mut Image, sigma_x: f32, sigma_y: f32, repeat_edge: bool) {
|
||||
let (rx, ry) = gaussian_boxes(img.w, img.h, sigma_x, sigma_y);
|
||||
box_blur(img, &rx, &ry, repeat_edge);
|
||||
}
|
||||
|
||||
/// The box radii [`gaussian`] runs on a `w`×`h` image, per axis (empty: that axis is untouched).
|
||||
pub(crate) fn gaussian_boxes(w: usize, h: usize, sigma_x: f32, sigma_y: f32) -> (Vec<u32>, Vec<u32>) {
|
||||
// beyond a few image sizes every radius gives the same (flat) result; cap to keep box sizes sane
|
||||
let sigma_x = if sigma_x.is_finite() { sigma_x.min(img.w.max(8) as f32 * 2.0) } else { 0.0 };
|
||||
let sigma_y = if sigma_y.is_finite() { sigma_y.min(img.h.max(8) as f32 * 2.0) } else { 0.0 };
|
||||
if sigma_x > 0.3 {
|
||||
for r in boxes_for_gauss(sigma_x, 3) {
|
||||
box_rows(&mut img.px, img.w, r, repeat_edge);
|
||||
}
|
||||
let sigma_x = if sigma_x.is_finite() { sigma_x.min(w.max(8) as f32 * 2.0) } else { 0.0 };
|
||||
let sigma_y = if sigma_y.is_finite() { sigma_y.min(h.max(8) as f32 * 2.0) } else { 0.0 };
|
||||
let radii = |s: f32| if s > 0.3 { boxes_for_gauss(s, 3).into_iter().map(|r| r.min(u32::MAX as usize) as u32).collect() } else { Vec::new() };
|
||||
(radii(sigma_x), radii(sigma_y))
|
||||
}
|
||||
|
||||
/// Box passes of radii `rx` along rows, then `ry` along columns.
|
||||
pub(crate) fn box_blur(img: &mut Image, rx: &[u32], ry: &[u32], repeat_edge: bool) {
|
||||
for r in rx {
|
||||
box_rows(&mut img.px, img.w, *r as usize, repeat_edge);
|
||||
}
|
||||
if sigma_y > 0.3 {
|
||||
if !ry.is_empty() {
|
||||
let mut t = transpose(img);
|
||||
for r in boxes_for_gauss(sigma_y, 3) {
|
||||
box_rows(&mut t.px, t.w, r, repeat_edge);
|
||||
for r in ry {
|
||||
box_rows(&mut t.px, t.w, *r as usize, repeat_edge);
|
||||
}
|
||||
*img = transpose(&t);
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn unsharp(img: &mut Image, radius: f32, amount: f32, threshold: f32) {
|
||||
let (rx, ry) = gaussian_boxes(img.w, img.h, radius, radius);
|
||||
unsharp_boxes(img, &rx, &ry, amount, threshold);
|
||||
}
|
||||
|
||||
/// Unsharp mask against a repeat-edge box-Gaussian of radii `rx` / `ry`.
|
||||
pub(crate) fn unsharp_boxes(img: &mut Image, rx: &[u32], ry: &[u32], amount: f32, threshold: f32) {
|
||||
let mut blurred = img.clone();
|
||||
gaussian(&mut blurred, radius, radius, true);
|
||||
box_blur(&mut blurred, rx, ry, true);
|
||||
img.px.par_chunks_mut(4).zip(blurred.px.par_chunks(4)).for_each(|(p, bq)| {
|
||||
for k in 0..3 {
|
||||
let d = p[k] - bq[k];
|
||||
@@ -1078,12 +821,12 @@ pub(crate) fn unsharp(img: &mut Image, radius: f32, amount: f32, threshold: f32)
|
||||
});
|
||||
}
|
||||
|
||||
pub(crate) fn directional_blur(img: &mut Image, len: f32, dir: f64) {
|
||||
if len < 0.5 {
|
||||
/// Average of `steps` edge-clamped bilinear taps along (dx, dy), centred on each pixel (no-op
|
||||
/// below 2 steps).
|
||||
pub(crate) fn directional_taps(img: &mut Image, dx: f32, dy: f32, steps: usize) {
|
||||
if steps < 2 {
|
||||
return;
|
||||
}
|
||||
let steps = (len.ceil() as usize).clamp(2, 64);
|
||||
let (dx, dy) = ((dir.sin() as f32) * len, (-dir.cos() as f32) * len);
|
||||
let src = img.clone();
|
||||
let w = img.w;
|
||||
img.px.par_chunks_mut(w * 4).enumerate().for_each(|(y, row)| {
|
||||
|
||||
@@ -0,0 +1,737 @@
|
||||
//! Standard effects with a GPU implementation (`filmcraft_gpu` runs them in WGSL).
|
||||
//!
|
||||
//! Each effect here is evaluated in two steps: [`FxOp::eval`] reads the effect's parameters at a
|
||||
//! time (keyframes, defaults, `px_scale`, the working image size) into plain numbers, and
|
||||
//! [`FxOp::apply`] runs the CPU reference on a working [`Image`]. `effects::apply` uses exactly
|
||||
//! these two steps, so the CPU render and the GPU plan agree on every parameter (and on how
|
||||
//! hostile values are clamped) by construction. The GPU compositor receives the evaluated
|
||||
//! [`FxOp`]s in a [`crate::plan::LayerFx`] and reproduces `apply` per pixel; the plan only hands an
|
||||
//! op to the GPU when [`FxOp::gpu_ok`] says the shader covers it (finite numbers, no minifying
|
||||
//! resample), otherwise the clip is rendered on the CPU as before.
|
||||
|
||||
use filmcraft_color::{hsl_to_rgb, linear_to_srgb, luma709, rgb_to_hsl};
|
||||
use filmcraft_geom::Affine;
|
||||
use filmcraft_project::EffectInstance;
|
||||
use rayon::prelude::*;
|
||||
|
||||
use crate::effects::{FxCtx, b, choice, color, dec, enc, f, gaussian_boxes, point};
|
||||
use crate::image::Image;
|
||||
|
||||
/// A bilinear resample of the working image through an affine map (`Image::transformed` at a
|
||||
/// magnification or mild minification: no mip pre-filter).
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct Resample {
|
||||
/// Destination pixel → source pixel (None: singular map, the result is transparent).
|
||||
pub inv: Option<Affine>,
|
||||
/// Destination pixels written: x0..x1, y0..y1 (the rest becomes transparent).
|
||||
pub rect: [u32; 4],
|
||||
/// Alpha scale applied afterwards (Transform's Opacity; 1 = none).
|
||||
pub opacity: f32,
|
||||
}
|
||||
|
||||
/// One evaluated effect.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub enum FxOp {
|
||||
BrightnessContrast {
|
||||
br: f32,
|
||||
co: f32,
|
||||
},
|
||||
ProcAmp {
|
||||
br: f32,
|
||||
co: f32,
|
||||
hue: f32,
|
||||
sat: f32,
|
||||
},
|
||||
Tint {
|
||||
black: [f32; 3],
|
||||
white: [f32; 3],
|
||||
amount: f32,
|
||||
},
|
||||
BlackWhite,
|
||||
ColorBalance {
|
||||
sh: [f32; 3],
|
||||
md: [f32; 3],
|
||||
hi: [f32; 3],
|
||||
preserve: bool,
|
||||
},
|
||||
LeaveColor {
|
||||
amount: f32,
|
||||
key_hue: f32,
|
||||
tol: f32,
|
||||
soft: f32,
|
||||
},
|
||||
ChangeToColor {
|
||||
from_hue: f32,
|
||||
to_hue: f32,
|
||||
tol: f32,
|
||||
soft: f32,
|
||||
},
|
||||
ColorPass {
|
||||
key: [f32; 3],
|
||||
sim: f32,
|
||||
reverse: bool,
|
||||
},
|
||||
Gamma {
|
||||
g: f32,
|
||||
},
|
||||
Levels {
|
||||
ib: f32,
|
||||
iw: f32,
|
||||
ob: f32,
|
||||
ow: f32,
|
||||
g: f32,
|
||||
},
|
||||
Extract {
|
||||
lo: f32,
|
||||
hi: f32,
|
||||
soft: f32,
|
||||
invert: bool,
|
||||
},
|
||||
Invert {
|
||||
channel: u32,
|
||||
blend: f32,
|
||||
},
|
||||
Posterize {
|
||||
n: f32,
|
||||
},
|
||||
/// Three box passes per axis approximating a Gaussian (radii per pass; empty: axis untouched).
|
||||
Gaussian {
|
||||
rx: Vec<u32>,
|
||||
ry: Vec<u32>,
|
||||
repeat: bool,
|
||||
},
|
||||
/// `steps` bilinear (edge-clamped) taps along `(dx, dy)` centred on the pixel.
|
||||
DirectionalBlur {
|
||||
dx: f32,
|
||||
dy: f32,
|
||||
steps: u32,
|
||||
},
|
||||
/// Unsharp mask over a repeat-edge Gaussian of the given box radii (per axis).
|
||||
Unsharp {
|
||||
rx: Vec<u32>,
|
||||
ry: Vec<u32>,
|
||||
amount: f32,
|
||||
threshold: f32,
|
||||
},
|
||||
/// Crop / Edge Feather: alpha falloff `feather` px inside the rectangle (0: half-pixel AA).
|
||||
Crop {
|
||||
x0: f32,
|
||||
x1: f32,
|
||||
y0: f32,
|
||||
y1: f32,
|
||||
feather: f32,
|
||||
},
|
||||
Resample(Resample),
|
||||
HFlip,
|
||||
VFlip,
|
||||
Mirror {
|
||||
cx: f32,
|
||||
cy: f32,
|
||||
nx: f32,
|
||||
ny: f32,
|
||||
},
|
||||
/// Wrap-around shift (dx, dy already reduced into 0..w, 0..h), mixed with the original.
|
||||
Offset {
|
||||
dx: f32,
|
||||
dy: f32,
|
||||
blend: f32,
|
||||
},
|
||||
/// ASC CDL on display-encoded colour (slope, offset, power, saturation).
|
||||
AscCdl {
|
||||
slope: [f32; 3],
|
||||
offset: [f32; 3],
|
||||
power: [f32; 3],
|
||||
sat: f32,
|
||||
},
|
||||
/// Channel Mixer: rows (r, g, b, constant) per output channel, on display-encoded colour.
|
||||
ChannelMix {
|
||||
m: [[f32; 4]; 3],
|
||||
},
|
||||
/// Color Replace: `sim` already ×1.2; `replace_hsl` the HSL of the replacement colour.
|
||||
ColorReplace {
|
||||
sim: f32,
|
||||
solid: bool,
|
||||
target: [f32; 3],
|
||||
replace: [f32; 3],
|
||||
replace_hsl: [f32; 3],
|
||||
},
|
||||
AlphaAdjust {
|
||||
opacity: f32,
|
||||
ignore: bool,
|
||||
invert: bool,
|
||||
mask_only: bool,
|
||||
},
|
||||
}
|
||||
|
||||
/// Effect ids [`FxOp::eval`] understands (the GPU-capable standard effects).
|
||||
pub const GPU_EFFECTS: &[&str] = &[
|
||||
"brightness_contrast",
|
||||
"proc_amp",
|
||||
"tint",
|
||||
"black_white",
|
||||
"color_balance",
|
||||
"leave_color",
|
||||
"change_to_color",
|
||||
"color_pass",
|
||||
"gamma_correction",
|
||||
"levels",
|
||||
"extract",
|
||||
"invert",
|
||||
"posterize",
|
||||
"gaussian_blur",
|
||||
"gaussian_blur_legacy",
|
||||
"camera_blur",
|
||||
"directional_blur",
|
||||
"directional_blur_legacy",
|
||||
"sharpen",
|
||||
"unsharp_mask",
|
||||
"crop",
|
||||
"edge_feather",
|
||||
"transform",
|
||||
"horizontal_flip",
|
||||
"vertical_flip",
|
||||
"mirror",
|
||||
"offset",
|
||||
"asc_cdl",
|
||||
"channel_mix",
|
||||
"color_replace",
|
||||
"alpha_adjust",
|
||||
];
|
||||
|
||||
/// `Image::transformed` without the mip path: the destination rectangle it writes and the inverse.
|
||||
fn resample(w: usize, h: usize, m: &Affine, opacity: f32) -> Resample {
|
||||
let Some(inv) = m.inverse() else { return Resample { inv: None, rect: [0; 4], opacity } };
|
||||
let b = m.bounds(&filmcraft_geom::Rect::new(0.0, 0.0, w as f64, h as f64));
|
||||
let y0 = (b.y.floor().max(0.0) as usize).min(h);
|
||||
let y1 = (b.bottom().ceil().max(0.0) as usize).min(h);
|
||||
let x0 = (b.x.floor().max(0.0) as usize).min(w);
|
||||
let x1 = (b.right().ceil().max(0.0) as usize).min(w);
|
||||
Resample { inv: Some(inv), rect: [x0 as u32, x1 as u32, y0.min(y1) as u32, y1 as u32], opacity }
|
||||
}
|
||||
|
||||
/// Whether `Image::transformed` would take its mip (downsample) path for `m` on a `w`×`h` image.
|
||||
fn minifies(m: &Affine, w: usize, h: usize) -> bool {
|
||||
let sx = (m.a * m.a + m.b * m.b).sqrt();
|
||||
let sy = (m.c * m.c + m.d * m.d).sqrt();
|
||||
let minify = 1.0 / sx.min(sy).max(1e-6);
|
||||
// (a non-finite factor: keep such maps off the GPU; `apply` still hands them to the CPU's
|
||||
// `Image::transformed`, which decides as before)
|
||||
!minify.is_finite() || minify >= 2.0 && w >= 4 && h >= 4
|
||||
}
|
||||
|
||||
impl FxOp {
|
||||
/// Evaluate `e` at `cx.t` for a `w`×`h` working image. None for effects without a GPU
|
||||
/// implementation (and for disabled effects).
|
||||
pub fn eval(e: &EffectInstance, cx: &FxCtx, w: usize, h: usize) -> Option<FxOp> {
|
||||
if !e.enabled {
|
||||
return None;
|
||||
}
|
||||
Some(match e.effect.as_str() {
|
||||
"brightness_contrast" => FxOp::BrightnessContrast { br: f(e, "brightness", cx) / 100.0 * 0.4, co: 1.0 + f(e, "contrast", cx) / 100.0 },
|
||||
"proc_amp" => FxOp::ProcAmp {
|
||||
br: f(e, "brightness", cx) / 100.0 * 0.4,
|
||||
co: f(e, "contrast", cx) / 100.0,
|
||||
hue: f(e, "hue", cx) / 360.0,
|
||||
sat: f(e, "saturation", cx) / 100.0,
|
||||
},
|
||||
"tint" => {
|
||||
let (bl, wh) = (color(e, "black", cx), color(e, "white", cx));
|
||||
FxOp::Tint { black: [bl[0], bl[1], bl[2]], white: [wh[0], wh[1], wh[2]], amount: f(e, "amount", cx) / 100.0 }
|
||||
}
|
||||
"black_white" => FxOp::BlackWhite,
|
||||
"color_balance" => {
|
||||
let g = |k: &str| f(e, k, cx) / 100.0 * 0.25;
|
||||
FxOp::ColorBalance {
|
||||
sh: [g("shadow_r"), g("shadow_g"), g("shadow_b")],
|
||||
md: [g("mid_r"), g("mid_g"), g("mid_b")],
|
||||
hi: [g("hi_r"), g("hi_g"), g("hi_b")],
|
||||
preserve: b(e, "preserve"),
|
||||
}
|
||||
}
|
||||
"leave_color" => {
|
||||
let key = color(e, "color", cx);
|
||||
FxOp::LeaveColor {
|
||||
amount: f(e, "amount", cx) / 100.0,
|
||||
key_hue: rgb_to_hsl(key[0], key[1], key[2])[0],
|
||||
tol: f(e, "tolerance", cx) / 100.0,
|
||||
soft: f(e, "softness", cx) / 100.0 + 1e-4,
|
||||
}
|
||||
}
|
||||
"change_to_color" => {
|
||||
let (from, to) = (color(e, "from", cx), color(e, "to", cx));
|
||||
FxOp::ChangeToColor {
|
||||
from_hue: rgb_to_hsl(from[0], from[1], from[2])[0],
|
||||
to_hue: rgb_to_hsl(to[0], to[1], to[2])[0],
|
||||
tol: f(e, "hue_tol", cx) / 100.0,
|
||||
soft: f(e, "softness", cx) / 100.0 * 0.3 + 1e-4,
|
||||
}
|
||||
}
|
||||
"color_pass" => {
|
||||
let key = color(e, "color", cx);
|
||||
FxOp::ColorPass { key: [key[0], key[1], key[2]], sim: f(e, "similarity", cx) / 100.0, reverse: b(e, "reverse") }
|
||||
}
|
||||
"gamma_correction" => FxOp::Gamma { g: f(e, "gamma", cx) / 10.0 },
|
||||
"levels" => {
|
||||
let ib = f(e, "in_black", cx) / 255.0;
|
||||
FxOp::Levels {
|
||||
ib,
|
||||
iw: (f(e, "in_white", cx) / 255.0).max(ib + 1e-3),
|
||||
ob: f(e, "out_black", cx) / 255.0,
|
||||
ow: f(e, "out_white", cx) / 255.0,
|
||||
g: 100.0 / f(e, "gamma", cx).max(1.0),
|
||||
}
|
||||
}
|
||||
"extract" => FxOp::Extract {
|
||||
lo: f(e, "black", cx) / 255.0,
|
||||
hi: f(e, "white", cx) / 255.0,
|
||||
soft: f(e, "softness", cx) / 100.0 * 0.2 + 1e-4,
|
||||
invert: b(e, "invert"),
|
||||
},
|
||||
"invert" => FxOp::Invert { channel: choice(e, "channel"), blend: f(e, "blend", cx) / 100.0 },
|
||||
"posterize" => FxOp::Posterize { n: f(e, "levels", cx).max(2.0) - 1.0 },
|
||||
"gaussian_blur" | "gaussian_blur_legacy" => {
|
||||
let r = f(e, "blurriness", cx) * cx.px_scale * 0.5;
|
||||
let dims = choice(e, "dimensions");
|
||||
let (rx, ry) = gaussian_boxes(w, h, if dims == 2 { 0.0 } else { r }, if dims == 1 { 0.0 } else { r });
|
||||
FxOp::Gaussian { rx, ry, repeat: b(e, "repeat_edge") }
|
||||
}
|
||||
"camera_blur" => {
|
||||
let r = f(e, "percent", cx) * cx.px_scale * 0.3;
|
||||
let (rx, ry) = gaussian_boxes(w, h, r, r);
|
||||
FxOp::Gaussian { rx, ry, repeat: true }
|
||||
}
|
||||
"directional_blur" | "directional_blur_legacy" => {
|
||||
let len = f(e, "length", cx) * cx.px_scale * 2.0;
|
||||
let dir = (f(e, "direction", cx) as f64).to_radians();
|
||||
if len < 0.5 {
|
||||
FxOp::DirectionalBlur { dx: 0.0, dy: 0.0, steps: 0 }
|
||||
} else {
|
||||
let steps = (len.ceil() as usize).clamp(2, 64) as u32;
|
||||
FxOp::DirectionalBlur { dx: (dir.sin() as f32) * len, dy: (-dir.cos() as f32) * len, steps }
|
||||
}
|
||||
}
|
||||
"sharpen" => {
|
||||
let r = cx.px_scale.max(0.35);
|
||||
let (rx, ry) = gaussian_boxes(w, h, r, r);
|
||||
FxOp::Unsharp { rx, ry, amount: f(e, "amount", cx) / 100.0, threshold: 0.0 }
|
||||
}
|
||||
"unsharp_mask" => {
|
||||
let r = f(e, "radius", cx) * cx.px_scale;
|
||||
let (rx, ry) = gaussian_boxes(w, h, r, r);
|
||||
FxOp::Unsharp { rx, ry, amount: f(e, "amount", cx) / 100.0, threshold: f(e, "threshold", cx) / 255.0 }
|
||||
}
|
||||
"crop" => {
|
||||
let l = f(e, "left", cx) / 100.0;
|
||||
let t = f(e, "top", cx) / 100.0;
|
||||
let r = f(e, "right", cx) / 100.0;
|
||||
let bt = f(e, "bottom", cx) / 100.0;
|
||||
if b(e, "zoom") && l + r < 0.99 && t + bt < 0.99 {
|
||||
let (fw, fh) = (w as f64, h as f64);
|
||||
let m = Affine::scale(1.0 / (1.0 - (l + r) as f64), 1.0 / (1.0 - (t + bt) as f64))
|
||||
.then_apply(&Affine::translate(-(l as f64) * fw, -(t as f64) * fh));
|
||||
if minifies(&m, w, h) {
|
||||
// negative crops zoom out: the CPU's mip path (see `transform`)
|
||||
return Some(FxOp::Resample(Resample { inv: Some(m), rect: [u32::MAX; 4], opacity: 1.0 }));
|
||||
}
|
||||
FxOp::Resample(resample(w, h, &m, 1.0))
|
||||
} else {
|
||||
FxOp::crop_rect(w, h, l, t, r, bt, f(e, "feather", cx) * cx.px_scale)
|
||||
}
|
||||
}
|
||||
"edge_feather" => {
|
||||
let amt = f(e, "amount", cx) / 100.0 * (w.min(h) as f32) * 0.5;
|
||||
FxOp::crop_rect(w, h, 0.0, 0.0, 0.0, 0.0, amt)
|
||||
}
|
||||
"transform" => {
|
||||
let img = Image { w, h, px: Vec::new() };
|
||||
let anchor = point(e, "anchor", cx, &img);
|
||||
let pos = point(e, "position", cx, &img);
|
||||
let sh = f(e, "scale_height", cx) as f64 / 100.0;
|
||||
let sw = if b(e, "uniform_scale") { sh } else { f(e, "scale_width", cx) as f64 / 100.0 };
|
||||
let rot = f(e, "rotation", cx) as f64;
|
||||
let skew = (f(e, "skew", cx) as f64).to_radians().tan();
|
||||
let skew_axis = f(e, "skew_axis", cx) as f64;
|
||||
let op = f(e, "opacity", cx) / 100.0;
|
||||
let sk = Affine::rotate_deg(skew_axis)
|
||||
.then_apply(&Affine { a: 1.0, b: 0.0, c: skew, d: 1.0, e: 0.0, f: 0.0 })
|
||||
.then_apply(&Affine::rotate_deg(-skew_axis));
|
||||
let m = Affine::translate(pos.x, pos.y)
|
||||
.then_apply(&Affine::rotate_deg(rot))
|
||||
.then_apply(&sk)
|
||||
.then_apply(&Affine::scale(sw, sh))
|
||||
.then_apply(&Affine::translate(-anchor.x, -anchor.y));
|
||||
if minifies(&m, w, h) {
|
||||
// the CPU's mip path (`Image::transformed` with the forward map; `rect`
|
||||
// u32::MAX marks it): CPU only
|
||||
return Some(FxOp::Resample(Resample { inv: Some(m), rect: [u32::MAX; 4], opacity: op }));
|
||||
}
|
||||
FxOp::Resample(resample(w, h, &m, op))
|
||||
}
|
||||
"horizontal_flip" => FxOp::HFlip,
|
||||
"vertical_flip" => FxOp::VFlip,
|
||||
"mirror" => {
|
||||
let img = Image { w, h, px: Vec::new() };
|
||||
let c = point(e, "center", cx, &img);
|
||||
let ang = (f(e, "angle", cx) as f64).to_radians();
|
||||
FxOp::Mirror { cx: c.x as f32, cy: c.y as f32, nx: ang.cos() as f32, ny: ang.sin() as f32 }
|
||||
}
|
||||
"offset" => {
|
||||
let img = Image { w, h, px: Vec::new() };
|
||||
let s = point(e, "shift", cx, &img);
|
||||
let (dx, dy) = (s.x - w as f64 / 2.0, s.y - h as f64 / 2.0);
|
||||
FxOp::Offset { dx: dx.rem_euclid(w.max(1) as f64) as f32, dy: dy.rem_euclid(h.max(1) as f64) as f32, blend: f(e, "blend", cx) / 100.0 }
|
||||
}
|
||||
"asc_cdl" => {
|
||||
use crate::vfx::fv;
|
||||
FxOp::AscCdl {
|
||||
slope: [fv(e, "r_slope", cx), fv(e, "g_slope", cx), fv(e, "b_slope", cx)],
|
||||
offset: [fv(e, "r_offset", cx), fv(e, "g_offset", cx), fv(e, "b_offset", cx)],
|
||||
power: [fv(e, "r_power", cx), fv(e, "g_power", cx), fv(e, "b_power", cx)],
|
||||
sat: fv(e, "saturation", cx),
|
||||
}
|
||||
}
|
||||
"channel_mix" => {
|
||||
use crate::vfx::{bv, fv};
|
||||
let g = |k: &str| fv(e, k, cx) / 100.0;
|
||||
let mut m = [[g("rr"), g("rg"), g("rb"), g("rc")], [g("gr"), g("gg"), g("gb"), g("gc")], [g("br"), g("bg"), g("bb"), g("bc")]];
|
||||
if bv(e, "monochrome") {
|
||||
m = [m[0]; 3];
|
||||
}
|
||||
FxOp::ChannelMix { m }
|
||||
}
|
||||
"color_replace" => {
|
||||
use crate::vfx::{bv, cv, fv};
|
||||
let t = cv(e, "target", cx);
|
||||
let r = cv(e, "replace", cx);
|
||||
FxOp::ColorReplace {
|
||||
sim: fv(e, "similarity", cx) / 100.0 * 1.2,
|
||||
solid: bv(e, "solid"),
|
||||
target: [t[0], t[1], t[2]],
|
||||
replace: [r[0], r[1], r[2]],
|
||||
replace_hsl: rgb_to_hsl(r[0], r[1], r[2]),
|
||||
}
|
||||
}
|
||||
"alpha_adjust" => {
|
||||
use crate::vfx::{bv, fv};
|
||||
FxOp::AlphaAdjust { opacity: fv(e, "opacity", cx) / 100.0, ignore: bv(e, "ignore"), invert: bv(e, "invert"), mask_only: bv(e, "mask_only") }
|
||||
}
|
||||
_ => return None,
|
||||
})
|
||||
}
|
||||
|
||||
fn crop_rect(w: usize, h: usize, l: f32, t: f32, r: f32, b: f32, feather: f32) -> FxOp {
|
||||
let (w, h) = (w as f32, h as f32);
|
||||
FxOp::Crop { x0: l * w, x1: w * (1.0 - r), y0: t * h, y1: h * (1.0 - b), feather: feather.max(0.0) }
|
||||
}
|
||||
|
||||
/// Whether the GPU shader reproduces [`apply`](Self::apply) for this op: every number finite
|
||||
/// (the CPU's NaN / infinity behaviour is left to the CPU) and resamples that the CPU does
|
||||
/// not pre-filter.
|
||||
pub fn gpu_ok(&self) -> bool {
|
||||
let fin = |v: &[f32]| v.iter().all(|x| x.is_finite());
|
||||
match self {
|
||||
FxOp::BrightnessContrast { br, co } => fin(&[*br, *co]),
|
||||
FxOp::ProcAmp { br, co, hue, sat } => fin(&[*br, *co, *hue, *sat]),
|
||||
FxOp::Tint { black, white, amount } => fin(black) && fin(white) && amount.is_finite(),
|
||||
FxOp::BlackWhite | FxOp::HFlip | FxOp::VFlip => true,
|
||||
FxOp::ColorBalance { sh, md, hi, .. } => fin(sh) && fin(md) && fin(hi),
|
||||
FxOp::LeaveColor { amount, key_hue, tol, soft } => fin(&[*amount, *key_hue, *tol, *soft]),
|
||||
FxOp::ChangeToColor { from_hue, to_hue, tol, soft } => fin(&[*from_hue, *to_hue, *tol, *soft]),
|
||||
FxOp::ColorPass { key, sim, .. } => fin(key) && sim.is_finite(),
|
||||
FxOp::Gamma { g } => g.is_finite(),
|
||||
FxOp::Levels { ib, iw, ob, ow, g } => fin(&[*ib, *iw, *ob, *ow, *g]),
|
||||
FxOp::Extract { lo, hi, soft, .. } => fin(&[*lo, *hi, *soft]),
|
||||
FxOp::Invert { blend, .. } => blend.is_finite(),
|
||||
FxOp::Posterize { n } => n.is_finite(),
|
||||
FxOp::Gaussian { .. } => true,
|
||||
FxOp::DirectionalBlur { dx, dy, .. } => fin(&[*dx, *dy]),
|
||||
FxOp::Unsharp { amount, threshold, .. } => fin(&[*amount, *threshold]),
|
||||
FxOp::Crop { x0, x1, y0, y1, feather } => fin(&[*x0, *x1, *y0, *y1, *feather]),
|
||||
FxOp::Resample(r) => {
|
||||
r.rect[0] != u32::MAX
|
||||
&& r.opacity.is_finite()
|
||||
&& r.inv.as_ref().is_none_or(|m| [m.a, m.b, m.c, m.d, m.e, m.f].iter().all(|v| v.is_finite() && v.abs() < 1e7))
|
||||
}
|
||||
FxOp::Mirror { cx, cy, nx, ny } => fin(&[*cx, *cy, *nx, *ny]),
|
||||
FxOp::Offset { dx, dy, blend } => fin(&[*dx, *dy, *blend]),
|
||||
FxOp::AscCdl { slope, offset, power, sat } => fin(slope) && fin(offset) && fin(power) && sat.is_finite(),
|
||||
FxOp::ChannelMix { m } => m.iter().all(|r| fin(r)),
|
||||
FxOp::ColorReplace { sim, target, replace, replace_hsl, .. } => sim.is_finite() && fin(target) && fin(replace) && fin(replace_hsl),
|
||||
FxOp::AlphaAdjust { opacity, .. } => opacity.is_finite(),
|
||||
}
|
||||
}
|
||||
|
||||
/// The CPU reference.
|
||||
pub fn apply(&self, img: &mut Image) {
|
||||
if img.w == 0 || img.h == 0 {
|
||||
return;
|
||||
}
|
||||
match self {
|
||||
FxOp::BrightnessContrast { br, co } => {
|
||||
let (br, co) = (*br, *co);
|
||||
img.map_rgb(|c, _, _| {
|
||||
let c = enc(c);
|
||||
dec(c.map(|v| (v - 0.5) * co + 0.5 + br))
|
||||
});
|
||||
}
|
||||
FxOp::ProcAmp { br, co, hue, sat } => {
|
||||
let (br, co, hue, sat) = (*br, *co, *hue, *sat);
|
||||
img.map_rgb(|c, _, _| {
|
||||
let c = enc(c);
|
||||
let mut hsl = rgb_to_hsl(c[0], c[1], c[2]);
|
||||
hsl[0] = (hsl[0] + hue).rem_euclid(1.0);
|
||||
hsl[1] = (hsl[1] * sat).clamp(0.0, 1.0);
|
||||
let c = hsl_to_rgb(hsl[0], hsl[1], hsl[2]);
|
||||
dec(c.map(|v| (v - 0.5) * co + 0.5 + br))
|
||||
});
|
||||
}
|
||||
FxOp::Tint { black: bl, white: wh, amount } => {
|
||||
let amt = *amount;
|
||||
img.map_rgb(|c, _, _| {
|
||||
let l = linear_to_srgb(luma709(c[0], c[1], c[2]).max(0.0));
|
||||
let t = [bl[0] + (wh[0] - bl[0]) * l, bl[1] + (wh[1] - bl[1]) * l, bl[2] + (wh[2] - bl[2]) * l];
|
||||
let e = enc(c);
|
||||
dec([e[0] + (t[0] - e[0]) * amt, e[1] + (t[1] - e[1]) * amt, e[2] + (t[2] - e[2]) * amt])
|
||||
});
|
||||
}
|
||||
FxOp::BlackWhite => img.map_rgb(|c, _, _| {
|
||||
let l = luma709(c[0], c[1], c[2]);
|
||||
[l, l, l]
|
||||
}),
|
||||
FxOp::ColorBalance { sh, md, hi, preserve } => {
|
||||
let preserve = *preserve;
|
||||
img.map_rgb(|c, _, _| {
|
||||
let c = enc(c);
|
||||
let l = 0.2126 * c[0] + 0.7152 * c[1] + 0.0722 * c[2];
|
||||
let ws = (1.0 - l).powi(2);
|
||||
let wh = l.powi(2);
|
||||
let wm = 1.0 - ws - wh;
|
||||
let mut o = [0.0; 3];
|
||||
for k in 0..3 {
|
||||
o[k] = c[k] + sh[k] * ws + md[k] * wm.max(0.0) + hi[k] * wh;
|
||||
}
|
||||
if preserve {
|
||||
let l2 = 0.2126 * o[0] + 0.7152 * o[1] + 0.0722 * o[2];
|
||||
let d = l - l2;
|
||||
o = o.map(|v| v + d);
|
||||
}
|
||||
dec(o)
|
||||
});
|
||||
}
|
||||
FxOp::LeaveColor { amount, key_hue, tol, soft } => {
|
||||
let (amt, kh, tol, soft) = (*amount, *key_hue, *tol, *soft);
|
||||
img.map_rgb(|c, _, _| {
|
||||
let ec = enc(c);
|
||||
let h = rgb_to_hsl(ec[0], ec[1], ec[2])[0];
|
||||
let d = (h - kh).abs().min(1.0 - (h - kh).abs()) * 2.0;
|
||||
let keep = 1.0 - ((d - tol) / soft).clamp(0.0, 1.0);
|
||||
let l = luma709(c[0], c[1], c[2]);
|
||||
let k = amt * (1.0 - keep);
|
||||
[c[0] + (l - c[0]) * k, c[1] + (l - c[1]) * k, c[2] + (l - c[2]) * k]
|
||||
});
|
||||
}
|
||||
FxOp::ChangeToColor { from_hue: fh, to_hue: th, tol, soft } => {
|
||||
let (fh, th, tol, soft) = (*fh, *th, *tol, *soft);
|
||||
img.map_rgb(|c, _, _| {
|
||||
let ec = enc(c);
|
||||
let mut hsl = rgb_to_hsl(ec[0], ec[1], ec[2]);
|
||||
let d = (hsl[0] - fh).abs().min(1.0 - (hsl[0] - fh).abs());
|
||||
let w = 1.0 - ((d - tol) / soft).clamp(0.0, 1.0);
|
||||
hsl[0] = (hsl[0] + (th - fh) * w).rem_euclid(1.0);
|
||||
dec(hsl_to_rgb(hsl[0], hsl[1], hsl[2]))
|
||||
});
|
||||
}
|
||||
FxOp::ColorPass { key, sim, reverse } => {
|
||||
let (sim, rev) = (*sim, *reverse);
|
||||
img.map_rgb(|c, _, _| {
|
||||
let ec = enc(c);
|
||||
let d = ((ec[0] - key[0]).powi(2) + (ec[1] - key[1]).powi(2) + (ec[2] - key[2]).powi(2)).sqrt();
|
||||
let pass = (d <= sim * 1.2) != rev;
|
||||
if pass {
|
||||
c
|
||||
} else {
|
||||
let l = luma709(c[0], c[1], c[2]);
|
||||
[l, l, l]
|
||||
}
|
||||
});
|
||||
}
|
||||
FxOp::Gamma { g } => {
|
||||
let g = *g;
|
||||
img.map_rgb(|c, _, _| dec(enc(c).map(|v| v.max(0.0).powf(g))));
|
||||
}
|
||||
FxOp::Levels { ib, iw, ob, ow, g } => {
|
||||
let (ib, iw, ob, ow, g) = (*ib, *iw, *ob, *ow, *g);
|
||||
img.map_rgb(|c, _, _| dec(enc(c).map(|v| ob + (((v - ib) / (iw - ib)).clamp(0.0, 1.0)).powf(g) * (ow - ob))));
|
||||
}
|
||||
FxOp::Extract { lo, hi, soft, invert } => {
|
||||
let (lo, hi, soft, inv) = (*lo, *hi, *soft, *invert);
|
||||
img.map_rgb(|c, _, _| {
|
||||
let l = linear_to_srgb(luma709(c[0], c[1], c[2]).max(0.0));
|
||||
let inside = ((l - lo) / soft).clamp(0.0, 1.0).min(((hi - l) / soft).clamp(0.0, 1.0));
|
||||
let v = if inv { 1.0 - inside } else { inside };
|
||||
[v, v, v]
|
||||
});
|
||||
}
|
||||
FxOp::Invert { channel, blend } => {
|
||||
let (ch, blend) = (*channel, *blend);
|
||||
if ch == 4 {
|
||||
img.px.par_chunks_mut(4).for_each(|p| {
|
||||
let a = p[3];
|
||||
let na = 1.0 - a;
|
||||
let k = if a > 1e-6 { na / a } else { 0.0 };
|
||||
for c in &mut p[..3] {
|
||||
*c *= k;
|
||||
}
|
||||
p[3] = na * (1.0 - blend) + a * blend;
|
||||
});
|
||||
} else {
|
||||
img.map_rgb(|c, _, _| {
|
||||
let ec = enc(c);
|
||||
let mut o = ec;
|
||||
for k in 0..3 {
|
||||
if ch == 0 || ch as usize == k + 1 {
|
||||
o[k] = 1.0 - ec[k];
|
||||
}
|
||||
}
|
||||
dec([o[0] + (ec[0] - o[0]) * blend, o[1] + (ec[1] - o[1]) * blend, o[2] + (ec[2] - o[2]) * blend])
|
||||
});
|
||||
}
|
||||
}
|
||||
FxOp::Posterize { n } => {
|
||||
let n = *n;
|
||||
img.map_rgb(|c, _, _| dec(enc(c).map(|v| (v * n).round() / n)));
|
||||
}
|
||||
FxOp::Gaussian { rx, ry, repeat } => crate::effects::box_blur(img, rx, ry, *repeat),
|
||||
FxOp::DirectionalBlur { dx, dy, steps } => crate::effects::directional_taps(img, *dx, *dy, *steps as usize),
|
||||
FxOp::Unsharp { rx, ry, amount, threshold } => crate::effects::unsharp_boxes(img, rx, ry, *amount, *threshold),
|
||||
FxOp::Crop { x0, x1, y0, y1, feather } => crate::effects::crop_px(img, *x0, *x1, *y0, *y1, *feather),
|
||||
FxOp::Resample(r) => {
|
||||
let out = match (&r.inv, r.rect[0]) {
|
||||
(Some(m), u32::MAX) => img.transformed(img.w, img.h, m), // mip path (CPU only)
|
||||
(Some(inv), _) => resample_cpu(img, inv, r.rect),
|
||||
(None, _) => Image::new(img.w, img.h),
|
||||
};
|
||||
*img = out;
|
||||
img.scale_alpha(r.opacity);
|
||||
}
|
||||
FxOp::HFlip => {
|
||||
let w = img.w;
|
||||
img.px.par_chunks_mut(w * 4).for_each(|row| {
|
||||
for x in 0..w / 2 {
|
||||
for k in 0..4 {
|
||||
row.swap(x * 4 + k, (w - 1 - x) * 4 + k);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
FxOp::VFlip => {
|
||||
let (w, h) = (img.w, img.h);
|
||||
for y in 0..h / 2 {
|
||||
let (a, bb) = img.px.split_at_mut((h - 1 - y) * w * 4);
|
||||
a[y * w * 4..(y + 1) * w * 4].swap_with_slice(&mut bb[..w * 4]);
|
||||
}
|
||||
}
|
||||
FxOp::Mirror { cx, cy, nx, ny } => {
|
||||
let (c, n) = ((*cx as f64, *cy as f64), (*nx as f64, *ny as f64));
|
||||
let src = img.clone();
|
||||
let w = img.w;
|
||||
img.px.par_chunks_mut(w * 4).enumerate().for_each(|(y, row)| {
|
||||
for x in 0..w {
|
||||
let (px, py) = (x as f64 + 0.5, y as f64 + 0.5);
|
||||
let d = (px - c.0) * n.0 + (py - c.1) * n.1;
|
||||
if d > 0.0 {
|
||||
let (rx, ry) = (px - 2.0 * d * n.0, py - 2.0 * d * n.1);
|
||||
row[x * 4..x * 4 + 4].copy_from_slice(&src.sample_bilinear(rx as f32, ry as f32));
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
FxOp::Offset { dx, dy, blend } => {
|
||||
let (dx, dy, blend) = (*dx as f64, *dy as f64, *blend);
|
||||
let src = img.clone();
|
||||
let (w, h) = (img.w as f64, img.h as f64);
|
||||
let wi = img.w;
|
||||
img.px.par_chunks_mut(wi * 4).enumerate().for_each(|(y, row)| {
|
||||
for x in 0..wi {
|
||||
let u = (x as f64 + 0.5 - dx).rem_euclid(w);
|
||||
let v = (y as f64 + 0.5 - dy).rem_euclid(h);
|
||||
let p = src.sample_bilinear_clamped(u as f32, v as f32);
|
||||
let o = src.get(x, y);
|
||||
for k in 0..4 {
|
||||
row[x * 4 + k] = p[k] + (o[k] - p[k]) * blend;
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
FxOp::AscCdl { slope, offset, power, sat } => {
|
||||
let (s, o, pw, sat) = (*slope, *offset, *power, *sat);
|
||||
img.map_rgb(|c, _, _| dec(crate::vfx::cdl(enc(c), s, o, pw, sat)));
|
||||
}
|
||||
FxOp::ChannelMix { m } => {
|
||||
let m = *m;
|
||||
img.map_rgb(|c, _, _| {
|
||||
let v = enc(c);
|
||||
dec([0, 1, 2].map(|k| (m[k][0] * v[0] + m[k][1] * v[1] + m[k][2] * v[2] + m[k][3]).clamp(0.0, 1.0)))
|
||||
});
|
||||
}
|
||||
FxOp::ColorReplace { sim, solid, target: t, replace: r, replace_hsl: rh } => {
|
||||
let (sim, solid) = (*sim, *solid);
|
||||
img.map_rgb(|c, _, _| {
|
||||
let v = enc(c);
|
||||
let d = ((v[0] - t[0]).powi(2) + (v[1] - t[1]).powi(2) + (v[2] - t[2]).powi(2)).sqrt();
|
||||
let k = 1.0 - crate::vfx::smoothstep(sim * 0.85, sim.max(1e-4), d);
|
||||
if k <= 0.0 {
|
||||
return c;
|
||||
}
|
||||
let target = if solid {
|
||||
[r[0], r[1], r[2]]
|
||||
} else {
|
||||
let l = rgb_to_hsl(v[0], v[1], v[2])[2];
|
||||
hsl_to_rgb(rh[0], rh[1], l)
|
||||
};
|
||||
dec(crate::vfx::lerp3(v, target, k))
|
||||
});
|
||||
}
|
||||
FxOp::AlphaAdjust { opacity, ignore, invert, mask_only } => {
|
||||
let (op, ignore, invert, mask_only) = (*opacity, *ignore, *invert, *mask_only);
|
||||
img.px.par_chunks_mut(4).for_each(|p| {
|
||||
let c = Image::unpremul([p[0], p[1], p[2], p[3]]);
|
||||
let mut a = if ignore { 1.0 } else { p[3] };
|
||||
if invert {
|
||||
a = 1.0 - a;
|
||||
}
|
||||
a = (a * op).clamp(0.0, 1.0);
|
||||
if mask_only {
|
||||
let g = filmcraft_color::srgb_to_linear(a);
|
||||
p.copy_from_slice(&[g, g, g, 1.0]);
|
||||
} else {
|
||||
p.copy_from_slice(&[c[0] * a, c[1] * a, c[2] * a, a]);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The body of `Image::transformed` (no mip path) for a precomputed inverse and rectangle.
|
||||
fn resample_cpu(src: &Image, inv: &Affine, rect: [u32; 4]) -> Image {
|
||||
let (w, h) = (src.w, src.h);
|
||||
let mut out = Image::new(w, h);
|
||||
let [x0, x1, y0, y1] = rect.map(|v| v as usize);
|
||||
let (x1, y1) = (x1.min(w), y1.min(h));
|
||||
let (x0, y0) = (x0.min(x1), y0.min(y1));
|
||||
let axis_aligned = inv.b == 0.0 && inv.c == 0.0;
|
||||
out.px.par_chunks_mut(w * 4).enumerate().skip(y0).take(y1 - y0).for_each(|(y, row)| {
|
||||
let py = y as f64 + 0.5;
|
||||
for x in x0..x1 {
|
||||
let px = x as f64 + 0.5;
|
||||
let (u, v) =
|
||||
if axis_aligned { (inv.a * px + inv.e, inv.d * py + inv.f) } else { (inv.a * px + inv.c * py + inv.e, inv.b * px + inv.d * py + inv.f) };
|
||||
if u < -1.0 || v < -1.0 || u > w as f64 + 1.0 || v > h as f64 + 1.0 {
|
||||
continue;
|
||||
}
|
||||
row[x * 4..x * 4 + 4].copy_from_slice(&src.sample_bilinear(u as f32, v as f32));
|
||||
}
|
||||
});
|
||||
out
|
||||
}
|
||||
@@ -101,7 +101,7 @@ impl Image {
|
||||
let x0 = (b.x.floor().max(0.0) as usize).min(w);
|
||||
let x1 = (b.right().ceil().max(0.0) as usize).min(w);
|
||||
let axis_aligned = m.b == 0.0 && m.c == 0.0;
|
||||
out.px.par_chunks_mut(w * 4).enumerate().skip(y0).take(y1 - y0).for_each(|(y, row)| {
|
||||
out.px.par_chunks_mut(w * 4).enumerate().skip(y0).take(y1.saturating_sub(y0)).for_each(|(y, row)| {
|
||||
let py = y as f64 + 0.5;
|
||||
for x in x0..x1 {
|
||||
let px = x as f64 + 0.5;
|
||||
|
||||
@@ -15,6 +15,7 @@ pub mod blend;
|
||||
pub mod color_match;
|
||||
pub mod colorman;
|
||||
pub mod effects;
|
||||
pub mod gpufx;
|
||||
pub mod graphic_clip;
|
||||
pub mod graphics;
|
||||
pub mod image;
|
||||
|
||||
+115
-29
@@ -2,11 +2,14 @@
|
||||
//!
|
||||
//! [`plan_frame`] resolves what is visible at a time into a list of layers the GPU can draw
|
||||
//! directly: a decoded source frame (YUV planes or RGBA), the matrix from source pixels to output
|
||||
//! pixels, an opacity and a blend mode (all 27 of [`Blend`] are composited by the GPU). Anything
|
||||
//! the shaders don't cover yet — standard effects, adjustment layers, nested sequences,
|
||||
//! non-dissolve transitions — is rendered on the CPU for that layer (or the whole frame) and
|
||||
//! handed over as a pre-composited image, so the GPU path is always exact with respect to the CPU
|
||||
//! reference.
|
||||
//! pixels, an opacity and a blend mode (all 27 of [`Blend`] are composited by the GPU). A media
|
||||
//! clip whose enabled standard effects all have a GPU implementation ([`crate::gpufx`]) carries
|
||||
//! them as a [`LayerFx`]: their parameters evaluated at the frame's time, run by the GPU on the
|
||||
//! clip's working image before Motion places it, in the CPU's order (effects → Motion → Opacity /
|
||||
//! blend). Anything the shaders don't cover yet — other standard effects, masks, adjustment
|
||||
//! layers, nested sequences, non-dissolve transitions — is rendered on the CPU for that layer (or
|
||||
//! the whole frame) and handed over as a pre-composited image, so the GPU path is always exact
|
||||
//! with respect to the CPU reference.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
@@ -16,17 +19,46 @@ use filmcraft_media::FrameRequest;
|
||||
use filmcraft_project::{ItemId, ItemKind, Project, Sequence, TrackItem};
|
||||
use filmcraft_time::Tick;
|
||||
|
||||
use crate::gpufx::FxOp;
|
||||
use crate::{Blend, RenderOptions, SourceProvider, motion_matrix, output_size};
|
||||
|
||||
/// One layer for the GPU, bottom to top.
|
||||
#[derive(Clone)]
|
||||
pub struct PlanLayer {
|
||||
pub frame: Arc<VideoFrame>,
|
||||
/// Maps frame pixels (0..w, 0..h) to output pixels.
|
||||
/// Maps frame pixels (0..w, 0..h) to output pixels — or, with [`fx`](Self::fx), working-image
|
||||
/// pixels (0..fx.size).
|
||||
pub matrix: Affine,
|
||||
pub opacity: f32,
|
||||
/// How the layer combines with what is under it ([`crate::blend::composite`]).
|
||||
pub blend: Blend,
|
||||
/// Standard effects to run on the frame before it is placed (None: draw the frame directly).
|
||||
pub fx: Option<Arc<LayerFx>>,
|
||||
}
|
||||
|
||||
/// The GPU effect stage of a layer: the frame is decoded into a working image of `size` (the
|
||||
/// frame box-decimated by `decimation`, as the CPU decodes it), `ops` run on it in order, and the
|
||||
/// result is placed with the layer's matrix, opacity and blend mode.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct LayerFx {
|
||||
pub size: (u32, u32),
|
||||
pub decimation: u32,
|
||||
pub ops: Vec<FxOp>,
|
||||
}
|
||||
|
||||
impl PlanLayer {
|
||||
/// A layer drawing `frame` directly.
|
||||
pub fn new(frame: Arc<VideoFrame>, matrix: Affine, opacity: f32, blend: Blend) -> Self {
|
||||
Self { frame, matrix, opacity, blend, fx: None }
|
||||
}
|
||||
|
||||
/// Size of the picture the matrix places: the working image with effects, else the frame.
|
||||
pub fn size(&self) -> (u32, u32) {
|
||||
match &self.fx {
|
||||
Some(fx) => fx.size,
|
||||
None => (self.frame.width, self.frame.height),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
@@ -45,12 +77,29 @@ fn simple_transition(id: &str) -> bool {
|
||||
matches!(id, "cross_dissolve" | "dip_to_black" | "dip_to_white" | "morph_cut")
|
||||
}
|
||||
|
||||
/// Whether a track item's source can be drawn by the GPU as-is (media without standard effects or
|
||||
/// opacity masks; any blend mode).
|
||||
fn gpu_simple(project: &Project, item: &TrackItem) -> bool {
|
||||
/// The standard effects of a media clip if the GPU can draw it (any blend mode, no opacity masks,
|
||||
/// every enabled standard effect unmasked and with a GPU implementation; empty without effects or
|
||||
/// with effects off). None: the clip is rendered on the CPU.
|
||||
fn gpu_chain<'a>(project: &Project, item: &'a TrackItem, opts: RenderOptions) -> Option<Vec<&'a filmcraft_project::EffectInstance>> {
|
||||
let is_media = project.item(item.item).is_some_and(|p| matches!(p.kind, ItemKind::Media(_) | ItemKind::Subclip { .. }));
|
||||
let no_fx = !item.has_standard_effects() && !item.has_opacity_masks();
|
||||
is_media && no_fx
|
||||
if !is_media || item.has_opacity_masks() {
|
||||
return None;
|
||||
}
|
||||
if !opts.effects {
|
||||
return Some(Vec::new());
|
||||
}
|
||||
let mut chain = Vec::new();
|
||||
for e in item.effects.iter().filter(|e| e.def().is_some_and(|d| !d.intrinsic) && !filmcraft_project::graphic::is_layer(e)) {
|
||||
if !e.enabled {
|
||||
continue;
|
||||
}
|
||||
let masked = e.masks.iter().any(|m| m.mode != filmcraft_project::MaskMode::None);
|
||||
if masked || !crate::gpufx::GPU_EFFECTS.contains(&e.effect.as_str()) {
|
||||
return None;
|
||||
}
|
||||
chain.push(e);
|
||||
}
|
||||
Some(chain)
|
||||
}
|
||||
|
||||
/// Plan the frame at timeline `t`.
|
||||
@@ -90,12 +139,7 @@ pub fn plan_frame(project: &Project, seq_id: ItemId, t: Tick, opts: RenderOption
|
||||
match trn.effect.effect.as_str() {
|
||||
"dip_to_black" | "dip_to_white" => {
|
||||
let col = if trn.effect.effect == "dip_to_black" { [0.0, 0.0, 0.0, 1.0] } else { [1.0, 1.0, 1.0, 1.0] };
|
||||
layers.push(PlanLayer {
|
||||
frame: Arc::new(VideoFrame::rgba_f32(1, 1, col.to_vec())),
|
||||
matrix: Affine::scale(w as f64, h as f64),
|
||||
opacity: 1.0,
|
||||
blend: Blend::Normal,
|
||||
});
|
||||
layers.push(PlanLayer::new(Arc::new(VideoFrame::rgba_f32(1, 1, col.to_vec())), Affine::scale(w as f64, h as f64), 1.0, Blend::Normal));
|
||||
let (it, k) = if p < 0.5 { (a, 1.0 - p * 2.0) } else { (b, (p - 0.5) * 2.0) };
|
||||
if let Some(it) = it {
|
||||
push_item(project, seq, it, t, opts, sources, k, Some(Blend::Normal), &mut layers);
|
||||
@@ -121,12 +165,12 @@ pub fn plan_frame(project: &Project, seq_id: ItemId, t: Tick, opts: RenderOption
|
||||
}
|
||||
if opts.captions {
|
||||
for o in crate::caption_overlays(seq, t, w, h) {
|
||||
layers.push(PlanLayer {
|
||||
frame: Arc::new(VideoFrame::rgba_f32(o.w as u32, o.h as u32, o.px)),
|
||||
matrix: Affine::translate(o.x as f64, o.y as f64),
|
||||
opacity: 1.0,
|
||||
blend: Blend::Normal,
|
||||
});
|
||||
layers.push(PlanLayer::new(
|
||||
Arc::new(VideoFrame::rgba_f32(o.w as u32, o.h as u32, o.px)),
|
||||
Affine::translate(o.x as f64, o.y as f64),
|
||||
1.0,
|
||||
Blend::Normal,
|
||||
));
|
||||
}
|
||||
}
|
||||
FramePlan::Layers { width: w, height: h, layers }
|
||||
@@ -179,11 +223,11 @@ fn push_item(
|
||||
let (w, h) = output_size(seq, opts.scale);
|
||||
let m = Affine::scale(opts.scale as f64, opts.scale as f64).then_apply(&motion_matrix(seq, item, size, mt));
|
||||
if let Some((img, x, y)) = crate::graphic_clip::render_graphic_tight(item, mt, size, &m, w, h) {
|
||||
out.push(PlanLayer { frame: cpu_frame(img), matrix: Affine::translate(x as f64, y as f64), opacity: op * extra_opacity, blend: bl });
|
||||
out.push(PlanLayer::new(cpu_frame(img), Affine::translate(x as f64, y as f64), op * extra_opacity, bl));
|
||||
}
|
||||
return;
|
||||
}
|
||||
if gpu_simple(project, item) {
|
||||
if let Some(chain) = gpu_chain(project, item, opts) {
|
||||
let Some(src) = sources.source(item.item) else { return };
|
||||
let Some(size) = crate::source_size(project, item.item) else { return };
|
||||
let motion = motion_matrix(seq, item, size, mt);
|
||||
@@ -193,7 +237,35 @@ fn push_item(
|
||||
let cs = crate::colorman::source_space(project, item.item, &frame);
|
||||
// log / HDR / wide-gamut media is converted on the CPU (below), and so are blended
|
||||
// in-between frames (Frame Blending / Optical Flow on speed-changed clips)
|
||||
if !crate::colorman::needs_management(&seq.settings.color, cs, &frame) && crate::interpolation_blend(item, t, src.info().frame_rate()).is_none() {
|
||||
let plain =
|
||||
!crate::colorman::needs_management(&seq.settings.color, cs, &frame) && crate::interpolation_blend(item, t, src.info().frame_rate()).is_none();
|
||||
if plain && !chain.is_empty() {
|
||||
// GPU effect stage: the working image the CPU would decode (`base_layer`), the
|
||||
// effects evaluated for it, placed as `item_layer` places it
|
||||
let n = crate::decimation(frame.width as f32, size.0 as f32 * want);
|
||||
let (lw, lh) = ((frame.width as usize / n).max(1), (frame.height as usize / n).max(1));
|
||||
let px_scale = lw as f32 / size.0.max(1) as f32;
|
||||
let tc = "";
|
||||
let cx = crate::effects::FxCtx {
|
||||
t: mt,
|
||||
px_scale,
|
||||
seconds: (t - item.start).seconds(),
|
||||
timecode: tc,
|
||||
clip_name: &item.name,
|
||||
project: Some(project),
|
||||
env: None,
|
||||
working: seq.settings.color.working,
|
||||
};
|
||||
let ops: Option<Vec<FxOp>> = chain.iter().map(|e| FxOp::eval(e, &cx, lw, lh).filter(FxOp::gpu_ok)).collect();
|
||||
if let Some(ops) = ops {
|
||||
let m = Affine::scale(opts.scale as f64, opts.scale as f64)
|
||||
.then_apply(&motion)
|
||||
.then_apply(&Affine::scale(1.0 / px_scale as f64, 1.0 / px_scale as f64));
|
||||
let fx = LayerFx { size: (lw as u32, lh as u32), decimation: n as u32, ops };
|
||||
out.push(PlanLayer { frame, matrix: m, opacity: op * extra_opacity, blend: bl, fx: Some(Arc::new(fx)) });
|
||||
return;
|
||||
}
|
||||
} else if plain {
|
||||
// Draft playback at reduced resolution: hand over planes box-filtered to the size
|
||||
// drawn instead of the full picture (a quarter at 1/2, a sixteenth at 1/4 of the
|
||||
// upload and sampling). The mean is taken over Y'CbCr codes, not linear light as
|
||||
@@ -205,14 +277,14 @@ fn push_item(
|
||||
};
|
||||
let px_scale = frame.width as f64 / size.0.max(1) as f64;
|
||||
let m = Affine::scale(opts.scale as f64, opts.scale as f64).then_apply(&motion).then_apply(&Affine::scale(1.0 / px_scale, 1.0 / px_scale));
|
||||
out.push(PlanLayer { frame, matrix: m, opacity: op * extra_opacity, blend: bl });
|
||||
out.push(PlanLayer::new(frame, m, op * extra_opacity, bl));
|
||||
return;
|
||||
}
|
||||
}
|
||||
// CPU-rendered layer (standard effects): drawn by the GPU as a pre-rendered canvas image.
|
||||
let tc = filmcraft_time::format_time(t, seq.settings.frame_rate, seq.settings.drop_frame, filmcraft_time::TimeDisplay::Timecode, 48_000);
|
||||
if let Some((img, op2, _)) = crate::item_layer(project, seq, item, t, opts, sources, &tc) {
|
||||
out.push(PlanLayer { frame: cpu_frame(img), matrix: Affine::IDENTITY, opacity: op2 * extra_opacity, blend: bl });
|
||||
out.push(PlanLayer::new(cpu_frame(img), Affine::IDENTITY, op2 * extra_opacity, bl));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -220,6 +292,17 @@ fn near_identity(m: &Affine) -> bool {
|
||||
(m.a - 1.0).abs() < 1e-9 && (m.d - 1.0).abs() < 1e-9 && m.b.abs() < 1e-9 && m.c.abs() < 1e-9 && m.e.abs() < 1e-6 && m.f.abs() < 1e-6
|
||||
}
|
||||
|
||||
/// A layer's working image with its effects applied, on the CPU (the reference for the GPU
|
||||
/// effect stage).
|
||||
pub fn effect_image(frame: &VideoFrame, fx: &LayerFx) -> crate::Image {
|
||||
let (w, h, px) = frame.to_linear_f32_decimated(fx.decimation.max(1) as usize);
|
||||
let mut img = crate::Image { w, h, px };
|
||||
for op in &fx.ops {
|
||||
op.apply(&mut img);
|
||||
}
|
||||
img
|
||||
}
|
||||
|
||||
/// Execute a plan on the CPU (reference for the GPU compositor).
|
||||
pub fn execute_cpu(plan: &FramePlan) -> crate::Image {
|
||||
match plan {
|
||||
@@ -227,7 +310,10 @@ pub fn execute_cpu(plan: &FramePlan) -> crate::Image {
|
||||
FramePlan::Layers { width, height, layers } => {
|
||||
let mut canvas = crate::Image::new(*width, *height);
|
||||
for l in layers {
|
||||
let src = crate::Image { w: l.frame.width as usize, h: l.frame.height as usize, px: l.frame.to_linear_f32() };
|
||||
let src = match &l.fx {
|
||||
Some(fx) => effect_image(&l.frame, fx),
|
||||
None => crate::Image { w: l.frame.width as usize, h: l.frame.height as usize, px: l.frame.to_linear_f32() },
|
||||
};
|
||||
let placed = if l.matrix == Affine::scale(*width as f64, *height as f64) && src.w == 1 && src.h == 1 {
|
||||
crate::Image::filled(*width, *height, src.get(0, 0))
|
||||
} else {
|
||||
|
||||
@@ -99,16 +99,8 @@ pub fn lighting(img: &mut Image, e: &EffectInstance, cx: &FxCtx) {
|
||||
});
|
||||
}
|
||||
|
||||
/// ASC CDL: out = clamp(in × slope + offset)^power per channel, then saturation (Rec. 709 luma),
|
||||
/// on display-encoded values (ASC CDL v1.2).
|
||||
pub fn asc_cdl(img: &mut Image, e: &EffectInstance, cx: &FxCtx) {
|
||||
let s = [fv(e, "r_slope", cx), fv(e, "g_slope", cx), fv(e, "b_slope", cx)];
|
||||
let o = [fv(e, "r_offset", cx), fv(e, "g_offset", cx), fv(e, "b_offset", cx)];
|
||||
let pw = [fv(e, "r_power", cx), fv(e, "g_power", cx), fv(e, "b_power", cx)];
|
||||
let sat = fv(e, "saturation", cx);
|
||||
img.map_rgb(|c, _, _| dec(cdl(enc(c), s, o, pw, sat)));
|
||||
}
|
||||
|
||||
/// ASC CDL (v1.2): clamp(in × slope + offset)^power per channel, then saturation (Rec. 709 luma),
|
||||
/// on display-encoded values (`gpufx::FxOp::AscCdl`).
|
||||
pub(crate) fn cdl(c: [f32; 3], s: [f32; 3], o: [f32; 3], p: [f32; 3], sat: f32) -> [f32; 3] {
|
||||
let mut v = [0.0; 3];
|
||||
for k in 0..3 {
|
||||
@@ -225,64 +217,6 @@ pub fn vignette(img: &mut Image, e: &EffectInstance, cx: &FxCtx) {
|
||||
});
|
||||
}
|
||||
|
||||
/// Channel Mix: each output channel = Σ input × weight / 100 + constant / 100 (display-encoded).
|
||||
pub fn channel_mix(img: &mut Image, e: &EffectInstance, cx: &FxCtx) {
|
||||
let g = |k: &str| fv(e, k, cx) / 100.0;
|
||||
let mut m = [[g("rr"), g("rg"), g("rb"), g("rc")], [g("gr"), g("gg"), g("gb"), g("gc")], [g("br"), g("bg"), g("bb"), g("bc")]];
|
||||
if bv(e, "monochrome") {
|
||||
m = [m[0]; 3];
|
||||
}
|
||||
img.map_rgb(|c, _, _| {
|
||||
let v = enc(c);
|
||||
dec([0, 1, 2].map(|k| (m[k][0] * v[0] + m[k][1] * v[1] + m[k][2] * v[2] + m[k][3]).clamp(0.0, 1.0)))
|
||||
});
|
||||
}
|
||||
|
||||
/// Color Replace: pixels within Similarity of Target take the Replace colour (keeping their
|
||||
/// lightness unless Solid Colors).
|
||||
pub fn color_replace(img: &mut Image, e: &EffectInstance, cx: &FxCtx) {
|
||||
let sim = fv(e, "similarity", cx) / 100.0 * 1.2;
|
||||
let solid = bv(e, "solid");
|
||||
let t = cv(e, "target", cx);
|
||||
let r = cv(e, "replace", cx);
|
||||
let rh = rgb_to_hsl(r[0], r[1], r[2]);
|
||||
img.map_rgb(|c, _, _| {
|
||||
let v = enc(c);
|
||||
let d = ((v[0] - t[0]).powi(2) + (v[1] - t[1]).powi(2) + (v[2] - t[2]).powi(2)).sqrt();
|
||||
let k = 1.0 - smoothstep(sim * 0.85, sim.max(1e-4), d);
|
||||
if k <= 0.0 {
|
||||
return c;
|
||||
}
|
||||
let target = if solid {
|
||||
[r[0], r[1], r[2]]
|
||||
} else {
|
||||
let l = rgb_to_hsl(v[0], v[1], v[2])[2];
|
||||
hsl_to_rgb(rh[0], rh[1], l)
|
||||
};
|
||||
dec(lerp3(v, target, k))
|
||||
});
|
||||
}
|
||||
|
||||
/// Alpha Adjust: opacity, ignore / invert alpha, show the matte only.
|
||||
pub fn alpha_adjust(img: &mut Image, e: &EffectInstance, cx: &FxCtx) {
|
||||
let op = fv(e, "opacity", cx) / 100.0;
|
||||
let (ignore, invert, mask_only) = (bv(e, "ignore"), bv(e, "invert"), bv(e, "mask_only"));
|
||||
img.px.par_chunks_mut(4).for_each(|p| {
|
||||
let c = Image::unpremul([p[0], p[1], p[2], p[3]]);
|
||||
let mut a = if ignore { 1.0 } else { p[3] };
|
||||
if invert {
|
||||
a = 1.0 - a;
|
||||
}
|
||||
a = (a * op).clamp(0.0, 1.0);
|
||||
if mask_only {
|
||||
let g = filmcraft_color::srgb_to_linear(a);
|
||||
p.copy_from_slice(&[g, g, g, 1.0]);
|
||||
} else {
|
||||
p.copy_from_slice(&[c[0] * a, c[1] * a, c[2] * a, a]);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// Logo Cutout: removes a flat white/black/custom background by un-multiplying it — the alpha of
|
||||
/// a pixel is how far it is from the background towards the gamut edge, and the colour is
|
||||
/// recovered as if it had been composited over that background.
|
||||
|
||||
@@ -13,6 +13,7 @@
|
||||
|
||||
mod blur;
|
||||
mod color;
|
||||
pub(crate) use color::cdl;
|
||||
mod distort;
|
||||
mod immersive;
|
||||
mod lights;
|
||||
@@ -459,12 +460,8 @@ pub fn apply(img: &mut Image, e: &EffectInstance, cx: &FxCtx) -> bool {
|
||||
match e.effect.as_str() {
|
||||
// Adjust / Color / Image Control / Keying / Utility colour
|
||||
"lighting_effects" => color::lighting(img, e, cx),
|
||||
"asc_cdl" => color::asc_cdl(img, e, cx),
|
||||
"video_limiter" => color::video_limiter(img, e, cx),
|
||||
"vignette" => color::vignette(img, e, cx),
|
||||
"channel_mix" => color::channel_mix(img, e, cx),
|
||||
"color_replace" => color::color_replace(img, e, cx),
|
||||
"alpha_adjust" => color::alpha_adjust(img, e, cx),
|
||||
"logo_cutout" => color::logo_cutout(img, e, cx),
|
||||
"ultra_key" => color::ultra_key(img, e, cx),
|
||||
"track_matte" => color::track_matte(img, e, cx),
|
||||
|
||||
@@ -626,7 +626,7 @@ fn parse_args() -> Args {
|
||||
"--json" => a.json = it.next(),
|
||||
"--help" | "-h" => {
|
||||
println!(
|
||||
"bench_playback [--scenario h264-1080,stack3,stack3-blend,h264-2160,hevc-2160,vp9-2160,av1-2160,demo,after-preview,seek-storm] [--res full,half] [--cpu|--gpu] [--seconds 8] [--refresh 60] [--workers N] [--repeat N] [--json out.json]"
|
||||
"bench_playback [--scenario h264-1080,stack3,stack3-blend,stack3-fx,h264-2160,hevc-2160,vp9-2160,av1-2160,demo,after-preview,seek-storm] [--res full,half] [--cpu|--gpu] [--seconds 8] [--refresh 60] [--workers N] [--repeat N] [--json out.json]"
|
||||
);
|
||||
std::process::exit(0);
|
||||
}
|
||||
@@ -634,8 +634,10 @@ fn parse_args() -> Args {
|
||||
}
|
||||
}
|
||||
if a.scenarios.is_empty() {
|
||||
a.scenarios =
|
||||
["h264-1080", "stack3", "stack3-blend", "h264-2160", "hevc-2160", "demo", "after-preview", "seek-storm"].iter().map(|s| s.to_string()).collect();
|
||||
a.scenarios = ["h264-1080", "stack3", "stack3-blend", "stack3-fx", "h264-2160", "hevc-2160", "demo", "after-preview", "seek-storm"]
|
||||
.iter()
|
||||
.map(|s| s.to_string())
|
||||
.collect();
|
||||
}
|
||||
a
|
||||
}
|
||||
@@ -648,7 +650,7 @@ pub fn scenario_session(name: &str) -> Option<(Session, ItemId)> {
|
||||
let a = import(&mut s, &fixture("a1080.mp4")?);
|
||||
build_sequence(&mut s, 1920, 1080, &[(a, 100.0, None, 0.0, 100.0)], &[], 20.0)
|
||||
}
|
||||
"stack3" | "stack3-blend" => {
|
||||
"stack3" | "stack3-blend" | "stack3-fx" => {
|
||||
let (a, b, c) = (fixture("a1080.mp4")?, fixture("b1080.mp4")?, fixture("c1080.mp4")?);
|
||||
let (a, b, c) = (import(&mut s, &a), import(&mut s, &b), import(&mut s, &c));
|
||||
let seq = build_sequence(
|
||||
@@ -675,6 +677,30 @@ pub fn scenario_session(name: &str) -> Option<(Session, ItemId)> {
|
||||
})
|
||||
.ok()?;
|
||||
}
|
||||
if name == "stack3-fx" {
|
||||
// V2: Brightness & Contrast + Gaussian Blur; V3: Tint
|
||||
s.edit("effects", |p, _| {
|
||||
let q = p.sequence_mut(seq).expect("seq");
|
||||
let fx: [(usize, &str, &[(&str, f64)]); 3] = [
|
||||
(1, "brightness_contrast", &[("brightness", 12.0), ("contrast", 25.0)]),
|
||||
(1, "gaussian_blur", &[("blurriness", 8.0)]),
|
||||
(2, "tint", &[("amount", 70.0)]),
|
||||
];
|
||||
for (track, id, params) in fx {
|
||||
for it in &mut q.video_tracks[track].items {
|
||||
let mut e = find_effect(id).expect("effect").instance();
|
||||
for (k, v) in params {
|
||||
if let Some(x) = e.params.get_mut(*k) {
|
||||
x.value = ParamValue::Float(*v);
|
||||
}
|
||||
}
|
||||
it.effects.push(e);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.ok()?;
|
||||
}
|
||||
seq
|
||||
}
|
||||
"h264-2160" | "seek-storm-2160" => {
|
||||
|
||||
@@ -162,6 +162,7 @@ pub fn playback(o: &Opts) -> Vec<Value> {
|
||||
("h264-1080", "full", 1.0, false),
|
||||
("stack3", "full", 1.0, false),
|
||||
("stack3-blend", "full", 1.0, false),
|
||||
("stack3-fx", "full", 1.0, false),
|
||||
("h264-2160", "full", 1.0, false),
|
||||
("h264-2160", "half", 0.5, false),
|
||||
("h264-2160", "half draft", 0.5, true),
|
||||
|
||||
@@ -883,13 +883,14 @@ fn plan_job(job: &Job, seq: ItemId, pool: &Arc<MediaPool>, services: &Arc<dyn Se
|
||||
// captions (they are not part of the preview hash), so captions are layered on live.
|
||||
let (w, h) = filmcraft_render::output_size(q, job.scale);
|
||||
let matrix = filmcraft_geom::Affine::scale(w as f64 / frame.width.max(1) as f64, h as f64 / frame.height.max(1) as f64);
|
||||
let mut layers = vec![filmcraft_render::plan::PlanLayer { frame, matrix, opacity: 1.0, blend: filmcraft_render::Blend::Normal }];
|
||||
let mut layers = vec![filmcraft_render::plan::PlanLayer { frame, matrix, opacity: 1.0, blend: filmcraft_render::Blend::Normal, fx: None }];
|
||||
for o in filmcraft_render::caption_overlays(q, job.time, w, h) {
|
||||
layers.push(filmcraft_render::plan::PlanLayer {
|
||||
frame: Arc::new(filmcraft_frame::VideoFrame::rgba_f32(o.w as u32, o.h as u32, o.px)),
|
||||
matrix: filmcraft_geom::Affine::translate(o.x as f64, o.y as f64),
|
||||
opacity: 1.0,
|
||||
blend: filmcraft_render::Blend::Normal,
|
||||
fx: None,
|
||||
});
|
||||
}
|
||||
return (filmcraft_render::plan::FramePlan::Layers { width: w, height: h, layers }, true);
|
||||
|
||||
@@ -1314,7 +1314,8 @@ mod gpu_fallback_tests {
|
||||
#[test]
|
||||
fn plan_side_covers_output_and_every_layer() {
|
||||
let frame = |w: u32, h: u32| std::sync::Arc::new(filmcraft_frame::VideoFrame::rgba_f32(w, h, vec![0.0; (w * h * 4) as usize]));
|
||||
let layer = |w, h| PlanLayer { frame: frame(w, h), matrix: filmcraft_geom::Affine::IDENTITY, opacity: 1.0, blend: filmcraft_render::Blend::Normal };
|
||||
let layer =
|
||||
|w, h| PlanLayer { frame: frame(w, h), matrix: filmcraft_geom::Affine::IDENTITY, opacity: 1.0, blend: filmcraft_render::Blend::Normal, fx: None };
|
||||
// a 4K source in an HD sequence still needs a 3840-wide texture on the GPU
|
||||
let p = FramePlan::Layers { width: 1920, height: 1080, layers: vec![layer(64, 64), layer(3840, 2160)] };
|
||||
assert_eq!(super::plan_side(&p), 3840);
|
||||
|
||||
+61
-2
@@ -39,6 +39,9 @@ claude mcp add filmcraft-headless -- /abs/path/filmcraft/target/release/filmcraf
|
||||
|---|---|---|
|
||||
| `command_list` | both | every command: id, label, menu, shortcut, params, enabled now (`filter`, `enabled_only`) |
|
||||
| `command_run` | both | run a command `{id, params}`; edits are undoable |
|
||||
| `command_batch` | both | run several commands in order `{steps: [{id, params}], stop_on_error}` → `{completed, failed, results}` |
|
||||
| `doc_inspect` | both | the project tree and the active sequence in one call (`project_inspect` + `sequence_inspect`) |
|
||||
| `render_preview` | both | same as `render_frame` |
|
||||
| `project_inspect` | both | bins and items with ids, types, durations; active sequence |
|
||||
| `sequence_inspect` | both | the active sequence: tracks, clips (ticks and frames), effects, transitions, markers, playhead, selection |
|
||||
| `media_import` | both | import files by absolute path (`text`, one path per line) |
|
||||
@@ -55,6 +58,47 @@ claude mcp add filmcraft-headless -- /abs/path/filmcraft/target/release/filmcraf
|
||||
Typical loop: `project_inspect` / `sequence_inspect` → get ids → `command_run` → `render_frame` or
|
||||
`ui_screenshot` → look at the result → `edit.undo` if it's wrong.
|
||||
|
||||
### Conventions
|
||||
|
||||
The server follows a few conventions that make it predictable for agents. They are not
|
||||
FilmCraft-specific: the sibling craft apps' MCP servers can follow the same ones, so an agent that has
|
||||
driven one app can drive the others.
|
||||
|
||||
- **Core tools.** `command_list`, `command_run`, `command_batch`, `doc_inspect` and `render_preview`
|
||||
work in both modes. The FilmCraft tools above stay as they are.
|
||||
- **Titles and annotations.** Every tool has a `title` and the four MCP hints (`readOnlyHint`,
|
||||
`destructiveHint`, `idempotentHint`, `openWorldHint`), so a client can let the read-only tools
|
||||
(lists, inspects, renders, screenshots) run without asking and ask before edits. When you add a
|
||||
tool, set them in its `#[tool(...)]` attribute; a test checks every listed tool.
|
||||
- **Strict arguments.** An unknown tool argument is a JSON-RPC `-32602` error that names it and the
|
||||
accepted ones (`unknown argument "filtr" for command_list; expected: enabled_only, filter`), not a
|
||||
silently ignored key. `command_run.params` goes to the command unchanged.
|
||||
- **Errors.** A failing command is an `isError` result with the engine's message. A line that is not
|
||||
JSON gets a `-32700` parse error (id `null`) and the server keeps serving. A command that panics is
|
||||
reported as `internal error: …`, and the session stays usable.
|
||||
- **Resources.** `filmcraft://document` (the project tree and active sequence, as `doc_inspect`) and
|
||||
`filmcraft://commands` (the command catalog, as `command_list`), both `application/json`. For MCP
|
||||
2026-07-28 clients such as current Claude Code, list and read results carry the `ttlMs` and
|
||||
`cacheScope` hints that revision requires.
|
||||
|
||||
### Long exports: progress and cancellation
|
||||
|
||||
`command_run {"id": "file.exportMedia", "params": {…, "wait": true}}` in headless mode blocks until
|
||||
the file is written. It uses the MCP progress and cancellation utilities, so a client can show
|
||||
progress and stop it:
|
||||
|
||||
- The export runs as the same background job `jobs.list` shows. The session is locked only for short
|
||||
polls, so other requests (`ping`, `doc_inspect`, `jobs.list`, …) are answered while it encodes.
|
||||
- With `"_meta": {"progressToken": T}` on the `tools/call`, the server sends
|
||||
`notifications/progress` `{progressToken: T, progress: <frames done>, total: <frames>, message:
|
||||
<job status>}` at most every 100 ms while frames advance. Without a token it sends none.
|
||||
- `notifications/cancelled` for the request stops the encode at the next batch (`jobs.cancel`),
|
||||
deletes the partial output (the movie, or the frames and caption sidecar written so far) and sends
|
||||
no response, as the MCP cancellation utility asks.
|
||||
- Without `wait` the command returns `{job, path}` at once, as before: poll `jobs.list` and stop it
|
||||
with `jobs.cancel`. Other exports (`file.exportFrame`, interchange formats, …) finish quickly and
|
||||
ignore the token. Bridge mode forwards the call unchanged; the app shows its own progress.
|
||||
|
||||
## 2. Control channel
|
||||
|
||||
`filmcraft --control 9876` (or `FILMCRAFT_CONTROL_PORT=9876`) listens on `127.0.0.1` only. Send one
|
||||
@@ -140,7 +184,8 @@ media, sidecar and `ATTRIBUTION.md` entry.
|
||||
|
||||
1. [AGENTS.md](../AGENTS.md): absolute rules (assets, clean-room, licences).
|
||||
2. [CLAUDE.md](../CLAUDE.md): working instructions and non-negotiables.
|
||||
3. [ROADMAP.md](../ROADMAP.md): milestones, what's done, what's running.
|
||||
3. [ROADMAP.md](../ROADMAP.md): read the **honest assessment** and **Where we are lacking** first, then
|
||||
milestones, what's done and what's running.
|
||||
4. [architecture.md](architecture.md), then the README and tests of the crate you'll touch.
|
||||
5. [contributing.md](contributing.md) (how to add things, gates) and [testing.md](testing.md).
|
||||
|
||||
@@ -151,7 +196,9 @@ everything you need to contribute is in the public docs above. Ask a maintainer
|
||||
## 5. Autonomous work loop
|
||||
|
||||
1. **Orient.** Pick the next task: the next unchecked task in the maintainer status file, or an open
|
||||
ROADMAP item. Read the relevant architecture section and crate README.
|
||||
ROADMAP item. Prefer the gaps in ROADMAP's *Where we are lacking* (speed, correctness on real
|
||||
media, measurement, Windows/Linux) over adding more checklist items. Read the relevant
|
||||
architecture section and crate README.
|
||||
2. **Plan tests first.** Write down the acceptance test before writing code.
|
||||
3. **Implement and test.**
|
||||
4. **Verify.** Run all gates (`cargo xtask ci`). For UI work, run the app with `--control`, drive it
|
||||
@@ -190,3 +237,15 @@ A feature is done when:
|
||||
- [ ] all gates pass (`cargo xtask ci`), and new assets have sidecars;
|
||||
- [ ] the crate README and ROADMAP.md are updated where relevant;
|
||||
- [ ] it is committed with its task id.
|
||||
|
||||
### Reporting progress honestly
|
||||
|
||||
ROADMAP.md keeps two numbers: the **feature checklist** (does it exist?) and **ready for real
|
||||
work** (does it hold up?). When you update either:
|
||||
|
||||
- say how a number was obtained: *measured* (a test, a diff, a benchmark you ran) or *estimated*;
|
||||
- count an approximation, a stub or a partly wired setting as such, never as done;
|
||||
- record known bugs and limitations in the crate README and the honest assessment, not only
|
||||
what works;
|
||||
- benchmarks state the machine, the load average and before/after on the same build;
|
||||
- don't raise a percentage without evidence a reviewer can rerun.
|
||||
+20
-3
@@ -253,9 +253,21 @@ file ──► codecs (MP4/MOV, MKV, audio) demux + decode, GOP-aware see
|
||||
turns a frame into GPU layers, each with its opacity and blend mode. All 27 blend modes run on the
|
||||
GPU (`filmcraft-gpu`): Normal and Dissolve with fixed-function "over" blending, the others by
|
||||
copying the accumulator under the layer into a backdrop texture and compositing in the fragment
|
||||
shader with the CPU reference's formulas. Standard effects, adjustment layers, nested sequences
|
||||
and non-dissolve transitions are rendered on the CPU for that layer or frame and handed to the
|
||||
GPU as an image (a layer image keeps its clip's blend mode), so both paths give the same picture. Setting `FILMCRAFT_CPU_COMPOSITE=1`
|
||||
shader with the CPU reference's formulas. The common standard effects run on the GPU too
|
||||
(`render::gpufx`, `gpu::fx`): Brightness & Contrast, ProcAmp, Tint, Black & White, Color
|
||||
Balance, Leave Color, Change to Color, Color Pass, Color Replace, Channel Mix, ASC CDL, Gamma
|
||||
Correction, Levels, Extract, Invert, Posterize, Alpha Adjust, Gaussian Blur and Directional Blur
|
||||
(and their legacy aliases), Camera Blur, Sharpen, Unsharp Mask, Crop, Edge Feather, Transform,
|
||||
Horizontal / Vertical Flip, Mirror and Offset. When every enabled effect of a media clip is in
|
||||
that set (unmasked, with finite parameters, and no Transform shrinking the picture below half
|
||||
size, which the CPU pre-filters), the plan hands the GPU the clip's source with
|
||||
the effects' parameters evaluated at that time: the source is drawn into an `Rgba32Float`
|
||||
working image at the size the CPU decodes it, each effect runs as compute passes with the CPU
|
||||
reference's math (effects, then Motion, then Opacity / blend, as on the CPU), and the result is
|
||||
placed like any layer. Other standard effects, effect and opacity masks, adjustment layers,
|
||||
nested sequences and non-dissolve transitions are rendered on the CPU for that layer or frame
|
||||
and handed to the GPU as an image (a layer image keeps its clip's blend mode), so both paths
|
||||
give the same picture. Setting `FILMCRAFT_CPU_COMPOSITE=1`
|
||||
forces the CPU path in the desktop app.
|
||||
- **Frame scheduling.** `crates/ui-egui/src/frames.rs` runs a small pool of worker threads with
|
||||
prioritised jobs: the frame on screen first, then playback prefetch, then thumbnails. The UI never
|
||||
@@ -589,6 +601,11 @@ file.exportMedia {path, preset?, settings?, format?, range?, …} export.qui
|
||||
| `png`, `tiff`, `bmp` | `image` | numbered stills `<name>000.<ext>`, `<name>001.<ext>` … |
|
||||
| `gif`, `wav`, `aiff` | built in / `image` | GIF / RIFF WAVE (`WAVE_FORMAT_EXTENSIBLE` for 5.1) / AIFF (16- or 24-bit PCM) |
|
||||
|
||||
- **Reproducible.** The same project and settings give the same file on every machine. Frames render
|
||||
in batches of one per core, but the muxer gets them in fixed groups of 16 output frames
|
||||
(`INTERLEAVE` in `crates/export/src/job.rs`), each followed by its audio, and H.264 uses one slice
|
||||
per four macroblock rows instead of one per core. MXF files are the exception: their UMIDs and
|
||||
modification date come from the clock, as SMPTE ST 330 / ST 377 expect.
|
||||
- **Settings.** `ExportSettings` (serde, camelCase, every field optional) holds Video (frame size or
|
||||
Match Source, frame rate, Scale to Fit / Fill / Stretch, pixel aspect, field order — progressive
|
||||
only, profile / level, bitrate encoding, target / maximum / adaptive bitrate, keyframe distance,
|
||||
|
||||
@@ -133,6 +133,10 @@ export CARGO_TARGET_DIR=target/agent-hevc # separate build dir, no lock figh
|
||||
so reduced-resolution playback matches full resolution. Unknown ids are a no-op.
|
||||
3. Add a test in `crates/render/src/tests.rs`.
|
||||
4. The GPU path needs no change: `render::plan` pre-renders layers with standard effects on the CPU.
|
||||
To run it on the GPU too, make it an `FxOp` in `crates/render/src/gpufx.rs` instead (parameter
|
||||
evaluation in `FxOp::eval`, the CPU reference in `FxOp::apply`, the id in `GPU_EFFECTS`), add
|
||||
its passes to `crates/gpu/src/fx.rs` and its math to `fx.wgsl`, and a parity case to
|
||||
`crates/gpu/src/fx_tests.rs`.
|
||||
|
||||
Audio effects implement `AudioEffect` in `crates/audio-dsp` and register in its `effects()`
|
||||
registry. Then add an `audio(…)` definition in `effect.rs` and a `Mapping` from project parameters
|
||||
|
||||
@@ -103,7 +103,10 @@ Ctrl), `Ctrl` (macOS ⌃), `Alt`, `Shift`. The dialog (Edit ▸ Keyboard Shortcu
|
||||
## MCP
|
||||
`filmcraft-cli mcp` serves MCP on stdio: headless (in-process session; `--demo` / `--project p.fcproj`)
|
||||
or `--bridge 127.0.0.1:9876` to drive the running app. Tools: `command_list`, `command_run`,
|
||||
`project_inspect`, `sequence_inspect`, `media_import`, `render_frame`, `ui_inspect`, `ui_elements`,
|
||||
`ui_click`, `ui_drag`, `ui_key`, `ui_type`, `ui_screenshot`, `ui_control`. `.mcp.json` registers both.
|
||||
`command_batch`, `doc_inspect`, `render_preview`, `project_inspect`, `sequence_inspect`, `media_import`,
|
||||
`render_frame`, `ui_inspect`, `ui_elements`, `ui_click`, `ui_drag`, `ui_key`, `ui_type`, `ui_screenshot`,
|
||||
`ui_control`; resources `filmcraft://document` and `filmcraft://commands`. `.mcp.json` registers both.
|
||||
Annotations, argument checking, errors and export progress / cancellation:
|
||||
[agents.md § Conventions](agents.md#conventions).
|
||||
|
||||
Time is in ticks (254 016 000 000 per second); commands also accept `seconds`, `frame` or `timecode`.
|
||||
@@ -13,6 +13,30 @@ the before/after comparison relies on. Each milestone below alternated base / af
|
||||
each); the most load-independent figure is the decoder's own cycle count (`proc_pid_rusage`
|
||||
instructions and cycles of a process, all threads summed).
|
||||
|
||||
## Results (GPU2: standard effects on the GPU compositor, #30, before → after)
|
||||
|
||||
Before = this change with clips that carry standard effects sent back to the CPU layer path in
|
||||
`render::plan` (same binary otherwise); after = GPU2. New `stack3-fx` scenario: `stack3` with
|
||||
Brightness & Contrast + Gaussian Blur (blurriness 8) on V2 and Tint (70 %) on V3. `bench_playback
|
||||
--scenario stack3-fx,stack3 --res full|half --gpu` (release), 8 s plays, two rounds alternating
|
||||
before / after on 2026-10-05 at load average 107–170 (1-minute; 5-minute 141–152). `stack3` in
|
||||
the same runs is the control; it shows how much the load alone moved the numbers.
|
||||
|
||||
| case | shown/dropped before (2 runs) | after (2 runs) | CPU ms/frame before → after | worker svc p50 before → after |
|
||||
|---|---|---|---|---|
|
||||
| stack3-fx Full | 0/192, 0/192 | **192/0, 132/60** | 84, 92 → 94, 89 | 245, 220 → **1.8, 93** ms |
|
||||
| stack3-fx 1/2 | 9/183, 15/177 | **181/11, 166/26** | 111, 110 → 92, 92 | 295, 307 → **6.5, 28** ms |
|
||||
| stack3 Full (control) | 180/12, 189/3 | 172/19, 80/112 | 92, 92 → 89, 80 | 7.2, 5.1 → 17.8, 185 ms |
|
||||
| stack3 1/2 (control) | 138/54, 181/11 | 116/76, 134/53 | 91, 92 → 83, 91 | 61, 15 → 123, 106 ms |
|
||||
|
||||
Before, every frame of the effect stack was rendered on the CPU (Gaussian Blur and the two colour
|
||||
effects on 1080p layers): ≈ 250 ms of worker time per frame, so playback showed nothing at Full
|
||||
and 5–8 % of frames at ½ (CPU ms/frame stays near 90 only because most jobs were cancelled
|
||||
before finishing). After, the workers only decode, as for `stack3`: the effect stage runs on the
|
||||
GPU at present time (UI present p50 11–24 ms vs 11–39 ms for the control in the same runs). The
|
||||
second "after Full" run coincided with a load spike (load 155 → 170; the control dropped 112
|
||||
frames in the same minute).
|
||||
|
||||
## Results (HW1: VideoToolbox hardware decoding, Off → Auto)
|
||||
|
||||
Same commit, Settings ▸ Playback ▸ Hardware decoding switched with the bench flag `--hw off` /
|
||||
|
||||
@@ -17,15 +17,15 @@
|
||||
Scope="perMachine"
|
||||
Compressed="yes"
|
||||
Language="1033">
|
||||
<SummaryInformation Description="FilmCraft image editor" Manufacturer="Learning Machines LLC" />
|
||||
<SummaryInformation Description="FilmCraft video editor" Manufacturer="Learning Machines LLC" />
|
||||
<!-- Same-version upgrades let successive pre-releases of one X.Y.Z replace each other
|
||||
(MSI versions are numeric, so 1.2.3-rc.1 and 1.2.3-rc.2 are both 1.2.3). -->
|
||||
<MajorUpgrade AllowSameVersionUpgrades="yes"
|
||||
DowngradeErrorMessage="A newer version of FilmCraft is already installed." />
|
||||
<MediaTemplate EmbedCab="yes" CompressionLevel="high" />
|
||||
|
||||
<Icon Id="FilmcraftIcon" SourceFile="$(var.IconPath)" />
|
||||
<Property Id="ARPPRODUCTICON" Value="FilmcraftIcon" />
|
||||
<Icon Id="FilmcraftIcon.ico" SourceFile="$(var.IconPath)" />
|
||||
<Property Id="ARPPRODUCTICON" Value="FilmcraftIcon.ico" />
|
||||
<Property Id="ARPURLINFOABOUT" Value="https://getartcraft.com/apps/filmcraft" />
|
||||
<Property Id="ARPURLUPDATEINFO" Value="https://getartcraft.com/apps/filmcraft" />
|
||||
<Property Id="ARPHELPLINK" Value="https://discord.gg/artcraft" />
|
||||
@@ -45,8 +45,8 @@
|
||||
<Component Id="FilmcraftApp" Guid="15933017-925B-4963-8992-52000EB26AAA">
|
||||
<File Id="FilmcraftExe" Source="$(var.BinDir)\filmcraft.exe" KeyPath="yes">
|
||||
<Shortcut Id="StartMenuShortcut" Directory="ProgramMenuFolder" Name="FilmCraft"
|
||||
Description="FilmCraft image editor" WorkingDirectory="INSTALLFOLDER" Advertise="yes"
|
||||
Icon="FilmcraftIcon" IconIndex="0" />
|
||||
Description="FilmCraft video editor" WorkingDirectory="INSTALLFOLDER" Advertise="yes"
|
||||
Icon="FilmcraftIcon.ico" IconIndex="0" />
|
||||
</File>
|
||||
|
||||
<RegistryValue Root="HKLM" Key="Software\Microsoft\Windows\CurrentVersion\App Paths\filmcraft.exe" Value="[#FilmcraftExe]" Type="string" />
|
||||
|
||||
Reference in new issue
Block a user