From 93b06a6eab9c1636524be8173d95b074f02b20b8 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Thu, 15 Oct 2026 22:56:38 +0200 Subject: [PATCH 01/12] Bump FilmCraft to 4e2d0bc2 (part of #804) Problem: the hardware-decoding work (#819) needs a newer FilmCraft than the pinned 2c5ba7a, and the pin bump should be reviewable apart from it. Fix: move all filmcraft-* deps to 4e2d0bc2. Adapt to MediaInfo.audio -> audio_streams and VideoStreamInfo.hdr; ask the pool for a quarter into the frame because FilmCraft now rounds requests half up; adapt rotation and the AAC downmix; use Decoder::with_threads(1) in the VP9 tests. No filmcraft-platform, no hardware setting. Co-Authored-By: Claude Sonnet 5.5 --- Cargo.lock | 115 ++++++++++++++------------ Cargo.toml | 32 +++---- crates/export/tests/vp9_roundtrip.rs | 6 +- crates/host/tests/timeline_interop.rs | 3 +- crates/interchange/src/export.rs | 6 +- crates/interchange/src/import.rs | 4 +- crates/interchange/src/tests.rs | 6 +- crates/media/src/downmix.rs | 39 ++++++++- crates/media/src/pool.rs | 11 ++- crates/media/src/probe.rs | 15 +++- 10 files changed, 151 insertions(+), 86 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ce8fe1231..98e00791d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2025,8 +2025,8 @@ checksum = "b97b65636e5b9ef369943878ac74335ba1c55c1cb6adbf1e2c293c624248d693" [[package]] name = "filmcraft-aac" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" dependencies = [ "filmcraft-bitstream", "thiserror 2.0.21", @@ -2034,16 +2034,26 @@ dependencies = [ [[package]] name = "filmcraft-ac3" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" dependencies = [ "thiserror 2.0.21", ] +[[package]] +name = "filmcraft-apv" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" +dependencies = [ + "filmcraft-bitstream", + "rayon", + "thiserror 2.0.21", +] + [[package]] name = "filmcraft-av1" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" dependencies = [ "filmcraft-bitstream", "rayon", @@ -2052,21 +2062,22 @@ dependencies = [ [[package]] name = "filmcraft-bitstream" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" [[package]] name = "filmcraft-cfb" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" [[package]] name = "filmcraft-codecs" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" dependencies = [ "filmcraft-aac", "filmcraft-ac3", + "filmcraft-apv", "filmcraft-av1", "filmcraft-bitstream", "filmcraft-color", @@ -2094,16 +2105,16 @@ dependencies = [ [[package]] name = "filmcraft-color" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" dependencies = [ "serde", ] [[package]] name = "filmcraft-dnx" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" dependencies = [ "filmcraft-bitstream", "rayon", @@ -2112,8 +2123,8 @@ dependencies = [ [[package]] name = "filmcraft-frame" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" dependencies = [ "filmcraft-color", "filmcraft-time", @@ -2123,16 +2134,16 @@ dependencies = [ [[package]] name = "filmcraft-geom" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" dependencies = [ "serde", ] [[package]] name = "filmcraft-h264" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" dependencies = [ "filmcraft-bitstream", "rayon", @@ -2141,8 +2152,8 @@ dependencies = [ [[package]] name = "filmcraft-h264enc" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" dependencies = [ "filmcraft-bitstream", "rayon", @@ -2151,8 +2162,8 @@ dependencies = [ [[package]] name = "filmcraft-hevc" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" dependencies = [ "filmcraft-bitstream", "rayon", @@ -2161,8 +2172,8 @@ dependencies = [ [[package]] name = "filmcraft-interchange" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" dependencies = [ "filmcraft-cfb", "filmcraft-color", @@ -2178,8 +2189,8 @@ dependencies = [ [[package]] name = "filmcraft-isobmff" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" dependencies = [ "filmcraft-bitstream", "thiserror 2.0.21", @@ -2187,13 +2198,13 @@ dependencies = [ [[package]] name = "filmcraft-matroska" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" [[package]] name = "filmcraft-media" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" dependencies = [ "filmcraft-color", "filmcraft-frame", @@ -2206,8 +2217,8 @@ dependencies = [ [[package]] name = "filmcraft-mpeg2v" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" dependencies = [ "rayon", "thiserror 2.0.21", @@ -2215,31 +2226,31 @@ dependencies = [ [[package]] name = "filmcraft-mpegts" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" dependencies = [ "thiserror 2.0.21", ] [[package]] name = "filmcraft-mxf" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" [[package]] name = "filmcraft-ogg" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" [[package]] name = "filmcraft-opus" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" [[package]] name = "filmcraft-project" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" dependencies = [ "filmcraft-color", "filmcraft-geom", @@ -2252,8 +2263,8 @@ dependencies = [ [[package]] name = "filmcraft-prores" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" dependencies = [ "filmcraft-bitstream", "rayon", @@ -2262,16 +2273,16 @@ dependencies = [ [[package]] name = "filmcraft-time" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" dependencies = [ "serde", ] [[package]] name = "filmcraft-vp9" -version = "0.1.1" -source = "git+https://github.com/storytold/filmcraft?rev=2c5ba7a72619935f0481f99d59409e0d1844b234#2c5ba7a72619935f0481f99d59409e0d1844b234" +version = "0.6.0" +source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" dependencies = [ "filmcraft-bitstream", "rayon", diff --git a/Cargo.toml b/Cargo.toml index 9f4e9d17f..cb8b7ea38 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -71,26 +71,26 @@ image = { version = "0.25", default-features = false, features = ["png", "jpeg"] # media codecs + containers: FilmCraft's pure-Rust crates, pinned to a revision (plan/adr/0001). # Only `crates/media`, `crates/export` and `crates/interchange` may depend on these. -filmcraft-codecs = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" } -filmcraft-media = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" } -filmcraft-frame = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" } -filmcraft-color = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" } -filmcraft-time = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" } +filmcraft-codecs = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } +filmcraft-media = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } +filmcraft-frame = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } +filmcraft-color = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } +filmcraft-time = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } # encoders + muxer (crates/export) -filmcraft-h264enc = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" } -filmcraft-prores = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" } -filmcraft-aac = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" } -filmcraft-isobmff = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" } +filmcraft-h264enc = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } +filmcraft-prores = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } +filmcraft-aac = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } +filmcraft-isobmff = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } # decoder used as the round-trip oracle of effectcraft-opusenc (crates/export dev-dependency) -filmcraft-opus = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" } -filmcraft-vp9 = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" } +filmcraft-opus = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } +filmcraft-vp9 = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } # decoders used as the round-trip oracles of effectcraft-hevcenc / effectcraft-av1enc (dev-dependencies only) -filmcraft-hevc = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" } -filmcraft-av1 = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" } +filmcraft-hevc = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } +filmcraft-av1 = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } # timeline interchange (EDL, FCP7 XML, FCPXML, OTIO) for Premiere Pro interop (crates/interchange only) -filmcraft-interchange = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" } -filmcraft-project = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" } -filmcraft-geom = { git = "https://github.com/storytold/filmcraft", rev = "2c5ba7a72619935f0481f99d59409e0d1844b234" } +filmcraft-interchange = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } +filmcraft-project = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } +filmcraft-geom = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } # gpu (the same wgpu eframe / egui-wgpu use) wgpu = { version = "30.0.1", features = ["fragile-send-sync-non-atomic-wasm"] } diff --git a/crates/export/tests/vp9_roundtrip.rs b/crates/export/tests/vp9_roundtrip.rs index e7c2b0346..1432e3892 100644 --- a/crates/export/tests/vp9_roundtrip.rs +++ b/crates/export/tests/vp9_roundtrip.rs @@ -41,7 +41,7 @@ fn source(w: usize, h: usize, kind: u32) -> (Vec, Vec, Vec) { } fn decode(frame: &[u8]) -> filmcraft_vp9::Picture { - let mut d = Decoder::new(); + let mut d = Decoder::with_threads(1); let mut pics = d.decode(frame, 0).expect("decode"); assert_eq!(pics.len(), 1); pics.remove(0) @@ -189,7 +189,7 @@ fn planes_of(p: &filmcraft_vp9::Picture, w: usize, h: usize) -> [Vec; 3] { fn run_sequence(cfg: EncoderConfig, n: usize, kind: u32) -> (usize, f64) { let (w, h) = (cfg.width as usize, cfg.height as usize); let mut e = Vp9Encoder::new(cfg); - let mut d = Decoder::new(); + let mut d = Decoder::with_threads(1); let (mut bytes, mut db) = (0usize, 0.0); for t in 0..n { let (y, u, v) = moving(w, h, t, kind); @@ -344,7 +344,7 @@ fn frames_never_end_in_a_superframe_marker() { } let Some((w, h, q, frames, recon)) = padded else { panic!("no frame needed the padding: widen the search") }; // The padded frames decode exactly, with FilmCraft's decoder and with ffmpeg. - let mut d = Decoder::new(); + let mut d = Decoder::with_threads(1); for (t, (f, r)) in frames.iter().zip(&recon).enumerate() { let mut pics = d.decode(f, t as i64).unwrap_or_else(|e| panic!("{w}x{h} q{q} frame {t}: {e:?}")); assert_eq!(planes_of(&pics.remove(0), w, h), *r, "{w}x{h} q{q} frame {t}"); diff --git a/crates/host/tests/timeline_interop.rs b/crates/host/tests/timeline_interop.rs index 5123dc721..9688a409f 100644 --- a/crates/host/tests/timeline_interop.rs +++ b/crates/host/tests/timeline_interop.rs @@ -56,8 +56,9 @@ fn still(p: &mut fp::Project, path: &str) -> fp::ItemId { color: Default::default(), has_alpha: false, bitrate: None, + hdr: None, }), - audio: None, + audio_streams: Vec::new(), container: String::new(), start_timecode: None, file_size: None, diff --git a/crates/interchange/src/export.rs b/crates/interchange/src/export.rs index 7da18913a..da6ef1382 100644 --- a/crates/interchange/src/export.rs +++ b/crates/interchange/src/export.rs @@ -684,8 +684,12 @@ fn media_info(name: &str, kind: MediaKind, duration: Tick, video: Option { MediaKind::AudioOnly => FootageKind::Audio, MediaKind::Still => FootageKind::Still, MediaKind::ImageSequence => FootageKind::Sequence, - _ if v.is_none() && info.audio.is_some() => FootageKind::Audio, + _ if v.is_none() && info.has_audio() => FootageKind::Audio, _ => FootageKind::Video, }; let _ = name; @@ -284,7 +284,7 @@ impl Importer<'_> { frame_rate: rate_in(info.frame_rate()), duration: tick_in(info.duration), has_video: kind != FootageKind::Audio, - has_audio: info.audio.is_some() || kind == FootageKind::Audio, + has_audio: info.has_audio() || kind == FootageKind::Audio, alpha: if v.is_some_and(|v| v.has_alpha) { AlphaMode::Straight } else { AlphaMode::Ignore }, codec: v.map(|v| v.codec.clone()).unwrap_or_default(), missing: true, diff --git a/crates/interchange/src/tests.rs b/crates/interchange/src/tests.rs index 3ad7fcaae..6e29c79d2 100644 --- a/crates/interchange/src/tests.rs +++ b/crates/interchange/src/tests.rs @@ -36,8 +36,12 @@ fn info(kind: filmcraft_media::MediaKind, video: bool, audio: bool) -> filmcraft color: Default::default(), has_alpha: false, bitrate: None, + hdr: None, }), - audio: audio.then(|| filmcraft_media::AudioStreamInfo { sample_rate: 48_000, channels: 2, codec: "pcm".into(), bits_per_sample: Some(16) }), + audio_streams: audio + .then(|| filmcraft_media::AudioStreamInfo { sample_rate: 48_000, channels: 2, codec: "pcm".into(), bits_per_sample: Some(16) }) + .into_iter() + .collect(), container: String::new(), start_timecode: None, file_size: None, diff --git a/crates/media/src/downmix.rs b/crates/media/src/downmix.rs index 2ef417dd7..e8a9237e0 100644 --- a/crates/media/src/downmix.rs +++ b/crates/media/src/downmix.rs @@ -64,11 +64,30 @@ pub fn stereo_gains(count: usize, aac: bool) -> Vec { out } +/// The gains for AAC as FilmCraft's decoder hands it over: channel configurations 3 to 6 (3 to 6 +/// channels) are already in WAV/SMPTE order (L R C [Cs] / [LFE] Ls Rs), everything else (mono, +/// stereo, 7 and 8 channels, program-config layouts) in AAC's element order. +pub fn decoded_aac_gains(count: usize) -> Vec { + match count { + // C L R Cs, reordered to L R C Cs: the same gains, permuted. + 4 => { + let g = stereo_gains(4, true); + [1, 2, 0, 3].iter().filter_map(|&i| g.get(i).copied()).collect() + } + 3 | 5 | 6 => stereo_gains(count, false), + _ => stereo_gains(count, true), + } +} + /// Downmix planar `channels` to interleaved stereo in `out` (`out.len() / 2` frames); frames a /// channel is too short for count as silence, extra frames in `out` stay as they are. pub fn downmix_into(channels: &[Vec], aac: bool, out: &mut [f32]) { - let gains = stereo_gains(channels.len(), aac); - for (ch, g) in channels.iter().zip(&gains) { + let gains = if aac { decoded_aac_gains(channels.len()) } else { stereo_gains(channels.len(), false) }; + mix(channels, &gains, out); +} + +fn mix(channels: &[Vec], gains: &[Pair], out: &mut [f32]) { + for (ch, g) in channels.iter().zip(gains) { if g[0] == 0.0 && g[1] == 0.0 { continue; } @@ -98,6 +117,22 @@ mod tests { } } + #[test] + fn decoded_aac_is_in_wav_order_for_three_to_six_channels() { + // FilmCraft hands AAC 3.0 to 5.1 over as L R C [LFE] Ls Rs, the order of the plain layouts. + for n in [3, 5, 6] { + assert_eq!(decoded_aac_gains(n), stereo_gains(n, false), "{n} channels"); + } + let g = decoded_aac_gains(4); + assert_eq!(g.len(), 4); + assert!(g[0][0] > 0.0 && g[0][1] == 0.0 && g[1][1] > 0.0 && g[1][0] == 0.0, "L R first"); + assert!(g[2][0] > 0.0 && g[2][0] == g[2][1], "the centre is third and feeds both sides"); + assert!(g[3][0] > 0.0 && g[3][0] < g[2][0], "the rear centre is last, below the centre"); + for n in [1, 2, 7, 8, 9] { + assert_eq!(decoded_aac_gains(n), stereo_gains(n, true), "{n} channels"); + } + } + #[test] fn five_one_matches_the_bs775_matrix() { let k = 1.0 / (1.0 + 2.0 * FRAC_1_SQRT_2); diff --git a/crates/media/src/pool.rs b/crates/media/src/pool.rs index 98c707108..33fb0b26f 100644 --- a/crates/media/src/pool.rs +++ b/crates/media/src/pool.rs @@ -509,7 +509,7 @@ impl MediaPool { let path: Arc = footage.path.as_str().into(); let src = self.inner.source(&path).ok_or("the file can't be opened")?; let s0 = start.to_units_floor(rate as i64); - let native = src.info().audio.as_ref().map_or(rate, |a| a.sample_rate); + let native = src.info().audio().map_or(rate, |a| a.sample_rate); if native == 0 || rate == 0 || native == rate { return read_stereo(&src, s0, frames, rate); } @@ -700,9 +700,12 @@ impl Inner { let rate = footage.frame_rate; let n = Self::frame_count(rate, footage.duration); let i = Self::frame_index(rate, t, n, footage.loop_count); - // Middle of the frame in the file's own timing (robust to timestamp rounding). + // A quarter into the frame in the file's own timing: FilmCraft rounds a request to + // the track's nearest timestamp unit (halves up), so the middle of a frame would + // land on the next one when the timescale is the frame rate; a quarter still + // tolerates timestamps rounded either way. let native = footage.native_rate.unwrap_or(rate); - let mt = native.tick_of(i) + Tick(native.frame_duration().0 / 2); + let mt = native.tick_of(i) + Tick(native.frame_duration().0 / 4); Loc { key: key(&footage.path, i), media_t: Some(mt), float } } } @@ -915,7 +918,7 @@ fn read_stereo(src: &SharedSource, s0: i64, frames: usize, rate: u32) -> std::re } let buf = src.audio(s0.max(0), frames - skip, rate).map_err(|e| e.to_string())?; let Some(dst) = skip.checked_mul(2).and_then(|i| out.get_mut(i..)) else { return Ok(out) }; - let aac = src.info().audio.as_ref().is_some_and(|a| a.codec.starts_with("AAC")); + let aac = src.info().audio().is_some_and(|a| a.codec.starts_with("AAC")); crate::downmix::downmix_into(&buf.channels, aac, dst); Ok(out) } diff --git a/crates/media/src/probe.rs b/crates/media/src/probe.rs index bae714c81..7c5d1c491 100644 --- a/crates/media/src/probe.rs +++ b/crates/media/src/probe.rs @@ -119,7 +119,7 @@ pub(crate) fn footage_from_info(path: &str, info: &filmcraft_media::MediaInfo) - (_, None) => FootageKind::Audio, }; let rate = v.map_or(FrameRate::FPS_30, |v| rate_from_fc(v.frame_rate)); - let codec = match (v, info.audio.as_ref()) { + let codec = match (v, info.audio()) { (Some(v), _) => v.codec.clone(), (None, Some(a)) => a.codec.clone(), _ => String::new(), @@ -134,7 +134,7 @@ pub(crate) fn footage_from_info(path: &str, info: &filmcraft_media::MediaInfo) - native_rate: None, duration: if kind == FootageKind::Still { Tick::ZERO } else { Tick(info.duration.0) }, has_video: v.is_some(), - has_audio: info.audio.is_some(), + has_audio: info.has_audio(), alpha: if v.is_some_and(|v| v.has_alpha) { AlphaMode::Straight } else { AlphaMode::Ignore }, premul_color: [0.0; 3], loop_count: 1, @@ -239,12 +239,19 @@ pub(crate) fn load_oriented(bytes: &[u8], fmt: Option) -> im /// The rotation an MP4/MOV movie is shown with (a phone held upright records sideways frames and /// says so in its video track's header): the quarter turn nearest the track's rotation. Only the /// rotation counts; any mirroring, scaling or offset in the header is ignored, and other -/// containers have none. +/// containers have none. FilmCraft turns the frames of a pure 0/90/180/270° header itself, so this +/// is only the quarter turn nearest to a header it does not turn. pub(crate) fn video_turn(bytes: &[u8]) -> Orientation { let Ok(file) = filmcraft_isobmff::open(bytes) else { return Orientation::NoTransforms }; - let Some(m) = file.tracks.iter().find(|t| t.kind == filmcraft_isobmff::TrackKind::Video).map(|t| t.matrix) else { + let Some(track) = file.tracks.iter().find(|t| t.kind == filmcraft_isobmff::TrackKind::Video) else { return Orientation::NoTransforms; }; + // A pure rotation is already applied by FilmCraft's decoder (frames and size come out + // upright); only the headers it leaves as stored (mirrored, scaled, sheared) are turned here. + if track.display_rotation().is_some() { + return Orientation::NoTransforms; + } + let m = track.matrix; // [a b u; c d v; x y w]: (a, b) is where the picture's x axis points. let (a, b) = (m[0] as i64, m[1] as i64); match (a.abs() >= b.abs(), a >= 0, b >= 0) { From ee17172fb688ffe1abb07de2db9c7d806f4e5198 Mon Sep 17 00:00:00 2001 From: tylerjenningsw Date: Sat, 17 Oct 2026 10:50:04 -0400 Subject: [PATCH 02/12] Media: update FilmCraft to 09aaf9b, which reads AVI (#1701) Moves the pin past FilmCraft's AVI / OpenDML reader (Motion JPEG, H.264, HEVC, uncompressed; PCM, MP3, MP2, AC-3). No EffectCraft API changes. --- Cargo.lock | 144 +++++++++++++++++++++++++++++++++-------------------- Cargo.toml | 32 ++++++------ 2 files changed, 106 insertions(+), 70 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 06da2cebd..1dea8187a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2043,8 +2043,8 @@ checksum = "b97b65636e5b9ef369943878ac74335ba1c55c1cb6adbf1e2c293c624248d693" [[package]] name = "filmcraft-aac" -version = "0.6.0" -source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" dependencies = [ "filmcraft-bitstream", "thiserror 2.0.21", @@ -2052,16 +2052,16 @@ dependencies = [ [[package]] name = "filmcraft-ac3" -version = "0.6.0" -source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" dependencies = [ "thiserror 2.0.21", ] [[package]] name = "filmcraft-apv" -version = "0.6.0" -source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" dependencies = [ "filmcraft-bitstream", "rayon", @@ -2070,33 +2070,39 @@ dependencies = [ [[package]] name = "filmcraft-av1" -version = "0.6.0" -source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" dependencies = [ "filmcraft-bitstream", "rayon", "thiserror 2.0.21", ] +[[package]] +name = "filmcraft-avi" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" + [[package]] name = "filmcraft-bitstream" -version = "0.6.0" -source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" [[package]] name = "filmcraft-cfb" -version = "0.6.0" -source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" [[package]] name = "filmcraft-codecs" -version = "0.6.0" -source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" dependencies = [ "filmcraft-aac", "filmcraft-ac3", "filmcraft-apv", "filmcraft-av1", + "filmcraft-avi", "filmcraft-bitstream", "filmcraft-color", "filmcraft-dnx", @@ -2106,6 +2112,7 @@ dependencies = [ "filmcraft-isobmff", "filmcraft-matroska", "filmcraft-media", + "filmcraft-mpa", "filmcraft-mpeg2v", "filmcraft-mpegts", "filmcraft-mxf", @@ -2113,9 +2120,11 @@ dependencies = [ "filmcraft-opus", "filmcraft-prores", "filmcraft-time", + "filmcraft-vp8", "filmcraft-vp9", "image", "log", + "rayon", "symphonia", "thiserror 2.0.21", "web-time", @@ -2123,16 +2132,25 @@ dependencies = [ [[package]] name = "filmcraft-color" -version = "0.6.0" -source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" dependencies = [ "serde", ] +[[package]] +name = "filmcraft-dng" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" +dependencies = [ + "rayon", + "thiserror 2.0.21", +] + [[package]] name = "filmcraft-dnx" -version = "0.6.0" -source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" dependencies = [ "filmcraft-bitstream", "rayon", @@ -2141,8 +2159,8 @@ dependencies = [ [[package]] name = "filmcraft-frame" -version = "0.6.0" -source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" dependencies = [ "filmcraft-color", "filmcraft-time", @@ -2152,16 +2170,16 @@ dependencies = [ [[package]] name = "filmcraft-geom" -version = "0.6.0" -source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" dependencies = [ "serde", ] [[package]] name = "filmcraft-h264" -version = "0.6.0" -source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" dependencies = [ "filmcraft-bitstream", "rayon", @@ -2170,8 +2188,8 @@ dependencies = [ [[package]] name = "filmcraft-h264enc" -version = "0.6.0" -source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" dependencies = [ "filmcraft-bitstream", "rayon", @@ -2180,8 +2198,8 @@ dependencies = [ [[package]] name = "filmcraft-hevc" -version = "0.6.0" -source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" dependencies = [ "filmcraft-bitstream", "rayon", @@ -2190,8 +2208,8 @@ dependencies = [ [[package]] name = "filmcraft-interchange" -version = "0.6.0" -source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" dependencies = [ "filmcraft-cfb", "filmcraft-color", @@ -2207,8 +2225,8 @@ dependencies = [ [[package]] name = "filmcraft-isobmff" -version = "0.6.0" -source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" dependencies = [ "filmcraft-bitstream", "thiserror 2.0.21", @@ -2216,15 +2234,16 @@ dependencies = [ [[package]] name = "filmcraft-matroska" -version = "0.6.0" -source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" [[package]] name = "filmcraft-media" -version = "0.6.0" -source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" dependencies = [ "filmcraft-color", + "filmcraft-dng", "filmcraft-frame", "filmcraft-time", "image", @@ -2233,10 +2252,19 @@ dependencies = [ "thiserror 2.0.21", ] +[[package]] +name = "filmcraft-mpa" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" +dependencies = [ + "filmcraft-bitstream", + "thiserror 2.0.21", +] + [[package]] name = "filmcraft-mpeg2v" -version = "0.6.0" -source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" dependencies = [ "rayon", "thiserror 2.0.21", @@ -2244,31 +2272,31 @@ dependencies = [ [[package]] name = "filmcraft-mpegts" -version = "0.6.0" -source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" dependencies = [ "thiserror 2.0.21", ] [[package]] name = "filmcraft-mxf" -version = "0.6.0" -source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" [[package]] name = "filmcraft-ogg" -version = "0.6.0" -source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" [[package]] name = "filmcraft-opus" -version = "0.6.0" -source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" [[package]] name = "filmcraft-project" -version = "0.6.0" -source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" dependencies = [ "filmcraft-color", "filmcraft-geom", @@ -2281,8 +2309,8 @@ dependencies = [ [[package]] name = "filmcraft-prores" -version = "0.6.0" -source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" dependencies = [ "filmcraft-bitstream", "rayon", @@ -2291,16 +2319,24 @@ dependencies = [ [[package]] name = "filmcraft-time" -version = "0.6.0" -source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" dependencies = [ "serde", ] +[[package]] +name = "filmcraft-vp8" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" +dependencies = [ + "thiserror 2.0.21", +] + [[package]] name = "filmcraft-vp9" -version = "0.6.0" -source = "git+https://github.com/storytold/filmcraft?rev=4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe#4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" +version = "0.8.0" +source = "git+https://github.com/storytold/filmcraft?rev=09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5#09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" dependencies = [ "filmcraft-bitstream", "rayon", diff --git a/Cargo.toml b/Cargo.toml index 7b9983788..8810e3c16 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -72,26 +72,26 @@ image = { version = "0.25", default-features = false, features = ["png", "jpeg"] # media codecs + containers: FilmCraft's pure-Rust crates, pinned to a revision (plan/adr/0001). # Only `crates/media`, `crates/export` and `crates/interchange` may depend on these. -filmcraft-codecs = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } -filmcraft-media = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } -filmcraft-frame = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } -filmcraft-color = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } -filmcraft-time = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } +filmcraft-codecs = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } +filmcraft-media = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } +filmcraft-frame = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } +filmcraft-color = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } +filmcraft-time = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } # encoders + muxer (crates/export) -filmcraft-h264enc = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } -filmcraft-prores = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } -filmcraft-aac = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } -filmcraft-isobmff = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } +filmcraft-h264enc = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } +filmcraft-prores = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } +filmcraft-aac = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } +filmcraft-isobmff = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } # decoder used as the round-trip oracle of effectcraft-opusenc (crates/export dev-dependency) -filmcraft-opus = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } -filmcraft-vp9 = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } +filmcraft-opus = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } +filmcraft-vp9 = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } # decoders used as the round-trip oracles of effectcraft-hevcenc / effectcraft-av1enc (dev-dependencies only) -filmcraft-hevc = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } -filmcraft-av1 = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } +filmcraft-hevc = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } +filmcraft-av1 = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } # timeline interchange (EDL, FCP7 XML, FCPXML, OTIO) for Premiere Pro interop (crates/interchange only) -filmcraft-interchange = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } -filmcraft-project = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } -filmcraft-geom = { git = "https://github.com/storytold/filmcraft", rev = "4e2d0bc2d3646f9cf27ef7aa8b2095c6858c7efe" } +filmcraft-interchange = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } +filmcraft-project = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } +filmcraft-geom = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } # The symphonia filmcraft-codecs already builds, named only to turn on its AIFF reader: the pinned # rev enables wav alone (FilmCraft main enables aiff too). Drop it when the pin moves past that. symphonia = { version = "0.5.5", default-features = false, features = ["aiff"] } From 01ad187b97888efedd588130f12d20887e7a610b Mon Sep 17 00:00:00 2001 From: tylerjenningsw Date: Sat, 17 Oct 2026 10:53:04 -0400 Subject: [PATCH 03/12] Media: drop the AIFF workaround dependency (#1701) FilmCraft's codecs now turn on symphonia's AIFF reader themselves. --- Cargo.lock | 1 - Cargo.toml | 3 --- crates/media/Cargo.toml | 2 -- 3 files changed, 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1dea8187a..8fc3183ec 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1455,7 +1455,6 @@ dependencies = [ "log", "rayon", "serde_json", - "symphonia", "thiserror 2.0.21", ] diff --git a/Cargo.toml b/Cargo.toml index 8810e3c16..b1c6caaf6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -92,9 +92,6 @@ filmcraft-av1 = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9b filmcraft-interchange = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } filmcraft-project = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } filmcraft-geom = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } -# The symphonia filmcraft-codecs already builds, named only to turn on its AIFF reader: the pinned -# rev enables wav alone (FilmCraft main enables aiff too). Drop it when the pin moves past that. -symphonia = { version = "0.5.5", default-features = false, features = ["aiff"] } # gpu (the same wgpu eframe / egui-wgpu use) wgpu = { version = "30.0.1", features = ["fragile-send-sync-non-atomic-wasm"] } diff --git a/crates/media/Cargo.toml b/crates/media/Cargo.toml index bb47dccb0..a3ff624b4 100644 --- a/crates/media/Cargo.toml +++ b/crates/media/Cargo.toml @@ -24,8 +24,6 @@ filmcraft-isobmff = { workspace = true } filmcraft-frame = { workspace = true } filmcraft-color = { workspace = true } filmcraft-time = { workspace = true } -# AIFF import (#1428): enables the aiff reader of filmcraft-codecs' symphonia. -symphonia = { workspace = true } exr = { workspace = true } gif = { workspace = true } image = { workspace = true, features = ["gif", "webp", "tiff", "bmp", "exr", "tga", "hdr"] } From ca724a3d84c8fcd1724b3de724ed4c5c4507ee53 Mon Sep 17 00:00:00 2001 From: tylerjenningsw Date: Sat, 17 Oct 2026 10:53:05 -0400 Subject: [PATCH 04/12] Media: downmix 7.1 AAC in the order FilmCraft now delivers it (#1701) FilmCraft reorders AAC channel configuration 12 (7.1) to WAVE/SMPTE order, as it does 3 to 6, so 7.1 AAC's left channel was mixed as the centre. The downmix oracle test gains 6.1 (a program config element, left in AAC's order) and 7.1 AAC fixtures. --- crates/media/src/downmix.rs | 16 +++++++++------- crates/media/tests/downmix.rs | 10 ++++++++++ 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/crates/media/src/downmix.rs b/crates/media/src/downmix.rs index e8a9237e0..1ad9721ba 100644 --- a/crates/media/src/downmix.rs +++ b/crates/media/src/downmix.rs @@ -65,8 +65,10 @@ pub fn stereo_gains(count: usize, aac: bool) -> Vec { } /// The gains for AAC as FilmCraft's decoder hands it over: channel configurations 3 to 6 (3 to 6 -/// channels) are already in WAV/SMPTE order (L R C [Cs] / [LFE] Ls Rs), everything else (mono, -/// stereo, 7 and 8 channels, program-config layouts) in AAC's element order. +/// channels) and 12 (7.1, 8 channels) are already in WAV/SMPTE order (L R C [Cs] / [LFE] Ls Rs, +/// L R C LFE Lb Rb Ls Rs), everything else (mono, stereo, program-config layouts such as the 6.1 +/// ffmpeg writes) in AAC's element order. Configuration 11 (6.1) and 7 (7.1 with front wides) +/// can't be told from those by the channel count and are read in the order of the commoner one. pub fn decoded_aac_gains(count: usize) -> Vec { match count { // C L R Cs, reordered to L R C Cs: the same gains, permuted. @@ -74,7 +76,7 @@ pub fn decoded_aac_gains(count: usize) -> Vec { let g = stereo_gains(4, true); [1, 2, 0, 3].iter().filter_map(|&i| g.get(i).copied()).collect() } - 3 | 5 | 6 => stereo_gains(count, false), + 3 | 5 | 6 | 8 => stereo_gains(count, false), _ => stereo_gains(count, true), } } @@ -118,9 +120,9 @@ mod tests { } #[test] - fn decoded_aac_is_in_wav_order_for_three_to_six_channels() { - // FilmCraft hands AAC 3.0 to 5.1 over as L R C [LFE] Ls Rs, the order of the plain layouts. - for n in [3, 5, 6] { + fn decoded_aac_is_in_wav_order_for_three_to_six_and_eight_channels() { + // FilmCraft hands AAC 3.0 to 5.1 and 7.1 over in the order of the plain layouts. + for n in [3, 5, 6, 8] { assert_eq!(decoded_aac_gains(n), stereo_gains(n, false), "{n} channels"); } let g = decoded_aac_gains(4); @@ -128,7 +130,7 @@ mod tests { assert!(g[0][0] > 0.0 && g[0][1] == 0.0 && g[1][1] > 0.0 && g[1][0] == 0.0, "L R first"); assert!(g[2][0] > 0.0 && g[2][0] == g[2][1], "the centre is third and feeds both sides"); assert!(g[3][0] > 0.0 && g[3][0] < g[2][0], "the rear centre is last, below the centre"); - for n in [1, 2, 7, 8, 9] { + for n in [1, 2, 7, 9] { assert_eq!(decoded_aac_gains(n), stereo_gains(n, true), "{n} channels"); } } diff --git a/crates/media/tests/downmix.rs b/crates/media/tests/downmix.rs index e0e7d2252..e2c95c251 100644 --- a/crates/media/tests/downmix.rs +++ b/crates/media/tests/downmix.rs @@ -94,8 +94,18 @@ fn cases() -> Vec { let (n71, g71) = seven_one(); let k5 = 1.0 / (1.0 + 2.0 * K); let kq = 1.0 / (1.0 + K); + let k61 = 1.0 / (1.5 + 2.0 * K); vec![ Case { file: "dm51.m4a", layout: "5.1", names: n51, codec: &["-c:a", "aac", "-b:a", "384k"], gains: g51.clone(), tol: 0.15 }, + Case { + file: "dm61.m4a", + layout: "6.1", + names: &["FL", "FR", "FC", "LFE", "BC", "SL", "SR"], + codec: &["-c:a", "aac", "-b:a", "448k"], + gains: vec![[k61, 0.0], [0.0, k61], [k61 * K, k61 * K], [0.0, 0.0], [k61 / 2.0, k61 / 2.0], [k61 * K, 0.0], [0.0, k61 * K]], + tol: 0.15, + }, + Case { file: "dm71.m4a", layout: "7.1", names: n71, codec: &["-c:a", "aac", "-b:a", "512k"], gains: g71.clone(), tol: 0.15 }, Case { file: "dm51.ac3.mp4", layout: "5.1", names: n51, codec: &["-c:a", "ac3", "-b:a", "640k"], gains: g51.clone(), tol: 0.15 }, Case { file: "dm51.wav", layout: "5.1", names: n51, codec: &["-c:a", "pcm_s16le"], gains: g51.clone(), tol: 0.02 }, Case { file: "dm51.flac", layout: "5.1", names: n51, codec: &["-c:a", "flac"], gains: g51, tol: 0.02 }, From 8bb5dbb81e8651072f7e19220342225f8bd2d7f1 Mon Sep 17 00:00:00 2001 From: tylerjenningsw Date: Thu, 29 Oct 2026 17:52:57 -0400 Subject: [PATCH 05/12] Media: downmix AAC in the order its channel configuration decodes in (#1701) FilmCraft reorders AAC configurations 3 to 6, 11 and 12 to WAVE/SMPTE order and leaves configuration 7 and program config elements as coded, so the channel count alone can't say which order arrives: ffmpeg writes 7.1 as configuration 7 (C L R first) and 6.1 as a PCE it fills in its own order (L R C LFE Cs Ls Rs), while FilmCraft's encoder writes a centre-first PCE. The order is now read from an MP4/MOV's AudioSpecificConfig when the source opens; other containers keep a channel-count guess. The ffmpeg oracle test gains configuration 12 (ffmpeg's 7.1 relabelled) and skips PCE files ffmpeg 4.2 marks as stereo, which FilmCraft reads as two channels. --- Cargo.lock | 1 + crates/media/Cargo.toml | 1 + crates/media/src/downmix.rs | 123 +++++++++++++++++++++++-------- crates/media/src/pool.rs | 77 ++++++++++++------- crates/media/src/probe.rs | 11 +++ crates/media/tests/it/downmix.rs | 72 ++++++++++++++---- 6 files changed, 211 insertions(+), 74 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d7b774557..627da7cf4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1632,6 +1632,7 @@ dependencies = [ "effectcraft-svg", "effectcraft-time", "exr", + "filmcraft-aac", "filmcraft-codecs", "filmcraft-color", "filmcraft-frame", diff --git a/crates/media/Cargo.toml b/crates/media/Cargo.toml index 923a28afa..d3f815bba 100644 --- a/crates/media/Cargo.toml +++ b/crates/media/Cargo.toml @@ -21,6 +21,7 @@ effectcraft-pdf = { workspace = true } effectcraft-model = { workspace = true } filmcraft-codecs = { workspace = true } filmcraft-media = { workspace = true } +filmcraft-aac = { workspace = true } filmcraft-isobmff = { workspace = true } filmcraft-frame = { workspace = true } filmcraft-color = { workspace = true } diff --git a/crates/media/src/downmix.rs b/crates/media/src/downmix.rs index 1ad9721ba..745f28065 100644 --- a/crates/media/src/downmix.rs +++ b/crates/media/src/downmix.rs @@ -12,9 +12,10 @@ //! //! Channel order follows the WAVE / SMPTE convention by channel count (what FilmCraft's AC-3, PCM, //! FLAC and Opus readers deliver): 3 = L R C, 4 = quad L R Ls Rs, 5 = L R C Ls Rs, -//! 6 = L R C LFE Ls Rs, 7 = L R C LFE Cs Ls Rs, 8 = L R C LFE Lb Rb Ls Rs. FilmCraft's AAC-LC -//! decoder instead delivers AAC's own order (C L R first, LFE last), so AAC gets its own tables. -//! More than 8 channels carry no known layout: only the first two (front L/R) are used. +//! 6 = L R C LFE Ls Rs, 7 = L R C LFE Cs Ls Rs, 8 = L R C LFE Lb Rb Ls Rs. AAC arrives in either +//! that order or AAC's own (C L R first, LFE last) depending on its channel configuration +//! ([`aac_order`]). More than 8 channels carry no known layout: only the first two (front L/R) +//! are used. use std::f32::consts::FRAC_1_SQRT_2; @@ -64,28 +65,59 @@ pub fn stereo_gains(count: usize, aac: bool) -> Vec { out } -/// The gains for AAC as FilmCraft's decoder hands it over: channel configurations 3 to 6 (3 to 6 -/// channels) and 12 (7.1, 8 channels) are already in WAV/SMPTE order (L R C [Cs] / [LFE] Ls Rs, -/// L R C LFE Lb Rb Ls Rs), everything else (mono, stereo, program-config layouts such as the 6.1 -/// ffmpeg writes) in AAC's element order. Configuration 11 (6.1) and 7 (7.1 with front wides) -/// can't be told from those by the channel count and are read in the order of the commoner one. -pub fn decoded_aac_gains(count: usize) -> Vec { - match count { - // C L R Cs, reordered to L R C Cs: the same gains, permuted. - 4 => { - let g = stereo_gains(4, true); - [1, 2, 0, 3].iter().filter_map(|&i| g.get(i).copied()).collect() - } - 3 | 5 | 6 | 8 => stereo_gains(count, false), - _ => stereo_gains(count, true), +/// The order a source's decoded channels arrive in. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum Order { + /// WAVE / SMPTE by channel count: PCM, FLAC, AC-3, Opus and the rest. + #[default] + Wav, + /// AAC that FilmCraft reorders to WAVE / SMPTE (configurations 3 to 6, 11 and 12; 4 becomes + /// L R C Cs) or that ffmpeg wrote in it (a program config element listing the front pair + /// first: ffmpeg fills a PCE's elements in its own channel order and reads them back so). + AacWav, + /// AAC in its element order (C L R … LFE), which FilmCraft keeps: configuration 7 and a PCE + /// listing the centre first. + AacElements, + /// AAC whose configuration the container doesn't show us (only MP4 / MOV headers are read): + /// as [`Order::AacWav`] up to 7 channels (configurations 3 to 6, 11 and ffmpeg's 6.1 PCE), + /// as [`Order::AacElements`] above (configuration 7, ffmpeg's 7.1). + Aac, +} + +/// How FilmCraft's decoder hands over the AAC described by the AudioSpecificConfig `asc`. +pub fn aac_order(asc: &[u8]) -> Order { + let Ok(asc) = filmcraft_aac::AudioSpecificConfig::parse(asc) else { return Order::Aac }; + match (asc.channel_config, asc.pce) { + (3..=6 | 11 | 12, _) => Order::AacWav, + (0, Some(pce)) if pce.front.first().is_some_and(|&(cpe, _)| cpe) => Order::AacWav, + (0, None) => Order::Aac, + _ => Order::AacElements, } } -/// Downmix planar `channels` to interleaved stereo in `out` (`out.len() / 2` frames); frames a -/// channel is too short for count as silence, extra frames in `out` stay as they are. -pub fn downmix_into(channels: &[Vec], aac: bool, out: &mut [f32]) { - let gains = if aac { decoded_aac_gains(channels.len()) } else { stereo_gains(channels.len(), false) }; - mix(channels, &gains, out); +/// The gains for `count` channels arriving in `order`. +pub fn decoded_gains(count: usize, order: Order) -> Vec { + let order = match order { + Order::Aac if (3..=7).contains(&count) => Order::AacWav, + Order::Aac => Order::AacElements, + o => o, + }; + match (order, count) { + // C L R Cs, reordered to L R C Cs: the same gains, permuted. + (Order::AacWav, 4) => { + let g = stereo_gains(4, true); + [1, 2, 0, 3].iter().filter_map(|&i| g.get(i).copied()).collect() + } + (Order::AacElements, _) => stereo_gains(count, true), + _ => stereo_gains(count, false), + } +} + +/// Downmix planar `channels` arriving in `order` to interleaved stereo in `out` (`out.len() / 2` +/// frames); frames a channel is too short for count as silence, extra frames in `out` stay as +/// they are. +pub fn downmix_into(channels: &[Vec], order: Order, out: &mut [f32]) { + mix(channels, &decoded_gains(channels.len(), order), out); } fn mix(channels: &[Vec], gains: &[Pair], out: &mut [f32]) { @@ -119,19 +151,46 @@ mod tests { } } + /// AudioSpecificConfigs as ffmpeg 8.1 and FilmCraft's encoder write them (48 kHz AAC-LC). + const PCE_FFMPEG_61: &str = "118004c848002000c4400d4c61766336322e32382e31303256e500"; + const PCE_FILMCRAFT_61: &str = "118004c845000108840000"; + + fn asc(hex: &str) -> Vec { + (0..hex.len() / 2).filter_map(|i| hex.get(2 * i..2 * i + 2).and_then(|b| u8::from_str_radix(b, 16).ok())).collect() + } + #[test] - fn decoded_aac_is_in_wav_order_for_three_to_six_and_eight_channels() { - // FilmCraft hands AAC 3.0 to 5.1 and 7.1 over in the order of the plain layouts. - for n in [3, 5, 6, 8] { - assert_eq!(decoded_aac_gains(n), stereo_gains(n, false), "{n} channels"); + fn aac_order_follows_the_channel_configuration() { + // AAC-LC at 48 kHz with channel configuration n + let config = |n: u8| aac_order(&[0x11, 0x80 | n << 3]); + for n in [3, 4, 5, 6, 11, 12] { + assert_eq!(config(n), Order::AacWav, "configuration {n}"); } - let g = decoded_aac_gains(4); + for n in [1, 2, 7] { + assert_eq!(config(n), Order::AacElements, "configuration {n}"); + } + assert_eq!(aac_order(&asc(PCE_FFMPEG_61)), Order::AacWav, "ffmpeg's 6.1 PCE lists L R first"); + assert_eq!(aac_order(&asc(PCE_FILMCRAFT_61)), Order::AacElements, "a PCE listing the centre first"); + assert_eq!(aac_order(&[]), Order::Aac); + } + + #[test] + fn decoded_aac_gains_follow_the_order() { + for n in [3, 5, 6, 7, 8] { + assert_eq!(decoded_gains(n, Order::AacWav), stereo_gains(n, false), "{n} channels"); + assert_eq!(decoded_gains(n, Order::AacElements), stereo_gains(n, true), "{n} channels"); + } + let g = decoded_gains(4, Order::AacWav); assert_eq!(g.len(), 4); assert!(g[0][0] > 0.0 && g[0][1] == 0.0 && g[1][1] > 0.0 && g[1][0] == 0.0, "L R first"); assert!(g[2][0] > 0.0 && g[2][0] == g[2][1], "the centre is third and feeds both sides"); assert!(g[3][0] > 0.0 && g[3][0] < g[2][0], "the rear centre is last, below the centre"); - for n in [1, 2, 7, 9] { - assert_eq!(decoded_aac_gains(n), stereo_gains(n, true), "{n} channels"); + assert_eq!(decoded_gains(4, Order::Wav), stereo_gains(4, false), "PCM 4 channels are quad"); + for n in [3, 4, 5, 6, 7] { + assert_eq!(decoded_gains(n, Order::Aac), decoded_gains(n, Order::AacWav), "{n} channels"); + } + for n in [1, 2, 8, 9] { + assert_eq!(decoded_gains(n, Order::Aac), stereo_gains(n, true), "{n} channels"); } } @@ -161,10 +220,10 @@ mod tests { #[test] fn ragged_and_empty_input_never_panics() { let mut out = vec![0.0; 8]; - downmix_into(&[], false, &mut out); - downmix_into(&[vec![1.0; 100], vec![1.0; 2], vec![], vec![1.0; 3], vec![1.0; 1], vec![1.0]], true, &mut out); + downmix_into(&[], Order::Wav, &mut out); + downmix_into(&[vec![1.0; 100], vec![1.0; 2], vec![], vec![1.0; 3], vec![1.0; 1], vec![1.0]], Order::Aac, &mut out); assert!(out.iter().all(|v| v.is_finite())); let mut none: [f32; 0] = []; - downmix_into(&vec![vec![1.0; 3]; 7], false, &mut none); + downmix_into(&vec![vec![1.0; 3]; 7], Order::Wav, &mut none); } } diff --git a/crates/media/src/pool.rs b/crates/media/src/pool.rs index ed89bf7e8..193804447 100644 --- a/crates/media/src/pool.rs +++ b/crates/media/src/pool.rs @@ -12,6 +12,7 @@ use filmcraft_media::{FrameRequest, SharedSource}; use image::metadata::Orientation; use crate::convert::{AlphaOp, dynamic_to_image, frame_to_image_in, frame_to_image_scaled, turned}; +use crate::downmix::Order; use crate::gif_anim::{self, Anim}; use crate::{MediaError, Result}; @@ -178,10 +179,37 @@ struct CacheState { last: HashMap, i64>, } -/// A movie/audio source by path and the turn its picture is shown with -/// ([`crate::probe::video_turn`]), opened by the first caller while the others wait for it -/// (`None`: failed to open; not retried until `forget`). -type SourceSlot = Arc>>; +/// A movie/audio source by path ([`Opened`]), opened by the first caller while the others wait +/// for it (`None`: failed to open; not retried until `forget`). +type SourceSlot = Arc>>; + +/// An opened movie/audio source, the turn its picture is shown with and the order its decoded +/// audio channels arrive in. +#[derive(Clone)] +struct Opened { + src: SharedSource, + turn: Orientation, + order: Order, +} + +impl Opened { + /// `src` with what its MP4/MOV `header` (ftyp and moov) says, if it has one. + fn new(src: SharedSource, header: Option<&[u8]>) -> Opened { + let turn = header.map_or(Orientation::NoTransforms, crate::probe::video_turn); + let order = channel_order(&src, header.and_then(crate::probe::aac_config).as_deref()); + Opened { src, turn, order } + } +} + +/// The order `src`'s audio channels decode in, AAC's decided by its AudioSpecificConfig `asc` +/// when the container's header was read. +fn channel_order(src: &SharedSource, asc: Option<&[u8]>) -> Order { + match (src.info().audio().is_some_and(|a| a.codec.starts_with("AAC")), asc) { + (false, _) => Order::Wav, + (true, Some(asc)) => crate::downmix::aac_order(asc), + (true, None) => Order::Aac, + } +} /// An animated GIF by path, as [`SourceSlot`] (the error message when it failed to load). type GifSlot = Arc, String>>>; @@ -514,7 +542,7 @@ impl MediaPool { /// [`MediaPool::decode_audio`], failing when the source can't be opened or read. fn try_decode_audio(&self, footage: &Footage, start: Tick, frames: usize, rate: u32) -> std::result::Result, String> { let path: Arc = footage.path.as_str().into(); - let src = self.inner.source(&path).ok_or("the file can't be opened")?; + let Opened { src, order, .. } = self.inner.opened(&path).ok_or("the file can't be opened")?; let s0 = start.to_units_floor(rate as i64); let native = src.info().audio().map_or(rate, |a| a.sample_rate); // The header's rate scales the samples read: 0xFFFFFFFF Hz made one conform chunk about @@ -523,14 +551,14 @@ impl MediaPool { return Err(format!("a sample rate of {native} Hz is outside {MIN_SAMPLE_RATE}–{MAX_SAMPLE_RATE} Hz")); } if native == 0 || rate == 0 || native == rate { - return read_stereo(&src, s0, frames, rate); + return read_stereo(&src, order, s0, frames, rate); } // Output sample `n` sits at source position n × native / rate. let (native_i, rate_i) = (i128::from(native), i128::from(rate)); let first = (i128::from(s0) * native_i).div_euclid(rate_i); let last = ((i128::from(s0) + frames as i128) * native_i).div_euclid(rate_i) + 1; let (Ok(first64), Ok(count)) = (i64::try_from(first), usize::try_from(last - first + 1)) else { return Ok(vec![0.0; frames * 2]) }; - let buf = read_stereo(&src, first64, count, native)?; + let buf = read_stereo(&src, order, first64, count, native)?; let mut out = Vec::with_capacity(frames * 2); for k in 0..frames as i128 { let pos = (i128::from(s0) + k) * native_i; @@ -593,17 +621,12 @@ impl Inner { crate::read_regular_file(std::path::Path::new(path)).map(Into::into).map_err(|e| MediaError::Io(format!("{path}: {e}"))) } - /// The opened movie/audio source for `path`. The first caller opens it, outside the map's - /// lock (reading and parsing a large file takes a moment; FLAC, MP3, Ogg and AIFF files are - /// decoded whole); callers asking for the same file meanwhile wait for that instead of opening - /// it again: a preview's frame workers, audio feeder, conform and waveform threads each - /// decoded a long FLAC in full at once (#408). - fn source(&self, path: &Arc) -> Option { - self.opened(path).map(|(src, _)| src) - } - - /// [`Self::source`] and the turn the movie's picture is shown with. - fn opened(&self, path: &Arc) -> Option<(SharedSource, Orientation)> { + /// The opened movie/audio source for `path` ([`Opened`]). The first caller opens it, outside + /// the map's lock (reading and parsing a large file takes a moment; FLAC, MP3, Ogg and AIFF + /// files are decoded whole); callers asking for the same file meanwhile wait for that instead + /// of opening it again: a preview's frame workers, audio feeder, conform and waveform threads + /// each decoded a long FLAC in full at once (#408). + fn opened(&self, path: &Arc) -> Option { let slot = lock(&self.sources).entry(path.clone()).or_default().clone(); slot.get_or_init(|| { self.opened.fetch_add(1, Ordering::Relaxed); @@ -613,14 +636,15 @@ impl Inner { let file = std::path::Path::new(&**path); let in_place = if in_memory { None } else { crate::stream::open_source(file) }; let in_place = in_place.map(|src| { - let turn = crate::stream::movie_header(file).map_or(Orientation::NoTransforms, |h| crate::probe::video_turn(&h)); - src.map(|src| (src, turn)) + let header = crate::stream::movie_header(file); + src.map(|src| Opened::new(src, header.as_deref())) }); let opened = in_place.unwrap_or_else(|| { self.read(path).and_then(|bytes| { let name = std::path::Path::new(&**path).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default(); - let turn = crate::probe::video_turn(&bytes); - filmcraft_codecs::open_bytes(&name, bytes).map(|src| (src, turn)).map_err(MediaError::from) + let (turn, asc) = (crate::probe::video_turn(&bytes), crate::probe::aac_config(&bytes)); + let src = filmcraft_codecs::open_bytes(&name, bytes).map_err(MediaError::from)?; + Ok(Opened { order: channel_order(&src, asc.as_deref()), src, turn }) }) }); match opened { @@ -854,7 +878,7 @@ impl Inner { Ok(dynamic_to_image(frame, op)) } Some(mt) => { - let (src, turn) = self.opened(path).ok_or_else(|| MediaError::Io(format!("{path}: cannot open")))?; + let Opened { src, turn, .. } = self.opened(path).ok_or_else(|| MediaError::Io(format!("{path}: cannot open")))?; let vf = src .video_frame(FrameRequest::full(filmcraft_time::Tick(mt.0))) .map_err(MediaError::from) @@ -953,8 +977,8 @@ impl FootageSource for MediaPool { /// `frames` stereo sample frames (interleaved) of `src` from sample `s0` at `rate` Hz, which /// FilmCraft serves as decoded when it is the source's own rate. Mono is duplicated to both -/// channels, multichannel sources are downmixed ([`crate::downmix`]); silence before the start. An error where decoding fails. -fn read_stereo(src: &SharedSource, s0: i64, frames: usize, rate: u32) -> std::result::Result, String> { +/// channels, multichannel sources (channels in `order`) are downmixed ([`crate::downmix`]); silence before the start. An error where decoding fails. +fn read_stereo(src: &SharedSource, order: Order, s0: i64, frames: usize, rate: u32) -> std::result::Result, String> { let mut out = vec![0.0; frames * 2]; let skip = usize::try_from(s0.saturating_neg()).unwrap_or(0); if skip >= frames { @@ -962,8 +986,7 @@ fn read_stereo(src: &SharedSource, s0: i64, frames: usize, rate: u32) -> std::re } let buf = src.audio(s0.max(0), frames - skip, rate).map_err(|e| e.to_string())?; let Some(dst) = skip.checked_mul(2).and_then(|i| out.get_mut(i..)) else { return Ok(out) }; - let aac = src.info().audio().is_some_and(|a| a.codec.starts_with("AAC")); - crate::downmix::downmix_into(&buf.channels, aac, dst); + crate::downmix::downmix_into(&buf.channels, order, dst); Ok(out) } diff --git a/crates/media/src/probe.rs b/crates/media/src/probe.rs index 9a7ebfa1d..435f9ce57 100644 --- a/crates/media/src/probe.rs +++ b/crates/media/src/probe.rs @@ -259,6 +259,17 @@ pub(crate) fn load_oriented(bytes: &[u8], fmt: Option) -> im Ok(img) } +/// The AudioSpecificConfig of an MP4/MOV movie's first audio track (the one FilmCraft plays), if +/// that is AAC. +pub(crate) fn aac_config(bytes: &[u8]) -> Option> { + let file = filmcraft_isobmff::open(bytes).ok()?; + let track = file.tracks.iter().find(|t| t.kind == filmcraft_isobmff::TrackKind::Audio)?; + match &track.entries.first()?.codec { + filmcraft_isobmff::CodecConfig::Aac(a) => Some(a.asc.clone()), + _ => None, + } +} + /// The rotation an MP4/MOV movie is shown with (a phone held upright records sideways frames and /// says so in its video track's header): the quarter turn nearest the track's rotation. Only the /// rotation counts; any mirroring, scaling or offset in the header is ignored, and other diff --git a/crates/media/tests/it/downmix.rs b/crates/media/tests/it/downmix.rs index e2c95c251..c139676d6 100644 --- a/crates/media/tests/it/downmix.rs +++ b/crates/media/tests/it/downmix.rs @@ -164,24 +164,66 @@ fn each_channel_reaches_stereo_at_its_bs775_gain() { for c in cases() { let freqs: Vec = FREQS.iter().copied().take(c.names.len()).collect(); let Some(p) = fixture(c.file, c.layout, c.names, &freqs, c.codec) else { continue }; - let s = ours(&p); - let sine = 0.125 * 1.0; // peak of ffmpeg's sine source (0.125 = 2^-3) - for (i, (name, g)) in c.names.iter().zip(&c.gains).enumerate() { - for (side, gain) in g.iter().enumerate() { - let got = amp(&s, side, freqs[i]); - let want = sine * gain; - assert!((got - want).abs() <= sine * c.tol.max(0.02), "{}: {name} -> {} is {got:.4}, expected {want:.4}", c.file, ["L", "R"][side]); - } - } - let theirs = ffmpeg_ac2(&p); - let (a, b) = (rms(&s), rms(&theirs)); - let db = 20.0 * (a / b).log10(); - assert!(db.abs() < 1.0, "{}: level {db:.2} dB off ffmpeg -ac 2 (rms {a:.4} vs {b:.4})", c.file); - let pdb = 20.0 * (peak(&s) / peak(&theirs)).log10(); - assert!(pdb.abs() < 1.5, "{}: peak {pdb:.2} dB off ffmpeg -ac 2", c.file); + check(&c, &p); } } +/// AAC 7.1 in channel configuration 12 (back surrounds), which FilmCraft reorders where it +/// leaves ffmpeg's configuration 7 as coded: ffmpeg's 7.1 with its configuration rewritten (both +/// code the same elements). +#[test] +fn seven_one_aac_in_configuration_12() { + let c = cases().into_iter().find(|c| c.file == "dm71.m4a").expect("the 7.1 AAC case"); + let freqs: Vec = FREQS.iter().copied().take(c.names.len()).collect(); + let Some(p) = fixture(c.file, c.layout, c.names, &freqs, c.codec) else { return }; + let mut bytes = std::fs::read(&p).expect("read"); + let (asc, _) = aac_entry(&bytes).expect("an AAC track"); + assert_eq!(asc.get(..2), Some(&[0x11, 0xB8][..]), "AAC-LC 48 kHz, configuration 7"); + let at = bytes.windows(asc.len()).position(|w| w == asc).expect("the configuration in the file"); + bytes[at + 1] = 0xE0; + let p12 = fixtures().join("dm71c12.m4a"); + std::fs::write(&p12, bytes).expect("write"); + check(&Case { file: "dm71c12.m4a", ..c }, &p12); +} + +/// The AudioSpecificConfig and sample entry channel count of an MP4's first AAC track. +fn aac_entry(bytes: &[u8]) -> Option<(Vec, u32)> { + let file = filmcraft_isobmff::open(bytes).ok()?; + file.tracks.iter().flat_map(|t| &t.entries).find_map(|e| match &e.codec { + filmcraft_isobmff::CodecConfig::Aac(a) => Some((a.asc.clone(), e.audio.as_ref()?.channels)), + _ => None, + }) +} + +fn check(c: &Case, p: &Path) { + let pce_as_stereo = aac_entry(&std::fs::read(p).expect("read")).is_some_and(|(asc, entry)| { + let config = filmcraft_aac::AudioSpecificConfig::parse(&asc).map(|a| a.channel_config); + config.ok() == Some(0) && entry == 2 + }); + if pce_as_stereo && c.names.len() > 2 { + // ffmpeg 4.2 writes 2 in the sample entry of an AAC PCE layout (8.1 the real count), and + // that is all FilmCraft decodes. + eprintln!("{}: FilmCraft reads 2 of its {} channels: skipping", c.file, c.names.len()); + return; + } + let s = ours(p); + let freqs: Vec = FREQS.iter().copied().take(c.names.len()).collect(); + let sine = 0.125 * 1.0; // peak of ffmpeg's sine source (0.125 = 2^-3) + for (i, (name, g)) in c.names.iter().zip(&c.gains).enumerate() { + for (side, gain) in g.iter().enumerate() { + let got = amp(&s, side, freqs[i]); + let want = sine * gain; + assert!((got - want).abs() <= sine * c.tol.max(0.02), "{}: {name} -> {} is {got:.4}, expected {want:.4}", c.file, ["L", "R"][side]); + } + } + let theirs = ffmpeg_ac2(p); + let (a, b) = (rms(&s), rms(&theirs)); + let db = 20.0 * (a / b).log10(); + assert!(db.abs() < 1.0, "{}: level {db:.2} dB off ffmpeg -ac 2 (rms {a:.4} vs {b:.4})", c.file); + let pdb = 20.0 * (peak(&s) / peak(&theirs)).log10(); + assert!(pdb.abs() < 1.5, "{}: peak {pdb:.2} dB off ffmpeg -ac 2", c.file); +} + /// Full-scale in every channel must not clip. #[test] fn loud_surround_does_not_clip() { From 393a7aa56529c7ff424116e9f02b27b9bf9bc5dd Mon Sep 17 00:00:00 2001 From: tylerjenningsw Date: Thu, 29 Oct 2026 17:57:08 -0400 Subject: [PATCH 06/12] Export tests: pin the H.264 bytes FilmCraft 09aaf9b writes (#1701) The encoder's first-frame user_data_unregistered SEI now lists its settings ("... - options: rc=vbr bitrate=10000 ... scenecut=1", 211 bytes where "filmcraft-h264enc 0.1.1 - clean-room H.264 encoder" took 50): +161 bytes. The same build gives (104322, 11051847956767423392), the pin Linux CI took on a base before the 19-byte nclx colr box, when that box is left out, so the bytes match across platforms. --- crates/export/tests/it/movie_bytes.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/export/tests/it/movie_bytes.rs b/crates/export/tests/it/movie_bytes.rs index 1f48efd1e..7a3fba611 100644 --- a/crates/export/tests/it/movie_bytes.rs +++ b/crates/export/tests/it/movie_bytes.rs @@ -94,7 +94,7 @@ fn export_bytes(format: OutputFormat, name: &str) -> Vec { #[test] fn h264_movie_bytes_are_pinned() { let data = export_bytes(OutputFormat::H264, "/out/pinned.mp4"); - assert_eq!((data.len(), fnv1a(&data)), (104_180, 8_728_430_532_616_484_528), "H.264 MP4"); + assert_eq!((data.len(), fnv1a(&data)), (104_341, 17_655_680_090_264_218_914), "H.264 MP4"); } #[test] From e675df7cabe9ccb4b6f0f1f159c7db784d8fac10 Mon Sep 17 00:00:00 2001 From: tylerjenningsw Date: Thu, 29 Oct 2026 18:20:08 -0400 Subject: [PATCH 07/12] Media: downmix AAC by where each decoded channel plays (#1701) The gains of an AAC source now come from the configuration FilmCraft decodes it with, read from MP4/MOV (esds), Matroska (CodecPrivate) and LATM in MPEG TS/PS (its first StreamMuxConfig). Channel configurations 3 to 6, 11 and 12 arrive reordered to WAVE order, 4 as L R C Cs, 7 in element order. A program config element goes by its element positions, except the six structures in which ffmpeg's encoder fills its own channel order across them (an LFE in a side SCE, a back centre in a side pair), recognised by its "Lavc" comment. ADTS can only signal configurations 1 to 7 that FilmCraft decodes, which the channel count tells apart. A decode of fewer channels than the layout keeps its first ones (7 of configuration 7's 8 in MPEG-TS LATM). --- Cargo.lock | 2 + Cargo.toml | 5 + crates/media/Cargo.toml | 2 + crates/media/src/aac.rs | 85 ++++++++++ crates/media/src/downmix.rs | 272 ++++++++++++++++++------------- crates/media/src/lib.rs | 1 + crates/media/src/pool.rs | 36 ++-- crates/media/src/probe.rs | 11 -- crates/media/src/stream.rs | 11 ++ crates/media/tests/it/footage.rs | 2 +- 10 files changed, 283 insertions(+), 144 deletions(-) create mode 100644 crates/media/src/aac.rs diff --git a/Cargo.lock b/Cargo.lock index 79ceb885c..21adef411 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1637,7 +1637,9 @@ dependencies = [ "filmcraft-color", "filmcraft-frame", "filmcraft-isobmff", + "filmcraft-matroska", "filmcraft-media", + "filmcraft-mpegts", "filmcraft-time", "gif", "image", diff --git a/Cargo.toml b/Cargo.toml index 57ea0d720..c101c489c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -87,6 +87,9 @@ filmcraft-h264enc = { git = "https://github.com/storytold/filmcraft", rev = "09a filmcraft-prores = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } filmcraft-aac = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } filmcraft-isobmff = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } +# container headers read for AAC channel layouts (crates/media) +filmcraft-matroska = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } +filmcraft-mpegts = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } # decoder used as the round-trip oracle of effectcraft-opusenc (crates/export dev-dependency) filmcraft-opus = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } filmcraft-vp9 = { git = "https://github.com/storytold/filmcraft", rev = "09aaf9bf15d510eda1c413c0ccabb1d7e08b41c5" } @@ -128,6 +131,8 @@ sys-locale = "0.3" # filmcraft-prores = { path = "../filmcraft/crates/prores" } # filmcraft-aac = { path = "../filmcraft/crates/aac" } # filmcraft-isobmff = { path = "../filmcraft/crates/isobmff" } +# filmcraft-matroska = { path = "../filmcraft/crates/matroska" } +# filmcraft-mpegts = { path = "../filmcraft/crates/mpegts" } [workspace.lints.rust] unsafe_code = "forbid" diff --git a/crates/media/Cargo.toml b/crates/media/Cargo.toml index d3f815bba..77bda60d0 100644 --- a/crates/media/Cargo.toml +++ b/crates/media/Cargo.toml @@ -23,6 +23,8 @@ filmcraft-codecs = { workspace = true } filmcraft-media = { workspace = true } filmcraft-aac = { workspace = true } filmcraft-isobmff = { workspace = true } +filmcraft-matroska = { workspace = true } +filmcraft-mpegts = { workspace = true } filmcraft-frame = { workspace = true } filmcraft-color = { workspace = true } filmcraft-time = { workspace = true } diff --git a/crates/media/src/aac.rs b/crates/media/src/aac.rs new file mode 100644 index 000000000..a2d7de37b --- /dev/null +++ b/crates/media/src/aac.rs @@ -0,0 +1,85 @@ +//! The AudioSpecificConfig a movie's first AAC track decodes with, from the containers that carry +//! it outside the audio frames: MP4 / MOV (`esds`), Matroska (`CodecPrivate`) and LATM in MPEG +//! TS / PS (its StreamMuxConfig). FilmCraft's other AAC is ADTS (in MPEG TS / PS or a raw `.aac`), +//! whose channel configurations FilmCraft decodes (1 to 7) its channel count tells apart +//! ([`crate::downmix::Order::Aac`]). + +use std::io; + +use filmcraft_aac::AudioSpecificConfig; +use filmcraft_media::ByteReader; + +/// How far into an MPEG TS / PS file its first LATM frame is looked for: the stream is indexed +/// as far as this, not through the whole file. +const MPEG_HEAD: u64 = 8 << 20; + +/// The configuration of the first audio track, if it is AAC: from an MP4 / MOV `header` (its +/// ftyp and moov, or the whole file), or else from `file`, read as Matroska or MPEG TS / PS. +pub(crate) fn config(header: Option<&[u8]>, file: Option<&dyn ByteReader>) -> Option { + let asc = match header.and_then(mp4) { + Some(asc) => asc, + None => { + let file = file?; + let mut magic = [0u8; 4]; + file.read_at(0, &mut magic).ok()?; + if magic == [0x1A, 0x45, 0xDF, 0xA3] { matroska(file)? } else { latm(file)? } + } + }; + AudioSpecificConfig::parse(&asc).ok() +} + +fn mp4(bytes: &[u8]) -> Option> { + let file = filmcraft_isobmff::open(bytes).ok()?; + let track = file.tracks.iter().find(|t| t.kind == filmcraft_isobmff::TrackKind::Audio)?; + match &track.entries.first()?.codec { + filmcraft_isobmff::CodecConfig::Aac(a) => Some(a.asc.clone()), + _ => None, + } +} + +fn matroska(file: &dyn ByteReader) -> Option> { + let opts = filmcraft_matroska::OpenOptions { index: false, ..Default::default() }; + let mkv = filmcraft_matroska::open_with(Head(file, file.len()), &opts).ok()?; + match &mkv.tracks.iter().find(|t| t.kind == filmcraft_matroska::TrackKind::Audio)?.codec { + filmcraft_matroska::Codec::Aac { asc } => Some(asc.clone()), + _ => None, + } +} + +fn latm(file: &dyn ByteReader) -> Option> { + let head = Head(file, file.len().min(MPEG_HEAD)); + let mpeg = filmcraft_mpegts::open(&head).ok()?; + let stream = mpeg.find(filmcraft_mpegts::Kind::Audio, |c| *c == filmcraft_mpegts::Codec::AacLatm)?; + let frame = mpeg.read_unit(&head, stream, 0).ok()?; + Some(filmcraft_codecs::audio::latm_config(&frame)?.asc) +} + +/// The first `.1` bytes of a file, as the byte source FilmCraft's demuxers read. +struct Head<'a>(&'a dyn ByteReader, u64); + +impl Head<'_> { + fn read(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> { + match offset.checked_add(buf.len() as u64) { + Some(end) if end <= self.1 => self.0.read_at(offset, buf), + _ => Err(io::Error::new(io::ErrorKind::UnexpectedEof, "read past the end")), + } + } +} + +impl filmcraft_matroska::ByteSource for Head<'_> { + fn len(&self) -> u64 { + self.1 + } + fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> { + self.read(offset, buf) + } +} + +impl filmcraft_mpegts::ByteSource for Head<'_> { + fn len(&self) -> u64 { + self.1 + } + fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> { + self.read(offset, buf) + } +} diff --git a/crates/media/src/downmix.rs b/crates/media/src/downmix.rs index 745f28065..09bb1e1f6 100644 --- a/crates/media/src/downmix.rs +++ b/crates/media/src/downmix.rs @@ -10,17 +10,19 @@ //! exceed full scale: 5.1 is scaled by 1/(1 + 0.707 + 0.707). Mono stays duplicated at unity and //! stereo is untouched. //! -//! Channel order follows the WAVE / SMPTE convention by channel count (what FilmCraft's AC-3, PCM, -//! FLAC and Opus readers deliver): 3 = L R C, 4 = quad L R Ls Rs, 5 = L R C Ls Rs, -//! 6 = L R C LFE Ls Rs, 7 = L R C LFE Cs Ls Rs, 8 = L R C LFE Lb Rb Ls Rs. AAC arrives in either -//! that order or AAC's own (C L R first, LFE last) depending on its channel configuration -//! ([`aac_order`]). More than 8 channels carry no known layout: only the first two (front L/R) -//! are used. +//! A decoded channel's gains come from where it plays. WAVE / SMPTE order by channel count is what +//! FilmCraft's AC-3, PCM, FLAC and Opus readers deliver: 3 = L R C, 4 = quad L R Ls Rs, +//! 5 = L R C Ls Rs, 6 = L R C LFE Ls Rs, 7 = L R C LFE Cs Ls Rs, 8 = L R C LFE Lb Rb Ls Rs. AAC +//! arrives in an order its configuration decides ([`aac_order`]). More than 8 channels in WAVE +//! order carry no known layout: only the first two (front L/R) are used. use std::f32::consts::FRAC_1_SQRT_2; +use std::sync::Arc; + +use filmcraft_aac::{AudioSpecificConfig, ProgramConfig}; /// A source channel's contribution to the left and right outputs, before normalisation. -type Pair = [f32; 2]; +pub type Pair = [f32; 2]; const FL: Pair = [1.0, 0.0]; const FR: Pair = [0.0, 1.0]; @@ -30,33 +32,18 @@ const SL: Pair = [FRAC_1_SQRT_2, 0.0]; const SR: Pair = [0.0, FRAC_1_SQRT_2]; const CS: Pair = [0.5, 0.5]; -/// The per-channel `[left, right]` gains for `count` channels in the given order, normalised. -/// `aac` selects AAC's native channel order. Empty for zero channels. -pub fn stereo_gains(count: usize, aac: bool) -> Vec { - let raw: &[Pair] = match (count, aac) { - (0, _) => &[], - (1, _) => &[[1.0, 1.0]], - (2, _) => &[FL, FR], - (3, false) => &[FL, FR, FC], - (3, true) => &[FC, FL, FR], - (4, false) => &[FL, FR, SL, SR], - (4, true) => &[FC, FL, FR, CS], - (5, false) => &[FL, FR, FC, SL, SR], - (5, true) => &[FC, FL, FR, SL, SR], - (6, false) => &[FL, FR, FC, LFE, SL, SR], - (6, true) => &[FC, FL, FR, SL, SR, LFE], - (7, false) => &[FL, FR, FC, LFE, CS, SL, SR], - (7, true) => &[FC, FL, FR, SL, SR, CS, LFE], - (8, false) => &[FL, FR, FC, LFE, SL, SR, SL, SR], - (8, true) => &[FC, FL, FR, SL, SR, SL, SR, LFE], - _ => &[FL, FR], - }; +/// AAC channel configuration 7 as FilmCraft decodes it (element order, not reordered). +const CONFIG_7: [Pair; 8] = [FC, FL, FR, SL, SR, SL, SR, LFE]; +/// AAC channel configuration 4 (C L R Cs), which FilmCraft reorders to L R C Cs. +const CONFIG_4: [Pair; 4] = [FL, FR, FC, CS]; + +/// `raw` scaled by the reciprocal of its larger row sum when that exceeds 1 (more than two +/// channels): full-scale input can never exceed full scale. +fn normalised(raw: &[Pair]) -> Vec { let mut out = raw.to_vec(); - // Channels beyond a known layout (more than 8) are silent. - out.resize(count, [0.0, 0.0]); let peak = out.iter().map(|g| g[0]).sum::().max(out.iter().map(|g| g[1]).sum::()); // Mono is deliberately left at unity on both sides; so is anything already within range. - if count > 2 && peak > 1.0 { + if out.len() > 2 && peak > 1.0 { for g in &mut out { g[0] /= peak; g[1] /= peak; @@ -65,63 +52,109 @@ pub fn stereo_gains(count: usize, aac: bool) -> Vec { out } -/// The order a source's decoded channels arrive in. -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +/// The per-channel `[left, right]` gains for `count` channels in WAVE / SMPTE order, normalised. +/// Empty for zero channels. +pub fn stereo_gains(count: usize) -> Vec { + let raw: &[Pair] = match count { + 0 => &[], + 1 => &[[1.0, 1.0]], + 2 => &[FL, FR], + 3 => &[FL, FR, FC], + 4 => &[FL, FR, SL, SR], + 5 => &[FL, FR, FC, SL, SR], + 6 => &[FL, FR, FC, LFE, SL, SR], + 7 => &[FL, FR, FC, LFE, CS, SL, SR], + 8 => &[FL, FR, FC, LFE, SL, SR, SL, SR], + _ => &[FL, FR], + }; + let mut raw = raw.to_vec(); + // Channels beyond a known layout (more than 8) are silent. + raw.resize(count, [0.0, 0.0]); + normalised(&raw) +} + +/// Where a source's decoded channels play. +#[derive(Clone, Debug, Default, PartialEq)] pub enum Order { /// WAVE / SMPTE by channel count: PCM, FLAC, AC-3, Opus and the rest. #[default] Wav, - /// AAC that FilmCraft reorders to WAVE / SMPTE (configurations 3 to 6, 11 and 12; 4 becomes - /// L R C Cs) or that ffmpeg wrote in it (a program config element listing the front pair - /// first: ffmpeg fills a PCE's elements in its own channel order and reads them back so). - AacWav, - /// AAC in its element order (C L R … LFE), which FilmCraft keeps: configuration 7 and a PCE - /// listing the centre first. - AacElements, - /// AAC whose configuration the container doesn't show us (only MP4 / MOV headers are read): - /// as [`Order::AacWav`] up to 7 channels (configurations 3 to 6, 11 and ffmpeg's 6.1 PCE), - /// as [`Order::AacElements`] above (configuration 7, ffmpeg's 7.1). + /// AAC whose configuration wasn't read: by channel count as the configurations 1 to 7 decode + /// (exact for ADTS, which can signal no other configuration FilmCraft decodes), 7 channels as + /// configuration 11. Aac, + /// Each decoded channel's gains, before normalisation. + Speakers(Arc<[Pair]>), } -/// How FilmCraft's decoder hands over the AAC described by the AudioSpecificConfig `asc`. -pub fn aac_order(asc: &[u8]) -> Order { - let Ok(asc) = filmcraft_aac::AudioSpecificConfig::parse(asc) else { return Order::Aac }; - match (asc.channel_config, asc.pce) { - (3..=6 | 11 | 12, _) => Order::AacWav, - (0, Some(pce)) if pce.front.first().is_some_and(|&(cpe, _)| cpe) => Order::AacWav, - (0, None) => Order::Aac, - _ => Order::AacElements, +/// How FilmCraft's decoder hands over AAC with the configuration `asc`: it reorders channel +/// configurations 3 to 6, 11 and 12 to WAVE / SMPTE and leaves 7 and PCE layouts in element order. +pub fn aac_order(asc: &AudioSpecificConfig) -> Order { + match (asc.channel_config, &asc.pce) { + (4, _) => Order::Speakers(CONFIG_4.into()), + (1..=6 | 11 | 12, _) => Order::Wav, + (7, _) => Order::Speakers(CONFIG_7.into()), + (0, Some(pce)) => Order::Speakers(pce_speakers(pce).into()), + _ => Order::Aac, } } -/// The gains for `count` channels arriving in `order`. -pub fn decoded_gains(count: usize, order: Order) -> Vec { - let order = match order { - Order::Aac if (3..=7).contains(&count) => Order::AacWav, - Order::Aac => Order::AacElements, - o => o, - }; - match (order, count) { - // C L R Cs, reordered to L R C Cs: the same gains, permuted. - (Order::AacWav, 4) => { - let g = stereo_gains(4, true); - [1, 2, 0, 3].iter().filter_map(|&i| g.get(i).copied()).collect() +/// ffmpeg's encoder fills a program config element's elements with its layout's channels in its +/// own order (L R C LFE …), and for these structures that order doesn't match the element +/// positions: an LFE in a side SCE, a back centre in a side pair. Front, side and back element +/// types (`true` for a pair) and the channels of the ffmpeg layout with that structure (the first +/// ffmpeg 8.1 lists, where several share it: 6.1 for 6.1(back) and 6.1(front), 7.1(wide) for +/// 7.1(wide-side)). +const FFMPEG_PCE: [(&[bool], &[bool], &[bool], &[Pair]); 6] = [ + (&[true, false], &[false], &[false], &[FL, FR, FC, LFE, CS]), + (&[true, false], &[false], &[true], &[FL, FR, FC, LFE, SL, SR]), + (&[true, false], &[true], &[false], &[FL, FR, FC, CS, SL, SR]), + (&[true, false], &[false], &[true, false], &[FL, FR, FC, LFE, CS, SL, SR]), + (&[true, false], &[false], &[true, true], &[FL, FR, FC, LFE, SL, SR, FL, FR]), + (&[true, false], &[true], &[true, false], &[FL, FR, FC, SL, SR, CS, SL, SR]), +]; + +/// The channels of a PCE layout in decoded (element) order: ffmpeg's for its structures above +/// (it names itself in the PCE's comment), otherwise by position (ISO/IEC 14496-3's program config element): +/// front pairs left and right, a front single the centre, side and back pairs surrounds, a side +/// or back single a back centre. +fn pce_speakers(pce: &ProgramConfig) -> Vec { + let pairs = |els: &[(bool, u8)]| els.iter().map(|e| e.0).collect::>(); + let (front, side, back) = (pairs(&pce.front), pairs(&pce.side), pairs(&pce.back)); + if pce.comment.starts_with(b"Lavc") && pce.lfe.is_empty() { + let known = FFMPEG_PCE.iter().find(|(f, s, b, _)| *f == front.as_slice() && *s == side.as_slice() && *b == back.as_slice()); + if let Some((.., speakers)) = known { + return speakers.to_vec(); } - (Order::AacElements, _) => stereo_gains(count, true), - _ => stereo_gains(count, false), + } + let mut out = Vec::new(); + for pair in front { + out.extend_from_slice(if pair { &[FL, FR] } else { &[FC] }); + } + for pair in side.into_iter().chain(back) { + out.extend_from_slice(if pair { &[SL, SR] } else { &[CS] }); + } + out.extend(pce.lfe.iter().map(|_| LFE)); + out +} + +/// The gains for `count` channels arriving in `order`. FilmCraft decodes as many channels as the +/// container names, the first ones of the layout where that is fewer (MPEG-TS LATM names 7 of +/// configuration 7's 8, ffmpeg 4.2 MP4s 2 of a PCE's); more go by count. +pub fn decoded_gains(count: usize, order: &Order) -> Vec { + match order { + Order::Speakers(s) if count <= s.len() => normalised(s.get(..count).unwrap_or_default()), + Order::Aac if count == 4 => normalised(&CONFIG_4), + Order::Aac if count == 8 => normalised(&CONFIG_7), + _ => stereo_gains(count), } } /// Downmix planar `channels` arriving in `order` to interleaved stereo in `out` (`out.len() / 2` /// frames); frames a channel is too short for count as silence, extra frames in `out` stay as /// they are. -pub fn downmix_into(channels: &[Vec], order: Order, out: &mut [f32]) { - mix(channels, &decoded_gains(channels.len(), order), out); -} - -fn mix(channels: &[Vec], gains: &[Pair], out: &mut [f32]) { - for (ch, g) in channels.iter().zip(gains) { +pub fn downmix_into(channels: &[Vec], order: &Order, out: &mut [f32]) { + for (ch, g) in channels.iter().zip(&decoded_gains(channels.len(), order)) { if g[0] == 0.0 && g[1] == 0.0 { continue; } @@ -138,13 +171,14 @@ mod tests { #[test] fn every_channel_count_has_a_bounded_defined_matrix() { - for aac in [false, true] { + let pce = Order::Speakers([FL, FR, FC, FC, SL, SR, SL, SR, CS, LFE].into()); + for order in [Order::Wav, Order::Aac, Order::Speakers(CONFIG_7.into()), pce] { for n in 0..40usize { - let g = stereo_gains(n, aac); + let g = decoded_gains(n, &order); assert_eq!(g.len(), n); let (l, r) = (g.iter().map(|p| p[0]).sum::(), g.iter().map(|p| p[1]).sum::()); if n > 2 { - assert!(l <= 1.0001 && r <= 1.0001, "{n} channels (aac {aac}): {l} {r}"); + assert!(l <= 1.0001 && r <= 1.0001, "{n} channels ({order:?}): {l} {r}"); assert!(l > 0.0 && r > 0.0); } } @@ -155,75 +189,87 @@ mod tests { const PCE_FFMPEG_61: &str = "118004c848002000c4400d4c61766336322e32382e31303256e500"; const PCE_FILMCRAFT_61: &str = "118004c845000108840000"; - fn asc(hex: &str) -> Vec { - (0..hex.len() / 2).filter_map(|i| hex.get(2 * i..2 * i + 2).and_then(|b| u8::from_str_radix(b, 16).ok())).collect() + fn order(hex: &str) -> Order { + let asc: Vec = (0..hex.len() / 2).filter_map(|i| hex.get(2 * i..2 * i + 2).and_then(|b| u8::from_str_radix(b, 16).ok())).collect(); + aac_order(&AudioSpecificConfig::parse(&asc).expect("an AudioSpecificConfig")) + } + + fn speakers(order: &Order) -> Vec { + match order { + Order::Speakers(s) => s.to_vec(), + o => panic!("{o:?} names no speakers"), + } } #[test] fn aac_order_follows_the_channel_configuration() { - // AAC-LC at 48 kHz with channel configuration n - let config = |n: u8| aac_order(&[0x11, 0x80 | n << 3]); - for n in [3, 4, 5, 6, 11, 12] { - assert_eq!(config(n), Order::AacWav, "configuration {n}"); + for n in [1, 2, 3, 5, 6, 11, 12] { + assert_eq!(aac_order(&AudioSpecificConfig::lc(3, n)), Order::Wav, "configuration {n}"); } - for n in [1, 2, 7] { - assert_eq!(config(n), Order::AacElements, "configuration {n}"); - } - assert_eq!(aac_order(&asc(PCE_FFMPEG_61)), Order::AacWav, "ffmpeg's 6.1 PCE lists L R first"); - assert_eq!(aac_order(&asc(PCE_FILMCRAFT_61)), Order::AacElements, "a PCE listing the centre first"); - assert_eq!(aac_order(&[]), Order::Aac); + assert_eq!(speakers(&aac_order(&AudioSpecificConfig::lc(3, 4))), [FL, FR, FC, CS]); + assert_eq!(speakers(&aac_order(&AudioSpecificConfig::lc(3, 7))), [FC, FL, FR, SL, SR, SL, SR, LFE]); + assert_eq!(speakers(&order(PCE_FFMPEG_61)), [FL, FR, FC, LFE, CS, SL, SR], "ffmpeg's 6.1 carries the LFE in a side SCE"); + assert_eq!(speakers(&order(PCE_FILMCRAFT_61)), [FC, FL, FR, SL, SR, CS, LFE], "a centre-first PCE by position"); } #[test] - fn decoded_aac_gains_follow_the_order() { - for n in [3, 5, 6, 7, 8] { - assert_eq!(decoded_gains(n, Order::AacWav), stereo_gains(n, false), "{n} channels"); - assert_eq!(decoded_gains(n, Order::AacElements), stereo_gains(n, true), "{n} channels"); - } - let g = decoded_gains(4, Order::AacWav); - assert_eq!(g.len(), 4); - assert!(g[0][0] > 0.0 && g[0][1] == 0.0 && g[1][1] > 0.0 && g[1][0] == 0.0, "L R first"); - assert!(g[2][0] > 0.0 && g[2][0] == g[2][1], "the centre is third and feeds both sides"); - assert!(g[3][0] > 0.0 && g[3][0] < g[2][0], "the rear centre is last, below the centre"); - assert_eq!(decoded_gains(4, Order::Wav), stereo_gains(4, false), "PCM 4 channels are quad"); - for n in [3, 4, 5, 6, 7] { - assert_eq!(decoded_gains(n, Order::Aac), decoded_gains(n, Order::AacWav), "{n} channels"); - } - for n in [1, 2, 8, 9] { - assert_eq!(decoded_gains(n, Order::Aac), stereo_gains(n, true), "{n} channels"); + fn only_ffmpeg_pce_structures_it_misplaces_take_its_channel_order() { + let pce = |comment: &[u8], side: Vec<(bool, u8)>, lfe: Vec| ProgramConfig { + front: vec![(true, 0), (false, 0)], + side, + back: vec![(true, 1)], + lfe, + comment: comment.to_vec(), + ..Default::default() + }; + // 5.1(side): ffmpeg's LFE sits in the side SCE + assert_eq!(pce_speakers(&pce(b"Lavc58.54.100", vec![(false, 1)], vec![])), [FL, FR, FC, LFE, SL, SR]); + // the same structure from another encoder: a back centre by position + assert_eq!(pce_speakers(&pce(b"", vec![(false, 1)], vec![])), [FL, FR, FC, CS, SL, SR]); + // 7.0: ffmpeg's order matches the positions + assert_eq!(pce_speakers(&pce(b"Lavc62.28.102", vec![(true, 2)], vec![])), [FL, FR, FC, SL, SR, SL, SR]); + // a real LFE element is never ffmpeg's misplaced one + assert_eq!(pce_speakers(&pce(b"Lavc62.28.102", vec![(false, 1)], vec![0])), [FL, FR, FC, CS, SL, SR, LFE]); + } + + #[test] + fn a_decode_of_fewer_channels_takes_the_first_of_the_layout_and_more_go_by_count() { + assert_eq!(decoded_gains(2, &order(PCE_FILMCRAFT_61)), [FC, FL]); + assert_eq!(decoded_gains(7, &Order::Speakers(CONFIG_7.into())), normalised(&CONFIG_7[..7])); + assert_eq!(decoded_gains(9, &Order::Speakers(CONFIG_7.into())), stereo_gains(9)); + assert_eq!(decoded_gains(4, &Order::Aac), normalised(&[FL, FR, FC, CS])); + assert_eq!(decoded_gains(8, &Order::Aac), normalised(&CONFIG_7)); + for n in [3, 5, 6, 7] { + assert_eq!(decoded_gains(n, &Order::Aac), stereo_gains(n), "{n} channels"); } } #[test] fn five_one_matches_the_bs775_matrix() { let k = 1.0 / (1.0 + 2.0 * FRAC_1_SQRT_2); - let g = stereo_gains(6, false); + let g = stereo_gains(6); assert!((g[0][0] - k).abs() < 1e-6 && g[0][1] == 0.0); assert!((g[2][0] - k * FRAC_1_SQRT_2).abs() < 1e-6 && (g[2][1] - k * FRAC_1_SQRT_2).abs() < 1e-6); assert_eq!(g[3], [0.0, 0.0], "LFE is left out"); assert!((g[4][0] - k * FRAC_1_SQRT_2).abs() < 1e-6 && g[4][1] == 0.0); assert!((g[5][1] - k * FRAC_1_SQRT_2).abs() < 1e-6 && g[5][0] == 0.0); - // AAC order: the centre is first, LFE last - let a = stereo_gains(6, true); - assert_eq!((a[0], a[5]), (g[2], g[3])); - assert_eq!((a[1], a[2]), (g[0], g[1])); } #[test] fn mono_and_stereo_are_untouched_and_wide_layouts_use_the_front_pair() { - assert_eq!(stereo_gains(1, false), vec![[1.0, 1.0]]); - assert_eq!(stereo_gains(2, true), vec![FL, FR]); - let g = stereo_gains(12, false); + assert_eq!(stereo_gains(1), vec![[1.0, 1.0]]); + assert_eq!(stereo_gains(2), vec![FL, FR]); + let g = stereo_gains(12); assert_eq!((g[0], g[1], g[2]), (FL, FR, [0.0, 0.0])); } #[test] fn ragged_and_empty_input_never_panics() { let mut out = vec![0.0; 8]; - downmix_into(&[], Order::Wav, &mut out); - downmix_into(&[vec![1.0; 100], vec![1.0; 2], vec![], vec![1.0; 3], vec![1.0; 1], vec![1.0]], Order::Aac, &mut out); + downmix_into(&[], &Order::Wav, &mut out); + downmix_into(&[vec![1.0; 100], vec![1.0; 2], vec![], vec![1.0; 3], vec![1.0; 1], vec![1.0]], &Order::Aac, &mut out); assert!(out.iter().all(|v| v.is_finite())); let mut none: [f32; 0] = []; - downmix_into(&vec![vec![1.0; 3]; 7], Order::Wav, &mut none); + downmix_into(&vec![vec![1.0; 3]; 7], &Order::Speakers(CONFIG_7.into()), &mut none); } } diff --git a/crates/media/src/lib.rs b/crates/media/src/lib.rs index 306f8d860..d9df8944a 100644 --- a/crates/media/src/lib.rs +++ b/crates/media/src/lib.rs @@ -15,6 +15,7 @@ #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +mod aac; mod convert; pub mod downmix; mod dpx; diff --git a/crates/media/src/pool.rs b/crates/media/src/pool.rs index 193804447..655ca4ed8 100644 --- a/crates/media/src/pool.rs +++ b/crates/media/src/pool.rs @@ -193,21 +193,20 @@ struct Opened { } impl Opened { - /// `src` with what its MP4/MOV `header` (ftyp and moov) says, if it has one. - fn new(src: SharedSource, header: Option<&[u8]>) -> Opened { + /// `src` with what its container says: `header` is an MP4/MOV's ftyp and moov (or the whole + /// file), `file` the file for other containers' AAC configuration. + fn new(src: SharedSource, header: Option<&[u8]>, file: Option<&dyn filmcraft_media::ByteReader>) -> Opened { let turn = header.map_or(Orientation::NoTransforms, crate::probe::video_turn); - let order = channel_order(&src, header.and_then(crate::probe::aac_config).as_deref()); - Opened { src, turn, order } + Opened { order: channel_order(&src, header, file), src, turn } } } -/// The order `src`'s audio channels decode in, AAC's decided by its AudioSpecificConfig `asc` -/// when the container's header was read. -fn channel_order(src: &SharedSource, asc: Option<&[u8]>) -> Order { - match (src.info().audio().is_some_and(|a| a.codec.starts_with("AAC")), asc) { - (false, _) => Order::Wav, - (true, Some(asc)) => crate::downmix::aac_order(asc), - (true, None) => Order::Aac, +/// The order `src`'s audio channels decode in; multichannel AAC's by its configuration, where +/// the container's header shows it ([`crate::aac::config`]). +fn channel_order(src: &SharedSource, header: Option<&[u8]>, file: Option<&dyn filmcraft_media::ByteReader>) -> Order { + match src.info().audio() { + Some(a) if a.codec.contains("AAC") && a.channels > 2 => crate::aac::config(header, file).map_or(Order::Aac, |asc| crate::downmix::aac_order(&asc)), + _ => Order::Wav, } } @@ -551,14 +550,14 @@ impl MediaPool { return Err(format!("a sample rate of {native} Hz is outside {MIN_SAMPLE_RATE}–{MAX_SAMPLE_RATE} Hz")); } if native == 0 || rate == 0 || native == rate { - return read_stereo(&src, order, s0, frames, rate); + return read_stereo(&src, &order, s0, frames, rate); } // Output sample `n` sits at source position n × native / rate. let (native_i, rate_i) = (i128::from(native), i128::from(rate)); let first = (i128::from(s0) * native_i).div_euclid(rate_i); let last = ((i128::from(s0) + frames as i128) * native_i).div_euclid(rate_i) + 1; let (Ok(first64), Ok(count)) = (i64::try_from(first), usize::try_from(last - first + 1)) else { return Ok(vec![0.0; frames * 2]) }; - let buf = read_stereo(&src, order, first64, count, native)?; + let buf = read_stereo(&src, &order, first64, count, native)?; let mut out = Vec::with_capacity(frames * 2); for k in 0..frames as i128 { let pos = (i128::from(s0) + k) * native_i; @@ -636,15 +635,14 @@ impl Inner { let file = std::path::Path::new(&**path); let in_place = if in_memory { None } else { crate::stream::open_source(file) }; let in_place = in_place.map(|src| { - let header = crate::stream::movie_header(file); - src.map(|src| Opened::new(src, header.as_deref())) + let (header, reader) = (crate::stream::movie_header(file), crate::stream::file_reader(file)); + src.map(|src| Opened::new(src, header.as_deref(), reader.as_deref())) }); let opened = in_place.unwrap_or_else(|| { self.read(path).and_then(|bytes| { let name = std::path::Path::new(&**path).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default(); - let (turn, asc) = (crate::probe::video_turn(&bytes), crate::probe::aac_config(&bytes)); - let src = filmcraft_codecs::open_bytes(&name, bytes).map_err(MediaError::from)?; - Ok(Opened { order: channel_order(&src, asc.as_deref()), src, turn }) + let src = filmcraft_codecs::open_bytes(&name, bytes.clone()).map_err(MediaError::from)?; + Ok(Opened::new(src, Some(&bytes), Some(&filmcraft_media::reader::MemReader(bytes.clone())))) }) }); match opened { @@ -978,7 +976,7 @@ impl FootageSource for MediaPool { /// `frames` stereo sample frames (interleaved) of `src` from sample `s0` at `rate` Hz, which /// FilmCraft serves as decoded when it is the source's own rate. Mono is duplicated to both /// channels, multichannel sources (channels in `order`) are downmixed ([`crate::downmix`]); silence before the start. An error where decoding fails. -fn read_stereo(src: &SharedSource, order: Order, s0: i64, frames: usize, rate: u32) -> std::result::Result, String> { +fn read_stereo(src: &SharedSource, order: &Order, s0: i64, frames: usize, rate: u32) -> std::result::Result, String> { let mut out = vec![0.0; frames * 2]; let skip = usize::try_from(s0.saturating_neg()).unwrap_or(0); if skip >= frames { diff --git a/crates/media/src/probe.rs b/crates/media/src/probe.rs index 435f9ce57..9a7ebfa1d 100644 --- a/crates/media/src/probe.rs +++ b/crates/media/src/probe.rs @@ -259,17 +259,6 @@ pub(crate) fn load_oriented(bytes: &[u8], fmt: Option) -> im Ok(img) } -/// The AudioSpecificConfig of an MP4/MOV movie's first audio track (the one FilmCraft plays), if -/// that is AAC. -pub(crate) fn aac_config(bytes: &[u8]) -> Option> { - let file = filmcraft_isobmff::open(bytes).ok()?; - let track = file.tracks.iter().find(|t| t.kind == filmcraft_isobmff::TrackKind::Audio)?; - match &track.entries.first()?.codec { - filmcraft_isobmff::CodecConfig::Aac(a) => Some(a.asc.clone()), - _ => None, - } -} - /// The rotation an MP4/MOV movie is shown with (a phone held upright records sideways frames and /// says so in its video track's header): the quarter turn nearest the track's rotation. Only the /// rotation counts; any mirroring, scaling or offset in the header is ignored, and other diff --git a/crates/media/src/stream.rs b/crates/media/src/stream.rs index 447e239d1..80db21fc4 100644 --- a/crates/media/src/stream.rs +++ b/crates/media/src/stream.rs @@ -85,6 +85,17 @@ pub(crate) fn open_source(_path: &Path) -> Option> { None } +/// `path` read in place (`None` when it can't be opened, or without a native file system). +#[cfg(any(unix, windows))] +pub(crate) fn file_reader(path: &Path) -> Option> { + Some(Box::new(FileReader::open(path).ok()?)) +} + +#[cfg(not(any(unix, windows)))] +pub(crate) fn file_reader(_path: &Path) -> Option> { + None +} + /// The leading `ftyp` and the `moov` box of an ISO-BMFF movie on disk (all a track's display /// matrix needs), without reading the samples: top-level box headers are walked with positional /// reads. `None` when the file isn't ISO-BMFF or its `moov` is missing or implausibly large. diff --git a/crates/media/tests/it/footage.rs b/crates/media/tests/it/footage.rs index 4d539c4e5..c57cec7eb 100644 --- a/crates/media/tests/it/footage.rs +++ b/crates/media/tests/it/footage.rs @@ -915,7 +915,7 @@ fn flac_audio_of_any_shape_reads_in_device_blocks() { // ffmpeg's sine source has an amplitude of 1/8 (mono is duplicated). The fixture carries // channels 0 and 1 at full level and the others at half, so a multichannel file sounds as // its stereo downmix (tests/it/downmix.rs checks that matrix channel by channel). - let gains = effectcraft_media::downmix::stereo_gains(channels as usize, false); + let gains = effectcraft_media::downmix::stereo_gains(channels as usize); let side = |s: usize| gains.iter().enumerate().map(|(c, g)| f64::from(g[s]) * if c < 2 { 1.0 } else { 0.5 }).sum::(); let want = 0.125 * ((side(0).powi(2) + side(1).powi(2)) / 2.0).sqrt() / 2f64.sqrt(); let rms = (sum / n.max(1) as f64).sqrt(); From ec86f24afeaf54da8a4ea661c00c5d7ee89b9f30 Mon Sep 17 00:00:00 2001 From: tylerjenningsw Date: Thu, 29 Oct 2026 18:20:09 -0400 Subject: [PATCH 08/12] Media tests: every AAC layout mixes by speaker in MP4, Matroska and ADTS (#1701) Tones encoded with FilmCraft's AAC encoder, with the configuration rewritten to 11, 12 or ffmpeg's 6.1 PCE where the coded elements fit it, so nothing depends on ffmpeg or skips. Expected gains come from each channel's speaker. --- crates/media/tests/it/aac_layout.rs | 224 ++++++++++++++++++++++++++++ crates/media/tests/it/main.rs | 1 + 2 files changed, 225 insertions(+) create mode 100644 crates/media/tests/it/aac_layout.rs diff --git a/crates/media/tests/it/aac_layout.rs b/crates/media/tests/it/aac_layout.rs new file mode 100644 index 000000000..9e5ddbb23 --- /dev/null +++ b/crates/media/tests/it/aac_layout.rs @@ -0,0 +1,224 @@ +//! Multichannel AAC reaches stereo by where each channel plays, whatever order its channel +//! configuration decodes in (#1701). Fixtures are tones encoded with FilmCraft's AAC encoder (no +//! ffmpeg needed, so nothing is skipped), with the configuration rewritten where the coded +//! elements fit another: configurations 11 and 12 code the same elements as the encoder's 7- and +//! 8-channel output, and a program config element can name any of them. + +use std::f64::consts::PI; +use std::io::Cursor; +use std::path::PathBuf; +use std::sync::Arc; + +use effectcraft_media::{MediaPool, probe, probe_bytes}; +use effectcraft_time::Tick; +use filmcraft_aac::{AudioSpecificConfig, Encoder, EncoderConfig, ProgramConfig}; + +const RATE: u32 = 48_000; +/// Tone frequencies by channel; an LFE gets [`LFE_TONE`], which its element can carry. +const FREQS: [u32; 8] = [300, 500, 700, 900, 1100, 1300, 1500, 1700]; +const LFE_TONE: u32 = 60; +const AMP: f32 = 0.25; +const K: f64 = std::f64::consts::FRAC_1_SQRT_2; + +/// A channel's content, by where it plays. +#[derive(Clone, Copy, Debug, PartialEq)] +enum Ch { + L, + R, + C, + Lfe, + Ls, + Rs, + Cs, +} + +impl Ch { + /// ITU-R BS.775 gains into `[left, right]`, before normalisation. + fn gains(self) -> [f64; 2] { + match self { + Ch::L => [1.0, 0.0], + Ch::R => [0.0, 1.0], + Ch::C => [K, K], + Ch::Lfe => [0.0, 0.0], + Ch::Ls => [K, 0.0], + Ch::Rs => [0.0, K], + Ch::Cs => [0.5, 0.5], + } + } +} + +struct Case { + name: &'static str, + /// What the encoder's input channels carry (its element order: C, L R, …, LFE last). + content: &'static [Ch], + /// The AudioSpecificConfig written to the container instead of the encoder's, if any. + asc: Option, +} + +fn lc(config: u8) -> Option { + Some(AudioSpecificConfig::lc(3, config)) +} + +fn pce(front: Vec<(bool, u8)>, side: Vec<(bool, u8)>, back: Vec<(bool, u8)>, lfe: Vec, comment: &[u8]) -> Option { + let pce = ProgramConfig { object_type: 1, sf_index: 3, front, side, back, lfe, comment: comment.to_vec(), ..Default::default() }; + Some(AudioSpecificConfig { pce: Some(pce), ..AudioSpecificConfig::lc(3, 0) }) +} + +use Ch::*; + +fn cases() -> Vec { + vec![ + Case { name: "config3", content: &[C, L, R], asc: None }, + Case { name: "config4", content: &[C, L, R, Cs], asc: None }, + Case { name: "config5", content: &[C, L, R, Ls, Rs], asc: None }, + Case { name: "config6", content: &[C, L, R, Ls, Rs, Lfe], asc: None }, + // configuration 7 as ffmpeg writes 7.1: the second and third pairs are surrounds + Case { name: "config7", content: &[C, L, R, Ls, Rs, Ls, Rs, Lfe], asc: None }, + Case { name: "config11", content: &[C, L, R, Ls, Rs, Cs, Lfe], asc: lc(11) }, + Case { name: "config12", content: &[C, L, R, Ls, Rs, Ls, Rs, Lfe], asc: lc(12) }, + // the encoder's own 6.1: a PCE listing the centre first + Case { name: "pce61-centre-first", content: &[C, L, R, Ls, Rs, Cs, Lfe], asc: None }, + // ffmpeg's 6.1 PCE: front pair and centre, a side SCE, a back pair and SCE, no LFE + // element, its channels in ffmpeg's order L R C LFE Cs Ls Rs. The decoder places the + // encoder's SCE 0 in the front SCE, pair 0 front, pair 1 back, SCE 1 in the back SCE (tag + // 1) and the LFE in the side SCE left over (tag 5). + Case { + name: "pce61-ffmpeg", + content: &[C, L, R, Cs, Ls, Rs, Lfe], + asc: pce(vec![(true, 0), (false, 0)], vec![(false, 5)], vec![(true, 1), (false, 1)], vec![], b"Lavc62.28.102"), + }, + ] +} + +fn freq(content: &[Ch], i: usize) -> u32 { + if content.get(i) == Some(&Lfe) { LFE_TONE } else { FREQS[i] } +} + +/// Two seconds of `content`'s tones encoded as AAC: the AudioSpecificConfig and access units. +fn encode(content: &[Ch], adts: bool) -> (Vec, Vec>) { + let n = content.len(); + let mut cfg = EncoderConfig::cbr(RATE, n, 96_000 * n as u32); + cfg.adts = adts; + let mut enc = Encoder::new(cfg).expect("encoder"); + let planes: Vec> = + (0..n).map(|c| (0..2 * RATE).map(|i| AMP * (2.0 * PI * f64::from(freq(content, c)) * f64::from(i) / f64::from(RATE)).sin() as f32).collect()).collect(); + let refs: Vec<&[f32]> = planes.iter().map(Vec::as_slice).collect(); + let mut units = enc.encode(&refs); + units.extend(enc.flush()); + (enc.audio_specific_config(), units) +} + +fn mp4(asc: Vec, channels: u32, units: &[Vec]) -> Vec { + use filmcraft_isobmff::{Brand, Mp4Writer, SampleEntry, TrackConfig, WriteSample, WriterOptions}; + let mut w = Mp4Writer::new(Cursor::new(Vec::new()), WriterOptions::new(Brand::Mp4)).expect("writer"); + let t = w.add_track(TrackConfig::new(SampleEntry::aac(asc, channels, RATE), RATE)).expect("track"); + for u in units { + w.write_sample(t, WriteSample { data: u, duration: 1024, composition_offset: 0, is_sync: true }).expect("sample"); + } + w.finish().expect("finish").into_inner() +} + +fn mkv(asc: Vec, channels: u32, units: &[Vec]) -> Vec { + use filmcraft_matroska::{MkvWriter, MuxOptions, TrackKind, TrackSpec}; + let mut t = TrackSpec::new(TrackKind::Audio, "A_AAC"); + t.codec_private = asc; + t.audio = Some((f64::from(RATE), u64::from(channels), None)); + let mut w = MkvWriter::new(Cursor::new(Vec::new()), vec![t], MuxOptions::default()).expect("writer"); + let frame_ns = 1_000_000_000 * 1024 / i64::from(RATE); + for (i, u) in units.iter().enumerate() { + w.write_frame(0, i as i64 * frame_ns, true, u, Some(frame_ns as u64)).expect("frame"); + } + w.finish().expect("finish").into_inner() +} + +/// The fixture written to disk as `name`. +fn write(name: &str, bytes: &[u8]) -> PathBuf { + let dir = std::env::temp_dir().join(format!("effectcraft-aac-layout-{}", std::process::id())); + std::fs::create_dir_all(&dir).expect("mkdir"); + let p = dir.join(name); + std::fs::write(&p, bytes).expect("write"); + p +} + +/// Amplitude of the tone at `freq` in one side of interleaved stereo. +fn amp(s: &[f32], side: usize, freq: u32) -> f64 { + let (mut re, mut im, mut n) = (0.0, 0.0, 0usize); + for (i, f) in s.as_chunks::<2>().0.iter().enumerate() { + let w = 2.0 * PI * f64::from(freq) * i as f64 / f64::from(RATE); + re += f64::from(f[side]) * w.cos(); + im += f64::from(f[side]) * w.sin(); + n += 1; + } + 2.0 * (re * re + im * im).sqrt() / n.max(1) as f64 +} + +/// Stereo of one second from 0.5 s in, through the media pool as footage reads it. +fn mix(pool: &MediaPool, footage: &effectcraft_project::Footage) -> Vec { + pool.audio_samples(footage, Tick::from_seconds_f64(0.5), RATE as usize, RATE) +} + +/// Each channel of `content` reaches each side at its BS.775 gain, normalised by the larger +/// side's sum (an LFE nowhere). +fn check(label: &str, content: &[Ch], s: &[f32]) { + let raw: Vec<[f64; 2]> = content.iter().map(|c| c.gains()).collect(); + let norm = raw.iter().map(|g| g[0]).sum::().max(raw.iter().map(|g| g[1]).sum::()).max(1.0); + for (i, (ch, g)) in content.iter().zip(&raw).enumerate() { + for side in 0..2 { + let (got, want) = (amp(s, side, freq(content, i)), f64::from(AMP) * g[side] / norm); + assert!( + (got - want).abs() <= f64::from(AMP) * 0.05, + "{label}: {ch:?} ({} Hz) -> {} is {got:.4}, expected {want:.4}", + freq(content, i), + ["L", "R"][side] + ); + } + } +} + +/// The LFE tone survives encoding, so its absence from the mix means it was left out. +fn lfe_is_coded(case: &Case, bytes: Vec) { + if !case.content.contains(&Lfe) { + return; + } + let src = filmcraft_codecs::open_bytes("lfe.mp4", bytes.into()).expect("open"); + let buf = src.audio(24_000, RATE as usize, RATE).expect("decode"); + let best = buf.channels.iter().map(|ch| amp(&ch.iter().flat_map(|&v| [v, v]).collect::>(), 0, LFE_TONE)).fold(0.0, f64::max); + assert!(best > 0.8 * f64::from(AMP), "{}: the LFE tone is coded ({best:.4})", case.name); +} + +#[test] +fn every_aac_layout_mixes_by_speaker_in_mp4() { + for case in cases() { + let (asc, units) = encode(case.content, false); + let asc = case.asc.as_ref().map_or(asc, AudioSpecificConfig::to_bytes); + let bytes = mp4(asc, case.content.len() as u32, &units); + lfe_is_coded(&case, bytes.clone()); + let p = write(&format!("{}.mp4", case.name), &bytes); + check(case.name, case.content, &mix(&MediaPool::new(), &probe(&p).expect("probe"))); + // in memory (`add_bytes`) as well as in place + let (path, bytes): (String, Arc<[u8]>) = (format!("/mem/{}.mp4", case.name), bytes.into()); + let pool = MediaPool::new(); + pool.add_bytes(&path, bytes.clone()); + check(&format!("{} in memory", case.name), case.content, &mix(&pool, &probe_bytes(&path, bytes).expect("probe"))); + } +} + +#[test] +fn matroska_aac_layouts_mix_by_speaker() { + for case in cases().into_iter().filter(|c| matches!(c.name, "config12" | "pce61-centre-first" | "pce61-ffmpeg")) { + let (asc, units) = encode(case.content, false); + let asc = case.asc.as_ref().map_or(asc, AudioSpecificConfig::to_bytes); + let p = write(&format!("{}.mkv", case.name), &mkv(asc, case.content.len() as u32, &units)); + check(case.name, case.content, &mix(&MediaPool::new(), &probe(&p).expect("probe"))); + } +} + +/// ADTS signals configurations 1 to 7 only, which the channel count tells apart. +#[test] +fn adts_aac_layouts_mix_by_speaker() { + for case in cases().into_iter().filter(|c| c.asc.is_none() && c.content.len() != 7) { + let (_, units) = encode(case.content, true); + let p = write(&format!("{}.aac", case.name), &units.concat()); + check(case.name, case.content, &mix(&MediaPool::new(), &probe(&p).expect("probe"))); + } +} diff --git a/crates/media/tests/it/main.rs b/crates/media/tests/it/main.rs index 5f7fb8037..69bed4a9f 100644 --- a/crates/media/tests/it/main.rs +++ b/crates/media/tests/it/main.rs @@ -1,5 +1,6 @@ //! The crate's integration tests, one binary (each module is a former `tests/.rs`). +mod aac_layout; mod downmix; mod exif_orientation; mod footage; From 799303ad68bcaac91bc197ecc7335d371efe23d5 Mon Sep 17 00:00:00 2001 From: tylerjenningsw Date: Thu, 29 Oct 2026 18:20:09 -0400 Subject: [PATCH 09/12] Media tests: read ffmpeg 4.2's PCE fixtures with their real channel count (#1701) ffmpeg 4.2 writes 2 in the sample entry of an AAC PCE layout and FilmCraft decodes as many channels as that names, so the 6.1 check now reads a copy naming all seven instead of skipping. Configuration 12 moves to the synthetic layout tests. --- crates/media/tests/it/downmix.rs | 44 +++++++++++++------------------- 1 file changed, 18 insertions(+), 26 deletions(-) diff --git a/crates/media/tests/it/downmix.rs b/crates/media/tests/it/downmix.rs index c139676d6..05205ab6a 100644 --- a/crates/media/tests/it/downmix.rs +++ b/crates/media/tests/it/downmix.rs @@ -168,22 +168,23 @@ fn each_channel_reaches_stereo_at_its_bs775_gain() { } } -/// AAC 7.1 in channel configuration 12 (back surrounds), which FilmCraft reorders where it -/// leaves ffmpeg's configuration 7 as coded: ffmpeg's 7.1 with its configuration rewritten (both -/// code the same elements). -#[test] -fn seven_one_aac_in_configuration_12() { - let c = cases().into_iter().find(|c| c.file == "dm71.m4a").expect("the 7.1 AAC case"); - let freqs: Vec = FREQS.iter().copied().take(c.names.len()).collect(); - let Some(p) = fixture(c.file, c.layout, c.names, &freqs, c.codec) else { return }; - let mut bytes = std::fs::read(&p).expect("read"); - let (asc, _) = aac_entry(&bytes).expect("an AAC track"); - assert_eq!(asc.get(..2), Some(&[0x11, 0xB8][..]), "AAC-LC 48 kHz, configuration 7"); - let at = bytes.windows(asc.len()).position(|w| w == asc).expect("the configuration in the file"); - bytes[at + 1] = 0xE0; - let p12 = fixtures().join("dm71c12.m4a"); - std::fs::write(&p12, bytes).expect("write"); - check(&Case { file: "dm71c12.m4a", ..c }, &p12); +/// `p`, or a copy whose AAC sample entry names all `channels` of its PCE layout: ffmpeg 4.2 writes +/// 2 there for a PCE layout (8.1 the real count), and FilmCraft decodes as many channels as the +/// sample entry names, so the other five of a 6.1 would never reach the mix. +fn whole_pce_entry(p: &Path, channels: usize) -> PathBuf { + let mut bytes = std::fs::read(p).expect("read"); + let Some((asc, 2)) = aac_entry(&bytes) else { return p.to_path_buf() }; + if channels <= 2 || filmcraft_aac::AudioSpecificConfig::parse(&asc).map(|a| a.channel_config).ok() != Some(0) { + return p.to_path_buf(); + } + // the `mp4a` entry's channelcount follows its 6 reserved bytes, data reference index and + // 8 reserved bytes + let at = bytes.windows(4).position(|w| w == b"mp4a").expect("an mp4a sample entry") + 4 + 6 + 2 + 8; + bytes[at..at + 2].copy_from_slice(&(channels as u16).to_be_bytes()); + eprintln!("{}: its sample entry names 2 of {channels} channels (ffmpeg 4.2): reading a copy naming all", p.display()); + let out = p.with_extension("entry.m4a"); + std::fs::write(&out, bytes).expect("write"); + out } /// The AudioSpecificConfig and sample entry channel count of an MP4's first AAC track. @@ -196,16 +197,7 @@ fn aac_entry(bytes: &[u8]) -> Option<(Vec, u32)> { } fn check(c: &Case, p: &Path) { - let pce_as_stereo = aac_entry(&std::fs::read(p).expect("read")).is_some_and(|(asc, entry)| { - let config = filmcraft_aac::AudioSpecificConfig::parse(&asc).map(|a| a.channel_config); - config.ok() == Some(0) && entry == 2 - }); - if pce_as_stereo && c.names.len() > 2 { - // ffmpeg 4.2 writes 2 in the sample entry of an AAC PCE layout (8.1 the real count), and - // that is all FilmCraft decodes. - eprintln!("{}: FilmCraft reads 2 of its {} channels: skipping", c.file, c.names.len()); - return; - } + let p = &whole_pce_entry(p, c.names.len()); let s = ours(p); let freqs: Vec = FREQS.iter().copied().take(c.names.len()).collect(); let sine = 0.125 * 1.0; // peak of ffmpeg's sine source (0.125 = 2^-3) From 4daff9a74adc5c875273abcbc11bf4b53438b7cc Mon Sep 17 00:00:00 2001 From: tylerjenningsw Date: Thu, 29 Oct 2026 18:20:09 -0400 Subject: [PATCH 10/12] Export tests: say why the H.264 pin moved with FilmCraft 09aaf9b (#1701) --- crates/export/tests/it/movie_bytes.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/export/tests/it/movie_bytes.rs b/crates/export/tests/it/movie_bytes.rs index 7a3fba611..f72d6fbf7 100644 --- a/crates/export/tests/it/movie_bytes.rs +++ b/crates/export/tests/it/movie_bytes.rs @@ -94,6 +94,8 @@ fn export_bytes(format: OutputFormat, name: &str) -> Vec { #[test] fn h264_movie_bytes_are_pinned() { let data = export_bytes(OutputFormat::H264, "/out/pinned.mp4"); + // FilmCraft 09aaf9b's encoder lists its settings in the first frame's user-data SEI (211 + // bytes where 0.1.1's name took 50): 161 bytes more than the FilmCraft 2c5ba7a pin. assert_eq!((data.len(), fnv1a(&data)), (104_341, 17_655_680_090_264_218_914), "H.264 MP4"); } From 800b555e514e541e25aecc84585d06d2aeb62da0 Mon Sep 17 00:00:00 2001 From: tylerjenningsw Date: Thu, 29 Oct 2026 18:44:05 -0400 Subject: [PATCH 11/12] Media: read the AAC configuration of the MP4 or Matroska track FilmCraft plays, not an empty one before it (#1701) --- crates/media/src/aac.rs | 40 ++++++++++++++++++------ crates/media/tests/it/aac_layout.rs | 48 ++++++++++++++++++++++++++--- 2 files changed, 73 insertions(+), 15 deletions(-) diff --git a/crates/media/src/aac.rs b/crates/media/src/aac.rs index a2d7de37b..cd33eaf7b 100644 --- a/crates/media/src/aac.rs +++ b/crates/media/src/aac.rs @@ -1,4 +1,4 @@ -//! The AudioSpecificConfig a movie's first AAC track decodes with, from the containers that carry +//! The AudioSpecificConfig of the AAC track FilmCraft plays from a movie, from the containers that carry //! it outside the audio frames: MP4 / MOV (`esds`), Matroska (`CodecPrivate`) and LATM in MPEG //! TS / PS (its StreamMuxConfig). FilmCraft's other AAC is ADTS (in MPEG TS / PS or a raw `.aac`), //! whose channel configurations FilmCraft decodes (1 to 7) its channel count tells apart @@ -13,10 +13,11 @@ use filmcraft_media::ByteReader; /// as far as this, not through the whole file. const MPEG_HEAD: u64 = 8 << 20; -/// The configuration of the first audio track, if it is AAC: from an MP4 / MOV `header` (its -/// ftyp and moov, or the whole file), or else from `file`, read as Matroska or MPEG TS / PS. +/// The configuration of the audio track FilmCraft plays, if it is AAC: from an MP4 / MOV `header` +/// (its ftyp and moov, or the whole file) and `file`, or else from `file`, read as Matroska or +/// MPEG TS / PS. pub(crate) fn config(header: Option<&[u8]>, file: Option<&dyn ByteReader>) -> Option { - let asc = match header.and_then(mp4) { + let asc = match header.and_then(|h| mp4(h, file)) { Some(asc) => asc, None => { let file = file?; @@ -28,19 +29,29 @@ pub(crate) fn config(header: Option<&[u8]>, file: Option<&dyn ByteReader>) -> Op AudioSpecificConfig::parse(&asc).ok() } -fn mp4(bytes: &[u8]) -> Option> { - let file = filmcraft_isobmff::open(bytes).ok()?; - let track = file.tracks.iter().find(|t| t.kind == filmcraft_isobmff::TrackKind::Audio)?; +/// The track FilmCraft's MP4 source plays: the first audio track with samples, a sample +/// description and a timescale, read from the whole file as it is (a fragmented movie's samples +/// are in its fragments, not the `moov`). +fn mp4(header: &[u8], file: Option<&dyn ByteReader>) -> Option> { + let mp4 = match file { + Some(f) => filmcraft_isobmff::open(Head(f, f.len())), + None => filmcraft_isobmff::open(header), + }; + let mp4 = mp4.ok()?; + let playable = + |t: &&filmcraft_isobmff::Track| t.kind == filmcraft_isobmff::TrackKind::Audio && !t.samples.is_empty() && !t.entries.is_empty() && t.timescale > 0; + let track = mp4.tracks.iter().find(playable)?; match &track.entries.first()?.codec { filmcraft_isobmff::CodecConfig::Aac(a) => Some(a.asc.clone()), _ => None, } } +/// The track FilmCraft's Matroska source plays: the first audio track with frames, found by +/// indexing the clusters as that source does. fn matroska(file: &dyn ByteReader) -> Option> { - let opts = filmcraft_matroska::OpenOptions { index: false, ..Default::default() }; - let mkv = filmcraft_matroska::open_with(Head(file, file.len()), &opts).ok()?; - match &mkv.tracks.iter().find(|t| t.kind == filmcraft_matroska::TrackKind::Audio)?.codec { + let mkv = filmcraft_matroska::open(Head(file, file.len())).ok()?; + match &mkv.tracks.iter().find(|t| t.kind == filmcraft_matroska::TrackKind::Audio && !t.samples.is_empty())?.codec { filmcraft_matroska::Codec::Aac { asc } => Some(asc.clone()), _ => None, } @@ -66,6 +77,15 @@ impl Head<'_> { } } +impl filmcraft_isobmff::ByteSource for Head<'_> { + fn len(&self) -> u64 { + self.1 + } + fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> { + self.read(offset, buf) + } +} + impl filmcraft_matroska::ByteSource for Head<'_> { fn len(&self) -> u64 { self.1 diff --git a/crates/media/tests/it/aac_layout.rs b/crates/media/tests/it/aac_layout.rs index 9e5ddbb23..c6403667b 100644 --- a/crates/media/tests/it/aac_layout.rs +++ b/crates/media/tests/it/aac_layout.rs @@ -109,8 +109,17 @@ fn encode(content: &[Ch], adts: bool) -> (Vec, Vec>) { } fn mp4(asc: Vec, channels: u32, units: &[Vec]) -> Vec { + mp4_after(None, asc, channels, units) +} + +/// As [`mp4`], after an audio track with the configuration and channel count `empty` and no +/// samples, if any. +fn mp4_after(empty: Option<(Vec, u32)>, asc: Vec, channels: u32, units: &[Vec]) -> Vec { use filmcraft_isobmff::{Brand, Mp4Writer, SampleEntry, TrackConfig, WriteSample, WriterOptions}; let mut w = Mp4Writer::new(Cursor::new(Vec::new()), WriterOptions::new(Brand::Mp4)).expect("writer"); + if let Some((asc, channels)) = empty { + w.add_track(TrackConfig::new(SampleEntry::aac(asc, channels, RATE), RATE)).expect("empty track"); + } let t = w.add_track(TrackConfig::new(SampleEntry::aac(asc, channels, RATE), RATE)).expect("track"); for u in units { w.write_sample(t, WriteSample { data: u, duration: 1024, composition_offset: 0, is_sync: true }).expect("sample"); @@ -119,14 +128,25 @@ fn mp4(asc: Vec, channels: u32, units: &[Vec]) -> Vec { } fn mkv(asc: Vec, channels: u32, units: &[Vec]) -> Vec { + mkv_after(None, asc, channels, units) +} + +/// As [`mkv`], after an audio track with the configuration and channel count `empty` and no +/// frames, if any. +fn mkv_after(empty: Option<(Vec, u32)>, asc: Vec, channels: u32, units: &[Vec]) -> Vec { use filmcraft_matroska::{MkvWriter, MuxOptions, TrackKind, TrackSpec}; - let mut t = TrackSpec::new(TrackKind::Audio, "A_AAC"); - t.codec_private = asc; - t.audio = Some((f64::from(RATE), u64::from(channels), None)); - let mut w = MkvWriter::new(Cursor::new(Vec::new()), vec![t], MuxOptions::default()).expect("writer"); + let track = |asc: Vec, channels: u32| { + let mut t = TrackSpec::new(TrackKind::Audio, "A_AAC"); + t.codec_private = asc; + t.audio = Some((f64::from(RATE), u64::from(channels), None)); + t + }; + let tracks: Vec = empty.into_iter().map(|(asc, n)| track(asc, n)).chain([track(asc, channels)]).collect(); + let played = tracks.len() - 1; + let mut w = MkvWriter::new(Cursor::new(Vec::new()), tracks, MuxOptions::default()).expect("writer"); let frame_ns = 1_000_000_000 * 1024 / i64::from(RATE); for (i, u) in units.iter().enumerate() { - w.write_frame(0, i as i64 * frame_ns, true, u, Some(frame_ns as u64)).expect("frame"); + w.write_frame(played, i as i64 * frame_ns, true, u, Some(frame_ns as u64)).expect("frame"); } w.finish().expect("finish").into_inner() } @@ -222,3 +242,21 @@ fn adts_aac_layouts_mix_by_speaker() { check(case.name, case.content, &mix(&MediaPool::new(), &probe(&p).expect("probe"))); } } + +/// Audio plays from the first track FilmCraft can play, so an audio track with no samples before +/// it lends it no configuration. Each case decodes in another order than the empty track's +/// configuration would (FilmCraft reorders 12 to WAVE / SMPTE but not 7, and 11 but not a PCE). +#[test] +fn an_empty_audio_track_lends_its_configuration_to_no_other() { + for (name, empty) in [("config12", 7), ("pce61-centre-first", 11)] { + let case = cases().into_iter().find(|c| c.name == name).expect("case"); + let n = case.content.len() as u32; + let empty = Some((AudioSpecificConfig::lc(3, empty).to_bytes(), n)); + let (asc, units) = encode(case.content, false); + let asc = case.asc.as_ref().map_or(asc, AudioSpecificConfig::to_bytes); + let p = write(&format!("{name}-after-empty.mp4"), &mp4_after(empty.clone(), asc.clone(), n, &units)); + check(&format!("{name} MP4"), case.content, &mix(&MediaPool::new(), &probe(&p).expect("probe"))); + let p = write(&format!("{name}-after-empty.mkv"), &mkv_after(empty, asc, n, &units)); + check(&format!("{name} Matroska"), case.content, &mix(&MediaPool::new(), &probe(&p).expect("probe"))); + } +} From aecdffecb09ef33c43d3e5350cf5537bfa2067aa Mon Sep 17 00:00:00 2001 From: tylerjenningsw Date: Thu, 29 Oct 2026 18:44:34 -0400 Subject: [PATCH 12/12] Media: read the LATM configuration only when FilmCraft plays that MPEG stream, not an ADTS one before it (#1701) --- crates/media/src/aac.rs | 21 +++- crates/media/tests/it/aac_layout.rs | 160 ++++++++++++++++++++++++++++ 2 files changed, 176 insertions(+), 5 deletions(-) diff --git a/crates/media/src/aac.rs b/crates/media/src/aac.rs index cd33eaf7b..625c37b43 100644 --- a/crates/media/src/aac.rs +++ b/crates/media/src/aac.rs @@ -9,8 +9,8 @@ use std::io; use filmcraft_aac::AudioSpecificConfig; use filmcraft_media::ByteReader; -/// How far into an MPEG TS / PS file its first LATM frame is looked for: the stream is indexed -/// as far as this, not through the whole file. +/// How far into an MPEG TS / PS file the audio streams are looked for: the stream is indexed as +/// far as this, not through the whole file. const MPEG_HEAD: u64 = 8 << 20; /// The configuration of the audio track FilmCraft plays, if it is AAC: from an MP4 / MOV `header` @@ -57,12 +57,23 @@ fn matroska(file: &dyn ByteReader) -> Option> { } } +/// The configuration of the stream FilmCraft's MPEG source plays: the first audio stream whose +/// first unit it can read. Only AAC is asked about, so the streams before that one were refused +/// whatever their codec, and an ADTS stream played has no configuration outside its frames. fn latm(file: &dyn ByteReader) -> Option> { + use filmcraft_mpegts::Codec; let head = Head(file, file.len().min(MPEG_HEAD)); let mpeg = filmcraft_mpegts::open(&head).ok()?; - let stream = mpeg.find(filmcraft_mpegts::Kind::Audio, |c| *c == filmcraft_mpegts::Codec::AacLatm)?; - let frame = mpeg.read_unit(&head, stream, 0).ok()?; - Some(filmcraft_codecs::audio::latm_config(&frame)?.asc) + let audio = mpeg.streams.iter().enumerate().filter(|(_, s)| s.kind() == filmcraft_mpegts::Kind::Audio && !s.units.is_empty()); + let played = audio.take(filmcraft_media::MAX_AUDIO_STREAMS).find_map(|(i, s)| { + let first = mpeg.read_unit(&head, i, 0).ok()?; + match s.codec { + Codec::AacAdts => filmcraft_mpegts::frame_info(&s.codec, &first).map(|_| None), + Codec::AacLatm => filmcraft_codecs::audio::latm_config(&first).map(|c| Some(c.asc)), + _ => None, + } + }); + played? } /// The first `.1` bytes of a file, as the byte source FilmCraft's demuxers read. diff --git a/crates/media/tests/it/aac_layout.rs b/crates/media/tests/it/aac_layout.rs index c6403667b..fb3e078a6 100644 --- a/crates/media/tests/it/aac_layout.rs +++ b/crates/media/tests/it/aac_layout.rs @@ -260,3 +260,163 @@ fn an_empty_audio_track_lends_its_configuration_to_no_other() { check(&format!("{name} Matroska"), case.content, &mix(&MediaPool::new(), &probe(&p).expect("probe"))); } } + +/// A transport stream plays its first audio stream and mixes by that stream's configuration, +/// whatever follows it. ADTS of configuration 7 decodes in element order, LATM of configuration +/// 12 reordered to WAVE / SMPTE; LATM of configuration 7 names 7 channels, which mix as +/// configuration 7's rather than as 7 channels by count. (FilmCraft decodes LATM of configuration +/// 12 as 12 channels, so it isn't played here.) +#[test] +fn mpeg_aac_mixes_by_the_stream_played() { + let case = cases().into_iter().find(|c| c.name == "config7").expect("case"); + let (_, adts) = encode(case.content, true); + let (_, raw) = encode(case.content, false); + for (label, latm_first, latm_config) in [("ADTS before LATM", false, 12), ("LATM before ADTS", true, 7)] { + let (adts_pid, latm_pid) = if latm_first { (0x102, 0x101) } else { (0x101, 0x102) }; + let mut ts = Ts::default(); + let mut streams = [(0x0F, adts_pid), (0x11, latm_pid)]; + streams.sort_by_key(|s| s.1); + ts.psi(&streams); + for (i, (a, r)) in adts.iter().zip(&raw).enumerate() { + let pts = 1920 * i as i64; + let mut units = [(adts_pid, pes(pts, a)), (latm_pid, pes(pts, &loas(latm_config, r)))]; + units.sort_by_key(|u| u.0); + for (pid, unit) in units { + ts.pes(pid, &unit); + } + } + let p = write(&format!("latm-first-{latm_first}.ts"), &ts.out); + check(label, case.content, &mix(&MediaPool::new(), &probe(&p).expect("probe"))); + } +} + +/// A LOAS frame of `payload` with its StreamMuxConfig: AAC LC, 48 kHz, channel configuration +/// `config`. +fn loas(config: u32, payload: &[u8]) -> Vec { + let mut bits = Bits::default(); + // useSameStreamMux, audioMuxVersion, allStreamsSameTimeFraming, numSubFrames, numProgram, numLayer + bits.put(0b001, 3); + bits.put(0, 13); + // AudioSpecificConfig and GASpecificConfig + bits.put(2, 5); + bits.put(3, 4); + bits.put(config, 4); + bits.put(0, 3); + // frameLengthType, latmBufferFullness, otherDataPresent, crcCheckPresent + bits.put(0, 3); + bits.put(0xFF, 8); + bits.put(0, 2); + let mut len = payload.len(); + while len >= 255 { + bits.put(255, 8); + len -= 255; + } + bits.put(len as u32, 8); + for &b in payload { + bits.put(u32::from(b), 8); + } + let body = bits.finish(); + let mut v = vec![0x56, 0xE0 | (body.len() >> 8) as u8, body.len() as u8]; + v.extend(body); + v +} + +/// A big-endian bit writer. +#[derive(Default)] +struct Bits { + out: Vec, + acc: u64, + n: u32, +} + +impl Bits { + fn put(&mut self, v: u32, n: u32) { + self.acc = self.acc << n | u64::from(v); + self.n += n; + while self.n >= 8 { + self.n -= 8; + self.out.push((self.acc >> self.n) as u8); + } + } + fn finish(mut self) -> Vec { + if self.n > 0 { + self.put(0, 8 - self.n); + } + self.out + } +} + +/// An audio PES packet (stream id 0xC0) presented at `pts` (90 kHz). +fn pes(pts: i64, payload: &[u8]) -> Vec { + let t = [0x21 | ((pts >> 29) & 0x0E) as u8, (pts >> 22) as u8, ((pts >> 14) & 0xFE) as u8 | 1, (pts >> 7) as u8, ((pts << 1) & 0xFE) as u8 | 1]; + let len = 8 + payload.len(); + let mut v = vec![0, 0, 1, 0xC0, (len >> 8) as u8, len as u8, 0x80, 0x80, 5]; + v.extend(t); + v.extend_from_slice(payload); + v +} + +/// A transport stream of one program (its PMT on PID 0x100), PES cut into 188-byte packets. +#[derive(Default)] +struct Ts { + out: Vec, + cc: std::collections::HashMap, +} + +impl Ts { + fn packet(&mut self, pid: u16, start: bool, payload: &[u8]) -> usize { + let cc = self.cc.entry(pid).or_insert(0); + let mut p = vec![0x47, u8::from(start) << 6 | (pid >> 8) as u8, pid as u8]; + let n = payload.len().min(184); + if n < 184 { + let stuffing = 183 - n; + p.extend([0x30 | *cc, stuffing as u8]); + if stuffing > 0 { + p.push(0); + p.extend(std::iter::repeat_n(0xFF, stuffing - 1)); + } + } else { + p.push(0x10 | *cc); + } + *cc = (*cc + 1) & 15; + p.extend_from_slice(&payload[..n]); + self.out.extend(p); + n + } + fn section(&mut self, pid: u16, body: &[u8]) { + let mut s = vec![0]; + s.extend_from_slice(body); + s.extend(crc32_mpeg(body).to_be_bytes()); + self.packet(pid, true, &s); + } + /// The PAT and a PMT of `streams` (stream type, PID). + fn psi(&mut self, streams: &[(u8, u16)]) { + self.section(0, &[0x00, 0xB0, 13, 0, 1, 0xC1, 0, 0, 0, 1, 0xE1, 0x00]); + let mut pmt = vec![0x02, 0xB0, 0, 0, 1, 0xC1, 0, 0, 0xE1, 0x01, 0xF0, 0]; + for &(kind, pid) in streams { + pmt.extend([kind, 0xE0 | (pid >> 8) as u8, pid as u8, 0xF0, 0]); + } + pmt[2] = (pmt.len() + 1) as u8; + self.section(0x100, &pmt); + } + fn pes(&mut self, pid: u16, data: &[u8]) { + let (mut rest, mut start) = (data, true); + while !rest.is_empty() { + let n = self.packet(pid, start, rest); + rest = &rest[n..]; + start = false; + } + } +} + +/// CRC-32 of PSI sections (MSB first, polynomial 0x04C11DB7, initial value all ones). +fn crc32_mpeg(data: &[u8]) -> u32 { + let mut crc = u32::MAX; + for &b in data { + crc ^= u32::from(b) << 24; + for _ in 0..8 { + crc = if crc & 0x8000_0000 != 0 { crc << 1 ^ 0x04C1_1DB7 } else { crc << 1 }; + } + } + crc +}