diff --git a/crates/media/tests/it/aac_layout.rs b/crates/media/tests/it/aac_layout.rs new file mode 100644 index 000000000..9e5ddbb23 --- /dev/null +++ b/crates/media/tests/it/aac_layout.rs @@ -0,0 +1,224 @@ +//! Multichannel AAC reaches stereo by where each channel plays, whatever order its channel +//! configuration decodes in (#1701). Fixtures are tones encoded with FilmCraft's AAC encoder (no +//! ffmpeg needed, so nothing is skipped), with the configuration rewritten where the coded +//! elements fit another: configurations 11 and 12 code the same elements as the encoder's 7- and +//! 8-channel output, and a program config element can name any of them. + +use std::f64::consts::PI; +use std::io::Cursor; +use std::path::PathBuf; +use std::sync::Arc; + +use effectcraft_media::{MediaPool, probe, probe_bytes}; +use effectcraft_time::Tick; +use filmcraft_aac::{AudioSpecificConfig, Encoder, EncoderConfig, ProgramConfig}; + +const RATE: u32 = 48_000; +/// Tone frequencies by channel; an LFE gets [`LFE_TONE`], which its element can carry. +const FREQS: [u32; 8] = [300, 500, 700, 900, 1100, 1300, 1500, 1700]; +const LFE_TONE: u32 = 60; +const AMP: f32 = 0.25; +const K: f64 = std::f64::consts::FRAC_1_SQRT_2; + +/// A channel's content, by where it plays. +#[derive(Clone, Copy, Debug, PartialEq)] +enum Ch { + L, + R, + C, + Lfe, + Ls, + Rs, + Cs, +} + +impl Ch { + /// ITU-R BS.775 gains into `[left, right]`, before normalisation. + fn gains(self) -> [f64; 2] { + match self { + Ch::L => [1.0, 0.0], + Ch::R => [0.0, 1.0], + Ch::C => [K, K], + Ch::Lfe => [0.0, 0.0], + Ch::Ls => [K, 0.0], + Ch::Rs => [0.0, K], + Ch::Cs => [0.5, 0.5], + } + } +} + +struct Case { + name: &'static str, + /// What the encoder's input channels carry (its element order: C, L R, …, LFE last). + content: &'static [Ch], + /// The AudioSpecificConfig written to the container instead of the encoder's, if any. + asc: Option, +} + +fn lc(config: u8) -> Option { + Some(AudioSpecificConfig::lc(3, config)) +} + +fn pce(front: Vec<(bool, u8)>, side: Vec<(bool, u8)>, back: Vec<(bool, u8)>, lfe: Vec, comment: &[u8]) -> Option { + let pce = ProgramConfig { object_type: 1, sf_index: 3, front, side, back, lfe, comment: comment.to_vec(), ..Default::default() }; + Some(AudioSpecificConfig { pce: Some(pce), ..AudioSpecificConfig::lc(3, 0) }) +} + +use Ch::*; + +fn cases() -> Vec { + vec![ + Case { name: "config3", content: &[C, L, R], asc: None }, + Case { name: "config4", content: &[C, L, R, Cs], asc: None }, + Case { name: "config5", content: &[C, L, R, Ls, Rs], asc: None }, + Case { name: "config6", content: &[C, L, R, Ls, Rs, Lfe], asc: None }, + // configuration 7 as ffmpeg writes 7.1: the second and third pairs are surrounds + Case { name: "config7", content: &[C, L, R, Ls, Rs, Ls, Rs, Lfe], asc: None }, + Case { name: "config11", content: &[C, L, R, Ls, Rs, Cs, Lfe], asc: lc(11) }, + Case { name: "config12", content: &[C, L, R, Ls, Rs, Ls, Rs, Lfe], asc: lc(12) }, + // the encoder's own 6.1: a PCE listing the centre first + Case { name: "pce61-centre-first", content: &[C, L, R, Ls, Rs, Cs, Lfe], asc: None }, + // ffmpeg's 6.1 PCE: front pair and centre, a side SCE, a back pair and SCE, no LFE + // element, its channels in ffmpeg's order L R C LFE Cs Ls Rs. The decoder places the + // encoder's SCE 0 in the front SCE, pair 0 front, pair 1 back, SCE 1 in the back SCE (tag + // 1) and the LFE in the side SCE left over (tag 5). + Case { + name: "pce61-ffmpeg", + content: &[C, L, R, Cs, Ls, Rs, Lfe], + asc: pce(vec![(true, 0), (false, 0)], vec![(false, 5)], vec![(true, 1), (false, 1)], vec![], b"Lavc62.28.102"), + }, + ] +} + +fn freq(content: &[Ch], i: usize) -> u32 { + if content.get(i) == Some(&Lfe) { LFE_TONE } else { FREQS[i] } +} + +/// Two seconds of `content`'s tones encoded as AAC: the AudioSpecificConfig and access units. +fn encode(content: &[Ch], adts: bool) -> (Vec, Vec>) { + let n = content.len(); + let mut cfg = EncoderConfig::cbr(RATE, n, 96_000 * n as u32); + cfg.adts = adts; + let mut enc = Encoder::new(cfg).expect("encoder"); + let planes: Vec> = + (0..n).map(|c| (0..2 * RATE).map(|i| AMP * (2.0 * PI * f64::from(freq(content, c)) * f64::from(i) / f64::from(RATE)).sin() as f32).collect()).collect(); + let refs: Vec<&[f32]> = planes.iter().map(Vec::as_slice).collect(); + let mut units = enc.encode(&refs); + units.extend(enc.flush()); + (enc.audio_specific_config(), units) +} + +fn mp4(asc: Vec, channels: u32, units: &[Vec]) -> Vec { + use filmcraft_isobmff::{Brand, Mp4Writer, SampleEntry, TrackConfig, WriteSample, WriterOptions}; + let mut w = Mp4Writer::new(Cursor::new(Vec::new()), WriterOptions::new(Brand::Mp4)).expect("writer"); + let t = w.add_track(TrackConfig::new(SampleEntry::aac(asc, channels, RATE), RATE)).expect("track"); + for u in units { + w.write_sample(t, WriteSample { data: u, duration: 1024, composition_offset: 0, is_sync: true }).expect("sample"); + } + w.finish().expect("finish").into_inner() +} + +fn mkv(asc: Vec, channels: u32, units: &[Vec]) -> Vec { + use filmcraft_matroska::{MkvWriter, MuxOptions, TrackKind, TrackSpec}; + let mut t = TrackSpec::new(TrackKind::Audio, "A_AAC"); + t.codec_private = asc; + t.audio = Some((f64::from(RATE), u64::from(channels), None)); + let mut w = MkvWriter::new(Cursor::new(Vec::new()), vec![t], MuxOptions::default()).expect("writer"); + let frame_ns = 1_000_000_000 * 1024 / i64::from(RATE); + for (i, u) in units.iter().enumerate() { + w.write_frame(0, i as i64 * frame_ns, true, u, Some(frame_ns as u64)).expect("frame"); + } + w.finish().expect("finish").into_inner() +} + +/// The fixture written to disk as `name`. +fn write(name: &str, bytes: &[u8]) -> PathBuf { + let dir = std::env::temp_dir().join(format!("effectcraft-aac-layout-{}", std::process::id())); + std::fs::create_dir_all(&dir).expect("mkdir"); + let p = dir.join(name); + std::fs::write(&p, bytes).expect("write"); + p +} + +/// Amplitude of the tone at `freq` in one side of interleaved stereo. +fn amp(s: &[f32], side: usize, freq: u32) -> f64 { + let (mut re, mut im, mut n) = (0.0, 0.0, 0usize); + for (i, f) in s.as_chunks::<2>().0.iter().enumerate() { + let w = 2.0 * PI * f64::from(freq) * i as f64 / f64::from(RATE); + re += f64::from(f[side]) * w.cos(); + im += f64::from(f[side]) * w.sin(); + n += 1; + } + 2.0 * (re * re + im * im).sqrt() / n.max(1) as f64 +} + +/// Stereo of one second from 0.5 s in, through the media pool as footage reads it. +fn mix(pool: &MediaPool, footage: &effectcraft_project::Footage) -> Vec { + pool.audio_samples(footage, Tick::from_seconds_f64(0.5), RATE as usize, RATE) +} + +/// Each channel of `content` reaches each side at its BS.775 gain, normalised by the larger +/// side's sum (an LFE nowhere). +fn check(label: &str, content: &[Ch], s: &[f32]) { + let raw: Vec<[f64; 2]> = content.iter().map(|c| c.gains()).collect(); + let norm = raw.iter().map(|g| g[0]).sum::().max(raw.iter().map(|g| g[1]).sum::()).max(1.0); + for (i, (ch, g)) in content.iter().zip(&raw).enumerate() { + for side in 0..2 { + let (got, want) = (amp(s, side, freq(content, i)), f64::from(AMP) * g[side] / norm); + assert!( + (got - want).abs() <= f64::from(AMP) * 0.05, + "{label}: {ch:?} ({} Hz) -> {} is {got:.4}, expected {want:.4}", + freq(content, i), + ["L", "R"][side] + ); + } + } +} + +/// The LFE tone survives encoding, so its absence from the mix means it was left out. +fn lfe_is_coded(case: &Case, bytes: Vec) { + if !case.content.contains(&Lfe) { + return; + } + let src = filmcraft_codecs::open_bytes("lfe.mp4", bytes.into()).expect("open"); + let buf = src.audio(24_000, RATE as usize, RATE).expect("decode"); + let best = buf.channels.iter().map(|ch| amp(&ch.iter().flat_map(|&v| [v, v]).collect::>(), 0, LFE_TONE)).fold(0.0, f64::max); + assert!(best > 0.8 * f64::from(AMP), "{}: the LFE tone is coded ({best:.4})", case.name); +} + +#[test] +fn every_aac_layout_mixes_by_speaker_in_mp4() { + for case in cases() { + let (asc, units) = encode(case.content, false); + let asc = case.asc.as_ref().map_or(asc, AudioSpecificConfig::to_bytes); + let bytes = mp4(asc, case.content.len() as u32, &units); + lfe_is_coded(&case, bytes.clone()); + let p = write(&format!("{}.mp4", case.name), &bytes); + check(case.name, case.content, &mix(&MediaPool::new(), &probe(&p).expect("probe"))); + // in memory (`add_bytes`) as well as in place + let (path, bytes): (String, Arc<[u8]>) = (format!("/mem/{}.mp4", case.name), bytes.into()); + let pool = MediaPool::new(); + pool.add_bytes(&path, bytes.clone()); + check(&format!("{} in memory", case.name), case.content, &mix(&pool, &probe_bytes(&path, bytes).expect("probe"))); + } +} + +#[test] +fn matroska_aac_layouts_mix_by_speaker() { + for case in cases().into_iter().filter(|c| matches!(c.name, "config12" | "pce61-centre-first" | "pce61-ffmpeg")) { + let (asc, units) = encode(case.content, false); + let asc = case.asc.as_ref().map_or(asc, AudioSpecificConfig::to_bytes); + let p = write(&format!("{}.mkv", case.name), &mkv(asc, case.content.len() as u32, &units)); + check(case.name, case.content, &mix(&MediaPool::new(), &probe(&p).expect("probe"))); + } +} + +/// ADTS signals configurations 1 to 7 only, which the channel count tells apart. +#[test] +fn adts_aac_layouts_mix_by_speaker() { + for case in cases().into_iter().filter(|c| c.asc.is_none() && c.content.len() != 7) { + let (_, units) = encode(case.content, true); + let p = write(&format!("{}.aac", case.name), &units.concat()); + check(case.name, case.content, &mix(&MediaPool::new(), &probe(&p).expect("probe"))); + } +} diff --git a/crates/media/tests/it/main.rs b/crates/media/tests/it/main.rs index 5f7fb8037..69bed4a9f 100644 --- a/crates/media/tests/it/main.rs +++ b/crates/media/tests/it/main.rs @@ -1,5 +1,6 @@ //! The crate's integration tests, one binary (each module is a former `tests/.rs`). +mod aac_layout; mod downmix; mod exif_orientation; mod footage;