diff --git a/.github/workflows/audio.yml b/.github/workflows/audio.yml index 21692c774..a488ce61b 100644 --- a/.github/workflows/audio.yml +++ b/.github/workflows/audio.yml @@ -35,7 +35,7 @@ jobs: # Each feature on its own, so code that needs another feature fails here. - name: Clippy, each feature alone run: | - for f in "" decode decode-native symphonia encode lossless-encode pcm-encode flac-encode peaks serde; do + for f in "" decode decode-native symphonia encode lossless-encode pcm-encode flac-encode peaks aac-decode serde; do echo "::group::features: ${f:-none}" cargo clippy -p craft-audio --all-targets --locked --no-default-features ${f:+--features "$f"} -- -D warnings echo "::endgroup::" @@ -87,8 +87,8 @@ jobs: test: timeout-minutes: 20 name: Audio tests with ffmpeg oracles - # Pinned for the ffmpeg its apt ships (8.0): the oracle's files come from this ffmpeg, and - # ffmpeg 6, on ubuntu-24.04, writes MP2 and E-AC-3 in Matroska with early ends. + # Pinned for the ffmpeg its apt ships (8.0), so the files the oracle tests write do not change + # with the runner image. runs-on: ubuntu-26.04 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 diff --git a/README.md b/README.md index 2881e2947..7ba48e385 100644 --- a/README.md +++ b/README.md @@ -78,8 +78,9 @@ craft- = { git = "https://github.com/storytold/craft-libs", rev = " = packets.iter().take(craft_audio::LOOKAHEAD_PACKETS).map(Vec::as_slice).collect(); let mut dec = TrackDecoder::new(¶ms, &lookahead)?; for p in &packets { @@ -166,12 +175,19 @@ for p in &packets { } ``` -It decodes PCM so far, and returns `AudioBuffer`s in WAV order when `layout()` is ordered. -`codec()`, `sample_rate()` and `layout()` describe the output. Container timing is the caller's. +It decodes AAC (finding HE-AAC in the lookahead) and PCM, and returns `AudioBuffer`s in WAV +order when `layout()` is ordered. `codec()`, `sample_rate()` and `layout()` describe the output. Container timing is the caller's. + +Streams without codec configuration work where the packets carry it. AAC with empty `extradata` +takes its configuration from the ADTS headers (MPEG-TS, raw `.aac`), which are stripped from each +packet. To seek: position your demuxer at the packet that starts `preroll()` frames or more before the target, call `reset()`, decode from there, and drop the output before the target. +The codec modules are also usable on their own. `aac::Decoder` returns the codec's channel order; +`aac::wav_order(channel_config)` maps it to WAV order. + ### Encode a file ```rust @@ -216,7 +232,8 @@ exact, O(log n). `rms_spans` returns one RMS value per column. ### Errors -Functions return `craft_audio::Result`. +Functions return `craft_audio::Result`. The codec modules have their own `Error` types, which +convert into `AudioError` with `?`. | `AudioError` | Cause | |---|---| @@ -243,23 +260,44 @@ match craft_audio::decode(&bytes, None) { | WAV, BWF, RF64/BW64 | native | native | | AIFF, AIFF-C | native | native | | FLAC | symphonia | native | -| AAC-LC, HE-AAC | symphonia | — | +| AAC-LC | native | — | +| HE-AAC v1/v2 | native, core only (see [Limits](#limits)) | — | | MP1, MP2, MP3 (MPEG audio Layer I/II/III) | symphonia | — | | ALAC, Vorbis, ADPCM, A-law, µ-law | symphonia | — | | Opus | — | — | | Container | Demuxer | Timing | |---|---|---| -| MP4, M4A, MOV | `craft_media::isobmff` for PCM; symphonia for the others (AAC, ALAC, MP3, FLAC) | edit lists | -| Matroska, WebM | `craft_media::matroska` for PCM; symphonia for the others (AAC, Vorbis, FLAC, MP3) | `CodecDelay`, `DiscardPadding` | +| MP4, M4A, MOV | `craft_media::isobmff` for native codecs; symphonia for the others (ALAC, MP3, FLAC) | edit lists (AAC priming) | +| Matroska, WebM | `craft_media::matroska` for native codecs; symphonia for the others (Vorbis, FLAC, MP3) | `CodecDelay`, `DiscardPadding` | | Ogg | symphonia (Vorbis, FLAC) | — | -| Raw ADTS AAC, MP1/MP2/MP3 | symphonia | — | +| Raw ADTS AAC | native frame sync | — | +| Raw MP1/MP2/MP3 | symphonia | — | | FLAC | symphonia | — | | CAF | symphonia | — | ## Limits -- AIFF with more than four channels decodes with an unordered layout, in the file's order. +- HE-AAC: the AAC-LC core is decoded. `decode`, `Decoder` and `TrackDecoder` upsample it 2×; + `aac::Decoder` alone outputs the core rate. SBR is not reconstructed, so there is no content + above the core's Nyquist frequency. +- HE-AAC v2: parametric stereo is not reconstructed either, so the mono core decodes as mono. +- AAC: Main, SSR and LTP profiles, coupling channels and 960-sample frames are unsupported. +- Raw ADTS: frames with several raw data blocks are unsupported: a stream of them is refused, and + one among other frames decodes as silence of its length and counts as damaged, as does a frame + that changes the stream's sample rate, profile or channel configuration. +- A seek into AAC with PNS gives the same samples each time, but its noise differs from a continuous + decode's, at the same band energy. +- MP4 gapless information outside an edit list is not read: a file from Apple's encoders with + `iTunSMPB` and no `elst` decodes with its priming and padding. +- AAC channel configuration 7 and program-config layouts, and AIFF with more than four channels, + decode with an unordered layout, in the codec's order. +- A Matroska track with no block durations (AAC from ffmpeg 6) ends at its `DURATION` tag, when + that ends inside the last packet. Otherwise all its audio is presented, including any padding. +- ffmpeg before 9 ends some codecs' tracks in MP4 and Matroska up to 11 ms early. The audio past + the end a file declares is not presented. +- A Matroska track's end comes from its timestamps, so it is known to one timestamp tick (usually + 1 ms) unless the last block has `DiscardPadding`. - A FLAC layout that decoders would not assume for the channel count depends on its `WAVEFORMATEXTENSIBLE_CHANNEL_MASK` comment. This includes the usual layouts for 5 and 6 channels: without the comment, back surrounds are read as side surrounds. It includes the usual 7-channel @@ -271,17 +309,41 @@ match craft_audio::decode(&bytes, None) { - `ChannelLayout::unordered` is written to FLAC as mask `0x0000`, as RFC 9639 §8.6.2 allows, but ffmpeg and `flac -d` ignore that mask and use their default layout for the channel count. - Not decoded at all: AC-3, E-AC-3, Opus and Speex. -- `TrackDecoder` returns `Unsupported` for every codec but PCM; `decode` and `Decoder` read the - other codecs in the table above through symphonia. -- MP4 edit lists: a single leading media edit (encoder priming) is applied. Empty edits and lists - of more than one edit are not; the media plays as it is stored. +- `TrackDecoder` returns `Unsupported` for MP3, FLAC, ALAC and Vorbis; `decode` and `Decoder` read + those through symphonia. +- MP4 edit lists: a single media edit (encoder priming and the end) is applied, alone or after one + empty edit; the empty edit's delay is not added as silence. Other edit lists are not applied, and + the media plays as it is stored. +- AAC in MP4 stored as ADTS frames, when the sample durations count fewer frames than the samples + hold: HE-AAC durations that count only the core frames, or several frames in a sample timed as + one. Such a file is broken: it is outside the MP4 specification in two ways. + - An MP4 AAC sample is one raw access unit, with the stream's configuration in the sample + description (ISO/IEC 14496-14 and 14496-3). ADTS is a transport framing for streams such as + MPEG-TS and raw `.aac`, and its headers do not belong in an MP4 sample. + - A sample's duration in the time-to-sample table is the time it decodes to (ISO/IEC 14496-12). + Durations that count fewer frames than the samples hold do not describe the track. + + ffmpeg, Apple's encoders, phones and cameras write raw AAC with matching durations, and a + correct remux from MPEG-TS or raw `.aac` converts the framing, so files within the specification + are not affected. In a file outside it: + - Read from the start, the track decodes in full: durations that fall short of the frames in + the samples read at open do not end it. + - `Decoder::seek` places samples by those durations, so it lands later than asked, and an edit + list measured in them can end the track early. + - These are not corrected because where each sample's audio starts is only known by counting + the ADTS frames in every sample before it: one read per sample, hundreds of thousands on a + two-hour track. Done at open, that would slow every ADTS-framed MP4 to work around files + that are outside the specification. ## Codecs +Each codec module's rustdoc lists the coding tools it implements. + | Codec | Spec | Scope | Verified against | |---|---|---|---| | WAV, AIFF | Microsoft RIFF, Apple AIFF/AIFF-C | read: integer 8–32-bit, float 32/64-bit, `WAVE_FORMAT_EXTENSIBLE` masks, RF64/BW64, BWF `bext`, truncated files; write: 16/24/32-bit integer and 32-bit float, RF64 above 4 GiB, any layout, TPDF dither | round trips; hostile-input tests | | FLAC encoder | RFC 9639 | 16, 24, 32 bits; 1–8 channels, layouts decoders would not assume for the count as a `WAVEFORMATEXTENSIBLE_CHANNEL_MASK` comment; up to 655,350 Hz; LPC, stereo decorrelation, wasted bits, STREAMINFO MD5 | bit-exact through ffmpeg, STREAMINFO MD5 checked against ffmpeg | +| AAC (`aac`) | ISO/IEC 14496-3 | AAC-LC decode, 1–8 channels; HE-AAC: see [Limits](#limits) | ffmpeg: ~135 dB SNR; PNS noise differs per sample at equal band energy | ## symphonia @@ -297,6 +359,8 @@ Notes while symphonia is in use: streams through `Decoder`. - symphonia allocates sizes taken from the file. Container size fields of WAV, FLAC, MP4 and CAF files are checked against the file length before symphonia reads them. +- AAC in CAF: the priming is not trimmed and `Decoder::seek` is not sample-exact, as CAF goes + through symphonia. - A panic inside symphonia is caught and returned as `AudioError::Malformed` only where panics unwind: native builds with the default `panic = "unwind"`. On `wasm32-unknown-unknown`, and in an app built with `panic = "abort"`, such a panic aborts. The native decoders do not panic. @@ -333,7 +397,7 @@ the full tests with ffmpeg required. | Part | Source | |---|---| | WAV, AIFF, `decode`/`probe`, peaks | SoundCraft `soundcraft-audio-io` [7dc56a4](https://github.com/storytold/soundcraft/commit/7dc56a4dbeebf57eacece09536726c57844fc59d) | -| FLAC encoder | FilmCraft [8fda2f2](https://github.com/storytold/filmcraft/commit/8fda2f232198c4c79ff9cfe5b6a347bb1375fa9e) | +| FLAC encoder, AAC | FilmCraft [8fda2f2](https://github.com/storytold/filmcraft/commit/8fda2f232198c4c79ff9cfe5b6a347bb1375fa9e) | Where apps had overlapping implementations, the one that measured better was kept, with the tests of both: @@ -342,12 +406,16 @@ tests of both: |---|---|---| | FLAC encoding | FilmCraft | 0.1–37% smaller output than SoundCraft's on 12 test signals; STREAMINFO MD5 | | WAV/AIFF writing | SoundCraft | only one with BWF, dither, arbitrary speaker masks, streaming without known length | +| AAC-LC decoding | FilmCraft (over symphonia) | equal SNR against ffmpeg; also decodes 5.1 AAC, which symphonia rejects | ## Migration notes for apps For the apps switching from their own code; to be removed once all three have switched. -What an app adjusts: +The codec modules' public items keep their names, so apps can re-export them +(`pub use craft_audio::aac::*;`). Names kept only for that are `#[doc(hidden)]`: +`aac::Upsample2x`, `aac::ProgramConfig::parse`, `aac::config_layout` and `aac::sample_rate_index`. +What else an app adjusts: - `AudioInfo` has `container`, `codec`, `layout` and `damaged` instead of SoundCraft's `format`. `detect_container` replaces `detect_format`. `probe` reports the layout instead of @@ -361,16 +429,18 @@ What an app adjusts: channels, frames)`, `AudioBuffer::new(sample_rate, channels, frames)`, `AudioBuffer::from_interleaved` and `num_channels`, and the `ChannelLayout` constructors and `channels()`. -- `AudioInfo`, `AudioError`, `EncodeOptions`, `BwfInfo`, `SampleFormat` and `BitDepth` are - `#[non_exhaustive]`. +- `AudioInfo`, `AudioError`, `EncodeOptions`, `BwfInfo`, `SampleFormat`, `BitDepth`, the codec + `Error` enums and the codec modules' header, frame and config structs (but for + `aac::ProgramConfig`) are `#[non_exhaustive]`. `Speaker::ALL` is a slice. - `TrackInfo::sample_rate` and `channels` are `Option`s. -- Features are per format. SoundCraft's `encode` (WAV, AIFF, FLAC) is now `lossless-encode`, made +- Features are per codec. SoundCraft's `encode` (WAV, AIFF, FLAC) is now `lossless-encode`, made of `pcm-encode` (WAV, AIFF) and `flac-encode`. `encode` turns on every encoder. Writing a format whose feature is off returns `AudioError::Unsupported`. - `FileEncoder::write` takes any per-channel slices, but refuses a block with missing channels or short slices instead of filling them with silence. - `DecodeOptions` is not `Copy`. - The FLAC encoder's vendor string is `craft-audio `. -- `decode` tries the native path first (WAV, AIFF, and PCM in MP4 and Matroska), which decodes - MP4 and Matroska files to presentation length. A failed source read is `AudioError::Io`. +- `decode` tries native decoders first. AAC no longer goes through symphonia, except in CAF. + MP4 and Matroska files decode to presentation length. A packet that fails is concealed, not + dropped. A failed source read is `AudioError::Io`. diff --git a/crates/audio/examples/packets.rs b/crates/audio/examples/packets.rs index 03fdd5884..c01ecbfa3 100644 --- a/crates/audio/examples/packets.rs +++ b/crates/audio/examples/packets.rs @@ -1,7 +1,7 @@ //! Demux an MP4 with craft-media and decode its audio packets with `TrackDecoder`: what an app //! with its own container code (or a network stream) does instead of `decode`. //! -//! `cargo run -p craft-audio --example packets --features decode -- clip.mov` +//! `cargo run -p craft-audio --example packets --features decode -- clip.m4a` use craft_audio::TrackDecoder; use craft_media::isobmff; @@ -17,7 +17,7 @@ fn main() -> Result<(), Box> { let params = t.codec_params(); let packets: Vec> = (0..t.samples.len()).map(|i| file.read_sample(bytes.as_slice(), track, i)).collect::>()?; - // Codecs that find their configuration in the stream read it from the first packets. + // The first packets reveal HE-AAC that the configuration does not announce. let lookahead: Vec<&[u8]> = packets.iter().take(craft_audio::LOOKAHEAD_PACKETS).map(Vec::as_slice).collect(); let mut dec = TrackDecoder::new(¶ms, &lookahead)?; println!("{} {} Hz {}", dec.codec().name(), dec.sample_rate(), dec.layout()); diff --git a/crates/audio/src/aac/config.rs b/crates/audio/src/aac/config.rs new file mode 100644 index 000000000..eb7ed7e2f --- /dev/null +++ b/crates/audio/src/aac/config.rs @@ -0,0 +1,635 @@ +//! `AudioSpecificConfig` (ISO/IEC 14496-3 §1.6.2.1), `program_config_element` (§4.4.1.1) and ADTS +//! framing (§1.A.2.2). + +use craft_media::bitstream::{BitReader, BitWriter}; + +use crate::aac::tables::{SAMPLE_RATES, table_index_for_rate}; +use crate::aac::{Error, Result}; + +/// Syntactic element types (`id_syn_ele`). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ElementType { + /// single_channel_element + Sce, + /// channel_pair_element + Cpe, + /// lfe_channel_element + Lfe, +} + +impl ElementType { + /// Channels the element carries: 2 for a CPE, 1 otherwise. + pub fn channels(self) -> usize { + if self == ElementType::Cpe { 2 } else { 1 } + } +} + +/// A `program_config_element` (only the parts that affect channel mapping are interpreted). +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct ProgramConfig { + /// `element_instance_tag` (4 bits). + pub element_instance_tag: u8, + /// `object_type` (2 bits): the profile, audio object type minus 1. + pub object_type: u8, + /// `sampling_frequency_index` (4 bits). + pub sf_index: u8, + /// (is_cpe, tag) for each front channel element. + pub front: Vec<(bool, u8)>, + /// (is_cpe, tag) for side channel elements. + pub side: Vec<(bool, u8)>, + /// (is_cpe, tag) for back channel elements. + pub back: Vec<(bool, u8)>, + /// Tags of the LFE channel elements. + pub lfe: Vec, + /// Tags of the associated data elements. + pub assoc_data: Vec, + /// (is_ind_sw, tag) + pub cc: Vec<(bool, u8)>, + /// Comment field bytes (at most 255 are written). + pub comment: Vec, +} + +impl ProgramConfig { + // Public for FilmCraft's re-export only; it takes craft-media's bit reader. + #[doc(hidden)] + pub fn parse(br: &mut BitReader) -> Result { + let mut p = ProgramConfig { + element_instance_tag: br.read_bits(4)? as u8, + object_type: br.read_bits(2)? as u8, + sf_index: br.read_bits(4)? as u8, + ..Default::default() + }; + let nf = br.read_bits(4)?; + let ns = br.read_bits(4)?; + let nb = br.read_bits(4)?; + let nl = br.read_bits(2)?; + let na = br.read_bits(3)?; + let nc = br.read_bits(4)?; + if br.read_bits(1)? == 1 { + br.skip(4)?; + } + if br.read_bits(1)? == 1 { + br.skip(4)?; + } + if br.read_bits(1)? == 1 { + br.skip(3)?; + } + for (n, v) in [(nf, &mut p.front), (ns, &mut p.side), (nb, &mut p.back)] { + for _ in 0..n { + let cpe = br.read_bits(1)? == 1; + v.push((cpe, br.read_bits(4)? as u8)); + } + } + for _ in 0..nl { + p.lfe.push(br.read_bits(4)? as u8); + } + for _ in 0..na { + p.assoc_data.push(br.read_bits(4)? as u8); + } + for _ in 0..nc { + let ind = br.read_bits(1)? == 1; + p.cc.push((ind, br.read_bits(4)? as u8)); + } + br.byte_align(); + let len = br.read_bits(8)?; + for _ in 0..len { + p.comment.push(br.read_bits(8)? as u8); + } + Ok(p) + } + + /// Serialise the element (no mixdown signalling; comment truncated to 255 bytes). More elements + /// than its count fields hold are refused, with nothing written. + pub fn write(&self, bw: &mut BitWriter) -> Result<()> { + if [self.front.len(), self.side.len(), self.back.len(), self.cc.len()].iter().any(|&n| n > 15) + || self.lfe.len() > 3 + || self.assoc_data.len() > 7 + { + return Err(Error::InvalidConfig("more elements than a program config element can list")); + } + bw.write_bits(self.element_instance_tag as u32, 4); + bw.write_bits(self.object_type as u32, 2); + bw.write_bits(self.sf_index as u32, 4); + bw.write_bits(self.front.len() as u32, 4); + bw.write_bits(self.side.len() as u32, 4); + bw.write_bits(self.back.len() as u32, 4); + bw.write_bits(self.lfe.len() as u32, 2); + bw.write_bits(self.assoc_data.len() as u32, 3); + bw.write_bits(self.cc.len() as u32, 4); + bw.write_bits(0, 3); // no mixdowns + for v in [&self.front, &self.side, &self.back] { + for &(cpe, tag) in v { + bw.write_bits(cpe as u32, 1); + bw.write_bits(tag as u32, 4); + } + } + for &t in &self.lfe { + bw.write_bits(t as u32, 4); + } + for &t in &self.assoc_data { + bw.write_bits(t as u32, 4); + } + for &(ind, t) in &self.cc { + bw.write_bits(ind as u32, 1); + bw.write_bits(t as u32, 4); + } + bw.align_zero(); + bw.write_bits(self.comment.len().min(255) as u32, 8); + for &b in self.comment.iter().take(255) { + bw.write_bits(b as u32, 8); + } + Ok(()) + } + + /// Elements in output channel order (front, side, back, LFE). + pub fn layout(&self) -> Vec<(ElementType, u8)> { + let mut out = Vec::new(); + for v in [&self.front, &self.side, &self.back] { + for &(cpe, tag) in v { + out.push((if cpe { ElementType::Cpe } else { ElementType::Sce }, tag)); + } + } + for &t in &self.lfe { + out.push((ElementType::Lfe, t)); + } + out + } +} + +/// Element layout of `channelConfiguration` 1..=7, 11 and 12 (tags as conventionally assigned; 11 and +/// 12 from ISO/IEC 14496-3 Amd 4: 6.1 with a back centre, 7.1 with back surrounds). +// Public for FilmCraft's re-export only. +#[doc(hidden)] +pub fn config_layout(channel_config: u8) -> Option> { + use ElementType::*; + Some(match channel_config { + 1 => vec![(Sce, 0)], + 2 => vec![(Cpe, 0)], + 3 => vec![(Sce, 0), (Cpe, 0)], + 4 => vec![(Sce, 0), (Cpe, 0), (Sce, 1)], + 5 => vec![(Sce, 0), (Cpe, 0), (Cpe, 1)], + 6 => vec![(Sce, 0), (Cpe, 0), (Cpe, 1), (Lfe, 0)], + 7 | 12 => vec![(Sce, 0), (Cpe, 0), (Cpe, 1), (Cpe, 2), (Lfe, 0)], + 11 => vec![(Sce, 0), (Cpe, 0), (Cpe, 1), (Sce, 1), (Lfe, 0)], + _ => return None, + }) +} + +/// Parsed `AudioSpecificConfig`. +#[non_exhaustive] +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AudioSpecificConfig { + /// Audio object type of the core (2 = AAC LC). + pub object_type: u8, + /// Core sampling rate. + pub sample_rate: u32, + /// Index into the scalefactor-band tables (the signalled index, or derived from an explicit rate). + pub sf_index: u8, + /// `channelConfiguration` (0 = channel layout from the program config element). + pub channel_config: u8, + /// `frameLengthFlag`: 960-sample frames when set (refused by [`Self::parse`]). + pub frame_length_flag: bool, + /// Explicitly signalled SBR/PS (HE-AAC v1/v2); only the AAC-LC core is decoded. + pub sbr: bool, + /// Explicitly signalled PS (audio object type 29, HE-AAC v2). + pub ps: bool, + /// Output (SBR) rate when `sbr` is set. + pub extension_sample_rate: Option, + /// The program config element, present when `channel_config` is 0. + pub pce: Option, +} + +fn read_aot(br: &mut BitReader) -> Result { + let a = br.read_bits(5)? as u8; + Ok(if a == 31 { 32 + br.read_bits(6)? as u8 } else { a }) +} + +fn read_rate(br: &mut BitReader) -> Result<(u8, u32)> { + let i = br.read_bits(4)? as u8; + if i == 15 { + let r = br.read_bits(24)?; + if r == 0 { + return Err(Error::InvalidConfig("sampling rate of 0")); + } + return Ok((table_index_for_rate(r), r)); + } + let r = *SAMPLE_RATES.get(i as usize).ok_or(Error::InvalidConfig("reserved sampling frequency index"))?; + Ok((i, r)) +} + +/// Backward-compatible explicit SBR signalling (ISO/IEC 14496-3 §1.6.6.2): after an AAC-LC +/// config, a sync extension `0x2b7` with extension object type 5 names SBR and its output rate, +/// and `0x548` after it names PS. Returns `(sbr, extension rate, ps)`, or `None` when the bits +/// that follow are not that extension (they are then ignored, as decoders that don't know it do). +fn sync_extension(br: &mut BitReader) -> Option<(bool, Option, bool)> { + if br.bits_left() < 16 || br.read_bits(11).ok()? != 0x2b7 || read_aot(br).ok()? != 5 { + return None; + } + if br.read_bits(1).ok()? != 1 { + return Some((false, None, false)); + } + let (_, rate) = read_rate(br).ok()?; + let ps = br.bits_left() >= 12 && br.read_bits(11).ok()? == 0x548 && br.read_bits(1).ok()? == 1; + Some((true, Some(rate), ps)) +} + +impl AudioSpecificConfig { + /// A plain AAC-LC configuration. A sampling frequency index with no rate (13 and above) gives a + /// sample rate of 0, which [`crate::aac::Decoder`] refuses. + pub fn lc(sf_index: u8, channel_config: u8) -> AudioSpecificConfig { + AudioSpecificConfig { + object_type: 2, + sample_rate: SAMPLE_RATES.get(usize::from(sf_index)).copied().unwrap_or(0), + sf_index, + channel_config, + frame_length_flag: false, + sbr: false, + ps: false, + extension_sample_rate: None, + pce: None, + } + } + + /// Parse an `AudioSpecificConfig` (§1.6.2.1). Only AAC-LC cores are accepted; explicit SBR/PS + /// signalling is recorded, both hierarchical (audio object types 5 and 29) and backward + /// compatible (a `0x2b7` sync extension after the config). + pub fn parse(data: &[u8]) -> Result { + let mut br = BitReader::new(data); + let mut aot = read_aot(&mut br)?; + let (sf_index, sample_rate) = read_rate(&mut br)?; + let channel_config = br.read_bits(4)? as u8; + let (mut sbr, mut ps, mut extension_sample_rate) = (false, false, None); + if aot == 5 || aot == 29 { + sbr = true; + ps = aot == 29; + let (_, ext) = read_rate(&mut br)?; + extension_sample_rate = Some(ext); + aot = read_aot(&mut br)?; + } + if aot != 2 { + return Err(Error::Unsupported(match aot { + 1 => "AAC Main profile", + 3 => "AAC SSR profile", + 4 => "AAC LTP profile", + 23 => "AAC-LD", + 39 => "AAC-ELD", + _ => "audio object type other than AAC LC", + })); + } + let frame_length_flag = br.read_bits(1)? == 1; + if br.read_bits(1)? == 1 { + br.skip(14)?; // coreCoderDelay + } + let extension_flag = br.read_bits(1)? == 1; + let pce = if channel_config == 0 { Some(ProgramConfig::parse(&mut br)?) } else { None }; + // GASpecificConfig ends with extensionFlag3 when extensionFlag is set (AAC-LC has no other + // extension fields); what follows it is the sync extension. + if extension_flag && br.bits_left() > 0 { + br.skip(1)?; + } + if frame_length_flag { + return Err(Error::Unsupported("960-sample frames")); + } + if !sbr && let Some((s, rate, p)) = sync_extension(&mut br) { + (sbr, extension_sample_rate, ps) = (s, rate, p); + } + let cfg = + AudioSpecificConfig { object_type: aot, sample_rate, sf_index, channel_config, frame_length_flag, sbr, ps, extension_sample_rate, pce }; + if cfg.channels() == 0 { + return Err(Error::InvalidConfig("no channels")); + } + Ok(cfg) + } + + /// Serialise (AAC-LC, no SBR signalling). Channel configuration 0 needs its program config + /// element, and the sampling rate must not be 0: otherwise the bytes would not parse back. + pub fn to_bytes(&self) -> Result> { + if self.channel_config == 0 && self.pce.is_none() { + return Err(Error::InvalidConfig("channel configuration 0 without a program config element")); + } + if self.sample_rate == 0 { + return Err(Error::InvalidConfig("a sampling rate of 0")); + } + let mut bw = BitWriter::new(); + bw.write_bits(self.object_type as u32, 5); + match crate::aac::tables::sample_rate_index(self.sample_rate) { + Some(i) => bw.write_bits(i as u32, 4), + None => { + bw.write_bits(15, 4); + bw.write_bits(self.sample_rate, 24); + } + } + bw.write_bits(self.channel_config as u32, 4); + bw.write_bits(0, 3); // frameLengthFlag, dependsOnCoreCoder, extensionFlag + if self.channel_config == 0 + && let Some(p) = &self.pce + { + p.write(&mut bw)?; + } + Ok(bw.finish()) + } + + /// Elements in output channel order. + pub fn layout(&self) -> Vec<(ElementType, u8)> { + match &self.pce { + Some(p) if self.channel_config == 0 => p.layout(), + _ => config_layout(self.channel_config).unwrap_or_default(), + } + } + + /// Total output channels of [`Self::layout`]. + pub fn channels(&self) -> usize { + self.layout().iter().map(|e| e.0.channels()).sum() + } +} + +/// A parsed ADTS header. +#[non_exhaustive] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct AdtsHeader { + /// Audio object type (`profile + 1`). + pub object_type: u8, + /// `sampling_frequency_index` (4 bits). + pub sf_index: u8, + /// `channel_configuration` (3 bits; 0 = program config element in the stream). + pub channel_config: u8, + /// Whole frame length including the header. + pub frame_length: usize, + /// `adts_buffer_fullness` (11 bits; 0x7FF signals variable bitrate). + pub buffer_fullness: u16, + /// Header length (7, or 9 with CRC). + pub header_length: usize, + /// Raw data blocks in the frame (`number_of_raw_data_blocks_in_frame` + 1, so 1..=4). + pub raw_data_blocks: u8, +} + +impl AdtsHeader { + /// Parse the ADTS fixed and variable header at the start of `data` (§1.A.2.2). + pub fn parse(data: &[u8]) -> Result { + let mut br = BitReader::new(data); + if br.read_bits(12)? != 0xFFF { + return Err(Error::Bitstream("missing ADTS syncword")); + } + br.skip(1)?; // ID + if br.read_bits(2)? != 0 { + return Err(Error::Bitstream("ADTS layer must be 0")); + } + let protection_absent = br.read_bits(1)? == 1; + let object_type = br.read_bits(2)? as u8 + 1; + let sf_index = br.read_bits(4)? as u8; + if sf_index as usize >= SAMPLE_RATES.len() { + return Err(Error::Bitstream("invalid ADTS sampling frequency index")); + } + br.skip(1)?; + let channel_config = br.read_bits(3)? as u8; + br.skip(4)?; + let frame_length = br.read_bits(13)? as usize; + let buffer_fullness = br.read_bits(11)? as u16; + let raw_data_blocks = br.read_bits(2)? as u8 + 1; + let header_length = if protection_absent { 7 } else { 9 }; + if frame_length < header_length { + return Err(Error::Bitstream("ADTS frame shorter than its header")); + } + Ok(AdtsHeader { object_type, sf_index, channel_config, frame_length, buffer_fullness, header_length, raw_data_blocks }) + } + + /// The equivalent `AudioSpecificConfig`. With channel configuration 0 it has no layout: the + /// decoder takes that from the program config element in the stream. + pub fn to_asc(&self) -> AudioSpecificConfig { + AudioSpecificConfig { object_type: self.object_type, ..AudioSpecificConfig::lc(self.sf_index, self.channel_config) } + } + + /// Write a 7-byte header (no CRC) for a frame whose raw payload is `payload_len` bytes. What the + /// header's fields cannot hold is refused: a frame too long for its 13-bit length, an object + /// type outside its 2-bit profile (1–4), a sampling frequency index with no rate, and channel + /// configurations above 7 (11 and 12 need a program config element in ADTS). + pub fn write(object_type: u8, sf_index: u8, channel_config: u8, payload_len: usize, buffer_fullness: u16) -> Result<[u8; 7]> { + if payload_len > 0x1FFF - 7 { + return Err(Error::InvalidConfig("an ADTS frame longer than its 13-bit length")); + } + if !(1..=4).contains(&object_type) { + return Err(Error::InvalidConfig("an ADTS profile holds audio object types 1 to 4")); + } + if usize::from(sf_index) >= SAMPLE_RATES.len() { + return Err(Error::InvalidConfig("a sampling frequency index with no rate")); + } + if channel_config > 7 { + return Err(Error::InvalidConfig("an ADTS channel configuration above 7")); + } + let len = (payload_len + 7) as u32; + let mut bw = BitWriter::new(); + bw.write_bits(0xFFF, 12); + bw.write_bits(0, 1); // MPEG-4 + bw.write_bits(0, 2); + bw.write_bits(1, 1); // protection absent + bw.write_bits(u32::from(object_type - 1), 2); + bw.write_bits(sf_index as u32, 4); + bw.write_bits(0, 1); + bw.write_bits(u32::from(channel_config), 3); + bw.write_bits(0, 4); + bw.write_bits(len & 0x1FFF, 13); + bw.write_bits(buffer_fullness as u32 & 0x7FF, 11); + bw.write_bits(0, 2); + let v = bw.finish(); + let mut out = [0u8; 7]; + out.copy_from_slice(&v); + Ok(out) + } +} + +/// Split an ADTS stream into `(header, raw payload)` frames. Stops at the first malformed or +/// truncated frame (returning what was parsed so far); errors only if nothing parses. +pub fn split_adts(data: &[u8]) -> Result> { + split_adts_upto(data, usize::MAX) +} + +/// [`split_adts`], stopping after `max` frames. +pub(crate) fn split_adts_upto(mut data: &[u8], max: usize) -> Result> { + let mut out = Vec::new(); + while data.len() >= 7 && out.len() < max { + let h = match AdtsHeader::parse(data) { + Ok(h) => h, + Err(e) => { + if out.is_empty() { + return Err(e); + } + break; + } + }; + if h.frame_length > data.len() { + break; + } + out.push((h, &data[h.header_length..h.frame_length])); + data = &data[h.frame_length..]; + } + if out.is_empty() { + return Err(Error::Bitstream("no ADTS frames")); + } + Ok(out) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Backward-compatible signalling, as craft-media's Matroska demuxer and some MP4 muxers + /// write it: an AAC-LC config, then `0x2b7`, object type 5 and the output rate. + #[test] + fn a_sync_extension_after_the_config_signals_sbr_and_ps() { + // AAC-LC, 22.05 kHz, stereo; 0x2b7, AOT 5, SBR, 44.1 kHz + let asc = AudioSpecificConfig::parse(&[0x13, 0x90, 0x56, 0xE5, 0xA0]).unwrap(); + assert_eq!((asc.object_type, asc.sample_rate, asc.sbr, asc.extension_sample_rate, asc.ps), (2, 22_050, true, Some(44_100), false)); + // ... then 0x548 and PS + let mut bw = BitWriter::new(); + for (value, bits) in [(2, 5), (7, 4), (1, 4), (0, 3), (0x2b7, 11), (5, 5), (1, 1), (4, 4), (0x548, 11), (1, 1)] { + bw.write_bits(value, bits); + } + let asc = AudioSpecificConfig::parse(&bw.finish()).unwrap(); + assert_eq!((asc.sbr, asc.extension_sample_rate, asc.ps), (true, Some(44_100), true)); + // with extensionFlag set, extensionFlag3 comes first + let mut bw = BitWriter::new(); + for (value, bits) in [(2, 5), (7, 4), (1, 4), (0b001, 3), (0, 1), (0x2b7, 11), (5, 5), (1, 1), (4, 4)] { + bw.write_bits(value, bits); + } + let asc = AudioSpecificConfig::parse(&bw.finish()).unwrap(); + assert_eq!((asc.sbr, asc.extension_sample_rate), (true, Some(44_100))); + // SBR flagged off, a different extension, or too few bits: plain AAC-LC + for bytes in [&[0x13, 0x90, 0x56, 0xE5, 0x00][..], &[0x13, 0x90, 0x12, 0x34], &[0x13, 0x90, 0x56]] { + let asc = AudioSpecificConfig::parse(bytes).unwrap(); + assert_eq!((asc.sbr, asc.extension_sample_rate), (false, None), "{bytes:02x?}"); + } + } + + #[test] + fn an_index_with_no_rate_does_not_panic_and_is_refused() { + for sf_index in 13..=15 { + let asc = AudioSpecificConfig::lc(sf_index, 2); + assert_eq!(asc.sample_rate, 0); + assert!(crate::aac::Decoder::from_config(asc).is_err()); + } + } + + #[test] + fn configs_that_would_not_parse_back_are_not_written() { + assert!(AudioSpecificConfig::lc(4, 0).to_bytes().is_err()); + assert!(AudioSpecificConfig::lc(13, 2).to_bytes().is_err(), "a sampling rate of 0"); + let pce = ProgramConfig { front: vec![(true, 0)], ..Default::default() }; + let asc = AudioSpecificConfig { pce: Some(pce), ..AudioSpecificConfig::lc(4, 0) }; + assert_eq!(AudioSpecificConfig::parse(&asc.to_bytes().unwrap()).unwrap(), asc); + } + + #[test] + fn adts_headers_refuse_fields_they_cannot_hold() { + for (object_type, sf_index, channel_config) in [(0, 4, 2), (5, 4, 2), (29, 4, 2), (2, 13, 2), (2, 15, 2), (2, 4, 8), (2, 4, 11), (2, 4, 12)] { + assert!(AdtsHeader::write(object_type, sf_index, channel_config, 100, 0x7FF).is_err(), "{object_type} {sf_index} {channel_config}"); + } + for object_type in 1..=4 { + let h = AdtsHeader::parse(&AdtsHeader::write(object_type, 12, 7, 100, 0x7FF).unwrap()).unwrap(); + assert_eq!((h.object_type, h.sf_index, h.channel_config, h.frame_length), (object_type, 12, 7, 107)); + } + } + + #[test] + fn program_configs_refuse_more_elements_than_their_counts_hold() { + let fits = ProgramConfig { front: vec![(false, 0); 15], lfe: vec![0; 3], assoc_data: vec![0; 7], ..Default::default() }; + assert!(fits.write(&mut BitWriter::new()).is_ok()); + for pce in [ + ProgramConfig { front: vec![(false, 0); 16], ..Default::default() }, + ProgramConfig { side: vec![(false, 0); 16], ..Default::default() }, + ProgramConfig { back: vec![(false, 0); 16], ..Default::default() }, + ProgramConfig { lfe: vec![0; 4], ..Default::default() }, + ProgramConfig { assoc_data: vec![0; 8], ..Default::default() }, + ProgramConfig { cc: vec![(false, 0); 16], ..Default::default() }, + ] { + assert!(pce.write(&mut BitWriter::new()).is_err(), "{pce:?}"); + let asc = AudioSpecificConfig { pce: Some(pce), ..AudioSpecificConfig::lc(4, 0) }; + assert!(asc.to_bytes().is_err()); + } + } + + #[test] + fn adts_headers_refuse_frames_too_long_for_their_length() { + let longest = AdtsHeader::write(2, 4, 2, 0x1FFF - 7, 0x7FF).unwrap(); + assert_eq!(AdtsHeader::parse(&longest).unwrap().frame_length, 0x1FFF); + for payload in [0x1FFF - 6, 0x1FFF, usize::MAX] { + assert!(AdtsHeader::write(2, 4, 2, payload, 0x7FF).is_err(), "{payload}"); + } + } + + /// An explicit sampling rate of 0 is not a rate. + #[test] + fn a_zero_explicit_rate_is_refused() { + let mut bw = BitWriter::new(); + bw.write_bits(2, 5); + bw.write_bits(15, 4); + bw.write_bits(0, 24); + bw.write_bits(2, 4); + bw.write_bits(0, 3); + assert!(AudioSpecificConfig::parse(&bw.finish()).is_err()); + } + + #[test] + fn asc_roundtrip() { + let a = AudioSpecificConfig::lc(4, 2); + let b = a.to_bytes().unwrap(); + assert_eq!(b, vec![0x12, 0x10]); + assert_eq!(AudioSpecificConfig::parse(&b).unwrap(), a); + // 48 kHz 5.1 + let a = AudioSpecificConfig::lc(3, 6); + assert_eq!(AudioSpecificConfig::parse(&a.to_bytes().unwrap()).unwrap().channels(), 6); + } + + #[test] + fn amd4_channel_configurations() { + // 6.1 and 7.1 back (#922); 22.2 and 7.1 with heights stay unsupported + let channels = |c| AudioSpecificConfig::parse(&AudioSpecificConfig::lc(3, c).to_bytes().unwrap()).map(|a| a.channels()).ok(); + assert_eq!([7, 11, 12].map(channels), [Some(8), Some(7), Some(8)]); + assert_eq!([13, 14].map(config_layout), [None, None]); + } + + #[test] + fn asc_with_pce() { + let pce = ProgramConfig { + sf_index: 3, + object_type: 1, + front: vec![(false, 0), (true, 0)], + side: vec![(true, 1)], + back: vec![(false, 1)], + lfe: vec![0], + ..Default::default() + }; + let a = AudioSpecificConfig { pce: Some(pce), ..AudioSpecificConfig::lc(3, 0) }; + let p = AudioSpecificConfig::parse(&a.to_bytes().unwrap()).unwrap(); + assert_eq!(p.channels(), 7); + assert_eq!(p, a); + } + + #[test] + fn he_aac_asc_parses_core() { + // AOT 5, 24 kHz core, stereo, ext 48 kHz, core AOT 2 + let mut bw = BitWriter::new(); + bw.write_bits(5, 5); + bw.write_bits(6, 4); + bw.write_bits(2, 4); + bw.write_bits(3, 4); + bw.write_bits(2, 5); + bw.write_bits(0, 3); + let a = AudioSpecificConfig::parse(&bw.finish()).unwrap(); + assert!(a.sbr); + assert_eq!(a.sample_rate, 24000); + assert_eq!(a.extension_sample_rate, Some(48000)); + } + + #[test] + fn adts_roundtrip() { + let h = AdtsHeader::write(2, 4, 2, 100, 0x7FF).unwrap(); + let mut frame = h.to_vec(); + frame.extend(std::iter::repeat_n(0u8, 100)); + let p = AdtsHeader::parse(&frame).unwrap(); + assert_eq!(p.frame_length, 107); + assert_eq!((p.object_type, p.sf_index, p.channel_config), (2, 4, 2)); + let frames = split_adts(&frame).unwrap(); + assert_eq!(frames.len(), 1); + assert_eq!(frames[0].1.len(), 100); + } +} diff --git a/crates/audio/src/aac/decoder.rs b/crates/audio/src/aac/decoder.rs new file mode 100644 index 000000000..d072796e9 --- /dev/null +++ b/crates/audio/src/aac/decoder.rs @@ -0,0 +1,717 @@ +//! AAC-LC decoder: raw access units (`raw_data_block`, ISO/IEC 14496-3 §4.4.2.1) → planar f32. + +use craft_media::bitstream::BitReader; + +use crate::aac::config::{AudioSpecificConfig, ElementType, ProgramConfig}; +use crate::aac::huffman::{self, INTENSITY_HCB, INTENSITY_HCB2, NOISE_HCB, ZERO_HCB}; +use crate::aac::ics::{IcsInfo, WindowSequence}; +use crate::aac::mdct::{WindowShape, mdct_long, mdct_short, window}; +use crate::aac::tables::{pow43, sf_gain}; +use crate::aac::tns::{self, TnsData}; +use crate::aac::{Error, Result}; + +const ID_SCE: u32 = 0; +const ID_CPE: u32 = 1; +const ID_CCE: u32 = 2; +const ID_LFE: u32 = 3; +const ID_DSE: u32 = 4; +const ID_PCE: u32 = 5; +const ID_FIL: u32 = 6; +#[cfg(test)] +const ID_END: u32 = 7; + +/// `extension_type` of a fill element carrying SBR data (ISO/IEC 14496-3 §4.5.2.8.2, Table 4.121). +const EXT_SBR_DATA: u32 = 13; +const EXT_SBR_DATA_CRC: u32 = 14; + +/// The noise generator's state at the start of a stream and after [`Decoder::reset`]. +const NOISE_SEED: u32 = 0x2545_F491; + +/// One decoded `individual_channel_stream` before stereo processing and synthesis. +struct Ics { + info: IcsInfo, + /// Band type per (group, sfb). + band_type: [[u8; 64]; 8], + /// Scalefactor / noise energy / intensity position per (group, sfb). + sf: [[i32; 64]; 8], + tns: Option, + /// Dequantised spectrum (short windows at stride 128). + spec: Vec, +} + +#[derive(Clone)] +struct ChannelState { + overlap: Vec, + prev_shape: WindowShape, +} + +/// AAC-LC decoder. +#[derive(Clone)] +pub struct Decoder { + asc: AudioSpecificConfig, + layout: Vec<(ElementType, u8)>, + /// First output channel of each layout entry. + offsets: Vec, + channels: Vec, + noise_state: u32, + /// A fill element carried SBR data (implicitly signalled HE-AAC). + sbr_seen: bool, +} + +impl Decoder { + /// Create a decoder from an `AudioSpecificConfig`. + pub fn new(asc: &[u8]) -> Result { + Self::from_config(AudioSpecificConfig::parse(asc)?) + } + + /// Create a decoder for an ADTS stream from its first header. With channel configuration 0 the + /// channel layout is taken from the program config element carried in the first frame that + /// decodes with it. + pub fn from_adts(header: &crate::aac::config::AdtsHeader) -> Result { + Self::from_config(header.to_asc()) + } + + /// Create a decoder from a parsed `AudioSpecificConfig`. Only audio object type 2 (AAC LC) is accepted. + pub fn from_config(asc: AudioSpecificConfig) -> Result { + if asc.object_type != 2 { + return Err(Error::Unsupported("audio object type other than AAC LC")); + } + if usize::from(asc.sf_index) >= crate::aac::SAMPLE_RATES.len() || asc.sample_rate == 0 { + return Err(Error::InvalidConfig("a sampling frequency index with no rate")); + } + let layout = asc.layout(); + if layout.is_empty() && asc.channel_config != 0 { + return Err(Error::InvalidConfig("unsupported channel configuration")); + } + let mut d = Decoder { asc, layout: Vec::new(), offsets: Vec::new(), channels: Vec::new(), noise_state: NOISE_SEED, sbr_seen: false }; + d.set_layout(layout); + Ok(d) + } + + fn set_layout(&mut self, layout: Vec<(ElementType, u8)>) { + let mut offsets = Vec::with_capacity(layout.len()); + let mut n = 0; + for e in &layout { + offsets.push(n); + n += e.0.channels(); + } + self.layout = layout; + self.offsets = offsets; + self.channels = vec![ChannelState { overlap: vec![0.0; 1024], prev_shape: WindowShape::Sine }; n]; + } + + /// The `AudioSpecificConfig` the decoder was created with. + pub fn config(&self) -> &AudioSpecificConfig { + &self.asc + } + /// Output sampling rate in Hz: the AAC-LC core rate. + pub fn sample_rate(&self) -> u32 { + self.asc.sample_rate + } + /// Whether the stream is HE-AAC: SBR signalled in the `AudioSpecificConfig` (explicit) or SBR + /// data seen in a fill element of a decoded access unit (implicit). Only the AAC-LC core is + /// decoded, at [`Self::sample_rate`]; the SBR high band is not reconstructed. + pub fn sbr(&self) -> bool { + self.asc.sbr || self.sbr_seen + } + /// Output channels of the current element layout. + pub fn channels(&self) -> usize { + self.channels.len() + } + /// Element layout in output channel order (C, L/R, surround pairs, LFE for configurations 1–7). + pub fn layout(&self) -> &[(ElementType, u8)] { + &self.layout + } + + /// Clear the overlap and noise state (e.g. after a seek), so a seek to the same place always + /// gives the same samples. + pub fn reset(&mut self) { + self.noise_state = NOISE_SEED; + for c in &mut self.channels { + c.overlap.fill(0.0); + c.prev_shape = WindowShape::Sine; + } + } + + fn sf_index(&self) -> u8 { + self.asc.sf_index + } + + /// Decode one access unit (one `raw_data_block`) into 1024 samples per channel. An access unit + /// that fails leaves the decoder as it was, but for [`Self::sbr`]: its damage, including a damaged + /// program config element, does not reach the access units after it. + pub fn decode(&mut self, au: &[u8]) -> Result>> { + let before = (self.asc.pce.clone(), self.layout.clone(), self.offsets.clone(), self.channels.clone(), self.noise_state); + let decoded = self.decode_block(au, true); + if decoded.is_err() { + (self.asc.pce, self.layout, self.offsets, self.channels, self.noise_state) = before; + } + decoded + } + + fn decode_block(&mut self, au: &[u8], allow_reconfig: bool) -> Result>> { + let mut br = BitReader::new(au); + let nch = self.channels.len(); + let mut saw_element = false; + let mut out: Vec>> = vec![None; nch]; + let mut used = vec![false; self.layout.len()]; + loop { + let id = br.read_bits(3)?; + saw_element |= matches!(id, ID_SCE | ID_CPE | ID_LFE); + match id { + ID_SCE | ID_LFE => { + let tag = br.read_bits(4)? as u8; + let ty = if id == ID_SCE { ElementType::Sce } else { ElementType::Lfe }; + let slot = self.find_slot(ty, tag, &used)?; + used[slot] = true; + let mut ics = self.decode_ics(&mut br, None)?; + let ch = self.offsets[slot]; + self.fill_noise(&mut ics, None); + out[ch] = Some(self.finish_channel(ch, &mut ics)); + } + ID_CPE => { + let tag = br.read_bits(4)? as u8; + let slot = self.find_slot(ElementType::Cpe, tag, &used)?; + used[slot] = true; + let ch = self.offsets[slot]; + let [a, b] = self.decode_cpe(&mut br, ch)?; + out[ch] = Some(a); + out[ch + 1] = Some(b); + } + ID_CCE => return Err(Error::Unsupported("coupling channel element")), + ID_DSE => { + br.skip(4)?; + let align = br.read_bits(1)? == 1; + let mut count = br.read_bits(8)? as usize; + if count == 255 { + count += br.read_bits(8)? as usize; + } + if align { + br.byte_align(); + } + br.skip(count * 8)?; + } + ID_PCE => { + let pce = ProgramConfig::parse(&mut br)?; + if allow_reconfig && !saw_element && self.asc.channel_config == 0 && self.asc.pce.as_ref() != Some(&pce) { + let layout = pce.layout(); + if !layout.is_empty() && layout.iter().map(|e| e.0.channels()).sum::() <= 64 { + self.asc.pce = Some(pce); + self.set_layout(layout); + return self.decode_block(au, false); + } + } + } + ID_FIL => { + let mut count = br.read_bits(4)? as usize; + if count == 15 { + count += br.read_bits(8)? as usize; + count -= 1; + } + if count > 0 { + let ext = br.read_bits(4)?; + self.sbr_seen |= matches!(ext, EXT_SBR_DATA | EXT_SBR_DATA_CRC); + br.skip(count * 8 - 4)?; + } + } + _ => break, // ID_END + } + } + if nch == 0 && !saw_element { + return Err(Error::InvalidConfig("channel configuration 0 without a program config element")); + } + let mut res = Vec::with_capacity(nch); + for o in out { + res.push(o.ok_or(Error::Bitstream("access unit is missing a channel element"))?); + } + Ok(res) + } + + fn find_slot(&self, ty: ElementType, tag: u8, used: &[bool]) -> Result { + let free = |want: &dyn Fn(ElementType, u8) -> bool| self.layout.iter().enumerate().position(|(i, &(t, g))| want(t, g) && !used[i]); + let slot = if self.asc.channel_config == 0 { + // A program config names its elements' tags. + free(&|t, g| t == ty && g == tag) + } else { + // An indexed configuration does not: its elements are placed by the order they arrive + // in, whatever their tags. Some encoders code an LFE as an SCE, as ffmpeg allows. + free(&|t, _| t == ty).or_else(|| free(&|t, _| ty == ElementType::Sce && t == ElementType::Lfe)) + }; + slot.ok_or(Error::Bitstream("unexpected channel element for the channel configuration")) + } + + fn decode_cpe(&mut self, br: &mut BitReader, ch: usize) -> Result<[Vec; 2]> { + let common_window = br.read_bits(1)? == 1; + let mut ms_mask_present = 0; + let mut ms_used = [[false; 64]; 8]; + let mut info = None; + if common_window { + let i = IcsInfo::parse(br, self.sf_index())?; + ms_mask_present = br.read_bits(2)?; + if ms_mask_present == 3 { + return Err(Error::Bitstream("reserved ms_mask_present")); + } + if ms_mask_present == 1 { + for g in 0..i.num_groups { + for sfb in 0..i.max_sfb { + ms_used[g][sfb] = br.read_bits(1)? == 1; + } + } + } else if ms_mask_present == 2 { + ms_used = [[true; 64]; 8]; + } + info = Some(i); + } + let mut l = self.decode_ics(br, info)?; + let mut r = self.decode_ics(br, info)?; + let ms = if ms_mask_present > 0 { Some(&ms_used) } else { None }; + self.fill_noise(&mut l, None); + self.fill_noise(&mut r, if common_window { Some((&l, ms)) } else { None }); + if common_window { + let swb = l.info.swb_offsets(self.sf_index()); + for g in 0..l.info.num_groups { + let w0 = l.info.group_start(g); + for sfb in 0..l.info.max_sfb { + let (bl, br_) = (l.band_type[g][sfb], r.band_type[g][sfb]); + let range = swb[sfb] as usize..swb[sfb + 1] as usize; + if br_ == INTENSITY_HCB || br_ == INTENSITY_HCB2 { + let mut c: f32 = if br_ == INTENSITY_HCB { 1.0 } else { -1.0 }; + if ms_mask_present > 0 && ms_used[g][sfb] { + c = -c; + } + let scale = c * (-0.25 * r.sf[g][sfb].clamp(-400, 400) as f64).exp2() as f32; + for w in w0..w0 + l.info.group_len[g] as usize { + for k in range.clone() { + r.spec[w * 128 + k] = l.spec[w * 128 + k] * scale; + } + } + } else if ms_mask_present > 0 && ms_used[g][sfb] && bl < NOISE_HCB && br_ < NOISE_HCB { + for w in w0..w0 + l.info.group_len[g] as usize { + for k in range.clone() { + let (m, s) = (l.spec[w * 128 + k], r.spec[w * 128 + k]); + l.spec[w * 128 + k] = m + s; + r.spec[w * 128 + k] = m - s; + } + } + } + } + } + } + Ok([self.finish_channel(ch, &mut l), self.finish_channel(ch + 1, &mut r)]) + } + + fn decode_ics(&mut self, br: &mut BitReader, common: Option) -> Result { + let sfi = self.sf_index(); + let global_gain = br.read_bits(8)? as i32; + let info = match common { + Some(i) => i, + None => IcsInfo::parse(br, sfi)?, + }; + let swb = info.swb_offsets(sfi); + let num_swb = swb.len() - 1; + let mut ics = Ics { info, band_type: [[0; 64]; 8], sf: [[0; 64]; 8], tns: None, spec: vec![0.0; 1024] }; + + // section_data + let (sect_bits, esc) = if info.is_short() { (3, 7) } else { (5, 31) }; + for g in 0..info.num_groups { + let mut k = 0; + while k < info.max_sfb { + let cb = br.read_bits(4)? as u8; + if cb == 12 { + return Err(Error::Bitstream("reserved codebook 12")); + } + let mut len = 0usize; + loop { + let inc = br.read_bits(sect_bits)?; + len += inc as usize; + if inc != esc { + break; + } + } + if k + len > info.max_sfb { + return Err(Error::Bitstream("section exceeds max_sfb")); + } + if len == 0 { + // zero-length sections are legal but useless; guard against endless loops + if br.bits_left() == 0 { + return Err(Error::Eof); + } + } + for sfb in k..k + len { + ics.band_type[g][sfb] = cb; + } + k += len; + } + } + + // scale_factor_data + let mut sf = global_gain; + let mut noise = global_gain - 90; + let mut is_pos = 0i32; + let mut noise_first = true; + for g in 0..info.num_groups { + for sfb in 0..info.max_sfb { + match ics.band_type[g][sfb] { + ZERO_HCB => ics.sf[g][sfb] = 0, + INTENSITY_HCB | INTENSITY_HCB2 => { + is_pos += huffman::decode_sf(br)?; + ics.sf[g][sfb] = is_pos; + } + NOISE_HCB => { + if noise_first { + noise += br.read_bits(9)? as i32 - 256; + noise_first = false; + } else { + noise += huffman::decode_sf(br)?; + } + ics.sf[g][sfb] = noise; + } + _ => { + sf += huffman::decode_sf(br)?; + if !(0..=255).contains(&sf) { + return Err(Error::Bitstream("scalefactor out of range")); + } + ics.sf[g][sfb] = sf; + } + } + } + } + + // pulse_data + let mut pulses: Vec<(usize, i32)> = Vec::new(); + if br.read_bits(1)? == 1 { + if info.is_short() { + return Err(Error::Bitstream("pulse data in short window")); + } + let n = br.read_bits(2)? as usize + 1; + let start = br.read_bits(6)? as usize; + if start >= num_swb { + return Err(Error::Bitstream("pulse_start_sfb out of range")); + } + let mut k = swb[start] as usize; + for _ in 0..n { + k += br.read_bits(5)? as usize; + let amp = br.read_bits(4)? as i32; + if k >= 1024 { + return Err(Error::Bitstream("pulse position out of range")); + } + pulses.push((k, amp)); + } + } + if br.read_bits(1)? == 1 { + ics.tns = Some(TnsData::parse(br, &info)?); + } + if br.read_bits(1)? == 1 { + return Err(Error::Unsupported("gain control data (AAC SSR)")); + } + + // spectral_data + let mut q = vec![0i32; 1024]; + let mut vals = [0i32; 4]; + for g in 0..info.num_groups { + let w0 = info.group_start(g); + let glen = info.group_len[g] as usize; + for sfb in 0..info.max_sfb { + let cb = ics.band_type[g][sfb]; + if cb == ZERO_HCB || cb >= NOISE_HCB { + continue; + } + let dim = huffman::book(cb).dim; + let (lo, hi) = (swb[sfb] as usize, swb[sfb + 1] as usize); + for w in w0..w0 + glen { + let base = w * 128; + let mut k = lo; + while k < hi { + huffman::decode_spectral(br, cb, &mut vals)?; + q[base + k..base + k + dim].copy_from_slice(&vals[..dim]); + k += dim; + } + } + } + } + for (k, amp) in pulses { + if q[k] > 0 { + q[k] += amp; + } else { + q[k] -= amp; + } + } + + // dequantise + for g in 0..info.num_groups { + let w0 = info.group_start(g); + for sfb in 0..info.max_sfb { + let cb = ics.band_type[g][sfb]; + if cb == ZERO_HCB || cb >= NOISE_HCB { + continue; + } + let gain = sf_gain(ics.sf[g][sfb]); + for w in w0..w0 + info.group_len[g] as usize { + for k in swb[sfb] as usize..swb[sfb + 1] as usize { + let v = q[w * 128 + k]; + let m = pow43(v.unsigned_abs()) * gain; + ics.spec[w * 128 + k] = if v < 0 { -m } else { m }; + } + } + } + } + Ok(ics) + } + + fn next_random(&mut self) -> f32 { + self.noise_state = self.noise_state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + (self.noise_state as i32) as f32 + } + + /// Perceptual noise substitution. `pair` = (left channel, ms mask) for the right channel of a CPE: + /// bands that are noise in both channels with `ms_used` reuse the left channel's noise vector. + fn fill_noise(&mut self, ics: &mut Ics, pair: Option<(&Ics, Option<&[[bool; 64]; 8]>)>) { + let info = ics.info; + let swb = info.swb_offsets(self.sf_index()); + for g in 0..info.num_groups { + let w0 = info.group_start(g); + for sfb in 0..info.max_sfb { + if ics.band_type[g][sfb] != NOISE_HCB { + continue; + } + let (lo, hi) = (swb[sfb] as usize, swb[sfb + 1] as usize); + let target = (0.25 * ics.sf[g][sfb].clamp(-400, 400) as f64).exp2(); + let correlated = match pair { + Some((l, Some(ms))) => ms[g][sfb] && l.band_type[g][sfb] == NOISE_HCB, + _ => false, + }; + for w in w0..w0 + info.group_len[g] as usize { + let band = w * 128 + lo..w * 128 + hi; + if correlated && let Some((l, _)) = pair { + ics.spec[band.clone()].copy_from_slice(&l.spec[band.clone()]); + } else { + for k in band.clone() { + ics.spec[k] = self.next_random(); + } + } + let energy: f64 = ics.spec[band.clone()].iter().map(|&v| v as f64 * v as f64).sum(); + let scale = if energy > 0.0 { (target / energy.sqrt()) as f32 } else { 0.0 }; + for k in band { + ics.spec[k] *= scale; + } + } + } + } + } + + /// TNS + filterbank for a single channel element; returns output samples. + fn finish_channel(&mut self, ch: usize, ics: &mut Ics) -> Vec { + if let Some(t) = &ics.tns { + tns::apply_decoder(&mut ics.spec, &ics.info, self.sf_index(), t); + } + synth(&mut self.channels[ch], &ics.info, &ics.spec) + } +} + +/// Inverse MDCT, windowing and overlap-add (§4.6.18), output scaled to ±1.0 full scale. +fn synth(state: &mut ChannelState, info: &IcsInfo, spec: &[f32]) -> Vec { + let mut buf = vec![0f32; 2048]; + let prev = state.prev_shape; + let cur = info.window_shape; + let norm = 1.0 / 32768.0; + match info.window_sequence { + WindowSequence::EightShort => { + let t = mdct_short(); + let mut y = vec![0f32; 256]; + let wc = window(cur, false); + for w in 0..8 { + t.inverse(&spec[w * 128..(w + 1) * 128], &mut y); + let wl = window(if w == 0 { prev } else { cur }, false); + let off = 448 + 128 * w; + let s = norm / 128.0; + for n in 0..128 { + buf[off + n] += y[n] * wl[n] * s; + buf[off + 128 + n] += y[128 + n] * wc[127 - n] * s; + } + } + } + seq => { + let mut y = vec![0f32; 2048]; + mdct_long().inverse(&spec[..1024], &mut y); + let s = norm / 1024.0; + match seq { + WindowSequence::LongStop => { + let ws = window(prev, false); + for n in 448..576 { + buf[n] = y[n] * ws[n - 448] * s; + } + for n in 576..1024 { + buf[n] = y[n] * s; + } + } + _ => { + let wl = window(prev, true); + for n in 0..1024 { + buf[n] = y[n] * wl[n] * s; + } + } + } + match seq { + WindowSequence::LongStart => { + let ws = window(cur, false); + for n in 1024..1472 { + buf[n] = y[n] * s; + } + for n in 1472..1600 { + buf[n] = y[n] * ws[127 - (n - 1472)] * s; + } + } + _ => { + let wl = window(cur, true); + for n in 1024..2048 { + buf[n] = y[n] * wl[2047 - n] * s; + } + } + } + } + } + // A hostile stream can overflow f32 (PNS at full energy through a resonant TNS filter); that must + // not reach the output or the next frame's overlap. + for s in buf.iter_mut().filter(|s| !s.is_finite()) { + *s = 0.0; + } + let out: Vec = (0..1024).map(|n| state.overlap[n] + buf[n]).collect(); + state.overlap.copy_from_slice(&buf[1024..]); + state.prev_shape = cur; + out +} + +#[cfg(test)] +mod tests { + use super::*; + use craft_media::bitstream::BitWriter; + + /// Implicitly signalled HE-AAC (an AAC-LC config) is recognised from the SBR fill element. + #[test] + fn sbr_fill_element_marks_implicit_he_aac() { + // AAC-LC, 22.05 kHz, stereo + let mut dec = Decoder::new(&[0x13, 0x90]).unwrap(); + assert!(!dec.sbr()); + // ID_FIL, count 1, extension_type EXT_SBR_DATA (+ 4 bits of payload), ID_END + let _ = dec.decode(&[0b1100_0011, 0b1010_0001, 0b1100_0000]); + assert!(dec.sbr()); + // other fill payloads (EXT_FILL) don't + let mut dec = Decoder::new(&[0x13, 0x90]).unwrap(); + let _ = dec.decode(&[0b1100_0010, 0b0000_0001, 0b1100_0000]); + assert!(!dec.sbr()); + // explicit signalling: AOT 5, core 22.05 kHz, stereo, extension 44.1 kHz, core AOT 2 + let dec = Decoder::new(&[0x2B, 0x92, 0x08, 0x00]).unwrap(); + assert!(dec.sbr()); + assert_eq!(dec.sample_rate(), 22_050); + } + + /// An access unit whose elements are all empty: `elements` are (id, tag), then ID_END. + fn silent_au(pce: Option<&ProgramConfig>, elements: &[(u32, u32)]) -> Vec { + let mut bw = BitWriter::new(); + if let Some(pce) = pce { + bw.write_bits(ID_PCE, 3); + pce.write(&mut bw).unwrap(); + } + for &(id, tag) in elements { + bw.write_bits(id, 3); + bw.write_bits(tag, 4); + if id == ID_CPE { + bw.write_bits(0, 1); // common_window + } + for _ in 0..if id == ID_CPE { 2 } else { 1 } { + bw.write_bits(0, 8); // global_gain + bw.write_bits(0, 11); // ics_info: long window, max_sfb 0, no predictor + bw.write_bits(0, 3); // no pulse, TNS or gain control data + } + } + bw.write_bits(ID_END, 3); + bw.finish() + } + + /// Indexed configurations place elements by arrival order, whatever their tags (as ffmpeg does). + #[test] + fn indexed_configurations_ignore_element_tags() { + let dec = Decoder::from_config(AudioSpecificConfig::lc(4, 5)).unwrap(); + // SCE, CPE, CPE: the first CPE to arrive is L/R even when tagged 1 + assert_eq!(dec.find_slot(ElementType::Cpe, 1, &[true, false, false]), Ok(1)); + assert_eq!(dec.find_slot(ElementType::Cpe, 0, &[true, true, false]), Ok(2)); + } + + #[test] + fn program_configs_place_elements_by_tag() { + let pce = ProgramConfig { front: vec![(true, 3), (true, 5)], ..Default::default() }; + let asc = AudioSpecificConfig { pce: Some(pce), ..AudioSpecificConfig::lc(4, 0) }; + let dec = Decoder::from_config(asc).unwrap(); + assert_eq!(dec.find_slot(ElementType::Cpe, 5, &[false, false]), Ok(1)); + } + + /// A program config lists its elements: one it does not list has no place. + #[test] + fn program_configs_refuse_unlisted_tags() { + let pce = ProgramConfig { front: vec![(true, 3), (true, 5)], ..Default::default() }; + let asc = AudioSpecificConfig { pce: Some(pce), ..AudioSpecificConfig::lc(4, 0) }; + let dec = Decoder::from_config(asc).unwrap(); + assert!(dec.find_slot(ElementType::Cpe, 9, &[false, false]).is_err()); + assert!(dec.find_slot(ElementType::Sce, 3, &[false, false]).is_err()); + } + + /// Some encoders code 5.1's LFE as an SCE, as ffmpeg allows. Nothing else changes type: an LFE + /// is not written to the centre channel. + #[test] + fn indexed_configurations_refuse_elements_of_another_type() { + let dec = Decoder::from_config(AudioSpecificConfig::lc(4, 6)).unwrap(); + // SCE, CPE, CPE, LFE + assert_eq!(dec.find_slot(ElementType::Sce, 1, &[true, false, false, false]), Ok(3)); + assert!(dec.find_slot(ElementType::Lfe, 1, &[false, false, false, true]).is_err()); + assert!(dec.find_slot(ElementType::Cpe, 2, &[false, true, true, false]).is_err()); + } + + /// A PCE in an access unit that then fails leaves the layout as it was. + #[test] + fn a_damaged_in_band_pce_does_not_change_the_layout() { + let two = ProgramConfig { front: vec![(true, 0)], ..Default::default() }; + let six = ProgramConfig { front: vec![(false, 0), (true, 0)], back: vec![(true, 1)], lfe: vec![0], ..Default::default() }; + let mut dec = Decoder::from_config(AudioSpecificConfig { pce: Some(two.clone()), ..AudioSpecificConfig::lc(4, 0) }).unwrap(); + let good = silent_au(None, &[(ID_CPE, 0)]); + assert_eq!(dec.decode(&good).unwrap().len(), 2); + // the new layout's elements are missing from this access unit + assert!(dec.decode(&silent_au(Some(&six), &[(ID_CPE, 0)])).is_err()); + assert_eq!((dec.channels(), dec.config().pce.as_ref()), (2, Some(&two))); + assert_eq!(dec.decode(&good).unwrap().len(), 2); + // one that decodes does change it + let full = silent_au(Some(&six), &[(ID_SCE, 0), (ID_CPE, 0), (ID_CPE, 1), (ID_LFE, 0)]); + assert_eq!(dec.decode(&full).unwrap().len(), 6); + assert_eq!(dec.channels(), 6); + } + + /// A failed access unit leaves no trace: the next one decodes as if it had not been there. + #[test] + fn a_failed_access_unit_leaves_the_state_as_it_was() { + let mut dec = Decoder::from_config(AudioSpecificConfig::lc(4, 1)).unwrap(); + dec.channels[0].overlap.fill(0.5); + dec.channels[0].prev_shape = WindowShape::Kbd; + let noise = dec.noise_state; + // a complete channel element, then the access unit ends without ID_END + let au = silent_au(None, &[(ID_SCE, 0)]); + assert!(dec.decode(&au[..au.len() - 1]).is_err()); + assert!(dec.channels[0].overlap.iter().all(|&s| s == 0.5)); + assert_eq!((dec.channels[0].prev_shape, dec.noise_state), (WindowShape::Kbd, noise)); + } + + #[test] + fn a_reset_restarts_the_noise_generator() { + let mut dec = Decoder::from_config(AudioSpecificConfig::lc(4, 1)).unwrap(); + let first = dec.next_random(); + dec.next_random(); + dec.reset(); + assert_eq!(dec.next_random(), first); + } + + #[test] + fn non_finite_spectra_do_not_reach_the_output_or_the_overlap() { + let mut state = ChannelState { overlap: vec![0.0; 1024], prev_shape: WindowShape::Sine }; + let info = IcsInfo::parse(&mut BitReader::new(&[0; 2]), 4).unwrap(); + let out = synth(&mut state, &info, &vec![f32::INFINITY; 1024]); + assert!(out.iter().chain(&state.overlap).all(|s| s.is_finite())); + } +} diff --git a/crates/audio/src/aac/huffman.rs b/crates/audio/src/aac/huffman.rs new file mode 100644 index 000000000..1de2bb194 --- /dev/null +++ b/crates/audio/src/aac/huffman.rs @@ -0,0 +1,306 @@ +//! Huffman coding of spectral values and scalefactors (ISO/IEC 14496-3 §4.6.3, Annex 4.A). + +use std::sync::OnceLock; + +use craft_media::bitstream::{BitReader, BitWriter}; + +use crate::aac::huffman_tables::{ + SCALEFACTOR, SPECTRUM_1, SPECTRUM_2, SPECTRUM_3, SPECTRUM_4, SPECTRUM_5, SPECTRUM_6, SPECTRUM_7, SPECTRUM_8, SPECTRUM_9, SPECTRUM_10, SPECTRUM_11, +}; +use crate::aac::{Error, Result}; + +/// `ZERO_HCB`: band quantised to zero, nothing transmitted. +pub const ZERO_HCB: u8 = 0; +/// `ESC_HCB`: codebook 11 with escape sequences. +pub const ESC_HCB: u8 = 11; +/// `NOISE_HCB`: perceptual noise substitution. +pub const NOISE_HCB: u8 = 13; +/// `INTENSITY_HCB2`: intensity stereo, out of phase. +pub const INTENSITY_HCB2: u8 = 14; +/// `INTENSITY_HCB`: intensity stereo, in phase. +pub const INTENSITY_HCB: u8 = 15; + +/// Parameters of one spectrum codebook. +pub struct Book { + /// Values per codeword (4 = quad, 2 = pair). + pub dim: usize, + /// Signed values in the codeword (no separate sign bits). + pub signed: bool, + /// Largest absolute value (16 = escape for codebook 11). + pub lav: u32, + /// Radix of the index (`2*lav+1` signed, `lav+1` unsigned). + pub modulo: u32, + pub codes: &'static [(u32, u8)], +} + +static BOOKS: [Book; 11] = [ + Book { dim: 4, signed: true, lav: 1, modulo: 3, codes: &SPECTRUM_1 }, + Book { dim: 4, signed: true, lav: 1, modulo: 3, codes: &SPECTRUM_2 }, + Book { dim: 4, signed: false, lav: 2, modulo: 3, codes: &SPECTRUM_3 }, + Book { dim: 4, signed: false, lav: 2, modulo: 3, codes: &SPECTRUM_4 }, + Book { dim: 2, signed: true, lav: 4, modulo: 9, codes: &SPECTRUM_5 }, + Book { dim: 2, signed: true, lav: 4, modulo: 9, codes: &SPECTRUM_6 }, + Book { dim: 2, signed: false, lav: 7, modulo: 8, codes: &SPECTRUM_7 }, + Book { dim: 2, signed: false, lav: 7, modulo: 8, codes: &SPECTRUM_8 }, + Book { dim: 2, signed: false, lav: 12, modulo: 13, codes: &SPECTRUM_9 }, + Book { dim: 2, signed: false, lav: 12, modulo: 13, codes: &SPECTRUM_10 }, + Book { dim: 2, signed: false, lav: 16, modulo: 17, codes: &SPECTRUM_11 }, +]; + +/// Spectrum codebook 1..=11. +#[inline] +pub fn book(cb: u8) -> &'static Book { + &BOOKS[(cb as usize).clamp(1, 11) - 1] +} + +const LEAF: u32 = 1 << 31; + +/// Binary decoding tree for one codebook. +struct Tree { + nodes: Vec<[u32; 2]>, +} + +impl Tree { + fn build(codes: &[(u32, u8)]) -> Tree { + let mut nodes: Vec<[u32; 2]> = vec![[0, 0]]; + for (sym, &(code, len)) in codes.iter().enumerate() { + let mut node = 0usize; + for i in (0..len).rev() { + let bit = ((code >> i) & 1) as usize; + if i == 0 { + nodes[node][bit] = LEAF | sym as u32; + } else { + if nodes[node][bit] == 0 { + nodes.push([0, 0]); + nodes[node][bit] = (nodes.len() - 1) as u32; + } + node = nodes[node][bit] as usize; + } + } + } + Tree { nodes } + } + + #[inline] + fn decode(&self, br: &mut BitReader) -> Result { + let mut node = 0usize; + loop { + let bit = br.read_bits(1)? as usize; + let next = self.nodes[node][bit]; + if next & LEAF != 0 { + return Ok(next & !LEAF); + } + if next == 0 { + return Err(Error::Bitstream("invalid Huffman codeword")); + } + node = next as usize; + } + } +} + +fn trees() -> &'static [Tree; 12] { + static TREES: OnceLock<[Tree; 12]> = OnceLock::new(); + TREES.get_or_init(|| std::array::from_fn(|i| if i == 0 { Tree::build(&SCALEFACTOR) } else { Tree::build(BOOKS[i - 1].codes) })) +} + +/// Decode one scalefactor difference (`-60..=60`). +#[inline] +pub fn decode_sf(br: &mut BitReader) -> Result { + Ok(trees()[0].decode(br)? as i32 - 60) +} + +/// Write one scalefactor difference (`-60..=60`). +#[inline] +pub fn write_sf(bw: &mut BitWriter, diff: i32) { + let (code, len) = SCALEFACTOR[(diff + 60).clamp(0, 120) as usize]; + bw.write_bits(code, len as u32); +} + +/// Bits of one scalefactor difference. +#[inline] +pub fn sf_bits(diff: i32) -> u32 { + SCALEFACTOR[(diff + 60).clamp(0, 120) as usize].1 as u32 +} + +/// Decode one codeword of codebook `cb` (1..=11) into `out[..dim]`, including sign bits and escapes. +pub fn decode_spectral(br: &mut BitReader, cb: u8, out: &mut [i32]) -> Result<()> { + let b = book(cb); + let mut idx = trees()[cb as usize].decode(br)?; + for i in (0..b.dim).rev() { + out[i] = (idx % b.modulo) as i32; + idx /= b.modulo; + } + if b.signed { + for v in out[..b.dim].iter_mut() { + *v -= b.lav as i32; + } + return Ok(()); + } + for v in out[..b.dim].iter_mut() { + if *v != 0 && br.read_bits(1)? == 1 { + *v = -*v; + } + } + if cb == ESC_HCB { + for v in out[..b.dim].iter_mut() { + if v.abs() == 16 { + let mut n = 0u32; + while br.read_bits(1)? == 1 { + n += 1; + if n > 8 { + return Err(Error::Bitstream("escape sequence too long")); + } + } + let mag = (1i32 << (n + 4)) + br.read_bits(n + 4)? as i32; + *v = if *v < 0 { -mag } else { mag }; + } + } + } + Ok(()) +} + +/// Codebook index of the (sign-stripped for unsigned books) values. +#[inline] +fn index_of(b: &Book, vals: &[i32]) -> usize { + let mut idx = 0u32; + for &v in &vals[..b.dim] { + let d = if b.signed { (v + b.lav as i32) as u32 } else { v.unsigned_abs().min(b.lav) }; + idx = idx * b.modulo + d; + } + idx as usize +} + +#[inline] +fn escape_bits(mag: u32) -> u32 { + // mag >= 16: N ones, a zero, N+4 bits where 2^(N+4) <= mag < 2^(N+5). + let n = 31 - mag.leading_zeros() - 4; + 2 * n + 5 +} + +/// Bits needed to code `vals` (a multiple of the codebook dimension) with codebook `cb` (1..=11). +/// The values must fit the codebook (`|v| <= lav`, or `<= 8191` for codebook 11). +pub fn count_bits(cb: u8, vals: &[i32]) -> u32 { + let b = book(cb); + let mut bits = 0u32; + for chunk in vals.chunks_exact(b.dim) { + bits += b.codes[index_of(b, chunk)].1 as u32; + if !b.signed { + for &v in chunk { + if v != 0 { + bits += 1; + if cb == ESC_HCB && v.unsigned_abs() >= 16 { + bits += escape_bits(v.unsigned_abs()); + } + } + } + } + } + bits +} + +/// Write `vals` with codebook `cb` (1..=11). +pub fn write_spectral(bw: &mut BitWriter, cb: u8, vals: &[i32]) { + let b = book(cb); + for chunk in vals.chunks_exact(b.dim) { + let (code, len) = b.codes[index_of(b, chunk)]; + bw.write_bits(code, len as u32); + if b.signed { + continue; + } + for &v in chunk { + if v != 0 { + bw.write_bits((v < 0) as u32, 1); + } + } + if cb == ESC_HCB { + for &v in chunk { + let mag = v.unsigned_abs(); + if mag >= 16 { + let n = 31 - mag.leading_zeros() - 4; + for _ in 0..n { + bw.write_bits(1, 1); + } + bw.write_bits(0, 1); + bw.write_bits(mag - (1 << (n + 4)), n + 4); + } + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn check_complete(codes: &[(u32, u8)]) { + // Kraft sum exactly one and prefix-free. + let max = codes.iter().map(|c| c.1).max().unwrap() as u32; + let mut kraft = 0u64; + for &(_, l) in codes { + kraft += 1u64 << (max - l as u32); + } + assert_eq!(kraft, 1u64 << max); + for (i, &(ci, li)) in codes.iter().enumerate() { + for (j, &(cj, lj)) in codes.iter().enumerate() { + if i != j && li <= lj { + assert_ne!(cj >> (lj - li), ci, "codeword {i} is a prefix of {j}"); + } + } + } + } + + #[test] + fn codebooks_are_complete_prefix_codes() { + check_complete(&SCALEFACTOR); + assert_eq!(SCALEFACTOR[60], (0, 1)); + for b in &BOOKS { + assert_eq!(b.codes.len() as u32, b.modulo.pow(b.dim as u32)); + check_complete(b.codes); + } + } + + #[test] + fn spectral_roundtrip_every_book() { + for cb in 1..=11u8 { + let b = book(cb); + let lim = if cb == 11 { 8191 } else { b.lav as i32 }; + let mut vals = Vec::new(); + let mut seed = 12345u32; + for _ in 0..400 { + seed = seed.wrapping_mul(1664525).wrapping_add(1013904223); + let r = (seed >> 8) as i32; + let v = if cb == 11 && r % 3 == 0 { r % (lim + 1) } else { r % (b.lav.min(16) as i32 + 1) }; + let v = v.min(lim); + vals.push(if b.signed || r & 1 == 0 { v } else { -v }); + } + if b.signed { + for v in vals.iter_mut() { + *v = (*v).clamp(-(b.lav as i32), b.lav as i32); + } + } + let mut bw = BitWriter::new(); + write_spectral(&mut bw, cb, &vals); + assert_eq!(bw.bit_len() as u32, count_bits(cb, &vals)); + let data = bw.finish(); + let mut br = BitReader::new(&data); + let mut out = vec![0i32; vals.len()]; + for c in out.chunks_mut(b.dim) { + decode_spectral(&mut br, cb, c).unwrap(); + } + assert_eq!(out, vals, "codebook {cb}"); + } + } + + #[test] + fn scalefactor_roundtrip() { + let mut bw = BitWriter::new(); + for d in -60..=60 { + write_sf(&mut bw, d); + } + let data = bw.finish(); + let mut br = BitReader::new(&data); + for d in -60..=60 { + assert_eq!(decode_sf(&mut br).unwrap(), d); + } + } +} diff --git a/crates/audio/src/aac/huffman_tables.rs b/crates/audio/src/aac/huffman_tables.rs new file mode 100644 index 000000000..c65c7df60 --- /dev/null +++ b/crates/audio/src/aac/huffman_tables.rs @@ -0,0 +1,1406 @@ +//! AAC Huffman codebooks (ISO/IEC 14496-3 §4.A.1, spectrum codebooks 1–11 and the scalefactor codebook). +//! +//! Each table is indexed by the codebook index defined in the standard and stores `(codeword, length)`. +//! Generated data: derived by black-box probing of an external decoder used purely as a test oracle +//! (every table is a complete prefix code — Kraft sum exactly 1 — covering every index once; see +//! `tests::codebooks_are_complete_prefix_codes`). + +#![allow(clippy::unreadable_literal)] + +pub(crate) static SCALEFACTOR: [(u32, u8); 121] = [ + (0x3ffe8, 18), + (0x3ffe6, 18), + (0x3ffe7, 18), + (0x3ffe5, 18), + (0x7fff5, 19), + (0x7fff1, 19), + (0x7ffed, 19), + (0x7fff6, 19), + (0x7ffee, 19), + (0x7ffef, 19), + (0x7fff0, 19), + (0x7fffc, 19), + (0x7fffd, 19), + (0x7ffff, 19), + (0x7fffe, 19), + (0x7fff7, 19), + (0x7fff8, 19), + (0x7fffb, 19), + (0x7fff9, 19), + (0x3ffe4, 18), + (0x7fffa, 19), + (0x3ffe3, 18), + (0x1ffef, 17), + (0x1fff0, 17), + (0xfff5, 16), + (0x1ffee, 17), + (0xfff2, 16), + (0xfff3, 16), + (0xfff4, 16), + (0xfff1, 16), + (0x7ff6, 15), + (0x7ff7, 15), + (0x3ff9, 14), + (0x3ff5, 14), + (0x3ff7, 14), + (0x3ff3, 14), + (0x3ff6, 14), + (0x3ff2, 14), + (0x1ff7, 13), + (0x1ff5, 13), + (0xff9, 12), + (0xff7, 12), + (0xff6, 12), + (0x7f9, 11), + (0xff4, 12), + (0x7f8, 11), + (0x3f9, 10), + (0x3f7, 10), + (0x3f5, 10), + (0x1f8, 9), + (0x1f7, 9), + (0xfa, 8), + (0xf8, 8), + (0xf6, 8), + (0x79, 7), + (0x3a, 6), + (0x38, 6), + (0x1a, 5), + (0xb, 4), + (0x4, 3), + (0x0, 1), + (0xa, 4), + (0xc, 4), + (0x1b, 5), + (0x39, 6), + (0x3b, 6), + (0x78, 7), + (0x7a, 7), + (0xf7, 8), + (0xf9, 8), + (0x1f6, 9), + (0x1f9, 9), + (0x3f4, 10), + (0x3f6, 10), + (0x3f8, 10), + (0x7f5, 11), + (0x7f4, 11), + (0x7f6, 11), + (0x7f7, 11), + (0xff5, 12), + (0xff8, 12), + (0x1ff4, 13), + (0x1ff6, 13), + (0x1ff8, 13), + (0x3ff8, 14), + (0x3ff4, 14), + (0xfff0, 16), + (0x7ff4, 15), + (0xfff6, 16), + (0x7ff5, 15), + (0x3ffe2, 18), + (0x7ffd9, 19), + (0x7ffda, 19), + (0x7ffdb, 19), + (0x7ffdc, 19), + (0x7ffdd, 19), + (0x7ffde, 19), + (0x7ffd8, 19), + (0x7ffd2, 19), + (0x7ffd3, 19), + (0x7ffd4, 19), + (0x7ffd5, 19), + (0x7ffd6, 19), + (0x7fff2, 19), + (0x7ffdf, 19), + (0x7ffe7, 19), + (0x7ffe8, 19), + (0x7ffe9, 19), + (0x7ffea, 19), + (0x7ffeb, 19), + (0x7ffe6, 19), + (0x7ffe0, 19), + (0x7ffe1, 19), + (0x7ffe2, 19), + (0x7ffe3, 19), + (0x7ffe4, 19), + (0x7ffe5, 19), + (0x7ffd7, 19), + (0x7ffec, 19), + (0x7fff4, 19), + (0x7fff3, 19), +]; + +pub(crate) static SPECTRUM_1: [(u32, u8); 81] = [ + (0x7f8, 11), + (0x1f1, 9), + (0x7fd, 11), + (0x3f5, 10), + (0x68, 7), + (0x3f0, 10), + (0x7f7, 11), + (0x1ec, 9), + (0x7f5, 11), + (0x3f1, 10), + (0x72, 7), + (0x3f4, 10), + (0x74, 7), + (0x11, 5), + (0x76, 7), + (0x1eb, 9), + (0x6c, 7), + (0x3f6, 10), + (0x7fc, 11), + (0x1e1, 9), + (0x7f1, 11), + (0x1f0, 9), + (0x61, 7), + (0x1f6, 9), + (0x7f2, 11), + (0x1ea, 9), + (0x7fb, 11), + (0x1f2, 9), + (0x69, 7), + (0x1ed, 9), + (0x77, 7), + (0x17, 5), + (0x6f, 7), + (0x1e6, 9), + (0x64, 7), + (0x1e5, 9), + (0x67, 7), + (0x15, 5), + (0x62, 7), + (0x12, 5), + (0x0, 1), + (0x14, 5), + (0x65, 7), + (0x16, 5), + (0x6d, 7), + (0x1e9, 9), + (0x63, 7), + (0x1e4, 9), + (0x6b, 7), + (0x13, 5), + (0x71, 7), + (0x1e3, 9), + (0x70, 7), + (0x1f3, 9), + (0x7fe, 11), + (0x1e7, 9), + (0x7f3, 11), + (0x1ef, 9), + (0x60, 7), + (0x1ee, 9), + (0x7f0, 11), + (0x1e2, 9), + (0x7fa, 11), + (0x3f3, 10), + (0x6a, 7), + (0x1e8, 9), + (0x75, 7), + (0x10, 5), + (0x73, 7), + (0x1f4, 9), + (0x6e, 7), + (0x3f7, 10), + (0x7f6, 11), + (0x1e0, 9), + (0x7f9, 11), + (0x3f2, 10), + (0x66, 7), + (0x1f5, 9), + (0x7ff, 11), + (0x1f7, 9), + (0x7f4, 11), +]; + +pub(crate) static SPECTRUM_2: [(u32, u8); 81] = [ + (0x1f3, 9), + (0x6f, 7), + (0x1fd, 9), + (0xeb, 8), + (0x23, 6), + (0xea, 8), + (0x1f7, 9), + (0xe8, 8), + (0x1fa, 9), + (0xf2, 8), + (0x2d, 6), + (0x70, 7), + (0x20, 6), + (0x6, 5), + (0x2b, 6), + (0x6e, 7), + (0x28, 6), + (0xe9, 8), + (0x1f9, 9), + (0x66, 7), + (0xf8, 8), + (0xe7, 8), + (0x1b, 6), + (0xf1, 8), + (0x1f4, 9), + (0x6b, 7), + (0x1f5, 9), + (0xec, 8), + (0x2a, 6), + (0x6c, 7), + (0x2c, 6), + (0xa, 5), + (0x27, 6), + (0x67, 7), + (0x1a, 6), + (0xf5, 8), + (0x24, 6), + (0x8, 5), + (0x1f, 6), + (0x9, 5), + (0x0, 3), + (0x7, 5), + (0x1d, 6), + (0xb, 5), + (0x30, 6), + (0xef, 8), + (0x1c, 6), + (0x64, 7), + (0x1e, 6), + (0xc, 5), + (0x29, 6), + (0xf3, 8), + (0x2f, 6), + (0xf0, 8), + (0x1fc, 9), + (0x71, 7), + (0x1f2, 9), + (0xf4, 8), + (0x21, 6), + (0xe6, 8), + (0xf7, 8), + (0x68, 7), + (0x1f8, 9), + (0xee, 8), + (0x22, 6), + (0x65, 7), + (0x31, 6), + (0x2, 4), + (0x26, 6), + (0xed, 8), + (0x25, 6), + (0x6a, 7), + (0x1fb, 9), + (0x72, 7), + (0x1fe, 9), + (0x69, 7), + (0x2e, 6), + (0xf6, 8), + (0x1ff, 9), + (0x6d, 7), + (0x1f6, 9), +]; + +pub(crate) static SPECTRUM_3: [(u32, u8); 81] = [ + (0x0, 1), + (0x9, 4), + (0xef, 8), + (0xb, 4), + (0x19, 5), + (0xf0, 8), + (0x1eb, 9), + (0x1e6, 9), + (0x3f2, 10), + (0xa, 4), + (0x35, 6), + (0x1ef, 9), + (0x34, 6), + (0x37, 6), + (0x1e9, 9), + (0x1ed, 9), + (0x1e7, 9), + (0x3f3, 10), + (0x1ee, 9), + (0x3ed, 10), + (0x1ffa, 13), + (0x1ec, 9), + (0x1f2, 9), + (0x7f9, 11), + (0x7f8, 11), + (0x3f8, 10), + (0xff8, 12), + (0x8, 4), + (0x38, 6), + (0x3f6, 10), + (0x36, 6), + (0x75, 7), + (0x3f1, 10), + (0x3eb, 10), + (0x3ec, 10), + (0xff4, 12), + (0x18, 5), + (0x76, 7), + (0x7f4, 11), + (0x39, 6), + (0x74, 7), + (0x3ef, 10), + (0x1f3, 9), + (0x1f4, 9), + (0x7f6, 11), + (0x1e8, 9), + (0x3ea, 10), + (0x1ffc, 13), + (0xf2, 8), + (0x1f1, 9), + (0xffb, 12), + (0x3f5, 10), + (0x7f3, 11), + (0xffc, 12), + (0xee, 8), + (0x3f7, 10), + (0x7ffe, 15), + (0x1f0, 9), + (0x7f5, 11), + (0x7ffd, 15), + (0x1ffb, 13), + (0x3ffa, 14), + (0xffff, 16), + (0xf1, 8), + (0x3f0, 10), + (0x3ffc, 14), + (0x1ea, 9), + (0x3ee, 10), + (0x3ffb, 14), + (0xff6, 12), + (0xffa, 12), + (0x7ffc, 15), + (0x7f2, 11), + (0xff5, 12), + (0xfffe, 16), + (0x3f4, 10), + (0x7f7, 11), + (0x7ffb, 15), + (0xff7, 12), + (0xff9, 12), + (0x7ffa, 15), +]; + +pub(crate) static SPECTRUM_4: [(u32, u8); 81] = [ + (0x7, 4), + (0x16, 5), + (0xf6, 8), + (0x18, 5), + (0x8, 4), + (0xef, 8), + (0x1ef, 9), + (0xf3, 8), + (0x7f8, 11), + (0x19, 5), + (0x17, 5), + (0xed, 8), + (0x15, 5), + (0x1, 4), + (0xe2, 8), + (0xf0, 8), + (0x70, 7), + (0x3f0, 10), + (0x1ee, 9), + (0xf1, 8), + (0x7fa, 11), + (0xee, 8), + (0xe4, 8), + (0x3f2, 10), + (0x7f6, 11), + (0x3ef, 10), + (0x7fd, 11), + (0x5, 4), + (0x14, 5), + (0xf2, 8), + (0x9, 4), + (0x4, 4), + (0xe5, 8), + (0xf4, 8), + (0xe8, 8), + (0x3f4, 10), + (0x6, 4), + (0x2, 4), + (0xe7, 8), + (0x3, 4), + (0x0, 4), + (0x6b, 7), + (0xe3, 8), + (0x69, 7), + (0x1f3, 9), + (0xeb, 8), + (0xe6, 8), + (0x3f6, 10), + (0x6e, 7), + (0x6a, 7), + (0x1f4, 9), + (0x3ec, 10), + (0x1f0, 9), + (0x3f9, 10), + (0xf5, 8), + (0xec, 8), + (0x7fb, 11), + (0xea, 8), + (0x6f, 7), + (0x3f7, 10), + (0x7f9, 11), + (0x3f3, 10), + (0xfff, 12), + (0xe9, 8), + (0x6d, 7), + (0x3f8, 10), + (0x6c, 7), + (0x68, 7), + (0x1f5, 9), + (0x3ee, 10), + (0x1f2, 9), + (0x7f4, 11), + (0x7f7, 11), + (0x3f1, 10), + (0xffe, 12), + (0x3ed, 10), + (0x1f1, 9), + (0x7f5, 11), + (0x7fe, 11), + (0x3f5, 10), + (0x7fc, 11), +]; + +pub(crate) static SPECTRUM_5: [(u32, u8); 81] = [ + (0x1fff, 13), + (0xff7, 12), + (0x7f4, 11), + (0x7e8, 11), + (0x3f1, 10), + (0x7ee, 11), + (0x7f9, 11), + (0xff8, 12), + (0x1ffd, 13), + (0xffd, 12), + (0x7f1, 11), + (0x3e8, 10), + (0x1e8, 9), + (0xf0, 8), + (0x1ec, 9), + (0x3ee, 10), + (0x7f2, 11), + (0xffa, 12), + (0xff4, 12), + (0x3ef, 10), + (0x1f2, 9), + (0xe8, 8), + (0x70, 7), + (0xec, 8), + (0x1f0, 9), + (0x3ea, 10), + (0x7f3, 11), + (0x7eb, 11), + (0x1eb, 9), + (0xea, 8), + (0x1a, 5), + (0x8, 4), + (0x19, 5), + (0xee, 8), + (0x1ef, 9), + (0x7ed, 11), + (0x3f0, 10), + (0xf2, 8), + (0x73, 7), + (0xb, 4), + (0x0, 1), + (0xa, 4), + (0x71, 7), + (0xf3, 8), + (0x7e9, 11), + (0x7ef, 11), + (0x1ee, 9), + (0xef, 8), + (0x18, 5), + (0x9, 4), + (0x1b, 5), + (0xeb, 8), + (0x1e9, 9), + (0x7ec, 11), + (0x7f6, 11), + (0x3eb, 10), + (0x1f3, 9), + (0xed, 8), + (0x72, 7), + (0xe9, 8), + (0x1f1, 9), + (0x3ed, 10), + (0x7f7, 11), + (0xff6, 12), + (0x7f0, 11), + (0x3e9, 10), + (0x1ed, 9), + (0xf1, 8), + (0x1ea, 9), + (0x3ec, 10), + (0x7f8, 11), + (0xff9, 12), + (0x1ffc, 13), + (0xffc, 12), + (0xff5, 12), + (0x7ea, 11), + (0x3f3, 10), + (0x3f2, 10), + (0x7f5, 11), + (0xffb, 12), + (0x1ffe, 13), +]; + +pub(crate) static SPECTRUM_6: [(u32, u8); 81] = [ + (0x7fe, 11), + (0x3fd, 10), + (0x1f1, 9), + (0x1eb, 9), + (0x1f4, 9), + (0x1ea, 9), + (0x1f0, 9), + (0x3fc, 10), + (0x7fd, 11), + (0x3f6, 10), + (0x1e5, 9), + (0xea, 8), + (0x6c, 7), + (0x71, 7), + (0x68, 7), + (0xf0, 8), + (0x1e6, 9), + (0x3f7, 10), + (0x1f3, 9), + (0xef, 8), + (0x32, 6), + (0x27, 6), + (0x28, 6), + (0x26, 6), + (0x31, 6), + (0xeb, 8), + (0x1f7, 9), + (0x1e8, 9), + (0x6f, 7), + (0x2e, 6), + (0x8, 4), + (0x4, 4), + (0x6, 4), + (0x29, 6), + (0x6b, 7), + (0x1ee, 9), + (0x1ef, 9), + (0x72, 7), + (0x2d, 6), + (0x2, 4), + (0x0, 4), + (0x3, 4), + (0x2f, 6), + (0x73, 7), + (0x1fa, 9), + (0x1e7, 9), + (0x6e, 7), + (0x2b, 6), + (0x7, 4), + (0x1, 4), + (0x5, 4), + (0x2c, 6), + (0x6d, 7), + (0x1ec, 9), + (0x1f9, 9), + (0xee, 8), + (0x30, 6), + (0x24, 6), + (0x2a, 6), + (0x25, 6), + (0x33, 6), + (0xec, 8), + (0x1f2, 9), + (0x3f8, 10), + (0x1e4, 9), + (0xed, 8), + (0x6a, 7), + (0x70, 7), + (0x69, 7), + (0x74, 7), + (0xf1, 8), + (0x3fa, 10), + (0x7ff, 11), + (0x3f9, 10), + (0x1f6, 9), + (0x1ed, 9), + (0x1f8, 9), + (0x1e9, 9), + (0x1f5, 9), + (0x3fb, 10), + (0x7fc, 11), +]; + +pub(crate) static SPECTRUM_7: [(u32, u8); 64] = [ + (0x0, 1), + (0x5, 3), + (0x37, 6), + (0x74, 7), + (0xf2, 8), + (0x1eb, 9), + (0x3ed, 10), + (0x7f7, 11), + (0x4, 3), + (0xc, 4), + (0x35, 6), + (0x71, 7), + (0xec, 8), + (0xee, 8), + (0x1ee, 9), + (0x1f5, 9), + (0x36, 6), + (0x34, 6), + (0x72, 7), + (0xea, 8), + (0xf1, 8), + (0x1e9, 9), + (0x1f3, 9), + (0x3f5, 10), + (0x73, 7), + (0x70, 7), + (0xeb, 8), + (0xf0, 8), + (0x1f1, 9), + (0x1f0, 9), + (0x3ec, 10), + (0x3fa, 10), + (0xf3, 8), + (0xed, 8), + (0x1e8, 9), + (0x1ef, 9), + (0x3ef, 10), + (0x3f1, 10), + (0x3f9, 10), + (0x7fb, 11), + (0x1ed, 9), + (0xef, 8), + (0x1ea, 9), + (0x1f2, 9), + (0x3f3, 10), + (0x3f8, 10), + (0x7f9, 11), + (0x7fc, 11), + (0x3ee, 10), + (0x1ec, 9), + (0x1f4, 9), + (0x3f4, 10), + (0x3f7, 10), + (0x7f8, 11), + (0xffd, 12), + (0xffe, 12), + (0x7f6, 11), + (0x3f0, 10), + (0x3f2, 10), + (0x3f6, 10), + (0x7fa, 11), + (0x7fd, 11), + (0xffc, 12), + (0xfff, 12), +]; + +pub(crate) static SPECTRUM_8: [(u32, u8); 64] = [ + (0xe, 5), + (0x5, 4), + (0x10, 5), + (0x30, 6), + (0x6f, 7), + (0xf1, 8), + (0x1fa, 9), + (0x3fe, 10), + (0x3, 4), + (0x0, 3), + (0x4, 4), + (0x12, 5), + (0x2c, 6), + (0x6a, 7), + (0x75, 7), + (0xf8, 8), + (0xf, 5), + (0x2, 4), + (0x6, 4), + (0x14, 5), + (0x2e, 6), + (0x69, 7), + (0x72, 7), + (0xf5, 8), + (0x2f, 6), + (0x11, 5), + (0x13, 5), + (0x2a, 6), + (0x32, 6), + (0x6c, 7), + (0xec, 8), + (0xfa, 8), + (0x71, 7), + (0x2b, 6), + (0x2d, 6), + (0x31, 6), + (0x6d, 7), + (0x70, 7), + (0xf2, 8), + (0x1f9, 9), + (0xef, 8), + (0x68, 7), + (0x33, 6), + (0x6b, 7), + (0x6e, 7), + (0xee, 8), + (0xf9, 8), + (0x3fc, 10), + (0x1f8, 9), + (0x74, 7), + (0x73, 7), + (0xed, 8), + (0xf0, 8), + (0xf6, 8), + (0x1f6, 9), + (0x1fd, 9), + (0x3fd, 10), + (0xf3, 8), + (0xf4, 8), + (0xf7, 8), + (0x1f7, 9), + (0x1fb, 9), + (0x1fc, 9), + (0x3ff, 10), +]; + +pub(crate) static SPECTRUM_9: [(u32, u8); 169] = [ + (0x0, 1), + (0x5, 3), + (0x37, 6), + (0xe7, 8), + (0x1de, 9), + (0x3ce, 10), + (0x3d9, 10), + (0x7c8, 11), + (0x7cd, 11), + (0xfc8, 12), + (0xfdd, 12), + (0x1fe4, 13), + (0x1fec, 13), + (0x4, 3), + (0xc, 4), + (0x35, 6), + (0x72, 7), + (0xea, 8), + (0xed, 8), + (0x1e2, 9), + (0x3d1, 10), + (0x3d3, 10), + (0x3e0, 10), + (0x7d8, 11), + (0xfcf, 12), + (0xfd5, 12), + (0x36, 6), + (0x34, 6), + (0x71, 7), + (0xe8, 8), + (0xec, 8), + (0x1e1, 9), + (0x3cf, 10), + (0x3dd, 10), + (0x3db, 10), + (0x7d0, 11), + (0xfc7, 12), + (0xfd4, 12), + (0xfe4, 12), + (0xe6, 8), + (0x70, 7), + (0xe9, 8), + (0x1dd, 9), + (0x1e3, 9), + (0x3d2, 10), + (0x3dc, 10), + (0x7cc, 11), + (0x7ca, 11), + (0x7de, 11), + (0xfd8, 12), + (0xfea, 12), + (0x1fdb, 13), + (0x1df, 9), + (0xeb, 8), + (0x1dc, 9), + (0x1e6, 9), + (0x3d5, 10), + (0x3de, 10), + (0x7cb, 11), + (0x7dd, 11), + (0x7dc, 11), + (0xfcd, 12), + (0xfe2, 12), + (0xfe7, 12), + (0x1fe1, 13), + (0x3d0, 10), + (0x1e0, 9), + (0x1e4, 9), + (0x3d6, 10), + (0x7c5, 11), + (0x7d1, 11), + (0x7db, 11), + (0xfd2, 12), + (0x7e0, 11), + (0xfd9, 12), + (0xfeb, 12), + (0x1fe3, 13), + (0x1fe9, 13), + (0x7c4, 11), + (0x1e5, 9), + (0x3d7, 10), + (0x7c6, 11), + (0x7cf, 11), + (0x7da, 11), + (0xfcb, 12), + (0xfda, 12), + (0xfe3, 12), + (0xfe9, 12), + (0x1fe6, 13), + (0x1ff3, 13), + (0x1ff7, 13), + (0x7d3, 11), + (0x3d8, 10), + (0x3e1, 10), + (0x7d4, 11), + (0x7d9, 11), + (0xfd3, 12), + (0xfde, 12), + (0x1fdd, 13), + (0x1fd9, 13), + (0x1fe2, 13), + (0x1fea, 13), + (0x1ff1, 13), + (0x1ff6, 13), + (0x7d2, 11), + (0x3d4, 10), + (0x3da, 10), + (0x7c7, 11), + (0x7d7, 11), + (0x7e2, 11), + (0xfce, 12), + (0xfdb, 12), + (0x1fd8, 13), + (0x1fee, 13), + (0x3ff0, 14), + (0x1ff4, 13), + (0x3ff2, 14), + (0x7e1, 11), + (0x3df, 10), + (0x7c9, 11), + (0x7d6, 11), + (0xfca, 12), + (0xfd0, 12), + (0xfe5, 12), + (0xfe6, 12), + (0x1feb, 13), + (0x1fef, 13), + (0x3ff3, 14), + (0x3ff4, 14), + (0x3ff5, 14), + (0xfe0, 12), + (0x7ce, 11), + (0x7d5, 11), + (0xfc6, 12), + (0xfd1, 12), + (0xfe1, 12), + (0x1fe0, 13), + (0x1fe8, 13), + (0x1ff0, 13), + (0x3ff1, 14), + (0x3ff8, 14), + (0x3ff6, 14), + (0x7ffc, 15), + (0xfe8, 12), + (0x7df, 11), + (0xfc9, 12), + (0xfd7, 12), + (0xfdc, 12), + (0x1fdc, 13), + (0x1fdf, 13), + (0x1fed, 13), + (0x1ff5, 13), + (0x3ff9, 14), + (0x3ffb, 14), + (0x7ffd, 15), + (0x7ffe, 15), + (0x1fe7, 13), + (0xfcc, 12), + (0xfd6, 12), + (0xfdf, 12), + (0x1fde, 13), + (0x1fda, 13), + (0x1fe5, 13), + (0x1ff2, 13), + (0x3ffa, 14), + (0x3ff7, 14), + (0x3ffc, 14), + (0x3ffd, 14), + (0x7fff, 15), +]; + +pub(crate) static SPECTRUM_10: [(u32, u8); 169] = [ + (0x22, 6), + (0x8, 5), + (0x1d, 6), + (0x26, 6), + (0x5f, 7), + (0xd3, 8), + (0x1cf, 9), + (0x3d0, 10), + (0x3d7, 10), + (0x3ed, 10), + (0x7f0, 11), + (0x7f6, 11), + (0xffd, 12), + (0x7, 5), + (0x0, 4), + (0x1, 4), + (0x9, 5), + (0x20, 6), + (0x54, 7), + (0x60, 7), + (0xd5, 8), + (0xdc, 8), + (0x1d4, 9), + (0x3cd, 10), + (0x3de, 10), + (0x7e7, 11), + (0x1c, 6), + (0x2, 4), + (0x6, 5), + (0xc, 5), + (0x1e, 6), + (0x28, 6), + (0x5b, 7), + (0xcd, 8), + (0xd9, 8), + (0x1ce, 9), + (0x1dc, 9), + (0x3d9, 10), + (0x3f1, 10), + (0x25, 6), + (0xb, 5), + (0xa, 5), + (0xd, 5), + (0x24, 6), + (0x57, 7), + (0x61, 7), + (0xcc, 8), + (0xdd, 8), + (0x1cc, 9), + (0x1de, 9), + (0x3d3, 10), + (0x3e7, 10), + (0x5d, 7), + (0x21, 6), + (0x1f, 6), + (0x23, 6), + (0x27, 6), + (0x59, 7), + (0x64, 7), + (0xd8, 8), + (0xdf, 8), + (0x1d2, 9), + (0x1e2, 9), + (0x3dd, 10), + (0x3ee, 10), + (0xd1, 8), + (0x55, 7), + (0x29, 6), + (0x56, 7), + (0x58, 7), + (0x62, 7), + (0xce, 8), + (0xe0, 8), + (0xe2, 8), + (0x1da, 9), + (0x3d4, 10), + (0x3e3, 10), + (0x7eb, 11), + (0x1c9, 9), + (0x5e, 7), + (0x5a, 7), + (0x5c, 7), + (0x63, 7), + (0xca, 8), + (0xda, 8), + (0x1c7, 9), + (0x1ca, 9), + (0x1e0, 9), + (0x3db, 10), + (0x3e8, 10), + (0x7ec, 11), + (0x1e3, 9), + (0xd2, 8), + (0xcb, 8), + (0xd0, 8), + (0xd7, 8), + (0xdb, 8), + (0x1c6, 9), + (0x1d5, 9), + (0x1d8, 9), + (0x3ca, 10), + (0x3da, 10), + (0x7ea, 11), + (0x7f1, 11), + (0x1e1, 9), + (0xd4, 8), + (0xcf, 8), + (0xd6, 8), + (0xde, 8), + (0xe1, 8), + (0x1d0, 9), + (0x1d6, 9), + (0x3d1, 10), + (0x3d5, 10), + (0x3f2, 10), + (0x7ee, 11), + (0x7fb, 11), + (0x3e9, 10), + (0x1cd, 9), + (0x1c8, 9), + (0x1cb, 9), + (0x1d1, 9), + (0x1d7, 9), + (0x1df, 9), + (0x3cf, 10), + (0x3e0, 10), + (0x3ef, 10), + (0x7e6, 11), + (0x7f8, 11), + (0xffa, 12), + (0x3eb, 10), + (0x1dd, 9), + (0x1d3, 9), + (0x1d9, 9), + (0x1db, 9), + (0x3d2, 10), + (0x3cc, 10), + (0x3dc, 10), + (0x3ea, 10), + (0x7ed, 11), + (0x7f3, 11), + (0x7f9, 11), + (0xff9, 12), + (0x7f2, 11), + (0x3ce, 10), + (0x1e4, 9), + (0x3cb, 10), + (0x3d8, 10), + (0x3d6, 10), + (0x3e2, 10), + (0x3e5, 10), + (0x7e8, 11), + (0x7f4, 11), + (0x7f5, 11), + (0x7f7, 11), + (0xffb, 12), + (0x7fa, 11), + (0x3ec, 10), + (0x3df, 10), + (0x3e1, 10), + (0x3e4, 10), + (0x3e6, 10), + (0x3f0, 10), + (0x7e9, 11), + (0x7ef, 11), + (0xff8, 12), + (0xffe, 12), + (0xffc, 12), + (0xfff, 12), +]; + +pub(crate) static SPECTRUM_11: [(u32, u8); 289] = [ + (0x0, 4), + (0x6, 5), + (0x19, 6), + (0x3d, 7), + (0x9c, 8), + (0xc6, 8), + (0x1a7, 9), + (0x390, 10), + (0x3c2, 10), + (0x3df, 10), + (0x7e6, 11), + (0x7f3, 11), + (0xffb, 12), + (0x7ec, 11), + (0xffa, 12), + (0xffe, 12), + (0x38e, 10), + (0x5, 5), + (0x1, 4), + (0x8, 5), + (0x14, 6), + (0x37, 7), + (0x42, 7), + (0x92, 8), + (0xaf, 8), + (0x191, 9), + (0x1a5, 9), + (0x1b5, 9), + (0x39e, 10), + (0x3c0, 10), + (0x3a2, 10), + (0x3cd, 10), + (0x7d6, 11), + (0xae, 8), + (0x17, 6), + (0x7, 5), + (0x9, 5), + (0x18, 6), + (0x39, 7), + (0x40, 7), + (0x8e, 8), + (0xa3, 8), + (0xb8, 8), + (0x199, 9), + (0x1ac, 9), + (0x1c1, 9), + (0x3b1, 10), + (0x396, 10), + (0x3be, 10), + (0x3ca, 10), + (0x9d, 8), + (0x3c, 7), + (0x15, 6), + (0x16, 6), + (0x1a, 6), + (0x3b, 7), + (0x44, 7), + (0x91, 8), + (0xa5, 8), + (0xbe, 8), + (0x196, 9), + (0x1ae, 9), + (0x1b9, 9), + (0x3a1, 10), + (0x391, 10), + (0x3a5, 10), + (0x3d5, 10), + (0x94, 8), + (0x9a, 8), + (0x36, 7), + (0x38, 7), + (0x3a, 7), + (0x41, 7), + (0x8c, 8), + (0x9b, 8), + (0xb0, 8), + (0xc3, 8), + (0x19e, 9), + (0x1ab, 9), + (0x1bc, 9), + (0x39f, 10), + (0x38f, 10), + (0x3a9, 10), + (0x3cf, 10), + (0x93, 8), + (0xbf, 8), + (0x3e, 7), + (0x3f, 7), + (0x43, 7), + (0x45, 7), + (0x9e, 8), + (0xa7, 8), + (0xb9, 8), + (0x194, 9), + (0x1a2, 9), + (0x1ba, 9), + (0x1c3, 9), + (0x3a6, 10), + (0x3a7, 10), + (0x3bb, 10), + (0x3d4, 10), + (0x9f, 8), + (0x1a0, 9), + (0x8f, 8), + (0x8d, 8), + (0x90, 8), + (0x98, 8), + (0xa6, 8), + (0xb6, 8), + (0xc4, 8), + (0x19f, 9), + (0x1af, 9), + (0x1bf, 9), + (0x399, 10), + (0x3bf, 10), + (0x3b4, 10), + (0x3c9, 10), + (0x3e7, 10), + (0xa8, 8), + (0x1b6, 9), + (0xab, 8), + (0xa4, 8), + (0xaa, 8), + (0xb2, 8), + (0xc2, 8), + (0xc5, 8), + (0x198, 9), + (0x1a4, 9), + (0x1b8, 9), + (0x38c, 10), + (0x3a4, 10), + (0x3c4, 10), + (0x3c6, 10), + (0x3dd, 10), + (0x3e8, 10), + (0xad, 8), + (0x3af, 10), + (0x192, 9), + (0xbd, 8), + (0xbc, 8), + (0x18e, 9), + (0x197, 9), + (0x19a, 9), + (0x1a3, 9), + (0x1b1, 9), + (0x38d, 10), + (0x398, 10), + (0x3b7, 10), + (0x3d3, 10), + (0x3d1, 10), + (0x3db, 10), + (0x7dd, 11), + (0xb4, 8), + (0x3de, 10), + (0x1a9, 9), + (0x19b, 9), + (0x19c, 9), + (0x1a1, 9), + (0x1aa, 9), + (0x1ad, 9), + (0x1b3, 9), + (0x38b, 10), + (0x3b2, 10), + (0x3b8, 10), + (0x3ce, 10), + (0x3e1, 10), + (0x3e0, 10), + (0x7d2, 11), + (0x7e5, 11), + (0xb7, 8), + (0x7e3, 11), + (0x1bb, 9), + (0x1a8, 9), + (0x1a6, 9), + (0x1b0, 9), + (0x1b2, 9), + (0x1b7, 9), + (0x39b, 10), + (0x39a, 10), + (0x3ba, 10), + (0x3b5, 10), + (0x3d6, 10), + (0x7d7, 11), + (0x3e4, 10), + (0x7d8, 11), + (0x7ea, 11), + (0xba, 8), + (0x7e8, 11), + (0x3a0, 10), + (0x1bd, 9), + (0x1b4, 9), + (0x38a, 10), + (0x1c4, 9), + (0x392, 10), + (0x3aa, 10), + (0x3b0, 10), + (0x3bc, 10), + (0x3d7, 10), + (0x7d4, 11), + (0x7dc, 11), + (0x7db, 11), + (0x7d5, 11), + (0x7f0, 11), + (0xc1, 8), + (0x7fb, 11), + (0x3c8, 10), + (0x3a3, 10), + (0x395, 10), + (0x39d, 10), + (0x3ac, 10), + (0x3ae, 10), + (0x3c5, 10), + (0x3d8, 10), + (0x3e2, 10), + (0x3e6, 10), + (0x7e4, 11), + (0x7e7, 11), + (0x7e0, 11), + (0x7e9, 11), + (0x7f7, 11), + (0x190, 9), + (0x7f2, 11), + (0x393, 10), + (0x1be, 9), + (0x1c0, 9), + (0x394, 10), + (0x397, 10), + (0x3ad, 10), + (0x3c3, 10), + (0x3c1, 10), + (0x3d2, 10), + (0x7da, 11), + (0x7d9, 11), + (0x7df, 11), + (0x7eb, 11), + (0x7f4, 11), + (0x7fa, 11), + (0x195, 9), + (0x7f8, 11), + (0x3bd, 10), + (0x39c, 10), + (0x3ab, 10), + (0x3a8, 10), + (0x3b3, 10), + (0x3b9, 10), + (0x3d0, 10), + (0x3e3, 10), + (0x3e5, 10), + (0x7e2, 11), + (0x7de, 11), + (0x7ed, 11), + (0x7f1, 11), + (0x7f9, 11), + (0x7fc, 11), + (0x193, 9), + (0xffd, 12), + (0x3dc, 10), + (0x3b6, 10), + (0x3c7, 10), + (0x3cc, 10), + (0x3cb, 10), + (0x3d9, 10), + (0x3da, 10), + (0x7d3, 11), + (0x7e1, 11), + (0x7ee, 11), + (0x7ef, 11), + (0x7f5, 11), + (0x7f6, 11), + (0xffc, 12), + (0xfff, 12), + (0x19d, 9), + (0x1c2, 9), + (0xb5, 8), + (0xa1, 8), + (0x96, 8), + (0x97, 8), + (0x95, 8), + (0x99, 8), + (0xa0, 8), + (0xa2, 8), + (0xac, 8), + (0xa9, 8), + (0xb1, 8), + (0xb3, 8), + (0xbb, 8), + (0xc0, 8), + (0x18f, 9), + (0x4, 5), +]; diff --git a/crates/audio/src/aac/ics.rs b/crates/audio/src/aac/ics.rs new file mode 100644 index 000000000..f50b8929c --- /dev/null +++ b/crates/audio/src/aac/ics.rs @@ -0,0 +1,151 @@ +//! `ics_info` and helpers shared by the encoder and decoder (ISO/IEC 14496-3 §4.4.2.1, §4.5.2.3). + +use craft_media::bitstream::{BitReader, BitWriter}; + +use crate::aac::mdct::WindowShape; +use crate::aac::tables::{swb_offsets_long, swb_offsets_short}; +use crate::aac::{Error, Result}; + +/// `window_sequence`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum WindowSequence { + #[default] + OnlyLong = 0, + LongStart = 1, + EightShort = 2, + LongStop = 3, +} + +impl WindowSequence { + pub fn from_bits(v: u32) -> Self { + match v & 3 { + 0 => WindowSequence::OnlyLong, + 1 => WindowSequence::LongStart, + 2 => WindowSequence::EightShort, + _ => WindowSequence::LongStop, + } + } +} + +/// Decoded/encoded `ics_info`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct IcsInfo { + pub window_sequence: WindowSequence, + pub window_shape: WindowShape, + pub max_sfb: usize, + pub num_groups: usize, + /// Windows per group (only the first `num_groups` entries are used; long windows: `[1]`). + pub group_len: [u8; 8], +} + +impl Default for IcsInfo { + fn default() -> Self { + IcsInfo { + window_sequence: WindowSequence::OnlyLong, + window_shape: WindowShape::Sine, + max_sfb: 0, + num_groups: 1, + group_len: [1, 0, 0, 0, 0, 0, 0, 0], + } + } +} + +impl IcsInfo { + #[inline] + pub fn is_short(&self) -> bool { + self.window_sequence == WindowSequence::EightShort + } + #[inline] + pub fn num_windows(&self) -> usize { + if self.is_short() { 8 } else { 1 } + } + /// Band offsets of one window. + pub fn swb_offsets(&self, sf_index: u8) -> &'static [u16] { + if self.is_short() { swb_offsets_short(sf_index) } else { swb_offsets_long(sf_index) } + } + /// Number of bands for this window length (`num_swb`). + pub fn num_swb(&self, sf_index: u8) -> usize { + self.swb_offsets(sf_index).len() - 1 + } + /// First window of group `g`. + pub fn group_start(&self, g: usize) -> usize { + self.group_len[..g].iter().map(|&l| l as usize).sum() + } + + pub fn parse(br: &mut BitReader, sf_index: u8) -> Result { + if br.read_bits(1)? != 0 { + // ics_reserved_bit; tolerated by most decoders + } + let window_sequence = WindowSequence::from_bits(br.read_bits(2)?); + let window_shape = WindowShape::from_bit(br.read_bits(1)? == 1); + let mut info = IcsInfo { window_sequence, window_shape, ..Default::default() }; + if info.is_short() { + info.max_sfb = br.read_bits(4)? as usize; + let grouping = br.read_bits(7)?; + info.num_groups = 1; + info.group_len = [1, 0, 0, 0, 0, 0, 0, 0]; + for w in 1..8 { + if grouping & (1 << (6 - (w - 1))) != 0 { + info.group_len[info.num_groups - 1] += 1; + } else { + info.num_groups += 1; + info.group_len[info.num_groups - 1] = 1; + } + } + } else { + info.max_sfb = br.read_bits(6)? as usize; + if br.read_bits(1)? == 1 { + return Err(Error::Unsupported("predictor data (AAC Main / LTP)")); + } + } + if info.max_sfb > info.num_swb(sf_index) { + return Err(Error::Bitstream("max_sfb exceeds the number of scalefactor bands")); + } + Ok(info) + } + + pub fn write(&self, bw: &mut BitWriter) { + bw.write_bits(0, 1); + bw.write_bits(self.window_sequence as u32, 2); + bw.write_bits(self.window_shape.bit(), 1); + if self.is_short() { + bw.write_bits(self.max_sfb as u32, 4); + let mut grouping = 0u32; + let mut w = 0; + for g in 0..self.num_groups { + for i in 0..self.group_len[g] as usize { + if w > 0 && i > 0 { + grouping |= 1 << (6 - (w - 1)); + } + w += 1; + } + } + bw.write_bits(grouping, 7); + } else { + bw.write_bits(self.max_sfb as u32, 6); + bw.write_bits(0, 1); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn grouping_roundtrip() { + let info = IcsInfo { + window_sequence: WindowSequence::EightShort, + window_shape: WindowShape::Kbd, + max_sfb: 12, + num_groups: 3, + group_len: [3, 1, 4, 0, 0, 0, 0, 0], + }; + let mut bw = BitWriter::new(); + info.write(&mut bw); + let d = bw.finish(); + let p = IcsInfo::parse(&mut BitReader::new(&d), 4).unwrap(); + assert_eq!(p, info); + assert_eq!(p.group_start(2), 4); + } +} diff --git a/crates/audio/src/aac/mdct.rs b/crates/audio/src/aac/mdct.rs new file mode 100644 index 000000000..b829030da --- /dev/null +++ b/crates/audio/src/aac/mdct.rs @@ -0,0 +1,278 @@ +//! MDCT / IMDCT (ISO/IEC 14496-3 §4.6.18) via a DCT-IV computed with an N/4-point complex FFT, +//! plus the sine and Kaiser–Bessel-derived windows. + +use std::f64::consts::PI; +use std::sync::OnceLock; + +/// Radix-2 complex FFT (forward, `e^{-2πi nk/P}`). +struct Fft { + n: usize, + twiddle: Vec<[f32; 2]>, + rev: Vec, +} + +impl Fft { + fn new(n: usize) -> Fft { + assert!(n.is_power_of_two()); + let bits = n.trailing_zeros(); + let rev = (0..n as u32).map(|i| if bits == 0 { 0 } else { i.reverse_bits() >> (32 - bits) }).collect(); + let twiddle = (0..n / 2) + .map(|k| { + let a = -2.0 * PI * k as f64 / n as f64; + [a.cos() as f32, a.sin() as f32] + }) + .collect(); + Fft { n, twiddle, rev } + } + + fn run(&self, re: &mut [f32], im: &mut [f32]) { + let n = self.n; + for i in 0..n { + let j = self.rev[i] as usize; + if j > i { + re.swap(i, j); + im.swap(i, j); + } + } + let mut len = 2; + while len <= n { + let half = len / 2; + let step = n / len; + for start in (0..n).step_by(len) { + for k in 0..half { + let [wr, wi] = self.twiddle[k * step]; + let a = start + k; + let b = a + half; + let xr = re[b] * wr - im[b] * wi; + let xi = re[b] * wi + im[b] * wr; + re[b] = re[a] - xr; + im[b] = im[a] - xi; + re[a] += xr; + im[a] += xi; + } + } + len <<= 1; + } + } +} + +/// MDCT of `2M` inputs to `M` outputs (unscaled sums): +/// `X[k] = Σ x[n] cos(π/M (n + ½ + M/2)(k + ½))`, and the matching unscaled inverse. +pub struct Mdct { + m: usize, + fft: Fft, + pre: Vec<[f32; 2]>, + post: Vec<[f32; 2]>, +} + +impl Mdct { + pub fn new(m: usize) -> Mdct { + let q = m / 2; + let pre = (0..q) + .map(|n| { + let a = -PI * n as f64 / m as f64; + [a.cos() as f32, a.sin() as f32] + }) + .collect(); + let post = (0..q) + .map(|k| { + let a = -PI * (k as f64 + 0.25) / m as f64; + [a.cos() as f32, a.sin() as f32] + }) + .collect(); + Mdct { m, fft: Fft::new(q), pre, post } + } + + /// DCT-IV: `out[k] = Σ u[n] cos(π/M (n+½)(k+½))`. + fn dct4(&self, u: &[f32], out: &mut [f32]) { + let m = self.m; + let q = m / 2; + let mut re = vec![0f32; q]; + let mut im = vec![0f32; q]; + for n in 0..q { + let (a, b) = (u[2 * n], u[m - 1 - 2 * n]); + let [c, s] = self.pre[n]; + re[n] = a * c - b * s; + im[n] = a * s + b * c; + } + self.fft.run(&mut re, &mut im); + for k in 0..q { + let [c, s] = self.post[k]; + let wr = re[k] * c - im[k] * s; + let wi = re[k] * s + im[k] * c; + out[2 * k] = wr; + out[m - 1 - 2 * k] = -wi; + } + } + + /// Forward MDCT of `x` (length 2M, already windowed) into `out` (length M). + pub fn forward(&self, x: &[f32], out: &mut [f32]) { + let m = self.m; + let h = m / 2; + let mut u = vec![0f32; m]; + for n in 0..h { + // quarters a, b, c, d of length M/2 + u[n] = -x[m + h - 1 - n] - x[m + h + n]; + u[h + n] = x[n] - x[m - 1 - n]; + } + self.dct4(&u, out); + } + + /// Inverse MDCT (unscaled): `y[n] = Σ X[k] cos(π/M (n + ½ + M/2)(k + ½))`, `y` has length 2M. + pub fn inverse(&self, spec: &[f32], y: &mut [f32]) { + let m = self.m; + let h = m / 2; + let mut u = vec![0f32; m]; + self.dct4(spec, &mut u); + for n in 0..h { + y[n] = u[h + n]; + y[m - 1 - n] = -u[h + n]; + y[m + h - 1 - n] = -u[n]; + y[m + h + n] = -u[n]; + } + } +} + +/// Cached transforms for 1024 and 128 lines. +pub fn mdct_long() -> &'static Mdct { + static M: OnceLock = OnceLock::new(); + M.get_or_init(|| Mdct::new(1024)) +} +pub fn mdct_short() -> &'static Mdct { + static M: OnceLock = OnceLock::new(); + M.get_or_init(|| Mdct::new(128)) +} + +/// Window shape (`window_shape` bit): 0 = sine, 1 = Kaiser–Bessel derived. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum WindowShape { + /// Sine window (`window_shape` 0). + #[default] + Sine, + /// Kaiser–Bessel derived window (`window_shape` 1). + Kbd, +} + +impl WindowShape { + /// The shape signalled by a `window_shape` bit. + pub fn from_bit(b: bool) -> Self { + if b { WindowShape::Kbd } else { WindowShape::Sine } + } + /// The `window_shape` bit for this shape. + pub fn bit(self) -> u32 { + (self == WindowShape::Kbd) as u32 + } +} + +fn sine_half(m: usize) -> Vec { + let n = 2 * m; + (0..m).map(|i| (PI / n as f64 * (i as f64 + 0.5)).sin() as f32).collect() +} + +fn bessel_i0(x: f64) -> f64 { + let mut sum = 1.0; + let mut term = 1.0; + let y = x * x / 4.0; + for k in 1..60 { + term *= y / (k as f64 * k as f64); + sum += term; + if term < sum * 1e-17 { + break; + } + } + sum +} + +fn kbd_half(m: usize, alpha: f64) -> Vec { + // w[n] = sqrt(Σ_{j<=n} W(j) / Σ_{j<=M} W(j)), W the Kaiser kernel of length M+1. + let kernel: Vec = (0..=m) + .map(|j| { + let r = (j as f64 - m as f64 / 2.0) / (m as f64 / 2.0); + bessel_i0(PI * alpha * (1.0 - r * r).max(0.0).sqrt()) + }) + .collect(); + let total: f64 = kernel.iter().sum(); + let mut acc = 0.0; + (0..m) + .map(|n| { + acc += kernel[n]; + (acc / total).sqrt() as f32 + }) + .collect() +} + +/// Rising half (length N/2) of the window of the given shape; `long` selects N = 2048, else N = 256. +pub fn window(shape: WindowShape, long: bool) -> &'static [f32] { + static W: OnceLock<[Vec; 4]> = OnceLock::new(); + let w = W.get_or_init(|| [sine_half(1024), kbd_half(1024, 4.0), sine_half(128), kbd_half(128, 6.0)]); + let i = match (long, shape) { + (true, WindowShape::Sine) => 0, + (true, WindowShape::Kbd) => 1, + (false, WindowShape::Sine) => 2, + (false, WindowShape::Kbd) => 3, + }; + &w[i] +} + +#[cfg(test)] +mod tests { + use super::*; + + fn direct(x: &[f32]) -> Vec { + let m = x.len() / 2; + (0..m).map(|k| (0..2 * m).map(|n| x[n] as f64 * (PI / m as f64 * (n as f64 + 0.5 + m as f64 / 2.0) * (k as f64 + 0.5)).cos()).sum()).collect() + } + + #[test] + fn forward_matches_definition() { + for m in [128usize, 1024] { + let x: Vec = (0..2 * m).map(|i| ((i * 7919) % 101) as f32 / 50.0 - 1.0).collect(); + let mut out = vec![0f32; m]; + Mdct::new(m).forward(&x, &mut out); + let d = direct(&x); + for k in 0..m { + assert!((out[k] as f64 - d[k]).abs() < 1e-3 * (m as f64).sqrt(), "m={m} k={k} {} {}", out[k], d[k]); + } + } + } + + #[test] + fn inverse_matches_definition() { + let m = 128; + let spec: Vec = (0..m).map(|i| ((i * 31) % 17) as f32 - 8.0).collect(); + let mut y = vec![0f32; 2 * m]; + Mdct::new(m).inverse(&spec, &mut y); + for n in 0..2 * m { + let d: f64 = (0..m).map(|k| spec[k] as f64 * (PI / m as f64 * (n as f64 + 0.5 + m as f64 / 2.0) * (k as f64 + 0.5)).cos()).sum(); + assert!((y[n] as f64 - d).abs() < 1e-3, "n={n}"); + } + } + + #[test] + fn tdac_perfect_reconstruction() { + let m = 1024; + let t = Mdct::new(m); + for shape in [WindowShape::Sine, WindowShape::Kbd] { + let w = window(shape, true); + let win = |n: usize| if n < m { w[n] } else { w[2 * m - 1 - n] }; + let x: Vec = (0..4 * m).map(|i| ((i as f32) * 0.013).sin() + ((i * 13) % 7) as f32 * 0.05).collect(); + let mut out = vec![0f32; 4 * m]; + for f in 0..3 { + let seg: Vec = (0..2 * m).map(|n| x[f * m + n] * win(n)).collect(); + let mut spec = vec![0f32; m]; + t.forward(&seg, &mut spec); + for v in spec.iter_mut() { + *v *= 2.0; + } + let mut y = vec![0f32; 2 * m]; + t.inverse(&spec, &mut y); + for n in 0..2 * m { + out[f * m + n] += y[n] * win(n) / m as f32; + } + } + for n in m..3 * m { + assert!((out[n] - x[n]).abs() < 1e-4, "{shape:?} n={n} {} {}", out[n], x[n]); + } + } + } +} diff --git a/crates/audio/src/aac/mod.rs b/crates/audio/src/aac/mod.rs new file mode 100644 index 000000000..23aef9361 --- /dev/null +++ b/crates/audio/src/aac/mod.rs @@ -0,0 +1,221 @@ +//! Clean-room AAC-LC decoder (ISO/IEC 14496-3, MPEG-4 Audio, Low Complexity profile). +//! +//! [`Decoder`] (feature `aac-decode`): `AudioSpecificConfig` + raw access units → planar f32. +//! Supports M/S, intensity stereo, PNS, TNS, pulse data, both window shapes and every window +//! sequence. +//! +//! HE-AAC (SBR/PS), Main/SSR/LTP profiles, coupling channels and 960-sample frames are not +//! supported; for HE-AAC only the AAC-LC core is decoded (at the core rate; [`Decoder::sbr`] says +//! when). +//! +//! From FilmCraft's `filmcraft-aac` (storytold/filmcraft@8fda2f2). Public items keep their names, +//! so FilmCraft can re-export this module. + +// Index loops follow the specification's pseudo-code (FilmCraft allows this lint workspace-wide). +#![allow(clippy::needless_range_loop)] +// Tables, Huffman coding, MDCT, window and TNS code shared by the decoder and the encoder, as +// ported; the encoder's half is unused until the encoder is added. +#![allow(dead_code)] + +mod config; +mod decoder; +mod huffman; +mod huffman_tables; +mod ics; +mod mdct; +mod tables; +mod tns; + +#[cfg(feature = "aac-decode")] +pub(crate) use config::split_adts_upto; +pub use config::{AdtsHeader, AudioSpecificConfig, ElementType, ProgramConfig, config_layout, split_adts}; +pub use decoder::Decoder; +pub use mdct::WindowShape; +pub use tables::{SAMPLE_RATES, sample_rate_index}; + +/// Samples per AAC-LC frame (per channel). +pub const FRAME_LEN: usize = 1024; + +/// Errors from configuration parsing and decoding. +#[non_exhaustive] +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum Error { + /// The configuration is malformed or outside the supported range. + #[error("invalid configuration: {0}")] + InvalidConfig(&'static str), + /// A valid stream uses a feature this crate does not implement. + #[error("unsupported feature: {0}")] + Unsupported(&'static str), + /// Malformed bitstream data. + #[error("invalid bitstream: {0}")] + Bitstream(&'static str), + /// The bit reader ran past the end of the data. + #[error("unexpected end of access unit")] + Eof, +} + +impl From for Error { + fn from(_: craft_media::bitstream::BitError) -> Self { + Error::Eof + } +} + +/// Result with this module's [`Error`]. +pub type Result = std::result::Result; + +/// Channel order for AAC surround (ISO/IEC 14496-3 Table 1.19, the decoder's element order: C, L R, +/// surrounds, LFE): output channel → decoded channel, giving the WAV/SMPTE order (L R C LFE Ls Rs) +/// the rest of craft-audio uses. Only the unambiguous channel configurations 3–6, 11 (6.1: … LFE, +/// back centre, sides) and 12 (7.1: … LFE, backs, sides); 1, 2, 7 and PCE layouts keep the decoded +/// order. From FilmCraft's `filmcraft-codecs`. +pub fn wav_order(channel_config: u8) -> Option<&'static [usize]> { + Some(match channel_config { + 3 => &[1, 2, 0], + 4 => &[1, 2, 0, 3], + 5 => &[1, 2, 0, 3, 4], + 6 => &[1, 2, 0, 5, 3, 4], + 11 => &[1, 2, 0, 6, 5, 3, 4], + 12 => &[1, 2, 0, 7, 5, 6, 3, 4], + _ => return None, + }) +} + +/// The output rate of an AAC stream: twice the core rate for HE-AAC (ISO/IEC 14496-3 §1.6.5). +/// Explicit signalling names the rate in the `AudioSpecificConfig`; with implicit signalling (an +/// AAC-LC config at 24 kHz or less) SBR is only visible as fill-element data in the access units, +/// so the first ones are decoded to look for it. `None` if `asc` is not one the decoder reads. +/// From FilmCraft's `filmcraft-codecs`. +#[cfg(feature = "aac-decode")] +pub fn output_rate<'a>(asc: &[u8], first_units: impl IntoIterator) -> Option { + let mut dec = Decoder::new(asc).ok()?; + let core = dec.sample_rate(); + if let Some(ext) = dec.config().extension_sample_rate { + return Some(ext); + } + if core <= 24_000 { + for au in first_units { + let _ = dec.decode(au); + if dec.sbr() { + return Some(core * 2); + } + } + } + Some(core) +} + +/// Taps on each side of the interpolation point in [`Upsample2x`]. +#[cfg(feature = "aac-decode")] +const UPSAMPLE_HALF: usize = 12; + +/// Input samples [`Upsample2x`] delays its output by: 481, the delay of the SBR filterbanks (962 +/// output samples), which HE-AAC encoders and the container's priming count on. +#[cfg(feature = "aac-decode")] +const UPSAMPLE_DELAY: usize = 481; + +/// Input samples [`Upsample2x`] keeps per channel. +#[cfg(feature = "aac-decode")] +const UPSAMPLE_HISTORY: usize = UPSAMPLE_DELAY + UPSAMPLE_HALF; + +/// 2× upsampler for HE-AAC. The decoder reconstructs the AAC-LC core at half the output rate and +/// not yet the SBR high band, so the core is interpolated to the output rate: timing and pitch are +/// right and the band above the core's Nyquist stays empty. Even outputs are the input samples, +/// odd ones a Blackman-windowed sinc half-band interpolation over 2 × `UPSAMPLE_HALF` inputs. +/// The output is delayed by `UPSAMPLE_DELAY` input samples, as an SBR decoder's is; the state +/// carries across packets, so a seek primed with the preceding packet gives the same samples as +/// continuous decoding. From FilmCraft's `filmcraft-codecs`. +#[cfg(feature = "aac-decode")] +#[derive(Clone)] +// Internal to HE-AAC output; public only so FilmCraft can re-export this module. +#[doc(hidden)] +pub struct Upsample2x { + taps: [f32; 2 * UPSAMPLE_HALF], + /// The last `UPSAMPLE_HISTORY` input samples per channel. + hist: Vec>, +} + +#[cfg(feature = "aac-decode")] +impl Upsample2x { + /// An upsampler for `channels` channels with empty history. + pub fn new(channels: usize) -> Self { + let m = UPSAMPLE_HALF as f64; + let mut taps = [0f32; 2 * UPSAMPLE_HALF]; + let mut sum = 0.0; + for (k, t) in taps.iter_mut().enumerate() { + // distance from the interpolation point (halfway between two inputs) + let x = k as f64 - m + 0.5; + let sinc = (std::f64::consts::PI * x).sin() / (std::f64::consts::PI * x); + let w = 0.42 + 0.5 * (std::f64::consts::PI * x / m).cos() + 0.08 * (2.0 * std::f64::consts::PI * x / m).cos(); + *t = (sinc * w) as f32; + sum += sinc * w; + } + for t in &mut taps { + *t = (*t as f64 / sum) as f32; + } + Self { taps, hist: vec![vec![0.0; UPSAMPLE_HISTORY]; channels] } + } + + /// Clear the history (e.g. after a seek). + pub fn reset(&mut self) { + for h in &mut self.hist { + h.fill(0.0); + } + } + + /// Upsample planar channels by two: each output channel has twice the input samples. + pub fn process(&mut self, input: Vec>) -> Vec> { + // A changed channel count is a new layout: history of channels that were absent meanwhile + // (or that held other speakers) would put a step into the output. + if self.hist.len() != input.len() { + self.hist = vec![vec![0.0; UPSAMPLE_HISTORY]; input.len()]; + } + let m = UPSAMPLE_HALF; + input + .into_iter() + .zip(&mut self.hist) + .map(|(x, hist)| { + let mut buf = std::mem::take(hist); + buf.extend_from_slice(&x); + let mut out = Vec::with_capacity(x.len() * 2); + // output pair i is centred on buf[i + m]: the sample itself, then the point halfway + // to the next one from buf[i + 1 ..= i + 2m] + for w in buf.windows(2 * m + 1).take(x.len()) { + out.push(w.get(m).copied().unwrap_or(0.0)); + out.push(w.get(1..).unwrap_or_default().iter().zip(&self.taps).map(|(a, b)| a * b).sum()); + } + *hist = buf.split_off(buf.len().saturating_sub(UPSAMPLE_HISTORY)); + out + }) + .collect() + } +} + +#[cfg(all(test, feature = "aac-decode"))] +mod tests { + use super::*; + + /// The output lags the input by the SBR filterbanks' delay, whatever the block size. + #[test] + fn the_upsampler_delays_by_the_sbr_filterbanks() { + let mut impulse = vec![0.0; 2000]; + impulse[100] = 1.0; + let mut up = Upsample2x::new(1); + let mut out = Vec::new(); + for block in impulse.chunks(300) { + out.extend(up.process(vec![block.to_vec()]).remove(0)); + } + assert_eq!(out.len(), 4000); + assert_eq!(UPSAMPLE_DELAY, 481); + assert_eq!(out[(100 + 481) * 2], 1.0); + assert!(out[..(100 + 481 - UPSAMPLE_HALF) * 2].iter().all(|&s| s == 0.0)); + // a change of channel count starts each channel's history afresh + let mut fresh = Upsample2x::new(2); + let block = vec![vec![0.5; 1024], vec![0.25; 1024]]; + let mut up = Upsample2x::new(2); + up.process(block.clone()); + up.process(vec![vec![0.5; 1024]]); + assert_eq!(up.process(block.clone()), fresh.process(block)); + // a reset forgets the history + up.reset(); + assert!(up.process(vec![vec![0.0; 600]]).remove(0).iter().all(|&s| s == 0.0)); + } +} diff --git a/crates/audio/src/aac/tables.rs b/crates/audio/src/aac/tables.rs new file mode 100644 index 000000000..6dc6fb6f8 --- /dev/null +++ b/crates/audio/src/aac/tables.rs @@ -0,0 +1,145 @@ +//! Sampling-rate dependent tables from ISO/IEC 14496-3 §4.5.4 (scalefactor bands) and §4.6.9 (TNS). + +/// Sampling frequencies indexed by `samplingFrequencyIndex`. +pub const SAMPLE_RATES: [u32; 13] = [96000, 88200, 64000, 48000, 44100, 32000, 24000, 22050, 16000, 12000, 11025, 8000, 7350]; + +/// Map an arbitrary sample rate to its `samplingFrequencyIndex`, if it is one of the standard rates. +// Public for FilmCraft's re-export only. +#[doc(hidden)] +pub fn sample_rate_index(rate: u32) -> Option { + SAMPLE_RATES.iter().position(|&r| r == rate).map(|i| i as u8) +} + +/// The table index used for scalefactor-band tables when a non-standard rate is signalled explicitly +/// (§4.5.4: rates are mapped to the nearest table by frequency ranges). +pub fn table_index_for_rate(rate: u32) -> u8 { + match rate { + 92017.. => 0, + 75132.. => 1, + 55426.. => 2, + 46009.. => 3, + 37566.. => 4, + 27713.. => 5, + 23004.. => 6, + 18783.. => 7, + 13856.. => 8, + 11502.. => 9, + 9391.. => 10, + _ => 11, + } +} + +const SWB_LONG_96: &[u16] = &[ + 0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 64, 72, 80, 88, 96, 108, 120, 132, 144, 156, 172, 188, 212, 240, 276, 320, 384, 448, + 512, 576, 640, 704, 768, 832, 896, 960, 1024, +]; +const SWB_LONG_64: &[u16] = &[ + 0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 64, 72, 80, 88, 100, 112, 124, 140, 156, 172, 192, 216, 240, 268, 304, 344, 384, 424, + 464, 504, 544, 584, 624, 664, 704, 744, 784, 824, 864, 904, 944, 984, 1024, +]; +const SWB_LONG_48: &[u16] = &[ + 0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 48, 56, 64, 72, 80, 88, 96, 108, 120, 132, 144, 160, 176, 196, 216, 240, 264, 292, 320, 352, 384, 416, + 448, 480, 512, 544, 576, 608, 640, 672, 704, 736, 768, 800, 832, 864, 896, 928, 1024, +]; +const SWB_LONG_32: &[u16] = &[ + 0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 48, 56, 64, 72, 80, 88, 96, 108, 120, 132, 144, 160, 176, 196, 216, 240, 264, 292, 320, 352, 384, 416, + 448, 480, 512, 544, 576, 608, 640, 672, 704, 736, 768, 800, 832, 864, 896, 928, 960, 992, 1024, +]; +const SWB_LONG_24: &[u16] = &[ + 0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 52, 60, 68, 76, 84, 92, 100, 108, 116, 124, 136, 148, 160, 172, 188, 204, 220, 240, 260, 284, 308, + 336, 364, 396, 432, 468, 508, 552, 600, 652, 704, 768, 832, 896, 960, 1024, +]; +const SWB_LONG_16: &[u16] = &[ + 0, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 100, 112, 124, 136, 148, 160, 172, 184, 196, 212, 228, 244, 260, 280, 300, 320, 344, 368, 396, 424, + 456, 492, 532, 572, 616, 664, 716, 772, 832, 896, 960, 1024, +]; +const SWB_LONG_8: &[u16] = &[ + 0, 12, 24, 36, 48, 60, 72, 84, 96, 108, 120, 132, 144, 156, 172, 188, 204, 220, 236, 252, 268, 288, 308, 328, 348, 372, 396, 420, 448, 476, 508, + 544, 580, 620, 664, 712, 764, 820, 880, 944, 1024, +]; + +const SWB_SHORT_96: &[u16] = &[0, 4, 8, 12, 16, 20, 24, 32, 40, 48, 64, 92, 128]; +const SWB_SHORT_48: &[u16] = &[0, 4, 8, 12, 16, 20, 28, 36, 44, 56, 68, 80, 96, 112, 128]; +const SWB_SHORT_24: &[u16] = &[0, 4, 8, 12, 16, 20, 24, 28, 36, 44, 52, 64, 76, 92, 108, 128]; +const SWB_SHORT_16: &[u16] = &[0, 4, 8, 12, 16, 20, 24, 28, 32, 40, 48, 60, 72, 88, 108, 128]; +const SWB_SHORT_8: &[u16] = &[0, 4, 8, 12, 16, 20, 24, 28, 36, 44, 52, 60, 72, 88, 108, 128]; + +/// Scalefactor band offsets (`swb_offset_long_window`, 1024-line frames); `len() == num_swb + 1`. +pub fn swb_offsets_long(sf_index: u8) -> &'static [u16] { + match sf_index { + 0 | 1 => SWB_LONG_96, + 2 => SWB_LONG_64, + 3 | 4 => SWB_LONG_48, + 5 => SWB_LONG_32, + 6 | 7 => SWB_LONG_24, + 8..=10 => SWB_LONG_16, + _ => SWB_LONG_8, + } +} + +/// Scalefactor band offsets for one short window (128 lines). +pub fn swb_offsets_short(sf_index: u8) -> &'static [u16] { + match sf_index { + 0..=2 => SWB_SHORT_96, + 3..=5 => SWB_SHORT_48, + 6 | 7 => SWB_SHORT_24, + 8..=10 => SWB_SHORT_16, + _ => SWB_SHORT_8, + } +} + +/// `TNS_MAX_BANDS` for AAC-LC (long, short). +pub fn tns_max_bands(sf_index: u8, short: bool) -> usize { + const LONG: [u8; 13] = [31, 31, 34, 40, 42, 51, 46, 46, 42, 42, 42, 39, 39]; + const SHORT: [u8; 13] = [9, 9, 10, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14]; + let i = (sf_index as usize).min(12); + if short { SHORT[i] as usize } else { LONG[i] as usize } +} + +/// `TNS_MAX_ORDER` for AAC-LC. +pub fn tns_max_order(short: bool) -> usize { + if short { 7 } else { 12 } +} + +/// `x^(4/3)` for the integer quantised magnitudes 0..=8191. +pub fn pow43(q: u32) -> f32 { + use std::sync::OnceLock; + static TABLE: OnceLock> = OnceLock::new(); + let t = TABLE.get_or_init(|| (0..8192u32).map(|i| (i as f64).powf(4.0 / 3.0) as f32).collect()); + match t.get(q as usize) { + Some(v) => *v, + None => (q as f64).powf(4.0 / 3.0) as f32, + } +} + +/// `2^(0.25 * (sf - 100))`, the dequantisation gain of scalefactor `sf`. +#[inline] +pub fn sf_gain(sf: i32) -> f32 { + (0.25 * (sf as f64 - 100.0)).exp2() as f32 +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn band_tables_are_monotonic_and_complete() { + for i in 0..13u8 { + let l = swb_offsets_long(i); + assert_eq!(*l.last().unwrap(), 1024); + assert!(l.windows(2).all(|w| w[0] < w[1] && (w[1] - w[0]) % 4 == 0)); + let s = swb_offsets_short(i); + assert_eq!(*s.last().unwrap(), 128); + assert!(s.windows(2).all(|w| w[0] < w[1] && (w[1] - w[0]) % 4 == 0)); + assert!(tns_max_bands(i, false) < l.len()); + assert!(tns_max_bands(i, true) < s.len()); + } + assert_eq!(swb_offsets_long(3).len() - 1, 49); + assert_eq!(swb_offsets_long(5).len() - 1, 51); + assert_eq!(swb_offsets_long(0).len() - 1, 41); + assert_eq!(swb_offsets_long(2).len() - 1, 47); + assert_eq!(swb_offsets_long(6).len() - 1, 47); + assert_eq!(swb_offsets_long(8).len() - 1, 43); + assert_eq!(swb_offsets_long(11).len() - 1, 40); + } +} diff --git a/crates/audio/src/aac/tns.rs b/crates/audio/src/aac/tns.rs new file mode 100644 index 000000000..a538207c7 --- /dev/null +++ b/crates/audio/src/aac/tns.rs @@ -0,0 +1,313 @@ +//! Temporal noise shaping (ISO/IEC 14496-3 §4.6.9): syntax, coefficient conversion, and the +//! decoder (all-pole) and encoder (all-zero) filters. + +use std::f64::consts::FRAC_PI_2; + +use craft_media::bitstream::{BitReader, BitWriter}; + +use crate::aac::ics::IcsInfo; +use crate::aac::tables::{tns_max_bands, tns_max_order}; +use crate::aac::{Error, Result}; + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct TnsFilter { + /// Length in scalefactor bands. + pub length: u8, + pub order: u8, + /// false = upward, true = downward. + pub direction: bool, + pub coef_compress: bool, + /// Quantised reflection coefficients (signed, `coef_res + 3 - coef_compress` bits each). + pub coef: [i8; 12], +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct TnsWindow { + pub n_filt: u8, + /// false = 3-bit, true = 4-bit coefficient resolution. + pub coef_res: bool, + pub filt: [TnsFilter; 4], +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct TnsData { + pub windows: [TnsWindow; 8], +} + +impl TnsData { + pub fn parse(br: &mut BitReader, info: &IcsInfo) -> Result { + let short = info.is_short(); + let mut t = TnsData::default(); + for w in 0..info.num_windows() { + let win = &mut t.windows[w]; + win.n_filt = br.read_bits(if short { 1 } else { 2 })? as u8; + if win.n_filt == 0 { + continue; + } + win.coef_res = br.read_bits(1)? == 1; + for f in 0..win.n_filt as usize { + let filt = &mut win.filt[f]; + filt.length = br.read_bits(if short { 4 } else { 6 })? as u8; + filt.order = br.read_bits(if short { 3 } else { 5 })? as u8; + if filt.order as usize > tns_max_order(short) { + return Err(Error::Bitstream("TNS order too large")); + } + if filt.order > 0 { + filt.direction = br.read_bits(1)? == 1; + filt.coef_compress = br.read_bits(1)? == 1; + let bits = 3 + win.coef_res as u32 - filt.coef_compress as u32; + for i in 0..filt.order as usize { + let v = br.read_bits(bits)? as i32; + let v = if v & (1 << (bits - 1)) != 0 { v - (1 << bits) } else { v }; + filt.coef[i] = v as i8; + } + } + } + } + Ok(t) + } + + pub fn write(&self, bw: &mut BitWriter, info: &IcsInfo) { + let short = info.is_short(); + for w in 0..info.num_windows() { + let win = &self.windows[w]; + bw.write_bits(win.n_filt as u32, if short { 1 } else { 2 }); + if win.n_filt == 0 { + continue; + } + bw.write_bits(win.coef_res as u32, 1); + for f in 0..win.n_filt as usize { + let filt = &win.filt[f]; + bw.write_bits(filt.length as u32, if short { 4 } else { 6 }); + bw.write_bits(filt.order as u32, if short { 3 } else { 5 }); + if filt.order > 0 { + bw.write_bits(filt.direction as u32, 1); + bw.write_bits(filt.coef_compress as u32, 1); + let bits = 3 + win.coef_res as u32 - filt.coef_compress as u32; + for i in 0..filt.order as usize { + bw.write_bits(filt.coef[i] as i32 as u32 & ((1 << bits) - 1), bits); + } + } + } + } + } + + /// Bits `write` would produce. + pub fn bit_count(&self, info: &IcsInfo) -> u32 { + let short = info.is_short(); + let mut n = 0; + for w in 0..info.num_windows() { + let win = &self.windows[w]; + n += if short { 1 } else { 2 }; + if win.n_filt == 0 { + continue; + } + n += 1; + for f in 0..win.n_filt as usize { + let filt = &win.filt[f]; + n += if short { 7 } else { 11 }; + if filt.order > 0 { + n += 2 + filt.order as u32 * (3 + win.coef_res as u32 - filt.coef_compress as u32); + } + } + } + n + } +} + +/// Dequantise one reflection coefficient. +pub fn dequant_coef(c: i8, coef_res: bool) -> f64 { + let res = 3 + coef_res as i32; + let iqfac = ((1 << (res - 1)) as f64 - 0.5) / FRAC_PI_2; + let iqfac_m = ((1 << (res - 1)) as f64 + 0.5) / FRAC_PI_2; + let c = c as f64; + (c / if c >= 0.0 { iqfac } else { iqfac_m }).sin() +} + +/// Quantise a reflection coefficient (inverse of [`dequant_coef`], nearest index). +pub fn quant_coef(k: f64, coef_res: bool) -> i8 { + let res = 3 + coef_res as i32; + let iqfac = ((1 << (res - 1)) as f64 - 0.5) / FRAC_PI_2; + let iqfac_m = ((1 << (res - 1)) as f64 + 0.5) / FRAC_PI_2; + let a = k.clamp(-1.0, 1.0).asin(); + let lo = -(1 << (res - 1)); + let hi = (1 << (res - 1)) - 1; + let q = if a >= 0.0 { (a * iqfac).round() } else { (a * iqfac_m).round() }; + (q as i32).clamp(lo, hi) as i8 +} + +/// Reflection coefficients → direct-form LPC `a[0..=order]` (`a[0] = 1`). +pub fn parcor_to_lpc(k: &[f64]) -> [f64; 13] { + let mut a = [0f64; 13]; + a[0] = 1.0; + let mut b = [0f64; 13]; + for m in 1..=k.len() { + for i in 1..m { + b[i] = a[i] + k[m - 1] * a[m - i]; + } + a[1..m].copy_from_slice(&b[1..m]); + a[m] = k[m - 1]; + } + a +} + +/// Filter region `[start, end)` of every TNS filter of window `w`, in spectrum lines of that window. +fn regions(info: &IcsInfo, sf_index: u8, win: &TnsWindow) -> Vec<(usize, usize, usize)> { + let short = info.is_short(); + let swb = info.swb_offsets(sf_index); + let num_swb = swb.len() - 1; + let lim = tns_max_bands(sf_index, short).min(info.max_sfb); + let mut out = Vec::new(); + let mut bottom = num_swb; + for f in 0..win.n_filt as usize { + let filt = &win.filt[f]; + let top = bottom; + bottom = top.saturating_sub(filt.length as usize); + if filt.order == 0 { + continue; + } + let start = swb[bottom.min(lim)] as usize; + let end = swb[top.min(lim)] as usize; + if end > start { + out.push((f, start, end)); + } + } + out +} + +fn lpc_of(filt: &TnsFilter, coef_res: bool) -> [f64; 13] { + let k: Vec = filt.coef[..filt.order as usize].iter().map(|&c| dequant_coef(c, coef_res)).collect(); + parcor_to_lpc(&k) +} + +/// Decoder: apply the all-pole TNS synthesis filters to `spec` (1024 lines; short windows at 128 stride). +pub fn apply_decoder(spec: &mut [f32], info: &IcsInfo, sf_index: u8, tns: &TnsData) { + let wlen = if info.is_short() { 128 } else { 1024 }; + for w in 0..info.num_windows() { + let win = &tns.windows[w]; + for (f, start, end) in regions(info, sf_index, win) { + let filt = &win.filt[f]; + let a = lpc_of(filt, win.coef_res); + let order = filt.order as usize; + let s = &mut spec[w * wlen..(w + 1) * wlen]; + let size = end - start; + let mut state = [0f64; 12]; + for m in 0..size { + let idx = if filt.direction { end - 1 - m } else { start + m }; + let mut y = s[idx] as f64; + for i in 0..order { + y -= a[i + 1] * state[i]; + } + for i in (1..order).rev() { + state[i] = state[i - 1]; + } + if order > 0 { + state[0] = y; + } + s[idx] = y as f32; + } + } + } +} + +/// Encoder: apply the all-zero TNS analysis filters (exact inverse of [`apply_decoder`]). +pub fn apply_encoder(spec: &mut [f32], info: &IcsInfo, sf_index: u8, tns: &TnsData) { + let wlen = if info.is_short() { 128 } else { 1024 }; + for w in 0..info.num_windows() { + let win = &tns.windows[w]; + for (f, start, end) in regions(info, sf_index, win) { + let filt = &win.filt[f]; + let a = lpc_of(filt, win.coef_res); + let order = filt.order as usize; + let s = &mut spec[w * wlen..(w + 1) * wlen]; + let size = end - start; + let mut state = [0f64; 12]; + for m in 0..size { + let idx = if filt.direction { end - 1 - m } else { start + m }; + let x = s[idx] as f64; + let mut y = x; + for i in 0..order { + y += a[i + 1] * state[i]; + } + for i in (1..order).rev() { + state[i] = state[i - 1]; + } + if order > 0 { + state[0] = x; + } + s[idx] = y as f32; + } + } + } +} + +/// Levinson–Durbin on autocorrelation `r[0..=order]`: returns reflection coefficients and the +/// prediction gain `r[0] / error`. +pub fn levinson(r: &[f64], order: usize) -> (Vec, f64) { + let mut a = vec![0f64; order + 1]; + a[0] = 1.0; + let mut err = r[0]; + let mut ks = Vec::with_capacity(order); + if err <= 0.0 { + return (vec![0.0; order], 1.0); + } + for m in 1..=order { + let mut acc = r[m]; + for i in 1..m { + acc += a[i] * r[m - i]; + } + let k = -acc / err; + let prev = a.clone(); + for i in 1..m { + a[i] = prev[i] + k * prev[m - i]; + } + a[m] = k; + err *= 1.0 - k * k; + ks.push(k); + if err <= r[0] * 1e-9 { + ks.resize(order, 0.0); + break; + } + } + (ks, r[0] / err.max(r[0] * 1e-9)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::aac::ics::WindowSequence; + use crate::aac::mdct::WindowShape; + + #[test] + fn coef_quant_roundtrip() { + for res in [false, true] { + let n = if res { 8 } else { 4 }; + for c in -n..n { + let k = dequant_coef(c as i8, res); + assert_eq!(quant_coef(k, res), c as i8); + } + } + } + + #[test] + fn encoder_decoder_filters_are_inverse() { + let info = IcsInfo { window_sequence: WindowSequence::OnlyLong, window_shape: WindowShape::Sine, max_sfb: 49, ..Default::default() }; + let mut t = TnsData::default(); + t.windows[0].n_filt = 1; + t.windows[0].coef_res = true; + t.windows[0].filt[0] = TnsFilter { length: 40, order: 4, direction: true, coef_compress: false, coef: [3, -2, 1, 5, 0, 0, 0, 0, 0, 0, 0, 0] }; + let orig: Vec = (0..1024).map(|i| ((i * 37 % 101) as f32 - 50.0) * 0.1).collect(); + let mut s = orig.clone(); + apply_encoder(&mut s, &info, 4, &t); + assert_ne!(s, orig); + apply_decoder(&mut s, &info, 4, &t); + for i in 0..1024 { + assert!((s[i] - orig[i]).abs() < 1e-3); + } + let mut bw = BitWriter::new(); + t.write(&mut bw, &info); + assert_eq!(bw.bit_len() as u32, t.bit_count(&info)); + let d = bw.finish(); + assert_eq!(TnsData::parse(&mut BitReader::new(&d), &info).unwrap(), t); + } +} diff --git a/crates/audio/src/layout.rs b/crates/audio/src/layout.rs index 5cd054ca8..24945c41e 100644 --- a/crates/audio/src/layout.rs +++ b/crates/audio/src/layout.rs @@ -145,7 +145,7 @@ impl TryFrom for ChannelLayout { } /// A channel count as the API holds it, saturating at `u16::MAX`. -#[cfg(any(feature = "decode-native", feature = "pcm-encode"))] +#[cfg(any(feature = "aac-decode", feature = "pcm-encode"))] pub(crate) fn count(n: usize) -> u16 { u16::try_from(n).unwrap_or(u16::MAX) } diff --git a/crates/audio/src/lib.rs b/crates/audio/src/lib.rs index b85a7611f..18ff06060 100644 --- a/crates/audio/src/lib.rs +++ b/crates/audio/src/lib.rs @@ -8,29 +8,33 @@ //! `wasm32-unknown-unknown`. //! //! - [`decode`], [`probe`], [`Decoder`] (streaming, seekable), [`tracks`], [`detect_container`] -//! (feature `decode`): WAV and AIFF are read natively; PCM in MP4/MOV and Matroska/WebM goes -//! through craft-media's demuxers and [`TrackDecoder`]; everything else (AAC, MP1/MP2/MP3, -//! FLAC, ALAC, Vorbis, CAF, ADPCM, A-law, µ-law) through `symphonia`, which is to be replaced -//! by native decoders. Feature `decode-native` is the same without symphonia. -//! - [`TrackDecoder`] (feature `decode-native`): one track's packets from any demuxer to planar +//! (feature `decode`): WAV and AIFF are read natively; AAC and PCM in MP4/MOV, +//! Matroska/WebM or raw ADTS streams go through craft-media's demuxers and +//! [`TrackDecoder`]; everything else (MP1/MP2/MP3, FLAC, ALAC, Vorbis, CAF, ADPCM, A-law, +//! µ-law) through `symphonia`, which is to be replaced by native decoders. Feature +//! `decode-native` is the same without symphonia. +//! - [`TrackDecoder`] (any decoder feature): one track's packets from any demuxer to planar //! `f32` in WAV channel order. //! - [`encode`], [`FileEncoder`]: WAV/BWF/RF64 and AIFF/AIFF-C (feature `pcm-encode`, which also //! gives [`wav_float_header`]) and FLAC (feature `flac-encode`), whole-buffer or incremental into //! any `Write + Seek`. Feature `lossless-encode` turns on both; feature `encode` is //! the same as `lossless-encode`. //! - [`peaks`] (feature `peaks`): min/max/RMS pyramid for waveform drawing. +//! - Codec modules for packet-level use, each with its own features: [`aac`] (`aac-decode`). //! //! The shared types below build with no features. +#[cfg(feature = "aac-decode")] +pub mod aac; mod buffer; mod layout; -#[cfg(feature = "decode-native")] +#[cfg(feature = "aac-decode")] mod pcm; #[cfg(feature = "peaks")] pub mod peaks; #[cfg(feature = "decode-native")] mod read; -#[cfg(feature = "decode-native")] +#[cfg(feature = "aac-decode")] mod track; #[cfg(any(feature = "decode-native", feature = "pcm-encode"))] mod wav; @@ -47,7 +51,7 @@ pub use layout::{ChannelLayout, Speaker}; pub use read::SharedSource; #[cfg(feature = "decode-native")] pub use read::{DEFAULT_MAX_SAMPLES, DecodeOptions, Decoder, TrackInfo, decode, decode_with, detect_container, probe, probe_with, tracks, tracks_of}; -#[cfg(feature = "decode-native")] +#[cfg(feature = "aac-decode")] pub use track::{LOOKAHEAD_PACKETS, TrackDecoder}; #[cfg(feature = "pcm-encode")] pub use write::wav_float_header; @@ -79,6 +83,20 @@ pub enum AudioError { /// Result alias for this module. pub type Result = std::result::Result; +// Codec errors convert into `AudioError`, so code mixing the file API with the codec modules can +// use `?` throughout: a codec's "unsupported" stays `Unsupported`, anything else in the input is +// `Malformed`, and an encoder's refusal is `Encode`. + +#[cfg(feature = "aac-decode")] +impl From for AudioError { + fn from(e: aac::Error) -> Self { + match e { + aac::Error::Unsupported(_) => AudioError::Unsupported(e.to_string()), + _ => AudioError::Malformed(e.to_string()), + } + } +} + /// The file formats [`encode`] writes. Each needs its feature: WAV and AIFF `pcm-encode`, FLAC /// `flac-encode`; asking for one that is not built returns [`AudioError::Unsupported`]. #[non_exhaustive] diff --git a/crates/audio/src/read/mod.rs b/crates/audio/src/read/mod.rs index f205e83b9..e223ddb49 100644 --- a/crates/audio/src/read/mod.rs +++ b/crates/audio/src/read/mod.rs @@ -164,9 +164,10 @@ impl DecodeOptions { pub struct TrackInfo { /// The value for [`DecodeOptions::track`]. pub index: usize, - /// The codec the container names (an AAC track that turns out to be HE-AAC is `Aac` here). + /// The codec the container names (an AAC track that turns out to be HE-AAC is `Aac` here). A + /// raw ADTS stream names none, so this is what decoding finds. pub codec: CodecId, - /// The sample rate the container declares, if it does. + /// The sample rate the container declares, if it does; for raw ADTS, the decoded rate. pub sample_rate: Option, /// The channel count the container declares, if it does. pub channels: Option, @@ -182,11 +183,12 @@ enum Inner<'a> { /// A file's audio, decoded block by block: the streaming counterpart of [`decode`], with seeking /// and track choice. /// -/// Native formats (WAV, AIFF, and PCM in MP4/MOV and Matroska/WebM) read their sample data from -/// the source packet by packet. The container's packet index is held in memory (tens of bytes per -/// packet), and opening a Matroska file reads through the whole file to build it; WAV and AIFF -/// need no index. Formats symphonia decodes (AAC, MP1/MP2/MP3, FLAC, ALAC, Vorbis, CAF, ADPCM, -/// A-law, µ-law) read the whole source into memory first; their decoded audio still streams. +/// Native formats (WAV, AIFF, AAC and PCM in MP4/MOV and Matroska/WebM, and raw ADTS streams) +/// read their sample data from the source packet by packet. The container's packet index is held +/// in memory (tens of bytes per packet), and opening a Matroska file or a raw ADTS stream reads +/// through the whole file to build it; WAV and AIFF need no index. Formats symphonia decodes (MP1/MP2/MP3, FLAC, ALAC, +/// Vorbis, CAF, ADPCM, A-law, µ-law) read the whole source into memory first; their decoded audio +/// still streams. /// /// ```no_run /// # fn demo() -> craft_audio::Result<()> { diff --git a/crates/audio/src/read/native.rs b/crates/audio/src/read/native.rs index 67588476c..4c30ce9a5 100644 --- a/crates/audio/src/read/native.rs +++ b/crates/audio/src/read/native.rs @@ -1,7 +1,7 @@ -//! The native decode path: craft-media's demuxers (ISO BMFF / QuickTime, Matroska / WebM) and -//! WAV / AIFF PCM, feeding [`TrackDecoder`]. -//! [`Native::open`] answers `None` for a stream with no native codec (AAC, MP1/MP2/MP3, FLAC, -//! ALAC, Vorbis, Opus, ...) or a container these demuxers do not read; the caller then uses symphonia. +//! The native decode path: craft-media's demuxers (ISO BMFF / QuickTime, Matroska / WebM), raw +//! ADTS streams, and WAV / AIFF PCM, feeding [`TrackDecoder`]. +//! [`Native::open`] answers `None` for a stream with no native codec (MP1/MP2/MP3, FLAC, ALAC, +//! Vorbis, Opus, ...) or a container these demuxers do not read; the caller then uses symphonia. //! //! Every container is a [`Packets`] source over a [`Src`], read packet by packet, so a decode //! streams. Timing follows the containers: MP4 edit lists (AAC priming) and Matroska `CodecDelay` and @@ -11,12 +11,16 @@ use super::aiff::{self, AiffParse}; use super::id3v2_len; use super::source::{Src, io_error}; use super::wav::{self, WavParse}; -use crate::track::TrackDecoder; -use crate::{AudioBuffer, AudioError, AudioInfo, CodecId, CodecParams, Container, PcmFormat, Result}; +use crate::track::{TrackDecoder, adts_format_of, adts_frame_count, adts_stream_header}; +use crate::{AudioBuffer, AudioError, AudioInfo, CodecId, CodecParams, Container, PcmFormat, Result, aac}; use craft_media::{isobmff, matroska, ogg}; /// Corrupt packets tolerated before a stream that has produced nothing is an error. const MAX_ERRORS: usize = 1000; +/// ID3v2 tags skipped at the start of a stream; real files have one. +const MAX_ID3_TAGS: usize = 16; +/// How far into a raw stream a first frame is looked for. +const MAX_SYNC_SCAN: u64 = 64 * 1024; /// The most frames a packet may declare (as on the symphonia path): decoders allocate a packet's /// frames up front. const MAX_PACKET_FRAMES: u64 = 1 << 20; @@ -40,6 +44,9 @@ enum Packets<'a> { /// Media timescale and output rate, for packet start times. timescale: u32, rate: u32, + /// Frames every packet decodes to, when the codec says: then a packet's place on the + /// decoded timeline is its index, whatever the sample table's durations claim. + per_packet: Option, }, /// Matroska is read packet by packet, which carries each block's `DiscardPadding`. Mkv { @@ -54,6 +61,8 @@ enum Packets<'a> { /// A packet read ahead by a seek, returned next. pending: Option, }, + /// Frames of a raw stream: `(offset, length, decoded frames)`. + Raw { frames: Vec<(u64, u32, u32)>, src: Src<'a>, next: usize }, /// Interleaved PCM in `start..start + frames * frame_bytes` (WAV, AIFF). Pcm { src: Src<'a>, start: u64, frames: u64, frame_bytes: u64, next_frame: u64 }, } @@ -70,6 +79,11 @@ impl Packets<'_> { file.read_sample(src, *track, i).map(whole).map_err(mp4_error) }) } + Packets::Raw { frames, src, next } => { + let &(at, len, _) = frames.get(*next)?; + *next += 1; + Some(src.read(at, len as usize).map(whole)) + } Packets::Pcm { src, start, frames, frame_bytes, next_frame } => { let n = (*frames - *next_frame).min(PCM_PACKET_FRAMES); if n == 0 { @@ -103,8 +117,13 @@ impl Packets<'_> { found } match self { - Packets::Mp4 { file, track, next, timescale, rate, .. } => { + Packets::Mp4 { file, track, next, timescale, rate, per_packet, .. } => { let samples = file.tracks.get(*track).map(|t| t.samples.as_slice()).unwrap_or_default(); + if let Some(n) = per_packet.filter(|&n| n > 0) { + let i = (frame / n).min((samples.len() as u64).saturating_sub(1)); + *next = i as usize; + return Ok(i * n); + } let first = samples.first().map_or(0, |s| s.dts); let starts = samples.iter().map(|s| rescale(u64::try_from(s.dts.saturating_sub(first)).unwrap_or(0), u64::from(*timescale), u64::from(*rate))); @@ -112,6 +131,17 @@ impl Packets<'_> { *next = i; Ok(s) } + Packets::Raw { frames, next, .. } => { + let mut acc = 0u64; + let starts = frames.iter().map(|&(_, _, n)| { + let s = acc; + acc += u64::from(n); + s + }); + let (i, s) = at_or_before(starts, frame); + *next = i; + Ok(s) + } Packets::Pcm { frames, next_frame, .. } => { *next_frame = frame.min(*frames); Ok(*next_frame) @@ -222,8 +252,11 @@ impl<'a> Native<'a> { Ok(head) => head, Err(e) => return Some(Err(e)), }; - let at = after_id3(&head); - let body = head.get(at..).unwrap_or_default(); + let at = after_id3(src); + // A tag longer than the head (cover art) hides what follows it, so look there. + let past_tag = usize::try_from(at).is_ok_and(|at| at >= head.len() && at > 0); + let sniffed = if past_tag { super::detect_container(&src.bytes_upto(at, 64 * 1024), None) } else { super::detect_container(&head, None) }; + let body = usize::try_from(at).ok().and_then(|at| head.get(at..)).unwrap_or_default(); // The bytes decide the container when they say; the detection (which may come from the // extension) only picks among raw streams. // Formats with one track: any other track index is an error, as for a multi-track file. @@ -231,11 +264,12 @@ impl<'a> Native<'a> { (Some(n), Some(Ok(_))) if n > 0 => Some(Err(AudioError::Unsupported(format!("no audio track {n}: the file has one")))), (_, opened) => opened, }; - match super::detect_container(&head, None).or(detected) { + match sniffed.or(detected) { Some(Container::Wav) if wav::is_wav(&head) => single(open_wav(src)), Some(Container::Aiff) if aiff::is_aiff(&head) => single(open_aiff(src)), Some(c @ (Container::Mp4 | Container::QuickTime)) if head.get(4..8) == Some(b"ftyp") => open_mp4(src, c, track), Some(c @ (Container::Matroska | Container::WebM)) if body.starts_with(&[0x1A, 0x45, 0xDF, 0xA3]) => open_mkv(src, c, track), + Some(Container::Adts) | None => single(open_adts(src, at)), _ => None, } } @@ -356,11 +390,14 @@ impl<'a> Native<'a> { } } -/// Where a stream starts: after any ID3v2 tags. -fn after_id3(b: &[u8]) -> usize { - let mut at = 0; - while let Some(len) = b.get(at..).and_then(id3v2_len) { - at = at.saturating_add(len); +/// Where a stream starts: after any ID3v2 tags, which may be longer than the head of the file. +fn after_id3(src: &Src<'_>) -> u64 { + let mut at = 0u64; + for _ in 0..MAX_ID3_TAGS { + match id3v2_len(&src.bytes_upto(at, 10)) { + Some(len) => at = at.saturating_add(len as u64), + None => break, + } } at } @@ -447,6 +484,121 @@ fn pcm_track<'a>( Some(Ok(n)) } +/// Reads a raw stream in windows, for [`raw_frames`]. +struct Scanner<'s, 'a, F> { + src: &'s Src<'a>, + total: u64, + frame_len: F, + /// The window's start and bytes. + window: (u64, Vec), + /// The source failed to read: the scan is over. + failed: Option, +} + +impl Option<(usize, T)>> Scanner<'_, '_, F> { + const WINDOW: usize = 1 << 16; + /// The most header bytes a frame-length function reads. + const HEADER: u64 = 16; + + /// The length and description of the frame at `p`, if one starts there and fits. + fn len_at(&mut self, p: u64) -> Option<(usize, T)> { + if p >= self.total || self.failed.is_some() { + return None; + } + if p < self.window.0 || p + Self::HEADER > self.window.0 + self.window.1.len() as u64 { + match self.src.read_upto(p, Self::WINDOW) { + Ok(bytes) => self.window = (p, bytes), + Err(e) => { + self.failed = Some(e); + return None; + } + } + } + let off = usize::try_from(p - self.window.0).ok()?; + let (n, frames) = self.window.1.get(off..).and_then(&self.frame_len)?; + (n > 0 && p + n as u64 <= self.total).then_some((n, frames)) + } + + /// Whether a frame at `p` is followed by another (or by the end of the stream). + fn paired(&mut self, p: u64) -> bool { + match self.len_at(p) { + Some((n, _)) => p + n as u64 == self.total || self.len_at(p + n as u64).is_some(), + None => false, + } + } +} + +/// Frame lengths of a raw stream of self-delimiting frames from `at`: `frame_len(rest)` gives the +/// length and a description of a frame starting at `rest`, or `None`. The first frame must be +/// followed by a second (or the end of the data), so a stray sync word does not make a stream; +/// damage later is skipped by scanning for the next pair. Reads the stream in windows. `None` when +/// no first frame is found; a source that fails to read is an error, not the end of the stream. +fn raw_frames(src: &Src<'_>, at: u64, frame_len: impl Fn(&[u8]) -> Option<(usize, T)>) -> Result>> { + let total = src.size(); + let mut sc = Scanner { src, total, frame_len, window: (u64::MAX, Vec::new()), failed: None }; + let first = (at..total.min(at.saturating_add(MAX_SYNC_SCAN))).find(|&p| sc.failed.is_some() || sc.paired(p)); + let Some(first) = first.filter(|_| sc.failed.is_none()) else { + return sc.failed.map_or(Ok(None), Err); + }; + let mut frames = Vec::new(); + let mut p = first; + while p < total { + match sc.len_at(p) { + Some((n, f)) => { + frames.push((p, n as u32, f)); + p += n as u64; + } + None => match (p + 1..total).find(|&q| sc.failed.is_some() || sc.paired(q)) { + Some(q) if sc.failed.is_none() => p = q, + _ => break, + }, + } + } + sc.failed.map_or(Ok(Some(frames)), Err) +} + +/// The first `n` frames of a raw stream. +fn raw_lookahead(src: &Src<'_>, frames: &[(u64, u32, T)], n: usize) -> Result>> { + frames.iter().take(n).map(|&(at, len, _)| src.read(at, len as usize)).collect() +} + +/// A raw ADTS AAC stream from `at`: its frames, headers included, are the packets. The stream's +/// format is the one most of its first frames share, so one damaged header does not decide it. +/// [`TrackDecoder`] conceals the frames that differ from it or have several raw data blocks; a +/// stream that starts with those is not opened here. +fn open_adts<'a>(src: &Src<'a>, at: u64) -> Option>> { + let frames = + match raw_frames(src, at, |rest| aac::AdtsHeader::parse(rest).ok().filter(|h| h.frame_length > h.header_length).map(|h| (h.frame_length, h))) + { + Ok(frames) => frames?, + Err(e) => return Some(Err(e)), + }; + let first = &frames[..frames.len().min(crate::track::LOOKAHEAD_PACKETS)]; + let headers: Vec = first.iter().map(|&(_, _, h)| h).collect(); + let stream = adts_format_of(&adts_stream_header(&headers)?); + if headers.iter().filter(|h| adts_format_of(h) == stream && h.raw_data_blocks == 1).count() * 2 <= first.len() { + return None; + } + // Without extradata the configuration comes from the frames themselves. + let params = CodecParams::new(CodecId::Aac); + let lookahead = match raw_lookahead(src, &frames, crate::track::LOOKAHEAD_PACKETS) { + Ok(lookahead) => lookahead, + Err(e) => return Some(Err(e)), + }; + // Decoded frames are set below, when the decoder has said what a frame decodes to. + let packets: Vec<(u64, u32, u32)> = frames.iter().map(|&(p, n, h)| (p, n, u32::from(h.raw_data_blocks))).collect(); + let mut n = native(Container::Adts, Packets::Raw { frames: packets, src: src.clone(), next: 0 }, ¶ms, &lookahead, 0, None, None)?; + let per_block = n.dec.frames_per_packet()? as u32; + if let Packets::Raw { frames, .. } = &mut n.packets { + for f in frames.iter_mut() { + f.2 *= per_block; + } + n.info.frames = frames.iter().map(|f| u64::from(f.2)).sum(); + n.frames_known = true; + } + Some(Ok(n)) +} + /// The `n`-th audio track (the first when `None`) among `kinds`, as an index into them. fn audio_track(kinds: impl Iterator, n: Option) -> Option> { let audio: Vec = kinds.enumerate().filter(|&(_, audio)| audio).map(|(i, _)| i).collect(); @@ -486,21 +638,61 @@ fn open_mp4<'a>(src: &Src<'a>, container: Container, track: Option) -> Op (0..t.samples.len().min(crate::track::LOOKAHEAD_PACKETS)).filter_map(|i| file.read_sample(src, track, i).ok()).collect(); let timescale = t.timescale; // Probe the decoder for the output rate before the timing is worked out. - let rate = TrackDecoder::new(¶ms, &lookahead.iter().map(Vec::as_slice).collect::>()).ok()?.sample_rate(); + let probe = TrackDecoder::new(¶ms, &lookahead.iter().map(Vec::as_slice).collect::>()).ok()?; + let rate = probe.sample_rate(); // A leading media edit (AAC priming) is skipped, and the edit's duration is the // presentation length. Empty edits and multi-segment lists play the media as it is. - let (skip, limit) = match t.edits.as_slice() { - [e] if e.media_time >= 0 && e.media_rate == 0x0001_0000 => { + // A leading empty edit (a delay before the track starts, ffmpeg's `-itsoffset` and muxers + // aligning audio to video) does not change which media is presented: the media edit after it + // still skips the priming and ends the track. The delay itself is not added as silence. + let media_edit = match t.edits.as_slice() { + [e] => Some(e), + [empty, e] if empty.media_time < 0 => Some(e), + _ => None, + }; + let (skip, limit) = match media_edit { + Some(e) if e.media_time >= 0 && e.media_rate == 0x0001_0000 => { let skip = rescale(e.media_time.unsigned_abs(), u64::from(timescale), u64::from(rate)); let limit = (e.segment_duration > 0).then(|| rescale(e.segment_duration, u64::from(file.timescale), u64::from(rate))); (skip, limit) } _ => (0, None), }; + // The samples' durations end the track when no edit does, and an edit cannot outlast the media. let media: u64 = t.samples.iter().map(|s| u64::from(s.duration)).sum(); - let frames = limit.unwrap_or_else(|| rescale(media, u64::from(timescale), u64::from(rate)).saturating_sub(skip)); - let packets = Packets::Mp4 { file: Box::new(file), src: src.clone(), track, next: 0, timescale, rate }; - native(container, packets, ¶ms, &lookahead, skip, limit, Some(frames)).map(Ok) + let mut stored = rescale(media, u64::from(timescale), u64::from(rate)).saturating_sub(skip); + // Durations only trim the last packet: ones that end anywhere else are wrong (some count a + // HE-AAC packet's core frames) and can't cut audio off. + // ADTS-framed samples (not standard) may hold several frames each, and finding out would mean + // reading every sample: their sample table's timing is kept instead, unless it falls short of + // the frames the samples hold (durations counting HE-AAC's core frames): short of one frame + // per sample, or of the frames in the samples already read. Then it can't end the track, and + // its length is unknown until decoded. + let adts_counts: Vec = lookahead.iter().map(|p| adts_frame_count(p) as u64).collect(); + let adts = adts_counts.iter().any(|&c| c > 0); + let per_packet = probe.frames_per_packet().map(|n| n as u64).filter(|_| !adts); + let mut length_known = true; + if let Some(n) = probe.frames_per_packet().map(|n| n as u64) { + let one_each = (t.samples.len() as u64).saturating_mul(n).saturating_sub(skip); + // The samples read ahead are the first ones unless one could not be read. The track's last + // sample is left out: its duration may trim any number of its frames. + let read_short = adts_counts.len() == t.samples.len().min(crate::track::LOOKAHEAD_PACKETS) && { + let counted = adts_counts.len().min(t.samples.len() - 1); + let held = adts_counts[..counted].iter().sum::().saturating_mul(n); + let timed: u64 = t.samples[..counted].iter().map(|s| u64::from(s.duration)).sum(); + held.saturating_sub(rescale(timed, u64::from(timescale), u64::from(rate))) >= n + }; + if !adts && (stored > one_each || one_each - stored >= n) { + stored = one_each; + } else if adts && (one_each.saturating_sub(stored) >= n || read_short) { + stored = 0; + length_known = limit.is_some(); + } + } + let frames = if stored > 0 { limit.map_or(stored, |l| l.min(stored)) } else { limit.unwrap_or(0) }; + let limit = (stored > 0).then_some(frames).or(limit); + let packets = Packets::Mp4 { file: Box::new(file), src: src.clone(), track, next: 0, timescale, rate, per_packet }; + native(container, packets, ¶ms, &lookahead, skip, limit, length_known.then_some(frames)).map(Ok) } /// A Matroska / WebM file's audio track, if its codec is native. @@ -543,8 +735,16 @@ fn open_mkv<'a>(src: &Src<'a>, container: Container, track: Option) -> Op let last = t.samples.iter().max_by_key(|s| s.pts); let delay_ticks = i64::try_from(rescale_round(t.codec_delay_ns.saturating_mul(den), num.saturating_mul(1_000_000_000), 1)).unwrap_or(i64::MAX); let start = t.samples.iter().map(|s| s.pts).min().unwrap_or(0).saturating_add(delay_ticks); - let frames = last.filter(|s| s.duration > 0).and_then(|s| { - let end = u64::try_from(s.pts.saturating_add(i64::try_from(s.duration).ok()?).saturating_sub(start)).ok()?; + let frames = last.and_then(|s| { + let duration = if s.duration > 0 { + s.duration + } else { + // ffmpeg 6 writes AAC without block durations, and records the end in a tag. + let MkvFrames::Constant(n) = frames_of else { return None }; + let packet = rescale_round(n.saturating_mul(den), u64::from(rate).saturating_mul(num), 1); + tagged_last_duration(track_duration_tag(&file.tags, t.uid)?, s.pts, delay_ticks, packet, t.timebase)? + }; + let end = u64::try_from(s.pts.saturating_add(i64::try_from(duration).ok()?).saturating_sub(start)).ok()?; Some(rescale_round(end.saturating_mul(num), den, u64::from(rate))) }); let mut demux = demux; @@ -559,6 +759,37 @@ fn open_mkv<'a>(src: &Src<'a>, container: Container, track: Option) -> Op native(container, packets, ¶ms, &lookahead, skip, frames, frames).map(Ok) } +/// The `DURATION` tag of the track with UID `uid`, in nanoseconds. ffmpeg and mkvmerge write it +/// as `HH:MM:SS.nnnnnnnnn`. +fn track_duration_tag(tags: &[matroska::Tag], uid: u64) -> Option { + tags.iter() + .filter(|tag| tag.targets.track_uids.contains(&uid)) + .flat_map(|tag| &tag.simple) + .find(|simple| simple.name.eq_ignore_ascii_case("DURATION")) + .and_then(|simple| parse_tag_duration(simple.string.as_deref()?)) +} + +/// `HH:MM:SS.nnnnnnnnn` in nanoseconds. +fn parse_tag_duration(text: &str) -> Option { + let (clock, fraction) = text.split_once('.').unwrap_or((text, "")); + let mut parts = clock.split(':').map(|p| p.parse::().ok()); + let (hours, minutes, seconds) = (parts.next()??, parts.next()??, parts.next()??); + if parts.next().is_some() || fraction.len() > 9 || !fraction.bytes().all(|b| b.is_ascii_digit()) { + return None; + } + let nanos = if fraction.is_empty() { 0 } else { fraction.parse::().ok()? * 10u64.pow(9 - fraction.len() as u32) }; + hours.checked_mul(3600)?.checked_add(minutes.checked_mul(60)?)?.checked_add(seconds)?.checked_mul(1_000_000_000)?.checked_add(nanos) +} + +/// The duration in ticks of a track's last packet, when `tag_ns` (which counts `CodecDelay`, as the +/// stored block timestamps do) ends inside that packet. A tag can go stale when a file is edited, +/// so one that ends anywhere else is ignored and can't cut audio off. +fn tagged_last_duration(tag_ns: u64, last_pts: i64, delay_ticks: i64, packet_ticks: u64, (num, den): (u64, u64)) -> Option { + let tag_ticks = i64::try_from(rescale_round(tag_ns.saturating_mul(den), num.saturating_mul(1_000_000_000), 1)).ok()?; + let duration = u64::try_from(tag_ticks.saturating_sub(delay_ticks).saturating_sub(last_pts)).ok()?; + (duration > 0 && duration <= packet_ticks).then_some(duration) +} + /// The audio tracks of `src` the native path reads, described for a track list; `None` when it /// does not read the container. pub(crate) fn tracks(src: &Src<'_>, detected: Option) -> Option>> { @@ -658,3 +889,246 @@ fn rescale(v: u64, from: u64, to: u64) -> u64 { fn rescale_round(v: u64, from: u64, to: u64) -> u64 { if from == 0 { 0 } else { u64::try_from((u128::from(v) * u128::from(to) + u128::from(from) / 2) / u128::from(from)).unwrap_or(u64::MAX) } } + +#[cfg(test)] +mod tag_tests { + use super::{parse_tag_duration, tagged_last_duration}; + + #[test] + fn tag_durations_parse_and_nonsense_does_not() { + assert_eq!(parse_tag_duration("00:00:05.021000000"), Some(5_021_000_000)); + assert_eq!(parse_tag_duration("01:02:03.5"), Some(3_723_500_000_000)); + assert_eq!(parse_tag_duration("00:00:05"), Some(5_000_000_000)); + for bad in ["", "5.021", "00:05.0", "00:00:00:05.0", "00:00:0x.0", "00:00:05.-1", "00:00:05.0000000001", "99999999999:00:00.0"] { + assert_eq!(parse_tag_duration(bad), None, "{bad}"); + } + } + + #[test] + fn only_a_tag_ending_inside_the_last_packet_counts() { + // 5 s of 48 kHz AAC at millisecond ticks: priming 21, last packet stored at 4992 (5013 with + // the delay), 21 ticks long + let last = |ms: u64| tagged_last_duration(ms * 1_000_000, 4992, 21, 21, (1, 1000)); + assert_eq!(last(5021), Some(8)); + assert_eq!(last(5034), Some(21)); + assert_eq!(last(5035), None, "past the packet"); + assert_eq!(last(5013), None, "at its start"); + assert_eq!(last(3000), None, "far before it"); + } +} + +#[cfg(test)] +mod adts_tests { + use crate::{AudioError, Container, DecodeOptions, Decoder, decode}; + use craft_media::ByteSource; + + /// A frame of mono silence at sampling frequency index `sf_index`. + fn frame(sf_index: u8) -> Vec { + // ID_SCE, tag 0, an empty channel stream, ID_END + let au = [0x00, 0x00, 0x00, 0x07]; + [&crate::aac::AdtsHeader::write(2, sf_index, 1, au.len(), 0x7FF).unwrap()[..], &au].concat() + } + + #[test] + fn a_stream_decodes_past_damage_between_frames() { + let good = frame(3).repeat(4); + let file = [&good[..], &[0x12; 100], &good].concat(); + let (info, buf) = decode(&file, None).unwrap(); + assert_eq!((info.container, buf.frames()), (Container::Adts, 8 * 1024)); + } + + #[test] + fn frames_with_several_raw_data_blocks_do_not_lose_audio_silently() { + let mut one = frame(3); + one[6] |= 1; // two raw data blocks + let file = one.repeat(8); + // The file is refused, not decoded at half its length. + assert!(decode(&file, Some("aac")).is_err()); + } + + /// Frames of several blocks after the stream has started are concealed at their whole length, + /// not skipped. + #[test] + fn several_raw_data_blocks_mid_stream_keep_the_timing() { + let mut two = frame(3); + two[6] |= 1; + let file = [frame(3).repeat(8), two, frame(3).repeat(2)].concat(); + let (info, buf) = decode(&file, None).unwrap(); + assert_eq!((info.damaged, info.frames, buf.frames()), (1, 12 * 1024, 12 * 1024)); + assert_eq!(crate::probe(&file, None).unwrap().frames, 12 * 1024); + } + + /// One flipped bit in the first header does not silence the stream. + #[test] + fn a_damaged_first_header_does_not_decide_the_format() { + let file = [frame(4), frame(3).repeat(9)].concat(); + let (info, buf) = decode(&file, None).unwrap(); + assert_eq!((info.sample_rate, info.damaged, buf.frames()), (48_000, 1, 10 * 1024)); + } + + #[test] + fn a_frame_that_changes_the_stream_format_is_damage() { + let file = [frame(3).repeat(4), frame(4).repeat(3)].concat(); + let (info, buf) = decode(&file, None).unwrap(); + assert_eq!((info.sample_rate, info.damaged, buf.frames()), (48_000, 3, 7 * 1024)); + } + + #[test] + fn a_stream_is_found_after_an_id3_tag_longer_than_the_head() { + let size = 300_000u32; + let tag = [ + b"ID3\x03\x00\x00".to_vec(), + [size >> 21, size >> 14, size >> 7, size].iter().map(|v| (v & 0x7F) as u8).collect(), + vec![0; size as usize], + ] + .concat(); + let file = [tag, frame(3).repeat(4)].concat(); + let (info, buf) = decode(&file, None).unwrap(); + assert_eq!((info.container, buf.frames()), (Container::Adts, 4 * 1024)); + } + + /// Bytes that fail to read past `good`. + struct Failing { + bytes: Vec, + good: u64, + } + + impl ByteSource for Failing { + fn len(&self) -> u64 { + self.bytes.len() as u64 + } + fn read_at(&self, offset: u64, buf: &mut [u8]) -> std::io::Result<()> { + if offset + buf.len() as u64 > self.good { + return Err(std::io::Error::other("the source failed")); + } + self.bytes.as_slice().read_at(offset, buf) + } + } + + /// A frame that reads in the scan and fails when read again. + #[test] + fn a_source_failing_during_the_lookahead_is_an_io_error() { + struct FailsFrames(Vec); + impl ByteSource for FailsFrames { + fn len(&self) -> u64 { + self.0.len() as u64 + } + fn read_at(&self, offset: u64, buf: &mut [u8]) -> std::io::Result<()> { + // A whole frame in the middle: the scan reads windows, to the end of the file. + if buf.len() == 11 && offset + 11 < self.len() { + return Err(std::io::Error::other("the source failed")); + } + self.0.as_slice().read_at(offset, buf) + } + } + let opened = Decoder::open(FailsFrames(frame(3).repeat(64)), Some("aac"), &DecodeOptions::default()); + assert!(matches!(opened, Err(AudioError::Io(_))), "{:?}", opened.err()); + } + + /// ADTS-framed HE-AAC whose sample durations count core frames is not cut to the durations' + /// half length. + #[test] + fn adts_framed_he_aac_in_mp4_with_core_durations_keeps_its_length() { + use craft_media::bitstream::BitWriter; + use craft_media::isobmff::{Brand, Mp4Writer, SampleEntry, TrackConfig, WriteSample, WriterOptions}; + // mono silence with SBR data in a fill element (implicit HE-AAC), at 22.05 kHz + let mut bw = BitWriter::new(); + for (value, bits) in [(0, 7), (0, 22), (6, 3), (1, 4), (13, 4), (0, 4), (7, 3)] { + bw.write_bits(value, bits); + } + let au = bw.finish(); + let frame = [&crate::aac::AdtsHeader::write(2, 7, 1, au.len(), 0x7FF).unwrap()[..], &au].concat(); + let asc = crate::aac::AudioSpecificConfig::lc(7, 1).to_bytes().unwrap(); + let mut w = Mp4Writer::new(std::io::Cursor::new(Vec::new()), WriterOptions::new(Brand::Mp4)).unwrap(); + let track = w.add_track(TrackConfig::new(SampleEntry::aac(asc, 1, 44_100), 44_100)).unwrap(); + for _ in 0..12 { + w.write_sample(track, WriteSample { data: &frame, duration: 1024, composition_offset: 0, is_sync: true }).unwrap(); + } + let file = w.finish().unwrap().into_inner(); + let (info, buf) = decode(&file, None).unwrap(); + assert_eq!((info.codec, info.sample_rate, buf.frames()), (crate::CodecId::HeAac, 44_100, 12 * 2048)); + assert_eq!(crate::probe(&file, None).unwrap().frames, 12 * 2048); + } + + /// Samples of two HE-AAC ADTS frames each, timed as two core frames: the durations match one + /// output frame per sample, and the frames in the samples read ahead show they fall short. + #[test] + fn adts_framed_he_aac_in_mp4_with_several_frames_per_sample_keeps_its_length() { + use craft_media::bitstream::BitWriter; + use craft_media::isobmff::{Brand, Mp4Writer, SampleEntry, TrackConfig, WriteSample, WriterOptions}; + let mut bw = BitWriter::new(); + for (value, bits) in [(0, 7), (0, 22), (6, 3), (1, 4), (13, 4), (0, 4), (7, 3)] { + bw.write_bits(value, bits); + } + let au = bw.finish(); + let frame = [&crate::aac::AdtsHeader::write(2, 7, 1, au.len(), 0x7FF).unwrap()[..], &au].concat().repeat(2); + let asc = crate::aac::AudioSpecificConfig::lc(7, 1).to_bytes().unwrap(); + let mut w = Mp4Writer::new(std::io::Cursor::new(Vec::new()), WriterOptions::new(Brand::Mp4)).unwrap(); + let track = w.add_track(TrackConfig::new(SampleEntry::aac(asc, 1, 44_100), 44_100)).unwrap(); + for _ in 0..12 { + w.write_sample(track, WriteSample { data: &frame, duration: 2048, composition_offset: 0, is_sync: true }).unwrap(); + } + let file = w.finish().unwrap().into_inner(); + let (info, buf) = decode(&file, None).unwrap(); + assert_eq!((info.codec, info.sample_rate, buf.frames()), (crate::CodecId::HeAac, 44_100, 12 * 4096)); + assert_eq!(crate::probe(&file, None).unwrap().frames, 12 * 4096); + } + + /// A short track's last sample, read ahead, can trim more than one of its frames. + #[test] + fn a_short_adts_framed_mp4_track_keeps_a_last_sample_trim_of_over_a_frame() { + use craft_media::isobmff::{Brand, Mp4Writer, SampleEntry, TrackConfig, WriteSample, WriterOptions}; + let asc = crate::aac::AudioSpecificConfig::lc(3, 1).to_bytes().unwrap(); + let mut w = Mp4Writer::new(std::io::Cursor::new(Vec::new()), WriterOptions::new(Brand::Mp4)).unwrap(); + let track = w.add_track(TrackConfig::new(SampleEntry::aac(asc, 1, 48_000), 48_000)).unwrap(); + for i in 0..4 { + let duration = if i == 3 { 2048 - 1500 } else { 2048 }; + w.write_sample(track, WriteSample { data: &frame(3).repeat(2), duration, composition_offset: 0, is_sync: true }).unwrap(); + } + let file = w.finish().unwrap().into_inner(); + let (_, buf) = decode(&file, None).unwrap(); + assert_eq!(buf.frames(), 4 * 2048 - 1500); + assert_eq!(crate::probe(&file, None).unwrap().frames, 4 * 2048 - 1500); + } + + /// AAC-LC samples of two ADTS frames each, timed correctly, still end at their durations. + #[test] + fn adts_framed_lc_in_mp4_with_several_frames_per_sample_keeps_its_trim() { + use craft_media::isobmff::{Brand, Mp4Writer, SampleEntry, TrackConfig, WriteSample, WriterOptions}; + let asc = crate::aac::AudioSpecificConfig::lc(3, 1).to_bytes().unwrap(); + let mut w = Mp4Writer::new(std::io::Cursor::new(Vec::new()), WriterOptions::new(Brand::Mp4)).unwrap(); + let track = w.add_track(TrackConfig::new(SampleEntry::aac(asc, 1, 48_000), 48_000)).unwrap(); + for i in 0..12 { + let duration = if i == 11 { 2048 - 500 } else { 2048 }; + w.write_sample(track, WriteSample { data: &frame(3).repeat(2), duration, composition_offset: 0, is_sync: true }).unwrap(); + } + let file = w.finish().unwrap().into_inner(); + let (_, buf) = decode(&file, None).unwrap(); + assert_eq!(buf.frames(), 12 * 2048 - 500); + assert_eq!(crate::probe(&file, None).unwrap().frames, 12 * 2048 - 500); + } + + /// A sample of several ADTS frames past the lookahead still counts them all. + #[test] + fn a_later_mp4_sample_of_several_adts_frames_keeps_its_length() { + use craft_media::isobmff::{Brand, Mp4Writer, SampleEntry, TrackConfig, WriteSample, WriterOptions}; + let asc = crate::aac::AudioSpecificConfig::lc(3, 1).to_bytes().unwrap(); + let mut w = Mp4Writer::new(std::io::Cursor::new(Vec::new()), WriterOptions::new(Brand::Mp4)).unwrap(); + let track = w.add_track(TrackConfig::new(SampleEntry::aac(asc, 1, 48_000), 48_000)).unwrap(); + for i in 0..12 { + let (data, duration) = if i == 10 { (frame(3).repeat(2), 2048) } else { (frame(3), 1024) }; + w.write_sample(track, WriteSample { data: &data, duration, composition_offset: 0, is_sync: true }).unwrap(); + } + let file = w.finish().unwrap().into_inner(); + let (_, buf) = decode(&file, None).unwrap(); + assert_eq!(buf.frames(), 13 * 1024); + } + + #[test] + fn a_source_failing_during_the_frame_scan_is_an_io_error() { + let bytes = frame(3).repeat(100_000); + // Past the head, so the failure comes during the scan rather than format detection. + let opened = Decoder::open(Failing { bytes, good: 300_000 }, Some("aac"), &DecodeOptions::default()); + assert!(matches!(opened, Err(AudioError::Io(_))), "{:?}", opened.err()); + } +} diff --git a/crates/audio/src/read/symph.rs b/crates/audio/src/read/symph.rs index 126bd5c81..e16c6e314 100644 --- a/crates/audio/src/read/symph.rs +++ b/crates/audio/src/read/symph.rs @@ -376,7 +376,9 @@ fn open(bytes: Cow<'_, [u8]>, detected: Option) -> Result { checked => (checked, None), }; let body = bytes.get(start..).unwrap_or_default(); - let ogg_streams = craft_media::ogg::sniff(body).then(|| super::flac::ogg_audio_streams(body)); + // symphonia finds the first page past leading bytes, so look for it the same way. + let first_page = body.windows(4).take(MAX_PREFIX_SCAN).position(|w| w == b"OggS"); + let ogg_streams = first_page.map(|at| super::flac::ogg_audio_streams(&body[at..])); // symphonia reads from an owned stream: a borrowed buffer is copied once, a buffer read from // a shared source is moved in. let mut mss = MediaSourceStream::new(Box::new(Cursor::new(bytes.into_owned())), MediaSourceStreamOptions::default()); diff --git a/crates/audio/src/track.rs b/crates/audio/src/track.rs index 34495b00c..5816e0993 100644 --- a/crates/audio/src/track.rs +++ b/crates/audio/src/track.rs @@ -2,6 +2,8 @@ //! codec. The file-level [`crate::decode`] and [`crate::Decoder`] use it for every native codec; //! an app with its own demuxer (MXF, MPEG-TS, a network stream) uses it directly. +#[cfg(feature = "aac-decode")] +use crate::aac; use crate::pcm::{PcmKind, deinterleave}; use crate::{AudioBuffer, AudioError, ChannelLayout, CodecId, CodecParams, Result, SampleFormat}; @@ -10,7 +12,18 @@ pub const LOOKAHEAD_PACKETS: usize = 8; /// The codec-specific decoder behind a [`TrackDecoder`]. enum Inner { - Pcm { kind: PcmKind, channels: usize }, + #[cfg(feature = "aac-decode")] + Aac { + dec: Box, + up: Option, + order: Option<&'static [usize]>, + /// The format ADTS packets must keep, when their headers gave the configuration. + adts_format: Option, + }, + Pcm { + kind: PcmKind, + channels: usize, + }, } /// Decodes the packets of one audio track to planar `f32`, in WAV channel order when @@ -18,8 +31,9 @@ enum Inner { /// /// It takes the track's [`CodecParams`] (from `isobmff::Track::codec_params`, /// `matroska::Track::codec_params`, `ogg::Stream::codec_params`, or built by hand for another -/// container) and decodes PCM. Container timing (priming, end trimming) is the caller's: every -/// decoded sample is returned. +/// container) and decodes AAC (LC; HE-AAC as its upsampled LC core) and PCM. +/// Container timing (AAC priming, end trimming) is the caller's: every decoded sample is +/// returned. /// /// ```no_run /// # fn demo(params: &craft_audio::CodecParams, packets: &[Vec]) -> craft_audio::Result<()> { @@ -60,19 +74,53 @@ impl std::fmt::Debug for TrackDecoder { impl TrackDecoder { /// A decoder for the track `params` describes. `lookahead` holds the track's first packets - /// (up to [`LOOKAHEAD_PACKETS`]; empty when they are not at hand), for codecs that find their - /// configuration in the stream; PCM needs none. Missing configuration is + /// (up to [`LOOKAHEAD_PACKETS`]; empty when they are not at hand): HE-AAC signalled only + /// inside the stream is found in them. + /// + /// Streams without codec configuration are accepted where the packets carry it: AAC with + /// empty `extradata` takes its configuration from the ADTS headers of the lookahead (MPEG-TS, + /// raw `.aac`), the one most of them share, and ADTS headers are stripped from every packet. Other missing configuration is /// [`AudioError::Malformed`]. /// - /// Codecs without a native packet decoder yet (AAC, MP3, FLAC, ALAC, Vorbis, Opus) are - /// [`AudioError::Unsupported`]; `craft_audio::decode` decodes them through symphonia when the + /// Codecs without a native packet decoder yet are [`AudioError::Unsupported`]; + /// `craft_audio::decode` decodes MP3, FLAC, ALAC and Vorbis through symphonia when the /// `symphonia` feature is on. pub fn new(params: &CodecParams, lookahead: &[&[u8]]) -> Result { let rate = params.sample_rate.unwrap_or(0); let channels = usize::from(params.channels.unwrap_or(0)); - // PCM needs no lookahead; the codec decoders take their configuration from it. - let _ = lookahead; let (inner, codec, sample_rate, layout, sample_format) = match params.codec { + #[cfg(feature = "aac-decode")] + CodecId::Aac | CodecId::HeAac => { + // ADTS-framed packets (MPEG-TS, raw `.aac`) carry their configuration in each + // header: without extradata, the headers of the lookahead give it. + let payloads: Vec<&[u8]> = lookahead + .iter() + .take(LOOKAHEAD_PACKETS) + .flat_map(|p| adts_frames(p).map_or_else(|| vec![*p], |frames| frames.into_iter().map(|(_, unit)| unit).collect())) + .collect(); + let mut adts_format = None; + let asc = if params.extradata.is_empty() { + let headers: Vec = lookahead.iter().filter_map(|p| aac::AdtsHeader::parse(p).ok()).collect(); + let h = adts_stream_header(&headers).ok_or_else(|| { + AudioError::Malformed("AAC track without an AudioSpecificConfig (extradata), and its packets are not ADTS".into()) + })?; + adts_format = Some(adts_format_of(&h)); + adts_config(&h, &payloads)? + } else { + params.extradata.clone() + }; + let asc = asc.as_slice(); + let dec = aac::Decoder::new(asc)?; + let out = aac::output_rate(asc, payloads.iter().copied()).unwrap_or(dec.sample_rate()); + let up = (dec.sample_rate().checked_mul(2) == Some(out)).then(|| aac::Upsample2x::new(dec.channels())); + let codec = if up.is_some() { CodecId::HeAac } else { CodecId::Aac }; + // Only twice the core rate is upsampled to; any other signalled rate plays the core. + let out = if up.is_some() { out } else { dec.sample_rate() }; + let cfg = dec.config().channel_config; + let layout = aac_layout(cfg, dec.channels()); + let order = layout.is_ordered().then(|| aac::wav_order(cfg)).flatten(); + (Inner::Aac { dec: Box::new(dec), up, order, adts_format }, codec, out, layout, SampleFormat::Compressed) + } CodecId::Pcm => { let pcm = params.pcm.ok_or_else(|| AudioError::Malformed("PCM track without a sample format".into()))?; let (kind, sample_format) = pcm_kind(pcm.bits, pcm.float, pcm.big_endian, !pcm.signed) @@ -87,12 +135,13 @@ impl TrackDecoder { Ok(TrackDecoder { inner, codec, sample_rate, layout, sample_format, last_frames: 0 }) } - /// The codec. + /// The codec, refined from the stream: [`CodecId::HeAac`] for AAC with spectral band + /// replication. pub fn codec(&self) -> CodecId { self.codec } - /// The rate of the decoded audio. + /// The rate of the decoded audio (for HE-AAC, twice the AAC-LC core rate). pub fn sample_rate(&self) -> u32 { self.sample_rate } @@ -116,6 +165,31 @@ impl TrackDecoder { /// its timing. pub fn decode(&mut self, packet: &[u8]) -> Result { let channels = match &mut self.inner { + #[cfg(feature = "aac-decode")] + Inner::Aac { dec, up, order, adts_format } => { + let mut out: Vec> = Vec::new(); + let units = match adts_frames(packet) { + Some(frames) => adts_units(frames, *adts_format)?, + None => vec![packet], + }; + // A failed frame leaves the decoder as it was, so the frames before it in the + // packet must too: their audio is replaced by the packet's concealment. + let before = (units.len() > 1).then(|| (dec.clone(), up.clone())); + let decoded = units.into_iter().try_for_each(|unit| { + let mut block = dec.decode(unit)?; + if let Some(up) = up.as_mut() { + block = up.process(block); + } + append(&mut out, block) + }); + if let Err(e) = decoded { + if let Some(before) = before { + (*dec, *up) = before; + } + return Err(e); + } + reorder(out, *order) + } Inner::Pcm { kind, channels } => { let frames = (packet.len() / (kind.bytes() * *channels).max(1)) as u64; deinterleave(packet, *kind, *channels, frames)? @@ -131,54 +205,187 @@ impl TrackDecoder { } /// What stands in for a lost or undecodable packet (`packet` when its bytes are at hand): - /// silence as long as the packet would have been (from its size, or the last packet). Empty - /// when the length is unknown. Fails with [`AudioError::TooLarge`](crate::AudioError::TooLarge) - /// when that silence would pass the decode limit (2^30 samples, as [`decode`](Self::decode) - /// has) or cannot be allocated, rather than return less than the packet's length. + /// silence as long as the packet would have been (from its frame + /// header, the codec's frame length, or the last packet). Empty when the length is unknown. + /// Fails with [`AudioError::TooLarge`](crate::AudioError::TooLarge) when that silence would pass + /// the decode limit (2^30 samples, as [`decode`](Self::decode) has) or cannot be allocated, + /// rather than return less than the packet's length. pub fn conceal(&mut self, packet: Option<&[u8]>) -> Result { let data = packet.unwrap_or_default(); let frames = match &mut self.inner { + #[cfg(feature = "aac-decode")] + Inner::Aac { up, .. } => { + let per_unit = aac::FRAME_LEN * if up.is_some() { 2 } else { 1 }; + match adts_frames(data) { + Some(f) => Some(f.iter().map(|(h, _)| usize::from(h.raw_data_blocks)).sum::().min(MAX_ADTS_FRAMES_PER_PACKET) * per_unit), + // A packet lost whole: as long as the last one, in a stream of multi-frame + // packets as in any other. + None if data.is_empty() && self.last_frames > 0 => Some(self.last_frames), + None => Some(per_unit), + } + } Inner::Pcm { kind, channels } => Some(data.len() / (kind.bytes() * *channels).max(1)), }; let frames = frames.filter(|&n| n > 0).unwrap_or(self.last_frames); Ok(AudioBuffer { sample_rate: self.sample_rate, channels: crate::pcm::silence(usize::from(self.layout.channels()), frames)? }) } - /// The frames every packet decodes to, for codecs whose packets all have one length; `None` - /// for PCM. + /// The frames every packet decodes to, for codecs whose packets all have one length: AAC + /// (1024, 2048 for HE-AAC; a packet of several ADTS frames decodes to that many times this); + /// `None` for PCM. pub fn frames_per_packet(&self) -> Option { match &self.inner { + #[cfg(feature = "aac-decode")] + Inner::Aac { up, .. } => Some(aac::FRAME_LEN * if up.is_some() { 2 } else { 1 }), Inner::Pcm { .. } => None, } } /// Frames to decode before a seek target and drop, so the decoder has settled when the target - /// is reached; none for PCM. + /// is reached: two AAC frames, none for PCM. pub fn preroll(&self) -> usize { match &self.inner { + #[cfg(feature = "aac-decode")] + Inner::Aac { up, .. } => 2 * aac::FRAME_LEN * if up.is_some() { 2 } else { 1 }, Inner::Pcm { .. } => 0, } } /// Forget the decoder state, for a seek: the next packet decodes as if it were the first - /// (codecs with overlap need [`TrackDecoder::preroll`] frames to settle). + /// (codecs with overlap, such as AAC, need [`TrackDecoder::preroll`] frames to + /// settle). pub fn reset(&mut self) { // the packet before a seek says nothing about the length of a packet lost after it self.last_frames = 0; match &mut self.inner { + #[cfg(feature = "aac-decode")] + Inner::Aac { dec, up, .. } => { + dec.reset(); + if let Some(up) = up { + up.reset(); + } + } Inner::Pcm { .. } => {} } } } fn unsupported(codec: CodecId) -> AudioError { - if cfg!(feature = "symphonia") { - AudioError::Unsupported(format!("no native {} packet decoder yet (craft_audio::decode reads it through symphonia)", codec.name())) - } else { - AudioError::Unsupported(format!("no native {} packet decoder yet", codec.name())) + let feature = match codec { + CodecId::Aac | CodecId::HeAac => Some("aac-decode"), + _ => None, + }; + match feature { + // A codec this crate decodes, in a build without its feature. + Some(f) => AudioError::Unsupported(format!("decoding {} needs the `{f}` feature", codec.name())), + None if cfg!(feature = "symphonia") && matches!(codec, CodecId::Mp3 | CodecId::Flac | CodecId::Alac | CodecId::Vorbis) => { + AudioError::Unsupported(format!("no native {} packet decoder yet (craft_audio::decode reads it through symphonia)", codec.name())) + } + None => AudioError::Unsupported(format!("no native {} packet decoder yet", codec.name())), } } +/// `channels` reordered by `order` (output channel → decoded channel); unchanged when the counts +/// differ. +#[cfg(feature = "aac-decode")] +fn reorder(mut channels: Vec>, order: Option<&[usize]>) -> Vec> { + match order { + Some(order) if order.len() == channels.len() => order.iter().map(|&c| channels.get_mut(c).map(std::mem::take).unwrap_or_default()).collect(), + _ => channels, + } +} + +/// Append `more` to `out` channel by channel. A packet whose frames differ in channel count is +/// malformed: one buffer cannot hold them. +#[cfg(feature = "aac-decode")] +fn append(out: &mut Vec>, more: Vec>) -> Result<()> { + if out.is_empty() { + *out = more; + } else if out.len() == more.len() { + for (o, c) in out.iter_mut().zip(more) { + o.extend(c); + } + } else { + return Err(AudioError::Malformed("the frames of a packet change the channel count".into())); + } + Ok(()) +} + +/// The most ADTS frames a packet may hold: its audio is decoded and held in memory before any +/// decode limit applies. 1024 frames is the 2^20 frames the demuxers allow a packet. +#[cfg(feature = "aac-decode")] +const MAX_ADTS_FRAMES_PER_PACKET: usize = 1024; + +/// The header of the format most of `headers` share, the earliest on a tie, so one damaged header +/// does not decide a stream's format. +#[cfg(feature = "aac-decode")] +pub(crate) fn adts_stream_header(headers: &[aac::AdtsHeader]) -> Option { + headers.iter().rev().max_by_key(|h| headers.iter().filter(|o| adts_format_of(o) == adts_format_of(h)).count()).copied() +} + +/// The ADTS frames `packet` holds, 0 when it is not ADTS-framed. +#[cfg(feature = "decode-native")] +pub(crate) fn adts_frame_count(packet: &[u8]) -> usize { + adts_frames(packet).map_or(0, |f| f.len()) +} + +/// The configuration of an AAC stream from its ADTS `header`. Channel configuration 0 leaves +/// the layout to a program config element in the stream, looked for in `first_units`. +#[cfg(feature = "aac-decode")] +fn adts_config(header: &aac::AdtsHeader, first_units: &[&[u8]]) -> Result> { + if header.channel_config != 0 { + return Ok(header.to_asc().to_bytes()?); + } + let mut dec = aac::Decoder::from_adts(header)?; + for unit in first_units { + let _ = dec.decode(unit); + if dec.config().pce.is_some() { + return Ok(dec.config().to_bytes()?); + } + } + Err(AudioError::Malformed("ADTS stream with channel configuration 0, and no program config element in its first packets".into())) +} + +/// The whole frames at the start of an ADTS-framed packet, without a damaged or truncated end, or +/// `None` when the packet does not start with one. A raw access unit never looks like one: its first +/// three bits would be 7, ID_END, leaving nothing to decode. +#[cfg(feature = "aac-decode")] +fn adts_frames(packet: &[u8]) -> Option> { + if packet.len() < 7 || packet[0] != 0xFF || packet[1] & 0xF6 != 0xF0 { + return None; + } + // Past the limit, what matters is only that there are too many: `adts_units` refuses them. + aac::split_adts_upto(packet, MAX_ADTS_FRAMES_PER_PACKET + 1).ok() +} + +/// What an ADTS frame says the stream's format is: profile, sampling frequency index and channel +/// configuration. +#[cfg(feature = "aac-decode")] +pub(crate) type AdtsFormat = (u8, u8, u8); + +#[cfg(feature = "aac-decode")] +pub(crate) fn adts_format_of(h: &aac::AdtsHeader) -> AdtsFormat { + (h.object_type, h.sf_index, h.channel_config) +} + +/// The raw access units of an ADTS-framed packet. In a stream whose headers gave its `format`, +/// a frame that changes it would play wrongly, so it is malformed; with extradata, that decides the +/// format and headers are not compared with it. Frames with several raw data blocks are +/// unsupported. +#[cfg(feature = "aac-decode")] +fn adts_units(frames: Vec<(aac::AdtsHeader, &[u8])>, format: Option) -> Result> { + if frames.len() > MAX_ADTS_FRAMES_PER_PACKET { + return Err(AudioError::Malformed(format!("a packet of more than {MAX_ADTS_FRAMES_PER_PACKET} ADTS frames"))); + } + if frames.iter().any(|(h, _)| h.raw_data_blocks > 1) { + return Err(AudioError::Unsupported("ADTS frames with several raw data blocks".into())); + } + if format.is_some_and(|f| frames.iter().any(|(h, _)| adts_format_of(h) != f)) { + return Err(AudioError::Malformed("an ADTS frame changes the stream's format".into())); + } + Ok(frames.into_iter().map(|(_, unit)| unit).collect()) +} + /// The sample layout and stored format of PCM samples, `None` for a depth we do not read. pub(crate) fn pcm_kind(bits: u16, float: bool, big_endian: bool, unsigned: bool) -> Option<(PcmKind, SampleFormat)> { let bytes = usize::from(bits.div_ceil(8)); @@ -198,6 +405,23 @@ pub(crate) fn pcm_kind(bits: u16, float: bool, big_endian: bool, unsigned: bool) } } +#[cfg(feature = "aac-decode")] +/// The speakers of an AAC channel configuration, in the WAV order [`aac::wav_order`] gives; +/// configuration 7 and program-config layouts keep the decoded order and are unordered. +pub(crate) fn aac_layout(channel_config: u8, channels: usize) -> ChannelLayout { + match channel_config { + 1 => ChannelLayout::MONO, + 2 => ChannelLayout::STEREO, + 3 => ChannelLayout::SURROUND_3_0, + 4 => ChannelLayout::from_mask(0x107), + 5 => ChannelLayout::from_mask(0x607), + 6 => ChannelLayout::SURROUND_5_1_SIDE, + 11 => ChannelLayout::SURROUND_6_1, + 12 => ChannelLayout::SURROUND_7_1, + _ => ChannelLayout::unordered(crate::layout::count(channels)), + } +} + #[cfg(test)] mod tests { use super::*; @@ -219,11 +443,283 @@ mod tests { assert!(matches!(dec.conceal(None), Err(AudioError::TooLarge(_)))); } + /// One ADTS frame (stereo, 48 kHz, or channel configuration 0 with a PCE) of silence. + #[cfg(feature = "aac-decode")] + fn adts_frame(channel_config: u8) -> Vec { + use craft_media::bitstream::BitWriter; + let mut bw = BitWriter::new(); + if channel_config == 0 { + bw.write_bits(5, 3); // ID_PCE + aac::ProgramConfig { front: vec![(false, 0), (true, 0)], ..Default::default() }.write(&mut bw).unwrap(); + bw.write_bits(0, 3); // ID_SCE + bw.write_bits(0, 4); + bw.write_bits(0, 22); // global_gain, ics_info, no pulse, TNS or gain control data + } + bw.write_bits(1, 3); // ID_CPE + bw.write_bits(0, 5); // tag, common_window + bw.write_bits(0, 44); // two such channel streams + bw.write_bits(7, 3); // ID_END + let au = bw.finish(); + [&aac::AdtsHeader::write(2, 3, channel_config, au.len(), 0x7FF).unwrap()[..], &au].concat() + } + + #[cfg(feature = "aac-decode")] + fn aac_params() -> CodecParams { + CodecParams::new(CodecId::Aac) + } + + #[cfg(feature = "aac-decode")] + #[test] + fn adts_packets_conceal_to_the_length_of_all_their_frames() { + let frame = adts_frame(2); + let mut dec = TrackDecoder::new(&aac_params(), &[&frame]).unwrap(); + let packet = frame.repeat(3); + assert_eq!(dec.decode(&packet).unwrap().frames(), 3072); + // the middle frame's payload is damaged: its element id is ID_CCE, which is unsupported + let mut damaged = packet.clone(); + damaged[frame.len() + 7] = 0b0100_0000; + assert!(dec.decode(&damaged).is_err()); + assert_eq!(dec.conceal(Some(&damaged)).unwrap().frames(), 3072); + assert_eq!(dec.conceal(Some(&frame)).unwrap().frames(), 1024); + // a packet lost whole lasts as long as the last one decoded: here, three frames + assert_eq!(dec.conceal(None).unwrap().frames(), 3072); + } + + /// Implicitly signalled HE-AAC: an AAC-LC configuration at 24 kHz or less, with SBR data in a + /// fill element. + #[cfg(feature = "aac-decode")] + #[test] + fn he_aac_is_found_in_the_lookahead_and_upsampled() { + use craft_media::bitstream::BitWriter; + let unit = |sbr: bool| { + let mut bw = BitWriter::new(); + bw.write_bits(0, 7); // ID_SCE, tag 0 + bw.write_bits(0, 22); // an empty channel stream + if sbr { + bw.write_bits(6, 3); // ID_FIL + bw.write_bits(1, 4); // count + bw.write_bits(13, 4); // EXT_SBR_DATA + bw.write_bits(0, 4); + } + bw.write_bits(7, 3); // ID_END + bw.finish() + }; + // 22.05 kHz, mono + let params = aac_params().with_extradata(vec![0x13, 0x88]); + let (plain, he) = (unit(false), unit(true)); + let dec = TrackDecoder::new(¶ms, &[&plain]).unwrap(); + assert_eq!((dec.codec(), dec.sample_rate(), dec.frames_per_packet()), (CodecId::Aac, 22_050, Some(1024))); + let mut dec = TrackDecoder::new(¶ms, &[&plain, &he]).unwrap(); + assert_eq!((dec.codec(), dec.sample_rate(), dec.frames_per_packet(), dec.preroll()), (CodecId::HeAac, 44_100, Some(2048), 4096)); + assert_eq!(dec.decode(&he).unwrap().frames(), 2048); + assert_eq!(dec.conceal(None).unwrap().frames(), 2048); + } + + /// A signalled output rate that is not twice the core's is not upsampled to: the core plays. + #[cfg(feature = "aac-decode")] + #[test] + fn an_sbr_rate_other_than_twice_the_core_plays_the_core() { + // AOT 5, 24 kHz, mono, extension 44.1 kHz, core AOT 2 + let asc = { + use craft_media::bitstream::BitWriter; + let mut bw = BitWriter::new(); + for (value, bits) in [(5, 5), (6, 4), (1, 4), (4, 4), (2, 5), (0, 3)] { + bw.write_bits(value, bits); + } + bw.finish() + }; + let dec = TrackDecoder::new(&aac_params().with_extradata(asc), &[]).unwrap(); + assert_eq!((dec.codec(), dec.sample_rate(), dec.frames_per_packet()), (CodecId::Aac, 24_000, Some(1024))); + } + + #[cfg(feature = "aac-decode")] + #[test] + fn adts_frames_of_another_format_or_several_blocks_conceal_at_their_length() { + let frame = adts_frame(2); + let mut dec = TrackDecoder::new(&aac_params(), &[&frame]).unwrap(); + // sampling frequency index 4 where the stream has 3 + let mut other = frame.clone(); + other[2] = (other[2] & 0xC3) | (4 << 2); + assert!(matches!(dec.decode(&other), Err(AudioError::Malformed(_)))); + assert_eq!(dec.conceal(Some(&other)).unwrap().frames(), 1024); + let mut blocks = frame.clone(); + blocks[6] |= 2; // three raw data blocks + assert!(matches!(dec.decode(&blocks), Err(AudioError::Unsupported(_)))); + assert_eq!(dec.conceal(Some(&blocks)).unwrap().frames(), 3072); + // the profile and the channel configuration are part of the format too + let mut main = frame.clone(); + main[2] &= 0x3F; // profile 0: Main + assert!(matches!(dec.decode(&main), Err(AudioError::Malformed(_)))); + let mut mono = frame.clone(); + mono[3] = (mono[3] & 0x3F) | (1 << 6); + assert!(matches!(dec.decode(&mono), Err(AudioError::Malformed(_)))); + } + + /// With extradata, a header's differing fields are not a format change (ADTS in MP4 with a + /// program config element in its AudioSpecificConfig, or channel configuration 0 in its headers). + #[cfg(feature = "aac-decode")] + #[test] + fn extradata_decides_the_format_of_adts_packets() { + let frame = adts_frame(0); + let mut stereo = adts_frame(2); + let params = aac_params().with_extradata(aac::AudioSpecificConfig::lc(3, 2).to_bytes().unwrap()); + let mut dec = TrackDecoder::new(¶ms, &[]).unwrap(); + assert_eq!(dec.decode(&stereo).unwrap().channels.len(), 2); + stereo[3] &= 0x3F; // channel configuration 0 in the header + stereo[2] &= 0xFE; + assert_eq!(dec.decode(&stereo).unwrap().channels.len(), 2); + assert!(dec.decode(&frame).is_err(), "elements the configuration does not have"); + } + + /// One damaged header at the start of a stream does not decide its format. + #[cfg(feature = "aac-decode")] + #[test] + fn the_adts_format_is_the_one_most_of_the_lookahead_shares() { + let good = adts_frame(2); + let mut damaged = good.clone(); + damaged[2] = (damaged[2] & 0xC3) | (4 << 2); // sampling frequency index 4 where the rest have 3 + let lookahead: [&[u8]; 4] = [&damaged, &good, &good, &good]; + let mut dec = TrackDecoder::new(&aac_params(), &lookahead).unwrap(); + assert_eq!(dec.sample_rate(), 48_000); + assert!(dec.decode(&damaged).is_err()); + assert_eq!(dec.decode(&good).unwrap().frames(), 1024); + // on a tie, the earlier one + let tied: [&[u8]; 2] = [&good, &damaged]; + assert_eq!(TrackDecoder::new(&aac_params(), &tied).unwrap().sample_rate(), 48_000); + #[cfg(feature = "decode-native")] + assert_eq!((adts_frame_count(&good), adts_frame_count(&good.repeat(2)), adts_frame_count(&[0; 8])), (1, 2, 0)); + } + + /// `au` as a stereo ADTS frame at 48 kHz (channel configuration `channel_config`). + #[cfg(feature = "aac-decode")] + fn adts_of(au: &[u8], channel_config: u8) -> Vec { + [&aac::AdtsHeader::write(2, 3, channel_config, au.len(), 0x7FF).unwrap()[..], au].concat() + } + + /// An access unit of `pce` and `elements` (element id, tag), all empty, with channel streams of + /// 22 bits (a CPE's are preceded by its `common_window` bit). + #[cfg(feature = "aac-decode")] + fn au_of(pce: Option<&aac::ProgramConfig>, elements: &[(u32, u32)]) -> Vec { + use craft_media::bitstream::BitWriter; + let mut bw = BitWriter::new(); + if let Some(pce) = pce { + bw.write_bits(5, 3); // ID_PCE + pce.write(&mut bw).unwrap(); + } + for &(id, tag) in elements { + bw.write_bits(id, 3); + bw.write_bits(tag, 4); + if id == 1 { + bw.write_bits(0, 1); + bw.write_bits(0, 22); + } + bw.write_bits(0, 22); + } + bw.write_bits(7, 3); // ID_END + bw.finish() + } + + #[cfg(feature = "aac-decode")] + fn stereo_pce() -> aac::ProgramConfig { + aac::ProgramConfig { front: vec![(true, 0)], ..Default::default() } + } + + /// HE-AAC whose SBR data first appears in a later frame of a lookahead packet is found. + #[cfg(feature = "aac-decode")] + #[test] + fn sbr_in_a_later_adts_frame_of_a_packet_is_found() { + use craft_media::bitstream::BitWriter; + let unit = |sbr: bool| { + let mut bw = BitWriter::new(); + bw.write_bits(0, 7); // ID_SCE, tag 0 + bw.write_bits(0, 22); + if sbr { + bw.write_bits(6, 3); // ID_FIL + bw.write_bits(1, 4); + bw.write_bits(13, 4); // EXT_SBR_DATA + bw.write_bits(0, 4); + } + bw.write_bits(7, 3); + bw.finish() + }; + let frame = |au: &[u8]| [&aac::AdtsHeader::write(2, 7, 1, au.len(), 0x7FF).unwrap()[..], au].concat(); + let packet = [frame(&unit(false)), frame(&unit(true))].concat(); + let params = aac_params().with_extradata(vec![0x13, 0x88]); + let dec = TrackDecoder::new(¶ms, &[&packet]).unwrap(); + assert_eq!((dec.codec(), dec.sample_rate()), (CodecId::HeAac, 44_100)); + } + + /// A packet whose later frame fails leaves the decoder as a failed single frame does. + #[cfg(feature = "aac-decode")] + #[test] + fn a_failed_frame_undoes_the_earlier_frames_of_its_packet() { + let first = adts_frame(0); + let mut dec = TrackDecoder::new(&aac_params(), &[&first]).unwrap(); + assert_eq!(dec.decode(&first).unwrap().channels.len(), 3); + // A valid frame that changes the layout, then one that fails. + let change = adts_of(&au_of(Some(&stereo_pce()), &[(1, 0)]), 0); + let bad = adts_of(&au_of(None, &[(2, 0)]), 0); // ID_CCE, unsupported + assert!(dec.decode(&[change, bad].concat()).is_err()); + // Still three channels: SCE and CPE with no program config of their own. + let plain = adts_of(&au_of(None, &[(0, 0), (1, 0)]), 0); + assert_eq!(dec.decode(&plain).unwrap().channels.len(), 3); + } + + /// Frames of one packet with different channel counts are an error, not a short packet. + #[cfg(feature = "aac-decode")] + #[test] + fn a_channel_count_change_inside_a_packet_is_an_error() { + let first = adts_frame(0); + let mut dec = TrackDecoder::new(&aac_params(), &[&first]).unwrap(); + let change = adts_of(&au_of(Some(&stereo_pce()), &[(1, 0)]), 0); + assert!(matches!(dec.decode(&[first.clone(), change.clone()].concat()), Err(AudioError::Malformed(_)))); + assert_eq!(dec.conceal(Some(&[first, change].concat())).unwrap().frames(), 2048); + } + + /// Whole frames before a damaged or truncated end of a packet still decode. + #[cfg(feature = "aac-decode")] + #[test] + fn the_whole_frames_before_a_damaged_tail_decode() { + let frame = adts_frame(2); + let mut dec = TrackDecoder::new(&aac_params(), &[&frame]).unwrap(); + let cut = [frame.repeat(2), frame[..frame.len() - 3].to_vec()].concat(); + assert_eq!(dec.decode(&cut).unwrap().frames(), 2048); + let garbage = [frame.repeat(2), vec![0x12; 9]].concat(); + assert_eq!(dec.decode(&garbage).unwrap().frames(), 2048); + } + + #[cfg(feature = "aac-decode")] + #[test] + fn a_packet_of_too_many_adts_frames_is_refused() { + let frame = adts_frame(2); + let mut dec = TrackDecoder::new(&aac_params(), &[&frame]).unwrap(); + assert_eq!(dec.decode(&frame.repeat(MAX_ADTS_FRAMES_PER_PACKET)).unwrap().frames(), MAX_ADTS_FRAMES_PER_PACKET * 1024); + let packet = frame.repeat(MAX_ADTS_FRAMES_PER_PACKET + 1); + assert!(matches!(dec.decode(&packet), Err(AudioError::Malformed(_)))); + assert_eq!(dec.conceal(Some(&packet)).unwrap().frames(), MAX_ADTS_FRAMES_PER_PACKET * 1024); + } + + #[cfg(feature = "aac-decode")] + #[test] + fn adts_channel_configuration_0_takes_its_layout_from_the_stream() { + let frame = adts_frame(0); + let mut dec = TrackDecoder::new(&aac_params(), &[&frame]).unwrap(); + assert_eq!(dec.channels(), 3); + assert_eq!(dec.decode(&frame).unwrap().channels.len(), 3); + // no program config element where one is needed + let bare = aac::AdtsHeader::write(2, 3, 0, 1, 0x7FF).unwrap(); + let bare = [&bare[..], &[0xE0]].concat(); + assert!(matches!(TrackDecoder::new(&aac_params(), &[&bare]), Err(AudioError::Malformed(_)))); + } + #[test] fn codecs_without_a_packet_decoder_are_unsupported() { - for codec in [CodecId::Aac, CodecId::Flac, CodecId::Vorbis, CodecId::Alac, CodecId::Mp3, CodecId::Opus, CodecId::H264] { + for codec in [CodecId::Flac, CodecId::Vorbis, CodecId::Alac, CodecId::Mp3, CodecId::Opus, CodecId::H264] { assert!(matches!(TrackDecoder::new(&CodecParams::new(codec), &[]), Err(AudioError::Unsupported(_))), "{codec:?}"); } + // A codec whose feature is off names the feature. + #[cfg(not(feature = "aac-decode"))] + assert!(matches!(TrackDecoder::new(&CodecParams::new(CodecId::Aac), &[]), Err(AudioError::Unsupported(m)) if m.contains("aac-decode"))); // PCM needs its sample format. assert!(TrackDecoder::new(&CodecParams::new(CodecId::Pcm).with_channels(2).with_sample_rate(48_000), &[]).is_err()); } diff --git a/crates/audio/src/write/flac/oracle_tests.rs b/crates/audio/src/write/flac/oracle_tests.rs index a9f80ecef..aaf2322e1 100644 --- a/crates/audio/src/write/flac/oracle_tests.rs +++ b/crates/audio/src/write/flac/oracle_tests.rs @@ -357,6 +357,58 @@ fn ogg_flac_before_opus_is_the_first_track() { } } +/// The first page of a Speex stream, which symphonia has no reader for. +#[cfg(feature = "symphonia")] +fn speex_ogg_page(serial: u32) -> Vec { + let mut packet = b"Speex ".to_vec(); + packet.resize(80, 0); + let mut page = b"OggS\0\x02".to_vec(); // the start of a stream + page.extend_from_slice(&[0; 8]); // granule position + page.extend_from_slice(&serial.to_le_bytes()); + page.extend_from_slice(&[0; 8]); // page number and checksum + page.extend_from_slice(&[1, packet.len() as u8]); + page.extend_from_slice(&packet); + let crc = craft_media::ogg::crc32(&page); + page[22..26].copy_from_slice(&crc.to_le_bytes()); + page +} + +/// A stream symphonia has no track for does not stop the default decode from reading the next one, +/// and it keeps its index. +#[cfg(feature = "symphonia")] +#[test] +fn ogg_flac_after_a_stream_with_no_reader_is_the_default_track() { + use crate::{CodecId, DecodeOptions}; + let Some(ff) = ffmpeg() else { return }; + let args = ["-c:a", "flac"]; + for reversed in [false, true] { + let Some(flac) = two_ogg_streams(&ff, "ogg-speex-flac", &args, reversed) else { return }; + for speex_serial in [0, 9] { + let file = [speex_ogg_page(speex_serial), flac.clone()].concat(); + let codecs: Vec<_> = crate::tracks(&file, None).unwrap().iter().map(|t| t.codec).collect(); + assert_eq!(codecs, [CodecId::Speex, CodecId::Flac, CodecId::Flac]); + let info = crate::decode(&file, None).unwrap().0; + assert_eq!((info.codec, info.sample_rate), (CodecId::Flac, 48_000), "reversed {reversed}, speex {speex_serial}"); + assert!(crate::decode_with(&file, None, &DecodeOptions::default().with_track(0)).is_err()); + let second = crate::decode_with(&file, None, &DecodeOptions::default().with_track(2)).unwrap().0; + assert_eq!(second.sample_rate, 44_100, "reversed {reversed}, speex {speex_serial}"); + } + } +} + +/// Bytes before the first page do not lose the file's track order or mask comments. +#[cfg(feature = "symphonia")] +#[test] +fn ogg_flac_after_leading_bytes_keeps_its_track_order() { + use crate::{ChannelLayout, DecodeOptions}; + let Some(ff) = ffmpeg() else { return }; + let args = ["-c:a", "flac", "-metadata:s:a:0", "WAVEFORMATEXTENSIBLE_CHANNEL_MASK=0x0107"]; + let Some(bytes) = two_ogg_streams(&ff, "ogg-leading-bytes", &args, true) else { return }; + let file = [vec![0; 64], bytes].concat(); + let info = crate::decode_with(&file, None, &DecodeOptions::default().with_track(0)).unwrap().0; + assert_eq!((info.sample_rate, info.layout), (48_000, ChannelLayout::from_mask(0x107))); +} + #[test] fn the_mask_comment_is_written_only_when_decoders_would_assume_otherwise() { use super::mask_comment; diff --git a/crates/audio/tests/codecs/aac/common.rs b/crates/audio/tests/codecs/aac/common.rs new file mode 100644 index 000000000..d63fc0740 --- /dev/null +++ b/crates/audio/tests/codecs/aac/common.rs @@ -0,0 +1,245 @@ +#![allow(dead_code)] + +use std::path::{Path, PathBuf}; +use std::process::Command; + +use craft_audio::aac::{Decoder, split_adts}; + +/// ffmpeg (see `crate::testkit::oracle`); `None` after printing `SKIPPED`. +pub fn ffmpeg() -> Option { + crate::testkit::ffmpeg_or_skip("aac oracle") +} + +pub fn fixtures() -> PathBuf { + crate::testkit::fixtures_dir("aac") +} + +pub struct Rng(pub u64); +impl Rng { + pub fn next(&mut self) -> f32 { + self.0 ^= self.0 << 13; + self.0 ^= self.0 >> 7; + self.0 ^= self.0 << 17; + ((self.0 >> 40) as f32 / (1u64 << 24) as f32) * 2.0 - 1.0 + } +} + +use std::f32::consts::PI; + +/// Logarithmic sine sweep 40 Hz → 16 kHz. +pub fn sweep(rate: u32, n: usize, amp: f32) -> Vec { + let (f0, f1) = (40f64, 16000f64.min(rate as f64 * 0.45)); + let t_total = n as f64 / rate as f64; + let k = (f1 / f0).ln(); + (0..n) + .map(|i| { + let t = i as f64 / rate as f64; + let phase = 2.0 * std::f64::consts::PI * f0 * t_total / k * ((t / t_total * k).exp() - 1.0); + (phase.sin() as f32) * amp + }) + .collect() +} + +pub fn noise(n: usize, amp: f32, seed: u64) -> Vec { + let mut r = Rng(seed); + (0..n).map(|_| r.next() * amp).collect() +} + +/// Speech-like: glottal pulse train (gliding pitch) through three resonant formant filters with a +/// syllabic envelope and pauses, plus a little fricative noise. +pub fn speech(rate: u32, n: usize, seed: u64) -> Vec { + let mut r = Rng(seed); + let fs = rate as f32; + let formants = [(700.0f32, 130.0f32), (1220.0, 70.0), (2600.0, 160.0)]; + let mut states = [[0f32; 2]; 3]; + let mut phase = 0f32; + let mut out = Vec::with_capacity(n); + for i in 0..n { + let t = i as f32 / fs; + let f0 = 120.0 + 30.0 * (2.0 * PI * 0.7 * t).sin(); + phase += f0 / fs; + let pulse = if phase >= 1.0 { + phase -= 1.0; + 1.0 + } else { + 0.0 + }; + let syll = (2.0 * PI * 3.0 * t).sin().max(0.0); + let voiced = if (t * 1.3).fract() < 0.8 { syll } else { 0.0 }; + let fric = if (t * 1.3).fract() > 0.85 { 0.05 * r.next() } else { 0.0 }; + let mut s = 0f32; + for (j, &(f, bw)) in formants.iter().enumerate() { + let rr = (-PI * bw / fs).exp(); + let a1 = 2.0 * rr * (2.0 * PI * f / fs).cos(); + let a2 = -rr * rr; + let y = pulse * voiced + a1 * states[j][0] + a2 * states[j][1]; + states[j][1] = states[j][0]; + states[j][0] = y; + s += y * [1.0, 0.6, 0.3][j]; + } + out.push(s * 0.08 + fric); + } + let peak = out.iter().fold(0f32, |m, v| m.max(v.abs())); + out.iter().map(|v| v / peak * 0.7).collect() +} + +/// Music-like: a sequence of decaying harmonic chords. +pub fn chords(rate: u32, n: usize, detune: f32) -> Vec { + let fs = rate as f32; + let roots = [220.0f32, 261.63, 196.0, 246.94]; + let chord_len = (fs * 0.5) as usize; + (0..n) + .map(|i| { + let c = (i / chord_len) % roots.len(); + let t = (i % chord_len) as f32 / fs; + let env = (-t * 3.0).exp() * (1.0 - (-t * 200.0).exp()); + let mut s = 0f32; + for (m, ratio) in [1.0f32, 1.25, 1.5, 2.0].iter().enumerate() { + let f = roots[c] * ratio * (1.0 + detune); + for h in 1..8 { + s += (2.0 * PI * f * h as f32 * t + m as f32).sin() / (h * h) as f32; + } + } + s * env * 0.18 + }) + .collect() +} + +/// Clicks: sharp impulses every 250 ms over a quiet low tone (transient test). +pub fn clicks(rate: u32, n: usize) -> Vec { + let fs = rate as f32; + let period = (fs * 0.25) as usize; + (0..n) + .map(|i| { + let t = i as f32 / fs; + let k = i % period; + let click = if k < 256 { 0.8 * (-(k as f32) / 24.0).exp() * (2.0 * PI * 1500.0 * k as f32 / fs).cos() } else { 0.0 }; + 0.02 * (2.0 * PI * 330.0 * t).sin() + click + }) + .collect() +} + +/// Decode with ffmpeg → planar f32 (ffmpeg's own channel order), plus stderr. +pub fn ffmpeg_decode(ff: &Path, file: &Path, channels: usize) -> (Vec>, String) { + let out = Command::new(ff).args(["-v", "error", "-i"]).arg(file).args(["-f", "f32le", "-"]).output().unwrap(); + let data = out.stdout; + let n = data.len() / 4 / channels; + let mut planar = vec![Vec::with_capacity(n); channels]; + for i in 0..n { + for c in 0..channels { + let o = (i * channels + c) * 4; + planar[c].push(f32::from_le_bytes([data[o], data[o + 1], data[o + 2], data[o + 3]])); + } + } + (planar, String::from_utf8_lossy(&out.stderr).into_owned()) +} + +/// Encode with ffmpeg's AAC encoder to ADTS. +pub fn ffmpeg_encode(ff: &Path, planar: &[Vec], rate: u32, args: &[&str], out: &Path) { + let raw = out.with_extension("f32"); + let ch = planar.len(); + let mut bytes = Vec::with_capacity(planar[0].len() * ch * 4); + for i in 0..planar[0].len() { + for c in planar { + bytes.extend_from_slice(&c[i].to_le_bytes()); + } + } + std::fs::write(&raw, bytes).unwrap(); + let layout = match ch { + 1 => "mono", + 2 => "stereo", + 3 => "3.0", + 4 => "4.0", + 5 => "5.0", + 6 => "5.1", + _ => "7.1(wide)", + }; + let st = Command::new(ff) + .args(["-v", "error", "-y", "-f", "f32le", "-ar", &rate.to_string(), "-ch_layout", layout, "-i"]) + .arg(&raw) + .args(["-c:a", "aac"]) + .args(args) + .args(["-f", "adts"]) + .arg(out) + .status() + .unwrap(); + assert!(st.success()); +} + +/// Decode an ADTS file with our decoder. +pub fn our_decode(data: &[u8]) -> Vec> { + let frames = split_adts(data).unwrap(); + let mut dec = Decoder::from_adts(&frames[0].0).unwrap(); + let mut out: Vec> = Vec::new(); + for (_, au) in frames { + let pcm = dec.decode(au).unwrap(); + out.resize(pcm.len(), Vec::new()); + for (o, p) in out.iter_mut().zip(pcm) { + o.extend(p); + } + } + out +} + +/// SNR in dB of `dec` against `refr`. +pub fn snr(refr: &[f32], dec: &[f32]) -> f64 { + let n = refr.len().min(dec.len()); + let (mut s, mut e) = (0f64, 0f64); + for i in 0..n { + s += (refr[i] as f64).powi(2); + e += (refr[i] as f64 - dec[i] as f64).powi(2); + } + 10.0 * (s / e.max(1e-20)).log10() +} + +/// Segmental SNR (1024-sample segments above -50 dBFS, each clamped to [-10, 60] dB). +pub fn seg_snr(refr: &[f32], dec: &[f32]) -> f64 { + let n = refr.len().min(dec.len()); + let mut acc = 0f64; + let mut cnt = 0; + for s in (0..n.saturating_sub(1024)).step_by(1024) { + let (mut p, mut e) = (0f64, 0f64); + for i in s..s + 1024 { + p += (refr[i] as f64).powi(2); + e += (refr[i] as f64 - dec[i] as f64).powi(2); + } + if p / 1024.0 < 1e-5 { + continue; + } + acc += (10.0 * (p / e.max(1e-20)).log10()).clamp(-10.0, 60.0); + cnt += 1; + } + if cnt == 0 { 0.0 } else { acc / cnt as f64 } +} + +pub fn max_abs_diff(a: &[f32], b: &[f32]) -> f32 { + a.iter().zip(b).fold(0f32, |m, (x, y)| m.max((x - y).abs())) +} + +/// Windowed-sinc (Blackman, 401 taps) low-pass at `fc` Hz, zero-phase (delay-compensated). +pub fn lowpass(x: &[f32], fc: f32, fs: f32) -> Vec { + let taps = 401usize; + let m = (taps - 1) as f32; + let wc = fc / fs; + let h: Vec = (0..taps) + .map(|i| { + let n = i as f32 - m / 2.0; + let sinc = if n == 0.0 { 2.0 * wc } else { (2.0 * PI * wc * n).sin() / (PI * n) }; + let w = 0.42 - 0.5 * (2.0 * PI * i as f32 / m).cos() + 0.08 * (4.0 * PI * i as f32 / m).cos(); + sinc * w + }) + .collect(); + let half = taps / 2; + (0..x.len()) + .map(|i| { + let mut acc = 0f32; + for (k, hk) in h.iter().enumerate() { + let j = i as isize + k as isize - half as isize; + if j >= 0 && (j as usize) < x.len() { + acc += hk * x[j as usize]; + } + } + acc + }) + .collect() +} diff --git a/crates/audio/tests/codecs/aac/decoder_oracle.rs b/crates/audio/tests/codecs/aac/decoder_oracle.rs new file mode 100644 index 000000000..3a4eb94b9 --- /dev/null +++ b/crates/audio/tests/codecs/aac/decoder_oracle.rs @@ -0,0 +1,175 @@ +//! Decoder accuracy against ffmpeg's decoder on streams produced by ffmpeg's AAC encoder (oracle +//! only; skipped when ffmpeg is absent). + +use super::common; + +use common::*; + +struct Case { + name: &'static str, + rate: u32, + channels: usize, + args: &'static [&'static str], +} + +fn input(rate: u32, channels: usize) -> Vec> { + let n = rate as usize * 3 / 2; + (0..channels) + .map(|c| { + let a = chords(rate, n, 0.003 * c as f32); + let b = clicks(rate, n); + let s = speech(rate, n, 5 + c as u64); + a.iter().zip(&b).zip(&s).map(|((x, y), z)| 0.6 * x + 0.3 * y + 0.3 * z * if c % 2 == 0 { 1.0 } else { -0.5 }).collect() + }) + .collect() +} + +/// Returns (max abs error, samples compared). +fn compare(case: &Case) -> Option<(f32, usize)> { + let ff = ffmpeg()?; + let path = fixtures().join(format!("ffenc_{}.aac", case.name)); + ffmpeg_encode(&ff, &input(case.rate, case.channels), case.rate, case.args, &path); + let data = std::fs::read(&path).unwrap(); + let (ref_pcm, err) = ffmpeg_decode(&ff, &path, case.channels); + assert!(err.trim().is_empty(), "{err}"); + let ours = our_decode(&data); + assert_eq!(ours.len(), case.channels); + // ffmpeg reorders channels to its native layouts; match each of our (AAC element order) + // channels to the closest ffmpeg channel and require a permutation. + let mut worst = 0f32; + let mut taken = vec![false; case.channels]; + for c in 0..case.channels { + let (best, err) = + (0..case.channels).filter(|&j| !taken[j]).map(|j| (j, max_abs_diff(&ours[c], &ref_pcm[j]))).min_by(|a, b| a.1.total_cmp(&b.1)).unwrap(); + taken[best] = true; + assert_eq!(ours[c].len(), ref_pcm[best].len(), "{}: length", case.name); + worst = worst.max(err); + } + Some((worst, ours[0].len())) +} + +#[test] +fn matches_ffmpeg_decoder() { + if ffmpeg().is_none() { + eprintln!("ffmpeg not found; skipping"); + return; + } + let cases = [ + Case { name: "stereo_44k_128k", rate: 44100, channels: 2, args: &["-b:a", "128k", "-aac_pns", "0"] }, + Case { name: "stereo_48k_256k", rate: 48000, channels: 2, args: &["-b:a", "256k", "-aac_pns", "0"] }, + Case { name: "mono_48k_64k", rate: 48000, channels: 1, args: &["-b:a", "64k", "-aac_pns", "0"] }, + Case { + name: "stereo_44k_96k_is_ms_tns", + rate: 44100, + channels: 2, + args: &["-b:a", "96k", "-aac_pns", "0", "-aac_is", "1", "-aac_ms", "1", "-aac_tns", "1"], + }, + Case { name: "stereo_32k_48k_is", rate: 32000, channels: 2, args: &["-b:a", "48k", "-aac_pns", "0", "-aac_is", "1"] }, + Case { name: "mono_22k_32k", rate: 22050, channels: 1, args: &["-b:a", "32k", "-aac_pns", "0"] }, + Case { name: "mono_16k_24k", rate: 16000, channels: 1, args: &["-b:a", "24k", "-aac_pns", "0"] }, + Case { name: "mono_8k_16k", rate: 8000, channels: 1, args: &["-b:a", "16k", "-aac_pns", "0"] }, + Case { name: "stereo_96k_256k", rate: 96000, channels: 2, args: &["-b:a", "256k", "-aac_pns", "0"] }, + Case { name: "c3_48k", rate: 48000, channels: 3, args: &["-b:a", "192k", "-aac_pns", "0"] }, + Case { name: "c51_48k_320k", rate: 48000, channels: 6, args: &["-b:a", "320k", "-aac_pns", "0"] }, + Case { name: "c71_48k_448k", rate: 48000, channels: 8, args: &["-b:a", "448k", "-aac_pns", "0"] }, + Case { name: "stereo_44k_fast_coder", rate: 44100, channels: 2, args: &["-b:a", "160k", "-aac_coder", "fast", "-aac_pns", "0"] }, + ]; + println!("{:<28} {:>10} {:>12}", "stream", "samples/ch", "max |err|"); + let mut fails = Vec::new(); + for c in &cases { + let (err, n) = compare(c).unwrap(); + println!("{:<28} {:>10} {:>12.3e}", c.name, n, err); + if err >= 1e-5 { + fails.push((c.name, err)); + } + } + assert!(fails.is_empty(), "decoder mismatches: {fails:?}"); +} + +/// PNS output is random by design, so samples cannot be compared: build noise-substituted access +/// units by hand (mono, and a common-window pair with M/S = correlated noise) and compare +/// per-frame energies and the L/R correlation with ffmpeg's decode. +#[test] +fn pns_matches_ffmpeg() { + use craft_audio::aac::AdtsHeader; + use craft_media::bitstream::BitWriter; + let Some(ff) = ffmpeg() else { return }; + fn ics(bw: &mut BitWriter, with_info: bool, nrg: u32) { + bw.write_bits(100, 8); // global_gain + if with_info { + bw.write_bits(0, 4); // reserved, ONLY_LONG, sine + bw.write_bits(40, 6); + bw.write_bits(0, 1); + } + bw.write_bits(13, 4); // NOISE_HCB for all 40 bands + bw.write_bits(31, 5); + bw.write_bits(9, 5); + bw.write_bits(nrg, 9); // first noise energy (PCM) + for _ in 1..40 { + bw.write_bits(0, 1); // difference 0 + } + bw.write_bits(0, 3); // pulse, tns, gain control + } + for channels in [1usize, 2] { + let mut stream = Vec::new(); + for f in 0..12u32 { + let mut bw = BitWriter::new(); + let nrg = 256 + 20 + (f % 6) * 7; + if channels == 1 { + bw.write_bits(0, 3); + bw.write_bits(0, 4); + ics(&mut bw, true, nrg); + } else { + bw.write_bits(1, 3); + bw.write_bits(0, 4); + bw.write_bits(1, 1); // common_window + bw.write_bits(0, 4); + bw.write_bits(40, 6); + bw.write_bits(0, 1); + bw.write_bits(2, 2); // ms_mask_present = all + ics(&mut bw, false, nrg); + ics(&mut bw, false, nrg - 8); + } + bw.write_bits(7, 3); + let au = bw.finish(); + stream.extend_from_slice(&AdtsHeader::write(2, 4, channels as u8, au.len(), 0x7FF).unwrap()); + stream.extend_from_slice(&au); + } + let path = fixtures().join(format!("handmade_pns_{channels}.aac")); + std::fs::write(&path, &stream).unwrap(); + let (ref_pcm, err) = ffmpeg_decode(&ff, &path, channels); + assert!(err.trim().is_empty(), "{err}"); + let ours = our_decode(&stream); + let mut worst_db = 0f64; + for c in 0..channels { + let e = |v: &[f32]| 10.0 * v.iter().map(|x| (*x as f64).powi(2)).sum::().log10(); + let (eo, ef) = (e(&ours[c]), e(&ref_pcm[c])); + println!("PNS {channels}ch ch{c}: total energy ours {eo:.2} dB, ffmpeg {ef:.2} dB"); + assert!((eo - ef).abs() < 0.5); + assert_eq!(ours[c].len(), ref_pcm[c].len()); + for s in (1024..ours[c].len() - 1024).step_by(1024) { + let ea: f64 = ours[c][s..s + 1024].iter().map(|v| (*v as f64).powi(2)).sum(); + let eb: f64 = ref_pcm[c][s..s + 1024].iter().map(|v| (*v as f64).powi(2)).sum(); + worst_db = worst_db.max((10.0 * (ea / eb).log10()).abs()); + } + } + let corr = |a: &[f32], b: &[f32]| { + let (mut ab, mut aa, mut bb) = (0f64, 0f64, 0f64); + for (x, y) in a.iter().zip(b) { + ab += *x as f64 * *y as f64; + aa += (*x as f64).powi(2); + bb += (*y as f64).powi(2); + } + ab / (aa * bb).sqrt() + }; + let (c_ours, c_ff) = if channels == 2 { (corr(&ours[0], &ours[1]), corr(&ref_pcm[0], &ref_pcm[1])) } else { (0.0, 0.0) }; + println!("PNS {channels}ch: per-frame energy max deviation {worst_db:.2} dB; L/R correlation ours {c_ours:.3} ffmpeg {c_ff:.3}"); + if channels == 1 { + assert!(worst_db < 1.0); + } else { + // ISO/IEC 14496-3 §4.6.13: noise bands with ms_used share one random vector (correlated + // noise). ffmpeg does not implement this and generates independent noise. + assert!(c_ours > 0.99); + } + } +} diff --git a/crates/audio/tests/codecs/aac/mod.rs b/crates/audio/tests/codecs/aac/mod.rs new file mode 100644 index 000000000..1a992c77b --- /dev/null +++ b/crates/audio/tests/codecs/aac/mod.rs @@ -0,0 +1,4 @@ +//! FilmCraft's `filmcraft-aac` integration tests. + +mod common; +mod decoder_oracle; diff --git a/crates/audio/tests/codecs/main.rs b/crates/audio/tests/codecs/main.rs index 5fddafb29..49c505800 100644 --- a/crates/audio/tests/codecs/main.rs +++ b/crates/audio/tests/codecs/main.rs @@ -7,4 +7,5 @@ use craft_media_testkit as common; #[macro_use] mod testkit; +mod aac; mod routing_oracle; diff --git a/crates/audio/tests/codecs/routing_oracle.rs b/crates/audio/tests/codecs/routing_oracle.rs index 6a01b9aa6..bcb60c1e6 100644 --- a/crates/audio/tests/codecs/routing_oracle.rs +++ b/crates/audio/tests/codecs/routing_oracle.rs @@ -1,6 +1,7 @@ //! `craft_audio::decode` routes each container and codec below to the home-grown demuxers and -//! decoders, or to symphonia (FLAC): each pair decodes, sample-aligned, to its presentation -//! length, within an SNR of ffmpeg's decode; truncated and corrupted copies never panic. Fixtures are written by ffmpeg +//! decoders, or to symphonia (FLAC): each pair decodes, sample-aligned, to its presentation length +//! (edit lists, Matroska `CodecDelay` and `DiscardPadding` applied), within an SNR +//! of ffmpeg's decode; truncated and corrupted copies never panic. Fixtures are written by ffmpeg //! (skipped without it). use craft_audio::{ChannelLayout, CodecId, Container, decode, probe}; @@ -44,6 +45,58 @@ const CASES: &[Case] = &[ frames: Some(240_000), min_snr: 140.0, }, + Case { + name: "aac.aac", + surround: false, + mono: false, + rate: 48_000, + args: &["-c:a", "aac", "-b:a", "128k", "-aac_pns", "0"], + container: Container::Adts, + codec: CodecId::Aac, + layout: ChannelLayout::STEREO, + channels: 2, + frames: None, + min_snr: 120.0, + }, + Case { + name: "aac.m4a", + surround: false, + mono: false, + rate: 44_100, + args: &["-c:a", "aac", "-b:a", "128k", "-aac_pns", "0"], + container: Container::Mp4, + codec: CodecId::Aac, + layout: ChannelLayout::STEREO, + channels: 2, + frames: Some(220_500), + min_snr: 120.0, + }, + Case { + name: "aac-5.1.m4a", + surround: true, + mono: false, + rate: 48_000, + args: &["-c:a", "aac", "-b:a", "384k", "-aac_pns", "0"], + container: Container::Mp4, + codec: CodecId::Aac, + layout: ChannelLayout::SURROUND_5_1_SIDE, + channels: 6, + frames: Some(240_000), + min_snr: 120.0, + }, + Case { + name: "aac.mkv", + surround: false, + mono: false, + rate: 48_000, + args: &["-c:a", "aac", "-b:a", "128k", "-aac_pns", "0"], + container: Container::Matroska, + codec: CodecId::Aac, + layout: ChannelLayout::STEREO, + channels: 2, + frames: Some(240_000), + min_snr: 120.0, + }, Case { name: "pcm-s24be.mov", surround: false, @@ -76,7 +129,8 @@ fn run(ff: &Path, args: &[&str]) -> bool { Command::new(ff).args(["-v", "error", "-y"]).args(args).output().is_ok_and(|o| o.status.success()) } -/// The fixture for `c`, written by ffmpeg (cached), or `None` when this ffmpeg lacks an encoder. +/// The fixture for `c`, written by ffmpeg (cached), or `None` when this ffmpeg lacks an encoder +/// (a failure with `CRAFT_MEDIA_REQUIRE_ORACLES=1`). fn fixture(ff: &Path, c: &Case) -> Option { let out = crate::testkit::fixtures_dir("routing").join(c.name); let source = match (c.mono, c.surround) { @@ -85,13 +139,15 @@ fn fixture(ff: &Path, c: &Case) -> Option { _ => STEREO.to_string(), }; let input = format!("{source}:s={}", c.rate); - crate::testkit::fixtures::generate(&out, |tmp| { + let made = crate::testkit::fixtures::generate(&out, |tmp| { let tmp = tmp.to_string_lossy(); let mut args = vec!["-f", "lavfi", "-i", &input, "-t", "5"]; args.extend_from_slice(c.args); args.push(&tmp); run(ff, &args) - }) + }); + assert!(made.is_some() || !craft_media_testkit::oracles_required(), "{}: ffmpeg could not write this fixture", c.name); + made } /// ffmpeg's decode of `file` as interleaved f32. @@ -101,6 +157,23 @@ fn reference(ff: &Path, file: &Path) -> Vec { out.stdout.as_chunks::<4>().0.iter().map(|c| f32::from_le_bytes(*c)).collect() } +/// The major version of this ffmpeg, or `None` for one that does not say (a git build). +fn ffmpeg_major(ff: &Path) -> Option { + let out = Command::new(ff).arg("-version").output().ok()?; + let text = String::from_utf8_lossy(&out.stdout); + let version = text.strip_prefix("ffmpeg version ")?.trim_start_matches('n'); + version.split(|c: char| !c.is_ascii_digit()).next()?.parse().ok() +} + +/// How many frames of priming ffmpeg before 9 decodes as audio at the start of `name`, which +/// `decode` drops; ffmpeg 9 drops them too. +fn reference_lead(name: &str) -> usize { + match name { + "aac.mkv" => 1024, + _ => 0, + } +} + /// SNR (dB) of `ours` (planar) against `reference` (interleaved), sample-aligned, over the /// frames both hold. fn snr(ours: &[Vec], reference: &[f32]) -> f64 { @@ -121,6 +194,7 @@ fn snr(ours: &[Vec], reference: &[f32]) -> f64 { #[test] fn every_container_and_codec_decodes_home_grown() { let ff = require_ffmpeg!(); + let major = ffmpeg_major(&ff); for c in CASES { let Some(path) = fixture(&ff, c) else { eprintln!("SKIPPED {}: this ffmpeg cannot write it", c.name); @@ -133,7 +207,13 @@ fn every_container_and_codec_decodes_home_grown() { assert_eq!(info.layout, c.layout, "{}: layout", c.name); let expected = c.frames.unwrap_or(reference.len() / c.channels); assert_eq!(buf.frames(), expected, "{}: presented frames", c.name); - let s = snr(&buf.channels, &reference); + // One alignment, so audio that keeps the priming, or loses more than it, cannot pass. + let lead = reference_lead(c.name) * c.channels; + let s = match major { + Some(9..) => snr(&buf.channels, &reference), + Some(_) => snr(&buf.channels, &reference[lead..]), + None => snr(&buf.channels, &reference).max(snr(&buf.channels, &reference[lead..])), + }; assert!(s >= c.min_snr, "{}: SNR {s:.1} dB < {}", c.name, c.min_snr); let p = probe(&bytes, None).unwrap(); assert_eq!( @@ -225,7 +305,7 @@ fn track_decoder_on_demuxed_packets_matches_decode() { use craft_audio::TrackDecoder; use craft_media::isobmff; let ff = require_ffmpeg!(); - for name in ["pcm-s24be.mov"] { + for name in ["aac.m4a", "aac-5.1.m4a", "pcm-s24be.mov"] { let c = CASES.iter().find(|c| c.name == name).unwrap(); let Some(path) = fixture(&ff, c) else { continue }; let bytes = std::fs::read(&path).unwrap(); @@ -257,8 +337,8 @@ fn track_decoder_on_demuxed_packets_matches_decode() { fn track_choice_and_the_sample_limit() { use craft_audio::{AudioError, DecodeOptions, decode_with, tracks}; let ff = require_ffmpeg!(); - for ext in ["mkv", "mov"] { - let out = crate::testkit::fixtures_dir("routing").join(format!("two-tracks-pcm.{ext}")); + for ext in ["mkv", "mp4"] { + let out = crate::testkit::fixtures_dir("routing").join(format!("two-tracks.{ext}")); let Some(path) = crate::testkit::fixtures::generate(&out, |tmp| { let tmp = tmp.to_string_lossy(); run( @@ -277,7 +357,9 @@ fn track_choice_and_the_sample_limit() { "-map", "1:a", "-c:a", - "pcm_s16le", + "aac", + "-b:a", + "96k", &tmp, ], ) @@ -287,7 +369,7 @@ fn track_choice_and_the_sample_limit() { let bytes = std::fs::read(&path).unwrap(); let list = tracks(&bytes, None).unwrap(); assert_eq!(list.len(), 2, "{ext}"); - assert_eq!((list[0].codec, list[0].sample_rate, list[0].channels), (CodecId::Pcm, Some(48_000), Some(1)), "{ext}"); + assert_eq!((list[0].codec, list[0].sample_rate, list[0].channels), (CodecId::Aac, Some(48_000), Some(1)), "{ext}"); assert_eq!((list[1].sample_rate, list[1].channels), (Some(44_100), Some(2)), "{ext}"); let (first, _) = decode_with(&bytes, None, &DecodeOptions::default()).unwrap(); let (second, buf) = decode_with(&bytes, None, &DecodeOptions::default().with_track(1)).unwrap(); @@ -306,7 +388,7 @@ fn track_choice_and_the_sample_limit() { fn decoder_reads_from_a_file() { use craft_audio::{DecodeOptions, Decoder}; let ff = require_ffmpeg!(); - let c = CASES.iter().find(|c| c.name == "pcm-s24be.mov").unwrap(); + let c = CASES.iter().find(|c| c.name == "aac.m4a").unwrap(); let Some(path) = fixture(&ff, c) else { return }; let (_, whole) = decode(&std::fs::read(&path).unwrap(), None).unwrap(); let mut d = Decoder::open(std::fs::File::open(&path).unwrap(), None, &DecodeOptions::default()).unwrap(); @@ -318,3 +400,216 @@ fn decoder_reads_from_a_file() { } assert_eq!(frames, whole.frames()); } + +/// `fixture`'s AAC packets in a Matroska file with no block durations, like ffmpeg 6's, +/// and a `DURATION` tag for the track when `tag` is given. +fn aac_mkv_without_durations(bytes: &[u8], tag: Option<&str>) -> Vec { + use craft_media::matroska::{Demuxer, LacingMode, MkvWriter, MuxOptions, TrackKind, TrackSpec}; + let mut demux = Demuxer::from_slice(bytes).unwrap(); + let track = demux.file().tracks[0].clone(); + let mut spec = TrackSpec::new(TrackKind::Audio, track.codec_id.clone()); + spec.codec_private = track.codec_private.clone(); + spec.audio = Some((48_000.0, 2, None)); + spec.codec_delay_ns = track.codec_delay_ns; + let opts = MuxOptions { lacing: LacingMode::None, ..MuxOptions::default() }; + let mut w = MkvWriter::new(std::io::Cursor::new(Vec::new()), vec![spec], opts).unwrap(); + while let Some(p) = demux.next_packet().unwrap() { + w.write_frame(0, p.pts_ns, true, &p.data, None).unwrap(); + } + let mut out = w.finish().unwrap().into_inner(); + let Some(tag) = tag else { return out }; + + fn el(id: &[u8], body: &[u8]) -> Vec { + assert!(body.len() < 127); + [id, &[0x80 | body.len() as u8], body].concat() + } + let uid = Demuxer::from_slice(&out).unwrap().file().tracks[0].uid.to_be_bytes(); + let targets = el(&[0x63, 0xC0], &el(&[0x63, 0xC5], &uid)); + let simple = el(&[0x67, 0xC8], &[el(&[0x45, 0xA3], b"DURATION"), el(&[0x44, 0x87], tag.as_bytes())].concat()); + let tags = el(&[0x12, 0x54, 0xC3, 0x67], &el(&[0x73, 0x73], &[targets, simple].concat())); + out.extend_from_slice(&tags); + // The Segment's 8-byte size grows by the tags. + let at = out.windows(4).position(|w| w == [0x18, 0x53, 0x80, 0x67]).unwrap() + 4; + let size = u64::from_be_bytes(out[at..at + 8].try_into().unwrap()) + tags.len() as u64; + out[at..at + 8].copy_from_slice(&size.to_be_bytes()); + out +} + +/// A track with no block durations ends where its `DURATION` tag says, when that is inside the +/// last packet. Otherwise all the file's audio is presented. +#[test] +fn matroska_aac_without_block_durations_ends_at_its_duration_tag() { + let ff = require_ffmpeg!(); + let case = CASES.iter().find(|c| c.name == "aac.mkv").unwrap(); + let Some(path) = fixture(&ff, case) else { return }; + let bytes = std::fs::read(path).unwrap(); + let frames = |tag| { + let file = aac_mkv_without_durations(&bytes, tag); + let (info, buf) = decode(&file, None).unwrap(); + assert_eq!(probe(&file, None).unwrap().frames, info.frames, "{tag:?}"); + buf.frames() + }; + // 5 s, plus the 21 ms of priming the stored timestamps count + assert_eq!(frames(Some("00:00:05.021000000")), 240_000); + // 236 packets less the priming + let all = 236 * 1024 - 1024; + assert_eq!(frames(None), all); + assert_eq!(frames(Some("00:00:03.000000000")), all, "ends before the last packet"); + assert_eq!(frames(Some("00:00:09.000000000")), all, "ends after the last packet"); + assert_eq!(frames(Some("not a duration")), all); +} + +/// ADTS with channel configuration 0 takes its layout from the program config element in the stream. +#[test] +fn adts_with_a_program_config_element_decodes() { + let ff = require_ffmpeg!(); + let c = Case { + name: "aac-pce.aac", + surround: true, + mono: false, + rate: 48_000, + args: &["-c:a", "aac", "-b:a", "384k", "-aac_pns", "0", "-aac_pce", "1"], + container: Container::Adts, + codec: CodecId::Aac, + layout: ChannelLayout::unordered(6), + channels: 6, + frames: None, + min_snr: 100.0, + }; + let Some(path) = fixture(&ff, &c) else { return }; + let bytes = std::fs::read(&path).unwrap(); + assert_eq!(bytes[3] >> 6 | (bytes[2] & 1) << 2, 0, "the fixture signals its layout in a PCE"); + let (info, buf) = decode(&bytes, None).unwrap(); + assert_eq!((info.container, info.codec, buf.channels.len()), (c.container, c.codec, c.channels)); + assert_eq!(info.layout, c.layout); + assert_eq!(buf.frames(), reference(&ff, &path).len() / c.channels); + // A program config's channels come in its own order, and which order ffmpeg writes differs by + // version: each of ours must match a different one of ffmpeg's. + let refr = reference(&ff, &path); + let mut matched = Vec::new(); + for plane in &buf.channels { + let (best, s) = (0..c.channels) + .map(|r| { + let other: Vec = refr.iter().skip(r).step_by(c.channels).copied().collect(); + (r, snr(std::slice::from_ref(plane), &other)) + }) + .max_by(|a, b| a.1.total_cmp(&b.1)) + .unwrap(); + assert!(s >= c.min_snr, "channel {} matches ffmpeg's {best} at only {s:.1} dB", matched.len()); + matched.push(best); + } + matched.sort_unstable(); + assert_eq!(matched, (0..c.channels).collect::>()); +} + +/// `probe` and `decode` agree on an MP4's length when no single edit gives it: the samples' +/// durations end the track. ffmpeg 9 does the same, but earlier versions present the whole last +/// packet, so the length is not compared with its decode. +#[test] +fn mp4_without_an_edit_list_ends_with_its_sample_durations() { + let ff = require_ffmpeg!(); + let c = Case { + name: "aac-no-edits.m4a", + surround: false, + mono: false, + rate: 44_100, + args: &["-c:a", "aac", "-b:a", "128k", "-aac_pns", "0", "-use_editlist", "0"], + container: Container::Mp4, + codec: CodecId::Aac, + layout: ChannelLayout::STEREO, + channels: 2, + frames: None, + min_snr: 100.0, + }; + let Some(path) = fixture(&ff, &c) else { return }; + let bytes = std::fs::read(&path).unwrap(); + let (info, buf) = decode(&bytes, None).unwrap(); + assert_eq!(probe(&bytes, None).unwrap().frames, buf.frames() as u64); + assert_eq!(info.frames, buf.frames() as u64); + // 5 s, and 1024 frames of priming with no edit list to skip them. + assert_eq!(buf.frames(), 5 * 44_100 + 1024); +} + +/// The big-endian `u32` at each of `offsets` after the first `box_name` in `file`, changed by `edit`. +fn patched(file: &[u8], box_name: &[u8; 4], offsets: &[usize], edit: impl Fn(u32) -> u32) -> Vec { + let mut out = file.to_vec(); + let at = file.windows(4).position(|w| w == box_name).unwrap(); + for &o in offsets { + let p = at + o; + let v: [u8; 4] = out[p..p + 4].try_into().unwrap(); + out[p..p + 4].copy_from_slice(&edit(u32::from_be_bytes(v)).to_be_bytes()); + } + out +} + +/// Wrong sample durations or edits don't cut audio or disagree with `probe`: durations only trim the +/// last packet, and an edit can't outlast the media. +#[test] +fn mp4_with_wrong_durations_or_edits_presents_its_audio() { + let ff = require_ffmpeg!(); + let c = CASES.iter().find(|c| c.name == "aac.m4a").unwrap(); + let Some(path) = fixture(&ff, c) else { return }; + let bytes = std::fs::read(path).unwrap(); + let (_, whole) = decode(&bytes, None).unwrap(); + // From the name's start: the name (4), version and flags (4), entries (4), then stts's first + // run's count (4) and delta, or elst v0's first duration. + let stts_delta = 4 + 4 + 4 + 4; + let elst_duration = 4 + 4 + 4; + for (name, file) in [ + ("durations halved", patched(&bytes, b"stts", &[stts_delta], |v| v / 2)), + ("durations of one", patched(&bytes, b"stts", &[stts_delta], |_| 1)), + ("durations doubled", patched(&bytes, b"stts", &[stts_delta], |v| v * 2)), + ("durations of zero", patched(&bytes, b"stts", &[stts_delta], |_| 0)), + ("last delta longer than a packet", patched(&bytes, b"stts", &[stts_delta + 8], |_| 1500)), + ] { + assert_ne!(file, bytes, "{name}"); + let (info, buf) = decode(&file, None).unwrap(); + assert!(buf.frames().abs_diff(whole.frames()) <= 1024, "{name}: {} frames", buf.frames()); + assert_eq!((probe(&file, None).unwrap().frames, info.frames), (buf.frames() as u64, buf.frames() as u64), "{name}"); + // A seek lands on the same audio: a packet's place comes from its index, not the durations. + let mut d = craft_audio::Decoder::from_bytes(&file, None, &craft_audio::DecodeOptions::default()).unwrap(); + assert_eq!(d.seek(50_000).unwrap(), 50_000, "{name}"); + let block = d.next_block().unwrap().unwrap(); + for (a, b) in block.channels[0].iter().zip(&buf.channels[0][50_000..]) { + assert!((a - b).abs() < 1e-4, "{name}: seek"); + } + } + let file = patched(&bytes, b"elst", &[elst_duration], |v| v * 2); + let (info, buf) = decode(&file, None).unwrap(); + assert!(buf.frames() < whole.frames() + 2048, "edit doubled: {} frames", buf.frames()); + assert_eq!((probe(&file, None).unwrap().frames, info.frames), (buf.frames() as u64, buf.frames() as u64), "edit doubled"); +} + +/// An empty edit before the media edit (QuickTime's and GPAC's delayed audio, ffmpeg's +/// `-itsoffset`) still has the media edit applied: its priming skip and its end, as ffmpeg's +/// decode does. ffmpeg writes such a media edit with no skip, so the test sets one. +#[test] +fn mp4_with_an_empty_edit_before_its_media_edit_applies_the_media_edit() { + let ff = require_ffmpeg!(); + let dir = crate::testkit::fixtures_dir("routing"); + let out = dir.join("aac-itsoffset.m4a"); + let input = format!("{MONO}:s=44100"); + let made = crate::testkit::fixtures::generate(&out, |tmp| { + let tmp = tmp.to_string_lossy(); + run(ff.as_path(), &["-itsoffset", "0.5", "-f", "lavfi", "-i", &input, "-t", "5", "-c:a", "aac", "-aac_pns", "0", &tmp]) + }); + assert!(made.is_some() || !craft_media_testkit::oracles_required(), "ffmpeg could not write aac-itsoffset.m4a"); + let Some(path) = made else { return }; + let bytes = std::fs::read(&path).unwrap(); + // elst v0 from the name's start: the name (4), version and flags (4), entries (4), then 12 + // bytes per edit: duration, media time, rate. + let elst = bytes.windows(4).position(|w| w == b"elst").unwrap(); + assert_eq!((bytes[elst + 4], u32::from_be_bytes(bytes[elst + 8..elst + 12].try_into().unwrap())), (0, 2), "an empty edit and a media edit"); + assert_eq!(i32::from_be_bytes(bytes[elst + 12 + 4..elst + 12 + 8].try_into().unwrap()), -1, "the first edit is empty"); + let media_time = elst + 12 + 12 + 4; + let mut file = bytes.clone(); + file[media_time..media_time + 4].copy_from_slice(&1024i32.to_be_bytes()); + let patched = dir.join("aac-itsoffset-priming.m4a"); + std::fs::write(&patched, &file).unwrap(); + let reference = reference(&ff, &patched); + let (info, buf) = decode(&file, None).unwrap(); + assert_eq!(buf.frames(), reference.len(), "presented frames"); + assert_eq!((probe(&file, None).unwrap().frames, info.frames), (buf.frames() as u64, buf.frames() as u64)); + let s = snr(&buf.channels, &reference); + assert!(s >= 100.0, "SNR {s:.1} dB: the priming skip is not applied"); +}