diff --git a/crates/video/Cargo.toml b/crates/video/Cargo.toml index b8bbd4b1c..b24c50dd3 100644 --- a/crates/video/Cargo.toml +++ b/crates/video/Cargo.toml @@ -23,6 +23,8 @@ apv-decode = ["craft-media/bitstream"] apv-encode = ["craft-media/bitstream"] # VP8 (RFC 6386): decoder. vp8 = [] +# MPEG-2 video (ITU-T H.262) and MPEG-1 video (ISO/IEC 11172-2): decoder. +mpeg2 = [] # Apple ProRes (SMPTE RDD 36): decoder and encoder. prores = ["craft-media/bitstream"] # H.264 / AVC: decoder and encoder; `h264-decode` or `h264-encode` for one of them. @@ -94,6 +96,10 @@ required-features = ["vp9-decode"] name = "vp9_vp9dec" required-features = ["vp9-decode"] +[[test]] +name = "mpeg2" +required-features = ["mpeg2"] + [[test]] name = "prores" required-features = ["prores"] diff --git a/crates/video/README.md b/crates/video/README.md index 71ae64532..8588b462e 100644 --- a/crates/video/README.md +++ b/crates/video/README.md @@ -21,6 +21,7 @@ No codec feature is on by default. | `dnx` | [`dnx`](src/dnx/README.md) | Avid DNxHD / DNxHR (SMPTE ST 2019-1 VC-3): decoder and encoder | | `apv` | [`apv`](src/apv/README.md) | APV (Advanced Professional Video, RFC 9924): decoder (`apv-decode` alone) and encoder (`apv-encode` alone) | | `vp8` | `vp8` | VP8 (RFC 6386): decoder | +| `mpeg2` | [`mpeg2`](src/mpeg2/README.md) | MPEG-2 video (ITU-T H.262) and MPEG-1 video (ISO/IEC 11172-2): decoder | | `prores` | `prores` | Apple ProRes (SMPTE RDD 36): decoder and encoder | | `h264` | [`h264`](src/h264/decode/README.md) | ITU-T H.264 / AVC: decoder (`h264-decode` alone) and encoder (`h264-encode` alone) | | `hevc` | [`hevc`](src/hevc/decode/README.md) | ITU-T H.265 / HEVC: decoder (`hevc-decode` alone) and encoder (`hevc-encode` alone) | diff --git a/crates/video/src/lib.rs b/crates/video/src/lib.rs index cfe8163a3..fc8e407b1 100644 --- a/crates/video/src/lib.rs +++ b/crates/video/src/lib.rs @@ -8,6 +8,7 @@ //! | `dnx` | [`dnx`] | Avid DNxHD / DNxHR (SMPTE ST 2019-1): decoder and encoder | //! | `apv` | [`apv`] | APV (RFC 9924): decoder (`apv-decode`) and encoder (`apv-encode`) | //! | `vp8` | [`vp8`] | VP8 (RFC 6386): decoder | +//! | `mpeg2` | [`mpeg2`] | MPEG-2 video (ITU-T H.262) and MPEG-1 video: decoder | //! | `prores` | [`prores`] | Apple ProRes (SMPTE RDD 36): decoder and encoder | //! | `h264` | [`h264`] | ITU-T H.264 / AVC: decoder (`h264-decode`) and encoder (`h264-encode`) | //! | `hevc` | [`hevc`] | ITU-T H.265 / HEVC: decoder (`hevc-decode`) and encoder (`hevc-encode`) | @@ -33,6 +34,9 @@ pub mod dnx; #[cfg(feature = "vp8")] pub mod vp8; +#[cfg(feature = "mpeg2")] +pub mod mpeg2; + #[cfg(feature = "prores")] pub mod prores; diff --git a/crates/video/src/mpeg2/README.md b/crates/video/src/mpeg2/README.md new file mode 100644 index 000000000..1c49d45cd --- /dev/null +++ b/crates/video/src/mpeg2/README.md @@ -0,0 +1,133 @@ +# MPEG-2 and MPEG-1 video decoder (`craft_video::mpeg2`, feature `mpeg2`) + +Clean-room MPEG-2 video and MPEG-1 video decoder in pure Rust. Depends only on +`thiserror` and (optional, feature `threads`) `rayon`. Builds for +`wasm32-unknown-unknown`; no `unsafe`. No GPL/LGPL code (FFmpeg, libmpeg2, libavcodec) was +consulted; FFmpeg is used only as an external fixture generator and test oracle. + +## Specifications + +| Document | Edition | Used for | +|---|---|---| +| ITU-T H.262 \| ISO/IEC 13818-2 | 02/2000 (second edition, with Amd. 1 and Cor. 1/2 as incorporated in the 2000 text) | sequence / GOP / picture headers and extensions, slices, macroblocks, VLC tables (Annex B), inverse quantisation and mismatch control (§7.4), IDCT (Annex A), motion vectors and prediction (§7.6) | +| ISO/IEC 11172-2 | 1993 | MPEG-1 video: slices spanning rows, MPEG-1 escape codes, oddification, full-pel vectors, D pictures | +| IEEE Std 1180 | 1990 | IDCT accuracy test procedure (run as a unit test) | +| ISO/IEC TR 13818-5 (MPEG Software Simulation Group decoder behaviour) | 1997 | two decoder behaviours the normative text leaves open (see "Interpretations") | + +## API + +```rust +let mut dec = craft_video::mpeg2::Decoder::new(); +for (i, au) in craft_video::mpeg2::access_units(&es).into_iter().enumerate() { + for pic in dec.decode(&es[au], i as i64)? { // display order + // pic.y / pic.cb / pic.cr: tight 8-bit planes; pic.chroma: 4:2:0 / 4:2:2 + // pic.field_order(), pic.picture_type, pic.pts, pic.info (SequenceInfo) + } +} +let rest = dec.flush(); +let info = craft_video::mpeg2::probe(&es); // sequence header + extensions +let au = craft_video::mpeg2::scan_access_unit(&es[..]); // picture types without decoding +``` + +`decode` takes any chunk holding whole pictures (an access unit from a TS/PS/MXF/MOV sample). +Pictures come out in display order (anchors are held until the next anchor) and carry the `pts` +of the chunk their first field came in. `reset` forgets pictures but keeps the sequence +parameters, so a random-access point without its own sequence header still decodes. + +## Decoder + +- Main Profile at Low/Main/High-1440/High Level, 4:2:2 Profile (IMX / D-10, XDCAM HD422), + Simple Profile, MPEG-1 (including D pictures). 4:4:4 is refused (no profile in use for it). +- Pictures above 2^25 luma samples (8192 x 4096; 8K UHD at 7680 x 4320 fits, and the largest MPEG-2 level, + High, is 1920 x 1152) are refused as `Unsupported`: the 14-bit header fields allow 16383 x 16383, which + would ask for gigabytes per frame from hostile input. +- Frame pictures with frame and field motion compensation, dual prime, frame and field DCT; + field pictures with field, 16x8 and dual-prime motion compensation, the second field of a + P frame predicted from the first; open and closed GOPs; B pictures without a forward + reference after a broken link are dropped (closed GOP: decoded from the backward reference). +- Both intra VLC tables, zigzag and alternate scan, linear and non-linear quantiser scale, + quantiser matrices from the sequence header and the quant matrix extension (chroma matrices + for 4:2:2), intra DC precision 8-11 bits, concealment motion vectors, saturation and + mismatch control. +- IDCT: the separable 2-D IDCT in `f32` with even/odd symmetry, the DC term added exactly, + row skipping and a DC-only path; rounded and saturated to [-256, 255]. `ieee_1180_accuracy` + runs the IEEE 1180 procedure (10 000 blocks × 3 ranges × 2 signs): peak error ≤ 1 and all mean + / mean-square limits met. +- Motion compensation clamps reference coordinates to the picture (only corrupt streams point + outside it); a missing reference predicts from mid-gray. +- Slices of different macroblock rows decode in parallel on rayon (each row into its own chunk + of the frame); MPEG-1 slices, which may span rows, decode in order. +- Corrupt slices are concealed (the rest of the picture still decodes) and counted + (`Decoder::errors`, `last_error`); nothing panics on truncated or mutated input. + +### Interpretations + +Two behaviours follow the MPEG Software Simulation Group reference decoder (and FFmpeg), which +the synthetic-stream tests confirmed against FFmpeg: + +- Skipped macroblocks in field pictures predict from the field of the same parity, in B + pictures as well as P pictures. +- Motion vector reconstruction (§7.6.3.1) brings the vector back into range only on the side the + delta moves it, and keeps the prediction for a zero delta. This differs from the literal + formula only when the prediction is out of range: a field vector's doubled vertical + component predicting a following frame vector. + +## Accuracy (vs FFmpeg as an external oracle) + +MPEG-2's IDCT is not bit-exact by definition (any IEEE 1180-accurate IDCT conforms, and the +encoder's reconstruction drifts from every decoder's). FFmpeg uses an integer IDCT, so frames +are compared within a tolerance: **per frame max |Δ| ≤ 4 and PSNR ≥ 58 dB, at most 3 % of +samples differing**. Frame count, display order, picture types, interlacing and field order are +exact (ffprobe). + +`tests/mpeg2/oracle.rs` (FFmpeg-encoded elementary streams): + +| Fixture | Content | Max Δ | Differing | PSNR | +|---|---|---|---|---| +| m2v_cif_ibbp | 352×288 progressive IBBP, Main@Main | 2 | 0.60 % | 70.3 dB | +| m2v_576i_tff | 720×576 interlaced TFF, field MC/DCT, alternate scan | 2 | 0.33 % | 73.0 dB | +| m2v_480i_bff_vlc1 | 720×480 BFF, intra VLC table 1, non-linear q, 10-bit DC | 2 | 0.38 % | 72.3 dB | +| m2v_imx_422_intra | 720×608 4:2:2 intra (IMX / D-10 style) | 1 | 0.23 % | 74.6 dB | +| m2v_hd422_longgop | 1920×1080 4:2:2 interlaced long GOP (4:2:2@High) | 2 | 0.20 % | 75.1 dB | +| m2v_1080i | 1920×1080 4:2:0 interlaced, 25 Mb/s (Main@High) | 3 | 0.25 % | 74.2 dB | +| m2v_escapes_qmatrix | q = 1 (escape codes), custom intra/inter matrices | 2 | 1.80 % | 65.6 dB | +| m2v_ipp | no B pictures | 2 | 1.67 % | 65.9 dB | +| m1v_sif | MPEG-1 352×240 IBBP | 2 | 0.59 % | 70.4 dB | +| m1v_q1 | MPEG-1 q = 1 (8/16-bit escapes) | 2 | 1.93 % | 65.2 dB | + +FFmpeg's encoder never writes field pictures, 16x8 or dual-prime prediction, so +`synth_tests.rs` writes random but valid streams with them (random macroblock types, +vectors, field selects, quantiser scales and coefficients; skipped macroblocks; concealment +vectors; 4:2:0 and 4:2:2; TFF and BFF) and FFmpeg decodes the same streams with `-xerror`: + +| Stream | Max Δ | PSNR | +|---|---|---| +| field pictures I/P/B + 16x8, TFF 4:2:0 / BFF 4:2:2 | 2 | 65.7–66.0 dB | +| field pictures + dual prime, TFF 4:2:0 / BFF 4:2:2 | 2 | 66.5–67.5 dB | +| interlaced frame pictures + field MC + dual prime, TFF / BFF | 2 | 64.8–65.7 dB | +| frame pictures with B, 4:2:2, alternate scan, non-linear q | 1 | 66.8 dB | +| seed sweep: 8 seeds × 4 configurations (160 streams checked during development) | ≤ 2 | ≥ 64.7 dB | + +Residuals are kept realistic (dequantised coefficients within ±300): with extreme residuals +FFmpeg's integer IDCT saturates differently from the exact one. + +## Performance + +`cargo test --release -p craft-video --features mpeg2,threads --test mpeg2 perf:: -- --ignored --nocapture` decodes the +1080i fixture (1920×1080 4:2:0, 25 Mb/s, IBBP, field MC/DCT; FFmpeg-encoded) on an M4 Pro +(14 cores) that was heavily loaded by other jobs (load average ≈ 190), best of five runs: +**271 fps on one thread, 274 fps with slice rows on rayon** (runs under heavier momentary load +fell to 77–150 fps). These are lower bounds. + +## Tests + +- `vlc.rs`: every table builds without prefix clashes; Kraft sums equal 1 minus exactly the + codes the standard reserves; every coded_block_pattern and motion_code value appears once; + motion codes are the address-increment codes plus a sign; 111 run/level pairs per DCT table. +- `idct.rs`: IEEE 1180; DC-only path equals the full transform for every DC value. +- `headers.rs`, `mc.rs`, `bits.rs`: scans, start codes, frame rates, half-sample + rounding, field addressing, edge clamping. +- `tests.rs`: a hand-built intra picture decodes to its DC; truncation at every byte and + 3000 random mutations never panic. +- `synth_tests.rs`, `tests/mpeg2/oracle.rs`: see above (single-threaded decode identical to the + threaded one). diff --git a/crates/video/src/mpeg2/bits.rs b/crates/video/src/mpeg2/bits.rs new file mode 100644 index 000000000..2df38941a --- /dev/null +++ b/crates/video/src/mpeg2/bits.rs @@ -0,0 +1,93 @@ +//! Big-endian bit reader over one start-code unit. Reads past the end return zero bits; callers +//! check [`Bits::overrun`] to detect truncated data. + +#[derive(Clone)] +pub(crate) struct Bits<'a> { + data: &'a [u8], + pos: usize, +} + +impl<'a> Bits<'a> { + pub fn new(data: &'a [u8]) -> Self { + Self { data, pos: 0 } + } + + /// The next 32 bits, MSB first (zeros past the end). + #[inline(always)] + pub fn peek32(&self) -> u32 { + let byte = self.pos >> 3; + let w = if let Some(&b) = self.data.get(byte..).and_then(|d| d.first_chunk::<8>()) { + u64::from_be_bytes(b) + } else { + let mut b = [0u8; 8]; + if byte < self.data.len() { + let n = self.data.len() - byte; + b[..n].copy_from_slice(&self.data[byte..]); + } + u64::from_be_bytes(b) + }; + ((w << (self.pos & 7)) >> 32) as u32 + } + + /// The next `n` (1..=32) bits without consuming them. + #[inline(always)] + pub fn peek(&self, n: u32) -> u32 { + debug_assert!((1..=32).contains(&n)); + ((self.peek32() as u64) >> (32 - n)) as u32 + } + + #[inline(always)] + pub fn skip(&mut self, n: u32) { + self.pos += n as usize; + } + + #[inline(always)] + pub fn read(&mut self, n: u32) -> u32 { + if n == 0 { + return 0; + } + let v = self.peek(n); + self.pos += n as usize; + v + } + + #[inline(always)] + pub fn bit(&mut self) -> bool { + self.read(1) == 1 + } + + /// A `n`-bit two's complement value. + #[inline(always)] + pub fn read_signed(&mut self, n: u32) -> i32 { + let v = self.read(n) as i32; + (v << (32 - n)) >> (32 - n) + } + + pub fn bits_left(&self) -> isize { + (self.data.len() * 8) as isize - self.pos as isize + } + + /// More than a few bytes have been read past the end: the data is truncated or corrupt. + pub fn overrun(&self) -> bool { + self.bits_left() < 0 + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn reads_across_bytes_and_past_the_end() { + let d = [0b1010_1100, 0xFF, 0x01]; + let mut b = Bits::new(&d); + assert_eq!(b.read(3), 0b101); + assert_eq!(b.read(7), 0b011_0011); + assert_eq!(b.peek(4), 0xF); + assert_eq!(b.read_signed(4), -1); + assert_eq!(b.read(10), 0b11_0000_0001); + assert!(!b.overrun()); + assert_eq!(b.read(8), 0); + assert!(b.overrun()); + } +} diff --git a/crates/video/src/mpeg2/headers.rs b/crates/video/src/mpeg2/headers.rs new file mode 100644 index 000000000..6b9438107 --- /dev/null +++ b/crates/video/src/mpeg2/headers.rs @@ -0,0 +1,466 @@ +//! Start codes and headers (H.262 §6.2 / §6.3; ISO/IEC 11172-2 §2.4.2). + +use crate::mpeg2::bits::Bits; +use crate::mpeg2::{Error, Result}; + +pub const PICTURE_START: u8 = 0x00; +pub const SLICE_MIN: u8 = 0x01; +pub const SLICE_MAX: u8 = 0xAF; +pub const USER_DATA: u8 = 0xB2; +pub const SEQUENCE_HEADER: u8 = 0xB3; +pub const SEQUENCE_ERROR: u8 = 0xB4; +pub const EXTENSION: u8 = 0xB5; +pub const SEQUENCE_END: u8 = 0xB7; +pub const GROUP_START: u8 = 0xB8; + +/// Start codes in `data`: (offset of the `00 00 01` prefix, start code value). A unit's payload +/// runs from `offset + 4` to the next start code. +pub fn start_codes(data: &[u8]) -> Vec<(usize, u8)> { + let mut out = Vec::new(); + let mut i = 0; + let n = data.len(); + while i + 3 < n { + // skip quickly to a zero byte at i + 2 + if data[i + 2] > 1 { + i += 3; + continue; + } + if data[i] == 0 && data[i + 1] == 0 && data[i + 2] == 1 { + out.push((i, data[i + 3])); + i += 4; + } else { + i += 1; + } + } + out +} + +/// Default intra quantiser matrix (H.262 §6.3.11), raster order. +pub const DEFAULT_INTRA: [u8; 64] = [ + 8, 16, 19, 22, 26, 27, 29, 34, // + 16, 16, 22, 24, 27, 29, 34, 37, // + 19, 22, 26, 27, 29, 34, 34, 38, // + 22, 22, 26, 27, 29, 34, 37, 40, // + 22, 26, 27, 29, 32, 35, 40, 48, // + 26, 27, 29, 32, 35, 40, 48, 58, // + 26, 27, 29, 34, 38, 46, 56, 69, // + 27, 29, 35, 38, 46, 56, 69, 83, +]; + +/// Scan index → raster position: zigzag (scan 0) and alternate (scan 1), H.262 Figure 7-2/7-3. +pub const SCAN: [[u8; 64]; 2] = [ + [ + 0, 1, 8, 16, 9, 2, 3, 10, 17, 24, 32, 25, 18, 11, 4, 5, 12, 19, 26, 33, 40, 48, 41, 34, 27, 20, 13, 6, 7, 14, 21, 28, 35, 42, 49, 56, 57, 50, + 43, 36, 29, 22, 15, 23, 30, 37, 44, 51, 58, 59, 52, 45, 38, 31, 39, 46, 53, 60, 61, 54, 47, 55, 62, 63, + ], + [ + 0, 8, 16, 24, 1, 9, 2, 10, 17, 25, 32, 40, 48, 56, 57, 49, 41, 33, 26, 18, 3, 11, 4, 12, 19, 27, 34, 42, 50, 58, 35, 43, 51, 59, 20, 28, 5, + 13, 6, 14, 21, 29, 36, 44, 52, 60, 37, 45, 53, 61, 22, 30, 7, 15, 23, 31, 38, 46, 54, 62, 39, 47, 55, 63, + ], +]; + +/// A quantiser matrix as transmitted (zigzag order) → raster order. +fn read_matrix(b: &mut Bits) -> [u8; 64] { + let mut m = [0u8; 64]; + for i in 0..64 { + m[SCAN[0][i] as usize] = b.read(8) as u8; + } + m +} + +/// sequence_header() (§6.2.2.1). +#[derive(Clone, Debug, PartialEq)] +pub struct SequenceHeader { + pub horizontal_size: u32, + pub vertical_size: u32, + pub aspect_ratio_information: u8, + pub frame_rate_code: u8, + pub bit_rate_value: u32, + pub vbv_buffer_size_value: u32, + pub constrained_parameters: bool, + /// Raster order; `None`: default. + pub intra_matrix: Option<[u8; 64]>, + pub non_intra_matrix: Option<[u8; 64]>, +} + +impl SequenceHeader { + pub fn parse(payload: &[u8]) -> Result { + if payload.len() < 8 { + return Err(Error::Invalid("short sequence header".into())); + } + let mut b = Bits::new(payload); + let horizontal_size = b.read(12); + let vertical_size = b.read(12); + let aspect_ratio_information = b.read(4) as u8; + let frame_rate_code = b.read(4) as u8; + let bit_rate_value = b.read(18); + b.skip(1); + let vbv_buffer_size_value = b.read(10); + let constrained_parameters = b.bit(); + let intra_matrix = b.bit().then(|| read_matrix(&mut b)); + let non_intra_matrix = b.bit().then(|| read_matrix(&mut b)); + if b.overrun() { + return Err(Error::Invalid("truncated sequence header".into())); + } + if horizontal_size == 0 || vertical_size == 0 { + return Err(Error::Invalid("zero picture size".into())); + } + Ok(Self { + horizontal_size, + vertical_size, + aspect_ratio_information, + frame_rate_code, + bit_rate_value, + vbv_buffer_size_value, + constrained_parameters, + intra_matrix, + non_intra_matrix, + }) + } +} + +/// sequence_extension() (§6.2.2.3). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct SequenceExtension { + pub profile_and_level_indication: u8, + pub progressive_sequence: bool, + /// 1 = 4:2:0, 2 = 4:2:2, 3 = 4:4:4. + pub chroma_format: u8, + pub horizontal_size_extension: u8, + pub vertical_size_extension: u8, + pub bit_rate_extension: u16, + pub vbv_buffer_size_extension: u8, + pub low_delay: bool, + pub frame_rate_extension_n: u8, + pub frame_rate_extension_d: u8, +} + +/// sequence_display_extension() (§6.2.2.4). +#[derive(Clone, Copy, Debug, PartialEq, Eq, Default)] +pub struct DisplayExtension { + pub video_format: u8, + /// (colour_primaries, transfer_characteristics, matrix_coefficients), H.262 Tables 6-7..6-9. + pub colour: Option<(u8, u8, u8)>, + pub display_horizontal_size: u16, + pub display_vertical_size: u16, +} + +/// quant_matrix_extension() (§6.2.3.2), raster order. +#[derive(Clone, Copy, Debug, Default)] +pub struct QuantMatrixExtension { + pub intra: Option<[u8; 64]>, + pub non_intra: Option<[u8; 64]>, + pub chroma_intra: Option<[u8; 64]>, + pub chroma_non_intra: Option<[u8; 64]>, +} + +/// picture_coding_extension() (§6.2.3.1). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct PictureCodingExtension { + /// `f_code[s][t]`: s 0 forward / 1 backward, t 0 horizontal / 1 vertical. + pub f_code: [[u8; 2]; 2], + pub intra_dc_precision: u8, + /// 1 top field, 2 bottom field, 3 frame picture. + pub picture_structure: u8, + pub top_field_first: bool, + pub frame_pred_frame_dct: bool, + pub concealment_motion_vectors: bool, + pub q_scale_type: bool, + pub intra_vlc_format: bool, + pub alternate_scan: bool, + pub repeat_first_field: bool, + pub chroma_420_type: bool, + pub progressive_frame: bool, +} + +impl PictureCodingExtension { + /// The values an MPEG-1 picture behaves as (progressive frame picture, zigzag, linear q). + pub fn mpeg1(ph: &PictureHeader) -> Self { + Self { + f_code: [[ph.forward_f_code; 2], [ph.backward_f_code; 2]], + intra_dc_precision: 0, + picture_structure: 3, + top_field_first: false, + frame_pred_frame_dct: true, + concealment_motion_vectors: false, + q_scale_type: false, + intra_vlc_format: false, + alternate_scan: false, + repeat_first_field: false, + chroma_420_type: true, + progressive_frame: true, + } + } +} + +/// An extension unit (§6.2.2.2 / §6.2.3). +#[derive(Clone, Debug)] +pub enum Extension { + Sequence(SequenceExtension), + Display(DisplayExtension), + QuantMatrix(Box), + PictureCoding(PictureCodingExtension), + /// Scalability, copyright, picture display… (not needed to decode the base layer). + Other(u8), +} + +impl Extension { + pub fn parse(payload: &[u8]) -> Result { + let mut b = Bits::new(payload); + let id = b.read(4) as u8; + let e = match id { + 1 => { + let profile_and_level_indication = b.read(8) as u8; + let progressive_sequence = b.bit(); + let chroma_format = b.read(2) as u8; + let horizontal_size_extension = b.read(2) as u8; + let vertical_size_extension = b.read(2) as u8; + let bit_rate_extension = b.read(12) as u16; + b.skip(1); + let vbv_buffer_size_extension = b.read(8) as u8; + let low_delay = b.bit(); + let frame_rate_extension_n = b.read(2) as u8; + let frame_rate_extension_d = b.read(5) as u8; + Extension::Sequence(SequenceExtension { + profile_and_level_indication, + progressive_sequence, + chroma_format, + horizontal_size_extension, + vertical_size_extension, + bit_rate_extension, + vbv_buffer_size_extension, + low_delay, + frame_rate_extension_n, + frame_rate_extension_d, + }) + } + 2 => { + let video_format = b.read(3) as u8; + let colour = b.bit().then(|| (b.read(8) as u8, b.read(8) as u8, b.read(8) as u8)); + let display_horizontal_size = b.read(14) as u16; + b.skip(1); + let display_vertical_size = b.read(14) as u16; + Extension::Display(DisplayExtension { video_format, colour, display_horizontal_size, display_vertical_size }) + } + 3 => { + let intra = b.bit().then(|| read_matrix(&mut b)); + let non_intra = b.bit().then(|| read_matrix(&mut b)); + let chroma_intra = b.bit().then(|| read_matrix(&mut b)); + let chroma_non_intra = b.bit().then(|| read_matrix(&mut b)); + Extension::QuantMatrix(Box::new(QuantMatrixExtension { intra, non_intra, chroma_intra, chroma_non_intra })) + } + 8 => { + let f_code = [[b.read(4) as u8, b.read(4) as u8], [b.read(4) as u8, b.read(4) as u8]]; + let intra_dc_precision = b.read(2) as u8; + let picture_structure = b.read(2) as u8; + let top_field_first = b.bit(); + let frame_pred_frame_dct = b.bit(); + let concealment_motion_vectors = b.bit(); + let q_scale_type = b.bit(); + let intra_vlc_format = b.bit(); + let alternate_scan = b.bit(); + let repeat_first_field = b.bit(); + let chroma_420_type = b.bit(); + let progressive_frame = b.bit(); + if picture_structure == 0 { + return Err(Error::Invalid("reserved picture_structure".into())); + } + Extension::PictureCoding(PictureCodingExtension { + f_code, + intra_dc_precision, + picture_structure, + top_field_first, + frame_pred_frame_dct, + concealment_motion_vectors, + q_scale_type, + intra_vlc_format, + alternate_scan, + repeat_first_field, + chroma_420_type, + progressive_frame, + }) + } + other => Extension::Other(other), + }; + if b.overrun() { + return Err(Error::Invalid(format!("truncated extension {id}"))); + } + Ok(e) + } +} + +/// group_of_pictures_header() (§6.2.2.6). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct GopHeader { + pub drop_frame: bool, + pub hours: u8, + pub minutes: u8, + pub seconds: u8, + pub pictures: u8, + pub closed_gop: bool, + pub broken_link: bool, +} + +impl GopHeader { + pub fn parse(payload: &[u8]) -> Result { + if payload.len() < 4 { + return Err(Error::Invalid("short GOP header".into())); + } + let mut b = Bits::new(payload); + let drop_frame = b.bit(); + let hours = b.read(5) as u8; + let minutes = b.read(6) as u8; + b.skip(1); + let seconds = b.read(6) as u8; + let pictures = b.read(6) as u8; + let closed_gop = b.bit(); + let broken_link = b.bit(); + Ok(Self { drop_frame, hours, minutes, seconds, pictures, closed_gop, broken_link }) + } +} + +/// picture_coding_type. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum PictureType { + I, + P, + B, + /// ISO/IEC 11172-2 DC-coded picture. + D, +} + +impl PictureType { + pub fn is_anchor(self) -> bool { + matches!(self, PictureType::I | PictureType::P | PictureType::D) + } +} + +/// picture_header() (§6.2.3). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct PictureHeader { + pub temporal_reference: u16, + pub coding_type: PictureType, + pub vbv_delay: u16, + pub full_pel_forward_vector: bool, + pub forward_f_code: u8, + pub full_pel_backward_vector: bool, + pub backward_f_code: u8, +} + +impl PictureHeader { + pub fn parse(payload: &[u8]) -> Result { + if payload.len() < 4 { + return Err(Error::Invalid("short picture header".into())); + } + let mut b = Bits::new(payload); + let temporal_reference = b.read(10) as u16; + let coding_type = match b.read(3) { + 1 => PictureType::I, + 2 => PictureType::P, + 3 => PictureType::B, + 4 => PictureType::D, + t => return Err(Error::Invalid(format!("picture_coding_type {t}"))), + }; + let vbv_delay = b.read(16) as u16; + let (mut full_pel_forward_vector, mut forward_f_code, mut full_pel_backward_vector, mut backward_f_code) = (false, 7, false, 7); + if matches!(coding_type, PictureType::P | PictureType::B) { + full_pel_forward_vector = b.bit(); + forward_f_code = b.read(3) as u8; + } + if coding_type == PictureType::B { + full_pel_backward_vector = b.bit(); + backward_f_code = b.read(3) as u8; + } + Ok(Self { temporal_reference, coding_type, vbv_delay, full_pel_forward_vector, forward_f_code, full_pel_backward_vector, backward_f_code }) + } +} + +/// Frame rate from frame_rate_code (Table 6-4) and the sequence extension's n/d (MPEG-2). +pub fn frame_rate(code: u8, ext_n: u8, ext_d: u8) -> Option<(u32, u32)> { + let (n, d) = match code { + 1 => (24000, 1001), + 2 => (24, 1), + 3 => (25, 1), + 4 => (30000, 1001), + 5 => (30, 1), + 6 => (50, 1), + 7 => (60000, 1001), + 8 => (60, 1), + _ => return None, + }; + let (n, d) = (n * (ext_n as u32 + 1), d * (ext_d as u32 + 1)); + let g = gcd(n, d); + Some((n / g, d / g)) +} + +pub(crate) fn gcd(a: u32, b: u32) -> u32 { + if b == 0 { a.max(1) } else { gcd(b, a % b) } +} + +/// Profile and level names (Tables 8-2, 8-3, 8-7). +pub fn profile_level_name(pli: u8) -> String { + if pli & 0x80 != 0 { + return match pli { + 0x82 => "4:2:2@High".into(), + 0x85 => "4:2:2@Main".into(), + 0x8A => "Multi-view@High".into(), + 0x8B => "Multi-view@High-1440".into(), + 0x8D => "Multi-view@Main".into(), + 0x8E => "Multi-view@Low".into(), + other => format!("profile/level 0x{other:02x}"), + }; + } + let profile = match (pli >> 4) & 7 { + 1 => "High", + 2 => "Spatially Scalable", + 3 => "SNR Scalable", + 4 => "Main", + 5 => "Simple", + _ => "Reserved", + }; + let level = match pli & 15 { + 4 => "High", + 6 => "High-1440", + 8 => "Main", + 10 => "Low", + _ => "Reserved", + }; + format!("{profile}@{level}") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn scans_are_permutations_and_inverse_of_the_figures() { + for s in SCAN { + let mut seen = [false; 64]; + for &p in &s { + assert!(!seen[p as usize]); + seen[p as usize] = true; + } + } + // Figure 7-3 row 0 of the alternate scan: 0 4 6 20 22 36 38 52 + let inv: Vec = (0..8).map(|u| SCAN[1].iter().position(|&p| p as usize == u).unwrap()).collect(); + assert_eq!(inv, vec![0, 4, 6, 20, 22, 36, 38, 52]); + // Figure 7-2 row 0 of the zigzag scan: 0 1 5 6 14 15 27 28 + let inv: Vec = (0..8).map(|u| SCAN[0].iter().position(|&p| p as usize == u).unwrap()).collect(); + assert_eq!(inv, vec![0, 1, 5, 6, 14, 15, 27, 28]); + } + + #[test] + fn start_codes_found_at_every_alignment() { + let d = [0, 0, 1, 0xB3, 9, 0, 0, 0, 1, 0x00, 0, 0, 1, 0x01, 0, 0]; + assert_eq!(start_codes(&d), vec![(0, 0xB3), (6, 0x00), (10, 0x01)]); + } + + #[test] + fn frame_rates() { + assert_eq!(frame_rate(4, 0, 0), Some((30000, 1001))); + assert_eq!(frame_rate(3, 1, 0), Some((50, 1))); + assert_eq!(frame_rate(0, 0, 0), None); + assert_eq!(profile_level_name(0x44), "Main@High"); + assert_eq!(profile_level_name(0x85), "4:2:2@Main"); + } +} diff --git a/crates/video/src/mpeg2/idct.rs b/crates/video/src/mpeg2/idct.rs new file mode 100644 index 000000000..f94b935f0 --- /dev/null +++ b/crates/video/src/mpeg2/idct.rs @@ -0,0 +1,179 @@ +//! 8×8 inverse DCT (H.262 Annex A): the separable 2-D IDCT evaluated in `f32` with even/odd +//! symmetry, rounded to the nearest integer and saturated to [-256, 255]. Its error is far inside +//! the IEEE Std 1180-1990 accuracy limits the standard requires (checked by the test procedure of +//! IEEE 1180 itself, below). + +use std::sync::OnceLock; + +/// `M[x][u] = C(u)/2 · cos((2x+1)uπ/16)`, `C(0) = 1/√2`, `C(u>0) = 1`. +fn basis() -> &'static [[f32; 8]; 8] { + static B: OnceLock<[[f32; 8]; 8]> = OnceLock::new(); + B.get_or_init(|| { + let mut m = [[0f32; 8]; 8]; + for (x, row) in m.iter_mut().enumerate() { + for (u, v) in row.iter_mut().enumerate() { + let c = if u == 0 { std::f64::consts::FRAC_1_SQRT_2 } else { 1.0 }; + *v = (c / 2.0 * (((2 * x + 1) * u) as f64 * std::f64::consts::PI / 16.0).cos()) as f32; + } + } + m + }) +} + +/// In-place IDCT of a block of dequantised coefficients (raster order). `dc_only`: only `blk[0]` +/// may be non-zero. +#[inline] +pub(crate) fn idct(blk: &mut [i32; 64], dc_only: bool) { + if dc_only { + let v = ((blk[0] + 4) >> 3).clamp(-256, 255); + blk.fill(v); + return; + } + let m = basis(); + // The DC term F[0][0]/8 is added exactly (so ties round as in the DC-only path). + let dc = blk[0] as f32 * 0.125; + blk[0] = 0; + // rows: tmp[v][x] = Σ_u M[x][u] F[v][u] + let mut tmp = [[0f32; 8]; 8]; + for v in 0..8 { + let r = &blk[v * 8..v * 8 + 8]; + if r.iter().all(|&c| c == 0) { + continue; + } + let f: [f32; 8] = std::array::from_fn(|u| r[u] as f32); + for x in 0..4 { + let mx = &m[x]; + let e = mx[0] * f[0] + mx[2] * f[2] + mx[4] * f[4] + mx[6] * f[6]; + let o = mx[1] * f[1] + mx[3] * f[3] + mx[5] * f[5] + mx[7] * f[7]; + tmp[v][x] = e + o; + tmp[v][7 - x] = e - o; + } + } + // columns, eight at a time: out[y][x] = Σ_v M[y][v] tmp[v][x] + for y in 0..4 { + let my = &m[y]; + let mut e = [0f32; 8]; + let mut o = [0f32; 8]; + for x in 0..8 { + e[x] = my[0] * tmp[0][x] + my[2] * tmp[2][x] + my[4] * tmp[4][x] + my[6] * tmp[6][x]; + o[x] = my[1] * tmp[1][x] + my[3] * tmp[3][x] + my[5] * tmp[5][x] + my[7] * tmp[7][x]; + } + for x in 0..8 { + blk[y * 8 + x] = ((dc + (e[x] + o[x]) + 0.5).floor() as i32).clamp(-256, 255); + blk[(7 - y) * 8 + x] = ((dc + (e[x] - o[x]) + 0.5).floor() as i32).clamp(-256, 255); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The pseudo-random generator of IEEE Std 1180-1990 (values in [-l, h]). + struct Rand(i64); + impl Rand { + fn next(&mut self, l: i64, h: i64) -> i64 { + self.0 = (self.0.wrapping_mul(1_103_515_245).wrapping_add(12_345)) & 0xFFFF_FFFF; + let i = self.0 & 0x7FFF_FFFE; + let x = i as f64 / 0x7FFF_FFFF as f64 * (l + h + 1) as f64; + x as i64 - l + } + } + + fn c(u: usize) -> f64 { + if u == 0 { std::f64::consts::FRAC_1_SQRT_2 } else { 1.0 } + } + + fn cosv(a: usize, b: usize) -> f64 { + static T: OnceLock<[[f64; 8]; 8]> = OnceLock::new(); + T.get_or_init(|| std::array::from_fn(|a| std::array::from_fn(|b| (((2 * a + 1) * b) as f64 * std::f64::consts::PI / 16.0).cos())))[a][b] + } + + /// Separable double-precision 1-D passes (exact up to f64 rounding). + fn fdct(p: &[i64; 64]) -> [i64; 64] { + let mut t = [0f64; 64]; + for y in 0..8 { + for u in 0..8 { + t[y * 8 + u] = (0..8).map(|x| p[y * 8 + x] as f64 * cosv(x, u)).sum::() * c(u) / 2.0; + } + } + let mut out = [0i64; 64]; + for v in 0..8 { + for u in 0..8 { + let s = (0..8).map(|y| t[y * 8 + u] * cosv(y, v)).sum::() * c(v) / 2.0; + out[v * 8 + u] = (s.round() as i64).clamp(-2048, 2047); + } + } + out + } + + fn reference_idct(f: &[i64; 64]) -> [i64; 64] { + let mut t = [0f64; 64]; + for v in 0..8 { + for x in 0..8 { + t[v * 8 + x] = (0..8).map(|u| c(u) * f[v * 8 + u] as f64 * cosv(x, u)).sum::() / 2.0; + } + } + let mut out = [0i64; 64]; + for y in 0..8 { + for x in 0..8 { + let s = (0..8).map(|v| c(v) * t[v * 8 + x] * cosv(y, v)).sum::() / 2.0; + out[y * 8 + x] = (s.round() as i64).clamp(-256, 255); + } + } + out + } + + /// IEEE 1180-1990 §3: 10 000 blocks for each range and sign; peak error ≤ 1, per-pixel mean + /// square error ≤ 0.06, overall ≤ 0.02, per-pixel mean error ≤ 0.015, overall ≤ 0.0015. + #[test] + fn ieee_1180_accuracy() { + let blocks = if cfg!(debug_assertions) { 2_000 } else { 10_000 }; + for (l, h) in [(256i64, 255i64), (5, 5), (300, 300)] { + for sign in [1i64, -1] { + let mut rng = Rand(1); + let mut sum_err = [0i64; 64]; + let mut sum_sq = [0i64; 64]; + for _ in 0..blocks { + let mut p = [0i64; 64]; + for v in p.iter_mut() { + *v = rng.next(l, h) * sign; + } + let f = fdct(&p); + let r = reference_idct(&f); + let mut blk: [i32; 64] = std::array::from_fn(|i| f[i] as i32); + idct(&mut blk, false); + for i in 0..64 { + let e = blk[i] as i64 - r[i]; + assert!(e.abs() <= 1, "peak error {e} (range {l}/{h} sign {sign})"); + sum_err[i] += e; + sum_sq[i] += e * e; + } + } + let n = blocks as f64; + for i in 0..64 { + assert!(sum_sq[i] as f64 / n <= 0.06); + assert!((sum_err[i] as f64 / n).abs() <= 0.015); + } + assert!(sum_sq.iter().sum::() as f64 / (64.0 * n) <= 0.02); + assert!((sum_err.iter().sum::() as f64 / (64.0 * n)).abs() <= 0.0015); + } + } + // all-zero input gives all-zero output + let mut z = [0i32; 64]; + idct(&mut z, false); + assert!(z.iter().all(|&v| v == 0)); + } + + #[test] + fn dc_only_matches_the_full_transform() { + for dc in -2048..=2047 { + let mut a = [0i32; 64]; + a[0] = dc; + let mut b = a; + idct(&mut a, true); + idct(&mut b, false); + assert_eq!(a, b, "dc {dc}"); + } + } +} diff --git a/crates/video/src/mpeg2/mc.rs b/crates/video/src/mpeg2/mc.rs new file mode 100644 index 000000000..2d39508e2 --- /dev/null +++ b/crates/video/src/mpeg2/mc.rs @@ -0,0 +1,165 @@ +//! Frame buffers and motion-compensated prediction (H.262 §7.6). + +/// A decoded frame, padded to whole macroblocks. +#[derive(Clone)] +pub(crate) struct Frame { + pub planes: [Vec; 3], + /// Luma width / height (multiples of 16). + pub w: usize, + pub h: usize, + /// Chroma width / height. + pub cw: usize, + pub ch: usize, +} + +impl Frame { + pub fn new(w: usize, h: usize, cw: usize, ch: usize) -> Frame { + Frame { planes: [vec![16; w * h], vec![128; cw * ch], vec![128; cw * ch]], w, h, cw, ch } + } + + /// Plane `c` (0 Y, 1 Cb, 2 Cr) as a reference: the whole frame, or one field (0 top, 1 bottom). + #[inline] + pub fn plane(&self, c: usize, field: Option) -> RefPlane<'_> { + let (w, h) = if c == 0 { (self.w, self.h) } else { (self.cw, self.ch) }; + match field { + None => RefPlane { data: &self.planes[c], off: 0, stride: w, w, h }, + Some(p) => RefPlane { data: &self.planes[c], off: p * w, stride: 2 * w, w, h: h / 2 }, + } + } +} + +/// A plane (or a field of one) to predict from. +#[derive(Clone, Copy)] +pub(crate) struct RefPlane<'a> { + pub data: &'a [u8], + pub off: usize, + pub stride: usize, + pub w: usize, + pub h: usize, +} + +/// A block to predict: integer position, half-sample flags and size. +#[derive(Clone, Copy)] +pub(crate) struct Block { + pub(crate) x: i32, + pub(crate) y: i32, + pub(crate) hx: bool, + pub(crate) hy: bool, + pub(crate) bw: usize, + pub(crate) bh: usize, +} + +/// Form the `bw`×`bh` prediction at integer position (`x`, `y`) with half-sample flags into +/// `dst` (row `r` at `dst[doff + r·dstride..]`), or average it with what is there (`avg`). +/// Positions outside the reference are clamped to its edges (not allowed in conforming streams; +/// this keeps corrupt ones safe). +#[inline] +pub(crate) fn predict(dst: &mut [u8], doff: usize, dstride: usize, src: RefPlane, blk: Block, avg: bool) { + let Block { x, y, hx, hy, bw, bh } = blk; + let (ex, ey) = (hx as i32, hy as i32); + let inside = x >= 0 && y >= 0 && (x + bw as i32 + ex) as usize <= src.w && (y + bh as i32 + ey) as usize <= src.h; + let mut tmp = [0u8; 16]; + if inside { + let s = src.stride; + let base = src.off + y as usize * s + x as usize; + for r in 0..bh { + let p = base + r * s; + let a = &src.data[p..p + bw + 1 - (!hx) as usize]; + let row = &mut tmp[..bw]; + match (hx, hy) { + (false, false) => row.copy_from_slice(&a[..bw]), + (true, false) => { + for i in 0..bw { + row[i] = ((a[i] as u16 + a[i + 1] as u16 + 1) >> 1) as u8; + } + } + (false, true) => { + let b = &src.data[p + s..p + s + bw]; + for i in 0..bw { + row[i] = ((a[i] as u16 + b[i] as u16 + 1) >> 1) as u8; + } + } + (true, true) => { + let b = &src.data[p + s..p + s + bw + 1]; + for i in 0..bw { + row[i] = ((a[i] as u16 + a[i + 1] as u16 + b[i] as u16 + b[i + 1] as u16 + 2) >> 2) as u8; + } + } + } + store(dst, doff + r * dstride, row, avg); + } + return; + } + let get = |xx: i32, yy: i32| -> u16 { + let xx = xx.clamp(0, src.w as i32 - 1) as usize; + let yy = yy.clamp(0, src.h as i32 - 1) as usize; + src.data[src.off + yy * src.stride + xx] as u16 + }; + for r in 0..bh { + let yy = y + r as i32; + let row = &mut tmp[..bw]; + for (i, v) in row.iter_mut().enumerate() { + let xx = x + i as i32; + *v = match (hx, hy) { + (false, false) => get(xx, yy), + (true, false) => (get(xx, yy) + get(xx + 1, yy) + 1) >> 1, + (false, true) => (get(xx, yy) + get(xx, yy + 1) + 1) >> 1, + (true, true) => (get(xx, yy) + get(xx + 1, yy) + get(xx, yy + 1) + get(xx + 1, yy + 1) + 2) >> 2, + } as u8; + } + store(dst, doff + r * dstride, row, avg); + } +} + +#[inline(always)] +fn store(dst: &mut [u8], at: usize, row: &[u8], avg: bool) { + let d = &mut dst[at..at + row.len()]; + if avg { + for (o, &p) in d.iter_mut().zip(row) { + *o = ((*o as u16 + p as u16 + 1) >> 1) as u8; + } + } else { + d.copy_from_slice(row); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn frame() -> Frame { + let mut f = Frame::new(32, 32, 16, 16); + for (i, v) in f.planes[0].iter_mut().enumerate() { + *v = (i % 32 * 3 + i / 32 * 5) as u8; + } + f + } + + #[test] + fn half_sample_interpolation_rounds_up() { + let f = frame(); + let p = f.plane(0, None); + let mut d = [0u8; 4]; + predict(&mut d, 0, 2, p, Block { x: 1, y: 1, hx: true, hy: true, bw: 2, bh: 2 }, false); + let s = |x: usize, y: usize| f.planes[0][y * 32 + x] as u16; + assert_eq!(d[0] as u16, (s(1, 1) + s(2, 1) + s(1, 2) + s(2, 2) + 2) >> 2); + predict(&mut d, 0, 2, p, Block { x: 1, y: 1, hx: true, hy: false, bw: 2, bh: 2 }, false); + assert_eq!(d[3] as u16, (s(2, 2) + s(3, 2) + 1) >> 1); + } + + #[test] + fn fields_and_clamped_edges() { + let f = frame(); + let mut d = [0u8; 4]; + // bottom field row 0 = frame row 1 + predict(&mut d, 0, 2, f.plane(0, Some(1)), Block { x: 0, y: 0, hx: false, hy: false, bw: 2, bh: 2 }, false); + assert_eq!(d, [f.planes[0][32], f.planes[0][33], f.planes[0][96], f.planes[0][97]]); + // outside: clamped to the corner + predict(&mut d, 0, 2, f.plane(0, None), Block { x: -10, y: -10, hx: false, hy: false, bw: 2, bh: 2 }, false); + assert_eq!(d, [f.planes[0][0]; 4]); + // averaging + let mut d = [10u8; 4]; + predict(&mut d, 0, 2, f.plane(0, None), Block { x: -10, y: -10, hx: false, hy: false, bw: 2, bh: 2 }, true); + assert_eq!(d[0], ((10 + f.planes[0][0] as u16 + 1) >> 1) as u8); + } +} diff --git a/crates/video/src/mpeg2/mod.rs b/crates/video/src/mpeg2/mod.rs new file mode 100644 index 000000000..5d88ea4e5 --- /dev/null +++ b/crates/video/src/mpeg2/mod.rs @@ -0,0 +1,920 @@ +//! Clean-room MPEG-2 video (ITU-T H.262 | ISO/IEC 13818-2) and MPEG-1 video (ISO/IEC 11172-2) +//! decoder. +//! +//! - Main Profile at Low/Main/High-1440/High Level, 4:2:2 Profile (XDCAM HD422, IMX / D-10), +//! Simple Profile, and MPEG-1 (including D pictures and constrained-parameters streams). +//! - Progressive and interlaced coding: frame pictures (frame / field / dual-prime motion +//! compensation, frame and field DCT) and field pictures (field / 16x8 / dual-prime motion +//! compensation, the second field predicted from the first), alternate scan, both intra VLC +//! tables, linear and non-linear quantiser scale, quantiser matrices (sequence header and quant +//! matrix extension), intra DC precision 8-11 bits, concealment motion vectors. +//! - Output in display order as planar 8-bit frames (two fields woven into one frame), with +//! field order, repeat-first-field, picture type and colour description. +//! +//! ```no_run +//! let mut dec = craft_video::mpeg2::Decoder::new(); +//! # let access_units: Vec<(Vec, i64)> = vec![]; +//! for (au, pts) in access_units { +//! for pic in dec.decode(&au, pts)? { +//! println!("{}x{} {:?} pts {}", pic.width, pic.height, pic.picture_type, pic.pts); +//! } +//! } +//! let rest = dec.flush(); +//! # Ok::<(), craft_video::mpeg2::Error>(()) +//! ``` + +#![allow(clippy::needless_range_loop, clippy::type_complexity)] + +mod bits; +pub mod headers; +mod idct; +mod mc; +mod slice; +mod vlc; + +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; + +pub use headers::{ + DisplayExtension, Extension, GopHeader, PictureCodingExtension, PictureHeader, PictureType, QuantMatrixExtension, SequenceExtension, + SequenceHeader, profile_level_name, start_codes, +}; + +use headers::{EXTENSION, GROUP_START, PICTURE_START, SEQUENCE_END, SEQUENCE_HEADER, SLICE_MAX, SLICE_MIN}; +use mc::Frame; +use slice::{PicParams, Target, decode_slice, slice_row}; + +/// Pictures a decoder holds at once, for the memory budget: two anchors, the frame being +/// decoded, the copy of its first field, the gray stand-in, and the output of one access unit. +const BUDGET_PICTURES: u64 = 5 + MAX_OUTPUT_PICTURES_PER_ACCESS_UNIT as u64; + +/// One access unit must not retain an unbounded number of cropped decoded pictures. +const MAX_OUTPUT_PICTURES_PER_ACCESS_UNIT: usize = 8; + +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("invalid MPEG video: {0}")] + Invalid(String), + #[error("unsupported MPEG video: {0}")] + Unsupported(String), +} + +pub type Result = std::result::Result; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ChromaFormat { + Yuv420, + Yuv422, + Yuv444, +} + +/// Stream-level properties (sequence header and its extensions). +#[derive(Clone, Debug, PartialEq)] +pub struct SequenceInfo { + /// ISO/IEC 13818-2 (has a sequence extension) or ISO/IEC 11172-2. + pub mpeg2: bool, + pub width: u32, + pub height: u32, + pub chroma: ChromaFormat, + pub profile_and_level: Option, + pub progressive_sequence: bool, + pub low_delay: bool, + /// Frames per second (num, den). + pub frame_rate: Option<(u32, u32)>, + /// Sample aspect ratio. + pub sar: (u32, u32), + /// Display aspect ratio when signalled (MPEG-2 aspect_ratio_information 2-4). + pub dar: Option<(u32, u32)>, + /// (colour_primaries, transfer_characteristics, matrix_coefficients) from the sequence + /// display extension. + pub colour: Option<(u8, u8, u8)>, + pub video_format: Option, + /// bits/s (0: unknown / variable). + pub bit_rate: u64, +} + +impl SequenceInfo { + /// "MPEG-2 Video (Main@High)", "MPEG-1 Video". + pub fn codec_name(&self) -> String { + match (self.mpeg2, self.profile_and_level) { + (false, _) => "MPEG-1 Video".into(), + (true, Some(p)) => format!("MPEG-2 Video ({})", profile_level_name(p)), + (true, None) => "MPEG-2 Video".into(), + } + } +} + +/// A decoded frame (both fields of field-coded pictures), cropped to the coded size. +#[derive(Clone, Debug)] +pub struct Picture { + pub width: u32, + pub height: u32, + pub chroma: ChromaFormat, + pub y: Vec, + pub cb: Vec, + pub cr: Vec, + /// The timestamp passed with the access unit the picture (its first field) came in. + pub pts: i64, + pub picture_type: PictureType, + pub temporal_reference: u16, + pub progressive_frame: bool, + pub top_field_first: bool, + pub repeat_first_field: bool, + /// Coded as two field pictures. + pub field_pictures: bool, + pub info: Arc, +} + +impl Picture { + pub fn chroma_width(&self) -> u32 { + match self.chroma { + ChromaFormat::Yuv444 => self.width, + _ => self.width.div_ceil(2), + } + } + pub fn chroma_height(&self) -> u32 { + match self.chroma { + ChromaFormat::Yuv420 => self.height.div_ceil(2), + _ => self.height, + } + } + /// Interlaced content and which field comes first: `None` for progressive frames. + pub fn field_order(&self) -> Option { + if self.progressive_frame { + None + } else if self.top_field_first { + Some(FieldOrder::TopFirst) + } else { + Some(FieldOrder::BottomFirst) + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum FieldOrder { + TopFirst, + BottomFirst, +} + +/// The active sequence. +#[derive(Clone)] +struct Seq { + hdr: SequenceHeader, + ext: Option, + display: Option, + info: Arc, + mb_width: usize, + mb_height: usize, +} + +/// MPEG-1 pel_aspect_ratio (height/width of a sample) ×10000, Table 2-D.2. +const MPEG1_PEL_ASPECT: [u32; 15] = [0, 10000, 6735, 7031, 7615, 8055, 8437, 8935, 9157, 9815, 10255, 10695, 10950, 11575, 12015]; + +/// Largest accepted luma area (8192 x 4096; 8K UHD is 7680 x 4320). +const MAX_PICTURE_AREA: u64 = 1 << 25; + +impl Seq { + fn new(hdr: SequenceHeader, ext: Option, display: Option) -> Result { + let (w, h) = match &ext { + Some(e) => { + (hdr.horizontal_size | (e.horizontal_size_extension as u32) << 12, hdr.vertical_size | (e.vertical_size_extension as u32) << 12) + } + None => (hdr.horizontal_size, hdr.vertical_size), + }; + let chroma = match ext.map_or(1, |e| e.chroma_format) { + 1 => ChromaFormat::Yuv420, + 2 => ChromaFormat::Yuv422, + 3 => ChromaFormat::Yuv444, + _ => return Err(Error::Invalid("reserved chroma_format".into())), + }; + // The 14-bit sizes allow 16383 x 16383 (gigabytes per frame, several frames and a gray reference + // held); no real stream exceeds 8K, so cap the area there. + if w > 16383 || h > 16383 || u64::from(w) * u64::from(h) > MAX_PICTURE_AREA { + return Err(Error::Unsupported(format!("{w}x{h}"))); + } + let progressive_sequence = ext.is_none_or(|e| e.progressive_sequence); + let mb_width = (w as usize).div_ceil(16); + let mb_height = if progressive_sequence { (h as usize).div_ceil(16) } else { 2 * (h as usize).div_ceil(32) }; + let frame_rate = + headers::frame_rate(hdr.frame_rate_code, ext.map_or(0, |e| e.frame_rate_extension_n), ext.map_or(0, |e| e.frame_rate_extension_d)); + let (sar, dar) = if ext.is_some() { + let (dw, dh) = display + .filter(|d| d.display_horizontal_size > 0 && d.display_vertical_size > 0) + .map_or((w, h), |d| (d.display_horizontal_size as u32, d.display_vertical_size as u32)); + let dar = match hdr.aspect_ratio_information { + 2 => Some((4, 3)), + 3 => Some((16, 9)), + 4 => Some((221, 100)), + _ => None, + }; + let sar = match dar { + Some((n, d)) => { + // the display rectangle (dw × dh samples) has this aspect: SAR = DAR · dh / dw + let (n, d) = (n as u64 * dh as u64, d as u64 * dw as u64); + let g = gcd64(n, d); + ((n / g) as u32, (d / g) as u32) + } + None => (1, 1), + }; + (reduce_sar(sar), dar) + } else { + let v = MPEG1_PEL_ASPECT.get(hdr.aspect_ratio_information as usize).copied().filter(|&v| v > 0).unwrap_or(10000); + let g = headers::gcd(10000, v); + ((10000 / g, v / g), None) + }; + let bit_rate = (hdr.bit_rate_value as u64 | (ext.map_or(0, |e| e.bit_rate_extension as u64) << 18)) * 400; + let info = SequenceInfo { + mpeg2: ext.is_some(), + width: w, + height: h, + chroma, + profile_and_level: ext.map(|e| e.profile_and_level_indication), + progressive_sequence, + low_delay: ext.is_some_and(|e| e.low_delay), + frame_rate, + sar, + dar, + colour: display.and_then(|d| d.colour), + video_format: display.map(|d| d.video_format), + bit_rate: if hdr.bit_rate_value == 0x3FFFF && ext.is_none() { 0 } else { bit_rate }, + }; + Ok(Seq { hdr, ext, display, info: Arc::new(info), mb_width, mb_height }) + } + + /// Whether the pictures a decoder holds for this sequence fit in `budget` bytes. + fn fits(&self, budget: u64) -> bool { + let (shift_x, shift_y) = self.shifts(); + let shape = crate::budget::PictureShape { + width: self.mb_width as u64 * 16, + height: self.mb_height as u64 * 16, + shift_x, + shift_y, + chroma_planes: 2, + bytes_per_sample: 1, + }; + shape.fits(BUDGET_PICTURES, budget) + } + + fn shifts(&self) -> (u32, u32) { + match self.info.chroma { + ChromaFormat::Yuv420 => (1, 1), + ChromaFormat::Yuv422 => (1, 0), + ChromaFormat::Yuv444 => (0, 0), + } + } +} + +fn gcd64(a: u64, b: u64) -> u64 { + if b == 0 { a.max(1) } else { gcd64(b, a % b) } +} + +fn reduce_sar((n, d): (u32, u32)) -> (u32, u32) { + if n == 0 || d == 0 { + return (1, 1); + } + let g = headers::gcd(n, d); + (n / g, d / g) +} + +/// Parse the first sequence header (and its extensions) in `data`. +pub fn probe(data: &[u8]) -> Option { + let codes = start_codes(data); + let mut seq: Option<(SequenceHeader, Option, Option)> = None; + for (k, &(pos, code)) in codes.iter().enumerate() { + let end = codes.get(k + 1).map_or(data.len(), |c| c.0); + let payload = &data[(pos + 4).min(end)..end]; + match code { + SEQUENCE_HEADER if seq.is_none() => seq = Some((SequenceHeader::parse(payload).ok()?, None, None)), + EXTENSION => { + if let Some(s) = seq.as_mut() { + match Extension::parse(payload) { + Ok(Extension::Sequence(e)) => s.1 = Some(e), + Ok(Extension::Display(d)) => s.2 = Some(d), + _ => {} + } + } + } + _ if seq.is_some() => break, + _ => {} + } + } + let (h, e, d) = seq?; + Seq::new(h, e, d).ok().map(|s| (*s.info).clone()) +} + +/// What an access unit holds, without decoding it. +#[derive(Clone, Debug, Default)] +pub struct AccessUnitInfo { + pub sequence_header: bool, + pub gop: Option, + pub pictures: Vec<(PictureHeader, Option)>, +} + +impl AccessUnitInfo { + /// Starts with an I picture (decoding can begin here when a sequence header has been seen). + pub fn is_intra(&self) -> bool { + self.pictures.first().is_some_and(|p| matches!(p.0.coding_type, PictureType::I | PictureType::D)) + } + /// Only B pictures (never used for prediction). + pub fn is_disposable(&self) -> bool { + !self.pictures.is_empty() && self.pictures.iter().all(|p| p.0.coding_type == PictureType::B) + } +} + +pub fn scan_access_unit(data: &[u8]) -> AccessUnitInfo { + let codes = start_codes(data); + let mut info = AccessUnitInfo::default(); + for (k, &(pos, code)) in codes.iter().enumerate() { + let end = codes.get(k + 1).map_or(data.len(), |c| c.0); + let payload = &data[(pos + 4).min(end)..end]; + match code { + SEQUENCE_HEADER => info.sequence_header = true, + GROUP_START => info.gop = GopHeader::parse(payload).ok(), + PICTURE_START => { + if let Ok(ph) = PictureHeader::parse(payload) { + info.pictures.push((ph, None)); + } + } + EXTENSION => { + if let (Ok(Extension::PictureCoding(p)), Some(last)) = (Extension::parse(payload), info.pictures.last_mut()) { + last.1 = Some(p); + } + } + _ => {} + } + } + info +} + +/// Split an elementary stream into access units: one coded frame each (a frame picture or a +/// pair of field pictures) with the sequence / GOP headers before it. Bytes before the first +/// header are skipped. +pub fn access_units(es: &[u8]) -> Vec> { + let codes = start_codes(es); + let mut out = Vec::new(); + let mut cur: Option = None; + let mut header: Option = None; + let mut open_field: Option = None; + for (k, &(pos, code)) in codes.iter().enumerate() { + match code { + SEQUENCE_HEADER | GROUP_START => { + header.get_or_insert(pos); + } + PICTURE_START => { + // picture_structure from the picture coding extension that follows + let structure = codes + .iter() + .enumerate() + .skip(k + 1) + .take_while(|(_, c)| c.1 == EXTENSION || c.1 == headers::USER_DATA) + .find_map(|(next, &(p, c))| { + // `next` is this unit's position in `codes`, so its successor gives the + // payload end directly. Searching `codes` from the beginning here made a + // stream of short picture/extension pairs quadratic to index. + let end = codes.get(next + 1).map_or(es.len(), |x| x.0); + (c == EXTENSION).then(|| Extension::parse(&es[(p + 4).min(end)..end]).ok()).flatten() + }) + .and_then(|e| match e { + Extension::PictureCoding(p) => Some(p.picture_structure), + _ => None, + }) + .unwrap_or(3); + let start = header.take().unwrap_or(pos); + let second = matches!(open_field, Some(s) if structure != 3 && structure != s); + if second { + open_field = None; + } else { + if let Some(c) = cur { + out.push(c..start); + } + cur = Some(start); + open_field = (structure != 3).then_some(structure); + } + } + _ => {} + } + } + if let Some(c) = cur { + out.push(c..es.len()); + } + out +} + +/// A picture whose header has been read and whose slices are being collected. +struct Pending { + ph: PictureHeader, + pce: Option, + /// (start code, payload range). + slices: Vec<(u8, std::ops::Range)>, +} + +#[derive(Clone)] +struct Meta { + pts: i64, + ptype: PictureType, + temporal_reference: u16, + pce: PictureCodingExtension, + field_pictures: bool, + info: Arc, +} + +/// A frame whose first field has been decoded. +struct Building { + frame: Frame, + parity: usize, + meta: Meta, +} + +struct Anchor { + frame: Arc, + meta: Meta, + shown: bool, +} + +/// A stateful decoder. Feed access units (or any chunks holding whole pictures) in decode order; +/// pictures come out in display order. +pub struct Decoder { + seq: Option, + /// A sequence header is waiting for its extensions (applied at the next non-extension unit). + seq_pending: Option<(SequenceHeader, Option, Option)>, + matrices: [[u8; 64]; 4], + closed_gop: bool, + /// Older and newer anchor (reference) frames. + fwd: Option, + bwd: Option, + building: Option, + /// Mid-gray stand-in for missing references. + gray: Option>, + pending: Option, + out: Vec, + output_overflow: bool, + /// After a refused access unit the references are gone: only an I picture restarts decoding. + need_intra: bool, + errors: usize, + last_error: Option<&'static str>, + threads: bool, + /// Bytes the picture store may take ([`crate::budget`]). + memory_budget: u64, +} + +impl Default for Decoder { + fn default() -> Self { + Self::new() + } +} + +impl Decoder { + pub fn new() -> Decoder { + Decoder { + seq: None, + seq_pending: None, + matrices: [headers::DEFAULT_INTRA, [16; 64], headers::DEFAULT_INTRA, [16; 64]], + closed_gop: false, + fwd: None, + bwd: None, + building: None, + gray: None, + pending: None, + out: Vec::new(), + output_overflow: false, + need_intra: false, + errors: 0, + last_error: None, + threads: cfg!(feature = "threads"), + memory_budget: crate::budget::DECODE_MEMORY_BUDGET, + } + } + + /// Decode slices on the rayon pool (default with the `threads` feature). + pub fn set_threads(&mut self, on: bool) { + self.threads = on && cfg!(feature = "threads"); + } + + /// Bytes the decoded pictures may take (default [`crate::budget::DECODE_MEMORY_BUDGET`]). A + /// sequence whose pictures (two anchors, the frame being decoded and its first-field copy, + /// the gray stand-in and the output of one access unit) do not fit is refused: it counts as + /// an error and the sequence does not become active. + pub fn set_memory_budget(&mut self, bytes: u64) { + self.memory_budget = bytes; + } + + /// Slices that failed to decode so far (corrupt data; the picture is still output). + pub fn errors(&self) -> usize { + self.errors + } + + /// What the most recent slice error was. + pub fn last_error(&self) -> Option<&'static str> { + self.last_error + } + + /// The active sequence. + pub fn sequence(&self) -> Option<&SequenceInfo> { + self.seq.as_ref().map(|s| &*s.info) + } + + /// Decode a chunk of the elementary stream holding whole pictures (an access unit) tagged + /// with `pts`. Returns the pictures that became displayable, in display order. + pub fn decode(&mut self, data: &[u8], pts: i64) -> Result> { + self.output_overflow = false; + let codes = start_codes(data); + let mut err = None; + for (k, &(pos, code)) in codes.iter().enumerate() { + let end = codes.get(k + 1).map_or(data.len(), |c| c.0); + let range = (pos + 4).min(end)..end; + let payload = &data[range.clone()]; + if code != EXTENSION { + self.apply_sequence(); + } + match code { + SEQUENCE_HEADER => { + self.finish_picture(data, pts); + match SequenceHeader::parse(payload) { + Ok(h) => self.seq_pending = Some((h, None, None)), + Err(e) => err = Some(e), + } + } + EXTENSION => match Extension::parse(payload) { + Ok(Extension::Sequence(e)) => { + if let Some(s) = self.seq_pending.as_mut() { + s.1 = Some(e); + } else if let Some(s) = &self.seq { + self.seq_pending = Some((s.hdr.clone(), Some(e), s.display)); + } + } + Ok(Extension::Display(d)) => { + if let Some(s) = self.seq_pending.as_mut() { + s.2 = Some(d); + } + } + Ok(Extension::QuantMatrix(q)) => { + if let Some(m) = q.intra { + self.matrices[0] = m; + self.matrices[2] = m; + } + if let Some(m) = q.non_intra { + self.matrices[1] = m; + self.matrices[3] = m; + } + if let Some(m) = q.chroma_intra { + self.matrices[2] = m; + } + if let Some(m) = q.chroma_non_intra { + self.matrices[3] = m; + } + } + Ok(Extension::PictureCoding(p)) => { + if let Some(pd) = self.pending.as_mut() { + pd.pce = Some(p); + } + } + Ok(Extension::Other(_)) => {} + Err(e) => err = Some(e), + }, + GROUP_START => { + self.finish_picture(data, pts); + if let Ok(g) = GopHeader::parse(payload) { + self.closed_gop = g.closed_gop || g.broken_link; + } + } + PICTURE_START => { + self.finish_picture(data, pts); + match PictureHeader::parse(payload) { + Ok(ph) => self.pending = Some(Pending { ph, pce: None, slices: Vec::new() }), + Err(e) => err = Some(e), + } + } + SLICE_MIN..=SLICE_MAX => { + if let Some(pd) = self.pending.as_mut() { + pd.slices.push((code, range)); + } + } + SEQUENCE_END => self.finish_picture(data, pts), + _ => {} + } + } + self.apply_sequence(); + self.finish_picture(data, pts); + if self.output_overflow { + // the references of a refused access unit are stale: drop them and wait for an I picture + self.reset(); + self.need_intra = true; + return Err(Error::Invalid("access unit produces too many decoded pictures".into())); + } + let out = std::mem::take(&mut self.out); + match err { + Some(e) if out.is_empty() && self.seq.is_none() => Err(e), + _ => Ok(out), + } + } + + /// Output the frames held back for reordering (end of stream). + pub fn flush(&mut self) -> Vec { + if let Some(b) = self.building.take() { + self.complete(b.frame, b.meta); + } + if let Some(a) = self.bwd.as_mut() + && !a.shown + { + a.shown = true; + let p = to_picture(&a.frame, &a.meta); + self.push_output(p); + } + std::mem::take(&mut self.out) + } + + /// Forget all pictures (before decoding from a new random-access point). The sequence + /// parameters are kept: a random-access point without a sequence header still decodes. + pub fn reset(&mut self) { + self.fwd = None; + self.bwd = None; + self.building = None; + self.pending = None; + self.out.clear(); + self.output_overflow = false; + self.need_intra = false; + self.closed_gop = false; + } + + fn apply_sequence(&mut self) { + let Some((h, e, d)) = self.seq_pending.take() else { return }; + // a sequence header resets the quantiser matrices (§6.3.11) + let intra = h.intra_matrix.unwrap_or(headers::DEFAULT_INTRA); + let non_intra = h.non_intra_matrix.unwrap_or([16; 64]); + self.matrices = [intra, non_intra, intra, non_intra]; + match Seq::new(h, e, d) { + Ok(s) if !s.fits(self.memory_budget) => { + self.errors += 1; + self.last_error = Some("sequence exceeds the decoder memory budget"); + } + Ok(s) => { + let changed = + self.seq.as_ref().is_none_or(|o| o.mb_width != s.mb_width || o.mb_height != s.mb_height || o.info.chroma != s.info.chroma); + if changed { + // the reset must not hide an overflow already seen in this access unit + let (overflow, need_intra) = (self.output_overflow, self.need_intra); + self.reset(); + self.output_overflow = overflow; + self.need_intra = need_intra; + } + self.seq = Some(s); + } + Err(_) => self.errors += 1, + } + } + + fn finish_picture(&mut self, data: &[u8], pts: i64) { + let Some(pd) = self.pending.take() else { return }; + if self.output_overflow { + return; // the access unit is already refused: do not allocate or decode more pictures for it + } + if pd.slices.is_empty() { + self.errors += 1; + self.last_error = Some("picture has no slices"); + return; + } + let Some(seq) = self.seq.clone() else { + self.errors += 1; + return; + }; + let mpeg2 = seq.ext.is_some(); + let pce = match (mpeg2, pd.pce) { + (true, Some(p)) => p, + (true, None) => { + self.errors += 1; + return; + } + (false, _) => PictureCodingExtension::mpeg1(&pd.ph), + }; + if seq.info.chroma == ChromaFormat::Yuv444 { + self.errors += 1; + return; + } + let ptype = pd.ph.coding_type; + if self.need_intra { + // an MPEG-1 D picture is intra coded, so it restarts decoding like an I picture + if ptype != PictureType::I && (mpeg2 || ptype != PictureType::D) { + self.errors += 1; + return; // its references went with the refused access unit + } + self.need_intra = false; + } + let (shx, shy) = seq.shifts(); + let (w, h) = (seq.mb_width * 16, seq.mb_height * 16); + let (cw, ch) = (w >> shx, h >> shy); + let structure = pce.picture_structure; + let parity = if structure == 2 { 1 } else { 0 }; + // field pairing + let mut second = None; + if structure == 3 { + if let Some(b) = self.building.take() { + self.complete(b.frame, b.meta); + } + } else if let Some(b) = self.building.take() { + if b.parity != parity && b.frame.w == w && b.frame.h == h { + second = Some(b); + } else { + self.complete(b.frame, b.meta); + } + } + if self.output_overflow { + return; // pairing an unpaired field output the last allowed picture: refuse before allocating and decoding this one + } + // B pictures need their forward reference unless the GOP is closed + if ptype == PictureType::B && second.is_none() && (self.bwd.is_none() || (self.fwd.is_none() && !self.closed_gop)) { + return; + } + let is_second = second.is_some(); + let (mut frame, meta) = match second { + Some(b) => (b.frame, b.meta), + None => ( + Frame::new(w, h, cw, ch), + Meta { pts, ptype, temporal_reference: pd.ph.temporal_reference, pce, field_pictures: structure != 3, info: seq.info.clone() }, + ), + }; + // a P second field may predict from the first field of its own frame + let first_field = (is_second && ptype == PictureType::P).then(|| frame.clone()); + // A missing reference (decoding started at a P picture, or a broken link) predicts from + // mid-gray. + if self.gray.as_ref().is_none_or(|g| g.w != w || g.h != h || g.cw != cw) { + let mut g = Frame::new(w, h, cw, ch); + g.planes[0].fill(128); + self.gray = Some(Arc::new(g)); + } + let gray = self.gray.as_deref(); + let latest = self.bwd.as_ref().map(|a| &*a.frame); + let older = self.fwd.as_ref().map(|a| &*a.frame); + let (fwd, bwd) = match ptype { + PictureType::P => (latest.or(gray), None), + PictureType::B => (older.or(latest).or(gray), latest.or(gray)), + _ => (None, None), + }; + let p = PicParams { + mpeg2, + ptype, + pce, + full_pel: [pd.ph.full_pel_forward_vector, pd.ph.full_pel_backward_vector], + shx, + shy, + mb_width: seq.mb_width, + mb_rows: if structure == 3 { seq.mb_height } else { seq.mb_height / 2 }, + matrices: self.matrices, + fwd, + bwd, + first_field: first_field.as_ref(), + parity, + vertical_size_extension: seq.info.height > 2800, + }; + let errors = SliceErrors::default(); + decode_picture(&p, data, &pd.slices, &mut frame, structure, self.threads, &errors); + self.errors += errors.count.load(Ordering::Relaxed); + if let Some(e) = errors.first.into_inner().unwrap_or(None) { + self.last_error = Some(e); + } + if structure != 3 && !is_second { + self.building = Some(Building { frame, parity, meta }); + } else { + self.complete(frame, meta); + } + } + + fn complete(&mut self, frame: Frame, meta: Meta) { + if meta.ptype.is_anchor() { + if let Some(mut prev) = self.bwd.take() { + if !prev.shown { + prev.shown = true; + self.push_output(to_picture(&prev.frame, &prev.meta)); + } + self.fwd = Some(prev); + } + self.bwd = Some(Anchor { frame: Arc::new(frame), meta, shown: false }); + } else { + self.push_output(to_picture(&frame, &meta)); + } + } + + fn push_output(&mut self, picture: Picture) { + if self.out.len() >= MAX_OUTPUT_PICTURES_PER_ACCESS_UNIT { + self.output_overflow = true; + } else { + self.out.push(picture); + } + } +} + +/// Slice errors of one picture (slices may decode on several threads). +#[derive(Default)] +struct SliceErrors { + count: AtomicUsize, + first: std::sync::Mutex>, +} + +impl SliceErrors { + fn add(&self, what: &'static str) { + self.count.fetch_add(1, Ordering::Relaxed); + let mut f = self.first.lock().unwrap_or_else(|e| e.into_inner()); + f.get_or_insert(what); + } +} + +/// Decode the slices of one picture into `frame`. +fn decode_picture( + p: &PicParams, + data: &[u8], + slices: &[(u8, std::ops::Range)], + frame: &mut Frame, + structure: u8, + threads: bool, + errors: &SliceErrors, +) { + let (w, cw) = (frame.w, frame.cw); + let field = (structure != 3).then_some(p.parity); + let lines = if structure == 3 { 16 } else { 32 }; + let clines = lines >> p.shy; + let [y, cb, cr] = &mut frame.planes; + if !p.mpeg2 { + // MPEG-1 slices may span rows: decode in order into the whole picture + let mut t = Target { planes: [&mut y[..], &mut cb[..], &mut cr[..]], strides: [w, cw], line0: [0, 0], field }; + for (code, r) in slices { + let payload = &data[r.clone()]; + if let Err(e) = decode_slice(p, payload, *code as usize - 1, false, &mut t) { + errors.add(e.0); + } + } + return; + } + let mut rows: Vec> = vec![Vec::new(); p.mb_rows]; + for (code, r) in slices { + let payload = &data[r.clone()]; + let row = slice_row(p.vertical_size_extension, *code, payload); + match rows.get_mut(row) { + Some(v) => v.push(payload), + None => { + errors.add("slice row out of range"); + } + } + } + let run = |r: usize, y: &mut [u8], cb: &mut [u8], cr: &mut [u8]| { + let Some(list) = rows.get(r) else { return }; + let mut t = Target { planes: [y, cb, cr], strides: [w, cw], line0: [r * lines, r * clines], field }; + for payload in list { + if let Err(e) = decode_slice(p, payload, r, true, &mut t) { + errors.add(e.0); + } + } + }; + #[cfg(feature = "threads")] + if threads && p.mb_rows >= 4 { + use rayon::prelude::*; + y.par_chunks_mut(lines * w) + .zip(cb.par_chunks_mut(clines * cw)) + .zip(cr.par_chunks_mut(clines * cw)) + .enumerate() + .for_each(|(r, ((y, cb), cr))| run(r, y, cb, cr)); + return; + } + let _ = threads; + for (r, ((y, cb), cr)) in y.chunks_mut(lines * w).zip(cb.chunks_mut(clines * cw)).zip(cr.chunks_mut(clines * cw)).enumerate() { + run(r, y, cb, cr); + } +} + +fn to_picture(f: &Frame, meta: &Meta) -> Picture { + let info = &meta.info; + let (w, h) = (info.width as usize, info.height as usize); + let (cw, ch) = match info.chroma { + ChromaFormat::Yuv420 => (w.div_ceil(2), h.div_ceil(2)), + ChromaFormat::Yuv422 => (w.div_ceil(2), h), + ChromaFormat::Yuv444 => (w, h), + }; + let crop = |src: &[u8], stride: usize, w: usize, h: usize| { + let mut v = Vec::with_capacity(w * h); + for r in 0..h { + v.extend_from_slice(&src[r * stride..r * stride + w]); + } + v + }; + let progressive_frame = meta.pce.progressive_frame; + Picture { + width: w as u32, + height: h as u32, + chroma: info.chroma, + y: crop(&f.planes[0], f.w, w, h), + cb: crop(&f.planes[1], f.cw, cw, ch), + cr: crop(&f.planes[2], f.cw, cw, ch), + pts: meta.pts, + picture_type: meta.ptype, + temporal_reference: meta.temporal_reference, + progressive_frame, + // field pictures: the first coded field is displayed first + top_field_first: if meta.field_pictures { meta.pce.picture_structure == 1 } else { meta.pce.top_field_first }, + repeat_first_field: meta.pce.repeat_first_field, + field_pictures: meta.field_pictures, + info: meta.info.clone(), + } +} + +#[cfg(test)] +mod synth_tests; +#[cfg(test)] +mod tests; diff --git a/crates/video/src/mpeg2/slice.rs b/crates/video/src/mpeg2/slice.rs new file mode 100644 index 000000000..f7a6389f0 --- /dev/null +++ b/crates/video/src/mpeg2/slice.rs @@ -0,0 +1,672 @@ +//! Slice, macroblock and block decoding (H.262 §6.2.4-§6.2.6, §7.2-§7.6; ISO/IEC 11172-2 +//! §2.4.2.7-§2.4.4). + +use crate::mpeg2::bits::Bits; +use crate::mpeg2::headers::{PictureCodingExtension, PictureType, SCAN}; +use crate::mpeg2::idct::idct; +use crate::mpeg2::mc::{Block, Frame, predict}; +use crate::mpeg2::vlc::{DCT_EOB, DCT_ESCAPE, MB_BWD, MB_FWD, MB_INTRA, MB_PATTERN, MB_QUANT, MBA_ESCAPE, MBA_STUFFING, tables}; + +/// Non-linear quantiser_scale (Table 7-6), indexed by quantiser_scale_code. +pub(crate) const NON_LINEAR_Q: [u8; 32] = + [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 14, 16, 18, 20, 22, 24, 28, 32, 36, 40, 44, 48, 52, 56, 64, 72, 80, 88, 96, 104, 112]; + +/// Motion types (frame_motion_type / field_motion_type values). +const MT_FIELD: u8 = 1; +const MT_FRAME: u8 = 2; // frame pictures; 16x8 MC in field pictures +const MT_16X8: u8 = 2; +const MT_DUAL: u8 = 3; + +/// Everything slices of one picture share (read-only while slices decode in parallel). +pub(crate) struct PicParams<'a> { + pub mpeg2: bool, + pub ptype: PictureType, + pub pce: PictureCodingExtension, + pub full_pel: [bool; 2], + /// Chroma subsampling shifts (4:2:0: 1, 1; 4:2:2: 1, 0). + pub shx: u32, + pub shy: u32, + pub mb_width: usize, + /// Macroblock rows of this picture (field rows for field pictures). + pub mb_rows: usize, + /// Intra luma, non-intra luma, intra chroma, non-intra chroma (raster order). + pub matrices: [[u8; 64]; 4], + /// Forward reference (P: the latest anchor; B: the older anchor) and backward reference. + pub fwd: Option<&'a Frame>, + pub bwd: Option<&'a Frame>, + /// The first field of this frame (when decoding its second field). + pub first_field: Option<&'a Frame>, + /// Field pictures: parity of this field (0 top, 1 bottom). + pub parity: usize, + pub vertical_size_extension: bool, +} + +impl PicParams<'_> { + fn frame_picture(&self) -> bool { + self.pce.picture_structure == 3 + } + fn chroma_blocks(&self) -> usize { + if self.shy == 1 { 2 } else { 4 } + } + /// Chroma macroblock size. + fn cmb(&self) -> (usize, usize) { + (16 >> self.shx, 16 >> self.shy) + } +} + +/// Where a row (or, for MPEG-1, the whole picture) of macroblocks is written. +pub(crate) struct Target<'a> { + pub planes: [&'a mut [u8]; 3], + pub strides: [usize; 2], + /// First luma / chroma frame line held by `planes`. + pub line0: [usize; 2], + /// Field pictures: parity written. + pub field: Option, +} + +impl Target<'_> { + fn put(&mut self, mbx: usize, mby: usize, mb: &MbBuf, cmb: (usize, usize)) { + for (c, plane) in self.planes.iter_mut().enumerate() { + let k = (c > 0) as usize; + let (bw, bh) = if c == 0 { (16, 16) } else { cmb }; + let stride = self.strides[k]; + let src = if c == 0 { &mb.y[..] } else { &mb.c[c - 1][..] }; + for l in 0..bh { + let line = match self.field { + None => bh * mby + l, + Some(p) => 2 * (bh * mby + l) + p, + }; + let Some(local) = line.checked_sub(self.line0[k]) else { continue }; + let at = local * stride + bw * mbx; + if at + bw <= plane.len() { + plane[at..at + bw].copy_from_slice(&src[l * bw..l * bw + bw]); + } + } + } + } +} + +/// One macroblock's samples (luma 16×16; chroma rows of `16 >> shx` samples). +pub(crate) struct MbBuf { + y: [u8; 256], + c: [[u8; 256]; 2], +} + +#[derive(Clone, Copy, Default)] +struct Motion { + flags: i16, + mtype: u8, + /// vectors[r][s][t] in half-sample units (vertical in field units for field prediction). + mv: [[[i32; 2]; 2]; 2], + select: [[usize; 2]; 2], + dmv: [i32; 2], +} + +struct SliceState { + qscale: i32, + dc_pred: [i32; 3], + pmv: [[[i32; 2]; 2]; 2], + last: Motion, +} + +fn dc_reset(p: &PicParams) -> i32 { + 1 << (7 + p.pce.intra_dc_precision) +} + +#[derive(Debug)] +pub(crate) struct SliceError(pub &'static str); + +type R = std::result::Result; + +/// Decode one slice (`data`: the bytes after its start code) into `t`. `row`: the slice's +/// macroblock row; `single_row`: MPEG-2 slices may not leave their row. +pub(crate) fn decode_slice(p: &PicParams, data: &[u8], row: usize, single_row: bool, t: &mut Target) -> R<()> { + let tb = tables(); + let mut b = Bits::new(data); + if p.vertical_size_extension { + b.skip(3); // already applied to `row` by the caller + } + let qcode = b.read(5) as usize; + if p.mpeg2 && b.peek(1) == 1 { + b.skip(9); // intra_slice_flag, intra_slice, reserved_bits + } + while b.bit() { + b.skip(8); // extra_information_slice + } + let mut st = SliceState { qscale: qscale(p, qcode), dc_pred: [dc_reset(p); 3], pmv: [[[0; 2]; 2]; 2], last: Motion::default() }; + let total = p.mb_width * p.mb_rows; + let row_end = if single_row { (row + 1) * p.mb_width } else { total }; + let mut addr: isize = (row * p.mb_width) as isize - 1; + let mut first = true; + let mut buf = MbBuf { y: [0; 256], c: [[0; 256]; 2] }; + let mut blk = [0i32; 64]; + loop { + // macroblock_address_increment + let mut inc = 0isize; + loop { + match tb.mba.decode(&mut b) { + Some(MBA_ESCAPE) => inc += 33, + Some(MBA_STUFFING) => {} + Some(n) => { + inc += n as isize; + break; + } + None => return Err(SliceError("bad macroblock_address_increment")), + } + if b.overrun() { + return Err(SliceError("truncated slice")); + } + } + if !first && inc > 1 { + // skipped macroblocks + for _ in 1..inc { + addr += 1; + if addr as usize >= row_end { + return Err(SliceError("macroblock address past the row")); + } + skipped_mb(p, &mut st, addr as usize, &mut buf, t)?; + } + addr += 1; + } else { + addr += inc; + } + first = false; + if addr < 0 || addr as usize >= row_end { + return Err(SliceError("macroblock address out of range")); + } + let a = addr as usize; + macroblock(p, &mut st, &mut b, a, &mut buf, &mut blk, t)?; + if b.overrun() { + return Err(SliceError("truncated macroblock")); + } + // next_start_code: 23 zero bits end the slice + if b.bits_left() <= 0 || b.peek(23) == 0 { + return Ok(()); + } + } +} + +fn qscale(p: &PicParams, code: usize) -> i32 { + if !p.mpeg2 { + code as i32 + } else if p.pce.q_scale_type { + NON_LINEAR_Q[code & 31] as i32 + } else { + 2 * code as i32 + } +} + +/// A skipped macroblock (§7.6.6): P pictures predict from the same position with a zero vector; +/// B pictures repeat the previous macroblock's prediction. +fn skipped_mb(p: &PicParams, st: &mut SliceState, addr: usize, buf: &mut MbBuf, t: &mut Target) -> R<()> { + st.dc_pred = [dc_reset(p); 3]; + let m = match p.ptype { + PictureType::P => { + st.pmv = [[[0; 2]; 2]; 2]; + let mut m = Motion { flags: MB_FWD, mtype: if p.frame_picture() { MT_FRAME } else { MT_FIELD }, ..Default::default() }; + m.select[0][0] = p.parity; + m + } + PictureType::B => { + let last = st.last; + if last.flags & MB_INTRA != 0 { + return Err(SliceError("skipped macroblock after an intra macroblock")); + } + // the previous macroblock's direction, one vector per direction (PMV[0][s]); field + // pictures predict from the field of the same parity (§7.6.6.4) + let mut m = + Motion { flags: last.flags & (MB_FWD | MB_BWD), mtype: if p.frame_picture() { MT_FRAME } else { MT_FIELD }, ..Default::default() }; + for s in 0..2 { + m.mv[0][s] = st.pmv[0][s]; + m.select[0][s] = p.parity; + } + m + } + _ => return Err(SliceError("skipped macroblock in an intra picture")), + }; + let (mbx, mby) = (addr % p.mb_width, addr / p.mb_width); + predict_mb(p, &m, mbx, mby, buf); + t.put(mbx, mby, buf, p.cmb()); + if p.ptype == PictureType::B { + st.last = Motion { flags: m.flags, ..st.last }; + } + Ok(()) +} + +fn macroblock(p: &PicParams, st: &mut SliceState, b: &mut Bits, addr: usize, buf: &mut MbBuf, blk: &mut [i32; 64], t: &mut Target) -> R<()> { + let tb = tables(); + let ti = match p.ptype { + PictureType::I => 0, + PictureType::P => 1, + PictureType::B => 2, + PictureType::D => 3, + }; + let flags = tb.mbtype[ti].decode(b).ok_or(SliceError("bad macroblock_type"))?; + let intra = flags & MB_INTRA != 0; + let frame_pic = p.frame_picture(); + let mut m = Motion { flags, ..Default::default() }; + if flags & (MB_FWD | MB_BWD) != 0 { + m.mtype = if frame_pic && (p.pce.frame_pred_frame_dct || !p.mpeg2) { MT_FRAME } else { b.read(2) as u8 }; + if m.mtype == 0 { + return Err(SliceError("reserved motion type")); + } + } else if intra && p.pce.concealment_motion_vectors { + m.mtype = if frame_pic { MT_FRAME } else { MT_FIELD }; + } + let dct_type = p.mpeg2 && frame_pic && !p.pce.frame_pred_frame_dct && (intra || flags & MB_PATTERN != 0) && b.bit(); + if flags & MB_QUANT != 0 { + st.qscale = qscale(p, b.read(5) as usize); + } + let concealment = intra && p.pce.concealment_motion_vectors; + read_motion_vectors(p, st, b, &mut m, concealment)?; + if concealment { + b.skip(1); // marker_bit + } + // coded_block_pattern + let nblocks = 4 + p.chroma_blocks(); + let cbp = read_cbp(p, b, flags, intra)?; + let (mbx, mby) = (addr % p.mb_width, addr / p.mb_width); + if intra { + if !concealment { + st.pmv = [[[0; 2]; 2]; 2]; + } + } else { + st.dc_pred = [dc_reset(p); 3]; + if p.ptype == PictureType::P && flags & MB_FWD == 0 { + // "No MC": zero vector from the same position (same-parity field in field pictures) + st.pmv = [[[0; 2]; 2]; 2]; + m.flags |= MB_FWD; + m.mtype = if frame_pic { MT_FRAME } else { MT_FIELD }; + m.select[0][0] = p.parity; + } + predict_mb(p, &m, mbx, mby, buf); + } + // blocks + let (cbw, cbh) = p.cmb(); + for i in 0..nblocks { + if cbp & (1 << (nblocks - 1 - i)) == 0 { + continue; + } + let cc = block_component(i); + let dc_only = block(p, st, b, cc, intra, blk)?; + idct(blk, dc_only); + place_block(buf, blk, i, (cbw, cbh), dct_type, intra); + } + if p.ptype == PictureType::D { + b.skip(1); // end_of_macroblock + } + t.put(mbx, mby, buf, (cbw, cbh)); + st.last = m; + Ok(()) +} + +/// Colour component (0 luma, 1 Cb, 2 Cr) of block `i` of a macroblock. +fn block_component(i: usize) -> usize { + if i < 4 { 0 } else { 1 + (i & 1) } +} + +/// How a macroblock's motion vectors are sent. +struct MvLayout { + /// Vectors per direction: 1, or 2 for field/16x8 prediction. + count: usize, + field_fmt: bool, + dual: bool, +} + +impl MvLayout { + fn of(frame_pic: bool, mtype: u8) -> MvLayout { + let (count, field_fmt, dual) = match (frame_pic, mtype) { + (true, MT_FIELD) => (2, true, false), + (true, MT_DUAL) | (false, MT_DUAL) => (1, true, true), + (true, _) => (1, false, false), + (false, MT_16X8) => (2, true, false), + (false, _) => (1, true, false), + }; + MvLayout { count, field_fmt, dual } + } +} + +/// Read the motion vectors (and dual-prime vectors) of a macroblock, updating the predictors. +fn read_motion_vectors(p: &PicParams, st: &mut SliceState, b: &mut Bits, m: &mut Motion, concealment: bool) -> R<()> { + let tb = tables(); + let frame_pic = p.frame_picture(); + let MvLayout { count, field_fmt, dual } = MvLayout::of(frame_pic, m.mtype); + let flags = m.flags; + for s in 0..2 { + let present = if s == 0 { flags & MB_FWD != 0 || concealment } else { flags & MB_BWD != 0 }; + if !present { + continue; + } + for r in 0..count { + if count == 2 || (field_fmt && !dual) { + m.select[r][s] = b.read(1) as usize; + } + for c in 0..2 { + let fc = p.pce.f_code[s][c] as i32; + let code = tb.motion.decode(b).ok_or(SliceError("bad motion_code"))? as i32; + let r_size = (fc - 1).max(0); + let residual = if fc != 1 && code != 0 { b.read(r_size as u32) as i32 } else { 0 }; + if dual { + m.dmv[c] = tb.dmv.decode(b).ok_or(SliceError("bad dmvector"))? as i32; + } + let f = 1 << r_size; + let delta = if f == 1 || code == 0 { code } else { (((code.abs() - 1) * f) + residual + 1) * code.signum() }; + let halve = p.mpeg2 && field_fmt && c == 1 && frame_pic; + let pred = if halve { st.pmv[r][s][c] >> 1 } else { st.pmv[r][s][c] }; + let v = wrap_vector(pred, delta, f); + st.pmv[r][s][c] = if halve { v * 2 } else { v }; + m.mv[r][s][c] = v; + } + } + if count == 1 { + st.pmv[1][s] = st.pmv[0][s]; + } + if !p.mpeg2 && p.full_pel[s] { + for c in 0..2 { + m.mv[0][s][c] *= 2; + } + } + } + Ok(()) +} + +/// coded_block_pattern: one bit per block, first block in the most significant bit. +fn read_cbp(p: &PicParams, b: &mut Bits, flags: i16, intra: bool) -> R { + let nblocks = 4 + p.chroma_blocks(); + Ok(if flags & MB_PATTERN != 0 { + let v = tables().cbp.decode(b).ok_or(SliceError("bad coded_block_pattern"))? as u32; + if nblocks == 8 { (v << 2) | b.read(2) } else { v } + } else if intra { + (1 << nblocks) - 1 + } else { + 0 + }) +} + +/// Write (intra) or add (inter) the decoded block `i` into the macroblock buffer. +fn place_block(buf: &mut MbBuf, blk: &[i32; 64], i: usize, cmb: (usize, usize), dct_type: bool, intra: bool) { + let (cbw, cbh) = cmb; + let cc = block_component(i); + let (dst, stride, x0, y0, step) = if i < 4 { + let x0 = (i & 1) * 8; + let (y0, step) = if dct_type { (i >> 1, 2) } else { ((i >> 1) * 8, 1) }; + (&mut buf.y[..], 16, x0, y0, step) + } else { + let k = (i - 4) >> 1; // 0 for 4:2:0; 0/1 (top/bottom) for 4:2:2 + let (y0, step) = if cbh == 16 && dct_type { (k, 2) } else { (k * 8, 1) }; + (&mut buf.c[cc - 1][..], cbw, 0, y0, step) + }; + for r in 0..8 { + let at = (y0 + r * step) * stride + x0; + let row = &mut dst[at..at + 8]; + let src = &blk[r * 8..r * 8 + 8]; + if intra { + for (o, &v) in row.iter_mut().zip(src) { + *o = v.clamp(0, 255) as u8; + } + } else { + for (o, &v) in row.iter_mut().zip(src) { + *o = (*o as i32 + v).clamp(0, 255) as u8; + } + } + } +} + +/// Decode and dequantise one block into `blk` (raster order). Returns whether only the DC +/// coefficient is non-zero. +fn block(p: &PicParams, st: &mut SliceState, b: &mut Bits, cc: usize, intra: bool, blk: &mut [i32; 64]) -> R { + let tb = tables(); + blk.fill(0); + let scan = &SCAN[p.pce.alternate_scan as usize]; + let w = &p.matrices[(cc > 0 && p.shy == 0) as usize * 2 + (!intra) as usize]; + let qs = st.qscale; + let mut sum: i32 = 0; + let mut i: usize; + let mut ac = false; + if intra { + let size = if cc == 0 { tb.dc_luma.decode(b) } else { tb.dc_chroma.decode(b) }.ok_or(SliceError("bad dct_dc_size"))? as u32; + let diff = if size == 0 { + 0 + } else { + let v = b.read(size) as i32; + if v & (1 << (size - 1)) == 0 { v - (1 << size) + 1 } else { v } + }; + st.dc_pred[cc] += diff; + let dc = if p.mpeg2 { st.dc_pred[cc] * (8 >> p.pce.intra_dc_precision) } else { st.dc_pred[cc] * 8 }; + blk[0] = dc; + sum = dc; + if p.ptype == PictureType::D { + return Ok(true); + } + i = 0; + } else { + i = usize::MAX; // the first coefficient lands at run + // "1s": run 0, level ±1 as the first coefficient of a non-intra block + if b.peek(1) == 1 { + b.skip(1); + let level = if b.bit() { -1 } else { 1 }; + i = 0; + let v = dequant(p, level, qs, w[scan[0] as usize] as i32, false); + blk[scan[0] as usize] = v; + sum += v; + ac = scan[0] != 0; + } + } + let table = if intra && p.pce.intra_vlc_format { &tb.dct_one } else { &tb.dct_zero }; + loop { + let code = table.decode(b).ok_or(SliceError("bad DCT coefficient code"))?; + if code == DCT_EOB { + break; + } + let (run, level) = if code == DCT_ESCAPE { + let run = b.read(6) as usize; + let level = if p.mpeg2 { + b.read_signed(12) + } else { + match b.read(8) as i32 { + 0 => b.read(8) as i32, + 128 => b.read(8) as i32 - 256, + v => (v << 24) >> 24, + } + }; + if level == 0 { + return Err(SliceError("zero escape level")); + } + (run, level) + } else { + let run = (code & 63) as usize; + let level = (code >> 6) as i32; + (run, if b.bit() { -level } else { level }) + }; + i = i.wrapping_add(run + 1); + if i > 63 { + return Err(SliceError("coefficient index past 63")); + } + let pos = scan[i] as usize; + let v = dequant(p, level, qs, w[pos] as i32, intra); + blk[pos] = v; + sum += v; + ac = true; + if b.overrun() { + return Err(SliceError("truncated block")); + } + } + if p.mpeg2 && sum & 1 == 0 { + // mismatch control (§7.4.4) + blk[63] ^= 1; + ac = true; + } + Ok(!ac && blk[1..].iter().all(|&v| v == 0)) +} + +/// `prediction + delta`, brought back into [-16f, 16f-1] (§7.6.3.1). The range is restored on +/// the side the delta moves the vector, and a zero delta keeps the prediction, as the MPEG +/// Software Simulation Group reference decoder (ISO/IEC TR 13818-5) does. This only matters when +/// the prediction itself is out of range: a field vector's doubled vertical predictor before a +/// frame vector. +#[inline(always)] +pub(crate) fn wrap_vector(pred: i32, delta: i32, f: i32) -> i32 { + let v = pred + delta; + if delta > 0 && v > 16 * f - 1 { + v - 32 * f + } else if delta < 0 && v < -16 * f { + v + 32 * f + } else { + v + } +} + +#[inline(always)] +fn dequant(p: &PicParams, level: i32, qs: i32, w: i32, intra: bool) -> i32 { + if p.mpeg2 { + let k = if intra { 0 } else { level.signum() }; + ((2 * level + k) * w * qs / 32).clamp(-2048, 2047) + } else { + let mut v = if intra { 2 * level * qs * w / 16 } else { (2 * level + level.signum()) * qs * w / 16 }; + if v & 1 == 0 { + v -= v.signum(); + } + v.clamp(-2048, 2047) + } +} + +/// The reference field (or frame) for direction `s` and field `sel` (field prediction). +fn reference<'a>(p: &PicParams<'a>, s: usize, sel: usize) -> Option<&'a Frame> { + if s == 1 { + return p.bwd.or(p.fwd); + } + if p.ptype == PictureType::P && !p.frame_picture() && sel != p.parity && p.first_field.is_some() { + return p.first_field; + } + p.fwd.or(p.bwd) +} + +/// Form the prediction of a non-intra macroblock in `buf` (§7.6). +fn predict_mb(p: &PicParams, m: &Motion, mbx: usize, mby: usize, buf: &mut MbBuf) { + let (cbw, cbh) = p.cmb(); + let mb = Pred { mv: [0; 2], mbx, mby, cbw, cbh, avg: false }; + let frame_pic = p.frame_picture(); + let mut avg = false; + for s in 0..2 { + let dir = if s == 0 { MB_FWD } else { MB_BWD }; + if m.flags & dir == 0 { + continue; + } + if frame_pic { + match m.mtype { + MT_FIELD => { + for r in 0..2 { + let Some(f) = reference(p, s, m.select[r][s]) else { return gray(buf) }; + pred_block(p, buf, f, Some(m.select[r][s]), Pred { mv: m.mv[r][s], avg, ..mb }, (8, r, 2)); + } + } + MT_DUAL => { + let Some(f) = reference(p, s, 0) else { return gray(buf) }; + for par in 0..2 { + let opp = 1 - par; + let (mm, e) = + if par == 0 { (if p.pce.top_field_first { 1 } else { 3 }, -1) } else { (if p.pce.top_field_first { 3 } else { 1 }, 1) }; + let d = dual_vector(m.mv[0][s], m.dmv, mm, e); + pred_block(p, buf, f, Some(par), Pred { mv: m.mv[0][s], avg, ..mb }, (8, par, 2)); + pred_block(p, buf, f, Some(opp), Pred { mv: d, avg: true, ..mb }, (8, par, 2)); + } + } + _ => { + let Some(f) = reference(p, s, 0) else { return gray(buf) }; + pred_block(p, buf, f, None, Pred { mv: m.mv[0][s], avg, ..mb }, (16, 0, 1)); + } + } + } else { + match m.mtype { + MT_16X8 => { + for r in 0..2 { + let Some(f) = reference(p, s, m.select[r][s]) else { return gray(buf) }; + pred_half(p, buf, f, m.select[r][s], Pred { mv: m.mv[r][s], avg, ..mb }, r); + } + } + MT_DUAL => { + let same = p.parity; + let opp = 1 - same; + let Some(fs) = reference(p, s, same) else { return gray(buf) }; + let Some(fo) = reference(p, s, opp) else { return gray(buf) }; + let e = if same == 1 { 1 } else { -1 }; + let d = dual_vector(m.mv[0][s], m.dmv, 1, e); + pred_block(p, buf, fs, Some(same), Pred { mv: m.mv[0][s], avg, ..mb }, (16, 0, 1)); + pred_block(p, buf, fo, Some(opp), Pred { mv: d, avg: true, ..mb }, (16, 0, 1)); + } + _ => { + let Some(f) = reference(p, s, m.select[0][s]) else { return gray(buf) }; + pred_block(p, buf, f, Some(m.select[0][s]), Pred { mv: m.mv[0][s], avg, ..mb }, (16, 0, 1)); + } + } + } + avg = true; + } +} + +fn gray(buf: &mut MbBuf) { + buf.y.fill(128); + buf.c[0].fill(128); + buf.c[1].fill(128); +} + +/// Dual-prime opposite-parity vector (§7.6.3.6). +fn dual_vector(v: [i32; 2], dmv: [i32; 2], m: i32, e: i32) -> [i32; 2] { + let scale = |x: i32| (x * m + (x > 0) as i32) >> 1; + [scale(v[0]) + dmv[0], scale(v[1]) + e + dmv[1]] +} + +/// One motion-compensated prediction: vector, macroblock position, chroma block size, and +/// whether to average with what the buffer already holds. +#[derive(Clone, Copy)] +struct Pred { + mv: [i32; 2], + mbx: usize, + mby: usize, + cbw: usize, + cbh: usize, + avg: bool, +} + +/// Predict a 16-wide luma block of `lh` lines (plus chroma) from `f` (one field when `field`), +/// into `buf` lines `first + k·step`; `lines` is `(lh, first, step)`. `lh` is 16 (frame or +/// field-picture prediction) or 8 (one field of a frame macroblock). +fn pred_block(p: &PicParams, buf: &mut MbBuf, f: &Frame, field: Option, pr: Pred, lines: (usize, usize, usize)) { + let Pred { mv, mbx, mby, cbw, cbh, avg } = pr; + let (lh, first, step) = lines; + // luma: base position in the reference plane's own (frame or field) lines + let by = if lh == 8 { mby * 8 } else { mby * 16 }; + let (x, y) = ((mbx * 16) as i32 + (mv[0] >> 1), by as i32 + (mv[1] >> 1)); + predict(&mut buf.y, first * 16, 16 * step, f.plane(0, field), Block { x, y, hx: mv[0] & 1 != 0, hy: mv[1] & 1 != 0, bw: 16, bh: lh }, avg); + let cmv = [if p.shx == 1 { mv[0] / 2 } else { mv[0] }, if p.shy == 1 { mv[1] / 2 } else { mv[1] }]; + let clh = if lh == 8 { cbh / 2 } else { cbh }; + let cby = mby * clh; + let (cx, cy) = ((mbx * cbw) as i32 + (cmv[0] >> 1), cby as i32 + (cmv[1] >> 1)); + let cb = Block { x: cx, y: cy, hx: cmv[0] & 1 != 0, hy: cmv[1] & 1 != 0, bw: cbw, bh: clh }; + for c in 0..2 { + predict(&mut buf.c[c], first * cbw, cbw * step, f.plane(1 + c, field), cb, avg); + } +} + +/// 16x8 motion compensation in a field picture: half `r` (0 upper, 1 lower) of the macroblock. +fn pred_half(p: &PicParams, buf: &mut MbBuf, f: &Frame, field: usize, pr: Pred, r: usize) { + let Pred { mv, mbx, mby, cbw, cbh, avg } = pr; + let (x, y) = ((mbx * 16) as i32 + (mv[0] >> 1), (mby * 16 + r * 8) as i32 + (mv[1] >> 1)); + predict(&mut buf.y, r * 8 * 16, 16, f.plane(0, Some(field)), Block { x, y, hx: mv[0] & 1 != 0, hy: mv[1] & 1 != 0, bw: 16, bh: 8 }, avg); + let cmv = [if p.shx == 1 { mv[0] / 2 } else { mv[0] }, if p.shy == 1 { mv[1] / 2 } else { mv[1] }]; + let ch = cbh / 2; + let (cx, cy) = ((mbx * cbw) as i32 + (cmv[0] >> 1), (mby * cbh + r * ch) as i32 + (cmv[1] >> 1)); + let cb = Block { x: cx, y: cy, hx: cmv[0] & 1 != 0, hy: cmv[1] & 1 != 0, bw: cbw, bh: ch }; + for c in 0..2 { + predict(&mut buf.c[c], r * ch * cbw, cbw, f.plane(1 + c, Some(field)), cb, avg); + } +} + +/// The macroblock row a slice starts in (slice_vertical_position and its extension). +pub(crate) fn slice_row(vertical_size_extension: bool, code: u8, payload: &[u8]) -> usize { + let mut row = code as usize - 1; + if vertical_size_extension && !payload.is_empty() { + row += ((payload[0] >> 5) as usize) << 7; + } + row +} diff --git a/crates/video/src/mpeg2/synth_tests.rs b/crates/video/src/mpeg2/synth_tests.rs new file mode 100644 index 000000000..f3c8648eb --- /dev/null +++ b/crates/video/src/mpeg2/synth_tests.rs @@ -0,0 +1,903 @@ +//! Synthetic MPEG-2 streams for coding tools ffmpeg's encoder never writes: field pictures (the +//! second field predicted from the first), 16x8 motion compensation, dual-prime prediction in +//! field and frame pictures, concealment motion vectors in every picture type, skipped +//! macroblocks of field pictures, 4:2:2 field DCT... A random but valid stream is written here +//! (random macroblock types, vectors, field selects and coefficients) and decoded by both our +//! decoder and ffmpeg (the oracle); the frames must agree up to IDCT rounding. + +use std::collections::HashMap; + +use crate::mpeg2::tests::W; +use crate::mpeg2::vlc::*; +use crate::mpeg2::*; + +struct Rng(u64); + +impl Rng { + fn next(&mut self) -> u64 { + self.0 ^= self.0 << 13; + self.0 ^= self.0 >> 7; + self.0 ^= self.0 << 17; + self.0 + } + fn below(&mut self, n: u32) -> u32 { + (self.next() % n.max(1) as u64) as u32 + } + fn range(&mut self, lo: i32, hi: i32) -> i32 { + lo + self.below((hi - lo + 1) as u32) as i32 + } + fn chance(&mut self, pct: u32) -> bool { + self.below(100) < pct + } +} + +fn code_of(table: &[(&'static str, i16)], v: i16) -> &'static str { + table.iter().find(|e| e.1 == v).map(|e| e.0).unwrap_or_else(|| panic!("no code for {v}")) +} + +#[derive(Clone, Copy)] +struct Cfg { + name: &'static str, + seed: u64, + chroma: u8, + w: usize, + h: usize, + /// Field pictures (else interlaced frame pictures with frame_pred_frame_dct = 0). + field_pics: bool, + /// The first field / top_field_first. + tff: bool, + b_frames: bool, + dual: bool, + alt_scan: bool, + intra_vlc: bool, + q_scale_type: bool, + dc_prec: u8, + concealment: bool, + frames: usize, +} + +#[derive(Clone, Copy, Default)] +struct M { + flags: i16, + mtype: u8, + mv: [[[i32; 2]; 2]; 2], + select: [[usize; 2]; 2], + dmv: [i32; 2], +} + +/// A prediction block: plane (0 luma, 1 chroma), field, position, half flags, size. +struct Req { + chroma: bool, + field: bool, + x: i32, + y: i32, + hx: i32, + hy: i32, + w: i32, + h: i32, +} + +struct Enc { + w: W, + rng: Rng, + cfg: Cfg, + mbw: usize, + mbh: usize, + shy: u32, + dct_codes: [HashMap<(u32, u32), &'static str>; 2], + /// The active quantiser_scale. + qs: i32, +} + +/// Picture-level coding state. +struct Pic { + ptype: PictureType, + structure: u8, + parity: usize, + fpfd: bool, + /// The P second field of the first frame: only the first field (opposite parity) exists. + lone: bool, +} + +struct St { + pmv: [[[i32; 2]; 2]; 2], + dc: [i32; 3], + last: M, +} + +const F_CODE: i32 = 2; + +impl Enc { + fn new(cfg: Cfg) -> Enc { + let mut dct_codes = [HashMap::new(), HashMap::new()]; + for (k, t) in [dct_table(DCT_ZERO, DCT_ZERO_LONG), dct_table(DCT_ONE, &[])].into_iter().enumerate() { + for (c, v) in t { + if v >= 0 { + dct_codes[k].insert(((v & 63) as u32, (v >> 6) as u32), c); + } + } + } + Enc { + w: W::new(), + rng: Rng(cfg.seed), + cfg, + mbw: cfg.w.div_ceil(16), + mbh: 2 * cfg.h.div_ceil(32), + shy: (cfg.chroma == 1) as u32, + dct_codes, + qs: 2, + } + } + + fn sequence(&mut self) { + let c = self.cfg; + let w = &mut self.w; + w.start(0xB3); + w.put(c.w as u32, 12); + w.put(c.h as u32, 12); + w.put(1, 4); + w.put(3, 4); + w.put(20000, 18); + w.put(1, 1); + w.put(112, 10); + w.put(0, 1); + w.put(0, 1); + w.put(0, 1); + w.start(0xB5); + w.put(1, 4); + w.put(if c.chroma == 2 { 0x85 } else { 0x48 }, 8); + w.put(0, 1); // interlaced sequence + w.put(c.chroma as u32, 2); + w.put(0, 2); + w.put(0, 2); + w.put(0, 12); + w.put(1, 1); + w.put(0, 8); + w.put(0, 1); + w.put(0, 2); + w.put(0, 5); + w.start(0xB8); + w.put(0, 12); + w.put(1, 1); // marker_bit + w.put(0, 12); + w.put(1, 1); // closed GOP + w.put(0, 1); + } + + fn picture(&mut self, tref: u16, ptype: PictureType, structure: u8) { + let c = self.cfg; + let w = &mut self.w; + w.start(0x00); + w.put(tref as u32, 10); + w.put( + match ptype { + PictureType::I => 1, + PictureType::P => 2, + _ => 3, + }, + 3, + ); + w.put(0xFFFF, 16); + if ptype != PictureType::I { + w.put(0, 1); + w.put(7, 3); + } + if ptype == PictureType::B { + w.put(0, 1); + w.put(7, 3); + } + w.put(0, 1); + w.start(0xB5); + w.put(8, 4); + let fwd = if ptype == PictureType::I && !c.concealment { 15 } else { F_CODE as u32 }; + let bwd = if ptype == PictureType::B { F_CODE as u32 } else { 15 }; + for v in [fwd, fwd, bwd, bwd] { + w.put(v, 4); + } + w.put(c.dc_prec as u32, 2); + w.put(structure as u32, 2); + w.put((structure == 3 && c.tff) as u32, 1); + w.put(0, 1); // frame_pred_frame_dct + w.put(c.concealment as u32, 1); + w.put(c.q_scale_type as u32, 1); + w.put(c.intra_vlc as u32, 1); + w.put(c.alt_scan as u32, 1); + w.put(0, 1); // repeat_first_field + w.put(0, 1); // chroma_420_type + w.put(0, 1); // progressive_frame + w.put(0, 1); + let lone = tref == 0 && ptype == PictureType::P; + let pic = Pic { ptype, structure, parity: (structure == 2) as usize, fpfd: false, lone }; + let rows = if structure == 3 { self.mbh } else { self.mbh / 2 }; + for r in 0..rows { + self.slice(&pic, r); + } + } + + fn set_q(&mut self, code: i32) { + self.qs = if self.cfg.q_scale_type { crate::mpeg2::slice::NON_LINEAR_Q[code as usize] as i32 } else { 2 * code }; + } + + fn dc_reset(&self) -> i32 { + 1 << (7 + self.cfg.dc_prec) + } + + fn slice(&mut self, p: &Pic, row: usize) { + self.w.start(1 + row as u8); + let q = self.rng.range(1, 31); + self.set_q(q); + self.w.put(q as u32, 5); + self.w.put(0, 1); + let mut st = St { pmv: [[[0; 2]; 2]; 2], dc: [self.dc_reset(); 3], last: M::default() }; + let mut skipped = 0; + for mbx in 0..self.mbw { + let can_skip = p.ptype != PictureType::I && !p.lone && mbx > 0 && mbx + 1 < self.mbw && self.rng.chance(15); + if can_skip && self.try_skip(p, &mut st, mbx, row) { + skipped += 1; + continue; + } + self.address_increment(skipped + 1); + skipped = 0; + self.macroblock(p, &mut st, mbx, row); + } + } + + fn address_increment(&mut self, mut n: usize) { + while n > 33 { + self.w.code(code_of(MBA, MBA_ESCAPE)); + n -= 33; + } + self.w.code(code_of(MBA, n as i16)); + } + + /// Skip the macroblock when its implied prediction stays inside the picture. + fn try_skip(&mut self, p: &Pic, st: &mut St, mbx: usize, mby: usize) -> bool { + let mut m = M { mtype: if p.structure == 3 { 2 } else { 1 }, ..Default::default() }; + match p.ptype { + PictureType::P => { + m.flags = MB_FWD; + m.select[0][0] = p.parity; + } + _ => { + if st.last.flags & MB_INTRA != 0 || st.last.flags & (MB_FWD | MB_BWD) == 0 { + return false; + } + m.flags = st.last.flags & (MB_FWD | MB_BWD); + for s in 0..2 { + m.mv[0][s] = st.pmv[0][s]; + m.select[0][s] = p.parity; + } + } + } + if !self.fits(p, &m, mbx, mby) { + return false; + } + st.dc = [self.dc_reset(); 3]; + if p.ptype == PictureType::P { + st.pmv = [[[0; 2]; 2]; 2]; + } + true + } + + /// Every prediction block of `m` lies inside its reference plane. + fn fits(&self, p: &Pic, m: &M, mbx: usize, mby: usize) -> bool { + let (lw, lh) = (self.mbw as i32 * 16, self.mbh as i32 * 16); + let (cw, ch) = (lw / 2, lh >> self.shy); + self.requests(p, m, mbx, mby).iter().all(|r| { + let (pw, ph) = if r.chroma { (cw, ch) } else { (lw, lh) }; + let ph = if r.field { ph / 2 } else { ph }; + r.x >= 0 && r.y >= 0 && r.x + r.w + r.hx <= pw && r.y + r.h + r.hy <= ph + }) + } + + /// The prediction blocks of `m` (mirrors the decoder's geometry). + fn requests(&self, p: &Pic, m: &M, mbx: usize, mby: usize) -> Vec { + let mut out = Vec::new(); + let cbh = 16 >> self.shy; + let mut add = |field: bool, mv: [i32; 2], lx: i32, ly: i32, lh: i32, cy: i32, ch: i32| { + out.push(Req { chroma: false, field, x: lx + (mv[0] >> 1), y: ly + (mv[1] >> 1), hx: mv[0] & 1, hy: mv[1] & 1, w: 16, h: lh }); + let cmv = [mv[0] / 2, if self.shy == 1 { mv[1] / 2 } else { mv[1] }]; + out.push(Req { chroma: true, field, x: lx / 2 + (cmv[0] >> 1), y: cy + (cmv[1] >> 1), hx: cmv[0] & 1, hy: cmv[1] & 1, w: 8, h: ch }); + }; + let x = (mbx * 16) as i32; + for s in 0..2 { + let dir = if s == 0 { MB_FWD } else { MB_BWD }; + if m.flags & dir == 0 { + continue; + } + let mby = mby as i32; + if p.structure == 3 { + match m.mtype { + 1 => { + for r in 0..2 { + add(true, m.mv[r][s], x, mby * 8, 8, mby * cbh / 2, cbh / 2); + } + } + 3 => { + for par in 0..2 { + let (mm, e) = if par == 0 { (if self.cfg.tff { 1 } else { 3 }, -1) } else { (if self.cfg.tff { 3 } else { 1 }, 1) }; + add(true, m.mv[0][s], x, mby * 8, 8, mby * cbh / 2, cbh / 2); + add(true, dual(m.mv[0][s], m.dmv, mm, e), x, mby * 8, 8, mby * cbh / 2, cbh / 2); + } + } + _ => add(false, m.mv[0][s], x, mby * 16, 16, mby * cbh, cbh), + } + } else { + match m.mtype { + 2 => { + for r in 0..2 { + add(true, m.mv[r][s], x, mby * 16 + 8 * r as i32, 8, mby * cbh + r as i32 * cbh / 2, cbh / 2); + } + } + 3 => { + let e = if p.parity == 1 { 1 } else { -1 }; + add(true, m.mv[0][s], x, mby * 16, 16, mby * cbh, cbh); + add(true, dual(m.mv[0][s], m.dmv, 1, e), x, mby * 16, 16, mby * cbh, cbh); + } + _ => add(true, m.mv[0][s], x, mby * 16, 16, mby * cbh, cbh), + } + } + } + out + } + + fn macroblock(&mut self, p: &Pic, st: &mut St, mbx: usize, mby: usize) { + let frame_pic = p.structure == 3; + let (table, flags) = self.pick_mb_type(p); + let intra = flags & MB_INTRA != 0; + let concealment = intra && self.cfg.concealment; + let mut pmv = st.pmv; + let moving = flags & (MB_FWD | MB_BWD) != 0; + let mut m = self.pick_motion(p, flags, concealment, (mbx, mby), pmv); + self.w.code(code_of(table, flags)); + if moving { + self.w.put(m.mtype as u32, 2); + } + let dct_type = frame_pic && !p.fpfd && (intra || flags & MB_PATTERN != 0); + if dct_type { + self.w.put(self.rng.below(2), 1); + } + if flags & MB_QUANT != 0 { + let q = self.rng.range(1, 31); + self.set_q(q); + self.w.put(q as u32, 5); + } + self.put_vectors(p, &m, concealment, &mut pmv); + // predictor resets + if intra { + if !concealment { + pmv = [[[0; 2]; 2]; 2]; + } + } else { + st.dc = [self.dc_reset(); 3]; + if p.ptype == PictureType::P && flags & MB_FWD == 0 { + pmv = [[[0; 2]; 2]; 2]; + m.select[0][0] = p.parity; + } + } + st.pmv = pmv; + self.put_blocks(st, flags, intra); + st.last = m; + } + + fn pick_mb_type(&mut self, p: &Pic) -> (&'static [(&'static str, i16)], i16) { + let quant = self.rng.chance(25); + let (table, mut flags): (&'static [(&'static str, i16)], i16) = match p.ptype { + PictureType::I => (MBTYPE_I, MB_INTRA), + PictureType::P => ( + MBTYPE_P, + match self.rng.below(100) { + 0..10 => MB_INTRA, + 10..55 => MB_FWD | MB_PATTERN, + 55..70 if !p.lone => MB_PATTERN, + 55..70 => MB_FWD | MB_PATTERN, + _ => MB_FWD, + }, + ), + _ => ( + MBTYPE_B, + match self.rng.below(100) { + 0..6 => MB_INTRA, + 6..30 => MB_FWD | MB_BWD | MB_PATTERN, + 30..45 => MB_FWD | MB_BWD, + 45..60 => MB_FWD | MB_PATTERN, + 60..70 => MB_FWD, + 70..85 => MB_BWD | MB_PATTERN, + _ => MB_BWD, + }, + ), + }; + if quant && (flags & (MB_INTRA | MB_PATTERN) != 0) { + flags |= MB_QUANT; + } + (table, flags) + } + + /// Pick vectors until the prediction fits the picture and the predictors can reach them. + fn pick_motion(&mut self, p: &Pic, flags: i16, concealment: bool, pos: (usize, usize), pmv: [[[i32; 2]; 2]; 2]) -> M { + let (mbx, mby) = pos; + let frame_pic = p.structure == 3; + let mut m = M { flags, ..Default::default() }; + let moving = flags & (MB_FWD | MB_BWD) != 0; + if moving || concealment { + let mut ok = false; + for _ in 0..40 { + let mut cand = M { flags, ..Default::default() }; + cand.mtype = if concealment { + if frame_pic { 2 } else { 1 } + } else { + let dual_ok = self.cfg.dual && p.ptype == PictureType::P && !p.lone; + match self.rng.below(if dual_ok { 3 } else { 2 }) { + 0 => 1, + 1 => 2, + _ => 3, + } + }; + let spread = if self.rng.chance(20) { 0 } else { 31 }; + + for r in 0..2 { + for s in 0..2 { + cand.select[r][s] = self.rng.below(2) as usize; + cand.mv[r][s] = [self.rng.range(-spread - 1, spread), self.rng.range(-spread - 1, spread)]; + } + } + cand.dmv = [self.rng.range(-1, 1), self.rng.range(-1, 1)]; + if cand.mtype == 3 { + cand.select = [[p.parity; 2]; 2]; + } + if p.lone { + cand.select = [[1 - p.parity; 2]; 2]; + } + if (concealment || self.fits(p, &cand, mbx, mby)) && representable(p, &cand, concealment, pmv) { + m = cand; + ok = true; + break; + } + } + if !ok && concealment { + // a vector the (possibly out-of-range) predictor reaches unambiguously + let mut c = M { flags, mtype: if frame_pic { 2 } else { 1 }, ..Default::default() }; + for t in 0..2 { + let pred = pmv[0][0][t]; + c.mv[0][0][t] = if pred < -32 { + pred + 32 + } else if pred > 31 { + pred - 32 + } else { + 0 + }; + } + m = c; + } else if !ok { + // zero vectors from the same parity always fit; field vectors (halved + // predictors) are always representable + let sel = if p.lone { 1 - p.parity } else { p.parity }; + m = if frame_pic { + M { flags, mtype: 1, select: [[0, 0], [1, 1]], ..Default::default() } + } else { + M { flags, mtype: 1, select: [[sel; 2]; 2], ..Default::default() } + }; + assert!(representable(p, &m, concealment, pmv)); + } + } + m + } + + /// Vectors, coded differentially against the predictors as the decoder forms them. + fn put_vectors(&mut self, p: &Pic, m: &M, concealment: bool, pmv: &mut [[[i32; 2]; 2]; 2]) { + let frame_pic = p.structure == 3; + let flags = m.flags; + let (count, field_fmt, dualp) = match (frame_pic, m.mtype) { + (true, 1) => (2, true, false), + (_, 3) => (1, true, true), + (true, _) => (1, false, false), + (false, 2) => (2, true, false), + (false, _) => (1, true, false), + }; + for s in 0..2 { + let present = if s == 0 { flags & MB_FWD != 0 || concealment } else { flags & MB_BWD != 0 }; + if !present { + continue; + } + for r in 0..count { + if count == 2 || (field_fmt && !dualp) { + self.w.put(m.select[r][s] as u32, 1); + } + for c in 0..2 { + let halve = field_fmt && c == 1 && frame_pic; + let pred = if halve { pmv[r][s][c] >> 1 } else { pmv[r][s][c] }; + let v = m.mv[r][s][c]; + self.motion_code(v - pred); + if dualp { + self.w.code(code_of(DMV, m.dmv[c] as i16)); + } + pmv[r][s][c] = if halve { v * 2 } else { v }; + } + } + if count == 1 { + pmv[1][s] = pmv[0][s]; + } + } + if concealment { + self.w.put(1, 1); + } + } + + /// coded_block_pattern and the blocks it selects. + fn put_blocks(&mut self, st: &mut St, flags: i16, intra: bool) { + let nblocks = if self.cfg.chroma == 2 { 8 } else { 6 }; + let cbp: u32 = if flags & MB_PATTERN != 0 { + // the coded_block_pattern code for 0 is only for 4:2:2 and 4:4:4 (Table B.9) + let v = self.rng.range(1, (1 << nblocks) - 1) as u32; + self.w.code(code_of(CBP, (v >> (nblocks - 6)) as i16)); + if nblocks == 8 { + self.w.put(v & 3, 2); + } + v + } else if intra { + (1 << nblocks) - 1 + } else { + 0 + }; + for i in 0..nblocks { + if cbp & (1 << (nblocks - 1 - i)) != 0 { + let cc = if i < 4 { 0 } else { 1 + (i & 1) }; + self.block(st, cc, intra); + } + } + } + + fn motion_code(&mut self, delta: i32) { + let f = 1 << (F_CODE - 1); + let range = 32 * f; + let mut d = delta; + if d < -16 * f { + d += range; + } else if d > 16 * f - 1 { + d -= range; + } + if d == 0 { + self.w.code(code_of(MOTION, 0)); + return; + } + let a = d.abs() - 1; + let code = (a >> (F_CODE - 1)) + 1; + self.w.code(code_of(MOTION, (code * d.signum()) as i16)); + self.w.put((a & (f - 1)) as u32, (F_CODE - 1) as u32); + } + + fn block(&mut self, st: &mut St, cc: usize, intra: bool) { + let tab = (intra && self.cfg.intra_vlc) as usize; + let mut first = !intra; + if intra { + let max = (1 << (8 + self.cfg.dc_prec)) - 1; + let target = (st.dc[cc] + self.rng.range(-40, 40)).clamp(0, max); + let diff = target - st.dc[cc]; + st.dc[cc] = target; + let size = 32 - diff.unsigned_abs().leading_zeros(); + self.w.code(code_of(if cc == 0 { DC_LUMA } else { DC_CHROMA }, size as i16)); + if size > 0 { + let v = if diff > 0 { diff } else { diff + (1 << size) - 1 }; + self.w.put(v as u32, size); + } + } + let n = if intra { self.rng.below(6) } else { 1 + self.rng.below(6) }; + let mut idx: i32 = if intra { 0 } else { -1 }; + for _ in 0..n { + let run = self.rng.range(0, 6); + if idx + run + 1 > 63 { + break; + } + idx += run + 1; + // keep dequantised values within ±300 (W ≤ 83 for the default intra matrix): realistic + // residuals (extreme ones overflow ffmpeg's integer IDCT, which then differs) + let cap = if intra { 300 * 16 / (self.qs * 83) } else { (300 * 32 / (self.qs * 16) - 1) / 2 }.clamp(1, 2047); + let mag = if self.rng.chance(8) { self.rng.range(cap.min(20), cap.min(400)) } else { self.rng.range(1, cap.min(4)) }; + let neg = self.rng.chance(50); + if first && run == 0 && mag == 1 { + self.w.code("1"); + self.w.put(neg as u32, 1); + } else if let Some(c) = self.dct_codes[tab].get(&(run as u32, mag as u32)).filter(|_| !self.rng.chance(5)) { + self.w.code(c); + self.w.put(neg as u32, 1); + } else { + self.w.code("0000 01"); + self.w.put(run as u32, 6); + let l = if neg { -mag } else { mag }; + self.w.put((l & 0xFFF) as u32, 12); + } + first = false; + } + if first { + // a coded non-intra block needs a coefficient + self.w.code("1"); + self.w.put(0, 1); + } + self.w.code(if tab == 1 { "0110" } else { "10" }); + } +} + +/// Whether every vector of `m` decodes to itself under both the literal §7.6.3.1 wrap and the +/// reference decoder's (they differ only for out-of-range predictors). +fn representable(p: &Pic, m: &M, concealment: bool, mut pmv: [[[i32; 2]; 2]; 2]) -> bool { + let frame_pic = p.structure == 3; + let f = 1 << (F_CODE - 1); + let (count, field_fmt, _) = match (frame_pic, m.mtype) { + (true, 1) => (2, true, false), + (_, 3) => (1, true, true), + (true, _) => (1, false, false), + (false, 2) => (2, true, false), + (false, _) => (1, true, false), + }; + for s in 0..2 { + let present = if s == 0 { m.flags & MB_FWD != 0 || concealment } else { m.flags & MB_BWD != 0 }; + if !present { + continue; + } + for r in 0..count { + for c in 0..2 { + let halve = field_fmt && c == 1 && frame_pic; + let pred = if halve { pmv[r][s][c] >> 1 } else { pmv[r][s][c] }; + let v = m.mv[r][s][c]; + let mut d = v - pred; + if d < -16 * f { + d += 32 * f; + } else if d > 16 * f - 1 { + d -= 32 * f; + } + let literal = { + let x = pred + d; + if x < -16 * f { + x + 32 * f + } else if x > 16 * f - 1 { + x - 32 * f + } else { + x + } + }; + if crate::mpeg2::slice::wrap_vector(pred, d, f) != v || literal != v { + return false; + } + pmv[r][s][c] = if halve { v * 2 } else { v }; + } + } + if count == 1 { + pmv[1][s] = pmv[0][s]; + } + } + true +} + +fn dual(v: [i32; 2], dmv: [i32; 2], m: i32, e: i32) -> [i32; 2] { + let s = |x: i32| (x * m + (x > 0) as i32) >> 1; + [s(v[0]) + dmv[0], s(v[1]) + e + dmv[1]] +} + +/// Write a stream: closed GOP, I/P/B frame order I0 P3 B1 B2 P6 B4 B5 … (or I0 P1 P2 … without +/// B pictures); field-coded frames are a pair of field pictures (I + P for the first frame). +fn stream(cfg: Cfg) -> Vec { + let mut e = Enc::new(cfg); + e.sequence(); + let mut order: Vec<(u16, PictureType)> = vec![(0, PictureType::I)]; + if cfg.b_frames { + let mut a = 0u16; + while (a as usize) + 3 < cfg.frames { + order.push((a + 3, PictureType::P)); + order.push((a + 1, PictureType::B)); + order.push((a + 2, PictureType::B)); + a += 3; + } + } else { + for k in 1..cfg.frames as u16 { + order.push((k, PictureType::P)); + } + } + let (first, second) = if cfg.tff { (1, 2) } else { (2, 1) }; + for (tref, t) in order { + if cfg.field_pics { + e.picture(tref, t, first); + e.picture(tref, if t == PictureType::I { PictureType::P } else { t }, second); + } else { + e.picture(tref, t, 3); + } + } + e.w.start(0xB7); + e.w.out +} + +fn check(cfg: Cfg) { + let es = stream(cfg); + let (pics, errors) = { + let mut d = Decoder::new(); + let mut out = Vec::new(); + for (i, r) in access_units(&es).into_iter().enumerate() { + out.extend(d.decode(&es[r], i as i64).unwrap()); + } + out.extend(d.flush()); + (out, d.errors()) + }; + assert_eq!(errors, 0, "{}: our decoder reports slice errors", cfg.name); + let Some(ff) = craft_media_testkit::ffmpeg_or_skip(cfg.name) else { return }; + let path = craft_media_testkit::fixtures_dir("mpeg2v").join(format!("{}_{:x}.m2v", cfg.name, cfg.seed)); + std::fs::write(&path, &es).unwrap(); + let fmt = if cfg.chroma == 2 { "yuv422p" } else { "yuv420p" }; + let o = std::process::Command::new(ff) + .args(["-hide_banner", "-loglevel", "error", "-xerror", "-i"]) + .arg(&path) + .args(["-fps_mode", "passthrough", "-f", "rawvideo", "-pix_fmt", fmt, "-"]) + .output() + .unwrap(); + assert!(o.status.success(), "{}: ffmpeg rejects the stream: {}", cfg.name, String::from_utf8_lossy(&o.stderr)); + let raw = o.stdout; + let (w, h) = (cfg.w, cfg.h); + let (cw, ch) = (w / 2, if cfg.chroma == 2 { h } else { h / 2 }); + let fs = w * h + 2 * cw * ch; + assert_eq!(raw.len() / fs, pics.len(), "{}: frame count", cfg.name); + let mut worst = 0u8; + let (mut sse, mut n, mut differ) = (0f64, 0u64, 0u64); + // (frame, type, macroblock x, y, line) of the first luma difference above 4 + let mut first_bad = None; + for (i, p) in pics.iter().enumerate() { + assert_eq!(p.temporal_reference as usize, i, "{}: display order", cfg.name); + assert_eq!(p.top_field_first, cfg.tff); + let f = &raw[i * fs..(i + 1) * fs]; + if first_bad.is_none() + && let Some(k) = (0..w * h).find(|&k| p.y[k].abs_diff(f[k]) > 4) + { + first_bad = Some((i, p.picture_type, k % w / 16, k / w / 16, k / w % 16)); + } + for (a, b) in p.y.iter().chain(&p.cb).chain(&p.cr).zip(f) { + let d = a.abs_diff(*b); + worst = worst.max(d); + sse += (d as f64).powi(2); + differ += (d != 0) as u64; + n += 1; + } + } + let psnr = 10.0 * (255f64 * 255.0 * n as f64 / sse.max(1e-9)).log10(); + println!("{}: {} frames, max diff {worst}, {:.3}% differ, PSNR {psnr:.2} dB", cfg.name, pics.len(), 100.0 * differ as f64 / n as f64); + // the same criteria as the ffmpeg-encoded fixtures (tests/mpeg2/oracle.rs) + assert!(worst <= 4 && psnr >= 58.0, "{} (seed {:#x}): max diff {worst}, PSNR {psnr:.2}, first at {first_bad:?}", cfg.name, cfg.seed); +} + +const BASE: Cfg = Cfg { + name: "", + seed: 1, + chroma: 1, + w: 96, + h: 64, + field_pics: true, + tff: true, + b_frames: true, + dual: false, + alt_scan: false, + intra_vlc: true, + q_scale_type: false, + dc_prec: 0, + concealment: true, + frames: 7, +}; + +/// Every configuration with more random streams. +#[test] +fn seed_sweep() { + let cfgs = [ + Cfg { name: "sweep_field_ibp", ..BASE }, + Cfg { name: "sweep_field_dual_422", b_frames: false, dual: true, chroma: 2, tff: false, frames: 4, ..BASE }, + Cfg { name: "sweep_frame_dual", field_pics: false, b_frames: false, dual: true, frames: 4, dc_prec: 3, ..BASE }, + Cfg { name: "sweep_frame_ib_422", field_pics: false, chroma: 2, tff: false, q_scale_type: true, ..BASE }, + ]; + for seed in 1..=8u64 { + for c in cfgs { + check(Cfg { seed: seed.wrapping_mul(0x9E37_79B9_7F4A_7C15) ^ c.w as u64, ..c }); + } + } +} + +#[test] +fn field_pictures_with_b_and_16x8() { + check(Cfg { name: "synth_field_ibp_tff", seed: 0x1234_5678_9abc, ..BASE }); + check(Cfg { + name: "synth_field_ibp_bff_422", + seed: 0xfeed_beef_1234, + tff: false, + chroma: 2, + alt_scan: true, + q_scale_type: true, + dc_prec: 2, + w: 112, + h: 96, + ..BASE + }); +} + +#[test] +fn dual_prime_in_field_pictures() { + check(Cfg { name: "synth_field_dual_tff", seed: 0x0bad_cafe_0042, b_frames: false, dual: true, frames: 5, ..BASE }); + check(Cfg { + name: "synth_field_dual_bff_422", + seed: 0x1357_9bdf_2468, + b_frames: false, + dual: true, + tff: false, + chroma: 2, + concealment: false, + frames: 5, + ..BASE + }); +} + +#[test] +fn frame_pictures_field_mc_and_dual_prime() { + check(Cfg { name: "synth_frame_dual_tff", seed: 0x5555_aaaa_1111, field_pics: false, b_frames: false, dual: true, frames: 6, ..BASE }); + check(Cfg { + name: "synth_frame_dual_bff_422", + seed: 0x7777_3333_9999, + field_pics: false, + b_frames: false, + dual: true, + tff: false, + chroma: 2, + dc_prec: 1, + frames: 6, + ..BASE + }); + check(Cfg { + name: "synth_frame_ib_422", + seed: 0x2468_ace0_1357, + field_pics: false, + chroma: 2, + alt_scan: true, + intra_vlc: false, + q_scale_type: true, + w: 128, + h: 96, + ..BASE + }); +} + +/// ffmpeg's encoder writes no field pictures, 16x8 or dual-prime prediction, so the hostile sweep +/// over its streams (`tests/mpeg2/fuzz_hostile.rs`) never reaches that code. Mutate and truncate +/// the synthetic streams instead; needs no ffmpeg. +#[test] +fn mutated_field_and_dual_prime_streams_never_panic() { + let cfgs = [ + Cfg { name: "mut_field_ibp", seed: 0x1234_5678_9abc, ..BASE }, + Cfg { name: "mut_field_dual_422", seed: 0x1357_9bdf_2468, b_frames: false, dual: true, chroma: 2, tff: false, frames: 4, ..BASE }, + Cfg { name: "mut_frame_dual", seed: 0x5555_aaaa_1111, field_pics: false, b_frames: false, dual: true, frames: 4, dc_prec: 3, ..BASE }, + Cfg { name: "mut_frame_ib_422", seed: 0x2468_ace0_1357, field_pics: false, chroma: 2, tff: false, q_scale_type: true, ..BASE }, + ]; + let mut state = 0x9E37_79B9_7F4A_7C15u64; + let mut next = move |n: usize| { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + (state % n.max(1) as u64) as usize + }; + for cfg in cfgs { + let es = stream(cfg); + for round in 0..400 { + let mut d = es.clone(); + for _ in 0..1 + next(6) { + // every fourth round stays in the sequence and picture headers + let p = if round % 4 == 0 { next(d.len().min(160)) } else { next(d.len()) }; + match next(3) { + 0 => d[p] ^= 1 << next(8), + 1 => d[p] = next(256) as u8, + _ => d[p] = [0, 0xff, 0x80, 1][next(4)], + } + } + if round % 5 == 4 { + d.truncate(next(d.len())); + } + for threads in [false, true] { + let mut dec = Decoder::new(); + dec.set_threads(threads); + for (i, r) in access_units(&d).into_iter().enumerate() { + let _ = dec.decode(&d[r], i as i64); + } + let _ = dec.flush(); + } + } + } +} diff --git a/crates/video/src/mpeg2/tests.rs b/crates/video/src/mpeg2/tests.rs new file mode 100644 index 000000000..864472e05 --- /dev/null +++ b/crates/video/src/mpeg2/tests.rs @@ -0,0 +1,171 @@ +//! Unit tests over hand-built streams (headers, robustness). Decoding accuracy against ffmpeg is +//! in `tests/mpeg2/oracle.rs`. + +use super::*; + +/// A minimal bit writer for building streams. +pub(crate) struct W { + pub(crate) out: Vec, + acc: u64, + n: u32, +} + +impl W { + pub(crate) fn new() -> W { + W { out: Vec::new(), acc: 0, n: 0 } + } + pub(crate) fn put(&mut self, v: u32, bits: u32) { + for i in (0..bits).rev() { + self.acc = (self.acc << 1) | ((v >> i) & 1) as u64; + self.n += 1; + if self.n == 8 { + self.out.push(self.acc as u8); + self.acc = 0; + self.n = 0; + } + } + } + pub(crate) fn code(&mut self, s: &str) { + for c in s.chars().filter(|c| *c != ' ') { + self.put((c == '1') as u32, 1); + } + } + pub(crate) fn align(&mut self) { + while self.n != 0 { + self.put(0, 1); + } + } + pub(crate) fn start(&mut self, code: u8) { + self.align(); + self.out.extend_from_slice(&[0, 0, 1, code]); + } +} + +/// An MPEG-2 4:2:0 progressive 32×16 I picture whose macroblocks all have DC `dc` (luma) and +/// no AC: two macroblocks, one slice. +fn intra_stream(dc_luma: i32) -> Vec { + let mut w = W::new(); + w.start(0xB3); + w.put(32, 12); + w.put(16, 12); + w.put(1, 4); // square samples + w.put(3, 4); // 25 fps + w.put(1000, 18); + w.put(1, 1); + w.put(10, 10); + w.put(0, 1); + w.put(0, 1); + w.put(0, 1); + w.start(0xB5); + w.put(1, 4); + w.put(0x48, 8); // Main@Main + w.put(1, 1); // progressive + w.put(1, 2); // 4:2:0 + w.put(0, 2); + w.put(0, 2); + w.put(0, 12); + w.put(1, 1); + w.put(0, 8); + w.put(0, 1); + w.put(0, 2); + w.put(0, 5); + w.start(0x00); + w.put(0, 10); + w.put(1, 3); // I + w.put(0xFFFF, 16); + w.put(0, 1); // extra_bit_picture + w.start(0xB5); + w.put(8, 4); + for _ in 0..4 { + w.put(15, 4); + } + w.put(0, 2); // 8-bit DC + w.put(3, 2); // frame picture + w.put(0, 1); + w.put(1, 1); // frame_pred_frame_dct + w.put(0, 1); + w.put(0, 1); + w.put(0, 1); + w.put(0, 1); + w.put(0, 1); + w.put(1, 1); + w.put(1, 1); // progressive_frame + w.put(0, 1); + w.start(0x01); + w.put(8, 5); // quantiser_scale_code + w.put(0, 1); // extra_bit_slice + for mb in 0..2 { + w.code("1"); // address increment 1 + w.code("1"); // macroblock_type: intra + for b in 0..6 { + // DC differential: the first luma block of the slice carries dc - 128 + let diff = if b == 0 && mb == 0 { dc_luma - 128 } else { 0 }; + if b < 4 { + if diff == 0 { + w.code("100"); + } else { + // size 6 covers |diff| < 64 + w.code("1111 0"); + let v = if diff > 0 { diff as u32 } else { (diff + 63) as u32 }; + w.put(v, 6); + } + } else { + w.code("00"); + } + w.code("10"); // EOB (table zero) + } + } + w.start(0xB7); + w.out +} + +#[test] +fn decodes_a_hand_built_intra_picture() { + let s = intra_stream(160); + let info = probe(&s).unwrap(); + assert_eq!((info.width, info.height, info.mpeg2, info.frame_rate), (32, 16, true, Some((25, 1)))); + assert_eq!(info.codec_name(), "MPEG-2 Video (Main@Main)"); + let mut d = Decoder::new(); + let mut pics = d.decode(&s, 7).unwrap(); + pics.extend(d.flush()); + assert_eq!(pics.len(), 1); + let p = &pics[0]; + assert_eq!(d.errors(), 0); + assert_eq!((p.width, p.height, p.pts, p.picture_type), (32, 16, 7, PictureType::I)); + // DC 160 everywhere (the mismatch-control toggle of F[7][7] is far below rounding) + assert!(p.y.iter().all(|&v| v == 160), "{:?}", &p.y[..8]); + assert!(p.cb.iter().chain(&p.cr).all(|&v| v == 128)); + assert_eq!(p.field_order(), None); +} + +#[test] +fn truncation_and_mutation_never_panic() { + let s = intra_stream(100); + for cut in 0..s.len() { + let mut d = Decoder::new(); + let _ = d.decode(&s[..cut], 0); + let _ = d.flush(); + } + let mut x = 0x1234_5678u32; + for _ in 0..3000 { + let mut m = s.clone(); + for _ in 0..3 { + x ^= x << 13; + x ^= x >> 17; + x ^= x << 5; + let i = x as usize % m.len(); + m[i] ^= (x >> 8) as u8; + } + let mut d = Decoder::new(); + let _ = d.decode(&m, 0); + let _ = d.flush(); + } +} + +#[test] +fn access_unit_scan() { + let s = intra_stream(128); + let au = scan_access_unit(&s); + assert!(au.sequence_header && au.is_intra() && !au.is_disposable()); + assert_eq!(au.pictures[0].1.unwrap().picture_structure, 3); +} diff --git a/crates/video/src/mpeg2/vlc.rs b/crates/video/src/mpeg2/vlc.rs new file mode 100644 index 000000000..192f86be0 --- /dev/null +++ b/crates/video/src/mpeg2/vlc.rs @@ -0,0 +1,628 @@ +//! Variable-length code tables of ITU-T H.262 Annex B (B.1-B.4, B.9-B.15), which include those +//! of ISO/IEC 11172-2 Annex B. Codes are written as in the standard (spaces ignored) and turned +//! into two-level lookup tables once. + +use std::sync::OnceLock; + +use crate::mpeg2::bits::Bits; + +#[derive(Clone, Copy, Default)] +struct Entry { + /// Decoded value, or the subtable offset when `sub > 0`. + val: i16, + /// Code length (0: invalid code). + len: u8, + /// Index bits of the subtable this entry points to. + sub: u8, +} + +/// A two-level lookup table. +pub(crate) struct Vlc { + root: u32, + table: Vec, +} + +fn parse_code(s: &str) -> (u32, u32) { + let mut v = 0u32; + let mut n = 0u32; + for c in s.chars() { + match c { + '0' | '1' => { + v = (v << 1) | (c as u32 - '0' as u32); + n += 1; + } + ' ' => {} + // Codes are literals in this crate; the table tests cover them. + _ => debug_assert!(false, "bad code {s}"), + } + } + (v, n) +} + +impl Vlc { + pub fn new(root: u32, codes: &[(&str, i16)]) -> Vlc { + let codes: Vec<(u32, u32, i16)> = codes + .iter() + .map(|(s, v)| { + let (c, n) = parse_code(s); + (c, n, *v) + }) + .collect(); + let mut table = vec![Entry::default(); 1 << root]; + // subtable sizes per root prefix + let mut sub_bits = vec![0u32; 1 << root]; + for &(c, n, _) in &codes { + if n > root { + let p = (c >> (n - root)) as usize; + sub_bits[p] = sub_bits[p].max(n - root); + } + } + for (p, &b) in sub_bits.iter().enumerate() { + if b > 0 { + let off = table.len(); + table.extend(std::iter::repeat_n(Entry::default(), 1 << b)); + table[p] = Entry { val: off as i16, len: 0, sub: b as u8 }; + } + } + for &(c, n, v) in &codes { + if n <= root { + let shift = root - n; + for k in 0..(1u32 << shift) { + let e = &mut table[((c << shift) | k) as usize]; + assert!(e.len == 0 && e.sub == 0, "VLC prefix clash"); + *e = Entry { val: v, len: n as u8, sub: 0 }; + } + } else { + let p = (c >> (n - root)) as usize; + let Entry { val: off, sub: b, .. } = table[p]; + let b = b as u32; + let rest = c & ((1 << (n - root)) - 1); + let shift = b - (n - root); + for k in 0..(1u32 << shift) { + let e = &mut table[off as usize + ((rest << shift) | k) as usize]; + assert!(e.len == 0, "VLC prefix clash"); + *e = Entry { val: v, len: (n - root) as u8, sub: 0 }; + } + } + } + Vlc { root, table } + } + + /// Decode one code; `None` for an invalid code. + #[inline(always)] + pub fn decode(&self, b: &mut Bits) -> Option { + let w = b.peek32(); + let e = self.table[(w >> (32 - self.root)) as usize]; + if e.sub == 0 { + if e.len == 0 { + return None; + } + b.skip(e.len as u32); + return Some(e.val); + } + let idx = ((w << self.root) >> (32 - e.sub as u32)) as usize; + let e2 = self.table[e.val as usize + idx]; + if e2.len == 0 { + return None; + } + b.skip(self.root + e2.len as u32); + Some(e2.val) + } +} + +/// macroblock_address_increment (B.1): 1..=33, [`MBA_ESCAPE`], [`MBA_STUFFING`]. +pub(crate) const MBA_ESCAPE: i16 = 34; +pub(crate) const MBA_STUFFING: i16 = 35; +pub(crate) const MBA: &[(&str, i16)] = &[ + ("1", 1), + ("011", 2), + ("010", 3), + ("0011", 4), + ("0010", 5), + ("0001 1", 6), + ("0001 0", 7), + ("0000 111", 8), + ("0000 110", 9), + ("0000 1011", 10), + ("0000 1010", 11), + ("0000 1001", 12), + ("0000 1000", 13), + ("0000 0111", 14), + ("0000 0110", 15), + ("0000 0101 11", 16), + ("0000 0101 10", 17), + ("0000 0101 01", 18), + ("0000 0101 00", 19), + ("0000 0100 11", 20), + ("0000 0100 10", 21), + ("0000 0100 011", 22), + ("0000 0100 010", 23), + ("0000 0100 001", 24), + ("0000 0100 000", 25), + ("0000 0011 111", 26), + ("0000 0011 110", 27), + ("0000 0011 101", 28), + ("0000 0011 100", 29), + ("0000 0011 011", 30), + ("0000 0011 010", 31), + ("0000 0011 001", 32), + ("0000 0011 000", 33), + ("0000 0001 000", MBA_ESCAPE), + ("0000 0001 111", MBA_STUFFING), +]; + +/// macroblock_type flags. +pub(crate) const MB_QUANT: i16 = 1; +pub(crate) const MB_FWD: i16 = 2; +pub(crate) const MB_BWD: i16 = 4; +pub(crate) const MB_PATTERN: i16 = 8; +pub(crate) const MB_INTRA: i16 = 16; + +/// B.2: I pictures. +pub(crate) const MBTYPE_I: &[(&str, i16)] = &[("1", MB_INTRA), ("01", MB_INTRA | MB_QUANT)]; +/// B.3: P pictures. +pub(crate) const MBTYPE_P: &[(&str, i16)] = &[ + ("1", MB_FWD | MB_PATTERN), + ("01", MB_PATTERN), + ("001", MB_FWD), + ("0001 1", MB_INTRA), + ("0001 0", MB_QUANT | MB_FWD | MB_PATTERN), + ("0000 1", MB_QUANT | MB_PATTERN), + ("0000 01", MB_QUANT | MB_INTRA), +]; +/// B.4: B pictures. +pub(crate) const MBTYPE_B: &[(&str, i16)] = &[ + ("10", MB_FWD | MB_BWD), + ("11", MB_FWD | MB_BWD | MB_PATTERN), + ("010", MB_BWD), + ("011", MB_BWD | MB_PATTERN), + ("0010", MB_FWD), + ("0011", MB_FWD | MB_PATTERN), + ("0001 1", MB_INTRA), + ("0001 0", MB_QUANT | MB_FWD | MB_BWD | MB_PATTERN), + ("0000 11", MB_QUANT | MB_FWD | MB_PATTERN), + ("0000 10", MB_QUANT | MB_BWD | MB_PATTERN), + ("0000 01", MB_QUANT | MB_INTRA), +]; +/// ISO/IEC 11172-2 D pictures (DC intra-coded). +pub(crate) const MBTYPE_D: &[(&str, i16)] = &[("1", MB_INTRA)]; + +/// B.9: coded_block_pattern (0 only in ISO/IEC 13818-2). +pub(crate) const CBP: &[(&str, i16)] = &[ + ("111", 60), + ("1101", 4), + ("1100", 8), + ("1011", 16), + ("1010", 32), + ("1001 1", 12), + ("1001 0", 48), + ("1000 1", 20), + ("1000 0", 40), + ("0111 1", 28), + ("0111 0", 44), + ("0110 1", 52), + ("0110 0", 56), + ("0101 1", 1), + ("0101 0", 61), + ("0100 1", 2), + ("0100 0", 62), + ("0011 11", 24), + ("0011 10", 36), + ("0011 01", 3), + ("0011 00", 63), + ("0010 111", 5), + ("0010 110", 9), + ("0010 101", 17), + ("0010 100", 33), + ("0010 011", 6), + ("0010 010", 10), + ("0010 001", 18), + ("0010 000", 34), + ("0001 1111", 7), + ("0001 1110", 11), + ("0001 1101", 19), + ("0001 1100", 35), + ("0001 1011", 13), + ("0001 1010", 49), + ("0001 1001", 21), + ("0001 1000", 41), + ("0001 0111", 14), + ("0001 0110", 50), + ("0001 0101", 22), + ("0001 0100", 42), + ("0001 0011", 15), + ("0001 0010", 51), + ("0001 0001", 23), + ("0001 0000", 43), + ("0000 1111", 25), + ("0000 1110", 37), + ("0000 1101", 26), + ("0000 1100", 38), + ("0000 1011", 29), + ("0000 1010", 45), + ("0000 1001", 53), + ("0000 1000", 57), + ("0000 0111", 30), + ("0000 0110", 46), + ("0000 0101", 54), + ("0000 0100", 58), + ("0000 0011 1", 31), + ("0000 0011 0", 47), + ("0000 0010 1", 55), + ("0000 0010 0", 59), + ("0000 0001 1", 27), + ("0000 0001 0", 39), + ("0000 0000 1", 0), +]; + +/// B.10: motion_code (-16..=16). +pub(crate) const MOTION: &[(&str, i16)] = &[ + ("0000 0011 001", -16), + ("0000 0011 011", -15), + ("0000 0011 101", -14), + ("0000 0011 111", -13), + ("0000 0100 001", -12), + ("0000 0100 011", -11), + ("0000 0100 11", -10), + ("0000 0101 01", -9), + ("0000 0101 11", -8), + ("0000 0111", -7), + ("0000 1001", -6), + ("0000 1011", -5), + ("0000 111", -4), + ("0001 1", -3), + ("0011", -2), + ("011", -1), + ("1", 0), + ("010", 1), + ("0010", 2), + ("0001 0", 3), + ("0000 110", 4), + ("0000 1010", 5), + ("0000 1000", 6), + ("0000 0110", 7), + ("0000 0101 10", 8), + ("0000 0101 00", 9), + ("0000 0100 10", 10), + ("0000 0100 010", 11), + ("0000 0100 000", 12), + ("0000 0011 110", 13), + ("0000 0011 100", 14), + ("0000 0011 010", 15), + ("0000 0011 000", 16), +]; + +/// B.11: dmvector. +pub(crate) const DMV: &[(&str, i16)] = &[("11", -1), ("0", 0), ("10", 1)]; + +/// B.12: dct_dc_size_luminance. +pub(crate) const DC_LUMA: &[(&str, i16)] = &[ + ("100", 0), + ("00", 1), + ("01", 2), + ("101", 3), + ("110", 4), + ("1110", 5), + ("1111 0", 6), + ("1111 10", 7), + ("1111 110", 8), + ("1111 1110", 9), + ("1111 1111 0", 10), + ("1111 1111 1", 11), +]; + +/// B.13: dct_dc_size_chrominance. +pub(crate) const DC_CHROMA: &[(&str, i16)] = &[ + ("00", 0), + ("01", 1), + ("10", 2), + ("110", 3), + ("1110", 4), + ("1111 0", 5), + ("1111 10", 6), + ("1111 110", 7), + ("1111 1110", 8), + ("1111 1111 0", 9), + ("1111 1111 10", 10), + ("1111 1111 11", 11), +]; + +/// DCT coefficient table values: `run | level << 6`, or these. +pub(crate) const DCT_EOB: i16 = -1; +pub(crate) const DCT_ESCAPE: i16 = -2; + +pub(crate) const fn rl(run: i16, level: i16) -> i16 { + run | (level << 6) +} + +/// Table zero's codes for levels 12-15 at run 0 (table one has shorter ones). +pub(crate) const DCT_ZERO_LONG: &[(&str, i16)] = + &[("0000 0000 1101 0", rl(0, 12)), ("0000 0000 1100 1", rl(0, 13)), ("0000 0000 1100 0", rl(0, 14)), ("0000 0000 1011 1", rl(0, 15))]; + +/// Codes from "0000 0000 1" on, common to tables zero and one (sign bit excluded). +pub(crate) const DCT_LONG: &[(&str, i16)] = &[ + ("0000 0000 1011 0", rl(1, 6)), + ("0000 0000 1010 1", rl(1, 7)), + ("0000 0000 1010 0", rl(2, 5)), + ("0000 0000 1001 1", rl(3, 4)), + ("0000 0000 1001 0", rl(5, 3)), + ("0000 0000 1000 1", rl(9, 2)), + ("0000 0000 1000 0", rl(10, 2)), + ("0000 0000 1111 1", rl(22, 1)), + ("0000 0000 1111 0", rl(23, 1)), + ("0000 0000 1110 1", rl(24, 1)), + ("0000 0000 1110 0", rl(25, 1)), + ("0000 0000 1101 1", rl(26, 1)), + ("0000 0000 0111 11", rl(0, 16)), + ("0000 0000 0111 10", rl(0, 17)), + ("0000 0000 0111 01", rl(0, 18)), + ("0000 0000 0111 00", rl(0, 19)), + ("0000 0000 0110 11", rl(0, 20)), + ("0000 0000 0110 10", rl(0, 21)), + ("0000 0000 0110 01", rl(0, 22)), + ("0000 0000 0110 00", rl(0, 23)), + ("0000 0000 0101 11", rl(0, 24)), + ("0000 0000 0101 10", rl(0, 25)), + ("0000 0000 0101 01", rl(0, 26)), + ("0000 0000 0101 00", rl(0, 27)), + ("0000 0000 0100 11", rl(0, 28)), + ("0000 0000 0100 10", rl(0, 29)), + ("0000 0000 0100 01", rl(0, 30)), + ("0000 0000 0100 00", rl(0, 31)), + ("0000 0000 0011 000", rl(0, 32)), + ("0000 0000 0010 111", rl(0, 33)), + ("0000 0000 0010 110", rl(0, 34)), + ("0000 0000 0010 101", rl(0, 35)), + ("0000 0000 0010 100", rl(0, 36)), + ("0000 0000 0010 011", rl(0, 37)), + ("0000 0000 0010 010", rl(0, 38)), + ("0000 0000 0010 001", rl(0, 39)), + ("0000 0000 0010 000", rl(0, 40)), + ("0000 0000 0011 111", rl(1, 8)), + ("0000 0000 0011 110", rl(1, 9)), + ("0000 0000 0011 101", rl(1, 10)), + ("0000 0000 0011 100", rl(1, 11)), + ("0000 0000 0011 011", rl(1, 12)), + ("0000 0000 0011 010", rl(1, 13)), + ("0000 0000 0011 001", rl(1, 14)), + ("0000 0000 0001 0011", rl(1, 15)), + ("0000 0000 0001 0010", rl(1, 16)), + ("0000 0000 0001 0001", rl(1, 17)), + ("0000 0000 0001 0000", rl(1, 18)), + ("0000 0000 0001 0100", rl(6, 3)), + ("0000 0000 0001 1010", rl(11, 2)), + ("0000 0000 0001 1001", rl(12, 2)), + ("0000 0000 0001 1000", rl(13, 2)), + ("0000 0000 0001 0111", rl(14, 2)), + ("0000 0000 0001 0110", rl(15, 2)), + ("0000 0000 0001 0101", rl(16, 2)), + ("0000 0000 0001 1111", rl(27, 1)), + ("0000 0000 0001 1110", rl(28, 1)), + ("0000 0000 0001 1101", rl(29, 1)), + ("0000 0000 0001 1100", rl(30, 1)), + ("0000 0000 0001 1011", rl(31, 1)), +]; + +/// "0000 0001 xxxx" codes of both tables (table one replaces the six marked `*` with shorter ones). +pub(crate) const DCT_MID_SHARED: &[(&str, i16)] = &[ + ("0000 0001 1100", rl(3, 3)), + ("0000 0001 0010", rl(4, 3)), + ("0000 0001 1110", rl(6, 2)), + ("0000 0001 0101", rl(7, 2)), + ("0000 0001 0001", rl(8, 2)), + ("0000 0001 1111", rl(17, 1)), + ("0000 0001 1010", rl(18, 1)), + ("0000 0001 1001", rl(19, 1)), + ("0000 0001 0111", rl(20, 1)), + ("0000 0001 0110", rl(21, 1)), +]; + +/// B.14: DCT coefficients table zero (the "1s" first-coefficient code is handled by the caller). +pub(crate) const DCT_ZERO: &[(&str, i16)] = &[ + ("10", DCT_EOB), + ("11", rl(0, 1)), + ("011", rl(1, 1)), + ("0100", rl(0, 2)), + ("0101", rl(2, 1)), + ("0010 1", rl(0, 3)), + ("0011 1", rl(3, 1)), + ("0011 0", rl(4, 1)), + ("0001 10", rl(1, 2)), + ("0001 11", rl(5, 1)), + ("0001 01", rl(6, 1)), + ("0001 00", rl(7, 1)), + ("0000 110", rl(0, 4)), + ("0000 100", rl(2, 2)), + ("0000 111", rl(8, 1)), + ("0000 101", rl(9, 1)), + ("0000 01", DCT_ESCAPE), + ("0010 0110", rl(0, 5)), + ("0010 0001", rl(0, 6)), + ("0010 0101", rl(1, 3)), + ("0010 0100", rl(3, 2)), + ("0010 0111", rl(10, 1)), + ("0010 0011", rl(11, 1)), + ("0010 0010", rl(12, 1)), + ("0010 0000", rl(13, 1)), + ("0000 0010 10", rl(0, 7)), + ("0000 0011 00", rl(1, 4)), + ("0000 0010 11", rl(2, 3)), + ("0000 0011 11", rl(4, 2)), + ("0000 0010 01", rl(5, 2)), + ("0000 0011 10", rl(14, 1)), + ("0000 0011 01", rl(15, 1)), + ("0000 0010 00", rl(16, 1)), + ("0000 0001 1101", rl(0, 8)), + ("0000 0001 1000", rl(0, 9)), + ("0000 0001 0011", rl(0, 10)), + ("0000 0001 0000", rl(0, 11)), + ("0000 0001 1011", rl(1, 5)), + ("0000 0001 0100", rl(2, 4)), +]; + +/// B.15: DCT coefficients table one (intra blocks with intra_vlc_format = 1). +pub(crate) const DCT_ONE: &[(&str, i16)] = &[ + ("0110", DCT_EOB), + ("10", rl(0, 1)), + ("010", rl(1, 1)), + ("110", rl(0, 2)), + ("0010 1", rl(2, 1)), + ("0111", rl(0, 3)), + ("0011 1", rl(3, 1)), + ("0001 10", rl(4, 1)), + ("0011 0", rl(1, 2)), + ("0001 11", rl(5, 1)), + ("0000 110", rl(6, 1)), + ("0000 100", rl(7, 1)), + ("1110 0", rl(0, 4)), + ("0000 111", rl(2, 2)), + ("0000 101", rl(8, 1)), + ("1111 000", rl(9, 1)), + ("0000 01", DCT_ESCAPE), + ("1110 1", rl(0, 5)), + ("0001 01", rl(0, 6)), + ("1111 001", rl(1, 3)), + ("0010 0110", rl(3, 2)), + ("1111 010", rl(10, 1)), + ("0010 0001", rl(11, 1)), + ("0010 0101", rl(12, 1)), + ("0010 0100", rl(13, 1)), + ("0001 00", rl(0, 7)), + ("0010 0111", rl(1, 4)), + ("1111 1100", rl(2, 3)), + ("1111 1101", rl(4, 2)), + ("0000 0010 0", rl(5, 2)), + ("0000 0010 1", rl(14, 1)), + ("0000 0011 1", rl(15, 1)), + ("0000 0011 01", rl(16, 1)), + ("1111 011", rl(0, 8)), + ("1111 100", rl(0, 9)), + ("0010 0011", rl(0, 10)), + ("0010 0010", rl(0, 11)), + ("0010 0000", rl(1, 5)), + ("0000 0011 00", rl(2, 4)), + ("1111 1010", rl(0, 12)), + ("1111 1011", rl(0, 13)), + ("1111 1110", rl(0, 14)), + ("1111 1111", rl(0, 15)), +]; + +pub(crate) struct Tables { + pub mba: Vlc, + pub mbtype: [Vlc; 4], + pub cbp: Vlc, + pub motion: Vlc, + pub dmv: Vlc, + pub dc_luma: Vlc, + pub dc_chroma: Vlc, + pub dct_zero: Vlc, + pub dct_one: Vlc, +} + +/// A complete DCT coefficient table: its own short codes plus the shared long ones. +pub(crate) fn dct_table(short: &'static [(&'static str, i16)], extra: &'static [(&'static str, i16)]) -> Vec<(&'static str, i16)> { + let mut v = short.to_vec(); + v.extend_from_slice(extra); + v.extend_from_slice(DCT_MID_SHARED); + v.extend_from_slice(DCT_LONG); + v +} + +pub(crate) fn tables() -> &'static Tables { + static T: OnceLock = OnceLock::new(); + T.get_or_init(|| Tables { + mba: Vlc::new(8, MBA), + mbtype: [Vlc::new(2, MBTYPE_I), Vlc::new(6, MBTYPE_P), Vlc::new(6, MBTYPE_B), Vlc::new(1, MBTYPE_D)], + cbp: Vlc::new(9, CBP), + motion: Vlc::new(8, MOTION), + dmv: Vlc::new(2, DMV), + dc_luma: Vlc::new(9, DC_LUMA), + dc_chroma: Vlc::new(10, DC_CHROMA), + dct_zero: Vlc::new(9, &dct_table(DCT_ZERO, DCT_ZERO_LONG)), + dct_one: Vlc::new(9, &dct_table(DCT_ONE, &[])), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Kraft sum of a code list (2^-len summed). + fn kraft(codes: &[(&str, i16)]) -> f64 { + codes.iter().map(|(c, _)| 0.5f64.powi(parse_code(c).1 as i32)).sum() + } + + #[test] + fn tables_build_without_prefix_clashes() { + let _ = tables(); + } + + #[test] + fn kraft_sums_match_the_reserved_codes() { + // B.1: "0000 0000", "0000 0010" and "0000 0001 001/010/011/100/101/110" are unused + let unused = 2.0 * 0.5f64.powi(8) + 6.0 * 0.5f64.powi(11); + assert!((kraft(MBA) + unused - 1.0).abs() < 1e-12); + assert!((kraft(MBTYPE_I) - 0.75).abs() < 1e-12, "01 is the last I code: 00 is forbidden"); + assert!((kraft(MBTYPE_P) + 0.5f64.powi(6) - 1.0).abs() < 1e-12); + assert!((kraft(MBTYPE_B) + 0.5f64.powi(6) - 1.0).abs() < 1e-12); + // B.9: only "0000 0000 0" is forbidden + assert!((kraft(CBP) + 0.5f64.powi(9) - 1.0).abs() < 1e-12); + // B.10: "0000 0000", "0000 0001" and "0000 0010" are unused + assert!((kraft(MOTION) + 3.0 * 0.5f64.powi(8) - 1.0).abs() < 1e-12); + assert!((kraft(DMV) - 1.0).abs() < 1e-12); + assert!((kraft(DC_LUMA) - 1.0).abs() < 1e-12); + assert!((kraft(DC_CHROMA) - 1.0).abs() < 1e-12); + // table zero is complete except "0000 0000 0000 xxxx" + let z = kraft(&dct_table(DCT_ZERO, DCT_ZERO_LONG)); + assert!((z + 0.5f64.powi(12) - 1.0).abs() < 1e-12, "{z}"); + // table one leaves six 12-bit and four 13-bit codes of table zero unused + let o = kraft(&dct_table(DCT_ONE, &[])); + assert!((o + 0.5f64.powi(12) + 6.0 * 0.5f64.powi(12) + 4.0 * 0.5f64.powi(13) - 1.0).abs() < 1e-12, "{o}"); + } + + #[test] + fn every_value_appears_once() { + let mut seen = [false; 64]; + for &(_, v) in CBP { + assert!(!seen[v as usize]); + seen[v as usize] = true; + } + assert!(seen.iter().all(|&s| s)); + let mut m: Vec = MOTION.iter().map(|x| x.1).collect(); + m.sort(); + assert_eq!(m, (-16..=16).collect::>()); + // motion codes are the address increment codes with a trailing sign bit + for &(c, v) in MOTION { + if v == 0 { + continue; + } + let (code, n) = parse_code(c); + let mba = MBA.iter().find(|(m, _)| parse_code(m) == (code, n)).map(|m| m.1).unwrap(); + assert_eq!(mba / 2, v.abs()); + assert_eq!(mba % 2 == 0, v < 0, "{c}"); + } + // run/level pairs are distinct within each table + for t in [dct_table(DCT_ZERO, DCT_ZERO_LONG), dct_table(DCT_ONE, &[])] { + let mut vals: Vec = t.iter().map(|x| x.1).filter(|&v| v >= 0).collect(); + let n = vals.len(); + vals.sort(); + vals.dedup(); + assert_eq!(vals.len(), n); + assert_eq!(n, 111, "111 run/level pairs besides EOB and escape"); + } + } + + #[test] + fn decodes_codes() { + let t = tables(); + // 0000 0011 000 (33) then 1 (1) then 0000 0001 000 (escape) + let d = [0b0000_0011, 0b0001_0000, 0b0001_0000, 0]; + let mut b = Bits::new(&d); + assert_eq!(t.mba.decode(&mut b), Some(33)); + assert_eq!(t.mba.decode(&mut b), Some(1)); + assert_eq!(t.mba.decode(&mut b), Some(MBA_ESCAPE)); + let d = [0b0000_0000, 0b0001_1111, 0]; + let mut b = Bits::new(&d); + assert_eq!(t.dct_zero.decode(&mut b), Some(rl(27, 1))); + } +} diff --git a/crates/video/tests/mpeg2/common/mod.rs b/crates/video/tests/mpeg2/common/mod.rs new file mode 100644 index 000000000..e39e3e8c7 --- /dev/null +++ b/crates/video/tests/mpeg2/common/mod.rs @@ -0,0 +1,138 @@ +//! Shared helpers for the MPEG video oracle tests. ffmpeg is used only as an external fixture +//! generator and reference decoder (never linked). +#![allow(dead_code)] + +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; + +use craft_video::mpeg2::{ChromaFormat, Decoder, Picture}; + +pub fn fixture_dir() -> PathBuf { + craft_media_testkit::fixtures_dir("mpeg2v") +} + +/// An elementary-stream fixture: a lavfi source encoded by ffmpeg's MPEG-1/2 video encoder. +pub struct Spec { + pub name: &'static str, + pub lavfi: &'static str, + pub frames: u32, + pub enc: &'static [&'static str], + /// ffmpeg rawvideo pixel format of the decoded frames. + pub pix_fmt: &'static str, +} + +/// 1080i 4:2:0 (decode benchmark; Main@High). +pub const HD1080I: Spec = Spec { + name: "m2v_1080i", + lavfi: "testsrc2=size=1920x1080:rate=50,tinterlace=mode=interleave_top,setfield=tff", + frames: 25, + enc: &["-c:v", "mpeg2video", "-flags", "+ilme+ildct", "-bf", "2", "-g", "12", "-b:v", "25M", "-maxrate", "25M", "-bufsize", "9M"], + pix_fmt: "yuv420p", +}; + +pub fn path(spec: &Spec) -> PathBuf { + fixture_dir().join(format!("{}.{}", spec.name, if spec.enc.contains(&"mpeg1video") { "m1v" } else { "m2v" })) +} + +/// Generate (or reuse) the fixture. +pub fn make(ff: &Path, spec: &Spec) -> Option { + let out = path(spec); + craft_media_testkit::fixtures::generate(&out, |tmp| { + let frames = spec.frames.to_string(); + let fmt = if spec.enc.contains(&"mpeg1video") { "mpeg1video" } else { "mpeg2video" }; + let st = Command::new(ff) + .args(["-hide_banner", "-loglevel", "error", "-y", "-f", "lavfi", "-i", spec.lavfi, "-frames:v", &frames]) + .args(spec.enc) + .args(["-f", fmt]) + .arg(tmp) + .stdin(Stdio::null()) + .status(); + matches!(st, Ok(s) if s.success()) + }) +} + +/// ffmpeg's decode of `file` as raw frames. +pub fn ffmpeg_frames(ff: &Path, file: &Path, pix_fmt: &str) -> Vec { + let o = Command::new(ff) + .args(["-hide_banner", "-loglevel", "error", "-i"]) + .arg(file) + .args(["-fps_mode", "passthrough", "-f", "rawvideo", "-pix_fmt", pix_fmt, "-"]) + .stdin(Stdio::null()) + .output() + .expect("run ffmpeg"); + assert!(o.status.success(), "ffmpeg: {}", String::from_utf8_lossy(&o.stderr)); + o.stdout +} + +/// Decode an elementary stream access unit by access unit (pts = index), flushing at the end. +pub fn decode_es(es: &[u8], threads: bool) -> (Vec, Decoder) { + let mut d = Decoder::new(); + d.set_threads(threads); + let mut out = Vec::new(); + for (i, r) in craft_video::mpeg2::access_units(es).into_iter().enumerate() { + out.extend(d.decode(&es[r], i as i64).expect("decode")); + } + out.extend(d.flush()); + (out, d) +} + +/// Per-frame comparison statistics against raw planes. +#[derive(Default, Debug, Clone, Copy)] +pub struct Stats { + pub max: u8, + pub sse: f64, + pub n: u64, + /// Samples differing at all. + pub diff: u64, +} + +impl Stats { + pub fn psnr(&self) -> f64 { + if self.sse == 0.0 { f64::INFINITY } else { 10.0 * (255.0 * 255.0 * self.n as f64 / self.sse).log10() } + } + pub fn add(&mut self, a: &[u8], b: &[u8]) { + assert_eq!(a.len(), b.len(), "plane size"); + for (x, y) in a.iter().zip(b) { + let d = x.abs_diff(*y); + self.max = self.max.max(d); + self.sse += (d as f64) * (d as f64); + self.diff += (d != 0) as u64; + } + self.n += a.len() as u64; + } + pub fn merge(&mut self, o: &Stats) { + self.max = self.max.max(o.max); + self.sse += o.sse; + self.n += o.n; + self.diff += o.diff; + } +} + +/// Compare decoded pictures with ffmpeg's raw frames; returns per-frame stats. +pub fn compare(pics: &[Picture], raw: &[u8]) -> Vec { + let p0 = &pics[0]; + let (w, h) = (p0.width as usize, p0.height as usize); + let (cw, ch) = (p0.chroma_width() as usize, p0.chroma_height() as usize); + let fsize = w * h + 2 * cw * ch; + assert_eq!(raw.len() % fsize, 0, "raw size"); + assert_eq!(raw.len() / fsize, pics.len(), "frame count = ffmpeg's"); + pics.iter() + .enumerate() + .map(|(i, p)| { + let f = &raw[i * fsize..(i + 1) * fsize]; + let mut s = Stats::default(); + s.add(&p.y, &f[..w * h]); + s.add(&p.cb, &f[w * h..w * h + cw * ch]); + s.add(&p.cr, &f[w * h + cw * ch..]); + s + }) + .collect() +} + +pub fn pix_fmt_of(c: ChromaFormat) -> &'static str { + match c { + ChromaFormat::Yuv420 => "yuv420p", + ChromaFormat::Yuv422 => "yuv422p", + ChromaFormat::Yuv444 => "yuv444p", + } +} diff --git a/crates/video/tests/mpeg2/fuzz_hostile.rs b/crates/video/tests/mpeg2/fuzz_hostile.rs new file mode 100644 index 000000000..598565784 --- /dev/null +++ b/crates/video/tests/mpeg2/fuzz_hostile.rs @@ -0,0 +1,199 @@ +//! Hostile-input robustness: truncation at every length, byte/bit mutation and header-field +//! sweeps over a small ffmpeg-generated stream must never panic or run away (errors are fine). +//! Skips when ffmpeg is missing. + +use crate::common; + +use std::panic::{AssertUnwindSafe, catch_unwind}; + +use craft_video::mpeg2::{Decoder, access_units, probe, scan_access_unit}; + +struct Rng(u64); +impl Rng { + fn next(&mut self) -> u64 { + self.0 ^= self.0 << 13; + self.0 ^= self.0 >> 7; + self.0 ^= self.0 << 17; + self.0 + } + fn below(&mut self, n: usize) -> usize { + (self.next() % n.max(1) as u64) as usize + } +} + +const SPECS: [common::Spec; 3] = [ + common::Spec { + name: "m2v_fuzz_seed", + lavfi: "testsrc2=size=64x48:rate=25", + frames: 8, + enc: &["-c:v", "mpeg2video", "-bf", "2", "-g", "6", "-b:v", "500k"], + pix_fmt: "yuv420p", + }, + common::Spec { + name: "m2v_fuzz_seed_interlaced", + lavfi: "testsrc2=size=96x64:rate=25,tinterlace=mode=interleave_top,setfield=tff", + frames: 6, + enc: &["-c:v", "mpeg2video", "-flags", "+ilme+ildct", "-bf", "1", "-g", "6", "-b:v", "800k"], + pix_fmt: "yuv420p", + }, + common::Spec { + name: "m2v_fuzz_seed_422", + lavfi: "testsrc2=size=64x48:rate=25", + frames: 4, + enc: &["-c:v", "mpeg2video", "-pix_fmt", "yuv422p", "-profile:v", "0", "-b:v", "1M"], + pix_fmt: "yuv422p", + }, +]; + +static PANICS: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); + +/// Decoder worker threads catch panics and turn them into errors; count them so they still fail the test. +fn count_panics() { + static ONCE: std::sync::Once = std::sync::Once::new(); + ONCE.call_once(|| { + std::panic::set_hook(Box::new(|info| { + PANICS.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + eprintln!("PANIC: {info}"); + })); + }); +} + +fn run_all(d: &[u8], label: &str) { + craft_media_testkit::set_progress(label); + count_panics(); + let before = PANICS.load(std::sync::atomic::Ordering::SeqCst); + let res = catch_unwind(AssertUnwindSafe(|| { + let _ = probe(d); + let _ = scan_access_unit(d); + for threads in [false, true] { + let mut dec = Decoder::new(); + dec.set_threads(threads); + for (i, r) in access_units(d).into_iter().enumerate() { + let _ = dec.decode(&d[r], i as i64); + } + // also the whole buffer as one unit + let _ = dec.decode(d, 99); + let _ = dec.flush(); + } + })); + assert!(res.is_ok() && PANICS.load(std::sync::atomic::Ordering::SeqCst) == before, "panic: {label}"); +} + +/// Seconds the whole sweep may take before it counts as a hang (override: CRAFT_TEST_TIMEOUT_SECS). +const DEADLINE_SECS: u64 = 600; + +/// CI runs a rotating share of every part of the sweep; `FILMCRAFT_FUZZ_ITERS=2500` runs all of it (see [`Depth`]). +#[test] +fn hostile_streams_never_panic() { + craft_media_testkit::with_deadline(DEADLINE_SECS, sweep); +} + +fn sweep() { + let Some(ff) = craft_media_testkit::ffmpeg_or_skip("mpeg2v fuzz") else { return }; + let depth = Depth::from_env(); + for spec in &SPECS { + if !craft_media_testkit::ffmpeg_has_encoder(&ff, "mpeg2video") { + craft_media_testkit::skip_test("mpeg2v fuzz", &format!("this ffmpeg has no mpeg2video encoder (seed {})", spec.name)); + continue; + } + let Some(path) = common::make(&ff, spec) else { + panic!("ffmpeg has mpeg2video but could not encode the {} seed", spec.name); + }; + fuzz(&depth, spec.name, &std::fs::read(path).unwrap()); + } +} + +fn fuzz(depth: &Depth, name: &str, seed: &[u8]) { + // every truncation length (stride keeps it quick on larger seeds) + for n in depth.truncations(seed.len()) { + run_all(&seed[..n], &format!("{name} truncate {n}")); + } + // header region: every byte to 0 / 0xff / 0x80 / each single bit + for p in 0..seed.len().min(96) { + for (i, v) in VALUES.into_iter().enumerate() { + if !depth.value(p, i) { + continue; + } + let mut d = seed.to_vec(); + d[p] = v; + run_all(&d, &format!("{name} byte {p}={v:#x}")); + } + for b in 0..8 { + if !depth.bit(p, b) { + continue; + } + let mut d = seed.to_vec(); + d[p] ^= 1 << b; + run_all(&d, &format!("{name} flip {p}.{b}")); + } + } + // random multi-mutations + let mut rng = Rng(0x9e3779b97f4a7c15); + // Skipped rounds still draw their mutations, so round i matches round i of a full run. + for i in 0..depth.rounds { + let mut d = seed.to_vec(); + for _ in 0..1 + rng.below(6) { + let p = rng.below(d.len()); + if rng.below(2) == 0 { + d[p] ^= 1 << rng.below(8); + } else { + d[p] = rng.next() as u8; + } + } + if depth.round(i) { + run_all(&d, &format!("{name} random {i}")); + } + } +} + +const VALUES: [u8; 4] = [0, 0xff, 0x80, 0x01]; + +/// How much of the sweep runs. `FILMCRAFT_FUZZ_ITERS=N` runs all of it (about 600 truncation +/// points spread over each seed, every header byte, overwrite value and bit flip) plus N random +/// rounds per seed: 2500 is the full depth, more hunts longer. Unset, CI runs a third of the +/// truncation points and of the full depth's random rounds and, for each header byte, one value +/// and two bits. A rotation offset shifts which share runs (`FILMCRAFT_FUZZ_SEED`, else the UTC +/// day number), so every case of the full sweep runs within a few days, and a random round's +/// mutations match the same round of a full run. +struct Depth { + full: bool, + truncations: usize, + rounds: usize, + rotation: usize, +} + +impl Depth { + fn from_env() -> Self { + match std::env::var("FILMCRAFT_FUZZ_ITERS").ok().and_then(|s| s.parse().ok()) { + Some(rounds) => Depth { full: true, truncations: 600, rounds, rotation: 0 }, + None => { + let rotation = rotation(); + eprintln!("fuzz subset rotation {rotation} (FILMCRAFT_FUZZ_SEED={rotation} replays it; FILMCRAFT_FUZZ_ITERS=2500 runs everything)"); + Depth { full: false, truncations: 600, rounds: 2500, rotation } + } + } + } + /// Truncation lengths for a `len`-byte input: every point of the full sweep, or every third one. + fn truncations(&self, len: usize) -> std::iter::StepBy> { + let stride = (len / self.truncations).max(1); + let (start, step) = if self.full { (0, stride) } else { ((self.rotation % 3) * stride, 3 * stride) }; + (start..len).step_by(step) + } + fn value(&self, p: usize, i: usize) -> bool { + self.full || i == (p + self.rotation % VALUES.len()) % VALUES.len() + } + fn bit(&self, p: usize, b: usize) -> bool { + self.full || (b + p + self.rotation % 4).is_multiple_of(4) + } + fn round(&self, i: usize) -> bool { + self.full || i % 3 == self.rotation % 3 + } +} + +/// The default subset's rotation offset: `FILMCRAFT_FUZZ_SEED`, else days since the Unix epoch (UTC). +fn rotation() -> usize { + std::env::var("FILMCRAFT_FUZZ_SEED").ok().and_then(|s| s.parse().ok()).unwrap_or_else(|| { + let secs = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map_or(0, |d| d.as_secs()); + usize::try_from(secs / 86_400).unwrap_or(0) + }) +} diff --git a/crates/video/tests/mpeg2/hostile_dims.rs b/crates/video/tests/mpeg2/hostile_dims.rs new file mode 100644 index 000000000..1e1788e54 --- /dev/null +++ b/crates/video/tests/mpeg2/hostile_dims.rs @@ -0,0 +1,190 @@ +//! A 14-byte sequence header must not make the decoder size its frame buffers at 16383 x 16383 +//! (gigabytes per picture). Hand-built, no ffmpeg. + +use craft_video::mpeg2::Decoder; + +#[derive(Default)] +struct Bits { + bytes: Vec, + n: usize, +} + +impl Bits { + fn put(&mut self, value: u32, bits: usize) { + for i in (0..bits).rev() { + if self.n.is_multiple_of(8) { + self.bytes.push(0); + } + if (value >> i) & 1 == 1 + && let Some(b) = self.bytes.last_mut() + { + *b |= 0x80 >> (self.n % 8); + } + self.n += 1; + } + } +} + +/// Sequence header plus sequence extension for a `w` x `h` 4:4:4 stream. +fn sequence(w: u32, h: u32, chroma_format: u32) -> Vec { + let mut s = Bits::default(); + s.put(w & 0xfff, 12); + s.put(h & 0xfff, 12); + s.put(1, 4); // aspect_ratio_information + s.put(4, 4); // frame_rate_code + s.put(1000, 18); // bit_rate_value + s.put(1, 1); // marker + s.put(100, 10); // vbv_buffer_size_value + s.put(0, 3); // constrained_parameters_flag, load_intra, load_non_intra + let mut e = Bits::default(); + e.put(1, 4); // sequence_extension + e.put(0x48, 8); // profile_and_level_indication + e.put(1, 1); // progressive_sequence + e.put(chroma_format, 2); + e.put(w >> 12, 2); + e.put(h >> 12, 2); + e.put(0, 12); // bit_rate_extension + e.put(1, 1); // marker + e.put(0, 8 + 1 + 2 + 5); // vbv_buffer_size_extension, low_delay, frame_rate_extension_n/d + let mut out = vec![0, 0, 1, 0xb3]; + out.extend(s.bytes); + out.extend([0, 0, 1, 0xb5]); + out.extend(e.bytes); + out +} + +fn accepted(w: u32, h: u32, chroma_format: u32) -> bool { + let mut dec = Decoder::new(); + let _ = dec.decode(&sequence(w, h, chroma_format), 0); + dec.sequence().is_some() +} + +#[test] +fn ordinary_sequences_are_accepted() { + assert!(accepted(1920, 1080, 1)); + assert!(accepted(7680, 4320, 3)); +} + +#[test] +fn gigantic_sequences_are_refused() { + assert!(!accepted(16383, 16383, 3)); + assert!(!accepted(16383, 16383, 1)); + assert!(!accepted(8192, 8192, 1)); +} + +/// A picture of `coding_type` (1 = I, 2 = P): picture header, coding extension, one slice with a junk payload. +fn picture_of(coding_type: u32) -> Vec { + let mut h = Bits::default(); + h.put(0, 10); // temporal_reference + h.put(coding_type, 3); // picture_coding_type + h.put(0xffff, 16); // vbv_delay + if coding_type == 2 { + h.put(0, 1); // full_pel_forward_vector + h.put(7, 3); // forward_f_code + } + let mut e = Bits::default(); + e.put(8, 4); // picture_coding_extension + e.put(0xffff, 16); // f_codes + e.put(0, 2); // intra_dc_precision + e.put(3, 2); // picture_structure: frame + e.put(0, 10); // flags up to progressive_frame + e.put(1, 1); // progressive_frame + e.put(0, 1); // composite_display_flag + let mut out = vec![0, 0, 1, 0x00]; + out.extend(h.bytes); + out.extend([0, 0, 1, 0xb5]); + out.extend(e.bytes); + out.extend([0, 0, 1, 0x01, 0x80, 0x80]); + out +} + +fn picture() -> Vec { + picture_of(1) +} + +#[test] +fn an_access_unit_of_many_pictures_stops_early() { + let mut data = sequence(64, 64, 1); + for _ in 0..200 { + data.extend(picture()); + } + let mut dec = Decoder::new(); + assert!(dec.decode(&data, 0).is_err(), "200 pictures in one access unit must be refused"); + // pictures after the refusal are not decoded: the per-picture slice errors stay near the output cap + assert!(dec.errors() <= 32, "{} picture errors", dec.errors()); +} + +#[test] +fn a_dimension_change_does_not_hide_an_overflow() { + let mut data = sequence(64, 64, 1); + for _ in 0..20 { + data.extend(picture()); + } + data.extend(sequence(128, 64, 1)); + data.extend(picture()); + let mut dec = Decoder::new(); + assert!(dec.decode(&data, 0).is_err(), "the reset for the new dimensions must not clear the overflow"); +} + +#[test] +fn a_refused_access_unit_leaves_no_stale_references() { + let mut data = sequence(64, 64, 1); + for _ in 0..200 { + data.extend(picture()); + } + let mut dec = Decoder::new(); + assert!(dec.decode(&data, 0).is_err()); + // a P picture cannot continue from the refused access unit's anchor + assert!(dec.decode(&picture_of(2), 1).unwrap().is_empty()); + assert!(dec.flush().is_empty(), "a P picture after the refusal was decoded from stale references"); + // an I picture restarts decoding; nothing from the refused access unit comes out before it + let two = [picture(), picture()].concat(); + assert_eq!(dec.decode(&two, 2).unwrap().len(), 1, "the first I picture is shown when the second arrives"); + assert_eq!(dec.flush().len(), 1); +} + +/// `data` without its extension units (an MPEG-1 stream): everything from a 00 00 01 b5 start code up to the next start code. +fn mpeg1(data: &[u8]) -> Vec { + let mut out = Vec::new(); + let mut i = 0; + while i < data.len() { + if data[i..].starts_with(&[0, 0, 1, 0xb5]) { + i += 4; + while i < data.len() && !data[i..].starts_with(&[0, 0, 1]) { + i += 1; + } + } else { + out.push(data[i]); + i += 1; + } + } + out +} + +#[test] +fn an_mpeg1_d_picture_restarts_decoding_after_a_refused_access_unit() { + let mut data = mpeg1(&sequence(64, 64, 1)); + for _ in 0..200 { + data.extend(mpeg1(&picture())); + } + let mut dec = Decoder::new(); + assert!(dec.decode(&data, 0).is_err()); + let two = [mpeg1(&picture_of(4)), mpeg1(&picture_of(4))].concat(); + assert_eq!(dec.decode(&two, 1).unwrap().len(), 1, "the first D picture is shown when the second arrives"); +} + +/// 352x288 4:2:0 pads to 352x288: 152064 bytes a picture, 13 pictures held. +#[test] +fn sequence_over_the_memory_budget_is_refused() { + let need = 13 * (352 * 288 * 3 / 2); + for (budget, ok) in [(need, true), (need - 1, false)] { + let mut dec = Decoder::new(); + dec.set_memory_budget(budget); + let _ = dec.decode(&sequence(352, 288, 1), 0); + assert_eq!(dec.sequence().is_some(), ok, "budget {budget}"); + assert_eq!(dec.errors(), usize::from(!ok)); + assert_eq!(dec.last_error().is_some(), !ok); + } + // 8K 4:4:4 (13 pictures of about 100 MB) fits the default budget. + assert!(accepted(7680, 4320, 3)); +} diff --git a/crates/video/tests/mpeg2/main.rs b/crates/video/tests/mpeg2/main.rs new file mode 100644 index 000000000..1864b4a22 --- /dev/null +++ b/crates/video/tests/mpeg2/main.rs @@ -0,0 +1,5 @@ +mod common; +mod fuzz_hostile; +mod hostile_dims; +mod oracle; +mod perf; diff --git a/crates/video/tests/mpeg2/oracle.rs b/crates/video/tests/mpeg2/oracle.rs new file mode 100644 index 000000000..14993660c --- /dev/null +++ b/crates/video/tests/mpeg2/oracle.rs @@ -0,0 +1,231 @@ +//! MPEG-1/2 video against ffmpeg: ffmpeg encodes lavfi test patterns into elementary streams, +//! ffmpeg's decode of the same file is the reference. MPEG-2's IDCT is not bit-exact by +//! definition (any IEEE 1180-accurate IDCT conforms), so frames are compared within a tolerance; +//! frame count and display order are exact. + +use crate::common; + +use common::*; + +const SPECS: &[Spec] = &[ + // progressive 4:2:0 IBBP, default VLCs + Spec { + name: "m2v_cif_ibbp", + lavfi: "testsrc2=size=352x288:rate=25", + frames: 30, + enc: &["-c:v", "mpeg2video", "-bf", "2", "-g", "12", "-b:v", "3M"], + pix_fmt: "yuv420p", + }, + // interlaced (frame pictures, field MC + field DCT), top field first, alternate scan + Spec { + name: "m2v_576i_tff", + lavfi: "testsrc2=size=720x576:rate=50,tinterlace=mode=interleave_top,setfield=tff", + frames: 16, + enc: &["-c:v", "mpeg2video", "-flags", "+ilme+ildct", "-alternate_scan", "1", "-bf", "2", "-g", "8", "-b:v", "6M"], + pix_fmt: "yuv420p", + }, + // bottom field first, intra VLC table one, non-linear quantiser, 10-bit DC + Spec { + name: "m2v_480i_bff_vlc1", + lavfi: "testsrc2=size=720x480:rate=60000/1001,tinterlace=mode=interleave_bottom,setfield=bff", + frames: 12, + enc: &[ + "-c:v", + "mpeg2video", + "-flags", + "+ilme+ildct", + "-intra_vlc", + "1", + "-non_linear_quant", + "1", + "-qmax", + "28", + "-dc", + "10", + "-bf", + "1", + "-g", + "6", + "-b:v", + "8M", + ], + pix_fmt: "yuv420p", + }, + // 4:2:2 intra-only, IMX / D-10 style (720x608, 9-bit... DC precision 10, low delay) + Spec { + name: "m2v_imx_422_intra", + lavfi: "testsrc2=size=720x608:rate=25", + frames: 6, + enc: &[ + "-c:v", + "mpeg2video", + "-pix_fmt", + "yuv422p", + "-g", + "1", + "-intra_vlc", + "1", + "-non_linear_quant", + "1", + "-dc", + "10", + "-flags", + "+ildct+low_delay", + "-qmin", + "1", + "-qmax", + "3", + "-b:v", + "30M", + "-minrate", + "30M", + "-maxrate", + "30M", + "-bufsize", + "1200000", + ], + pix_fmt: "yuv422p", + }, + // 4:2:2 long GOP interlaced HD (XDCAM HD422 style) + Spec { + name: "m2v_hd422_longgop", + lavfi: "testsrc2=size=1920x1080:rate=50,tinterlace=mode=interleave_top,setfield=tff", + frames: 8, + enc: &["-c:v", "mpeg2video", "-pix_fmt", "yuv422p", "-flags", "+ilme+ildct", "-bf", "2", "-g", "6", "-b:v", "50M"], + pix_fmt: "yuv422p", + }, + // low quantiser: large levels (escape codes), custom quantiser matrices + Spec { + name: "m2v_escapes_qmatrix", + lavfi: "mandelbrot=size=320x240:rate=25", + frames: 10, + enc: &[ + "-c:v", + "mpeg2video", + "-qscale:v", + "1", + "-qmin", + "1", + "-bf", + "2", + "-g", + "5", + "-intra_matrix", + "8,16,16,16,17,18,21,24,16,16,16,16,17,19,22,25,16,16,17,18,20,22,25,29,16,16,18,21,24,27,31,36,17,17,20,24,30,35,41,47,18,19,22,27,35,44,54,65,21,22,25,31,41,54,70,88,24,25,29,36,47,65,88,115", + "-inter_matrix", + "18,18,18,18,19,21,23,27,18,18,18,18,19,21,24,29,18,18,19,20,22,24,28,32,18,18,20,24,27,30,35,40,19,19,22,27,33,39,46,53,21,21,24,30,39,50,61,73,23,24,28,35,46,61,79,98,27,29,32,40,53,73,98,129", + ], + pix_fmt: "yuv420p", + }, + // no B pictures + Spec { + name: "m2v_ipp", + lavfi: "testsrc2=size=176x144:rate=30", + frames: 20, + enc: &["-c:v", "mpeg2video", "-bf", "0", "-g", "10"], + pix_fmt: "yuv420p", + }, + // MPEG-1 IBBP, and MPEG-1 at q 1 (8/16-bit escapes) + Spec { + name: "m1v_sif", + lavfi: "testsrc2=size=352x240:rate=30000/1001", + frames: 24, + enc: &["-c:v", "mpeg1video", "-bf", "2", "-g", "12", "-b:v", "1500k"], + pix_fmt: "yuv420p", + }, + Spec { + name: "m1v_q1", + lavfi: "mandelbrot=size=192x144:rate=25", + frames: 8, + enc: &["-c:v", "mpeg1video", "-qscale:v", "1", "-qmin", "1", "-bf", "1", "-g", "4"], + pix_fmt: "yuv420p", + }, +]; + +/// Tolerance against ffmpeg (whose integer IDCT differs from our IEEE 1180-accurate one, and the +/// difference propagates through prediction): per frame max |Δ| ≤ 4 and PSNR ≥ 58 dB; overall +/// at most 3 % of samples differ at all. +pub const TOL_MAX: u8 = 4; +pub const MIN_PSNR: f64 = 58.0; +pub const MAX_DIFFERING: f64 = 0.03; + +/// Per displayed frame: (pict_type, interlaced_frame, top_field_first) from ffprobe. +fn ffprobe_frames(file: &std::path::Path) -> Option> { + let fp = craft_media_testkit::ffprobe()?; + let o = std::process::Command::new(fp) + .args(["-v", "error", "-select_streams", "v:0", "-show_entries", "frame=pict_type,interlaced_frame,top_field_first", "-of", "csv=p=0"]) + .arg(file) + .output() + .ok()?; + Some( + String::from_utf8_lossy(&o.stdout) + .lines() + .filter(|l| !l.is_empty()) + .map(|l| { + let v: Vec<&str> = l.split(',').collect(); + (v[0].to_string(), v.get(1) == Some(&"1"), v.get(2) == Some(&"1")) + }) + .collect(), + ) +} + +fn check(spec: &Spec) { + let Some(ff) = craft_media_testkit::ffmpeg_or_skip("mpeg2 oracle") else { return }; + let f = make(&ff, spec).unwrap_or_else(|| panic!("could not generate {}", spec.name)); + let es = std::fs::read(&f).unwrap(); + let (pics, dec) = decode_es(&es, true); + assert_eq!(dec.errors(), 0, "{}: slice errors ({:?})", spec.name, dec.last_error()); + let raw = ffmpeg_frames(&ff, &f, spec.pix_fmt); + let stats = compare(&pics, &raw); + let mut all = Stats::default(); + for (i, s) in stats.iter().enumerate() { + all.merge(s); + assert!(s.max <= TOL_MAX && s.psnr() >= MIN_PSNR, "{} frame {i} ({:?}): max {} psnr {:.2}", spec.name, pics[i].picture_type, s.max, s.psnr()); + } + assert!((all.diff as f64) / (all.n as f64) <= MAX_DIFFERING, "{}: {} of {} samples differ", spec.name, all.diff, all.n); + // display order: picture types, interlacing and field order as ffprobe reports them + if let Some(fr) = ffprobe_frames(&f) { + assert_eq!(fr.len(), pics.len()); + for (i, (p, (t, il, tff))) in pics.iter().zip(&fr).enumerate() { + assert_eq!(format!("{:?}", p.picture_type), *t, "{} frame {i} type", spec.name); + assert_eq!(!p.progressive_frame, *il, "{} frame {i} interlaced", spec.name); + if *il { + assert_eq!(p.top_field_first, *tff, "{} frame {i} field order", spec.name); + } + } + } + println!( + "{}: {} frames, {} ({}), max diff {}, {:.4}% samples differ, PSNR {:.2} dB", + spec.name, + pics.len(), + pics[0].info.codec_name(), + pix_fmt_of(pics[0].chroma), + all.max, + 100.0 * all.diff as f64 / all.n as f64, + all.psnr() + ); + // the single-threaded decode is identical + let (single, _) = decode_es(&es, false); + assert!(single.iter().zip(&pics).all(|(a, b)| a.y == b.y && a.cb == b.cb && a.cr == b.cr)); +} + +#[test] +fn ffmpeg_fixtures_match() { + for s in SPECS { + check(s); + } +} + +#[test] +fn hd_1080i_matches() { + check(&HD1080I); +} + +#[test] +#[ignore] +fn generate_fixtures() { + let Some(ff) = craft_media_testkit::ffmpeg_or_skip("mpeg2v fixtures") else { return }; + for s in SPECS.iter().chain([&HD1080I]) { + craft_media_testkit::generate_and_report(&format!("mpeg2v/{}", s.name), &[path(s)], || make(&ff, s)); + } +} diff --git a/crates/video/tests/mpeg2/perf.rs b/crates/video/tests/mpeg2/perf.rs new file mode 100644 index 000000000..bc5372415 --- /dev/null +++ b/crates/video/tests/mpeg2/perf.rs @@ -0,0 +1,37 @@ +//! Decode speed (ignored; run with `cargo test --release -p craft-video --features mpeg2,threads --test mpeg2 perf:: -- +//! --ignored --nocapture`). Uses the 1080i fixture of `tests/mpeg2/oracle.rs` (ffmpeg-encoded, 25 Mb/s). + +use crate::common; + +use common::*; + +#[test] +#[ignore] +fn perf_1080i_decode_fps() { + let Some(ff) = craft_media_testkit::ffmpeg_or_skip("mpeg2 perf") else { return }; + let f = make(&ff, &HD1080I).expect("fixture"); + let es = std::fs::read(&f).unwrap(); + let aus = craft_video::mpeg2::access_units(&es); + for threads in [false, true] { + let mut best = f64::MAX; + let mut frames = 0; + for _ in 0..5 { + let mut d = craft_video::mpeg2::Decoder::new(); + d.set_threads(threads); + let t0 = std::time::Instant::now(); + let mut n = 0; + for (i, r) in aus.iter().enumerate() { + n += d.decode(&es[r.clone()], i as i64).unwrap().len(); + } + n += d.flush().len(); + best = best.min(t0.elapsed().as_secs_f64()); + frames = n; + } + println!( + "1080i25 MPEG-2 (4:2:0, 25 Mb/s, IBBP): {frames} frames in {:.1} ms = {:.1} fps ({})", + best * 1e3, + frames as f64 / best, + if threads { "slice rows on rayon" } else { "single thread" } + ); + } +}