diff --git a/crates/video/README.md b/crates/video/README.md index 74b8c6414..7de348bb6 100644 --- a/crates/video/README.md +++ b/crates/video/README.md @@ -21,7 +21,7 @@ No codec feature is on by default. | `dnx` | [`dnx`](src/dnx/README.md) | Avid DNxHD / DNxHR (SMPTE ST 2019-1 VC-3): decoder and encoder | | `apv` | [`apv`](src/apv/README.md) | APV (Advanced Professional Video, RFC 9924): decoder (`apv-decode` alone) and encoder (`apv-encode` alone) | | `vp8` | `vp8` | VP8 (RFC 6386): decoder | -| `mpeg2` | [`mpeg2`](src/mpeg2/README.md) | MPEG-2 video (ITU-T H.262) and MPEG-1 video (ISO/IEC 11172-2): decoder (`mpeg2-decode` alone); MPEG-2 encoder with I and P pictures (`mpeg2-encode`, which also turns on the decoder) | +| `mpeg2` | [`mpeg2`](src/mpeg2/README.md) | MPEG-2 video (ITU-T H.262) and MPEG-1 video (ISO/IEC 11172-2): decoder (`mpeg2-decode` alone); MPEG-2 encoder with I, P and B pictures (`mpeg2-encode`, which also turns on the decoder) | | `mpeg4` | [`mpeg4`](src/mpeg4/README.md) | MPEG-4 Part 2 video (ISO/IEC 14496-2, the Xvid/DivX format): Simple profile encoder (`mpeg4-encode`) | | `prores` | `prores` | Apple ProRes (SMPTE RDD 36): decoder and encoder | | `h264` | [`h264`](src/h264/decode/README.md) | ITU-T H.264 / AVC: decoder (`h264-decode` alone) and encoder (`h264-encode` alone) | diff --git a/crates/video/src/lib.rs b/crates/video/src/lib.rs index d9723a81d..0d8625bbd 100644 --- a/crates/video/src/lib.rs +++ b/crates/video/src/lib.rs @@ -8,7 +8,7 @@ //! | `dnx` | [`dnx`] | Avid DNxHD / DNxHR (SMPTE ST 2019-1): decoder and encoder | //! | `apv` | [`apv`] | APV (RFC 9924): decoder (`apv-decode`) and encoder (`apv-encode`) | //! | `vp8` | [`vp8`] | VP8 (RFC 6386): decoder (`vp8-decode`) and key-frame encoder (`vp8-encode`) | -//! | `mpeg2` | [`mpeg2`] | MPEG-2 video (ITU-T H.262) and MPEG-1 video: decoder (`mpeg2-decode`) and MPEG-2 I/P encoder (`mpeg2-encode`) | +//! | `mpeg2` | [`mpeg2`] | MPEG-2 video (ITU-T H.262) and MPEG-1 video: decoder (`mpeg2-decode`) and MPEG-2 I/P/B encoder (`mpeg2-encode`) | //! | `mpeg4` | [`mpeg4`] | MPEG-4 Part 2 video (ISO/IEC 14496-2, Xvid/DivX): Simple profile encoder (`mpeg4-encode`) | //! | `prores` | [`prores`] | Apple ProRes (SMPTE RDD 36): decoder and encoder | //! | `h264` | [`h264`] | ITU-T H.264 / AVC: decoder (`h264-decode`) and encoder (`h264-encode`) | diff --git a/crates/video/src/mpeg2/README.md b/crates/video/src/mpeg2/README.md index b68dc800b..069fd67bd 100644 --- a/crates/video/src/mpeg2/README.md +++ b/crates/video/src/mpeg2/README.md @@ -1,7 +1,7 @@ # MPEG-2 and MPEG-1 video decoder, MPEG-2 encoder (`craft_video::mpeg2`, features `mpeg2-decode`, `mpeg2-encode`) -Clean-room MPEG-2 video and MPEG-1 video decoder, and the first step of an MPEG-2 video encoder -(I and P pictures, see "Encoder"), in pure Rust. Depends only on +Clean-room MPEG-2 video and MPEG-1 video decoder, and the first steps of an MPEG-2 video encoder +(I, P and B pictures, see "Encoder"), in pure Rust. Depends only on `thiserror` and (optional, feature `threads`) `rayon`. Builds for `wasm32-unknown-unknown`; no `unsafe`. No GPL/LGPL code (FFmpeg, libmpeg2, libavcodec) was consulted; FFmpeg is used only as an external fixture generator and test oracle. @@ -77,8 +77,8 @@ the synthetic-stream tests confirmed against FFmpeg: ## Encoder (feature `mpeg2-encode`) -The encoder is the first step of MPEG-2 encoding in this crate, tracked in -storytold/craft-libs#1133. It writes valid MPEG-2 elementary streams, but not yet DVD or +The encoder is built in steps, tracked in storytold/craft-libs#1133: I and P pictures came in +#1136, B pictures in the second step. It writes valid MPEG-2 elementary streams, but not yet DVD or broadcast deliverables: those need the later steps listed below. ```rust @@ -86,13 +86,15 @@ use craft_video::mpeg2::{Encoder, EncoderConfig, RateControl, YuvFrame}; let mut cfg = EncoderConfig::new(720, 576, 25, 1); cfg.rate = RateControl::Abr { kbps: 6000 }; // or RateControl::Quantiser(1..=31) -cfg.gop_size = 12; // one I picture, then P pictures +cfg.gop_size = 12; // pictures from one I picture to the next +cfg.b_pictures = 2; // I B B P B B P ...; 0 for I and P only +cfg.closed_gop = true; // or open GOPs let mut enc = Encoder::new(cfg)?; for (i, f) in frames.iter().enumerate() { let frame = YuvFrame { y: &f.y, cb: &f.cb, cr: &f.cr, y_stride: 720, c_stride: 360 }; for p in enc.encode(&frame, i as i64)? { /* p.data, p.pts, p.dts, p.keyframe, p.picture_type */ } } -for p in enc.flush() { /* none yet: no B pictures */ } +for p in enc.flush() { /* the pictures held for reordering */ } es.extend_from_slice(enc.end_of_sequence()); ``` @@ -101,9 +103,18 @@ What it does: - Main Profile (4:2:0) at Low, Main, High-1440 and High Level, and 4:2:2 Profile at Main and High Level. The lowest level that holds the size, frame rate and (average) bitrate is chosen; sizes beyond High Level are refused. Any even picture size; edges are padded by repetition. -- Progressive frame pictures, I and P, in closed GOPs. Every GOP starts with a sequence header, - sequence extension and GOP header (with a time code), so every I picture is a random access - point. Packets come out in decoding order, which equals display order without B pictures. +- Progressive frame pictures, I, P and B. Every GOP starts with a sequence header, sequence + extension and GOP header (with a time code), so every I picture is a random access point. +- B pictures (0-4 between anchors) and reordering: pictures go in in display order and packets + come out in decoding order; a B picture is held until the anchor after it is coded, and + `flush` codes the last held picture as a P picture. With B pictures, `dts` is the presentation + times in decoding order one picture interval earlier, so presentation times must step evenly. + A single picture flushed before a second one gets `dts = pts - 1`. With average bitrate, a + GOP that `flush` cuts short gives back the budget of the pictures it did not get. + `low_delay` is set only without B pictures. +- GOPs are closed or open. The B pictures shown before a GOP's I picture (coded after it) predict + backward only in a closed GOP, and from the previous GOP's last anchor too in an open one. + temporal_reference counts display order from the GOP's first picture. - Forward DCT on the transpose of the decoder's IDCT basis; intra quantisation with a small dead zone, non-intra quantisation by truncation; default or custom quantiser matrices (sequence header); linear or non-linear quantiser scale; intra DC precision 8-10 bits; @@ -112,9 +123,14 @@ What it does: predictor, the previous picture's vectors), then half-sample refinement; cost SAD + λ·bits. Each macroblock is coded intra, motion-compensated with or without residual, "no MC" with residual, or skipped. The search range sets f_code (1-4). +- B pictures: a search per direction (candidates include the P picture's vectors scaled to the + B picture's distance), then forward, backward, interpolated or the skip prediction (the + previous macroblock's directions with the predictors), whichever costs least, or intra. + Interpolated predictions are averaged in the decoder's order. - Adaptive quantisation by macroblock activity (Test Model 5 §5.2.3). -- Rate control: a constant quantiser, or single-pass average bitrate with Test Model 5's - target bit allocation at picture level. Streams are signalled as variable bitrate +- Rate control: a constant quantiser (B pictures at 1.25 times it), or single-pass average + bitrate with Test Model 5's target bit allocation at picture level (B pictures weighted by + K_b = 1.4). Streams are signalled as variable bitrate (`vbv_delay` 0xFFFF, the level's maximum bitrate and VBV size in the sequence header); the VBV buffer is not modelled. - The reconstruction uses the decoder's own dequantisation, mismatch control, IDCT and motion @@ -122,7 +138,7 @@ What it does: - Slices (one per macroblock row) are coded in parallel on rayon with `threads`; the stream is identical either way. -Later steps (#1133): B pictures and reordering; CBR and VBR with a VBV guarantee; DVD presets +Later steps (#1133): CBR and VBR with a VBV guarantee; DVD presets (sizes, aspect, GOP length, 9.8 Mb/s, 23.976 soft telecine); interlaced coding (field/frame DCT, field prediction, alternate scan) and 4:2:2 High Level CBR for XDCAM HD422. MPEG-2 essence in the MXF writer and a program stream muxer belong to `craft-media`. @@ -191,9 +207,15 @@ fell to 77–150 fps). These are lower bounds. rate control converges on its target. `encode/tests.rs`: our decoder outputs exactly the encoder's reconstruction (intra only, a 30-picture P GOP, 4:2:2, linear and non-linear quantisers, quantisers 1 and 31, DC precision 8 and 10, custom matrices, odd macroblock counts, - average bitrate); quality and size follow the quantiser; repeated pictures skip; levels and - aspect in the sequence header; bad configurations and short or inconsistent planes are refused. -- `tests/mpeg2/encode.rs`: FFmpeg decodes our streams of `testsrc2` (CIF and 576-line 4:2:2) - without errors and within the decoder suite's tolerance of our reconstruction, ffprobe reports - the profile, level, pixel format and I/P picture types, and average bitrate lands within 10 % of - 800 and 2500 kbit/s targets. + average bitrate, each with and without B pictures; 1-4 B pictures with closed and open GOPs, + GOP lengths that are and are not multiples of the anchor distance, and streams that end inside + a group of B pictures); decoding order, decoding times and temporal_reference; a closed GOP + decodes from its I picture with its leading B pictures and an open one without them; B pictures + are smaller than P pictures; uneven presentation times are refused with B pictures; quality and + size follow the quantiser; repeated pictures skip; levels and aspect in the sequence header; + bad configurations and short or inconsistent planes are refused. +- `tests/mpeg2/encode.rs`: FFmpeg decodes our streams of `testsrc2` (CIF and 576-line 4:2:2; + without B pictures, with 2 and 4, closed and open GOPs) without errors and within the decoder + suite's tolerance of our reconstruction, ffprobe reports the profile, level, pixel format and + picture types in display order, and average bitrate lands within 10 % of 800 and 2500 kbit/s + targets. diff --git a/crates/video/src/mpeg2/encode/mod.rs b/crates/video/src/mpeg2/encode/mod.rs index fcd1bd8d1..af32b271d 100644 --- a/crates/video/src/mpeg2/encode/mod.rs +++ b/crates/video/src/mpeg2/encode/mod.rs @@ -1,9 +1,9 @@ //! MPEG-2 video encoder (ITU-T H.262 | ISO/IEC 13818-2), feature `mpeg2-encode`. //! -//! This is the first step of MPEG-2 encoding in this crate (storytold/craft-libs#1133): I and P -//! pictures, progressive frame pictures, 4:2:0 (Main Profile) and 4:2:2 (4:2:2 Profile), closed -//! GOPs, constant quantiser or single-pass average bitrate. B pictures, a VBV guarantee, -//! interlaced coding and DVD / broadcast presets come later. +//! MPEG-2 encoding in this crate is built in steps (storytold/craft-libs#1133). So far: I, P and B +//! pictures, progressive frame pictures, 4:2:0 (Main Profile) and 4:2:2 (4:2:2 Profile), closed or +//! open GOPs, constant quantiser or single-pass average bitrate. A VBV guarantee, interlaced +//! coding and DVD / broadcast presets come later. //! //! ```no_run //! use craft_video::mpeg2::encode::{Encoder, EncoderConfig, RateControl, YuvFrame}; @@ -36,6 +36,8 @@ mod transform; #[cfg(test)] mod tests; +use std::collections::VecDeque; + use crate::mpeg2::ChromaFormat; use crate::mpeg2::headers::{DEFAULT_INTRA, PictureType, SCAN}; use crate::mpeg2::mc::Frame; @@ -50,6 +52,10 @@ pub enum EncodeError { Config(String), #[error("bad input frame: {0}")] BadFrame(String), + /// With B pictures, presentation times must increase by a constant step (the decoding times + /// are the presentation times one step earlier). + #[error("bad presentation time: {0}")] + BadTimestamp(String), } /// aspect_ratio_information (H.262 Table 6-3). @@ -97,8 +103,15 @@ pub struct EncoderConfig { /// 4:2:0 (Main Profile) or 4:2:2 (4:2:2 Profile). pub chroma: ChromaFormat, pub aspect: AspectRatio, - /// Pictures per GOP: one I picture, then P pictures. 1 codes every picture as I. 1..=1023. + /// Pictures from one I picture to the next (display order). 1 codes every picture as I. + /// 1..=1023, and at most 1024 with `b_pictures` added. pub gop_size: u32, + /// B pictures between two anchor (I or P) pictures, 0..=4. 0 gives I and P pictures only and a + /// low-delay stream (no reordering). + pub b_pictures: u32, + /// Closed GOPs: the B pictures shown before a GOP's I picture predict from it alone, so the + /// GOP decodes on its own. Open GOPs let them predict from the previous GOP too. + pub closed_gop: bool, pub rate: RateControl, /// q_scale_type: the non-linear quantiser scale (finer steps at low quantisers). pub nonlinear_quant: bool, @@ -125,6 +138,8 @@ impl EncoderConfig { chroma: ChromaFormat::Yuv420, aspect: AspectRatio::Square, gop_size: 12, + b_pictures: 2, + closed_gop: true, rate: RateControl::Quantiser(4), nonlinear_quant: true, adaptive_quant: true, @@ -155,9 +170,12 @@ pub struct Packet { /// the picture. pub data: Vec, pub pts: i64, - /// Decoding timestamp; equal to `pts` while there are no B pictures. + /// Decoding timestamp: `pts` without B pictures; with them, the presentation times in + /// decoding order one picture interval earlier, so `dts <= pts`. A stream of one picture has + /// no interval: its `dts` is `pts - 1`. pub dts: i64, - /// An I picture starting a closed GOP (random access point). + /// An I picture (random access point; in an open GOP the B pictures after it in decoding + /// order also need the previous GOP). pub keyframe: bool, pub picture_type: PictureType, /// Average quantiser_scale of the picture's macroblocks. @@ -260,6 +278,13 @@ fn validate(cfg: &EncoderConfig) -> Result<(), EncodeError> { if !(1..=1023).contains(&cfg.gop_size) { return bad(format!("GOP size {} (1..=1023)", cfg.gop_size)); } + if cfg.b_pictures > 4 { + return bad(format!("{} B pictures between anchors (0..=4)", cfg.b_pictures)); + } + // temporal_reference (10 bits) counts a GOP's leading B pictures too + if cfg.gop_size + cfg.b_pictures > 1024 { + return bad(format!("GOP size {} with {} B pictures: more than 1024 pictures to number", cfg.gop_size, cfg.b_pictures)); + } match cfg.rate { RateControl::Quantiser(q) if !(1..=31).contains(&q) => return bad(format!("quantiser_scale_code {q} (1..=31)")), RateControl::Abr { kbps: 0 } => return bad("bitrate 0".into()), @@ -284,6 +309,26 @@ fn put_matrix(w: &mut BitWriter, m: &[u8; 64]) { } } +/// A source picture waiting for the anchor after it (a B picture). +struct Held { + src: Frame, + pts: i64, + /// Display index. + n: u64, +} + +/// A coded picture before rate control and timestamps. +struct Coded { + packet: Packet, + recon: Frame, + mvs: Vec<[i32; 2]>, + avg_q: f64, +} + +/// Quantiser ratio of B to P pictures with a constant quantiser (TM5 uses 1.4 for its rate +/// allocation; a constant quantiser keeps B pictures a little closer to the anchors). +const B_QUANT_FACTOR: f64 = 1.25; + /// A stateful encoder. Feed pictures in display order; packets come out in decoding order. pub struct Encoder { cfg: EncoderConfig, @@ -297,16 +342,32 @@ pub struct Encoder { inter_matrix: [u8; 64], /// Sequence header and sequence extension, sent before every GOP. seq_header: Vec, - /// The reconstruction of the previous picture. - reference: Option, - prev_mvs: Vec<[i32; 2]>, - /// Pictures coded so far. + /// The reconstructions of the last two anchor pictures and their display indices: `latest` + /// is the backward reference of the B pictures held, and the forward reference of the next + /// P picture. + older: Option<(Frame, u64)>, + latest: Option<(Frame, u64)>, + /// Forward vectors of the latest anchor when it is a P picture (search candidates). + anchor_mvs: Vec<[i32; 2]>, + /// B pictures waiting for their backward reference. + held: Vec, + /// Display index of the next picture. count: u64, - /// Position of the next picture in its GOP. - gop_pos: u32, + /// Display indices of the current GOP's I picture and of its first picture in display order + /// (a leading B picture in front of the I picture, or the I picture). + gop_start: u64, + gop_first: u64, + /// The current GOP is closed: its leading B pictures predict backward only. + gop_closed: bool, + /// Presentation times not yet given to a coded picture as its decoding time, in input order. + display_pts: VecDeque, + last_pts: Option, + pts_step: Option, + /// Packets waiting for the presentation time step (the decoding time offset). + out: Vec, rc: Option, recon_capture: bool, - recon: Vec, + recon: Vec<(u64, ReconFrame)>, } impl Encoder { @@ -330,10 +391,18 @@ impl Encoder { intra_matrix: cfg.intra_matrix.unwrap_or(DEFAULT_INTRA), inter_matrix: cfg.non_intra_matrix.unwrap_or([16; 64]), seq_header: Vec::new(), - reference: None, - prev_mvs: Vec::new(), + older: None, + latest: None, + anchor_mvs: Vec::new(), + held: Vec::new(), count: 0, - gop_pos: 0, + gop_start: 0, + gop_first: 0, + gop_closed: true, + display_pts: VecDeque::new(), + last_pts: None, + pts_step: None, + out: Vec::new(), rc, recon_capture: false, recon: Vec::new(), @@ -367,13 +436,32 @@ impl Encoder { self.recon_capture = on; } + /// The reconstructions captured so far, in display order. pub fn take_recon(&mut self) -> Vec { - std::mem::take(&mut self.recon) + let mut r = std::mem::take(&mut self.recon); + r.sort_by_key(|(n, _)| *n); + r.into_iter().map(|(_, f)| f).collect() } - /// Pictures held for reordering (none yet: there are no B pictures). + /// Code the pictures held for reordering (the last of them becomes a P picture) and return + /// every packet not yet returned. The next picture starts a new GOP. pub fn flush(&mut self) -> Vec { - Vec::new() + let mut out = Vec::new(); + if let Some(last) = self.held.pop() { + self.code_anchor(last, PictureType::P, &mut out); + } + // the GOP ends here: its budget covered `gop_size` pictures, so take back the share of + // those that never came, or every flushed GOP would raise the bitrate of the next + if self.latest.is_some() { + let missing = u64::from(self.cfg.gop_size).saturating_sub(self.count - self.gop_start); + if let Some(rc) = self.rc.as_mut() { + rc.end_gop(missing as u32); + } + } + self.older = None; + self.latest = None; + self.anchor_mvs.clear(); + self.release(out, true) } fn sequence_header_bytes(&self) -> Vec { @@ -406,16 +494,15 @@ impl Encoder { w.put((bit_rate >> 18) as u32 & 0xFFF, 12); w.bit(true); // marker_bit w.put((vbv >> 10) as u32 & 0xFF, 8); - w.bit(true); // low_delay: no B pictures + w.bit(c.b_pictures == 0); // low_delay: no B pictures, no reordering w.put(0, 2); // frame_rate_extension_n w.put(0, 5); // frame_rate_extension_d w.finish() } - /// group_of_pictures_header with the time code of picture `self.count`. - fn gop_header(&self, w: &mut BitWriter) { + /// group_of_pictures_header with the time code of display picture `n`. + fn gop_header(&self, w: &mut BitWriter, n: u64, closed: bool) { let fps = u64::from(self.cfg.fps_num.div_ceil(self.cfg.fps_den)).max(1); - let n = self.count; let secs = n / fps; w.start_code(0xB8); w.bit(false); // drop_frame_flag @@ -424,24 +511,40 @@ impl Encoder { w.bit(true); // marker_bit w.put((secs % 60) as u32, 6); w.put((n % fps) as u32, 6); - w.bit(true); // closed_gop + w.bit(closed); // closed_gop w.bit(false); // broken_link } - fn picture_header(&self, w: &mut BitWriter, intra: bool) { + fn picture_header(&self, w: &mut BitWriter, ptype: PictureType, temporal_reference: u64) { w.start_code(0x00); - w.put(self.gop_pos & 0x3FF, 10); // temporal_reference: display order within the GOP - w.put(if intra { 1 } else { 2 }, 3); + w.put((temporal_reference & 0x3FF) as u32, 10); // display order within the GOP + w.put( + match ptype { + PictureType::P => 2, + PictureType::B => 3, + _ => 1, + }, + 3, + ); w.put(0xFFFF, 16); // vbv_delay: variable bitrate - if !intra { - w.bit(false); // full_pel_forward_vector - w.put(7, 3); // forward_f_code (MPEG-2: in the extension) + for dir in [PictureType::P, PictureType::B] { + if ptype == dir || ptype == PictureType::B { + // full_pel_{forward,backward}_vector and {forward,backward}_f_code: MPEG-2 + // carries them in the extension + w.bit(false); + w.put(7, 3); + } } w.bit(false); // extra_bit_picture w.start_code(0xB5); w.put(8, 4); // picture coding extension - let fc = if intra { 15 } else { u32::from(self.f_code) }; - for v in [fc, fc, 15, 15] { + let fc = u32::from(self.f_code); + let f_codes = match ptype { + PictureType::P => [fc, fc, 15, 15], + PictureType::B => [fc; 4], + _ => [15; 4], + }; + for v in f_codes { w.put(v, 4); } w.put(u32::from(self.cfg.intra_dc_bits - 8), 2); @@ -502,28 +605,113 @@ impl Encoder { acts.iter().map(|&a| qcode_for(nl, base * (2.0 * a + avg) / (a + 2.0 * avg))).collect() } - /// Encode one picture. Returns the packets that became ready (one per picture for now). + /// With B pictures, presentation times must step by a constant amount, and the decoding + /// times (one step earlier) must fit in an `i64`. Checked before any state changes. + fn check_pts(&self, pts: i64) -> Result, EncodeError> { + if self.cfg.b_pictures == 0 { + return Ok(None); + } + let no_dts = |pts: i64, step: i64| EncodeError::BadTimestamp(format!("{pts}: no decoding time {step} earlier")); + let Some(last) = self.last_pts else { + // flushed before a second picture, this one decodes at `pts - 1` (see `release`) + return if pts == i64::MIN { Err(no_dts(pts, 1)) } else { Ok(None) }; + }; + let step = pts.checked_sub(last).filter(|&d| d > 0).ok_or_else(|| EncodeError::BadTimestamp(format!("{pts} after {last}")))?; + match self.pts_step { + Some(s) if s != step => Err(EncodeError::BadTimestamp(format!("{pts} after {last}: step {step}, earlier {s}"))), + // the earliest decoding time is the earlier picture's time less the step + None if last.checked_sub(step).is_none() => Err(no_dts(last, step)), + _ => Ok(Some(step)), + } + } + + /// Encode one picture (display order). Returns the packets that became ready, in decoding + /// order: with B pictures, a B picture is held until the anchor after it is coded. pub fn encode(&mut self, frame: &YuvFrame, pts: i64) -> Result, EncodeError> { let src = self.load(frame)?; - let reference = if self.gop_pos == 0 { None } else { self.reference.take() }; - let intra = reference.is_none(); + if let Some(step) = self.check_pts(pts)? { + self.pts_step = Some(step); + } + self.last_pts = Some(pts); + let n = self.count; + self.count += 1; + self.display_pts.push_back(pts); + let held = Held { src, pts, n }; + let mut out = Vec::new(); + let m = u64::from(self.cfg.b_pictures) + 1; + if self.latest.is_none() || n - self.gop_start >= u64::from(self.cfg.gop_size) { + self.code_anchor(held, PictureType::I, &mut out); + } else if (n - self.gop_start).is_multiple_of(m) { + self.code_anchor(held, PictureType::P, &mut out); + } else { + self.held.push(held); + } + Ok(self.release(out, false)) + } + + /// Code an anchor picture, then the B pictures held in front of it. + fn code_anchor(&mut self, h: Held, ptype: PictureType, out: &mut Vec) { + let intra = ptype == PictureType::I; if intra { - self.gop_pos = 0; + self.gop_start = h.n; + self.gop_first = self.held.first().map_or(h.n, |b| b.n); + self.gop_closed = self.cfg.closed_gop || self.held.is_empty(); + let n = u64::from(self.cfg.gop_size); + let m = u64::from(self.cfg.b_pictures) + 1; + let np = (n - 1) / m; if let Some(rc) = self.rc.as_mut() { - rc.start_gop(self.cfg.gop_size); + rc.start_gop(np as u32, (n - 1 - np) as u32); } } + let fwd = if intra { None } else { self.latest.as_ref().map(|r| &r.0) }; + let cand: &[[i32; 2]] = if intra { &[] } else { &self.anchor_mvs }; + let c = self.code_picture(&h, ptype, (fwd, None), [cand, &[]]); + let (recon, mvs) = self.finish(c, h.n, out); + self.older = self.latest.take(); + self.latest = Some((recon, h.n)); + self.anchor_mvs = if intra { Vec::new() } else { mvs }; + for b in std::mem::take(&mut self.held) { + self.code_b(b, out); + } + } + + /// Code a B picture between the two latest anchors. The leading B pictures of a closed GOP + /// (shown before its I picture) predict backward only. + fn code_b(&mut self, b: Held, out: &mut Vec) { + let Some((bwd, bn)) = self.latest.as_ref() else { return }; + let leading = b.n < self.gop_start; + let fwd = self.older.as_ref().filter(|_| !(leading && self.gop_closed)); + // search candidates: the latest anchor's vectors (which span the two anchors) scaled to + // this picture's distance from each + let span = self.older.as_ref().map_or(1, |o| bn.saturating_sub(o.1).max(1)) as i64; + let scaled = |d: i64| -> Vec<[i32; 2]> { + self.anchor_mvs.iter().map(|v| [(i64::from(v[0]) * d / span) as i32, (i64::from(v[1]) * d / span) as i32]).collect() + }; + let cf = fwd.map_or(Vec::new(), |f| scaled(b.n as i64 - f.1 as i64)); + let cb = scaled(b.n as i64 - *bn as i64); + let c = self.code_picture(&b, PictureType::B, (fwd.map(|f| &f.0), Some(bwd)), [&cf, &cb]); + self.finish(c, b.n, out); + } + + /// Code one picture with the given references and search candidates. + fn code_picture(&self, h: &Held, ptype: PictureType, (fwd, bwd): (Option<&Frame>, Option<&Frame>), cand: [&[[i32; 2]]; 2]) -> Coded { let base = match (&self.rc, self.cfg.rate) { - (Some(rc), _) => rc.qscale(intra), - (None, RateControl::Quantiser(q)) => f64::from(qscale(self.cfg.nonlinear_quant, q)), + (Some(rc), _) => rc.qscale(ptype), + (None, RateControl::Quantiser(q)) => { + let qs = f64::from(qscale(self.cfg.nonlinear_quant, q)); + if ptype == PictureType::B { qs * B_QUANT_FACTOR } else { qs } + } (None, _) => 8.0, }; - let mb_qcode = self.mb_quant(&src, base); + let src = &h.src; + let mb_qcode = self.mb_quant(src, base); let mut recon = Frame::new(src.w, src.h, src.cw, src.ch); let ctx = PicCtx { - src: &src, - reference: reference.as_ref(), - prev_mvs: &self.prev_mvs, + ptype, + src, + fwd, + bwd, + cand, mb_w: self.mb_w, shx: self.shx, shy: self.shy, @@ -536,11 +724,12 @@ impl Encoder { mb_qcode: &mb_qcode, }; let rows = encode_rows(&ctx, &mut recon, self.cfg.threads); + let intra = ptype == PictureType::I; let mut w = BitWriter::new(); if intra { - self.gop_header(&mut w); + self.gop_header(&mut w, self.gop_first, self.gop_closed); } - self.picture_header(&mut w, intra); + self.picture_header(&mut w, ptype, h.n - self.gop_first); let mut data = if intra { self.seq_header.clone() } else { Vec::new() }; data.extend_from_slice(&w.finish()); let mut qsum = 0u64; @@ -551,18 +740,47 @@ impl Encoder { mvs.extend_from_slice(&r.mvs); } let avg_q = qsum as f64 / (self.mb_w * self.mb_h).max(1) as f64; + let packet = Packet { data, pts: h.pts, dts: h.pts, keyframe: intra, picture_type: ptype, quantiser_scale: avg_q as f32 }; + Coded { packet, recon, mvs, avg_q } + } + + /// Account for a coded picture (display index `n`): rate control, its decoding time, the + /// reconstruction capture. Returns the reconstruction and the picture's vectors. + fn finish(&mut self, c: Coded, n: u64, out: &mut Vec) -> (Frame, Vec<[i32; 2]>) { + let Coded { mut packet, recon, mvs, avg_q } = c; if let Some(rc) = self.rc.as_mut() { - rc.update(intra, data.len() as u64 * 8, avg_q); + rc.update(packet.picture_type, packet.data.len() as u64 * 8, avg_q); } + // the k-th coded picture decodes at the k-th presentation time (less the offset that + // `release` subtracts); one time is queued per input picture + packet.dts = self.display_pts.pop_front().unwrap_or(packet.pts); if self.recon_capture { - self.recon.push(self.crop(&recon)); + self.recon.push((n, self.crop(&recon))); } - self.reference = Some(recon); - self.prev_mvs = mvs; - self.count += 1; - self.gop_pos = (self.gop_pos + 1) % self.cfg.gop_size; - let picture_type = if intra { PictureType::I } else { PictureType::P }; - Ok(vec![Packet { data, pts, dts: pts, keyframe: intra, picture_type, quantiser_scale: avg_q as f32 }]) + out.push(packet); + (recon, mvs) + } + + /// Hand out packets once their decoding times are known: with B pictures they are the + /// presentation times one step earlier, so packets wait for the second picture's time. + fn release(&mut self, out: Vec, flushing: bool) -> Vec { + self.out.extend(out); + if self.cfg.b_pictures == 0 { + return std::mem::take(&mut self.out); + } + if self.pts_step.is_none() && !flushing { + return Vec::new(); + } + // A lone picture flushed before a second one set the step decodes one unit early, the + // smallest step there is: if encoding goes on, the next picture decodes at this one's + // presentation time whatever the step turns out to be, and that must be later. + let off = self.pts_step.unwrap_or(1); + let mut res = std::mem::take(&mut self.out); + for p in &mut res { + // `check_pts` refused the times this would not fit for + p.dts = p.dts.saturating_sub(off); + } + res } fn crop(&self, f: &Frame) -> ReconFrame { diff --git a/crates/video/src/mpeg2/encode/motion.rs b/crates/video/src/mpeg2/encode/motion.rs index a7bedc80b..8242faafe 100644 --- a/crates/video/src/mpeg2/encode/motion.rs +++ b/crates/video/src/mpeg2/encode/motion.rs @@ -47,7 +47,7 @@ impl Search<'_> { } /// A half-sample vector the bitstream can carry whose prediction stays inside the reference. - fn valid(&self, mv: [i32; 2]) -> bool { + pub(crate) fn valid(&self, mv: [i32; 2]) -> bool { let f = 1i32 << self.f_code.saturating_sub(1); let (ox, oy) = self.origin(); let size = [self.reference.w as i32, self.reference.h as i32]; @@ -58,7 +58,8 @@ impl Search<'_> { }) } - fn mv_cost(&self, mv: [i32; 2]) -> u32 { + /// λ·(bits of `mv` against the predictor). + pub(crate) fn mv_cost(&self, mv: [i32; 2]) -> u32 { let t = enc_tables(); let bits: u32 = (0..2).map(|c| t.motion_bits(wrap_delta(mv[c], self.pmv[c], self.f_code), self.f_code)).sum(); self.lambda * bits diff --git a/crates/video/src/mpeg2/encode/picture.rs b/crates/video/src/mpeg2/encode/picture.rs index e7bb84b7e..0be5dbafd 100644 --- a/crates/video/src/mpeg2/encode/picture.rs +++ b/crates/video/src/mpeg2/encode/picture.rs @@ -1,11 +1,11 @@ //! Slices and macroblocks of one frame picture (H.262 §6.2.4-§6.2.6), with a reconstruction //! that matches the decoder exactly. One slice per macroblock row; slices only depend on the -//! reference picture, so rows can be coded in parallel. +//! reference pictures, so rows can be coded in parallel. -use crate::mpeg2::headers::SCAN; +use crate::mpeg2::headers::{PictureType, SCAN}; use crate::mpeg2::mc::{Block, Frame, predict}; use crate::mpeg2::slice::NON_LINEAR_Q; -use crate::mpeg2::vlc::{MB_FWD, MB_INTRA, MB_PATTERN, MB_QUANT}; +use crate::mpeg2::vlc::{MB_BWD, MB_FWD, MB_INTRA, MB_PATTERN, MB_QUANT}; use super::bitwriter::BitWriter; use super::motion::Search; @@ -35,12 +35,17 @@ pub(crate) fn qcode_for(nonlinear: bool, qs: f64) -> u8 { /// What every slice of one picture shares (read-only). pub(crate) struct PicCtx<'a> { + /// I, P or B. + pub ptype: PictureType, /// The padded source picture. pub src: &'a Frame, - /// The reconstructed previous picture (P pictures); `None` codes an I picture. - pub reference: Option<&'a Frame>, - /// Vectors of the previous picture (motion search candidates), one per macroblock. - pub prev_mvs: &'a [[i32; 2]], + /// The reconstructed reference pictures: forward (P and B pictures) and backward (B + /// pictures). A B picture without a forward reference (the leading B pictures of a closed + /// GOP) predicts backward only. + pub fwd: Option<&'a Frame>, + pub bwd: Option<&'a Frame>, + /// Motion search candidates per macroblock, forward and backward (empty: none). + pub cand: [&'a [[i32; 2]]; 2], pub mb_w: usize, pub shx: u32, pub shy: u32, @@ -166,11 +171,22 @@ impl MbPix { struct SliceSt { qcode: u8, dc: [i32; 3], - pmv: [i32; 2], + /// Motion vector predictors, forward and backward. + pmv: [[i32; 2]; 2], /// Skipped macroblocks not yet covered by an address increment. skipped: usize, - /// The vector of the macroblock to the left (motion search candidate). - left_mv: [i32; 2], + /// The vectors of the macroblock to the left (motion search candidates). + left_mv: [[i32; 2]; 2], + /// Prediction directions (`MB_FWD` / `MB_BWD`) of the previous macroblock, or `MB_INTRA`: + /// a skipped macroblock in a B picture repeats them. + last_dirs: i16, +} + +/// A B macroblock's prediction: its directions and a vector per direction. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct BPred { + dirs: i16, + mvs: [[i32; 2]; 2], } /// One macroblock being coded. @@ -188,7 +204,7 @@ pub(crate) fn encode_row(ctx: &PicCtx, row: usize, out: &mut RowPlanes) -> RowOu let q0 = ctx.mb_qcode.get(row * ctx.mb_w).copied().unwrap_or(1); w.put(u32::from(q0), 5); w.bit(false); // extra_bit_slice - let mut st = SliceSt { qcode: q0, dc: [ctx.dc_reset(); 3], pmv: [0; 2], skipped: 0, left_mv: [0; 2] }; + let mut st = SliceSt { qcode: q0, dc: [ctx.dc_reset(); 3], pmv: [[0; 2]; 2], skipped: 0, left_mv: [[0; 2]; 2], last_dirs: MB_INTRA }; let mut ro = RowOut { bytes: Vec::new(), mvs: vec![[0; 2]; ctx.mb_w], stats: RowStats::default() }; for mbx in 0..ctx.mb_w { macroblock(ctx, &mut st, &mut w, (mbx, row), out, &mut ro); @@ -205,31 +221,105 @@ fn macroblock(ctx: &PicCtx, st: &mut SliceSt, w: &mut BitWriter, (mbx, mby): (us let qs = qscale(ctx.nonlinear, qcode); ro.stats.qsum += qs as u64; let planes = (ctx.src.w, ctx.src.cw); - let Some(reference) = ctx.reference else { - intra_mb(ctx, st, w, &mb).put(out, mbx, ctx.cmb(), planes); - ro.stats.intra += 1; - return; + let lambda = ((qs * 3 + 4) / 8).max(1) as u32; + let rec = match (ctx.ptype, ctx.fwd) { + (PictureType::P, Some(reference)) => { + let search = Search { reference, src: &src.y, mbx, mby, f_code: ctx.f_code, range: ctx.search, pmv: st.pmv[0], lambda }; + let found = search.run(&candidates(ctx, st, addr, 0)); + if intra_cost(&src.y).saturating_add(INTRA_BIAS) < found.sad { + None + } else { + let (rec, skipped) = inter_mb(ctx, st, w, &mb, reference, found.mv); + ro.stats.skipped += skipped as u32; + ro.mvs[mbx] = found.mv; + st.left_mv[0] = found.mv; + Some(rec) + } + } + (PictureType::B, _) if ctx.bwd.is_some() => b_decision(ctx, st, &mb, lambda).map(|pred| { + let (rec, skipped) = b_mb(ctx, st, w, &mb, pred); + ro.stats.skipped += skipped as u32; + st.left_mv = pred.mvs; + rec + }), + _ => None, }; - let mut candidates = vec![st.left_mv, st.pmv]; + let rec = rec.unwrap_or_else(|| { + ro.stats.intra += 1; + st.left_mv = [[0; 2]; 2]; + intra_mb(ctx, st, w, &mb) + }); + rec.put(out, mbx, ctx.cmb(), planes); +} + +/// Motion search candidates for direction `s` of macroblock `addr`: the left neighbour's vector, +/// the predictor, and the co-located and neighbouring candidates of the picture. +fn candidates(ctx: &PicCtx, st: &SliceSt, addr: usize, s: usize) -> Vec<[i32; 2]> { + let mut c = vec![st.left_mv[s], st.pmv[s]]; for a in [Some(addr), addr.checked_sub(ctx.mb_w), Some(addr + 1), Some(addr + ctx.mb_w)].into_iter().flatten() { - if let Some(&v) = ctx.prev_mvs.get(a) { - candidates.push(v); + if let Some(&v) = ctx.cand[s].get(a) { + c.push(v); } } - let lambda = ((qs * 3 + 4) / 8).max(1) as u32; - let search = Search { reference, src: &src.y, mbx, mby, f_code: ctx.f_code, range: ctx.search, pmv: st.pmv, lambda }; - let found = search.run(&candidates); - if intra_cost(&src.y).saturating_add(INTRA_BIAS) < found.sad { - intra_mb(ctx, st, w, &mb).put(out, mbx, ctx.cmb(), planes); - ro.stats.intra += 1; - st.left_mv = [0; 2]; - return; + c +} + +/// Luma SAD of a prediction against the source. +fn sad(a: &[u8; 256], b: &[u8; 256]) -> u32 { + a.iter().zip(b).map(|(x, y)| u32::from(x.abs_diff(*y))).sum() +} + +/// Choose a B macroblock's prediction: forward, backward or interpolated, from a motion search +/// per direction, or the skip prediction (the previous macroblock's directions with the +/// predictors). `None`: code it intra. +fn b_decision(ctx: &PicCtx, st: &SliceSt, mb: &MbIn, lambda: u32) -> Option { + let addr = mb.mby * ctx.mb_w + mb.mbx; + let mut found = [None, None]; + for (s, r) in [ctx.fwd, ctx.bwd].into_iter().enumerate() { + if let Some(reference) = r { + let search = + Search { reference, src: &mb.src.y, mbx: mb.mbx, mby: mb.mby, f_code: ctx.f_code, range: ctx.search, pmv: st.pmv[s], lambda }; + let f = search.run(&candidates(ctx, st, addr, s)); + found[s] = Some((f.mv, f.sad, search.mv_cost(f.mv))); + } } - let (rec, skipped) = inter_mb(ctx, st, w, &mb, reference, found.mv); - rec.put(out, mbx, ctx.cmb(), planes); - ro.stats.skipped += skipped as u32; - ro.mvs[mbx] = found.mv; - st.left_mv = found.mv; + let mut best: Option<(BPred, u32)> = None; + let mut consider = |pred: BPred, cost: u32| { + if best.is_none_or(|(_, c)| cost < c) { + best = Some((pred, cost)); + } + }; + for (s, dir) in [(0, MB_FWD), (1, MB_BWD)] { + if let Some((mv, sad, bits)) = found[s] { + let mut mvs = [[0; 2]; 2]; + mvs[s] = mv; + consider(BPred { dirs: dir, mvs }, sad.saturating_add(bits)); + } + } + if let (Some((f, _, bits_f)), Some((b, _, bits_b))) = (found[0], found[1]) { + let pred = BPred { dirs: MB_FWD | MB_BWD, mvs: [f, b] }; + let p = predict_b(ctx, pred, mb.mbx, mb.mby); + consider(pred, sad(&p.y, &mb.src.y).saturating_add(bits_f).saturating_add(bits_b)); + } + // the skip prediction costs no vector bits; the predictors came from the macroblock to the + // left, so they must be checked to stay inside the reference from here + let edge = mb.mbx == 0 || mb.mbx + 1 == ctx.mb_w; + if !edge && st.last_dirs & MB_INTRA == 0 && st.last_dirs != 0 { + let pred = BPred { dirs: st.last_dirs, mvs: st.pmv }; + let usable = [(0, MB_FWD, ctx.fwd), (1, MB_BWD, ctx.bwd)].into_iter().all(|(s, dir, r)| { + st.last_dirs & dir == 0 + || r.is_some_and(|reference| { + let search = + Search { reference, src: &mb.src.y, mbx: mb.mbx, mby: mb.mby, f_code: ctx.f_code, range: ctx.search, pmv: st.pmv[s], lambda }; + search.valid(st.pmv[s]) + }) + }); + if usable { + consider(pred, sad(&predict_b(ctx, pred, mb.mbx, mb.mby).y, &mb.src.y)); + } + } + let (pred, cost) = best?; + (intra_cost(&mb.src.y).saturating_add(INTRA_BIAS) >= cost).then_some(pred) } /// Σ |sample − block mean| over the four luma blocks: the cost of coding them intra. @@ -270,8 +360,12 @@ fn put_coefficients(w: &mut BitWriter, t: &EncTables, levels: &[i32; 64], table: t.dct_eob[table].put(w); } -fn put_mb_type(w: &mut BitWriter, t: &EncTables, predicted: bool, flags: i16) { - let table = if predicted { &t.mbtype_p } else { &t.mbtype_i }; +fn put_mb_type(w: &mut BitWriter, t: &EncTables, ptype: PictureType, flags: i16) { + let table = match ptype { + PictureType::P => &t.mbtype_p, + PictureType::B => &t.mbtype_b, + _ => &t.mbtype_i, + }; if let Some(c) = table.get(flags as usize).copied().flatten() { c.put(w); } @@ -283,12 +377,13 @@ fn intra_mb(ctx: &PicCtx, st: &mut SliceSt, w: &mut BitWriter, mb: &MbIn) -> MbP let quant = mb.qcode != st.qcode; put_increment(w, t, st.skipped + 1); st.skipped = 0; - put_mb_type(w, t, ctx.reference.is_some(), if quant { MB_INTRA | MB_QUANT } else { MB_INTRA }); + put_mb_type(w, t, ctx.ptype, if quant { MB_INTRA | MB_QUANT } else { MB_INTRA }); if quant { w.put(u32::from(mb.qcode), 5); st.qcode = mb.qcode; } - st.pmv = [0; 2]; + st.pmv = [[0; 2]; 2]; + st.last_dirs = MB_INTRA; let qs = qscale(ctx.nonlinear, st.qcode); let dc_mult = 8 >> ctx.dc_prec; let dc_max = (1 << (8 + ctx.dc_prec)) - 1; @@ -321,28 +416,16 @@ fn intra_mb(ctx: &PicCtx, st: &mut SliceSt, w: &mut BitWriter, mb: &MbIn) -> MbP /// whether it was skipped. fn inter_mb(ctx: &PicCtx, st: &mut SliceSt, w: &mut BitWriter, mb: &MbIn, reference: &Frame, mv: [i32; 2]) -> (MbPix, bool) { let t = enc_tables(); - let mut rec = predict_mb(ctx, reference, mv, mb.mbx, mb.mby); + let mut rec = predict_mb(ctx, Some(reference), mv, mb.mbx, mb.mby); let qs = qscale(ctx.nonlinear, mb.qcode); - let (cbw, _) = ctx.cmb(); - let n = ctx.blocks(); - let mut levels = [[0i32; 64]; 8]; - let mut cbp = 0u32; - for i in 0..n { - let (s, p) = (mb.src.block(i, cbw), rec.block(i, cbw)); - let f = fdct(&std::array::from_fn(|k| s[k] - p[k])); - for pos in 0..64 { - levels[i][pos] = quant_inter(f[pos], ctx.inter_matrix[pos], qs); - } - if levels[i].iter().any(|&l| l != 0) { - cbp |= 1 << (n - 1 - i); - } - } + let (levels, cbp) = residual(ctx, mb, &rec, qs); st.dc = [ctx.dc_reset(); 3]; + st.last_dirs = MB_FWD; // the first and last macroblock of a slice are always coded let edge = mb.mbx == 0 || mb.mbx + 1 == ctx.mb_w; if mv == [0, 0] && cbp == 0 && !edge { st.skipped += 1; - st.pmv = [0; 2]; + st.pmv = [[0; 2]; 2]; return (rec, true); } put_increment(w, t, st.skipped + 1); @@ -358,44 +441,140 @@ fn inter_mb(ctx: &PicCtx, st: &mut SliceSt, w: &mut BitWriter, mb: &MbIn, refere if quant { flags |= MB_QUANT; } - put_mb_type(w, t, true, flags); + put_mb_type(w, t, PictureType::P, flags); if quant { w.put(u32::from(mb.qcode), 5); st.qcode = mb.qcode; } if flags & MB_FWD != 0 { for c in 0..2 { - t.put_motion(w, wrap_delta(mv[c], st.pmv[c], ctx.f_code), ctx.f_code); + t.put_motion(w, wrap_delta(mv[c], st.pmv[0][c], ctx.f_code), ctx.f_code); } - st.pmv = mv; + st.pmv[0] = mv; } else { - st.pmv = [0; 2]; // "No MC" resets the predictors - } - if cbp != 0 { - if n == 8 { - t.cbp[(cbp >> 2) as usize].put(w); - w.put(cbp & 3, 2); - } else { - t.cbp[cbp as usize].put(w); - } - for i in 0..n { - if cbp & (1 << (n - 1 - i)) != 0 { - put_coefficients(w, t, &levels[i], 0, 0); - rec.store(i, cbw, &reconstruct(&levels[i], ctx.inter_matrix, qs, false, 0), true); - } - } + st.pmv = [[0; 2]; 2]; // "No MC" resets the predictors } + put_residual(ctx, w, &levels, cbp, &mut rec, qs); (rec, false) } -/// The frame prediction of a macroblock from `f` with vector `mv`, formed as the decoder forms -/// it (§7.6.4: chroma vectors are the luma ones halved towards zero). -fn predict_mb(ctx: &PicCtx, f: &Frame, mv: [i32; 2], mbx: usize, mby: usize) -> MbPix { - let (cbw, cbh) = ctx.cmb(); +/// Quantised residual of every block of `mb` against `pred`, and the coded_block_pattern. +fn residual(ctx: &PicCtx, mb: &MbIn, pred: &MbPix, qs: i32) -> ([[i32; 64]; 8], u32) { + let (cbw, _) = ctx.cmb(); + let n = ctx.blocks(); + let mut levels = [[0i32; 64]; 8]; + let mut cbp = 0u32; + for i in 0..n { + let (s, p) = (mb.src.block(i, cbw), pred.block(i, cbw)); + let f = fdct(&std::array::from_fn(|k| s[k] - p[k])); + for pos in 0..64 { + levels[i][pos] = quant_inter(f[pos], ctx.inter_matrix[pos], qs); + } + if levels[i].iter().any(|&l| l != 0) { + cbp |= 1 << (n - 1 - i); + } + } + (levels, cbp) +} + +/// Write coded_block_pattern and the coded blocks, adding their reconstruction to `rec`. +fn put_residual(ctx: &PicCtx, w: &mut BitWriter, levels: &[[i32; 64]; 8], cbp: u32, rec: &mut MbPix, qs: i32) { + if cbp == 0 { + return; + } + let t = enc_tables(); + let (cbw, _) = ctx.cmb(); + let n = ctx.blocks(); + if n == 8 { + t.cbp[(cbp >> 2) as usize].put(w); + w.put(cbp & 3, 2); + } else { + t.cbp[cbp as usize].put(w); + } + for i in 0..n { + if cbp & (1 << (n - 1 - i)) != 0 { + put_coefficients(w, t, &levels[i], 0, 0); + rec.store(i, cbw, &reconstruct(&levels[i], ctx.inter_matrix, qs, false, 0), true); + } + } +} + +/// Code a B macroblock with prediction `pred` (or skip it); returns its reconstruction and +/// whether it was skipped. +fn b_mb(ctx: &PicCtx, st: &mut SliceSt, w: &mut BitWriter, mb: &MbIn, pred: BPred) -> (MbPix, bool) { + let t = enc_tables(); + let mut rec = predict_b(ctx, pred, mb.mbx, mb.mby); + let qs = qscale(ctx.nonlinear, mb.qcode); + let (levels, cbp) = residual(ctx, mb, &rec, qs); + st.dc = [ctx.dc_reset(); 3]; + // a skipped B macroblock repeats the previous one's directions with the predictors as its + // vectors (§7.6.6.3); the first and last macroblock of a slice are always coded + let edge = mb.mbx == 0 || mb.mbx + 1 == ctx.mb_w; + let same = |s: usize, dir: i16| pred.dirs & dir == 0 || pred.mvs[s] == st.pmv[s]; + if cbp == 0 && !edge && st.last_dirs & MB_INTRA == 0 && pred.dirs == st.last_dirs && same(0, MB_FWD) && same(1, MB_BWD) { + st.skipped += 1; + return (rec, true); + } + put_increment(w, t, st.skipped + 1); + st.skipped = 0; + let quant = cbp != 0 && mb.qcode != st.qcode; + let mut flags = pred.dirs; + if cbp != 0 { + flags |= MB_PATTERN; + } + if quant { + flags |= MB_QUANT; + } + put_mb_type(w, t, PictureType::B, flags); + if quant { + w.put(u32::from(mb.qcode), 5); + st.qcode = mb.qcode; + } + for (s, dir) in [(0, MB_FWD), (1, MB_BWD)] { + if pred.dirs & dir != 0 { + for c in 0..2 { + t.put_motion(w, wrap_delta(pred.mvs[s][c], st.pmv[s][c], ctx.f_code), ctx.f_code); + } + st.pmv[s] = pred.mvs[s]; + } + } + st.last_dirs = pred.dirs; + put_residual(ctx, w, &levels, cbp, &mut rec, qs); + (rec, false) +} + +/// The prediction of a B macroblock: forward, backward, or the average of both, formed in the +/// decoder's order (forward first, then the backward prediction averaged in). +fn predict_b(ctx: &PicCtx, pred: BPred, mbx: usize, mby: usize) -> MbPix { let mut p = MbPix { y: [0; 256], c: [[0; 256]; 2] }; + let mut avg = false; + for (s, dir, r) in [(0, MB_FWD, ctx.fwd), (1, MB_BWD, ctx.bwd)] { + if pred.dirs & dir != 0 { + if let Some(f) = r { + predict_into(ctx, &mut p, f, pred.mvs[s], (mbx, mby), avg); + } + avg = true; + } + } + p +} + +/// The frame prediction of a macroblock from `f` with vector `mv` (gray without a reference). +fn predict_mb(ctx: &PicCtx, f: Option<&Frame>, mv: [i32; 2], mbx: usize, mby: usize) -> MbPix { + let mut p = MbPix { y: [128; 256], c: [[128; 256]; 2] }; + if let Some(f) = f { + predict_into(ctx, &mut p, f, mv, (mbx, mby), false); + } + p +} + +/// Form the frame prediction from `f` with vector `mv` into `p`, or average it with what `p` +/// holds, as the decoder forms it (§7.6.4: chroma vectors are the luma ones halved towards zero). +fn predict_into(ctx: &PicCtx, p: &mut MbPix, f: &Frame, mv: [i32; 2], (mbx, mby): (usize, usize), avg: bool) { + let (cbw, cbh) = ctx.cmb(); let blk = Block { x: (mbx * 16) as i32 + (mv[0] >> 1), y: (mby * 16) as i32 + (mv[1] >> 1), hx: mv[0] & 1 != 0, hy: mv[1] & 1 != 0, bw: 16, bh: 16 }; - predict(&mut p.y, 0, 16, f.plane(0, None), blk, false); + predict(&mut p.y, 0, 16, f.plane(0, None), blk, avg); let cmv = [if ctx.shx == 1 { mv[0] / 2 } else { mv[0] }, if ctx.shy == 1 { mv[1] / 2 } else { mv[1] }]; let cb = Block { x: (mbx * cbw) as i32 + (cmv[0] >> 1), @@ -406,7 +585,48 @@ fn predict_mb(ctx: &PicCtx, f: &Frame, mv: [i32; 2], mbx: usize, mby: usize) -> bh: cbh, }; for c in 0..2 { - predict(&mut p.c[c], 0, cbw, f.plane(1 + c, None), cb, false); + predict(&mut p.c[c], 0, cbw, f.plane(1 + c, None), cb, avg); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_skip_prediction_never_leaves_the_reference() { + // three macroblocks across; the left neighbour's forward vector (+17 samples) fits its + // own position but points past the right edge from the middle macroblock + let mut f = Frame::new(48, 16, 24, 8); + for (i, v) in f.planes[0].iter_mut().enumerate() { + *v = ((i % 48) * 5 + (i / 48) * 11) as u8; + } + let ctx = PicCtx { + ptype: PictureType::B, + src: &f, + fwd: Some(&f), + bwd: Some(&f), + cand: [&[], &[]], + mb_w: 3, + shx: 1, + shy: 1, + intra_matrix: &crate::mpeg2::headers::DEFAULT_INTRA, + inter_matrix: &[16; 64], + nonlinear: true, + dc_prec: 1, + f_code: 3, + search: 31, + mb_qcode: &[4; 3], + }; + let bad = BPred { dirs: MB_FWD, mvs: [[34, 0], [0, 0]] }; + // the bitstream can carry the vector (f_code 3: -64..64): only the position rules it out + let at = |mbx| Search { reference: &f, src: &[0; 256], mbx, mby: 0, f_code: ctx.f_code, range: ctx.search, pmv: bad.mvs[0], lambda: 1 }; + assert!(at(0).valid(bad.mvs[0]) && !at(1).valid(bad.mvs[0])); + // a source that the out-of-range (edge-clamped) prediction would match exactly + let src = predict_b(&ctx, bad, 1, 0); + let st = SliceSt { qcode: 4, dc: [256; 3], pmv: bad.mvs, skipped: 0, left_mv: bad.mvs, last_dirs: MB_FWD }; + let mb = MbIn { src: &src, mbx: 1, mby: 0, qcode: 4 }; + let pred = b_decision(&ctx, &st, &mb, 1); + assert_ne!(pred, Some(bad)); } - p } diff --git a/crates/video/src/mpeg2/encode/rate.rs b/crates/video/src/mpeg2/encode/rate.rs index 65c23efc3..96c6f2aa1 100644 --- a/crates/video/src/mpeg2/encode/rate.rs +++ b/crates/video/src/mpeg2/encode/rate.rs @@ -1,18 +1,26 @@ //! Single-pass average bitrate control at picture level, after the target bit allocation of the //! MPEG-2 Test Model 5: each picture type keeps a complexity `X = bits · average quantiser`, the -//! bits left for the GOP are shared out by complexity, and a picture's quantiser is `X / target`. -//! There is no VBV model: the average converges on the target, single pictures may not. +//! bits left for the GOP are shared out by complexity (B pictures weighted down by `KB`), and a +//! picture's quantiser is `X / target`. There is no VBV model: the average converges on the +//! target, single pictures may not. + +use crate::mpeg2::headers::PictureType; + +/// TM5's quantiser ratio of B to P pictures (K_b / K_p). +const KB: f64 = 1.4; pub(crate) struct RateCtl { bps: f64, fps: f64, - /// Complexity of I and P pictures. + /// Complexity of I, P and B pictures. xi: f64, xp: f64, + xb: f64, /// Bits left for the current GOP (carries over between GOPs). remaining: f64, - /// P pictures left in the current GOP. + /// P and B pictures left in the current GOP. np: u32, + nb: u32, /// Range of quantiser_scale values. q_min: f64, q_max: f64, @@ -21,34 +29,60 @@ pub(crate) struct RateCtl { impl RateCtl { pub fn new(bps: f64, fps: f64, q_range: (f64, f64)) -> Self { // TM5's starting complexities - Self { bps, fps, xi: 160.0 * bps / 115.0, xp: 60.0 * bps / 115.0, remaining: 0.0, np: 0, q_min: q_range.0, q_max: q_range.1 } + Self { + bps, + fps, + xi: 160.0 * bps / 115.0, + xp: 60.0 * bps / 115.0, + xb: 42.0 * bps / 115.0, + remaining: 0.0, + np: 0, + nb: 0, + q_min: q_range.0, + q_max: q_range.1, + } } - /// A GOP of `pictures` pictures (one I, the rest P) starts. - pub fn start_gop(&mut self, pictures: u32) { - self.remaining += self.bps * f64::from(pictures) / self.fps; - self.np = pictures.saturating_sub(1); + /// A GOP of one I, `np` P and `nb` B pictures starts. + pub fn start_gop(&mut self, np: u32, nb: u32) { + self.remaining += self.bps * f64::from(1 + np + nb) / self.fps; + self.np = np; + self.nb = nb; } - /// quantiser_scale for the next picture. - pub fn qscale(&self, intra: bool) -> f64 { + /// The GOP ends `missing` pictures short of what `start_gop` budgeted (the encoder was + /// flushed): their bits are not carried into the next GOP. + pub fn end_gop(&mut self, missing: u32) { + self.remaining -= self.bps * f64::from(missing) / self.fps; + self.np = 0; + self.nb = 0; + } + + /// quantiser_scale for the next picture of type `ptype`. + pub fn qscale(&self, ptype: PictureType) -> f64 { let floor = self.bps / (8.0 * self.fps); - let (target, x) = if intra { - (self.remaining / (1.0 + f64::from(self.np) * self.xp / self.xi), self.xi) - } else { - (self.remaining / f64::from(self.np.max(1)), self.xp) + let (np, nb) = (f64::from(self.np), f64::from(self.nb)); + let (target, x) = match ptype { + PictureType::I => (self.remaining / (1.0 + np * self.xp / self.xi + nb * self.xb / (self.xi * KB)), self.xi), + PictureType::B => (self.remaining / (nb.max(1.0) + np * KB * self.xp / self.xb), self.xb), + _ => (self.remaining / (np.max(1.0) + nb * self.xb / (KB * self.xp)), self.xp), }; (x / target.max(floor)).clamp(self.q_min, self.q_max) } - /// The picture used `bits` at average quantiser_scale `avg_q`. - pub fn update(&mut self, intra: bool, bits: u64, avg_q: f64) { - let x = bits as f64 * avg_q; - if intra { - self.xi = x.max(1.0); - } else { - self.xp = x.max(1.0); - self.np = self.np.saturating_sub(1); + /// A picture of type `ptype` used `bits` at average quantiser_scale `avg_q`. + pub fn update(&mut self, ptype: PictureType, bits: u64, avg_q: f64) { + let x = (bits as f64 * avg_q).max(1.0); + match ptype { + PictureType::I => self.xi = x, + PictureType::B => { + self.xb = x; + self.nb = self.nb.saturating_sub(1); + } + _ => { + self.xp = x; + self.np = self.np.saturating_sub(1); + } } self.remaining -= bits as f64; } @@ -58,18 +92,26 @@ impl RateCtl { mod tests { use super::*; - /// A picture whose size is `complexity / q` (the model the controller assumes). - fn run(complexity_i: f64, complexity_p: f64, bps: f64, gops: u32) -> f64 { + /// A picture whose size is `complexity / q` (the model the controller assumes), in GOPs of + /// 12 pictures with `b` B pictures between anchors. + fn run(c: [f64; 3], b: usize, bps: f64, gops: u32) -> f64 { let fps = 25.0; + let types: Vec = (0..12) + .map(|k| match k { + 0 => PictureType::I, + k if k % (b + 1) == 0 => PictureType::P, + _ => PictureType::B, + }) + .collect(); + let np = types.iter().filter(|&&t| t == PictureType::P).count() as u32; let mut rc = RateCtl::new(bps, fps, (1.0, 112.0)); let mut total = 0.0; for _ in 0..gops { - rc.start_gop(12); - for k in 0..12 { - let intra = k == 0; - let q = rc.qscale(intra); - let bits = (if intra { complexity_i } else { complexity_p }) / q; - rc.update(intra, bits as u64, q); + rc.start_gop(np, 11 - np); + for &t in &types { + let q = rc.qscale(t); + let bits = c[t as usize] / q; + rc.update(t, bits as u64, q); total += bits; } } @@ -79,24 +121,40 @@ mod tests { #[test] fn converges_on_the_target() { // complexities whose target quantiser is inside the allowed range - for (ci, cp) in [(4e7, 1e7), (8e7, 2e6), (1e7, 9e6)] { - let got = run(ci, cp, 4e6, 20); - assert!((got / 4e6 - 1.0).abs() < 0.03, "{ci} {cp}: {got}"); + for (c, b) in [([4e7, 1e7, 7e6], 0), ([8e7, 2e6, 1e6], 0), ([1e7, 9e6, 8e6], 0), ([4e7, 1e7, 7e6], 2), ([6e7, 2e7, 5e6], 1)] { + let got = run(c, b, 4e6, 20); + assert!((got / 4e6 - 1.0).abs() < 0.03, "{c:?} b {b}: {got}"); } } + #[test] + fn a_gop_cut_short_gives_its_budget_back() { + let mut rc = RateCtl::new(4e6, 25.0, (1.0, 112.0)); + for _ in 0..20 { + // a GOP of 12 that ends after its I picture and two P pictures + rc.start_gop(11, 0); + for t in [PictureType::I, PictureType::P, PictureType::P] { + let q = rc.qscale(t); + rc.update(t, (1e7 / q) as u64, q); + } + rc.end_gop(9); + } + // what is left is the three pictures' rounding, not nine pictures' budget per GOP + assert!(rc.remaining.abs() < 3.0 * 4e6 / 25.0, "{}", rc.remaining); + } + #[test] fn the_quantiser_stays_in_range() { let mut rc = RateCtl::new(1e6, 25.0, (2.0, 62.0)); - rc.start_gop(2); + rc.start_gop(1, 0); // a picture far larger than its budget pushes the next quantiser to the top - rc.update(true, 50_000_000, 10.0); - rc.start_gop(2); - assert_eq!(rc.qscale(true), 62.0); + rc.update(PictureType::I, 50_000_000, 10.0); + rc.start_gop(1, 0); + assert_eq!(rc.qscale(PictureType::I), 62.0); // and a tiny one to the bottom let mut rc = RateCtl::new(1e6, 25.0, (2.0, 62.0)); - rc.start_gop(2); - rc.update(true, 10, 2.0); - assert_eq!(rc.qscale(true), 2.0); + rc.start_gop(1, 0); + rc.update(PictureType::I, 10, 2.0); + assert_eq!(rc.qscale(PictureType::I), 2.0); } } diff --git a/crates/video/src/mpeg2/encode/tables.rs b/crates/video/src/mpeg2/encode/tables.rs index b79dd92ea..e23929450 100644 --- a/crates/video/src/mpeg2/encode/tables.rs +++ b/crates/video/src/mpeg2/encode/tables.rs @@ -4,7 +4,7 @@ use std::sync::OnceLock; use crate::mpeg2::vlc::{ - CBP, DC_CHROMA, DC_LUMA, DCT_EOB, DCT_ESCAPE, DCT_ONE, DCT_ZERO, DCT_ZERO_LONG, MBA, MBA_ESCAPE, MBTYPE_I, MBTYPE_P, MOTION, dct_table, + CBP, DC_CHROMA, DC_LUMA, DCT_EOB, DCT_ESCAPE, DCT_ONE, DCT_ZERO, DCT_ZERO_LONG, MBA, MBA_ESCAPE, MBTYPE_B, MBTYPE_I, MBTYPE_P, MOTION, dct_table, parse_code, }; @@ -38,9 +38,10 @@ pub(crate) struct EncTables { /// macroblock_address_increment 1..=33 (index 0 unused) and the escape. pub mba: [Code; 34], pub mba_escape: Code, - /// macroblock_type by flags: I pictures, P pictures (index: flags, 0..32). + /// macroblock_type by flags: I, P and B pictures (index: flags, 0..32). pub mbtype_i: [Option; 32], pub mbtype_p: [Option; 32], + pub mbtype_b: [Option; 32], /// coded_block_pattern_420 (the first six blocks). pub cbp: [Code; 64], /// motion_code -16..=16 at index code + 16. @@ -105,6 +106,7 @@ pub(crate) fn enc_tables() -> &'static EncTables { mba_escape, mbtype_i: mb_types(MBTYPE_I), mbtype_p: mb_types(MBTYPE_P), + mbtype_b: mb_types(MBTYPE_B), cbp: by_value(CBP, 0), motion: by_value(MOTION, 16), dc_luma: by_value(DC_LUMA, 0), @@ -185,11 +187,25 @@ mod tests { use super::*; use crate::mpeg2::bits::Bits; use crate::mpeg2::slice::wrap_vector; - use crate::mpeg2::vlc::{MB_FWD, MB_INTRA, MB_PATTERN, MB_QUANT, rl, tables}; + use crate::mpeg2::vlc::{MB_BWD, MB_FWD, MB_INTRA, MB_PATTERN, MB_QUANT, rl, tables}; - /// Macroblock types the encoder sends (frame pictures, frame prediction, no B pictures). + /// Macroblock types the encoder sends in P pictures (frame pictures, frame prediction). const ENC_MB_TYPES: [i16; 7] = [MB_INTRA, MB_INTRA | MB_QUANT, MB_FWD | MB_PATTERN, MB_PATTERN, MB_FWD, MB_QUANT | MB_FWD | MB_PATTERN, MB_QUANT | MB_PATTERN]; + /// And in B pictures: every combination of directions, with and without a residual. + const ENC_MB_TYPES_B: [i16; 11] = [ + MB_INTRA, + MB_INTRA | MB_QUANT, + MB_FWD, + MB_BWD, + MB_FWD | MB_BWD, + MB_FWD | MB_PATTERN, + MB_BWD | MB_PATTERN, + MB_FWD | MB_BWD | MB_PATTERN, + MB_QUANT | MB_FWD | MB_PATTERN, + MB_QUANT | MB_BWD | MB_PATTERN, + MB_QUANT | MB_FWD | MB_BWD | MB_PATTERN, + ]; #[test] fn every_code_decodes_to_its_value() { @@ -223,6 +239,9 @@ mod tests { } check(t.mbtype_p[f as usize].unwrap_or_default(), &d.mbtype[1], f); } + for f in ENC_MB_TYPES_B { + check(t.mbtype_b[f as usize].unwrap_or_default(), &d.mbtype[2], f); + } for (k, vlc) in [&d.dct_zero, &d.dct_one].into_iter().enumerate() { for run in 0..=MAX_RUN { for level in 1..=MAX_LEVEL { diff --git a/crates/video/src/mpeg2/encode/tests.rs b/crates/video/src/mpeg2/encode/tests.rs index 5352161b3..01aac2b32 100644 --- a/crates/video/src/mpeg2/encode/tests.rs +++ b/crates/video/src/mpeg2/encode/tests.rs @@ -3,7 +3,7 @@ //! comparison is in `tests/mpeg2/encode.rs`. use super::*; -use crate::mpeg2::{Decoder, access_units, probe}; +use crate::mpeg2::{Decoder, probe}; /// A moving, textured test sequence: a gradient with a sharp pattern drifting by a fraction of a /// sample per picture, and a block moving across it. @@ -54,16 +54,21 @@ fn encode(cfg: EncoderConfig, frames: usize) -> (Vec, Vec, Vec

Vec { +/// Decode packets (each tagged with its presentation time); returns the pictures in display +/// order. +fn decode(packets: &[Packet]) -> Vec { let mut d = Decoder::new(); let mut out = Vec::new(); - for (i, r) in access_units(es).into_iter().enumerate() { - out.extend(d.decode(&es[r], i as i64).unwrap()); + for p in packets { + out.extend(d.decode(&p.data, p.pts).unwrap()); } out.extend(d.flush()); assert_eq!(d.errors(), 0, "decoder errors: {:?}", d.last_error()); @@ -71,9 +76,11 @@ fn decode(es: &[u8]) -> Vec { } fn assert_decodes_to_recon(name: &str, cfg: EncoderConfig, frames: usize) -> Vec { - let (es, recon, packets) = encode(cfg, frames); - let pics = decode(&es); + let (_, recon, packets) = encode(cfg, frames); + assert!(packets.windows(2).all(|w| w[0].dts < w[1].dts) && packets.iter().all(|p| p.dts <= p.pts), "{name}: decoding times"); + let pics = decode(&packets); assert_eq!(pics.len(), frames, "{name}: picture count"); + assert_eq!(recon.len(), frames, "{name}: reconstruction count"); for (i, (p, r)) in pics.iter().zip(&recon).enumerate() { assert_eq!(p.pts, i as i64, "{name}: display order"); for (plane, (a, b)) in [("Y", (&p.y, &r.y)), ("Cb", (&p.cb, &r.cb)), ("Cr", (&p.cr, &r.cr))] { @@ -101,6 +108,7 @@ fn intra_only_decodes_to_the_reconstruction() { fn long_p_gop_does_not_drift() { let mut cfg = EncoderConfig::new(96, 64, 25, 1); cfg.gop_size = 30; + cfg.b_pictures = 0; let packets = assert_decodes_to_recon("long gop", cfg, 30); assert_eq!(packets.iter().filter(|p| p.keyframe).count(), 1); assert!(packets[1..].iter().all(|p| p.picture_type == PictureType::P)); @@ -127,7 +135,201 @@ fn every_coding_option_decodes_to_the_reconstruction() { ("422 odd size", EncoderConfig { width: 70, height: 30, chroma: ChromaFormat::Yuv422, ..base.clone() }), ]; for (name, cfg) in cases { - assert_decodes_to_recon(name, cfg, 8); + assert_decodes_to_recon(name, cfg.clone(), 8); + assert_decodes_to_recon(name, EncoderConfig { b_pictures: 0, ..cfg }, 8); + } +} + +#[test] +fn b_pictures_decode_to_the_reconstruction() { + let base = EncoderConfig::new(96, 64, 25, 1); + for b in 1..=4 { + for closed in [true, false] { + for (gop, frames) in [(12, 30), (7, 17), (2, 6), (1, 4), (15, 16)] { + let name = format!("b {b} closed {closed} gop {gop} frames {frames}"); + let cfg = EncoderConfig { b_pictures: b, closed_gop: closed, gop_size: gop, ..base.clone() }; + let packets = assert_decodes_to_recon(&name, cfg, frames); + if gop > 1 { + assert!(packets.iter().any(|p| p.picture_type == PictureType::B), "{name}: no B pictures"); + } + } + } + } + let cases = [ + ("422", EncoderConfig { chroma: ChromaFormat::Yuv422, ..base.clone() }), + ("abr", EncoderConfig { rate: RateControl::Abr { kbps: 400 }, ..base.clone() }), + ("odd size", EncoderConfig { width: 50, height: 34, ..base.clone() }), + ("small range", EncoderConfig { search_range: 2, ..base.clone() }), + ("linear coarse", EncoderConfig { nonlinear_quant: false, rate: RateControl::Quantiser(25), ..base.clone() }), + ]; + for (name, cfg) in cases { + assert_decodes_to_recon(name, cfg, 14); + } +} + +#[test] +fn pictures_are_reordered_into_decoding_order() { + let cfg = EncoderConfig { gop_size: 12, b_pictures: 2, ..EncoderConfig::new(64, 48, 25, 1) }; + let (_, _, packets) = encode(cfg, 15); + let order: Vec<(PictureType, i64)> = packets.iter().map(|p| (p.picture_type, p.pts)).collect(); + use PictureType::{B, I, P}; + #[rustfmt::skip] + let want = [(I, 0), (P, 3), (B, 1), (B, 2), (P, 6), (B, 4), (B, 5), (P, 9), (B, 7), (B, 8), + // B10 and B11 lead the second GOP; at the end the last held picture becomes a P picture + (I, 12), (B, 10), (B, 11), (P, 14), (B, 13)]; + assert_eq!(order, want); + // decoding times: the presentation times in order, one interval early + let dts: Vec = packets.iter().map(|p| p.dts).collect(); + assert_eq!(dts, (-1..14).collect::>()); + // temporal_reference counts display order from each GOP's first picture + let pics = decode(&packets); + let tr: Vec = pics.iter().map(|p| p.temporal_reference).collect(); + assert_eq!(tr, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4]); +} + +#[test] +fn a_closed_gop_decodes_on_its_own() { + for closed in [true, false] { + let cfg = EncoderConfig { gop_size: 6, b_pictures: 2, closed_gop: closed, ..EncoderConfig::new(64, 48, 25, 1) }; + let (_, recon, packets) = encode(cfg, 12); + // start decoding at the second GOP's I picture (display 6): B4 and B5 lead it + let at = packets.iter().position(|p| p.pts == 6).unwrap(); + assert!(packets[at].keyframe); + let mut d = Decoder::new(); + let mut pics = Vec::new(); + for p in &packets[at..] { + pics.extend(d.decode(&p.data, p.pts).unwrap()); + } + pics.extend(d.flush()); + let shown: Vec = pics.iter().map(|p| p.pts).collect(); + if closed { + assert_eq!(shown, (4..12).collect::>()); + for p in &pics { + assert_eq!(p.y, recon[p.pts as usize].y, "closed GOP: picture {}", p.pts); + } + } else { + // the leading B pictures need the previous GOP's last P picture + assert_eq!(shown, (6..12).collect::>()); + } + } +} + +#[test] +fn b_pictures_are_smaller_than_p_pictures() { + let cfg = EncoderConfig { gop_size: 12, b_pictures: 2, ..EncoderConfig::new(128, 96, 25, 1) }; + let (_, _, packets) = encode(cfg, 12); + let avg = |t: PictureType| { + let s: Vec = packets.iter().filter(|p| p.picture_type == t).map(|p| p.data.len()).collect(); + s.iter().sum::() as f64 / s.len().max(1) as f64 + }; + let (p, b) = (avg(PictureType::P), avg(PictureType::B)); + assert!(b * 1.5 < p, "P {p:.0} bytes, B {b:.0} bytes"); + // static content: the B pictures skip every macroblock but the first and last of a slice + let mut enc = Encoder::new(EncoderConfig { gop_size: 12, b_pictures: 2, ..EncoderConfig::new(128, 64, 25, 1) }).unwrap(); + let (y, cb, cr) = scene(128, 64, 64, 32, 0); + let f = YuvFrame { y: &y, cb: &cb, cr: &cr, y_stride: 128, c_stride: 64 }; + let mut packets = Vec::new(); + for t in 0..7 { + packets.extend(enc.encode(&f, t).unwrap()); + } + let i = packets[0].data.len(); + let bs: Vec<&Packet> = packets.iter().filter(|p| p.picture_type == PictureType::B).collect(); + assert_eq!(bs.len(), 4); + for p in bs { + // a picture header, four slice headers and eight edge macroblocks + assert!(p.data.len() < 64, "static B picture {} of {} bytes (I {i})", p.pts, p.data.len()); + } +} + +#[test] +fn presentation_times_must_step_evenly_with_b_pictures() { + let mut enc = Encoder::new(EncoderConfig::new(32, 32, 25, 1)).unwrap(); + let (y, c) = (vec![0u8; 32 * 32], vec![0u8; 16 * 16]); + let f = YuvFrame { y: &y, cb: &c, cr: &c, y_stride: 32, c_stride: 16 }; + // the I picture waits for the second time, which sets the step + assert!(enc.encode(&f, 10).unwrap().is_empty()); + let mut packets = enc.encode(&f, 12).unwrap(); + for bad in [12, 11, 15] { + assert!(matches!(enc.encode(&f, bad), Err(EncodeError::BadTimestamp(_))), "{bad}"); + } + packets.extend(enc.encode(&f, 14).unwrap()); + packets.extend(enc.flush()); + assert_eq!(packets.iter().map(|p| (p.pts, p.dts)).collect::>(), [(10, 8), (14, 10), (12, 12)]); + // without B pictures any increasing (or not) times pass through + let mut enc = Encoder::new(EncoderConfig { b_pictures: 0, ..EncoderConfig::new(32, 32, 25, 1) }).unwrap(); + for pts in [5, 3, 9] { + let p = enc.encode(&f, pts).unwrap().remove(0); + assert_eq!((p.pts, p.dts), (pts, pts)); + } +} + +#[test] +fn decoding_times_increase_after_a_lone_picture_is_flushed() { + let mut enc = Encoder::new(EncoderConfig::new(32, 32, 25, 1)).unwrap(); + let (y, c) = (vec![0u8; 32 * 32], vec![0u8; 16 * 16]); + let f = YuvFrame { y: &y, cb: &c, cr: &c, y_stride: 32, c_stride: 16 }; + // one picture, flushed before a second sets the step: it decodes one unit early + assert!(enc.encode(&f, 100).unwrap().is_empty()); + let mut packets = enc.flush(); + assert_eq!(packets.iter().map(|p| (p.pts, p.dts)).collect::>(), [(100, 99)]); + // encoding goes on: the next picture decodes at the first one's presentation time + for pts in [105, 110] { + packets.extend(enc.encode(&f, pts).unwrap()); + } + packets.extend(enc.flush()); + assert_eq!(packets.iter().map(|p| (p.pts, p.dts)).collect::>(), [(100, 99), (105, 100), (110, 105)]); + assert!(packets[1].keyframe); +} + +#[test] +fn presentation_times_without_a_decoding_time_are_refused() { + let (y, c) = (vec![0u8; 32 * 32], vec![0u8; 16 * 16]); + let f = YuvFrame { y: &y, cb: &c, cr: &c, y_stride: 32, c_stride: 16 }; + // the decoding times are one step earlier: they must fit in an i64 + for (first, second) in [(i64::MIN + 1, i64::MIN + 3), (i64::MIN + 5, i64::MIN + 20)] { + let mut enc = Encoder::new(EncoderConfig::new(32, 32, 25, 1)).unwrap(); + assert!(matches!(enc.encode(&f, i64::MIN), Err(EncodeError::BadTimestamp(_)))); + assert!(enc.encode(&f, first).unwrap().is_empty()); + assert!(matches!(enc.encode(&f, second), Err(EncodeError::BadTimestamp(_))), "{second} after {first}"); + // a refused time leaves the encoder usable + assert_eq!(enc.flush().iter().map(|p| (p.pts, p.dts)).collect::>(), [(first, first - 1)]); + } + // the lowest times that have one + let mut enc = Encoder::new(EncoderConfig::new(32, 32, 25, 1)).unwrap(); + let mut packets = enc.encode(&f, i64::MIN + 1).unwrap(); + packets.extend(enc.encode(&f, i64::MIN + 2).unwrap()); + packets.extend(enc.flush()); + assert_eq!(packets.iter().map(|p| p.dts).collect::>(), [i64::MIN, i64::MIN + 1]); + // without B pictures every time passes through + let mut enc = Encoder::new(EncoderConfig { b_pictures: 0, ..EncoderConfig::new(32, 32, 25, 1) }).unwrap(); + assert_eq!(enc.encode(&f, i64::MIN).unwrap()[0].dts, i64::MIN); +} + +/// Average bitrate (bit/s) of the scene coded in groups of `group` pictures, flushed after each. +fn flushed_bitrate(cfg: EncoderConfig, group: usize, groups: usize) -> f64 { + let (w, h) = (cfg.width as usize, cfg.height as usize); + let fps = f64::from(cfg.fps_num) / f64::from(cfg.fps_den); + let mut enc = Encoder::new(cfg).unwrap(); + let mut bytes = 0; + for t in 0..group * groups { + let (y, cb, cr) = scene(w, h, w / 2, h / 2, t); + let f = YuvFrame { y: &y, cb: &cb, cr: &cr, y_stride: w, c_stride: w / 2 }; + bytes += enc.encode(&f, t as i64).unwrap().iter().map(|p| p.data.len()).sum::(); + if (t + 1) % group == 0 { + bytes += enc.flush().iter().map(|p| p.data.len()).sum::(); + } + } + bytes as f64 * 8.0 * fps / (group * groups) as f64 +} + +#[test] +fn flushing_short_gops_keeps_the_average_bitrate() { + // every flush ends a GOP of 12 after 4 pictures: the 8 that never came must not leave their + // bits to the next GOP + for b in [0, 2] { + let cfg = EncoderConfig { gop_size: 12, b_pictures: b, rate: RateControl::Abr { kbps: 600 }, ..EncoderConfig::new(176, 144, 25, 1) }; + let got = flushed_bitrate(cfg, 4, 15); + assert!((got / 600e3 - 1.0).abs() < 0.15, "b {b}: {got:.0} bit/s for 600000"); } } @@ -161,7 +363,7 @@ fn quality_follows_the_quantiser() { #[test] fn p_pictures_are_smaller_and_static_content_skips() { - let cfg = EncoderConfig { gop_size: 8, ..EncoderConfig::new(128, 64, 25, 1) }; + let cfg = EncoderConfig { gop_size: 8, b_pictures: 0, ..EncoderConfig::new(128, 64, 25, 1) }; let mut enc = Encoder::new(cfg).unwrap(); let (y, cb, cr) = scene(128, 64, 64, 32, 0); let f = YuvFrame { y: &y, cb: &cb, cr: &cr, y_stride: 128, c_stride: 64 }; @@ -191,7 +393,7 @@ fn the_sequence_header_describes_the_stream() { let enc = Encoder::new(cfg).unwrap(); assert_eq!(enc.profile_and_level(), pli, "{name}"); let info = probe(enc.sequence_header()).unwrap(); - assert_eq!((info.width, info.height, info.mpeg2, info.progressive_sequence, info.low_delay), (w, h, true, true, true)); + assert_eq!((info.width, info.height, info.mpeg2, info.progressive_sequence, info.low_delay), (w, h, true, true, false)); assert_eq!(info.codec_name(), format!("MPEG-2 Video ({name})")); assert_eq!(aspect, AspectRatio::Square); assert_eq!(info.sar, (1, 1)); @@ -199,6 +401,9 @@ fn the_sequence_header_describes_the_stream() { let enc = Encoder::new(EncoderConfig { aspect: AspectRatio::Dar16x9, ..EncoderConfig::new(720, 576, 30000, 1001) }).unwrap(); let info = probe(enc.sequence_header()).unwrap(); assert_eq!((info.dar, info.frame_rate), (Some((16, 9)), Some((30000, 1001)))); + // without B pictures the stream is low delay + let enc = Encoder::new(EncoderConfig { b_pictures: 0, ..EncoderConfig::new(720, 576, 25, 1) }).unwrap(); + assert!(probe(enc.sequence_header()).unwrap().low_delay); } #[test] @@ -214,6 +419,8 @@ fn bad_configurations_are_refused() { EncoderConfig { fps_den: 0, ..ok.clone() }, EncoderConfig { gop_size: 0, ..ok.clone() }, EncoderConfig { gop_size: 1024, ..ok.clone() }, + EncoderConfig { b_pictures: 5, ..ok.clone() }, + EncoderConfig { gop_size: 1023, b_pictures: 2, ..ok.clone() }, EncoderConfig { rate: RateControl::Quantiser(0), ..ok.clone() }, EncoderConfig { rate: RateControl::Quantiser(32), ..ok.clone() }, EncoderConfig { rate: RateControl::Abr { kbps: 0 }, ..ok.clone() }, @@ -231,7 +438,7 @@ fn bad_configurations_are_refused() { #[test] fn short_or_inconsistent_planes_are_refused() { - let mut enc = Encoder::new(EncoderConfig::new(32, 32, 25, 1)).unwrap(); + let mut enc = Encoder::new(EncoderConfig { b_pictures: 0, ..EncoderConfig::new(32, 32, 25, 1) }).unwrap(); let (y, c) = (vec![0u8; 32 * 32], vec![0u8; 16 * 16]); let ok = YuvFrame { y: &y, cb: &c, cr: &c, y_stride: 32, c_stride: 16 }; assert!(enc.encode(&ok, 0).is_ok()); diff --git a/crates/video/src/mpeg2/mod.rs b/crates/video/src/mpeg2/mod.rs index ae2bb1682..4fd59dced 100644 --- a/crates/video/src/mpeg2/mod.rs +++ b/crates/video/src/mpeg2/mod.rs @@ -1,5 +1,5 @@ //! Clean-room MPEG-2 video (ITU-T H.262 | ISO/IEC 13818-2) and MPEG-1 video (ISO/IEC 11172-2) -//! decoder, and the start of an MPEG-2 encoder ([`encode`], feature `mpeg2-encode`: I and P +//! decoder, and the start of an MPEG-2 encoder ([`encode`], feature `mpeg2-encode`: I, P and B //! pictures). //! //! - Main Profile at Low/Main/High-1440/High Level, 4:2:2 Profile (XDCAM HD422, IMX / D-10), diff --git a/crates/video/tests/mpeg2/encode.rs b/crates/video/tests/mpeg2/encode.rs index 5cb2377dc..122fe2d71 100644 --- a/crates/video/tests/mpeg2/encode.rs +++ b/crates/video/tests/mpeg2/encode.rs @@ -6,7 +6,7 @@ use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; -use craft_video::mpeg2::{ChromaFormat, Encoder, EncoderConfig, RateControl, ReconFrame, YuvFrame}; +use craft_video::mpeg2::{ChromaFormat, Encoder, EncoderConfig, Packet, RateControl, ReconFrame, YuvFrame}; use crate::common::{Stats, fixture_dir, pix_fmt_of}; use crate::oracle::{MAX_DIFFERING, MIN_PSNR, TOL_MAX}; @@ -39,24 +39,22 @@ fn sizes(cfg: &EncoderConfig) -> (usize, usize, usize) { (w * h, w / 2 * ch, w / 2) } -/// Encode `raw` frames; returns the elementary stream and the reconstruction. -fn encode(cfg: &EncoderConfig, raw: &[u8]) -> (Vec, Vec) { +/// Encode `raw` frames; returns the elementary stream, the reconstruction (display order) and +/// the packets (decoding order). +fn encode(cfg: &EncoderConfig, raw: &[u8]) -> (Vec, Vec, Vec) { let (ys, cs, cw) = sizes(cfg); let fsize = ys + 2 * cs; let mut enc = Encoder::new(cfg.clone()).expect("config"); enc.set_recon_capture(true); - let mut es = Vec::new(); + let mut packets = Vec::new(); for (i, f) in raw.chunks_exact(fsize).enumerate() { let frame = YuvFrame { y: &f[..ys], cb: &f[ys..ys + cs], cr: &f[ys + cs..], y_stride: cfg.width as usize, c_stride: cw }; - for p in enc.encode(&frame, i as i64).expect("encode") { - es.extend_from_slice(&p.data); - } - } - for p in enc.flush() { - es.extend_from_slice(&p.data); + packets.extend(enc.encode(&frame, i as i64).expect("encode")); } + packets.extend(enc.flush()); + let mut es: Vec = packets.iter().flat_map(|p| p.data.iter().copied()).collect(); es.extend_from_slice(enc.end_of_sequence()); - (es, enc.take_recon()) + (es, enc.take_recon(), packets) } /// ffmpeg's decode of `file`, failing on any decoding error. @@ -121,10 +119,11 @@ fn check(ff: &Path, c: &Case) -> (Vec, f64) { let raw = source(ff, c.cfg.width, c.cfg.height, c.frames, pix_fmt); let (ys, cs, _) = sizes(&c.cfg); let fsize = ys + 2 * cs; - let (es, recon) = encode(&c.cfg, &raw); + let (es, recon, packets) = encode(&c.cfg, &raw); let path = write(c.name, &es); let decoded = ffmpeg_decode(ff, &path, pix_fmt); assert_eq!(decoded.len(), c.frames * fsize, "{}: ffmpeg's frame count", c.name); + assert_eq!(recon.len(), c.frames, "{}: reconstruction count", c.name); let mut all = Stats::default(); for (i, (r, f)) in recon.iter().zip(decoded.chunks_exact(fsize)).enumerate() { let mut s = Stats::default(); @@ -139,15 +138,21 @@ fn check(ff: &Path, c: &Case) -> (Vec, f64) { assert!(quality >= c.min_psnr, "{}: luma PSNR {quality:.2} dB against the source", c.name); if let Some((stream, types)) = ffprobe(&path) { assert_eq!(stream, c.probe, "{}: ffprobe", c.name); - let gop = c.cfg.gop_size as usize; - let want: Vec<&str> = (0..c.frames).map(|i| if i % gop == 0 { "I" } else { "P" }).collect(); + // ffprobe lists the pictures in display order + let mut shown: Vec<&Packet> = packets.iter().collect(); + shown.sort_by_key(|p| p.pts); + let want: Vec = shown.iter().map(|p| format!("{:?}", p.picture_type)).collect(); assert_eq!(types, want, "{}: picture types", c.name); } + let count = |t: &str| packets.iter().filter(|p| format!("{:?}", p.picture_type) == t).count(); let kbps = es.len() as f64 * 8.0 * 25.0 / c.frames as f64 / 1000.0; println!( - "{}: {} frames, {kbps:.0} kbit/s, luma PSNR {quality:.2} dB, max diff to ffmpeg {}, {:.3}% differ", + "{}: {} frames ({} I, {} P, {} B), {kbps:.0} kbit/s, luma PSNR {quality:.2} dB, max diff to ffmpeg {}, {:.3}% differ", c.name, c.frames, + count("I"), + count("P"), + count("B"), all.max, 100.0 * all.diff as f64 / all.n as f64 ); @@ -160,6 +165,21 @@ fn ffmpeg_decodes_our_streams() { let cif = EncoderConfig { gop_size: 12, ..EncoderConfig::new(352, 288, 25, 1) }; let cases = [ Case { name: "cif_q4", cfg: cif.clone(), frames: 25, probe: "mpeg2video,Main,10,yuv420p", min_psnr: 36.0 }, + Case { name: "cif_ip", cfg: EncoderConfig { b_pictures: 0, ..cif.clone() }, frames: 25, probe: "mpeg2video,Main,10,yuv420p", min_psnr: 36.0 }, + Case { + name: "cif_open_gop", + cfg: EncoderConfig { closed_gop: false, gop_size: 15, ..cif.clone() }, + frames: 31, + probe: "mpeg2video,Main,10,yuv420p", + min_psnr: 35.0, + }, + Case { + name: "cif_b4", + cfg: EncoderConfig { b_pictures: 4, gop_size: 10, ..cif.clone() }, + frames: 23, + probe: "mpeg2video,Main,10,yuv420p", + min_psnr: 34.0, + }, Case { name: "cif_intra", cfg: EncoderConfig { gop_size: 1, rate: RateControl::Quantiser(8), ..cif.clone() }, @@ -191,10 +211,14 @@ fn ffmpeg_decodes_our_streams() { #[test] fn average_bitrate_reaches_its_target() { let Some(ff) = craft_media_testkit::ffmpeg_or_skip("mpeg2 encoder bitrate") else { return }; - for kbps in [800u32, 2500] { + for (kbps, b) in [(800u32, 2), (2500, 2), (800, 0)] { let c = Case { - name: if kbps == 800 { "cif_abr_800" } else { "cif_abr_2500" }, - cfg: EncoderConfig { rate: RateControl::Abr { kbps }, gop_size: 12, ..EncoderConfig::new(352, 288, 25, 1) }, + name: match (kbps, b) { + (800, 0) => "cif_abr_800_ip", + (800, _) => "cif_abr_800", + _ => "cif_abr_2500", + }, + cfg: EncoderConfig { rate: RateControl::Abr { kbps }, gop_size: 12, b_pictures: b, ..EncoderConfig::new(352, 288, 25, 1) }, frames: 72, probe: "mpeg2video,Main,10,yuv420p", min_psnr: 26.0,