From c83e8ba250df957c7e12377689ae074f3ee19727 Mon Sep 17 00:00:00 2001 From: alehel Date: Sun, 15 Nov 2026 00:07:33 +0100 Subject: [PATCH] craft-video: MPEG-2 encoder: flush and timestamp fixes from review (#1133) - flush ends the GOP for average bitrate too: the budget of the pictures the GOP did not get is taken back, so flushed short GOPs no longer raise the bitrate. - A lone picture flushed before a second sets the step gets dts = pts - 1, so decoding times still increase if encoding goes on. - Presentation times whose decoding time does not fit in an i64 are refused with BadTimestamp before any state changes. - Tests: the skip range test uses an f_code that can carry its vector, and the reconstruction count is checked before comparing pictures. --- crates/video/src/mpeg2/README.md | 2 + crates/video/src/mpeg2/encode/mod.rs | 31 +++++++++-- crates/video/src/mpeg2/encode/picture.rs | 7 ++- crates/video/src/mpeg2/encode/rate.rs | 24 ++++++++ crates/video/src/mpeg2/encode/tests.rs | 71 ++++++++++++++++++++++++ crates/video/tests/mpeg2/encode.rs | 1 + 6 files changed, 128 insertions(+), 8 deletions(-) diff --git a/crates/video/src/mpeg2/README.md b/crates/video/src/mpeg2/README.md index a8502f696..069fd67bd 100644 --- a/crates/video/src/mpeg2/README.md +++ b/crates/video/src/mpeg2/README.md @@ -109,6 +109,8 @@ What it does: come out in decoding order; a B picture is held until the anchor after it is coded, and `flush` codes the last held picture as a P picture. With B pictures, `dts` is the presentation times in decoding order one picture interval earlier, so presentation times must step evenly. + A single picture flushed before a second one gets `dts = pts - 1`. With average bitrate, a + GOP that `flush` cuts short gives back the budget of the pictures it did not get. `low_delay` is set only without B pictures. - GOPs are closed or open. The B pictures shown before a GOP's I picture (coded after it) predict backward only in a closed GOP, and from the previous GOP's last anchor too in an open one. diff --git a/crates/video/src/mpeg2/encode/mod.rs b/crates/video/src/mpeg2/encode/mod.rs index 073847f8c..af32b271d 100644 --- a/crates/video/src/mpeg2/encode/mod.rs +++ b/crates/video/src/mpeg2/encode/mod.rs @@ -171,7 +171,8 @@ pub struct Packet { pub data: Vec, pub pts: i64, /// Decoding timestamp: `pts` without B pictures; with them, the presentation times in - /// decoding order one picture interval earlier, so `dts <= pts`. + /// decoding order one picture interval earlier, so `dts <= pts`. A stream of one picture has + /// no interval: its `dts` is `pts - 1`. pub dts: i64, /// An I picture (random access point; in an open GOP the B pictures after it in decoding /// order also need the previous GOP). @@ -449,6 +450,14 @@ impl Encoder { if let Some(last) = self.held.pop() { self.code_anchor(last, PictureType::P, &mut out); } + // the GOP ends here: its budget covered `gop_size` pictures, so take back the share of + // those that never came, or every flushed GOP would raise the bitrate of the next + if self.latest.is_some() { + let missing = u64::from(self.cfg.gop_size).saturating_sub(self.count - self.gop_start); + if let Some(rc) = self.rc.as_mut() { + rc.end_gop(missing as u32); + } + } self.older = None; self.latest = None; self.anchor_mvs.clear(); @@ -596,16 +605,22 @@ impl Encoder { acts.iter().map(|&a| qcode_for(nl, base * (2.0 * a + avg) / (a + 2.0 * avg))).collect() } - /// With B pictures, presentation times must step by a constant amount. Checked before any - /// state changes. + /// With B pictures, presentation times must step by a constant amount, and the decoding + /// times (one step earlier) must fit in an `i64`. Checked before any state changes. fn check_pts(&self, pts: i64) -> Result, EncodeError> { if self.cfg.b_pictures == 0 { return Ok(None); } - let Some(last) = self.last_pts else { return Ok(None) }; + let no_dts = |pts: i64, step: i64| EncodeError::BadTimestamp(format!("{pts}: no decoding time {step} earlier")); + let Some(last) = self.last_pts else { + // flushed before a second picture, this one decodes at `pts - 1` (see `release`) + return if pts == i64::MIN { Err(no_dts(pts, 1)) } else { Ok(None) }; + }; let step = pts.checked_sub(last).filter(|&d| d > 0).ok_or_else(|| EncodeError::BadTimestamp(format!("{pts} after {last}")))?; match self.pts_step { Some(s) if s != step => Err(EncodeError::BadTimestamp(format!("{pts} after {last}: step {step}, earlier {s}"))), + // the earliest decoding time is the earlier picture's time less the step + None if last.checked_sub(step).is_none() => Err(no_dts(last, step)), _ => Ok(Some(step)), } } @@ -756,10 +771,14 @@ impl Encoder { if self.pts_step.is_none() && !flushing { return Vec::new(); } - let off = self.pts_step.unwrap_or(0); + // A lone picture flushed before a second one set the step decodes one unit early, the + // smallest step there is: if encoding goes on, the next picture decodes at this one's + // presentation time whatever the step turns out to be, and that must be later. + let off = self.pts_step.unwrap_or(1); let mut res = std::mem::take(&mut self.out); for p in &mut res { - p.dts -= off; + // `check_pts` refused the times this would not fit for + p.dts = p.dts.saturating_sub(off); } res } diff --git a/crates/video/src/mpeg2/encode/picture.rs b/crates/video/src/mpeg2/encode/picture.rs index b6c177eec..0be5dbafd 100644 --- a/crates/video/src/mpeg2/encode/picture.rs +++ b/crates/video/src/mpeg2/encode/picture.rs @@ -614,11 +614,14 @@ mod tests { inter_matrix: &[16; 64], nonlinear: true, dc_prec: 1, - f_code: 2, - search: 15, + f_code: 3, + search: 31, mb_qcode: &[4; 3], }; let bad = BPred { dirs: MB_FWD, mvs: [[34, 0], [0, 0]] }; + // the bitstream can carry the vector (f_code 3: -64..64): only the position rules it out + let at = |mbx| Search { reference: &f, src: &[0; 256], mbx, mby: 0, f_code: ctx.f_code, range: ctx.search, pmv: bad.mvs[0], lambda: 1 }; + assert!(at(0).valid(bad.mvs[0]) && !at(1).valid(bad.mvs[0])); // a source that the out-of-range (edge-clamped) prediction would match exactly let src = predict_b(&ctx, bad, 1, 0); let st = SliceSt { qcode: 4, dc: [256; 3], pmv: bad.mvs, skipped: 0, left_mv: bad.mvs, last_dirs: MB_FWD }; diff --git a/crates/video/src/mpeg2/encode/rate.rs b/crates/video/src/mpeg2/encode/rate.rs index cd7de9680..96c6f2aa1 100644 --- a/crates/video/src/mpeg2/encode/rate.rs +++ b/crates/video/src/mpeg2/encode/rate.rs @@ -50,6 +50,14 @@ impl RateCtl { self.nb = nb; } + /// The GOP ends `missing` pictures short of what `start_gop` budgeted (the encoder was + /// flushed): their bits are not carried into the next GOP. + pub fn end_gop(&mut self, missing: u32) { + self.remaining -= self.bps * f64::from(missing) / self.fps; + self.np = 0; + self.nb = 0; + } + /// quantiser_scale for the next picture of type `ptype`. pub fn qscale(&self, ptype: PictureType) -> f64 { let floor = self.bps / (8.0 * self.fps); @@ -119,6 +127,22 @@ mod tests { } } + #[test] + fn a_gop_cut_short_gives_its_budget_back() { + let mut rc = RateCtl::new(4e6, 25.0, (1.0, 112.0)); + for _ in 0..20 { + // a GOP of 12 that ends after its I picture and two P pictures + rc.start_gop(11, 0); + for t in [PictureType::I, PictureType::P, PictureType::P] { + let q = rc.qscale(t); + rc.update(t, (1e7 / q) as u64, q); + } + rc.end_gop(9); + } + // what is left is the three pictures' rounding, not nine pictures' budget per GOP + assert!(rc.remaining.abs() < 3.0 * 4e6 / 25.0, "{}", rc.remaining); + } + #[test] fn the_quantiser_stays_in_range() { let mut rc = RateCtl::new(1e6, 25.0, (2.0, 62.0)); diff --git a/crates/video/src/mpeg2/encode/tests.rs b/crates/video/src/mpeg2/encode/tests.rs index c8dfc1205..01aac2b32 100644 --- a/crates/video/src/mpeg2/encode/tests.rs +++ b/crates/video/src/mpeg2/encode/tests.rs @@ -80,6 +80,7 @@ fn assert_decodes_to_recon(name: &str, cfg: EncoderConfig, frames: usize) -> Vec assert!(packets.windows(2).all(|w| w[0].dts < w[1].dts) && packets.iter().all(|p| p.dts <= p.pts), "{name}: decoding times"); let pics = decode(&packets); assert_eq!(pics.len(), frames, "{name}: picture count"); + assert_eq!(recon.len(), frames, "{name}: reconstruction count"); for (i, (p, r)) in pics.iter().zip(&recon).enumerate() { assert_eq!(p.pts, i as i64, "{name}: display order"); for (plane, (a, b)) in [("Y", (&p.y, &r.y)), ("Cb", (&p.cb, &r.cb)), ("Cr", (&p.cr, &r.cr))] { @@ -262,6 +263,76 @@ fn presentation_times_must_step_evenly_with_b_pictures() { } } +#[test] +fn decoding_times_increase_after_a_lone_picture_is_flushed() { + let mut enc = Encoder::new(EncoderConfig::new(32, 32, 25, 1)).unwrap(); + let (y, c) = (vec![0u8; 32 * 32], vec![0u8; 16 * 16]); + let f = YuvFrame { y: &y, cb: &c, cr: &c, y_stride: 32, c_stride: 16 }; + // one picture, flushed before a second sets the step: it decodes one unit early + assert!(enc.encode(&f, 100).unwrap().is_empty()); + let mut packets = enc.flush(); + assert_eq!(packets.iter().map(|p| (p.pts, p.dts)).collect::>(), [(100, 99)]); + // encoding goes on: the next picture decodes at the first one's presentation time + for pts in [105, 110] { + packets.extend(enc.encode(&f, pts).unwrap()); + } + packets.extend(enc.flush()); + assert_eq!(packets.iter().map(|p| (p.pts, p.dts)).collect::>(), [(100, 99), (105, 100), (110, 105)]); + assert!(packets[1].keyframe); +} + +#[test] +fn presentation_times_without_a_decoding_time_are_refused() { + let (y, c) = (vec![0u8; 32 * 32], vec![0u8; 16 * 16]); + let f = YuvFrame { y: &y, cb: &c, cr: &c, y_stride: 32, c_stride: 16 }; + // the decoding times are one step earlier: they must fit in an i64 + for (first, second) in [(i64::MIN + 1, i64::MIN + 3), (i64::MIN + 5, i64::MIN + 20)] { + let mut enc = Encoder::new(EncoderConfig::new(32, 32, 25, 1)).unwrap(); + assert!(matches!(enc.encode(&f, i64::MIN), Err(EncodeError::BadTimestamp(_)))); + assert!(enc.encode(&f, first).unwrap().is_empty()); + assert!(matches!(enc.encode(&f, second), Err(EncodeError::BadTimestamp(_))), "{second} after {first}"); + // a refused time leaves the encoder usable + assert_eq!(enc.flush().iter().map(|p| (p.pts, p.dts)).collect::>(), [(first, first - 1)]); + } + // the lowest times that have one + let mut enc = Encoder::new(EncoderConfig::new(32, 32, 25, 1)).unwrap(); + let mut packets = enc.encode(&f, i64::MIN + 1).unwrap(); + packets.extend(enc.encode(&f, i64::MIN + 2).unwrap()); + packets.extend(enc.flush()); + assert_eq!(packets.iter().map(|p| p.dts).collect::>(), [i64::MIN, i64::MIN + 1]); + // without B pictures every time passes through + let mut enc = Encoder::new(EncoderConfig { b_pictures: 0, ..EncoderConfig::new(32, 32, 25, 1) }).unwrap(); + assert_eq!(enc.encode(&f, i64::MIN).unwrap()[0].dts, i64::MIN); +} + +/// Average bitrate (bit/s) of the scene coded in groups of `group` pictures, flushed after each. +fn flushed_bitrate(cfg: EncoderConfig, group: usize, groups: usize) -> f64 { + let (w, h) = (cfg.width as usize, cfg.height as usize); + let fps = f64::from(cfg.fps_num) / f64::from(cfg.fps_den); + let mut enc = Encoder::new(cfg).unwrap(); + let mut bytes = 0; + for t in 0..group * groups { + let (y, cb, cr) = scene(w, h, w / 2, h / 2, t); + let f = YuvFrame { y: &y, cb: &cb, cr: &cr, y_stride: w, c_stride: w / 2 }; + bytes += enc.encode(&f, t as i64).unwrap().iter().map(|p| p.data.len()).sum::(); + if (t + 1) % group == 0 { + bytes += enc.flush().iter().map(|p| p.data.len()).sum::(); + } + } + bytes as f64 * 8.0 * fps / (group * groups) as f64 +} + +#[test] +fn flushing_short_gops_keeps_the_average_bitrate() { + // every flush ends a GOP of 12 after 4 pictures: the 8 that never came must not leave their + // bits to the next GOP + for b in [0, 2] { + let cfg = EncoderConfig { gop_size: 12, b_pictures: b, rate: RateControl::Abr { kbps: 600 }, ..EncoderConfig::new(176, 144, 25, 1) }; + let got = flushed_bitrate(cfg, 4, 15); + assert!((got / 600e3 - 1.0).abs() < 0.15, "b {b}: {got:.0} bit/s for 600000"); + } +} + #[cfg(feature = "threads")] #[test] fn threaded_slices_give_the_same_stream() { diff --git a/crates/video/tests/mpeg2/encode.rs b/crates/video/tests/mpeg2/encode.rs index eb4ec947a..122fe2d71 100644 --- a/crates/video/tests/mpeg2/encode.rs +++ b/crates/video/tests/mpeg2/encode.rs @@ -123,6 +123,7 @@ fn check(ff: &Path, c: &Case) -> (Vec, f64) { let path = write(c.name, &es); let decoded = ffmpeg_decode(ff, &path, pix_fmt); assert_eq!(decoded.len(), c.frames * fsize, "{}: ffmpeg's frame count", c.name); + assert_eq!(recon.len(), c.frames, "{}: reconstruction count", c.name); let mut all = Stats::default(); for (i, (r, f)) in recon.iter().zip(decoded.chunks_exact(fsize)).enumerate() { let mut s = Stats::default();