mirror of
https://github.com/storytold/craft-libs.git
synced 2026-11-16 23:42:27 +00:00
craft-video: MPEG-2 encoder: flush and timestamp fixes from review (#1133)
- flush ends the GOP for average bitrate too: the budget of the pictures the GOP did not get is taken back, so flushed short GOPs no longer raise the bitrate. - A lone picture flushed before a second sets the step gets dts = pts - 1, so decoding times still increase if encoding goes on. - Presentation times whose decoding time does not fit in an i64 are refused with BadTimestamp before any state changes. - Tests: the skip range test uses an f_code that can carry its vector, and the reconstruction count is checked before comparing pictures.
This commit is contained in:
6 files changed
+128
-8
No files matched your search
@@ -109,6 +109,8 @@ What it does:
|
||||
come out in decoding order; a B picture is held until the anchor after it is coded, and
|
||||
`flush` codes the last held picture as a P picture. With B pictures, `dts` is the presentation
|
||||
times in decoding order one picture interval earlier, so presentation times must step evenly.
|
||||
A single picture flushed before a second one gets `dts = pts - 1`. With average bitrate, a
|
||||
GOP that `flush` cuts short gives back the budget of the pictures it did not get.
|
||||
`low_delay` is set only without B pictures.
|
||||
- GOPs are closed or open. The B pictures shown before a GOP's I picture (coded after it) predict
|
||||
backward only in a closed GOP, and from the previous GOP's last anchor too in an open one.
|
||||
|
||||
@@ -171,7 +171,8 @@ pub struct Packet {
|
||||
pub data: Vec<u8>,
|
||||
pub pts: i64,
|
||||
/// Decoding timestamp: `pts` without B pictures; with them, the presentation times in
|
||||
/// decoding order one picture interval earlier, so `dts <= pts`.
|
||||
/// decoding order one picture interval earlier, so `dts <= pts`. A stream of one picture has
|
||||
/// no interval: its `dts` is `pts - 1`.
|
||||
pub dts: i64,
|
||||
/// An I picture (random access point; in an open GOP the B pictures after it in decoding
|
||||
/// order also need the previous GOP).
|
||||
@@ -449,6 +450,14 @@ impl Encoder {
|
||||
if let Some(last) = self.held.pop() {
|
||||
self.code_anchor(last, PictureType::P, &mut out);
|
||||
}
|
||||
// the GOP ends here: its budget covered `gop_size` pictures, so take back the share of
|
||||
// those that never came, or every flushed GOP would raise the bitrate of the next
|
||||
if self.latest.is_some() {
|
||||
let missing = u64::from(self.cfg.gop_size).saturating_sub(self.count - self.gop_start);
|
||||
if let Some(rc) = self.rc.as_mut() {
|
||||
rc.end_gop(missing as u32);
|
||||
}
|
||||
}
|
||||
self.older = None;
|
||||
self.latest = None;
|
||||
self.anchor_mvs.clear();
|
||||
@@ -596,16 +605,22 @@ impl Encoder {
|
||||
acts.iter().map(|&a| qcode_for(nl, base * (2.0 * a + avg) / (a + 2.0 * avg))).collect()
|
||||
}
|
||||
|
||||
/// With B pictures, presentation times must step by a constant amount. Checked before any
|
||||
/// state changes.
|
||||
/// With B pictures, presentation times must step by a constant amount, and the decoding
|
||||
/// times (one step earlier) must fit in an `i64`. Checked before any state changes.
|
||||
fn check_pts(&self, pts: i64) -> Result<Option<i64>, EncodeError> {
|
||||
if self.cfg.b_pictures == 0 {
|
||||
return Ok(None);
|
||||
}
|
||||
let Some(last) = self.last_pts else { return Ok(None) };
|
||||
let no_dts = |pts: i64, step: i64| EncodeError::BadTimestamp(format!("{pts}: no decoding time {step} earlier"));
|
||||
let Some(last) = self.last_pts else {
|
||||
// flushed before a second picture, this one decodes at `pts - 1` (see `release`)
|
||||
return if pts == i64::MIN { Err(no_dts(pts, 1)) } else { Ok(None) };
|
||||
};
|
||||
let step = pts.checked_sub(last).filter(|&d| d > 0).ok_or_else(|| EncodeError::BadTimestamp(format!("{pts} after {last}")))?;
|
||||
match self.pts_step {
|
||||
Some(s) if s != step => Err(EncodeError::BadTimestamp(format!("{pts} after {last}: step {step}, earlier {s}"))),
|
||||
// the earliest decoding time is the earlier picture's time less the step
|
||||
None if last.checked_sub(step).is_none() => Err(no_dts(last, step)),
|
||||
_ => Ok(Some(step)),
|
||||
}
|
||||
}
|
||||
@@ -756,10 +771,14 @@ impl Encoder {
|
||||
if self.pts_step.is_none() && !flushing {
|
||||
return Vec::new();
|
||||
}
|
||||
let off = self.pts_step.unwrap_or(0);
|
||||
// A lone picture flushed before a second one set the step decodes one unit early, the
|
||||
// smallest step there is: if encoding goes on, the next picture decodes at this one's
|
||||
// presentation time whatever the step turns out to be, and that must be later.
|
||||
let off = self.pts_step.unwrap_or(1);
|
||||
let mut res = std::mem::take(&mut self.out);
|
||||
for p in &mut res {
|
||||
p.dts -= off;
|
||||
// `check_pts` refused the times this would not fit for
|
||||
p.dts = p.dts.saturating_sub(off);
|
||||
}
|
||||
res
|
||||
}
|
||||
|
||||
@@ -614,11 +614,14 @@ mod tests {
|
||||
inter_matrix: &[16; 64],
|
||||
nonlinear: true,
|
||||
dc_prec: 1,
|
||||
f_code: 2,
|
||||
search: 15,
|
||||
f_code: 3,
|
||||
search: 31,
|
||||
mb_qcode: &[4; 3],
|
||||
};
|
||||
let bad = BPred { dirs: MB_FWD, mvs: [[34, 0], [0, 0]] };
|
||||
// the bitstream can carry the vector (f_code 3: -64..64): only the position rules it out
|
||||
let at = |mbx| Search { reference: &f, src: &[0; 256], mbx, mby: 0, f_code: ctx.f_code, range: ctx.search, pmv: bad.mvs[0], lambda: 1 };
|
||||
assert!(at(0).valid(bad.mvs[0]) && !at(1).valid(bad.mvs[0]));
|
||||
// a source that the out-of-range (edge-clamped) prediction would match exactly
|
||||
let src = predict_b(&ctx, bad, 1, 0);
|
||||
let st = SliceSt { qcode: 4, dc: [256; 3], pmv: bad.mvs, skipped: 0, left_mv: bad.mvs, last_dirs: MB_FWD };
|
||||
|
||||
@@ -50,6 +50,14 @@ impl RateCtl {
|
||||
self.nb = nb;
|
||||
}
|
||||
|
||||
/// The GOP ends `missing` pictures short of what `start_gop` budgeted (the encoder was
|
||||
/// flushed): their bits are not carried into the next GOP.
|
||||
pub fn end_gop(&mut self, missing: u32) {
|
||||
self.remaining -= self.bps * f64::from(missing) / self.fps;
|
||||
self.np = 0;
|
||||
self.nb = 0;
|
||||
}
|
||||
|
||||
/// quantiser_scale for the next picture of type `ptype`.
|
||||
pub fn qscale(&self, ptype: PictureType) -> f64 {
|
||||
let floor = self.bps / (8.0 * self.fps);
|
||||
@@ -119,6 +127,22 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_gop_cut_short_gives_its_budget_back() {
|
||||
let mut rc = RateCtl::new(4e6, 25.0, (1.0, 112.0));
|
||||
for _ in 0..20 {
|
||||
// a GOP of 12 that ends after its I picture and two P pictures
|
||||
rc.start_gop(11, 0);
|
||||
for t in [PictureType::I, PictureType::P, PictureType::P] {
|
||||
let q = rc.qscale(t);
|
||||
rc.update(t, (1e7 / q) as u64, q);
|
||||
}
|
||||
rc.end_gop(9);
|
||||
}
|
||||
// what is left is the three pictures' rounding, not nine pictures' budget per GOP
|
||||
assert!(rc.remaining.abs() < 3.0 * 4e6 / 25.0, "{}", rc.remaining);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_quantiser_stays_in_range() {
|
||||
let mut rc = RateCtl::new(1e6, 25.0, (2.0, 62.0));
|
||||
|
||||
@@ -80,6 +80,7 @@ fn assert_decodes_to_recon(name: &str, cfg: EncoderConfig, frames: usize) -> Vec
|
||||
assert!(packets.windows(2).all(|w| w[0].dts < w[1].dts) && packets.iter().all(|p| p.dts <= p.pts), "{name}: decoding times");
|
||||
let pics = decode(&packets);
|
||||
assert_eq!(pics.len(), frames, "{name}: picture count");
|
||||
assert_eq!(recon.len(), frames, "{name}: reconstruction count");
|
||||
for (i, (p, r)) in pics.iter().zip(&recon).enumerate() {
|
||||
assert_eq!(p.pts, i as i64, "{name}: display order");
|
||||
for (plane, (a, b)) in [("Y", (&p.y, &r.y)), ("Cb", (&p.cb, &r.cb)), ("Cr", (&p.cr, &r.cr))] {
|
||||
@@ -262,6 +263,76 @@ fn presentation_times_must_step_evenly_with_b_pictures() {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoding_times_increase_after_a_lone_picture_is_flushed() {
|
||||
let mut enc = Encoder::new(EncoderConfig::new(32, 32, 25, 1)).unwrap();
|
||||
let (y, c) = (vec![0u8; 32 * 32], vec![0u8; 16 * 16]);
|
||||
let f = YuvFrame { y: &y, cb: &c, cr: &c, y_stride: 32, c_stride: 16 };
|
||||
// one picture, flushed before a second sets the step: it decodes one unit early
|
||||
assert!(enc.encode(&f, 100).unwrap().is_empty());
|
||||
let mut packets = enc.flush();
|
||||
assert_eq!(packets.iter().map(|p| (p.pts, p.dts)).collect::<Vec<_>>(), [(100, 99)]);
|
||||
// encoding goes on: the next picture decodes at the first one's presentation time
|
||||
for pts in [105, 110] {
|
||||
packets.extend(enc.encode(&f, pts).unwrap());
|
||||
}
|
||||
packets.extend(enc.flush());
|
||||
assert_eq!(packets.iter().map(|p| (p.pts, p.dts)).collect::<Vec<_>>(), [(100, 99), (105, 100), (110, 105)]);
|
||||
assert!(packets[1].keyframe);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn presentation_times_without_a_decoding_time_are_refused() {
|
||||
let (y, c) = (vec![0u8; 32 * 32], vec![0u8; 16 * 16]);
|
||||
let f = YuvFrame { y: &y, cb: &c, cr: &c, y_stride: 32, c_stride: 16 };
|
||||
// the decoding times are one step earlier: they must fit in an i64
|
||||
for (first, second) in [(i64::MIN + 1, i64::MIN + 3), (i64::MIN + 5, i64::MIN + 20)] {
|
||||
let mut enc = Encoder::new(EncoderConfig::new(32, 32, 25, 1)).unwrap();
|
||||
assert!(matches!(enc.encode(&f, i64::MIN), Err(EncodeError::BadTimestamp(_))));
|
||||
assert!(enc.encode(&f, first).unwrap().is_empty());
|
||||
assert!(matches!(enc.encode(&f, second), Err(EncodeError::BadTimestamp(_))), "{second} after {first}");
|
||||
// a refused time leaves the encoder usable
|
||||
assert_eq!(enc.flush().iter().map(|p| (p.pts, p.dts)).collect::<Vec<_>>(), [(first, first - 1)]);
|
||||
}
|
||||
// the lowest times that have one
|
||||
let mut enc = Encoder::new(EncoderConfig::new(32, 32, 25, 1)).unwrap();
|
||||
let mut packets = enc.encode(&f, i64::MIN + 1).unwrap();
|
||||
packets.extend(enc.encode(&f, i64::MIN + 2).unwrap());
|
||||
packets.extend(enc.flush());
|
||||
assert_eq!(packets.iter().map(|p| p.dts).collect::<Vec<_>>(), [i64::MIN, i64::MIN + 1]);
|
||||
// without B pictures every time passes through
|
||||
let mut enc = Encoder::new(EncoderConfig { b_pictures: 0, ..EncoderConfig::new(32, 32, 25, 1) }).unwrap();
|
||||
assert_eq!(enc.encode(&f, i64::MIN).unwrap()[0].dts, i64::MIN);
|
||||
}
|
||||
|
||||
/// Average bitrate (bit/s) of the scene coded in groups of `group` pictures, flushed after each.
|
||||
fn flushed_bitrate(cfg: EncoderConfig, group: usize, groups: usize) -> f64 {
|
||||
let (w, h) = (cfg.width as usize, cfg.height as usize);
|
||||
let fps = f64::from(cfg.fps_num) / f64::from(cfg.fps_den);
|
||||
let mut enc = Encoder::new(cfg).unwrap();
|
||||
let mut bytes = 0;
|
||||
for t in 0..group * groups {
|
||||
let (y, cb, cr) = scene(w, h, w / 2, h / 2, t);
|
||||
let f = YuvFrame { y: &y, cb: &cb, cr: &cr, y_stride: w, c_stride: w / 2 };
|
||||
bytes += enc.encode(&f, t as i64).unwrap().iter().map(|p| p.data.len()).sum::<usize>();
|
||||
if (t + 1) % group == 0 {
|
||||
bytes += enc.flush().iter().map(|p| p.data.len()).sum::<usize>();
|
||||
}
|
||||
}
|
||||
bytes as f64 * 8.0 * fps / (group * groups) as f64
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn flushing_short_gops_keeps_the_average_bitrate() {
|
||||
// every flush ends a GOP of 12 after 4 pictures: the 8 that never came must not leave their
|
||||
// bits to the next GOP
|
||||
for b in [0, 2] {
|
||||
let cfg = EncoderConfig { gop_size: 12, b_pictures: b, rate: RateControl::Abr { kbps: 600 }, ..EncoderConfig::new(176, 144, 25, 1) };
|
||||
let got = flushed_bitrate(cfg, 4, 15);
|
||||
assert!((got / 600e3 - 1.0).abs() < 0.15, "b {b}: {got:.0} bit/s for 600000");
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "threads")]
|
||||
#[test]
|
||||
fn threaded_slices_give_the_same_stream() {
|
||||
|
||||
@@ -123,6 +123,7 @@ fn check(ff: &Path, c: &Case) -> (Vec<u8>, f64) {
|
||||
let path = write(c.name, &es);
|
||||
let decoded = ffmpeg_decode(ff, &path, pix_fmt);
|
||||
assert_eq!(decoded.len(), c.frames * fsize, "{}: ffmpeg's frame count", c.name);
|
||||
assert_eq!(recon.len(), c.frames, "{}: reconstruction count", c.name);
|
||||
let mut all = Stats::default();
|
||||
for (i, (r, f)) in recon.iter().zip(decoded.chunks_exact(fsize)).enumerate() {
|
||||
let mut s = Stats::default();
|
||||
|
||||
Reference in new issue
Block a user