From 77f244cb96cedd8248b4bad0b9b9a254010f61ff Mon Sep 17 00:00:00 2001 From: Aleksander Helgaker Date: Tue, 17 Nov 2026 03:31:05 +0100 Subject: [PATCH] craft-video: VideoFrame, a shared frame type for decoder output (#1231) (#1768) --- crates/video/README.md | 20 + crates/video/src/frame/convert.rs | 371 ++++++++++++++++++ crates/video/src/frame/mod.rs | 344 +++++++++++++++++ crates/video/src/frame/tests.rs | 404 ++++++++++++++++++++ crates/video/src/hevc/decode/decoder.rs | 1 + crates/video/src/hevc/decode/mod.rs | 2 + crates/video/src/lib.rs | 4 + crates/video/tests/vp9_decode/common/mod.rs | 2 + crates/video/tests/vp9_decode/frame.rs | 46 +++ crates/video/tests/vp9_decode/main.rs | 1 + 10 files changed, 1195 insertions(+) create mode 100644 crates/video/src/frame/convert.rs create mode 100644 crates/video/src/frame/mod.rs create mode 100644 crates/video/src/frame/tests.rs create mode 100644 crates/video/tests/vp9_decode/frame.rs diff --git a/crates/video/README.md b/crates/video/README.md index a354f245e..309b2b8f8 100644 --- a/crates/video/README.md +++ b/crates/video/README.md @@ -31,3 +31,23 @@ No codec feature is on by default. | `vp9` | [`vp9`](src/vp9/decode/README.md) | VP9: decoder (`vp9-decode` alone) and encoder (`vp9-encode` alone) | | `av1` | [`av1`](src/av1/decode/README.md) | AV1 (AOMedia Video 1): decoder (`av1-decode` alone) and clean-room encoder (`av1-encode`); tiled rav1e fallback for large frames (`av1-large-encode`) | | `threads` | — | Multi-threaded slice/row parallelism via `rayon` | + +## Decoded frames + +Each decoder returns its own picture type. `frame::VideoFrame` is a codec-agnostic form that +every decoder's output converts into with `VideoFrame::try_from` (always built; each conversion +comes with its codec's feature): + +- Planes are `[Y, Cb, Cr]`, `[R, G, B]` or `[Y]` (`ColorFamily`), plus an optional alpha plane. + DNx, VP9 and AV1 (identity matrix) store RGB as G, B, R; the conversion reorders them. +- Samples keep the decoder's storage, `u8` or `u16`, with a bit depth per plane (HEVC chroma may + differ from luma). Converting moves the buffers and copies no samples. +- Chroma subsampling is a log2 shift per axis, 0 to 3 (FFV1 codes up to 3). +- Colour is given as ITU-T H.273 code points (`ColorTags`), mapped from each codec's own + signalling. Codecs that do not signal a value leave it unspecified (2). +- `pts`, sample aspect ratio, field order and the decoder's draft flag are carried over where the + decoder reports them. ProRes signals the picture's shape (4:3 or 16:9); the sample aspect + ratio is worked out from it and the frame size. + +The constructors check that every plane's buffer, stride and size fit the frame, so a picture +with inconsistent fields is an error, not an out-of-bounds read. diff --git a/crates/video/src/frame/convert.rs b/crates/video/src/frame/convert.rs new file mode 100644 index 000000000..4089f7148 --- /dev/null +++ b/crates/video/src/frame/convert.rs @@ -0,0 +1,371 @@ +//! `TryFrom` from each decoder's own output into [`VideoFrame`]. Each conversion is built with +//! the feature of its decoder. + +// Each conversion uses some of these; a build with no decoder feature uses none. +#[allow(unused_imports)] +use super::{ColorFamily, ColorTags, FieldOrder, FrameError, Plane, Samples, Subsampling, VideoFrame}; + +/// A bit depth a decoder reports as `u32`. +#[cfg(any(feature = "hevc-decode", feature = "vp9-decode"))] +fn depth(bits: u32) -> Result { + u8::try_from(bits).map_err(|_| FrameError::Plane("bit depth does not fit the sample storage")) +} + +/// A sample aspect ratio with 0 meaning not signalled. +#[cfg(any(feature = "h264-decode", feature = "hevc-decode", feature = "mpeg2-decode", feature = "dnx"))] +fn sar(num: u32, den: u32) -> Option<(u32, u32)> { + (num != 0 && den != 0).then_some((num, den)) +} + +/// The sample aspect ratio of a ProRes `aspect_ratio_information` code: 1 is square pixels, 2 and +/// 3 give the picture's shape (4:3 and 16:9), 0 and the reserved codes say nothing. +#[cfg(feature = "prores")] +pub(super) fn prores_sar(code: u8, width: u32, height: u32) -> Option<(u32, u32)> { + let (dar_w, dar_h): (u64, u64) = match code { + 1 => return Some((1, 1)), + 2 => (4, 3), + 3 => (16, 9), + _ => return None, + }; + // sar = dar * height / width, in lowest terms. Neither product can overflow `u64`. + let (mut num, mut den) = (dar_w * u64::from(height), dar_h * u64::from(width)); + let (mut a, mut b) = (num, den); + while b != 0 { + (a, b) = (b, a % b); + } + if a == 0 { + return None; + } + num /= a; + den /= a; + match (u32::try_from(num), u32::try_from(den)) { + (Ok(n), Ok(d)) if n != 0 && d != 0 => Some((n, d)), + _ => None, + } +} + +/// ProRes stores `0` for "unspecified" where H.273 uses `2`. +#[cfg(feature = "prores")] +fn h273(code: u8) -> u8 { + if code == 0 { 2 } else { code } +} + +#[cfg(feature = "h264-decode")] +impl TryFrom for VideoFrame { + type Error = FrameError; + + /// An 8-bit 4:2:0 frame. + fn try_from(p: crate::h264::decode::Picture) -> Result { + let planes = vec![ + Plane::new(p.width, p.height, p.y_stride, 8, Samples::U8(p.y))?, + Plane::new(p.chroma_width, p.chroma_height, p.uv_stride, 8, Samples::U8(p.u))?, + Plane::new(p.chroma_width, p.chroma_height, p.uv_stride, 8, Samples::U8(p.v))?, + ]; + let mut f = VideoFrame::new(p.width, p.height, ColorFamily::YCbCr, Subsampling::S420, planes, None)?; + f.color = ColorTags { primaries: p.color.primaries, transfer: p.color.transfer, matrix: p.color.matrix, full_range: p.color.full_range }; + f.pts = Some(p.pts); + f.sar = sar(p.sar.0.into(), p.sar.1.into()); + f.draft = p.draft; + Ok(f) + } +} + +#[cfg(feature = "hevc-decode")] +impl TryFrom for VideoFrame { + type Error = FrameError; + + /// A 4:2:0 or 4:2:2 frame, 8-bit storage at bit depth 8 and 16-bit above. The chroma planes + /// carry the stream's chroma bit depth, which may differ from the luma depth. + fn try_from(p: crate::hevc::decode::Picture) -> Result { + use crate::hevc::decode::{ChromaFormat, Plane as P}; + let bits = depth(p.bit_depth)?; + let chroma_bits = depth(p.bit_depth_chroma)?; + let samples = |pl: P| match pl { + P::U8(v) => Samples::U8(v), + P::U16(v) => Samples::U16(v), + }; + let planes = vec![ + Plane::new(p.width, p.height, p.y_stride, bits, samples(p.y))?, + Plane::new(p.chroma_width, p.chroma_height, p.uv_stride, chroma_bits, samples(p.u))?, + Plane::new(p.chroma_width, p.chroma_height, p.uv_stride, chroma_bits, samples(p.v))?, + ]; + let sub = match p.chroma { + ChromaFormat::C420 => Subsampling::S420, + ChromaFormat::C422 => Subsampling::S422, + }; + let mut f = VideoFrame::new(p.width, p.height, ColorFamily::YCbCr, sub, planes, None)?; + f.color = ColorTags { primaries: p.color.primaries, transfer: p.color.transfer, matrix: p.color.matrix, full_range: p.color.full_range }; + f.pts = Some(p.pts); + f.sar = sar(p.sar.0.into(), p.sar.1.into()); + f.draft = p.draft; + Ok(f) + } +} + +/// VP9 `color_space` as an H.273 matrix: 1 BT.601 (625-line), 2 BT.709, 3 SMPTE 170M, +/// 4 SMPTE 240M, 5 BT.2020 non-constant luminance, 7 RGB. +#[cfg(feature = "vp9-decode")] +pub(super) fn vp9_matrix(color_space: u8) -> u8 { + match color_space { + 1 => 5, + 2 => 1, + 3 => 6, + 4 => 7, + 5 => 9, + 7 => 0, + _ => 2, + } +} + +#[cfg(feature = "vp9-decode")] +impl TryFrom for VideoFrame { + type Error = FrameError; + + /// A YCbCr frame, or an RGB one for `color_space` 7, whose coded G, B, R planes are reordered + /// to R, G, B. VP9 signals only the matrix, so primaries and transfer are unspecified. + fn try_from(p: crate::vp9::decode::Picture) -> Result { + use crate::vp9::decode::Plane as P; + let bits = depth(p.bit_depth)?; + let samples = |pl: P| match pl { + P::U8(v) => Samples::U8(v), + P::U16(v) => Samples::U16(v), + }; + let y = Plane::new(p.width, p.height, p.y_stride, bits, samples(p.y))?; + let u = Plane::new(p.chroma_width, p.chroma_height, p.uv_stride, bits, samples(p.u))?; + let v = Plane::new(p.chroma_width, p.chroma_height, p.uv_stride, bits, samples(p.v))?; + let sub = Subsampling { x: u8::from(p.subsampling_x), y: u8::from(p.subsampling_y) }; + let rgb = p.color.color_space == 7; + let (family, planes) = if rgb { (ColorFamily::Rgb, vec![v, y, u]) } else { (ColorFamily::YCbCr, vec![y, u, v]) }; + let mut f = VideoFrame::new(p.width, p.height, family, sub, planes, None)?; + f.color = ColorTags { matrix: vp9_matrix(p.color.color_space), full_range: p.color.full_range, ..ColorTags::UNSPECIFIED }; + f.pts = Some(p.pts); + f.draft = p.draft; + Ok(f) + } +} + +#[cfg(feature = "av1-decode")] +impl TryFrom for VideoFrame { + type Error = FrameError; + + /// A YCbCr frame, a gray one for a monochrome stream, or an RGB one for a 4:4:4 stream with + /// the identity matrix (`matrix_coefficients` 0), whose coded G, B, R planes are reordered to + /// R, G, B; 16-bit storage at every bit depth. + fn try_from(p: crate::av1::decode::Picture) -> Result { + let [y, u, v] = p.planes; + let y = Plane::packed(p.width, p.height, p.bit_depth, Samples::U16(y))?; + let (family, sub, planes) = if p.mono_chrome { + (ColorFamily::Gray, Subsampling::S444, vec![y]) + } else { + let sub = Subsampling { x: p.subsampling_x, y: p.subsampling_y }; + let (cw, ch) = sub.chroma_size(p.width, p.height); + let u = Plane::packed(cw, ch, p.bit_depth, Samples::U16(u))?; + let v = Plane::packed(cw, ch, p.bit_depth, Samples::U16(v))?; + // AV1 requires 4:4:4 with the identity matrix; a subsampled picture tagged with it + // cannot be RGB and stays as coded. + if p.matrix_coefficients == 0 && sub == Subsampling::S444 { + (ColorFamily::Rgb, sub, vec![v, y, u]) + } else { + (ColorFamily::YCbCr, sub, vec![y, u, v]) + } + }; + let mut f = VideoFrame::new(p.width, p.height, family, sub, planes, None)?; + f.color = + ColorTags { primaries: p.color_primaries, transfer: p.transfer_characteristics, matrix: p.matrix_coefficients, full_range: p.full_range }; + f.pts = Some(p.pts); + f.draft = p.draft; + Ok(f) + } +} + +#[cfg(feature = "mpeg2-decode")] +impl TryFrom for VideoFrame { + type Error = FrameError; + + /// An 8-bit frame. MPEG-1 and MPEG-2 have no full-range flag, so the range is limited. + fn try_from(p: crate::mpeg2::Picture) -> Result { + use crate::mpeg2::{ChromaFormat, FieldOrder as F}; + let sub = match p.chroma { + ChromaFormat::Yuv420 => Subsampling::S420, + ChromaFormat::Yuv422 => Subsampling::S422, + ChromaFormat::Yuv444 => Subsampling::S444, + }; + let (cw, ch) = (p.chroma_width(), p.chroma_height()); + let field_order = match p.field_order() { + None => FieldOrder::Progressive, + Some(F::TopFirst) => FieldOrder::TopFieldFirst, + Some(F::BottomFirst) => FieldOrder::BottomFieldFirst, + }; + let planes = vec![ + Plane::packed(p.width, p.height, 8, Samples::U8(p.y))?, + Plane::packed(cw, ch, 8, Samples::U8(p.cb))?, + Plane::packed(cw, ch, 8, Samples::U8(p.cr))?, + ]; + let mut f = VideoFrame::new(p.width, p.height, ColorFamily::YCbCr, sub, planes, None)?; + if let Some((primaries, transfer, matrix)) = p.info.colour { + f.color = ColorTags { primaries, transfer, matrix, full_range: false }; + } + f.pts = Some(p.pts); + f.sar = sar(p.info.sar.0, p.info.sar.1); + f.field_order = field_order; + Ok(f) + } +} + +#[cfg(feature = "vp8-decode")] +impl TryFrom for VideoFrame { + type Error = FrameError; + + /// An 8-bit 4:2:0 frame. `color_space` 0 (BT.601 YCbCr) maps to matrix 6; VP8 signals no + /// primaries, transfer or range. + fn try_from(p: crate::vp8::Picture) -> Result { + let planes = vec![ + Plane::packed(p.width, p.height, 8, Samples::U8(p.y))?, + Plane::packed(p.chroma_width, p.chroma_height, 8, Samples::U8(p.u))?, + Plane::packed(p.chroma_width, p.chroma_height, 8, Samples::U8(p.v))?, + ]; + let mut f = VideoFrame::new(p.width, p.height, ColorFamily::YCbCr, Subsampling::S420, planes, None)?; + if p.color_space == 0 { + f.color.matrix = 6; + } + f.pts = Some(p.pts); + Ok(f) + } +} + +#[cfg(feature = "prores")] +impl TryFrom for VideoFrame { + type Error = FrameError; + + /// A 4:2:2 or 4:4:4 frame with optional alpha, 16-bit storage. The range is limited. The + /// sample aspect ratio follows from `aspect_ratio` and the frame size. + fn try_from(p: crate::prores::Frame) -> Result { + use crate::prores::{ChromaFormat, Interlace}; + let sub = match p.chroma { + ChromaFormat::Yuv422 => Subsampling::S422, + ChromaFormat::Yuv444 => Subsampling::S444, + }; + let (cw, ch) = sub.chroma_size(p.width, p.height); + let planes = vec![ + Plane::packed(p.width, p.height, p.bit_depth, Samples::U16(p.y))?, + Plane::packed(cw, ch, p.bit_depth, Samples::U16(p.cb))?, + Plane::packed(cw, ch, p.bit_depth, Samples::U16(p.cr))?, + ]; + let alpha = p.alpha.map(|a| Plane::packed(p.width, p.height, p.bit_depth, Samples::U16(a))).transpose()?; + let mut f = VideoFrame::new(p.width, p.height, ColorFamily::YCbCr, sub, planes, alpha)?; + f.color = ColorTags { primaries: h273(p.color.primaries), transfer: h273(p.color.transfer), matrix: h273(p.color.matrix), full_range: false }; + f.sar = prores_sar(p.aspect_ratio, p.width, p.height); + f.field_order = match p.interlace { + Interlace::Progressive => FieldOrder::Progressive, + Interlace::TopFieldFirst => FieldOrder::TopFieldFirst, + Interlace::BottomFieldFirst => FieldOrder::BottomFieldFirst, + }; + Ok(f) + } +} + +#[cfg(feature = "dnx")] +impl TryFrom for VideoFrame { + type Error = FrameError; + + /// A YCbCr frame, or an RGB one whose stored G, B, R planes are reordered to R, G, B; optional + /// alpha, 16-bit storage. The colour volume (`CLV`) gives only the matrix; primaries, transfer + /// and range are not signalled. + fn try_from(p: crate::dnx::Frame) -> Result { + use crate::dnx::{ChromaFormat, ColorVolume, Interlace}; + let sub = match p.chroma { + ChromaFormat::Yuv420 => Subsampling::S420, + ChromaFormat::Yuv422 => Subsampling::S422, + ChromaFormat::Yuv444 => Subsampling::S444, + }; + let (cw, ch) = sub.chroma_size(p.width, p.height); + let y = Plane::packed(p.width, p.height, p.bit_depth, Samples::U16(p.y))?; + let cb = Plane::packed(cw, ch, p.bit_depth, Samples::U16(p.cb))?; + let cr = Plane::packed(cw, ch, p.bit_depth, Samples::U16(p.cr))?; + let (family, planes) = if p.rgb { (ColorFamily::Rgb, vec![cr, y, cb]) } else { (ColorFamily::YCbCr, vec![y, cb, cr]) }; + let alpha = p.alpha.map(|a| Plane::packed(p.width, p.height, p.bit_depth, Samples::U16(a))).transpose()?; + let mut f = VideoFrame::new(p.width, p.height, family, sub, planes, alpha)?; + f.color.matrix = match p.color_volume { + _ if p.rgb => 0, + ColorVolume::Bt709 => 1, + ColorVolume::Bt2020Ncl => 9, + ColorVolume::Bt2020Cl => 10, + ColorVolume::OutOfBand => 2, + }; + f.sar = sar(p.par.0.into(), p.par.1.into()); + f.field_order = match p.interlace { + Interlace::Progressive => FieldOrder::Progressive, + Interlace::TopFieldFirst => FieldOrder::TopFieldFirst, + Interlace::BottomFieldFirst => FieldOrder::BottomFieldFirst, + }; + Ok(f) + } +} + +#[cfg(any(feature = "apv-decode", feature = "apv-encode"))] +impl TryFrom for VideoFrame { + type Error = FrameError; + + /// A YCbCr or gray frame, 16-bit storage. 4:4:4:4 is 4:4:4 with its fourth component as alpha. + fn try_from(p: crate::apv::Frame) -> Result { + use crate::apv::ChromaFormat; + let y = Plane::packed(p.width, p.height, p.bit_depth, Samples::U16(p.y))?; + let (family, sub, planes) = match p.chroma { + ChromaFormat::Monochrome => (ColorFamily::Gray, Subsampling::S444, vec![y]), + c => { + let sub = if c == ChromaFormat::Yuv422 { Subsampling::S422 } else { Subsampling::S444 }; + let (cw, ch) = sub.chroma_size(p.width, p.height); + let cb = Plane::packed(cw, ch, p.bit_depth, Samples::U16(p.cb))?; + let cr = Plane::packed(cw, ch, p.bit_depth, Samples::U16(p.cr))?; + (ColorFamily::YCbCr, sub, vec![y, cb, cr]) + } + }; + let alpha = p.alpha.map(|a| Plane::packed(p.width, p.height, p.bit_depth, Samples::U16(a))).transpose()?; + let mut f = VideoFrame::new(p.width, p.height, family, sub, planes, alpha)?; + f.color = ColorTags { primaries: p.color.primaries, transfer: p.color.transfer, matrix: p.color.matrix, full_range: p.color.full_range }; + Ok(f) + } +} + +#[cfg(feature = "ffv1")] +impl TryFrom for VideoFrame { + type Error = FrameError; + + /// A YCbCr, RGB or gray frame with optional transparency, 16-bit storage. FFV1 carries no + /// colour description; RGB frames are tagged with the identity matrix and full range. + fn try_from(p: crate::ffv1::Frame) -> Result { + use crate::ffv1::Colorspace; + let (w, h, bits) = (p.width, p.height, p.bit_depth); + let full = |v: Vec| Plane::packed(w, h, bits, Samples::U16(v)); + let mut it = p.planes.into_iter(); + let count = it.len(); + let mut next = || it.next().ok_or(FrameError::Shape("plane count does not fit the colour family")); + let (family, sub, planes, alpha) = match (p.colorspace, count) { + (Colorspace::YCbCr, 1 | 2) => { + let y = full(next()?)?; + let a = if count == 2 { Some(full(next()?)?) } else { None }; + (ColorFamily::Gray, Subsampling::S444, vec![y], a) + } + (Colorspace::YCbCr, 3 | 4) => { + let sub = Subsampling { x: p.chroma_shift.0, y: p.chroma_shift.1 }; + let (cw, ch) = sub.chroma_size(w, h); + let y = full(next()?)?; + let cb = Plane::packed(cw, ch, bits, Samples::U16(next()?))?; + let cr = Plane::packed(cw, ch, bits, Samples::U16(next()?))?; + let a = if count == 4 { Some(full(next()?)?) } else { None }; + (ColorFamily::YCbCr, sub, vec![y, cb, cr], a) + } + (Colorspace::Rgb, 3 | 4) => { + let planes = vec![full(next()?)?, full(next()?)?, full(next()?)?]; + let a = if count == 4 { Some(full(next()?)?) } else { None }; + (ColorFamily::Rgb, Subsampling::S444, planes, a) + } + _ => return Err(FrameError::Shape("plane count does not fit the colour family")), + }; + let mut f = VideoFrame::new(w, h, family, sub, planes, alpha)?; + if family == ColorFamily::Rgb { + f.color = ColorTags { matrix: 0, full_range: true, ..ColorTags::UNSPECIFIED }; + } + Ok(f) + } +} diff --git a/crates/video/src/frame/mod.rs b/crates/video/src/frame/mod.rs new file mode 100644 index 000000000..4ac896cdb --- /dev/null +++ b/crates/video/src/frame/mod.rs @@ -0,0 +1,344 @@ +//! A codec-agnostic decoded video frame. +//! +//! Each decoder in this crate returns its own picture type. [`VideoFrame`] holds what they have in +//! common: planar samples at their stored width (`u8` or `u16`), per-plane bit depth, chroma +//! subsampling, an optional alpha plane, colour tags as ITU-T H.273 code points, and timing. +//! Every decoder output converts into it with `VideoFrame::try_from`, behind the codec's feature. +//! A conversion moves the sample buffers; it does not copy or rescale samples. +//! +//! Plane order is fixed by [`ColorFamily`]: `[Y, Cb, Cr]`, `[R, G, B]` or `[Y]`. Decoders that +//! store RGB in another order (DNx, VP9 and identity-matrix AV1 store G, B, R) are reordered by +//! the conversion. +//! +//! The constructors check that the planes and their buffers agree in size, so reading a row +//! never goes out of bounds. They do not check that sample values fit the bit depth. + +mod convert; + +/// Why a frame or plane was rejected. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum FrameError { + /// A plane's buffer, stride or bit depth does not fit its size. + #[error("invalid plane: {0}")] + Plane(&'static str), + /// The planes do not fit the frame size, colour family or subsampling. + #[error("invalid frame: {0}")] + Shape(&'static str), +} + +/// The samples of one plane, at the width the decoder stores them. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Samples { + /// 8-bit storage (bit depth 1 to 8). + U8(Vec), + /// 16-bit storage (bit depth 1 to 16), values below `1 << bit_depth`. + U16(Vec), +} + +impl Samples { + /// Number of stored samples. + pub fn len(&self) -> usize { + match self { + Samples::U8(v) => v.len(), + Samples::U16(v) => v.len(), + } + } + + /// Whether no samples are stored. + pub fn is_empty(&self) -> bool { + self.len() == 0 + } +} + +/// One plane of samples: `height` rows of `width` samples, each row starting `stride` samples +/// after the previous one. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Plane { + width: u32, + height: u32, + stride: usize, + bit_depth: u8, + samples: Samples, +} + +impl Plane { + /// A plane with the given row stride. Fails when `stride < width`, when `bit_depth` is 0, above + /// 16, or above 8 for [`Samples::U8`], or when `samples` is shorter than the rows need. + pub fn new(width: u32, height: u32, stride: usize, bit_depth: u8, samples: Samples) -> Result { + let w = usize::try_from(width).map_err(|_| FrameError::Plane("width does not fit in usize"))?; + let h = usize::try_from(height).map_err(|_| FrameError::Plane("height does not fit in usize"))?; + if stride < w { + return Err(FrameError::Plane("stride is less than the width")); + } + let max_depth = match samples { + Samples::U8(_) => 8, + Samples::U16(_) => 16, + }; + if bit_depth == 0 || bit_depth > max_depth { + return Err(FrameError::Plane("bit depth does not fit the sample storage")); + } + let needed = match h.checked_sub(1) { + None => 0, + Some(last) => last.checked_mul(stride).and_then(|n| n.checked_add(w)).ok_or(FrameError::Plane("plane size overflows"))?, + }; + if samples.len() < needed { + return Err(FrameError::Plane("buffer is shorter than the plane")); + } + Ok(Plane { width, height, stride, bit_depth, samples }) + } + + /// A plane whose rows are packed (`stride == width`). + pub fn packed(width: u32, height: u32, bit_depth: u8, samples: Samples) -> Result { + let stride = usize::try_from(width).map_err(|_| FrameError::Plane("width does not fit in usize"))?; + Plane::new(width, height, stride, bit_depth, samples) + } + + /// Samples per row. + pub fn width(&self) -> u32 { + self.width + } + + /// Rows. + pub fn height(&self) -> u32 { + self.height + } + + /// Distance between the starts of two rows, in samples. + pub fn stride(&self) -> usize { + self.stride + } + + /// Significant bits per sample. + pub fn bit_depth(&self) -> u8 { + self.bit_depth + } + + /// The stored samples, rows `stride` apart. + pub fn samples(&self) -> &Samples { + &self.samples + } + + /// The stored samples, giving up the plane. + pub fn into_samples(self) -> Samples { + self.samples + } + + /// Row `y` of an 8-bit plane: `None` for a 16-bit plane or a row past the end. + pub fn row_u8(&self, y: u32) -> Option<&[u8]> { + match &self.samples { + Samples::U8(v) => v.get(self.row_range(y)?), + Samples::U16(_) => None, + } + } + + /// Row `y` of a 16-bit plane: `None` for an 8-bit plane or a row past the end. + pub fn row_u16(&self, y: u32) -> Option<&[u16]> { + match &self.samples { + Samples::U16(v) => v.get(self.row_range(y)?), + Samples::U8(_) => None, + } + } + + fn row_range(&self, y: u32) -> Option> { + if y >= self.height { + return None; + } + let start = usize::try_from(y).ok()?.checked_mul(self.stride)?; + Some(start..start.checked_add(usize::try_from(self.width).ok()?)?) + } +} + +/// What the colour planes hold, and in which order. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum ColorFamily { + /// `[Y, Cb, Cr]`, chroma subsampled by [`VideoFrame::subsampling`]. + YCbCr, + /// `[R, G, B]`, all at full size. + Rgb, + /// `[Y]` alone. + Gray, +} + +/// Chroma subsampling as log2 factors: the chroma planes are `width.div_ceil(1 << x)` by +/// `height.div_ceil(1 << y)`. FFV1 codes shifts up to 3; the other codecs stop at 1. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct Subsampling { + /// log2 of the horizontal factor (0 to 3). + pub x: u8, + /// log2 of the vertical factor (0 to 3). + pub y: u8, +} + +impl Subsampling { + /// 4:2:0. + pub const S420: Subsampling = Subsampling { x: 1, y: 1 }; + /// 4:2:2. + pub const S422: Subsampling = Subsampling { x: 1, y: 0 }; + /// 4:4:4, also used for RGB and gray frames. + pub const S444: Subsampling = Subsampling { x: 0, y: 0 }; + + /// Largest shift a frame accepts. + pub const MAX_SHIFT: u8 = 3; + + /// Chroma plane size for a luma size. A shift above [`Subsampling::MAX_SHIFT`] counts as that. + pub fn chroma_size(self, width: u32, height: u32) -> (u32, u32) { + (width.div_ceil(1 << self.x.min(Self::MAX_SHIFT)), height.div_ceil(1 << self.y.min(Self::MAX_SHIFT))) + } +} + +/// Colour description as ITU-T H.273 code points. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct ColorTags { + /// `ColourPrimaries` (2 = unspecified). + pub primaries: u8, + /// `TransferCharacteristics` (2 = unspecified). + pub transfer: u8, + /// `MatrixCoefficients` (0 = identity, used for RGB; 2 = unspecified). + pub matrix: u8, + /// `VideoFullRangeFlag`: full range when true. False when the stream does not say. + pub full_range: bool, +} + +impl ColorTags { + /// Nothing signalled. + pub const UNSPECIFIED: ColorTags = ColorTags { primaries: 2, transfer: 2, matrix: 2, full_range: false }; +} + +impl Default for ColorTags { + fn default() -> Self { + ColorTags::UNSPECIFIED + } +} + +/// Scan structure of a frame. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Default)] +pub enum FieldOrder { + /// Progressive, or not signalled. + #[default] + Progressive, + /// Two interlaced fields; the top field (even rows) comes first. + TopFieldFirst, + /// Two interlaced fields; the bottom field (odd rows) comes first. + BottomFieldFirst, +} + +/// A decoded frame in planar form, independent of the codec. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct VideoFrame { + width: u32, + height: u32, + family: ColorFamily, + subsampling: Subsampling, + planes: Vec, + alpha: Option, + /// Colour description. + pub color: ColorTags, + /// The timestamp passed to the decoder with the packet that produced the frame; `None` for + /// codecs whose decoders take no timestamp (the intra-only codecs). + pub pts: Option, + /// Sample aspect ratio, `None` when not signalled. + pub sar: Option<(u32, u32)>, + /// Field order. + pub field_order: FieldOrder, + /// Decoded in a decoder's draft mode (some filtering skipped): the samples are approximate. + pub draft: bool, +} + +impl VideoFrame { + /// A frame from its planes, in the order [`ColorFamily`] gives. Fails when the number of planes + /// does not fit `family`, a plane's size does not follow from `width`, `height` and + /// `subsampling`, a shift is above 3, or an RGB or gray frame is subsampled. The alpha plane is + /// full size. The colour tags and timing start unset. + pub fn new( + width: u32, + height: u32, + family: ColorFamily, + subsampling: Subsampling, + planes: Vec, + alpha: Option, + ) -> Result { + if subsampling.x > Subsampling::MAX_SHIFT || subsampling.y > Subsampling::MAX_SHIFT { + return Err(FrameError::Shape("subsampling shift above 3")); + } + if family != ColorFamily::YCbCr && subsampling != Subsampling::S444 { + return Err(FrameError::Shape("only YCbCr frames are subsampled")); + } + let expected = if family == ColorFamily::Gray { 1 } else { 3 }; + if planes.len() != expected { + return Err(FrameError::Shape("plane count does not fit the colour family")); + } + let chroma = subsampling.chroma_size(width, height); + for (i, p) in planes.iter().enumerate() { + let size = if i == 0 { (width, height) } else { chroma }; + if (p.width, p.height) != size { + return Err(FrameError::Shape("plane size does not fit the frame")); + } + } + if let Some(a) = &alpha + && (a.width, a.height) != (width, height) + { + return Err(FrameError::Shape("alpha plane size does not fit the frame")); + } + Ok(VideoFrame { + width, + height, + family, + subsampling, + planes, + alpha, + color: ColorTags::UNSPECIFIED, + pts: None, + sar: None, + field_order: FieldOrder::Progressive, + draft: false, + }) + } + + /// Luma width (the size of plane 0). + pub fn width(&self) -> u32 { + self.width + } + + /// Luma height. + pub fn height(&self) -> u32 { + self.height + } + + /// What the planes hold. + pub fn family(&self) -> ColorFamily { + self.family + } + + /// Chroma subsampling ([`Subsampling::S444`] for RGB and gray). + pub fn subsampling(&self) -> Subsampling { + self.subsampling + } + + /// The colour planes, in the order [`ColorFamily`] gives. + pub fn planes(&self) -> &[Plane] { + &self.planes + } + + /// Colour plane `i`, `None` past the last one. + pub fn plane(&self, i: usize) -> Option<&Plane> { + self.planes.get(i) + } + + /// The alpha plane (0 = transparent, the bit depth's maximum = opaque). + pub fn alpha(&self) -> Option<&Plane> { + self.alpha.as_ref() + } + + /// Bit depth of plane 0. Chroma may differ; see [`Plane::bit_depth`]. + pub fn bit_depth(&self) -> u8 { + self.planes.first().map_or(0, Plane::bit_depth) + } + + /// The colour planes and the alpha plane, giving up the frame. + pub fn into_planes(self) -> (Vec, Option) { + (self.planes, self.alpha) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/video/src/frame/tests.rs b/crates/video/src/frame/tests.rs new file mode 100644 index 000000000..cba664af4 --- /dev/null +++ b/crates/video/src/frame/tests.rs @@ -0,0 +1,404 @@ +use super::*; + +fn u8_plane(w: u32, h: u32, fill: u8) -> Plane { + Plane::packed(w, h, 8, Samples::U8(vec![fill; (w * h) as usize])).unwrap() +} + +#[test] +fn plane_checks_buffer_stride_and_depth() { + assert_eq!(Plane::new(4, 2, 3, 8, Samples::U8(vec![0; 8])), Err(FrameError::Plane("stride is less than the width"))); + // Two rows at stride 6 need 6 + 4 samples: the last row's padding is not required. + assert!(Plane::new(4, 2, 6, 8, Samples::U8(vec![0; 10])).is_ok()); + assert_eq!(Plane::new(4, 2, 6, 8, Samples::U8(vec![0; 9])), Err(FrameError::Plane("buffer is shorter than the plane"))); + assert!(Plane::packed(4, 2, 9, Samples::U8(vec![0; 8])).is_err()); + assert!(Plane::packed(4, 2, 0, Samples::U16(vec![0; 8])).is_err()); + assert!(Plane::packed(4, 2, 17, Samples::U16(vec![0; 8])).is_err()); + assert!(Plane::packed(4, 2, 16, Samples::U16(vec![0; 8])).is_ok()); + assert!(Plane::packed(0, 0, 8, Samples::U8(Vec::new())).is_ok()); +} + +#[test] +fn plane_size_overflow_is_an_error() { + assert_eq!(Plane::new(1, u32::MAX, usize::MAX, 8, Samples::U8(Vec::new())), Err(FrameError::Plane("plane size overflows"))); +} + +#[test] +fn rows_honour_the_stride() { + let p = Plane::new(2, 3, 4, 8, Samples::U8(vec![0, 1, 9, 9, 2, 3, 9, 9, 4, 5])).unwrap(); + assert_eq!(p.row_u8(0), Some(&[0, 1][..])); + assert_eq!(p.row_u8(1), Some(&[2, 3][..])); + assert_eq!(p.row_u8(2), Some(&[4, 5][..])); + assert_eq!(p.row_u8(3), None); + assert_eq!(p.row_u16(0), None); + let q = Plane::packed(1, 2, 10, Samples::U16(vec![7, 1023])).unwrap(); + assert_eq!(q.row_u16(1), Some(&[1023][..])); + assert_eq!(q.row_u8(0), None); +} + +#[test] +fn frame_checks_planes_against_its_shape() { + let planes = || vec![u8_plane(5, 3, 0), u8_plane(3, 2, 0), u8_plane(3, 2, 0)]; + // Odd sizes round the chroma up. + let f = VideoFrame::new(5, 3, ColorFamily::YCbCr, Subsampling::S420, planes(), None).unwrap(); + assert_eq!((f.width(), f.height(), f.bit_depth()), (5, 3, 8)); + assert_eq!(f.color, ColorTags::UNSPECIFIED); + assert_eq!(f.pts, None); + assert!(VideoFrame::new(5, 3, ColorFamily::YCbCr, Subsampling::S422, planes(), None).is_err()); + assert!(VideoFrame::new(5, 3, ColorFamily::Gray, Subsampling::S444, planes(), None).is_err()); + assert!(VideoFrame::new(5, 3, ColorFamily::Rgb, Subsampling::S420, planes(), None).is_err()); + assert_eq!( + VideoFrame::new(5, 3, ColorFamily::YCbCr, Subsampling { x: 4, y: 0 }, planes(), None), + Err(FrameError::Shape("subsampling shift above 3")) + ); + assert!(VideoFrame::new(5, 3, ColorFamily::YCbCr, Subsampling::S420, planes(), Some(u8_plane(3, 2, 0))).is_err()); + let a = VideoFrame::new(5, 3, ColorFamily::YCbCr, Subsampling::S420, planes(), Some(u8_plane(5, 3, 255))).unwrap(); + assert_eq!(a.alpha().map(Plane::width), Some(5)); + let g = VideoFrame::new(5, 3, ColorFamily::Gray, Subsampling::S444, vec![u8_plane(5, 3, 0)], None).unwrap(); + assert_eq!(g.planes().len(), 1); + assert!(g.plane(1).is_none()); +} + +#[test] +fn chroma_planes_may_have_their_own_depth() { + let y = Plane::packed(2, 2, 10, Samples::U16(vec![0; 4])).unwrap(); + let c = || Plane::packed(1, 1, 8, Samples::U16(vec![0])).unwrap(); + let f = VideoFrame::new(2, 2, ColorFamily::YCbCr, Subsampling::S420, vec![y, c(), c()], None).unwrap(); + assert_eq!(f.bit_depth(), 10); + assert_eq!(f.planes()[1].bit_depth(), 8); +} + +#[cfg(feature = "h264-decode")] +fn h264_picture(y_len: usize) -> crate::h264::decode::Picture { + crate::h264::decode::Picture { + width: 4, + height: 2, + chroma_width: 2, + chroma_height: 1, + y: vec![16; y_len], + u: vec![128; 2], + v: vec![128; 2], + y_stride: 4, + uv_stride: 2, + pts: 7, + poc: 0, + key: true, + color: crate::h264::decode::ColorInfo { full_range: true, primaries: 1, transfer: 1, matrix: 1 }, + sar: (0, 0), + draft: false, + } +} + +#[cfg(feature = "h264-decode")] +#[test] +fn h264_picture_converts() { + let f = VideoFrame::try_from(h264_picture(8)).unwrap(); + assert_eq!((f.family(), f.subsampling()), (ColorFamily::YCbCr, Subsampling::S420)); + assert_eq!(f.color, ColorTags { primaries: 1, transfer: 1, matrix: 1, full_range: true }); + assert_eq!((f.pts, f.sar), (Some(7), None)); + let mut p = h264_picture(8); + p.sar = (4, 3); + assert_eq!(VideoFrame::try_from(p).unwrap().sar, Some((4, 3))); +} + +/// The decoder's pictures have public fields; one whose buffer does not fit its size is an error, +/// not an out-of-bounds read later. +#[cfg(feature = "h264-decode")] +#[test] +fn h264_picture_with_a_short_buffer_is_an_error() { + assert_eq!(VideoFrame::try_from(h264_picture(7)), Err(FrameError::Plane("buffer is shorter than the plane"))); +} + +#[cfg(feature = "hevc-decode")] +#[test] +fn hevc_picture_converts() { + use crate::hevc::decode::{ChromaFormat, ColorInfo, Picture, Plane as P}; + let p = Picture { + width: 4, + height: 2, + chroma_width: 2, + chroma_height: 2, + chroma: ChromaFormat::C422, + bit_depth: 10, + bit_depth_chroma: 10, + y: P::U16(vec![64; 8]), + u: P::U16(vec![512; 4]), + v: P::U16(vec![512; 4]), + y_stride: 4, + uv_stride: 2, + pts: 3, + poc: 0, + key: true, + color: ColorInfo { full_range: false, primaries: 9, transfer: 16, matrix: 9 }, + sar: (1, 1), + draft: true, + }; + let f = VideoFrame::try_from(p.clone()).unwrap(); + assert_eq!((f.subsampling(), f.bit_depth()), (Subsampling::S422, 10)); + assert_eq!(f.planes()[2].row_u16(1), Some(&[512, 512][..])); + assert_eq!(f.color, ColorTags { primaries: 9, transfer: 16, matrix: 9, full_range: false }); + assert_eq!((f.sar, f.draft), (Some((1, 1)), true)); + + // 10-bit luma with 8-bit chroma: both stored as `u16`, each plane with its own depth. + let mixed = VideoFrame::try_from(Picture { bit_depth_chroma: 8, u: P::U16(vec![128; 4]), v: P::U16(vec![128; 4]), ..p }).unwrap(); + assert_eq!(mixed.planes().iter().map(Plane::bit_depth).collect::>(), [10, 8, 8]); + assert_eq!(mixed.planes()[1].row_u16(0), Some(&[128, 128][..])); +} + +#[cfg(feature = "av1-decode")] +#[test] +fn av1_monochrome_is_gray() { + let p = crate::av1::decode::Picture { + width: 3, + height: 2, + bit_depth: 8, + subsampling_x: 1, + subsampling_y: 1, + mono_chrome: true, + planes: [vec![1; 6], Vec::new(), Vec::new()], + color_primaries: 1, + transfer_characteristics: 13, + matrix_coefficients: 1, + full_range: true, + pts: 5, + draft: false, + }; + let f = VideoFrame::try_from(p).unwrap(); + assert_eq!((f.family(), f.subsampling(), f.planes().len()), (ColorFamily::Gray, Subsampling::S444, 1)); + assert_eq!(f.color, ColorTags { primaries: 1, transfer: 13, matrix: 1, full_range: true }); +} + +#[cfg(feature = "av1-decode")] +#[test] +fn av1_420_rounds_chroma_up() { + let p = crate::av1::decode::Picture { + width: 3, + height: 3, + bit_depth: 10, + subsampling_x: 1, + subsampling_y: 1, + mono_chrome: false, + planes: [vec![0; 9], vec![0; 4], vec![0; 4]], + color_primaries: 2, + transfer_characteristics: 2, + matrix_coefficients: 2, + full_range: false, + pts: 0, + draft: false, + }; + let f = VideoFrame::try_from(p).unwrap(); + assert_eq!((f.planes()[1].width(), f.planes()[1].height()), (2, 2)); +} + +/// The identity matrix codes G, B, R in the Y, U, V planes. +#[cfg(feature = "av1-decode")] +#[test] +fn av1_identity_matrix_is_rgb_reordered_to_r_g_b() { + let p = crate::av1::decode::Picture { + width: 2, + height: 2, + bit_depth: 8, + subsampling_x: 0, + subsampling_y: 0, + mono_chrome: false, + planes: [vec![1; 4], vec![2; 4], vec![3; 4]], + color_primaries: 1, + transfer_characteristics: 13, + matrix_coefficients: 0, + full_range: true, + pts: 0, + draft: false, + }; + let f = VideoFrame::try_from(p.clone()).unwrap(); + assert_eq!(f.family(), ColorFamily::Rgb); + let first = |f: &VideoFrame, i: usize| f.planes()[i].row_u16(0).and_then(|r| r.first().copied()); + assert_eq!([first(&f, 0), first(&f, 1), first(&f, 2)], [Some(3), Some(1), Some(2)]); + assert_eq!(f.color, ColorTags { primaries: 1, transfer: 13, matrix: 0, full_range: true }); + + // Any other matrix keeps the coded order. + let y = VideoFrame::try_from(crate::av1::decode::Picture { matrix_coefficients: 1, ..p }).unwrap(); + assert_eq!(y.family(), ColorFamily::YCbCr); + assert_eq!([first(&y, 0), first(&y, 1), first(&y, 2)], [Some(1), Some(2), Some(3)]); +} + +#[cfg(feature = "vp9-decode")] +#[test] +fn vp9_color_space_maps_to_a_matrix() { + assert_eq!([0, 1, 2, 3, 4, 5, 6, 7].map(convert::vp9_matrix), [2, 5, 1, 6, 7, 9, 2, 0]); +} + +#[cfg(feature = "mpeg2-decode")] +#[test] +fn mpeg2_picture_converts() { + use crate::mpeg2::{ChromaFormat, Picture, PictureType, SequenceInfo}; + let info = SequenceInfo { + mpeg2: true, + width: 5, + height: 4, + chroma: ChromaFormat::Yuv420, + profile_and_level: None, + progressive_sequence: false, + low_delay: false, + frame_rate: None, + sar: (16, 15), + dar: None, + colour: Some((5, 5, 5)), + video_format: None, + bit_rate: 0, + }; + let p = Picture { + width: 5, + height: 4, + chroma: ChromaFormat::Yuv420, + y: vec![16; 20], + cb: vec![128; 6], + cr: vec![128; 6], + pts: 9, + picture_type: PictureType::I, + temporal_reference: 0, + progressive_frame: false, + top_field_first: false, + repeat_first_field: false, + field_pictures: false, + info: std::sync::Arc::new(info), + }; + let f = VideoFrame::try_from(p).unwrap(); + assert_eq!(f.field_order, FieldOrder::BottomFieldFirst); + assert_eq!(f.color, ColorTags { primaries: 5, transfer: 5, matrix: 5, full_range: false }); + assert_eq!((f.sar, f.pts), (Some((16, 15)), Some(9))); +} + +#[cfg(feature = "vp8-decode")] +#[test] +fn vp8_picture_converts() { + let p = crate::vp8::Picture { + width: 3, + height: 3, + chroma_width: 2, + chroma_height: 2, + y: vec![0; 9], + u: vec![0; 4], + v: vec![0; 4], + pts: 1, + key: true, + color_space: 0, + }; + let f = VideoFrame::try_from(p).unwrap(); + assert_eq!(f.color, ColorTags { matrix: 6, ..ColorTags::UNSPECIFIED }); + assert_eq!(f.pts, Some(1)); +} + +#[cfg(feature = "prores")] +#[test] +fn prores_frame_converts() { + use crate::prores::{ChromaFormat, ColorInfo, Frame, Interlace}; + let mut p = Frame::new(5, 2, ChromaFormat::Yuv422, 10, true).unwrap(); + p.color = ColorInfo { primaries: 0, transfer: 1, matrix: 0 }; + p.interlace = Interlace::TopFieldFirst; + let f = VideoFrame::try_from(p).unwrap(); + assert_eq!((f.planes()[1].width(), f.planes()[1].height()), (3, 2)); + assert_eq!(f.alpha().map(Plane::bit_depth), Some(10)); + assert_eq!(f.color, ColorTags { primaries: 2, transfer: 1, matrix: 2, full_range: false }); + assert_eq!((f.field_order, f.pts), (FieldOrder::TopFieldFirst, None)); + assert_eq!(f.sar, None); +} + +#[cfg(feature = "prores")] +#[test] +fn prores_aspect_ratio_gives_the_sample_aspect_ratio() { + use crate::prores::{ChromaFormat, Frame}; + let sar = |code: u8, w: u32, h: u32| { + let mut p = Frame::new(w, h, ChromaFormat::Yuv422, 10, false).unwrap(); + p.aspect_ratio = code; + VideoFrame::try_from(p).unwrap().sar + }; + // 0 is unknown and 4 to 15 are reserved. + assert_eq!([sar(0, 720, 486), sar(4, 720, 486)], [None, None]); + assert_eq!(sar(1, 720, 486), Some((1, 1))); + // 720 x 486 shown at 4:3 and at 16:9. + assert_eq!(sar(2, 720, 486), Some((9, 10))); + assert_eq!(sar(3, 720, 486), Some((6, 5))); + // 1920 x 1080 at 16:9 has square pixels. + assert_eq!(sar(3, 1920, 1080), Some((1, 1))); + assert_eq!(convert::prores_sar(2, u32::MAX, u32::MAX), Some((4, 3))); + assert_eq!(convert::prores_sar(2, 0, 4), None); +} + +#[cfg(feature = "dnx")] +#[test] +fn dnx_rgb_is_reordered_to_r_g_b() { + use crate::dnx::{ChromaFormat, Frame}; + let mut p = Frame::new(2, 2, ChromaFormat::Yuv444, 10, false); + p.rgb = true; + // Stored G, B, R. + p.y = vec![1; 4]; + p.cb = vec![2; 4]; + p.cr = vec![3; 4]; + let f = VideoFrame::try_from(p).unwrap(); + assert_eq!(f.family(), ColorFamily::Rgb); + let first = |i: usize| f.planes()[i].row_u16(0).and_then(|r| r.first().copied()); + assert_eq!([first(0), first(1), first(2)], [Some(3), Some(1), Some(2)]); + assert_eq!(f.color, ColorTags { matrix: 0, ..ColorTags::UNSPECIFIED }); +} + +#[cfg(feature = "dnx")] +#[test] +fn dnx_ycbcr_keeps_its_order() { + use crate::dnx::{ChromaFormat, ColorVolume, Frame}; + let mut p = Frame::new(4, 2, ChromaFormat::Yuv420, 8, true); + p.color_volume = ColorVolume::Bt2020Ncl; + p.par = (0, 0); + let f = VideoFrame::try_from(p).unwrap(); + assert_eq!((f.family(), f.subsampling()), (ColorFamily::YCbCr, Subsampling::S420)); + assert_eq!(f.color, ColorTags { matrix: 9, ..ColorTags::UNSPECIFIED }); + assert_eq!(f.sar, None); + assert!(f.alpha().is_some()); +} + +#[cfg(any(feature = "apv-decode", feature = "apv-encode"))] +#[test] +fn apv_frames_convert() { + use crate::apv::{ChromaFormat, Frame}; + let mono = VideoFrame::try_from(Frame::new(3, 2, ChromaFormat::Monochrome, 10, false)).unwrap(); + assert_eq!((mono.family(), mono.planes().len()), (ColorFamily::Gray, 1)); + let rgba = VideoFrame::try_from(Frame::new(3, 2, ChromaFormat::Yuv4444, 12, false)).unwrap(); + assert_eq!((rgba.family(), rgba.subsampling()), (ColorFamily::YCbCr, Subsampling::S444)); + assert!(rgba.alpha().is_some()); + assert_eq!(rgba.color, ColorTags { primaries: 1, transfer: 1, matrix: 1, full_range: false }); + let c422 = VideoFrame::try_from(Frame::new(3, 2, ChromaFormat::Yuv422, 10, false)).unwrap(); + assert_eq!(c422.planes()[1].width(), 2); +} + +#[cfg(feature = "ffv1")] +fn ffv1_frame(colorspace: crate::ffv1::Colorspace, shift: (u8, u8), sizes: &[usize]) -> crate::ffv1::Frame { + crate::ffv1::Frame { width: 3, height: 3, colorspace, bit_depth: 8, chroma_shift: shift, planes: sizes.iter().map(|&n| vec![0; n]).collect() } +} + +#[cfg(feature = "ffv1")] +#[test] +fn ffv1_frames_convert() { + use crate::ffv1::Colorspace; + let g = VideoFrame::try_from(ffv1_frame(Colorspace::YCbCr, (0, 0), &[9, 9])).unwrap(); + assert_eq!((g.family(), g.alpha().is_some()), (ColorFamily::Gray, true)); + let y = VideoFrame::try_from(ffv1_frame(Colorspace::YCbCr, (1, 1), &[9, 4, 4])).unwrap(); + assert_eq!((y.family(), y.subsampling(), y.alpha().is_some()), (ColorFamily::YCbCr, Subsampling::S420, false)); + assert_eq!(y.color, ColorTags::UNSPECIFIED); + let r = VideoFrame::try_from(ffv1_frame(Colorspace::Rgb, (0, 0), &[9, 9, 9, 9])).unwrap(); + assert_eq!((r.family(), r.alpha().is_some()), (ColorFamily::Rgb, true)); + assert_eq!(r.color, ColorTags { matrix: 0, full_range: true, ..ColorTags::UNSPECIFIED }); + assert!(VideoFrame::try_from(ffv1_frame(Colorspace::Rgb, (0, 0), &[9])).is_err()); + assert!(VideoFrame::try_from(ffv1_frame(Colorspace::YCbCr, (0, 0), &[])).is_err()); + // FFV1 codes shifts up to 3: an 8 x 1 frame at (3, 0) has one chroma sample per plane. + let wide = crate::ffv1::Frame { + width: 8, + height: 1, + colorspace: Colorspace::YCbCr, + bit_depth: 8, + chroma_shift: (3, 0), + planes: vec![vec![0; 8], vec![0; 1], vec![0; 1]], + }; + let wide = VideoFrame::try_from(wide).unwrap(); + assert_eq!(wide.subsampling(), Subsampling { x: 3, y: 0 }); + assert_eq!((wide.planes()[1].width(), wide.planes()[1].height(), wide.planes()[2].samples().len()), (1, 1, 1)); + // A 2 x 2 chroma plane needs 4 samples. + assert!(VideoFrame::try_from(ffv1_frame(Colorspace::YCbCr, (1, 1), &[9, 3, 4])).is_err()); +} diff --git a/crates/video/src/hevc/decode/decoder.rs b/crates/video/src/hevc/decode/decoder.rs index a09e87ab9..2f226d571 100644 --- a/crates/video/src/hevc/decode/decoder.rs +++ b/crates/video/src/hevc/decode/decoder.rs @@ -710,6 +710,7 @@ fn make_picture(o: &Output) -> Picture { chroma_height: cch as u32, chroma, bit_depth: bd, + bit_depth_chroma: o.frame.bit_depth_c, y, u, v, diff --git a/crates/video/src/hevc/decode/mod.rs b/crates/video/src/hevc/decode/mod.rs index 00ec97738..9a731d3b9 100644 --- a/crates/video/src/hevc/decode/mod.rs +++ b/crates/video/src/hevc/decode/mod.rs @@ -150,6 +150,8 @@ pub struct Picture { pub chroma: ChromaFormat, /// Luma bit depth (chroma uses the same storage type). pub bit_depth: u32, + /// Chroma bit depth; a stream may code it apart from the luma depth. + pub bit_depth_chroma: u32, pub y: Plane, pub u: Plane, pub v: Plane, diff --git a/crates/video/src/lib.rs b/crates/video/src/lib.rs index 8085c42fa..ec2243913 100644 --- a/crates/video/src/lib.rs +++ b/crates/video/src/lib.rs @@ -24,10 +24,14 @@ //! //! The decoders bound the pictures they hold with a memory budget ([`budget`]). //! +//! Each decoder returns its own picture type. [`frame::VideoFrame`] is the codec-agnostic form +//! they all convert into (`VideoFrame::try_from`). +//! //! Containers and bit-level I/O are in `craft-media`; the codecs here share its `bitstream` //! module and describe their streams with `craft_media::CodecParams`. pub mod budget; +pub mod frame; #[cfg(any(feature = "apv-decode", feature = "apv-encode"))] pub mod apv; diff --git a/crates/video/tests/vp9_decode/common/mod.rs b/crates/video/tests/vp9_decode/common/mod.rs index 07eaeffef..e3847bdc1 100644 --- a/crates/video/tests/vp9_decode/common/mod.rs +++ b/crates/video/tests/vp9_decode/common/mod.rs @@ -44,6 +44,8 @@ pub const FIXTURES: &[Fixture] = &[ fx!("profile3_444_10bit", "testsrc2", 176, 144, 6, NOISE, "yuv444p10le", ["-b:v", "500k", "-profile:v", "3", "-frame-parallel", "0"]), fx!("profile3_422_12bit", "mandelbrot", 176, 144, 6, NOISE, "yuv422p12le", ["-b:v", "500k", "-profile:v", "3", "-frame-parallel", "0"]), fx!("lossless", "testsrc2", 176, 144, 5, NOISE, P420, ["-lossless", "1", "-frame-parallel", "0"]), + // RGB (`color_space` 7): libvpx takes it as gbrp, so the coded planes are G, B, R. + fx!("profile1_rgb", "testsrc2", 176, 144, 2, "", "gbrp", ["-lossless", "1", "-profile:v", "1", "-frame-parallel", "0"]), fx!("lossless_10bit", "mandelbrot", 176, 144, 4, NOISE, P420_10, ["-lossless", "1", "-profile:v", "2", "-frame-parallel", "0"]), fx!("tiles_cols4", "testsrc2", 1280, 720, 6, NOISE, P420, ["-b:v", "2M", "-tile-columns", "2", "-speed", "4", "-frame-parallel", "0"]), fx!( diff --git a/crates/video/tests/vp9_decode/frame.rs b/crates/video/tests/vp9_decode/frame.rs new file mode 100644 index 000000000..c8e94db57 --- /dev/null +++ b/crates/video/tests/vp9_decode/frame.rs @@ -0,0 +1,46 @@ +//! Conversion of decoded VP9 pictures into `craft_video::frame::VideoFrame`. + +use crate::common; +use craft_video::frame::{ColorFamily, Samples, VideoFrame}; + +/// An RGB stream converts to R, G, B. The reference decode is ffmpeg's `gbrp` (G, B, R), so the +/// frame's planes 0, 1, 2 must equal reference planes 2, 0, 1. +#[test] +fn rgb_planes_come_out_as_r_g_b() { + let f = common::fixture("profile1_rgb"); + let Some((ivf, yuv)) = common::ensure(f) else { return }; + let reference = std::fs::read(yuv).unwrap(); + let pics = common::decode_file_threads(&ivf, 1).map_err(|e| e.1).unwrap(); + assert_eq!(pics.len(), f.frames as usize); + let n = (f.width * f.height) as usize; + for (i, pic) in pics.into_iter().enumerate() { + assert_eq!(pic.color.color_space, 7); + let frame = VideoFrame::try_from(pic).unwrap(); + assert_eq!(frame.family(), ColorFamily::Rgb); + assert_eq!(frame.color.matrix, 0); + assert!(frame.color.full_range); + let refp = &reference[3 * n * i..3 * n * (i + 1)]; + let (g, b, r) = (&refp[..n], &refp[n..2 * n], &refp[2 * n..]); + for (plane, want, name) in [(0, r, "R"), (1, g, "G"), (2, b, "B")] { + let Samples::U8(got) = frame.planes()[plane].samples() else { panic!("8-bit storage") }; + assert_eq!(got.as_slice(), want, "frame {i} plane {name}"); + } + // The test source is not gray, so a swapped order would not pass by accident. + assert_ne!(r, g); + } +} + +/// A 4:2:2 12-bit stream keeps its subsampling and 16-bit storage. +#[test] +fn ycbcr_422_12bit() { + let f = common::fixture("profile3_422_12bit"); + let Some((ivf, _)) = common::ensure(f) else { return }; + let pics = common::decode_file_threads(&ivf, 1).map_err(|e| e.1).unwrap(); + let frame = VideoFrame::try_from(pics.into_iter().next().unwrap()).unwrap(); + assert_eq!(frame.family(), ColorFamily::YCbCr); + assert_eq!(frame.subsampling(), craft_video::frame::Subsampling::S422); + assert_eq!(frame.bit_depth(), 12); + assert_eq!((frame.planes()[1].width(), frame.planes()[1].height()), (88, 144)); + assert!(matches!(frame.planes()[0].samples(), Samples::U16(_))); + assert_eq!(frame.pts, Some(0)); +} diff --git a/crates/video/tests/vp9_decode/main.rs b/crates/video/tests/vp9_decode/main.rs index 66697e009..b05b5fdc3 100644 --- a/crates/video/tests/vp9_decode/main.rs +++ b/crates/video/tests/vp9_decode/main.rs @@ -1,5 +1,6 @@ mod common; mod conformance; +mod frame; mod fuzz_hostile; mod perf; mod robustness;