From 92c0b49b5e01762e0f2944a286865584fae009d2 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sat, 14 Nov 2026 20:38:28 -0500 Subject: [PATCH] feat(video): support tiled AV1 for large frames (#1453) * feat(video): add tiled AV1 large-frame fallback * feat(video): expose tiled AV1 speed preset * fix(video): use clamped AV1 keyframe interval --- Cargo.lock | 1 + crates/video/Cargo.toml | 5 + crates/video/README.md | 2 +- crates/video/src/av1/encode/README.md | 9 ++ crates/video/src/av1/encode/large.rs | 155 ++++++++++++++++++++++++++ crates/video/src/av1/encode/mod.rs | 5 + 6 files changed, 176 insertions(+), 1 deletion(-) create mode 100644 crates/video/src/av1/encode/large.rs diff --git a/Cargo.lock b/Cargo.lock index 6e48813da..fb1a3f862 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2732,6 +2732,7 @@ version = "0.1.0" dependencies = [ "craft-media", "craft-media-testkit", + "rav1e", "rayon", "thiserror 2.0.21", ] diff --git a/crates/video/Cargo.toml b/crates/video/Cargo.toml index e5bbf1c94..c3ebfd722 100644 --- a/crates/video/Cargo.toml +++ b/crates/video/Cargo.toml @@ -52,6 +52,8 @@ vp9-encode = ["craft-media/isobmff"] av1 = ["av1-decode", "av1-encode"] av1-decode = [] av1-encode = ["craft-media/isobmff"] +# Tiled AV1 fallback for frames that exceed the clean-room encoder's single-tile limits. +av1-large-encode = ["av1-encode", "dep:rav1e"] # Decode and encode across worker threads (rayon) in the codecs that support it. threads = ["dep:rayon"] @@ -60,6 +62,9 @@ craft-media = { workspace = true } thiserror = { workspace = true } rayon = { workspace = true, optional = true } +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +rav1e = { version = "0.8", default-features = false, optional = true } + [dev-dependencies] craft-media-testkit = { workspace = true } # The AVI muxer, for the MPEG-4 Part 2 encoder's end-to-end test. diff --git a/crates/video/README.md b/crates/video/README.md index 7de348bb6..248e922b3 100644 --- a/crates/video/README.md +++ b/crates/video/README.md @@ -27,5 +27,5 @@ No codec feature is on by default. | `h264` | [`h264`](src/h264/decode/README.md) | ITU-T H.264 / AVC: decoder (`h264-decode` alone) and encoder (`h264-encode` alone) | | `hevc` | [`hevc`](src/hevc/decode/README.md) | ITU-T H.265 / HEVC: decoder (`hevc-decode` alone) and encoder (`hevc-encode` alone) | | `vp9` | [`vp9`](src/vp9/decode/README.md) | VP9: decoder (`vp9-decode` alone) and encoder (`vp9-encode` alone) | -| `av1` | [`av1`](src/av1/decode/README.md) | AV1 (AOMedia Video 1): decoder (`av1-decode` alone) and encoder (`av1-encode` alone) | +| `av1` | [`av1`](src/av1/decode/README.md) | AV1 (AOMedia Video 1): decoder (`av1-decode` alone) and clean-room encoder (`av1-encode`); tiled rav1e fallback for large frames (`av1-large-encode`) | | `threads` | — | Multi-threaded slice/row parallelism via `rayon` | diff --git a/crates/video/src/av1/encode/README.md b/crates/video/src/av1/encode/README.md index aa1604706..2df88ba64 100644 --- a/crates/video/src/av1/encode/README.md +++ b/crates/video/src/av1/encode/README.md @@ -3,6 +3,15 @@ Clean-room, pure-Rust AV1 encoder for MP4 / WebM export (layer L0, `std` only, no `unsafe`, builds for `wasm32-unknown-unknown`). +## Large frame fallback + +The optional `av1-large-encode` feature exposes `LargeEncoder`, a tiled rav1e-backed encoder for +8-bit 4:2:0 frames up to 8192 samples per side. Use it when the clean-room `Encoder`'s one-tile +limits (4096 samples per side and 2304 superblocks) would reject the frame. It supports constant +quantizer rate control and emits one low-latency packet per input frame. The fallback is native +only; the clean-room encoder remains available without the feature and supports 8- and 10-bit +input. + ## What is implemented - **Stream**: Main profile (`seq_profile` 0), 4:2:0, 8 or 10 bit, the configured colour description (BT.709 by default), diff --git a/crates/video/src/av1/encode/large.rs b/crates/video/src/av1/encode/large.rs new file mode 100644 index 000000000..109a536dc --- /dev/null +++ b/crates/video/src/av1/encode/large.rs @@ -0,0 +1,155 @@ +//! Tiled AV1 encoding for frames outside the clean-room encoder's single-tile limits. + +use super::{EncoderConfig, Error, Frame, Packet, RateControl}; +use rav1e::prelude::{ + ChromaSampling, ColorDescription, ColorPrimaries, Config, Context, EncoderConfig as Rav1eConfig, FrameParameters, FrameType, FrameTypeOverride, + MatrixCoefficients, PixelRange, Rational, TransferCharacteristics, +}; + +/// Tiled AV1 encoder used when a frame is too large for [`super::Encoder`]. +pub struct LargeEncoder { + ctx: Context, + width: usize, + height: usize, + seq_header: Option>, +} + +impl LargeEncoder { + /// Creates a tiled 8-bit 4:2:0 encoder for dimensions up to 8192 samples per side. + pub fn new(cfg: &EncoderConfig) -> Result { + Self::with_speed(cfg, 6) + } + + /// Creates the tiled encoder with a rav1e speed preset from 1 (slower) to 10 (faster). + pub fn with_speed(cfg: &EncoderConfig, speed: u8) -> Result { + if cfg.width < 16 || cfg.height < 16 || cfg.width > 8192 || cfg.height > 8192 { + return Err(Error::InvalidConfig("large AV1 frame dimensions must be 16..=8192".into())); + } + if cfg.bit_depth != 8 || cfg.fps_num == 0 || cfg.fps_den == 0 { + return Err(Error::InvalidConfig("large AV1 encoding requires 8-bit input and a valid frame rate".into())); + } + let RateControl::ConstantQ(quality) = cfg.rate else { + return Err(Error::InvalidConfig("large AV1 encoding currently supports constant quantizer rate control".into())); + }; + let mut ec = Rav1eConfig::with_speed_preset(speed.clamp(1, 10)); + ec.width = cfg.width as usize; + ec.height = cfg.height as usize; + ec.time_base = Rational::new(u64::from(cfg.fps_den), u64::from(cfg.fps_num)); + ec.bit_depth = 8; + ec.chroma_sampling = ChromaSampling::Cs420; + ec.pixel_range = if cfg.full_range { PixelRange::Full } else { PixelRange::Limited }; + ec.color_description = Some(ColorDescription { + color_primaries: ColorPrimaries::BT709, + transfer_characteristics: TransferCharacteristics::BT709, + matrix_coefficients: MatrixCoefficients::BT709, + }); + ec.low_latency = true; + ec.quantizer = quality.max(1) as usize; + ec.min_quantizer = ec.quantizer.min(255) as u8; + ec.min_key_frame_interval = 1; + ec.max_key_frame_interval = cfg.keyint.clamp(1, 120) as u64; + ec.tiles = std::thread::available_parallelism().map_or(4, |n| n.get()).clamp(2, 16); + let ctx = Config::new().with_encoder_config(ec).new_context().map_err(|e| Error::InvalidConfig(format!("rav1e: {e}")))?; + Ok(Self { ctx, width: cfg.width as usize, height: cfg.height as usize, seq_header: None }) + } + + /// Encodes one frame. The frame planes must match the configured 4:2:0 dimensions. + pub fn encode_with_key(&mut self, frame: &Frame<'_>, force_key: bool) -> Result { + if frame.y_stride < self.width + || frame.uv_stride < self.width / 2 + || frame.y.len() < frame.y_stride.saturating_mul(self.height) + || frame.u.len() < frame.uv_stride.saturating_mul(self.height / 2) + || frame.v.len() < frame.uv_stride.saturating_mul(self.height / 2) + { + return Err(Error::InvalidFrame("plane dimensions or strides do not match the configured frame".into())); + } + let mut input = self.ctx.new_frame(); + copy_plane(&mut input.planes[0], frame.y, frame.y_stride, self.width, self.height)?; + copy_plane(&mut input.planes[1], frame.u, frame.uv_stride, self.width / 2, self.height / 2)?; + copy_plane(&mut input.planes[2], frame.v, frame.uv_stride, self.width / 2, self.height / 2)?; + let params = + FrameParameters { frame_type_override: if force_key { FrameTypeOverride::Key } else { FrameTypeOverride::No }, ..Default::default() }; + self.ctx.send_frame((input, params)).map_err(|e| Error::InvalidFrame(format!("rav1e: {e}")))?; + loop { + match self.ctx.receive_packet() { + Ok(packet) => { + let (data, seq) = sample_data(&packet.data)?; + if self.seq_header.is_none() { + self.seq_header = seq; + } + return Ok(Packet { data, keyframe: packet.frame_type == FrameType::KEY }); + } + Err(rav1e::prelude::EncoderStatus::Encoded) => continue, + Err(rav1e::prelude::EncoderStatus::NeedMoreData | rav1e::prelude::EncoderStatus::LimitReached) => { + return Err(Error::InvalidFrame("rav1e did not emit a low-latency frame".into())); + } + Err(e) => return Err(Error::InvalidFrame(format!("rav1e: {e}"))), + } + } + } + + /// Returns the AV1 configuration record after the first frame has been encoded. + pub fn av1c(&self) -> Option> { + self.seq_header.as_ref().map(|obu| { + let mut config = vec![0x81, 31, 0x0c, 0]; + config.extend_from_slice(obu); + config + }) + } +} + +fn copy_plane(plane: &mut rav1e::prelude::Plane, source: &[u16], stride: usize, width: usize, height: usize) -> Result<(), Error> { + let mut packed = Vec::with_capacity(width * height); + for row in 0..height { + let start = row * stride; + let pixels = &source[start..start + width]; + if pixels.iter().any(|&value| value > u8::MAX as u16) { + return Err(Error::InvalidFrame("8-bit AV1 samples must be in 0..=255".into())); + } + packed.extend(pixels.iter().map(|&value| value as u8)); + } + plane.copy_from_raw_u8(&packed, width, 1); + Ok(()) +} + +fn sample_data(data: &[u8]) -> Result<(Vec, Option>), Error> { + let mut pos = 0usize; + let mut sample = Vec::with_capacity(data.len()); + let mut sequence = None; + while pos < data.len() { + let start = pos; + let header = data[pos]; + if header & 0x80 != 0 { + return Err(Error::InvalidFrame("rav1e emitted an invalid OBU".into())); + } + let kind = (header >> 3) & 0x0f; + pos += 1 + usize::from(header & 0x04 != 0); + let end = if header & 0x02 != 0 { + let mut size = 0usize; + let mut shift = 0usize; + loop { + let byte = *data.get(pos).ok_or_else(|| Error::InvalidFrame("truncated OBU size".into()))?; + pos += 1; + size |= usize::from(byte & 0x7f).checked_shl(shift as u32).ok_or_else(|| Error::InvalidFrame("OBU size overflow".into()))?; + if byte & 0x80 == 0 { + break; + } + shift += 7; + if shift >= usize::BITS as usize { + return Err(Error::InvalidFrame("OBU size overflow".into())); + } + } + pos.checked_add(size).filter(|end| *end <= data.len()).ok_or_else(|| Error::InvalidFrame("OBU exceeds packet bounds".into()))? + } else { + data.len() + }; + if kind == 1 && sequence.is_none() { + sequence = Some(data[start..end].to_vec()); + } + if kind != 2 && kind != 15 { + sample.extend_from_slice(&data[start..end]); + } + pos = end; + } + Ok((sample, sequence)) +} diff --git a/crates/video/src/av1/encode/mod.rs b/crates/video/src/av1/encode/mod.rs index 10e88d0ad..3a954cef2 100644 --- a/crates/video/src/av1/encode/mod.rs +++ b/crates/video/src/av1/encode/mod.rs @@ -20,6 +20,11 @@ mod tables; mod transform; mod writer; +#[cfg(all(feature = "av1-large-encode", not(target_arch = "wasm32")))] +mod large; +#[cfg(all(feature = "av1-large-encode", not(target_arch = "wasm32")))] +pub use large::LargeEncoder; + use bitw::obu; use craft_media::bitstream::BitWriter; use craft_media::isobmff::Av1Config;