diff --git a/Cargo.lock b/Cargo.lock index d1e5d3cec..480e000e5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2956,6 +2956,7 @@ dependencies = [ name = "craft-video" version = "0.1.0" dependencies = [ + "craft-jpeg", "craft-media", "craft-media-testkit", "rav1e", diff --git a/crates/video/Cargo.toml b/crates/video/Cargo.toml index bb771e419..31504f309 100644 --- a/crates/video/Cargo.toml +++ b/crates/video/Cargo.toml @@ -36,6 +36,8 @@ mpeg4 = ["mpeg4-encode"] mpeg4-encode = ["craft-media/bitstream"] # FFV1 lossless video (RFC 9043): version 3, range coder, decoder and encoder. ffv1 = [] +# Motion JPEG: a frame-level decoder and encoder over `craft-jpeg`. +mjpeg = ["dep:craft-jpeg"] # Apple ProRes (SMPTE RDD 36): decoder and encoder. prores = ["craft-media/bitstream"] # H.264 / AVC: decoder and encoder; `h264-decode` or `h264-encode` for one of them. @@ -61,6 +63,7 @@ threads = ["dep:rayon", "rav1e?/threading"] [dependencies] craft-media = { workspace = true } +craft-jpeg = { workspace = true, optional = true } thiserror = { workspace = true } rayon = { workspace = true, optional = true } @@ -72,6 +75,10 @@ craft-media-testkit = { workspace = true } # The AVI muxer, for the MPEG-4 Part 2 encoder's end-to-end test. craft-media = { workspace = true, features = ["avi"] } +[[test]] +name = "mjpeg" +required-features = ["mjpeg"] + [[test]] name = "dnx" required-features = ["dnx"] diff --git a/crates/video/README.md b/crates/video/README.md index 248e922b3..14f1e3f44 100644 --- a/crates/video/README.md +++ b/crates/video/README.md @@ -23,6 +23,7 @@ No codec feature is on by default. | `vp8` | `vp8` | VP8 (RFC 6386): decoder | | `mpeg2` | [`mpeg2`](src/mpeg2/README.md) | MPEG-2 video (ITU-T H.262) and MPEG-1 video (ISO/IEC 11172-2): decoder (`mpeg2-decode` alone); MPEG-2 encoder with I, P and B pictures (`mpeg2-encode`, which also turns on the decoder) | | `mpeg4` | [`mpeg4`](src/mpeg4/README.md) | MPEG-4 Part 2 video (ISO/IEC 14496-2, the Xvid/DivX format): Simple profile encoder (`mpeg4-encode`) | +| `mjpeg` | `mjpeg` | Motion JPEG: a decoder (baseline, extended and progressive Huffman JPEG; the Annex K tables for AVI `MJPG` packets that have none) and a baseline encoder, both over `craft-jpeg`. No two-field (MJPEG-A/B) packets and no lossless JPEG. | | `prores` | `prores` | Apple ProRes (SMPTE RDD 36): decoder and encoder | | `h264` | [`h264`](src/h264/decode/README.md) | ITU-T H.264 / AVC: decoder (`h264-decode` alone) and encoder (`h264-encode` alone) | | `hevc` | [`hevc`](src/hevc/decode/README.md) | ITU-T H.265 / HEVC: decoder (`hevc-decode` alone) and encoder (`hevc-encode` alone) | diff --git a/crates/video/src/lib.rs b/crates/video/src/lib.rs index 17ca960bb..da4e32006 100644 --- a/crates/video/src/lib.rs +++ b/crates/video/src/lib.rs @@ -11,6 +11,7 @@ //! | `mpeg2` | [`mpeg2`] | MPEG-2 video (ITU-T H.262) and MPEG-1 video: decoder (`mpeg2-decode`) and MPEG-2 I/P/B encoder (`mpeg2-encode`) | //! | `mpeg4` | [`mpeg4`] | MPEG-4 Part 2 video (ISO/IEC 14496-2, Xvid/DivX): Simple profile encoder (`mpeg4-encode`) | //! | `ffv1` | [`ffv1`] | FFV1 lossless video (RFC 9043): version 3, range coder, decoder and encoder | +//! | `mjpeg` | [`mjpeg`] | Motion JPEG: frame-level decoder and baseline encoder over `craft-jpeg` | //! | `prores` | [`prores`] | Apple ProRes (SMPTE RDD 36): decoder and encoder | //! | `h264` | [`h264`] | ITU-T H.264 / AVC: decoder (`h264-decode`) and encoder (`h264-encode`) | //! | `hevc` | [`hevc`] | ITU-T H.265 / HEVC: decoder (`hevc-decode`) and encoder (`hevc-encode`) | @@ -45,6 +46,9 @@ pub mod mpeg4; #[cfg(feature = "ffv1")] pub mod ffv1; +#[cfg(feature = "mjpeg")] +pub mod mjpeg; + #[cfg(feature = "prores")] pub mod prores; diff --git a/crates/video/src/mjpeg/mod.rs b/crates/video/src/mjpeg/mod.rs new file mode 100644 index 000000000..2a3b1e4b5 --- /dev/null +++ b/crates/video/src/mjpeg/mod.rs @@ -0,0 +1,176 @@ +//! Motion JPEG (MJPEG): every picture of the stream is a JPEG image (ITU-T T.81 | ISO/IEC 10918-1), +//! so this module is a frame-level wrapper over `craft-jpeg`, which does the decoding and the +//! baseline encoding. There is no second JPEG implementation here. +//! +//! - [`Decoder`] decodes one packet to a [`Frame`]: baseline, extended-sequential (8 and 12 bit) +//! and progressive Huffman JPEG, grayscale or YCbCr (as RGB) with any chroma subsampling. Packets +//! of AVI `MJPG` streams (the "AVI1" framing) often leave out the Huffman tables: the Annex K +//! tables are put in when a packet has none. +//! - [`Encoder`] writes each frame as a complete baseline JPEG (SOI to EOI, tables in the packet) +//! with Huffman tables made for the picture, which is what AVI `MJPG`, QuickTime `jpeg`, Matroska +//! `V_MJPEG` and the other containers carry as a sample. +//! - [`is_packet`] tells a JPEG packet from other data. +//! +//! Not here: the two-field packets of interlaced MJPEG (QuickTime MJPEG-A and -B, AVI field +//! pairs), lossless JPEG (SOF3), arithmetic coding and the hierarchical process, which +//! `craft-jpeg` does not decode. Such packets are errors, never panics. + +use craft_jpeg::Limits; +pub use craft_jpeg::{ChromaSubsampling, Samples}; + +use crate::budget::DECODE_MEMORY_BUDGET; + +/// Codec errors. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum Error { + /// The packet is not a decodable JPEG. + #[error("invalid MJPEG packet: {0}")] + Invalid(String), + /// A JPEG process this module does not decode. + #[error("unsupported MJPEG feature: {0}")] + Unsupported(String), + /// The picture is larger than the memory budget or [`Decoder::max_pixels`] allows. + #[error("MJPEG picture above the limits: {0}")] + Limit(String), + /// The encoder was given a frame or settings it cannot encode. + #[error("invalid encoder input: {0}")] + Input(String), +} + +/// `Result` of this module. +pub type Result = std::result::Result; + +impl From for Error { + fn from(e: craft_jpeg::Error) -> Self { + match e { + craft_jpeg::Error::Malformed(m) => Error::Invalid(m), + craft_jpeg::Error::Unsupported(m) => Error::Unsupported(m), + craft_jpeg::Error::LimitExceeded(m) => Error::Limit(m), + _ => Error::Invalid("undecodable JPEG".to_string()), + } + } +} + +/// One decoded picture: interleaved samples, rows top to bottom. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Frame { + pub width: u32, + pub height: u32, + /// Samples per pixel: 1 for grayscale, 3 for RGB (a YCbCr picture is converted, JFIF / T.871). + pub channels: u8, + /// `U8` for 8-bit pictures; `U16` for 12-bit ones, scaled to the 16-bit range. + pub samples: Samples, + /// The entropy-coded data was damaged or cut short: what could be decoded is kept, the rest + /// of that picture is grey, and the stream goes on with the next packet. + pub damaged: bool, +} + +/// Whether `packet` starts like a JPEG image (the SOI marker followed by another marker). +pub fn is_packet(packet: &[u8]) -> bool { + matches!(packet, [0xFF, 0xD8, 0xFF, ..]) +} + +/// A Motion JPEG decoder. Every packet is independent, so one decoder can decode packets in any +/// order and from several threads. +#[derive(Clone, Debug)] +pub struct Decoder { + limits: Limits, +} + +impl Default for Decoder { + fn default() -> Self { + Self::new() + } +} + +impl Decoder { + /// A decoder whose pictures stay within the decoding memory budget of the crate. + pub fn new() -> Self { + let mut limits = Limits::default(); + limits.max_alloc = DECODE_MEMORY_BUDGET; + Decoder { limits } + } + + /// Refuse pictures of more than `pixels` pixels (`width × height`). + pub fn max_pixels(mut self, pixels: u64) -> Self { + self.limits.max_pixels = pixels; + self + } + + /// Decode one packet. + pub fn decode(&self, packet: &[u8]) -> Result { + if !is_packet(packet) { + return Err(Error::Invalid("the packet does not start with a JPEG SOI marker".to_string())); + } + // AVI `MJPG` packets may leave out the Huffman tables; every other packet keeps its own + let packet = craft_jpeg::with_avi1_default_huffman(packet); + let d = craft_jpeg::decode(&packet, &self.limits, None)?; + Ok(Frame { width: d.width, height: d.height, channels: d.components, samples: d.samples, damaged: d.damaged }) + } +} + +/// Encoder settings. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct EncoderConfig { + /// JPEG quality, 1 (smallest) to 100 (best); the usual scale on the Annex K tables. + pub quality: u8, + /// Chroma sampling of colour pictures; ignored for grayscale. + pub subsampling: ChromaSubsampling, +} + +impl Default for EncoderConfig { + fn default() -> Self { + EncoderConfig { quality: 90, subsampling: ChromaSubsampling::S420 } + } +} + +/// A Motion JPEG encoder: each frame becomes one baseline JPEG packet. +#[derive(Clone, Copy, Debug, Default)] +pub struct Encoder { + cfg: EncoderConfig, +} + +impl Encoder { + /// An encoder, or the setting it cannot carry. + pub fn new(cfg: EncoderConfig) -> Result { + if !(1..=100).contains(&cfg.quality) { + return Err(Error::Input(format!("quality {} is not in 1..=100", cfg.quality))); + } + Ok(Encoder { cfg }) + } + + /// Encode `width × height` interleaved 8-bit pixels of `channels` samples (1 gray, 3 RGB, or 4 + /// RGBA with the alpha dropped). + pub fn encode(&self, width: u32, height: u32, channels: u8, pixels: &[u8]) -> Result> { + craft_jpeg::encode_baseline(pixels, width as usize, height as usize, usize::from(channels), self.cfg.quality, self.cfg.subsampling, &[]) + .map_err(|e| Error::Input(e.to_string())) + } + + /// Encode a decoded [`Frame`] of 8-bit samples. + pub fn encode_frame(&self, frame: &Frame) -> Result> { + match &frame.samples { + Samples::U8(p) => self.encode(frame.width, frame.height, frame.channels, p), + _ => Err(Error::Input("only 8-bit frames can be encoded".to_string())), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn packets_start_with_soi() { + assert!(is_packet(&[0xFF, 0xD8, 0xFF, 0xE0])); + assert!(!is_packet(&[0xFF, 0xD8])); + assert!(!is_packet(b"RIFF")); + assert!(Decoder::new().decode(b"not a jpeg").is_err()); + } + + #[test] + fn quality_is_checked() { + assert!(Encoder::new(EncoderConfig { quality: 0, ..Default::default() }).is_err()); + assert!(Encoder::new(EncoderConfig { quality: 101, ..Default::default() }).is_err()); + assert!(Encoder::new(EncoderConfig::default()).is_ok()); + } +} diff --git a/crates/video/tests/mjpeg.rs b/crates/video/tests/mjpeg.rs new file mode 100644 index 000000000..d83b61438 --- /dev/null +++ b/crates/video/tests/mjpeg.rs @@ -0,0 +1,226 @@ +//! Motion JPEG: the encoder and decoder against each other, against ffmpeg as the reference +//! (oracle; skipped when ffmpeg is absent, a failure with `CRAFT_MEDIA_REQUIRE_ORACLES=1`), and on +//! hostile packets. + +use craft_video::mjpeg::{ChromaSubsampling, Decoder, Encoder, EncoderConfig, Frame, Samples, is_packet}; +use std::path::Path; +use std::process::Command; + +/// A smooth picture with a diagonal edge, `channels` samples per pixel. +fn picture(w: usize, h: usize, channels: usize) -> Vec { + let mut v = Vec::with_capacity(w * h * channels); + for y in 0..h { + for x in 0..w { + let edge = if x + y > (w + h) / 2 { 40.0 } else { 0.0 }; + for c in 0..channels { + let base = 90.0 + 60.0 * ((x as f32 * 0.11 + c as f32).sin() + (y as f32 * 0.07 - c as f32).cos()); + v.push((base + edge + 20.0 * c as f32).clamp(0.0, 255.0) as u8); + } + } + } + v +} + +fn psnr(a: &[u8], b: &[u8]) -> f64 { + assert_eq!(a.len(), b.len()); + let mse = a.iter().zip(b).map(|(x, y)| (f64::from(*x) - f64::from(*y)).powi(2)).sum::() / a.len().max(1) as f64; + if mse == 0.0 { 99.0 } else { 10.0 * (255.0f64 * 255.0 / mse).log10() } +} + +fn bytes(f: &Frame) -> &[u8] { + match &f.samples { + Samples::U8(v) => v, + _ => panic!("8-bit expected"), + } +} + +#[test] +fn encoded_frames_decode_back_close_to_the_input() { + let dec = Decoder::new(); + for (w, h, ch, sub, min) in [ + (64, 48, 3, ChromaSubsampling::S444, 38.0), + (67, 41, 3, ChromaSubsampling::S422, 33.0), + (67, 41, 3, ChromaSubsampling::S420, 32.0), + (1, 1, 3, ChromaSubsampling::S420, 25.0), + (33, 17, 1, ChromaSubsampling::S420, 38.0), + ] { + let pixels = picture(w, h, ch); + let enc = Encoder::new(EncoderConfig { quality: 92, subsampling: sub }).expect("encoder"); + let packet = enc.encode(w as u32, h as u32, ch as u8, &pixels).expect("encode"); + assert!(is_packet(&packet)); + assert_eq!(&packet[packet.len() - 2..], &[0xFF, 0xD9], "a packet runs from SOI to EOI"); + let f = dec.decode(&packet).expect("decode"); + assert_eq!((f.width as usize, f.height as usize, f.channels as usize, f.damaged), (w, h, ch, false)); + let p = psnr(&pixels, bytes(&f)); + assert!(p >= min, "{w}x{h} {ch}ch {sub:?}: {p:.1} dB under {min}"); + } +} + +#[test] +fn rgba_drops_alpha_and_bad_input_is_refused() { + let enc = Encoder::default(); + let rgba: Vec = (0..16 * 16 * 4).map(|i| (i * 7) as u8).collect(); + let f = Decoder::new().decode(&enc.encode(16, 16, 4, &rgba).expect("encode")).expect("decode"); + assert_eq!(f.channels, 3); + assert!(enc.encode(16, 16, 3, &[0; 10]).is_err()); + assert!(enc.encode(0, 16, 3, &[]).is_err()); + assert!(enc.encode(16, 16, 2, &rgba).is_err()); + assert!(enc.encode(70_000, 1, 1, &[0; 70_000]).is_err()); +} + +#[test] +fn the_pixel_limit_is_applied_before_allocating() { + let enc = Encoder::default(); + let packet = enc.encode(64, 64, 1, &[7; 64 * 64]).expect("encode"); + assert!(Decoder::new().max_pixels(64 * 64).decode(&packet).is_ok()); + assert!(matches!(Decoder::new().max_pixels(64 * 64 - 1).decode(&packet), Err(craft_video::mjpeg::Error::Limit(_)))); + // a header that claims 65535 x 65535 pixels of a tiny file + let mut huge = packet.clone(); + let sof = huge.windows(2).position(|w| w == [0xFF, 0xC0]).expect("SOF0"); + huge[sof + 5..sof + 9].copy_from_slice(&[0xFF, 0xFF, 0xFF, 0xFF]); + assert!(Decoder::new().max_pixels(1 << 20).decode(&huge).is_err()); +} + +#[test] +fn hostile_packets_are_errors_never_panics() { + let enc = Encoder::default(); + let packet = enc.encode(48, 40, 3, &picture(48, 40, 3)).expect("encode"); + let dec = Decoder::new(); + // every truncation + for n in 0..packet.len() { + let _ = dec.decode(&packet[..n]); + } + // single byte flips, and a run of noise after the headers + let mut x = 0x2545_F491u32; + for i in 0..packet.len() { + let mut p = packet.clone(); + x ^= x << 13; + x ^= x >> 17; + x ^= x << 5; + p[i] ^= (x >> 8) as u8 | 1; + let _ = dec.decode(&p); + } + let mut noisy = packet.clone(); + let from = noisy.len() / 2; + for b in &mut noisy[from..] { + x ^= x << 13; + x ^= x >> 17; + x ^= x << 5; + *b = (x >> 8) as u8; + } + let _ = dec.decode(&noisy); + for junk in [&[][..], &[0xFF, 0xD8], &[0xFF, 0xD8, 0xFF, 0xD9], &[0xFF, 0xD8, 0xFF, 0xC3, 0, 2], b"\xFF\xD8\xFF\xDA\x00\x02"] { + assert!(dec.decode(junk).is_err()); + } +} + +#[test] +fn packets_that_are_not_baseline_or_progressive_huffman_are_unsupported() { + // SOF3 (lossless): a frame header, no more + let sof3 = [0xFF, 0xD8, 0xFF, 0xC3, 0, 11, 8, 0, 8, 0, 8, 1, 1, 0x11, 0, 0xFF, 0xD9]; + assert!(matches!(Decoder::new().decode(&sof3), Err(craft_video::mjpeg::Error::Unsupported(_))), "{:?}", Decoder::new().decode(&sof3)); +} + +fn ffmpeg(ff: &Path, args: &[&str]) { + let out = Command::new(ff).args(["-v", "error", "-nostdin", "-y"]).args(args).output().expect("run ffmpeg"); + assert!(out.status.success(), "ffmpeg {args:?}: {}", String::from_utf8_lossy(&out.stderr)); +} + +/// The SOS-preceding DHT segments cut out of a JPEG, as AVI `MJPG` packets often come. +fn without_huffman_tables(jpeg: &[u8]) -> Vec { + let mut out = vec![0xFF, 0xD8]; + let mut at = 2; + while at + 3 < jpeg.len() && jpeg[at] == 0xFF { + let marker = jpeg[at + 1]; + let len = usize::from(u16::from_be_bytes([jpeg[at + 2], jpeg[at + 3]])); + if marker == 0xDA { + break; + } + if marker != 0xC4 { + out.extend_from_slice(&jpeg[at..at + 2 + len]); + } + at += 2 + len; + } + out.extend_from_slice(&jpeg[at..]); + out +} + +#[test] +fn we_decode_what_ffmpeg_encodes_to_what_ffmpeg_decodes() { + let Some(ff) = craft_media_testkit::ffmpeg() else { return }; + let dir = craft_media_testkit::fixture_subdir("mjpeg"); + let s = |p: &Path| p.to_str().expect("path").to_string(); + for (name, pix_fmt, rgb_pix, channels) in [("420", "yuvj420p", "rgb24", 3), ("422", "yuvj422p", "rgb24", 3), ("444", "yuvj444p", "rgb24", 3)] { + let jpg = dir.join(format!("{name}.jpg")); + ffmpeg( + &ff, + &["-f", "lavfi", "-i", "gradients=size=67x41:rate=1", "-frames:v", "1", "-c:v", "mjpeg", "-pix_fmt", pix_fmt, "-q:v", "2", &s(&jpg)], + ); + let reference = dir.join(format!("{name}.raw")); + ffmpeg(&ff, &["-i", &s(&jpg), "-f", "rawvideo", "-pix_fmt", rgb_pix, &s(&reference)]); + let packet = std::fs::read(&jpg).expect("read"); + let f = Decoder::new().decode(&packet).expect("decode"); + assert_eq!((f.width, f.height, f.channels as usize), (67, 41, channels), "{name}"); + let p = psnr(&std::fs::read(&reference).expect("read"), bytes(&f)); + println!("{name}: {p:.1} dB against ffmpeg's decode"); + assert!(p >= 38.0, "{name}: {p:.1} dB"); + + // the same stream with the Huffman tables left out, as AVI MJPG often has it, if ffmpeg + // wrote the Annex K ones + let plain = dir.join(format!("{name}_default_tables.jpg")); + ffmpeg( + &ff, + &[ + "-f", + "lavfi", + "-i", + "gradients=size=67x41:rate=1", + "-frames:v", + "1", + "-c:v", + "mjpeg", + "-pix_fmt", + pix_fmt, + "-q:v", + "2", + "-huffman", + "default", + &s(&plain), + ], + ); + let stripped = without_huffman_tables(&std::fs::read(&plain).expect("read")); + let with_tables = Decoder::new().decode(&std::fs::read(&plain).expect("read")).expect("decode"); + let without = Decoder::new().decode(&stripped).expect("decode without DHT"); + assert_eq!(with_tables, without, "{name}: the default tables are put in"); + } +} + +#[test] +fn ffmpeg_decodes_what_we_encode() { + let Some(ff) = craft_media_testkit::ffmpeg() else { return }; + let dir = craft_media_testkit::fixture_subdir("mjpeg"); + let s = |p: &Path| p.to_str().expect("path").to_string(); + for (name, w, h, ch, sub, pix_fmt) in [ + ("o420", 67usize, 41usize, 3usize, ChromaSubsampling::S420, "rgb24"), + ("o422", 67, 41, 3, ChromaSubsampling::S422, "rgb24"), + ("o444", 64, 48, 3, ChromaSubsampling::S444, "rgb24"), + ("ogray", 33, 17, 1, ChromaSubsampling::S420, "gray"), + ] { + let pixels = picture(w, h, ch); + let packet = Encoder::new(EncoderConfig { quality: 90, subsampling: sub }) + .expect("encoder") + .encode(w as u32, h as u32, ch as u8, &pixels) + .expect("encode"); + let jpg = dir.join(format!("{name}.jpg")); + std::fs::write(&jpg, &packet).expect("write"); + let out = dir.join(format!("{name}.raw")); + ffmpeg(&ff, &["-i", &s(&jpg), "-f", "rawvideo", "-pix_fmt", pix_fmt, &s(&out)]); + let theirs = std::fs::read(&out).expect("read"); + let ours = Decoder::new().decode(&packet).expect("decode"); + let p = psnr(&theirs, bytes(&ours)); + let q = psnr(&pixels, &theirs); + println!("{name}: ffmpeg decode against ours {p:.1} dB, against the input {q:.1} dB"); + assert!(p >= 38.0, "{name}: ffmpeg's decode of our packet differs from ours: {p:.1} dB"); + assert!(q >= 28.0, "{name}: ffmpeg's decode is far from the input: {q:.1} dB"); + } +}