craft-video: Motion JPEG decoder and encoder (mjpeg)

A frame-level wrapper over craft-jpeg: packets decode to gray or RGB
frames (Annex K Huffman tables put in for AVI MJPG packets that have
none) and frames encode to baseline JPEG packets. Checked against
ffmpeg as a black-box oracle in both directions.

Part of #1510.
This commit is contained in:
el-j committed 2026-11-15 22:19:12 +01:00
1 parent 30596f73c5
commit 0281d0951b
6 files changed
+415

No files matched your search

Generated
+1
View File
@@ -2887,6 +2887,7 @@ dependencies = [
name = "craft-video"
version = "0.1.0"
dependencies = [
"craft-jpeg",
"craft-media",
"craft-media-testkit",
"rav1e",
+7
View File
@@ -36,6 +36,8 @@ mpeg4 = ["mpeg4-encode"]
mpeg4-encode = ["craft-media/bitstream"]
# FFV1 lossless video (RFC 9043): version 3, range coder, decoder and encoder.
ffv1 = []
# Motion JPEG: a frame-level decoder and encoder over `craft-jpeg`.
mjpeg = ["dep:craft-jpeg"]
# Apple ProRes (SMPTE RDD 36): decoder and encoder.
prores = ["craft-media/bitstream"]
# H.264 / AVC: decoder and encoder; `h264-decode` or `h264-encode` for one of them.
@@ -61,6 +63,7 @@ threads = ["dep:rayon"]
[dependencies]
craft-media = { workspace = true }
craft-jpeg = { workspace = true, optional = true }
thiserror = { workspace = true }
rayon = { workspace = true, optional = true }
@@ -72,6 +75,10 @@ craft-media-testkit = { workspace = true }
# The AVI muxer, for the MPEG-4 Part 2 encoder's end-to-end test.
craft-media = { workspace = true, features = ["avi"] }
[[test]]
name = "mjpeg"
required-features = ["mjpeg"]
[[test]]
name = "dnx"
required-features = ["dnx"]
+1
View File
@@ -23,6 +23,7 @@ No codec feature is on by default.
| `vp8` | `vp8` | VP8 (RFC 6386): decoder |
| `mpeg2` | [`mpeg2`](src/mpeg2/README.md) | MPEG-2 video (ITU-T H.262) and MPEG-1 video (ISO/IEC 11172-2): decoder (`mpeg2-decode` alone); MPEG-2 encoder with I, P and B pictures (`mpeg2-encode`, which also turns on the decoder) |
| `mpeg4` | [`mpeg4`](src/mpeg4/README.md) | MPEG-4 Part 2 video (ISO/IEC 14496-2, the Xvid/DivX format): Simple profile encoder (`mpeg4-encode`) |
| `mjpeg` | `mjpeg` | Motion JPEG: a decoder (baseline, extended and progressive Huffman JPEG; the Annex K tables for AVI `MJPG` packets that have none) and a baseline encoder, both over `craft-jpeg`. No two-field (MJPEG-A/B) packets and no lossless JPEG. |
| `prores` | `prores` | Apple ProRes (SMPTE RDD 36): decoder and encoder |
| `h264` | [`h264`](src/h264/decode/README.md) | ITU-T H.264 / AVC: decoder (`h264-decode` alone) and encoder (`h264-encode` alone) |
| `hevc` | [`hevc`](src/hevc/decode/README.md) | ITU-T H.265 / HEVC: decoder (`hevc-decode` alone) and encoder (`hevc-encode` alone) |
+4
View File
@@ -11,6 +11,7 @@
//! | `mpeg2` | [`mpeg2`] | MPEG-2 video (ITU-T H.262) and MPEG-1 video: decoder (`mpeg2-decode`) and MPEG-2 I/P/B encoder (`mpeg2-encode`) |
//! | `mpeg4` | [`mpeg4`] | MPEG-4 Part 2 video (ISO/IEC 14496-2, Xvid/DivX): Simple profile encoder (`mpeg4-encode`) |
//! | `ffv1` | [`ffv1`] | FFV1 lossless video (RFC 9043): version 3, range coder, decoder and encoder |
//! | `mjpeg` | [`mjpeg`] | Motion JPEG: frame-level decoder and baseline encoder over `craft-jpeg` |
//! | `prores` | [`prores`] | Apple ProRes (SMPTE RDD 36): decoder and encoder |
//! | `h264` | [`h264`] | ITU-T H.264 / AVC: decoder (`h264-decode`) and encoder (`h264-encode`) |
//! | `hevc` | [`hevc`] | ITU-T H.265 / HEVC: decoder (`hevc-decode`) and encoder (`hevc-encode`) |
@@ -45,6 +46,9 @@ pub mod mpeg4;
#[cfg(feature = "ffv1")]
pub mod ffv1;
#[cfg(feature = "mjpeg")]
pub mod mjpeg;
#[cfg(feature = "prores")]
pub mod prores;
+176
View File
@@ -0,0 +1,176 @@
//! Motion JPEG (MJPEG): every picture of the stream is a JPEG image (ITU-T T.81 | ISO/IEC 10918-1),
//! so this module is a frame-level wrapper over `craft-jpeg`, which does the decoding and the
//! baseline encoding. There is no second JPEG implementation here.
//!
//! - [`Decoder`] decodes one packet to a [`Frame`]: baseline, extended-sequential (8 and 12 bit)
//! and progressive Huffman JPEG, grayscale or YCbCr (as RGB) with any chroma subsampling. Packets
//! of AVI `MJPG` streams (the "AVI1" framing) often leave out the Huffman tables: the Annex K
//! tables are put in when a packet has none.
//! - [`Encoder`] writes each frame as a complete baseline JPEG (SOI to EOI, tables in the packet)
//! with Huffman tables made for the picture, which is what AVI `MJPG`, QuickTime `jpeg`, Matroska
//! `V_MJPEG` and the other containers carry as a sample.
//! - [`is_packet`] tells a JPEG packet from other data.
//!
//! Not here: the two-field packets of interlaced MJPEG (QuickTime MJPEG-A and -B, AVI field
//! pairs), lossless JPEG (SOF3), arithmetic coding and the hierarchical process, which
//! `craft-jpeg` does not decode. Such packets are errors, never panics.
use craft_jpeg::Limits;
pub use craft_jpeg::{ChromaSubsampling, Samples};
use crate::budget::DECODE_MEMORY_BUDGET;
/// Codec errors.
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
pub enum Error {
/// The packet is not a decodable JPEG.
#[error("invalid MJPEG packet: {0}")]
Invalid(String),
/// A JPEG process this module does not decode.
#[error("unsupported MJPEG feature: {0}")]
Unsupported(String),
/// The picture is larger than the memory budget or [`Decoder::max_pixels`] allows.
#[error("MJPEG picture above the limits: {0}")]
Limit(String),
/// The encoder was given a frame or settings it cannot encode.
#[error("invalid encoder input: {0}")]
Input(String),
}
/// `Result` of this module.
pub type Result<T> = std::result::Result<T, Error>;
impl From<craft_jpeg::Error> for Error {
fn from(e: craft_jpeg::Error) -> Self {
match e {
craft_jpeg::Error::Malformed(m) => Error::Invalid(m),
craft_jpeg::Error::Unsupported(m) => Error::Unsupported(m),
craft_jpeg::Error::LimitExceeded(m) => Error::Limit(m),
_ => Error::Invalid("undecodable JPEG".to_string()),
}
}
}
/// One decoded picture: interleaved samples, rows top to bottom.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Frame {
pub width: u32,
pub height: u32,
/// Samples per pixel: 1 for grayscale, 3 for RGB (a YCbCr picture is converted, JFIF / T.871).
pub channels: u8,
/// `U8` for 8-bit pictures; `U16` for 12-bit ones, scaled to the 16-bit range.
pub samples: Samples,
/// The entropy-coded data was damaged or cut short: what could be decoded is kept, the rest
/// of that picture is grey, and the stream goes on with the next packet.
pub damaged: bool,
}
/// Whether `packet` starts like a JPEG image (the SOI marker followed by another marker).
pub fn is_packet(packet: &[u8]) -> bool {
matches!(packet, [0xFF, 0xD8, 0xFF, ..])
}
/// A Motion JPEG decoder. Every packet is independent, so one decoder can decode packets in any
/// order and from several threads.
#[derive(Clone, Debug)]
pub struct Decoder {
limits: Limits,
}
impl Default for Decoder {
fn default() -> Self {
Self::new()
}
}
impl Decoder {
/// A decoder whose pictures stay within the decoding memory budget of the crate.
pub fn new() -> Self {
let mut limits = Limits::default();
limits.max_alloc = DECODE_MEMORY_BUDGET;
Decoder { limits }
}
/// Refuse pictures of more than `pixels` pixels (`width × height`).
pub fn max_pixels(mut self, pixels: u64) -> Self {
self.limits.max_pixels = pixels;
self
}
/// Decode one packet.
pub fn decode(&self, packet: &[u8]) -> Result<Frame> {
if !is_packet(packet) {
return Err(Error::Invalid("the packet does not start with a JPEG SOI marker".to_string()));
}
// AVI `MJPG` packets may leave out the Huffman tables; every other packet keeps its own
let packet = craft_jpeg::with_avi1_default_huffman(packet);
let d = craft_jpeg::decode(&packet, &self.limits, None)?;
Ok(Frame { width: d.width, height: d.height, channels: d.components, samples: d.samples, damaged: d.damaged })
}
}
/// Encoder settings.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct EncoderConfig {
/// JPEG quality, 1 (smallest) to 100 (best); the usual scale on the Annex K tables.
pub quality: u8,
/// Chroma sampling of colour pictures; ignored for grayscale.
pub subsampling: ChromaSubsampling,
}
impl Default for EncoderConfig {
fn default() -> Self {
EncoderConfig { quality: 90, subsampling: ChromaSubsampling::S420 }
}
}
/// A Motion JPEG encoder: each frame becomes one baseline JPEG packet.
#[derive(Clone, Copy, Debug, Default)]
pub struct Encoder {
cfg: EncoderConfig,
}
impl Encoder {
/// An encoder, or the setting it cannot carry.
pub fn new(cfg: EncoderConfig) -> Result<Encoder> {
if !(1..=100).contains(&cfg.quality) {
return Err(Error::Input(format!("quality {} is not in 1..=100", cfg.quality)));
}
Ok(Encoder { cfg })
}
/// Encode `width × height` interleaved 8-bit pixels of `channels` samples (1 gray, 3 RGB, or 4
/// RGBA with the alpha dropped).
pub fn encode(&self, width: u32, height: u32, channels: u8, pixels: &[u8]) -> Result<Vec<u8>> {
craft_jpeg::encode_baseline(pixels, width as usize, height as usize, usize::from(channels), self.cfg.quality, self.cfg.subsampling, &[])
.map_err(|e| Error::Input(e.to_string()))
}
/// Encode a decoded [`Frame`] of 8-bit samples.
pub fn encode_frame(&self, frame: &Frame) -> Result<Vec<u8>> {
match &frame.samples {
Samples::U8(p) => self.encode(frame.width, frame.height, frame.channels, p),
_ => Err(Error::Input("only 8-bit frames can be encoded".to_string())),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn packets_start_with_soi() {
assert!(is_packet(&[0xFF, 0xD8, 0xFF, 0xE0]));
assert!(!is_packet(&[0xFF, 0xD8]));
assert!(!is_packet(b"RIFF"));
assert!(Decoder::new().decode(b"not a jpeg").is_err());
}
#[test]
fn quality_is_checked() {
assert!(Encoder::new(EncoderConfig { quality: 0, ..Default::default() }).is_err());
assert!(Encoder::new(EncoderConfig { quality: 101, ..Default::default() }).is_err());
assert!(Encoder::new(EncoderConfig::default()).is_ok());
}
}
+226
View File
@@ -0,0 +1,226 @@
//! Motion JPEG: the encoder and decoder against each other, against ffmpeg as the reference
//! (oracle; skipped when ffmpeg is absent, a failure with `CRAFT_MEDIA_REQUIRE_ORACLES=1`), and on
//! hostile packets.
use craft_video::mjpeg::{ChromaSubsampling, Decoder, Encoder, EncoderConfig, Frame, Samples, is_packet};
use std::path::Path;
use std::process::Command;
/// A smooth picture with a diagonal edge, `channels` samples per pixel.
fn picture(w: usize, h: usize, channels: usize) -> Vec<u8> {
let mut v = Vec::with_capacity(w * h * channels);
for y in 0..h {
for x in 0..w {
let edge = if x + y > (w + h) / 2 { 40.0 } else { 0.0 };
for c in 0..channels {
let base = 90.0 + 60.0 * ((x as f32 * 0.11 + c as f32).sin() + (y as f32 * 0.07 - c as f32).cos());
v.push((base + edge + 20.0 * c as f32).clamp(0.0, 255.0) as u8);
}
}
}
v
}
fn psnr(a: &[u8], b: &[u8]) -> f64 {
assert_eq!(a.len(), b.len());
let mse = a.iter().zip(b).map(|(x, y)| (f64::from(*x) - f64::from(*y)).powi(2)).sum::<f64>() / a.len().max(1) as f64;
if mse == 0.0 { 99.0 } else { 10.0 * (255.0f64 * 255.0 / mse).log10() }
}
fn bytes(f: &Frame) -> &[u8] {
match &f.samples {
Samples::U8(v) => v,
_ => panic!("8-bit expected"),
}
}
#[test]
fn encoded_frames_decode_back_close_to_the_input() {
let dec = Decoder::new();
for (w, h, ch, sub, min) in [
(64, 48, 3, ChromaSubsampling::S444, 38.0),
(67, 41, 3, ChromaSubsampling::S422, 33.0),
(67, 41, 3, ChromaSubsampling::S420, 32.0),
(1, 1, 3, ChromaSubsampling::S420, 25.0),
(33, 17, 1, ChromaSubsampling::S420, 38.0),
] {
let pixels = picture(w, h, ch);
let enc = Encoder::new(EncoderConfig { quality: 92, subsampling: sub }).expect("encoder");
let packet = enc.encode(w as u32, h as u32, ch as u8, &pixels).expect("encode");
assert!(is_packet(&packet));
assert_eq!(&packet[packet.len() - 2..], &[0xFF, 0xD9], "a packet runs from SOI to EOI");
let f = dec.decode(&packet).expect("decode");
assert_eq!((f.width as usize, f.height as usize, f.channels as usize, f.damaged), (w, h, ch, false));
let p = psnr(&pixels, bytes(&f));
assert!(p >= min, "{w}x{h} {ch}ch {sub:?}: {p:.1} dB under {min}");
}
}
#[test]
fn rgba_drops_alpha_and_bad_input_is_refused() {
let enc = Encoder::default();
let rgba: Vec<u8> = (0..16 * 16 * 4).map(|i| (i * 7) as u8).collect();
let f = Decoder::new().decode(&enc.encode(16, 16, 4, &rgba).expect("encode")).expect("decode");
assert_eq!(f.channels, 3);
assert!(enc.encode(16, 16, 3, &[0; 10]).is_err());
assert!(enc.encode(0, 16, 3, &[]).is_err());
assert!(enc.encode(16, 16, 2, &rgba).is_err());
assert!(enc.encode(70_000, 1, 1, &[0; 70_000]).is_err());
}
#[test]
fn the_pixel_limit_is_applied_before_allocating() {
let enc = Encoder::default();
let packet = enc.encode(64, 64, 1, &[7; 64 * 64]).expect("encode");
assert!(Decoder::new().max_pixels(64 * 64).decode(&packet).is_ok());
assert!(matches!(Decoder::new().max_pixels(64 * 64 - 1).decode(&packet), Err(craft_video::mjpeg::Error::Limit(_))));
// a header that claims 65535 x 65535 pixels of a tiny file
let mut huge = packet.clone();
let sof = huge.windows(2).position(|w| w == [0xFF, 0xC0]).expect("SOF0");
huge[sof + 5..sof + 9].copy_from_slice(&[0xFF, 0xFF, 0xFF, 0xFF]);
assert!(Decoder::new().max_pixels(1 << 20).decode(&huge).is_err());
}
#[test]
fn hostile_packets_are_errors_never_panics() {
let enc = Encoder::default();
let packet = enc.encode(48, 40, 3, &picture(48, 40, 3)).expect("encode");
let dec = Decoder::new();
// every truncation
for n in 0..packet.len() {
let _ = dec.decode(&packet[..n]);
}
// single byte flips, and a run of noise after the headers
let mut x = 0x2545_F491u32;
for i in 0..packet.len() {
let mut p = packet.clone();
x ^= x << 13;
x ^= x >> 17;
x ^= x << 5;
p[i] ^= (x >> 8) as u8 | 1;
let _ = dec.decode(&p);
}
let mut noisy = packet.clone();
let from = noisy.len() / 2;
for b in &mut noisy[from..] {
x ^= x << 13;
x ^= x >> 17;
x ^= x << 5;
*b = (x >> 8) as u8;
}
let _ = dec.decode(&noisy);
for junk in [&[][..], &[0xFF, 0xD8], &[0xFF, 0xD8, 0xFF, 0xD9], &[0xFF, 0xD8, 0xFF, 0xC3, 0, 2], b"\xFF\xD8\xFF\xDA\x00\x02"] {
assert!(dec.decode(junk).is_err());
}
}
#[test]
fn packets_that_are_not_baseline_or_progressive_huffman_are_unsupported() {
// SOF3 (lossless): a frame header, no more
let sof3 = [0xFF, 0xD8, 0xFF, 0xC3, 0, 11, 8, 0, 8, 0, 8, 1, 1, 0x11, 0, 0xFF, 0xD9];
assert!(matches!(Decoder::new().decode(&sof3), Err(craft_video::mjpeg::Error::Unsupported(_))), "{:?}", Decoder::new().decode(&sof3));
}
fn ffmpeg(ff: &Path, args: &[&str]) {
let out = Command::new(ff).args(["-v", "error", "-nostdin", "-y"]).args(args).output().expect("run ffmpeg");
assert!(out.status.success(), "ffmpeg {args:?}: {}", String::from_utf8_lossy(&out.stderr));
}
/// The SOS-preceding DHT segments cut out of a JPEG, as AVI `MJPG` packets often come.
fn without_huffman_tables(jpeg: &[u8]) -> Vec<u8> {
let mut out = vec![0xFF, 0xD8];
let mut at = 2;
while at + 3 < jpeg.len() && jpeg[at] == 0xFF {
let marker = jpeg[at + 1];
let len = usize::from(u16::from_be_bytes([jpeg[at + 2], jpeg[at + 3]]));
if marker == 0xDA {
break;
}
if marker != 0xC4 {
out.extend_from_slice(&jpeg[at..at + 2 + len]);
}
at += 2 + len;
}
out.extend_from_slice(&jpeg[at..]);
out
}
#[test]
fn we_decode_what_ffmpeg_encodes_to_what_ffmpeg_decodes() {
let Some(ff) = craft_media_testkit::ffmpeg() else { return };
let dir = craft_media_testkit::fixture_subdir("mjpeg");
let s = |p: &Path| p.to_str().expect("path").to_string();
for (name, pix_fmt, rgb_pix, channels) in [("420", "yuvj420p", "rgb24", 3), ("422", "yuvj422p", "rgb24", 3), ("444", "yuvj444p", "rgb24", 3)] {
let jpg = dir.join(format!("{name}.jpg"));
ffmpeg(
&ff,
&["-f", "lavfi", "-i", "gradients=size=67x41:rate=1", "-frames:v", "1", "-c:v", "mjpeg", "-pix_fmt", pix_fmt, "-q:v", "2", &s(&jpg)],
);
let reference = dir.join(format!("{name}.raw"));
ffmpeg(&ff, &["-i", &s(&jpg), "-f", "rawvideo", "-pix_fmt", rgb_pix, &s(&reference)]);
let packet = std::fs::read(&jpg).expect("read");
let f = Decoder::new().decode(&packet).expect("decode");
assert_eq!((f.width, f.height, f.channels as usize), (67, 41, channels), "{name}");
let p = psnr(&std::fs::read(&reference).expect("read"), bytes(&f));
println!("{name}: {p:.1} dB against ffmpeg's decode");
assert!(p >= 38.0, "{name}: {p:.1} dB");
// the same stream with the Huffman tables left out, as AVI MJPG often has it, if ffmpeg
// wrote the Annex K ones
let plain = dir.join(format!("{name}_default_tables.jpg"));
ffmpeg(
&ff,
&[
"-f",
"lavfi",
"-i",
"gradients=size=67x41:rate=1",
"-frames:v",
"1",
"-c:v",
"mjpeg",
"-pix_fmt",
pix_fmt,
"-q:v",
"2",
"-huffman",
"default",
&s(&plain),
],
);
let stripped = without_huffman_tables(&std::fs::read(&plain).expect("read"));
let with_tables = Decoder::new().decode(&std::fs::read(&plain).expect("read")).expect("decode");
let without = Decoder::new().decode(&stripped).expect("decode without DHT");
assert_eq!(with_tables, without, "{name}: the default tables are put in");
}
}
#[test]
fn ffmpeg_decodes_what_we_encode() {
let Some(ff) = craft_media_testkit::ffmpeg() else { return };
let dir = craft_media_testkit::fixture_subdir("mjpeg");
let s = |p: &Path| p.to_str().expect("path").to_string();
for (name, w, h, ch, sub, pix_fmt) in [
("o420", 67usize, 41usize, 3usize, ChromaSubsampling::S420, "rgb24"),
("o422", 67, 41, 3, ChromaSubsampling::S422, "rgb24"),
("o444", 64, 48, 3, ChromaSubsampling::S444, "rgb24"),
("ogray", 33, 17, 1, ChromaSubsampling::S420, "gray"),
] {
let pixels = picture(w, h, ch);
let packet = Encoder::new(EncoderConfig { quality: 90, subsampling: sub })
.expect("encoder")
.encode(w as u32, h as u32, ch as u8, &pixels)
.expect("encode");
let jpg = dir.join(format!("{name}.jpg"));
std::fs::write(&jpg, &packet).expect("write");
let out = dir.join(format!("{name}.raw"));
ffmpeg(&ff, &["-i", &s(&jpg), "-f", "rawvideo", "-pix_fmt", pix_fmt, &s(&out)]);
let theirs = std::fs::read(&out).expect("read");
let ours = Decoder::new().decode(&packet).expect("decode");
let p = psnr(&theirs, bytes(&ours));
let q = psnr(&pixels, &theirs);
println!("{name}: ffmpeg decode against ours {p:.1} dB, against the input {q:.1} dB");
assert!(p >= 38.0, "{name}: ffmpeg's decode of our packet differs from ours: {p:.1} dB");
assert!(q >= 28.0, "{name}: ffmpeg's decode is far from the input: {q:.1} dB");
}
}