diff --git a/crates/av1enc/src/lib.rs b/crates/av1enc/src/lib.rs index b86db0fa2..2962833d9 100644 --- a/crates/av1enc/src/lib.rs +++ b/crates/av1enc/src/lib.rs @@ -52,11 +52,28 @@ pub struct EncoderConfig { pub keyint: u32, /// BT.709 limited range is signalled unless this is set. pub full_range: bool, + /// The colour description of `color_config` (ITU-T H.273 code points); BT.709 by default. + pub color_primaries: u8, + pub transfer_characteristics: u8, + pub matrix_coefficients: u8, } impl EncoderConfig { pub fn new(width: u32, height: u32, fps_num: u32, fps_den: u32) -> Self { - EncoderConfig { width, height, fps_num, fps_den, bit_depth: 8, level_idx: None, rate: RateControl::ConstantQ(100), keyint: 60, full_range: false } + EncoderConfig { + width, + height, + fps_num, + fps_den, + bit_depth: 8, + level_idx: None, + rate: RateControl::ConstantQ(100), + keyint: 60, + full_range: false, + color_primaries: 1, + transfer_characteristics: 1, + matrix_coefficients: 1, + } } } @@ -364,9 +381,9 @@ impl Encoder { w.f(1, (c.bit_depth > 8) as u32); // high_bitdepth w.f(1, 0); // mono_chrome w.f(1, 1); // color_description_present_flag - w.f(8, 1); // color_primaries: BT.709 - w.f(8, 1); // transfer_characteristics: BT.709 - w.f(8, 1); // matrix_coefficients: BT.709 + w.f(8, c.color_primaries as u32); // color_primaries + w.f(8, c.transfer_characteristics as u32); // transfer_characteristics + w.f(8, c.matrix_coefficients as u32); // matrix_coefficients w.f(1, c.full_range as u32); // color_range w.f(2, 0); // chroma_sample_position: unknown w.f(1, 0); // separate_uv_delta_q diff --git a/crates/export/src/encode.rs b/crates/export/src/encode.rs index c9888a035..dd972d326 100644 --- a/crates/export/src/encode.rs +++ b/crates/export/src/encode.rs @@ -19,14 +19,41 @@ fn mux_err(e: impl std::fmt::Display) -> ExportError { ExportError::Io(e.to_string()) } -/// Tag a video [`SampleEntry`] with the BT.709 limited-range `colr` (nclx) box every video -/// encoder here outputs (`rgba_to_yuv420`, `rgba_to_yuv_10` and `rgba_to_yuv420_bits` all use -/// the 0.2126/0.7152/0.0722 luma and a 16–235 luma range). Without it a re-import falls back to -/// the container's resolution-based `default_matrix` — BT.601 below 1024×577 — and shifts the +/// What a render's frames are, as ITU-T H.273 code points, for the `colr` box and the encoders' +/// own colour description. The matrix is BT.709, the one every encoder here converts with +/// (`rgba_to_yuv420`, `rgba_to_yuv_10` and `rgba_to_yuv420_bits` all use the 0.2126/0.7152/0.0722 +/// luma). Primaries and transfer follow the render path (`Pipe::output`): only a working space +/// converts to the output space, so unmanaged frames pass through. sRGB output keeps transfer 1 +/// rather than 13 (#340), and an alpha matte is not in the output space. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Codes { + pub(crate) primaries: u8, + pub(crate) transfer: u8, + pub(crate) matrix: u8, +} + +pub(crate) fn output_codes(settings: &ProjectSettings, channels: Channels) -> Codes { + use effectcraft_project::ColorSpace as Space; + let output = settings.working_space.and(settings.output_space).filter(|_| channels != Channels::Alpha); + let (primaries, transfer) = match output { + None | Some(Space::Srgb | Space::Rec709) => (1, 1), + Some(Space::Rec2020) => (9, 1), + Some(Space::DisplayP3) => (12, 13), + Some(Space::Rec2100Pq) => (9, 16), + Some(Space::Rec2100Hlg) => (9, 18), + // H.273 has no code for the ACES primaries: unspecified, in linear light. + Some(Space::AcesCg | Space::Aces2065) => (2, 8), + }; + Codes { primaries, transfer, matrix: 1 } +} + +/// Tag a video [`SampleEntry`] with the limited-range (16–235 luma, as every encoder here +/// writes) `colr` (nclx) box for `c` (#2879). Without it a re-import falls back to the +/// container's resolution-based `default_matrix` — BT.601 below 1024×577 — and shifts the /// colours (pure red came back as [232,0,1]). -pub(crate) fn tag_bt709_limited(entry: &mut SampleEntry) { +pub(crate) fn tag_nclx_limited(entry: &mut SampleEntry, c: Codes) { if let Some(v) = entry.video.as_mut() { - v.color = Some(ColorInfo::Nclx { primaries: 1, transfer: 1, matrix: 1, full_range: false }); + v.color = Some(ColorInfo::Nclx { primaries: c.primaries.into(), transfer: c.transfer.into(), matrix: c.matrix.into(), full_range: false }); } } @@ -94,21 +121,23 @@ struct H264 { w: u32, h: u32, rate: FrameRate, + codes: Codes, y: Vec, u: Vec, v: Vec, } impl H264 { - fn new(w: u32, h: u32, rate: FrameRate, kbps: u32) -> Result { + fn new(w: u32, h: u32, rate: FrameRate, kbps: u32, codes: Codes) -> Result { let mut cfg = filmcraft_h264enc::EncoderConfig::new(w, h, rate.num as u32, rate.den as u32); cfg.format = filmcraft_h264enc::PacketFormat::LengthPrefixed; cfg.aud = false; cfg.keyint = (rate.as_f64() * 2.0).round().max(1.0) as u32; let kbps = kbps.max(100); cfg.rate = filmcraft_h264enc::RateControl::Vbr { target_kbps: kbps, max_kbps: kbps * 3 / 2 }; + cfg.color = filmcraft_h264enc::ColorConfig { primaries: codes.primaries, transfer: codes.transfer, matrix: codes.matrix, full_range: false }; let enc = filmcraft_h264enc::Encoder::new(cfg).map_err(enc)?; - Ok(H264 { enc, w, h, rate, y: vec![], u: vec![], v: vec![] }) + Ok(H264 { enc, w, h, rate, codes, y: vec![], u: vec![], v: vec![] }) } fn packets(&self, ps: Vec) -> Vec { ps.into_iter().map(|p| Packet { data: p.data, key: p.keyframe, cto: (p.pts - p.dts) as i32 }).collect() @@ -122,7 +151,7 @@ impl VideoEncoder for H264 { filmcraft_isobmff::AvcConfig::new(vec![sps], vec![pps], 4) }); let mut entry = SampleEntry::avc(cfg, self.w as u16, self.h as u16); - tag_bt709_limited(&mut entry); + tag_nclx_limited(&mut entry, self.codes); entry } fn encode(&mut self, rgba: &[u8], index: u64) -> Result> { @@ -167,25 +196,10 @@ struct ProRes { colr: ColorInfo, } -/// The `colr` atom of a ProRes render: what its frames are, in H.273 code points. MOV uses -/// QuickTime's `nclc`, which has no range flag; ProRes frames are limited range. The matrix is the -/// one [`rgba_to_yuv_10`] always uses. Primaries and transfer follow the render path -/// (`Pipe::output`): only a working space converts to the output space, so unmanaged frames pass -/// through. sRGB output keeps transfer 1 rather than 13 (#340), and an alpha matte is not in the -/// output space. -fn prores_colr(settings: &ProjectSettings, channels: Channels) -> ColorInfo { - use effectcraft_project::ColorSpace as Space; - let output = settings.working_space.and(settings.output_space).filter(|_| channels != Channels::Alpha); - let (primaries, transfer) = match output { - None | Some(Space::Srgb | Space::Rec709) => (1, 1), - Some(Space::Rec2020) => (9, 1), - Some(Space::DisplayP3) => (12, 13), - Some(Space::Rec2100Pq) => (9, 16), - Some(Space::Rec2100Hlg) => (9, 18), - // H.273 has no code for the ACES primaries: unspecified, in linear light. - Some(Space::AcesCg | Space::Aces2065) => (2, 8), - }; - ColorInfo::Nclc { primaries, transfer, matrix: 1 } +/// The `colr` atom of a ProRes render ([`output_codes`]). MOV uses QuickTime's `nclc`, which has +/// no range flag; ProRes frames are limited range. +fn prores_colr(c: Codes) -> ColorInfo { + ColorInfo::Nclc { primaries: c.primaries.into(), transfer: c.transfer.into(), matrix: c.matrix.into() } } fn prores_profile(p: ProResProfile) -> filmcraft_prores::Profile { @@ -328,15 +342,13 @@ pub(crate) async fn movie(job: &Cx<'_>, comp: &Comp, w: u32, h: u32, st: &mut St Channels::Rgba if !matches!(fmt, OutputFormat::ProRes | OutputFormat::Uncompressed) => Channels::Rgb, c => c, }; + let codes = output_codes(&job.project.settings, channels); let mut venc: Box = match fmt { - OutputFormat::H264 => Box::new(H264::new(w, h, rate, job.output.bitrate_kbps)?), - OutputFormat::Hevc => Box::new(crate::hevc_av1::Hevc::new(w, h, rate, job.output)?), - OutputFormat::Av1 => Box::new(crate::hevc_av1::Av1::new(w, h, rate, job.output)?), + OutputFormat::H264 => Box::new(H264::new(w, h, rate, job.output.bitrate_kbps, codes)?), + OutputFormat::Hevc => Box::new(crate::hevc_av1::Hevc::new(w, h, rate, job.output, codes)?), + OutputFormat::Av1 => Box::new(crate::hevc_av1::Av1::new(w, h, rate, job.output, codes)?), OutputFormat::Uncompressed => Box::new(crate::uncompressed::Raw::new(w, h, channels == Channels::Rgba)?), - _ => { - let colr = prores_colr(&job.project.settings, channels); - Box::new(ProRes::new(w, h, job.output.prores_profile, if channels == Channels::Alpha { Channels::Rgb } else { channels }, colr)) - } + _ => Box::new(ProRes::new(w, h, job.output.prores_profile, if channels == Channels::Alpha { Channels::Rgb } else { channels }, prores_colr(codes))), }; let brand = if matches!(fmt, OutputFormat::ProRes | OutputFormat::Uncompressed) { Brand::Mov } else { Brand::Mp4 }; let batch = batch_size(); diff --git a/crates/export/src/hevc_av1.rs b/crates/export/src/hevc_av1.rs index c363dff35..d43b1c5ed 100644 --- a/crates/export/src/hevc_av1.rs +++ b/crates/export/src/hevc_av1.rs @@ -7,7 +7,7 @@ use effectcraft_time::FrameRate; use filmcraft_isobmff::{Av1Config, CodecConfig, FourCc, HevcConfig, SampleEntry}; use rayon::prelude::*; -use crate::encode::{Packet, VideoEncoder}; +use crate::encode::{Codes, Packet, VideoEncoder}; use crate::{ExportError, Result}; fn enc_err(e: impl std::fmt::Display) -> ExportError { @@ -59,6 +59,7 @@ pub(crate) struct Hevc { w: u32, h: u32, bits: u8, + codes: Codes, planes: Planes, } @@ -81,11 +82,12 @@ pub(crate) fn hevc_config(w: u32, h: u32, rate: FrameRate, om: &OutputModule) -> } impl Hevc { - pub(crate) fn new(w: u32, h: u32, rate: FrameRate, om: &OutputModule) -> Result { - let cfg = hevc_config(w, h, rate, om); + pub(crate) fn new(w: u32, h: u32, rate: FrameRate, om: &OutputModule, codes: Codes) -> Result { + let mut cfg = hevc_config(w, h, rate, om); + (cfg.colour_primaries, cfg.transfer_characteristics, cfg.matrix_coeffs) = (codes.primaries, codes.transfer, codes.matrix); let bits = cfg.profile.bit_depth(); let enc = effectcraft_hevcenc::Encoder::new(cfg).map_err(enc_err)?; - Ok(Hevc { enc, w, h, bits, planes: Planes::default() }) + Ok(Hevc { enc, w, h, bits, codes, planes: Planes::default() }) } } @@ -93,7 +95,7 @@ impl VideoEncoder for Hevc { fn sample_entry(&self) -> SampleEntry { let cfg = HevcConfig::parse(&self.enc.hvcc()).unwrap_or_default(); let mut entry = SampleEntry::hevc(cfg, self.w as u16, self.h as u16); - crate::encode::tag_bt709_limited(&mut entry); + crate::encode::tag_nclx_limited(&mut entry, self.codes); entry } fn encode(&mut self, rgba: &[u8], _index: u64) -> Result> { @@ -116,6 +118,7 @@ pub(crate) struct Av1 { w: u32, h: u32, bits: u8, + codes: Codes, planes: Planes, } @@ -135,11 +138,12 @@ pub(crate) fn av1_config(w: u32, h: u32, rate: FrameRate, om: &OutputModule) -> } impl Av1 { - pub(crate) fn new(w: u32, h: u32, rate: FrameRate, om: &OutputModule) -> Result { - let cfg = av1_config(w, h, rate, om); + pub(crate) fn new(w: u32, h: u32, rate: FrameRate, om: &OutputModule, codes: Codes) -> Result { + let mut cfg = av1_config(w, h, rate, om); + (cfg.color_primaries, cfg.transfer_characteristics, cfg.matrix_coefficients) = (codes.primaries, codes.transfer, codes.matrix); let bits = cfg.bit_depth; let enc = effectcraft_av1enc::Encoder::new(cfg).map_err(enc_err)?; - Ok(Av1 { enc, w, h, bits, planes: Planes::default() }) + Ok(Av1 { enc, w, h, bits, codes, planes: Planes::default() }) } /// The `av1C` record (also the WebM CodecPrivate). pub(crate) fn av1c(&self) -> Vec { @@ -151,7 +155,7 @@ impl VideoEncoder for Av1 { fn sample_entry(&self) -> SampleEntry { let cfg = Av1Config::parse(&self.enc.av1c()).unwrap_or_default(); let mut entry = SampleEntry::video(FourCc(*b"av01"), CodecConfig::Av1(cfg), self.w as u16, self.h as u16); - crate::encode::tag_bt709_limited(&mut entry); + crate::encode::tag_nclx_limited(&mut entry, self.codes); entry } fn encode(&mut self, rgba: &[u8], _index: u64) -> Result> { diff --git a/crates/export/src/webm_av1.rs b/crates/export/src/webm_av1.rs index ca06a3e51..a85a5c9ec 100644 --- a/crates/export/src/webm_av1.rs +++ b/crates/export/src/webm_av1.rs @@ -42,7 +42,8 @@ fn cluster(blocks: &mut [Block]) -> Vec { pub(crate) async fn webm_av1(job: &Cx<'_>, comp: &Comp, w: u32, h: u32, st: &mut State<'_>) -> Result { let rate = job.settings.rate(comp); - let mut venc = crate::hevc_av1::Av1::new(w, h, rate, job.output)?; + let codes = crate::encode::output_codes(&job.project.settings, job.output.channels); + let mut venc = crate::hevc_av1::Av1::new(w, h, rate, job.output, codes)?; let with_audio = wants_audio(job); let opus_channels = if job.output.audio_channels == 1 { 1usize } else { 2 }; let mut opus = with_audio.then(|| opus_encoder(job)); diff --git a/crates/export/tests/it/main.rs b/crates/export/tests/it/main.rs index a5b9fa99a..f7bfa7a1c 100644 --- a/crates/export/tests/it/main.rs +++ b/crates/export/tests/it/main.rs @@ -7,6 +7,7 @@ mod hevc_av1_roundtrip; #[cfg(not(target_arch = "wasm32"))] mod mov_audio_sample_positions; mod movie_bytes; +mod movie_color; mod ms_rounded_footage; #[cfg(not(target_arch = "wasm32"))] mod native_audio_sample_positions; diff --git a/crates/export/tests/it/movie_color.rs b/crates/export/tests/it/movie_color.rs new file mode 100644 index 000000000..1a5371d91 --- /dev/null +++ b/crates/export/tests/it/movie_color.rs @@ -0,0 +1,95 @@ +//! H.264, HEVC and AV1 MP4s describe the frames the render path made (#2879): the `colr` box, read +//! back with FilmCraft's demuxer, and the bitstream's own colour description (the SPS VUI, AV1's +//! `color_config`), which ffprobe reports for the first frame of the bare elementary stream when +//! ffmpeg can copy it out and decode it. + +use std::path::Path; +use std::process::Command; + +use effectcraft_color::Label; +use effectcraft_export::render_queue::{OutputFormat, OutputModule, RenderSettings, TimeSpan}; +use effectcraft_export::{Job, export}; +use effectcraft_project::{ColorSpace, Comp, ItemKind, LayerSource, Project, Solid, build}; +use effectcraft_render::NoFootage; +use effectcraft_time::{FrameRate, Tick}; +use filmcraft_isobmff::{ColorInfo, TrackKind}; + +/// (name, Output color space, H.273 primaries / transfer / matrix, the same as ffprobe names them) +const ROWS: &[(&str, Option, [u16; 3], [&str; 3])] = &[ + ("default", None, [1, 1, 1], ["bt709", "bt709", "bt709"]), + ("rec2020", Some(ColorSpace::Rec2020), [9, 1, 1], ["bt2020", "bt709", "bt709"]), + ("p3", Some(ColorSpace::DisplayP3), [12, 13, 1], ["smpte432", "iec61966-2-1", "bt709"]), + ("pq", Some(ColorSpace::Rec2100Pq), [9, 16, 1], ["bt2020", "smpte2084", "bt709"]), + ("hlg", Some(ColorSpace::Rec2100Hlg), [9, 18, 1], ["bt2020", "arib-std-b67", "bt709"]), +]; + +fn render(output: Option, format: OutputFormat, path: &Path) { + let mut p = Project::default(); + p.settings.working_space = Some(ColorSpace::Srgb); + p.settings.output_space = output; + let mut comp = Comp::new(64, 48, FrameRate::new(10, 1), Tick::from_seconds_f64(0.2)); + let sid = p.add_item("Solid", Label::Red, None, ItemKind::Solid(Solid { color: [0.75, 0.25, 0.5], width: 64, height: 48, pixel_aspect: 1.0 })); + comp.layers.push(build::layer(&mut p, &comp, "Solid", LayerSource::Solid { item: sid }, (64, 48), None)); + let comp = p.add_item("Comp", Label::Sandstone, None, ItemKind::Comp(comp.into())); + let settings = RenderSettings { time_span: TimeSpan::LengthOfComp, ..Default::default() }; + let om = OutputModule::for_format(format); + let out = path.to_string_lossy().to_string(); + let job = Job { + project: &p, + footage: &NoFootage, + expr: None, + accel: None, + comp, + settings: &settings, + output: &om, + path: &out, + sink: None, + options: Default::default(), + nested_switches: true, + }; + export(&job, &mut |_| true).expect("export"); +} + +/// ffprobe's primaries, transfer and matrix of the first frame of the movie's video copied out as a +/// bare elementary stream (`h264`, `hevc` or `obu`), so no container metadata can stand in for the +/// bitstream's own; `None` without ffmpeg or when its build can't copy or decode the stream. +fn bitstream_tags(path: &Path, format: &str) -> Option> { + let es = path.with_extension(format); + let copied = + Command::new("ffmpeg").args(["-v", "error", "-y", "-i"]).arg(path).args(["-map", "0:v:0", "-c", "copy", "-f", format]).arg(&es).status().ok()?; + if !copied.success() { + return None; + } + let out = Command::new("ffprobe") + .args(["-v", "error", "-read_intervals", "%+#1", "-show_entries", "frame=color_primaries,color_transfer,color_space", "-of", "default=nw=1"]) + .arg(&es) + .output() + .ok()?; + let text = String::from_utf8_lossy(&out.stdout).to_string(); + let tag = |key: &str| text.lines().find_map(|l| l.strip_prefix(key)?.strip_prefix('=')).map(str::to_string); + Some(vec![tag("color_primaries")?, tag("color_transfer")?, tag("color_space")?]) +} + +#[test] +fn mp4_tags_describe_the_frames() { + let dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../target/test-out/export/movie_color"); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).expect("mkdir"); + for (format, codec, stream) in [(OutputFormat::H264, "h264", "h264"), (OutputFormat::Hevc, "hevc", "hevc"), (OutputFormat::Av1, "av1", "obu")] { + for &(name, output, [primaries, transfer, matrix], names) in ROWS { + let path = dir.join(format!("{codec}_{name}.mp4")); + render(output, format, &path); + + let bytes = std::fs::read(&path).expect("read movie"); + let mp4 = filmcraft_isobmff::open(bytes.as_slice()).expect("demux"); + let track = mp4.tracks.iter().find(|t| t.kind == TrackKind::Video).expect("video track"); + let colr = track.entries[0].video.as_ref().and_then(|v| v.color.clone()); + assert_eq!(colr, Some(ColorInfo::Nclx { primaries, transfer, matrix, full_range: false }), "{codec} {name}: colr"); + + match bitstream_tags(&path, stream) { + Some(tags) => assert_eq!(tags, names, "{codec} {name}: the bitstream's colour description"), + None => eprintln!("{codec} {name}: ffprobe can't decode it here, bitstream not checked"), + } + } + } +} diff --git a/crates/hevcenc/src/lib.rs b/crates/hevcenc/src/lib.rs index 4d38f84d9..1934143a0 100644 --- a/crates/hevcenc/src/lib.rs +++ b/crates/hevcenc/src/lib.rs @@ -83,6 +83,10 @@ pub struct EncoderConfig { pub keyint: u32, /// BT.709 limited range is signalled in the VUI unless this is set. pub full_range: bool, + /// The VUI colour description (ITU-T H.273 code points); BT.709 by default. + pub colour_primaries: u8, + pub transfer_characteristics: u8, + pub matrix_coeffs: u8, } impl EncoderConfig { @@ -97,6 +101,9 @@ impl EncoderConfig { rate: RateControl::ConstantQp(28), keyint: 60, full_range: false, + colour_primaries: 1, + transfer_characteristics: 1, + matrix_coeffs: 1, } } } @@ -234,6 +241,9 @@ impl Encoder { profile_idc: cfg.profile.idc(), level_idc, full_range: cfg.full_range, + colour_primaries: cfg.colour_primaries, + transfer_characteristics: cfg.transfer_characteristics, + matrix_coeffs: cfg.matrix_coeffs, fps_num: cfg.fps_num, fps_den: cfg.fps_den, }; diff --git a/crates/hevcenc/src/params.rs b/crates/hevcenc/src/params.rs index f1b89f731..89bac938e 100644 --- a/crates/hevcenc/src/params.rs +++ b/crates/hevcenc/src/params.rs @@ -25,6 +25,9 @@ pub struct SeqParams { pub profile_idc: u8, pub level_idc: u8, pub full_range: bool, + pub colour_primaries: u8, + pub transfer_characteristics: u8, + pub matrix_coeffs: u8, pub fps_num: u32, pub fps_den: u32, } @@ -166,9 +169,9 @@ fn vui(w: &mut BitWriter, p: &SeqParams) { w.put(5, 3); // video_format: unspecified w.flag(p.full_range); w.put(1, 1); // colour_description_present_flag - w.put(1, 8); // colour_primaries BT.709 - w.put(1, 8); // transfer_characteristics BT.709 - w.put(1, 8); // matrix_coeffs BT.709 + w.put(p.colour_primaries as u32, 8); // colour_primaries + w.put(p.transfer_characteristics as u32, 8); // transfer_characteristics + w.put(p.matrix_coeffs as u32, 8); // matrix_coeffs w.put(0, 1); // chroma_loc_info_present_flag w.put(0, 1); // neutral_chroma_indication_flag w.put(0, 1); // field_seq_flag