Export: H.264, HEVC and AV1 tag the frames' own colour (#2879)

Every MP4 export said BT.709 whatever the Output color space, in the
colr box and in the bitstream, so Rec. 2020, P3, PQ and HLG renders
were labelled as BT.709.

Move the ProRes colr mapping to output_codes and use it for every movie
path: the nclx colr box, FilmCraft's H.264 ColorConfig (VUI), and new
colour fields in effectcraft-hevcenc (VUI) and effectcraft-av1enc
(color_config), which wrote 1-1-1 as constants. The matrix stays BT.709,
the coefficients every path converts with. Default renders don't change.

crates/av1enc/src/lib.rs

@@ -52,11 +52,28 @@ pub struct EncoderConfig {
     pub keyint: u32,
     /// BT.709 limited range is signalled unless this is set.
     pub full_range: bool,
+    /// The colour description of `color_config` (ITU-T H.273 code points); BT.709 by default.
+    pub color_primaries: u8,
+    pub transfer_characteristics: u8,
+    pub matrix_coefficients: u8,
 }
 
 impl EncoderConfig {
     pub fn new(width: u32, height: u32, fps_num: u32, fps_den: u32) -> Self {
-        EncoderConfig { width, height, fps_num, fps_den, bit_depth: 8, level_idx: None, rate: RateControl::ConstantQ(100), keyint: 60, full_range: false }
+        EncoderConfig {
+            width,
+            height,
+            fps_num,
+            fps_den,
+            bit_depth: 8,
+            level_idx: None,
+            rate: RateControl::ConstantQ(100),
+            keyint: 60,
+            full_range: false,
+            color_primaries: 1,
+            transfer_characteristics: 1,
+            matrix_coefficients: 1,
+        }
     }
 }
 
@@ -364,9 +381,9 @@ impl Encoder {
         w.f(1, (c.bit_depth > 8) as u32); // high_bitdepth
         w.f(1, 0); // mono_chrome
         w.f(1, 1); // color_description_present_flag
-        w.f(8, 1); // color_primaries: BT.709
-        w.f(8, 1); // transfer_characteristics: BT.709
-        w.f(8, 1); // matrix_coefficients: BT.709
+        w.f(8, c.color_primaries as u32); // color_primaries
+        w.f(8, c.transfer_characteristics as u32); // transfer_characteristics
+        w.f(8, c.matrix_coefficients as u32); // matrix_coefficients
         w.f(1, c.full_range as u32); // color_range
         w.f(2, 0); // chroma_sample_position: unknown
         w.f(1, 0); // separate_uv_delta_q

crates/export/src/encode.rs

@@ -19,14 +19,41 @@ fn mux_err(e: impl std::fmt::Display) -> ExportError {
     ExportError::Io(e.to_string())
 }
 
-/// Tag a video [`SampleEntry`] with the BT.709 limited-range `colr` (nclx) box every video
-/// encoder here outputs (`rgba_to_yuv420`, `rgba_to_yuv_10` and `rgba_to_yuv420_bits` all use
-/// the 0.2126/0.7152/0.0722 luma and a 16–235 luma range). Without it a re-import falls back to
-/// the container's resolution-based `default_matrix` — BT.601 below 1024×577 — and shifts the
+/// What a render's frames are, as ITU-T H.273 code points, for the `colr` box and the encoders'
+/// own colour description. The matrix is BT.709, the one every encoder here converts with
+/// (`rgba_to_yuv420`, `rgba_to_yuv_10` and `rgba_to_yuv420_bits` all use the 0.2126/0.7152/0.0722
+/// luma). Primaries and transfer follow the render path (`Pipe::output`): only a working space
+/// converts to the output space, so unmanaged frames pass through. sRGB output keeps transfer 1
+/// rather than 13 (#340), and an alpha matte is not in the output space.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub(crate) struct Codes {
+    pub(crate) primaries: u8,
+    pub(crate) transfer: u8,
+    pub(crate) matrix: u8,
+}
+
+pub(crate) fn output_codes(settings: &ProjectSettings, channels: Channels) -> Codes {
+    use effectcraft_project::ColorSpace as Space;
+    let output = settings.working_space.and(settings.output_space).filter(|_| channels != Channels::Alpha);
+    let (primaries, transfer) = match output {
+        None | Some(Space::Srgb | Space::Rec709) => (1, 1),
+        Some(Space::Rec2020) => (9, 1),
+        Some(Space::DisplayP3) => (12, 13),
+        Some(Space::Rec2100Pq) => (9, 16),
+        Some(Space::Rec2100Hlg) => (9, 18),
+        // H.273 has no code for the ACES primaries: unspecified, in linear light.
+        Some(Space::AcesCg | Space::Aces2065) => (2, 8),
+    };
+    Codes { primaries, transfer, matrix: 1 }
+}
+
+/// Tag a video [`SampleEntry`] with the limited-range (16–235 luma, as every encoder here
+/// writes) `colr` (nclx) box for `c` (#2879). Without it a re-import falls back to the
+/// container's resolution-based `default_matrix` — BT.601 below 1024×577 — and shifts the
 /// colours (pure red came back as [232,0,1]).
-pub(crate) fn tag_bt709_limited(entry: &mut SampleEntry) {
+pub(crate) fn tag_nclx_limited(entry: &mut SampleEntry, c: Codes) {
     if let Some(v) = entry.video.as_mut() {
-        v.color = Some(ColorInfo::Nclx { primaries: 1, transfer: 1, matrix: 1, full_range: false });
+        v.color = Some(ColorInfo::Nclx { primaries: c.primaries.into(), transfer: c.transfer.into(), matrix: c.matrix.into(), full_range: false });
     }
 }
 
@@ -94,21 +121,23 @@ struct H264 {
     w: u32,
     h: u32,
     rate: FrameRate,
+    codes: Codes,
     y: Vec<u8>,
     u: Vec<u8>,
     v: Vec<u8>,
 }
 
 impl H264 {
-    fn new(w: u32, h: u32, rate: FrameRate, kbps: u32) -> Result<H264> {
+    fn new(w: u32, h: u32, rate: FrameRate, kbps: u32, codes: Codes) -> Result<H264> {
         let mut cfg = filmcraft_h264enc::EncoderConfig::new(w, h, rate.num as u32, rate.den as u32);
         cfg.format = filmcraft_h264enc::PacketFormat::LengthPrefixed;
         cfg.aud = false;
         cfg.keyint = (rate.as_f64() * 2.0).round().max(1.0) as u32;
         let kbps = kbps.max(100);
         cfg.rate = filmcraft_h264enc::RateControl::Vbr { target_kbps: kbps, max_kbps: kbps * 3 / 2 };
+        cfg.color = filmcraft_h264enc::ColorConfig { primaries: codes.primaries, transfer: codes.transfer, matrix: codes.matrix, full_range: false };
         let enc = filmcraft_h264enc::Encoder::new(cfg).map_err(enc)?;
-        Ok(H264 { enc, w, h, rate, y: vec![], u: vec![], v: vec![] })
+        Ok(H264 { enc, w, h, rate, codes, y: vec![], u: vec![], v: vec![] })
     }
     fn packets(&self, ps: Vec<filmcraft_h264enc::Packet>) -> Vec<Packet> {
         ps.into_iter().map(|p| Packet { data: p.data, key: p.keyframe, cto: (p.pts - p.dts) as i32 }).collect()
@@ -122,7 +151,7 @@ impl VideoEncoder for H264 {
             filmcraft_isobmff::AvcConfig::new(vec![sps], vec![pps], 4)
         });
         let mut entry = SampleEntry::avc(cfg, self.w as u16, self.h as u16);
-        tag_bt709_limited(&mut entry);
+        tag_nclx_limited(&mut entry, self.codes);
         entry
     }
     fn encode(&mut self, rgba: &[u8], index: u64) -> Result<Vec<Packet>> {
@@ -167,25 +196,10 @@ struct ProRes {
     colr: ColorInfo,
 }
 
-/// The `colr` atom of a ProRes render: what its frames are, in H.273 code points. MOV uses
-/// QuickTime's `nclc`, which has no range flag; ProRes frames are limited range. The matrix is the
-/// one [`rgba_to_yuv_10`] always uses. Primaries and transfer follow the render path
-/// (`Pipe::output`): only a working space converts to the output space, so unmanaged frames pass
-/// through. sRGB output keeps transfer 1 rather than 13 (#340), and an alpha matte is not in the
-/// output space.
-fn prores_colr(settings: &ProjectSettings, channels: Channels) -> ColorInfo {
-    use effectcraft_project::ColorSpace as Space;
-    let output = settings.working_space.and(settings.output_space).filter(|_| channels != Channels::Alpha);
-    let (primaries, transfer) = match output {
-        None | Some(Space::Srgb | Space::Rec709) => (1, 1),
-        Some(Space::Rec2020) => (9, 1),
-        Some(Space::DisplayP3) => (12, 13),
-        Some(Space::Rec2100Pq) => (9, 16),
-        Some(Space::Rec2100Hlg) => (9, 18),
-        // H.273 has no code for the ACES primaries: unspecified, in linear light.
-        Some(Space::AcesCg | Space::Aces2065) => (2, 8),
-    };
-    ColorInfo::Nclc { primaries, transfer, matrix: 1 }
+/// The `colr` atom of a ProRes render ([`output_codes`]). MOV uses QuickTime's `nclc`, which has
+/// no range flag; ProRes frames are limited range.
+fn prores_colr(c: Codes) -> ColorInfo {
+    ColorInfo::Nclc { primaries: c.primaries.into(), transfer: c.transfer.into(), matrix: c.matrix.into() }
 }
 
 fn prores_profile(p: ProResProfile) -> filmcraft_prores::Profile {
@@ -328,15 +342,13 @@ pub(crate) async fn movie(job: &Cx<'_>, comp: &Comp, w: u32, h: u32, st: &mut St
         Channels::Rgba if !matches!(fmt, OutputFormat::ProRes | OutputFormat::Uncompressed) => Channels::Rgb,
         c => c,
     };
+    let codes = output_codes(&job.project.settings, channels);
     let mut venc: Box<dyn VideoEncoder> = match fmt {
-        OutputFormat::H264 => Box::new(H264::new(w, h, rate, job.output.bitrate_kbps)?),
-        OutputFormat::Hevc => Box::new(crate::hevc_av1::Hevc::new(w, h, rate, job.output)?),
-        OutputFormat::Av1 => Box::new(crate::hevc_av1::Av1::new(w, h, rate, job.output)?),
+        OutputFormat::H264 => Box::new(H264::new(w, h, rate, job.output.bitrate_kbps, codes)?),
+        OutputFormat::Hevc => Box::new(crate::hevc_av1::Hevc::new(w, h, rate, job.output, codes)?),
+        OutputFormat::Av1 => Box::new(crate::hevc_av1::Av1::new(w, h, rate, job.output, codes)?),
         OutputFormat::Uncompressed => Box::new(crate::uncompressed::Raw::new(w, h, channels == Channels::Rgba)?),
-        _ => {
-            let colr = prores_colr(&job.project.settings, channels);
-            Box::new(ProRes::new(w, h, job.output.prores_profile, if channels == Channels::Alpha { Channels::Rgb } else { channels }, colr))
-        }
+        _ => Box::new(ProRes::new(w, h, job.output.prores_profile, if channels == Channels::Alpha { Channels::Rgb } else { channels }, prores_colr(codes))),
     };
     let brand = if matches!(fmt, OutputFormat::ProRes | OutputFormat::Uncompressed) { Brand::Mov } else { Brand::Mp4 };
     let batch = batch_size();

crates/export/src/hevc_av1.rs

@@ -7,7 +7,7 @@ use effectcraft_time::FrameRate;
 use filmcraft_isobmff::{Av1Config, CodecConfig, FourCc, HevcConfig, SampleEntry};
 use rayon::prelude::*;
 
-use crate::encode::{Packet, VideoEncoder};
+use crate::encode::{Codes, Packet, VideoEncoder};
 use crate::{ExportError, Result};
 
 fn enc_err(e: impl std::fmt::Display) -> ExportError {
@@ -59,6 +59,7 @@ pub(crate) struct Hevc {
     w: u32,
     h: u32,
     bits: u8,
+    codes: Codes,
     planes: Planes,
 }
 
@@ -81,11 +82,12 @@ pub(crate) fn hevc_config(w: u32, h: u32, rate: FrameRate, om: &OutputModule) ->
 }
 
 impl Hevc {
-    pub(crate) fn new(w: u32, h: u32, rate: FrameRate, om: &OutputModule) -> Result<Hevc> {
-        let cfg = hevc_config(w, h, rate, om);
+    pub(crate) fn new(w: u32, h: u32, rate: FrameRate, om: &OutputModule, codes: Codes) -> Result<Hevc> {
+        let mut cfg = hevc_config(w, h, rate, om);
+        (cfg.colour_primaries, cfg.transfer_characteristics, cfg.matrix_coeffs) = (codes.primaries, codes.transfer, codes.matrix);
         let bits = cfg.profile.bit_depth();
         let enc = effectcraft_hevcenc::Encoder::new(cfg).map_err(enc_err)?;
-        Ok(Hevc { enc, w, h, bits, planes: Planes::default() })
+        Ok(Hevc { enc, w, h, bits, codes, planes: Planes::default() })
     }
 }
 
@@ -93,7 +95,7 @@ impl VideoEncoder for Hevc {
     fn sample_entry(&self) -> SampleEntry {
         let cfg = HevcConfig::parse(&self.enc.hvcc()).unwrap_or_default();
         let mut entry = SampleEntry::hevc(cfg, self.w as u16, self.h as u16);
-        crate::encode::tag_bt709_limited(&mut entry);
+        crate::encode::tag_nclx_limited(&mut entry, self.codes);
         entry
     }
     fn encode(&mut self, rgba: &[u8], _index: u64) -> Result<Vec<Packet>> {
@@ -116,6 +118,7 @@ pub(crate) struct Av1 {
     w: u32,
     h: u32,
     bits: u8,
+    codes: Codes,
     planes: Planes,
 }
 
@@ -135,11 +138,12 @@ pub(crate) fn av1_config(w: u32, h: u32, rate: FrameRate, om: &OutputModule) ->
 }
 
 impl Av1 {
-    pub(crate) fn new(w: u32, h: u32, rate: FrameRate, om: &OutputModule) -> Result<Av1> {
-        let cfg = av1_config(w, h, rate, om);
+    pub(crate) fn new(w: u32, h: u32, rate: FrameRate, om: &OutputModule, codes: Codes) -> Result<Av1> {
+        let mut cfg = av1_config(w, h, rate, om);
+        (cfg.color_primaries, cfg.transfer_characteristics, cfg.matrix_coefficients) = (codes.primaries, codes.transfer, codes.matrix);
         let bits = cfg.bit_depth;
         let enc = effectcraft_av1enc::Encoder::new(cfg).map_err(enc_err)?;
-        Ok(Av1 { enc, w, h, bits, planes: Planes::default() })
+        Ok(Av1 { enc, w, h, bits, codes, planes: Planes::default() })
     }
     /// The `av1C` record (also the WebM CodecPrivate).
     pub(crate) fn av1c(&self) -> Vec<u8> {
@@ -151,7 +155,7 @@ impl VideoEncoder for Av1 {
     fn sample_entry(&self) -> SampleEntry {
         let cfg = Av1Config::parse(&self.enc.av1c()).unwrap_or_default();
         let mut entry = SampleEntry::video(FourCc(*b"av01"), CodecConfig::Av1(cfg), self.w as u16, self.h as u16);
-        crate::encode::tag_bt709_limited(&mut entry);
+        crate::encode::tag_nclx_limited(&mut entry, self.codes);
         entry
     }
     fn encode(&mut self, rgba: &[u8], _index: u64) -> Result<Vec<Packet>> {

crates/export/src/webm_av1.rs

@@ -42,7 +42,8 @@ fn cluster(blocks: &mut [Block]) -> Vec<u8> {
 
 pub(crate) async fn webm_av1(job: &Cx<'_>, comp: &Comp, w: u32, h: u32, st: &mut State<'_>) -> Result<Report> {
     let rate = job.settings.rate(comp);
-    let mut venc = crate::hevc_av1::Av1::new(w, h, rate, job.output)?;
+    let codes = crate::encode::output_codes(&job.project.settings, job.output.channels);
+    let mut venc = crate::hevc_av1::Av1::new(w, h, rate, job.output, codes)?;
     let with_audio = wants_audio(job);
     let opus_channels = if job.output.audio_channels == 1 { 1usize } else { 2 };
     let mut opus = with_audio.then(|| opus_encoder(job));

crates/export/tests/it/main.rs

@@ -7,6 +7,7 @@ mod hevc_av1_roundtrip;
 #[cfg(not(target_arch = "wasm32"))]
 mod mov_audio_sample_positions;
 mod movie_bytes;
+mod movie_color;
 mod ms_rounded_footage;
 #[cfg(not(target_arch = "wasm32"))]
 mod native_audio_sample_positions;

crates/export/tests/it/movie_color.rs

@@ -0,0 +1,98 @@
+//! H.264, HEVC and AV1 MP4s describe the frames the render path made (#2879): the `colr` box, read
+//! back with FilmCraft's demuxer, and the bitstream's own colour description (the SPS VUI, AV1's
+//! `color_config`), which ffprobe reports for the first decoded frame when it can decode it.
+
+use std::path::Path;
+use std::process::Command;
+
+use effectcraft_color::Label;
+use effectcraft_export::render_queue::{OutputFormat, OutputModule, RenderSettings, TimeSpan};
+use effectcraft_export::{Job, export};
+use effectcraft_project::{ColorSpace, Comp, ItemKind, LayerSource, Project, Solid, build};
+use effectcraft_render::NoFootage;
+use effectcraft_time::{FrameRate, Tick};
+use filmcraft_isobmff::{ColorInfo, TrackKind};
+
+/// (name, Output color space, H.273 primaries / transfer / matrix, the same as ffprobe names them)
+const ROWS: &[(&str, Option<ColorSpace>, [u16; 3], [&str; 3])] = &[
+    ("default", None, [1, 1, 1], ["bt709", "bt709", "bt709"]),
+    ("rec2020", Some(ColorSpace::Rec2020), [9, 1, 1], ["bt2020", "bt709", "bt709"]),
+    ("p3", Some(ColorSpace::DisplayP3), [12, 13, 1], ["smpte432", "iec61966-2-1", "bt709"]),
+    ("pq", Some(ColorSpace::Rec2100Pq), [9, 16, 1], ["bt2020", "smpte2084", "bt709"]),
+    ("hlg", Some(ColorSpace::Rec2100Hlg), [9, 18, 1], ["bt2020", "arib-std-b67", "bt709"]),
+];
+
+fn render(output: Option<ColorSpace>, format: OutputFormat, path: &Path) {
+    let mut p = Project::default();
+    p.settings.working_space = Some(ColorSpace::Srgb);
+    p.settings.output_space = output;
+    let mut comp = Comp::new(64, 48, FrameRate::new(10, 1), Tick::from_seconds_f64(0.2));
+    let sid = p.add_item("Solid", Label::Red, None, ItemKind::Solid(Solid { color: [0.75, 0.25, 0.5], width: 64, height: 48, pixel_aspect: 1.0 }));
+    comp.layers.push(build::layer(&mut p, &comp, "Solid", LayerSource::Solid { item: sid }, (64, 48), None));
+    let comp = p.add_item("Comp", Label::Sandstone, None, ItemKind::Comp(comp.into()));
+    let settings = RenderSettings { time_span: TimeSpan::LengthOfComp, ..Default::default() };
+    let om = OutputModule::for_format(format);
+    let out = path.to_string_lossy().to_string();
+    let job = Job {
+        project: &p,
+        footage: &NoFootage,
+        expr: None,
+        accel: None,
+        comp,
+        settings: &settings,
+        output: &om,
+        path: &out,
+        sink: None,
+        options: Default::default(),
+        nested_switches: true,
+    };
+    export(&job, &mut |_| true).expect("export");
+}
+
+/// ffprobe's primaries, transfer and matrix of the first decoded frame, or `None` without ffprobe
+/// or when its build can't decode the stream.
+fn frame_tags(path: &Path) -> Option<Vec<String>> {
+    let out = Command::new("ffprobe")
+        .args([
+            "-v",
+            "error",
+            "-select_streams",
+            "v:0",
+            "-read_intervals",
+            "%+#1",
+            "-show_entries",
+            "frame=color_primaries,color_transfer,color_space",
+            "-of",
+            "default=nw=1",
+        ])
+        .arg(path)
+        .output()
+        .ok()?;
+    let text = String::from_utf8_lossy(&out.stdout).to_string();
+    let tag = |key: &str| text.lines().find_map(|l| l.strip_prefix(key)?.strip_prefix('=')).map(str::to_string);
+    Some(vec![tag("color_primaries")?, tag("color_transfer")?, tag("color_space")?])
+}
+
+#[test]
+fn mp4_tags_describe_the_frames() {
+    let dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../target/test-out/export/movie_color");
+    let _ = std::fs::remove_dir_all(&dir);
+    std::fs::create_dir_all(&dir).expect("mkdir");
+    for (format, codec) in [(OutputFormat::H264, "h264"), (OutputFormat::Hevc, "hevc"), (OutputFormat::Av1, "av1")] {
+        for &(name, output, [primaries, transfer, matrix], names) in ROWS {
+            let path = dir.join(format!("{codec}_{name}.mp4"));
+            render(output, format, &path);
+
+            let bytes = std::fs::read(&path).expect("read movie");
+            let mp4 = filmcraft_isobmff::open(bytes.as_slice()).expect("demux");
+            let track = mp4.tracks.iter().find(|t| t.kind == TrackKind::Video).expect("video track");
+            let colr = track.entries[0].video.as_ref().and_then(|v| v.color.clone());
+            assert_eq!(colr, Some(ColorInfo::Nclx { primaries, transfer, matrix, full_range: false }), "{codec} {name}: colr");
+
+            match frame_tags(&path) {
+                Some(tags) => assert_eq!(tags, names, "{codec} {name}: the bitstream's colour description"),
+                None => eprintln!("{codec} {name}: ffprobe can't decode it here, bitstream not checked"),
+            }
+        }
+    }
+}

crates/hevcenc/src/lib.rs

@@ -83,6 +83,10 @@ pub struct EncoderConfig {
     pub keyint: u32,
     /// BT.709 limited range is signalled in the VUI unless this is set.
     pub full_range: bool,
+    /// The VUI colour description (ITU-T H.273 code points); BT.709 by default.
+    pub colour_primaries: u8,
+    pub transfer_characteristics: u8,
+    pub matrix_coeffs: u8,
 }
 
 impl EncoderConfig {
@@ -97,6 +101,9 @@ impl EncoderConfig {
             rate: RateControl::ConstantQp(28),
             keyint: 60,
             full_range: false,
+            colour_primaries: 1,
+            transfer_characteristics: 1,
+            matrix_coeffs: 1,
         }
     }
 }
@@ -234,6 +241,9 @@ impl Encoder {
             profile_idc: cfg.profile.idc(),
             level_idc,
             full_range: cfg.full_range,
+            colour_primaries: cfg.colour_primaries,
+            transfer_characteristics: cfg.transfer_characteristics,
+            matrix_coeffs: cfg.matrix_coeffs,
             fps_num: cfg.fps_num,
             fps_den: cfg.fps_den,
         };

crates/hevcenc/src/params.rs

@@ -25,6 +25,9 @@ pub struct SeqParams {
     pub profile_idc: u8,
     pub level_idc: u8,
     pub full_range: bool,
+    pub colour_primaries: u8,
+    pub transfer_characteristics: u8,
+    pub matrix_coeffs: u8,
     pub fps_num: u32,
     pub fps_den: u32,
 }
@@ -166,9 +169,9 @@ fn vui(w: &mut BitWriter, p: &SeqParams) {
     w.put(5, 3); // video_format: unspecified
     w.flag(p.full_range);
     w.put(1, 1); // colour_description_present_flag
-    w.put(1, 8); // colour_primaries BT.709
-    w.put(1, 8); // transfer_characteristics BT.709
-    w.put(1, 8); // matrix_coeffs BT.709
+    w.put(p.colour_primaries as u32, 8); // colour_primaries
+    w.put(p.transfer_characteristics as u32, 8); // transfer_characteristics
+    w.put(p.matrix_coeffs as u32, 8); // matrix_coeffs
     w.put(0, 1); // chroma_loc_info_present_flag
     w.put(0, 1); // neutral_chroma_indication_flag
     w.put(0, 1); // field_seq_flag