Merge pull request #3134 from vangel76/aep-tdsb3-values

.aep import: read effect values stored as default-state records

crates/aep/src/container/effects.rs

@@ -375,11 +375,25 @@ impl<'f, 'a> EffectParam<'f, 'a> {
     /// colours, popups and layer controls. A record counts when its `tdsb` is 1 and it has no
     /// keyframes; a layer control reads the `tdpi` chunk; a button has no value.
     pub fn stored_value(&self) -> Option<ParamValue> {
+        let p = self.stored?;
+        if self.kind != ParamKind::Layer && p.flags() != Some(1) {
+            return None;
+        }
+        self.recorded_value()
+    }
+
+    /// The value in the value record when the record is written (`tdsb` 1) or in the default
+    /// state (`tdsb` 3): what [`EffectParam::stored_value`] reads, also from a default-state
+    /// record. Lumetri Color writes the values of its hidden parameters so (High Dynamic Range,
+    /// HDR Specular, Look: 1200 `lumetri-01-set-all`, `-02-sec-top`, `-06-sec-light`; After
+    /// Effects reads back the recorded values). `None` with keyframes, without a record or for a
+    /// record in any other state (no sample has one; the importer reports it as damaged).
+    pub fn recorded_value(&self) -> Option<ParamValue> {
         let p = self.stored?;
         if self.kind == ParamKind::Layer {
             return be_u32(p.chunk().child(b"tdpi")?.bytes()?, 0).map(ParamValue::Layer);
         }
-        if p.flags() != Some(1) || p.is_animated() {
+        if !matches!(p.flags(), Some(1 | 3)) || p.is_animated() {
             return None;
         }
         let v = p.static_value()?;
@@ -443,8 +457,9 @@ impl<'f, 'a> EffectParam<'f, 'a> {
 
     /// True when the parameter is at its default: it has no value record (a record exists exactly
     /// when the value differs from the `pard` default; setting a value back to the default removes
-    /// the record again), or the record equals the default. `None` when that cannot be decided
-    /// (unknown parameter type).
+    /// the record again), or the record equals the default. A record in the default state (`tdsb`
+    /// 3) counts with its value ([`EffectParam::recorded_value`]). `None` when that cannot be
+    /// decided (unknown parameter type, keyframes).
     pub fn is_default(&self) -> Option<bool> {
         match (self.stored, self.kind) {
             // A Layer Control always has its record, and a popup selection overwrites the default
@@ -453,7 +468,7 @@ impl<'f, 'a> EffectParam<'f, 'a> {
             (None, _) => Some(true),
             (Some(_), ParamKind::Popup) => Some(false),
             (Some(_), _) => {
-                let stored = self.stored_value()?;
+                let stored = self.recorded_value()?;
                 // A checkbox's default is in `pard` too (on for Repeat Edge Pixels, 412).
                 Some(self.pard_default().is_some_and(|d| d == stored))
             }
@@ -541,10 +556,11 @@ impl<'f, 'a> EffectParam<'f, 'a> {
         self.items.as_ref()?.get(index).map(String::as_str)
     }
 
-    /// The effective value: the stored value when there is one, else the `pard` default (absent =
+    /// The effective value: the value in the record when there is one, also in a record in the
+    /// default state (`tdsb` 3, [`EffectParam::recorded_value`]), else the `pard` default (absent =
     /// default); a checkbox without a record is off.
     pub fn value(&self) -> Option<ParamValue> {
-        self.stored_value().or_else(|| self.pard_default()).or(match self.kind {
+        self.recorded_value().or_else(|| self.pard_default()).or(match self.kind {
             ParamKind::Checkbox if self.stored.is_none() => Some(ParamValue::Bool(false)),
             _ => None,
         })

crates/aep/src/import/effects.rs

@@ -133,6 +133,8 @@ struct ParamOut<'a> {
     col: &'a mut Collector,
 }
 
+/// Imports one parameter of `fx` into its EffectCraft property: its keyframes, the value in its
+/// record or the `pard` default, with a loss for what cannot be read or mapped.
 fn import_param(fx: &AepEffect<'_, '_>, map: &EffectMap, p: &EffectParam<'_, '_>, cx: &LayerCtx<'_>, shown: &str, out: ParamOut<'_>) {
     let ParamOut { group, refs, col } = out;
     if carries_nothing(p) {
@@ -269,7 +271,9 @@ fn import_param(fx: &AepEffect<'_, '_>, map: &EffectMap, p: &EffectParam<'_, '_>
         }
         return;
     }
-    let value = match (p.stored(), p.stored_value()) {
+    // A record in the default state (`tdsb` 3) still holds the value After Effects uses (1200,
+    // 1400: every such record reads back as recorded).
+    let value = match (p.stored(), p.recorded_value()) {
         (Some(_), Some(v)) => Some(v),
         (Some(_), None) => {
             col.lose(LossCategory::DamagedData, format!("{here}: parameter {label} holds an unreadable value, default used"));

crates/aep/tests/it/import/synthetic.rs

@@ -295,3 +295,111 @@ fn unknown_time_display_and_frame_start_codes_are_reported() {
         assert!(losses(&r.unwrap(), LossCategory::ProjectSettings).is_empty(), "nhed {at}: {bytes:?}");
     }
 }
+
+/// The project with the value records named `param` (`<effect>-NNNN`: the instance's and the one in
+/// the project's copy of the effect, `EfdG`) set to state `tdsb` and holding `value` (the `cdat`
+/// starts with the big-endian `f64`).
+fn with_record(param: &str, tdsb: u32, value: f64) -> Vec<u8> {
+    let mut file: AepFile<'_> = parse(SYNTHETIC).unwrap();
+    let mut patched = 0;
+    visit(&mut file.chunks, &mut |c| {
+        if !c.is_list(b"tdgp") {
+            return;
+        }
+        let kids = c.children_mut();
+        let Some(at) = kids.iter().position(|k| k.is(b"tdmn") && k.bytes().is_some_and(|b| b.split(|x| *x == 0).next() == Some(param.as_bytes()))) else {
+            return;
+        };
+        let Some(record) = kids.get_mut(at + 1).filter(|k| k.is_list(b"tdbs")) else { return };
+        for k in record.children_mut() {
+            if k.is(b"tdsb") {
+                set_data(k, 0, &tdsb.to_be_bytes());
+            } else if k.is(b"cdat") {
+                set_data(k, 0, &value.to_be_bytes());
+            }
+        }
+        patched += 1;
+    });
+    assert_eq!(patched, 2, "{param}: the instance and the EfdG copy");
+    write(&file).unwrap()
+}
+
+/// The values of the imported effect parameters, by property id, in file order.
+fn effect_values(p: &Project) -> Vec<(String, effectcraft_keyframe::Value)> {
+    let mut out = Vec::new();
+    for item in p.items.values() {
+        if let ItemKind::Comp(c) = &item.kind {
+            for layer in &c.layers {
+                if let Some(fx) = layer.effects() {
+                    fx.walk("", &mut |_, prop| out.push((prop.match_id.clone(), prop.value.clone())));
+                }
+            }
+        }
+    }
+    out
+}
+
+/// A default-state record (`tdsb` 3) reads like a written one, except through `stored_value`.
+#[test]
+fn a_default_state_effect_record_holds_the_value() {
+    // After Effects writes the values of some parameters as records in the default state (`tdsb`
+    // 3): Lumetri Color's hidden controls (1200) and Cryptomatte's Selection Mode (1400), each
+    // read back as recorded. The reader reads them like a written record (`tdsb` 1).
+    let bytes = with_record("ADBE Glo2-0011", 3, 0.75);
+    let file = parse(&bytes).unwrap();
+    let project = effectcraft_aep::Project::new(&file).unwrap();
+    let defs = project.effect_definition_index();
+    let mut seen = 0;
+    for item in project.all_items() {
+        let Some(comp) = item.comp() else { continue };
+        for layer in comp.layers() {
+            for fx in layer.effects().iter().filter(|fx| fx.match_name() == "ADBE Glo2") {
+                let params = defs.params(fx);
+                let p = params.iter().find(|p| p.match_name == "ADBE Glo2-0011").unwrap();
+                assert_eq!(p.stored().and_then(|s| s.flags()), Some(3));
+                // `stored_value` keeps reading written records only.
+                assert_eq!(p.stored_value(), None);
+                assert_eq!(p.recorded_value(), Some(effectcraft_aep::ParamValue::Number(0.75)));
+                assert_eq!(p.value(), Some(effectcraft_aep::ParamValue::Number(0.75)));
+                assert_eq!(p.is_default(), Some(false));
+                seen += 1;
+            }
+        }
+    }
+    assert_eq!(seen, 1);
+}
+
+/// Default-state records import with their values and no loss, like written records.
+#[test]
+fn default_state_effect_records_import_like_written_ones() {
+    // Before, such a record was reported as an unreadable value and the default used.
+    for (param, value) in [("ADBE Glo2-0011", 0.75), ("ADBE Easy Levels2-0008", 1.0)] {
+        let (written, r1) = run(&with_record(param, 1, value), true);
+        let (default_state, r3) = run(&with_record(param, 3, value), true);
+        let (r1, r3) = (r1.unwrap(), r3.unwrap());
+        assert!(losses(&r3, LossCategory::DamagedData).is_empty(), "{param}: {:?}", r3.report.losses);
+        assert_eq!(r3.report.losses, r1.report.losses, "{param}");
+        assert_eq!(effect_values(&default_state), effect_values(&written), "{param}");
+    }
+    // Glow's A & B Midpoint 0.75 is 75 % (the default is 50).
+    let (p, _) = run(&with_record("ADBE Glo2-0011", 3, 0.75), true);
+    let mid: Vec<_> = effect_values(&p).into_iter().filter(|(id, _)| id == "abMidpoint").collect();
+    assert_eq!(mid, vec![("abMidpoint".to_string(), effectcraft_keyframe::Value::Scalar(75.0))]);
+    // Levels' Clip To Output Black (`-0008`) recorded as After Effects' item 1, On: EffectCraft's
+    // On is item 1 of Off, On, Off for 32 bpc Color (the default is item 2).
+    let (p, _) = run(&with_record("ADBE Easy Levels2-0008", 3, 1.0), true);
+    let clip: Vec<_> = effect_values(&p).into_iter().filter(|(id, _)| id == "clipToOutputBlack").collect();
+    assert_eq!(clip, vec![("clipToOutputBlack".to_string(), effectcraft_keyframe::Value::Enum(1))]);
+}
+
+/// A record in a state other than 1 or 3 is not read and is reported.
+#[test]
+fn an_effect_record_in_another_state_is_reported_as_damaged() {
+    // Only the written (1) and the default (3) state hold a value; any other state is not read.
+    for tdsb in [0, 2, 5] {
+        let (_, r) = run(&with_record("ADBE Glo2-0011", tdsb, 0.75), true);
+        let r = r.unwrap();
+        let damaged = losses(&r, LossCategory::DamagedData);
+        assert!(damaged.iter().any(|l| l.contains("unreadable value")), "tdsb {tdsb}: {:?}", r.report.losses);
+    }
+}

docs/aep-format.md

@@ -328,7 +328,7 @@ Property record (`LIST tdbs`: `tdsb`, `tdsn`, `tdb4`, then `cdat` or a `LIST lis
 | keyed spatial-template flag | `tdb4` | 79 | u8 | 1 on a static colour or point effect value, 0 once it is keyed (359 / 364, 369 / 382); 0 on keyed and static sliders (362 / 363, 391 / 397); meaning **?** | 359, 362-364, 369, 382, 391, 397 | V (values) |
 | expression flags | `tdb4` | 76, 77, 120 | u8 | all three are 1 when an expression exists. 76 and 77 are also 1 on a camera's or light's computed (auto) Position, which has no expression | 060, 213, 251 | V |
 | **expression disabled** | `tdb4` | 119 | u8 | 1 = the expression is disabled; the text and bytes 76, 77, 120 stay. On the built-in view layers the byte flips by itself after the first save, so it counts only together with an expression | 060 (0), 213 (1) | V |
-| state | `tdsb` | 0 | u32 | 1 = written (non-default value, keys or expression), 3 = default state | 040, 071, 073 (1); 3 on the unwritten properties of 000 | V (1), E (3) |
+| state | `tdsb` | 0 | u32 | 1 = written (non-default value, keys or expression), 3 = default state. **A layer, camera, light, shape or text property in the default state holds After Effects' default** or a value with no effect on that layer (survey of 839 corpus and sample files: Material Options, cone 90 / 50 and falloff 500 / 500 of point lights, Position `[0, 0, -1333.33]` of ambient and environment lights, Opacity 1 and Scale `[1, 1, 1]` of cameras and lights, a polygon's Inner Radius 50, Miter Limit 4, the inert Time Remap 0), so the importer skips it. **An effect parameter's record in the default state holds the effective value** (3.7, stored value) | 040, 071, 073 (1); 3 on the unwritten properties of 000; 1200, 1400 (effects) | V (1), E (3), V (effects) |
 | static value | `cdat` | 0 | `components` x f64 | the first `components` doubles are the value | 040, 041, 043, 071 | V |
 | user name | `tdsn/Utf8` | | text | `-_0_/-` means "no custom name" | 091 (`Mask 1`), 238 (`Group 1`) | V |
 
@@ -437,9 +437,9 @@ The expression engine is project-wide (`LIST ExEn/Utf8`, `javascript-1.0`). API:
 | **float slider valid minimum, slider range** | `pard` | 104; 112 / 116 | f32; f32 x 2 | the smallest valid value (Exposure -100, Offset -2, Gamma Correction 0.1 as f32) and the range the slider covers (Exposure -4..4, Offset -0.5..0.5, Gamma 0.1..10); the slider range is repeated as `tdum` / `tduM` (f64) in each value record (391-394). @120 holds the default again as f32 (Gamma 1.0, Blurriness 25), u16 @124 the display decimals (**H**: Exposure 2, Offset 4) | 389, 391-394 | V |
 | **definitions of a second instance** | top-level `LIST EfdG` | | | one `EfDf` per effect TYPE (`tdmn` = match name, then a copy of ONE instance's `sspc`, including that instance's value records); the `EfDC` count before them is a little-endian u32. When two layers carry the same effect type, only one instance keeps a full `parT`; **the other has an empty `LIST parT` (4 bytes)** but its own `tdgp` (input layer `-0000` with its own `tdpi`, the button record, and its value records). Its definitions come from the `EfdG` copy by match name, its values from its own records, never from the copy (383: the copy holds S100's Start [0.1, 0.2]; L's Ramp reads the 368 defaults). Which instance keeps the full `parT` is **?** (in 383 S100 is both first in the file and newest). The `EfDf` order follows the stack order (365 / 366); `Pefl/pjef` does not | 365, 366, 383 | V |
 | popup items | `parT/pdnm/Utf8` | | text | items separated by `\|` | 077 (`Item 1\|Item 2\|Item 3`), 214 | V |
-| stored value | `sspc/tdgp/<match>-NNNN` | | `tdbs` | present only for a parameter that differs from its default; `tdsb` 1, `cdat` as in 3.4. **Absent record = the `pard` default** | see below | V |
+| stored value | `sspc/tdgp/<match>-NNNN` | | `tdbs` | present only for a parameter that differs from its default; `tdsb` 1, `cdat` as in 3.4. **Absent record = the `pard` default.** Some parameters are recorded in the default state (`tdsb` 3) and the record still holds the value After Effects uses: every one of the 25 such records in the corpus and the samples differs from its `pard` default and reads back as recorded (Lumetri Color's hidden controls High Dynamic Range, HDR Specular, the second Look, Legacy Tone And Saturation, Face Detection, Auto Color, Auto Intensity and SemanticAutoTone Progress, 1200; Cryptomatte's Selection Mode on, 1400). No effect record in the default state holds a value that does not count. The reader takes the value in either state (`EffectParam::recorded_value`, `EffectParam::value`); `EffectParam::stored_value` keeps to written records, and a record in any other state (no sample has one) is not read: the importer reports it as damaged data. That other effects record values so is H (no sample shows one) | see below; 1200, 1400 | V (1200, 1400), H (other effects) |
 
-Stored values by type (all `tdsb` 1):
+Stored values by type (`tdsb` 1, or 3 as above):
 
 | Type | Stored | Evidence | Conf. |
 |---|---|---|---|
@@ -823,7 +823,7 @@ another value).
 **Masks, shapes**
 
 - Masks: locked, colour, feather falloff, rotobezier, keyed expansion, a path whose last vertex has only handles, the counter after a mask is deleted, a mask on a scaled layer (whether the box stays layer-relative), an animated path with three or more keys.
-- Shape layers: open shape paths (in shape groups), stroke dashes, taper and wave with values, merge paths, repeater, offset paths and the other path operators, item blend modes and composite order, shape direction, gradient colour stops other than the default (**?**: they appear only as an `ADBE Vector Grad Colors` entry), highlight length and angle other than 0, gradient scale and rotation; why a radial gradient writes its highlight records with `tdsb` 1 and a polygon its inner radius with `tdsb` 3 (**?**).
+- Shape layers: open shape paths (in shape groups), stroke dashes, taper and wave with values, merge paths, repeater, offset paths and the other path operators, item blend modes and composite order, shape direction, gradient colour stops other than the default (**?**: they appear only as an `ADBE Vector Grad Colors` entry), highlight length and angle other than 0, gradient scale and rotation; why a radial gradient writes its highlight records with `tdsb` 1 and a polygon its inner radius with `tdsb` 3 (**?**; the polygon's records hold the defaults 50 and 0, as every non-effect record in the default state does: 3.4, state).
 
 **Text**