craft-psd: bound the builder's default composite (#985)

Without a composite, PsdBuilder::build made a white placeholder of width x height x depth x
channels bytes with vec!, before checking the geometry: a 300000 x 300000 canvas asked for
270 GB, and an overflowing size wrapped. The header is validated first, the size is checked
against MAX_DECODED_BYTES (the encoder's own limit), and the buffer is reserved fallibly.

Ported from storytold/photocraft#13309.

crates/psd/src/builder.rs

@@ -420,7 +420,7 @@ impl PsdBuilder {
         }
 
         let (w, h) = (self.width as usize, self.height as usize);
-        let n = w * h;
+        let n = w.checked_mul(h).ok_or(PsdError::LimitExceeded("image dimensions"))?;
         let (real, planes, with_alpha) = match &self.composite {
             Some(p) => {
                 self.check_pixels(p, n)?;
@@ -432,9 +432,19 @@ impl PsdBuilder {
                 (true, planes, with_alpha)
             }
             None => {
+                // Geometry first, so unsupported dimensions get the validation error, not an allocation.
+                Header::new(self.version, self.width, self.height, cc as u16, self.depth, self.color_mode).validate()?;
                 let bps = usize::from(self.depth / 8);
+                let len = n
+                    .checked_mul(bps)
+                    .and_then(|v| v.checked_mul(cc))
+                    .filter(|&v| v as u64 <= crate::compression::MAX_DECODED_BYTES)
+                    .ok_or(PsdError::LimitExceeded("default composite exceeds MAX_DECODED_BYTES"))?;
+                let mut plane = Vec::new();
+                plane.try_reserve_exact(len).map_err(|_| PsdError::LimitExceeded("default composite allocation"))?;
                 // White: 0xff in RGB/gray; no ink (stored inverted as 0xff) in CMYK.
-                (false, vec![0xff; n * bps * cc], false)
+                plane.resize(len, 0xff);
+                (false, plane, false)
             }
         };
         let channels = (cc + usize::from(with_alpha)) as u16;

crates/psd/tests/it/builder.rs

@@ -312,3 +312,20 @@ fn iter_layers_indices() {
     assert!(f.layer(3).is_none());
     assert_eq!(f.layer(0).unwrap().version(), Version::Psd);
 }
+
+#[test]
+fn a_huge_default_composite_is_an_error_not_an_allocation() {
+    // 300000 x 300000 x 3 bytes is 270 GB: refused before any buffer is made.
+    assert!(PsdBuilder::new(300_000, 300_000).build().is_err());
+    // Unsupported dimensions get the validation error.
+    assert!(PsdBuilder::new(u32::MAX, u32::MAX).build().is_err());
+    assert!(PsdBuilder::new(0, 5).build().is_err());
+}
+
+#[test]
+fn the_default_composite_follows_the_decoded_size_limit() {
+    // 40000 x 40000 CMYK at 16-bit is 12.8 GB: over the encoder's limit, refused before the buffer.
+    let e = PsdBuilder::new(40_000, 40_000).color_mode(ColorMode::Cmyk).depth(16).build().unwrap_err();
+    // The early refusal, not the encoder's later one.
+    assert!(matches!(e, PsdError::LimitExceeded("default composite exceeds MAX_DECODED_BYTES")), "{e:?}");
+}