calc: cap array broadcast result size (#2182)

zip2 (binary operators, TEXT), zip3 (array IF) and MAP now compute the
broadcast cell count with checked multiplication and return #NUM! when it
exceeds MAX_CELLS, instead of allocating (abort) or wrapping on wasm32.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

crates/calc/src/eval.rs

@@ -1012,10 +1012,8 @@ impl<'h> Evaluator<'h> {
                 let arrays: Vec<Arc<Array>> = args[..args.len() - 1].iter().map(|a| as_array(self.value(a))).collect();
                 let rows = arrays.iter().map(|a| a.rows).max().unwrap_or(1);
                 let cols = arrays.iter().map(|a| a.cols).max().unwrap_or(1);
-                if (rows * cols) as u64 > MAX_CELLS {
+                let Some(n) = broadcast_len(rows, cols) else { return err(CellError::Num) };
-                    return err(CellError::Num);
+                let mut out = Vec::with_capacity(n);
-                }
-                let mut out = Vec::with_capacity(rows * cols);
                 for r in 0..rows {
                     for c in 0..cols {
                         let bound = l.params.iter().zip(arrays.iter()).map(|(p, a)| (p.clone(), Ev::V(a.get_broadcast(r, c)))).collect();
@@ -1615,7 +1613,8 @@ pub fn zip2(a: &Value, b: &Value, f: impl Fn(&Value, &Value) -> Value) -> Value
         (Value::Array(x), Value::Array(y)) => {
             let rows = broadcast_dim(x.rows, y.rows);
             let cols = broadcast_dim(x.cols, y.cols);
-            let mut data = Vec::with_capacity(rows * cols);
+            let Some(n) = broadcast_len(rows, cols) else { return Value::Error(CellError::Num) };
+            let mut data = Vec::with_capacity(n);
             for r in 0..rows {
                 for c in 0..cols {
                     data.push(f(&x.get_broadcast(r, c), &y.get_broadcast(r, c)));
@@ -1629,6 +1628,13 @@ pub fn zip2(a: &Value, b: &Value, f: impl Fn(&Value, &Value) -> Value) -> Value
     }
 }
 
+/// Cell count of a `rows`×`cols` broadcast result, or `None` when it would exceed [`MAX_CELLS`].
+/// Each operand passes its own size cap, but their broadcast product can be far larger
+/// (`SEQUENCE(1048576)*SEQUENCE(1,16384)` is 17 billion cells), so it is checked before allocating.
+fn broadcast_len(rows: usize, cols: usize) -> Option<usize> {
+    rows.checked_mul(cols).filter(|&n| n as u64 <= MAX_CELLS)
+}
+
 fn broadcast_dim(a: usize, b: usize) -> usize {
     if a == 1 {
         b
@@ -1645,8 +1651,9 @@ fn zip3(c: &Array, t: &Value, f: &Value, g: impl Fn(&Value, &Value, &Value) -> V
     let (fr, fc) = dims(f);
     let rows = broadcast_dim(broadcast_dim(c.rows, tr), fr);
     let cols = broadcast_dim(broadcast_dim(c.cols, tc), fc);
+    let Some(n) = broadcast_len(rows, cols) else { return Value::Error(CellError::Num) };
     let get = |v: &Value, r: usize, col: usize| if let Value::Array(a) = v { a.get_broadcast(r, col) } else { v.clone() };
-    let mut data = Vec::with_capacity(rows * cols);
+    let mut data = Vec::with_capacity(n);
     for r in 0..rows {
         for col in 0..cols {
             data.push(g(&c.get_broadcast(r, col), &get(t, r, col), &get(f, r, col)));

crates/calc/src/tests.rs

@@ -684,6 +684,34 @@ fn legacy_array_formulas_fill_their_range() {
     assert_eq!(t.get("H1"), Value::Error(CellError::Num));
 }
 
+#[test]
+fn array_broadcast_result_is_capped() {
+    // A full column against a full row broadcasts to 1048576 x 16384 (17 billion cells, about
+    // 400 GB): each operand is within its own cap, so only the product check stops the abort.
+    let mut t = T::new();
+    let col = "SEQUENCE(1048576)";
+    let row = "SEQUENCE(1,16384)";
+    for (at, formula) in [
+        ("A1", format!("=ROWS({col}*{row})")),
+        ("A2", format!("=ROWS({col}&{row})")),
+        ("A3", format!("=ROWS({col}>{row})")),
+        ("A4", format!("=ROWS(IF({col},{row}))")),
+        ("A5", format!("=ROWS(IF({col}>0,1,{row}))")),
+        ("A6", format!("=ROWS(TEXT({col},{row}))")),
+        ("A7", format!("=ROWS(MAP({col},{row},LAMBDA(a,b,a)))")),
+    ] {
+        t.set(at, &formula);
+        assert_eq!(t.get(at), Value::Error(CellError::Num), "{formula}");
+    }
+    // Broadcasts within the cap are unchanged.
+    t.set("C1", "=SUM(SEQUENCE(3)*SEQUENCE(1,4))");
+    assert_eq!(t.num("C1"), 60.0);
+    t.set("C2", "=SUM(IF(SEQUENCE(2)>1,SEQUENCE(1,3),0))");
+    assert_eq!(t.num("C2"), 6.0);
+    t.set("C3", "=COLUMNS(TEXT(SEQUENCE(2),SEQUENCE(1,5)))");
+    assert_eq!(t.num("C3"), 5.0);
+}
+
 #[test]
 fn text_function() {
     let mut t = T::new();