Merge pull request #2694 from vangel76/perf/calc-spill-index-2683

perf(calc): index spill areas so cell and range reads don't scan every spill

crates/calc/src/lib.rs

@@ -6,6 +6,7 @@
 pub mod eval;
 mod precision;
 pub mod recalc;
+mod spill_index;
 mod trimrange;
 
 pub use eval::{Area, SPECIAL_FUNCTIONS, is_known_function};

crates/calc/src/recalc.rs

@@ -17,6 +17,17 @@ use gridcraft_formula::Expr;
 use gridcraft_model::{CalcMode, Workbook};
 
 use crate::eval::{Evaluator, Host, precedents, stack_position};
+use crate::spill_index::{SpillIndex, indexing_pays};
+
+/// Changed cells (in one `cells_changed`) from which a sheet's spill areas may be indexed.
+const INDEX_SPILLS_FROM: usize = 16;
+
+/// Whether a sheet's spill areas are indexed for a batch of `batch` changed cells of which
+/// `on_sheet` are on that sheet: the batch must be a paste, and registering the areas must cost
+/// less than walking them once for each of the sheet's own changed cells.
+fn index_spills<'a>(batch: usize, on_sheet: usize, areas: impl ExactSizeIterator<Item = &'a RangeRef>) -> bool {
+    batch > INDEX_SPILLS_FROM && areas.len() > 0 && indexing_pays(areas, on_sheet)
+}
 
 mod graph;
 mod pass_host;
@@ -263,11 +274,35 @@ impl Calc {
         let mut emptied: Vec<Key> = Vec::new();
         // Typing into (or clearing) a cell a dynamic array spills over (or would spill over)
         // re-evaluates the anchor, which may now be blocked or unblocked.
+        // With many changed cells (a paste) a sheet's spill areas are indexed once instead of being
+        // walked for each one, when registering them costs less than walking them would.
+        let mut areas = SpillIndex::default();
+        let mut strategy: HashMap<usize, bool> = HashMap::default();
+        let mut per_sheet: HashMap<usize, usize> = HashMap::default();
+        for k in changed {
+            *per_sheet.entry(k.0).or_insert(0) += 1;
+        }
+        let mut at = Vec::new();
         for &k in changed {
             if let Some(sh) = wb.sheet(k.0) {
-                for (anchor, r) in &sh.spill_ranges {
-                    if *anchor != k.1 && r.contains(k.1) {
-                        seeds.push((k.0, *anchor));
+                let indexed = *strategy.entry(k.0).or_insert_with(|| {
+                    let indexed = index_spills(changed.len(), per_sheet[&k.0], sh.spill_ranges.values());
+                    if indexed {
+                        for (anchor, r) in &sh.spill_ranges {
+                            areas.insert_rect((k.0, *anchor), *r, false);
+                        }
+                    }
+                    indexed
+                });
+                if indexed {
+                    // Anchor order, as walking `spill_ranges` gives.
+                    areas.meeting(k.0, RangeRef::new(k.1, k.1), &mut at);
+                    seeds.extend(at.iter().filter(|a| **a != k.1).map(|a| (k.0, *a)));
+                } else {
+                    for (anchor, r) in &sh.spill_ranges {
+                        if *anchor != k.1 && r.contains(k.1) {
+                            seeds.push((k.0, *anchor));
+                        }
                     }
                 }
                 for (c, cell) in sh.cells.iter_range(RangeRef::new(CellRef::new(k.1.row.saturating_sub(64), k.1.col.saturating_sub(64)), k.1)) {
@@ -576,7 +611,7 @@ pub(crate) mod tests {
     use gridcraft_core::{CellRef, RangeRef, Value};
     use gridcraft_model::{Cell, Formula, Workbook};
 
-    use super::{Calc, HashMap, Host, Intervals, Key, PassHost};
+    use super::{Calc, HashMap, Host, Intervals, Key, PassHost, index_spills};
 
     /// xorshift64 for repeatable pseudo-random cases.
     fn rng(state: &mut u64) -> u64 {
@@ -593,6 +628,19 @@ pub(crate) mod tests {
         v
     }
 
+    /// One changed cell on a sheet with many small spills does not index them just because the
+    /// batch (mostly on another sheet) is large; the sheet's own share of the batch decides.
+    #[test]
+    fn the_spill_index_is_chosen_from_the_sheets_own_changed_cells() {
+        let small: Vec<RangeRef> = (0..1_000).map(|i| RangeRef::new(CellRef::new(i * 3, 0), CellRef::new(i * 3 + 2, 0))).collect();
+        // Same areas, same batch of 5000: indexed when 5000 of them are on the sheet, walked when 1 is.
+        assert!(index_spills(5_000, 5_000, small.iter()));
+        assert!(!index_spills(5_000, 1, small.iter()));
+        // Not a paste, or nothing to index: never.
+        assert!(!index_spills(16, 16, small.iter()));
+        assert!(!index_spills(5_000, 5_000, [].iter()));
+    }
+
     /// A 1,000 x 1,000 Python result laid out from A1 in a pass, with the same values already
     /// spilled on the sheet from the last calculation, as a full recalculation finds them.
     fn big_python_spill() -> (Workbook, HashMap<Key, Value>) {
@@ -635,6 +683,39 @@ pub(crate) mod tests {
         assert!(!host.visit_range(0, inside, &mut |_| true), "an overlapping read leaves the block to range_values");
     }
 
+    /// A Python array laid out in this pass is seen by the indexed readers, whether the sheet
+    /// has no spill from before or an older, smaller one.
+    #[test]
+    fn a_python_spill_found_in_the_pass_is_read_through_the_index() {
+        for stale in [false, true] {
+            let mut wb = Workbook::new();
+            let sh = wb.sheet_mut(0).unwrap();
+            let anchor = CellRef::new(0, 0);
+            let py = gridcraft_model::PyCode::new("grid");
+            sh.set_cell(anchor, Cell { value: Value::Number(0.0), py: Some(std::sync::Arc::new(py)), ..Cell::default() });
+            if stale {
+                // The last calculation spilled a 2 x 2 block of 100s.
+                sh.spill_ranges.insert(anchor, RangeRef::new(anchor, CellRef::new(1, 1)));
+                for c in RangeRef::new(anchor, CellRef::new(1, 1)).iter().skip(1) {
+                    sh.spill.insert(c, Value::Number(100.0));
+                }
+            }
+            let data: Vec<Value> = (1..=9).map(|i| Value::Number(f64::from(i))).collect();
+            let array = gridcraft_core::Array::new(3, 3, data).unwrap();
+            let mut python = HashMap::default();
+            python.insert((0, anchor), Value::Array(std::sync::Arc::new(array)));
+            let exprs = HashMap::default();
+            let mut rng = 1u64;
+            let mut host = PassHost::new(&wb, &exprs, &mut rng, 0.0).with_python(python);
+            assert_eq!(host.cell_value(0, anchor), Value::Number(1.0));
+            let all = RangeRef::parse("A1:C3").unwrap();
+            let sum: f64 = host.range_values(0, all).iter().map(|v| if let Value::Number(n) = v { *n } else { 0.0 }).sum();
+            assert_eq!(sum, 45.0, "stale={stale}: a SUM over the new 3 x 3 spill reads all of it");
+            assert_eq!(host.cell_value(0, CellRef::new(2, 2)), Value::Number(9.0), "stale={stale}");
+            assert!(!host.visit_range(0, all, &mut |_| true), "stale={stale}: the new spill leaves the block to range_values");
+        }
+    }
+
     #[test]
     fn intervals_match_a_linear_scan() {
         let mut iv: Intervals<u32> = Intervals::default();

crates/calc/src/recalc/pass_host.rs

@@ -1,5 +1,6 @@
 use super::graph::{HashMap, HashSet, Key};
 use super::{Evaluator, Host, Workbook, stack_position};
+use crate::spill_index::{Cover, SpillIndex};
 use gridcraft_core::{Array, CellError, CellRef, RangeRef, Value};
 use gridcraft_formula::Expr;
 
@@ -24,6 +25,8 @@ pub(super) struct PassHost<'a> {
     in_progress: HashSet<Key>,
     /// New spills found during the pass: anchor → array.
     pub(super) spills: HashMap<Key, std::sync::Arc<Array>>,
+    /// Where the areas in `spills` are, so a read finds the ones it meets without scanning them all.
+    spill_index: SpillIndex,
     /// Used range per sheet: the workbook doesn't change during a pass, and working it out walks
     /// every row, so once per sheet rather than once per range read.
     used: HashMap<usize, Option<RangeRef>>,
@@ -93,6 +96,7 @@ impl<'a> PassHost<'a> {
             pending: exprs.keys().copied().collect(),
             in_progress: HashSet::default(),
             spills: HashMap::default(),
+            spill_index: SpillIndex::default(),
             used: HashMap::default(),
             rng,
             now,
@@ -131,6 +135,30 @@ impl<'a> PassHost<'a> {
     }
 }
 
+/// Most spill areas a range read meets before they are checked for overlapping one another by
+/// walking all of the pass's spills instead of comparing pairs.
+const MAX_UNORDERED_HITS: usize = 16;
+
+/// Whether an array at `anchor` covers `c`.
+fn spill_covers(anchor: CellRef, arr: &Array, c: CellRef) -> bool {
+    c.row >= anchor.row && c.col >= anchor.col && ((c.row - anchor.row) as usize) < arr.rows && ((c.col - anchor.col) as usize) < arr.cols
+}
+
+/// Whether two of the spill areas anchored at `hits` share a cell inside `range`.
+fn spill_hits_overlap(index: &SpillIndex, sheet: usize, hits: &[CellRef], range: RangeRef) -> bool {
+    let rects: Vec<RangeRef> = hits.iter().filter_map(|a| index.area(sheet, *a)).filter(|(_, empty)| !empty).map(|(r, _)| r).collect();
+    rects.iter().enumerate().any(|(i, a)| {
+        rects.iter().skip(i + 1).any(|b| {
+            a.intersects(b)
+                && RangeRef::new(
+                    CellRef::new(a.start.row.max(b.start.row), a.start.col.max(b.start.col)),
+                    CellRef::new(a.end.row.min(b.end.row), a.end.col.min(b.end.col)),
+                )
+                .intersects(&range)
+        })
+    })
+}
+
 /// Whether an array result at `k` can't spill: it is in a table, or its area runs off the sheet,
 /// overlaps merged cells, holds other cells, or another formula's spill (unless that formula is
 /// recalculated in this pass, and so may spill elsewhere now).
@@ -169,13 +197,6 @@ pub(super) fn spills_as_before(sheet: &gridcraft_model::Sheet, anchor: CellRef,
             .all(|(i, c)| c == anchor || (!sheet.cells.has(c) && sheet.spill.get(&c) == Some(arr.data.get(i).unwrap_or(&Value::Empty))))
 }
 
-/// The area an array anchored at `anchor` covers.
-fn array_area(anchor: CellRef, a: &Array) -> RangeRef {
-    let end =
-        CellRef::new(anchor.row.saturating_add((a.rows as u32).saturating_sub(1)), anchor.col.saturating_add((a.cols as u32).saturating_sub(1)));
-    RangeRef::new(anchor, end)
-}
-
 /// The spilled values `sheet` stores inside `range`, passed to `f` until it returns false. Values
 /// outside the range are not walked: with fewer rows in the range than spilled values, each row
 /// is looked up on its own; otherwise one ordered scan visits each spilled value at most once.
@@ -377,6 +398,7 @@ impl PassHost<'_> {
                 // value -- a FILTER matching one row, a SORT or UNIQUE over one item -- so `C1#`
                 // is the 1 x 1 range and `ROWS(C1#)` is 1, as in Excel (#1491). A formula that
                 // returns a scalar never reaches this arm, so it does not become an anchor.
+                self.spill_index.insert(k, a.rows, a.cols);
                 self.spills.insert(k, a.clone());
                 a.data.first().cloned().unwrap_or_default()
             }
@@ -391,6 +413,7 @@ impl PassHost<'_> {
         let v = match self.spills.get(&k) {
             Some(a) if spill_blocked(self, k, a) => {
                 self.spills.remove(&k);
+                self.spill_index.remove(k);
                 Value::Error(CellError::Spill)
             }
             _ => v,
@@ -410,6 +433,7 @@ impl PassHost<'_> {
                     && a.cols >= 1
                     && a.rows.checked_mul(a.cols).is_some_and(|n| n == a.data.len() && n <= gridcraft_model::python::MAX_RESULT_CELLS);
                 if sane {
+                    self.spill_index.insert(k, a.rows, a.cols);
                     self.spills.insert(k, a.clone());
                     a.data.first().cloned().unwrap_or_default()
                 } else {
@@ -421,6 +445,7 @@ impl PassHost<'_> {
         let v = match self.spills.get(&k) {
             Some(a) if spill_blocked(self, k, a) => {
                 self.spills.remove(&k);
+                self.spill_index.remove(k);
                 Value::Error(CellError::Spill)
             }
             _ => v,
@@ -467,7 +492,11 @@ impl PassHost<'_> {
         while passes < max_iterations.min(max_passes) && !self.converged(max_change) {
             self.prev = again.iter().filter_map(|k| self.results.remove(k).map(|v| (*k, v))).collect();
             self.pending = set.clone();
-            self.spills.retain(|a, _| !set.contains(a));
+            let gone: Vec<Key> = self.spills.keys().filter(|a| set.contains(a)).copied().collect();
+            for a in gone {
+                self.spills.remove(&a);
+                self.spill_index.remove(a);
+            }
             for k in &again {
                 self.calculate(*k);
             }
@@ -551,17 +580,19 @@ impl Host for PassHost<'_> {
         if self.pending.contains(&k) || self.results.contains_key(&k) {
             return self.compute(k);
         }
-        // A cell covered by a spill computed in this pass.
-        for (anchor, arr) in &self.spills {
-            if anchor.0 == sheet && c.row >= anchor.1.row && c.col >= anchor.1.col {
-                let (dr, dc) = ((c.row - anchor.1.row) as usize, (c.col - anchor.1.col) as usize);
-                if dr < arr.rows && dc < arr.cols {
-                    if self.iterate && self.tainted.contains(anchor) {
-                        self.taint = true;
-                    }
-                    return arr.get(dr, dc).cloned().unwrap_or_default();
-                }
+        // A cell covered by a spill computed in this pass. Where several overlap, the first in
+        // `spills`' iteration order counts, as it always has.
+        let covering = match self.spill_index.covering(sheet, c) {
+            Cover::None => None,
+            Cover::One(a) => self.spills.get_key_value(&(sheet, a)),
+            Cover::Many => self.spills.iter().find(|(a, arr)| a.0 == sheet && spill_covers(a.1, arr, c)),
+        };
+        if let Some((anchor, arr)) = covering {
+            let (dr, dc) = ((c.row - anchor.1.row) as usize, (c.col - anchor.1.col) as usize);
+            if self.iterate && self.tainted.contains(anchor) {
+                self.taint = true;
             }
+            return arr.get(dr, dc).cloned().unwrap_or_default();
         }
         self.wb.sheet(sheet).map(|s| s.value(c)).unwrap_or_default()
     }
@@ -600,46 +631,44 @@ impl Host for PassHost<'_> {
             }
         }
         if !self.spills.is_empty() {
-            // Only the part of each array inside the range is walked: a large array beside a small
-            // read costs nothing.
-            let mut new: Vec<(usize, Value)> = Vec::new();
-            for (a, arr) in self.spills.iter().filter(|(a, _)| a.0 == sheet) {
-                let area = array_area(a.1, arr);
-                if !area.intersects(&range) {
-                    continue;
-                }
-                let (r0, r1) = (area.start.row.max(range.start.row), area.end.row.min(range.end.row));
-                let (c0, c1) = (area.start.col.max(range.start.col), area.end.col.min(range.end.col));
-                for row in r0..=r1 {
-                    for col in c0..=c1 {
-                        let c = CellRef::new(row, col);
-                        if c == a.1 {
-                            continue;
-                        }
-                        let v = arr.get((row - a.1.row) as usize, (col - a.1.col) as usize).cloned().unwrap_or_default();
-                        new.push(((row - range.start.row) as usize * w + (col - range.start.col) as usize, v));
-                    }
-                }
-            }
-            #[cfg(test)]
-            {
-                self.spill_visits += new.len();
-            }
+            let mut hits = Vec::new();
+            self.spill_index.meeting(sheet, range, &mut hits);
             // Iterating: an array spilled by a formula that read a previous value.
-            if self.iterate
-                && self.spills.iter().any(|(a, arr)| {
-                    let end = CellRef::new(
-                        a.1.row.saturating_add((arr.rows as u32).saturating_sub(1)),
-                        a.1.col.saturating_add((arr.cols as u32).saturating_sub(1)),
-                    );
-                    a.0 == sheet && self.tainted.contains(a) && RangeRef::new(a.1, end).intersects(&range)
-                })
-            {
+            if self.iterate && hits.iter().any(|a| self.tainted.contains(&(sheet, *a))) {
                 self.taint = true;
             }
-            for (i, v) in new {
-                if let Some(slot) = out.get_mut(i) {
-                    *slot = v;
+            // Where areas overlap inside the range, the last in `spills`' iteration order wins,
+            // as it always has; otherwise the order doesn't matter.
+            let ordered = hits.len() > MAX_UNORDERED_HITS || spill_hits_overlap(&self.spill_index, sheet, &hits, range);
+            let mut put = |anchor: CellRef, arr: &Array| {
+                if arr.rows == 0 || arr.cols == 0 {
+                    return;
+                }
+                let (r0, c0) = (range.start.row.max(anchor.row), range.start.col.max(anchor.col));
+                let r1 = range.end.row.min(anchor.row.saturating_add((arr.rows as u32).saturating_sub(1)));
+                let c1 = range.end.col.min(anchor.col.saturating_add((arr.cols as u32).saturating_sub(1)));
+                for r in r0..=r1 {
+                    for cc in c0..=c1 {
+                        if (r, cc) != (anchor.row, anchor.col)
+                            && let Some(slot) = out.get_mut((r - range.start.row) as usize * w + (cc - range.start.col) as usize)
+                        {
+                            *slot = arr.get((r - anchor.row) as usize, (cc - anchor.col) as usize).cloned().unwrap_or_default();
+                        }
+                    }
+                }
+            };
+            if ordered {
+                let wanted: HashSet<Key> = hits.iter().map(|a| (sheet, *a)).collect();
+                for (a, arr) in &self.spills {
+                    if wanted.contains(a) {
+                        put(a.1, arr);
+                    }
+                }
+            } else {
+                for a in &hits {
+                    if let Some(arr) = self.spills.get(&(sheet, *a)) {
+                        put(*a, arr);
+                    }
                 }
             }
         }
@@ -658,7 +687,8 @@ impl Host for PassHost<'_> {
         }
         // Spilled values sit between the stored cells (and the formulas just calculated may have
         // produced some): leave such blocks to `range_values`.
-        let arrays = self.spills.iter().any(|(a, arr)| a.0 == sheet && array_area(a.1, arr).intersects(&range));
+        let mut arrays = Vec::new();
+        self.spill_index.meeting(sheet, range, &mut arrays);
         let mut stored = false;
         let _seen = stored_spill_in(sh, range, &mut |_, _| {
             stored = true;
@@ -668,7 +698,7 @@ impl Host for PassHost<'_> {
         {
             self.spill_visits += _seen;
         }
-        if arrays || stored {
+        if !arrays.is_empty() || stored {
             return false;
         }
         for (c, cell) in sh.cells.iter_range(range) {

crates/calc/src/spill_index.rs

@@ -0,0 +1,296 @@
+//! A spatial index over spill areas: which areas cover a cell, and which meet a range, without
+//! looking at every area. Areas are registered in 16 x 16 cell blocks, so a lookup touches one
+//! block (a cell) or the blocks a range covers (a range) rather than every spill on the sheet.
+//! An area that would take more than `BIG_AREA_BLOCKS` blocks (a full-height column spill takes
+//! 65,536) is not registered in blocks but kept in a short list that every lookup tests directly,
+//! so registering costs what the area's size warrants only for small areas.
+
+use crate::recalc::{FxHasher, Key};
+use gridcraft_core::{CellRef, RangeRef};
+use std::hash::BuildHasherDefault;
+
+type HashMap<K, V> = std::collections::HashMap<K, V, BuildHasherDefault<FxHasher>>;
+
+/// Block edge in cells (rows and columns).
+const BLOCK: u32 = 16;
+
+/// Areas spanning more blocks than this are tested directly instead of being registered.
+const BIG_AREA_BLOCKS: u64 = 1024;
+
+/// One indexed area: its rectangle, and whether it covers no cells (an array with no rows or no
+/// columns still meets the ranges its anchor is in, as a 1 x 1 area does for taint tracking).
+#[derive(Clone, Copy)]
+struct Entry {
+    rect: RangeRef,
+    empty: bool,
+}
+
+#[derive(Default)]
+struct SheetAreas {
+    entries: HashMap<CellRef, Entry>,
+    blocks: HashMap<(u32, u32), Vec<CellRef>>,
+    /// Anchors of the areas too big to register in blocks.
+    big: Vec<CellRef>,
+}
+
+/// What covers a cell.
+pub(crate) enum Cover {
+    None,
+    One(CellRef),
+    /// Several areas overlap here: the caller decides which one counts.
+    Many,
+}
+
+#[derive(Default)]
+pub(crate) struct SpillIndex {
+    sheets: HashMap<usize, SheetAreas>,
+}
+
+fn block_span(rect: &RangeRef) -> (u32, u32, u32, u32) {
+    (rect.start.row / BLOCK, rect.end.row / BLOCK, rect.start.col / BLOCK, rect.end.col / BLOCK)
+}
+
+fn block_count(rect: &RangeRef) -> u64 {
+    let (r0, r1, c0, c1) = block_span(rect);
+    u64::from(r1 - r0 + 1) * u64::from(c1 - c0 + 1)
+}
+
+/// Whether registering `areas` in blocks costs less than checking each of them against each of
+/// `lookups` cells would: a few block entries cost about one containment check each. A tall or
+/// wide area is not registered at all (see `BIG_AREA_BLOCKS`) and so costs nothing here.
+pub(crate) fn indexing_pays<'a>(areas: impl ExactSizeIterator<Item = &'a RangeRef>, lookups: usize) -> bool {
+    const BLOCK_COST: u64 = 16;
+    let budget = (lookups as u64).saturating_mul(areas.len() as u64) / BLOCK_COST;
+    let mut blocks = 0u64;
+    for r in areas {
+        let n = block_count(r);
+        if n > BIG_AREA_BLOCKS {
+            continue;
+        }
+        blocks = blocks.saturating_add(n);
+        if blocks > budget {
+            return false;
+        }
+    }
+    true
+}
+
+impl SpillIndex {
+    /// Whether any area is indexed on `sheet`.
+    #[cfg(test)]
+    pub(crate) fn has_sheet(&self, sheet: usize) -> bool {
+        self.sheets.get(&sheet).is_some_and(|s| !s.entries.is_empty())
+    }
+
+    /// Indexes the area an array of `rows` x `cols` anchored at `anchor` covers (replacing the
+    /// one already indexed for that anchor).
+    pub(crate) fn insert(&mut self, (sheet, anchor): Key, rows: usize, cols: usize) {
+        let end =
+            CellRef::new(anchor.row.saturating_add((rows as u32).saturating_sub(1)), anchor.col.saturating_add((cols as u32).saturating_sub(1)));
+        self.insert_rect((sheet, anchor), RangeRef::new(anchor, end), rows == 0 || cols == 0);
+    }
+
+    /// Indexes an area given as a range.
+    pub(crate) fn insert_rect(&mut self, (sheet, anchor): Key, rect: RangeRef, empty: bool) {
+        self.remove((sheet, anchor));
+        let areas = self.sheets.entry(sheet).or_default();
+        if block_count(&rect) > BIG_AREA_BLOCKS {
+            areas.big.push(anchor);
+        } else {
+            let (r0, r1, c0, c1) = block_span(&rect);
+            for br in r0..=r1 {
+                for bc in c0..=c1 {
+                    areas.blocks.entry((br, bc)).or_default().push(anchor);
+                }
+            }
+        }
+        areas.entries.insert(anchor, Entry { rect, empty });
+    }
+
+    pub(crate) fn remove(&mut self, (sheet, anchor): Key) {
+        let Some(areas) = self.sheets.get_mut(&sheet) else { return };
+        let Some(old) = areas.entries.remove(&anchor) else { return };
+        if block_count(&old.rect) > BIG_AREA_BLOCKS {
+            areas.big.retain(|a| *a != anchor);
+            return;
+        }
+        let (r0, r1, c0, c1) = block_span(&old.rect);
+        for br in r0..=r1 {
+            for bc in c0..=c1 {
+                if let Some(list) = areas.blocks.get_mut(&(br, bc)) {
+                    list.retain(|a| *a != anchor);
+                    if list.is_empty() {
+                        areas.blocks.remove(&(br, bc));
+                    }
+                }
+            }
+        }
+    }
+
+    /// The areas that cover `c`.
+    pub(crate) fn covering(&self, sheet: usize, c: CellRef) -> Cover {
+        let Some(areas) = self.sheets.get(&sheet) else { return Cover::None };
+        let in_block = areas.blocks.get(&(c.row / BLOCK, c.col / BLOCK)).map_or(&[][..], |l| l.as_slice());
+        let mut found = Cover::None;
+        for a in in_block.iter().chain(&areas.big) {
+            if let Some(e) = areas.entries.get(a)
+                && !e.empty
+                && e.rect.contains(c)
+            {
+                found = match found {
+                    Cover::None => Cover::One(*a),
+                    _ => return Cover::Many,
+                };
+            }
+        }
+        found
+    }
+
+    /// The anchors of the areas whose rectangle meets `range` (an empty area counts through its
+    /// anchor cell), sorted by anchor.
+    pub(crate) fn meeting(&self, sheet: usize, range: RangeRef, out: &mut Vec<CellRef>) {
+        out.clear();
+        let Some(areas) = self.sheets.get(&sheet) else { return };
+        let (r0, r1, c0, c1) = block_span(&range);
+        let touched = u64::from(r1 - r0 + 1) * u64::from(c1 - c0 + 1);
+        if touched > 4 * areas.entries.len() as u64 + 16 {
+            // A range wider than the areas are many: look at each area once.
+            out.extend(areas.entries.iter().filter(|(_, e)| e.rect.intersects(&range)).map(|(a, _)| *a));
+        } else {
+            for br in r0..=r1 {
+                for bc in c0..=c1 {
+                    if let Some(list) = areas.blocks.get(&(br, bc)) {
+                        out.extend(list.iter().copied().filter(|a| areas.entries.get(a).is_some_and(|e| e.rect.intersects(&range))));
+                    }
+                }
+            }
+            out.extend(areas.big.iter().copied().filter(|a| areas.entries.get(a).is_some_and(|e| e.rect.intersects(&range))));
+        }
+        out.sort_unstable();
+        out.dedup();
+    }
+
+    /// The rectangle indexed for `anchor` and whether it covers no cells.
+    pub(crate) fn area(&self, sheet: usize, anchor: CellRef) -> Option<(RangeRef, bool)> {
+        self.sheets.get(&sheet)?.entries.get(&anchor).map(|e| (e.rect, e.empty))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn at(r: u32, c: u32) -> CellRef {
+        CellRef::new(r, c)
+    }
+
+    #[test]
+    fn finds_the_areas_covering_a_cell_and_meeting_a_range() {
+        let mut ix = SpillIndex::default();
+        ix.insert((0, at(0, 0)), 40, 3);
+        ix.insert((0, at(30, 1)), 1, 40);
+        ix.insert((1, at(0, 0)), 5, 5);
+        assert!(matches!(ix.covering(0, at(39, 2)), Cover::One(a) if a == at(0, 0)));
+        assert!(matches!(ix.covering(0, at(30, 2)), Cover::Many));
+        assert!(matches!(ix.covering(0, at(40, 0)), Cover::None));
+        assert!(matches!(ix.covering(2, at(0, 0)), Cover::None));
+        let mut hits = Vec::new();
+        ix.meeting(0, RangeRef::new(at(30, 30), at(50, 60)), &mut hits);
+        assert_eq!(hits, vec![at(30, 1)]);
+        ix.meeting(0, RangeRef::new(at(0, 0), at(1_000_000, 16_000)), &mut hits);
+        assert_eq!(hits, vec![at(0, 0), at(30, 1)]);
+        ix.remove((0, at(30, 1)));
+        assert!(matches!(ix.covering(0, at(30, 2)), Cover::One(_)));
+        assert!(ix.has_sheet(1) && !ix.has_sheet(2));
+        ix.insert((0, at(0, 0)), 1, 1);
+        assert!(matches!(ix.covering(0, at(39, 2)), Cover::None));
+    }
+
+    #[test]
+    fn tall_or_wide_areas_are_not_registered_in_blocks() {
+        let column = RangeRef::new(at(0, 0), at(1_048_575, 0));
+        assert!(block_count(&column) > BIG_AREA_BLOCKS);
+        let mut ix = SpillIndex::default();
+        ix.insert_rect((0, at(0, 0)), column, false);
+        assert!(ix.sheets[&0].blocks.is_empty());
+        assert!(matches!(ix.covering(0, at(900_000, 0)), Cover::One(a) if a == at(0, 0)));
+        assert!(matches!(ix.covering(0, at(900_000, 1)), Cover::None));
+        let mut hits = Vec::new();
+        ix.meeting(0, RangeRef::new(at(5, 0), at(6, 3)), &mut hits);
+        assert_eq!(hits, vec![at(0, 0)]);
+        ix.remove((0, at(0, 0)));
+        assert!(matches!(ix.covering(0, at(900_000, 0)), Cover::None) && ix.sheets[&0].big.is_empty());
+        // Registering a big area costs nothing (lookups test it directly), so a small paste still indexes next to many small areas.
+        assert!(indexing_pays([column].iter(), 17));
+        // Many small areas pay back for a paste of a hundred cells, but not for a handful.
+        let small: Vec<RangeRef> = (0..1_000).map(|i| RangeRef::new(at(i * 3, 0), at(i * 3 + 2, 0))).collect();
+        assert!(indexing_pays(small.iter(), 100));
+        assert!(!indexing_pays(small.iter(), 17));
+    }
+
+    /// The index against the old way: test every area for every lookup. Mixed shapes: small,
+    /// thin and tall, wide, big (kept out of the blocks), empty, on two sheets, replaced and removed.
+    #[test]
+    fn matches_a_scan_of_every_area_on_mixed_shapes() {
+        let mut seed = 0x2545_F491_4F6C_DD1Du64;
+        let mut next = |n: u32| {
+            seed ^= seed << 13;
+            seed ^= seed >> 7;
+            seed ^= seed << 17;
+            (seed >> 11) as u32 % n
+        };
+        let mut ix = SpillIndex::default();
+        let mut model: std::collections::BTreeMap<(usize, CellRef), (RangeRef, bool)> = Default::default();
+        for step in 0..3_000 {
+            let sheet = next(2) as usize;
+            let anchor = at(next(400) * 5, next(40) * 3);
+            if next(4) == 0 {
+                ix.remove((sheet, anchor));
+                model.remove(&(sheet, anchor));
+            } else {
+                let (rows, cols) = match next(8) {
+                    0 => (0, 1 + next(5) as usize),
+                    1 => (1 + next(5) as usize, 0),
+                    2 => (5_000 + next(100_000) as usize, 1),
+                    3 => (1, 1 + next(40_000) as usize),
+                    4 => (200 + next(300) as usize, 40 + next(40) as usize),
+                    _ => (1 + next(60) as usize, 1 + next(40) as usize),
+                };
+                ix.insert((sheet, anchor), rows, cols);
+                let end = at(anchor.row + (rows as u32).saturating_sub(1), anchor.col + (cols as u32).saturating_sub(1));
+                model.insert((sheet, anchor), (RangeRef::new(anchor, end), rows == 0 || cols == 0));
+            }
+            if step % 7 != 0 {
+                continue;
+            }
+            let sheet = next(3) as usize;
+            for _ in 0..20 {
+                let cell = at(next(2_100), next(130));
+                let want: Vec<CellRef> =
+                    model.iter().filter(|((s, _), (r, e))| *s == sheet && !e && r.contains(cell)).map(|((_, a), _)| *a).collect();
+                match ix.covering(sheet, cell) {
+                    Cover::None => assert!(want.is_empty(), "{cell:?}"),
+                    Cover::One(a) => assert_eq!(want, vec![a], "{cell:?}"),
+                    Cover::Many => assert!(want.len() > 1, "{cell:?}"),
+                }
+                let (r0, c0) = (next(2_100), next(130));
+                let (tall, wide) = (if next(5) == 0 { 3_000 } else { 40 }, if next(5) == 0 { 130 } else { 20 });
+                let range = RangeRef::new(at(r0, c0), at(r0 + next(tall), c0 + next(wide)));
+                let want: Vec<CellRef> = model.iter().filter(|((s, _), (r, _))| *s == sheet && r.intersects(&range)).map(|((_, a), _)| *a).collect();
+                let mut got = Vec::new();
+                ix.meeting(sheet, range, &mut got);
+                assert_eq!(got, want, "{range:?}");
+            }
+        }
+    }
+
+    #[test]
+    fn an_area_without_cells_covers_none_but_is_met_at_its_anchor() {
+        let mut ix = SpillIndex::default();
+        ix.insert((0, at(3, 3)), 0, 4);
+        assert!(matches!(ix.covering(0, at(3, 3)), Cover::None));
+        let mut hits = Vec::new();
+        ix.meeting(0, RangeRef::new(at(3, 3), at(3, 3)), &mut hits);
+        assert_eq!(hits, vec![at(3, 3)]);
+    }
+}

crates/calc/src/tests.rs

@@ -2379,6 +2379,125 @@ fn loan_schedule_residue_keeps_excels_sign() {
     assert_eq!(t.get("P7"), Value::Text("".into()));
 }
 
+/// A full-height column spill (inside the 10,000,000-cell cap) and a paste of 17 cells elsewhere:
+/// the paste must not register the spill's 65,536 blocks, and a paste over the spill must still
+/// block it.
+fn tall_spill_book() -> T {
+    let mut t = T::new();
+    t.set("A1", "=SEQUENCE(1048576)");
+    t.calc.recalc_all(&mut t.wb);
+    assert_eq!(t.num("A1048576"), 1_048_576.0);
+    t
+}
+
+fn paste_into(t: &mut T, cells: &[(u32, u32)]) {
+    let changed: Vec<(usize, CellRef)> = cells.iter().map(|&(r, c)| (0, CellRef::new(r, c))).collect();
+    for k in &changed {
+        t.wb.sheet_mut(0).unwrap().cells.set(k.1, Cell::value(Value::Number(1.0)));
+    }
+    t.calc.cells_changed(&mut t.wb, &changed);
+}
+
+#[test]
+fn a_paste_beside_a_tall_spill_leaves_it_alone_and_one_over_it_blocks_it() {
+    let mut t = tall_spill_book();
+    let beside: Vec<(u32, u32)> = (0..17).map(|i| (i, 5)).collect();
+    paste_into(&mut t, &beside);
+    assert_eq!(t.num("A1048576"), 1_048_576.0);
+    let over: Vec<(u32, u32)> = (0..17).map(|i| (500_000 + i, if i == 9 { 0 } else { 5 })).collect();
+    paste_into(&mut t, &over);
+    assert_eq!(t.get("A1"), Value::Error(CellError::Spill));
+}
+
+/// Probe: `cells_changed` for a 17-cell paste in column F beside the tall spill.
+/// `cargo test -p gridcraft-calc --release -- --ignored --nocapture tall_spill_paste_probe`
+#[test]
+#[ignore]
+#[allow(clippy::disallowed_methods)]
+fn tall_spill_paste_probe() {
+    let mut t = tall_spill_book();
+    let mut best = std::time::Duration::MAX;
+    for round in 0..9u32 {
+        let beside: Vec<(u32, u32)> = (0..17).map(|i| (round * 20 + i, 5)).collect();
+        let start = std::time::Instant::now();
+        paste_into(&mut t, &beside);
+        best = best.min(start.elapsed());
+    }
+    eprintln!("tall spill, 17-cell paste beside it: best of 9 = {best:?}");
+}
+
+#[test]
+fn many_spills_are_read_and_blocked_by_a_paste() {
+    let mut t = T::new();
+    // 40 arrays of 3 rows side by side in rows 1-3, and a reader of a range over two of them.
+    for i in 0..40u32 {
+        let anchor = CellRef::new(0, 2 * i);
+        t.wb.sheet_mut(0).unwrap().cells.set(anchor, Cell::formula(Formula::new(&format!("=SEQUENCE(3,1,{})", 10 * i))));
+    }
+    t.set("A10", "=SUM(C1:F3)");
+    t.set("A12", "=COUNT(C1:F3)");
+    t.set("A11", "=C2*1");
+    t.calc.recalc_all(&mut t.wb);
+    assert_eq!(t.num("A10"), (10 + 11 + 12 + 20 + 21 + 22) as f64);
+    assert_eq!(t.num("A11"), 11.0);
+    assert_eq!(t.num("A12"), 6.0);
+    // Pasting 20 cells (more than one spill's width worth, so the changed set is indexed) over
+    // the second row of every other array blocks those arrays; the rest are unchanged.
+    let changed: Vec<(usize, CellRef)> = (0..20u32).map(|i| (0, CellRef::new(1, 4 * i))).collect();
+    for k in &changed {
+        t.wb.sheet_mut(0).unwrap().cells.set(k.1, Cell::value(Value::Number(1.0)));
+    }
+    t.calc.cells_changed(&mut t.wb, &changed);
+    assert_eq!(t.get("E1"), Value::Error(CellError::Spill));
+    assert_eq!(t.get("C1"), Value::Number(10.0));
+    assert_eq!(t.get("G2"), Value::Number(31.0));
+    assert_eq!(t.get("A10"), Value::Error(CellError::Spill));
+    assert_eq!(t.num("A12"), 4.0);
+}
+
+#[test]
+fn a_paste_over_small_and_tall_spills_blocks_those_it_touches() {
+    let mut t = T::new();
+    for i in 0..40u32 {
+        t.wb.sheet_mut(0).unwrap().cells.set(CellRef::new(0, 2 * i), Cell::formula(Formula::new("=SEQUENCE(3)")));
+    }
+    t.wb.sheet_mut(0).unwrap().cells.set(CellRef::new(0, 200), Cell::formula(Formula::new("=SEQUENCE(1048576)")));
+    t.calc.recalc_all(&mut t.wb);
+    // 20 cells: the middle row of the small arrays in columns 0, 8, 16 ... 72 (columns 80 and up hit
+    // nothing), and one deep inside the tall one.
+    let mut cells: Vec<(u32, u32)> = (0..19u32).map(|i| (1, 8 * i)).collect();
+    cells.push((700_000, 200));
+    paste_into(&mut t, &cells);
+    assert_eq!(t.get("A1"), Value::Error(CellError::Spill));
+    assert_eq!(t.get("GS1"), Value::Error(CellError::Spill));
+    assert_eq!(t.num("C1"), 1.0);
+    assert_eq!(t.num("C3"), 3.0);
+    assert_eq!(t.get("I1"), Value::Error(CellError::Spill));
+    assert_eq!(t.num("E3"), 3.0);
+}
+
+#[test]
+fn a_big_paste_on_one_sheet_and_one_cell_over_many_spills_on_another() {
+    let mut t = T::new();
+    t.wb.sheets.push(std::sync::Arc::new(gridcraft_model::Sheet::new("Data")));
+    for i in 0..1_500u32 {
+        t.wb.sheet_mut(1).unwrap().cells.set(CellRef::new(4 * i, 0), Cell::formula(Formula::new(&format!("=SEQUENCE(3,1,{})", 10 * i))));
+    }
+    t.calc.recalc_all(&mut t.wb);
+    assert_eq!(t.wb.sheet(1).unwrap().value(CellRef::new(4 * 700 + 2, 0)), Value::Number(7002.0));
+    // 5000 cells pasted on the first sheet, one over the middle of the 701st array on the second.
+    let mut changed: Vec<(usize, CellRef)> = (0..5_000u32).map(|i| (0, CellRef::new(i / 50, i % 50))).collect();
+    changed.push((1, CellRef::new(4 * 700 + 1, 0)));
+    for k in &changed {
+        t.wb.sheet_mut(k.0).unwrap().cells.set(k.1, Cell::value(Value::Number(1.0)));
+    }
+    t.calc.cells_changed(&mut t.wb, &changed);
+    let data = t.wb.sheet(1).unwrap();
+    assert_eq!(data.value(CellRef::new(4 * 700, 0)), Value::Error(CellError::Spill));
+    assert_eq!(data.value(CellRef::new(4 * 699 + 2, 0)), Value::Number(6992.0));
+    assert_eq!(data.value(CellRef::new(4 * 701 + 2, 0)), Value::Number(7012.0));
+}
+
 /// `SUM`, `COUNT` and `AVERAGE` over blocks of constants share one pass-wide answer per block, and a
 /// running total extends the shorter block's. Every result must equal the left-to-right sum of the
 /// cells, bit for bit, with text, blanks, errors, formulas and a nonzero start in the mix.