From 542aa176cee615abcade390c701df28f6855884e Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 05:19:24 -0500 Subject: [PATCH] Wildcards: match * and ? patterns in one pass over the text (#3205) * Wildcards: match * and ? patterns in one pass over the text After a mismatch the matcher restarted the pattern after the last star at the next text position, so a long repetitive cell against a near-matching pattern cost text x pattern comparisons. AutoFilter custom criteria, Find and Replace all call it once per cell. wildcard() now runs the pattern as a bit-parallel automaton: one pass over the text, one step per 64 pattern positions for each character. Results are unchanged; a test compares it with the old matcher on every short text and pattern over a small alphabet. * address review: build wildcard literal masks lazily with a bounded cache * address review: keep masks of the most frequent wildcard literals past the cache cap --- crates/engine/src/cmd/edit/find.rs | 100 ++++++++++++++++++++++------ crates/engine/src/cmd/edit/tests.rs | 97 +++++++++++++++++++++++++++ 2 files changed, 176 insertions(+), 21 deletions(-) diff --git a/crates/engine/src/cmd/edit/find.rs b/crates/engine/src/cmd/edit/find.rs index 3dfbc6cbe..3efcdaaf4 100644 --- a/crates/engine/src/cmd/edit/find.rs +++ b/crates/engine/src/cmd/edit/find.rs @@ -14,8 +14,13 @@ fn matches(hay: &str, what: &str, case: bool, whole: bool) -> bool { } /// `*`/`?` wildcard match with `~` escapes. +/// +/// The pattern runs as a bit-parallel automaton over the text: one pass, each character costing +/// one step per 64 pattern positions. Backtracking to the last `*` after a mismatch rescans the +/// text and costs text × pattern comparisons on a long, repetitive cell, for every cell a +/// filter or Find visits. pub(crate) fn wildcard(text: &str, pat: &str) -> bool { - let t: Vec = text.chars().collect(); + // The pattern as (character, escaped) items, runs of `*` folded into one. let mut p: Vec<(char, bool)> = Vec::new(); let mut it = pat.chars(); while let Some(c) = it.next() { @@ -23,32 +28,85 @@ pub(crate) fn wildcard(text: &str, pat: &str) -> bool { if let Some(n) = it.next() { p.push((n, true)); } - } else { + } else if c != '*' || p.last() != Some(&('*', false)) { p.push((c, false)); } } - let (mut ti, mut pi) = (0usize, 0usize); - let (mut star, mut mark) = (None, 0usize); - while ti < t.len() { - if pi < p.len() && (p[pi] == ('?', false) || (p[pi].0 == t[ti] && p[pi] != ('*', false))) { - ti += 1; - pi += 1; - } else if pi < p.len() && p[pi] == ('*', false) { - star = Some(pi); - mark = ti; - pi += 1; - } else if let Some(sp) = star { - pi = sp + 1; - mark += 1; - ti = mark; - } else { - return false; + let words = (p.len() + 1).div_ceil(64); + let bit = |set: &mut [u64], i: usize| set[i / 64] |= 1 << (i % 64); + // Positions holding a `*` and a `?`, and where each literal character sits in the pattern. + let mut stars = vec![0u64; words]; + let mut any = vec![0u64; words]; + let mut at: std::collections::HashMap> = std::collections::HashMap::new(); + for (i, item) in p.iter().enumerate() { + match item { + ('*', false) => bit(&mut stars, i), + ('?', false) => bit(&mut any, i), + (c, _) => at.entry(*c).or_default().push(i), } } - while pi < p.len() && p[pi] == ('*', false) { - pi += 1; + // A literal's bit mask is built when the text first reads it, and kept for at most + // `MAX_CACHED_MASKS` literals: a long pattern of mostly distinct characters would otherwise + // allocate a mask of the pattern's width for each one up front. When there are more literals + // than that, the ones kept are those at the most pattern positions (the dearest to rebuild); + // each of the rest has at most pattern length / `MAX_CACHED_MASKS` positions, so rebuilding + // its mask in `scratch` costs less than one step of the scan. + const MAX_CACHED_MASKS: usize = 256; + let cacheable: Option> = (at.len() > MAX_CACHED_MASKS).then(|| { + let mut by_count: Vec<(&char, usize)> = at.iter().map(|(c, v)| (c, v.len())).collect(); + by_count.sort_unstable_by_key(|&(_, n)| std::cmp::Reverse(n)); + by_count.into_iter().take(MAX_CACHED_MASKS).map(|(c, _)| *c).collect() + }); + let mut masks: std::collections::HashMap> = std::collections::HashMap::new(); + let mut scratch = vec![0u64; words]; + // A set bit i: the text read so far matches the first i pattern items. A `*` matches nothing + // as well, so a state at a `*` is also at the item after it. + let close = |set: &mut Vec| { + let mut carry = 0u64; + for w in 0..words { + let s = set[w] & stars[w]; + let next = (s << 1) | carry; + carry = s >> 63; + set[w] |= next; + } + }; + let mut state = vec![0u64; words]; + bit(&mut state, 0); + close(&mut state); + let mut next = vec![0u64; words]; + for ch in text.chars() { + let lit: Option<&[u64]> = match at.get(&ch) { + None => None, + Some(positions) if cacheable.as_ref().is_none_or(|k| k.contains(&ch)) => { + let mask = masks.entry(ch).or_insert_with(|| { + let mut m = vec![0u64; words]; + positions.iter().for_each(|&i| bit(&mut m, i)); + m + }); + Some(mask) + } + Some(positions) => { + scratch.fill(0); + positions.iter().for_each(|&i| bit(&mut scratch, i)); + Some(&scratch) + } + }; + let mut carry = 0u64; + let mut alive = 0u64; + for w in 0..words { + // Items that read `ch` advance; a `*` reads it and stays. + let m = state[w] & (any[w] | lit.map_or(0, |l| l[w])); + next[w] = (m << 1) | carry | (state[w] & stars[w]); + carry = m >> 63; + alive |= next[w]; + } + if alive == 0 { + return false; + } + std::mem::swap(&mut state, &mut next); + close(&mut state); } - pi == p.len() + state[p.len() / 64] & (1 << (p.len() % 64)) != 0 } pub(super) fn find(s: &mut Session, p: &Json) -> Result { diff --git a/crates/engine/src/cmd/edit/tests.rs b/crates/engine/src/cmd/edit/tests.rs index 0400e61e8..160ce9157 100644 --- a/crates/engine/src/cmd/edit/tests.rs +++ b/crates/engine/src/cmd/edit/tests.rs @@ -324,6 +324,103 @@ fn wildcards() { assert!(wildcard("", "*")); } +/// The backtracking matcher `wildcard` replaced, as the reference for its results. +fn wildcard_backtracking(text: &str, pat: &str) -> bool { + let t: Vec = text.chars().collect(); + let mut p: Vec<(char, bool)> = Vec::new(); + let mut it = pat.chars(); + while let Some(c) = it.next() { + if c == '~' { + if let Some(n) = it.next() { + p.push((n, true)); + } + } else { + p.push((c, false)); + } + } + let (mut ti, mut pi) = (0usize, 0usize); + let (mut star, mut mark) = (None, 0usize); + while ti < t.len() { + if pi < p.len() && (p[pi] == ('?', false) || (p[pi].0 == t[ti] && p[pi] != ('*', false))) { + ti += 1; + pi += 1; + } else if pi < p.len() && p[pi] == ('*', false) { + star = Some(pi); + mark = ti; + pi += 1; + } else if let Some(sp) = star { + pi = sp + 1; + mark += 1; + ti = mark; + } else { + return false; + } + } + while pi < p.len() && p[pi] == ('*', false) { + pi += 1; + } + pi == p.len() +} + +#[test] +fn wildcard_agrees_with_backtracking() { + // Every text up to 5 characters over a small alphabet against every pattern up to 4 items. + let alphabet = ['a', 'b', '*', '?', '~']; + let all = |n: usize, chars: &[char]| -> Vec { + let mut out = vec![String::new()]; + let mut level = vec![String::new()]; + for _ in 0..n { + level = level.iter().flat_map(|s| chars.iter().map(move |c| format!("{s}{c}"))).collect(); + out.extend(level.iter().cloned()); + } + out + }; + let texts = all(5, &['a', 'b', '*']); + for pat in all(4, &alphabet) { + for text in &texts { + assert_eq!(wildcard(text, &pat), wildcard_backtracking(text, &pat), "{text:?} ~ {pat:?}"); + } + } + // Patterns longer than one 64-bit word. + let long = format!("*{}*b?", "a".repeat(70)); + for text in [format!("x{}yb1", "a".repeat(70)), format!("{}b", "a".repeat(71)), "a".repeat(200)] { + assert_eq!(wildcard(&text, &long), wildcard_backtracking(&text, &long), "{text}"); + } +} + +#[test] +fn wildcard_with_many_distinct_literals_agrees_with_backtracking() { + // 600 distinct characters in the pattern: more than the masks kept, so the rest are rebuilt. + let chars: Vec = (0x4E00u32..0x4E00 + 600).filter_map(char::from_u32).collect(); + let word: String = chars.iter().collect(); + let pat = format!("*{word}?{}", chars[3]); + for text in [format!("x{word}yq{}", chars[3]), word.clone(), format!("{word}{word}"), chars.iter().rev().collect::()] { + assert_eq!(wildcard(&text, &pat), wildcard_backtracking(&text, &pat), "{} chars", text.chars().count()); + } + assert!(wildcard(&format!("{word}zz{}", chars[3]).replace("zz", "z"), &pat)); +} + +#[test] +fn wildcard_keeps_masks_of_frequent_literals_past_the_cache_limit() { + // 300 distinct characters come before the repeated `a`: past the cached masks, `a` (5,000 positions) + // still has to be read from a kept mask, or it is rebuilt for each of the 20,000 `a` in the text. + let rare: String = (0x4E00u32..0x4E00 + 300).filter_map(char::from_u32).collect(); + let pat = format!("*{rare}{}b", "a".repeat(5_000)); + let text = "a".repeat(20_000); + assert!(!wildcard(&text, &pat)); + assert!(wildcard(&format!("{rare}{}b", "a".repeat(5_000)), &pat)); + assert!(!wildcard(&format!("{rare}{}", "a".repeat(5_000)), &pat)); +} + +#[test] +fn wildcard_on_long_repetitive_text_is_linear() { + // The backtracking matcher compared about text × pattern characters here (10^8). + let text = "a".repeat(20_000); + let pat = format!("*{}b", "a".repeat(5_000)); + assert!(!wildcard(&text, &pat)); + assert!(wildcard(&format!("{text}b"), &pat)); +} + #[test] fn tsv() { assert_eq!(parse_tsv("a\tb\n1\t2\n"), vec![vec!["a", "b"], vec!["1", "2"]]);