From 68d7b4cf3b31516672f124e1a8bf65e8df150714 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Tue, 13 Oct 2026 21:20:48 +0200 Subject: [PATCH 1/3] Redact: mark text by caller-supplied regex with optional checksum (#1886) Co-Authored-By: Claude Sonnet 5.5 --- Cargo.lock | 33 ++++ Cargo.toml | 2 + ROADMAP.md | 1 + crates/automation/src/redact.rs | 30 +++- crates/automation/src/tools.rs | 4 +- crates/automation/tests/automation.rs | 28 ++++ crates/engine/src/lib.rs | 1 + crates/redact/Cargo.toml | 1 + crates/redact/src/lib.rs | 1 + crates/redact/src/user_regex.rs | 211 ++++++++++++++++++++++++++ parity/acrobat-features.toml | 12 ++ 11 files changed, 317 insertions(+), 7 deletions(-) create mode 100644 crates/redact/src/user_regex.rs diff --git a/Cargo.lock b/Cargo.lock index 91ed3c4904..268f3c745d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -148,6 +148,15 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + [[package]] name = "aligned-vec" version = "0.6.4" @@ -4284,6 +4293,7 @@ dependencies = [ "pdfcraft-model", "pdfcraft-testkit", "pdfcraft-xfa", + "regex", "thiserror 2.0.21", ] @@ -5136,6 +5146,29 @@ dependencies = [ "bitflags 2.13.2", ] +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + [[package]] name = "regex-syntax" version = "0.8.11" diff --git a/Cargo.toml b/Cargo.toml index 597dd316fa..cc0ceaa4d9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -62,6 +62,8 @@ weezl = "0.2" # foundation thiserror = "2" +# Linear-time matching (no backtracking) for caller-supplied redaction patterns. +regex = { version = "1", default-features = false, features = ["std", "unicode"] } serde = { version = "1", features = ["derive", "rc"] } serde_json = "1" log = "0.4" diff --git a/ROADMAP.md b/ROADMAP.md index 66b9677569..d1e1c6febf 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -157,6 +157,7 @@ Ranked; detail and the full milestone table in [docs/roadmap.md](docs/roadmap.md | 8 | 1.0 polish: performance, right-to-left, missing languages, keyboard-only (M14) | 55–95 | ## Progress log +- **2026-10-13 (M8, redact by regex):** `redact_mark` takes `regex` (caller expression, linear-time engine, length/size/nesting caps, empty-matching refused) with optional `validate` (luhn, mod97, iban). Headless only; issue #1886. - **2026-10-15 (M2, gap 12):** `cargo xtask text-oracle` scores reading order as well as word F1: the longest common subsequence of the two word sequences over the longer one. F1 is a bag of words and cannot see order at all -- measuring #2431, a change that moves text and nothing else, it called 26 of the 29 files whose output changed identical, where the order score separated 18 improvements from 6 regressions. Reports, does not gate; a disagreement with Poppler, not a verdict. Estimates unchanged. - **2026-10-15 (M2, #2305 follow-up):** WinAnsiEncoding 0xA0 and 0xAD and MacRomanEncoding 0xCA extract as the ordinary space and hyphen Annex D.2 names them, in the vendored hayro tables (patch 30) as well as `crates/fonts`, so search, copy and redaction see characters that were hidden behind U+00A0 and U+00AD. Across the pdf.js corpus the soft hyphens we extract fall 117 to 13 and the non-breaking spaces 30 to 26; the rest come from `/ToUnicode`. An explicit `/Differences` name keeps its glyph list value, as pdftotext does. Estimates unchanged. - **2026-10-13 (community, M3, #1569 review):** Pixel-only Wayland wheel, wheel-tilt and continuous input use standalone moved events, so Ctrl zoom does not remain latched after the modifier is released (#991); vendored winit patch (13), with a policy test. Estimates unchanged. diff --git a/crates/automation/src/redact.rs b/crates/automation/src/redact.rs index 1b6283c0ad..1367cb819c 100644 --- a/crates/automation/src/redact.rs +++ b/crates/automation/src/redact.rs @@ -2,8 +2,8 @@ //! (removing what they cover for good) or clear them. use pdfcraft_engine::{ - Edit, Hidden, NewAnnotation, REDACT_MAX_WORDS, REDACTION_CODE_SETS, RedactPattern, RedactionCodeSet, Shape, Style, find_pattern, rect_quad, - redact_word_list, + Edit, Hidden, NewAnnotation, REDACT_MAX_WORDS, REDACT_VALIDATORS, REDACTION_CODE_SETS, RedactPattern, RedactRegex, RedactValidator, + RedactionCodeSet, Shape, Style, find_pattern, rect_quad, redact_word_list, }; use serde_json::{Value, json}; @@ -49,16 +49,20 @@ impl Automation { a.get("find").is_some(), a.get("words").is_some(), a.get("pattern").is_some(), + a.get("regex").is_some(), a.opt_bool("whole_pages")?.unwrap_or(false), ]; if chosen.iter().filter(|c| **c).count() != 1 { - return Err(ToolError::InvalidArgs("pass exactly one of rect (with page), find, words, pattern or whole_pages: true".into())); + return Err(ToolError::InvalidArgs("pass exactly one of rect (with page), find, words, pattern, regex or whole_pages: true".into())); } let case_sensitive = a.opt_bool("case_sensitive")?.unwrap_or(false); let whole_words = a.opt_bool("whole_words")?.unwrap_or(false); if (case_sensitive || whole_words) && a.get("find").is_none() && a.get("words").is_none() { return Err(ToolError::InvalidArgs("case_sensitive and whole_words apply only to find or words".into())); } + if a.get("validate").is_some() && a.get("regex").is_none() { + return Err(ToolError::InvalidArgs("validate applies only to regex".into())); + } // Per searched word or phrase: how many marks it made. let mut matched_words: Vec<(String, usize)> = Vec::new(); if a.get("rect").is_some() { @@ -79,6 +83,19 @@ impl Automation { Some(p) => Some(RedactPattern::from_id(p).ok_or_else(|| ToolError::InvalidArgs(format!("unknown pattern {p:?}")))?), None => None, }; + let user_re = match a.opt_str("regex")? { + Some(src) => { + let validator = match a.opt_str("validate")? { + Some(v) => Some(RedactValidator::from_id(v).ok_or_else(|| { + let ids: Vec<_> = REDACT_VALIDATORS.iter().map(|v| v.id()).collect(); + ToolError::InvalidArgs(format!("unknown validate {v:?}; allowed: {}", ids.join(", "))) + })?), + None => None, + }; + Some(RedactRegex::new(src, validator).map_err(|e| ToolError::InvalidArgs(e.to_string()))?) + } + None => None, + }; let needles: Vec = match (a.opt_str("find")?, a.get("words")) { (Some(f), _) => vec![f.to_owned()], (None, Some(_)) => { @@ -98,9 +115,10 @@ impl Automation { let texts = self.page_texts(id, &pages)?; let info = self.doc(a)?.info.clone(); for (&p, text) in pages.iter().zip(&texts) { - let hits: Vec<(Option, _)> = match pattern { - Some(pat) => text.find_with(|chars| find_pattern(pat, chars)).into_iter().map(|h| (None, h)).collect(), - None => needles + let hits: Vec<(Option, _)> = match (&user_re, pattern) { + (Some(re), _) => text.find_with(|chars| re.find(chars)).into_iter().map(|h| (None, h)).collect(), + (None, Some(pat)) => text.find_with(|chars| find_pattern(pat, chars)).into_iter().map(|h| (None, h)).collect(), + (None, None) => needles .iter() .enumerate() .flat_map(|(i, w)| text.find_opts(w, case_sensitive, whole_words).into_iter().map(move |h| (Some(i), h))) diff --git a/crates/automation/src/tools.rs b/crates/automation/src/tools.rs index 5e89aef78a..84831b79a4 100644 --- a/crates/automation/src/tools.rs +++ b/crates/automation/src/tools.rs @@ -416,7 +416,7 @@ pub fn tools() -> Vec { t( "redact_mark", "Mark for redaction", - "Mark content for redaction (nothing is removed until redact_apply). One of: rect [x0, y0, x1, y1] (points from the top-left of the displayed page) with page; find (text, every match); words (a list of words or phrases, every match of each; the result's matched_words says how many marks each made); pattern (phone, email, credit-card, ssn, date: every match, Acrobat's Search & Redact patterns); whole_pages: true. find and words accept case_sensitive and whole_words (both default false). find, words, pattern and whole_pages work on pages (default all). overlay: text shown on the box once applied, or code_set (foia: U.S. FOIA (b)(1)(A)…(b)(9); privacy-act: U.S. Privacy Act (d)(5), (j)(1)…(k)(7)) with codes, shown as \"(b)(6), (b)(7)(C)\"; fill: box colour (default black). Undoable.", + "Mark content for redaction (nothing is removed until redact_apply). One of: rect [x0, y0, x1, y1] (points from the top-left of the displayed page) with page; find (text, every match); words (a list of words or phrases, every match of each; the result's matched_words says how many marks each made); pattern (phone, email, credit-card, ssn, date: every match, Acrobat's Search & Redact patterns); regex (your own regular expression, every match, on the page text with words and lines separated by one space; Rust regex syntax: linear-time, no look-around or back-references, case-sensitive unless (?i), at most 1024 bytes, matches of the empty string are refused) with optional validate (luhn, mod97, iban: keep a match only if its checksum holds); whole_pages: true. find and words accept case_sensitive and whole_words (both default false). find, words, pattern and whole_pages work on pages (default all). overlay: text shown on the box once applied, or code_set (foia: U.S. FOIA (b)(1)(A)…(b)(9); privacy-act: U.S. Privacy Act (d)(5), (j)(1)…(k)(7)) with codes, shown as \"(b)(6), (b)(7)(C)\"; fill: box colour (default black). Undoable.", ) .with(schema( json!({ @@ -428,6 +428,8 @@ pub fn tools() -> Vec { "case_sensitive": { "type": "boolean", "description": "Match letter case for find or words (default false)." }, "whole_words": { "type": "boolean", "description": "Match whole words for find or words (default false)." }, "pattern": { "type": "string", "enum": ["phone", "email", "credit-card", "ssn", "date"] }, + "regex": { "type": "string", "minLength": 1, "maxLength": 1024, "description": "Regular expression (Rust regex syntax); every match is marked." }, + "validate": { "type": "string", "enum": ["luhn", "mod97", "iban"], "description": "With regex: keep a match only if its checksum is valid (spaces and hyphens ignored)." }, "whole_pages": { "type": "boolean" }, "pages": pages("to search or mark (default: all)"), "overlay": { "type": "string" }, diff --git a/crates/automation/tests/automation.rs b/crates/automation/tests/automation.rs index 1e662d8f38..b1cb1920cb 100644 --- a/crates/automation/tests/automation.rs +++ b/crates/automation/tests/automation.rs @@ -3295,6 +3295,34 @@ fn redacting_through_tools() { assert!(matches!(a.call("redact_mark", &json!({ "doc": doc })), Err(ToolError::InvalidArgs(_)))); } +#[test] +fn redacting_with_a_caller_regex_through_tools() { + let dir = workdir("redact-regex"); + std::fs::write( + dir.join("ids.txt"), + "Ref 1 85 05 78 006 084 36 end\nRef 2 91 12 34 567 890 12 end\nCard 4539 1488 0343 6467 and 4539 1488 0343 6468", + ) + .unwrap(); + let mut a = auto(&dir); + let doc = ok(&mut a, "doc_create", json!({ "from": "text", "path": "ids.txt" }))["doc"].as_u64().unwrap(); + let bad = |a: &mut Automation, args: Value| matches!(a.call("redact_mark", &args), Err(ToolError::InvalidArgs(_))); + assert!(bad(&mut a, json!({ "doc": doc, "regex": "(unclosed" }))); + assert!(bad(&mut a, json!({ "doc": doc, "regex": "a*" }))); + assert!(bad(&mut a, json!({ "doc": doc, "regex": "x".repeat(2000) }))); + assert!(bad(&mut a, json!({ "doc": doc, "regex": "x", "pattern": "ssn" }))); + assert!(bad(&mut a, json!({ "doc": doc, "regex": "x", "validate": "crc" }))); + assert!(bad(&mut a, json!({ "doc": doc, "find": "x", "validate": "luhn" }))); + assert!(matches!(a.call("redact_mark", &json!({ "doc": doc, "regex": "Z{3}" })), Err(ToolError::Failed(_)))); + let r = ok(&mut a, "redact_mark", json!({ "doc": doc, "regex": r"\d \d\d \d\d \d\d \d{3} \d{3} \d\d" })); + assert_eq!(r["marked"].as_u64(), Some(2)); + let r = ok(&mut a, "redact_mark", json!({ "doc": doc, "regex": r"\d{4}( \d{4}){3}", "validate": "luhn" })); + assert_eq!(r["marked"].as_u64(), Some(1), "only the number with a valid checksum"); + ok(&mut a, "redact_apply", json!({ "doc": doc })); + let text = page_text(&mut a, doc)[0].clone(); + assert!(!text.contains("85 05") && !text.contains("91 12") && !text.contains("6467"), "{text}"); + assert!(text.contains("6468"), "{text}"); +} + #[test] fn redacting_with_codes_through_tools() { let dir = workdir("redact-codes"); diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 01f162ae7d..22743fdc65 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -152,6 +152,7 @@ pub use pdfcraft_redact::patterns::{ MAX_WORDS as REDACT_MAX_WORDS, PATTERNS as REDACT_PATTERNS, Pattern as RedactPattern, find as find_pattern, word_list as redact_word_list, }; pub use pdfcraft_redact::sanitize::{HIDDEN, Hidden}; +pub use pdfcraft_redact::user_regex::{UserRegex as RedactRegex, VALIDATORS as REDACT_VALIDATORS, Validator as RedactValidator}; pub use pdfcraft_sign as sign; pub use pdfcraft_sign::{SignOptions, SignatureInfo, Status as SignatureStatus, TrustStore}; pub use pdfcraft_xfdf::Format as DataFormat; diff --git a/crates/redact/Cargo.toml b/crates/redact/Cargo.toml index 49593855e2..f6f99995ac 100644 --- a/crates/redact/Cargo.toml +++ b/crates/redact/Cargo.toml @@ -17,6 +17,7 @@ pdfcraft-forms = { workspace = true } pdfcraft-edit = { workspace = true } pdfcraft-annot = { workspace = true } pdfcraft-xfa = { workspace = true } +regex = { workspace = true } thiserror = { workspace = true } [dev-dependencies] diff --git a/crates/redact/src/lib.rs b/crates/redact/src/lib.rs index 61231bd982..fc44d941d1 100644 --- a/crates/redact/src/lib.rs +++ b/crates/redact/src/lib.rs @@ -27,6 +27,7 @@ pub mod sanitize; mod tags; #[cfg(test)] mod tests; +pub mod user_regex; use interp::{Mode, Scope, process}; diff --git a/crates/redact/src/user_regex.rs b/crates/redact/src/user_regex.rs new file mode 100644 index 0000000000..043b51d0b0 --- /dev/null +++ b/crates/redact/src/user_regex.rs @@ -0,0 +1,211 @@ +//! Caller-supplied regular expressions for Search & Redact (`redact_mark regex`). +//! +//! The expression is untrusted input. It runs on the `regex` crate, which matches in time +//! linear in the text (no backtracking, no look-around or back-references), and is compiled +//! under hard limits on source length, nesting and compiled size, so a hostile expression is +//! refused with an error instead of exhausting time or memory. An optional [`Validator`] +//! keeps a match only when its checksum holds. + +use std::ops::Range; + +use regex::RegexBuilder; +use thiserror::Error; + +/// Longest expression accepted, in bytes. +pub const MAX_REGEX_LEN: usize = 1024; +/// Most matches returned for one page; the rest are ignored. +pub const MAX_MATCHES_PER_TEXT: usize = 10_000; +/// Cap on the compiled program and on the lazy-DFA cache, in bytes. +const SIZE_LIMIT: usize = 1 << 20; +/// Cap on parenthesis nesting. +const NEST_LIMIT: u32 = 32; + +#[derive(Debug, Error, PartialEq, Eq)] +pub enum RegexError { + #[error("regex is empty")] + Empty, + #[error("regex is {0} bytes long; at most {MAX_REGEX_LEN} are allowed")] + TooLong(usize), + #[error("invalid or too complex regex: {0}")] + Invalid(String), + #[error("regex matches the empty string, so it would mark nothing useful; require at least one character")] + MatchesEmpty, +} + +/// Checksum a match must pass to be kept. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Validator { + /// Luhn (ISO/IEC 7812): digits only, spaces and hyphens ignored. + Luhn, + /// ISO 7064 MOD 97-10: letters count as 10..35 and the number mod 97 must be 1. Letters and + /// digits only, spaces and hyphens ignored. + Mod97, + /// IBAN check (ISO 13616): the first four characters move to the end, then MOD 97-10. + Iban, +} + +/// Ids accepted for `validate`. +pub const VALIDATORS: [Validator; 3] = [Validator::Luhn, Validator::Mod97, Validator::Iban]; + +impl Validator { + pub fn id(self) -> &'static str { + match self { + Validator::Luhn => "luhn", + Validator::Mod97 => "mod97", + Validator::Iban => "iban", + } + } + + pub fn from_id(id: &str) -> Option { + VALIDATORS.into_iter().find(|v| v.id() == id) + } + + /// Whether `s` passes the checksum. + pub fn check(self, s: &str) -> bool { + match self { + Validator::Luhn => luhn(s), + Validator::Mod97 => mod97(s.chars()), + Validator::Iban => { + let kept: Vec = s.chars().filter(|c| *c != ' ' && *c != '-').collect(); + let (head, tail) = kept.split_at(kept.len().min(4)); + mod97(tail.iter().chain(head).copied()) + } + } + } +} + +fn luhn(s: &str) -> bool { + let (mut sum, mut digits, mut double) = (0u32, 0usize, false); + for c in s.chars().rev() { + if c == ' ' || c == '-' { + continue; + } + let Some(d) = c.to_digit(10).filter(|_| c.is_ascii_digit()) else { return false }; + let d = if double { if d * 2 > 9 { d * 2 - 9 } else { d * 2 } } else { d }; + sum = sum.saturating_add(d); + digits += 1; + double = !double; + } + digits >= 2 && sum % 10 == 0 +} + +fn mod97(s: impl Iterator) -> bool { + let (mut rem, mut n) = (0u32, 0usize); + for c in s { + if c == ' ' || c == '-' { + continue; + } + if !c.is_ascii_alphanumeric() { + return false; + } + // Digits stay one decimal digit, letters become two (A = 10 .. Z = 35). + let v = c.to_digit(36).unwrap_or(0); + rem = if v >= 10 { (rem * 100 + v) % 97 } else { (rem * 10 + v) % 97 }; + n += 1; + } + n >= 2 && rem == 1 +} + +/// A compiled caller expression with an optional checksum. +#[derive(Debug, Clone)] +pub struct UserRegex { + re: regex::Regex, + validator: Option, +} + +impl UserRegex { + pub fn new(source: &str, validator: Option) -> Result { + if source.is_empty() { + return Err(RegexError::Empty); + } + if source.len() > MAX_REGEX_LEN { + return Err(RegexError::TooLong(source.len())); + } + let re = RegexBuilder::new(source) + .size_limit(SIZE_LIMIT) + .dfa_size_limit(SIZE_LIMIT) + .nest_limit(NEST_LIMIT) + .build() + .map_err(|e| RegexError::Invalid(e.to_string()))?; + if re.is_match("") { + return Err(RegexError::MatchesEmpty); + } + Ok(Self { re, validator }) + } + + /// Character ranges of `text` the expression matches (non-overlapping, left to right). + pub fn find(&self, text: &[char]) -> Vec> { + let s: String = text.iter().collect(); + // Char index of every byte offset that starts a char (and of the end). + let mut byte_to_char = vec![usize::MAX; s.len().saturating_add(1)]; + for (ci, (bi, _)) in s.char_indices().enumerate() { + if let Some(slot) = byte_to_char.get_mut(bi) { + *slot = ci; + } + } + if let Some(slot) = byte_to_char.get_mut(s.len()) { + *slot = text.len(); + } + let mut out = Vec::new(); + for m in self.re.find_iter(&s) { + if out.len() >= MAX_MATCHES_PER_TEXT { + break; + } + if m.is_empty() || self.validator.is_some_and(|v| !v.check(m.as_str())) { + continue; + } + if let (Some(&a), Some(&b)) = (byte_to_char.get(m.start()), byte_to_char.get(m.end())) + && a < b + && b != usize::MAX + { + out.push(a..b); + } + } + out + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn chars(s: &str) -> Vec { + s.chars().collect() + } + + #[test] + fn finds_matches_as_char_ranges_with_multibyte_text() { + let r = UserRegex::new(r"\d{2}( \d{2}){2}", None).unwrap(); + let t = chars("Ré 85 05 78 et 91 12 34 fin"); + let found: Vec = r.find(&t).into_iter().map(|g| t[g].iter().collect()).collect(); + assert_eq!(found, ["85 05 78", "91 12 34"]); + } + + #[test] + fn validators_filter_matches() { + let t = chars("ok 4539 1488 0343 6467 bad 4539 1488 0343 6468"); + let r = UserRegex::new(r"\d{4}( \d{4}){3}", Some(Validator::Luhn)).unwrap(); + assert_eq!(r.find(&t).len(), 1); + let t = chars("GB82 WEST 1234 5698 7654 32 and GB83 WEST 1234 5698 7654 32"); + let r = UserRegex::new(r"GB\d\d WEST( \d{4}){3} \d\d", Some(Validator::Iban)).unwrap(); + assert_eq!(r.find(&t).len(), 1); + assert!(Validator::Mod97.check("98") && !Validator::Mod97.check("99")); + assert!(!Validator::Luhn.check("12a4") && !Validator::Mod97.check("é1") && !Validator::Luhn.check("0")); + } + + #[test] + fn hostile_expressions_are_refused_or_stay_linear() { + assert_eq!(UserRegex::new("", None).unwrap_err(), RegexError::Empty); + assert!(matches!(UserRegex::new(&"a".repeat(MAX_REGEX_LEN + 1), None), Err(RegexError::TooLong(_)))); + assert!(matches!(UserRegex::new("(unclosed", None), Err(RegexError::Invalid(_)))); + assert!(matches!(UserRegex::new(r"(a)\1", None), Err(RegexError::Invalid(_))), "no back-references"); + assert_eq!(UserRegex::new("a*", None).unwrap_err(), RegexError::MatchesEmpty); + assert!(matches!(UserRegex::new(r"\p{L}{1000}{1000}", None), Err(RegexError::Invalid(_))), "compiled size is capped"); + assert!(matches!(UserRegex::new(&format!("{}a{}", "(".repeat(100), ")".repeat(100)), None), Err(RegexError::Invalid(_)))); + // Classic catastrophic pattern finishes at once on a long non-matching input. + let r = UserRegex::new(r"(a+)+$", None).unwrap(); + let mut t = chars(&"a".repeat(50_000)); + t.push('!'); + assert!(r.find(&t).is_empty()); + } +} diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index 533013b547..159f580635 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -5611,6 +5611,18 @@ tools = ["redact_mark"] evidence = ["crates/redact/src/patterns.rs::phone_numbers", "crates/redact/src/patterns.rs::credit_cards_need_a_valid_checksum", "crates/automation/tests/automation.rs::redacting_through_tools"] notes = "phone (North American, E.164 with + country code, national with trunk 0), e-mail, credit card (Luhn), SSN, dates (English US forms); text drawn in a font whose built-in encoding we hold no table for (Symbol, ZapfDingbats, MacExpert) has no Unicode mapping, so a pattern search cannot see it -- it is skipped rather than mis-decoded" +[[feature]] +id = "protect.redact-regex" +title = "Redact by caller-supplied regular expression" +area = "G" +tier = "P2" +milestone = "M8" +status = "shipped" +commands = [] +tools = ["redact_mark"] +evidence = ["crates/redact/src/user_regex.rs::hostile_expressions_are_refused_or_stay_linear", "crates/redact/src/user_regex.rs::validators_filter_matches", "crates/automation/tests/automation.rs::redacting_with_a_caller_regex_through_tools"] +notes = "Headless only (redact_mark regex, optional validate luhn, mod97 or iban; issue #1886): no UI entry yet. Rust regex syntax, matched in linear time with no look-around or back-references; expression capped at 1024 bytes, compiled size and nesting capped, empty-matching expressions refused. Matches run on the page text as the pattern matchers see it (words and lines joined by one space), so a match cannot span a page break. Missing: a UI field, a mask for checksums that need more than the whole match (for example the French social security key), and a check against Acrobat's own custom-pattern syntax (not observed)." + [[feature]] id = "protect.redact-pattern-locales" title = "Pattern sets per locale" From 589c4af8e7138aa22eb51322353835d2ab73b772 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sun, 25 Oct 2026 10:19:17 +0100 Subject: [PATCH 2/3] Redact regex: fail when a page exceeds the match cap UserRegex::find now returns an error instead of silently stopping at 10,000 matches, so redact_mark cannot report success while later matches stay readable. Empty-match detection probes several short texts so patterns like \b are refused with the clear message. Schema notes that the 1024 limit is in bytes. Co-Authored-By: Claude Sonnet 5.5 --- crates/automation/src/redact.rs | 19 ++++++++++++- crates/automation/src/tools.rs | 4 +-- crates/redact/src/user_regex.rs | 48 ++++++++++++++++++++++++--------- 3 files changed, 56 insertions(+), 15 deletions(-) diff --git a/crates/automation/src/redact.rs b/crates/automation/src/redact.rs index 840020c570..808b6eabb7 100644 --- a/crates/automation/src/redact.rs +++ b/crates/automation/src/redact.rs @@ -126,7 +126,24 @@ impl Automation { let info = self.doc(a)?.info.clone(); for (&p, text) in pages.iter().zip(&texts) { let hits: Vec<(Option, _)> = match (&user_re, pattern) { - (Some(re), _) => text.find_with(|chars| re.find(chars)).into_iter().map(|h| (None, h)).collect(), + (Some(re), _) => { + // A page with too many matches fails the call: marking only the first ones + // would leave the rest readable after redact_apply. + let too_many = std::cell::Cell::new(false); + let found = text.find_with(|chars| { + re.find(chars).unwrap_or_else(|_| { + too_many.set(true); + Vec::new() + }) + }); + if too_many.get() { + return Err(ToolError::InvalidArgs(format!( + "regex has more than 10000 matches on page {}; nothing was marked. Narrow the expression so that every match can be marked", + p + 1 + ))); + } + found.into_iter().map(|h| (None, h)).collect() + } (None, Some(pat)) => text.find_with(|chars| find_pattern(pat, chars)).into_iter().map(|h| (None, h)).collect(), (None, None) => needles .iter() diff --git a/crates/automation/src/tools.rs b/crates/automation/src/tools.rs index 4c651c00dd..0a41cacc0a 100644 --- a/crates/automation/src/tools.rs +++ b/crates/automation/src/tools.rs @@ -486,7 +486,7 @@ pub fn tools() -> Vec { t( "redact_mark", "Mark for redaction", - "Mark content for redaction (nothing is removed until redact_apply). One of: rect [x0, y0, x1, y1] (points from the top-left of the displayed page) with page; find (text, every match); words (a list of words or phrases, every match of each; the result's matched_words says how many marks each made); pattern (phone, email, credit-card, ssn, date: every match, Acrobat's Search & Redact patterns); regex (your own regular expression, every match, on the page text with words and lines separated by one space; Rust regex syntax: linear-time, no look-around or back-references, case-sensitive unless (?i), at most 1024 bytes, matches of the empty string are refused) with optional validate (luhn, mod97, iban: keep a match only if its checksum holds); whole_pages: true. find and words accept case_sensitive and whole_words (both default false). Select one page with page or multiple with pages (default all); do not pass both. rect requires page. overlay: text shown on the box once applied, or code_set (foia: U.S. FOIA (b)(1)(A)…(b)(9); privacy-act: U.S. Privacy Act (d)(5), (j)(1)…(k)(7)) with codes, shown as \"(b)(6), (b)(7)(C)\"; fill: box colour (default black). Undoable.", + "Mark content for redaction (nothing is removed until redact_apply). One of: rect [x0, y0, x1, y1] (points from the top-left of the displayed page) with page; find (text, every match); words (a list of words or phrases, every match of each; the result's matched_words says how many marks each made); pattern (phone, email, credit-card, ssn, date: every match, Acrobat's Search & Redact patterns); regex (your own regular expression, every match, on the page text with words and lines separated by one space; Rust regex syntax: linear-time, no look-around or back-references, case-sensitive unless (?i), at most 1024 bytes, more than 10000 matches on one page fail the call, matches of the empty string are refused) with optional validate (luhn, mod97, iban: keep a match only if its checksum holds); whole_pages: true. find and words accept case_sensitive and whole_words (both default false). Select one page with page or multiple with pages (default all); do not pass both. rect requires page. overlay: text shown on the box once applied, or code_set (foia: U.S. FOIA (b)(1)(A)…(b)(9); privacy-act: U.S. Privacy Act (d)(5), (j)(1)…(k)(7)) with codes, shown as \"(b)(6), (b)(7)(C)\"; fill: box colour (default black). Undoable.", ) .with(schema( json!({ @@ -498,7 +498,7 @@ pub fn tools() -> Vec { "case_sensitive": { "type": "boolean", "description": "Match letter case for find or words (default false)." }, "whole_words": { "type": "boolean", "description": "Match whole words for find or words (default false)." }, "pattern": { "type": "string", "enum": ["phone", "email", "credit-card", "ssn", "date"] }, - "regex": { "type": "string", "minLength": 1, "maxLength": 1024, "description": "Regular expression (Rust regex syntax); every match is marked." }, + "regex": { "type": "string", "minLength": 1, "maxLength": 1024, "description": "Regular expression (Rust regex syntax), at most 1024 bytes (maxLength counts characters, the limit is bytes); every match is marked." }, "validate": { "type": "string", "enum": ["luhn", "mod97", "iban"], "description": "With regex: keep a match only if its checksum is valid (spaces and hyphens ignored)." }, "whole_pages": { "type": "boolean" }, "pages": pages("to search or mark (default: all); mutually exclusive with page, and not valid with rect"), diff --git a/crates/redact/src/user_regex.rs b/crates/redact/src/user_regex.rs index 043b51d0b0..9d1c1b0004 100644 --- a/crates/redact/src/user_regex.rs +++ b/crates/redact/src/user_regex.rs @@ -13,7 +13,8 @@ use thiserror::Error; /// Longest expression accepted, in bytes. pub const MAX_REGEX_LEN: usize = 1024; -/// Most matches returned for one page; the rest are ignored. +/// Most matches accepted for one page; one more makes [`UserRegex::find`] fail, so a page is never +/// marked only in part. pub const MAX_MATCHES_PER_TEXT: usize = 10_000; /// Cap on the compiled program and on the lazy-DFA cache, in bytes. const SIZE_LIMIT: usize = 1 << 20; @@ -30,6 +31,8 @@ pub enum RegexError { Invalid(String), #[error("regex matches the empty string, so it would mark nothing useful; require at least one character")] MatchesEmpty, + #[error("regex has more than {MAX_MATCHES_PER_TEXT} matches on one page; narrow it so that every match can be marked")] + TooManyMatches, } /// Checksum a match must pass to be kept. @@ -106,6 +109,12 @@ fn mod97(s: impl Iterator) -> bool { n >= 2 && rem == 1 } +/// Whether `re` can match the empty string, which `is_match("")` alone misses for assertions such +/// as `\b`. Tries a few short texts with letters, digits, space, newline and a non-ASCII letter. +fn can_match_empty(re: ®ex::Regex) -> bool { + ["", " ", "a", "1", "a a", "a\n", "\n", "é", "aa 11"].iter().any(|p| re.find_iter(p).any(|m| m.is_empty())) +} + /// A compiled caller expression with an optional checksum. #[derive(Debug, Clone)] pub struct UserRegex { @@ -127,14 +136,17 @@ impl UserRegex { .nest_limit(NEST_LIMIT) .build() .map_err(|e| RegexError::Invalid(e.to_string()))?; - if re.is_match("") { + if can_match_empty(&re) { return Err(RegexError::MatchesEmpty); } Ok(Self { re, validator }) } /// Character ranges of `text` the expression matches (non-overlapping, left to right). - pub fn find(&self, text: &[char]) -> Vec> { + /// + /// Fails with [`RegexError::TooManyMatches`] when more than [`MAX_MATCHES_PER_TEXT`] matches + /// are kept, instead of returning a partial list the caller would take for complete. + pub fn find(&self, text: &[char]) -> Result>, RegexError> { let s: String = text.iter().collect(); // Char index of every byte offset that starts a char (and of the end). let mut byte_to_char = vec![usize::MAX; s.len().saturating_add(1)]; @@ -148,9 +160,6 @@ impl UserRegex { } let mut out = Vec::new(); for m in self.re.find_iter(&s) { - if out.len() >= MAX_MATCHES_PER_TEXT { - break; - } if m.is_empty() || self.validator.is_some_and(|v| !v.check(m.as_str())) { continue; } @@ -158,10 +167,13 @@ impl UserRegex { && a < b && b != usize::MAX { + if out.len() >= MAX_MATCHES_PER_TEXT { + return Err(RegexError::TooManyMatches); + } out.push(a..b); } } - out + Ok(out) } } @@ -177,7 +189,7 @@ mod tests { fn finds_matches_as_char_ranges_with_multibyte_text() { let r = UserRegex::new(r"\d{2}( \d{2}){2}", None).unwrap(); let t = chars("Ré 85 05 78 et 91 12 34 fin"); - let found: Vec = r.find(&t).into_iter().map(|g| t[g].iter().collect()).collect(); + let found: Vec = r.find(&t).unwrap().into_iter().map(|g| t[g].iter().collect()).collect(); assert_eq!(found, ["85 05 78", "91 12 34"]); } @@ -185,10 +197,10 @@ mod tests { fn validators_filter_matches() { let t = chars("ok 4539 1488 0343 6467 bad 4539 1488 0343 6468"); let r = UserRegex::new(r"\d{4}( \d{4}){3}", Some(Validator::Luhn)).unwrap(); - assert_eq!(r.find(&t).len(), 1); + assert_eq!(r.find(&t).unwrap().len(), 1); let t = chars("GB82 WEST 1234 5698 7654 32 and GB83 WEST 1234 5698 7654 32"); let r = UserRegex::new(r"GB\d\d WEST( \d{4}){3} \d\d", Some(Validator::Iban)).unwrap(); - assert_eq!(r.find(&t).len(), 1); + assert_eq!(r.find(&t).unwrap().len(), 1); assert!(Validator::Mod97.check("98") && !Validator::Mod97.check("99")); assert!(!Validator::Luhn.check("12a4") && !Validator::Mod97.check("é1") && !Validator::Luhn.check("0")); } @@ -199,13 +211,25 @@ mod tests { assert!(matches!(UserRegex::new(&"a".repeat(MAX_REGEX_LEN + 1), None), Err(RegexError::TooLong(_)))); assert!(matches!(UserRegex::new("(unclosed", None), Err(RegexError::Invalid(_)))); assert!(matches!(UserRegex::new(r"(a)\1", None), Err(RegexError::Invalid(_))), "no back-references"); - assert_eq!(UserRegex::new("a*", None).unwrap_err(), RegexError::MatchesEmpty); + for empty in ["a*", r"\b", "(?m)^", r"\B", "(?m)$", r"\d*|x"] { + assert_eq!(UserRegex::new(empty, None).unwrap_err(), RegexError::MatchesEmpty, "{empty}"); + } + assert!(UserRegex::new(r"\bfoo\b", None).is_ok()); assert!(matches!(UserRegex::new(r"\p{L}{1000}{1000}", None), Err(RegexError::Invalid(_))), "compiled size is capped"); assert!(matches!(UserRegex::new(&format!("{}a{}", "(".repeat(100), ")".repeat(100)), None), Err(RegexError::Invalid(_)))); // Classic catastrophic pattern finishes at once on a long non-matching input. let r = UserRegex::new(r"(a+)+$", None).unwrap(); let mut t = chars(&"a".repeat(50_000)); t.push('!'); - assert!(r.find(&t).is_empty()); + assert!(r.find(&t).unwrap().is_empty()); + } + + #[test] + fn more_than_the_cap_of_matches_is_an_error_not_a_partial_list() { + let r = UserRegex::new("x", None).unwrap(); + let at_cap = chars(&"x ".repeat(MAX_MATCHES_PER_TEXT)); + assert_eq!(r.find(&at_cap).unwrap().len(), MAX_MATCHES_PER_TEXT); + let over = chars(&"x ".repeat(MAX_MATCHES_PER_TEXT + 1)); + assert_eq!(r.find(&over).unwrap_err(), RegexError::TooManyMatches); } } From cc4bd4914caa11f6db73d029edd7ff7dcafcf57d Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sat, 7 Nov 2026 16:40:37 +0100 Subject: [PATCH 3/3] regex: enable perf-dfa so the dfa_size_limit on user patterns applies Without the feature the lazy-DFA cache does not exist and the limit set in UserRegex::new does nothing. Matching stays linear-time. Co-Authored-By: Claude Sonnet 5.5 --- Cargo.toml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 29d28051d8..a684ca9857 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -80,8 +80,9 @@ hayro-jbig2 = "0.3" # foundation thiserror = "2" -# Linear-time matching (no backtracking) for caller-supplied redaction patterns. -regex = { version = "1", default-features = false, features = ["std", "unicode"] } +# Linear-time matching (no backtracking) for caller-supplied redaction patterns. `perf-dfa` gives +# the lazy-DFA cache that `RegexBuilder::dfa_size_limit` bounds (the limit is inert without it). +regex = { version = "1", default-features = false, features = ["std", "unicode", "perf-dfa"] } serde = { version = "1", features = ["derive", "rc"] } serde_json = "1" log = "0.4"