mirror of
https://github.com/storytold/pdfcraft.git
synced 2026-11-13 06:12:19 +00:00
Redact: mark text by caller-supplied regex with optional checksum (#1886)
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
vangel76andClaude Sonnet 5.5 committed
1 parent
99941daa5a
commit
68d7b4cf3b
11 files changed
+317
-7
No files matched your search
Generated
+33
@@ -148,6 +148,15 @@ dependencies = [
|
||||
"zerocopy",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aho-corasick"
|
||||
version = "1.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aligned-vec"
|
||||
version = "0.6.4"
|
||||
@@ -4284,6 +4293,7 @@ dependencies = [
|
||||
"pdfcraft-model",
|
||||
"pdfcraft-testkit",
|
||||
"pdfcraft-xfa",
|
||||
"regex",
|
||||
"thiserror 2.0.21",
|
||||
]
|
||||
|
||||
@@ -5136,6 +5146,29 @@ dependencies = [
|
||||
"bitflags 2.13.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex"
|
||||
version = "1.13.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d"
|
||||
dependencies = [
|
||||
"aho-corasick",
|
||||
"memchr",
|
||||
"regex-automata",
|
||||
"regex-syntax",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex-automata"
|
||||
version = "0.4.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2"
|
||||
dependencies = [
|
||||
"aho-corasick",
|
||||
"memchr",
|
||||
"regex-syntax",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex-syntax"
|
||||
version = "0.8.11"
|
||||
|
||||
@@ -62,6 +62,8 @@ weezl = "0.2"
|
||||
|
||||
# foundation
|
||||
thiserror = "2"
|
||||
# Linear-time matching (no backtracking) for caller-supplied redaction patterns.
|
||||
regex = { version = "1", default-features = false, features = ["std", "unicode"] }
|
||||
serde = { version = "1", features = ["derive", "rc"] }
|
||||
serde_json = "1"
|
||||
log = "0.4"
|
||||
|
||||
@@ -157,6 +157,7 @@ Ranked; detail and the full milestone table in [docs/roadmap.md](docs/roadmap.md
|
||||
| 8 | 1.0 polish: performance, right-to-left, missing languages, keyboard-only (M14) | 55–95 |
|
||||
|
||||
## Progress log
|
||||
- **2026-10-13 (M8, redact by regex):** `redact_mark` takes `regex` (caller expression, linear-time engine, length/size/nesting caps, empty-matching refused) with optional `validate` (luhn, mod97, iban). Headless only; issue #1886.
|
||||
- **2026-10-15 (M2, gap 12):** `cargo xtask text-oracle` scores reading order as well as word F1: the longest common subsequence of the two word sequences over the longer one. F1 is a bag of words and cannot see order at all -- measuring #2431, a change that moves text and nothing else, it called 26 of the 29 files whose output changed identical, where the order score separated 18 improvements from 6 regressions. Reports, does not gate; a disagreement with Poppler, not a verdict. Estimates unchanged.
|
||||
- **2026-10-15 (M2, #2305 follow-up):** WinAnsiEncoding 0xA0 and 0xAD and MacRomanEncoding 0xCA extract as the ordinary space and hyphen Annex D.2 names them, in the vendored hayro tables (patch 30) as well as `crates/fonts`, so search, copy and redaction see characters that were hidden behind U+00A0 and U+00AD. Across the pdf.js corpus the soft hyphens we extract fall 117 to 13 and the non-breaking spaces 30 to 26; the rest come from `/ToUnicode`. An explicit `/Differences` name keeps its glyph list value, as pdftotext does. Estimates unchanged.
|
||||
- **2026-10-13 (community, M3, #1569 review):** Pixel-only Wayland wheel, wheel-tilt and continuous input use standalone moved events, so Ctrl zoom does not remain latched after the modifier is released (#991); vendored winit patch (13), with a policy test. Estimates unchanged.
|
||||
|
||||
@@ -2,8 +2,8 @@
|
||||
//! (removing what they cover for good) or clear them.
|
||||
|
||||
use pdfcraft_engine::{
|
||||
Edit, Hidden, NewAnnotation, REDACT_MAX_WORDS, REDACTION_CODE_SETS, RedactPattern, RedactionCodeSet, Shape, Style, find_pattern, rect_quad,
|
||||
redact_word_list,
|
||||
Edit, Hidden, NewAnnotation, REDACT_MAX_WORDS, REDACT_VALIDATORS, REDACTION_CODE_SETS, RedactPattern, RedactRegex, RedactValidator,
|
||||
RedactionCodeSet, Shape, Style, find_pattern, rect_quad, redact_word_list,
|
||||
};
|
||||
use serde_json::{Value, json};
|
||||
|
||||
@@ -49,16 +49,20 @@ impl Automation {
|
||||
a.get("find").is_some(),
|
||||
a.get("words").is_some(),
|
||||
a.get("pattern").is_some(),
|
||||
a.get("regex").is_some(),
|
||||
a.opt_bool("whole_pages")?.unwrap_or(false),
|
||||
];
|
||||
if chosen.iter().filter(|c| **c).count() != 1 {
|
||||
return Err(ToolError::InvalidArgs("pass exactly one of rect (with page), find, words, pattern or whole_pages: true".into()));
|
||||
return Err(ToolError::InvalidArgs("pass exactly one of rect (with page), find, words, pattern, regex or whole_pages: true".into()));
|
||||
}
|
||||
let case_sensitive = a.opt_bool("case_sensitive")?.unwrap_or(false);
|
||||
let whole_words = a.opt_bool("whole_words")?.unwrap_or(false);
|
||||
if (case_sensitive || whole_words) && a.get("find").is_none() && a.get("words").is_none() {
|
||||
return Err(ToolError::InvalidArgs("case_sensitive and whole_words apply only to find or words".into()));
|
||||
}
|
||||
if a.get("validate").is_some() && a.get("regex").is_none() {
|
||||
return Err(ToolError::InvalidArgs("validate applies only to regex".into()));
|
||||
}
|
||||
// Per searched word or phrase: how many marks it made.
|
||||
let mut matched_words: Vec<(String, usize)> = Vec::new();
|
||||
if a.get("rect").is_some() {
|
||||
@@ -79,6 +83,19 @@ impl Automation {
|
||||
Some(p) => Some(RedactPattern::from_id(p).ok_or_else(|| ToolError::InvalidArgs(format!("unknown pattern {p:?}")))?),
|
||||
None => None,
|
||||
};
|
||||
let user_re = match a.opt_str("regex")? {
|
||||
Some(src) => {
|
||||
let validator = match a.opt_str("validate")? {
|
||||
Some(v) => Some(RedactValidator::from_id(v).ok_or_else(|| {
|
||||
let ids: Vec<_> = REDACT_VALIDATORS.iter().map(|v| v.id()).collect();
|
||||
ToolError::InvalidArgs(format!("unknown validate {v:?}; allowed: {}", ids.join(", ")))
|
||||
})?),
|
||||
None => None,
|
||||
};
|
||||
Some(RedactRegex::new(src, validator).map_err(|e| ToolError::InvalidArgs(e.to_string()))?)
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
let needles: Vec<String> = match (a.opt_str("find")?, a.get("words")) {
|
||||
(Some(f), _) => vec![f.to_owned()],
|
||||
(None, Some(_)) => {
|
||||
@@ -98,9 +115,10 @@ impl Automation {
|
||||
let texts = self.page_texts(id, &pages)?;
|
||||
let info = self.doc(a)?.info.clone();
|
||||
for (&p, text) in pages.iter().zip(&texts) {
|
||||
let hits: Vec<(Option<usize>, _)> = match pattern {
|
||||
Some(pat) => text.find_with(|chars| find_pattern(pat, chars)).into_iter().map(|h| (None, h)).collect(),
|
||||
None => needles
|
||||
let hits: Vec<(Option<usize>, _)> = match (&user_re, pattern) {
|
||||
(Some(re), _) => text.find_with(|chars| re.find(chars)).into_iter().map(|h| (None, h)).collect(),
|
||||
(None, Some(pat)) => text.find_with(|chars| find_pattern(pat, chars)).into_iter().map(|h| (None, h)).collect(),
|
||||
(None, None) => needles
|
||||
.iter()
|
||||
.enumerate()
|
||||
.flat_map(|(i, w)| text.find_opts(w, case_sensitive, whole_words).into_iter().map(move |h| (Some(i), h)))
|
||||
|
||||
@@ -416,7 +416,7 @@ pub fn tools() -> Vec<ToolDef> {
|
||||
t(
|
||||
"redact_mark",
|
||||
"Mark for redaction",
|
||||
"Mark content for redaction (nothing is removed until redact_apply). One of: rect [x0, y0, x1, y1] (points from the top-left of the displayed page) with page; find (text, every match); words (a list of words or phrases, every match of each; the result's matched_words says how many marks each made); pattern (phone, email, credit-card, ssn, date: every match, Acrobat's Search & Redact patterns); whole_pages: true. find and words accept case_sensitive and whole_words (both default false). find, words, pattern and whole_pages work on pages (default all). overlay: text shown on the box once applied, or code_set (foia: U.S. FOIA (b)(1)(A)…(b)(9); privacy-act: U.S. Privacy Act (d)(5), (j)(1)…(k)(7)) with codes, shown as \"(b)(6), (b)(7)(C)\"; fill: box colour (default black). Undoable.",
|
||||
"Mark content for redaction (nothing is removed until redact_apply). One of: rect [x0, y0, x1, y1] (points from the top-left of the displayed page) with page; find (text, every match); words (a list of words or phrases, every match of each; the result's matched_words says how many marks each made); pattern (phone, email, credit-card, ssn, date: every match, Acrobat's Search & Redact patterns); regex (your own regular expression, every match, on the page text with words and lines separated by one space; Rust regex syntax: linear-time, no look-around or back-references, case-sensitive unless (?i), at most 1024 bytes, matches of the empty string are refused) with optional validate (luhn, mod97, iban: keep a match only if its checksum holds); whole_pages: true. find and words accept case_sensitive and whole_words (both default false). find, words, pattern and whole_pages work on pages (default all). overlay: text shown on the box once applied, or code_set (foia: U.S. FOIA (b)(1)(A)…(b)(9); privacy-act: U.S. Privacy Act (d)(5), (j)(1)…(k)(7)) with codes, shown as \"(b)(6), (b)(7)(C)\"; fill: box colour (default black). Undoable.",
|
||||
)
|
||||
.with(schema(
|
||||
json!({
|
||||
@@ -428,6 +428,8 @@ pub fn tools() -> Vec<ToolDef> {
|
||||
"case_sensitive": { "type": "boolean", "description": "Match letter case for find or words (default false)." },
|
||||
"whole_words": { "type": "boolean", "description": "Match whole words for find or words (default false)." },
|
||||
"pattern": { "type": "string", "enum": ["phone", "email", "credit-card", "ssn", "date"] },
|
||||
"regex": { "type": "string", "minLength": 1, "maxLength": 1024, "description": "Regular expression (Rust regex syntax); every match is marked." },
|
||||
"validate": { "type": "string", "enum": ["luhn", "mod97", "iban"], "description": "With regex: keep a match only if its checksum is valid (spaces and hyphens ignored)." },
|
||||
"whole_pages": { "type": "boolean" },
|
||||
"pages": pages("to search or mark (default: all)"),
|
||||
"overlay": { "type": "string" },
|
||||
|
||||
@@ -3295,6 +3295,34 @@ fn redacting_through_tools() {
|
||||
assert!(matches!(a.call("redact_mark", &json!({ "doc": doc })), Err(ToolError::InvalidArgs(_))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn redacting_with_a_caller_regex_through_tools() {
|
||||
let dir = workdir("redact-regex");
|
||||
std::fs::write(
|
||||
dir.join("ids.txt"),
|
||||
"Ref 1 85 05 78 006 084 36 end\nRef 2 91 12 34 567 890 12 end\nCard 4539 1488 0343 6467 and 4539 1488 0343 6468",
|
||||
)
|
||||
.unwrap();
|
||||
let mut a = auto(&dir);
|
||||
let doc = ok(&mut a, "doc_create", json!({ "from": "text", "path": "ids.txt" }))["doc"].as_u64().unwrap();
|
||||
let bad = |a: &mut Automation, args: Value| matches!(a.call("redact_mark", &args), Err(ToolError::InvalidArgs(_)));
|
||||
assert!(bad(&mut a, json!({ "doc": doc, "regex": "(unclosed" })));
|
||||
assert!(bad(&mut a, json!({ "doc": doc, "regex": "a*" })));
|
||||
assert!(bad(&mut a, json!({ "doc": doc, "regex": "x".repeat(2000) })));
|
||||
assert!(bad(&mut a, json!({ "doc": doc, "regex": "x", "pattern": "ssn" })));
|
||||
assert!(bad(&mut a, json!({ "doc": doc, "regex": "x", "validate": "crc" })));
|
||||
assert!(bad(&mut a, json!({ "doc": doc, "find": "x", "validate": "luhn" })));
|
||||
assert!(matches!(a.call("redact_mark", &json!({ "doc": doc, "regex": "Z{3}" })), Err(ToolError::Failed(_))));
|
||||
let r = ok(&mut a, "redact_mark", json!({ "doc": doc, "regex": r"\d \d\d \d\d \d\d \d{3} \d{3} \d\d" }));
|
||||
assert_eq!(r["marked"].as_u64(), Some(2));
|
||||
let r = ok(&mut a, "redact_mark", json!({ "doc": doc, "regex": r"\d{4}( \d{4}){3}", "validate": "luhn" }));
|
||||
assert_eq!(r["marked"].as_u64(), Some(1), "only the number with a valid checksum");
|
||||
ok(&mut a, "redact_apply", json!({ "doc": doc }));
|
||||
let text = page_text(&mut a, doc)[0].clone();
|
||||
assert!(!text.contains("85 05") && !text.contains("91 12") && !text.contains("6467"), "{text}");
|
||||
assert!(text.contains("6468"), "{text}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn redacting_with_codes_through_tools() {
|
||||
let dir = workdir("redact-codes");
|
||||
|
||||
@@ -152,6 +152,7 @@ pub use pdfcraft_redact::patterns::{
|
||||
MAX_WORDS as REDACT_MAX_WORDS, PATTERNS as REDACT_PATTERNS, Pattern as RedactPattern, find as find_pattern, word_list as redact_word_list,
|
||||
};
|
||||
pub use pdfcraft_redact::sanitize::{HIDDEN, Hidden};
|
||||
pub use pdfcraft_redact::user_regex::{UserRegex as RedactRegex, VALIDATORS as REDACT_VALIDATORS, Validator as RedactValidator};
|
||||
pub use pdfcraft_sign as sign;
|
||||
pub use pdfcraft_sign::{SignOptions, SignatureInfo, Status as SignatureStatus, TrustStore};
|
||||
pub use pdfcraft_xfdf::Format as DataFormat;
|
||||
|
||||
@@ -17,6 +17,7 @@ pdfcraft-forms = { workspace = true }
|
||||
pdfcraft-edit = { workspace = true }
|
||||
pdfcraft-annot = { workspace = true }
|
||||
pdfcraft-xfa = { workspace = true }
|
||||
regex = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
|
||||
@@ -27,6 +27,7 @@ pub mod sanitize;
|
||||
mod tags;
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
pub mod user_regex;
|
||||
|
||||
use interp::{Mode, Scope, process};
|
||||
|
||||
|
||||
@@ -0,0 +1,211 @@
|
||||
//! Caller-supplied regular expressions for Search & Redact (`redact_mark regex`).
|
||||
//!
|
||||
//! The expression is untrusted input. It runs on the `regex` crate, which matches in time
|
||||
//! linear in the text (no backtracking, no look-around or back-references), and is compiled
|
||||
//! under hard limits on source length, nesting and compiled size, so a hostile expression is
|
||||
//! refused with an error instead of exhausting time or memory. An optional [`Validator`]
|
||||
//! keeps a match only when its checksum holds.
|
||||
|
||||
use std::ops::Range;
|
||||
|
||||
use regex::RegexBuilder;
|
||||
use thiserror::Error;
|
||||
|
||||
/// Longest expression accepted, in bytes.
|
||||
pub const MAX_REGEX_LEN: usize = 1024;
|
||||
/// Most matches returned for one page; the rest are ignored.
|
||||
pub const MAX_MATCHES_PER_TEXT: usize = 10_000;
|
||||
/// Cap on the compiled program and on the lazy-DFA cache, in bytes.
|
||||
const SIZE_LIMIT: usize = 1 << 20;
|
||||
/// Cap on parenthesis nesting.
|
||||
const NEST_LIMIT: u32 = 32;
|
||||
|
||||
#[derive(Debug, Error, PartialEq, Eq)]
|
||||
pub enum RegexError {
|
||||
#[error("regex is empty")]
|
||||
Empty,
|
||||
#[error("regex is {0} bytes long; at most {MAX_REGEX_LEN} are allowed")]
|
||||
TooLong(usize),
|
||||
#[error("invalid or too complex regex: {0}")]
|
||||
Invalid(String),
|
||||
#[error("regex matches the empty string, so it would mark nothing useful; require at least one character")]
|
||||
MatchesEmpty,
|
||||
}
|
||||
|
||||
/// Checksum a match must pass to be kept.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum Validator {
|
||||
/// Luhn (ISO/IEC 7812): digits only, spaces and hyphens ignored.
|
||||
Luhn,
|
||||
/// ISO 7064 MOD 97-10: letters count as 10..35 and the number mod 97 must be 1. Letters and
|
||||
/// digits only, spaces and hyphens ignored.
|
||||
Mod97,
|
||||
/// IBAN check (ISO 13616): the first four characters move to the end, then MOD 97-10.
|
||||
Iban,
|
||||
}
|
||||
|
||||
/// Ids accepted for `validate`.
|
||||
pub const VALIDATORS: [Validator; 3] = [Validator::Luhn, Validator::Mod97, Validator::Iban];
|
||||
|
||||
impl Validator {
|
||||
pub fn id(self) -> &'static str {
|
||||
match self {
|
||||
Validator::Luhn => "luhn",
|
||||
Validator::Mod97 => "mod97",
|
||||
Validator::Iban => "iban",
|
||||
}
|
||||
}
|
||||
|
||||
pub fn from_id(id: &str) -> Option<Self> {
|
||||
VALIDATORS.into_iter().find(|v| v.id() == id)
|
||||
}
|
||||
|
||||
/// Whether `s` passes the checksum.
|
||||
pub fn check(self, s: &str) -> bool {
|
||||
match self {
|
||||
Validator::Luhn => luhn(s),
|
||||
Validator::Mod97 => mod97(s.chars()),
|
||||
Validator::Iban => {
|
||||
let kept: Vec<char> = s.chars().filter(|c| *c != ' ' && *c != '-').collect();
|
||||
let (head, tail) = kept.split_at(kept.len().min(4));
|
||||
mod97(tail.iter().chain(head).copied())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn luhn(s: &str) -> bool {
|
||||
let (mut sum, mut digits, mut double) = (0u32, 0usize, false);
|
||||
for c in s.chars().rev() {
|
||||
if c == ' ' || c == '-' {
|
||||
continue;
|
||||
}
|
||||
let Some(d) = c.to_digit(10).filter(|_| c.is_ascii_digit()) else { return false };
|
||||
let d = if double { if d * 2 > 9 { d * 2 - 9 } else { d * 2 } } else { d };
|
||||
sum = sum.saturating_add(d);
|
||||
digits += 1;
|
||||
double = !double;
|
||||
}
|
||||
digits >= 2 && sum % 10 == 0
|
||||
}
|
||||
|
||||
fn mod97(s: impl Iterator<Item = char>) -> bool {
|
||||
let (mut rem, mut n) = (0u32, 0usize);
|
||||
for c in s {
|
||||
if c == ' ' || c == '-' {
|
||||
continue;
|
||||
}
|
||||
if !c.is_ascii_alphanumeric() {
|
||||
return false;
|
||||
}
|
||||
// Digits stay one decimal digit, letters become two (A = 10 .. Z = 35).
|
||||
let v = c.to_digit(36).unwrap_or(0);
|
||||
rem = if v >= 10 { (rem * 100 + v) % 97 } else { (rem * 10 + v) % 97 };
|
||||
n += 1;
|
||||
}
|
||||
n >= 2 && rem == 1
|
||||
}
|
||||
|
||||
/// A compiled caller expression with an optional checksum.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct UserRegex {
|
||||
re: regex::Regex,
|
||||
validator: Option<Validator>,
|
||||
}
|
||||
|
||||
impl UserRegex {
|
||||
pub fn new(source: &str, validator: Option<Validator>) -> Result<Self, RegexError> {
|
||||
if source.is_empty() {
|
||||
return Err(RegexError::Empty);
|
||||
}
|
||||
if source.len() > MAX_REGEX_LEN {
|
||||
return Err(RegexError::TooLong(source.len()));
|
||||
}
|
||||
let re = RegexBuilder::new(source)
|
||||
.size_limit(SIZE_LIMIT)
|
||||
.dfa_size_limit(SIZE_LIMIT)
|
||||
.nest_limit(NEST_LIMIT)
|
||||
.build()
|
||||
.map_err(|e| RegexError::Invalid(e.to_string()))?;
|
||||
if re.is_match("") {
|
||||
return Err(RegexError::MatchesEmpty);
|
||||
}
|
||||
Ok(Self { re, validator })
|
||||
}
|
||||
|
||||
/// Character ranges of `text` the expression matches (non-overlapping, left to right).
|
||||
pub fn find(&self, text: &[char]) -> Vec<Range<usize>> {
|
||||
let s: String = text.iter().collect();
|
||||
// Char index of every byte offset that starts a char (and of the end).
|
||||
let mut byte_to_char = vec![usize::MAX; s.len().saturating_add(1)];
|
||||
for (ci, (bi, _)) in s.char_indices().enumerate() {
|
||||
if let Some(slot) = byte_to_char.get_mut(bi) {
|
||||
*slot = ci;
|
||||
}
|
||||
}
|
||||
if let Some(slot) = byte_to_char.get_mut(s.len()) {
|
||||
*slot = text.len();
|
||||
}
|
||||
let mut out = Vec::new();
|
||||
for m in self.re.find_iter(&s) {
|
||||
if out.len() >= MAX_MATCHES_PER_TEXT {
|
||||
break;
|
||||
}
|
||||
if m.is_empty() || self.validator.is_some_and(|v| !v.check(m.as_str())) {
|
||||
continue;
|
||||
}
|
||||
if let (Some(&a), Some(&b)) = (byte_to_char.get(m.start()), byte_to_char.get(m.end()))
|
||||
&& a < b
|
||||
&& b != usize::MAX
|
||||
{
|
||||
out.push(a..b);
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn chars(s: &str) -> Vec<char> {
|
||||
s.chars().collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn finds_matches_as_char_ranges_with_multibyte_text() {
|
||||
let r = UserRegex::new(r"\d{2}( \d{2}){2}", None).unwrap();
|
||||
let t = chars("Ré 85 05 78 et 91 12 34 fin");
|
||||
let found: Vec<String> = r.find(&t).into_iter().map(|g| t[g].iter().collect()).collect();
|
||||
assert_eq!(found, ["85 05 78", "91 12 34"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validators_filter_matches() {
|
||||
let t = chars("ok 4539 1488 0343 6467 bad 4539 1488 0343 6468");
|
||||
let r = UserRegex::new(r"\d{4}( \d{4}){3}", Some(Validator::Luhn)).unwrap();
|
||||
assert_eq!(r.find(&t).len(), 1);
|
||||
let t = chars("GB82 WEST 1234 5698 7654 32 and GB83 WEST 1234 5698 7654 32");
|
||||
let r = UserRegex::new(r"GB\d\d WEST( \d{4}){3} \d\d", Some(Validator::Iban)).unwrap();
|
||||
assert_eq!(r.find(&t).len(), 1);
|
||||
assert!(Validator::Mod97.check("98") && !Validator::Mod97.check("99"));
|
||||
assert!(!Validator::Luhn.check("12a4") && !Validator::Mod97.check("é1") && !Validator::Luhn.check("0"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hostile_expressions_are_refused_or_stay_linear() {
|
||||
assert_eq!(UserRegex::new("", None).unwrap_err(), RegexError::Empty);
|
||||
assert!(matches!(UserRegex::new(&"a".repeat(MAX_REGEX_LEN + 1), None), Err(RegexError::TooLong(_))));
|
||||
assert!(matches!(UserRegex::new("(unclosed", None), Err(RegexError::Invalid(_))));
|
||||
assert!(matches!(UserRegex::new(r"(a)\1", None), Err(RegexError::Invalid(_))), "no back-references");
|
||||
assert_eq!(UserRegex::new("a*", None).unwrap_err(), RegexError::MatchesEmpty);
|
||||
assert!(matches!(UserRegex::new(r"\p{L}{1000}{1000}", None), Err(RegexError::Invalid(_))), "compiled size is capped");
|
||||
assert!(matches!(UserRegex::new(&format!("{}a{}", "(".repeat(100), ")".repeat(100)), None), Err(RegexError::Invalid(_))));
|
||||
// Classic catastrophic pattern finishes at once on a long non-matching input.
|
||||
let r = UserRegex::new(r"(a+)+$", None).unwrap();
|
||||
let mut t = chars(&"a".repeat(50_000));
|
||||
t.push('!');
|
||||
assert!(r.find(&t).is_empty());
|
||||
}
|
||||
}
|
||||
@@ -5611,6 +5611,18 @@ tools = ["redact_mark"]
|
||||
evidence = ["crates/redact/src/patterns.rs::phone_numbers", "crates/redact/src/patterns.rs::credit_cards_need_a_valid_checksum", "crates/automation/tests/automation.rs::redacting_through_tools"]
|
||||
notes = "phone (North American, E.164 with + country code, national with trunk 0), e-mail, credit card (Luhn), SSN, dates (English US forms); text drawn in a font whose built-in encoding we hold no table for (Symbol, ZapfDingbats, MacExpert) has no Unicode mapping, so a pattern search cannot see it -- it is skipped rather than mis-decoded"
|
||||
|
||||
[[feature]]
|
||||
id = "protect.redact-regex"
|
||||
title = "Redact by caller-supplied regular expression"
|
||||
area = "G"
|
||||
tier = "P2"
|
||||
milestone = "M8"
|
||||
status = "shipped"
|
||||
commands = []
|
||||
tools = ["redact_mark"]
|
||||
evidence = ["crates/redact/src/user_regex.rs::hostile_expressions_are_refused_or_stay_linear", "crates/redact/src/user_regex.rs::validators_filter_matches", "crates/automation/tests/automation.rs::redacting_with_a_caller_regex_through_tools"]
|
||||
notes = "Headless only (redact_mark regex, optional validate luhn, mod97 or iban; issue #1886): no UI entry yet. Rust regex syntax, matched in linear time with no look-around or back-references; expression capped at 1024 bytes, compiled size and nesting capped, empty-matching expressions refused. Matches run on the page text as the pattern matchers see it (words and lines joined by one space), so a match cannot span a page break. Missing: a UI field, a mask for checksums that need more than the whole match (for example the French social security key), and a check against Acrobat's own custom-pattern syntax (not observed)."
|
||||
|
||||
[[feature]]
|
||||
id = "protect.redact-pattern-locales"
|
||||
title = "Pattern sets per locale"
|
||||
|
||||
Reference in new issue
Block a user