mirror of
https://github.com/storytold/pdfcraft.git
synced 2026-11-09 04:22:20 +00:00
Merge pull request #1977 from vangel76/feat/redact-mark-regex
Redact: mark text by caller-supplied regex (#1886)
This commit is contained in:
11 files changed
+350
-7
No files matched your search
Generated
+24
@@ -4641,6 +4641,7 @@ dependencies = [
|
||||
"pdfcraft-render",
|
||||
"pdfcraft-testkit",
|
||||
"pdfcraft-xfa",
|
||||
"regex",
|
||||
"sha2 0.11.1",
|
||||
"thiserror 2.0.21",
|
||||
]
|
||||
@@ -5521,6 +5522,29 @@ dependencies = [
|
||||
"bitflags 2.13.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex"
|
||||
version = "1.13.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d"
|
||||
dependencies = [
|
||||
"aho-corasick",
|
||||
"memchr",
|
||||
"regex-automata",
|
||||
"regex-syntax",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex-automata"
|
||||
version = "0.4.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2"
|
||||
dependencies = [
|
||||
"aho-corasick",
|
||||
"memchr",
|
||||
"regex-syntax",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex-syntax"
|
||||
version = "0.8.11"
|
||||
|
||||
@@ -84,6 +84,9 @@ hayro-jbig2 = "0.3"
|
||||
|
||||
# foundation
|
||||
thiserror = "2"
|
||||
# Linear-time matching (no backtracking) for caller-supplied redaction patterns. `perf-dfa` gives
|
||||
# the lazy-DFA cache that `RegexBuilder::dfa_size_limit` bounds (the limit is inert without it).
|
||||
regex = { version = "1", default-features = false, features = ["std", "unicode", "perf-dfa"] }
|
||||
serde = { version = "1", features = ["derive", "rc"] }
|
||||
serde_json = "1"
|
||||
log = "0.4"
|
||||
|
||||
@@ -231,6 +231,7 @@ Ranked; detail and the full milestone table in [docs/roadmap.md](docs/roadmap.md
|
||||
| 8 | 1.0 polish: performance, right-to-left, missing languages, keyboard-only (M14) | 55–95 |
|
||||
|
||||
## Progress log
|
||||
- **2026-10-13 (M8, redact by regex):** `redact_mark` takes `regex` (caller expression, linear-time engine, length/size/nesting caps, empty-matching refused) with optional `validate` (luhn, mod97, iban). Headless only; issue #1886.
|
||||
|
||||
- **2026-10-28 (M10, #4119 partial):** Scan & OCR ▸ Enhance scanned file cleans scanned pages in place: background removal (level 0–100, as the camera clean-up), edge shadow removal (book gutters, copier borders) and despeckle. Each scan image is decoded, cleaned and written back into its own object at the same size and colour space (JPEG stays JPEG, others lossless, 1-bit stays 1-bit), so the OCR text layer, comments and links stay; one undo step, shared images once, unsupported images (CMYK, indexed, JPEG 2000, JBIG2, CCITT, broken data) reported. `page_enhance_scan` drives it headlessly. Not yet an option of Recognize text. Estimates unchanged.
|
||||
- **2026-11-03 (M11, shared JBIG2 dictionary):** With Monochrome Images set to JBIG2, Optimize has up to 16 JBIG2 images share one symbol dictionary (`/JBIG2Globals`), the letters found on more than one page stored once, when that is smaller (`pdfcraft_filters::encode_jbig2_shared`). Eight pages of 10 pt text rendered at 300 dpi take 41% less in Helvetica and 42% in Times than coded alone, 10% in Courier; groups of 16 save 45%, of 32 only 47%. Only images redaction can store again without the dictionary share one (image masks and DeviceGray named directly, at most 64 Mi pixels), and redacting one stores the rest of its group so, which the group bounds to 15.
|
||||
|
||||
@@ -2,8 +2,8 @@
|
||||
//! (removing what they cover for good) or clear them.
|
||||
|
||||
use pdfcraft_engine::{
|
||||
Edit, Hidden, NewAnnotation, REDACT_MAX_WORDS, REDACTION_CODE_SETS, RedactPattern, RedactionCodeSet, Shape, Style, find_pattern, rect_quad,
|
||||
redact_word_list,
|
||||
Edit, Hidden, NewAnnotation, REDACT_MAX_WORDS, REDACT_VALIDATORS, REDACTION_CODE_SETS, RedactPattern, RedactRegex, RedactValidator,
|
||||
RedactionCodeSet, Shape, Style, find_pattern, rect_quad, redact_word_list,
|
||||
};
|
||||
use serde_json::{Value, json};
|
||||
|
||||
@@ -59,16 +59,20 @@ impl Automation {
|
||||
a.get("find").is_some(),
|
||||
a.get("words").is_some(),
|
||||
a.get("pattern").is_some(),
|
||||
a.get("regex").is_some(),
|
||||
a.opt_bool("whole_pages")?.unwrap_or(false),
|
||||
];
|
||||
if chosen.iter().filter(|c| **c).count() != 1 {
|
||||
return Err(ToolError::InvalidArgs("pass exactly one of rect (with page), find, words, pattern or whole_pages: true".into()));
|
||||
return Err(ToolError::InvalidArgs("pass exactly one of rect (with page), find, words, pattern, regex or whole_pages: true".into()));
|
||||
}
|
||||
let case_sensitive = a.opt_bool("case_sensitive")?.unwrap_or(false);
|
||||
let whole_words = a.opt_bool("whole_words")?.unwrap_or(false);
|
||||
if (case_sensitive || whole_words) && a.get("find").is_none() && a.get("words").is_none() {
|
||||
return Err(ToolError::InvalidArgs("case_sensitive and whole_words apply only to find or words".into()));
|
||||
}
|
||||
if a.get("validate").is_some() && a.get("regex").is_none() {
|
||||
return Err(ToolError::InvalidArgs("validate applies only to regex".into()));
|
||||
}
|
||||
// Per searched word or phrase: how many marks it made.
|
||||
let mut matched_words: Vec<(String, usize)> = Vec::new();
|
||||
if a.get("rect").is_some() {
|
||||
@@ -89,6 +93,19 @@ impl Automation {
|
||||
Some(p) => Some(RedactPattern::from_id(p).ok_or_else(|| ToolError::InvalidArgs(format!("unknown pattern {p:?}")))?),
|
||||
None => None,
|
||||
};
|
||||
let user_re = match a.opt_str("regex")? {
|
||||
Some(src) => {
|
||||
let validator = match a.opt_str("validate")? {
|
||||
Some(v) => Some(RedactValidator::from_id(v).ok_or_else(|| {
|
||||
let ids: Vec<_> = REDACT_VALIDATORS.iter().map(|v| v.id()).collect();
|
||||
ToolError::InvalidArgs(format!("unknown validate {v:?}; allowed: {}", ids.join(", ")))
|
||||
})?),
|
||||
None => None,
|
||||
};
|
||||
Some(RedactRegex::new(src, validator).map_err(|e| ToolError::InvalidArgs(e.to_string()))?)
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
let needles: Vec<String> = match (a.opt_str("find")?, a.get("words")) {
|
||||
(Some(f), _) => vec![f.to_owned()],
|
||||
(None, Some(_)) => {
|
||||
@@ -108,9 +125,27 @@ impl Automation {
|
||||
let texts = self.page_texts(id, &pages)?;
|
||||
let info = self.doc(a)?.info.clone();
|
||||
for (&p, text) in pages.iter().zip(&texts) {
|
||||
let hits: Vec<(Option<usize>, _)> = match pattern {
|
||||
Some(pat) => text.find_with(|chars| find_pattern(pat, chars)).into_iter().map(|h| (None, h)).collect(),
|
||||
None => needles
|
||||
let hits: Vec<(Option<usize>, _)> = match (&user_re, pattern) {
|
||||
(Some(re), _) => {
|
||||
// A page with too many matches fails the call: marking only the first ones
|
||||
// would leave the rest readable after redact_apply.
|
||||
let too_many = std::cell::Cell::new(false);
|
||||
let found = text.find_with(|chars| {
|
||||
re.find(chars).unwrap_or_else(|_| {
|
||||
too_many.set(true);
|
||||
Vec::new()
|
||||
})
|
||||
});
|
||||
if too_many.get() {
|
||||
return Err(ToolError::InvalidArgs(format!(
|
||||
"regex has more than 10000 matches on page {}; nothing was marked. Narrow the expression so that every match can be marked",
|
||||
p + 1
|
||||
)));
|
||||
}
|
||||
found.into_iter().map(|h| (None, h)).collect()
|
||||
}
|
||||
(None, Some(pat)) => text.find_with(|chars| find_pattern(pat, chars)).into_iter().map(|h| (None, h)).collect(),
|
||||
(None, None) => needles
|
||||
.iter()
|
||||
.enumerate()
|
||||
.flat_map(|(i, w)| text.find_opts(w, case_sensitive, whole_words).into_iter().map(move |h| (Some(i), h)))
|
||||
|
||||
@@ -620,7 +620,7 @@ pub fn tools() -> Vec<ToolDef> {
|
||||
t(
|
||||
"redact_mark",
|
||||
"Mark for redaction",
|
||||
"Mark content for redaction (nothing is removed until redact_apply). One of: rect [x0, y0, x1, y1] (points from the top-left of the displayed page) with page; find (text, every match); words (a list of words or phrases, every match of each; the result's matched_words says how many marks each made); pattern (phone, email, credit-card, ssn, date, iban: every match, Acrobat's Search & Redact patterns); whole_pages: true. find and words accept case_sensitive and whole_words (both default false). Select one page with page or multiple with pages (default all); do not pass both. rect requires page. overlay: text shown on the box once applied, or code_set (foia: U.S. FOIA (b)(1)(A)…(b)(9); privacy-act: U.S. Privacy Act (d)(5), (j)(1)…(k)(7)) with codes, shown as \"(b)(6), (b)(7)(C)\"; fill: box colour (default black). Undoable.",
|
||||
"Mark content for redaction (nothing is removed until redact_apply). One of: rect [x0, y0, x1, y1] (points from the top-left of the displayed page) with page; find (text, every match); words (a list of words or phrases, every match of each; the result's matched_words says how many marks each made); pattern (phone, email, credit-card, ssn, date, iban: every match, Acrobat's Search & Redact patterns); regex (your own regular expression, every match, on the page text with words and lines separated by one space; Rust regex syntax: linear-time, no look-around or back-references, case-sensitive unless (?i), at most 1024 bytes, more than 10000 matches on one page fail the call, matches of the empty string are refused) with optional validate (luhn, mod97, iban: keep a match only if its checksum holds); whole_pages: true. find and words accept case_sensitive and whole_words (both default false). Select one page with page or multiple with pages (default all); do not pass both. rect requires page. overlay: text shown on the box once applied, or code_set (foia: U.S. FOIA (b)(1)(A)…(b)(9); privacy-act: U.S. Privacy Act (d)(5), (j)(1)…(k)(7)) with codes, shown as \"(b)(6), (b)(7)(C)\"; fill: box colour (default black). Undoable.",
|
||||
)
|
||||
.with(schema(
|
||||
json!({
|
||||
@@ -632,6 +632,8 @@ pub fn tools() -> Vec<ToolDef> {
|
||||
"case_sensitive": { "type": "boolean", "description": "Match letter case for find or words (default false)." },
|
||||
"whole_words": { "type": "boolean", "description": "Match whole words for find or words (default false)." },
|
||||
"pattern": pattern_enum(),
|
||||
"regex": { "type": "string", "minLength": 1, "maxLength": 1024, "description": "Regular expression (Rust regex syntax), at most 1024 bytes (maxLength counts characters, the limit is bytes); every match is marked." },
|
||||
"validate": { "type": "string", "enum": ["luhn", "mod97", "iban"], "description": "With regex: keep a match only if its checksum is valid (spaces and hyphens ignored)." },
|
||||
"whole_pages": { "type": "boolean" },
|
||||
"pages": pages("to search or mark (default: all); mutually exclusive with page, and not valid with rect"),
|
||||
"overlay": { "type": "string" },
|
||||
|
||||
@@ -4818,6 +4818,34 @@ fn redacting_through_tools() {
|
||||
assert!(matches!(a.call("redact_mark", &json!({ "doc": doc })), Err(ToolError::InvalidArgs(_))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn redacting_with_a_caller_regex_through_tools() {
|
||||
let dir = workdir("redact-regex");
|
||||
std::fs::write(
|
||||
dir.join("ids.txt"),
|
||||
"Ref 1 85 05 78 006 084 36 end\nRef 2 91 12 34 567 890 12 end\nCard 4539 1488 0343 6467 and 4539 1488 0343 6468",
|
||||
)
|
||||
.unwrap();
|
||||
let mut a = auto(&dir);
|
||||
let doc = ok(&mut a, "doc_create", json!({ "from": "text", "path": "ids.txt" }))["doc"].as_u64().unwrap();
|
||||
let bad = |a: &mut Automation, args: Value| matches!(a.call("redact_mark", &args), Err(ToolError::InvalidArgs(_)));
|
||||
assert!(bad(&mut a, json!({ "doc": doc, "regex": "(unclosed" })));
|
||||
assert!(bad(&mut a, json!({ "doc": doc, "regex": "a*" })));
|
||||
assert!(bad(&mut a, json!({ "doc": doc, "regex": "x".repeat(2000) })));
|
||||
assert!(bad(&mut a, json!({ "doc": doc, "regex": "x", "pattern": "ssn" })));
|
||||
assert!(bad(&mut a, json!({ "doc": doc, "regex": "x", "validate": "crc" })));
|
||||
assert!(bad(&mut a, json!({ "doc": doc, "find": "x", "validate": "luhn" })));
|
||||
assert!(matches!(a.call("redact_mark", &json!({ "doc": doc, "regex": "Z{3}" })), Err(ToolError::Failed(_))));
|
||||
let r = ok(&mut a, "redact_mark", json!({ "doc": doc, "regex": r"\d \d\d \d\d \d\d \d{3} \d{3} \d\d" }));
|
||||
assert_eq!(r["marked"].as_u64(), Some(2));
|
||||
let r = ok(&mut a, "redact_mark", json!({ "doc": doc, "regex": r"\d{4}( \d{4}){3}", "validate": "luhn" }));
|
||||
assert_eq!(r["marked"].as_u64(), Some(1), "only the number with a valid checksum");
|
||||
ok(&mut a, "redact_apply", json!({ "doc": doc }));
|
||||
let text = page_text(&mut a, doc)[0].clone();
|
||||
assert!(!text.contains("85 05") && !text.contains("91 12") && !text.contains("6467"), "{text}");
|
||||
assert!(text.contains("6468"), "{text}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
/// Mark, apply and save remove the complete international number while retaining labels and controls.
|
||||
fn redacting_italian_international_slash_numbers_through_tools() {
|
||||
|
||||
@@ -129,6 +129,7 @@ pub use pdfcraft_redact::patterns::{
|
||||
MAX_WORDS as REDACT_MAX_WORDS, PATTERNS as REDACT_PATTERNS, Pattern as RedactPattern, find as find_pattern, word_list as redact_word_list,
|
||||
};
|
||||
pub use pdfcraft_redact::sanitize::{HIDDEN, Hidden};
|
||||
pub use pdfcraft_redact::user_regex::{UserRegex as RedactRegex, VALIDATORS as REDACT_VALIDATORS, Validator as RedactValidator};
|
||||
pub use pdfcraft_sign as sign;
|
||||
pub use pdfcraft_sign::{SignOptions, SignatureInfo, Status as SignatureStatus, StatusCaptions, TrustStore};
|
||||
pub use pdfcraft_xfdf::Format as DataFormat;
|
||||
|
||||
@@ -20,6 +20,7 @@ pdfcraft-annot = { workspace = true }
|
||||
pdfcraft-xfa = { workspace = true }
|
||||
# CCITT Group 4 and JBIG2 encoding for cleared one-bit images.
|
||||
pdfcraft-filters = { workspace = true }
|
||||
regex = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
sha2 = "0.11"
|
||||
|
||||
|
||||
@@ -32,6 +32,7 @@ mod tests;
|
||||
mod tests_verify;
|
||||
pub(crate) mod type3;
|
||||
pub(crate) mod type3_tables;
|
||||
pub mod user_regex;
|
||||
pub mod verify;
|
||||
|
||||
use interp::{Mode, Scope, process};
|
||||
|
||||
@@ -0,0 +1,235 @@
|
||||
//! Caller-supplied regular expressions for Search & Redact (`redact_mark regex`).
|
||||
//!
|
||||
//! The expression is untrusted input. It runs on the `regex` crate, which matches in time
|
||||
//! linear in the text (no backtracking, no look-around or back-references), and is compiled
|
||||
//! under hard limits on source length, nesting and compiled size, so a hostile expression is
|
||||
//! refused with an error instead of exhausting time or memory. An optional [`Validator`]
|
||||
//! keeps a match only when its checksum holds.
|
||||
|
||||
use std::ops::Range;
|
||||
|
||||
use regex::RegexBuilder;
|
||||
use thiserror::Error;
|
||||
|
||||
/// Longest expression accepted, in bytes.
|
||||
pub const MAX_REGEX_LEN: usize = 1024;
|
||||
/// Most matches accepted for one page; one more makes [`UserRegex::find`] fail, so a page is never
|
||||
/// marked only in part.
|
||||
pub const MAX_MATCHES_PER_TEXT: usize = 10_000;
|
||||
/// Cap on the compiled program and on the lazy-DFA cache, in bytes.
|
||||
const SIZE_LIMIT: usize = 1 << 20;
|
||||
/// Cap on parenthesis nesting.
|
||||
const NEST_LIMIT: u32 = 32;
|
||||
|
||||
#[derive(Debug, Error, PartialEq, Eq)]
|
||||
pub enum RegexError {
|
||||
#[error("regex is empty")]
|
||||
Empty,
|
||||
#[error("regex is {0} bytes long; at most {MAX_REGEX_LEN} are allowed")]
|
||||
TooLong(usize),
|
||||
#[error("invalid or too complex regex: {0}")]
|
||||
Invalid(String),
|
||||
#[error("regex matches the empty string, so it would mark nothing useful; require at least one character")]
|
||||
MatchesEmpty,
|
||||
#[error("regex has more than {MAX_MATCHES_PER_TEXT} matches on one page; narrow it so that every match can be marked")]
|
||||
TooManyMatches,
|
||||
}
|
||||
|
||||
/// Checksum a match must pass to be kept.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum Validator {
|
||||
/// Luhn (ISO/IEC 7812): digits only, spaces and hyphens ignored.
|
||||
Luhn,
|
||||
/// ISO 7064 MOD 97-10: letters count as 10..35 and the number mod 97 must be 1. Letters and
|
||||
/// digits only, spaces and hyphens ignored.
|
||||
Mod97,
|
||||
/// IBAN check (ISO 13616): the first four characters move to the end, then MOD 97-10.
|
||||
Iban,
|
||||
}
|
||||
|
||||
/// Ids accepted for `validate`.
|
||||
pub const VALIDATORS: [Validator; 3] = [Validator::Luhn, Validator::Mod97, Validator::Iban];
|
||||
|
||||
impl Validator {
|
||||
pub fn id(self) -> &'static str {
|
||||
match self {
|
||||
Validator::Luhn => "luhn",
|
||||
Validator::Mod97 => "mod97",
|
||||
Validator::Iban => "iban",
|
||||
}
|
||||
}
|
||||
|
||||
pub fn from_id(id: &str) -> Option<Self> {
|
||||
VALIDATORS.into_iter().find(|v| v.id() == id)
|
||||
}
|
||||
|
||||
/// Whether `s` passes the checksum.
|
||||
pub fn check(self, s: &str) -> bool {
|
||||
match self {
|
||||
Validator::Luhn => luhn(s),
|
||||
Validator::Mod97 => mod97(s.chars()),
|
||||
Validator::Iban => {
|
||||
let kept: Vec<char> = s.chars().filter(|c| *c != ' ' && *c != '-').collect();
|
||||
let (head, tail) = kept.split_at(kept.len().min(4));
|
||||
mod97(tail.iter().chain(head).copied())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn luhn(s: &str) -> bool {
|
||||
let (mut sum, mut digits, mut double) = (0u32, 0usize, false);
|
||||
for c in s.chars().rev() {
|
||||
if c == ' ' || c == '-' {
|
||||
continue;
|
||||
}
|
||||
let Some(d) = c.to_digit(10).filter(|_| c.is_ascii_digit()) else { return false };
|
||||
let d = if double { if d * 2 > 9 { d * 2 - 9 } else { d * 2 } } else { d };
|
||||
sum = sum.saturating_add(d);
|
||||
digits += 1;
|
||||
double = !double;
|
||||
}
|
||||
digits >= 2 && sum % 10 == 0
|
||||
}
|
||||
|
||||
fn mod97(s: impl Iterator<Item = char>) -> bool {
|
||||
let (mut rem, mut n) = (0u32, 0usize);
|
||||
for c in s {
|
||||
if c == ' ' || c == '-' {
|
||||
continue;
|
||||
}
|
||||
if !c.is_ascii_alphanumeric() {
|
||||
return false;
|
||||
}
|
||||
// Digits stay one decimal digit, letters become two (A = 10 .. Z = 35).
|
||||
let v = c.to_digit(36).unwrap_or(0);
|
||||
rem = if v >= 10 { (rem * 100 + v) % 97 } else { (rem * 10 + v) % 97 };
|
||||
n += 1;
|
||||
}
|
||||
n >= 2 && rem == 1
|
||||
}
|
||||
|
||||
/// Whether `re` can match the empty string, which `is_match("")` alone misses for assertions such
|
||||
/// as `\b`. Tries a few short texts with letters, digits, space, newline and a non-ASCII letter.
|
||||
fn can_match_empty(re: ®ex::Regex) -> bool {
|
||||
["", " ", "a", "1", "a a", "a\n", "\n", "é", "aa 11"].iter().any(|p| re.find_iter(p).any(|m| m.is_empty()))
|
||||
}
|
||||
|
||||
/// A compiled caller expression with an optional checksum.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct UserRegex {
|
||||
re: regex::Regex,
|
||||
validator: Option<Validator>,
|
||||
}
|
||||
|
||||
impl UserRegex {
|
||||
pub fn new(source: &str, validator: Option<Validator>) -> Result<Self, RegexError> {
|
||||
if source.is_empty() {
|
||||
return Err(RegexError::Empty);
|
||||
}
|
||||
if source.len() > MAX_REGEX_LEN {
|
||||
return Err(RegexError::TooLong(source.len()));
|
||||
}
|
||||
let re = RegexBuilder::new(source)
|
||||
.size_limit(SIZE_LIMIT)
|
||||
.dfa_size_limit(SIZE_LIMIT)
|
||||
.nest_limit(NEST_LIMIT)
|
||||
.build()
|
||||
.map_err(|e| RegexError::Invalid(e.to_string()))?;
|
||||
if can_match_empty(&re) {
|
||||
return Err(RegexError::MatchesEmpty);
|
||||
}
|
||||
Ok(Self { re, validator })
|
||||
}
|
||||
|
||||
/// Character ranges of `text` the expression matches (non-overlapping, left to right).
|
||||
///
|
||||
/// Fails with [`RegexError::TooManyMatches`] when more than [`MAX_MATCHES_PER_TEXT`] matches
|
||||
/// are kept, instead of returning a partial list the caller would take for complete.
|
||||
pub fn find(&self, text: &[char]) -> Result<Vec<Range<usize>>, RegexError> {
|
||||
let s: String = text.iter().collect();
|
||||
// Char index of every byte offset that starts a char (and of the end).
|
||||
let mut byte_to_char = vec![usize::MAX; s.len().saturating_add(1)];
|
||||
for (ci, (bi, _)) in s.char_indices().enumerate() {
|
||||
if let Some(slot) = byte_to_char.get_mut(bi) {
|
||||
*slot = ci;
|
||||
}
|
||||
}
|
||||
if let Some(slot) = byte_to_char.get_mut(s.len()) {
|
||||
*slot = text.len();
|
||||
}
|
||||
let mut out = Vec::new();
|
||||
for m in self.re.find_iter(&s) {
|
||||
if m.is_empty() || self.validator.is_some_and(|v| !v.check(m.as_str())) {
|
||||
continue;
|
||||
}
|
||||
if let (Some(&a), Some(&b)) = (byte_to_char.get(m.start()), byte_to_char.get(m.end()))
|
||||
&& a < b
|
||||
&& b != usize::MAX
|
||||
{
|
||||
if out.len() >= MAX_MATCHES_PER_TEXT {
|
||||
return Err(RegexError::TooManyMatches);
|
||||
}
|
||||
out.push(a..b);
|
||||
}
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn chars(s: &str) -> Vec<char> {
|
||||
s.chars().collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn finds_matches_as_char_ranges_with_multibyte_text() {
|
||||
let r = UserRegex::new(r"\d{2}( \d{2}){2}", None).unwrap();
|
||||
let t = chars("Ré 85 05 78 et 91 12 34 fin");
|
||||
let found: Vec<String> = r.find(&t).unwrap().into_iter().map(|g| t[g].iter().collect()).collect();
|
||||
assert_eq!(found, ["85 05 78", "91 12 34"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validators_filter_matches() {
|
||||
let t = chars("ok 4539 1488 0343 6467 bad 4539 1488 0343 6468");
|
||||
let r = UserRegex::new(r"\d{4}( \d{4}){3}", Some(Validator::Luhn)).unwrap();
|
||||
assert_eq!(r.find(&t).unwrap().len(), 1);
|
||||
let t = chars("GB82 WEST 1234 5698 7654 32 and GB83 WEST 1234 5698 7654 32");
|
||||
let r = UserRegex::new(r"GB\d\d WEST( \d{4}){3} \d\d", Some(Validator::Iban)).unwrap();
|
||||
assert_eq!(r.find(&t).unwrap().len(), 1);
|
||||
assert!(Validator::Mod97.check("98") && !Validator::Mod97.check("99"));
|
||||
assert!(!Validator::Luhn.check("12a4") && !Validator::Mod97.check("é1") && !Validator::Luhn.check("0"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hostile_expressions_are_refused_or_stay_linear() {
|
||||
assert_eq!(UserRegex::new("", None).unwrap_err(), RegexError::Empty);
|
||||
assert!(matches!(UserRegex::new(&"a".repeat(MAX_REGEX_LEN + 1), None), Err(RegexError::TooLong(_))));
|
||||
assert!(matches!(UserRegex::new("(unclosed", None), Err(RegexError::Invalid(_))));
|
||||
assert!(matches!(UserRegex::new(r"(a)\1", None), Err(RegexError::Invalid(_))), "no back-references");
|
||||
for empty in ["a*", r"\b", "(?m)^", r"\B", "(?m)$", r"\d*|x"] {
|
||||
assert_eq!(UserRegex::new(empty, None).unwrap_err(), RegexError::MatchesEmpty, "{empty}");
|
||||
}
|
||||
assert!(UserRegex::new(r"\bfoo\b", None).is_ok());
|
||||
assert!(matches!(UserRegex::new(r"\p{L}{1000}{1000}", None), Err(RegexError::Invalid(_))), "compiled size is capped");
|
||||
assert!(matches!(UserRegex::new(&format!("{}a{}", "(".repeat(100), ")".repeat(100)), None), Err(RegexError::Invalid(_))));
|
||||
// Classic catastrophic pattern finishes at once on a long non-matching input.
|
||||
let r = UserRegex::new(r"(a+)+$", None).unwrap();
|
||||
let mut t = chars(&"a".repeat(50_000));
|
||||
t.push('!');
|
||||
assert!(r.find(&t).unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn more_than_the_cap_of_matches_is_an_error_not_a_partial_list() {
|
||||
let r = UserRegex::new("x", None).unwrap();
|
||||
let at_cap = chars(&"x ".repeat(MAX_MATCHES_PER_TEXT));
|
||||
assert_eq!(r.find(&at_cap).unwrap().len(), MAX_MATCHES_PER_TEXT);
|
||||
let over = chars(&"x ".repeat(MAX_MATCHES_PER_TEXT + 1));
|
||||
assert_eq!(r.find(&over).unwrap_err(), RegexError::TooManyMatches);
|
||||
}
|
||||
}
|
||||
@@ -5826,6 +5826,18 @@ tools = ["redact_mark"]
|
||||
evidence = ["crates/redact/src/patterns.rs::labelled_italian_international_slash_numbers_match_whole", "crates/automation/tests/it/automation.rs::redacting_italian_international_slash_numbers_through_tools", "crates/redact/src/patterns.rs::phone_numbers", "crates/redact/src/patterns.rs::slashed_and_labelled_unseparated_phone_numbers", "crates/redact/src/patterns.rs::credit_cards_need_a_valid_checksum", "crates/automation/tests/it/automation.rs::redacting_through_tools"]
|
||||
notes = "phone (North American, E.164 with + country code, national with trunk 0, a slash after a trunk-0 area code; labelled Italian +39 geographic numbers with one slash after the area code match with their complete country code (#3205 follow-up), with or without one space after +39; international slash numbers without a phone label, or with country codes other than +39, remain unsupported; numbers without separators only after a phone label such as Tel. or cell.), e-mail, credit card (Luhn), SSN, dates (English US forms); text drawn in a font whose built-in encoding we hold no table for (Symbol, ZapfDingbats, MacExpert) has no Unicode mapping, so a pattern search cannot see it -- it is skipped rather than mis-decoded"
|
||||
|
||||
[[feature]]
|
||||
id = "protect.redact-regex"
|
||||
title = "Redact by caller-supplied regular expression"
|
||||
area = "G"
|
||||
tier = "P2"
|
||||
milestone = "M8"
|
||||
status = "shipped"
|
||||
commands = []
|
||||
tools = ["redact_mark"]
|
||||
evidence = ["crates/redact/src/user_regex.rs::hostile_expressions_are_refused_or_stay_linear", "crates/redact/src/user_regex.rs::validators_filter_matches", "crates/automation/tests/it/automation.rs::redacting_with_a_caller_regex_through_tools"]
|
||||
notes = "Headless only (redact_mark regex, optional validate luhn, mod97 or iban; issue #1886): no UI entry yet. Rust regex syntax, matched in linear time with no look-around or back-references; expression capped at 1024 bytes, compiled size and nesting capped, empty-matching expressions refused. Matches run on the page text as the pattern matchers see it (words and lines joined by one space), so a match cannot span a page break. Missing: a UI field, a mask for checksums that need more than the whole match (for example the French social security key), and a check against Acrobat's own custom-pattern syntax (not observed)."
|
||||
|
||||
[[feature]]
|
||||
id = "protect.redact-pattern-locales"
|
||||
title = "Pattern sets per locale"
|
||||
|
||||
Reference in new issue
Block a user