diff --git a/crates/automation/src/tools.rs b/crates/automation/src/tools.rs index bf2abea185..54d94ff4bf 100644 --- a/crates/automation/src/tools.rs +++ b/crates/automation/src/tools.rs @@ -5,6 +5,8 @@ use serde_json::{Value, json}; +use pdfcraft_engine::REDACT_PATTERNS; + use crate::ToolError; #[derive(Clone, Debug)] @@ -30,6 +32,12 @@ fn pages(what: &str) -> Value { json!({ "type": "array", "items": { "type": "integer", "minimum": 1 }, "minItems": 1, "description": format!("1-based page numbers {what}.") }) } +/// The Search & Redact pattern ids, from the engine's registry, so the advertised schema cannot +/// drift from the patterns the runtime accepts. +fn pattern_enum() -> Value { + json!({ "type": "string", "enum": REDACT_PATTERNS.iter().map(|p| p.id()).collect::>() }) +} + fn path(what: &str) -> Value { json!({ "type": "array", "items": { "type": "integer", "minimum": 1 }, "description": format!("{what}: 1-based positions from the top level, e.g. [2, 1] = the first child of the second bookmark.") }) } @@ -605,7 +613,7 @@ pub fn tools() -> Vec { t( "redact_mark", "Mark for redaction", - "Mark content for redaction (nothing is removed until redact_apply). One of: rect [x0, y0, x1, y1] (points from the top-left of the displayed page) with page; find (text, every match); words (a list of words or phrases, every match of each; the result's matched_words says how many marks each made); pattern (phone, email, credit-card, ssn, date: every match, Acrobat's Search & Redact patterns); whole_pages: true. find and words accept case_sensitive and whole_words (both default false). Select one page with page or multiple with pages (default all); do not pass both. rect requires page. overlay: text shown on the box once applied, or code_set (foia: U.S. FOIA (b)(1)(A)…(b)(9); privacy-act: U.S. Privacy Act (d)(5), (j)(1)…(k)(7)) with codes, shown as \"(b)(6), (b)(7)(C)\"; fill: box colour (default black). Undoable.", + "Mark content for redaction (nothing is removed until redact_apply). One of: rect [x0, y0, x1, y1] (points from the top-left of the displayed page) with page; find (text, every match); words (a list of words or phrases, every match of each; the result's matched_words says how many marks each made); pattern (phone, email, credit-card, ssn, date, iban: every match, Acrobat's Search & Redact patterns); whole_pages: true. find and words accept case_sensitive and whole_words (both default false). Select one page with page or multiple with pages (default all); do not pass both. rect requires page. overlay: text shown on the box once applied, or code_set (foia: U.S. FOIA (b)(1)(A)…(b)(9); privacy-act: U.S. Privacy Act (d)(5), (j)(1)…(k)(7)) with codes, shown as \"(b)(6), (b)(7)(C)\"; fill: box colour (default black). Undoable.", ) .with(schema( json!({ @@ -616,7 +624,7 @@ pub fn tools() -> Vec { "words": { "type": "array", "items": { "type": "string", "minLength": 1 }, "minItems": 1, "maxItems": 1000 }, "case_sensitive": { "type": "boolean", "description": "Match letter case for find or words (default false)." }, "whole_words": { "type": "boolean", "description": "Match whole words for find or words (default false)." }, - "pattern": { "type": "string", "enum": ["phone", "email", "credit-card", "ssn", "date"] }, + "pattern": pattern_enum(), "whole_pages": { "type": "boolean" }, "pages": pages("to search or mark (default: all); mutually exclusive with page, and not valid with rect"), "overlay": { "type": "string" }, diff --git a/crates/automation/tests/it/automation.rs b/crates/automation/tests/it/automation.rs index 6f493b824d..5ed32360fd 100644 --- a/crates/automation/tests/it/automation.rs +++ b/crates/automation/tests/it/automation.rs @@ -1715,6 +1715,25 @@ fn mcp_default_lists_every_tool_and_has_no_meta_tools() { assert_eq!(rpc(&mut s, 3, "tools/call", json!({ "name": "tool_search" }))["error"]["code"], -32602); } +#[cfg(feature = "mcp")] +#[test] +/// `tools/list` advertises exactly the Search & Redact patterns the registry supports, in `pattern` +/// and by name in the description, so a schema-obeying client can select every one of them. +fn the_advertised_redact_mark_schema_offers_every_pattern() { + use pdfcraft_engine::REDACT_PATTERNS; + let registry: Vec<&str> = REDACT_PATTERNS.iter().map(|p| p.id()).collect(); + let mut s = McpServer::new(Automation::new()); + let list = rpc(&mut s, 1, "tools/list", json!({})); + let mark = list["result"]["tools"].as_array().unwrap().iter().find(|t| t["name"] == "redact_mark").expect("redact_mark is listed"); + let advertised: Vec<&str> = + mark["inputSchema"]["properties"]["pattern"]["enum"].as_array().unwrap().iter().map(|v| v.as_str().unwrap()).collect(); + assert_eq!(advertised, registry, "the advertised pattern enum must be the registry's ids"); + let description = mark["description"].as_str().unwrap(); + for id in ®istry { + assert!(description.contains(id), "the description must name {id}: {description}"); + } +} + #[cfg(feature = "mcp")] #[test] fn mcp_compact_lists_the_core_tools_and_two_meta_tools() { @@ -4809,6 +4828,35 @@ fn redacting_italian_international_slash_numbers_through_tools() { std::fs::remove_dir_all(dir).unwrap(); } +#[test] +/// A client obeying the advertised schema can select iban, and the mark made through the tool +/// covers the spaced number and nothing else. +fn redacting_an_iban_through_tools() { + let table = tools(); + let advertised: Vec<&str> = table.iter().find(|t| t.name == "redact_mark").unwrap().input_schema["properties"]["pattern"]["enum"] + .as_array() + .unwrap() + .iter() + .map(|v| v.as_str().unwrap()) + .collect(); + assert!(advertised.contains(&"iban"), "the advertised redact_mark schema must offer iban: {advertised:?}"); + let dir = workdir("redact-iban"); + std::fs::write(dir.join("invoice.txt"), "Pay DE89 3704 0044 0532 0130 00 today\nAccount 12345678 is ours\nTotal due").unwrap(); + let mut a = auto(&dir); + let doc = ok(&mut a, "doc_create", json!({ "from": "text", "path": "invoice.txt" }))["doc"].as_u64().unwrap(); + let marked = ok(&mut a, "redact_mark", json!({ "doc": doc, "pattern": "iban" })); + assert_eq!(marked["marked"], 1, "the spaced German sample IBAN needs one mark"); + assert!(page_text(&mut a, doc)[0].contains("DE89 3704 0044 0532 0130 00"), "marks alone remove nothing"); + assert_eq!(ok(&mut a, "redact_apply", json!({ "doc": doc }))["applied"], 1); + ok(&mut a, "doc_save", json!({ "doc": doc, "path": "redacted.pdf" })); + let reopened = ok(&mut a, "doc_open", json!({ "path": "redacted.pdf" }))["doc"].as_u64().unwrap(); + let text = &page_text(&mut a, reopened)[0]; + assert!(!text.contains("DE89") && !text.contains("0532"), "{text}"); + assert!(text.lines().any(|line| line.trim() == "Pay"), "only the label before the number remains: {text}"); + assert!(text.contains("Account 12345678") && text.contains("Total due"), "text without an IBAN remains: {text}"); + std::fs::remove_dir_all(dir).unwrap(); +} + /// One page with the text "Secret 1234" and a Redact annotation that has neither a usable /// QuadPoints array nor a Rect, so there is no area to redact. fn redact_pdf_with_unreadable_mark() -> Vec { diff --git a/crates/redact/src/patterns.rs b/crates/redact/src/patterns.rs index 59e623a7e7..9662602671 100644 --- a/crates/redact/src/patterns.rs +++ b/crates/redact/src/patterns.rs @@ -1,5 +1,5 @@ //! Search & Redact patterns (Acrobat's Find Text ▸ Patterns): phone numbers, email addresses, -//! credit card numbers, US Social Security numbers and dates. Each matcher works on a page's +//! credit card numbers, US Social Security numbers, dates and IBANs. Each matcher works on a page's //! text as characters and returns character ranges; callers map them to glyphs and areas. use std::ops::Range; @@ -11,9 +11,10 @@ pub enum Pattern { CreditCard, Ssn, Date, + Iban, } -pub const PATTERNS: [Pattern; 5] = [Pattern::Phone, Pattern::Email, Pattern::CreditCard, Pattern::Ssn, Pattern::Date]; +pub const PATTERNS: [Pattern; 6] = [Pattern::Phone, Pattern::Email, Pattern::CreditCard, Pattern::Ssn, Pattern::Date, Pattern::Iban]; impl Pattern { pub fn label(self) -> &'static str { @@ -23,6 +24,7 @@ impl Pattern { Pattern::CreditCard => "Credit Cards", Pattern::Ssn => "Social Security Numbers", Pattern::Date => "Dates", + Pattern::Iban => "IBAN Numbers", } } @@ -33,6 +35,7 @@ impl Pattern { Pattern::CreditCard => "credit-card", Pattern::Ssn => "ssn", Pattern::Date => "date", + Pattern::Iban => "iban", } } @@ -397,6 +400,173 @@ fn date(s: &[char], i: usize) -> Option { Some(k2 + y - i) } +/// An IBAN (ISO 13616): a two-letter country code, two check digits and the rest +/// of the country's exact length in letters and digits, either compact or in the +/// printed groups of four. A space is accepted only after a full group, so an +/// ordinary word after a code ("BA37 leaves at noon") ends the run. Valid iff the +/// number is one modulo 97 after moving the first four characters to the end. The +/// country code must be in the registry, and an alphanumeric character glued to +/// the end rejects the match. +fn iban(s: &[char], i: usize) -> Option { + if !(s.get(i).copied().is_some_and(|c| c.is_ascii_alphabetic()) + && s.get(i + 1).copied().is_some_and(|c| c.is_ascii_alphabetic()) + && s.get(i + 2).copied().is_some_and(|c| c.is_ascii_digit()) + && s.get(i + 3).copied().is_some_and(|c| c.is_ascii_digit())) + { + return None; + } + // Not the tail of a longer alphanumeric run. + if s.get(i.wrapping_sub(1)).is_some_and(|c| c.is_alphanumeric()) { + return None; + } + let country: String = s.iter().skip(i).take(2).map(char::to_ascii_uppercase).collect(); + let len = iban_length(&country)?; + // The IBAN's characters uppercased, read up to the country's length. + let mut digits = Vec::with_capacity(len); + let mut j = i; + let mut after_space = false; + while digits.len() < len { + match s.get(j) { + Some(c) if c.is_ascii_alphanumeric() => { + digits.push(c.to_ascii_uppercase()); + after_space = false; + } + // A single space between two groups of four. + Some(&' ') if digits.len() % 4 == 0 && !after_space => after_space = true, + _ => return None, + } + j += 1; + } + // Reject an IBAN glued directly to trailing alphanumerics ("...00ABC"). + // A space-separated suffix ("...00 123") is not part of the IBAN. + if s.get(j).is_some_and(|c| c.is_alphanumeric()) { + return None; + } + iban_check(&digits).then_some(j - i) +} + +/// Whether `digits` is a valid IBAN: move the first four characters to the end, +/// replace letters by two digits (A = 10, ..., Z = 35), and the number must be +/// one modulo 97. +fn iban_check(digits: &[char]) -> bool { + let mut rem = 0u32; + for c in digits.iter().skip(4).chain(digits.iter().take(4)).copied() { + let value = iban_value(c); + // A letter stands for two decimal digits (A = 10, ..., Z = 35), + // a digit for one. + rem = if c.is_ascii_digit() { (rem * 10 + value) % 97 } else { (rem * 100 + value) % 97 }; + } + rem == 1 +} + +/// The value of an IBAN character: a digit is itself, a letter is 10 (A) to 35 (Z). +fn iban_value(c: char) -> u32 { + match c { + '0'..='9' => c as u32 - '0' as u32, + _ => c as u32 - 'A' as u32 + 10, + } +} + +/// The exact length of an IBAN for each country code of the SWIFT IBAN registry +/// (release 99, December 2024), counting letters and digits only. A country code +/// that is not listed has no IBANs, so nothing starting with it is matched. +const IBAN_LENGTHS: [(&str, usize); 89] = [ + ("AD", 24), + ("AE", 23), + ("AL", 28), + ("AT", 20), + ("AZ", 28), + ("BA", 20), + ("BE", 16), + ("BG", 22), + ("BH", 22), + ("BI", 27), + ("BR", 29), + ("BY", 28), + ("CH", 21), + ("CR", 22), + ("CY", 28), + ("CZ", 24), + ("DE", 22), + ("DJ", 27), + ("DK", 18), + ("DO", 28), + ("EE", 20), + ("EG", 29), + ("ES", 24), + ("FI", 18), + ("FK", 18), + ("FO", 18), + ("FR", 27), + ("GB", 22), + ("GE", 22), + ("GI", 23), + ("GL", 18), + ("GR", 27), + ("GT", 28), + ("HN", 28), + ("HR", 21), + ("HU", 28), + ("IE", 22), + ("IL", 23), + ("IQ", 23), + ("IS", 26), + ("IT", 27), + ("JO", 30), + ("KW", 30), + ("KZ", 20), + ("LB", 28), + ("LC", 32), + ("LI", 21), + ("LT", 20), + ("LU", 20), + ("LV", 21), + ("LY", 25), + ("MC", 27), + ("MD", 24), + ("ME", 22), + ("MK", 19), + ("MN", 20), + ("MR", 27), + ("MT", 31), + ("MU", 30), + ("NI", 28), + ("NL", 18), + ("NO", 15), + ("OM", 23), + ("PK", 24), + ("PL", 28), + ("PS", 29), + ("PT", 25), + ("QA", 29), + ("RO", 24), + ("RS", 22), + ("RU", 33), + ("SA", 24), + ("SC", 31), + ("SD", 18), + ("SE", 24), + ("SI", 19), + ("SK", 24), + ("SM", 27), + ("SO", 23), + ("ST", 25), + ("SV", 28), + ("TL", 23), + ("TN", 24), + ("TR", 26), + ("UA", 29), + ("VA", 22), + ("VG", 24), + ("XK", 20), + ("YE", 30), +]; + +/// The length of an IBAN of `country`, if the registry lists it. +fn iban_length(country: &str) -> Option { + IBAN_LENGTHS.iter().find(|(code, _)| *code == country).map(|&(_, len)| len) +} + /// Character ranges of `text` matching `pattern`. pub fn find(pattern: Pattern, text: &[char]) -> Vec> { let f: fn(&[char], usize) -> Option = match pattern { @@ -405,6 +575,7 @@ pub fn find(pattern: Pattern, text: &[char]) -> Vec> { Pattern::CreditCard => credit_card, Pattern::Ssn => ssn, Pattern::Date => date, + Pattern::Iban => iban, }; let mut out = Vec::new(); let mut i = 0; @@ -665,4 +836,187 @@ mod tests { let s: Vec = "\u{130}\u{131}I\u{15E}".chars().collect(); assert_eq!(s.iter().copied().map(fold).collect::(), "iii\u{15F}"); } + + #[test] + fn iban_matches_compact_spaced_and_rejects_near_matches() { + assert_eq!(Pattern::from_id("iban").map(Pattern::id), Some("iban")); + + // Spaced and compact IBANs match, a wrong check digit is rejected ("DE89 + // 3704 0044 0532 0130 00" and "DE89370400440532013000" match, "...01" + // does not). + if let Some(iban_pattern) = Pattern::from_id("iban") { + // Spaced IBAN (valid German IBAN: DE89 3704 0044 0532 0130 00) + assert_eq!(found(iban_pattern, "DE89 3704 0044 0532 0130 00"), vec!["DE89 3704 0044 0532 0130 00"]); + // Compact IBAN (same number, no spaces) + assert_eq!(found(iban_pattern, "DE89370400440532013000"), vec!["DE89370400440532013000"]); + // Wrong check digit should not match (DE89...01 instead of DE89...00) + assert_eq!(found(iban_pattern, "DE89 3704 0044 0532 0130 01"), Vec::::new()); + // Near match: IBAN followed by extra digits should match only the IBAN + assert_eq!(found(iban_pattern, "DE89 3704 0044 0532 0130 00 123"), vec!["DE89 3704 0044 0532 0130 00"]); + } + } + + #[test] + fn iban_rejects_unknown_country_code_wrong_length_and_glued_tail() { + if let Some(iban_pattern) = Pattern::from_id("iban") { + // Unknown country code: the check digits pass mod-97, but ISO 13616 defines no length for XX. + assert_eq!(found(iban_pattern, "XX65000000000000000000"), Vec::::new()); + // DE is 22 characters long. This 15-character number passes mod-97 but has + // the wrong national length, so it must be rejected. + assert_eq!(found(iban_pattern, "DE8300000000000"), Vec::::new()); + // A valid IBAN glued directly to trailing alphanumerics is not an IBAN. + assert_eq!(found(iban_pattern, "DE89370400440532013000ABC"), Vec::::new()); + assert_eq!(found(iban_pattern, "ABCDE89370400440532013000"), Vec::::new()); + // Letters in place of the two check digits. + assert_eq!(found(iban_pattern, "DEAB 3704 0044 0532 0130 00"), Vec::::new()); + // A space-separated suffix does not glue to the IBAN. + assert_eq!(found(iban_pattern, "DE89 3704 0044 0532 0130 00 123"), vec!["DE89 3704 0044 0532 0130 00"]); + } + } + + #[test] + fn iban_accepts_spaces_only_between_groups_of_four() { + // A code followed by ordinary words: the words end the run, so nothing is marked. + for s in [ + "Flight BA37 leaves at noon from gate four", + "Flight BA30 leaves at noon", + "Gate BA45 leaves at noon", + "BA72 have water are word", + "GB00 their an her make that", + "SI30 write can all more", + ] { + assert_eq!(found(Pattern::Iban, s), Vec::::new(), "{s}"); + } + // A valid IBAN with a space in the wrong place, or two spaces, is not marked. + for s in ["DE89 3704 0044 0532 013 000", "DE89 3704 0044 0532 0 13000", "DE8 9370 4004 4053 2013 000", "DE89 3704 0044 0532 0130 00"] { + assert_eq!(found(Pattern::Iban, s), Vec::::new(), "{s}"); + } + // Any group boundary may be a space or not, and the IBAN inside a sentence is marked alone. + assert_eq!(found(Pattern::Iban, "DE893704 0044 0532013000"), vec!["DE893704 0044 0532013000"]); + assert_eq!(found(Pattern::Iban, "Pay to DE89 3704 0044 0532 0130 00 today"), vec!["DE89 3704 0044 0532 0130 00"]); + } + + /// One example IBAN per country of the SWIFT IBAN registry (release 99). + const IBAN_REGISTRY_EXAMPLES: [&str; IBAN_LENGTHS.len()] = [ + "AD1200012030200359100100", + "AE070331234567890123456", + "AL47212110090000000235698741", + "AT611904300234573201", + "AZ21NABZ00000000137010001944", + "BA391290079401028494", + "BE68539007547034", + "BG80BNBG96611020345678", + "BH67BMAG00001299123456", + "BI4210000100010000332045181", + "BR1800360305000010009795493C1", + "BY13NBRB3600900000002Z00AB00", + "CH9300762011623852957", + "CR05015202001026284066", + "CY17002001280000001200527600", + "CZ6508000000192000145399", + "DE89370400440532013000", + "DJ2100010000000154000100186", + "DK5000400440116243", + "DO28BAGR00000001212453611324", + "EE382200221020145685", + "EG380019000500000000263180002", + "ES9121000418450200051332", + "FI2112345600000785", + "FK88SC123456789012", + "FO6264600001631634", + "FR1420041010050500013M02606", + "GB29NWBK60161331926819", + "GE29NB0000000101904917", + "GI75NWBK000000007099453", + "GL8964710001000206", + "GR1601101250000000012300695", + "GT82TRAJ01020000001210029690", + "HN88CABF00000000000250005469", + "HR1210010051863000160", + "HU42117730161111101800000000", + "IE29AIBK93115212345678", + "IL620108000000099999999", + "IQ98NBIQ850123456789012", + "IS140159260076545510730339", + "IT60X0542811101000000123456", + "JO94CBJO0010000000000131000302", + "KW81CBKU0000000000001234560101", + "KZ86125KZT5004100100", + "LB62099900000001001901229114", + "LC55HEMM000100010012001200023015", + "LI21088100002324013AA", + "LT121000011101001000", + "LU280019400644750000", + "LV80BANK0000435195001", + "LY83002048000020100120361", + "MC5811222000010123456789030", + "MD24AG000225100013104168", + "ME25505000012345678951", + "MK07250120000058984", + "MN121234123456789123", + "MR1300020001010000123456753", + "MT84MALT011000012345MTLCAST001S", + "MU17BOMM0101101030300200000MUR", + "NI45BAPR00000013000003558124", + "NL91ABNA0417164300", + "NO9386011117947", + "OM810180000001299123456", + "PK36SCBL0000001123456702", + "PL61109010140000071219812874", + "PS92PALS000000000400123456702", + "PT50000201231234567890154", + "QA58DOHB00001234567890ABCDEFG", + "RO49AAAA1B31007593840000", + "RS35260005601001611379", + "RU0304452522540817810538091310419", + "SA0380000000608010167519", + "SC18SSCB11010000000000001497USD", + "SD2129010501234001", + "SE4550000000058398257466", + "SI56263300012039086", + "SK3112000000198742637541", + "SM86U0322509800000000270100", + "SO211000001001000100141", + "ST68000100010051845310112", + "SV62CENR00000000000000700025", + "TL380080012345678910157", + "TN5910006035183598478831", + "TR330006100519786457841326", + "UA213223130000026007233566001", + "VA59001123000012345678", + "VG96VPVG0000012345678901", + "XK051212012345678906", + "YE15CBYE0001018861234567891234", + ]; + + /// `iban` in the printed form: groups of four separated by single spaces. + fn printed(iban: &str) -> String { + iban.chars().collect::>().chunks(4).map(|group| group.iter().collect::()).collect::>().join(" ") + } + + #[test] + fn iban_registry_examples_cover_every_country_and_length() { + let from_examples: Vec<(&str, usize)> = IBAN_REGISTRY_EXAMPLES.iter().map(|e| (&e[..2], e.len())).collect(); + assert_eq!(from_examples.as_slice(), IBAN_LENGTHS.as_slice()); + } + + #[test] + fn iban_matches_the_registry_example_of_every_country() { + for example in IBAN_REGISTRY_EXAMPLES { + assert_eq!(found(Pattern::Iban, example), vec![example], "compact {example}"); + let spaced = printed(example); + assert_eq!(found(Pattern::Iban, &spaced), vec![spaced.clone()], "printed {spaced}"); + } + } + + #[test] + fn iban_rejects_a_registry_example_with_one_digit_changed() { + for example in IBAN_REGISTRY_EXAMPLES { + // The last digit becomes the next digit, so the country, the length and the layout stay the same. + let Some(at) = example.rfind(|c: char| c.is_ascii_digit()) else { continue }; + let digit = example[at..=at].parse::().unwrap_or(0); + let changed = format!("{}{}{}", &example[..at], (digit + 1) % 10, &example[at + 1..]); + assert_eq!(found(Pattern::Iban, &changed), Vec::::new(), "{changed}"); + } + } }